From 35c10b327e49a43c14b6f5ba11c1ff331596cb4b Mon Sep 17 00:00:00 2001 From: Aikins Laryea Date: Sat, 1 Aug 2026 20:19:52 +0000 Subject: [PATCH 01/31] migration: harden complete lifecycle (#8) * chore: run ci on dev pushes (#4) include the dev integration branch in push-triggered CI while keeping main covered for release merges. * security: scrub secret argv, require HTTPS, lock installer inputs * ci: replace deprecated brews with homebrew_casks (#6) * migration: harden complete lifecycle --- README.md | 237 ++- go.mod | 7 +- go.sum | 2 - internal/cli/applied.go | 46 +- internal/cli/applied_test.go | 73 +- internal/cli/apply_lock_unix.go | 56 +- internal/cli/apply_lock_windows.go | 95 +- internal/cli/cleanup.go | 1345 ++++++++++++++++- internal/cli/cleanup_test.go | 748 ++++++++- internal/cli/cockpit.go | 99 +- internal/cli/cockpit_test.go | 232 +++ internal/cli/commit.go | 131 +- internal/cli/commit_test.go | 300 ++++ internal/cli/data.go | 19 +- internal/cli/env.go | 19 +- internal/cli/guide.go | 245 ++- internal/cli/guide_test.go | 328 +++- internal/cli/init_target.go | 13 +- internal/cli/issues.go | 8 +- internal/cli/issues_test.go | 11 + internal/cli/progress.go | 17 +- internal/cli/rollback.go | 28 +- internal/cli/rollback_test.go | 74 + internal/cli/root.go | 47 +- internal/cli/run.go | 973 ++++++++++-- internal/cli/run_test.go | 1317 +++++++++++++++- internal/cli/scan.go | 2 +- internal/cli/state.go | 91 +- internal/cli/style.go | 12 + internal/cli/wizard.go | 277 ++-- internal/safepath/safepath.go | 7 +- internal/safepath/safepath_other.go | 4 + internal/safepath/safepath_unix.go | 10 + internal/target/dokploy/apply.go | 110 +- internal/target/dokploy/apply_test.go | 190 ++- internal/target/dokploy/cleanup.go | 829 ++++++++++ internal/target/dokploy/cleanup_path_unix.go | 62 + .../target/dokploy/cleanup_path_windows.go | 18 + internal/target/dokploy/cleanup_test.go | 509 +++++++ internal/target/dokploy/datamove_test.go | 5 + 40 files changed, 7937 insertions(+), 659 deletions(-) create mode 100644 internal/target/dokploy/cleanup_path_unix.go create mode 100644 internal/target/dokploy/cleanup_path_windows.go diff --git a/README.md b/README.md index 5a94ed6..b1801e8 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,8 @@ shaped: - **private local artifacts** so migration bundles, state, and target API keys stay on the server. - **safe cleanup** that inventories leftovers and only applies narrow metadata - cleanup after a database backup. + cleanup after a database backup, plus a separate confirmed source purge path + for post-acceptance Coolify leftovers. ## Why Bort exists @@ -67,8 +68,13 @@ Bort is at the local-first same-VPS migration stage. handling. - Show a linear status view with per-app health, attention items, and `fix:` commands. +- Persist one explicitly selected current run in `.bort/state.json` so live apply, + acceptance, and cleanup operate on the run you reviewed instead of guessing by + file modification time. - Record env answers and data-store strategies in `.bort/state.json` so resolved issues disappear from the next run. +- Keep each source-created run self-contained under `.bort/runs/` with its + private manifest, bundle, plans, progress, and apply ledger. - Prepare Dokploy resources in dry-run form, then create/reuse projects and compose apps in explicit live mode. - Upload sanitized env and deploy the captured raw compose/image snapshot without @@ -80,6 +86,10 @@ Bort is at the local-first same-VPS migration stage. apply. - Run non-destructive cleanup inventory and, when explicitly applied, remove only safe empty zero-domain Dokploy metadata after backing up the Dokploy database. +- Dry-run a source purge, then remove eligible source containers and networks + only with an explicit scope and confirmation phrase. When named volumes or host + paths are present, every listed source resource must instead be removed manually + before Bort verifies the completed purge. Not implemented yet: @@ -88,9 +98,8 @@ Not implemented yet: - Continuous volume delta sync. - Database replication adapters beyond the current local dump/restore and stopped-copy paths. -- Destructive source purge. Non-destructive cleanup inventory and metadata-only - Dokploy cleanup are implemented, but Bort does not delete Coolify containers, - volumes, networks, credentials, or target apps. +- Automatic source image pruning. `cleanup purge` intentionally leaves Docker + images alone because image tags/layers can be shared with target workloads. - Reverse and third-party platform adapters listed in the roadmap below. ## Quick start @@ -114,67 +123,177 @@ mkdir -p bin go build -o bin/bort ./cmd/bort ``` -Run the safest same-VPS audit from the source Coolify server: +The examples below use an installed `bort`; substitute `./bin/bort` when using +the local build. + +For a guided same-VPS Coolify → Dokploy migration, run Bort from the source +server in a terminal: ```sh -sudo bin/bort scan --source coolify-local --output manifest.json -bin/bort export --manifest manifest.json --output-dir bort-bundle -bin/bort +sudo bort ``` -`bort` with no subcommand is the normal product surface. It resumes the latest run -when one exists, creates a dry-run from `bort-bundle` when that default bundle -exists, or prompts interactively when it needs a source and run name. +`bort` is the normal migration cockpit. It starts guided setup when needed and +otherwise resumes the current run. Review each app, follow the shown `fix:` and +`next:` guidance, and rerun `sudo bort` until the cockpit shows `READY`. Target +setup is offered inline when credentials are needed. -When Bort asks for missing values or data decisions, use the fix commands it -prints: +Use one privilege model for the whole lifecycle. Same-VPS discovery, state copy, +proxy cutover, source retirement, and purge need host-level Docker and filesystem +access, so the examples consistently use `sudo`. Bort keeps `.bort` private to +that OS identity (`0700` directories and `0600` files) and preserves the `sudo` +prefix in copy-paste guidance. If your regular account already has all required +Docker and source-path access, omit `sudo` from the first command and every later +lifecycle command instead; do not switch identities within one workspace. + +The noninteractive start command performs discovery, exports a private bundle, +creates all run artifacts, and selects the new run in one step: ```sh -bin/bort env demo-app API_TOKEN=secret DATABASE_URL=postgres://... -bin/bort data demo-app postgres --migrate -bin/bort +sudo bort migrate --source coolify-local +sudo bort +sudo bort migrate --live +sudo bort commit --apply +sudo bort cleanup ``` -Create or refresh a named dry-run run: +The lifecycle is deliberately explicit: + +1. `sudo bort` or `sudo bort migrate --source ...` starts or resumes a reviewed + dry-run. +2. `sudo bort` shows setup blockers and records interactive fixes. +3. `sudo bort migrate --live` applies only the selected reviewed run. +4. `sudo bort commit --apply` accepts the target and retires source app + containers. +5. `sudo bort cleanup` audits leftovers; optional apply and purge commands remain + separately gated. + +For an existing manifest, use the same consolidated path: ```sh -bin/bort migrate --bundle bort-bundle --target dokploy --run demo-run -bin/bort status --run demo-run -bin/bort next --run demo-run +sudo bort migrate --manifest manifest.json ``` -Only after `bort next` says the gates are clear, bootstrap target credentials and -apply live: +Source and manifest starts create a self-contained directory under +`.bort/runs/` containing the private manifest, exported bundle, reviewed +plans, progress, and apply ledger. `.bort/state.json` stores the current run along +with recorded env/data decisions and target credentials. Pass `--run ` to +select another run explicitly. Mutating commands never choose a run by mtime. + +When Bort asks for missing values or data decisions, use the fix commands it +prints: ```sh -bin/bort init-target dokploy --install --dokploy-url http://127.0.0.1:3030 -bin/bort migrate --live --run demo-run +sudo bort env demo-app API_TOKEN=secret DATABASE_URL=postgres://... +sudo bort data demo-app postgres --migrate +sudo bort ``` After you accept the target, retire the source app stack and audit leftovers: ```sh -bin/bort commit --apply --run demo-run -bin/bort cleanup --run demo-run -bin/bort cleanup --apply --run demo-run +sudo bort commit --apply +sudo bort cleanup +sudo bort cleanup --apply +sudo bort cleanup purge +sudo bort cleanup purge --apply --app demo-app --confirm "purge " ``` `cleanup` is the non-destructive cleanup surface. `cleanup --apply` is metadata-only: it does not remove source containers, volumes, networks, source-control credentials, or Dokploy target apps. +`cleanup purge` is the destructive source cleanup surface. It is still dry-run by +default, and `--apply` refuses to run unless you pass an explicit scope +(`--app`, `--project`, or `--all-apps`), the run has a successful live apply +ledger, the target has been accepted with `sudo bort commit --apply`, and you provide +the confirmation phrase shown in the dry-run output. It writes a private +purge-plan backup before deleting eligible source containers and source networks +by their exact inspected Docker IDs. Bort never deletes named volumes or host +source paths. If either is included, the dry run requires manual removal of every +listed container, volume, network, and path; `--apply` only verifies that the +whole scope remains absent instead of combining absence checks with automatic +deletion. Verification-only purges do not set `PurgedAt`, because Bort cannot +lock external tools out of recreating an absent resource. `--all-apps` +covers non-platform apps by default; platform-role leftovers such as Coolify +source/proxy resources require `--include-platform`. It does not remove +source-control credentials, target Dokploy resources, or Docker images. A full +`--all-apps` purge marks the migration complete; app- or project-scoped purges do +not claim that the whole run is finished. + +## Acceptance trace + +The automated lifecycle trace uses a disposable workspace, fake Dokploy HTTP +API, and fake Docker executable. It creates a self-contained run, performs live +apply, verifies `TARGET LIVE`, commits, verifies `COMMITTED`, inventories cleanup, +applies a confirmed all-app purge, and verifies `COMPLETE`: + +```sh +go test ./internal/cli -run '^TestLifecycleAcceptanceTraceWithFakeDokploy$' -v +``` + +Real container, volume, network, bind-path, and proxy behavior still needs a +throwaway same-VPS host. Never run this destructive acceptance pass on a host +containing data you intend to keep. On a fresh VM or restorable snapshot: + +1. Install Coolify and Dokploy, then deploy one stateless disposable Coolify app + with a route, source network, and non-secret test env value. +2. Record the source container, network, and target project names before migration. +3. Run the complete lifecycle under one identity: + + ```sh + run="bort-acceptance-$(date +%Y%m%d-%H%M%S)" + sudo bort migrate --source coolify-local --run "$run" + sudo bort + sudo bort migrate --live --run "$run" + sudo bort status --run "$run" + sudo bort commit --apply --run "$run" + sudo bort cleanup --run "$run" + sudo bort cleanup purge --run "$run" --all-apps + ``` + + Review the purge plan and verify the target still serves the disposable + workload before applying it: + + ```sh + sudo bort cleanup purge --apply --run "$run" --all-apps --confirm "purge $run" + sudo bort status --run "$run" + ``` + +4. Confirm the target app remains healthy, `run.json` contains all three + lifecycle timestamps, the selected source container and network are absent, + and target resources plus source-control credentials were not removed. +5. Restore the clean snapshot and repeat with a disposable app that has a named + volume and bind mount under a disposable directory. After the purge dry run, + manually remove every listed source resource. Confirm `--apply` performs only + absence verification, leaves the run committed rather than complete, and does + not delete any replacement resource introduced before verification. +6. Destroy the VM or restore the snapshot after saving only the redacted command + output and lifecycle metadata needed for release evidence. + ## Safety model Bort is designed for production boxes where the safest default is "look first." -- **dry-run first:** scan, plan, validate, prepare, sync planning, cutover planning, - rollback planning, commit planning, and cleanup inventory are non-mutating by - default. -- **explicit live path:** target creation and traffic movement only happen through - `bort migrate --live` after readiness gates are clear. -- **resume instead of restart:** live apply writes an apply lock and - `.bort/runs//applied.json`, so reruns can resume or attach without - replaying completed steps blindly. +- **dry-run first:** source setup, plan, validate, prepare, sync, cutover, + rollback, commit, cleanup, and purge planning are non-mutating by default. +- **explicit live path:** target creation and traffic movement happen only through + `sudo bort migrate --live` after setup blockers are clear. Live mode never creates a + run implicitly and rejects source/planning flags. +- **explicit run ownership:** `.bort/state.json` names the current reviewed run. + `--run` overrides it explicitly, and mutating commands never infer a destructive + target from whichever `run.json` has the newest mtime. +- **immutable live plans:** after live execution begins, Bort will not regenerate + that run's reviewed plan or reorder its apply steps. Create a new run when the + plan itself must change. +- **serialized run mutation:** planning, live apply, target acceptance, cleanup + apply, and purge apply share a per-run operation lock. Live apply also writes a + separate attachable apply lock and `.bort/runs//applied.json`, so reruns + can attach or resume without racing another mutation or replaying completed + steps blindly. +- **persisted lifecycle:** `run.json` records when the target went live, when the + target was accepted, and when a full source purge completed. Older runs remain + compatible through apply-ledger inference. - **private files:** migration bundles, state, env, and target credentials are local artifacts with private permissions. - **no token flags:** Coolify API tokens are read from `BORT_COOLIFY_TOKEN`, not from @@ -191,23 +310,48 @@ Bort is designed for production boxes where the safest default is "look first." - **narrow cleanup:** `cleanup --apply` backs up the Dokploy database and only deletes stale Dokploy platform metadata records named `coolify-proxy`, `proxy`, or `source` when the project is empty and still has zero domains. +- **separate purge:** destructive source deletion lives under `cleanup purge`, + never under regular app entrypoints or metadata cleanup. `--apply` requires a + selected `--app`, `--project`, or `--all-apps` scope, a successful live apply + ledger, a committed target, and the exact `purge ` confirmation + phrase. Bort removes identified containers and networks only when the scope has + no absence-only prerequisite. A scope containing named volumes or host paths + requires manual removal of every listed source resource, which Bort verifies + remains absent without performing destructive operations or recording durable + lifecycle completion. +- **rollback stays inspectable:** `sudo bort rollback` prints the selected run's stored + rollback plan. Executable rollback is not implemented yet. ## Product surface -The main product is eight verbs: +The primary product follows one lifecycle: + +```text +sudo bort # start or resume the current run and review fixes +sudo bort migrate --live # apply only the reviewed current run +sudo bort rollback # inspect the current run's source rollback plan +sudo bort commit --apply # accept the target and retire source containers +sudo bort cleanup # audit leftovers; --apply removes safe metadata only +sudo bort cleanup purge # review source purge and manual prerequisites +``` + +Setup and automation commands remain available for noninteractive use and future +source/target adapters: ```text -bort # show app-first migration status and fix hints -bort env # record env values for an app -bort data # record a data-store strategy for an app -bort migrate # create/update a run; --live applies after gates are clear -bort rollback # plan rollback to the source -bort commit # accept the target; --apply stops source app containers -bort cleanup # inventory leftovers; --apply removes safe metadata only -bort init-target # bootstrap target credentials for live execution +sudo bort migrate --source # scan, export, and create a current run +sudo bort migrate --manifest # create a current run from a manifest +sudo bort migrate --bundle # create or refresh an advanced bundle run +sudo bort env KEY=value ... # record env values noninteractively +sudo bort data --migrate|--recreate|--managed # record a data-store strategy +sudo bort init-target dokploy # bootstrap target credentials explicitly +sudo bort status # compatibility alias for the current cockpit +sudo bort next # compatibility helper for one next action ``` -The power-user pipeline is still available when you want each artifact explicitly: +The artifact pipeline is also retained for inspection, scripting, compatibility, +and adapter development. Every command below is local and dry-run only, so it can +run unprivileged when its input and output paths are accessible: ```text bort scan # extract source platform state @@ -217,9 +361,6 @@ bort validate # validate compose, env, routes, storage, and portability bort prepare # plan target resources before creating anything bort sync # plan state copy or replication work bort cutover # plan traffic movement and rollback windows -bort status # summarize a persisted run -bort next # show the next safe action for a run -bort cleanup # inventory and safely apply metadata-only cleanup ``` ## Roadmap and status diff --git a/go.mod b/go.mod index c8627c1..0c5520d 100644 --- a/go.mod +++ b/go.mod @@ -3,11 +3,10 @@ module github.com/aikins01/bort go 1.26.2 require ( - github.com/charmbracelet/bubbles v0.21.1-0.20250623103423-23b8fd6302d7 - github.com/charmbracelet/bubbletea v1.3.6 github.com/charmbracelet/huh v1.0.0 github.com/charmbracelet/lipgloss v1.1.0 golang.org/x/crypto v0.50.0 + golang.org/x/sys v0.43.0 golang.org/x/term v0.42.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -16,8 +15,9 @@ require ( github.com/atotto/clipboard v0.1.4 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/catppuccin/go v0.3.0 // indirect + github.com/charmbracelet/bubbles v0.21.1-0.20250623103423-23b8fd6302d7 // indirect + github.com/charmbracelet/bubbletea v1.3.6 // indirect github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc // indirect - github.com/charmbracelet/harmonica v0.2.0 // indirect github.com/charmbracelet/x/ansi v0.9.3 // indirect github.com/charmbracelet/x/cellbuf v0.0.13 // indirect github.com/charmbracelet/x/exp/strings v0.0.0-20240722160745-212f7b056ed0 // indirect @@ -37,7 +37,6 @@ require ( github.com/rogpeppe/go-internal v1.14.1 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect golang.org/x/sync v0.20.0 // indirect - golang.org/x/sys v0.43.0 // indirect golang.org/x/text v0.36.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect ) diff --git a/go.sum b/go.sum index 93ef530..f893ab8 100644 --- a/go.sum +++ b/go.sum @@ -14,8 +14,6 @@ github.com/charmbracelet/bubbletea v1.3.6 h1:VkHIxPJQeDt0aFJIsVxw8BQdh/F/L2KKZGs github.com/charmbracelet/bubbletea v1.3.6/go.mod h1:oQD9VCRQFF8KplacJLo28/jofOI2ToOfGYeFgBBxHOc= github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc h1:4pZI35227imm7yK2bGPcfpFEmuY1gc2YSTShr4iJBfs= github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc/go.mod h1:X4/0JoqgTIPSFcRA/P6INZzIuyqdFY5rm8tb41s9okk= -github.com/charmbracelet/harmonica v0.2.0 h1:8NxJWRWg/bzKqqEaaeFNipOu77YR5t8aSwG4pgaUBiQ= -github.com/charmbracelet/harmonica v0.2.0/go.mod h1:KSri/1RMQOZLbw7AHqgcBycp8pgJnQMYYT8QZRqZ1Ao= github.com/charmbracelet/huh v1.0.0 h1:wOnedH8G4qzJbmhftTqrpppyqHakl/zbbNdXIWJyIxw= github.com/charmbracelet/huh v1.0.0/go.mod h1:5YVc+SlZ1IhQALxRPpkGwwEKftN/+OlJlnJYlDRFqN4= github.com/charmbracelet/lipgloss v1.1.0 h1:vYXsiLHVkK7fp74RkV7b2kq9+zDLoEU4MZoFqR/noCY= diff --git a/internal/cli/applied.go b/internal/cli/applied.go index 3ce6d78..c052905 100644 --- a/internal/cli/applied.go +++ b/internal/cli/applied.go @@ -19,13 +19,14 @@ const appliedAPIVersion = "bort.applied/v1alpha1" // step the live applier walks through so a partial run can be inspected and // safely retried. type runApplied struct { - APIVersion string `json:"apiVersion"` - RunName string `json:"runName"` - BundleDir string `json:"bundleDir,omitempty"` - Target string `json:"target,omitempty"` - UpdatedAt time.Time `json:"updatedAt"` - Steps []appliedStep `json:"steps,omitempty"` - Apps map[string]appliedApp `json:"apps,omitempty"` + APIVersion string `json:"apiVersion"` + RunName string `json:"runName"` + BundleDir string `json:"bundleDir,omitempty"` + Target string `json:"target,omitempty"` + UpdatedAt time.Time `json:"updatedAt"` + SucceededAt *time.Time `json:"succeededAt,omitempty"` + Steps []appliedStep `json:"steps,omitempty"` + Apps map[string]appliedApp `json:"apps,omitempty"` } type appliedStep struct { @@ -133,6 +134,7 @@ type appliedLedger struct { path string state runApplied persist func(string, runApplied) error + err error } func newAppliedLedger(path string, run migrationRun) (*appliedLedger, error) { @@ -146,8 +148,36 @@ func newAppliedLedger(path string, run migrationRun) (*appliedLedger, error) { func (l *appliedLedger) Record(progress dokploy.StepProgress) error { l.mu.Lock() defer l.mu.Unlock() + if l.err != nil { + return l.err + } l.state = recordAppliedStep(l.state, progress) - return l.persist(l.path, l.state) + if err := l.persist(l.path, l.state); err != nil { + l.err = err + return err + } + return nil +} + +func (l *appliedLedger) Err() error { + l.mu.Lock() + defer l.mu.Unlock() + return l.err +} + +func (l *appliedLedger) MarkSucceeded() error { + l.mu.Lock() + defer l.mu.Unlock() + if l.err != nil { + return l.err + } + now := time.Now().UTC() + l.state.SucceededAt = &now + if err := l.persist(l.path, l.state); err != nil { + l.err = err + return err + } + return nil } func (l *appliedLedger) Snapshot() runApplied { diff --git a/internal/cli/applied_test.go b/internal/cli/applied_test.go index 6aeb023..d8adbdc 100644 --- a/internal/cli/applied_test.go +++ b/internal/cli/applied_test.go @@ -6,7 +6,6 @@ import ( "testing" "github.com/aikins01/bort/internal/target/dokploy" - "github.com/charmbracelet/bubbles/progress" ) func TestAppliedLedgerRecordsAndPersists(t *testing.T) { @@ -76,6 +75,47 @@ func TestAppliedLedgerOverwritesByIndex(t *testing.T) { } } +func TestAppliedLedgerRetainsPersistenceFailure(t *testing.T) { + ledger := &appliedLedger{ + state: newRunApplied(migrationRun{Name: "run-failure"}), + persist: func(string, runApplied) error { + return errors.New("persist failed") + }, + } + progress := dokploy.StepProgress{Index: 0, Step: dokploy.Step{Kind: dokploy.StepCreateProject, App: "api", Ref: "api"}, Status: dokploy.StepStatusStarted} + if err := ledger.Record(progress); err == nil || err.Error() != "persist failed" { + t.Fatalf("expected persistence failure, got %v", err) + } + if err := ledger.Err(); err == nil || err.Error() != "persist failed" { + t.Fatalf("expected retained persistence failure, got %v", err) + } + if err := ledger.Record(progress); err == nil || err.Error() != "persist failed" { + t.Fatalf("expected later records to stop at the retained failure, got %v", err) + } + if err := ledger.MarkSucceeded(); err == nil || err.Error() != "persist failed" { + t.Fatalf("expected success marking to stop at the retained failure, got %v", err) + } +} + +func TestAppliedLedgerPersistsSuccessfulOutcome(t *testing.T) { + path := filepath.Join(t.TempDir(), "applied.json") + run := migrationRun{Name: "run-success", BundleDir: "bundle", Target: "dokploy"} + ledger, err := newAppliedLedger(path, run) + if err != nil { + t.Fatal(err) + } + if err := ledger.MarkSucceeded(); err != nil { + t.Fatal(err) + } + applied, err := readRunApplied(path, run) + if err != nil { + t.Fatal(err) + } + if applied.SucceededAt == nil { + t.Fatal("expected a durable successful live-apply outcome") + } +} + func TestReadRunAppliedRejectsUnknownAPIVersion(t *testing.T) { dir := t.TempDir() path := filepath.Join(dir, "applied.json") @@ -136,34 +176,3 @@ func TestCompletedApplyPrefixRequiresContiguousIndexedSteps(t *testing.T) { t.Fatalf("expected prefix to stop at mismatched index 1, got %d", got) } } - -func TestProgressModelUsesStepIndexWhenResuming(t *testing.T) { - model := &progressModel{bar: progress.New(progress.WithoutPercentage()), total: 5, done: 3} - model.Update(progressTick{progress: dokploy.StepProgress{ - Index: 3, - Total: 5, - Step: dokploy.Step{Kind: dokploy.StepRestoreDataStore, App: "api", Ref: "data-store:db"}, - Status: dokploy.StepStatusStarted, - }}) - if model.done != 3 { - t.Fatalf("expected started step to preserve resumed count 3, got %d", model.done) - } - model.Update(progressTick{progress: dokploy.StepProgress{ - Index: 3, - Total: 5, - Step: dokploy.Step{Kind: dokploy.StepRestoreDataStore, App: "api", Ref: "data-store:db"}, - Status: dokploy.StepStatusOK, - }}) - if model.done != 4 { - t.Fatalf("expected completed resumed step to advance to 4, got %d", model.done) - } - model.Update(progressTick{progress: dokploy.StepProgress{ - Index: 5, - Total: 5, - Step: dokploy.Step{Kind: dokploy.StepResumeSource, App: "api", Ref: "api"}, - Status: dokploy.StepStatusOK, - }}) - if model.done != 4 { - t.Fatalf("expected out-of-plan cleanup step not to advance count, got %d", model.done) - } -} diff --git a/internal/cli/apply_lock_unix.go b/internal/cli/apply_lock_unix.go index ee5769b..9ef7162 100644 --- a/internal/cli/apply_lock_unix.go +++ b/internal/cli/apply_lock_unix.go @@ -7,7 +7,8 @@ import ( "fmt" "os" "strconv" - "syscall" + + "golang.org/x/sys/unix" ) var errApplyAlreadyRunning = errors.New("live apply already running") @@ -17,13 +18,13 @@ type applyLock struct { } func acquireApplyLock(path string) (*applyLock, error) { - file, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0o600) + file, err := openLockFile(path, unix.O_CREAT|unix.O_RDWR) if err != nil { return nil, err } - if err := syscall.Flock(int(file.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil { + if err := unix.Flock(int(file.Fd()), unix.LOCK_EX|unix.LOCK_NB); err != nil { file.Close() - if errors.Is(err, syscall.EWOULDBLOCK) || errors.Is(err, syscall.EAGAIN) { + if errors.Is(err, unix.EWOULDBLOCK) || errors.Is(err, unix.EAGAIN) { return nil, errApplyAlreadyRunning } return nil, err @@ -47,14 +48,49 @@ func (l *applyLock) Release() { if l == nil || l.file == nil { return } - _ = syscall.Flock(int(l.file.Fd()), syscall.LOCK_UN) + _ = unix.Flock(int(l.file.Fd()), unix.LOCK_UN) _ = l.file.Close() } -func applyLockPIDRunning(pid int) bool { - if pid <= 0 { - return false +func applyLockActive(path string) (bool, error) { + file, err := openLockFile(path, unix.O_RDWR) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return false, nil + } + return false, err + } + defer file.Close() + if err := unix.Flock(int(file.Fd()), unix.LOCK_EX|unix.LOCK_NB); err != nil { + if errors.Is(err, unix.EWOULDBLOCK) || errors.Is(err, unix.EAGAIN) { + return true, nil + } + return false, err + } + if err := unix.Flock(int(file.Fd()), unix.LOCK_UN); err != nil { + return false, err + } + return false, nil +} + +func openLockFile(path string, flags int) (*os.File, error) { + fd, err := unix.Open(path, flags|unix.O_CLOEXEC|unix.O_NOFOLLOW, 0o600) + if err != nil { + return nil, err + } + file := os.NewFile(uintptr(fd), path) + if file == nil { + _ = unix.Close(fd) + return nil, fmt.Errorf("open lock file %s", path) + } + var stat unix.Stat_t + if err := unix.Fstat(fd, &stat); err != nil { + _ = file.Close() + return nil, err + } + if stat.Mode&unix.S_IFMT != unix.S_IFREG || stat.Nlink != 1 { + _ = file.Close() + return nil, fmt.Errorf("lock file %s must be a single-link regular file", path) } - err := syscall.Kill(pid, 0) - return err == nil || errors.Is(err, syscall.EPERM) + return file, nil } diff --git a/internal/cli/apply_lock_windows.go b/internal/cli/apply_lock_windows.go index b8611a5..9bcc3b8 100644 --- a/internal/cli/apply_lock_windows.go +++ b/internal/cli/apply_lock_windows.go @@ -6,56 +6,107 @@ import ( "errors" "fmt" "os" - "syscall" + + "golang.org/x/sys/windows" ) var errApplyAlreadyRunning = errors.New("live apply already running") type applyLock struct { - file *os.File + file *os.File + overlapped windows.Overlapped } func acquireApplyLock(path string) (*applyLock, error) { - file, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0o600) + file, err := openLockFile(path, windows.OPEN_ALWAYS) if err != nil { - if errors.Is(err, os.ErrExist) { + return nil, err + } + lock := &applyLock{file: file} + if err := windows.LockFileEx(windows.Handle(file.Fd()), windows.LOCKFILE_EXCLUSIVE_LOCK|windows.LOCKFILE_FAIL_IMMEDIATELY, 0, 1, 0, &lock.overlapped); err != nil { + _ = file.Close() + if errors.Is(err, windows.ERROR_LOCK_VIOLATION) { return nil, errApplyAlreadyRunning } return nil, err } + if err := file.Truncate(0); err != nil { + lock.Release() + return nil, err + } + if _, err := file.Seek(0, 0); err != nil { + lock.Release() + return nil, err + } if _, err := fmt.Fprintln(file, os.Getpid()); err != nil { - _ = file.Close() - _ = os.Remove(path) + lock.Release() return nil, err } - return &applyLock{file: file}, nil + return lock, nil } func (l *applyLock) Release() { if l == nil || l.file == nil { return } - path := l.file.Name() + _ = windows.UnlockFileEx(windows.Handle(l.file.Fd()), 0, 1, 0, &l.overlapped) _ = l.file.Close() - _ = os.Remove(path) + l.file = nil +} + +func applyLockActive(path string) (bool, error) { + file, err := openLockFile(path, windows.OPEN_EXISTING) + if err != nil { + if errors.Is(err, os.ErrNotExist) || errors.Is(err, windows.ERROR_FILE_NOT_FOUND) || errors.Is(err, windows.ERROR_PATH_NOT_FOUND) { + return false, nil + } + return false, err + } + defer file.Close() + var overlapped windows.Overlapped + err = windows.LockFileEx(windows.Handle(file.Fd()), windows.LOCKFILE_EXCLUSIVE_LOCK|windows.LOCKFILE_FAIL_IMMEDIATELY, 0, 1, 0, &overlapped) + if err != nil { + if errors.Is(err, windows.ERROR_LOCK_VIOLATION) { + return true, nil + } + return false, err + } + if err := windows.UnlockFileEx(windows.Handle(file.Fd()), 0, 1, 0, &overlapped); err != nil { + return false, err + } + return false, nil } -func applyLockPIDRunning(pid int) bool { - if pid <= 0 { - return false +func openLockFile(path string, disposition uint32) (*os.File, error) { + name, err := windows.UTF16PtrFromString(path) + if err != nil { + return nil, err } - const ( - processQueryLimitedInformation = 0x1000 - stillActive = 259 + handle, err := windows.CreateFile( + name, + windows.GENERIC_READ|windows.GENERIC_WRITE, + windows.FILE_SHARE_READ|windows.FILE_SHARE_WRITE, + nil, + disposition, + windows.FILE_ATTRIBUTE_NORMAL|windows.FILE_FLAG_OPEN_REPARSE_POINT, + 0, ) - handle, err := syscall.OpenProcess(processQueryLimitedInformation, false, uint32(pid)) if err != nil { - return false + return nil, err + } + var info windows.ByHandleFileInformation + if err := windows.GetFileInformationByHandle(handle, &info); err != nil { + _ = windows.CloseHandle(handle) + return nil, err + } + if info.FileAttributes&(windows.FILE_ATTRIBUTE_REPARSE_POINT|windows.FILE_ATTRIBUTE_DIRECTORY) != 0 || info.NumberOfLinks != 1 { + _ = windows.CloseHandle(handle) + return nil, fmt.Errorf("lock file %s must be a single-link regular file and cannot be a reparse point", path) } - defer syscall.CloseHandle(handle) - var code uint32 - if err := syscall.GetExitCodeProcess(handle, &code); err != nil { - return false + file := os.NewFile(uintptr(handle), path) + if file == nil { + _ = windows.CloseHandle(handle) + return nil, fmt.Errorf("open lock file %s", path) } - return code == stillActive + return file, nil } diff --git a/internal/cli/cleanup.go b/internal/cli/cleanup.go index d98630a..042fbbc 100644 --- a/internal/cli/cleanup.go +++ b/internal/cli/cleanup.go @@ -1,22 +1,32 @@ package cli import ( + "bufio" "context" "encoding/json" "flag" "fmt" "io" + "os" "path/filepath" "sort" "strings" + "time" "github.com/aikins01/bort/internal/analyzer" + "github.com/aikins01/bort/internal/planutil" "github.com/aikins01/bort/internal/preparer" "github.com/aikins01/bort/internal/safepath" "github.com/aikins01/bort/internal/target/dokploy" ) const cleanupAPIVersion = "bort.cleanup/v1alpha1" +const cleanupPurgeAPIVersion = "bort.cleanup.purge/v1alpha1" + +var cleanupPurgeLiveApplySkipKinds = map[dokploy.StepKind]struct{}{ + dokploy.StepResumeSource: {}, + dokploy.StepResumeTarget: {}, +} var defaultStaleDokployPlatformProjects = []string{"coolify-proxy", "proxy", "source"} @@ -65,19 +75,22 @@ type cleanupSourceContainer struct { } type cleanupSourceVolume struct { - App string `json:"app"` - Service string `json:"service,omitempty"` - Type string `json:"type"` - Name string `json:"name,omitempty"` - Source string `json:"source,omitempty"` - Target string `json:"target"` - Action string `json:"action"` + App string `json:"app"` + Service string `json:"service,omitempty"` + Type string `json:"type"` + Name string `json:"name,omitempty"` + Source string `json:"source,omitempty"` + Target string `json:"target"` + ExpectedAbsent bool `json:"expectedAbsent,omitempty"` + Action string `json:"action"` } type cleanupSourceNetwork struct { - App string `json:"app"` - Name string `json:"name"` - Action string `json:"action"` + App string `json:"app"` + Name string `json:"name"` + ExpectedIdentity string `json:"expectedIdentity,omitempty"` + ExpectedAbsent bool `json:"expectedAbsent,omitempty"` + Action string `json:"action"` } type cleanupTargetArtifact struct { @@ -95,7 +108,57 @@ type cleanupAction struct { Message string `json:"message"` } +type cleanupPurgeResult struct { + APIVersion string `json:"apiVersion"` + RunName string `json:"runName,omitempty"` + RunDir string `json:"runDir,omitempty"` + Target string `json:"target"` + DryRun bool `json:"dryRun"` + Applied bool `json:"applied,omitempty"` + CompletesLifecycle bool `json:"completesLifecycle,omitempty"` + LifecycleCompleted bool `json:"lifecycleCompleted,omitempty"` + ManualCompletion bool `json:"manualCompletion,omitempty"` + ExecutionStartedAt *time.Time `json:"executionStartedAt,omitempty"` + BackupDir string `json:"backupDir,omitempty"` + BackupPath string `json:"backupPath,omitempty"` + Filters cleanupPurgeFilters `json:"filters"` + SourceControls []cleanupSourceControl `json:"sourceControls,omitempty"` + SourceContainers []cleanupSourceContainer `json:"sourceContainers,omitempty"` + SourceVolumes []cleanupSourceVolume `json:"sourceVolumes,omitempty"` + SourceNetworks []cleanupSourceNetwork `json:"sourceNetworks,omitempty"` + SourcePaths []cleanupSourcePath `json:"sourcePaths,omitempty"` + Actions []cleanupAction `json:"actions"` + PurgeResult *dokploy.SourcePurgeResult `json:"purgeResult,omitempty"` + Warnings []string `json:"warnings,omitempty"` +} + +type cleanupPurgeFilters struct { + Apps []string `json:"apps,omitempty"` + Projects []string `json:"projects,omitempty"` + AllApps bool `json:"allApps,omitempty"` + IncludePlatform bool `json:"includePlatform,omitempty"` +} + +type cleanupSourcePath struct { + App string `json:"app,omitempty"` + Path string `json:"path"` + Source string `json:"source"` + AllowPlatform bool `json:"allowPlatform,omitempty"` + ExpectedAbsent bool `json:"expectedAbsent,omitempty"` + Action string `json:"action"` +} + +type stringListFlag []string + func runCleanup(ctx context.Context, args []string, stdout, stderr io.Writer) error { + return runCleanupWithInput(ctx, args, nil, stdout, stderr) +} + +func runCleanupWithInput(ctx context.Context, args []string, stdin io.Reader, stdout, stderr io.Writer) error { + if len(args) > 0 && args[0] == "purge" { + return runCleanupPurge(ctx, args[1:], stdin, stdout, stderr) + } + fs := flag.NewFlagSet("cleanup", flag.ContinueOnError) fs.SetOutput(stderr) @@ -116,6 +179,12 @@ func runCleanup(ctx context.Context, args []string, stdout, stderr io.Writer) er if err := fs.Parse(args); err != nil { return err } + if fs.NArg() > 0 { + if fs.Arg(0) == "purge" { + return fmt.Errorf("cleanup purge must immediately follow cleanup; use `%s`", bortCommand("cleanup purge")) + } + return fmt.Errorf("cleanup does not accept positional argument %q", fs.Arg(0)) + } if err := checkOutputFormat("cleanup", format); err != nil { return err } @@ -123,7 +192,19 @@ func runCleanup(ctx context.Context, args []string, stdout, stderr io.Writer) er return fmt.Errorf("cleanup currently supports target dokploy only, got %q", target) } - run, err := loadCleanupRun(runRef) + resolvedRunRef, err := resolveRunRef(runRef, false) + if err != nil { + return err + } + var operationLock *applyLock + if apply { + operationLock, err = acquireRunOperationLock(resolvedRunRef) + if err != nil { + return fmt.Errorf("cleanup run %q: %w", resolvedRunRef, err) + } + defer operationLock.Release() + } + run, err := loadMigrationRun(resolvedRunRef) if err != nil { return err } @@ -167,14 +248,919 @@ func runCleanup(ctx context.Context, args []string, stdout, stderr io.Writer) er return writeFormattedOutput(stdout, outputPath, format, result, writeCleanupText) } -func loadCleanupRun(runRef string) (loadedMigrationRun, error) { - if strings.TrimSpace(runRef) != "" { - return loadMigrationRun(runRef) +func runCleanupPurge(ctx context.Context, args []string, stdin io.Reader, stdout, stderr io.Writer) error { + fs := flag.NewFlagSet("cleanup purge", flag.ContinueOnError) + fs.SetOutput(stderr) + + var runRef string + var target string + var format string + var outputPath string + var apply bool + var backupDir string + var confirm string + var allApps bool + var includePlatform bool + var apps stringListFlag + var projects stringListFlag + var sourceDirs stringListFlag + + fs.StringVar(&runRef, "run", "", "run name under .bort/runs, or a run directory path") + fs.StringVar(&target, "target", "dokploy", "target platform") + fs.StringVar(&format, "format", "text", "output format: text, json") + fs.StringVar(&outputPath, "output", "-", "output path, or - for stdout") + fs.BoolVar(&apply, "apply", false, "remove eligible source containers and networks, or verify all resources absent when manual completion is required") + fs.StringVar(&backupDir, "backup-dir", filepath.Join(".bort", "backups"), "directory for purge plan backups before destructive cleanup") + fs.StringVar(&confirm, "confirm", "", "required with --apply; must equal `purge `") + fs.BoolVar(&allApps, "all-apps", false, "purge all non-platform apps in the run") + fs.BoolVar(&includePlatform, "include-platform", false, "allow purge planning for platform-role apps such as coolify-proxy/source") + fs.Var(&apps, "app", "app name to purge (repeatable or comma-separated)") + fs.Var(&projects, "project", "source or target project name to purge (repeatable or comma-separated)") + fs.Var(&sourceDirs, "source-dir", "extra absolute host source directory that must be absent before apply (repeatable or comma-separated)") + + if err := fs.Parse(args); err != nil { + return err + } + if fs.NArg() > 0 { + return fmt.Errorf("cleanup purge does not accept positional argument %q", fs.Arg(0)) + } + if err := checkOutputFormat("cleanup purge", format); err != nil { + return err + } + if target != "dokploy" { + return fmt.Errorf("cleanup purge currently supports target dokploy only, got %q", target) + } + if allApps && (len(cleanupStringListValues(apps)) > 0 || len(cleanupStringListValues(projects)) > 0) { + return fmt.Errorf("cleanup purge accepts either --all-apps or explicit --app/--project filters, not both") + } + + resolvedRunRef, err := resolveRunRef(runRef, false) + if err != nil { + return err + } + var operationLock *applyLock + if apply { + operationLock, err = acquireRunOperationLock(resolvedRunRef) + if err != nil { + return fmt.Errorf("cleanup purge run %q: %w", resolvedRunRef, err) + } + defer operationLock.Release() + } + run, err := loadMigrationRun(resolvedRunRef) + if err != nil { + return err + } + filters := cleanupPurgeFilters{ + Apps: cleanupStringListValues(apps), + Projects: cleanupStringListValues(projects), + AllApps: allApps, + IncludePlatform: includePlatform, + } + result, err := planCleanupPurge(run, target, filters, cleanupStringListValues(sourceDirs)) + if err != nil { + return err + } + if flagSet(fs, "backup-dir") { + result.BackupDir = backupDir + } + result.DryRun = !apply + if !apply { + return writeFormattedOutput(stdout, outputPath, format, result, writeCleanupPurgeText) + } + + if err := validateCleanupPurgeApplyScope(filters); err != nil { + return err + } + if len(result.SourceContainers) == 0 && len(cleanupPurgeVolumeChecks(result.SourceVolumes)) == 0 && len(result.SourceNetworks) == 0 && len(result.SourcePaths) == 0 { + return fmt.Errorf("cleanup purge --apply found no selected source resources to remove") + } + selectedApps, err := cleanupPurgeSelectedApps(run, filters) + if err != nil { + return err + } + if err := requireLiveApplySucceededForAppsSkipping(run, cleanupSelectedAppNames(selectedApps), cleanupPurgeLiveApplySkipKinds); err != nil { + return fmt.Errorf("refusing cleanup purge --apply before a successful live apply: %w", err) + } + if run.Run.CommittedAt == nil { + return fmt.Errorf("refusing cleanup purge --apply before `%s`; the source is still the rollback path", runScopedCommand(run, "commit --apply")) + } + client := &dokploy.Client{} + identified, err := client.IdentifySourcePurgeResources(ctx, cleanupPurgeOptions(result)) + if err != nil { + return fmt.Errorf("identify selected source resources before confirmation: %w", err) + } + applyCleanupPurgeIdentities(&result, identified) + if err := confirmCleanupPurgeApply(stdin, stderr, run.Run.Name, confirm); err != nil { + return err + } + backupPath, err := writeCleanupPurgeBackup(backupDir, result) + if err != nil { + return err + } + result.BackupPath = backupPath + recorder := cleanupPurgeBackupRecorder{path: backupPath, result: &result, persist: updateCleanupPurgeBackup} + if err := recorder.Start(); err != nil { + return fmt.Errorf("record source purge execution start in private backup %s: %w", backupPath, err) + } + options := cleanupPurgeOptions(result) + options.OnProgress = recorder.Record + purged, err := client.PurgeSourceResources(ctx, options) + if err != nil { + result.DryRun = false + result.PurgeResult = &purged + result.Warnings = append(result.Warnings, "source purge stopped before completion; earlier resources may already have been removed") + result.Actions = append(result.Actions, cleanupAction{ + Kind: "source_purge", + Ref: cleanupPurgeScopeLabel(filters), + Safety: "destructive_confirmed", + Status: "error", + Message: "purge stopped after partial execution; inspect the per-resource results and private backup before retrying", + }) + purgeErr := fmt.Errorf("source purge stopped after partial execution; inspect the reported results and backup %s before retrying: %w", backupPath, err) + if backupErr := recorder.Persist(); backupErr != nil { + result.Warnings = append(result.Warnings, "private purge backup could not be updated with the partial resource results") + purgeErr = fmt.Errorf("%v; update private purge backup: %w", purgeErr, backupErr) + } + if outputErr := writeFormattedOutput(stdout, outputPath, format, result, writeCleanupPurgeText); outputErr != nil { + return fmt.Errorf("%v; write partial purge results: %w", purgeErr, outputErr) + } + return purgeErr + } + result.Applied = true + result.DryRun = false + result.PurgeResult = &purged + purgeSafety := "destructive_confirmed" + if result.ManualCompletion { + purgeSafety = "manual_absence_verified" + } + result.Actions = append(result.Actions, cleanupAction{ + Kind: "source_purge", + Ref: cleanupPurgeScopeLabel(filters), + Safety: purgeSafety, + Status: "applied", + Message: "removed eligible source containers and networks, or verified all selected resources absent when manual completion was required, after writing a private purge plan backup", + }) + if err := recorder.Persist(); err != nil { + result.Warnings = append(result.Warnings, "source purge completed, but the private purge backup could not be updated with the resource results") + purgeErr := fmt.Errorf("source purge completed and initial backup is available at %s, but its resource results could not be persisted: %w", backupPath, err) + if outputErr := writeFormattedOutput(stdout, outputPath, format, result, writeCleanupPurgeText); outputErr != nil { + return fmt.Errorf("%v; write completed purge results: %w", purgeErr, outputErr) + } + return purgeErr + } + if result.CompletesLifecycle { + if err := markRunPurgedLocked(run.Run); err != nil { + result.Warnings = append(result.Warnings, "source purge completed, but migration lifecycle completion was not recorded") + result.Actions = append(result.Actions, cleanupAction{Kind: "lifecycle", Ref: run.Run.Name, Safety: "metadata", Status: "error", Message: "source resources were purged, but PurgedAt could not be persisted"}) + purgeErr := fmt.Errorf("source purge completed and backup is available at %s, but migration lifecycle completion could not be recorded: %w", backupPath, err) + if backupErr := recorder.Persist(); backupErr != nil { + result.Warnings = append(result.Warnings, "private purge backup could not be updated with the completed resource results") + purgeErr = fmt.Errorf("%v; update private purge backup: %w", purgeErr, backupErr) + } + if outputErr := writeFormattedOutput(stdout, outputPath, format, result, writeCleanupPurgeText); outputErr != nil { + return fmt.Errorf("%v; write completed purge results: %w", purgeErr, outputErr) + } + return purgeErr + } + result.LifecycleCompleted = true + if err := recorder.Persist(); err != nil { + result.Warnings = append(result.Warnings, "migration lifecycle completion was recorded, but the private purge backup could not be updated with that outcome") + purgeErr := fmt.Errorf("migration lifecycle completion was recorded, but backup %s could not be finalized: %w", backupPath, err) + if outputErr := writeFormattedOutput(stdout, outputPath, format, result, writeCleanupPurgeText); outputErr != nil { + return fmt.Errorf("%v; write completed purge results: %w", purgeErr, outputErr) + } + return purgeErr + } + } + return writeFormattedOutput(stdout, outputPath, format, result, writeCleanupPurgeText) +} + +func planCleanupPurge(run loadedMigrationRun, target string, filters cleanupPurgeFilters, extraSourceDirs []string) (cleanupPurgeResult, error) { + selected, err := cleanupPurgeSelectedApps(run, filters) + if err != nil { + return cleanupPurgeResult{}, err + } + selectedNames := cleanupSelectedAppNames(selected) + owners, err := cleanupPurgeResourceOwners(run) + if err != nil { + return cleanupPurgeResult{}, err + } + result := cleanupPurgeResult{ + APIVersion: cleanupPurgeAPIVersion, + RunName: run.Run.Name, + RunDir: run.Run.RunDir, + Target: target, + DryRun: true, + CompletesLifecycle: cleanupPurgeCoversAllRunApps(run, filters), + Filters: filters, + } + + for _, app := range selected { + if control := cleanupSourceControlForApp(app); control != nil { + control.Action = "credentials_not_touched_by_source_purge" + result.SourceControls = append(result.SourceControls, *control) + } + for _, unresolved := range cleanupUnresolvedContainersForApp(app) { + result.CompletesLifecycle = false + result.Warnings = append(result.Warnings, fmt.Sprintf("source container for %s/%s has no stable container ID and is not scheduled for purge", app.Name, unresolved)) + } + containers, err := cleanupContainersForApp(app) + if err != nil { + return cleanupPurgeResult{}, err + } + for _, container := range containers { + if sharedWith := cleanupContainerOwnersOutsideSelected(owners, container, selectedNames); len(sharedWith) > 0 { + result.CompletesLifecycle = false + result.Warnings = append(result.Warnings, fmt.Sprintf("source container %s for %s is also referenced by unselected app(s): %s", firstCleanupValue(container.ContainerName, container.ContainerID), app.Name, strings.Join(sharedWith, ", "))) + continue + } + container.Action = "remove_on_purge_apply" + result.SourceContainers = append(result.SourceContainers, container) + } + for _, volume := range cleanupVolumesForApp(app) { + switch volume.Type { + case "volume": + if strings.TrimSpace(volume.Name) == "" { + result.CompletesLifecycle = false + volume.Action = "inspect_manually_before_purge" + result.Warnings = append(result.Warnings, fmt.Sprintf("named volume for %s/%s has no recorded name and is not scheduled for purge", app.Name, volume.Service)) + } else if sharedWith := cleanupOwnersOutsideSelected(owners.NamedVolumes[volume.Name], selectedNames); len(sharedWith) > 0 { + result.CompletesLifecycle = false + volume.Action = "preserve_shared_with_unselected_app" + result.Warnings = append(result.Warnings, fmt.Sprintf("named volume %s for %s is also referenced by unselected app(s): %s", volume.Name, app.Name, strings.Join(sharedWith, ", "))) + } else { + volume.Action = "require_absent_before_purge_apply" + result.Warnings = append(result.Warnings, fmt.Sprintf("named volume %s for %s must be removed manually before cleanup purge --apply; Bort will verify it remains absent", volume.Name, app.Name)) + } + case "bind": + volume.Action = "require_absent_before_purge_apply" + path, warning := cleanupSourcePathForBindMount(volume, filters.IncludePlatform) + if warning != "" { + result.CompletesLifecycle = false + result.Warnings = append(result.Warnings, warning) + } else if path == nil { + result.CompletesLifecycle = false + volume.Action = "inspect_manually_before_purge" + } else if sharedWith := cleanupPathOwnersOutsideSelected(owners.BindPaths, path.Path, selectedNames); len(sharedWith) > 0 { + result.CompletesLifecycle = false + volume.Action = "preserve_shared_with_unselected_app" + result.Warnings = append(result.Warnings, fmt.Sprintf("bind mount source %s for %s is also referenced by unselected app(s): %s", path.Path, app.Name, strings.Join(sharedWith, ", "))) + } else { + path.Action = "require_absent_before_purge_apply" + result.SourcePaths = append(result.SourcePaths, *path) + result.Warnings = append(result.Warnings, fmt.Sprintf("bind mount source %s for %s must be removed manually before cleanup purge --apply; Bort will verify it remains absent", path.Path, app.Name)) + } + default: + result.CompletesLifecycle = false + volume.Action = "inspect_manually_before_purge" + } + result.SourceVolumes = append(result.SourceVolumes, volume) + } + networks, err := cleanupNetworksForApp(run, app) + if err != nil { + return cleanupPurgeResult{}, fmt.Errorf("inspect source networks for %s before purge: %w", app.Name, err) + } + for _, network := range networks { + if dokploy.IsProtectedSourcePurgeNetwork(network.Name) { + result.Warnings = append(result.Warnings, fmt.Sprintf("source network %s for %s is a Docker built-in network and is not scheduled for purge", network.Name, app.Name)) + continue + } + if cleanupPlatformNetwork(network.Name) && !filters.IncludePlatform { + result.CompletesLifecycle = false + result.Warnings = append(result.Warnings, fmt.Sprintf("source network %s for %s is a platform network and requires --include-platform for purge", network.Name, app.Name)) + continue + } + if sharedWith := cleanupOwnersOutsideSelected(owners.Networks[network.Name], selectedNames); len(sharedWith) > 0 { + result.CompletesLifecycle = false + result.Warnings = append(result.Warnings, fmt.Sprintf("source network %s for %s is also referenced by unselected app(s): %s", network.Name, app.Name, strings.Join(sharedWith, ", "))) + continue + } + network.Action = "remove_on_purge_apply" + result.SourceNetworks = append(result.SourceNetworks, network) + } + } + for _, path := range extraSourceDirs { + path = strings.TrimSpace(path) + if path == "" { + continue + } + cleaned := filepath.Clean(path) + if err := dokploy.ValidateSourcePurgePath(cleaned, filters.IncludePlatform); err != nil { + return cleanupPurgeResult{}, err + } + if sharedWith := cleanupPathOwnersOutsideSelected(owners.BindPaths, cleaned, selectedNames); len(sharedWith) > 0 { + return cleanupPurgeResult{}, fmt.Errorf("refusing source-dir %s because it overlaps bind mount(s) for unselected app(s): %s", cleaned, strings.Join(sharedWith, ", ")) + } + result.SourcePaths = append(result.SourcePaths, cleanupSourcePath{Path: cleaned, Source: "explicit", AllowPlatform: filters.IncludePlatform, Action: "require_absent_before_purge_apply"}) + result.Warnings = append(result.Warnings, fmt.Sprintf("source directory %s must be removed manually before cleanup purge --apply; Bort will verify it remains absent", cleaned)) + } + if filters.AllApps && !cleanupPurgeCoversAllRunApps(run, filters) { + result.Warnings = append(result.Warnings, "--all-apps excludes platform-role apps unless --include-platform is set; this purge will not complete the migration lifecycle") + } + if len(cleanupPurgeVolumeChecks(result.SourceVolumes)) > 0 || len(result.SourcePaths) > 0 { + setCleanupPurgeManualCompletion(&result) + } + if filters.AllApps && !result.CompletesLifecycle { + result.Warnings = append(result.Warnings, "the purge plan does not cover every removable source resource; successful apply will not complete the migration lifecycle") + } + result.Warnings = append(result.Warnings, "source-control credentials and target Dokploy artifacts are not removed by cleanup purge") + result.Warnings = append(result.Warnings, "Docker images are not removed by cleanup purge; remove unused images separately after confirming they are not shared") + sortCleanupPurgeResult(&result) + addCleanupPurgeActions(&result) + return result, nil +} + +type cleanupPurgeOwners struct { + ContainerIDs map[string][]string + ContainerNames map[string][]string + NamedVolumes map[string][]string + BindPaths map[string][]string + Networks map[string][]string +} + +func cleanupSelectedAppNames(apps []preparer.AppPlan) map[string]struct{} { + names := map[string]struct{}{} + for _, app := range apps { + if strings.TrimSpace(app.Name) != "" { + names[app.Name] = struct{}{} + } + } + return names +} + +func cleanupPurgeResourceOwners(run loadedMigrationRun) (cleanupPurgeOwners, error) { + owners := cleanupPurgeOwners{ + ContainerIDs: map[string][]string{}, + ContainerNames: map[string][]string{}, + NamedVolumes: map[string][]string{}, + BindPaths: map[string][]string{}, + Networks: map[string][]string{}, + } + for _, app := range run.Prepare.Apps { + appName := strings.TrimSpace(app.Name) + if appName == "" { + continue + } + containers, err := cleanupContainersForApp(app) + if err != nil { + return cleanupPurgeOwners{}, err + } + for _, container := range containers { + if containerID := strings.TrimSpace(container.ContainerID); containerID != "" { + owners.ContainerIDs[containerID] = append(owners.ContainerIDs[containerID], appName) + } + if containerName := cleanupContainerName(container.ContainerName); containerName != "" { + owners.ContainerNames[containerName] = append(owners.ContainerNames[containerName], appName) + } + } + for _, volume := range cleanupVolumesForApp(app) { + switch volume.Type { + case "volume": + if strings.TrimSpace(volume.Name) != "" { + owners.NamedVolumes[volume.Name] = append(owners.NamedVolumes[volume.Name], appName) + } + case "bind": + if strings.TrimSpace(volume.Source) != "" { + path := filepath.Clean(volume.Source) + owners.BindPaths[path] = append(owners.BindPaths[path], appName) + } + } + } + networks, err := cleanupNetworksForApp(run, app) + if err != nil { + return cleanupPurgeOwners{}, fmt.Errorf("inspect source networks for %s before purge: %w", app.Name, err) + } + for _, network := range networks { + if strings.TrimSpace(network.Name) != "" { + owners.Networks[network.Name] = append(owners.Networks[network.Name], appName) + } + } + } + return owners, nil +} + +func cleanupContainerOwnersOutsideSelected(owners cleanupPurgeOwners, container cleanupSourceContainer, selected map[string]struct{}) []string { + containerOwners := []string{} + if containerID := strings.TrimSpace(container.ContainerID); containerID != "" { + for ownedID, ownedBy := range owners.ContainerIDs { + if cleanupContainerIDsMatch(containerID, ownedID) { + containerOwners = append(containerOwners, ownedBy...) + } + } + } + if containerName := cleanupContainerName(container.ContainerName); containerName != "" { + containerOwners = append(containerOwners, owners.ContainerNames[containerName]...) + } + return cleanupOwnersOutsideSelected(containerOwners, selected) +} + +func cleanupContainerIDsMatch(a, b string) bool { + a = strings.TrimSpace(a) + b = strings.TrimSpace(b) + if a == b { + return a != "" + } + if len(a) > len(b) { + a, b = b, a + } + return len(a) >= 12 && strings.HasPrefix(b, a) +} + +func cleanupContainerName(name string) string { + return strings.TrimPrefix(strings.TrimSpace(name), "/") +} + +func cleanupOwnersOutsideSelected(owners []string, selected map[string]struct{}) []string { + outside := []string{} + seen := map[string]struct{}{} + for _, owner := range owners { + owner = strings.TrimSpace(owner) + if owner == "" { + continue + } + if _, ok := selected[owner]; ok { + continue + } + if _, ok := seen[owner]; ok { + continue + } + seen[owner] = struct{}{} + outside = append(outside, owner) + } + sort.Strings(outside) + return outside +} + +func cleanupPathOwnersOutsideSelected(pathOwners map[string][]string, path string, selected map[string]struct{}) []string { + owners := []string{} + for ownedPath, pathOwners := range pathOwners { + if cleanupPathsOverlap(ownedPath, path) { + owners = append(owners, pathOwners...) + } + } + return cleanupOwnersOutsideSelected(owners, selected) +} + +func cleanupPathsOverlap(a, b string) bool { + a = filepath.Clean(strings.TrimSpace(a)) + b = filepath.Clean(strings.TrimSpace(b)) + if a == "" || b == "" { + return false + } + if a == b { + return true + } + return cleanupPathContains(a, b) || cleanupPathContains(b, a) +} + +func cleanupPathContains(parent, child string) bool { + rel, err := filepath.Rel(parent, child) + return err == nil && rel != "." && rel != "" && rel != ".." && !strings.HasPrefix(rel, ".."+string(os.PathSeparator)) +} + +func cleanupPurgeSelectedApps(run loadedMigrationRun, filters cleanupPurgeFilters) ([]preparer.AppPlan, error) { + appFilters := cleanupStringListValues(filters.Apps) + projectFilters := cleanupStringListValues(filters.Projects) + filterActive := len(appFilters) > 0 || len(projectFilters) > 0 + matchedApps := map[string][]string{} + matchedProjects := map[string]bool{} + selected := []preparer.AppPlan{} + for _, app := range run.Prepare.Apps { + if isPlatformRunApp(app.Role) && !filters.IncludePlatform { + continue + } + appMatch := false + projectMatch := false + for _, name := range appFilters { + if cleanupPurgeAppMatches(app, name) { + appMatch = true + matchedApps[name] = append(matchedApps[name], app.Name) + } + } + for _, name := range projectFilters { + if cleanupPurgeProjectMatches(app, name) { + projectMatch = true + matchedProjects[name] = true + } + } + if filters.AllApps || !filterActive || appMatch || projectMatch { + selected = append(selected, app) + } + } + missing := []string{} + for _, name := range appFilters { + if len(matchedApps[name]) == 0 { + missing = append(missing, "app="+name) + } + } + for _, name := range projectFilters { + if !matchedProjects[name] { + missing = append(missing, "project="+name) + } + } + if len(missing) > 0 { + return nil, fmt.Errorf("cleanup purge filter(s) matched no selected app: %s", strings.Join(missing, ", ")) + } + for _, name := range appFilters { + matches := matchedApps[name] + if len(matches) > 1 { + sort.Strings(matches) + return nil, fmt.Errorf("cleanup purge --app %q is ambiguous; it matches: %s", name, strings.Join(matches, ", ")) + } + } + return selected, nil +} + +func cleanupPurgeCoversAllRunApps(run loadedMigrationRun, filters cleanupPurgeFilters) bool { + if !filters.AllApps { + return false + } + if filters.IncludePlatform { + return true + } + for _, app := range run.Prepare.Apps { + if isPlatformRunApp(app.Role) { + return false + } + } + return true +} + +func cleanupPlatformNetwork(name string) bool { + switch strings.TrimSpace(name) { + case "coolify", "coolify-proxy", "proxy": + return true + default: + return false + } +} + +func cleanupPurgeAppMatches(app preparer.AppPlan, name string) bool { + want := strings.ToLower(strings.TrimSpace(name)) + if want == "" { + return false + } + for _, candidate := range []string{app.Name, app.Directory, planutil.Slug(app.Name)} { + if strings.ToLower(strings.TrimSpace(candidate)) == want { + return true + } + } + return false +} + +func cleanupPurgeProjectMatches(app preparer.AppPlan, name string) bool { + want := strings.ToLower(strings.TrimSpace(name)) + if want == "" { + return false + } + candidates := []string{} + if app.ProjectGroup != nil { + candidates = append(candidates, app.ProjectGroup.Name, app.ProjectGroup.Source) + } + if app.TargetResources != nil && app.TargetResources.Dokploy != nil { + candidates = append(candidates, app.TargetResources.Dokploy.Project.Name) + } + for _, candidate := range candidates { + if strings.ToLower(strings.TrimSpace(candidate)) == want { + return true + } + } + return false +} + +func cleanupSourcePathForBindMount(volume cleanupSourceVolume, allowPlatform bool) (*cleanupSourcePath, string) { + path := strings.TrimSpace(volume.Source) + if path == "" { + return nil, fmt.Sprintf("bind mount for %s/%s -> %s has no host source path; not scheduled for purge", volume.App, volume.Service, volume.Target) + } + cleaned := filepath.Clean(path) + if err := dokploy.ValidateSourcePurgePath(cleaned, allowPlatform); err != nil { + return nil, fmt.Sprintf("bind mount source %s for %s/%s not scheduled for purge: %v", cleaned, volume.App, volume.Service, err) + } + return &cleanupSourcePath{App: volume.App, Path: cleaned, Source: "bind_mount", AllowPlatform: allowPlatform, Action: "require_absent_before_purge_apply"}, "" +} + +func validateCleanupPurgeApplyScope(filters cleanupPurgeFilters) error { + if filters.AllApps || len(filters.Apps) > 0 || len(filters.Projects) > 0 { + return nil + } + return fmt.Errorf("cleanup purge --apply requires an explicit scope: pass --app, --project, or --all-apps") +} + +func confirmCleanupPurgeApply(stdin io.Reader, stderr io.Writer, runName, provided string) error { + expected := cleanupPurgeConfirmPhrase(runName) + provided = strings.TrimSpace(provided) + if provided == expected { + return nil + } + if provided != "" { + return fmt.Errorf("cleanup purge --apply confirmation mismatch: pass --confirm %q", expected) + } + if stdin != nil && stdinIsTerminal(stdin) { + fmt.Fprintf(stderr, "cleanup purge is destructive. Type %q to continue: ", expected) + line, err := bufio.NewReader(stdin).ReadString('\n') + if err != nil && strings.TrimSpace(line) == "" { + return err + } + if strings.TrimSpace(line) == expected { + return nil + } + } + return fmt.Errorf("cleanup purge --apply requires --confirm %q", expected) +} + +func cleanupPurgeConfirmPhrase(runName string) string { + return "purge " + strings.TrimSpace(runName) +} + +func writeCleanupPurgeBackup(backupDir string, result cleanupPurgeResult) (string, error) { + backupDir = strings.TrimSpace(backupDir) + if backupDir == "" { + backupDir = filepath.Join(".bort", "backups") + } + if err := os.MkdirAll(backupDir, 0o700); err != nil { + return "", err + } + if err := os.Chmod(backupDir, 0o700); err != nil { + return "", err + } + path := filepath.Join(backupDir, fmt.Sprintf("source-purge-%s-%s.json", cleanupBackupFilePart(result.RunName), time.Now().UTC().Format("20060102-150405.000000000"))) + return path, updateCleanupPurgeBackup(path, result) +} + +func updateCleanupPurgeBackup(path string, result cleanupPurgeResult) error { + result.BackupPath = path + return writeJSONArtifact(path, result) +} + +type cleanupPurgeBackupRecorder struct { + path string + result *cleanupPurgeResult + persist func(string, cleanupPurgeResult) error + err error +} + +func (r *cleanupPurgeBackupRecorder) Start() error { + now := time.Now().UTC() + r.result.ExecutionStartedAt = &now + return r.Persist() +} + +func (r *cleanupPurgeBackupRecorder) Record(progress dokploy.SourcePurgeResult) error { + r.result.PurgeResult = &progress + return r.Persist() +} + +func (r *cleanupPurgeBackupRecorder) Persist() error { + if r.err != nil { + return r.err + } + if err := r.persist(r.path, *r.result); err != nil { + r.err = err + return err + } + return nil +} + +func cleanupBackupFilePart(value string) string { + value = strings.TrimSpace(value) + if value == "" { + value = "run" + } + var builder strings.Builder + for _, r := range value { + switch { + case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9', r == '-', r == '_': + builder.WriteRune(r) + default: + builder.WriteByte('-') + } + } + part := strings.Trim(builder.String(), "-") + if part == "" { + return "run" + } + return part +} + +func cleanupPurgeOptions(result cleanupPurgeResult) dokploy.SourcePurgeOptions { + opts := dokploy.SourcePurgeOptions{} + for _, container := range result.SourceContainers { + opts.Containers = append(opts.Containers, dokploy.SourcePurgeContainer{ + App: container.App, + Service: container.Service, + ContainerID: container.ContainerID, + ContainerName: container.ContainerName, + }) + } + for _, volume := range cleanupPurgeVolumeChecks(result.SourceVolumes) { + opts.Volumes = append(opts.Volumes, dokploy.SourcePurgeVolume{App: volume.App, Service: volume.Service, Name: volume.Name, ExpectedAbsent: volume.ExpectedAbsent}) + } + for _, network := range result.SourceNetworks { + opts.Networks = append(opts.Networks, dokploy.SourcePurgeNetwork{App: network.App, Name: network.Name, ExpectedIdentity: network.ExpectedIdentity, ExpectedAbsent: network.ExpectedAbsent}) + } + for _, path := range result.SourcePaths { + opts.Paths = append(opts.Paths, dokploy.SourcePurgePath{App: path.App, Source: path.Source, Path: path.Path, AllowPlatform: path.AllowPlatform, ExpectedAbsent: path.ExpectedAbsent}) + } + return opts +} + +func applyCleanupPurgeIdentities(result *cleanupPurgeResult, identified dokploy.SourcePurgeOptions) { + result.SourceContainers = nil + for _, container := range identified.Containers { + result.SourceContainers = append(result.SourceContainers, cleanupSourceContainer{ + App: container.App, + Service: container.Service, + ContainerID: container.ContainerID, + ContainerName: container.ContainerName, + Action: "remove_on_purge_apply", + }) + } + volumes := map[string]dokploy.SourcePurgeVolume{} + for _, volume := range identified.Volumes { + volumes[volume.Name] = volume + } + for i := range result.SourceVolumes { + if volume, ok := volumes[result.SourceVolumes[i].Name]; ok { + result.SourceVolumes[i].ExpectedAbsent = volume.ExpectedAbsent + } + } + networks := map[string]dokploy.SourcePurgeNetwork{} + for _, network := range identified.Networks { + networks[network.Name] = network + } + for i := range result.SourceNetworks { + if network, ok := networks[result.SourceNetworks[i].Name]; ok { + result.SourceNetworks[i].ExpectedIdentity = network.ExpectedIdentity + result.SourceNetworks[i].ExpectedAbsent = network.ExpectedAbsent + } + } + paths := map[string]dokploy.SourcePurgePath{} + for _, path := range identified.Paths { + paths[path.Path] = path + } + for i := range result.SourcePaths { + if path, ok := paths[result.SourcePaths[i].Path]; ok { + result.SourcePaths[i].ExpectedAbsent = path.ExpectedAbsent + } + } + manualCompletion := len(identified.Volumes) > 0 || len(identified.Paths) > 0 + for _, network := range identified.Networks { + manualCompletion = manualCompletion || network.ExpectedAbsent + } + if manualCompletion { + setCleanupPurgeManualCompletion(result) + } +} + +func setCleanupPurgeManualCompletion(result *cleanupPurgeResult) { + result.ManualCompletion = true + result.CompletesLifecycle = false + for i := range result.SourceContainers { + result.SourceContainers[i].Action = "require_absent_before_purge_apply" } - if latest, ok := latestRunRef(); ok { - return loadMigrationRun(latest) + for i := range result.SourceNetworks { + result.SourceNetworks[i].Action = "require_absent_before_purge_apply" + } + for i := range result.Actions { + switch result.Actions[i].Kind { + case "source_containers", "source_networks": + result.Actions[i].Safety = "manual_removal_required" + result.Actions[i].Status = "prerequisite" + result.Actions[i].Message = "selected resources must be removed manually because this scope contains absence-only prerequisites" + } + } + warning := "this purge has absence-only prerequisites, so Bort will not automatically delete any selected resource in this scope; remove the listed containers and networks manually too, then rerun apply to verify everything remains absent. Because external recreation cannot be locked out, verification-only purges do not record lifecycle completion" + for _, existing := range result.Warnings { + if existing == warning { + return + } + } + result.Warnings = append(result.Warnings, warning) +} + +func cleanupPurgeVolumeChecks(volumes []cleanupSourceVolume) []cleanupSourceVolume { + named := []cleanupSourceVolume{} + for _, volume := range volumes { + if volume.Type == "volume" && strings.TrimSpace(volume.Name) != "" && volume.Action == "require_absent_before_purge_apply" { + named = append(named, volume) + } + } + return named +} + +func addCleanupPurgeActions(result *cleanupPurgeResult) { + if len(result.SourceContainers) > 0 { + action := cleanupAction{Kind: "source_containers", Ref: "run", Safety: "destructive_confirm_required", Status: "planned", Message: "selected source containers will be removed only by cleanup purge --apply"} + if result.ManualCompletion { + action.Safety = "manual_removal_required" + action.Status = "prerequisite" + action.Message = "selected source containers must be removed manually because this scope contains absence-only prerequisites" + } + result.Actions = append(result.Actions, action) + } + if len(cleanupPurgeVolumeChecks(result.SourceVolumes)) > 0 { + result.Actions = append(result.Actions, cleanupAction{Kind: "source_named_volumes", Ref: "run", Safety: "manual_removal_required", Status: "prerequisite", Message: "selected source named volumes must be absent before cleanup purge --apply and are rechecked after confirmation"}) + } + if len(result.SourcePaths) > 0 { + result.Actions = append(result.Actions, cleanupAction{Kind: "source_paths", Ref: "run", Safety: "manual_removal_required", Status: "prerequisite", Message: "selected host source paths must be absent before cleanup purge --apply and are rechecked after confirmation"}) + } + if len(result.SourceNetworks) > 0 { + action := cleanupAction{Kind: "source_networks", Ref: "run", Safety: "destructive_confirm_required", Status: "planned", Message: "selected source networks will be removed only by cleanup purge --apply"} + if result.ManualCompletion { + action.Safety = "manual_removal_required" + action.Status = "prerequisite" + action.Message = "selected source networks must be removed manually because this scope contains absence-only prerequisites" + } + result.Actions = append(result.Actions, action) + } + if len(result.SourceControls) > 0 { + result.Actions = append(result.Actions, cleanupAction{Kind: "source_control", Ref: "run", Safety: "credentials_not_touched", Status: "preserved", Message: "source-control credentials are not copied, revoked, or removed by cleanup purge"}) + } + result.Actions = append(result.Actions, cleanupAction{Kind: "target_artifacts", Ref: "run", Safety: "preserve_target", Status: "preserved", Message: "Dokploy target projects, compose apps, domains, and volumes are not removed by cleanup purge"}) +} + +func sortCleanupPurgeResult(result *cleanupPurgeResult) { + sort.Slice(result.SourceControls, func(i, j int) bool { + return cleanupSortKey(result.SourceControls[i].App, result.SourceControls[i].Repository) < cleanupSortKey(result.SourceControls[j].App, result.SourceControls[j].Repository) + }) + sort.Slice(result.SourceContainers, func(i, j int) bool { + return cleanupSortKey(result.SourceContainers[i].App, result.SourceContainers[i].Service, result.SourceContainers[i].ContainerName) < cleanupSortKey(result.SourceContainers[j].App, result.SourceContainers[j].Service, result.SourceContainers[j].ContainerName) + }) + sort.Slice(result.SourceVolumes, func(i, j int) bool { + return cleanupSortKey(result.SourceVolumes[i].App, result.SourceVolumes[i].Service, result.SourceVolumes[i].Target) < cleanupSortKey(result.SourceVolumes[j].App, result.SourceVolumes[j].Service, result.SourceVolumes[j].Target) + }) + sort.Slice(result.SourceNetworks, func(i, j int) bool { + return cleanupSortKey(result.SourceNetworks[i].App, result.SourceNetworks[i].Name) < cleanupSortKey(result.SourceNetworks[j].App, result.SourceNetworks[j].Name) + }) + sort.Slice(result.SourcePaths, func(i, j int) bool { + return cleanupSortKey(result.SourcePaths[i].App, result.SourcePaths[i].Path) < cleanupSortKey(result.SourcePaths[j].App, result.SourcePaths[j].Path) + }) + sort.Strings(result.Warnings) +} + +func cleanupPurgeScopeLabel(filters cleanupPurgeFilters) string { + if filters.AllApps { + return "all-apps" + } + parts := []string{} + for _, app := range filters.Apps { + parts = append(parts, "app="+app) + } + for _, project := range filters.Projects { + parts = append(parts, "project="+project) + } + if len(parts) == 0 { + return "run" + } + sort.Strings(parts) + return strings.Join(parts, ",") +} + +func (f *stringListFlag) Set(value string) error { + for _, item := range strings.Split(value, ",") { + item = strings.TrimSpace(item) + if item != "" { + *f = append(*f, item) + } + } + return nil +} + +func (f *stringListFlag) String() string { + if f == nil { + return "" + } + return strings.Join(*f, ",") +} + +func cleanupStringListValues(values []string) []string { + seen := map[string]struct{}{} + cleaned := []string{} + for _, value := range values { + for _, item := range strings.Split(value, ",") { + item = strings.TrimSpace(item) + if item == "" { + continue + } + key := strings.ToLower(item) + if _, ok := seen[key]; ok { + continue + } + seen[key] = struct{}{} + cleaned = append(cleaned, item) + } + } + sort.Strings(cleaned) + return cleaned +} + +func loadCleanupRun(runRef string, allowLatest bool) (loadedMigrationRun, error) { + resolved, err := resolveRunRef(runRef, allowLatest) + if err != nil { + return loadedMigrationRun{}, err } - return loadedMigrationRun{}, fmt.Errorf("no migration run found; run `bort` or pass --run before cleanup") + return loadMigrationRun(resolved) } func cleanupStaleProjectNameCollisions(run loadedMigrationRun, names []string) []string { @@ -259,9 +1245,19 @@ func planCleanup(ctx context.Context, run loadedMigrationRun, target string) cle if control := cleanupSourceControlForApp(app); control != nil { result.SourceControls = append(result.SourceControls, *control) } - result.SourceContainers = append(result.SourceContainers, cleanupContainersForApp(app)...) + containers, err := cleanupContainersForApp(app) + if err != nil { + result.Warnings = append(result.Warnings, err.Error()) + } else { + result.SourceContainers = append(result.SourceContainers, containers...) + } result.SourceVolumes = append(result.SourceVolumes, cleanupVolumesForApp(app)...) - result.SourceNetworks = append(result.SourceNetworks, cleanupNetworksForApp(run, app)...) + networks, err := cleanupNetworksForApp(run, app) + if err != nil { + result.Warnings = append(result.Warnings, fmt.Sprintf("inspect source networks for %s: %v", app.Name, err)) + } else { + result.SourceNetworks = append(result.SourceNetworks, networks...) + } result.TargetArtifacts = append(result.TargetArtifacts, cleanupTargetArtifactsForApp(app)...) } sortCleanupResult(&result) @@ -362,27 +1358,125 @@ func cleanupSourceControlForApp(app preparer.AppPlan) *cleanupSourceControl { } } -func cleanupContainersForApp(app preparer.AppPlan) []cleanupSourceContainer { +func cleanupContainersForApp(app preparer.AppPlan) ([]cleanupSourceContainer, error) { containers := []cleanupSourceContainer{} - seen := map[string]struct{}{} - for _, service := range app.Resources.SourceServices { - ref := service.ContainerID + "/" + service.ContainerName - if ref == "/" { - continue + seenIDs := map[string]int{} + seenNames := map[string]int{} + idsByName := map[string]string{} + add := func(service, containerID, containerName string) error { + service = strings.TrimSpace(service) + containerID = strings.TrimSpace(containerID) + containerName = cleanupContainerName(containerName) + if containerID == "" && containerName == "" { + return nil } - if _, ok := seen[ref]; ok { - continue + if containerID != "" { + if existingID, ok := idsByName[containerName]; containerName != "" && ok { + if !cleanupContainerIDsMatch(existingID, containerID) { + return fmt.Errorf("source container name %q for app %s refers to conflicting container IDs %q and %q", containerName, app.Name, existingID, containerID) + } + if len(existingID) > len(containerID) { + containerID = existingID + } + } + if containerName != "" { + idsByName[containerName] = containerID + } + index, ok := seenIDs[containerID] + matchedID := containerID + if !ok { + for seenID, seenIndex := range seenIDs { + if cleanupContainerIDsMatch(seenID, containerID) { + index, ok, matchedID = seenIndex, true, seenID + break + } + } + } + if ok { + existing := containers[index] + if existing.ContainerName != "" && containerName != "" && existing.ContainerName != containerName { + return fmt.Errorf("source container ID %q for app %s refers to conflicting names %q and %q", containerID, app.Name, existing.ContainerName, containerName) + } + if existing.Service != "" && service != "" && existing.Service != service { + return fmt.Errorf("source container ID %q for app %s refers to conflicting services %q and %q", containerID, app.Name, existing.Service, service) + } + if existing.ContainerName == "" { + existing.ContainerName = containerName + } + if existing.Service == "" { + existing.Service = service + } + if len(containerID) > len(existing.ContainerID) { + existing.ContainerID = containerID + delete(seenIDs, matchedID) + seenIDs[containerID] = index + } + if existing.ContainerName != "" { + idsByName[existing.ContainerName] = existing.ContainerID + } + containers[index] = existing + return nil + } + } else { + if index, ok := seenNames[containerName]; ok { + existing := containers[index] + if existing.Service != "" && service != "" && existing.Service != service { + return fmt.Errorf("source container name %q for app %s refers to conflicting services %q and %q", containerName, app.Name, existing.Service, service) + } + return nil + } } - seen[ref] = struct{}{} + index := len(containers) containers = append(containers, cleanupSourceContainer{ App: app.Name, - Service: service.ServiceName, - ContainerID: service.ContainerID, - ContainerName: service.ContainerName, + Service: service, + ContainerID: containerID, + ContainerName: containerName, Action: "inventory_only_cleanup_apply_does_not_remove", }) + if containerID != "" { + seenIDs[containerID] = index + } else { + seenNames[containerName] = index + } + return nil + } + for _, service := range app.Resources.SourceServices { + if err := add(service.ServiceName, service.ContainerID, service.ContainerName); err != nil { + return nil, err + } + } + for _, volume := range app.Resources.Volumes { + if err := add(volume.Service, volume.SourceContainerID, volume.SourceContainerName); err != nil { + return nil, err + } + } + for _, store := range app.Resources.DataStores { + if err := add(store.Service, store.SourceContainerID, store.SourceContainerName); err != nil { + return nil, err + } + } + return containers, nil +} + +func cleanupUnresolvedContainersForApp(app preparer.AppPlan) []string { + unresolved := []string{} + add := func(service, containerID, containerName string) { + if strings.TrimSpace(containerID) == "" { + unresolved = append(unresolved, firstCleanupValue(service, "unknown-service")) + } + } + for _, service := range app.Resources.SourceServices { + add(service.ServiceName, service.ContainerID, service.ContainerName) + } + for _, volume := range app.Resources.Volumes { + add(volume.Service, volume.SourceContainerID, volume.SourceContainerName) + } + for _, store := range app.Resources.DataStores { + add(store.Service, store.SourceContainerID, store.SourceContainerName) } - return containers + sort.Strings(unresolved) + return unresolved } func cleanupVolumesForApp(app preparer.AppPlan) []cleanupSourceVolume { @@ -401,10 +1495,13 @@ func cleanupVolumesForApp(app preparer.AppPlan) []cleanupSourceVolume { return volumes } -func cleanupNetworksForApp(run loadedMigrationRun, app preparer.AppPlan) []cleanupSourceNetwork { - topology := cleanupReadTopology(run, app) +func cleanupNetworksForApp(run loadedMigrationRun, app preparer.AppPlan) ([]cleanupSourceNetwork, error) { + topology, err := cleanupReadTopology(run, app) + if err != nil { + return nil, err + } if len(topology.Networks) == 0 { - return nil + return nil, nil } networks := make([]cleanupSourceNetwork, 0, len(topology.Networks)) for _, network := range topology.Networks { @@ -413,28 +1510,28 @@ func cleanupNetworksForApp(run loadedMigrationRun, app preparer.AppPlan) []clean } networks = append(networks, cleanupSourceNetwork{App: app.Name, Name: network, Action: "preserve_until_manual_purge_after_acceptance"}) } - return networks + return networks, nil } -func cleanupReadTopology(run loadedMigrationRun, app preparer.AppPlan) analyzer.Topology { +func cleanupReadTopology(run loadedMigrationRun, app preparer.AppPlan) (analyzer.Topology, error) { bundleDir := run.Prepare.BundleDir appDir := filepath.Join(bundleDir, filepath.FromSlash(app.Directory)) if err := safepath.ContainedPath(bundleDir, appDir); err != nil { - return analyzer.Topology{} + return analyzer.Topology{}, err } path := filepath.Join(appDir, "topology.json") if err := safepath.ContainedPath(appDir, path); err != nil { - return analyzer.Topology{} + return analyzer.Topology{}, err } contents, err := safepath.ReadFileNoFollow(path) if err != nil { - return analyzer.Topology{} + return analyzer.Topology{}, err } var topology analyzer.Topology if err := json.Unmarshal(contents, &topology); err != nil { - return analyzer.Topology{} + return analyzer.Topology{}, err } - return topology + return topology, nil } func cleanupTargetArtifactsForApp(app preparer.AppPlan) []cleanupTargetArtifact { @@ -487,6 +1584,167 @@ func firstCleanupValue(values ...string) string { return "unknown" } +func writeCleanupPurgeText(w io.Writer, result cleanupPurgeResult) { + mode := "dry run" + if !result.DryRun { + mode = "incomplete" + } + if result.Applied { + mode = "applied" + } + fmt.Fprintf(w, "Cleanup purge plan: %s -> %s\n", firstCleanupValue(result.RunDir, result.RunName, "run"), result.Target) + fmt.Fprintf(w, "Mode: %s\n", mode) + fmt.Fprintf(w, "Scope: %s\n\n", cleanupPurgeScopeLabel(result.Filters)) + + if result.BackupPath != "" { + fmt.Fprintf(w, "Purge plan backup: %s\n\n", result.BackupPath) + } + + if len(result.SourceContainers) > 0 { + heading := "Source containers scheduled for purge:" + if result.ManualCompletion { + heading = "Source containers that must be absent before apply:" + } + fmt.Fprintln(w, heading) + for _, container := range result.SourceContainers { + fmt.Fprintf(w, " - %s/%s %s\n", container.App, container.Service, firstCleanupValue(container.ContainerName, container.ContainerID)) + } + fmt.Fprintln(w) + } + if named := cleanupPurgeVolumeChecks(result.SourceVolumes); len(named) > 0 { + fmt.Fprintln(w, "Named source volumes that must be absent before apply:") + for _, volume := range named { + fmt.Fprintf(w, " - %s %s -> %s\n", volume.App, volume.Name, volume.Target) + } + fmt.Fprintln(w) + } + if len(result.SourcePaths) > 0 { + fmt.Fprintln(w, "Host source paths that must be absent before apply:") + for _, path := range result.SourcePaths { + app := firstCleanupValue(path.App, "explicit") + fmt.Fprintf(w, " - %s %s (%s)\n", app, path.Path, path.Source) + } + fmt.Fprintln(w) + } + if len(result.SourceNetworks) > 0 { + heading := "Source networks scheduled for purge:" + if result.ManualCompletion { + heading = "Source networks that must be absent before apply:" + } + fmt.Fprintln(w, heading) + for _, network := range result.SourceNetworks { + fmt.Fprintf(w, " - %s %s\n", network.App, network.Name) + } + fmt.Fprintln(w) + } + if len(result.SourceControls) > 0 { + fmt.Fprintln(w, "Source-control credentials left untouched:") + for _, control := range result.SourceControls { + fmt.Fprintf(w, " - %s %s (%s/%s)\n", control.App, firstCleanupValue(control.Repository, "repository"), firstCleanupValue(control.Provider, "git"), firstCleanupValue(control.Auth, "unknown-auth")) + } + fmt.Fprintln(w) + } + + if result.PurgeResult != nil { + writeCleanupPurgeAppliedResults(w, *result.PurgeResult) + } + + if len(result.Warnings) > 0 { + fmt.Fprintln(w, "Warnings:") + for _, warning := range result.Warnings { + fmt.Fprintf(w, " - %s\n", warning) + } + fmt.Fprintln(w) + } + + if result.Applied { + if result.ManualCompletion { + if result.LifecycleCompleted { + fmt.Fprintln(w, "Verified the manually completed source purge and completed the migration lifecycle. Target Dokploy artifacts and source-control credentials were not removed.") + } else { + fmt.Fprintln(w, "Verified the manually completed source purge for the selected scope. The migration lifecycle remains incomplete; target Dokploy artifacts and source-control credentials were not removed.") + } + return + } + if result.LifecycleCompleted { + fmt.Fprintln(w, "Applied destructive source purge and completed the migration lifecycle. Target Dokploy artifacts and source-control credentials were not removed.") + } else { + fmt.Fprintln(w, "Applied destructive source purge for the selected scope. The migration lifecycle remains incomplete; target Dokploy artifacts and source-control credentials were not removed.") + } + return + } + if !result.DryRun { + fmt.Fprintln(w, "Source purge stopped before completion. Earlier resources may already have been removed; inspect the results and backup before retrying.") + return + } + command := cleanupPurgeApplyCommand(result) + if command == "" { + fmt.Fprintln(w, "Dry run only: rerun with an explicit --app, --project, or --all-apps scope before applying a purge.") + return + } + action := "remove the reviewed source-resource scope" + if result.ManualCompletion { + action = "verify the manually removed source-resource scope remains absent" + } + fmt.Fprintf(w, "Dry run only: run `%s` to %s.\n", command, action) + fmt.Fprintln(w, "Use --project or --all-apps instead of --app when that is the intended explicit scope.") +} + +func cleanupPurgeApplyCommand(result cleanupPurgeResult) string { + args := []string{"cleanup", "purge", "--apply"} + filters := result.Filters + if !filters.AllApps && len(filters.Apps) == 0 && len(filters.Projects) == 0 { + return "" + } + if filters.AllApps { + args = append(args, "--all-apps") + } else { + for _, app := range cleanupStringListValues(filters.Apps) { + args = append(args, "--app", shellQuote(app)) + } + for _, project := range cleanupStringListValues(filters.Projects) { + args = append(args, "--project", shellQuote(project)) + } + } + if filters.IncludePlatform { + args = append(args, "--include-platform") + } + if strings.TrimSpace(result.BackupDir) != "" { + args = append(args, "--backup-dir", shellQuote(result.BackupDir)) + } + for _, path := range result.SourcePaths { + if path.Source == "explicit" { + args = append(args, "--source-dir", shellQuote(path.Path)) + } + } + args = append(args, "--confirm", shellQuote(cleanupPurgeConfirmPhrase(result.RunName))) + run := loadedMigrationRun{Run: migrationRun{Name: result.RunName, RunDir: result.RunDir}} + return runScopedCommand(run, strings.Join(args, " ")) +} + +func writeCleanupPurgeAppliedResults(w io.Writer, result dokploy.SourcePurgeResult) { + fmt.Fprintln(w, "Purge results:") + writeCleanupPurgeResourceResults(w, "containers", result.Containers) + writeCleanupPurgeResourceResults(w, "volumes", result.Volumes) + writeCleanupPurgeResourceResults(w, "networks", result.Networks) + writeCleanupPurgeResourceResults(w, "paths", result.Paths) + fmt.Fprintln(w) +} + +func writeCleanupPurgeResourceResults(w io.Writer, label string, items []dokploy.SourcePurgeResourceResult) { + if len(items) == 0 { + return + } + fmt.Fprintf(w, " %s:\n", label) + for _, item := range items { + message := item.Message + if message != "" { + message = ": " + message + } + fmt.Fprintf(w, " [%s] %s%s\n", item.Status, item.Ref, message) + } +} + func writeCleanupText(w io.Writer, result cleanupResult) { mode := "dry run" if result.Applied { @@ -570,6 +1828,7 @@ func writeCleanupText(w io.Writer, result cleanupResult) { fmt.Fprintln(w, "Applied only the safe Dokploy metadata cleanup. Source containers, volumes, networks, source-control credentials, and target apps were not removed.") return } - fmt.Fprintln(w, "Dry run only: run `bort cleanup --apply` to remove only stale zero-domain Dokploy platform metadata after a DB backup.") + run := loadedMigrationRun{Run: migrationRun{Name: result.RunName, RunDir: result.RunDir}} + fmt.Fprintf(w, "Dry run only: run `%s` to remove only stale zero-domain Dokploy platform metadata after a DB backup.\n", runScopedCommand(run, "cleanup --apply")) fmt.Fprintln(w, "Source containers, volumes, networks, source-control credentials, and target apps are inventoried only and are preserved by this command.") } diff --git a/internal/cli/cleanup_test.go b/internal/cli/cleanup_test.go index 297b632..9bdbe1d 100644 --- a/internal/cli/cleanup_test.go +++ b/internal/cli/cleanup_test.go @@ -4,17 +4,153 @@ import ( "bytes" "context" "encoding/json" + "errors" + "io" "net/http" "net/http/httptest" + "os" "path/filepath" "strings" "testing" + "time" "github.com/aikins01/bort/internal/manifest" "github.com/aikins01/bort/internal/preparer" "github.com/aikins01/bort/internal/target/dokploy" ) +func TestLifecycleAcceptanceTraceWithFakeDokploy(t *testing.T) { + composeCreated := false + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && r.URL.Path == "/api/project.all": + _ = json.NewEncoder(w).Encode([]dokploy.Project{{ + ProjectID: "project-api", + Name: "api", + Environments: []dokploy.ProjectEnvironment{{ + EnvironmentID: "environment-production", + Name: "production", + }}, + }}) + case r.Method == http.MethodGet && r.URL.Path == "/api/compose.search": + items := []map[string]string{} + if composeCreated { + items = append(items, map[string]string{"composeId": "compose-api", "name": "api", "appName": "compose-api", "environmentId": "environment-production"}) + } + _ = json.NewEncoder(w).Encode(map[string]any{"items": items, "total": len(items)}) + case r.Method == http.MethodPost && r.URL.Path == "/api/compose.create": + composeCreated = true + _ = json.NewEncoder(w).Encode(dokploy.Compose{ComposeID: "compose-api", Name: "api", AppName: "compose-api", EnvironmentID: "environment-production"}) + case r.Method == http.MethodPost && (r.URL.Path == "/api/compose.update" || r.URL.Path == "/api/compose.deploy"): + w.WriteHeader(http.StatusOK) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + t.Setenv(dokploy.EnvBaseURL, server.URL) + t.Setenv(dokploy.EnvToken, "test-token") + + workDir := t.TempDir() + t.Chdir(workDir) + binDir := filepath.Join(workDir, "bin") + if err := os.MkdirAll(binDir, 0o700); err != nil { + t.Fatal(err) + } + dockerPath := filepath.Join(binDir, "docker") + dockerStub := "#!/bin/sh\nif [ \"$1\" = inspect ]; then\n echo 'Error: No such object' >&2\n exit 1\nfi\nif [ \"$1\" = ps ]; then\n echo dokploy-postgres\nfi\nexit 0\n" + if err := os.WriteFile(dockerPath, []byte(dockerStub), 0o700); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", binDir+string(os.PathListSeparator)+os.Getenv("PATH")) + + manifestPath := filepath.Join(workDir, "manifest.json") + if err := writeJSONArtifact(manifestPath, manifest.Manifest{ + Source: manifest.Source{Platform: "coolify-local"}, + Apps: []manifest.App{{ + Name: "api", + Services: []manifest.Service{{ + ID: "source-api", + Name: "api", + Image: "example/api:latest", + }}, + }}, + }); err != nil { + t.Fatal(err) + } + runCommand(t, runMigrate, []string{"--manifest", manifestPath, "--run", "acceptance", "--observation-window", "0", "--rollback-window", "0"}) + planned, err := loadMigrationRun("acceptance") + if err != nil { + t.Fatal(err) + } + if !strings.Contains(filepath.ToSlash(planned.Run.BundleDir), ".bort/runs/acceptance/bundle") { + t.Fatalf("expected a self-contained bundle, got %s", planned.Run.BundleDir) + } + for _, decision := range openSetupDecisions(planned) { + if err := recordReviewDecision(planned, decision, planned.Run.UpdatedAt.Add(time.Second)); err != nil { + t.Fatal(err) + } + } + + runCommand(t, runMigrate, []string{"--live", "--run", "acceptance"}) + live, err := loadMigrationRun("acceptance") + if err != nil { + t.Fatal(err) + } + if live.Run.LiveAppliedAt == nil || requireLiveApplySucceeded(live) != nil { + t.Fatalf("expected completed live lifecycle, metadata=%#v applied=%#v", live.Run, live.Applied) + } + var cockpit bytes.Buffer + writeAppFirstCockpit(&cockpit, live) + if !strings.Contains(cockpit.String(), "TARGET LIVE") { + t.Fatalf("expected target-live cockpit, got:\n%s", cockpit.String()) + } + + runCommand(t, runCommit, []string{"--apply", "--run", "acceptance"}) + committed, err := loadMigrationRun("acceptance") + if err != nil { + t.Fatal(err) + } + if committed.Run.CommittedAt == nil { + t.Fatal("expected committed lifecycle timestamp") + } + cockpit.Reset() + writeAppFirstCockpit(&cockpit, committed) + if !strings.Contains(cockpit.String(), "COMMITTED") { + t.Fatalf("expected committed cockpit, got:\n%s", cockpit.String()) + } + + var cleanupOut bytes.Buffer + if err := runCleanup(context.Background(), []string{"--run", "acceptance"}, &cleanupOut, io.Discard); err != nil { + t.Fatal(err) + } + if !strings.Contains(cleanupOut.String(), "Source containers inventoried for later purge") { + t.Fatalf("expected cleanup inventory, got:\n%s", cleanupOut.String()) + } + var purgeOut bytes.Buffer + if err := runCleanup(context.Background(), []string{"purge", "--run", "acceptance", "--all-apps"}, &purgeOut, io.Discard); err != nil { + t.Fatal(err) + } + if !strings.Contains(purgeOut.String(), "Source containers scheduled for purge") { + t.Fatalf("expected purge inventory, got:\n%s", purgeOut.String()) + } + if err := runCleanup(context.Background(), []string{"purge", "--run", "acceptance", "--apply", "--all-apps", "--confirm", "purge acceptance"}, io.Discard, io.Discard); err != nil { + t.Fatal(err) + } + completed, err := loadMigrationRun("acceptance") + if err != nil { + t.Fatal(err) + } + if completed.Run.PurgedAt == nil { + t.Fatal("expected full purge to mark the lifecycle complete") + } + cockpit.Reset() + writeAppFirstCockpit(&cockpit, completed) + if !strings.Contains(cockpit.String(), "COMPLETE") { + t.Fatalf("expected complete cockpit, got:\n%s", cockpit.String()) + } +} + func TestRunCleanupInventoriesLeftoversAndSafeMetadata(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch { @@ -81,7 +217,7 @@ func TestRunCleanupInventoriesLeftoversAndSafeMetadata(t *testing.T) { "Source networks preserved:", "api api-net", "Target artifacts kept by default:", - "Dry run only: run `bort cleanup --apply`", + "Dry run only: run `bort cleanup --apply --run cleanup-run`", } { if !strings.Contains(output, want) { t.Fatalf("expected cleanup output to contain %q, got:\n%s", want, output) @@ -92,6 +228,413 @@ func TestRunCleanupInventoriesLeftoversAndSafeMetadata(t *testing.T) { } } +func TestRunCleanupPurgeInventoriesDestructiveSourceResources(t *testing.T) { + workDir := t.TempDir() + t.Chdir(workDir) + bundleDir := filepath.Join(workDir, "bort-bundle") + writeTestBundle(t, bundleDir, manifest.Manifest{ + Source: manifest.Source{Platform: "coolify-local"}, + Apps: []manifest.App{{ + Name: "api", + Git: &manifest.GitSource{ + Repository: "https://github.com/example/api", + Branch: "main", + Provider: "github", + SourceType: "App\\Models\\GithubApp", + SourceID: "42", + }, + Services: []manifest.Service{{ + ID: "cid123", + Name: "web", + Image: "example/api:latest", + Mounts: []manifest.Mount{ + {Type: "volume", Name: "api-data", Target: "/data"}, + {Type: "bind", Source: "/data/coolify/applications/app-1/storage", Target: "/storage"}, + }, + Networks: []manifest.ServiceNetwork{{Name: "api-net"}}, + }}, + }}, + }) + runCommand(t, runMigrate, []string{"--bundle", bundleDir, "--run", "purge-run", "--observation-window", "0", "--rollback-window", "0"}) + + var stdout bytes.Buffer + var stderr bytes.Buffer + if err := runCleanup(context.Background(), []string{"purge", "--run", "purge-run", "--source-dir", "/data/coolify/services/service-1"}, &stdout, &stderr); err != nil { + t.Fatalf("cleanup purge failed: %v\nstderr:\n%s", err, stderr.String()) + } + + output := stdout.String() + for _, want := range []string{ + "Cleanup purge plan: .bort/runs/purge-run -> dokploy", + "Mode: dry run", + "Source containers that must be absent before apply:", + "api/web web", + "Named source volumes that must be absent before apply:", + "api api-data -> /data", + "Host source paths that must be absent before apply:", + "api /data/coolify/applications/app-1/storage (bind_mount)", + "explicit /data/coolify/services/service-1 (explicit)", + "Source networks that must be absent before apply:", + "api api-net", + "Source-control credentials left untouched:", + "Bort will not automatically delete any selected resource in this scope", + "Dry run only: rerun with an explicit --app, --project, or --all-apps scope before applying a purge.", + } { + if !strings.Contains(output, want) { + t.Fatalf("expected cleanup purge output to contain %q, got:\n%s", want, output) + } + } + if strings.Contains(output, "Applied destructive source purge") { + t.Fatalf("cleanup purge dry run claimed destructive removal:\n%s", output) + } +} + +func TestRunCleanupPurgeApplyRequiresExplicitScopeAndConfirmation(t *testing.T) { + workDir := t.TempDir() + t.Chdir(workDir) + bundleDir := filepath.Join(workDir, "bort-bundle") + writeTestBundle(t, bundleDir, manifest.Manifest{ + Source: manifest.Source{Platform: "coolify-local"}, + Apps: []manifest.App{{ + Name: "api", + Services: []manifest.Service{{ + ID: "cid123", + Name: "web", + Image: "example/api:latest", + Mounts: []manifest.Mount{{Type: "volume", Name: "api-data", Target: "/data"}}, + }}, + }}, + }) + runCommand(t, runMigrate, []string{"--bundle", bundleDir, "--run", "purge-run", "--observation-window", "0", "--rollback-window", "0"}) + + var stdout bytes.Buffer + var stderr bytes.Buffer + err := runCleanup(context.Background(), []string{"purge", "--run", "purge-run", "--apply", "--confirm", "purge purge-run"}, &stdout, &stderr) + if err == nil || !strings.Contains(err.Error(), "requires an explicit scope") { + t.Fatalf("expected explicit scope error, got err=%v stdout=%s stderr=%s", err, stdout.String(), stderr.String()) + } + + stdout.Reset() + stderr.Reset() + err = runCleanup(context.Background(), []string{"purge", "--run", "purge-run", "--apply", "--app", "api"}, &stdout, &stderr) + if err == nil || !strings.Contains(err.Error(), "before a successful live apply") { + t.Fatalf("expected lifecycle eligibility before confirmation, got err=%v stdout=%s stderr=%s", err, stdout.String(), stderr.String()) + } + run, err := loadMigrationRun("purge-run") + if err != nil { + t.Fatal(err) + } + steps := dokploy.PlanFromArtifacts(run.Prepare, run.Sync, run.Cutover).Steps + applied := newRunApplied(run.Run) + succeededAt := time.Now().UTC() + applied.SucceededAt = &succeededAt + for index, step := range steps { + applied.Steps = append(applied.Steps, appliedStep{Index: index, Kind: string(step.Kind), App: step.App, Ref: step.Ref, Status: string(dokploy.StepStatusOK)}) + } + appliedPath, err := safeRunArtifactPath(run.Run.RunDir, run.Run.Artifacts.Applied) + if err != nil { + t.Fatal(err) + } + if err := writeRunApplied(appliedPath, applied); err != nil { + t.Fatal(err) + } + + stdout.Reset() + stderr.Reset() + err = runCleanup(context.Background(), []string{"purge", "--run", "purge-run", "--apply", "--app", "api"}, &stdout, &stderr) + if err == nil || !strings.Contains(err.Error(), "before `bort commit --apply --run purge-run`") { + t.Fatalf("expected target acceptance before confirmation, got err=%v stdout=%s stderr=%s", err, stdout.String(), stderr.String()) + } + + operationLock, err := acquireRunOperationLock(run.Run.RunDir) + if err != nil { + t.Fatal(err) + } + markErr := markRunCommittedLocked(run.Run) + operationLock.Release() + if markErr != nil { + t.Fatal(markErr) + } + stdout.Reset() + stderr.Reset() + err = runCleanup(context.Background(), []string{"purge", "--run", "purge-run", "--apply", "--app", "api"}, &stdout, &stderr) + if err == nil || !strings.Contains(err.Error(), "--confirm \"purge purge-run\"") { + t.Fatalf("expected confirmation after lifecycle eligibility, got err=%v stdout=%s stderr=%s", err, stdout.String(), stderr.String()) + } + + stdout.Reset() + stderr.Reset() + err = runCleanup(context.Background(), []string{"purge", "--run", "purge-run", "--apply", "--app", "api", "--confirm", "wrong"}, &stdout, &stderr) + if err == nil || !strings.Contains(err.Error(), "confirmation mismatch") { + t.Fatalf("expected confirmation mismatch after lifecycle eligibility, got err=%v stdout=%s stderr=%s", err, stdout.String(), stderr.String()) + } +} + +func TestRunCleanupRejectsMisplacedPurgeSubcommand(t *testing.T) { + err := runCleanup(context.Background(), []string{"--run", "prod", "--apply", "purge", "--all-apps"}, io.Discard, io.Discard) + if err == nil || !strings.Contains(err.Error(), "purge must immediately follow cleanup") { + t.Fatalf("expected misplaced purge to be rejected before metadata apply, got %v", err) + } +} + +func TestRunCleanupPurgePreservesResourcesSharedWithUnselectedApps(t *testing.T) { + workDir := t.TempDir() + t.Chdir(workDir) + bundleDir := filepath.Join(workDir, "bort-bundle") + writeTestBundle(t, bundleDir, manifest.Manifest{ + Source: manifest.Source{Platform: "coolify-local"}, + Apps: []manifest.App{ + { + Name: "api", + Services: []manifest.Service{{ + ID: "api-id", + Name: "api-web", + Mounts: []manifest.Mount{ + {Type: "volume", Name: "shared-data", Target: "/data"}, + {Type: "bind", Source: "/data/coolify/applications/shared/storage", Target: "/storage"}, + }, + Networks: []manifest.ServiceNetwork{{Name: "shared-net"}, {Name: "coolify"}}, + }}, + }, + { + Name: "worker", + Services: []manifest.Service{{ + ID: "api-id", + Name: "api-web", + Mounts: []manifest.Mount{ + {Type: "volume", Name: "shared-data", Target: "/data"}, + {Type: "bind", Source: "/data/coolify/applications/shared/storage", Target: "/storage"}, + }, + Networks: []manifest.ServiceNetwork{{Name: "shared-net"}}, + }}, + }, + }, + }) + runCommand(t, runMigrate, []string{"--bundle", bundleDir, "--run", "purge-run", "--observation-window", "0", "--rollback-window", "0"}) + + var stdout bytes.Buffer + var stderr bytes.Buffer + if err := runCleanup(context.Background(), []string{"purge", "--run", "purge-run", "--app", "api"}, &stdout, &stderr); err != nil { + t.Fatalf("cleanup purge failed: %v\nstderr:\n%s", err, stderr.String()) + } + + output := stdout.String() + for _, want := range []string{ + "source container api-web for api is also referenced by unselected app(s): worker", + "named volume shared-data for api is also referenced by unselected app(s): worker", + "bind mount source /data/coolify/applications/shared/storage for api is also referenced by unselected app(s): worker", + "source network shared-net for api is also referenced by unselected app(s): worker", + "source network coolify for api is a platform network and requires --include-platform for purge", + } { + if !strings.Contains(output, want) { + t.Fatalf("expected cleanup purge output to contain %q, got:\n%s", want, output) + } + } + for _, notWant := range []string{ + "Source containers scheduled for purge:", + "Named source volumes that must be absent before apply:", + "Host source paths that must be absent before apply:", + "Source networks scheduled for purge:", + } { + if strings.Contains(output, notWant) { + t.Fatalf("did not expect shared resource section %q, got:\n%s", notWant, output) + } + } +} + +func TestCleanupPurgeFailsClosedWhenTopologyIsUnavailable(t *testing.T) { + workDir := t.TempDir() + t.Chdir(workDir) + bundleDir := filepath.Join(workDir, "bort-bundle") + writeTestBundle(t, bundleDir, manifest.Manifest{ + Source: manifest.Source{Platform: "coolify-local"}, + Apps: []manifest.App{{Name: "api", Services: []manifest.Service{{ID: "api-id", Name: "api"}}}}, + }) + runCommand(t, runMigrate, []string{"--bundle", bundleDir, "--run", "purge-run"}) + run, err := loadMigrationRun("purge-run") + if err != nil { + t.Fatal(err) + } + topologyPath := filepath.Join(run.Prepare.BundleDir, filepath.FromSlash(run.Prepare.Apps[0].Directory), "topology.json") + topology, err := os.ReadFile(topologyPath) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(topologyPath, []byte("{"), 0o600); err != nil { + t.Fatal(err) + } + + _, err = planCleanupPurge(run, "dokploy", cleanupPurgeFilters{AllApps: true}, nil) + if err == nil || !strings.Contains(err.Error(), "inspect source networks for api before purge") { + t.Fatalf("expected malformed topology to block purge planning, got %v", err) + } + + result := planCleanup(context.Background(), run, "dokploy") + if !strings.Contains(strings.Join(result.Warnings, "\n"), "inspect source networks for api") { + t.Fatalf("expected read-only cleanup to report the topology warning, got %#v", result.Warnings) + } + + if err := os.WriteFile(topologyPath, topology, 0o600); err != nil { + t.Fatal(err) + } + run.Prepare.Apps[0].Resources.SourceServices = []preparer.SourceServiceRef{{ServiceName: "api"}} + incomplete, err := planCleanupPurge(run, "dokploy", cleanupPurgeFilters{AllApps: true}, nil) + if err != nil { + t.Fatal(err) + } + if incomplete.CompletesLifecycle || !strings.Contains(strings.Join(incomplete.Warnings, "\n"), "has no stable container ID") { + t.Fatalf("expected unresolved source coordinates to keep lifecycle incomplete, got %#v", incomplete) + } +} + +func TestCleanupPurgeAllAppsCompletenessRequiresIncludedPlatformApps(t *testing.T) { + run := loadedMigrationRun{Prepare: preparer.Result{Apps: []preparer.AppPlan{ + {Name: "api"}, + {Name: "proxy", Role: "platform"}, + }}} + withoutPlatform := cleanupPurgeFilters{AllApps: true} + if cleanupPurgeCoversAllRunApps(run, withoutPlatform) { + t.Fatal("expected excluded platform apps to keep the lifecycle incomplete") + } + selected, err := cleanupPurgeSelectedApps(run, withoutPlatform) + if err != nil || len(selected) != 1 || selected[0].Name != "api" { + t.Fatalf("unexpected --all-apps selection without platform apps: selected=%#v err=%v", selected, err) + } + + withPlatform := cleanupPurgeFilters{AllApps: true, IncludePlatform: true} + if !cleanupPurgeCoversAllRunApps(run, withPlatform) { + t.Fatal("expected --all-apps --include-platform to cover every run app") + } + selected, err = cleanupPurgeSelectedApps(run, withPlatform) + if err != nil || len(selected) != 2 { + t.Fatalf("unexpected --all-apps --include-platform selection: selected=%#v err=%v", selected, err) + } + if cleanupPurgeCoversAllRunApps(run, cleanupPurgeFilters{IncludePlatform: true}) { + t.Fatal("expected explicit app scope never to mark a full purge") + } +} + +func TestCleanupPurgeManualCompletionDoesNotRecordLifecycleEligibility(t *testing.T) { + result := cleanupPurgeResult{ + CompletesLifecycle: true, + SourceContainers: []cleanupSourceContainer{{ContainerID: "cid123"}}, + SourceVolumes: []cleanupSourceVolume{{Type: "volume", Name: "api-data", Action: "require_absent_before_purge_apply"}}, + } + setCleanupPurgeManualCompletion(&result) + if !result.ManualCompletion || result.CompletesLifecycle { + t.Fatalf("expected verification-only purge to remain ineligible for lifecycle completion, got %#v", result) + } +} + +func TestCleanupContainerOwnersMatchReviewedShortAndCanonicalIDs(t *testing.T) { + shortID := "123456789abc" + fullID := shortID + strings.Repeat("d", 52) + owners := cleanupPurgeOwners{ContainerIDs: map[string][]string{fullID: {"worker"}}} + outside := cleanupContainerOwnersOutsideSelected(owners, cleanupSourceContainer{ContainerID: shortID}, map[string]struct{}{"api": {}}) + if len(outside) != 1 || outside[0] != "worker" { + t.Fatalf("expected canonical ID owner to keep reviewed short-ID container shared, got %#v", outside) + } +} + +func TestCleanupPurgePlatformNetworkRequiresExplicitInclusion(t *testing.T) { + workDir := t.TempDir() + t.Chdir(workDir) + bundleDir := filepath.Join(workDir, "bort-bundle") + writeTestBundle(t, bundleDir, manifest.Manifest{ + Source: manifest.Source{Platform: "coolify-local"}, + Apps: []manifest.App{{ + Name: "api", + Services: []manifest.Service{{ + ID: "api-id", + Name: "api", + Networks: []manifest.ServiceNetwork{{Name: "coolify"}}, + }}, + }}, + }) + runCommand(t, runMigrate, []string{"--bundle", bundleDir, "--run", "platform-network"}) + run, err := loadMigrationRun("platform-network") + if err != nil { + t.Fatal(err) + } + + withoutPlatform, err := planCleanupPurge(run, "dokploy", cleanupPurgeFilters{AllApps: true}, nil) + if err != nil { + t.Fatal(err) + } + if withoutPlatform.CompletesLifecycle || len(withoutPlatform.SourceNetworks) != 0 { + t.Fatalf("expected platform network exclusion to keep lifecycle incomplete, got %#v", withoutPlatform) + } + withPlatform, err := planCleanupPurge(run, "dokploy", cleanupPurgeFilters{AllApps: true, IncludePlatform: true}, nil) + if err != nil { + t.Fatal(err) + } + if !withPlatform.CompletesLifecycle || len(withPlatform.SourceNetworks) != 1 || withPlatform.SourceNetworks[0].Name != "coolify" { + t.Fatalf("expected included platform network to be scheduled, got %#v", withPlatform) + } +} + +func TestCleanupPurgeProjectFilterDoesNotAliasAppName(t *testing.T) { + app := preparer.AppPlan{ + Name: "api", + ProjectGroup: &preparer.ProjectGroup{Name: "source-project"}, + TargetResources: &preparer.TargetResources{Dokploy: &preparer.DokployResources{ + Project: preparer.DokployProject{Name: "target-project"}, + }}, + } + if cleanupPurgeProjectMatches(app, "api") { + t.Fatal("expected --project not to match an app name") + } + for _, project := range []string{"source-project", "target-project"} { + if !cleanupPurgeProjectMatches(app, project) { + t.Fatalf("expected --project to match %q", project) + } + } +} + +func TestCleanupPurgeRejectsAmbiguousAppSelector(t *testing.T) { + run := loadedMigrationRun{Prepare: preparer.Result{Apps: []preparer.AppPlan{ + {Name: "My App"}, + {Name: "my-app"}, + }}} + _, err := cleanupPurgeSelectedApps(run, cleanupPurgeFilters{Apps: []string{"my-app"}}) + if err == nil || !strings.Contains(err.Error(), "ambiguous") || !strings.Contains(err.Error(), "My App, my-app") { + t.Fatalf("expected ambiguous app selector to fail closed, got %v", err) + } +} + +func TestCleanupPurgeIncompleteResourceCoordinatesBlockLifecycleCompletion(t *testing.T) { + app := preparer.AppPlan{Name: "api"} + app.Resources.SourceServices = []preparer.SourceServiceRef{{ServiceName: "web"}} + app.Resources.Volumes = []preparer.VolumeResource{{Service: "files", Type: "volume", Target: "/data"}} + if unresolved := cleanupUnresolvedContainersForApp(app); len(unresolved) != 2 { + t.Fatalf("expected unresolved source service and volume coordinates, got %#v", unresolved) + } +} + +func TestCleanupPurgeContainerNameDoesNotInheritStableIDFromMatchingRecord(t *testing.T) { + app := preparer.AppPlan{Name: "api"} + app.Resources.SourceServices = []preparer.SourceServiceRef{{ServiceName: "web", ContainerID: "cid123", ContainerName: "api-web"}} + app.Resources.Volumes = []preparer.VolumeResource{{Service: "files", SourceContainerName: "api-web"}} + if unresolved := cleanupUnresolvedContainersForApp(app); len(unresolved) != 1 || unresolved[0] != "files" { + t.Fatalf("expected name-only coordinate to remain unresolved, got %#v", unresolved) + } +} + +func TestApplyCleanupPurgeIdentitiesUsesIdentifiedContainers(t *testing.T) { + result := cleanupPurgeResult{SourceContainers: []cleanupSourceContainer{ + {App: "api", Service: "web", ContainerID: "cid123", ContainerName: "api-web"}, + {App: "api", Service: "web", ContainerName: "api-web"}, + }} + identified := dokploy.SourcePurgeOptions{Containers: []dokploy.SourcePurgeContainer{ + {App: "api", Service: "web", ContainerID: "cid123", ContainerName: "api-web"}, + }} + applyCleanupPurgeIdentities(&result, identified) + options := cleanupPurgeOptions(result) + if len(result.SourceContainers) != 1 || len(options.Containers) != 1 || options.Containers[0].ContainerID != "cid123" { + t.Fatalf("expected backup and execution to use the identified container, result=%#v options=%#v", result.SourceContainers, options.Containers) + } +} + func TestCleanupStaleProjectNameCollisions(t *testing.T) { run := loadedMigrationRun{Prepare: preparer.Result{Apps: []preparer.AppPlan{ {Name: "api", TargetResources: &preparer.TargetResources{Dokploy: &preparer.DokployResources{Project: preparer.DokployProject{Name: "proxy"}}}}, @@ -102,3 +645,206 @@ func TestCleanupStaleProjectNameCollisions(t *testing.T) { t.Fatalf("unexpected collisions: %#v", collisions) } } + +func TestRunCleanupCommandsDoNotUseMtimeFallback(t *testing.T) { + workDir := t.TempDir() + t.Chdir(workDir) + bundleDir := filepath.Join(workDir, "bort-bundle") + writeTestBundle(t, bundleDir, manifest.Manifest{ + Source: manifest.Source{Platform: "docker"}, + Apps: []manifest.App{{Name: "mtime-only", Services: []manifest.Service{{Name: "mtime-only", Image: "example/mtime:latest"}}}}, + }) + runCommand(t, runMigrate, []string{"--bundle", bundleDir, "--run", "mtime-only"}) + if err := os.Remove(filepath.Join(workDir, ".bort", "state.json")); err != nil { + t.Fatal(err) + } + + for _, args := range [][]string{nil, {"purge"}} { + if err := runCleanup(context.Background(), args, io.Discard, io.Discard); err == nil || !strings.Contains(err.Error(), "no current migration run") { + t.Fatalf("expected cleanup %v to reject an mtime-only run, got %v", args, err) + } + } +} + +func TestCleanupContainersIncludeVolumeAndDataStoreSources(t *testing.T) { + app := preparer.AppPlan{Name: "api"} + app.Resources.Volumes = []preparer.VolumeResource{{Service: "files", SourceContainerID: "volume-container", SourceContainerName: "files"}} + app.Resources.DataStores = []preparer.DataStoreResource{{Service: "db", SourceContainerID: "database-container", SourceContainerName: "db"}} + containers, err := cleanupContainersForApp(app) + if err != nil { + t.Fatal(err) + } + if len(containers) != 2 || containers[0].ContainerID != "volume-container" || containers[1].ContainerID != "database-container" { + t.Fatalf("expected all source container coordinates in purge inventory, got %#v", containers) + } +} + +func TestCleanupContainersRejectConflictingCoordinatesForSameID(t *testing.T) { + for _, test := range []struct { + name string + second preparer.VolumeResource + want string + }{ + {name: "name", second: preparer.VolumeResource{Service: "web", SourceContainerID: "cid123", SourceContainerName: "replacement"}, want: "conflicting names"}, + {name: "service", second: preparer.VolumeResource{Service: "worker", SourceContainerID: "cid123", SourceContainerName: "web"}, want: "conflicting services"}, + } { + t.Run(test.name, func(t *testing.T) { + app := preparer.AppPlan{Name: "api"} + app.Resources.SourceServices = []preparer.SourceServiceRef{{ServiceName: "web", ContainerID: "cid123", ContainerName: "web"}} + app.Resources.Volumes = []preparer.VolumeResource{test.second} + if _, err := cleanupContainersForApp(app); err == nil || !strings.Contains(err.Error(), test.want) { + t.Fatalf("expected %s conflict, got %v", test.name, err) + } + }) + } +} + +func TestCleanupContainersDeduplicateReviewedShortAndCanonicalIDs(t *testing.T) { + shortID := "123456789abc" + fullID := shortID + strings.Repeat("d", 52) + app := preparer.AppPlan{Name: "api"} + app.Resources.SourceServices = []preparer.SourceServiceRef{{ServiceName: "web", ContainerID: shortID, ContainerName: "web"}} + app.Resources.Volumes = []preparer.VolumeResource{{Service: "web", SourceContainerID: fullID, SourceContainerName: "web"}} + containers, err := cleanupContainersForApp(app) + if err != nil { + t.Fatal(err) + } + if len(containers) != 1 || containers[0].ContainerID != fullID { + t.Fatalf("expected one canonical container coordinate, got %#v", containers) + } +} + +func TestCleanupPathContainsAllowsDotDotPrefixedChildNames(t *testing.T) { + parent := filepath.Join("data", "coolify", "applications", "api") + if !cleanupPathContains(parent, filepath.Join(parent, "..data")) { + t.Fatal("expected a child name beginning with two dots to remain inside its parent") + } + if cleanupPathContains(parent, filepath.Join(parent, "..", "other")) { + t.Fatal("expected parent traversal to remain outside") + } +} + +func TestCleanupRecoveryCommandsPreserveExternalRunAndPurgeScope(t *testing.T) { + externalRunDir := filepath.Join(t.TempDir(), "external-run") + result := cleanupPurgeResult{ + RunName: "external-run", + RunDir: externalRunDir, + DryRun: true, + BackupDir: "/private/purge backups", + Filters: cleanupPurgeFilters{Apps: []string{"api"}, IncludePlatform: true}, + SourcePaths: []cleanupSourcePath{{ + Path: "/data/coolify/services/service-1", + Source: "explicit", + }}, + } + command := cleanupPurgeApplyCommand(result) + for _, want := range []string{"cleanup purge --apply", "--app api", "--include-platform", "--backup-dir '/private/purge backups'", "--source-dir /data/coolify/services/service-1", "--confirm 'purge external-run'", "--run " + shellQuote(externalRunDir)} { + if !strings.Contains(command, want) { + t.Fatalf("expected purge recovery command to contain %q, got %q", want, command) + } + } + if strings.Contains(command, "") || strings.Contains(command, "--all-apps") { + t.Fatalf("purge recovery command replaced its reviewed scope: %q", command) + } + + var output strings.Builder + writeCleanupText(&output, cleanupResult{RunName: "external-run", RunDir: externalRunDir, DryRun: true}) + if !strings.Contains(output.String(), "cleanup --apply --run "+shellQuote(externalRunDir)) { + t.Fatalf("metadata cleanup recovery command lost external run path:\n%s", output.String()) + } +} + +func TestWriteCleanupPurgeTextWarnsAfterPartialApply(t *testing.T) { + result := cleanupPurgeResult{ + RunName: "partial", + RunDir: filepath.Join(".bort", "runs", "partial"), + DryRun: false, + PurgeResult: &dokploy.SourcePurgeResult{Containers: []dokploy.SourcePurgeResourceResult{ + {Ref: "web", Status: "removed"}, + {Ref: "worker", Status: "error"}, + }}, + } + var output strings.Builder + writeCleanupPurgeText(&output, result) + for _, want := range []string{"Mode: incomplete", "[removed] web", "[error] worker", "Earlier resources may already have been removed"} { + if !strings.Contains(output.String(), want) { + t.Fatalf("expected partial purge output to contain %q, got:\n%s", want, output.String()) + } + } + if strings.Contains(output.String(), "Dry run only") { + t.Fatalf("partial purge was mislabeled as a dry run:\n%s", output.String()) + } +} + +func TestWriteCleanupPurgeTextReportsIncompleteScopedApply(t *testing.T) { + var output strings.Builder + writeCleanupPurgeText(&output, cleanupPurgeResult{Applied: true, CompletesLifecycle: false}) + if !strings.Contains(output.String(), "migration lifecycle remains incomplete") { + t.Fatalf("expected scoped apply to report incomplete lifecycle, got:\n%s", output.String()) + } +} + +func TestWriteCleanupPurgeTextRequiresRecordedLifecycleCompletion(t *testing.T) { + var output strings.Builder + writeCleanupPurgeText(&output, cleanupPurgeResult{Applied: true, CompletesLifecycle: true}) + if strings.Contains(output.String(), "completed the migration lifecycle") || !strings.Contains(output.String(), "migration lifecycle remains incomplete") { + t.Fatalf("expected unrecorded lifecycle completion to remain incomplete, got:\n%s", output.String()) + } +} + +func TestCleanupPurgeApplyCommandRejectsUnscopedReview(t *testing.T) { + if command := cleanupPurgeApplyCommand(cleanupPurgeResult{RunName: "unscoped"}); command != "" { + t.Fatalf("expected no apply command for an unscoped review, got %q", command) + } +} + +func TestUpdateCleanupPurgeBackupPersistsPartialResults(t *testing.T) { + backupDir := t.TempDir() + result := cleanupPurgeResult{RunName: "partial", DryRun: false, PurgeResult: &dokploy.SourcePurgeResult{ + Containers: []dokploy.SourcePurgeResourceResult{{Ref: "web", Status: "removed"}}, + }} + path, err := writeCleanupPurgeBackup(backupDir, cleanupPurgeResult{RunName: result.RunName, DryRun: true}) + if err != nil { + t.Fatal(err) + } + recorder := cleanupPurgeBackupRecorder{path: path, result: &result, persist: updateCleanupPurgeBackup} + if err := recorder.Start(); err != nil { + t.Fatal(err) + } + if err := recorder.Record(*result.PurgeResult); err != nil { + t.Fatal(err) + } + contents, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + var persisted cleanupPurgeResult + if err := json.Unmarshal(contents, &persisted); err != nil { + t.Fatal(err) + } + if persisted.DryRun || persisted.ExecutionStartedAt == nil || persisted.PurgeResult == nil || len(persisted.PurgeResult.Containers) != 1 || persisted.PurgeResult.Containers[0].Status != "removed" { + t.Fatalf("expected partial results in private backup, got %#v", persisted) + } +} + +func TestCleanupPurgeBackupRecorderKeepsPersistenceFailureSticky(t *testing.T) { + result := cleanupPurgeResult{RunName: "partial"} + writes := 0 + recorder := cleanupPurgeBackupRecorder{ + path: filepath.Join(t.TempDir(), "purge.json"), + result: &result, + persist: func(string, cleanupPurgeResult) error { + writes++ + return errors.New("disk unavailable") + }, + } + if err := recorder.Start(); err == nil || !strings.Contains(err.Error(), "disk unavailable") { + t.Fatalf("expected initial persistence failure, got %v", err) + } + if err := recorder.Record(dokploy.SourcePurgeResult{}); err == nil || !strings.Contains(err.Error(), "disk unavailable") { + t.Fatalf("expected sticky persistence failure, got %v", err) + } + if writes != 1 { + t.Fatalf("expected persistence not to retry after failure, got %d writes", writes) + } +} diff --git a/internal/cli/cockpit.go b/internal/cli/cockpit.go index 1e07196..c285c14 100644 --- a/internal/cli/cockpit.go +++ b/internal/cli/cockpit.go @@ -14,7 +14,7 @@ import ( func writeAppFirstCockpit(w io.Writer, run loadedMigrationRun) { apps := appsFromRun(run) summary := summarizeMigrationRun(run) - overall := overallAppHealth(apps) + phase := migrationRunPhase(run) st := newStyler(w) ready, blocked := 0, 0 @@ -36,7 +36,7 @@ func writeAppFirstCockpit(w io.Writer, run loadedMigrationRun) { if blocked > 0 { header += " · " + st.glyph(fmt.Sprintf("%d blocked", blocked), sevBad) } - header += " " + st.pill("DRY RUN", sevWarn) + header += " " + st.pill(migrationRunPhaseLabel(phase), severityForMigrationRunPhase(phase)) fmt.Fprintln(w, header) fmt.Fprintln(w) @@ -99,10 +99,91 @@ func writeAppFirstCockpit(w io.Writer, run loadedMigrationRun) { if line := appliedFooter(run.Applied); line != "" { fmt.Fprintln(w, st.muted(line)) } - if overall != appHealthReady { + downstreamBlockers := openDownstreamBlockingDecisions(run) + if len(downstreamBlockers) > 0 { + fmt.Fprintf(w, "%s\n", st.muted(fmt.Sprintf("Downstream blockers: %d open", len(downstreamBlockers)))) + writeCockpitDecisions(w, st, downstreamBlockers) + } + reviewDecisions := openReviewDecisions(run) + if len(reviewDecisions) > 0 { + fmt.Fprintf(w, "%s\n", st.muted(fmt.Sprintf("Review-only decisions: %d open (non-blocking before live apply)", len(reviewDecisions)))) + writeCockpitDecisions(w, st, reviewDecisions) + } + switch phase { + case "lock-error": + fmt.Fprintln(w, st.muted("Live apply lock state could not be verified. Inspect the run's apply.lock before retrying.")) + case "applying": + fmt.Fprintf(w, "%s\n", st.muted(fmt.Sprintf("Live apply is running. Run `%s` to attach to its current status.", liveApplyCommand(run)))) + case "partial": + fmt.Fprintf(w, "%s\n", st.muted(fmt.Sprintf("Live apply is incomplete. Run `%s` to resume safely.", liveApplyCommand(run)))) + case "applied": + fmt.Fprintf(w, "%s\n", st.muted(fmt.Sprintf("Target is live. Verify it through the rollback window, then run `%s` to retire the source.", runScopedCommand(run, "commit --apply")))) + case "committed": + fmt.Fprintf(w, "%s\n", st.muted(fmt.Sprintf("Target accepted and source containers retired. Run `%s` to audit leftovers.", runScopedCommand(run, "cleanup")))) + case "purged": + fmt.Fprintln(w, st.muted("Migration complete. Target resources and source-control credentials were preserved.")) + case "planning": fmt.Fprintln(w, st.muted(issueActionFooter(apps))) - } else { - fmt.Fprintf(w, "%s\n", st.muted(fmt.Sprintf("All app inputs ready. Run `%s` to apply, or run `bort` interactively to continue.", liveApplyCommand(run)))) + default: + fmt.Fprintf(w, "%s\n", st.muted(fmt.Sprintf("All app inputs ready. Run `%s` to apply, or run `%s` interactively to continue.", liveApplyCommand(run), bortCommand("")))) + } +} + +func writeCockpitDecisions(w io.Writer, st *styler, decisions []runDecision) { + limit := min(len(decisions), 3) + for _, decision := range decisions[:limit] { + fmt.Fprintf(w, " %s %s: %s (%d item(s))\n", decision.Kind, decision.Readiness, decision.Action, decision.Count) + } + if remaining := len(decisions) - limit; remaining > 0 { + fmt.Fprintf(w, " %s\n", st.muted(fmt.Sprintf("and %d more", remaining))) + } +} + +func migrationRunPhase(run loadedMigrationRun) string { + applyActive, applyActiveErr := applyRunActive(run.Run.RunDir) + switch { + case run.Run.PurgedAt != nil: + return "purged" + case run.Run.CommittedAt != nil: + return "committed" + case run.Run.LiveAppliedAt != nil || liveApplySucceeded(run): + return "applied" + case applyActiveErr != nil: + return "lock-error" + case applyActive: + return "applying" + case len(run.Applied.Steps) > 0: + return "partial" + case len(openSetupDecisions(run)) > 0: + return "planning" + case len(liveApplyBlockingDecisions(run)) == 0: + return "ready" + default: + return "planning" + } +} + +func migrationRunPhaseLabel(phase string) string { + switch phase { + case "applied": + return "TARGET LIVE" + case "purged": + return "COMPLETE" + case "lock-error": + return "LOCK ERROR" + default: + return strings.ToUpper(phase) + } +} + +func severityForMigrationRunPhase(phase string) severity { + switch phase { + case "ready", "applied", "committed", "purged": + return sevGood + case "partial", "lock-error": + return sevBad + default: + return sevWarn } } @@ -119,13 +200,13 @@ func issueActionFooter(apps []appView) string { } switch { case hasFix && hasNext: - return "Run the shown `fix:` commands and use the `next:` notes as a checklist, then re-run `bort` to recheck." + return fmt.Sprintf("Run the shown `fix:` commands and use the `next:` notes as a checklist, then re-run `%s` to recheck.", bortCommand("")) case hasFix: - return "Run the shown `fix:` commands, then re-run `bort` to recheck." + return fmt.Sprintf("Run the shown `fix:` commands, then re-run `%s` to recheck.", bortCommand("")) case hasNext: - return "Use the `next:` notes as a checklist. Re-run `bort` after changing Coolify/Dokploy settings or the bundle." + return fmt.Sprintf("Use the `next:` notes as a checklist. Re-run `%s` after changing Coolify/Dokploy settings or the bundle.", bortCommand("")) default: - return "Resolve the issues above, then re-run `bort` to recheck." + return fmt.Sprintf("Resolve the issues above, then re-run `%s` to recheck.", bortCommand("")) } } diff --git a/internal/cli/cockpit_test.go b/internal/cli/cockpit_test.go index da89114..e17392d 100644 --- a/internal/cli/cockpit_test.go +++ b/internal/cli/cockpit_test.go @@ -3,9 +3,11 @@ package cli import ( "os" "path/filepath" + "strings" "testing" "time" + "github.com/aikins01/bort/internal/manifest" "github.com/aikins01/bort/internal/preparer" ) @@ -98,6 +100,127 @@ func TestReadRunProgressIgnoresStaleOrMismatchedProgress(t *testing.T) { } } +func TestRecordReviewDecisionPersistsWithoutTTY(t *testing.T) { + workDir := t.TempDir() + t.Chdir(workDir) + bundleDir := filepath.Join(workDir, "bort-bundle") + writeTestBundle(t, bundleDir, manifest.Manifest{ + Source: manifest.Source{Platform: "docker"}, + Apps: []manifest.App{{ + Name: "api", + Services: []manifest.Service{{Name: "api", Image: "example/api:latest"}}, + Routes: []manifest.Route{{Host: "api.example.com", ServiceName: "api"}}, + }}, + }) + runCommand(t, runMigrate, []string{"--bundle", bundleDir, "--run", "review-progress"}) + run, err := loadMigrationRun("review-progress") + if err != nil { + t.Fatal(err) + } + if len(run.Decisions.Decisions) == 0 { + t.Fatal("expected a review decision") + } + decision := run.Decisions.Decisions[0] + if err := recordReviewDecision(run, decision, run.Run.UpdatedAt.Add(time.Second)); err != nil { + t.Fatal(err) + } + reloaded, err := loadMigrationRun("review-progress") + if err != nil { + t.Fatal(err) + } + if reloaded.Progress.Decisions[decision.Kind].Status != progressStatusResolved { + t.Fatalf("expected review progress to persist, got %#v", reloaded.Progress.Decisions) + } + refreshed, err := refreshMigrationRun("review-progress") + if err != nil { + t.Fatal(err) + } + if refreshed.Progress.Decisions[decision.Kind].Status != progressStatusResolved { + t.Fatalf("expected review progress to survive plan refresh, got %#v", refreshed.Progress.Decisions) + } + for _, open := range openRunDecisions(refreshed) { + if open.Kind == decision.Kind { + t.Fatalf("refreshed plan reopened resolved decision %q", decision.Kind) + } + } +} + +func TestRecordReviewDecisionRejectsChangedPlanGeneration(t *testing.T) { + workDir := t.TempDir() + t.Chdir(workDir) + bundleDir := filepath.Join(workDir, "bort-bundle") + writeTestBundle(t, bundleDir, manifest.Manifest{ + Source: manifest.Source{Platform: "docker"}, + Apps: []manifest.App{{ + Name: "api", + Services: []manifest.Service{{Name: "api", Image: "example/api:latest"}}, + Routes: []manifest.Route{{Host: "api.example.com", ServiceName: "api"}}, + }}, + }) + runCommand(t, runMigrate, []string{"--bundle", bundleDir, "--run", "stale-review"}) + stale, err := loadMigrationRun("stale-review") + if err != nil { + t.Fatal(err) + } + if len(stale.Decisions.Decisions) == 0 { + t.Fatal("expected a review decision") + } + if _, err := refreshMigrationRun("stale-review"); err != nil { + t.Fatal(err) + } + err = recordReviewDecision(stale, stale.Decisions.Decisions[0], time.Now().UTC()) + if err == nil || !strings.Contains(err.Error(), "plan changed while review was open") { + t.Fatalf("expected stale review to be rejected, got %v", err) + } +} + +func TestNextWizardDecisionsIncludesDownstreamReview(t *testing.T) { + run := loadedMigrationRun{Decisions: runDecisions{ + APIVersion: decisionsAPIVersion, + Decisions: []runDecision{{ + ID: "cutover", + Kind: "cutover", + Readiness: preparer.ReadinessNeedsDecision, + Items: []runDecisionItem{{ + Stage: "cutover", + App: "api", + Code: "cutover.review", + Readiness: preparer.ReadinessNeedsDecision, + }}, + }}, + }} + decisions, reviewOnly := nextWizardDecisions(run) + if !reviewOnly || len(decisions) != 1 || decisions[0].Kind != "cutover" { + t.Fatalf("expected downstream review decision, got reviewOnly=%t decisions=%#v", reviewOnly, decisions) + } +} + +func TestMarkReviewDecisionDoneKeepsExcludedBlockersOpen(t *testing.T) { + reviewItem := runDecisionItem{Stage: "cutover", App: "api", Code: "cutover.review", Readiness: preparer.ReadinessNeedsDecision} + blockedItem := runDecisionItem{Stage: "cutover", App: "api", Code: "cutover.blocked", Readiness: preparer.ReadinessBlocked} + run := loadedMigrationRun{ + Decisions: runDecisions{ + APIVersion: decisionsAPIVersion, + Decisions: []runDecision{{ + ID: "cutover", + Kind: "cutover", + Readiness: preparer.ReadinessBlocked, + Items: []runDecisionItem{reviewItem, blockedItem}, + }}, + }, + Progress: emptyRunProgress(migrationRun{}), + } + selected := runDecision{ID: "cutover", Kind: "cutover", Readiness: preparer.ReadinessNeedsDecision, Items: []runDecisionItem{reviewItem}} + run.Progress = markReviewDecisionDone(run, selected, time.Now().UTC()) + if run.Progress.Decisions["cutover"].Status != progressStatusOpen { + t.Fatalf("expected partially reviewed decision kind to remain open, got %#v", run.Progress.Decisions["cutover"]) + } + open := openRunDecisions(run) + if len(open) != 1 || len(open[0].Items) != 1 || open[0].Items[0].Code != blockedItem.Code { + t.Fatalf("expected only the excluded blocker to remain open, got %#v", open) + } +} + func TestAppliedFooterEmptyWhenNoSteps(t *testing.T) { if got := appliedFooter(runApplied{}); got != "" { t.Fatalf("expected empty footer, got %q", got) @@ -115,3 +238,112 @@ func TestAppliedFooterCountsOkAndError(t *testing.T) { t.Fatalf("unexpected footer: %q", got) } } + +func TestCockpitShowsDownstreamDecisionsAsReviewOnly(t *testing.T) { + run := loadedMigrationRun{ + Run: migrationRun{Name: "reviewed", RunDir: t.TempDir(), Target: "dokploy"}, + Prepare: preparer.Result{Apps: []preparer.AppPlan{{ + Name: "api", + }}}, + Decisions: runDecisions{ + APIVersion: decisionsAPIVersion, + Decisions: []runDecision{{ + ID: "cutover", + Kind: "cutover", + Readiness: preparer.ReadinessNeedsDecision, + Items: []runDecisionItem{{ + Stage: "cutover", + App: "api", + Code: "cutover.review", + Readiness: preparer.ReadinessNeedsDecision, + }}, + }}, + }, + } + var output strings.Builder + writeAppFirstCockpit(&output, run) + for _, want := range []string{ + "READY", + "Review-only decisions: 1 open (non-blocking before live apply)", + "cutover needs_decision", + } { + if !strings.Contains(output.String(), want) { + t.Fatalf("expected cockpit to contain %q, got:\n%s", want, output.String()) + } + } +} + +func TestCockpitShowsDownstreamBlockersAsBlocking(t *testing.T) { + run := loadedMigrationRun{ + Run: migrationRun{Name: "blocked", RunDir: t.TempDir(), Target: "dokploy"}, + Prepare: preparer.Result{Apps: []preparer.AppPlan{{ + Name: "api", + }}}, + Decisions: runDecisions{ + APIVersion: decisionsAPIVersion, + Decisions: []runDecision{{ + ID: "cutover", + Kind: "cutover", + Readiness: preparer.ReadinessBlocked, + Items: []runDecisionItem{{ + Stage: "cutover", + App: "api", + Code: "cutover.blocked", + Readiness: preparer.ReadinessBlocked, + }}, + }}, + }, + } + var output strings.Builder + writeAppFirstCockpit(&output, run) + for _, want := range []string{"PLANNING", "Downstream blockers: 1 open", "cutover blocked"} { + if !strings.Contains(output.String(), want) { + t.Fatalf("expected cockpit to contain %q, got:\n%s", want, output.String()) + } + } + if strings.Contains(output.String(), "Review-only decisions") { + t.Fatalf("downstream blocker was mislabeled as review-only:\n%s", output.String()) + } +} + +func TestCockpitAndNextSurfaceApplyLockProbeErrors(t *testing.T) { + for _, fixture := range []struct { + name string + link func(string, string) error + }{ + {name: "symlink", link: os.Symlink}, + {name: "hard link", link: os.Link}, + } { + t.Run(fixture.name, func(t *testing.T) { + workDir := t.TempDir() + t.Chdir(workDir) + bundleDir := filepath.Join(workDir, "bort-bundle") + writeTestBundle(t, bundleDir, manifest.Manifest{ + Source: manifest.Source{Platform: "docker"}, + Apps: []manifest.App{{Name: "api", Services: []manifest.Service{{Name: "api", Image: "example/api:latest"}}}}, + }) + runCommand(t, runMigrate, []string{"--bundle", bundleDir, "--run", "lock-error"}) + run, err := loadMigrationRun("lock-error") + if err != nil { + t.Fatal(err) + } + targetPath := filepath.Join(run.Run.RunDir, "lock-target") + if err := os.WriteFile(targetPath, []byte("preserve\n"), 0o600); err != nil { + t.Fatal(err) + } + if err := fixture.link(targetPath, filepath.Join(run.Run.RunDir, "apply.lock")); err != nil { + t.Skipf("%s fixture is unavailable: %v", fixture.name, err) + } + + var output strings.Builder + writeAppFirstCockpit(&output, run) + if !strings.Contains(output.String(), "LOCK ERROR") || strings.Contains(output.String(), " READY") { + t.Fatalf("expected cockpit to surface the apply-lock probe error, got:\n%s", output.String()) + } + next := nextSafeStep(run, nil) + if !strings.Contains(next.Action, "inspect the live-apply lock") || strings.Contains(next.Action, "migrate --live") || next.Reason == "" { + t.Fatalf("expected next step to require lock inspection, got %#v", next) + } + }) + } +} diff --git a/internal/cli/commit.go b/internal/cli/commit.go index 8313c30..ead601f 100644 --- a/internal/cli/commit.go +++ b/internal/cli/commit.go @@ -34,21 +34,52 @@ func runCommit(ctx context.Context, args []string, stdout, stderr io.Writer) err fs.StringVar(&outputPath, "output", "-", "output path, or - for stdout") fs.StringVar(&cutoverPlanPath, "from-cutover", "", "read a prior cutover JSON plan artifact") fs.BoolVar(&apply, "apply", false, "execute commit cleanup (stop source containers and coolify-proxy)") - fs.StringVar(&runRef, "run", "", "run name under .bort/runs, or a run directory path (with --apply)") + fs.StringVar(&runRef, "run", "", "run name under .bort/runs, or a run directory path") fs.IntVar(&rollbackWindowSeconds, "rollback-window", commitplan.DefaultRollbackWindowSeconds, "rollback window in seconds") if err := fs.Parse(args); err != nil { return err } + if fs.NArg() > 0 { + return fmt.Errorf("commit does not accept positional argument %q", fs.Arg(0)) + } if apply { + for _, name := range []string{"app", "bundle", "format", "from-cutover", "output", "rollback-window", "target"} { + if flagSet(fs, name) { + return fmt.Errorf("commit --apply does not accept --%s; select the reviewed run with --run", name) + } + } return applyCommitFromArgs(ctx, runRef, stderr) } if err := checkOutputFormat("commit", format); err != nil { return err } + if strings.TrimSpace(runRef) != "" { + for _, name := range []string{"app", "bundle", "from-cutover", "rollback-window", "target"} { + if flagSet(fs, name) { + return fmt.Errorf("commit --run does not accept --%s; the run already owns its reviewed commit plan", name) + } + } + } var result commitplan.Result - if cutoverPlanPath != "" { + useCurrentRun := cutoverPlanPath == "" && !flagSet(fs, "bundle") && !flagSet(fs, "target") && !flagSet(fs, "app") && !flagSet(fs, "rollback-window") + resolvedRun := strings.TrimSpace(runRef) + useReviewedRun := resolvedRun != "" + if !useReviewedRun && useCurrentRun { + var err error + resolvedRun, useReviewedRun, err = selectedRunRef(false) + if err != nil { + return err + } + } + if useReviewedRun { + run, err := loadMigrationRun(resolvedRun) + if err != nil { + return err + } + result = run.Commit + } else if cutoverPlanPath != "" { expect := artifactExpectations{AppName: appName} if flagWasSet(fs, "bundle") { expect.BundleDir = bundleDir @@ -131,19 +162,17 @@ func writeCommitText(w io.Writer, result commitplan.Result) { fmt.Fprintln(w, "Dry run only: no target ownership was committed and no source resources were retired.") } -// applyCommitFromArgs executes the commit-time cleanup against the -// requested run (or the latest run if none was named). it stops every -// source app container and reasserts that coolify-proxy is stopped, but -// does not remove anything — operators on a long rollback window can -// still docker start the source manually. func applyCommitFromArgs(ctx context.Context, runRef string, stderr io.Writer) error { - if strings.TrimSpace(runRef) == "" { - latest, ok := latestRunRef() - if !ok { - return fmt.Errorf("no migration run found; create one before commit --apply") - } - runRef = latest + var err error + runRef, err = resolveRunRef(runRef, false) + if err != nil { + return err } + operationLock, err := acquireRunOperationLock(runRef) + if err != nil { + return fmt.Errorf("commit run %q: %w", runRef, err) + } + defer operationLock.Release() run, err := loadMigrationRun(runRef) if err != nil { return err @@ -161,37 +190,79 @@ func applyCommitFromArgs(ctx context.Context, runRef string, stderr io.Writer) e plan := dokploy.PlanForCommit(run.Prepare, run.Cutover) plan.RunName = run.Run.Name plan.RunDir = run.Run.RunDir - fmt.Fprintf(stderr, "commit apply: planned %d step(s) to retire source\n", len(plan.Steps)) - return client.Apply(ctx, plan) + fmt.Fprintf(stderr, "commit apply: run %s; planned %d step(s) to retire source\n", run.Run.Name, len(plan.Steps)) + if err := client.Apply(ctx, plan); err != nil { + return err + } + if err := markRunCommittedLocked(run.Run); err != nil { + return fmt.Errorf("source retirement completed, but migration commit metadata could not be recorded: %w", err) + } + return nil } -// requireLiveApplySucceeded blocks commit --apply on runs whose live -// migrate phase never finished. without this guard, commit --apply on a -// freshly-planned (or partially-applied) run would stop production -// source containers before the target was actually serving traffic. -// each step in the computed live plan must have a matching ledger entry -// at the same index with a completed status and identical kind/app/ref — -// anything else means the live apply is stale or incomplete. func requireLiveApplySucceeded(run loadedMigrationRun) error { + return requireLiveApplySucceededForAppsSkipping(run, nil, nil) +} + +func liveApplySucceeded(run loadedMigrationRun) bool { + return requireLiveApplySucceeded(run) == nil +} + +func requireLiveApplySucceededForApps(run loadedMigrationRun, apps map[string]struct{}) error { + return requireLiveApplySucceededForAppsSkipping(run, apps, nil) +} + +func requireLiveApplySucceededForAppsSkipping(run loadedMigrationRun, apps map[string]struct{}, skipKinds map[dokploy.StepKind]struct{}) error { + if run.Run.ApplyOutcomeRequired && run.Applied.SucceededAt == nil { + return fmt.Errorf("run %q has no successful live-apply outcome recorded; run `%s` first", run.Run.Name, liveApplyCommand(run)) + } live := dokploy.PlanFromArtifacts(run.Prepare, run.Sync, run.Cutover) if len(live.Steps) == 0 { return fmt.Errorf("run %q has no live apply steps; nothing to commit", run.Run.Name) } byIndex := map[int]appliedStep{} + latest := map[string]appliedStep{} for _, step := range run.Applied.Steps { - byIndex[step.Index] = step + if current, ok := byIndex[step.Index]; !ok || !step.UpdatedAt.Before(current.UpdatedAt) { + byIndex[step.Index] = step + } + key := appliedStepKey(step.Kind, step.App, step.Ref) + if current, ok := latest[key]; !ok || !step.UpdatedAt.Before(current.UpdatedAt) { + latest[key] = step + } } + checked := 0 for index, step := range live.Steps { - got, ok := byIndex[index] - if !ok { - return fmt.Errorf("run %q has not been live-applied; missing applied step %d %s. run `bort migrate --live --run %s` first", run.Run.Name, index, step.Kind, run.Run.Name) + if _, skip := skipKinds[step.Kind]; skip { + continue + } + if !liveApplyStepInScope(step, apps) { + continue } - if got.Kind != string(step.Kind) || got.App != step.App || got.Ref != step.Ref { - return fmt.Errorf("run %q live apply ledger is stale at step %d (expected %s %s/%s, got %s %s/%s); rerun the live apply before commit", run.Run.Name, index, step.Kind, step.App, step.Ref, got.Kind, got.App, got.Ref) + checked++ + key := appliedStepKey(string(step.Kind), step.App, step.Ref) + recorded, ok := byIndex[index] + if !ok || !appliedStepCompleted(recorded) || !appliedStepMatches(recorded, step) { + return fmt.Errorf("run %q has not been live-applied; missing completed applied step %s %s/%s at plan index %d. run `%s` first", run.Run.Name, step.Kind, step.App, step.Ref, index, liveApplyCommand(run)) } - if !appliedStepCompleted(got) { - return fmt.Errorf("run %q live apply step %d (%s %s/%s) finished with status %q, not complete; resolve before commit", run.Run.Name, index, step.Kind, step.App, step.Ref, got.Status) + if current, ok := latest[key]; !ok || !appliedStepCompleted(current) { + return fmt.Errorf("run %q has not been live-applied; latest applied outcome for step %s %s/%s is not complete. run `%s` first", run.Run.Name, step.Kind, step.App, step.Ref, liveApplyCommand(run)) } } + if checked == 0 { + return fmt.Errorf("run %q has no live apply steps for the selected purge scope", run.Run.Name) + } return nil } + +func liveApplyStepInScope(step dokploy.Step, apps map[string]struct{}) bool { + if len(apps) == 0 || step.App == "" { + return true + } + _, ok := apps[step.App] + return ok +} + +func appliedStepKey(kind, app, ref string) string { + return kind + "\x00" + app + "\x00" + ref +} diff --git a/internal/cli/commit_test.go b/internal/cli/commit_test.go index d887139..9ddcc38 100644 --- a/internal/cli/commit_test.go +++ b/internal/cli/commit_test.go @@ -4,8 +4,11 @@ import ( "bytes" "context" "encoding/json" + "io" + "path/filepath" "strings" "testing" + "time" commitplan "github.com/aikins01/bort/internal/commit" "github.com/aikins01/bort/internal/exporter" @@ -98,6 +101,194 @@ func TestRunCommitWritesJSONPlan(t *testing.T) { } } +func TestRunCommitApplyRejectsIgnoredPlanningFlags(t *testing.T) { + var stdout bytes.Buffer + var stderr bytes.Buffer + err := runCommit(context.Background(), []string{"--apply", "--app", "api"}, &stdout, &stderr) + if err == nil || !strings.Contains(err.Error(), "does not accept --app") { + t.Fatalf("expected commit apply to reject ignored app scope, got %v", err) + } +} + +func TestRunCommitApplyRejectsPositionalArguments(t *testing.T) { + err := runCommit(context.Background(), []string{"--apply", "purge"}, io.Discard, io.Discard) + if err == nil || !strings.Contains(err.Error(), "does not accept positional argument") { + t.Fatalf("expected positional argument to be rejected before source retirement, got %v", err) + } +} + +func TestRunCommitDefaultsToCurrentRun(t *testing.T) { + workDir := t.TempDir() + t.Chdir(workDir) + bundleDir := filepath.Join(workDir, "reviewed-bundle") + writeTestBundle(t, bundleDir, manifest.Manifest{ + Source: manifest.Source{Platform: "docker"}, + Apps: []manifest.App{{Name: "reviewed-app", Services: []manifest.Service{{Name: "reviewed-app", Image: "example/reviewed:latest"}}}}, + }) + runCommand(t, runMigrate, []string{"--bundle", bundleDir, "--run", "reviewed-run"}) + reviewed, err := loadMigrationRun("reviewed-run") + if err != nil { + t.Fatal(err) + } + writeTestBundle(t, filepath.Join(workDir, "bort-bundle"), manifest.Manifest{ + Source: manifest.Source{Platform: "docker"}, + Apps: []manifest.App{{Name: "stale-app", Services: []manifest.Service{{Name: "stale-app", Image: "example/stale:latest"}}}}, + }) + + var stdout bytes.Buffer + var stderr bytes.Buffer + if err := runCommit(context.Background(), nil, &stdout, &stderr); err != nil { + t.Fatalf("commit failed: %v\nstderr:\n%s", err, stderr.String()) + } + if !strings.Contains(stdout.String(), "Commit plan: "+reviewed.Run.BundleDir+" -> dokploy") || !strings.Contains(stdout.String(), "reviewed-app") { + t.Fatalf("expected commit to use the current run artifact, got:\n%s", stdout.String()) + } + if strings.Contains(stdout.String(), "stale-app") { + t.Fatalf("commit planned from the default bundle instead of the current run:\n%s", stdout.String()) + } +} + +func TestRunCommitHonorsExplicitRun(t *testing.T) { + workDir := t.TempDir() + t.Chdir(workDir) + explicitBundle := filepath.Join(workDir, "explicit-bundle") + currentBundle := filepath.Join(workDir, "current-bundle") + writeTestBundle(t, explicitBundle, manifest.Manifest{ + Source: manifest.Source{Platform: "docker"}, + Apps: []manifest.App{{Name: "explicit-app", Services: []manifest.Service{{Name: "explicit-app", Image: "example/explicit:latest"}}}}, + }) + writeTestBundle(t, currentBundle, manifest.Manifest{ + Source: manifest.Source{Platform: "docker"}, + Apps: []manifest.App{{Name: "current-app", Services: []manifest.Service{{Name: "current-app", Image: "example/current:latest"}}}}, + }) + runCommand(t, runMigrate, []string{"--bundle", explicitBundle, "--run", "explicit-run"}) + runCommand(t, runMigrate, []string{"--bundle", currentBundle, "--run", "current-run"}) + + var stdout bytes.Buffer + var stderr bytes.Buffer + if err := runCommit(context.Background(), []string{"--run", "explicit-run"}, &stdout, &stderr); err != nil { + t.Fatalf("commit failed: %v\nstderr:\n%s", err, stderr.String()) + } + if !strings.Contains(stdout.String(), "explicit-app") || strings.Contains(stdout.String(), "current-app") { + t.Fatalf("expected explicit run to override current run, got:\n%s", stdout.String()) + } +} + +func TestRunCommitUsesDefaultBundleWithoutCurrentRun(t *testing.T) { + workDir := t.TempDir() + t.Chdir(workDir) + reviewedBundle := filepath.Join(workDir, "reviewed-bundle") + writeTestBundle(t, reviewedBundle, manifest.Manifest{ + Source: manifest.Source{Platform: "docker"}, + Apps: []manifest.App{{Name: "reviewed-app", Services: []manifest.Service{{Name: "reviewed-app", Image: "example/reviewed:latest"}}}}, + }) + runCommand(t, runMigrate, []string{"--bundle", reviewedBundle, "--run", "reviewed-run"}) + writeTestBundle(t, filepath.Join(workDir, "bort-bundle"), manifest.Manifest{ + Source: manifest.Source{Platform: "docker"}, + Apps: []manifest.App{{Name: "default-app", Services: []manifest.Service{{Name: "default-app", Image: "example/default:latest"}}}}, + }) + if err := mutateBortState(defaultStatePath(), func(state *bortState) bool { + state.CurrentRun = "" + return true + }); err != nil { + t.Fatal(err) + } + + var stdout bytes.Buffer + if err := runCommit(context.Background(), nil, &stdout, io.Discard); err != nil { + t.Fatal(err) + } + if !strings.Contains(stdout.String(), "default-app") || strings.Contains(stdout.String(), "reviewed-app") { + t.Fatalf("expected bare commit to plan from the default bundle without a current run, got:\n%s", stdout.String()) + } +} + +func TestNewRunRequiresSuccessfulOutcomeBeforeCommit(t *testing.T) { + workDir := t.TempDir() + t.Chdir(workDir) + bundleDir := filepath.Join(workDir, "bort-bundle") + writeTestBundle(t, bundleDir, manifest.Manifest{ + Source: manifest.Source{Platform: "docker"}, + Apps: []manifest.App{{Name: "api", Services: []manifest.Service{{Name: "api", Image: "example/api:latest"}}}}, + }) + runCommand(t, runMigrate, []string{"--bundle", bundleDir, "--run", "missing-outcome"}) + run, err := loadMigrationRun("missing-outcome") + if err != nil { + t.Fatal(err) + } + if !run.Run.ApplyOutcomeRequired { + t.Fatal("expected new run metadata to require a successful apply outcome") + } + steps := dokploy.PlanFromArtifacts(run.Prepare, run.Sync, run.Cutover).Steps + applied := newRunApplied(run.Run) + for index, step := range steps { + applied.Steps = append(applied.Steps, appliedStep{ + Index: index, + Kind: string(step.Kind), + App: step.App, + Ref: step.Ref, + Status: string(dokploy.StepStatusOK), + }) + } + appliedPath, err := safeRunArtifactPath(run.Run.RunDir, run.Run.Artifacts.Applied) + if err != nil { + t.Fatal(err) + } + if err := writeRunApplied(appliedPath, applied); err != nil { + t.Fatal(err) + } + run, err = loadMigrationRun("missing-outcome") + if err != nil { + t.Fatal(err) + } + var cockpit bytes.Buffer + writeAppFirstCockpit(&cockpit, run) + if strings.Contains(cockpit.String(), "TARGET LIVE") { + t.Fatalf("complete steps without a durable outcome were shown as target live:\n%s", cockpit.String()) + } + if err := applyCommitFromArgs(context.Background(), "missing-outcome", io.Discard); err == nil || !strings.Contains(err.Error(), "no successful live-apply outcome") { + t.Fatalf("expected commit to reject complete steps without a durable outcome, got %v", err) + } +} + +func TestLegacyCompleteLedgerDoesNotRequireSuccessfulOutcomeMarker(t *testing.T) { + run := loadedMigrationRun{ + Run: migrationRun{Name: "legacy-run"}, + Prepare: preparer.Result{Apps: []preparer.AppPlan{{Name: "api"}}}, + Cutover: gateway.Result{Apps: []gateway.AppPlan{{Name: "api", Routes: []gateway.Route{{Host: "api.example.com"}}}}}, + } + steps := dokploy.PlanFromArtifacts(run.Prepare, run.Sync, run.Cutover).Steps + for index, step := range steps { + run.Applied.Steps = append(run.Applied.Steps, appliedStep{ + Index: index, + Kind: string(step.Kind), + App: step.App, + Ref: step.Ref, + Status: string(dokploy.StepStatusOK), + }) + } + if err := requireLiveApplySucceeded(run); err != nil { + t.Fatalf("expected a legacy complete ledger to remain accepted: %v", err) + } +} + +func TestLiveApplyRecoveryErrorsPreserveExternalRunDirectory(t *testing.T) { + externalRunDir := filepath.Join(t.TempDir(), "selected-run") + run := loadedMigrationRun{ + Run: migrationRun{Name: "selected-run", RunDir: externalRunDir, ApplyOutcomeRequired: true}, + Prepare: preparer.Result{Apps: []preparer.AppPlan{{Name: "api"}}}, + Cutover: gateway.Result{Apps: []gateway.AppPlan{{Name: "api", Routes: []gateway.Route{{Host: "api.example.com"}}}}}, + } + want := liveApplyCommand(run) + if err := requireLiveApplySucceeded(run); err == nil || !strings.Contains(err.Error(), want) { + t.Fatalf("expected missing-outcome recovery command %q, got %v", want, err) + } + run.Run.ApplyOutcomeRequired = false + if err := requireLiveApplySucceeded(run); err == nil || !strings.Contains(err.Error(), want) { + t.Fatalf("expected missing-step recovery command %q, got %v", want, err) + } +} + func TestRequireLiveApplySucceededAcceptsSkippedPlatformSteps(t *testing.T) { run := loadedMigrationRun{ Run: migrationRun{Name: "run-1"}, @@ -111,6 +302,8 @@ func TestRequireLiveApplySucceededAcceptsSkippedPlatformSteps(t *testing.T) { }}}, } steps := dokploy.PlanFromArtifacts(run.Prepare, run.Sync, run.Cutover).Steps + succeededAt := time.Now().UTC() + run.Applied.SucceededAt = &succeededAt for index, step := range steps { status := string(dokploy.StepStatusOK) if step.App == "proxy" { @@ -128,3 +321,110 @@ func TestRequireLiveApplySucceededAcceptsSkippedPlatformSteps(t *testing.T) { t.Fatalf("expected skipped platform steps to count as completed: %v", err) } } + +func TestRequireLiveApplySucceededRejectsReorderedLedgerIndexes(t *testing.T) { + run := loadedMigrationRun{ + Run: migrationRun{Name: "run-1"}, + Prepare: preparer.Result{Apps: []preparer.AppPlan{{Name: "api"}, {Name: "worker"}}}, + Cutover: gateway.Result{Apps: []gateway.AppPlan{ + {Name: "api", Routes: []gateway.Route{{Host: "api.example.com"}}}, + {Name: "worker", Routes: []gateway.Route{{Host: "worker.example.com"}}}, + }}, + } + steps := dokploy.PlanFromArtifacts(run.Prepare, run.Sync, run.Cutover).Steps + for index := len(steps) - 1; index >= 0; index-- { + step := steps[index] + run.Applied.Steps = append(run.Applied.Steps, appliedStep{ + Index: len(steps) - 1 - index, + Kind: string(step.Kind), + App: step.App, + Ref: step.Ref, + Status: string(dokploy.StepStatusOK), + }) + } + if err := requireLiveApplySucceeded(run); err == nil || !strings.Contains(err.Error(), "plan index") { + t.Fatalf("expected reordered completed ledger steps to be rejected, got %v", err) + } +} + +func TestRequireLiveApplySucceededRejectsLaterFailedLedgerEntry(t *testing.T) { + run := loadedMigrationRun{ + Run: migrationRun{Name: "run-1"}, + Prepare: preparer.Result{Apps: []preparer.AppPlan{{Name: "api"}}}, + Cutover: gateway.Result{Apps: []gateway.AppPlan{{Name: "api", Routes: []gateway.Route{{Host: "api.example.com"}}}}}, + } + steps := dokploy.PlanFromArtifacts(run.Prepare, run.Sync, run.Cutover).Steps + for index, step := range steps { + run.Applied.Steps = append(run.Applied.Steps, appliedStep{Index: index, Kind: string(step.Kind), App: step.App, Ref: step.Ref, Status: string(dokploy.StepStatusOK), UpdatedAt: time.Unix(1, 0)}) + } + failed := steps[0] + run.Applied.Steps = append(run.Applied.Steps, appliedStep{Index: len(steps), Kind: string(failed.Kind), App: failed.App, Ref: failed.Ref, Status: string(dokploy.StepStatusError), UpdatedAt: time.Unix(2, 0)}) + if err := requireLiveApplySucceeded(run); err == nil || !strings.Contains(err.Error(), string(failed.Kind)) { + t.Fatalf("expected later failed ledger entry to invalidate historical success, got %v", err) + } +} + +func TestRequireLiveApplySucceededForAppsIgnoresUnselectedApps(t *testing.T) { + run := loadedMigrationRun{ + Run: migrationRun{Name: "run-1"}, + Prepare: preparer.Result{Apps: []preparer.AppPlan{{Name: "api"}, {Name: "worker"}}}, + Cutover: gateway.Result{Apps: []gateway.AppPlan{ + {Name: "api", Routes: []gateway.Route{{Host: "api.example.com"}}}, + {Name: "worker", Routes: []gateway.Route{{Host: "worker.example.com"}}}, + }}, + } + steps := dokploy.PlanFromArtifacts(run.Prepare, run.Sync, run.Cutover).Steps + succeededAt := time.Now().UTC() + run.Applied.SucceededAt = &succeededAt + for index, step := range steps { + if step.App != "" && step.App != "api" { + continue + } + run.Applied.Steps = append(run.Applied.Steps, appliedStep{ + Index: index, + Kind: string(step.Kind), + App: step.App, + Ref: step.Ref, + Status: string(dokploy.StepStatusOK), + }) + } + if err := requireLiveApplySucceededForApps(run, map[string]struct{}{"api": {}}); err != nil { + t.Fatalf("expected selected app's completed ledger steps to count as successful live apply: %v", err) + } + if err := requireLiveApplySucceeded(run); err == nil || !strings.Contains(err.Error(), "worker") { + t.Fatalf("expected all-app guard to still reject missing worker steps, got %v", err) + } +} + +func TestRequireLiveApplySucceededForAppsSkippingAllowsMissingSkippedKinds(t *testing.T) { + run := loadedMigrationRun{ + Run: migrationRun{Name: "run-1"}, + Prepare: preparer.Result{Apps: []preparer.AppPlan{{Name: "api"}}}, + Cutover: gateway.Result{Apps: []gateway.AppPlan{{Name: "api", Routes: []gateway.Route{{Host: "api.example.com"}}}}}, + } + steps := dokploy.PlanFromArtifacts(run.Prepare, run.Sync, run.Cutover).Steps + if len(steps) == 0 { + t.Fatal("expected live plan steps") + } + succeededAt := time.Now().UTC() + run.Applied.SucceededAt = &succeededAt + skipKind := steps[0].Kind + for index, step := range steps { + if step.Kind == skipKind { + continue + } + run.Applied.Steps = append(run.Applied.Steps, appliedStep{ + Index: index, + Kind: string(step.Kind), + App: step.App, + Ref: step.Ref, + Status: string(dokploy.StepStatusOK), + }) + } + if err := requireLiveApplySucceededForAppsSkipping(run, nil, map[dokploy.StepKind]struct{}{skipKind: {}}); err != nil { + t.Fatalf("expected missing skipped step kind to be ignored: %v", err) + } + if err := requireLiveApplySucceeded(run); err == nil || !strings.Contains(err.Error(), string(skipKind)) { + t.Fatalf("expected regular guard to reject missing %s step, got %v", skipKind, err) + } +} diff --git a/internal/cli/data.go b/internal/cli/data.go index a258b3b..2a319b4 100644 --- a/internal/cli/data.go +++ b/internal/cli/data.go @@ -15,8 +15,13 @@ import ( // Exactly one strategy flag must be supplied. The recorded strategy is read // back by `bort` so the user's choice persists across runs. func runData(_ context.Context, args []string, stdout, stderr io.Writer) error { + usage := "usage: " + bortCommand("data --recreate|--migrate|--managed") + if len(args) == 1 && (args[0] == "-h" || args[0] == "--help") { + fmt.Fprintln(stdout, usage) + return nil + } if len(args) < 2 { - return fmt.Errorf("usage: bort data --recreate|--migrate|--managed") + return fmt.Errorf("%s", usage) } app := strings.TrimSpace(args[0]) store := strings.TrimSpace(args[1]) @@ -42,12 +47,10 @@ func runData(_ context.Context, args []string, stdout, stderr io.Writer) error { } statePath := defaultStatePath() - state, err := readBortState(statePath) - if err != nil { - return err - } - state = setAppDataStrategy(state, app, store, strategy) - if err := writeBortState(statePath, state); err != nil { + if err := mutateBortState(statePath, func(state *bortState) bool { + *state = setAppDataStrategy(*state, app, store, strategy) + return true + }); err != nil { return err } @@ -57,7 +60,7 @@ func runData(_ context.Context, args []string, stdout, stderr io.Writer) error { st.emph(strategy), st.emph(app+"/"+store), ) - fmt.Fprintln(stdout, st.muted("Stored in .bort/state.json. Run `bort` to recheck.")) + fmt.Fprintf(stdout, "%s\n", st.muted(fmt.Sprintf("Stored in .bort/state.json. Run `%s` to recheck.", bortCommand("")))) return nil } diff --git a/internal/cli/env.go b/internal/cli/env.go index 91b4c27..18fc4c0 100644 --- a/internal/cli/env.go +++ b/internal/cli/env.go @@ -15,8 +15,13 @@ import ( // modify the source apps; subsequent `bort` runs read the recorded values // instead of asking again. func runEnv(_ context.Context, args []string, stdout, _ io.Writer) error { + usage := "usage: " + bortCommand("env KEY=value [KEY=value ...]") + if len(args) == 1 && (args[0] == "-h" || args[0] == "--help") { + fmt.Fprintln(stdout, usage) + return nil + } if len(args) < 2 { - return fmt.Errorf("usage: bort env KEY=value [KEY=value ...]") + return fmt.Errorf("%s", usage) } app := strings.TrimSpace(args[0]) if app == "" { @@ -31,12 +36,10 @@ func runEnv(_ context.Context, args []string, stdout, _ io.Writer) error { } statePath := defaultStatePath() - state, err := readBortState(statePath) - if err != nil { - return err - } - state = setAppEnv(state, app, values) - if err := writeBortState(statePath, state); err != nil { + if err := mutateBortState(statePath, func(state *bortState) bool { + *state = setAppEnv(*state, app, values) + return true + }); err != nil { return err } @@ -48,7 +51,7 @@ func runEnv(_ context.Context, args []string, stdout, _ io.Writer) error { st.emph(app), strings.Join(keys, ", "), ) - fmt.Fprintln(stdout, st.muted("Stored in .bort/state.json (mode 0600). Run `bort` to recheck.")) + fmt.Fprintf(stdout, "%s\n", st.muted(fmt.Sprintf("Stored in .bort/state.json (mode 0600). Run `%s` to recheck.", bortCommand("")))) return nil } diff --git a/internal/cli/guide.go b/internal/cli/guide.go index 2f89afe..60d1899 100644 --- a/internal/cli/guide.go +++ b/internal/cli/guide.go @@ -37,20 +37,40 @@ type guidePromptInput interface { } func runGuide(ctx context.Context, stdin io.Reader, stdout, stderr io.Writer) error { - if runRef, ok := latestRunRef(); ok { - if applyRunActive(runRef) { + runRef, currentSelected, ok, err := guideRunRef() + if err != nil { + return err + } + if ok { + if !currentSelected { run, err := loadMigrationRun(runRef) if err != nil { return err } writeAppFirstCockpit(stdout, run) + fmt.Fprintf(stdout, "This existing run is not selected for mutation. Run `%s` to make it current and refresh its dry-run.\n", bortCommand("migrate --run "+shellQuote(run.Run.Name))) + return nil + } + applyActive, applyActiveErr := applyRunActive(runRef) + if applyActive || applyActiveErr != nil { + run, err := loadMigrationRun(runRef) + if err != nil { + return err + } + if err := rememberCurrentRun(run.Run); err != nil { + return err + } + writeAppFirstCockpit(stdout, run) return nil } run, err := refreshGuideRun(ctx, runRef, stdin, stdout) if err != nil { return err } - if isRealTTY(stdin, stdout) && readyToOfferLiveApply(run) { + if err := rememberCurrentRun(run.Run); err != nil { + return err + } + if isRealTTY(stdin, stdout) { return runWizard(ctx, run, stdin, stdout, stderr) } writeAppFirstCockpit(stdout, run) @@ -67,6 +87,9 @@ func runGuide(ctx context.Context, stdin io.Reader, stdout, stderr io.Writer) er if err != nil { return err } + if err := rememberCurrentRun(run.Run); err != nil { + return err + } writeAppFirstCockpit(stdout, run) return nil } @@ -92,6 +115,9 @@ func runGuide(ctx context.Context, stdin io.Reader, stdout, stderr io.Writer) er if err != nil { return err } + if err := rememberCurrentRun(run.Run); err != nil { + return err + } if isRealTTY(stdin, stdout) { return runWizard(ctx, run, stdin, stdout, stderr) } @@ -102,29 +128,45 @@ func runGuide(ctx context.Context, stdin io.Reader, stdout, stderr io.Writer) er return writeGuideStart(stdout) } -func readyToOfferLiveApply(run loadedMigrationRun) bool { - return overallAppHealth(appsFromRun(run)) == appHealthReady -} - func refreshGuideRun(ctx context.Context, runRef string, stdin io.Reader, stdout io.Writer) (loadedMigrationRun, error) { + operationLock, err := acquireRunOperationLock(runRef) + if err != nil { + return loadedMigrationRun{}, err + } + defer operationLock.Release() runDir := filepath.FromSlash(runRef) existing, err := readRunMetadata(filepath.Join(runDir, "run.json")) if err != nil { return loadedMigrationRun{}, err } - if shouldAutoRescanRun(existing) && !runHasAppliedSteps(existing) { + hasAppliedSteps, err := runHasAppliedSteps(existing) + if err != nil { + return loadedMigrationRun{}, fmt.Errorf("read applied migration progress: %w", err) + } + if existing.LiveAppliedAt != nil || existing.CommittedAt != nil || existing.PurgedAt != nil || hasAppliedSteps { + return loadMigrationRun(runRef) + } + if shouldAutoRescanRun(existing) { + var refreshedBundle, refreshedManifest string if isRealTTY(stdin, stdout) { stopStatus := startScanStatus(stdout) - err = refreshRunSourceBundle(ctx, existing) + refreshedBundle, refreshedManifest, err = refreshRunSourceBundle(ctx, existing) stopStatus(err) } else { - err = refreshRunSourceBundle(ctx, existing) + refreshedBundle, refreshedManifest, err = refreshRunSourceBundle(ctx, existing) } if err != nil { return loadedMigrationRun{}, err } + defer os.RemoveAll(refreshedBundle) + refreshed, err := refreshMigrationRunLockedWithInputs(runRef, refreshedBundle, refreshedManifest) + if err != nil { + _ = os.Remove(refreshedManifest) + return loadedMigrationRun{}, err + } + return refreshed, nil } - return refreshMigrationRun(runRef) + return refreshMigrationRunLocked(runRef) } func shouldAutoRescanRun(run migrationRun) bool { @@ -140,36 +182,36 @@ func shouldAutoRescanRun(run migrationRun) bool { return containedPath(runDir, bundleDir) == nil } -func runHasAppliedSteps(run migrationRun) bool { +func runHasAppliedSteps(run migrationRun) (bool, error) { run.Artifacts = run.Artifacts.withDefaults() runDir := filepath.FromSlash(run.RunDir) if runDir == "" { - return false + return false, fmt.Errorf("migration run has no run directory") } appliedPath, err := safeRunArtifactPath(runDir, run.Artifacts.Applied) if err != nil { - return false + return false, err } applied, err := readRunApplied(appliedPath, run) - return err == nil && len(applied.Steps) > 0 + if err != nil { + return false, err + } + return len(applied.Steps) > 0, nil } -func refreshRunSourceBundle(ctx context.Context, run migrationRun) error { +func refreshRunSourceBundle(ctx context.Context, run migrationRun) (string, string, error) { runDir := filepath.FromSlash(run.RunDir) - bundleDir := filepath.FromSlash(run.BundleDir) setup := guidedSetup{Source: run.Source, Target: run.Target, ManifestPath: run.ManifestPath} - m, _, err := guidedManifest(ctx, setup, runDir) + m, manifestPath, err := guidedManifest(ctx, setup, runDir) if err != nil { - return err + return "", "", err } - if err := containedPath(runDir, bundleDir); err != nil { - return err - } - if err := os.RemoveAll(bundleDir); err != nil { - return err + bundleDir, err := exportRunSourceBundle(runDir, m, run.AppName) + if err != nil { + _ = os.Remove(manifestPath) + return "", "", err } - _, err = exporter.Export(m, exporter.Options{OutputDir: bundleDir, AppName: run.AppName, IncludeEnvValues: true}) - return err + return bundleDir, manifestPath, nil } // isRealTTY reports whether stdin and stdout are both attached to an @@ -262,36 +304,84 @@ func promptGuidedSetupWithReader(reader *bufio.Reader, stdout io.Writer, now tim } func createGuidedMigrationRun(ctx context.Context, setup guidedSetup) (loadedMigrationRun, error) { - runDir, _ := newRunDir(setup.RunName, "", "", time.Now().UTC()) + return createMigrationRunFromSource(ctx, setup, gateway.DefaultObservationWindowSeconds, gateway.DefaultRollbackWindowSeconds) +} + +func createMigrationRunFromSource(ctx context.Context, setup guidedSetup, observationWindowSeconds, rollbackWindowSeconds int) (loadedMigrationRun, error) { + now := time.Now().UTC() + runDir, runName := newRunDir(setup.RunName, "", "", now) if err := ensurePrivateRunDir(runDir); err != nil { return loadedMigrationRun{}, err } + operationLock, err := acquireRunOperationLock(runDir) + if err != nil { + return loadedMigrationRun{}, err + } + defer operationLock.Release() + if _, err := existingMutableMigrationRun(runDir, runName); err != nil { + return loadedMigrationRun{}, err + } m, manifestPath, err := guidedManifest(ctx, setup, runDir) if err != nil { return loadedMigrationRun{}, err } + removeManifest := true + defer func() { + if removeManifest { + _ = os.Remove(manifestPath) + } + }() - bundleDir := filepath.Join(runDir, "bundle") - if _, err := exporter.Export(m, exporter.Options{OutputDir: bundleDir, IncludeEnvValues: true}); err != nil { + bundleDir, err := exportRunSourceBundle(runDir, m, "") + if err != nil { return loadedMigrationRun{}, err } + defer os.RemoveAll(bundleDir) - return createMigrationRun(migrationRunOptions{ + run, err := createMigrationRunLocked(migrationRunOptions{ BundleDir: bundleDir, Target: setup.Target, RunRef: runDir, Source: setup.Source, ManifestPath: manifestPath, - ObservationWindowSeconds: gateway.DefaultObservationWindowSeconds, - RollbackWindowSeconds: gateway.DefaultRollbackWindowSeconds, - }) + ObservationWindowSeconds: observationWindowSeconds, + RollbackWindowSeconds: rollbackWindowSeconds, + }, runDir, runName, now) + if err != nil { + return loadedMigrationRun{}, err + } + removeManifest = false + return run, nil +} + +func exportRunSourceBundle(runDir string, m manifest.Manifest, appName string) (string, error) { + bundleDir, err := os.MkdirTemp(runDir, "source-bundle-") + if err != nil { + return "", err + } + if err := os.Chmod(bundleDir, 0o700); err != nil { + _ = os.RemoveAll(bundleDir) + return "", err + } + if _, err := exporter.Export(m, exporter.Options{OutputDir: bundleDir, AppName: appName, IncludeEnvValues: true}); err != nil { + _ = os.RemoveAll(bundleDir) + return "", err + } + return bundleDir, nil } func guidedManifest(ctx context.Context, setup guidedSetup, runDir string) (manifest.Manifest, string, error) { if setup.Source == "manifest" { m, err := readManifestFile(setup.ManifestPath) - return m, setup.ManifestPath, err + if err != nil { + return manifest.Manifest{}, "", err + } + manifestPath, err := writeRunSourceManifest(runDir, m) + if err != nil { + return manifest.Manifest{}, "", err + } + return m, manifestPath, nil } scanOptions := source.ScanOptions{ @@ -309,13 +399,34 @@ func guidedManifest(ctx context.Context, setup guidedSetup, runDir string) (mani if err != nil { return manifest.Manifest{}, "", err } - manifestPath := filepath.Join(runDir, "manifest.json") - if err := writeJSONArtifact(manifestPath, m); err != nil { + manifestPath, err := writeRunSourceManifest(runDir, m) + if err != nil { return manifest.Manifest{}, "", err } return m, manifestPath, nil } +func writeRunSourceManifest(runDir string, m manifest.Manifest) (string, error) { + file, err := os.CreateTemp(runDir, "manifest-*.json") + if err != nil { + return "", err + } + path := file.Name() + if err := file.Close(); err != nil { + _ = os.Remove(path) + return "", err + } + if err := os.Chmod(path, 0o600); err != nil { + _ = os.Remove(path) + return "", err + } + if err := writeJSONArtifact(path, m); err != nil { + _ = os.Remove(path) + return "", err + } + return path, nil +} + func readManifestFile(path string) (manifest.Manifest, error) { file, err := os.Open(path) if err != nil { @@ -445,6 +556,53 @@ func latestRunRef() (string, bool) { return runs[0].Path, true } +func guideRunRef() (string, bool, bool, error) { + current, ok, err := currentRunRef() + if err != nil { + return "", false, false, err + } + if ok { + if !migrationRunMetadataExists(current) { + return "", false, false, fmt.Errorf("current migration run %q no longer exists; run `%s` to select another run", current, bortCommand("migrate --run ")) + } + return current, true, true, nil + } + latest, found := latestRunRef() + return latest, false, found, nil +} + +func selectedRunRef(allowLatest bool) (string, bool, error) { + current, ok, err := currentRunRef() + if err != nil { + return "", false, err + } + if ok && migrationRunMetadataExists(current) { + return current, true, nil + } + if ok && !allowLatest { + return "", false, fmt.Errorf("current migration run %q no longer exists; pass --run to select a run", current) + } + if allowLatest { + latest, found := latestRunRef() + return latest, found, nil + } + return "", false, nil +} + +func resolveRunRef(explicit string, allowLatest bool) (string, error) { + if ref := strings.TrimSpace(explicit); ref != "" { + return ref, nil + } + ref, ok, err := selectedRunRef(allowLatest) + if err != nil { + return "", err + } + if ok { + return ref, nil + } + return "", fmt.Errorf("no current migration run; run `%s` to start one or pass --run", bortCommand("")) +} + type runCandidate struct { Path string UpdatedAt time.Time @@ -459,17 +617,12 @@ func writeGuideStart(w io.Writer) error { st := newStyler(w) fmt.Fprintln(w, st.emph("bort")+" "+st.muted("— migrate self-hosted apps between PaaS platforms")) fmt.Fprintln(w) - fmt.Fprintln(w, st.muted("No local migration run or default bundle was found.")) + fmt.Fprintf(w, "%s\n", st.muted(fmt.Sprintf("No migration run was found. Run `%s` in a terminal for guided setup.", bortCommand("")))) fmt.Fprintln(w) - fmt.Fprintln(w, st.emph("To get started:")) - for _, cmd := range []string{ - "bort scan --output manifest.json", - "bort export --manifest manifest.json --output-dir bort-bundle", - "bort", - } { - fmt.Fprintf(w, " %s\n", st.emph(cmd)) - } + fmt.Fprintln(w, st.emph("For non-interactive setup:")) + fmt.Fprintf(w, " %s\n", st.emph(bortCommand("migrate --source coolify-local"))) + fmt.Fprintf(w, " %s\n", st.muted("or: "+bortCommand("migrate --manifest manifest.json"))) fmt.Fprintln(w) - fmt.Fprintln(w, st.muted("See `bort help` for more.")) + fmt.Fprintln(w, st.muted("This single command scans, exports a private bundle, and creates the current run.")) return nil } diff --git a/internal/cli/guide_test.go b/internal/cli/guide_test.go index d5137ca..854f78f 100644 --- a/internal/cli/guide_test.go +++ b/internal/cli/guide_test.go @@ -29,13 +29,39 @@ func TestRunWithoutArgsShowsGuidedStartWhenNoBundleOrRunExists(t *testing.T) { } output := stdout.String() - for _, want := range []string{"No local migration run or default bundle was found.", "bort export --manifest manifest.json --output-dir bort-bundle", "bort help"} { + for _, want := range []string{"No migration run was found.", "bort migrate --source coolify-local", "bort migrate --manifest manifest.json"} { if !strings.Contains(output, want) { t.Fatalf("expected guide output to contain %q, got:\n%s", want, output) } } } +func TestCommandHelpReturnsSuccess(t *testing.T) { + for _, args := range [][]string{ + {"help"}, + {"help", "--advanced"}, + {"migrate", "-h"}, + {"rollback", "-h"}, + {"commit", "-h"}, + {"cleanup", "-h"}, + {"cleanup", "purge", "-h"}, + {"status", "-h"}, + {"env", "-h"}, + {"data", "-h"}, + } { + t.Run(strings.Join(args, "_"), func(t *testing.T) { + var stdout bytes.Buffer + var stderr bytes.Buffer + if err := RunWithInput(context.Background(), args, strings.NewReader(""), &stdout, &stderr); err != nil { + t.Fatalf("help failed: %v\nstdout:\n%s\nstderr:\n%s", err, stdout.String(), stderr.String()) + } + if strings.Contains(stdout.String(), "flag: help requested") || strings.Contains(stderr.String(), "flag: help requested") { + t.Fatalf("help leaked flag.ErrHelp:\nstdout:\n%s\nstderr:\n%s", stdout.String(), stderr.String()) + } + }) + } +} + func TestRunWithoutArgsCreatesRunFromDefaultBundle(t *testing.T) { workDir := t.TempDir() t.Chdir(workDir) @@ -53,7 +79,7 @@ func TestRunWithoutArgsCreatesRunFromDefaultBundle(t *testing.T) { } output := stdout.String() - for _, want := range []string{"local bundle → dokploy", "api", "DRY RUN"} { + for _, want := range []string{"local bundle → dokploy", "api", "READY"} { if !strings.Contains(output, want) { t.Fatalf("expected guide output to contain %q, got:\n%s", want, output) } @@ -87,7 +113,7 @@ func TestRunWithoutArgsResumesLatestRun(t *testing.T) { } output := stdout.String() - for _, want := range []string{"local bundle → dokploy", "DRY RUN"} { + for _, want := range []string{"local bundle → dokploy", "READY"} { if !strings.Contains(output, want) { t.Fatalf("expected resumed guide output to contain %q, got:\n%s", want, output) } @@ -99,6 +125,78 @@ func TestRunWithoutArgsResumesLatestRun(t *testing.T) { } } +func TestRunWithoutArgsDoesNotPromoteOrRefreshMtimeFallback(t *testing.T) { + workDir := t.TempDir() + t.Chdir(workDir) + bundleDir := filepath.Join(workDir, "bort-bundle") + writeTestBundle(t, bundleDir, manifest.Manifest{ + Source: manifest.Source{Platform: "docker"}, + Apps: []manifest.App{{Name: "legacy-app", Services: []manifest.Service{{Name: "legacy-app", Image: "example/legacy:latest"}}}}, + }) + runCommand(t, runMigrate, []string{"--bundle", bundleDir, "--run", "legacy-run"}) + statePath := filepath.Join(workDir, ".bort", "state.json") + if err := os.Remove(statePath); err != nil { + t.Fatal(err) + } + metadataPath := filepath.Join(workDir, ".bort", "runs", "legacy-run", "run.json") + before, err := os.ReadFile(metadataPath) + if err != nil { + t.Fatal(err) + } + + var stdout bytes.Buffer + var stderr bytes.Buffer + if err := RunWithInput(context.Background(), nil, strings.NewReader(""), &stdout, &stderr); err != nil { + t.Fatalf("guide failed: %v\nstderr:\n%s", err, stderr.String()) + } + after, err := os.ReadFile(metadataPath) + if err != nil { + t.Fatal(err) + } + if string(after) != string(before) { + t.Fatalf("bare bort refreshed an mtime-selected run") + } + if _, err := os.Stat(statePath); !os.IsNotExist(err) { + t.Fatalf("bare bort promoted an mtime-selected run to current: %v", err) + } + for _, want := range []string{"legacy-app", "not selected for mutation", "bort migrate --run legacy-run"} { + if !strings.Contains(stdout.String(), want) { + t.Fatalf("expected fallback output to contain %q, got:\n%s", want, stdout.String()) + } + } +} + +func TestRunWithoutArgsSurfacesApplyLockProbeError(t *testing.T) { + workDir := t.TempDir() + t.Chdir(workDir) + bundleDir := filepath.Join(workDir, "bort-bundle") + writeTestBundle(t, bundleDir, manifest.Manifest{ + Source: manifest.Source{Platform: "docker"}, + Apps: []manifest.App{{Name: "api", Services: []manifest.Service{{Name: "api", Image: "example/api:latest"}}}}, + }) + runCommand(t, runMigrate, []string{"--bundle", bundleDir, "--run", "lock-error"}) + run, err := loadMigrationRun("lock-error") + if err != nil { + t.Fatal(err) + } + target := filepath.Join(workDir, "unexpected-lock") + if err := os.WriteFile(target, nil, 0o600); err != nil { + t.Fatal(err) + } + if err := os.Symlink(target, filepath.Join(run.Run.RunDir, "apply.lock")); err != nil { + t.Fatal(err) + } + + var stdout bytes.Buffer + var stderr bytes.Buffer + if err := RunWithInput(context.Background(), nil, strings.NewReader(""), &stdout, &stderr); err != nil { + t.Fatalf("guide failed instead of surfacing lock recovery: %v", err) + } + if !strings.Contains(stdout.String(), "LOCK ERROR") || !strings.Contains(stdout.String(), "apply.lock") { + t.Fatalf("expected lock recovery guidance, got:\n%s", stdout.String()) + } +} + func TestRunWithoutArgsPromptsForManifestAndPrivateEnvMode(t *testing.T) { workDir := t.TempDir() t.Chdir(workDir) @@ -146,10 +244,16 @@ func TestRunWithoutArgsPromptsForManifestAndPrivateEnvMode(t *testing.T) { } runName := entries[0].Name() run := readJSONFile[migrationRun](t, filepath.Join(workDir, ".bort", "runs", runName, "run.json")) - if run.Source != "manifest" || run.ManifestPath != manifestPath { + if run.Source != "manifest" || run.ManifestPath == manifestPath || !strings.HasPrefix(filepath.Base(run.ManifestPath), "manifest-") { t.Fatalf("unexpected guided run metadata: %#v", run) } - envPath := filepath.Join(workDir, ".bort", "runs", runName, "bundle", "api", ".env.api") + if err := containedPath(run.RunDir, run.ManifestPath); err != nil { + t.Fatalf("expected guided manifest to be a private run artifact: %v", err) + } + if err := containedPath(run.RunDir, run.BundleDir); err != nil { + t.Fatalf("expected guided source bundle to be a private run snapshot: %v", err) + } + envPath := filepath.Join(run.BundleDir, "api", ".env.api") envFile, err := os.ReadFile(envPath) if err != nil { t.Fatal(err) @@ -157,7 +261,7 @@ func TestRunWithoutArgsPromptsForManifestAndPrivateEnvMode(t *testing.T) { if !strings.Contains(string(envFile), "API_TOKEN=private-token") { t.Fatalf("expected private env file to preserve known value, got %q", string(envFile)) } - envExamplePath := filepath.Join(workDir, ".bort", "runs", runName, "bundle", "api", ".env.api.example") + envExamplePath := filepath.Join(run.BundleDir, "api", ".env.api.example") envExample, err := os.ReadFile(envExamplePath) if err != nil { t.Fatal(err) @@ -166,3 +270,215 @@ func TestRunWithoutArgsPromptsForManifestAndPrivateEnvMode(t *testing.T) { t.Fatalf("example env file exposed private value: %q", string(envExample)) } } + +func TestSourceRefreshPublishesVersionedManifestWithPlan(t *testing.T) { + workDir := t.TempDir() + t.Chdir(workDir) + before := prepareScannableGuideRun(t, workDir, "source-refresh") + metadataPath := filepath.Join(before.Run.RunDir, "run.json") + metadataBefore, err := os.ReadFile(metadataPath) + if err != nil { + t.Fatal(err) + } + manifestBefore, err := os.ReadFile(before.Run.ManifestPath) + if err != nil { + t.Fatal(err) + } + + refreshedBundle, refreshedManifest, err := refreshRunSourceBundle(context.Background(), before.Run) + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(refreshedBundle) + if refreshedManifest == before.Run.ManifestPath || !strings.HasPrefix(filepath.Base(refreshedManifest), "manifest-") { + t.Fatalf("expected a new versioned source manifest, old=%s new=%s", before.Run.ManifestPath, refreshedManifest) + } + publishedBefore, err := readRunMetadata(metadataPath) + if err != nil { + t.Fatal(err) + } + if publishedBefore.ManifestPath != before.Run.ManifestPath { + t.Fatalf("new source manifest became visible before plan publication: %#v", publishedBefore) + } + if current, err := os.ReadFile(metadataPath); err != nil || !bytes.Equal(current, metadataBefore) { + t.Fatalf("source scan changed published metadata before plan publication: err=%v", err) + } + if current, err := os.ReadFile(before.Run.ManifestPath); err != nil || !bytes.Equal(current, manifestBefore) { + t.Fatalf("source scan changed the published manifest: err=%v", err) + } + + operationLock, err := acquireRunOperationLock(before.Run.RunDir) + if err != nil { + t.Fatal(err) + } + after, refreshErr := refreshMigrationRunLockedWithInputs(before.Run.RunDir, refreshedBundle, refreshedManifest) + operationLock.Release() + if refreshErr != nil { + t.Fatal(refreshErr) + } + if after.Run.ManifestPath != refreshedManifest { + t.Fatalf("plan publication did not switch to the new manifest: %#v", after.Run) + } + for _, name := range []string{ + after.Run.Artifacts.Prepare, + after.Run.Artifacts.Sync, + after.Run.Artifacts.Cutover, + after.Run.Artifacts.Rollback, + after.Run.Artifacts.Commit, + after.Run.Artifacts.Decisions, + after.Run.Artifacts.Progress, + after.Run.Artifacts.Applied, + } { + if _, err := os.Stat(runArtifactPath(after.Run.RunDir, name)); err != nil { + t.Fatalf("published artifact %s does not exist: %v", name, err) + } + } + if current, err := os.ReadFile(before.Run.ManifestPath); err != nil || !bytes.Equal(current, manifestBefore) { + t.Fatalf("source refresh changed the previous manifest generation: err=%v", err) + } +} + +func TestFailedSourceRefreshPreservesPublishedManifestAndMetadata(t *testing.T) { + workDir := t.TempDir() + t.Chdir(workDir) + before := prepareScannableGuideRun(t, workDir, "failed-source-refresh") + metadataPath := filepath.Join(before.Run.RunDir, "run.json") + metadataBefore, err := os.ReadFile(metadataPath) + if err != nil { + t.Fatal(err) + } + manifestBefore, err := os.ReadFile(before.Run.ManifestPath) + if err != nil { + t.Fatal(err) + } + progressPath := runArtifactPath(before.Run.RunDir, before.Run.Artifacts.Progress) + if err := os.WriteFile(progressPath, []byte("{"), 0o600); err != nil { + t.Fatal(err) + } + + if _, err := refreshGuideRun(context.Background(), before.Run.RunDir, strings.NewReader(""), io.Discard); err == nil { + t.Fatal("expected source refresh publication to fail") + } + metadataAfter, err := os.ReadFile(metadataPath) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(metadataAfter, metadataBefore) { + t.Fatal("failed source refresh changed published run metadata") + } + manifestAfter, err := os.ReadFile(before.Run.ManifestPath) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(manifestAfter, manifestBefore) { + t.Fatal("failed source refresh changed the published manifest") + } + manifests, err := filepath.Glob(filepath.Join(before.Run.RunDir, "manifest-*.json")) + if err != nil { + t.Fatal(err) + } + if len(manifests) != 1 || manifests[0] != before.Run.ManifestPath { + t.Fatalf("failed source refresh left an unpublished manifest: %#v", manifests) + } + entries, err := os.ReadDir(before.Run.RunDir) + if err != nil { + t.Fatal(err) + } + for _, entry := range entries { + if strings.HasPrefix(entry.Name(), "source-bundle-") || strings.HasPrefix(entry.Name(), "plan-") { + t.Fatalf("failed source refresh left unpublished generation %s", entry.Name()) + } + } +} + +func TestSourceRefreshRejectsInvalidAppliedLedger(t *testing.T) { + workDir := t.TempDir() + t.Chdir(workDir) + run := prepareScannableGuideRun(t, workDir, "invalid-applied") + metadataPath := filepath.Join(run.Run.RunDir, "run.json") + before, err := os.ReadFile(metadataPath) + if err != nil { + t.Fatal(err) + } + appliedPath := runArtifactPath(run.Run.RunDir, run.Run.Artifacts.Applied) + if err := os.WriteFile(appliedPath, []byte("{"), 0o600); err != nil { + t.Fatal(err) + } + + if _, err := refreshGuideRun(context.Background(), run.Run.RunDir, strings.NewReader(""), io.Discard); err == nil || !strings.Contains(err.Error(), "read applied migration progress") { + t.Fatalf("expected invalid applied ledger to block refresh, got %v", err) + } + after, err := os.ReadFile(metadataPath) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(after, before) { + t.Fatal("invalid applied ledger allowed the reviewed plan to be regenerated") + } +} + +func prepareScannableGuideRun(t *testing.T, workDir, runName string) loadedMigrationRun { + t.Helper() + binDir := t.TempDir() + dockerPath := filepath.Join(binDir, "docker") + dockerScript := `#!/bin/sh +if [ "$1" = "ps" ]; then + printf 'reviewed-container\n' +elif [ "$1" = "inspect" ]; then + printf '%s\n' '[{"Id":"reviewed-container","Name":"/reviewed","Image":"sha256:reviewed","Config":{"Image":"example/reviewed:v2","Labels":{}},"State":{"Status":"running"},"Mounts":[],"NetworkSettings":{"Ports":{},"Networks":{}}}]' +elif [ "$1" = "image" ]; then + printf '%s\n' '[{"Id":"sha256:reviewed","RepoDigests":[]}]' +fi +` + if err := os.WriteFile(dockerPath, []byte(dockerScript), 0o700); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", binDir+string(os.PathListSeparator)+os.Getenv("PATH")) + bundleDir := filepath.Join(workDir, "legacy-source-bundle") + writeTestBundle(t, bundleDir, manifest.Manifest{ + Source: manifest.Source{Platform: "docker"}, + Apps: []manifest.App{{Name: "reviewed", Services: []manifest.Service{{Name: "reviewed", Image: "example/reviewed:v1"}}}}, + }) + runCommand(t, runMigrate, []string{"--bundle", bundleDir, "--run", runName}) + run, err := loadMigrationRun(runName) + if err != nil { + t.Fatal(err) + } + manifestPath, err := writeRunSourceManifest(run.Run.RunDir, manifest.Manifest{ + Source: manifest.Source{Platform: "docker"}, + Apps: []manifest.App{{Name: "reviewed", Services: []manifest.Service{{Name: "reviewed", Image: "example/reviewed:v1"}}}}, + }) + if err != nil { + t.Fatal(err) + } + run.Run.Source = "docker" + run.Run.BundleDir = bundleDir + run.Run.SourceBundleDir = "" + run.Run.ManifestPath = manifestPath + run.Prepare.BundleDir = bundleDir + run.Sync.BundleDir = bundleDir + run.Cutover.BundleDir = bundleDir + run.Rollback.BundleDir = bundleDir + run.Commit.BundleDir = bundleDir + run.Decisions.BundleDir = bundleDir + run.Applied.BundleDir = bundleDir + for path, value := range map[string]any{ + run.Run.Artifacts.Prepare: run.Prepare, + run.Run.Artifacts.Sync: run.Sync, + run.Run.Artifacts.Cutover: run.Cutover, + run.Run.Artifacts.Rollback: run.Rollback, + run.Run.Artifacts.Commit: run.Commit, + run.Run.Artifacts.Decisions: run.Decisions, + } { + if err := writeJSONArtifact(runArtifactPath(run.Run.RunDir, path), value); err != nil { + t.Fatal(err) + } + } + if err := writeRunApplied(runArtifactPath(run.Run.RunDir, run.Run.Artifacts.Applied), run.Applied); err != nil { + t.Fatal(err) + } + if err := writeJSONArtifact(filepath.Join(run.Run.RunDir, "run.json"), run.Run); err != nil { + t.Fatal(err) + } + return run +} diff --git a/internal/cli/init_target.go b/internal/cli/init_target.go index 58f9fb8..ce4113a 100644 --- a/internal/cli/init_target.go +++ b/internal/cli/init_target.go @@ -233,18 +233,17 @@ func runInitTargetWith(ctx context.Context, args []string, stdin io.Reader, stdo return fmt.Errorf("dokploy createApiKey: %w", err) } - state, err := readBortState(deps.statePath) - if err != nil { - return err - } - state = setTargetCredentials(state, target, targetCredentials{ + creds := targetCredentials{ URL: client.BaseURL, Token: apiKey, AdminEmail: admin.Email, BootstrappedAt: time.Now().UTC(), APIKeyName: apiKeyName, - }) - if err := writeBortState(deps.statePath, state); err != nil { + } + if err := mutateBortState(deps.statePath, func(state *bortState) bool { + *state = setTargetCredentials(*state, target, creds) + return true + }); err != nil { return err } diff --git a/internal/cli/issues.go b/internal/cli/issues.go index a672bc8..55916b9 100644 --- a/internal/cli/issues.go +++ b/internal/cli/issues.go @@ -61,7 +61,7 @@ func (i appIssue) FixCommand(app string) string { case issueKindEnv: keys := envKeysFromItems(i.Items) if len(keys) == 0 { - return fmt.Sprintf("bort env %s KEY=value", quotedApp) + return fmt.Sprintf("%s %s KEY=value", bortCommand("env"), quotedApp) } const maxKeys = 5 shown := keys @@ -78,7 +78,7 @@ func (i appIssue) FixCommand(app string) string { parts = append(parts, "KEY=value") } } - cmd := fmt.Sprintf("bort env %s %s", quotedApp, strings.Join(parts, " ")) + cmd := fmt.Sprintf("%s %s %s", bortCommand("env"), quotedApp, strings.Join(parts, " ")) if more > 0 { cmd += fmt.Sprintf(" # +%d more key(s)", more) } @@ -86,9 +86,9 @@ func (i appIssue) FixCommand(app string) string { case issueKindData: stores := dataStoresFromItems(i.Items) if len(stores) == 0 { - return fmt.Sprintf("bort data %s --migrate # or choose --recreate / --managed", quotedApp) + return fmt.Sprintf("%s %s --migrate # or choose --recreate / --managed", bortCommand("data"), quotedApp) } - return fmt.Sprintf("bort data %s %s --migrate # or choose --recreate / --managed", quotedApp, shellQuote(stores[0])) + return fmt.Sprintf("%s %s %s --migrate # or choose --recreate / --managed", bortCommand("data"), quotedApp, shellQuote(stores[0])) default: return "" } diff --git a/internal/cli/issues_test.go b/internal/cli/issues_test.go index 56e56d3..bd46ea3 100644 --- a/internal/cli/issues_test.go +++ b/internal/cli/issues_test.go @@ -101,3 +101,14 @@ func TestFixCommandShellQuotesUntrustedNames(t *testing.T) { t.Fatalf("expected unsafe key replaced with KEY=value, got %q", got) } } + +func TestFixAndLiveCommandsPreserveSudoLifecycle(t *testing.T) { + t.Setenv("SUDO_UID", "501") + issue := appIssue{Kind: issueKindEnv, Items: []runDecisionItem{{Code: "env.values_required", ResourceRef: "env:KEY"}}} + if got := issue.FixCommand("api"); got != "sudo bort env api KEY=value" { + t.Fatalf("expected sudo fix command, got %q", got) + } + if got := liveApplyCommand(loadedMigrationRun{Run: migrationRun{Name: "demo"}}); got != "sudo bort migrate --live --run demo" { + t.Fatalf("expected sudo live command, got %q", got) + } +} diff --git a/internal/cli/progress.go b/internal/cli/progress.go index e299b6e..4bb9b1c 100644 --- a/internal/cli/progress.go +++ b/internal/cli/progress.go @@ -8,7 +8,6 @@ import ( "time" "github.com/aikins01/bort/internal/planfile" - "github.com/aikins01/bort/internal/preparer" ) const progressAPIVersion = "bort.progress/v1alpha1" @@ -133,21 +132,7 @@ func applyProgressToDecisions(decisions []runDecision, progress runProgress) []r continue } - decision.Items = items - decision.Count = len(items) - decision.Apps = nil - decision.Codes = nil - decision.Readiness = preparer.ReadinessReadyToCreate - for _, item := range items { - decision.Apps = uniqueAppend(decision.Apps, item.App) - decision.Codes = uniqueAppend(decision.Codes, item.Code) - decision.Readiness = preparer.WorseReadiness(decision.Readiness, item.Readiness) - } - decision.Apps = sortedStrings(decision.Apps) - decision.Codes = sortedStrings(decision.Codes) - decision.Action = decisionAction(decision) - decision.Reason = decisionReason(decision) - filtered = append(filtered, decision) + filtered = append(filtered, runDecisionWithItems(decision, items)) } sortRunDecisions(filtered) return filtered diff --git a/internal/cli/rollback.go b/internal/cli/rollback.go index 7bc6fdd..05acaa7 100644 --- a/internal/cli/rollback.go +++ b/internal/cli/rollback.go @@ -5,6 +5,7 @@ import ( "flag" "fmt" "io" + "strings" rollbackplan "github.com/aikins01/bort/internal/rollback" ) @@ -19,6 +20,7 @@ func runRollback(_ context.Context, args []string, stdout, stderr io.Writer) err var format string var outputPath string var cutoverPlanPath string + var runRef string observationWindowSeconds := rollbackplan.DefaultObservationWindowSeconds fs.StringVar(&bundleDir, "bundle", "bort-bundle", "migration bundle directory") @@ -27,6 +29,7 @@ func runRollback(_ context.Context, args []string, stdout, stderr io.Writer) err fs.StringVar(&format, "format", "text", "output format: text, json") fs.StringVar(&outputPath, "output", "-", "output path, or - for stdout") fs.StringVar(&cutoverPlanPath, "from-cutover", "", "read a prior cutover JSON plan artifact") + fs.StringVar(&runRef, "run", "", "run name under .bort/runs, or a run directory path") fs.IntVar(&observationWindowSeconds, "observation-window", rollbackplan.DefaultObservationWindowSeconds, "observation window in seconds") if err := fs.Parse(args); err != nil { @@ -35,9 +38,32 @@ func runRollback(_ context.Context, args []string, stdout, stderr io.Writer) err if err := checkOutputFormat("rollback", format); err != nil { return err } + if strings.TrimSpace(runRef) != "" { + for _, name := range []string{"app", "bundle", "from-cutover", "observation-window", "target"} { + if flagSet(fs, name) { + return fmt.Errorf("rollback --run does not accept --%s; the run already owns its reviewed rollback plan", name) + } + } + } var result rollbackplan.Result - if cutoverPlanPath != "" { + useCurrentRun := cutoverPlanPath == "" && !flagSet(fs, "bundle") && !flagSet(fs, "target") && !flagSet(fs, "app") && !flagSet(fs, "observation-window") + resolvedRun := strings.TrimSpace(runRef) + useReviewedRun := resolvedRun != "" + if !useReviewedRun && useCurrentRun { + var err error + resolvedRun, useReviewedRun, err = selectedRunRef(false) + if err != nil { + return err + } + } + if useReviewedRun { + run, err := loadMigrationRun(resolvedRun) + if err != nil { + return err + } + result = run.Rollback + } else if cutoverPlanPath != "" { expect := artifactExpectations{AppName: appName} if flagWasSet(fs, "bundle") { expect.BundleDir = bundleDir diff --git a/internal/cli/rollback_test.go b/internal/cli/rollback_test.go index 994bcad..6bd00d3 100644 --- a/internal/cli/rollback_test.go +++ b/internal/cli/rollback_test.go @@ -4,6 +4,8 @@ import ( "bytes" "context" "encoding/json" + "io" + "path/filepath" "strings" "testing" @@ -89,3 +91,75 @@ func TestRunRollbackWritesJSONPlan(t *testing.T) { t.Fatalf("expected explicit zero observation window, got %#v", result.Apps[0]) } } + +func TestRunRollbackDefaultsToCurrentRun(t *testing.T) { + workDir := t.TempDir() + t.Chdir(workDir) + bundleDir := filepath.Join(workDir, "reviewed-bundle") + writeTestBundle(t, bundleDir, manifest.Manifest{ + Source: manifest.Source{Platform: "docker"}, + Apps: []manifest.App{{ + Name: "reviewed-app", + Services: []manifest.Service{{Name: "reviewed-app", Image: "example/reviewed:latest"}}, + Routes: []manifest.Route{{Host: "reviewed.example.com", ServiceName: "reviewed-app"}}, + }}, + }) + runCommand(t, runMigrate, []string{"--bundle", bundleDir, "--run", "reviewed-run"}) + reviewed, err := loadMigrationRun("reviewed-run") + if err != nil { + t.Fatal(err) + } + writeTestBundle(t, filepath.Join(workDir, "bort-bundle"), manifest.Manifest{ + Source: manifest.Source{Platform: "docker"}, + Apps: []manifest.App{{Name: "stale-app", Services: []manifest.Service{{Name: "stale-app", Image: "example/stale:latest"}}}}, + }) + + var stdout bytes.Buffer + var stderr bytes.Buffer + if err := runRollback(context.Background(), nil, &stdout, &stderr); err != nil { + t.Fatalf("rollback failed: %v\nstderr:\n%s", err, stderr.String()) + } + if !strings.Contains(stdout.String(), "Rollback plan: "+reviewed.Run.BundleDir+" -> dokploy") || !strings.Contains(stdout.String(), "reviewed-app") { + t.Fatalf("expected rollback to use the current run artifact, got:\n%s", stdout.String()) + } + if strings.Contains(stdout.String(), "stale-app") { + t.Fatalf("rollback planned from the default bundle instead of the current run:\n%s", stdout.String()) + } +} + +func TestRunRollbackUsesDefaultBundleWithoutCurrentRun(t *testing.T) { + workDir := t.TempDir() + t.Chdir(workDir) + reviewedBundle := filepath.Join(workDir, "reviewed-bundle") + writeTestBundle(t, reviewedBundle, manifest.Manifest{ + Source: manifest.Source{Platform: "docker"}, + Apps: []manifest.App{{ + Name: "reviewed-app", + Services: []manifest.Service{{Name: "reviewed-app", Image: "example/reviewed:latest"}}, + Routes: []manifest.Route{{Host: "reviewed.example.com", ServiceName: "reviewed-app"}}, + }}, + }) + runCommand(t, runMigrate, []string{"--bundle", reviewedBundle, "--run", "reviewed-run"}) + writeTestBundle(t, filepath.Join(workDir, "bort-bundle"), manifest.Manifest{ + Source: manifest.Source{Platform: "docker"}, + Apps: []manifest.App{{ + Name: "default-app", + Services: []manifest.Service{{Name: "default-app", Image: "example/default:latest"}}, + Routes: []manifest.Route{{Host: "default.example.com", ServiceName: "default-app"}}, + }}, + }) + if err := mutateBortState(defaultStatePath(), func(state *bortState) bool { + state.CurrentRun = "" + return true + }); err != nil { + t.Fatal(err) + } + + var stdout bytes.Buffer + if err := runRollback(context.Background(), nil, &stdout, io.Discard); err != nil { + t.Fatal(err) + } + if !strings.Contains(stdout.String(), "default-app") || strings.Contains(stdout.String(), "reviewed-app") { + t.Fatalf("expected bare rollback to plan from the default bundle without a current run, got:\n%s", stdout.String()) + } +} diff --git a/internal/cli/root.go b/internal/cli/root.go index 7f761dd..502c796 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -2,6 +2,8 @@ package cli import ( "context" + "errors" + "flag" "fmt" "io" "os" @@ -14,7 +16,12 @@ func Run(ctx context.Context, args []string, stdout, stderr io.Writer) error { return RunWithInput(ctx, args, os.Stdin, stdout, stderr) } -func RunWithInput(ctx context.Context, args []string, stdin io.Reader, stdout, stderr io.Writer) error { +func RunWithInput(ctx context.Context, args []string, stdin io.Reader, stdout, stderr io.Writer) (err error) { + defer func() { + if errors.Is(err, flag.ErrHelp) { + err = nil + } + }() if len(args) == 0 { return runGuide(ctx, stdin, stdout, stderr) } @@ -37,7 +44,7 @@ func RunWithInput(ctx context.Context, args []string, stdin io.Reader, stdout, s case "commit": return runCommit(ctx, args[1:], stdout, stderr) case "cleanup": - return runCleanup(ctx, args[1:], stdout, stderr) + return runCleanupWithInput(ctx, args[1:], stdin, stdout, stderr) case "scan": return runScan(ctx, args[1:], stdout, stderr) case "plan": @@ -77,30 +84,26 @@ func printHelp(w io.Writer, advanced bool) error { func writePrimaryHelp(w io.Writer, st *styler) { writeHelpSection(w, st, "Usage:", []helpLine{ - {verb: "bort", desc: "scan and show app-first migration status"}, - {verb: "bort env KEY=value ...", desc: "record env values for an app in .bort/state.json"}, - {verb: "bort data --recreate|--migrate|--managed", desc: "record a data store strategy in .bort/state.json"}, - {verb: "bort migrate", desc: "create/update a run; --live applies after gates are clear"}, - {verb: "bort rollback", desc: "plan a rollback to the source"}, - {verb: "bort commit", desc: "plan final target acceptance; --apply stops source app containers"}, - {verb: "bort cleanup", desc: "inventory leftovers; --apply removes safe metadata only"}, - {verb: "bort init-target dokploy", desc: "bootstrap target credentials for live execution"}, + {verb: bortCommand(""), desc: "start or resume the current migration"}, + {verb: bortCommand("migrate --live"), desc: "apply the reviewed current run to its target"}, + {verb: bortCommand("rollback"), desc: "inspect the source rollback plan"}, + {verb: bortCommand("commit --apply"), desc: "accept the target and retire source containers"}, + {verb: bortCommand("cleanup"), desc: "audit leftovers; --apply removes safe metadata only"}, + {verb: bortCommand("cleanup purge"), desc: "purge eligible source leftovers with confirmation"}, }) writeHelpSection(w, st, "Other:", []helpLine{ - {verb: "bort help [--advanced]", desc: "show this help (advanced lists power-user verbs)"}, + {verb: "bort help --advanced", desc: "show setup, automation, and artifact commands"}, {verb: "bort version", desc: "print the version"}, }) } func writeAdvancedHelp(w io.Writer, st *styler) { - writeHelpSection(w, st, "Primary:", []helpLine{ - {verb: "bort", desc: "start or resume a migration (linear, app-first)"}, - {verb: "bort env", desc: "record env values for an app in .bort/state.json"}, - {verb: "bort data", desc: "record a data store strategy in .bort/state.json"}, - {verb: "bort migrate", desc: "create/update a run; --live applies after gates are clear"}, - {verb: "bort rollback", desc: "plan a rollback to the source"}, - {verb: "bort commit", desc: "plan final target acceptance; --apply stops source app containers"}, - {verb: "bort cleanup", desc: "inventory leftovers; --apply removes safe metadata only"}, + writeHelpSection(w, st, "Setup and automation:", []helpLine{ + {verb: bortCommand("migrate --source "), desc: "scan, export, and create a reviewed run in one command"}, + {verb: bortCommand("migrate --manifest "), desc: "create a run from an existing manifest"}, + {verb: bortCommand("env KEY=value ..."), desc: "record env values non-interactively"}, + {verb: bortCommand("data --migrate|--recreate|--managed"), desc: "record a data strategy non-interactively"}, + {verb: bortCommand("init-target dokploy"), desc: "bootstrap target credentials before live execution"}, }) writeHelpSection(w, st, "Power-user pipeline (each step is local and dry-run only):", []helpLine{ {verb: "bort scan", desc: "discover local resources and write a migration manifest"}, @@ -111,9 +114,9 @@ func writeAdvancedHelp(w io.Writer, st *styler) { {verb: "bort sync", desc: "plan state sync work from prepared target resources"}, {verb: "bort cutover", desc: "plan route cutover and rollback"}, }) - writeHelpSection(w, st, "Scripting helpers (rarely needed; the linear status view shows the same info):", []helpLine{ - {verb: "bort status", desc: "print a run summary as text"}, - {verb: "bort next", desc: "print the next safe action for a run"}, + writeHelpSection(w, st, "Compatibility helpers (the default cockpit already includes both):", []helpLine{ + {verb: bortCommand("status"), desc: "show the app-first cockpit for a selected run"}, + {verb: bortCommand("next"), desc: "print only the next safe action"}, }) fmt.Fprintln(w, st.muted(`Run "bort -h" for command-specific flags.`)) } diff --git a/internal/cli/run.go b/internal/cli/run.go index 7347802..b6a1d27 100644 --- a/internal/cli/run.go +++ b/internal/cli/run.go @@ -2,6 +2,7 @@ package cli import ( "context" + "crypto/sha256" "encoding/json" "errors" "flag" @@ -30,14 +31,21 @@ const ( decisionsAPIVersion = "bort.decisions/v1alpha1" ) +var errRunOperationActive = errors.New("migration run operation already in progress") + type migrationRun struct { APIVersion string `json:"apiVersion"` Name string `json:"name"` RunDir string `json:"runDir"` CreatedAt time.Time `json:"createdAt"` UpdatedAt time.Time `json:"updatedAt"` + LiveAppliedAt *time.Time `json:"liveAppliedAt,omitempty"` + CommittedAt *time.Time `json:"committedAt,omitempty"` + PurgedAt *time.Time `json:"purgedAt,omitempty"` + ApplyOutcomeRequired bool `json:"applyOutcomeRequired,omitempty"` Source string `json:"source,omitempty"` BundleDir string `json:"bundleDir"` + SourceBundleDir string `json:"sourceBundleDir,omitempty"` ManifestPath string `json:"manifest,omitempty"` Target string `json:"target"` AppName string `json:"app,omitempty"` @@ -175,6 +183,8 @@ func runMigrate(ctx context.Context, args []string, stdout, stderr io.Writer) er var target string var appName string var runRef string + var sourceName string + var manifestPath string var live bool observationWindowSeconds := gateway.DefaultObservationWindowSeconds rollbackWindowSeconds := gateway.DefaultRollbackWindowSeconds @@ -183,6 +193,8 @@ func runMigrate(ctx context.Context, args []string, stdout, stderr io.Writer) er fs.StringVar(&target, "target", "dokploy", "target platform") fs.StringVar(&appName, "app", "", "optional app name to include in the run") fs.StringVar(&runRef, "run", "", "run name under .bort/runs, or a run directory path") + fs.StringVar(&sourceName, "source", "", "scan a source directly: docker, coolify, coolify-local, or manifest") + fs.StringVar(&manifestPath, "manifest", "", "existing manifest path (implies --source manifest)") fs.BoolVar(&live, "live", false, "execute against the target platform (default dry-run only)") fs.IntVar(&observationWindowSeconds, "observation-window", gateway.DefaultObservationWindowSeconds, "observation window in seconds") fs.IntVar(&rollbackWindowSeconds, "rollback-window", gateway.DefaultRollbackWindowSeconds, "rollback window in seconds") @@ -191,6 +203,50 @@ func runMigrate(ctx context.Context, args []string, stdout, stderr io.Writer) er return err } bundleFlagSet := flagSet(fs, "bundle") + if live { + for _, name := range []string{"app", "bundle", "manifest", "observation-window", "rollback-window", "source", "target"} { + if flagSet(fs, name) { + return fmt.Errorf("--live applies an existing reviewed run and does not accept --%s; create or update the dry-run first, then apply it with --run or the current-run default", name) + } + } + } + if strings.TrimSpace(manifestPath) != "" && strings.TrimSpace(sourceName) == "" { + sourceName = "manifest" + } + if strings.TrimSpace(sourceName) != "" { + if bundleFlagSet { + return fmt.Errorf("migrate accepts either --source/--manifest or --bundle, not both") + } + if appName != "" { + return fmt.Errorf("--app is only supported with --bundle; source scans create a complete run") + } + if live { + return fmt.Errorf("--live cannot create a run from --source; run the dry-run command first, review it with `%s`, then apply", bortCommand("")) + } + if sourceName == "manifest" && strings.TrimSpace(manifestPath) == "" { + return fmt.Errorf("--manifest is required with --source manifest") + } + if sourceName != "manifest" && strings.TrimSpace(manifestPath) != "" { + return fmt.Errorf("--manifest can only be used with --source manifest") + } + if strings.TrimSpace(runRef) == "" { + runRef = defaultGuideRunName(sourceName, time.Now().UTC()) + } + loadedRun, err := createMigrationRunFromSource(ctx, guidedSetup{ + Source: sourceName, + Target: target, + RunName: runRef, + ManifestPath: manifestPath, + }, observationWindowSeconds, rollbackWindowSeconds) + if err != nil { + return err + } + if err := rememberCurrentRun(loadedRun.Run); err != nil { + return err + } + writeAppFirstCockpit(stdout, loadedRun) + return nil + } opts := migrationRunOptions{ BundleDir: bundleDir, @@ -201,81 +257,193 @@ func runMigrate(ctx context.Context, args []string, stdout, stderr io.Writer) er RollbackWindowSeconds: rollbackWindowSeconds, } if live { - if activeRef, ok := activeApplyRunRefForMigrate(opts, bundleFlagSet); ok { + activeRef, ok, err := activeApplyRunRefForMigrate(opts, bundleFlagSet) + if err != nil { + return err + } + if ok { loadedRun, err := loadMigrationRun(activeRef) if err != nil { return err } + if err := rememberCurrentRun(loadedRun.Run); err != nil { + return err + } summary := summarizeMigrationRun(loadedRun) writeLiveMigrationRunText(stdout, "Migration run loaded", summary) - return applyLiveMigration(ctx, loadedRun, stderr, nil) + return attachLiveMigrationRun(ctx, loadedRun, stderr, nil) } + resolved, err := resolveRunRef(opts.RunRef, false) + if err != nil { + return err + } + if !migrationRunMetadataExists(resolved) { + return fmt.Errorf("migration run %q does not exist; run `%s` to create or select a run before --live", resolved, bortCommand("")) + } + operationLock, err := acquireRunOperationLock(resolved) + if err != nil { + applyActive, applyActiveErr := applyRunActive(resolved) + if applyActiveErr != nil { + return fmt.Errorf("check live migration for run %q: %w", resolved, applyActiveErr) + } + if errors.Is(err, errRunOperationActive) && applyActive { + loadedRun, loadErr := loadMigrationRun(resolved) + if loadErr != nil { + return loadErr + } + writeLiveMigrationRunText(stdout, "Migration run loaded", summarizeMigrationRun(loadedRun)) + return attachLiveMigrationRun(ctx, loadedRun, stderr, nil) + } + return fmt.Errorf("start live migration for run %q: %w", resolved, err) + } + defer operationLock.Release() + loadedRun, err := loadMigrationRun(resolved) + if err != nil { + return err + } + if err := rememberCurrentRun(loadedRun.Run); err != nil { + return err + } + writeLiveMigrationRunText(stdout, "Migration run loaded", summarizeMigrationRun(loadedRun)) + if err := validateLiveApplyReady(loadedRun); err != nil { + return err + } + return applyLiveMigrationLocked(ctx, loadedRun, stderr, nil) } - loadedRun, err := migrationRunForMigrateCommand(opts, live, bundleFlagSet) + loadedRun, err := migrationRunForMigrateCommand(opts, bundleFlagSet) if err != nil { return err } + if err := rememberCurrentRun(loadedRun.Run); err != nil { + return err + } summary := summarizeMigrationRun(loadedRun) - if live { - writeLiveMigrationRunText(stdout, "Migration run created", summary) - if err := validateLiveApplyReady(loadedRun); err != nil { - return err - } - return applyLiveMigration(ctx, loadedRun, stderr, nil) - } writeMigrationRunText(stdout, "Migration run created", summary) return nil } -func migrationRunForMigrateCommand(opts migrationRunOptions, live, bundleFlagSet bool) (loadedMigrationRun, error) { +func migrationRunForMigrateCommand(opts migrationRunOptions, bundleFlagSet bool) (loadedMigrationRun, error) { if strings.TrimSpace(opts.RunRef) != "" && !bundleFlagSet && migrationRunMetadataExists(opts.RunRef) { - return refreshMigrationRun(opts.RunRef) - } - if live && strings.TrimSpace(opts.RunRef) == "" && !bundleFlagSet && opts.BundleDir == "bort-bundle" && !defaultBundleExists() { - if latestRef, ok := latestRunRef(); ok { - return refreshMigrationRun(latestRef) - } + return refreshMigrationRunSafely(opts.RunRef) } return createMigrationRun(opts) } -func activeApplyRunRefForMigrate(opts migrationRunOptions, bundleFlagSet bool) (string, bool) { - if strings.TrimSpace(opts.RunRef) != "" && migrationRunMetadataExists(opts.RunRef) && applyRunActive(opts.RunRef) { - return opts.RunRef, true +func activeApplyRunRefForMigrate(opts migrationRunOptions, bundleFlagSet bool) (string, bool, error) { + if strings.TrimSpace(opts.RunRef) != "" && migrationRunMetadataExists(opts.RunRef) { + active, err := applyRunActive(opts.RunRef) + if err != nil { + return "", false, err + } + if active { + return opts.RunRef, true, nil + } } if strings.TrimSpace(opts.RunRef) == "" && !bundleFlagSet { - if latestRef, ok := latestRunRef(); ok && applyRunActive(latestRef) { - return latestRef, true + current, err := resolveRunRef("", false) + if err != nil { + return "", false, err } + active, err := applyRunActive(current) + if err != nil { + return "", false, err + } + if active { + return current, true, nil + } + } + return "", false, nil +} + +func refreshMigrationRunSafely(runRef string) (loadedMigrationRun, error) { + operationLock, err := acquireRunOperationLock(runRef) + if err != nil { + return loadedMigrationRun{}, err + } + defer operationLock.Release() + return refreshMigrationRunSafelyLocked(runRef) +} + +func refreshMigrationRunSafelyLocked(runRef string) (loadedMigrationRun, error) { + runDir, err := existingRunDir(runRef) + if err != nil { + return loadedMigrationRun{}, err } - return "", false + run, err := readRunMetadata(filepath.Join(runDir, "run.json")) + if err != nil { + return loadedMigrationRun{}, err + } + hasAppliedSteps, err := runHasAppliedSteps(run) + if err != nil { + return loadedMigrationRun{}, fmt.Errorf("read applied migration progress: %w", err) + } + if run.LiveAppliedAt != nil || run.CommittedAt != nil || run.PurgedAt != nil || hasAppliedSteps { + return loadMigrationRun(runRef) + } + return refreshMigrationRunLocked(runRef) } func validateLiveApplyReady(run loadedMigrationRun) error { - if decisions := openRunDecisions(run); len(decisions) > 0 { + if decisions := liveApplyBlockingDecisions(run); len(decisions) > 0 { decision := decisions[0] action := strings.TrimSpace(decision.Action) if action == "" { action = "resolve the open migration decision" } - runRef := strings.TrimSpace(run.Run.Name) - if runRef == "" { - runRef = run.Run.RunDir - } - return fmt.Errorf("live apply is blocked by %d open decision(s); next safe step: %s (run `bort next --run %s`)", len(decisions), action, shellQuote(runRef)) + return fmt.Errorf("live apply is blocked by %d unresolved requirement(s); next safe step: %s (run `%s` to review this run)", len(decisions), action, runScopedCommand(run, "status")) } return nil } -func applyRunActive(runRef string) bool { +func liveApplyBlockingDecisions(run loadedMigrationRun) []runDecision { + return openFilteredDecisions(run, func(item runDecisionItem) bool { + if item.Stage == "prepare" || item.Readiness == preparer.ReadinessBlocked || item.Readiness == preparer.ReadinessNeedsInput { + return true + } + if item.Readiness != preparer.ReadinessNeedsDecision { + return false + } + switch item.Stage { + case "cutover", "rollback", "commit": + return false + default: + return true + } + }) +} + +func applyRunActive(runRef string) (bool, error) { runDir, err := existingRunDir(runRef) if err != nil { - return false + return false, err } - pid := readApplyLockPID(filepath.Join(runDir, "apply.lock")) - return pid > 0 && applyLockPIDRunning(pid) + return applyLockActive(filepath.Join(runDir, "apply.lock")) +} + +func acquireRunOperationLock(runRef string) (*applyLock, error) { + runDir, err := existingRunDir(runRef) + if err != nil { + return nil, err + } + lockPath, err := safeRunArtifactPath(runDir, "operation.lock") + if err != nil { + return nil, err + } + lock, err := acquireApplyLock(lockPath) + if errors.Is(err, errApplyAlreadyRunning) { + return nil, errRunOperationActive + } + return lock, err +} + +func runOperationActive(runRef string) (bool, error) { + runDir, err := existingRunDir(runRef) + if err != nil { + return false, err + } + return applyLockActive(filepath.Join(runDir, "operation.lock")) } func migrationRunMetadataExists(runRef string) bool { @@ -297,10 +465,30 @@ func flagSet(fs *flag.FlagSet, name string) bool { return found } -func applyLiveMigration(ctx context.Context, run loadedMigrationRun, stderr io.Writer, onProgress func(dokploy.StepProgress)) error { +func attachLiveMigrationRun(ctx context.Context, run loadedMigrationRun, stderr io.Writer, onProgress func(dokploy.StepProgress)) error { + appliedPath, err := safeRunArtifactPath(run.Run.RunDir, run.Run.Artifacts.Applied) + if err != nil { + return err + } + lockPath, err := safeRunArtifactPath(run.Run.RunDir, "apply.lock") + if err != nil { + return err + } + if err := attachLiveMigration(ctx, run, appliedPath, lockPath, stderr, onProgress); err != nil { + return err + } + return finalizeAttachedLiveMigration(ctx, run.Run.RunDir) +} + +func applyLiveMigrationLocked(ctx context.Context, run loadedMigrationRun, stderr io.Writer, onProgress func(dokploy.StepProgress)) error { if run.Run.Target != "dokploy" { return fmt.Errorf("--live is only supported for target dokploy, got %q", run.Run.Target) } + var err error + run, err = ensureSelfContainedLiveRunLocked(run) + if err != nil { + return err + } appliedPath, err := safeRunArtifactPath(run.Run.RunDir, run.Run.Artifacts.Applied) if err != nil { return err @@ -312,7 +500,17 @@ func applyLiveMigration(ctx context.Context, run loadedMigrationRun, stderr io.W lock, err := acquireApplyLock(lockPath) if err != nil { if err == errApplyAlreadyRunning { - return attachLiveMigration(ctx, run, appliedPath, lockPath, stderr, onProgress) + if err := attachLiveMigration(ctx, run, appliedPath, lockPath, stderr, onProgress); err != nil { + return err + } + current, err := loadMigrationRun(run.Run.RunDir) + if err != nil { + return err + } + if err := requireLiveApplySucceeded(current); err != nil { + return err + } + return markRunLiveAppliedLocked(current.Run) } return fmt.Errorf("lock live migration run: %w", err) } @@ -320,6 +518,10 @@ func applyLiveMigration(ctx context.Context, run loadedMigrationRun, stderr io.W if err := validateLiveApplyReady(run); err != nil { return err } + plan := dokploy.PlanFromArtifacts(run.Prepare, run.Sync, run.Cutover) + if len(plan.Steps) == 0 { + return fmt.Errorf("live migration plan has no executable steps") + } client, err := ensureDokployClient(ctx, run.Run.Target, os.Stdin, stderr, stderr) if err != nil { if err == errDokploySetupSkipped { @@ -331,11 +533,14 @@ func applyLiveMigration(ctx context.Context, run loadedMigrationRun, stderr io.W if err != nil { return err } - plan := dokploy.PlanFromArtifacts(run.Prepare, run.Sync, run.Cutover) plan.RunName = run.Run.Name plan.RunDir = run.Run.RunDir resumeFrom := completedApplyPrefix(plan.Steps, ledger.Snapshot()) plan.ResumeFrom = resumeFrom + beforeStep := func(p dokploy.StepProgress) error { + return ledger.Record(p) + } + plan.BeforeStep = &beforeStep fn := func(p dokploy.StepProgress) { if p.Status == dokploy.StepStatusOK || p.Status == dokploy.StepStatusError || p.Status == dokploy.StepStatusSkipped { if recordErr := ledger.Record(p); recordErr != nil { @@ -354,7 +559,153 @@ func applyLiveMigration(ctx context.Context, run loadedMigrationRun, stderr io.W if resumeFrom == len(plan.Steps) && len(plan.Steps) > 0 { fmt.Fprintln(stderr, "live mode: all planned dokploy steps are already recorded as complete") } - return client.Apply(ctx, plan) + if err := client.Apply(ctx, plan); err != nil { + return err + } + if err := ledger.Err(); err != nil { + return fmt.Errorf("live apply completed but applied ledger could not be persisted: %w", err) + } + if err := ledger.MarkSucceeded(); err != nil { + return fmt.Errorf("live apply completed but its successful outcome could not be persisted: %w", err) + } + return markRunLiveAppliedLocked(run.Run) +} + +func ensureSelfContainedLiveRunLocked(run loadedMigrationRun) (loadedMigrationRun, error) { + if err := containedPath(run.Run.RunDir, run.Prepare.BundleDir); err == nil { + return run, nil + } else if run.Run.ApplyOutcomeRequired { + return loadedMigrationRun{}, fmt.Errorf("run %q does not have a self-contained reviewed bundle; run `%s` to refresh it before live apply: %w", run.Run.Name, runScopedCommand(run, "migrate"), err) + } + + runDir := filepath.FromSlash(run.Run.RunDir) + bundleDir, err := snapshotMigrationBundle(run.Prepare.BundleDir, runDir) + if err != nil { + return loadedMigrationRun{}, fmt.Errorf("snapshot legacy run bundle: %w", err) + } + removeBundle := true + defer func() { + if removeBundle { + _ = os.RemoveAll(bundleDir) + } + }() + artifacts, artifactDir, err := nextRunArtifacts(runDir, run.Run) + if err != nil { + return loadedMigrationRun{}, err + } + removeArtifacts := true + defer func() { + if removeArtifacts { + _ = os.RemoveAll(artifactDir) + } + }() + + now := time.Now().UTC() + if strings.TrimSpace(run.Run.Source) == "" && strings.TrimSpace(run.Run.SourceBundleDir) == "" { + run.Run.SourceBundleDir = run.Run.BundleDir + } + run.Run.BundleDir = bundleDir + run.Run.ApplyOutcomeRequired = true + run.Run.UpdatedAt = now + run.Run.Artifacts = artifacts + run.Prepare.BundleDir = bundleDir + run.Sync.BundleDir = bundleDir + run.Cutover.BundleDir = bundleDir + run.Rollback.BundleDir = bundleDir + run.Commit.BundleDir = bundleDir + run.Decisions.RunName = run.Run.Name + run.Decisions.RunDir = run.Run.RunDir + run.Decisions.BundleDir = bundleDir + for decisionIndex := range run.Decisions.Decisions { + for itemIndex := range run.Decisions.Decisions[decisionIndex].Items { + item := &run.Decisions.Decisions[decisionIndex].Items[itemIndex] + switch item.Stage { + case "prepare": + item.Artifact = runArtifactPath(runDir, artifacts.Prepare) + case "sync": + item.Artifact = runArtifactPath(runDir, artifacts.Sync) + case "cutover": + item.Artifact = runArtifactPath(runDir, artifacts.Cutover) + case "rollback": + item.Artifact = runArtifactPath(runDir, artifacts.Rollback) + case "commit": + item.Artifact = runArtifactPath(runDir, artifacts.Commit) + } + } + } + run.Progress.RunName = run.Run.Name + run.Progress.RunDir = run.Run.RunDir + run.Progress.UpdatedAt = now + run.Applied.RunName = run.Run.Name + run.Applied.BundleDir = bundleDir + run.Applied.Target = run.Run.Target + + if err := writeJSONArtifact(runArtifactPath(runDir, artifacts.Prepare), run.Prepare); err != nil { + return loadedMigrationRun{}, err + } + if err := writeJSONArtifact(runArtifactPath(runDir, artifacts.Sync), run.Sync); err != nil { + return loadedMigrationRun{}, err + } + if err := writeJSONArtifact(runArtifactPath(runDir, artifacts.Cutover), run.Cutover); err != nil { + return loadedMigrationRun{}, err + } + if err := writeJSONArtifact(runArtifactPath(runDir, artifacts.Rollback), run.Rollback); err != nil { + return loadedMigrationRun{}, err + } + if err := writeJSONArtifact(runArtifactPath(runDir, artifacts.Commit), run.Commit); err != nil { + return loadedMigrationRun{}, err + } + if err := writeJSONArtifact(runArtifactPath(runDir, artifacts.Decisions), run.Decisions); err != nil { + return loadedMigrationRun{}, err + } + if err := writeRunProgress(runArtifactPath(runDir, artifacts.Progress), run.Progress); err != nil { + return loadedMigrationRun{}, err + } + if err := writeRunApplied(runArtifactPath(runDir, artifacts.Applied), run.Applied); err != nil { + return loadedMigrationRun{}, err + } + if err := writeJSONArtifact(filepath.Join(runDir, "run.json"), run.Run); err != nil { + return loadedMigrationRun{}, err + } + removeBundle = false + removeArtifacts = false + return run, nil +} + +const liveFinalizePollInterval = 50 * time.Millisecond + +func finalizeAttachedLiveMigration(ctx context.Context, runRef string) error { + ticker := time.NewTicker(liveFinalizePollInterval) + defer ticker.Stop() + for { + run, err := loadMigrationRun(runRef) + if err != nil { + return err + } + if run.Run.LiveAppliedAt != nil { + return nil + } + operationLock, err := acquireRunOperationLock(runRef) + if err == nil { + current, loadErr := loadMigrationRun(runRef) + if loadErr == nil { + loadErr = requireLiveApplySucceeded(current) + } + if loadErr == nil { + loadErr = markRunLiveAppliedLocked(current.Run) + } + operationLock.Release() + return loadErr + } + if !errors.Is(err, errRunOperationActive) { + return err + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-ticker.C: + } + } } const liveAttachPollInterval = 2 * time.Second @@ -415,13 +766,11 @@ func attachLiveMigration(ctx context.Context, run loadedMigrationRun, appliedPat if completedApplyPrefix(plan.Steps, applied) >= len(plan.Steps) { return nil } - if pid <= 0 { - if _, err := os.Stat(lockPath); errors.Is(err, os.ErrNotExist) { - return attachExitResult(run, plan.Steps, applied, startedAt) - } - continue + active, err := applyLockActive(lockPath) + if err != nil { + return fmt.Errorf("check live migration lock: %w", err) } - if applyLockPIDRunning(pid) { + if active { continue } return attachExitResult(run, plan.Steps, applied, startedAt) @@ -530,7 +879,7 @@ func ensureDokployClient(ctx context.Context, target string, stdin io.Reader, st } return pingConfiguredDokployClient(ctx, target) } else { - return nil, fmt.Errorf("dokploy is not reachable at %s: %w. run `bort init-target dokploy --install --dokploy-url %s`", client.BaseURL, pingErr, client.BaseURL) + return nil, fmt.Errorf("dokploy is not reachable at %s: %w. run `%s`", client.BaseURL, pingErr, bortCommand("init-target dokploy --install --dokploy-url "+shellQuote(client.BaseURL))) } } if target == "dokploy" && stdinIsTerminal(stdin) { @@ -568,7 +917,7 @@ func lookupDokployClient(target string) (*dokploy.Client, error) { } return &dokploy.Client{BaseURL: creds.URL, Token: creds.Token, HTTPClient: &http.Client{Timeout: 30 * time.Second}}, nil } - return nil, fmt.Errorf("no dokploy credentials available: set %s and %s, or run `bort init-target dokploy --install`", dokploy.EnvBaseURL, dokploy.EnvToken) + return nil, fmt.Errorf("no dokploy credentials available: set %s and %s, or run `%s`", dokploy.EnvBaseURL, dokploy.EnvToken, bortCommand("init-target dokploy --install")) } func resolveDokployClient(ctx context.Context, target string, stdin io.Reader, stderr io.Writer) (*dokploy.Client, error) { @@ -587,7 +936,7 @@ func resolveDokployClient(ctx context.Context, target string, stdin io.Reader, s return &dokploy.Client{BaseURL: creds.URL, Token: creds.Token, HTTPClient: &http.Client{Timeout: 30 * time.Second}}, nil } if target == "dokploy" && stdinIsTerminal(stdin) { - fmt.Fprintln(stderr, "no dokploy credentials found; running `bort init-target dokploy` interactively") + fmt.Fprintf(stderr, "no dokploy credentials found; running `%s` interactively\n", bortCommand("init-target dokploy")) if err := runInitTarget(ctx, nil, stdin, stderr, stderr); err != nil { return nil, err } @@ -603,7 +952,7 @@ func resolveDokployClient(ctx context.Context, target string, stdin io.Reader, s return &dokploy.Client{BaseURL: creds.URL, Token: creds.Token, HTTPClient: &http.Client{Timeout: 30 * time.Second}}, nil } } - return nil, fmt.Errorf("no dokploy credentials available: set %s and %s, or run `bort init-target dokploy` first", dokploy.EnvBaseURL, dokploy.EnvToken) + return nil, fmt.Errorf("no dokploy credentials available: set %s and %s, or run `%s` first", dokploy.EnvBaseURL, dokploy.EnvToken, bortCommand("init-target dokploy")) } func stdinIsTerminal(stdin io.Reader) bool { @@ -614,6 +963,56 @@ func stdinIsTerminal(stdin io.Reader) bool { return isInteractiveTerminal(file) } +func existingMutableMigrationRun(runDir, runName string) (migrationRun, error) { + immutableError := func() error { + return fmt.Errorf("run %q has started live execution and its reviewed plan is immutable; create a new run instead", runName) + } + metadataPath := filepath.Join(runDir, "run.json") + if _, err := os.Stat(metadataPath); err != nil { + if !errors.Is(err, os.ErrNotExist) { + return migrationRun{}, err + } + appliedPath := filepath.Join(runDir, defaultRunArtifacts().Applied) + if _, err := os.Stat(appliedPath); err != nil { + if errors.Is(err, os.ErrNotExist) { + return migrationRun{}, nil + } + return migrationRun{}, err + } + probe := migrationRun{Name: runName, RunDir: filepath.ToSlash(filepath.Clean(runDir)), Artifacts: defaultRunArtifacts()} + applied, err := readRunApplied(appliedPath, probe) + if err != nil { + return migrationRun{}, fmt.Errorf("refusing to rewrite run %q because its apply ledger cannot be verified: %w", runName, err) + } + if len(applied.Steps) > 0 { + return migrationRun{}, immutableError() + } + return migrationRun{}, nil + } + + existing, err := readRunMetadata(metadataPath) + if err != nil { + return migrationRun{}, fmt.Errorf("refusing to rewrite run %q because its metadata cannot be read: %w", runName, err) + } + existing.RunDir = filepath.ToSlash(filepath.Clean(runDir)) + existing.Artifacts = existing.Artifacts.withDefaults() + if existing.LiveAppliedAt != nil || existing.CommittedAt != nil || existing.PurgedAt != nil { + return migrationRun{}, immutableError() + } + appliedPath, err := safeRunArtifactPath(runDir, existing.Artifacts.Applied) + if err != nil { + return migrationRun{}, fmt.Errorf("refusing to rewrite run %q because its apply ledger path is invalid: %w", runName, err) + } + applied, err := readRunApplied(appliedPath, existing) + if err != nil { + return migrationRun{}, fmt.Errorf("refusing to rewrite run %q because its apply ledger cannot be verified: %w", runName, err) + } + if len(applied.Steps) > 0 { + return migrationRun{}, immutableError() + } + return existing, nil +} + func createMigrationRun(opts migrationRunOptions) (loadedMigrationRun, error) { if opts.BundleDir == "" { opts.BundleDir = "bort-bundle" @@ -627,21 +1026,229 @@ func createMigrationRun(opts migrationRunOptions) (loadedMigrationRun, error) { if err := ensurePrivateRunDir(runDir); err != nil { return loadedMigrationRun{}, err } + operationLock, err := acquireRunOperationLock(runDir) + if err != nil { + return loadedMigrationRun{}, err + } + defer operationLock.Release() + return createMigrationRunLocked(opts, runDir, runName, now) +} + +func snapshotMigrationBundle(sourceDir, runDir string) (string, error) { + info, err := os.Lstat(sourceDir) + if err != nil { + return "", err + } + if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() { + return "", fmt.Errorf("migration bundle %s must be a directory and cannot be a symlink", sourceDir) + } + sourceAbs, err := filepath.Abs(sourceDir) + if err != nil { + return "", err + } + runAbs, err := filepath.Abs(runDir) + if err != nil { + return "", err + } + relRun, err := filepath.Rel(sourceAbs, runAbs) + if err != nil { + return "", err + } + if relRun == "." || (relRun != ".." && !strings.HasPrefix(relRun, ".."+string(os.PathSeparator))) { + return "", fmt.Errorf("migration bundle %s cannot contain run directory %s", sourceDir, runDir) + } + sourceDigest, err := digestMigrationBundle(sourceDir) + if err != nil { + return "", err + } + + snapshotDir, err := os.MkdirTemp(runDir, "bundle-") + if err != nil { + return "", err + } + removeSnapshot := true + defer func() { + if removeSnapshot { + _ = os.RemoveAll(snapshotDir) + } + }() + if err := os.Chmod(snapshotDir, 0o700); err != nil { + return "", err + } + if err := filepath.WalkDir(sourceDir, func(path string, entry os.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + rel, err := filepath.Rel(sourceDir, path) + if err != nil { + return err + } + if rel == "." { + return nil + } + if entry.Type()&os.ModeSymlink != 0 { + return fmt.Errorf("migration bundle entry %s cannot be a symlink", path) + } + target := filepath.Join(snapshotDir, rel) + if entry.IsDir() { + if err := os.MkdirAll(target, 0o700); err != nil { + return err + } + return os.Chmod(target, 0o700) + } + entryInfo, err := entry.Info() + if err != nil { + return err + } + if !entryInfo.Mode().IsRegular() { + return fmt.Errorf("migration bundle entry %s must be a regular file", path) + } + contents, err := readFileNoFollow(path) + if err != nil { + return err + } + return writeFileAtomic(target, contents, 0o600) + }); err != nil { + return "", err + } + if err := validateMigrationBundleSnapshot(sourceDir, snapshotDir, sourceDigest); err != nil { + return "", err + } + removeSnapshot = false + return filepath.Clean(snapshotDir), nil +} + +func validateMigrationBundleSnapshot(sourceDir, snapshotDir string, sourceDigest [sha256.Size]byte) error { + afterDigest, err := digestMigrationBundle(sourceDir) + if err != nil { + return err + } + snapshotDigest, err := digestMigrationBundle(snapshotDir) + if err != nil { + return err + } + if afterDigest != sourceDigest || snapshotDigest != sourceDigest { + return fmt.Errorf("migration bundle %s changed while it was being snapshotted; retry after its files are stable", sourceDir) + } + return nil +} + +func digestMigrationBundle(root string) ([sha256.Size]byte, error) { + digest := sha256.New() + err := filepath.WalkDir(root, func(path string, entry os.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + rel, err := filepath.Rel(root, path) + if err != nil { + return err + } + if entry.Type()&os.ModeSymlink != 0 { + return fmt.Errorf("migration bundle entry %s cannot be a symlink", path) + } + if rel == "." { + if !entry.IsDir() { + return fmt.Errorf("migration bundle %s must be a directory", root) + } + return nil + } + if err := writeMigrationBundleDigestField(digest, []byte(filepath.ToSlash(rel))); err != nil { + return err + } + if entry.IsDir() { + return writeMigrationBundleDigestField(digest, []byte("directory")) + } + entryInfo, err := entry.Info() + if err != nil { + return err + } + if !entryInfo.Mode().IsRegular() { + return fmt.Errorf("migration bundle entry %s must be a regular file", path) + } + contents, err := readFileNoFollow(path) + if err != nil { + return err + } + if err := writeMigrationBundleDigestField(digest, []byte("file")); err != nil { + return err + } + return writeMigrationBundleDigestField(digest, contents) + }) + var result [sha256.Size]byte + if err != nil { + return result, err + } + copy(result[:], digest.Sum(nil)) + return result, nil +} + +func writeMigrationBundleDigestField(w io.Writer, value []byte) error { + if _, err := fmt.Fprintf(w, "%d:", len(value)); err != nil { + return err + } + _, err := w.Write(value) + return err +} + +func nextRunArtifacts(runDir string, existing migrationRun) (runArtifacts, string, error) { + artifacts := defaultRunArtifacts() + if existing.APIVersion == "" { + return artifacts, "", nil + } + dir, err := os.MkdirTemp(runDir, "plan-") + if err != nil { + return runArtifacts{}, "", err + } + if err := os.Chmod(dir, 0o700); err != nil { + _ = os.RemoveAll(dir) + return runArtifacts{}, "", err + } + prefix := filepath.Base(dir) + artifacts.Prepare = filepath.ToSlash(filepath.Join(prefix, artifacts.Prepare)) + artifacts.Sync = filepath.ToSlash(filepath.Join(prefix, artifacts.Sync)) + artifacts.Cutover = filepath.ToSlash(filepath.Join(prefix, artifacts.Cutover)) + artifacts.Rollback = filepath.ToSlash(filepath.Join(prefix, artifacts.Rollback)) + artifacts.Commit = filepath.ToSlash(filepath.Join(prefix, artifacts.Commit)) + artifacts.Decisions = filepath.ToSlash(filepath.Join(prefix, artifacts.Decisions)) + artifacts.Progress = filepath.ToSlash(filepath.Join(prefix, artifacts.Progress)) + artifacts.Applied = filepath.ToSlash(filepath.Join(prefix, artifacts.Applied)) + return artifacts, dir, nil +} + +func createMigrationRunLocked(opts migrationRunOptions, runDir, runName string, now time.Time) (loadedMigrationRun, error) { + existingRun, err := existingMutableMigrationRun(runDir, runName) + if err != nil { + return loadedMigrationRun{}, err + } + bundleDir, err := snapshotMigrationBundle(opts.BundleDir, runDir) + if err != nil { + return loadedMigrationRun{}, err + } + sourceBundleDir := "" + removeSnapshot := true + if strings.TrimSpace(opts.Source) == "" { + sourceBundleDir = opts.BundleDir + } + defer func() { + if removeSnapshot { + _ = os.RemoveAll(bundleDir) + } + }() createdAt := now - if existing, err := readRunMetadata(filepath.Join(runDir, "run.json")); err == nil && !existing.CreatedAt.IsZero() { - createdAt = existing.CreatedAt + if !existingRun.CreatedAt.IsZero() { + createdAt = existingRun.CreatedAt } state, err := readBortState(defaultStatePath()) if err != nil { return loadedMigrationRun{}, err } - if _, err := applyStateEnvToBundle(state, opts.BundleDir); err != nil { + if _, err := applyStateEnvToBundle(state, bundleDir); err != nil { return loadedMigrationRun{}, err } - preparePlan, err := preparer.Plan(preparer.Options{BundleDir: opts.BundleDir, Target: opts.Target, AppName: opts.AppName}) + preparePlan, err := preparer.Plan(preparer.Options{BundleDir: bundleDir, Target: opts.Target, AppName: opts.AppName}) if err != nil { return loadedMigrationRun{}, err } @@ -659,6 +1266,16 @@ func createMigrationRun(opts migrationRunOptions) (loadedMigrationRun, error) { if err != nil { return loadedMigrationRun{}, err } + artifacts, artifactDir, err := nextRunArtifacts(runDir, existingRun) + if err != nil { + return loadedMigrationRun{}, err + } + removeArtifacts := artifactDir != "" + defer func() { + if removeArtifacts { + _ = os.RemoveAll(artifactDir) + } + }() run := migrationRun{ APIVersion: runAPIVersion, @@ -666,15 +1283,20 @@ func createMigrationRun(opts migrationRunOptions) (loadedMigrationRun, error) { RunDir: filepath.ToSlash(filepath.Clean(runDir)), CreatedAt: createdAt, UpdatedAt: now, + LiveAppliedAt: existingRun.LiveAppliedAt, + CommittedAt: existingRun.CommittedAt, + PurgedAt: existingRun.PurgedAt, + ApplyOutcomeRequired: true, Source: opts.Source, - BundleDir: opts.BundleDir, + BundleDir: bundleDir, + SourceBundleDir: sourceBundleDir, ManifestPath: opts.ManifestPath, Target: opts.Target, AppName: opts.AppName, DryRun: true, ObservationWindowSeconds: opts.ObservationWindowSeconds, RollbackWindowSeconds: opts.RollbackWindowSeconds, - Artifacts: defaultRunArtifacts(), + Artifacts: artifacts, } if err := writeJSONArtifact(runArtifactPath(runDir, run.Artifacts.Prepare), preparePlan); err != nil { @@ -701,8 +1323,21 @@ func createMigrationRun(opts migrationRunOptions) (loadedMigrationRun, error) { if err != nil { return loadedMigrationRun{}, err } - progress, err := readRunProgress(progressPath, run) - if err != nil { + progress := emptyRunProgress(run) + if existingRun.APIVersion != "" { + existingProgressPath, err := safeRunArtifactPath(runDir, existingRun.Artifacts.Progress) + if err != nil { + return loadedMigrationRun{}, err + } + progress, err = readRunProgress(existingProgressPath, existingRun) + if err != nil { + return loadedMigrationRun{}, err + } + } + progress.RunName = run.Name + progress.RunDir = run.RunDir + progress.UpdatedAt = now + if err := writeRunProgress(progressPath, progress); err != nil { return loadedMigrationRun{}, err } loadedRun.Progress = progress @@ -714,15 +1349,33 @@ func createMigrationRun(opts migrationRunOptions) (loadedMigrationRun, error) { if err != nil { return loadedMigrationRun{}, err } + if err := writeRunApplied(appliedPath, applied); err != nil { + return loadedMigrationRun{}, err + } loadedRun.Applied = applied if err := writeJSONArtifact(filepath.Join(runDir, "run.json"), run); err != nil { return loadedMigrationRun{}, err } + removeSnapshot = false + removeArtifacts = false return loadedRun, nil } func refreshMigrationRun(runRef string) (loadedMigrationRun, error) { + operationLock, err := acquireRunOperationLock(runRef) + if err != nil { + return loadedMigrationRun{}, err + } + defer operationLock.Release() + return refreshMigrationRunLocked(runRef) +} + +func refreshMigrationRunLocked(runRef string) (loadedMigrationRun, error) { + return refreshMigrationRunLockedWithInputs(runRef, "", "") +} + +func refreshMigrationRunLockedWithInputs(runRef, bundleOverride, manifestOverride string) (loadedMigrationRun, error) { runDir, err := existingRunDir(runRef) if err != nil { return loadedMigrationRun{}, err @@ -733,11 +1386,21 @@ func refreshMigrationRun(runRef string) (loadedMigrationRun, error) { } existing.Artifacts = existing.Artifacts.withDefaults() manifestPath := existing.ManifestPath + if strings.TrimSpace(manifestOverride) != "" { + manifestPath = manifestOverride + } if manifestPath == "" && strings.TrimSpace(existing.Source) != "" { manifestPath = runArtifactPath(runDir, "manifest.json") } - return createMigrationRun(migrationRunOptions{ - BundleDir: existing.BundleDir, + bundleDir := strings.TrimSpace(bundleOverride) + if bundleDir == "" { + bundleDir = existing.BundleDir + } + if strings.TrimSpace(bundleOverride) == "" && strings.TrimSpace(existing.Source) == "" && strings.TrimSpace(existing.SourceBundleDir) != "" { + bundleDir = existing.SourceBundleDir + } + return createMigrationRunLocked(migrationRunOptions{ + BundleDir: bundleDir, Target: existing.Target, AppName: existing.AppName, RunRef: runDir, @@ -745,20 +1408,20 @@ func refreshMigrationRun(runRef string) (loadedMigrationRun, error) { ManifestPath: manifestPath, ObservationWindowSeconds: existing.ObservationWindowSeconds, RollbackWindowSeconds: existing.RollbackWindowSeconds, - }) + }, runDir, runNameFromDir(runDir), time.Now().UTC()) } func runStatus(_ context.Context, args []string, stdout, stderr io.Writer) error { - run, err := loadRunFromArgs("status", args, stderr, false) + run, err := loadRunFromArgs("status", args, stderr) if err != nil { return err } - writeMigrationRunText(stdout, "Migration run", summarizeMigrationRun(run)) + writeAppFirstCockpit(stdout, run) return nil } func runNext(_ context.Context, args []string, stdout, stderr io.Writer) error { - run, err := loadRunFromArgs("next", args, stderr, true) + run, err := loadRunFromArgs("next", args, stderr) if err != nil { return err } @@ -766,7 +1429,7 @@ func runNext(_ context.Context, args []string, stdout, stderr io.Writer) error { return nil } -func loadRunFromArgs(command string, args []string, stderr io.Writer, allowLatest bool) (loadedMigrationRun, error) { +func loadRunFromArgs(command string, args []string, stderr io.Writer) (loadedMigrationRun, error) { fs := flag.NewFlagSet(command, flag.ContinueOnError) fs.SetOutput(stderr) @@ -778,16 +1441,11 @@ func loadRunFromArgs(command string, args []string, stderr io.Writer, allowLates if runRef == "" && fs.NArg() == 1 { runRef = fs.Arg(0) } - if strings.TrimSpace(runRef) == "" { - if allowLatest { - if latestRef, ok := latestRunRef(); ok { - return loadMigrationRun(latestRef) - } - return loadedMigrationRun{}, fmt.Errorf("no migration run found; run bort to start one") - } - return loadedMigrationRun{}, fmt.Errorf("--run is required") + resolved, err := resolveRunRef(runRef, false) + if err != nil { + return loadedMigrationRun{}, err } - return loadMigrationRun(runRef) + return loadMigrationRun(resolved) } func loadMigrationRun(runRef string) (loadedMigrationRun, error) { @@ -810,11 +1468,6 @@ func loadMigrationRun(runRef string) (loadedMigrationRun, error) { if err != nil { return loadedMigrationRun{}, err } - state, err := readBortState(defaultStatePath()) - if err != nil { - return loadedMigrationRun{}, err - } - applyStateOverridesToPrepare(state, &preparePlan) syncResult, err := readSyncArtifact(runArtifactPath(runDir, run.Artifacts.Sync), expect) if err != nil { return loadedMigrationRun{}, err @@ -919,10 +1572,11 @@ func summarizeMigrationRun(run loadedMigrationRun) migrationRunSummary { } } + blockingDecisions := liveApplyBlockingDecisions(run) summary.Decisions = openRunDecisions(run) summary.Progress = progressSummary(run.Progress) summary.FirstGates = firstUnresolvedRunGates(run, 3) - summary.Next = nextSafeStep(run, summary.Decisions) + summary.Next = nextSafeStep(run, blockingDecisions) return summary } @@ -940,8 +1594,27 @@ func firstUnresolvedRunGates(run loadedMigrationRun, limit int) []runGateSummary } func nextSafeStep(run loadedMigrationRun, decisions []runDecision) runNextStep { + if run.Run.PurgedAt != nil { + return runNextStep{Action: "migration complete", Reason: "selected source leftovers were purged after target acceptance"} + } + if run.Run.CommittedAt != nil { + return runNextStep{Action: fmt.Sprintf("run `%s` to audit remaining metadata and source leftovers", runScopedCommand(run, "cleanup")), Reason: "the target is accepted and source app containers are retired"} + } + if run.Run.LiveAppliedAt != nil || liveApplySucceeded(run) { + return runNextStep{Action: fmt.Sprintf("verify the target, then run `%s` after the rollback window", runScopedCommand(run, "commit --apply")), Reason: "the live apply completed and the source remains available for rollback"} + } + applyActive, applyActiveErr := applyRunActive(run.Run.RunDir) + if applyActiveErr != nil { + return runNextStep{Action: fmt.Sprintf("inspect the live-apply lock for run %s before retrying", shellQuote(run.Run.Name)), Reason: applyActiveErr.Error()} + } + if applyActive { + return runNextStep{Action: fmt.Sprintf("run `%s` to view the active apply", liveApplyCommand(run)), Reason: "another process is applying this run"} + } + if len(run.Applied.Steps) > 0 { + return runNextStep{Action: fmt.Sprintf("run `%s` to resume the interrupted apply", liveApplyCommand(run)), Reason: "the apply ledger contains incomplete work"} + } if decisions == nil { - decisions = openRunDecisions(run) + decisions = liveApplyBlockingDecisions(run) } if len(decisions) > 0 { decision := decisions[0] @@ -953,36 +1626,32 @@ func nextSafeStep(run loadedMigrationRun, decisions []runDecision) runNextStep { } } - for _, readiness := range []preparer.Readiness{preparer.ReadinessBlocked, preparer.ReadinessNeedsInput, preparer.ReadinessNeedsDecision} { - gates := gatesWithReadiness(run, readiness) - if len(gates) == 0 { - continue - } - gate := gates[0] - return runNextStep{Action: nextAction(gate), Reason: gateReason(gate), Artifact: gate.Artifact} - } - if run.Run.Target == "dokploy" && !hasDokployCredentials(run.Run.Target) { - return runNextStep{ - Action: "run `bort init-target dokploy` to bootstrap the target api key", - Reason: "all gates are clear but no dokploy credentials are stored in .bort/state.json", - } - } return runNextStep{ Action: fmt.Sprintf("run `%s` to apply the planned steps against the target", liveApplyCommand(run)), - Reason: "all gates are clear and target credentials are stored", + Reason: "all setup requirements are resolved; interactive target setup runs inline if needed", Artifact: runArtifactPath(run.Run.RunDir, run.Run.Artifacts.Commit), } } func liveApplyCommand(run loadedMigrationRun) string { - runRef := strings.TrimSpace(run.Run.Name) - if runRef == "" { - runRef = run.Run.RunDir + return runScopedCommand(run, "migrate --live") +} + +func runScopedCommand(run loadedMigrationRun, args string) string { + runRef := strings.TrimSpace(run.Run.RunDir) + runName := strings.TrimSpace(run.Run.Name) + if runName != "" { + expectedDir := filepath.Join(".bort", "runs", runName) + runDirAbs, runDirErr := filepath.Abs(runRef) + expectedDirAbs, expectedDirErr := filepath.Abs(expectedDir) + if runRef == "" || (runDirErr == nil && expectedDirErr == nil && filepath.Clean(runDirAbs) == filepath.Clean(expectedDirAbs)) { + runRef = runName + } } if runRef == "" { - return "bort migrate --live" + return bortCommand(args) } - return "bort migrate --live --run " + shellQuote(runRef) + return bortCommand(args + " --run " + shellQuote(runRef)) } func hasDokployCredentials(target string) bool { @@ -1004,6 +1673,68 @@ func openRunDecisions(run loadedMigrationRun) []runDecision { return applyProgressToDecisions(decisions, run.Progress) } +func openSetupDecisions(run loadedMigrationRun) []runDecision { + return openFilteredDecisions(run, func(item runDecisionItem) bool { + return item.Stage == "prepare" + }) +} + +func openReviewDecisions(run loadedMigrationRun) []runDecision { + return openFilteredDecisions(run, func(item runDecisionItem) bool { + if item.Readiness != preparer.ReadinessNeedsDecision { + return false + } + switch item.Stage { + case "cutover", "rollback", "commit": + return true + default: + return false + } + }) +} + +func openDownstreamBlockingDecisions(run loadedMigrationRun) []runDecision { + return openFilteredDecisions(run, func(item runDecisionItem) bool { + return item.Stage != "prepare" && (item.Readiness == preparer.ReadinessBlocked || item.Readiness == preparer.ReadinessNeedsInput) + }) +} + +func openFilteredDecisions(run loadedMigrationRun, include func(runDecisionItem) bool) []runDecision { + decisions := []runDecision{} + for _, decision := range openRunDecisions(run) { + items := make([]runDecisionItem, 0, len(decision.Items)) + for _, item := range decision.Items { + if include(item) { + items = append(items, item) + } + } + if len(items) == 0 { + continue + } + decisions = append(decisions, runDecisionWithItems(decision, items)) + } + sortRunDecisions(decisions) + return decisions +} + +func runDecisionWithItems(decision runDecision, items []runDecisionItem) runDecision { + decision.Items = items + decision.Count = len(items) + decision.Apps = nil + decision.Codes = nil + decision.Readiness = preparer.ReadinessReadyToCreate + for _, item := range items { + decision.Apps = uniqueAppend(decision.Apps, item.App) + decision.Codes = uniqueAppend(decision.Codes, item.Code) + decision.Readiness = preparer.WorseReadiness(decision.Readiness, item.Readiness) + } + decision.Apps = sortedStrings(decision.Apps) + decision.Codes = sortedStrings(decision.Codes) + decision.Action = decisionAction(decision) + decision.Reason = decisionReason(decision) + return decision +} + func generateRunDecisions(run loadedMigrationRun, generatedAt time.Time) runDecisions { if generatedAt.IsZero() { generatedAt = time.Now().UTC() @@ -1625,6 +2356,41 @@ func readRunMetadata(path string) (migrationRun, error) { return run, nil } +func markRunLiveAppliedLocked(run migrationRun) error { + return updateRunLifecycleLocked(run, func(current *migrationRun, now time.Time) { + if current.LiveAppliedAt == nil { + current.LiveAppliedAt = &now + } + }) +} + +func markRunCommittedLocked(run migrationRun) error { + return updateRunLifecycleLocked(run, func(current *migrationRun, now time.Time) { + if current.CommittedAt == nil { + current.CommittedAt = &now + } + }) +} + +func markRunPurgedLocked(run migrationRun) error { + return updateRunLifecycleLocked(run, func(current *migrationRun, now time.Time) { + if current.PurgedAt == nil { + current.PurgedAt = &now + } + }) +} + +func updateRunLifecycleLocked(run migrationRun, update func(*migrationRun, time.Time)) error { + runDir := filepath.FromSlash(run.RunDir) + path := filepath.Join(runDir, "run.json") + current, err := readRunMetadata(path) + if err != nil { + return err + } + update(¤t, time.Now().UTC()) + return writeJSONArtifact(path, current) +} + func readRunDecisions(path string, run migrationRun) (runDecisions, error) { var decisions runDecisions if err := planfile.Read(path, &decisions); err != nil { @@ -1646,9 +2412,10 @@ func readRunDecisions(path string, run migrationRun) (runDecisions, error) { } func writeJSONArtifact(path string, value any) error { - return writeOutput(io.Discard, path, func(out io.Writer) error { - encoder := json.NewEncoder(out) - encoder.SetIndent("", " ") - return encoder.Encode(value) - }) + contents, err := json.MarshalIndent(value, "", " ") + if err != nil { + return err + } + contents = append(contents, '\n') + return writeFileAtomic(path, contents, 0o600) } diff --git a/internal/cli/run_test.go b/internal/cli/run_test.go index 2ad8086..32cf3c4 100644 --- a/internal/cli/run_test.go +++ b/internal/cli/run_test.go @@ -3,10 +3,16 @@ package cli import ( "bytes" "context" + "errors" + "io" + "net/http" + "net/http/httptest" "os" "path/filepath" + "reflect" "strings" "testing" + "time" commitplan "github.com/aikins01/bort/internal/commit" "github.com/aikins01/bort/internal/exporter" @@ -15,6 +21,7 @@ import ( "github.com/aikins01/bort/internal/preparer" rollbackplan "github.com/aikins01/bort/internal/rollback" syncplan "github.com/aikins01/bort/internal/sync" + "github.com/aikins01/bort/internal/target/dokploy" ) func TestRunMigrateCreatesLocalRunArtifactsAndSummary(t *testing.T) { @@ -35,16 +42,19 @@ func TestRunMigrateCreatesLocalRunArtifactsAndSummary(t *testing.T) { } runDir := filepath.Join(workDir, ".bort", "runs", "demo-app") - for _, name := range []string{"run.json", "prepare.json", "sync.json", "cutover.json", "rollback.json", "commit.json", "decisions.json"} { + for _, name := range []string{"run.json", "prepare.json", "sync.json", "cutover.json", "rollback.json", "commit.json", "decisions.json", "progress.json", "applied.json"} { if _, err := os.Stat(filepath.Join(runDir, name)); err != nil { t.Fatalf("expected %s to exist: %v", name, err) } } run := readJSONFile[migrationRun](t, filepath.Join(runDir, "run.json")) - if run.APIVersion != runAPIVersion || !run.DryRun || run.Name != "demo-app" || run.BundleDir != bundleDir || run.Target != "dokploy" { + if run.APIVersion != runAPIVersion || !run.DryRun || run.Name != "demo-app" || run.SourceBundleDir != bundleDir || run.Target != "dokploy" { t.Fatalf("unexpected run metadata: %#v", run) } + if err := containedPath(runDir, run.BundleDir); err != nil { + t.Fatalf("expected a self-contained reviewed bundle: %v", err) + } prepareResult := readJSONFile[preparer.Result](t, filepath.Join(runDir, "prepare.json")) syncResult := readJSONFile[syncplan.Result](t, filepath.Join(runDir, "sync.json")) cutoverResult := readJSONFile[gateway.Result](t, filepath.Join(runDir, "cutover.json")) @@ -60,6 +70,13 @@ func TestRunMigrateCreatesLocalRunArtifactsAndSummary(t *testing.T) { if decisions.APIVersion != decisionsAPIVersion || !decisions.DryRun || len(decisions.Decisions) == 0 || decisions.Decisions[0].ID != "cutover" { t.Fatalf("unexpected decisions artifact: %#v", decisions) } + loaded, err := loadMigrationRun("demo-app") + if err != nil { + t.Fatal(err) + } + if err := validateLiveApplyReady(loaded); err != nil { + t.Fatalf("expected downstream review decision to remain visible without blocking live apply: %v", err) + } output := stdout.String() for _, want := range []string{ @@ -69,10 +86,7 @@ func TestRunMigrateCreatesLocalRunArtifactsAndSummary(t *testing.T) { "Routes: 1 cutover, 1 rollback, 1 commit", "Decisions: 3 open", "Open decisions:", - "cutover needs_decision: confirm cutover readiness for 1 app(s) (2 item(s))", - "Next safe step: confirm cutover readiness for 1 app(s)", - "Next decision: cutover", - "Next artifact: .bort/runs/demo-app/decisions.json", + "Next safe step: run `bort migrate --live --run demo-app`", "Dry run only: no target resources, sync operations, route changes, ownership commits, or source cleanup were executed.", } { if !strings.Contains(output, want) { @@ -81,7 +95,7 @@ func TestRunMigrateCreatesLocalRunArtifactsAndSummary(t *testing.T) { } } -func TestRunMigrateLiveUsesLatestRunWhenDefaultBundleMissing(t *testing.T) { +func TestRunMigrateLiveUsesCurrentRunInsteadOfDefaultBundle(t *testing.T) { t.Setenv("BORT_DOKPLOY_URL", "") t.Setenv("BORT_DOKPLOY_TOKEN", "") workDir := t.TempDir() @@ -94,6 +108,10 @@ func TestRunMigrateLiveUsesLatestRunWhenDefaultBundleMissing(t *testing.T) { }, }) runCommand(t, runMigrate, []string{"--bundle", bundleDir, "--run", "coolify-local", "--observation-window", "0", "--rollback-window", "0"}) + writeTestBundle(t, filepath.Join(workDir, "bort-bundle"), manifest.Manifest{ + Source: manifest.Source{Platform: "docker"}, + Apps: []manifest.App{{Name: "stale", Services: []manifest.Service{{Name: "stale", Image: "example/stale:latest"}}}}, + }) var stdout bytes.Buffer var stderr bytes.Buffer @@ -101,50 +119,1287 @@ func TestRunMigrateLiveUsesLatestRunWhenDefaultBundleMissing(t *testing.T) { if err == nil { t.Fatalf("expected live mode to stop at missing dokploy credentials") } - if strings.Contains(err.Error(), "bort-bundle") || strings.Contains(stderr.String(), "bort-bundle") { - t.Fatalf("expected latest run to be used instead of default bundle, err=%v stderr=%s", err, stderr.String()) + if strings.Contains(stdout.String(), "stale") { + t.Fatalf("expected current run to be used instead of stale default bundle, stdout=%s", stdout.String()) + } + if !strings.Contains(err.Error(), "no dokploy credentials available") { + t.Fatalf("expected the reviewed run to reach target setup, got err=%v stderr=%s stdout=%s", err, stderr.String(), stdout.String()) + } + if !strings.Contains(stdout.String(), "Migration run loaded: .bort/runs/coolify-local") { + t.Fatalf("expected live migrate to load the current run, got stdout=%s", stdout.String()) } - if !strings.Contains(err.Error(), "live apply is blocked") || !strings.Contains(err.Error(), "bort next --run coolify-local") { - t.Fatalf("expected live apply to stop at open gates after loading latest run, got err=%v stderr=%s stdout=%s", err, stderr.String(), stdout.String()) + if active, activeErr := runOperationActive("coolify-local"); activeErr != nil || active { + t.Fatalf("live apply failure left the run operation lock held: active=%t err=%v", active, activeErr) } - if !strings.Contains(stdout.String(), "Migration run created: .bort/runs/coolify-local") { - t.Fatalf("expected live migrate to refresh latest run, got stdout=%s", stdout.String()) + if _, loadErr := loadMigrationRun("coolify-local"); loadErr != nil { + t.Fatalf("live apply failure left unreadable run metadata: %v", loadErr) } } -func TestRunStatusAndNextReadExistingRun(t *testing.T) { +func TestRunMigrateLivePreservesReviewedArtifactsAfterBundleChanges(t *testing.T) { + t.Setenv("BORT_DOKPLOY_URL", "") + t.Setenv("BORT_DOKPLOY_TOKEN", "") workDir := t.TempDir() t.Chdir(workDir) bundleDir := filepath.Join(workDir, "bort-bundle") writeTestBundle(t, bundleDir, manifest.Manifest{ Source: manifest.Source{Platform: "docker"}, - Apps: []manifest.App{ - {Name: "api", Services: []manifest.Service{{Name: "api", Image: "example/api:latest"}}, Routes: []manifest.Route{{Host: "api.example.com", ServiceName: "api"}}}, + Apps: []manifest.App{{ + Name: "reviewed", + Services: []manifest.Service{{Name: "reviewed", Image: "example/reviewed:latest"}}, + Routes: []manifest.Route{{Host: "reviewed.example.com", ServiceName: "reviewed", Port: "3000"}}, + }}, + }) + runCommand(t, runMigrate, []string{"--bundle", bundleDir, "--run", "reviewed-plan"}) + reviewed, err := loadMigrationRun("reviewed-plan") + if err != nil { + t.Fatal(err) + } + if reviewed.Run.SourceBundleDir != bundleDir { + t.Fatalf("expected source bundle %s, got %s", bundleDir, reviewed.Run.SourceBundleDir) + } + if err := containedPath(reviewed.Run.RunDir, reviewed.Run.BundleDir); err != nil { + t.Fatalf("expected reviewed bundle inside the run directory: %v", err) + } + composePath := filepath.Join(reviewed.Run.BundleDir, filepath.FromSlash(reviewed.Prepare.Apps[0].Directory), reviewed.Prepare.Apps[0].Resources.App.ComposePath) + composeBefore, err := os.ReadFile(composePath) + if err != nil { + t.Fatal(err) + } + preparePath := filepath.Join(workDir, ".bort", "runs", "reviewed-plan", "prepare.json") + before, err := os.ReadFile(preparePath) + if err != nil { + t.Fatal(err) + } + writeTestBundle(t, bundleDir, manifest.Manifest{ + Source: manifest.Source{Platform: "docker"}, + Apps: []manifest.App{{ + Name: "reviewed", + Services: []manifest.Service{{Name: "reviewed", Image: "example/replacement:latest"}}, + Routes: []manifest.Route{{Host: "reviewed.example.com", ServiceName: "reviewed", Port: "3000"}}, + }}, + }) + + err = runMigrate(context.Background(), []string{"--live", "--run", "reviewed-plan"}, io.Discard, io.Discard) + if err == nil || !strings.Contains(err.Error(), "no dokploy credentials available") { + t.Fatalf("expected live apply to stop at missing dokploy credentials, got %v", err) + } + after, err := os.ReadFile(preparePath) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(after, before) { + t.Fatal("live apply rewrote the reviewed prepare artifact after the source bundle changed") + } + composeAfter, err := os.ReadFile(composePath) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(composeAfter, composeBefore) { + t.Fatal("live apply changed the compose file in the self-contained reviewed bundle") + } +} + +func TestLoadMigrationRunDoesNotApplyLaterWorkspaceState(t *testing.T) { + workDir := t.TempDir() + t.Chdir(workDir) + bundleDir := filepath.Join(workDir, "bort-bundle") + writeTestBundle(t, bundleDir, manifest.Manifest{ + Source: manifest.Source{Platform: "docker"}, + Apps: []manifest.App{{ + Name: "api", + Services: []manifest.Service{{Name: "postgres", Image: "postgres:16-alpine"}}, + }}, + }) + runCommand(t, runMigrate, []string{"--bundle", bundleDir, "--run", "reviewed-state"}) + run, err := loadMigrationRun("reviewed-state") + if err != nil { + t.Fatal(err) + } + if len(run.Prepare.Apps) != 1 || len(run.Prepare.Apps[0].Resources.DataStores) == 0 { + t.Fatalf("expected a reviewed data-store requirement, got %#v", run.Prepare.Apps) + } + preparePath, err := safeRunArtifactPath(run.Run.RunDir, run.Run.Artifacts.Prepare) + if err != nil { + t.Fatal(err) + } + reviewed := readJSONFile[preparer.Result](t, preparePath) + if err := mutateBortState(defaultStatePath(), func(state *bortState) bool { + *state = setAppDataStrategy(*state, "api", "postgres", dataStrategyMigrate) + return true + }); err != nil { + t.Fatal(err) + } + + reloaded, err := loadMigrationRun("reviewed-state") + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(reloaded.Prepare, reviewed) { + t.Fatalf("workspace state changed the reviewed prepare artifact in memory: reviewed=%#v reloaded=%#v", reviewed, reloaded.Prepare) + } +} + +func TestValidateLiveApplyReadyBlocksEveryPrepareRequirement(t *testing.T) { + run := loadedMigrationRun{ + Run: migrationRun{Name: "selected-run", RunDir: t.TempDir()}, + Decisions: runDecisions{ + APIVersion: decisionsAPIVersion, + Decisions: []runDecision{{ + ID: "prepare-review", + Kind: "route_review", + Readiness: preparer.ReadinessNeedsDecision, + Action: "review the prepared route", + Items: []runDecisionItem{{ + Stage: "prepare", + Readiness: preparer.ReadinessNeedsDecision, + }}, + }}, + }, + } + if err := validateLiveApplyReady(run); err == nil || !strings.Contains(err.Error(), "live apply is blocked") || !strings.Contains(err.Error(), runScopedCommand(run, "status")) { + t.Fatalf("expected prepare-stage needs_decision to block live apply with run-scoped review guidance, got %v", err) + } + + for _, stage := range []string{"cutover", "rollback", "commit"} { + run.Decisions.Decisions[0].Items[0].Stage = stage + if err := validateLiveApplyReady(run); err != nil { + t.Fatalf("expected %s-stage review decision not to block live apply, got %v", stage, err) + } + } + for _, kind := range []string{"data_stores", "database_review"} { + run.Decisions.Decisions[0].Kind = kind + if err := validateLiveApplyReady(run); err != nil { + t.Fatalf("expected downstream %s review decision not to block live apply, got %v", kind, err) + } + } + run.Decisions.Decisions[0].Items[0].Stage = "" + if err := validateLiveApplyReady(run); err == nil { + t.Fatal("expected needs_decision with an unknown stage to block live apply") + } + if decisions := openReviewDecisions(run); len(decisions) != 0 { + t.Fatalf("expected needs_decision with an unknown stage to stay out of review-only decisions, got %#v", decisions) + } + run.Decisions.Decisions[0].Items[0].Stage = "cutover" + for _, readiness := range []preparer.Readiness{preparer.ReadinessBlocked, preparer.ReadinessNeedsInput} { + run.Decisions.Decisions[0].Readiness = readiness + run.Decisions.Decisions[0].Items[0].Readiness = readiness + if err := validateLiveApplyReady(run); err == nil { + t.Fatalf("expected downstream %s item to block live apply", readiness) + } + } + next := nextSafeStep(run, nil) + if strings.Contains(next.Action, "migrate --live") || next.DecisionID != "prepare-review" { + t.Fatalf("expected next to surface the downstream blocker, got %#v", next) + } +} + +func TestEnsureDokployClientShellQuotesRepairURL(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + http.Error(w, "unavailable", http.StatusServiceUnavailable) + })) + defer server.Close() + baseURL := server.URL + "?source=a&mode=b" + t.Setenv(dokploy.EnvBaseURL, baseURL) + t.Setenv(dokploy.EnvToken, "secret") + + _, err := ensureDokployClient(context.Background(), "dokploy", strings.NewReader(""), io.Discard, io.Discard) + if err == nil || !strings.Contains(err.Error(), "--dokploy-url '"+baseURL+"'") { + t.Fatalf("expected shell-quoted repair URL, got %v", err) + } +} + +func TestApplyLiveMigrationRejectsEmptyPlan(t *testing.T) { + runDir := t.TempDir() + bundleDir := filepath.Join(runDir, "bundle") + if err := os.Mkdir(bundleDir, 0o700); err != nil { + t.Fatal(err) + } + run := loadedMigrationRun{ + Run: migrationRun{ + Name: "empty", + RunDir: runDir, + BundleDir: bundleDir, + Target: "dokploy", + DryRun: true, + Artifacts: defaultRunArtifacts(), + }, + Prepare: preparer.Result{BundleDir: bundleDir}, + } + if err := applyLiveMigrationLocked(context.Background(), run, io.Discard, nil); err == nil || !strings.Contains(err.Error(), "no executable steps") { + t.Fatalf("expected empty live plan to fail before recording success, got %v", err) + } +} + +func TestOpenSetupDecisionsRecomputesFilteredMetadata(t *testing.T) { + run := loadedMigrationRun{ + Decisions: runDecisions{ + APIVersion: decisionsAPIVersion, + Decisions: []runDecision{{ + ID: "blockers", + Kind: "blockers", + Readiness: preparer.ReadinessBlocked, + Action: "stale action", + Reason: "stale reason", + Apps: []string{"api", "worker"}, + Codes: []string{"cutover.blocked", "prepare.review"}, + Count: 2, + Items: []runDecisionItem{ + {Stage: "prepare", App: "api", Code: "prepare.review", Readiness: preparer.ReadinessNeedsDecision}, + {Stage: "cutover", App: "worker", Code: "cutover.blocked", Readiness: preparer.ReadinessBlocked}, + }, + }}, }, + } + decisions := openSetupDecisions(run) + if len(decisions) != 1 { + t.Fatalf("expected one setup decision, got %#v", decisions) + } + decision := decisions[0] + if decision.Count != 1 || decision.Readiness != preparer.ReadinessNeedsDecision || len(decision.Apps) != 1 || decision.Apps[0] != "api" || len(decision.Codes) != 1 || decision.Codes[0] != "prepare.review" { + t.Fatalf("expected metadata recomputed from the prepare item, got %#v", decision) + } + if decision.Action != "fix 1 blocking issue(s) across 1 app(s)" || decision.Reason != "1 item(s), 1 app(s): prepare.review" { + t.Fatalf("expected action and reason recomputed from the prepare item, got %#v", decision) + } +} + +func TestNextSafeStepScopesLifecycleCommandsToSelectedRun(t *testing.T) { + now := time.Now().UTC() + tests := []struct { + name string + run migrationRun + want string + }{ + { + name: "target live", + run: migrationRun{Name: "selected-run", LiveAppliedAt: &now}, + want: "bort commit --apply --run selected-run", + }, + { + name: "committed", + run: migrationRun{Name: "selected-run", CommittedAt: &now}, + want: "bort cleanup --run selected-run", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + step := nextSafeStep(loadedMigrationRun{Run: tt.run}, nil) + if !strings.Contains(step.Action, tt.want) { + t.Fatalf("expected next step to contain %q, got %q", tt.want, step.Action) + } + }) + } +} + +func TestRunScopedCommandPreservesExternalRunDirectory(t *testing.T) { + workDir := t.TempDir() + t.Chdir(workDir) + externalRunDir := filepath.Join(t.TempDir(), "selected-run") + run := loadedMigrationRun{Run: migrationRun{Name: "selected-run", RunDir: externalRunDir}} + got := runScopedCommand(run, "migrate --live") + want := bortCommand("migrate --live --run " + shellQuote(externalRunDir)) + if got != want { + t.Fatalf("expected external run command %q, got %q", want, got) + } +} + +func TestRunMigrateLiveRequiresReviewedRun(t *testing.T) { + workDir := t.TempDir() + t.Chdir(workDir) + writeTestBundle(t, filepath.Join(workDir, "bort-bundle"), manifest.Manifest{ + Source: manifest.Source{Platform: "docker"}, + Apps: []manifest.App{{Name: "api", Services: []manifest.Service{{Name: "api", Image: "example/api:latest"}}}}, }) - runCommand(t, runMigrate, []string{"--bundle", bundleDir, "--run", "demo-app", "--observation-window", "0", "--rollback-window", "0"}) - var statusOut bytes.Buffer - var statusErr bytes.Buffer - if err := runStatus(context.Background(), []string{"--run", "demo-app"}, &statusOut, &statusErr); err != nil { - t.Fatalf("status failed: %v\nstderr:\n%s", err, statusErr.String()) + var stdout bytes.Buffer + var stderr bytes.Buffer + err := runMigrate(context.Background(), []string{"--live"}, &stdout, &stderr) + if err == nil || !strings.Contains(err.Error(), "no current migration run") { + t.Fatalf("expected bare live apply to require a reviewed run, got err=%v stdout=%s stderr=%s", err, stdout.String(), stderr.String()) } - for _, want := range []string{"Migration run: .bort/runs/demo-app", "Run: demo-app", "Overall: needs_decision (yellow)", "Next decision: cutover", "Next artifact: .bort/runs/demo-app/decisions.json"} { - if !strings.Contains(statusOut.String(), want) { - t.Fatalf("expected status output to contain %q, got:\n%s", want, statusOut.String()) + if _, statErr := os.Stat(filepath.Join(workDir, ".bort", "runs")); !os.IsNotExist(statErr) { + t.Fatalf("live apply created a run from the default bundle: %v", statErr) + } +} + +func TestRunMigrateLiveDoesNotUseMtimeFallback(t *testing.T) { + workDir := t.TempDir() + t.Chdir(workDir) + bundleDir := filepath.Join(workDir, "bort-bundle") + writeTestBundle(t, bundleDir, manifest.Manifest{ + Source: manifest.Source{Platform: "docker"}, + Apps: []manifest.App{{Name: "latest-only", Services: []manifest.Service{{Name: "latest-only", Image: "example/latest:latest"}}}}, + }) + runCommand(t, runMigrate, []string{"--bundle", bundleDir, "--run", "latest-only"}) + if err := os.Remove(filepath.Join(workDir, ".bort", "state.json")); err != nil { + t.Fatal(err) + } + + var stdout bytes.Buffer + var stderr bytes.Buffer + err := runMigrate(context.Background(), []string{"--live"}, &stdout, &stderr) + if err == nil || !strings.Contains(err.Error(), "no current migration run") { + t.Fatalf("expected live mode to reject an mtime-only run, got err=%v stdout=%s stderr=%s", err, stdout.String(), stderr.String()) + } +} + +func TestApplyRunActiveTracksHeldLock(t *testing.T) { + runDir := t.TempDir() + lock, err := acquireApplyLock(filepath.Join(runDir, "apply.lock")) + if err != nil { + t.Fatal(err) + } + active, err := applyRunActive(runDir) + if err != nil || !active { + lock.Release() + t.Fatalf("expected held apply lock to report active: active=%t err=%v", active, err) + } + lock.Release() + active, err = applyRunActive(runDir) + if err != nil || active { + t.Fatalf("expected released apply lock to report inactive: active=%t err=%v", active, err) + } +} + +func TestApplyLockActiveSurfacesOpenErrors(t *testing.T) { + active, err := applyLockActive(filepath.Join(t.TempDir(), "missing.lock")) + if err != nil || active { + t.Fatalf("expected a missing lock file to report inactive: active=%t err=%v", active, err) + } + parentFile := filepath.Join(t.TempDir(), "not-a-directory") + if err := os.WriteFile(parentFile, []byte("lock probe fixture"), 0o600); err != nil { + t.Fatal(err) + } + if active, err := applyLockActive(filepath.Join(parentFile, "apply.lock")); err == nil || active { + t.Fatalf("expected an unexpected lock open error to be surfaced: active=%t err=%v", active, err) + } +} + +func TestApplyLockRejectsSymlinkWithoutChangingTarget(t *testing.T) { + dir := t.TempDir() + targetPath := filepath.Join(dir, "run.json") + want := []byte("preserve reviewed metadata\n") + if err := os.WriteFile(targetPath, want, 0o600); err != nil { + t.Fatal(err) + } + lockPath := filepath.Join(dir, "operation.lock") + if err := os.Symlink(targetPath, lockPath); err != nil { + t.Skipf("symlink fixture is unavailable: %v", err) + } + lock, err := acquireApplyLock(lockPath) + if lock != nil { + lock.Release() + } + if err == nil { + t.Fatal("expected a symlink lock path to be rejected") + } + if active, err := applyLockActive(lockPath); err == nil || active { + t.Fatalf("expected a symlink lock probe error: active=%t err=%v", active, err) + } + got, err := os.ReadFile(targetPath) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, want) { + t.Fatalf("lock acquisition changed symlink target: got %q want %q", got, want) + } +} + +func TestApplyLockRejectsHardLinkWithoutChangingTarget(t *testing.T) { + dir := t.TempDir() + targetPath := filepath.Join(dir, "run.json") + want := []byte("preserve reviewed metadata\n") + if err := os.WriteFile(targetPath, want, 0o600); err != nil { + t.Fatal(err) + } + lockPath := filepath.Join(dir, "operation.lock") + if err := os.Link(targetPath, lockPath); err != nil { + t.Skipf("hard-link fixture is unavailable: %v", err) + } + lock, err := acquireApplyLock(lockPath) + if lock != nil { + lock.Release() + } + if err == nil { + t.Fatal("expected a hard-linked lock path to be rejected") + } + if active, err := applyLockActive(lockPath); err == nil || active { + t.Fatalf("expected a hard-linked lock probe error: active=%t err=%v", active, err) + } + got, err := os.ReadFile(targetPath) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, want) { + t.Fatalf("lock acquisition changed hard-link target: got %q want %q", got, want) + } +} + +func TestRunOperationLockTracksContentionAndRelease(t *testing.T) { + runDir := t.TempDir() + lock, err := acquireRunOperationLock(runDir) + if err != nil { + t.Fatal(err) + } + active, err := runOperationActive(runDir) + if err != nil || !active { + lock.Release() + t.Fatalf("expected held run operation lock to report active: active=%t err=%v", active, err) + } + if second, err := acquireRunOperationLock(runDir); !errors.Is(err, errRunOperationActive) { + if second != nil { + second.Release() } + lock.Release() + t.Fatalf("expected second acquisition to fail with operation contention, got %v", err) + } + lock.Release() + active, err = runOperationActive(runDir) + if err != nil || active { + t.Fatalf("expected released run operation lock to report inactive: active=%t err=%v", active, err) } + reacquired, err := acquireRunOperationLock(runDir) + if err != nil { + t.Fatalf("expected lock reacquisition after release: %v", err) + } + reacquired.Release() +} - var nextOut bytes.Buffer - var nextErr bytes.Buffer - if err := runNext(context.Background(), []string{"demo-app"}, &nextOut, &nextErr); err != nil { - t.Fatalf("next failed: %v\nstderr:\n%s", err, nextErr.String()) +func TestBortStateMutationsSerializeReadModifyWrite(t *testing.T) { + statePath := filepath.Join(t.TempDir(), ".bort", "state.json") + firstEntered := make(chan struct{}) + releaseFirst := make(chan struct{}) + firstDone := make(chan error, 1) + go func() { + firstDone <- mutateBortState(statePath, func(state *bortState) bool { + close(firstEntered) + <-releaseFirst + *state = setAppEnv(*state, "api", map[string]string{"TOKEN": "value"}) + return true + }) + }() + <-firstEntered + + secondStarted := make(chan struct{}) + secondEntered := make(chan struct{}) + secondDone := make(chan error, 1) + go func() { + close(secondStarted) + secondDone <- mutateBortState(statePath, func(state *bortState) bool { + close(secondEntered) + *state = setAppDataStrategy(*state, "api", "postgres", dataStrategyMigrate) + return true + }) + }() + <-secondStarted + select { + case <-secondEntered: + close(releaseFirst) + t.Fatal("second state mutation entered before the first released its lock") + case <-time.After(50 * time.Millisecond): } - for _, want := range []string{"Next safe step: confirm cutover readiness for 1 app(s)", "Artifact: .bort/runs/demo-app/decisions.json", "Decision: cutover", "Dry run only: no live migration action is executed by this command."} { - if !strings.Contains(nextOut.String(), want) { - t.Fatalf("expected next output to contain %q, got:\n%s", want, nextOut.String()) + close(releaseFirst) + if err := <-firstDone; err != nil { + t.Fatal(err) + } + if err := <-secondDone; err != nil { + t.Fatal(err) + } + state, err := readBortState(statePath) + if err != nil { + t.Fatal(err) + } + if state.Apps["api"].Env["TOKEN"] != "value" || state.Apps["api"].Data["postgres"].Strategy != dataStrategyMigrate { + t.Fatalf("expected both serialized state updates, got %#v", state.Apps["api"]) + } +} + +func TestRunOperationLockBlocksMutationsButNotStatus(t *testing.T) { + workDir := t.TempDir() + t.Chdir(workDir) + bundleDir := filepath.Join(workDir, "bort-bundle") + writeTestBundle(t, bundleDir, manifest.Manifest{ + Source: manifest.Source{Platform: "docker"}, + Apps: []manifest.App{{Name: "api", Services: []manifest.Service{{Name: "api", Image: "example/api:latest"}}}}, + }) + runCommand(t, runMigrate, []string{"--bundle", bundleDir, "--run", "locked"}) + lock, err := acquireRunOperationLock("locked") + if err != nil { + t.Fatal(err) + } + defer lock.Release() + + for name, command := range map[string]func() error{ + "refresh": func() error { + return runMigrate(context.Background(), []string{"--run", "locked"}, io.Discard, io.Discard) + }, + "commit": func() error { + return runCommit(context.Background(), []string{"--apply", "--run", "locked"}, io.Discard, io.Discard) + }, + "cleanup": func() error { + return runCleanup(context.Background(), []string{"--apply", "--run", "locked"}, io.Discard, io.Discard) + }, + "purge": func() error { + return runCleanup(context.Background(), []string{"purge", "--apply", "--run", "locked", "--app", "api"}, io.Discard, io.Discard) + }, + } { + if err := command(); !errors.Is(err, errRunOperationActive) { + t.Fatalf("expected held operation lock to block %s, got %v", name, err) } } + if err := runStatus(context.Background(), []string{"--run", "locked"}, io.Discard, io.Discard); err != nil { + t.Fatalf("read-only status was blocked by operation lock: %v", err) + } +} + +func TestRunMigrateLiveAttachesToHeldApplyDespiteOperationLock(t *testing.T) { + workDir := t.TempDir() + t.Chdir(workDir) + bundleDir := filepath.Join(workDir, "bort-bundle") + writeTestBundle(t, bundleDir, manifest.Manifest{ + Source: manifest.Source{Platform: "docker"}, + Apps: []manifest.App{{Name: "api", Services: []manifest.Service{{Name: "api", Image: "example/api:latest"}}}}, + }) + runCommand(t, runMigrate, []string{"--bundle", bundleDir, "--run", "attaching"}) + operationLock, err := acquireRunOperationLock("attaching") + if err != nil { + t.Fatal(err) + } + defer operationLock.Release() + applyLock, err := acquireApplyLock(filepath.Join(workDir, ".bort", "runs", "attaching", "apply.lock")) + if err != nil { + t.Fatal(err) + } + defer applyLock.Release() + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + var stderr bytes.Buffer + err = runMigrate(ctx, []string{"--live", "--run", "attaching"}, io.Discard, &stderr) + if !errors.Is(err, context.Canceled) { + t.Fatalf("expected canceled attach, got %v", err) + } + if !strings.Contains(stderr.String(), "attaching to progress") { + t.Fatalf("expected second live invocation to attach, got:\n%s", stderr.String()) + } +} + +func TestFinalizeAttachedLiveMigrationRecordsLifecycle(t *testing.T) { + workDir := t.TempDir() + t.Chdir(workDir) + bundleDir := filepath.Join(workDir, "bort-bundle") + writeTestBundle(t, bundleDir, manifest.Manifest{ + Source: manifest.Source{Platform: "docker"}, + Apps: []manifest.App{{Name: "api", Services: []manifest.Service{{Name: "api", Image: "example/api:latest"}}}}, + }) + runCommand(t, runMigrate, []string{"--bundle", bundleDir, "--run", "attached-complete"}) + run, err := loadMigrationRun("attached-complete") + if err != nil { + t.Fatal(err) + } + steps := dokploy.PlanFromArtifacts(run.Prepare, run.Sync, run.Cutover).Steps + applied := newRunApplied(run.Run) + for index, step := range steps { + applied.Steps = append(applied.Steps, appliedStep{Index: index, Kind: string(step.Kind), App: step.App, Ref: step.Ref, Status: string(dokploy.StepStatusOK)}) + } + appliedPath, err := safeRunArtifactPath(run.Run.RunDir, run.Run.Artifacts.Applied) + if err != nil { + t.Fatal(err) + } + if err := writeRunApplied(appliedPath, applied); err != nil { + t.Fatal(err) + } + applyLock, err := acquireApplyLock(filepath.Join(run.Run.RunDir, "apply.lock")) + if err != nil { + t.Fatal(err) + } + if err := applyLiveMigrationLocked(context.Background(), run, io.Discard, nil); err == nil || !strings.Contains(err.Error(), "no successful live-apply outcome") { + applyLock.Release() + t.Fatalf("expected raced attach without a successful outcome to remain incomplete, got %v", err) + } + applyLock.Release() + if err := finalizeAttachedLiveMigration(context.Background(), run.Run.RunDir); err == nil || !strings.Contains(err.Error(), "no successful live-apply outcome") { + t.Fatalf("expected complete steps without a successful outcome to remain incomplete, got %v", err) + } + withoutOutcome, err := loadMigrationRun("attached-complete") + if err != nil { + t.Fatal(err) + } + if withoutOutcome.Run.LiveAppliedAt != nil { + t.Fatal("complete steps without a successful outcome marked the run live") + } + succeededAt := time.Now().UTC() + applied.SucceededAt = &succeededAt + if err := writeRunApplied(appliedPath, applied); err != nil { + t.Fatal(err) + } + if err := finalizeAttachedLiveMigration(context.Background(), run.Run.RunDir); err != nil { + t.Fatal(err) + } + completed, err := loadMigrationRun("attached-complete") + if err != nil { + t.Fatal(err) + } + if completed.Run.LiveAppliedAt == nil { + t.Fatal("expected attached completion to record live lifecycle") + } +} + +func TestRunRefreshPublishesVersionedArtifactsTogether(t *testing.T) { + workDir := t.TempDir() + t.Chdir(workDir) + bundleDir := filepath.Join(workDir, "bort-bundle") + writeTestBundle(t, bundleDir, manifest.Manifest{ + Source: manifest.Source{Platform: "docker"}, + Apps: []manifest.App{{Name: "api", Services: []manifest.Service{{Name: "api", Image: "example/api:v1"}}}}, + }) + runCommand(t, runMigrate, []string{"--bundle", bundleDir, "--run", "atomic-refresh"}) + before, err := loadMigrationRun("atomic-refresh") + if err != nil { + t.Fatal(err) + } + oldArtifacts := []string{ + before.Run.Artifacts.Prepare, + before.Run.Artifacts.Sync, + before.Run.Artifacts.Cutover, + before.Run.Artifacts.Rollback, + before.Run.Artifacts.Commit, + before.Run.Artifacts.Decisions, + before.Run.Artifacts.Progress, + before.Run.Artifacts.Applied, + } + + writeTestBundle(t, bundleDir, manifest.Manifest{ + Source: manifest.Source{Platform: "docker"}, + Apps: []manifest.App{{Name: "api", Services: []manifest.Service{{Name: "api", Image: "example/api:v2"}}}}, + }) + runCommand(t, runMigrate, []string{"--run", "atomic-refresh"}) + after, err := loadMigrationRun("atomic-refresh") + if err != nil { + t.Fatal(err) + } + newArtifacts := []string{ + after.Run.Artifacts.Prepare, + after.Run.Artifacts.Sync, + after.Run.Artifacts.Cutover, + after.Run.Artifacts.Rollback, + after.Run.Artifacts.Commit, + after.Run.Artifacts.Decisions, + after.Run.Artifacts.Progress, + after.Run.Artifacts.Applied, + } + artifactDir := filepath.Dir(filepath.FromSlash(newArtifacts[0])) + if artifactDir == "." { + t.Fatalf("expected refreshed artifacts in a versioned directory, got %#v", after.Run.Artifacts) + } + for index, name := range newArtifacts { + if name == oldArtifacts[index] || filepath.Dir(filepath.FromSlash(name)) != artifactDir { + t.Fatalf("expected one new versioned artifact set, old=%#v new=%#v", oldArtifacts, newArtifacts) + } + if _, err := os.Stat(runArtifactPath(after.Run.RunDir, name)); err != nil { + t.Fatalf("published artifact %s does not exist: %v", name, err) + } + } + for _, name := range oldArtifacts { + if _, err := os.Stat(runArtifactPath(before.Run.RunDir, name)); err != nil { + t.Fatalf("previous reviewed artifact %s was not preserved: %v", name, err) + } + } +} + +func TestRunMigrateManifestCreatesSelfContainedCurrentRun(t *testing.T) { + workDir := t.TempDir() + t.Chdir(workDir) + manifestPath := filepath.Join(workDir, "manifest.json") + if err := writeJSONArtifact(manifestPath, manifest.Manifest{ + Source: manifest.Source{Platform: "docker"}, + Apps: []manifest.App{{Name: "api", Services: []manifest.Service{{Name: "api", Image: "example/api:latest"}}}}, + }); err != nil { + t.Fatal(err) + } + + var stdout bytes.Buffer + var stderr bytes.Buffer + if err := runMigrate(context.Background(), []string{"--manifest", manifestPath, "--run", "direct"}, &stdout, &stderr); err != nil { + t.Fatalf("migrate from manifest failed: %v\nstderr:\n%s", err, stderr.String()) + } + run := readJSONFile[migrationRun](t, filepath.Join(workDir, ".bort", "runs", "direct", "run.json")) + if run.Source != "manifest" || run.ManifestPath == manifestPath || !strings.HasPrefix(filepath.Base(run.ManifestPath), "manifest-") || !run.ApplyOutcomeRequired { + t.Fatalf("unexpected direct migration metadata: %#v", run) + } + if err := containedPath(run.RunDir, run.ManifestPath); err != nil { + t.Fatalf("expected a private manifest generation: %v", err) + } + if err := containedPath(run.RunDir, run.BundleDir); err != nil || !strings.HasPrefix(filepath.Base(run.BundleDir), "bundle-") { + t.Fatalf("expected a private self-contained bundle snapshot, path=%s err=%v", run.BundleDir, err) + } + entries, err := os.ReadDir(run.RunDir) + if err != nil { + t.Fatal(err) + } + for _, entry := range entries { + if strings.HasPrefix(entry.Name(), "source-bundle-") { + t.Fatalf("temporary source bundle was not removed: %s", entry.Name()) + } + } + state := readJSONFile[bortState](t, filepath.Join(workDir, ".bort", "state.json")) + if state.CurrentRun != filepath.ToSlash(filepath.Join(".bort", "runs", "direct")) { + t.Fatalf("expected direct run to become current, got %#v", state) + } + for _, want := range []string{"Existing manifest → dokploy", "api"} { + if !strings.Contains(stdout.String(), want) { + t.Fatalf("expected output to contain %q, got:\n%s", want, stdout.String()) + } + } +} + +func TestMigrationBundleSnapshotValidationRejectsSourceChanges(t *testing.T) { + sourceDir := filepath.Join(t.TempDir(), "source") + snapshotDir := filepath.Join(t.TempDir(), "snapshot") + if err := os.MkdirAll(sourceDir, 0o700); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(snapshotDir, 0o700); err != nil { + t.Fatal(err) + } + sourcePath := filepath.Join(sourceDir, "compose.yaml") + snapshotPath := filepath.Join(snapshotDir, "compose.yaml") + if err := os.WriteFile(sourcePath, []byte("image: example/api:v1\n"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(snapshotPath, []byte("image: example/api:v1\n"), 0o600); err != nil { + t.Fatal(err) + } + sourceDigest, err := digestMigrationBundle(sourceDir) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(sourcePath, []byte("image: example/api:v2\n"), 0o600); err != nil { + t.Fatal(err) + } + if err := validateMigrationBundleSnapshot(sourceDir, snapshotDir, sourceDigest); err == nil || !strings.Contains(err.Error(), "changed while it was being snapshotted") { + t.Fatalf("expected a changing source bundle to be rejected, got %v", err) + } +} + +func TestRunSourceRefreshPublishesNewBundleAndArtifactsTogether(t *testing.T) { + workDir := t.TempDir() + t.Chdir(workDir) + manifestPath := filepath.Join(workDir, "manifest.json") + writeManifest := func(app, image string) { + t.Helper() + if err := writeJSONArtifact(manifestPath, manifest.Manifest{ + Source: manifest.Source{Platform: "docker"}, + Apps: []manifest.App{{Name: app, Services: []manifest.Service{{Name: app, Image: image}}}}, + }); err != nil { + t.Fatal(err) + } + } + writeManifest("reviewed", "example/reviewed:v1") + runCommand(t, runMigrate, []string{"--manifest", manifestPath, "--run", "source-refresh"}) + before, err := loadMigrationRun("source-refresh") + if err != nil { + t.Fatal(err) + } + oldManifest, err := os.ReadFile(before.Run.ManifestPath) + if err != nil { + t.Fatal(err) + } + oldIndexPath := filepath.Join(before.Run.BundleDir, "index.json") + oldIndex, err := os.ReadFile(oldIndexPath) + if err != nil { + t.Fatal(err) + } + oldArtifacts := []string{ + before.Run.Artifacts.Prepare, + before.Run.Artifacts.Sync, + before.Run.Artifacts.Cutover, + before.Run.Artifacts.Rollback, + before.Run.Artifacts.Commit, + before.Run.Artifacts.Decisions, + before.Run.Artifacts.Progress, + before.Run.Artifacts.Applied, + } + + writeManifest("replacement", "example/replacement:v2") + runCommand(t, runMigrate, []string{"--manifest", manifestPath, "--run", "source-refresh"}) + after, err := loadMigrationRun("source-refresh") + if err != nil { + t.Fatal(err) + } + if after.Run.BundleDir == before.Run.BundleDir { + t.Fatalf("source refresh reused the reviewed bundle path %s", after.Run.BundleDir) + } + if after.Run.ManifestPath == before.Run.ManifestPath || !strings.HasPrefix(filepath.Base(after.Run.ManifestPath), "manifest-") { + t.Fatalf("source refresh did not publish a new manifest generation: before=%s after=%s", before.Run.ManifestPath, after.Run.ManifestPath) + } + if err := containedPath(after.Run.RunDir, after.Run.ManifestPath); err != nil { + t.Fatalf("expected refreshed manifest inside the run: %v", err) + } + if err := containedPath(after.Run.RunDir, after.Run.BundleDir); err != nil || !strings.HasPrefix(filepath.Base(after.Run.BundleDir), "bundle-") { + t.Fatalf("expected refreshed source bundle to be a private snapshot, path=%s err=%v", after.Run.BundleDir, err) + } + if len(after.Prepare.Apps) != 1 || after.Prepare.Apps[0].Name != "replacement" { + t.Fatalf("expected refreshed artifacts to use replacement source, got %#v", after.Prepare.Apps) + } + preservedIndex, err := os.ReadFile(oldIndexPath) + if err != nil { + t.Fatalf("old reviewed bundle was removed: %v", err) + } + if !bytes.Equal(preservedIndex, oldIndex) { + t.Fatal("old reviewed bundle changed during source refresh") + } + preservedManifest, err := os.ReadFile(before.Run.ManifestPath) + if err != nil { + t.Fatalf("old reviewed manifest was removed: %v", err) + } + if !bytes.Equal(preservedManifest, oldManifest) { + t.Fatal("old reviewed manifest changed during source refresh") + } + for _, name := range oldArtifacts { + if _, err := os.Stat(runArtifactPath(before.Run.RunDir, name)); err != nil { + t.Fatalf("old reviewed artifact %s was removed: %v", name, err) + } + } + for _, name := range []string{ + after.Run.Artifacts.Prepare, + after.Run.Artifacts.Sync, + after.Run.Artifacts.Cutover, + after.Run.Artifacts.Rollback, + after.Run.Artifacts.Commit, + after.Run.Artifacts.Decisions, + after.Run.Artifacts.Progress, + after.Run.Artifacts.Applied, + } { + if _, err := os.Stat(runArtifactPath(after.Run.RunDir, name)); err != nil { + t.Fatalf("published artifact %s does not exist: %v", name, err) + } + } + if after.Run.Artifacts.Prepare == before.Run.Artifacts.Prepare { + t.Fatalf("source refresh did not switch to versioned artifacts: before=%#v after=%#v", before.Run.Artifacts, after.Run.Artifacts) + } + entries, err := os.ReadDir(after.Run.RunDir) + if err != nil { + t.Fatal(err) + } + for _, entry := range entries { + if strings.HasPrefix(entry.Name(), "source-bundle-") { + t.Fatalf("temporary refreshed source bundle was not removed: %s", entry.Name()) + } + } +} + +func TestLegacyRunBundleIsSnapshottedBeforeLiveResume(t *testing.T) { + workDir := t.TempDir() + t.Chdir(workDir) + bundleDir := filepath.Join(workDir, "legacy-bundle") + writeTestBundle(t, bundleDir, manifest.Manifest{ + Source: manifest.Source{Platform: "docker"}, + Apps: []manifest.App{{Name: "api", Services: []manifest.Service{{Name: "api", Image: "example/api:v1"}}}}, + }) + runCommand(t, runMigrate, []string{"--bundle", bundleDir, "--run", "legacy-resume"}) + legacy, err := loadMigrationRun("legacy-resume") + if err != nil { + t.Fatal(err) + } + oldArtifacts := legacy.Run.Artifacts + legacy.Run.ApplyOutcomeRequired = false + legacy.Run.SourceBundleDir = "" + legacy.Run.BundleDir = bundleDir + legacy.Prepare.BundleDir = bundleDir + legacy.Sync.BundleDir = bundleDir + legacy.Cutover.BundleDir = bundleDir + legacy.Rollback.BundleDir = bundleDir + legacy.Commit.BundleDir = bundleDir + legacy.Decisions.BundleDir = bundleDir + steps := dokploy.PlanFromArtifacts(legacy.Prepare, legacy.Sync, legacy.Cutover).Steps + if len(steps) == 0 { + t.Fatal("expected legacy live steps") + } + legacy.Applied = newRunApplied(legacy.Run) + legacy.Applied.Steps = []appliedStep{{ + Index: 0, + Kind: string(steps[0].Kind), + App: steps[0].App, + Ref: steps[0].Ref, + Status: string(dokploy.StepStatusOK), + }} + runDir := legacy.Run.RunDir + for path, value := range map[string]any{ + oldArtifacts.Prepare: legacy.Prepare, + oldArtifacts.Sync: legacy.Sync, + oldArtifacts.Cutover: legacy.Cutover, + oldArtifacts.Rollback: legacy.Rollback, + oldArtifacts.Commit: legacy.Commit, + oldArtifacts.Decisions: legacy.Decisions, + } { + if err := writeJSONArtifact(runArtifactPath(runDir, path), value); err != nil { + t.Fatal(err) + } + } + if err := writeRunApplied(runArtifactPath(runDir, oldArtifacts.Applied), legacy.Applied); err != nil { + t.Fatal(err) + } + if err := writeJSONArtifact(filepath.Join(runDir, "run.json"), legacy.Run); err != nil { + t.Fatal(err) + } + legacy, err = loadMigrationRun("legacy-resume") + if err != nil { + t.Fatal(err) + } + operationLock, err := acquireRunOperationLock(runDir) + if err != nil { + t.Fatal(err) + } + upgraded, upgradeErr := ensureSelfContainedLiveRunLocked(legacy) + operationLock.Release() + if upgradeErr != nil { + t.Fatal(upgradeErr) + } + if !upgraded.Run.ApplyOutcomeRequired || upgraded.Run.SourceBundleDir != bundleDir || upgraded.Run.BundleDir == bundleDir { + t.Fatalf("unexpected upgraded legacy metadata: %#v", upgraded.Run) + } + if err := containedPath(upgraded.Run.RunDir, upgraded.Run.BundleDir); err != nil { + t.Fatalf("legacy run bundle was not snapshotted into the run: %v", err) + } + if upgraded.Run.Artifacts.Prepare == oldArtifacts.Prepare || len(upgraded.Applied.Steps) != 1 { + t.Fatalf("legacy resume state was not moved to a new generation: artifacts=%#v applied=%#v", upgraded.Run.Artifacts, upgraded.Applied) + } + indexPath := filepath.Join(upgraded.Run.BundleDir, "index.json") + indexBefore, err := os.ReadFile(indexPath) + if err != nil { + t.Fatal(err) + } + writeTestBundle(t, bundleDir, manifest.Manifest{ + Source: manifest.Source{Platform: "docker"}, + Apps: []manifest.App{{Name: "replacement", Services: []manifest.Service{{Name: "replacement", Image: "example/replacement:v2"}}}}, + }) + indexAfter, err := os.ReadFile(indexPath) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(indexAfter, indexBefore) { + t.Fatal("upgraded legacy run still consumed its external mutable bundle") + } + reloaded, err := loadMigrationRun("legacy-resume") + if err != nil { + t.Fatal(err) + } + if reloaded.Run.BundleDir != upgraded.Run.BundleDir || len(reloaded.Applied.Steps) != 1 { + t.Fatalf("upgraded legacy run did not reload coherently: %#v", reloaded) + } +} + +func TestRunPlanBecomesImmutableAfterLiveExecutionStarts(t *testing.T) { + workDir := t.TempDir() + t.Chdir(workDir) + bundleDir := filepath.Join(workDir, "bort-bundle") + writeTestBundle(t, bundleDir, manifest.Manifest{ + Source: manifest.Source{Platform: "docker"}, + Apps: []manifest.App{{Name: "api", Services: []manifest.Service{{Name: "api", Image: "example/api:latest"}}}}, + }) + runCommand(t, runMigrate, []string{"--bundle", bundleDir, "--run", "immutable"}) + run, err := loadMigrationRun("immutable") + if err != nil { + t.Fatal(err) + } + operationLock, err := acquireRunOperationLock(run.Run.RunDir) + if err != nil { + t.Fatal(err) + } + markErr := markRunLiveAppliedLocked(run.Run) + operationLock.Release() + if markErr != nil { + t.Fatal(markErr) + } + + var stdout bytes.Buffer + var stderr bytes.Buffer + err = runMigrate(context.Background(), []string{"--bundle", bundleDir, "--run", "immutable"}, &stdout, &stderr) + if err == nil || !strings.Contains(err.Error(), "reviewed plan is immutable") { + t.Fatalf("expected applied run plan to be immutable, got err=%v", err) + } +} + +func TestRunSourceBundleBecomesImmutableAfterLiveExecutionStarts(t *testing.T) { + workDir := t.TempDir() + t.Chdir(workDir) + manifestPath := filepath.Join(workDir, "manifest.json") + if err := writeJSONArtifact(manifestPath, manifest.Manifest{ + Source: manifest.Source{Platform: "docker"}, + Apps: []manifest.App{{Name: "reviewed", Services: []manifest.Service{{Name: "reviewed", Image: "example/reviewed:latest"}}}}, + }); err != nil { + t.Fatal(err) + } + runCommand(t, runMigrate, []string{"--manifest", manifestPath, "--run", "source-immutable"}) + run, err := loadMigrationRun("source-immutable") + if err != nil { + t.Fatal(err) + } + operationLock, err := acquireRunOperationLock(run.Run.RunDir) + if err != nil { + t.Fatal(err) + } + markErr := markRunLiveAppliedLocked(run.Run) + operationLock.Release() + if markErr != nil { + t.Fatal(markErr) + } + indexPath := filepath.Join(run.Run.BundleDir, "index.json") + before, err := os.ReadFile(indexPath) + if err != nil { + t.Fatal(err) + } + if err := writeJSONArtifact(manifestPath, manifest.Manifest{ + Source: manifest.Source{Platform: "docker"}, + Apps: []manifest.App{{Name: "replacement", Services: []manifest.Service{{Name: "replacement", Image: "example/replacement:latest"}}}}, + }); err != nil { + t.Fatal(err) + } + + var stdout bytes.Buffer + var stderr bytes.Buffer + err = runMigrate(context.Background(), []string{"--manifest", manifestPath, "--run", "source-immutable"}, &stdout, &stderr) + if err == nil || !strings.Contains(err.Error(), "reviewed plan is immutable") { + t.Fatalf("expected source-created run to be immutable, got err=%v", err) + } + after, err := os.ReadFile(indexPath) + if err != nil { + t.Fatal(err) + } + if string(after) != string(before) { + t.Fatalf("source-created live bundle was overwritten") + } +} + +func TestRunPlanBecomesImmutableAfterLiveStartLedger(t *testing.T) { + workDir := t.TempDir() + t.Chdir(workDir) + bundleDir := filepath.Join(workDir, "bort-bundle") + writeTestBundle(t, bundleDir, manifest.Manifest{ + Source: manifest.Source{Platform: "docker"}, + Apps: []manifest.App{{Name: "api", Services: []manifest.Service{{Name: "api", Image: "example/api:latest"}}}}, + }) + runCommand(t, runMigrate, []string{"--bundle", bundleDir, "--run", "partial"}) + run, err := loadMigrationRun("partial") + if err != nil { + t.Fatal(err) + } + steps := dokploy.PlanFromArtifacts(run.Prepare, run.Sync, run.Cutover).Steps + if len(steps) == 0 { + t.Fatal("expected live apply steps") + } + applied := newRunApplied(run.Run) + applied.Steps = []appliedStep{{Index: 0, Kind: string(steps[0].Kind), App: steps[0].App, Ref: steps[0].Ref, Status: string(dokploy.StepStatusStarted)}} + appliedPath, err := safeRunArtifactPath(run.Run.RunDir, run.Run.Artifacts.Applied) + if err != nil { + t.Fatal(err) + } + if err := writeRunApplied(appliedPath, applied); err != nil { + t.Fatal(err) + } + + var stdout bytes.Buffer + var stderr bytes.Buffer + err = runMigrate(context.Background(), []string{"--bundle", bundleDir, "--run", "partial"}, &stdout, &stderr) + if err == nil || !strings.Contains(err.Error(), "reviewed plan is immutable") { + t.Fatalf("expected partial live ledger to make the plan immutable, got err=%v", err) + } +} + +func TestRunPlanRefusesToOverwriteUnreadableMetadata(t *testing.T) { + workDir := t.TempDir() + t.Chdir(workDir) + bundleDir := filepath.Join(workDir, "bort-bundle") + writeTestBundle(t, bundleDir, manifest.Manifest{ + Source: manifest.Source{Platform: "docker"}, + Apps: []manifest.App{{Name: "api", Services: []manifest.Service{{Name: "api", Image: "example/api:latest"}}}}, + }) + runCommand(t, runMigrate, []string{"--bundle", bundleDir, "--run", "damaged"}) + metadataPath := filepath.Join(workDir, ".bort", "runs", "damaged", "run.json") + if err := os.WriteFile(metadataPath, []byte("{\n"), 0o600); err != nil { + t.Fatal(err) + } + + var stdout bytes.Buffer + var stderr bytes.Buffer + err := runMigrate(context.Background(), []string{"--bundle", bundleDir, "--run", "damaged"}, &stdout, &stderr) + if err == nil || !strings.Contains(err.Error(), "metadata cannot be read") { + t.Fatalf("expected unreadable run metadata to block rewrite, got err=%v", err) + } + contents, readErr := os.ReadFile(metadataPath) + if readErr != nil || string(contents) != "{\n" { + t.Fatalf("damaged run metadata was overwritten: contents=%q err=%v", contents, readErr) + } +} + +func TestRunStatusAndNextReadExistingRun(t *testing.T) { + workDir := t.TempDir() + t.Chdir(workDir) + bundleDir := filepath.Join(workDir, "bort-bundle") + writeTestBundle(t, bundleDir, manifest.Manifest{ + Source: manifest.Source{Platform: "docker"}, + Apps: []manifest.App{ + {Name: "api", Services: []manifest.Service{{Name: "api", Image: "example/api:latest"}}, Routes: []manifest.Route{{Host: "api.example.com", ServiceName: "api"}}}, + }, + }) + runCommand(t, runMigrate, []string{"--bundle", bundleDir, "--run", "demo-app", "--observation-window", "0", "--rollback-window", "0"}) + + var statusOut bytes.Buffer + var statusErr bytes.Buffer + if err := runStatus(context.Background(), []string{"--run", "demo-app"}, &statusOut, &statusErr); err != nil { + t.Fatalf("status failed: %v\nstderr:\n%s", err, statusErr.String()) + } + for _, want := range []string{"local bundle → dokploy", "api", "READY", "bort migrate --live --run demo-app"} { + if !strings.Contains(statusOut.String(), want) { + t.Fatalf("expected status output to contain %q, got:\n%s", want, statusOut.String()) + } + } + + var nextOut bytes.Buffer + var nextErr bytes.Buffer + if err := runNext(context.Background(), []string{"demo-app"}, &nextOut, &nextErr); err != nil { + t.Fatalf("next failed: %v\nstderr:\n%s", err, nextErr.String()) + } + for _, want := range []string{"Next safe step: run `bort migrate --live --run demo-app`", "Run: demo-app", "Dry run only: no live migration action is executed by this command."} { + if !strings.Contains(nextOut.String(), want) { + t.Fatalf("expected next output to contain %q, got:\n%s", want, nextOut.String()) + } + } +} + +func TestRunStatusUsesCurrentRunBeforeNewerMtime(t *testing.T) { + workDir := t.TempDir() + t.Chdir(workDir) + mtimeBundle := filepath.Join(workDir, "mtime-bundle") + currentBundle := filepath.Join(workDir, "current-bundle") + writeTestBundle(t, mtimeBundle, manifest.Manifest{ + Source: manifest.Source{Platform: "docker"}, + Apps: []manifest.App{{Name: "mtime-only", Services: []manifest.Service{{Name: "mtime-only", Image: "example/mtime:latest"}}}}, + }) + writeTestBundle(t, currentBundle, manifest.Manifest{ + Source: manifest.Source{Platform: "docker"}, + Apps: []manifest.App{{Name: "selected-current", Services: []manifest.Service{{Name: "selected-current", Image: "example/current:latest"}}}}, + }) + runCommand(t, runMigrate, []string{"--bundle", mtimeBundle, "--run", "mtime-run"}) + runCommand(t, runMigrate, []string{"--bundle", currentBundle, "--run", "current-run"}) + future := time.Now().Add(time.Hour) + if err := os.Chtimes(filepath.Join(workDir, ".bort", "runs", "mtime-run", "run.json"), future, future); err != nil { + t.Fatal(err) + } + + var stdout bytes.Buffer + var stderr bytes.Buffer + if err := runStatus(context.Background(), nil, &stdout, &stderr); err != nil { + t.Fatalf("status failed: %v\nstderr:\n%s", err, stderr.String()) + } + if !strings.Contains(stdout.String(), "selected-current") { + t.Fatalf("expected status to use the persisted current run, got:\n%s", stdout.String()) + } + if strings.Contains(stdout.String(), "mtime-only") { + t.Fatalf("status selected a newer-mtime run instead of the current run:\n%s", stdout.String()) + } +} + +func TestRunStatusAndNextDoNotUseMtimeFallback(t *testing.T) { + workDir := t.TempDir() + t.Chdir(workDir) + bundleDir := filepath.Join(workDir, "bort-bundle") + writeTestBundle(t, bundleDir, manifest.Manifest{ + Source: manifest.Source{Platform: "docker"}, + Apps: []manifest.App{{Name: "mtime-only", Services: []manifest.Service{{Name: "mtime-only", Image: "example/mtime:latest"}}}}, + }) + runCommand(t, runMigrate, []string{"--bundle", bundleDir, "--run", "mtime-only"}) + if err := os.Remove(filepath.Join(workDir, ".bort", "state.json")); err != nil { + t.Fatal(err) + } + + for _, test := range []struct { + name string + run cliRunner + }{ + {name: "status", run: runStatus}, + {name: "next", run: runNext}, + } { + t.Run(test.name, func(t *testing.T) { + var stdout bytes.Buffer + var stderr bytes.Buffer + err := test.run(context.Background(), nil, &stdout, &stderr) + if err == nil || !strings.Contains(err.Error(), "no current migration run") { + t.Fatalf("expected %s to reject an mtime-only run, got err=%v stdout=%s stderr=%s", test.name, err, stdout.String(), stderr.String()) + } + }) + } +} + +func TestRunLifecycleTimestampsDriveCockpitLabels(t *testing.T) { + workDir := t.TempDir() + t.Chdir(workDir) + bundleDir := filepath.Join(workDir, "bort-bundle") + writeTestBundle(t, bundleDir, manifest.Manifest{ + Source: manifest.Source{Platform: "docker"}, + Apps: []manifest.App{{Name: "api", Services: []manifest.Service{{Name: "api", Image: "example/api:latest"}}}}, + }) + runCommand(t, runMigrate, []string{"--bundle", bundleDir, "--run", "lifecycle"}) + run, err := loadMigrationRun("lifecycle") + if err != nil { + t.Fatal(err) + } + operationLock, err := acquireRunOperationLock(run.Run.RunDir) + if err != nil { + t.Fatal(err) + } + defer operationLock.Release() + + if err := markRunLiveAppliedLocked(run.Run); err != nil { + t.Fatal(err) + } + run, err = loadMigrationRun("lifecycle") + if err != nil { + t.Fatal(err) + } + if run.Run.LiveAppliedAt == nil || run.Run.CommittedAt != nil || run.Run.PurgedAt != nil { + t.Fatalf("unexpected live-applied lifecycle metadata: %#v", run.Run) + } + liveAppliedAt := *run.Run.LiveAppliedAt + var cockpit bytes.Buffer + writeAppFirstCockpit(&cockpit, run) + if !strings.Contains(cockpit.String(), "TARGET LIVE") { + t.Fatalf("expected target-live cockpit, got:\n%s", cockpit.String()) + } + + if err := markRunCommittedLocked(run.Run); err != nil { + t.Fatal(err) + } + run, err = loadMigrationRun("lifecycle") + if err != nil { + t.Fatal(err) + } + if run.Run.LiveAppliedAt == nil || !run.Run.LiveAppliedAt.Equal(liveAppliedAt) || run.Run.CommittedAt == nil || run.Run.PurgedAt != nil { + t.Fatalf("unexpected committed lifecycle metadata: %#v", run.Run) + } + cockpit.Reset() + writeAppFirstCockpit(&cockpit, run) + if !strings.Contains(cockpit.String(), "COMMITTED") { + t.Fatalf("expected committed cockpit, got:\n%s", cockpit.String()) + } + + if err := markRunPurgedLocked(run.Run); err != nil { + t.Fatal(err) + } + run, err = loadMigrationRun("lifecycle") + if err != nil { + t.Fatal(err) + } + if run.Run.LiveAppliedAt == nil || run.Run.CommittedAt == nil || run.Run.PurgedAt == nil { + t.Fatalf("expected all lifecycle timestamps to persist: %#v", run.Run) + } + cockpit.Reset() + writeAppFirstCockpit(&cockpit, run) + if !strings.Contains(cockpit.String(), "COMPLETE") { + t.Fatalf("expected complete cockpit, got:\n%s", cockpit.String()) + } } func TestRunMigratePreservesPrepareBlockersBeforeDecisions(t *testing.T) { diff --git a/internal/cli/scan.go b/internal/cli/scan.go index b4eaa42..8c78cc2 100644 --- a/internal/cli/scan.go +++ b/internal/cli/scan.go @@ -83,7 +83,7 @@ func runScan(ctx context.Context, args []string, stdout, stderr io.Writer) error if !includeEnvValues { st := newStyler(stderr) - fmt.Fprintln(stderr, st.muted("note: env values were not captured (default). Re-run with --include-env-values, or use `bort` (no args) which captures by default.")) + fmt.Fprintln(stderr, st.muted(fmt.Sprintf("note: env values were not captured (default). Re-run with --include-env-values, or use `%s` (no args) which captures by default.", bortCommand("")))) } return nil diff --git a/internal/cli/state.go b/internal/cli/state.go index 089e45b..5da11f8 100644 --- a/internal/cli/state.go +++ b/internal/cli/state.go @@ -13,19 +13,23 @@ import ( const stateAPIVersion = "bort.state/v1alpha1" +const ( + stateLockRetryInterval = 10 * time.Millisecond + stateLockWait = 5 * time.Second +) + +var errBortStateActive = errors.New("workspace state update already in progress") + const ( dataStrategyRecreate = "recreate" dataStrategyMigrate = "migrate" dataStrategyManaged = "managed" ) -// bortState is the single user-edited file the linear UX writes to. It -// captures everything the user has decided locally (env values, data store -// strategies) so subsequent `bort` runs see the same answers without an -// interactive cockpit. type bortState struct { APIVersion string `json:"apiVersion"` UpdatedAt time.Time `json:"updatedAt"` + CurrentRun string `json:"currentRun,omitempty"` Apps map[string]appStateData `json:"apps,omitempty"` Targets map[string]targetCredentials `json:"targets,omitempty"` } @@ -91,14 +95,8 @@ func writeBortState(path string, state bortState) error { if state.UpdatedAt.IsZero() { state.UpdatedAt = time.Now().UTC() } - dir := filepath.Dir(path) - if dir != "" && dir != "." { - if err := os.MkdirAll(dir, 0o700); err != nil { - return err - } - if err := os.Chmod(dir, 0o700); err != nil { - return err - } + if err := ensurePrivateStateDir(path); err != nil { + return err } contents, err := json.MarshalIndent(state, "", " ") if err != nil { @@ -111,6 +109,51 @@ func writeBortState(path string, state bortState) error { return os.Chmod(path, 0o600) } +func ensurePrivateStateDir(path string) error { + dir := filepath.Dir(path) + if dir == "" || dir == "." { + return nil + } + if err := os.MkdirAll(dir, 0o700); err != nil { + return err + } + return os.Chmod(dir, 0o700) +} + +func acquireBortStateLock(path string) (*applyLock, error) { + if err := ensurePrivateStateDir(path); err != nil { + return nil, err + } + lockPath := filepath.Join(filepath.Dir(path), "state.lock") + deadline := time.Now().Add(stateLockWait) + for { + lock, err := acquireApplyLock(lockPath) + if !errors.Is(err, errApplyAlreadyRunning) { + return lock, err + } + if !time.Now().Before(deadline) { + return nil, errBortStateActive + } + time.Sleep(stateLockRetryInterval) + } +} + +func mutateBortState(path string, mutate func(*bortState) bool) error { + lock, err := acquireBortStateLock(path) + if err != nil { + return err + } + defer lock.Release() + state, err := readBortState(path) + if err != nil { + return err + } + if !mutate(&state) { + return nil + } + return writeBortState(path, state) +} + func emptyBortState() bortState { return bortState{ APIVersion: stateAPIVersion, @@ -157,6 +200,30 @@ func setTargetCredentials(state bortState, target string, creds targetCredential return state } +func rememberCurrentRun(run migrationRun) error { + ref := filepath.ToSlash(filepath.Clean(run.RunDir)) + if ref == "" || ref == "." { + return fmt.Errorf("cannot remember migration run with empty directory") + } + return mutateBortState(defaultStatePath(), func(state *bortState) bool { + if state.CurrentRun == ref { + return false + } + state.CurrentRun = ref + state.UpdatedAt = time.Now().UTC() + return true + }) +} + +func currentRunRef() (string, bool, error) { + state, err := readBortState(defaultStatePath()) + if err != nil { + return "", false, err + } + ref := strings.TrimSpace(state.CurrentRun) + return ref, ref != "", nil +} + func parseKeyValueArgs(args []string) (map[string]string, error) { values := map[string]string{} for _, arg := range args { diff --git a/internal/cli/style.go b/internal/cli/style.go index 5cfad31..c0675c8 100644 --- a/internal/cli/style.go +++ b/internal/cli/style.go @@ -3,10 +3,22 @@ package cli import ( "io" "os" + "strings" "github.com/charmbracelet/lipgloss" ) +func bortCommand(args string) string { + command := "bort" + if strings.TrimSpace(os.Getenv("SUDO_UID")) != "" { + command = "sudo bort" + } + if strings.TrimSpace(args) == "" { + return command + } + return command + " " + args +} + type styler struct { color bool diff --git a/internal/cli/wizard.go b/internal/cli/wizard.go index 4784cc9..2a8f3c7 100644 --- a/internal/cli/wizard.go +++ b/internal/cli/wizard.go @@ -10,31 +10,41 @@ import ( "strings" "time" - "github.com/aikins01/bort/internal/target/dokploy" - "github.com/charmbracelet/bubbles/progress" - tea "github.com/charmbracelet/bubbletea" + "github.com/aikins01/bort/internal/preparer" "github.com/charmbracelet/huh" ) -// runWizard drives the interactive flow after a migration run is created. -// it loops over open decisions, persisting each answer to .bort/state.json -// and re-planning, then offers an apply preview when everything is green. func runWizard(ctx context.Context, run loadedMigrationRun, stdin io.Reader, stdout, stderr io.Writer) error { current := run + if len(current.Applied.Steps) > 0 { + writeAppFirstCockpit(stdout, current) + return nil + } for { - decisions := openRunDecisions(current) + decisions, reviewOnly := nextWizardDecisions(current) if len(decisions) == 0 { break } decision := decisions[0] - handled, err := promptWizardDecision(ctx, current, decision, stdout) + var handled, replan bool + var err error + if reviewOnly { + handled, err = promptReviewDecision(current, decision, stdout) + } else { + handled, replan, err = promptWizardDecision(ctx, current, decision, stdout) + } if err != nil { return err } if !handled { break } - refreshed, err := refreshMigrationRun(current.Run.RunDir) + var refreshed loadedMigrationRun + if replan { + refreshed, err = refreshMigrationRun(current.Run.RunDir) + } else { + refreshed, err = loadMigrationRun(current.Run.RunDir) + } if err != nil { return err } @@ -46,46 +56,32 @@ func runWizard(ctx context.Context, run loadedMigrationRun, stdin io.Reader, std return nil } -// confirmCommitApply gates the wizard's source-retirement step behind an -// explicit confirm. it runs only after the live apply succeeded so the -// operator can poke at the target before stopping the source — this is -// the last reversible moment without a manual docker start. -func confirmCommitApply(run loadedMigrationRun) (bool, error) { - plan := dokploy.PlanForCommit(run.Prepare, run.Cutover) - confirm := false - form := huh.NewForm(huh.NewGroup( - huh.NewConfirm(). - Title("Retire source containers now?"). - Description(fmt.Sprintf("Will stop %d source container group(s) plus coolify-proxy. Source is left in place — `docker start` recovers it.", len(plan.Steps)-1)). - Affirmative("Retire"). - Negative("Skip"). - Value(&confirm), - )) - if err := form.Run(); err != nil { - return false, err +func nextWizardDecisions(run loadedMigrationRun) ([]runDecision, bool) { + if decisions := openSetupDecisions(run); len(decisions) > 0 { + return decisions, false } - return confirm, nil + return openReviewDecisions(run), true } -func promptWizardDecision(ctx context.Context, run loadedMigrationRun, decision runDecision, stdout io.Writer) (bool, error) { +func promptWizardDecision(ctx context.Context, run loadedMigrationRun, decision runDecision, stdout io.Writer) (bool, bool, error) { statePath := defaultStatePath() - state, err := readBortState(statePath) - if err != nil { - return false, err - } switch decision.Kind { case "environment": - return promptEnvDecision(ctx, decision, state, statePath, stdout) + handled, err := promptEnvDecision(ctx, decision, statePath, stdout) + return handled, handled, err case "data_stores": - return promptDataStoreDecision(ctx, decision, state, statePath, stdout) - case "routes": - return promptRoutesDecision(decision, stdout) + handled, err := promptDataStoreDecision(ctx, decision, statePath, stdout) + return handled, handled, err + } + if decision.Readiness != preparer.ReadinessNeedsDecision { + fmt.Fprintf(stdout, "%s.\n%s.\nOpen %s for details and re-run when resolved.\n", decisionAction(decision), decisionReason(decision), runArtifactPath(run.Run.RunDir, run.Run.Artifacts.Decisions)) + return false, false, nil } - fmt.Fprintf(stdout, "%s.\n%s.\nOpen %s for details and re-run when resolved.\n", decisionAction(decision), decisionReason(decision), runArtifactPath(run.Run.RunDir, run.Run.Artifacts.Decisions)) - return false, nil + handled, err := promptReviewDecision(run, decision, stdout) + return handled, false, err } -func promptEnvDecision(_ context.Context, decision runDecision, state bortState, statePath string, stdout io.Writer) (bool, error) { +func promptEnvDecision(_ context.Context, decision runDecision, statePath string, stdout io.Writer) (bool, error) { app := firstApp(decision) if app == "" { return false, nil @@ -107,15 +103,17 @@ func promptEnvDecision(_ context.Context, decision runDecision, state bortState, fmt.Fprintln(stdout, "No values entered; skipping.") return false, nil } - state = setAppEnv(state, app, values) - if err := writeBortState(statePath, state); err != nil { + if err := mutateBortState(statePath, func(state *bortState) bool { + *state = setAppEnv(*state, app, values) + return true + }); err != nil { return false, err } fmt.Fprintf(stdout, "Recorded %d env value(s) for %s.\n", len(values), app) return true, nil } -func promptDataStoreDecision(_ context.Context, decision runDecision, state bortState, statePath string, stdout io.Writer) (bool, error) { +func promptDataStoreDecision(_ context.Context, decision runDecision, statePath string, stdout io.Writer) (bool, error) { stores := storesFromDataItems(decision.Items) if len(stores) == 0 { return false, nil @@ -136,8 +134,10 @@ func promptDataStoreDecision(_ context.Context, decision runDecision, state bort if err := form.Run(); err != nil { return updated, err } - state = setAppDataStrategy(state, item.app, item.store, strategy) - if err := writeBortState(statePath, state); err != nil { + if err := mutateBortState(statePath, func(state *bortState) bool { + *state = setAppDataStrategy(*state, item.app, item.store, strategy) + return true + }); err != nil { return updated, err } fmt.Fprintf(stdout, "Recorded strategy %s for %s/%s.\n", strategy, item.app, item.store) @@ -146,41 +146,75 @@ func promptDataStoreDecision(_ context.Context, decision runDecision, state bort return updated, nil } -func promptRoutesDecision(decision runDecision, stdout io.Writer) (bool, error) { +func promptReviewDecision(run loadedMigrationRun, decision runDecision, stdout io.Writer) (bool, error) { confirm := false form := huh.NewForm(huh.NewGroup( huh.NewConfirm(). - Title("Confirm route plan"). + Title(decisionAction(decision)). Description(decisionAction(decision) + "\n" + decisionReason(decision)). - Affirmative("Continue"). - Negative("Skip for now"). + Affirmative("Reviewed"). + Negative("Not yet"). Value(&confirm), )) if err := form.Run(); err != nil { return false, err } if !confirm { - fmt.Fprintln(stdout, "Routes left as-is; rerun the wizard after editing the bundle if needed.") return false, nil } - return false, nil + if err := recordReviewDecision(run, decision, time.Now().UTC()); err != nil { + return false, err + } + fmt.Fprintf(stdout, "Review recorded. You can re-run `%s` at any time to revisit the plan.\n", bortCommand("")) + return true, nil } -func confirmApply(run loadedMigrationRun) (bool, error) { - confirm := false - planned := dokploy.PlanFromArtifacts(run.Prepare, run.Sync, run.Cutover) - form := huh.NewForm(huh.NewGroup( - huh.NewConfirm(). - Title("Apply this migration to dokploy now?"). - Description(fmt.Sprintf("Will execute %d step(s) against %s. You can also skip and run `%s` later.", len(planned.Steps), run.Run.Target, liveApplyCommand(run))). - Affirmative("Apply"). - Negative("Skip"). - Value(&confirm), - )) - if err := form.Run(); err != nil { - return false, err +func recordReviewDecision(run loadedMigrationRun, decision runDecision, at time.Time) error { + operationLock, err := acquireRunOperationLock(run.Run.RunDir) + if err != nil { + return err + } + defer operationLock.Release() + current, err := loadMigrationRun(run.Run.RunDir) + if err != nil { + return err + } + if run.Run.Artifacts.withDefaults() != current.Run.Artifacts.withDefaults() || !run.Run.UpdatedAt.Equal(current.Run.UpdatedAt) { + return fmt.Errorf("migration run plan changed while review was open; review the current plan and confirm again") } - return confirm, nil + progress := markReviewDecisionDone(current, decision, at) + path, err := safeRunArtifactPath(current.Run.RunDir, current.Run.Artifacts.Progress) + if err != nil { + return err + } + if err := writeRunProgress(path, progress); err != nil { + return err + } + return nil +} + +func markReviewDecisionDone(run loadedMigrationRun, decision runDecision, at time.Time) runProgress { + progress := markDecisionDone(run.Progress, decision, progressStatusResolved, "confirmed in guided review", at) + selected := map[string]struct{}{} + for _, item := range decision.Items { + selected[progressItemKey(item)] = struct{}{} + } + for _, current := range openRunDecisions(run) { + if current.Kind != decision.Kind { + continue + } + for _, item := range current.Items { + if _, ok := selected[progressItemKey(item)]; ok { + continue + } + state := progress.Decisions[decision.Kind] + state.Status = progressStatusOpen + state.Note = "" + progress.Decisions[decision.Kind] = state + return progress + } + } + return progress } var errDokploySetupSkipped = errors.New("dokploy setup skipped") @@ -203,7 +237,7 @@ func promptInstallAndBootstrapDokploy(ctx context.Context, stdin io.Reader, stdo return err } if !install { - fmt.Fprintln(stdout, "Skipped Dokploy setup. Run `bort` again when you're ready.") + fmt.Fprintf(stdout, "Skipped Dokploy setup. Run `%s` again when you're ready.\n", bortCommand("")) return errDokploySetupSkipped } return promptInlineInitTargetWithOptions(ctx, stdin, stdout, stderr, inlineInitTargetOptions{Install: true, DefaultURL: defaultURL}) @@ -250,119 +284,6 @@ func promptInlineInitTargetWithOptions(ctx context.Context, stdin io.Reader, std return runInitTarget(ctx, args, stdin, stdout, stderr) } -func executeWithProgress(ctx context.Context, run loadedMigrationRun, stdout, stderr io.Writer) error { - planned := dokploy.PlanFromArtifacts(run.Prepare, run.Sync, run.Cutover) - total := len(planned.Steps) - if total == 0 { - return applyLiveMigration(ctx, run, stderr, nil) - } - - bar := progress.New(progress.WithDefaultGradient(), progress.WithoutPercentage()) - bar.Width = 40 - model := &progressModel{bar: bar, total: total, done: completedApplyPrefix(planned.Steps, run.Applied)} - program := tea.NewProgram(model, tea.WithOutput(stderr)) - - var applyErr error - go func() { - applyErr = applyLiveMigration(ctx, run, stderr, func(p dokploy.StepProgress) { - program.Send(progressTick{progress: p}) - }) - program.Send(progressDone{err: applyErr}) - }() - - if _, err := program.Run(); err != nil && !errors.Is(err, tea.ErrInterrupted) { - return err - } - if applyErr != nil { - return applyErr - } - return nil -} - -type progressTick struct { - progress dokploy.StepProgress -} - -type progressDone struct { - err error -} - -type progressModel struct { - bar progress.Model - total int - done int - current dokploy.StepProgress - err error - closed bool -} - -func (m *progressModel) Init() tea.Cmd { - if m.total <= 0 || m.done <= 0 { - return nil - } - percent := float64(m.done) / float64(m.total) - if percent > 1 { - percent = 1 - } - return m.bar.SetPercent(percent) -} - -func (m *progressModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { - switch msg := msg.(type) { - case progressTick: - m.current = msg.progress - if m.total > 0 && msg.progress.Index >= 0 && msg.progress.Index < m.total { - done := msg.progress.Index - if msg.progress.Status == dokploy.StepStatusOK || msg.progress.Status == dokploy.StepStatusSkipped { - done++ - } - if done > m.done { - m.done = done - } - } - percent := 0.0 - if m.total > 0 { - percent = float64(m.done) / float64(m.total) - if percent > 1 { - percent = 1 - } - } - return m, m.bar.SetPercent(percent) - case progressDone: - m.err = msg.err - m.closed = true - return m, tea.Quit - case progress.FrameMsg: - newModel, cmd := m.bar.Update(msg) - m.bar = newModel.(progress.Model) - return m, cmd - case tea.KeyMsg: - if msg.String() == "ctrl+c" { - return m, tea.Quit - } - } - return m, nil -} - -func (m *progressModel) View() string { - var b strings.Builder - b.WriteString(m.bar.View()) - b.WriteString("\n") - if m.current.Step.Kind != "" { - b.WriteString(fmt.Sprintf(" %d/%d %s %s/%s", m.done, m.total, m.current.Status, m.current.Step.App, m.current.Step.Kind)) - } - if m.closed { - if m.err != nil { - b.WriteString("\n failed: ") - b.WriteString(m.err.Error()) - } else { - b.WriteString("\n done") - } - } - b.WriteString("\n") - return b.String() -} - type appStorePair struct { app string store string diff --git a/internal/safepath/safepath.go b/internal/safepath/safepath.go index eaa3c69..b183b55 100644 --- a/internal/safepath/safepath.go +++ b/internal/safepath/safepath.go @@ -76,6 +76,11 @@ func WriteFileAtomicNoFollow(path string, data []byte, mode os.FileMode) error { os.Remove(tmp) return err } + if err := f.Sync(); err != nil { + f.Close() + os.Remove(tmp) + return err + } if err := f.Close(); err != nil { os.Remove(tmp) return err @@ -84,7 +89,7 @@ func WriteFileAtomicNoFollow(path string, data []byte, mode os.FileMode) error { os.Remove(tmp) return err } - return nil + return syncParentDir(path) } func evalExistingSymlinks(path string) (string, error) { diff --git a/internal/safepath/safepath_other.go b/internal/safepath/safepath_other.go index 5ae0480..c16cd6c 100644 --- a/internal/safepath/safepath_other.go +++ b/internal/safepath/safepath_other.go @@ -34,3 +34,7 @@ func rejectFinalSymlink(path string) error { } return nil } + +func syncParentDir(string) error { + return nil +} diff --git a/internal/safepath/safepath_unix.go b/internal/safepath/safepath_unix.go index ac0e57f..84cd65e 100644 --- a/internal/safepath/safepath_unix.go +++ b/internal/safepath/safepath_unix.go @@ -4,6 +4,7 @@ package safepath import ( "os" + "path/filepath" "syscall" ) @@ -14,3 +15,12 @@ func openNoFollowRead(path string) (*os.File, error) { func openNoFollowWrite(path string, mode os.FileMode) (*os.File, error) { return os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC|syscall.O_NOFOLLOW, mode) } + +func syncParentDir(path string) error { + dir, err := os.Open(filepath.Dir(path)) + if err != nil { + return err + } + defer dir.Close() + return dir.Sync() +} diff --git a/internal/target/dokploy/apply.go b/internal/target/dokploy/apply.go index e3fb959..2c78cd0 100644 --- a/internal/target/dokploy/apply.go +++ b/internal/target/dokploy/apply.go @@ -1,7 +1,9 @@ package dokploy import ( + "bytes" "context" + "encoding/json" "errors" "fmt" "os" @@ -79,6 +81,7 @@ type Plan struct { RunName string RunDir string ResumeFrom int + BeforeStep *func(StepProgress) error OnProgress *func(StepProgress) } @@ -264,13 +267,13 @@ func validatePlanReadyForLiveApply(plan Plan) error { continue } switch app.Readiness { - case "", preparer.ReadinessReadyToCreate: + case "", preparer.ReadinessReadyToCreate, preparer.ReadinessNeedsDecision: default: return fmt.Errorf("live apply blocked by prepare readiness for app %s: %s", app.Name, app.Readiness) } for _, gate := range app.Gates { switch gate.Readiness { - case preparer.ReadinessReadyToCreate: + case preparer.ReadinessReadyToCreate, preparer.ReadinessNeedsDecision: default: return fmt.Errorf("live apply blocked by prepare gate %s/%s: %s", app.Name, gate.Code, gate.Message) } @@ -279,6 +282,91 @@ func validatePlanReadyForLiveApply(plan Plan) error { return nil } +type activeBortOverridePatch struct { + PatchID string `json:"patchId"` + FilePath string `json:"filePath"` + ComposeName string `json:"composeName"` + SourceType string `json:"sourceType"` + Repository string `json:"repository"` + Branch string `json:"branch"` +} + +func (c *Client) validateNoActiveBortOverrides(ctx context.Context, composeID string) error { + composeID = strings.TrimSpace(composeID) + if composeID == "" { + return fmt.Errorf("inspect active Dokploy patches: missing composeId") + } + patches, err := c.activeBortOverridePatches(ctx, composeID) + if err != nil { + return err + } + if len(patches) == 0 { + return nil + } + details := make([]string, 0, len(patches)) + for _, patch := range patches { + repo := strings.Trim(strings.TrimSpace(patch.Repository)+"@"+strings.TrimSpace(patch.Branch), "@") + if repo == "" { + repo = strings.TrimSpace(patch.SourceType) + } + details = append(details, fmt.Sprintf("%s on %s (%s, %s)", patch.PatchID, patch.ComposeName, patch.FilePath, repo)) + } + sort.Strings(details) + return fmt.Errorf("active Bort-owned Dokploy patch(es) override Git-backed compose app(s): %s; merge the patch contents into the repo or disable the patch before live apply so Git remains the source of truth", strings.Join(details, "; ")) +} + +func (c *Client) activeBortOverridePatches(ctx context.Context, composeID string) ([]activeBortOverridePatch, error) { + runner := c.dockerRunner() + pg, err := findDokployPostgresContainer(ctx, runner) + if err != nil { + return nil, fmt.Errorf("locate Dokploy postgres for active patch inspection: %w", err) + } + var out bytes.Buffer + if err := runner.Run(ctx, strings.NewReader(activeBortOverridePatchesSQL(composeID)), &out, + "exec", "-i", pg, "psql", "-U", "dokploy", "-d", "dokploy", "-v", "ON_ERROR_STOP=1", "-At"); err != nil { + return nil, fmt.Errorf("inspect active Dokploy patches: %w", err) + } + return parseActiveBortOverridePatches(out.String()) +} + +func activeBortOverridePatchesSQL(composeID string) string { + return fmt.Sprintf(`select json_build_object( + 'patchId', p."patchId", + 'filePath', p."filePath", + 'composeName', c.name, + 'sourceType', coalesce(c."sourceType"::text, ''), + 'repository', concat_ws('/', nullif(c.owner, ''), nullif(c.repository, '')), + 'branch', coalesce(c.branch, '') + )::text +from patch p +join compose c on c."composeId" = p."composeId" +where p.enabled = true + and p."patchId" like 'bort-%%' + and c."composeId" = %s + and coalesce(c."sourceType"::text, '') not in ('', 'raw') +order by c.name, p."patchId"; +`, sqlStringLiteral(composeID)) +} + +func parseActiveBortOverridePatches(output string) ([]activeBortOverridePatch, error) { + patches := []activeBortOverridePatch{} + for _, line := range strings.Split(output, "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + var patch activeBortOverridePatch + if err := json.Unmarshal([]byte(line), &patch); err != nil { + return nil, fmt.Errorf("parse active Dokploy patch row: %w", err) + } + if strings.TrimSpace(patch.PatchID) == "" || strings.TrimSpace(patch.FilePath) == "" || strings.TrimSpace(patch.ComposeName) == "" { + return nil, fmt.Errorf("parse active Dokploy patch row: missing required identity") + } + patches = append(patches, patch) + } + return patches, nil +} + func (c *Client) Apply(ctx context.Context, plan Plan) error { if err := validatePlanReadyForLiveApply(plan); err != nil { return err @@ -301,7 +389,14 @@ func (c *Client) Apply(ctx context.Context, plan Plan) error { } for index := resumeFrom; index < len(plan.Steps); index++ { step := plan.Steps[index] - emitProgress(plan.OnProgress, StepProgress{Index: index, Total: total, Step: step, Status: StepStatusStarted}) + started := StepProgress{Index: index, Total: total, Step: step, Status: StepStatusStarted} + if plan.BeforeStep != nil { + if err := (*plan.BeforeStep)(started); err != nil { + c.bestEffortResume(ctx, actx, plan, total, pausedApps, coolifyProxyStopped, false) + return fmt.Errorf("before dokploy step %s for %s (%s): %w", step.Kind, step.App, step.Ref, err) + } + } + emitProgress(plan.OnProgress, started) if shouldSkipApplyStep(plan, step) { emitProgress(plan.OnProgress, StepProgress{Index: index, Total: total, Step: step, Status: StepStatusSkipped}) continue @@ -603,6 +698,9 @@ func (c *Client) applyUploadEnv(ctx context.Context, actx *applyContext, step St if err != nil { return err } + if err := c.validateNoActiveBortOverrides(ctx, entry.ComposeID); err != nil { + return err + } return c.UpdateCompose(ctx, entry.ComposeID, composeFile, envContent) } @@ -618,6 +716,9 @@ func (c *Client) applyPushImage(ctx context.Context, actx *applyContext, step St if err := ensureComposeImagesAvailable(ctx, c.dockerRunner(), actx.plan, step.App, composeFile); err != nil { return err } + if err := c.validateNoActiveBortOverrides(ctx, entry.ComposeID); err != nil { + return err + } if err := c.DeployCompose(ctx, entry.ComposeID, deployComposeTitle(actx.plan)); err != nil { if safetyErr := c.validateMigratedVolumeMountsAfterDeploy(ctx, actx, step.App); safetyErr != nil && isUnsafeTargetResumeError(safetyErr) { return fmt.Errorf("deploy dokploy compose: %w; post-deploy safety check: %v", err, safetyErr) @@ -812,6 +913,9 @@ func (c *Client) applyActivateRoutes(ctx context.Context, actx *applyContext, st if err != nil { return err } + if err := c.validateNoActiveBortOverrides(ctx, entry.ComposeID); err != nil { + return err + } if err := c.UpdateCompose(ctx, entry.ComposeID, composeFile, envContent); err != nil { return err } diff --git a/internal/target/dokploy/apply_test.go b/internal/target/dokploy/apply_test.go index b5d12df..9bc1c28 100644 --- a/internal/target/dokploy/apply_test.go +++ b/internal/target/dokploy/apply_test.go @@ -100,7 +100,19 @@ func TestApplyResumeFromPrimesCompletedCreateSteps(t *testing.T) { })) defer server.Close() - client := &Client{BaseURL: server.URL, Token: "secret", HTTPClient: server.Client()} + client := &Client{ + BaseURL: server.URL, + Token: "secret", + HTTPClient: server.Client(), + Docker: &fakeDockerRunner{ + outputs: map[string][]byte{ + "ps --format {{.Names}}": []byte("dokploy-postgres\n"), + }, + runOutputs: map[string][]byte{ + "exec -i dokploy-postgres psql -U dokploy -d dokploy -v ON_ERROR_STOP=1 -At": {}, + }, + }, + } plan := Plan{ ResumeFrom: 2, Steps: []Step{ @@ -155,6 +167,154 @@ func TestApplySkipsPlatformAppSteps(t *testing.T) { } } +func TestApplyStopsBeforeStepWhenPersistenceHookFails(t *testing.T) { + sentinel := errors.New("persist started step") + beforeStep := func(StepProgress) error { + return sentinel + } + var progress []StepProgress + onProgress := func(p StepProgress) { + progress = append(progress, p) + } + client := &Client{} + err := client.Apply(context.Background(), Plan{ + Steps: []Step{{Kind: StepCreateProject, App: "source", Ref: "source"}}, + Prepare: preparer.Result{Apps: []preparer.AppPlan{{Name: "source", Role: "platform"}}}, + BeforeStep: &beforeStep, + OnProgress: &onProgress, + }) + if !errors.Is(err, sentinel) { + t.Fatalf("expected before-step failure, got %v", err) + } + if len(progress) != 0 { + t.Fatalf("expected no progress after before-step failure, got %#v", progress) + } +} + +func TestApplyResumesPausedSourceWhenPersistenceFailsBetweenSteps(t *testing.T) { + app := preparer.AppPlan{Name: "api"} + app.Resources.Volumes = []preparer.VolumeResource{{Service: "web", Type: "volume", SourceContainerID: "web-id"}} + runner := &fakeDockerRunner{outputs: map[string][]byte{ + "inspect --type container web-id": []byte(`[{"Id":"web-id","Name":"/web","State":{"Running":true,"Status":"running"}}]`), + "stop web-id": []byte("web-id\n"), + }} + var beforeCalls int + beforeStep := func(StepProgress) error { + beforeCalls++ + if beforeCalls == 2 { + return errors.New("persist failed") + } + return nil + } + var progress []StepProgress + onProgress := func(p StepProgress) { + progress = append(progress, p) + } + client := &Client{Docker: runner} + err := client.Apply(context.Background(), Plan{ + Steps: []Step{{Kind: StepPauseSource, App: "api", Ref: "api"}, {Kind: StepCreateVolume, App: "api", Ref: "data"}}, + Prepare: preparer.Result{Apps: []preparer.AppPlan{app}}, + BeforeStep: &beforeStep, + OnProgress: &onProgress, + }) + if err == nil || !strings.Contains(err.Error(), "persist failed") { + t.Fatalf("expected persistence failure, got %v", err) + } + for _, p := range progress { + if p.Step.Kind == StepResumeSource && p.Status == StepStatusOK { + return + } + } + t.Fatalf("expected cleanup to resume the paused source, got %#v", progress) +} + +func TestActivePatchGuardBlocksGitBackedCompose(t *testing.T) { + runner := &fakeDockerRunner{ + outputs: map[string][]byte{ + "ps --format {{.Names}}": []byte("dokploy-postgres.1.task\n"), + }, + runOutputs: map[string][]byte{ + "exec -i dokploy-postgres.1.task psql -U dokploy -d dokploy -v ON_ERROR_STOP=1 -At": []byte(`{"patchId":"bort-api-compose","filePath":"docker|compose.yml","composeName":"api","sourceType":"github","repository":"owner/repo","branch":"main"}` + "\n"), + }, + } + client := &Client{Docker: runner} + err := client.validateNoActiveBortOverrides(context.Background(), "compose-api") + if err == nil || !strings.Contains(err.Error(), "active Bort-owned Dokploy patch") || !strings.Contains(err.Error(), "bort-api-compose") || !strings.Contains(err.Error(), "docker|compose.yml") { + t.Fatalf("expected active patch guard, got %v", err) + } + if len(runner.runs) != 1 { + t.Fatalf("expected one Dokploy DB patch inspection, got %#v", runner.runs) + } + sql := string(runner.runs[0].Stdin) + for _, want := range []string{"c.\"composeId\" = 'compose-api'", "json_build_object", "p.\"patchId\" like 'bort-%'", "sourceType"} { + if !strings.Contains(sql, want) { + t.Fatalf("expected patch guard sql to contain %q, got:\n%s", want, sql) + } + } +} + +func TestApplyPushImageChecksActivePatchWithoutEnvOrRoutes(t *testing.T) { + bundleDir := t.TempDir() + appDir := filepath.Join(bundleDir, "api") + if err := os.MkdirAll(appDir, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(appDir, "compose.yaml"), []byte("services: {}\n"), 0o600); err != nil { + t.Fatal(err) + } + runner := &fakeDockerRunner{ + outputs: map[string][]byte{"ps --format {{.Names}}": []byte("dokploy-postgres.1.task\n")}, + runOutputs: map[string][]byte{ + "exec -i dokploy-postgres.1.task psql -U dokploy -d dokploy -v ON_ERROR_STOP=1 -At": []byte(`{"patchId":"bort-api-compose","filePath":"docker|compose.yml","composeName":"api","sourceType":"github","repository":"owner/repo","branch":"main"}` + "\n"), + }, + } + client := &Client{Docker: runner} + app := preparer.AppPlan{Name: "api", Directory: "api", TargetResources: &preparer.TargetResources{Dokploy: &preparer.DokployResources{ + Project: preparer.DokployProject{Name: "api", Environment: "production"}, + ComposeApp: preparer.DokployComposeApp{Name: "api", ComposePath: "compose.yaml"}, + }}} + actx := &applyContext{plan: Plan{Prepare: preparer.Result{BundleDir: bundleDir, Apps: []preparer.AppPlan{app}}}, cache: map[string]*appCache{"api": {ComposeID: "compose-1"}}} + err := client.applyPushImage(context.Background(), actx, Step{Kind: StepPushImage, App: "api", Ref: "api"}) + if err == nil || !strings.Contains(err.Error(), "active Bort-owned Dokploy patch") { + t.Fatalf("expected unconditional deployment path to block active patch, got %v", err) + } +} + +func TestActivePatchGuardRejectsMissingComposeID(t *testing.T) { + client := &Client{Docker: &fakeDockerRunner{}} + if err := client.validateNoActiveBortOverrides(context.Background(), ""); err == nil || !strings.Contains(err.Error(), "missing composeId") { + t.Fatalf("expected missing composeId to fail closed, got %v", err) + } +} + +func TestParseActiveBortOverridePatchesFailsClosedOnMalformedRow(t *testing.T) { + if _, err := parseActiveBortOverridePatches("not-json\n"); err == nil { + t.Fatal("expected malformed patch row to fail closed") + } +} + +func TestActivePatchGuardFailsClosedWhenInspectionCannotFindDokploy(t *testing.T) { + client := &Client{Docker: &fakeDockerRunner{}} + err := client.validateNoActiveBortOverrides(context.Background(), "compose-api") + if err == nil || !strings.Contains(err.Error(), "active patch inspection") || !strings.Contains(err.Error(), "docker output not stubbed") { + t.Fatalf("expected active patch inspection to fail closed, got %v", err) + } +} + +func TestActivePatchGuardFailsClosedWhenDokployDatabaseIsUnavailable(t *testing.T) { + runner := &fakeDockerRunner{outputs: map[string][]byte{ + "ps --format {{.Names}}": []byte("unrelated\n"), + }} + client := &Client{Docker: runner} + + if err := client.validateNoActiveBortOverrides(context.Background(), "compose-api"); err == nil || !strings.Contains(err.Error(), "dokploy postgres container was not found") { + t.Fatalf("expected missing Dokploy DB to fail closed, got %v", err) + } + if len(runner.runs) != 0 { + t.Fatalf("expected no Dokploy DB query when postgres is absent, got %#v", runner.runs) + } +} + func TestPlanFromArtifactsUsesGroupedProjectAndStableComposeNames(t *testing.T) { prepare := preparer.Result{Apps: []preparer.AppPlan{ { @@ -232,7 +392,19 @@ func TestApplyCreateStepsReuseGroupedProjectPerApp(t *testing.T) { })) defer server.Close() - client := &Client{BaseURL: server.URL, Token: "secret", HTTPClient: server.Client()} + client := &Client{ + BaseURL: server.URL, + Token: "secret", + HTTPClient: server.Client(), + Docker: &fakeDockerRunner{ + outputs: map[string][]byte{ + "ps --format {{.Names}}": []byte("dokploy-postgres\n"), + }, + runOutputs: map[string][]byte{ + "exec -i dokploy-postgres psql -U dokploy -d dokploy -v ON_ERROR_STOP=1 -At": {}, + }, + }, + } plan := Plan{ Steps: []Step{ {Kind: StepCreateProject, App: "demo-app", Ref: "demo-project"}, @@ -629,7 +801,19 @@ func TestApplyActivateRoutesUpdatesDomainsAndRedeploys(t *testing.T) { })) defer server.Close() - client := &Client{BaseURL: server.URL, Token: "secret", HTTPClient: server.Client(), Docker: &fakeDockerRunner{outputs: map[string][]byte{}}} + client := &Client{ + BaseURL: server.URL, + Token: "secret", + HTTPClient: server.Client(), + Docker: &fakeDockerRunner{ + outputs: map[string][]byte{ + "ps --format {{.Names}}": []byte("dokploy-postgres\n"), + }, + runOutputs: map[string][]byte{ + "exec -i dokploy-postgres psql -U dokploy -d dokploy -v ON_ERROR_STOP=1 -At": {}, + }, + }, + } plan := Plan{ Prepare: preparer.Result{BundleDir: bundleDir, Apps: []preparer.AppPlan{{ Name: "example-app", diff --git a/internal/target/dokploy/cleanup.go b/internal/target/dokploy/cleanup.go index 7b3a420..63e7fae 100644 --- a/internal/target/dokploy/cleanup.go +++ b/internal/target/dokploy/cleanup.go @@ -3,8 +3,10 @@ package dokploy import ( "bytes" "context" + "encoding/json" "fmt" "os" + pathpkg "path" "path/filepath" "sort" "strings" @@ -28,6 +30,57 @@ type StalePlatformCleanupResult struct { Deleted []StalePlatformProject `json:"deleted"` } +type SourcePurgeContainer struct { + App string `json:"app,omitempty"` + Service string `json:"service,omitempty"` + ContainerID string `json:"containerId,omitempty"` + ContainerName string `json:"containerName,omitempty"` +} + +type SourcePurgeVolume struct { + App string `json:"app,omitempty"` + Service string `json:"service,omitempty"` + Name string `json:"name"` + ExpectedAbsent bool `json:"expectedAbsent,omitempty"` +} + +type SourcePurgeNetwork struct { + App string `json:"app,omitempty"` + Name string `json:"name"` + ExpectedIdentity string `json:"expectedIdentity,omitempty"` + ExpectedAbsent bool `json:"expectedAbsent,omitempty"` +} + +type SourcePurgePath struct { + App string `json:"app,omitempty"` + Source string `json:"source,omitempty"` + Path string `json:"path"` + AllowPlatform bool `json:"allowPlatform,omitempty"` + ExpectedAbsent bool `json:"expectedAbsent,omitempty"` +} + +type SourcePurgeOptions struct { + Containers []SourcePurgeContainer + Volumes []SourcePurgeVolume + Networks []SourcePurgeNetwork + Paths []SourcePurgePath + OnProgress func(SourcePurgeResult) error +} + +type SourcePurgeResult struct { + Containers []SourcePurgeResourceResult `json:"containers,omitempty"` + Volumes []SourcePurgeResourceResult `json:"volumes,omitempty"` + Networks []SourcePurgeResourceResult `json:"networks,omitempty"` + Paths []SourcePurgeResourceResult `json:"paths,omitempty"` +} + +type SourcePurgeResourceResult struct { + App string `json:"app,omitempty"` + Ref string `json:"ref"` + Status string `json:"status"` + Message string `json:"message,omitempty"` +} + func (c *Client) CleanupStalePlatformProjects(ctx context.Context, opts StalePlatformCleanupOptions) (StalePlatformCleanupResult, error) { names := cleanupProjectNames(opts.ProjectNames) if len(names) == 0 { @@ -49,6 +102,782 @@ func (c *Client) CleanupStalePlatformProjects(ctx context.Context, opts StalePla return StalePlatformCleanupResult{BackupPath: backupPath, Deleted: deleted}, nil } +func (c *Client) PurgeSourceResources(ctx context.Context, opts SourcePurgeOptions) (SourcePurgeResult, error) { + runner := c.dockerRunner() + result := SourcePurgeResult{} + + containers, err := cleanupSourcePurgeContainers(opts.Containers) + if err != nil { + return result, err + } + volumes := cleanupSourcePurgeVolumes(opts.Volumes) + paths := cleanupSourcePurgePaths(opts.Paths) + networks := cleanupSourcePurgeNetworks(opts.Networks) + if sourcePurgeRequiresManualCompletion(volumes, paths, networks) { + for _, volume := range volumes { + if err := runSourcePurgeResource(&result, &result.Volumes, opts.OnProgress, SourcePurgeResourceResult{App: volume.App, Ref: volume.Name}, func() (SourcePurgeResourceResult, error) { + return purgeSourceVolume(ctx, runner, volume) + }); err != nil { + return result, err + } + } + for _, path := range paths { + if err := runSourcePurgeResource(&result, &result.Paths, opts.OnProgress, SourcePurgeResourceResult{App: path.App, Ref: path.Path}, func() (SourcePurgeResourceResult, error) { + return purgeSourcePath(path) + }); err != nil { + return result, err + } + } + for _, container := range containers { + if err := runSourcePurgeResource(&result, &result.Containers, opts.OnProgress, SourcePurgeResourceResult{App: container.App, Ref: container.ContainerID}, func() (SourcePurgeResourceResult, error) { + return verifySourcePurgeContainerAbsent(ctx, runner, container) + }); err != nil { + return result, err + } + } + for _, network := range networks { + if err := runSourcePurgeResource(&result, &result.Networks, opts.OnProgress, SourcePurgeResourceResult{App: network.App, Ref: network.Name}, func() (SourcePurgeResourceResult, error) { + return verifySourcePurgeNetworkAbsent(ctx, runner, network) + }); err != nil { + return result, err + } + } + return result, nil + } + for _, volume := range volumes { + if err := runSourcePurgeResource(&result, &result.Volumes, opts.OnProgress, SourcePurgeResourceResult{App: volume.App, Ref: volume.Name}, func() (SourcePurgeResourceResult, error) { + return purgeSourceVolume(ctx, runner, volume) + }); err != nil { + return result, err + } + } + for _, path := range paths { + if err := runSourcePurgeResource(&result, &result.Paths, opts.OnProgress, SourcePurgeResourceResult{App: path.App, Ref: path.Path}, func() (SourcePurgeResourceResult, error) { + return purgeSourcePath(path) + }); err != nil { + return result, err + } + } + for _, network := range networks { + if !network.ExpectedAbsent { + continue + } + if err := runSourcePurgeResource(&result, &result.Networks, opts.OnProgress, SourcePurgeResourceResult{App: network.App, Ref: network.Name}, func() (SourcePurgeResourceResult, error) { + return purgeSourceNetwork(ctx, runner, network) + }); err != nil { + return result, err + } + } + for _, container := range containers { + if err := runSourcePurgeResource(&result, &result.Containers, opts.OnProgress, SourcePurgeResourceResult{App: container.App, Ref: container.ContainerID}, func() (SourcePurgeResourceResult, error) { + return purgeSourceContainer(ctx, runner, container) + }); err != nil { + return result, err + } + } + for _, network := range networks { + if network.ExpectedAbsent { + continue + } + if err := runSourcePurgeResource(&result, &result.Networks, opts.OnProgress, SourcePurgeResourceResult{App: network.App, Ref: network.Name}, func() (SourcePurgeResourceResult, error) { + return purgeSourceNetwork(ctx, runner, network) + }); err != nil { + return result, err + } + } + return result, nil +} + +func sourcePurgeRequiresManualCompletion(volumes []SourcePurgeVolume, paths []SourcePurgePath, networks []SourcePurgeNetwork) bool { + if len(volumes) > 0 || len(paths) > 0 { + return true + } + for _, network := range networks { + if network.ExpectedAbsent { + return true + } + } + return false +} + +func runSourcePurgeResource(result *SourcePurgeResult, resources *[]SourcePurgeResourceResult, onProgress func(SourcePurgeResult) error, started SourcePurgeResourceResult, operation func() (SourcePurgeResourceResult, error)) error { + started.Status = "started" + *resources = append(*resources, started) + if err := publishSourcePurgeProgress(*result, onProgress); err != nil { + return err + } + outcome, operationErr := operation() + (*resources)[len(*resources)-1] = outcome + if err := publishSourcePurgeProgress(*result, onProgress); err != nil { + if operationErr != nil { + return fmt.Errorf("%v; %w", operationErr, err) + } + return err + } + return operationErr +} + +func publishSourcePurgeProgress(result SourcePurgeResult, onProgress func(SourcePurgeResult) error) error { + if onProgress == nil { + return nil + } + snapshot := SourcePurgeResult{ + Containers: append([]SourcePurgeResourceResult{}, result.Containers...), + Volumes: append([]SourcePurgeResourceResult{}, result.Volumes...), + Networks: append([]SourcePurgeResourceResult{}, result.Networks...), + Paths: append([]SourcePurgeResourceResult{}, result.Paths...), + } + if err := onProgress(snapshot); err != nil { + return fmt.Errorf("persist source purge progress: %w", err) + } + return nil +} + +func (c *Client) IdentifySourcePurgeResources(ctx context.Context, opts SourcePurgeOptions) (SourcePurgeOptions, error) { + runner := c.dockerRunner() + identified := opts + containers, err := cleanupSourcePurgeContainers(opts.Containers) + if err != nil { + return SourcePurgeOptions{}, err + } + for _, container := range containers { + if strings.TrimSpace(container.ContainerID) == "" { + return SourcePurgeOptions{}, fmt.Errorf("refusing source container %q without a stable container ID", container.ContainerName) + } + } + identified.Containers = containers + identified.Volumes = nil + for _, volume := range cleanupSourcePurgeVolumes(opts.Volumes) { + absent, err := inspectSourcePurgeVolumeAbsent(ctx, runner, volume.Name) + if err != nil { + return SourcePurgeOptions{}, err + } + if !absent { + return SourcePurgeOptions{}, fmt.Errorf("named volume %q still exists; remove it manually before cleanup purge --apply", volume.Name) + } + volume.ExpectedAbsent = true + identified.Volumes = append(identified.Volumes, volume) + } + identified.Networks = nil + for _, network := range cleanupSourcePurgeNetworks(opts.Networks) { + identity, absent, err := inspectSourcePurgeNetworkIdentity(ctx, runner, network.Name) + if err != nil { + return SourcePurgeOptions{}, err + } + network.ExpectedIdentity = identity + network.ExpectedAbsent = absent + identified.Networks = append(identified.Networks, network) + } + identified.Paths = nil + for _, path := range cleanupSourcePurgePaths(opts.Paths) { + absent, err := sourcePurgePathAbsentNoFollow(path.Path, path.AllowPlatform) + if err != nil { + return SourcePurgeOptions{}, err + } + if !absent { + return SourcePurgeOptions{}, fmt.Errorf("source path %q still exists; remove it manually before cleanup purge --apply", path.Path) + } + path.ExpectedAbsent = true + identified.Paths = append(identified.Paths, path) + } + if sourcePurgeRequiresManualCompletion(identified.Volumes, identified.Paths, identified.Networks) { + for _, container := range identified.Containers { + if _, err := verifySourcePurgeContainerAbsent(ctx, runner, container); err != nil { + return SourcePurgeOptions{}, fmt.Errorf("source purge has absence-only prerequisites and cannot remove other resources safely; remove all listed containers and networks manually first: %w", err) + } + } + for _, network := range identified.Networks { + if !network.ExpectedAbsent { + return SourcePurgeOptions{}, fmt.Errorf("source purge has absence-only prerequisites and cannot remove other resources safely; remove source network %q manually before cleanup purge --apply", network.Name) + } + } + } else { + for i, container := range identified.Containers { + canonical, err := identifySourcePurgeContainer(ctx, runner, container) + if err != nil { + return SourcePurgeOptions{}, err + } + identified.Containers[i] = canonical + } + } + return identified, nil +} + +func identifySourcePurgeContainer(ctx context.Context, runner dockerRunner, item SourcePurgeContainer) (SourcePurgeContainer, error) { + containerID := strings.TrimSpace(item.ContainerID) + container, err := inspectContainer(ctx, runner, containerID) + if err != nil { + if isContainerMissingErr(err) { + return item, nil + } + return SourcePurgeContainer{}, err + } + canonicalID := strings.TrimSpace(container.ID) + if !sourcePurgeContainerIDMatches(containerID, canonicalID) { + return SourcePurgeContainer{}, fmt.Errorf("refusing source container %s: docker returned container ID %q, not the reviewed ID %q", firstNonEmpty(item.ContainerName, containerID), canonicalID, containerID) + } + if item.ContainerName != "" && container.Name != "" && item.ContainerName != container.Name { + return SourcePurgeContainer{}, fmt.Errorf("refusing source container %s: expected name %q, found %q", containerID, item.ContainerName, container.Name) + } + item.ContainerID = canonicalID + if item.ContainerName == "" { + item.ContainerName = container.Name + } + return item, nil +} + +func sourcePurgeContainerIDMatches(reviewed, inspected string) bool { + reviewed = strings.TrimSpace(reviewed) + inspected = strings.TrimSpace(inspected) + return inspected == reviewed || len(reviewed) >= 12 && strings.HasPrefix(inspected, reviewed) +} + +func sourcePurgeContainerIDsEquivalent(a, b string) bool { + a = strings.TrimSpace(a) + b = strings.TrimSpace(b) + if a == b { + return a != "" + } + if len(a) > len(b) { + a, b = b, a + } + return len(a) >= 12 && strings.HasPrefix(b, a) +} + +func cleanupSourcePurgeContainers(containers []SourcePurgeContainer) ([]SourcePurgeContainer, error) { + idsByName := map[string]string{} + for _, container := range containers { + container.ContainerID = strings.TrimSpace(container.ContainerID) + container.ContainerName = trimDockerName(container.ContainerName) + if container.ContainerID == "" || container.ContainerName == "" { + continue + } + if id, ok := idsByName[container.ContainerName]; ok { + if !sourcePurgeContainerIDsEquivalent(id, container.ContainerID) { + return nil, fmt.Errorf("source container name %q refers to conflicting container IDs", container.ContainerName) + } + if len(id) > len(container.ContainerID) { + container.ContainerID = id + } + } + idsByName[container.ContainerName] = container.ContainerID + } + seen := map[string]int{} + cleaned := []SourcePurgeContainer{} + for _, container := range containers { + container.ContainerID = strings.TrimSpace(container.ContainerID) + container.ContainerName = trimDockerName(container.ContainerName) + ref := firstNonEmpty(container.ContainerID, container.ContainerName) + if ref == "" { + continue + } + key := "name:" + container.ContainerName + if container.ContainerID != "" { + key = "id:" + container.ContainerID + } + index, ok := seen[key] + matchedKey := key + if !ok && container.ContainerID != "" { + for seenKey, seenIndex := range seen { + if strings.HasPrefix(seenKey, "id:") && sourcePurgeContainerIDsEquivalent(strings.TrimPrefix(seenKey, "id:"), container.ContainerID) { + index, ok, matchedKey = seenIndex, true, seenKey + break + } + } + } + if ok { + if cleaned[index].ContainerName != "" && container.ContainerName != "" && cleaned[index].ContainerName != container.ContainerName { + return nil, fmt.Errorf("source container ID %q refers to conflicting container names", container.ContainerID) + } + if cleaned[index].Service != "" && container.Service != "" && cleaned[index].Service != container.Service { + return nil, fmt.Errorf("source container ID %q refers to conflicting services", container.ContainerID) + } + if cleaned[index].ContainerName == "" { + cleaned[index].ContainerName = container.ContainerName + } + if cleaned[index].Service == "" { + cleaned[index].Service = container.Service + } + if len(container.ContainerID) > len(cleaned[index].ContainerID) { + cleaned[index].ContainerID = container.ContainerID + delete(seen, matchedKey) + seen["id:"+container.ContainerID] = index + } + continue + } + seen[key] = len(cleaned) + cleaned = append(cleaned, container) + } + return cleaned, nil +} + +func cleanupSourcePurgeVolumes(volumes []SourcePurgeVolume) []SourcePurgeVolume { + seen := map[string]struct{}{} + cleaned := []SourcePurgeVolume{} + for _, volume := range volumes { + volume.Name = strings.TrimSpace(volume.Name) + if volume.Name == "" { + continue + } + if _, ok := seen[volume.Name]; ok { + continue + } + seen[volume.Name] = struct{}{} + cleaned = append(cleaned, volume) + } + sort.Slice(cleaned, func(i, j int) bool { return cleaned[i].Name < cleaned[j].Name }) + return cleaned +} + +func cleanupSourcePurgeNetworks(networks []SourcePurgeNetwork) []SourcePurgeNetwork { + seen := map[string]struct{}{} + cleaned := []SourcePurgeNetwork{} + for _, network := range networks { + network.Name = strings.TrimSpace(network.Name) + if network.Name == "" { + continue + } + if _, ok := seen[network.Name]; ok { + continue + } + seen[network.Name] = struct{}{} + cleaned = append(cleaned, network) + } + sort.Slice(cleaned, func(i, j int) bool { return cleaned[i].Name < cleaned[j].Name }) + return cleaned +} + +func cleanupSourcePurgePaths(paths []SourcePurgePath) []SourcePurgePath { + seen := map[string]struct{}{} + cleaned := []SourcePurgePath{} + for _, path := range paths { + path.Path = pathpkg.Clean(strings.TrimSpace(path.Path)) + if path.Path == "." || path.Path == "" { + continue + } + if _, ok := seen[path.Path]; ok { + continue + } + seen[path.Path] = struct{}{} + cleaned = append(cleaned, path) + } + sort.Slice(cleaned, func(i, j int) bool { return cleaned[i].Path < cleaned[j].Path }) + return cleaned +} + +func purgeSourceContainer(ctx context.Context, runner dockerRunner, item SourcePurgeContainer) (SourcePurgeResourceResult, error) { + ref := firstNonEmpty(item.ContainerID, item.ContainerName) + result := SourcePurgeResourceResult{App: item.App, Ref: ref} + containerID := strings.TrimSpace(item.ContainerID) + if containerID == "" { + result.Status = "blocked" + return result, fmt.Errorf("refusing to remove source container %q without a stable container ID", item.ContainerName) + } + container, err := inspectContainer(ctx, runner, containerID) + if err != nil { + if isContainerMissingErr(err) { + containerName := strings.TrimSpace(item.ContainerName) + if containerName != "" { + replacement, nameErr := inspectContainer(ctx, runner, containerName) + if nameErr == nil { + result.Status = "blocked" + return result, fmt.Errorf("refusing source container name %q because it now refers to container ID %q", containerName, replacement.ID) + } + if !isContainerMissingErr(nameErr) { + result.Status = "error" + return result, nameErr + } + } + result.Status = "skipped" + result.Message = "container is already absent" + return result, nil + } + result.Status = "error" + return result, err + } + if strings.TrimSpace(container.ID) != containerID { + result.Status = "blocked" + return result, fmt.Errorf("refusing to remove container %s: docker returned container ID %q, not the reviewed ID %q", ref, container.ID, containerID) + } + if item.ContainerName != "" && container.Name != "" && container.Name != item.ContainerName { + result.Status = "blocked" + return result, fmt.Errorf("refusing to remove container %s: expected name %q, found %q", ref, item.ContainerName, container.Name) + } + if _, err := runner.Output(ctx, "rm", "-f", containerID); err != nil { + if isContainerMissingErr(err) { + result.Status = "skipped" + result.Message = "container is already absent" + return result, nil + } + result.Status = "error" + return result, err + } + result.Ref = firstNonEmpty(container.Name, containerID) + result.Status = "removed" + return result, nil +} + +func verifySourcePurgeContainerAbsent(ctx context.Context, runner dockerRunner, item SourcePurgeContainer) (SourcePurgeResourceResult, error) { + containerID := strings.TrimSpace(item.ContainerID) + result := SourcePurgeResourceResult{App: item.App, Ref: firstNonEmpty(containerID, item.ContainerName)} + if containerID == "" { + result.Status = "blocked" + return result, fmt.Errorf("refusing source container %q without a stable container ID", item.ContainerName) + } + container, err := inspectContainer(ctx, runner, containerID) + if err != nil { + if isContainerMissingErr(err) { + containerName := strings.TrimSpace(item.ContainerName) + if containerName != "" { + replacement, nameErr := inspectContainer(ctx, runner, containerName) + if nameErr == nil { + result.Status = "blocked" + return result, fmt.Errorf("source container name %q now refers to container ID %q; remove it manually before cleanup purge --apply", containerName, replacement.ID) + } + if !isContainerMissingErr(nameErr) { + result.Status = "error" + return result, nameErr + } + } + result.Status = "skipped" + result.Message = "container remains absent after confirmation" + return result, nil + } + result.Status = "error" + return result, err + } + if !sourcePurgeContainerIDMatches(containerID, container.ID) { + result.Status = "blocked" + return result, fmt.Errorf("refusing source container %s: docker returned container ID %q, not the reviewed ID %q", result.Ref, container.ID, containerID) + } + result.Status = "blocked" + return result, fmt.Errorf("source container %q still exists; remove it manually before cleanup purge --apply", firstNonEmpty(container.Name, containerID)) +} + +func purgeSourceVolume(ctx context.Context, runner dockerRunner, item SourcePurgeVolume) (SourcePurgeResourceResult, error) { + name := strings.TrimSpace(item.Name) + result := SourcePurgeResourceResult{App: item.App, Ref: name} + if err := validateDockerPurgeName("volume", name); err != nil { + result.Status = "blocked" + return result, err + } + if !item.ExpectedAbsent { + result.Status = "blocked" + return result, fmt.Errorf("refusing source volume %q without pre-confirmation absence validation", name) + } + absent, err := inspectSourcePurgeVolumeAbsent(ctx, runner, name) + if err != nil { + result.Status = "error" + return result, err + } + if !absent { + result.Status = "blocked" + return result, fmt.Errorf("refusing source volume %q because it appeared after confirmation", name) + } + result.Status = "skipped" + result.Message = "volume remains absent after confirmation" + return result, nil +} + +func purgeSourceNetwork(ctx context.Context, runner dockerRunner, item SourcePurgeNetwork) (SourcePurgeResourceResult, error) { + name := strings.TrimSpace(item.Name) + result := SourcePurgeResourceResult{App: item.App, Ref: name} + if err := validateDockerPurgeName("network", name); err != nil { + result.Status = "blocked" + return result, err + } + if IsProtectedSourcePurgeNetwork(name) { + result.Status = "blocked" + return result, fmt.Errorf("refusing to remove protected docker network %q", name) + } + if item.ExpectedAbsent { + _, absent, err := inspectSourcePurgeNetworkIdentity(ctx, runner, name) + if err != nil { + result.Status = "error" + return result, err + } + if !absent { + result.Status = "blocked" + return result, fmt.Errorf("refusing source network %q because it appeared after confirmation", name) + } + result.Status = "skipped" + result.Message = "network remains absent after confirmation" + return result, nil + } + identity := strings.TrimSpace(item.ExpectedIdentity) + if identity == "" { + result.Status = "blocked" + return result, fmt.Errorf("refusing to remove docker network %q without a pre-confirmation identity", name) + } + if _, err := runner.Output(ctx, "network", "rm", identity); err != nil { + if isDockerVolumeOrNetworkMissingErr(err) { + _, absent, inspectErr := inspectSourcePurgeNetworkIdentity(ctx, runner, name) + if inspectErr != nil { + result.Status = "error" + return result, fmt.Errorf("recheck source network %q after reviewed network %s disappeared: %w", name, identity, inspectErr) + } + if !absent { + result.Status = "blocked" + return result, fmt.Errorf("refusing source network %q because a network with that name appeared after confirmation", name) + } + result.Status = "skipped" + result.Message = "network is already absent" + return result, nil + } + result.Status = "error" + return result, err + } + result.Status = "removed" + return result, nil +} + +func verifySourcePurgeNetworkAbsent(ctx context.Context, runner dockerRunner, item SourcePurgeNetwork) (SourcePurgeResourceResult, error) { + name := strings.TrimSpace(item.Name) + result := SourcePurgeResourceResult{App: item.App, Ref: name} + if err := validateDockerPurgeName("network", name); err != nil { + result.Status = "blocked" + return result, err + } + if IsProtectedSourcePurgeNetwork(name) { + result.Status = "blocked" + return result, fmt.Errorf("refusing protected docker network %q", name) + } + _, absent, err := inspectSourcePurgeNetworkIdentity(ctx, runner, name) + if err != nil { + result.Status = "error" + return result, err + } + if !absent { + result.Status = "blocked" + return result, fmt.Errorf("source network %q still exists; remove it manually before cleanup purge --apply", name) + } + result.Status = "skipped" + result.Message = "network remains absent after confirmation" + return result, nil +} + +func purgeSourcePath(item SourcePurgePath) (SourcePurgeResourceResult, error) { + path := pathpkg.Clean(strings.TrimSpace(item.Path)) + result := SourcePurgeResourceResult{App: item.App, Ref: path} + if err := ValidateSourcePurgePath(path, item.AllowPlatform); err != nil { + result.Status = "blocked" + return result, err + } + if !item.ExpectedAbsent { + result.Status = "blocked" + return result, fmt.Errorf("refusing source path %q without pre-confirmation absence validation", path) + } + absent, err := sourcePurgePathAbsentNoFollow(path, item.AllowPlatform) + if err != nil { + result.Status = "error" + return result, err + } + if !absent { + result.Status = "blocked" + return result, fmt.Errorf("refusing source path %q because it appeared after confirmation", path) + } + result.Status = "skipped" + result.Message = "path remains absent after confirmation" + return result, nil +} + +type sourcePurgeVolumeState struct { + Name string `json:"Name"` +} + +type sourcePurgeNetworkIdentity struct { + ID string `json:"Id"` + Name string `json:"Name"` +} + +func inspectSourcePurgeVolumeAbsent(ctx context.Context, runner dockerRunner, name string) (bool, error) { + out, err := runner.Output(ctx, "volume", "inspect", name) + if err != nil { + if isDockerVolumeOrNetworkMissingErr(err) { + return true, nil + } + return false, err + } + var volumes []sourcePurgeVolumeState + if err := json.Unmarshal(out, &volumes); err != nil { + return false, fmt.Errorf("decode docker volume inspect %s: %w", name, err) + } + if len(volumes) != 1 || strings.TrimSpace(volumes[0].Name) != name { + return false, fmt.Errorf("docker volume inspect %s returned an unexpected resource", name) + } + return false, nil +} + +func inspectSourcePurgeNetworkIdentity(ctx context.Context, runner dockerRunner, name string) (string, bool, error) { + out, err := runner.Output(ctx, "network", "inspect", name) + if err != nil { + if isDockerVolumeOrNetworkMissingErr(err) { + return "", true, nil + } + return "", false, err + } + var networks []sourcePurgeNetworkIdentity + if err := json.Unmarshal(out, &networks); err != nil { + return "", false, fmt.Errorf("decode docker network inspect %s: %w", name, err) + } + if len(networks) != 1 || strings.TrimSpace(networks[0].Name) != name || strings.TrimSpace(networks[0].ID) == "" { + return "", false, fmt.Errorf("docker network inspect %s returned an unexpected resource", name) + } + return strings.TrimSpace(networks[0].ID), false, nil +} + +func validateDockerPurgeName(kind, name string) error { + if strings.TrimSpace(name) == "" { + return fmt.Errorf("docker %s name is empty", kind) + } + if name == "/" || strings.Contains(name, "\x00") { + return fmt.Errorf("refusing to remove invalid docker %s %q", kind, name) + } + return nil +} + +func IsProtectedSourcePurgeNetwork(name string) bool { + switch strings.TrimSpace(name) { + case "bridge", "host", "none", "ingress": + return true + default: + return false + } +} + +func ValidateSourcePurgePath(path string, allowPlatform bool) error { + if err := validateSourcePurgePathLexical(path, allowPlatform); err != nil { + return err + } + cleaned := pathpkg.Clean(path) + localPath := filepath.FromSlash(cleaned) + resolved, ok, err := sourcePurgeResolveExistingPath(localPath) + if err != nil { + return err + } + if ok && resolved != filepath.Clean(localPath) { + return fmt.Errorf("source purge path %q contains or resolves through a symbolic link (%s)", cleaned, resolved) + } + return nil +} + +func validateSourcePurgePathLexical(path string, allowPlatform bool) error { + if strings.TrimSpace(path) == "" { + return fmt.Errorf("source purge path is empty") + } + if !pathpkg.IsAbs(path) { + return fmt.Errorf("source purge path must be absolute (got %q)", path) + } + cleaned := pathpkg.Clean(path) + if strings.Contains(cleaned, "\x00") { + return fmt.Errorf("source purge path contains invalid null byte") + } + if cleaned == "/" { + return fmt.Errorf("refusing to remove host root") + } + for _, protected := range []string{"/bin", "/boot", "/data", "/dev", "/etc", "/home", "/lib", "/opt", "/proc", "/root", "/run", "/sbin", "/srv", "/sys", "/tmp", "/usr", "/var"} { + if cleaned == protected { + return fmt.Errorf("refusing to remove protected host path %q", cleaned) + } + } + roots := []string{pathpkg.Clean("/data/coolify")} + if !sourcePurgeCoolifyPathAllowed(cleaned, roots, allowPlatform) { + return fmt.Errorf("source purge path %q must be inside /data/coolify applications/, services/, or databases/", cleaned) + } + return nil +} + +func sourcePurgeCoolifyPathAllowed(path string, roots []string, allowPlatform bool) bool { + for _, root := range roots { + if sourcePurgeCoolifyPathAllowedAtRoot(path, root, allowPlatform) { + return true + } + } + return false +} + +func sourcePurgeCoolifyPathAllowedAtRoot(path, root string, allowPlatform bool) bool { + cleanedRoot := pathpkg.Clean(root) + cleanedPath := pathpkg.Clean(path) + prefix := strings.TrimSuffix(cleanedRoot, "/") + "/" + if cleanedPath == cleanedRoot || !strings.HasPrefix(cleanedPath, prefix) { + return false + } + rel := strings.TrimPrefix(cleanedPath, prefix) + parts := strings.Split(rel, "/") + if len(parts) >= 2 && parts[1] != "" { + switch parts[0] { + case "applications", "services", "databases": + return true + } + } + if allowPlatform && len(parts) >= 1 { + switch parts[0] { + case "source", "proxy": + return true + } + } + return false +} + +func sourcePurgeResolveExistingPath(path string) (string, bool, error) { + cleaned := filepath.Clean(path) + if resolved, err := filepath.EvalSymlinks(cleaned); err == nil { + return filepath.Clean(resolved), true, nil + } else if !os.IsNotExist(err) { + return "", false, err + } + + missing := []string{} + parent := cleaned + for { + if _, err := os.Lstat(parent); err != nil { + if os.IsNotExist(err) { + next := filepath.Dir(parent) + if next == parent || next == "." || next == "" { + return "", false, nil + } + missing = append([]string{filepath.Base(parent)}, missing...) + parent = next + continue + } + return "", false, err + } + resolvedParent, err := filepath.EvalSymlinks(parent) + if err != nil { + if os.IsNotExist(err) { + return "", false, nil + } + return "", false, err + } + parts := append([]string{filepath.Clean(resolvedParent)}, missing...) + return filepath.Join(parts...), true, nil + } +} + +func trimDockerName(name string) string { + return strings.TrimPrefix(strings.TrimSpace(name), "/") +} + +func firstNonEmpty(values ...string) string { + for _, value := range values { + if strings.TrimSpace(value) != "" { + return strings.TrimSpace(value) + } + } + return "" +} + +func isDockerVolumeOrNetworkMissingErr(err error) bool { + if err == nil { + return false + } + message := strings.ToLower(err.Error()) + return strings.Contains(message, "no such volume") || + strings.Contains(message, "no such network") || + (strings.Contains(message, "error response from daemon: network ") && strings.Contains(message, " not found")) +} + func cleanupProjectNames(names []string) []string { seen := map[string]struct{}{} cleaned := []string{} diff --git a/internal/target/dokploy/cleanup_path_unix.go b/internal/target/dokploy/cleanup_path_unix.go new file mode 100644 index 0000000..eb344a9 --- /dev/null +++ b/internal/target/dokploy/cleanup_path_unix.go @@ -0,0 +1,62 @@ +//go:build !windows + +package dokploy + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "strings" + + "golang.org/x/sys/unix" +) + +func sourcePurgePathAbsentNoFollow(path string, allowPlatform bool) (bool, error) { + if err := ValidateSourcePurgePath(path, allowPlatform); err != nil { + return false, err + } + return pathAbsentNoFollow(path) +} + +func pathAbsentNoFollow(path string) (bool, error) { + cleaned := filepath.Clean(path) + if !filepath.IsAbs(cleaned) || cleaned == string(filepath.Separator) { + return false, fmt.Errorf("refusing invalid absolute source path %q", path) + } + parts := strings.Split(strings.TrimPrefix(cleaned, string(filepath.Separator)), string(filepath.Separator)) + parentFD, err := unix.Open(string(filepath.Separator), unix.O_RDONLY|unix.O_DIRECTORY|unix.O_CLOEXEC|unix.O_NOFOLLOW, 0) + if err != nil { + return false, err + } + parent := os.NewFile(uintptr(parentFD), string(filepath.Separator)) + if parent == nil { + _ = unix.Close(parentFD) + return false, fmt.Errorf("open filesystem root for source purge") + } + defer func() { _ = parent.Close() }() + + for _, part := range parts { + if part == "" || part == "." || part == ".." { + return false, fmt.Errorf("refusing invalid source purge path component %q", part) + } + nextFD, err := unix.Openat(int(parent.Fd()), part, unix.O_RDONLY|unix.O_DIRECTORY|unix.O_CLOEXEC|unix.O_NOFOLLOW, 0) + if err != nil { + if errors.Is(err, unix.ENOENT) { + return true, nil + } + return false, fmt.Errorf("open source purge path component %s without following links: %w", part, err) + } + next := os.NewFile(uintptr(nextFD), part) + if next == nil { + _ = unix.Close(nextFD) + return false, fmt.Errorf("open source purge path component %s", part) + } + if err := parent.Close(); err != nil { + next.Close() + return false, err + } + parent = next + } + return false, nil +} diff --git a/internal/target/dokploy/cleanup_path_windows.go b/internal/target/dokploy/cleanup_path_windows.go new file mode 100644 index 0000000..d5e96c9 --- /dev/null +++ b/internal/target/dokploy/cleanup_path_windows.go @@ -0,0 +1,18 @@ +//go:build windows + +package dokploy + +import ( + "fmt" +) + +func sourcePurgePathAbsentNoFollow(path string, allowPlatform bool) (bool, error) { + if err := ValidateSourcePurgePath(path, allowPlatform); err != nil { + return false, err + } + return pathAbsentNoFollow(path) +} + +func pathAbsentNoFollow(path string) (bool, error) { + return false, fmt.Errorf("source path absence verification for %q is unavailable on Windows; run cleanup purge on the source host", path) +} diff --git a/internal/target/dokploy/cleanup_test.go b/internal/target/dokploy/cleanup_test.go index dbb263a..5c218f9 100644 --- a/internal/target/dokploy/cleanup_test.go +++ b/internal/target/dokploy/cleanup_test.go @@ -2,6 +2,7 @@ package dokploy import ( "context" + "errors" "os" "path/filepath" "strings" @@ -58,6 +59,514 @@ func TestCleanupStalePlatformProjectsBacksUpThenDeletesMetadata(t *testing.T) { } } +func TestPurgeSourceResourcesRemovesDockerResources(t *testing.T) { + runner := &fakeDockerRunner{outputs: map[string][]byte{ + "inspect --type container cid123": []byte(`[{"Id":"cid123","Name":"/web","Config":{"Labels":{"coolify.managed":"true"}},"State":{"Running":false,"Status":"exited"}}]`), + "rm -f cid123": []byte("cid123\n"), + "network inspect api-net": []byte(`[{"Id":"network-id","Name":"api-net"}]`), + "network rm network-id": []byte("network-id\n"), + }} + client := &Client{Docker: runner} + + options, err := client.IdentifySourcePurgeResources(context.Background(), SourcePurgeOptions{ + Containers: []SourcePurgeContainer{{App: "api", Service: "web", ContainerID: "cid123", ContainerName: "web"}}, + Networks: []SourcePurgeNetwork{{App: "api", Name: "api-net"}}, + }) + if err != nil { + t.Fatalf("IdentifySourcePurgeResources: %v", err) + } + result, err := client.PurgeSourceResources(context.Background(), options) + if err != nil { + t.Fatalf("PurgeSourceResources: %v", err) + } + if len(result.Containers) != 1 || result.Containers[0].Status != "removed" || result.Containers[0].Ref != "web" { + t.Fatalf("unexpected container result: %#v", result.Containers) + } + if len(result.Networks) != 1 || result.Networks[0].Status != "removed" { + t.Fatalf("unexpected network result: %#v", result.Networks) + } + gotArgs := []string{} + for _, args := range runner.outputArgs { + gotArgs = append(gotArgs, strings.Join(args, " ")) + } + for _, want := range []string{"inspect --type container cid123", "rm -f cid123", "network inspect api-net", "network rm network-id"} { + found := false + for _, got := range gotArgs { + if got == want { + found = true + break + } + } + if !found { + t.Fatalf("expected docker command %q in %#v", want, gotArgs) + } + } +} + +func TestIdentifySourcePurgeResourcesRejectsNameOnlyDuplicate(t *testing.T) { + client := &Client{Docker: &fakeDockerRunner{}} + _, err := client.IdentifySourcePurgeResources(context.Background(), SourcePurgeOptions{Containers: []SourcePurgeContainer{ + {ContainerID: "cid123", ContainerName: "web"}, + {ContainerName: "web"}, + }}) + if err == nil || !strings.Contains(err.Error(), "without a stable container ID") { + t.Fatalf("expected name-only coordinate to remain unresolved, got %v", err) + } +} + +func TestIdentifySourcePurgeResourcesRejectsConflictingDuplicateContainerID(t *testing.T) { + client := &Client{Docker: &fakeDockerRunner{}} + for _, containers := range [][]SourcePurgeContainer{ + {{Service: "web", ContainerID: "cid123", ContainerName: "web"}, {Service: "web", ContainerID: "cid123", ContainerName: "replacement"}}, + {{Service: "web", ContainerID: "cid123", ContainerName: "web"}, {Service: "worker", ContainerID: "cid123", ContainerName: "web"}}, + } { + if _, err := client.IdentifySourcePurgeResources(context.Background(), SourcePurgeOptions{Containers: containers}); err == nil || !strings.Contains(err.Error(), "conflicting") { + t.Fatalf("expected duplicate container ID conflict, got %v", err) + } + } +} + +func TestIdentifySourcePurgeResourcesCanonicalizesReviewedShortContainerID(t *testing.T) { + shortID := "123456789abc" + fullID := shortID + strings.Repeat("d", 52) + runner := &fakeDockerRunner{outputs: map[string][]byte{ + "inspect --type container " + shortID: []byte(`[{"Id":"` + fullID + `","Name":"/web","State":{"Running":false,"Status":"exited"}}]`), + }} + client := &Client{Docker: runner} + identified, err := client.IdentifySourcePurgeResources(context.Background(), SourcePurgeOptions{ + Containers: []SourcePurgeContainer{{ContainerID: shortID, ContainerName: "web"}}, + }) + if err != nil { + t.Fatalf("IdentifySourcePurgeResources: %v", err) + } + if len(identified.Containers) != 1 || identified.Containers[0].ContainerID != fullID { + t.Fatalf("expected canonical reviewed container ID %q, got %#v", fullID, identified.Containers) + } +} + +func TestCleanupSourcePurgeContainersDeduplicatesShortAndCanonicalIDs(t *testing.T) { + shortID := "123456789abc" + fullID := shortID + strings.Repeat("d", 52) + for _, ids := range [][2]string{{shortID, fullID}, {fullID, shortID}} { + containers, err := cleanupSourcePurgeContainers([]SourcePurgeContainer{ + {Service: "web", ContainerID: ids[0], ContainerName: "web"}, + {Service: "web", ContainerID: ids[1], ContainerName: "web"}, + }) + if err != nil { + t.Fatal(err) + } + if len(containers) != 1 || containers[0].ContainerID != fullID { + t.Fatalf("expected one canonical container coordinate for order %#v, got %#v", ids, containers) + } + } +} + +func TestPurgeSourceResourcesSkipsAbsentAllowedPath(t *testing.T) { + client := &Client{Docker: &fakeDockerRunner{}} + options, err := client.IdentifySourcePurgeResources(context.Background(), SourcePurgeOptions{Paths: []SourcePurgePath{{Path: "/data/coolify/applications/app-1"}}}) + if err != nil { + t.Fatalf("IdentifySourcePurgeResources: %v", err) + } + result, err := client.PurgeSourceResources(context.Background(), options) + if err != nil { + t.Fatalf("PurgeSourceResources: %v", err) + } + if len(result.Paths) != 1 || result.Paths[0].Status != "skipped" || !strings.Contains(result.Paths[0].Message, "remains absent") { + t.Fatalf("unexpected path result: %#v", result.Paths) + } +} + +func TestPurgeSourceResourcesDoesNotFallbackFromStaleIDToName(t *testing.T) { + runner := &fakeDockerRunner{outputs: map[string][]byte{ + "inspect --type container web": []byte(`[{"Id":"new-id","Name":"/web","State":{"Running":true,"Status":"running"}}]`), + "rm -f new-id": []byte("new-id\n"), + }} + client := &Client{Docker: runner} + + result, err := client.PurgeSourceResources(context.Background(), SourcePurgeOptions{Containers: []SourcePurgeContainer{{App: "api", ContainerID: "missing-id", ContainerName: "web"}}}) + if err == nil || !strings.Contains(err.Error(), "docker inspect missing-id") { + t.Fatalf("expected stale ID inspect error, got err=%v result=%#v", err, result) + } + for _, args := range runner.outputArgs { + if strings.Join(args, " ") == "inspect --type container web" || strings.Join(args, " ") == "rm -f new-id" { + t.Fatalf("purge fell back to name after stale ID: %#v", runner.outputArgs) + } + } +} + +func TestPurgeSourceResourcesPreservesReplacementAfterReviewedContainerDisappears(t *testing.T) { + runner := &cleanupDockerRunner{ + fakeDockerRunner: &fakeDockerRunner{outputs: map[string][]byte{ + "inspect --type container web": []byte(`[{"Id":"replacement-id","Name":"/web","State":{"Running":true,"Status":"running"}}]`), + }}, + outputErrors: map[string]error{ + "inspect --type container reviewed-id": errors.New("Error: No such container: reviewed-id"), + }, + } + client := &Client{Docker: runner} + result, err := client.PurgeSourceResources(context.Background(), SourcePurgeOptions{Containers: []SourcePurgeContainer{{ContainerID: "reviewed-id", ContainerName: "web"}}}) + if err == nil || !strings.Contains(err.Error(), "now refers to container ID") { + t.Fatalf("expected replacement container to block purge, got result=%#v err=%v", result, err) + } + for _, args := range runner.outputArgs { + if strings.HasPrefix(strings.Join(args, " "), "rm -f ") { + t.Fatalf("replacement container was removed: %#v", runner.outputArgs) + } + } +} + +func TestPurgeSourceResourcesRejectsMismatchedInspectedContainerID(t *testing.T) { + runner := &fakeDockerRunner{outputs: map[string][]byte{ + "inspect --type container reviewed-id": []byte(`[{"Id":"replacement-id","Name":"/web","State":{"Running":true,"Status":"running"}}]`), + "rm -f replacement-id": []byte("replacement-id\n"), + }} + client := &Client{Docker: runner} + result, err := client.PurgeSourceResources(context.Background(), SourcePurgeOptions{Containers: []SourcePurgeContainer{{ContainerID: "reviewed-id", ContainerName: "web"}}}) + if err == nil || !strings.Contains(err.Error(), "not the reviewed ID") { + t.Fatalf("expected mismatched inspected ID to block purge, got result=%#v err=%v", result, err) + } + for _, args := range runner.outputArgs { + if strings.HasPrefix(strings.Join(args, " "), "rm -f ") { + t.Fatalf("mismatched inspected container was removed: %#v", runner.outputArgs) + } + } +} + +func TestPurgeSourceNetworkPreservesReplacementAfterReviewedIDDisappears(t *testing.T) { + runner := &cleanupDockerRunner{ + fakeDockerRunner: &fakeDockerRunner{outputs: map[string][]byte{ + "network inspect api-net": []byte(`[{"Id":"replacement-id","Name":"api-net"}]`), + }}, + outputErrors: map[string]error{ + "network rm reviewed-id": errors.New("Error response from daemon: network reviewed-id not found"), + }, + } + client := &Client{Docker: runner} + result, err := client.PurgeSourceResources(context.Background(), SourcePurgeOptions{ + Networks: []SourcePurgeNetwork{{Name: "api-net", ExpectedIdentity: "reviewed-id"}}, + }) + if err == nil || !strings.Contains(err.Error(), "appeared after confirmation") { + t.Fatalf("expected replacement network to be preserved, got result=%#v err=%v", result, err) + } + for _, args := range runner.outputArgs { + if strings.Join(args, " ") == "network rm replacement-id" { + t.Fatalf("replacement network was removed: %#v", runner.outputArgs) + } + } +} + +func TestPurgeSourceResourcesReturnsPartialResults(t *testing.T) { + runner := &fakeDockerRunner{outputs: map[string][]byte{ + "inspect --type container cid123": []byte(`[{"Id":"cid123","Name":"/web","State":{"Running":false,"Status":"exited"}}]`), + "rm -f cid123": []byte("cid123\n"), + "network inspect api-net": []byte(`[{"Id":"network-id","Name":"api-net"}]`), + }} + client := &Client{Docker: runner} + options, err := client.IdentifySourcePurgeResources(context.Background(), SourcePurgeOptions{ + Containers: []SourcePurgeContainer{{App: "api", ContainerID: "cid123", ContainerName: "web"}}, + Networks: []SourcePurgeNetwork{{App: "api", Name: "api-net"}}, + }) + if err != nil { + t.Fatalf("IdentifySourcePurgeResources: %v", err) + } + result, err := client.PurgeSourceResources(context.Background(), options) + if err == nil { + t.Fatal("expected purge to stop on the network failure") + } + if len(result.Containers) != 1 || result.Containers[0].Status != "removed" || len(result.Networks) != 1 || result.Networks[0].Status != "error" { + t.Fatalf("expected completed and failed resource results, got %#v", result) + } +} + +func TestSourcePurgeRequiresNamedVolumeToRemainAbsent(t *testing.T) { + runner := &fakeDockerRunner{outputs: map[string][]byte{ + "volume inspect api-data": []byte(`[{"Name":"api-data"}]`), + }} + client := &Client{Docker: runner} + if _, err := client.IdentifySourcePurgeResources(context.Background(), SourcePurgeOptions{Volumes: []SourcePurgeVolume{{Name: "api-data"}}}); err == nil || !strings.Contains(err.Error(), "remove it manually") { + t.Fatalf("expected existing named volume to block before confirmation, got %v", err) + } + result, err := client.PurgeSourceResources(context.Background(), SourcePurgeOptions{ + Volumes: []SourcePurgeVolume{{Name: "api-data", ExpectedAbsent: true}}, + }) + if err == nil || !strings.Contains(err.Error(), "appeared after confirmation") { + t.Fatalf("expected recreated volume to remain preserved, got result=%#v err=%v", result, err) + } + for _, args := range runner.outputArgs { + if strings.Join(args, " ") == "volume rm api-data" { + t.Fatalf("changed volume was removed: %#v", runner.outputArgs) + } + } +} + +func TestSourcePurgeRechecksAbsenceBeforeRemovingContainers(t *testing.T) { + runner := &fakeDockerRunner{outputs: map[string][]byte{ + "volume inspect api-data": []byte(`[{"Name":"api-data"}]`), + "inspect --type container cid123": []byte(`[{"Id":"cid123","Name":"/web","State":{"Running":false,"Status":"exited"}}]`), + "rm -f cid123": []byte("cid123\n"), + }} + client := &Client{Docker: runner} + result, err := client.PurgeSourceResources(context.Background(), SourcePurgeOptions{ + Containers: []SourcePurgeContainer{{ContainerID: "cid123", ContainerName: "web"}}, + Volumes: []SourcePurgeVolume{{Name: "api-data", ExpectedAbsent: true}}, + }) + if err == nil || !strings.Contains(err.Error(), "appeared after confirmation") { + t.Fatalf("expected recreated volume to block before container removal, got result=%#v err=%v", result, err) + } + for _, args := range runner.outputArgs { + if strings.Join(args, " ") == "rm -f cid123" { + t.Fatalf("container was removed before prerequisites passed: %#v", runner.outputArgs) + } + } +} + +func TestSourcePurgeWithAbsencePrerequisitesRequiresManualContainerRemoval(t *testing.T) { + runner := &stagedPurgeRunner{fakeDockerRunner: fakeDockerRunner{outputs: map[string][]byte{ + "inspect --type container cid123": []byte(`[{"Id":"cid123","Name":"/web","State":{"Running":false,"Status":"exited"}}]`), + "rm -f cid123": []byte("cid123\n"), + }}} + client := &Client{Docker: runner} + result, err := client.PurgeSourceResources(context.Background(), SourcePurgeOptions{ + Containers: []SourcePurgeContainer{{ContainerID: "cid123", ContainerName: "web"}}, + Volumes: []SourcePurgeVolume{{Name: "api-data", ExpectedAbsent: true}}, + }) + if err == nil || !strings.Contains(err.Error(), "remove it manually") { + t.Fatalf("expected manual-completion strategy to block automatic removal, got result=%#v err=%v", result, err) + } + for _, args := range runner.outputArgs { + if strings.Join(args, " ") == "rm -f cid123" { + t.Fatalf("container was removed after a prerequisite reappeared: %#v", runner.outputArgs) + } + } +} + +func TestIdentifySourcePurgeResourcesRejectsMixedAutomaticAndAbsenceOnlyPurge(t *testing.T) { + shortID := "123456789abc" + fullID := shortID + strings.Repeat("d", 52) + runner := &cleanupDockerRunner{ + fakeDockerRunner: &fakeDockerRunner{outputs: map[string][]byte{ + "inspect --type container " + shortID: []byte(`[{"Id":"` + fullID + `","Name":"/web","State":{"Running":false,"Status":"exited"}}]`), + }}, + outputErrors: map[string]error{ + "volume inspect api-data": errors.New("Error response from daemon: get api-data: no such volume"), + }, + } + client := &Client{Docker: runner} + _, err := client.IdentifySourcePurgeResources(context.Background(), SourcePurgeOptions{ + Containers: []SourcePurgeContainer{{ContainerID: shortID, ContainerName: "web"}}, + Volumes: []SourcePurgeVolume{{Name: "api-data"}}, + }) + if err == nil || !strings.Contains(err.Error(), "remove all listed containers and networks manually") { + t.Fatalf("expected mixed automatic and absence-only purge to fail before confirmation, got %v", err) + } + if strings.Contains(err.Error(), "not the reviewed ID") { + t.Fatalf("short reviewed ID was treated as a different container: %v", err) + } + for _, args := range runner.outputArgs { + if strings.HasPrefix(strings.Join(args, " "), "rm -f ") { + t.Fatalf("container was removed while identifying an ineligible purge: %#v", runner.outputArgs) + } + } +} + +func TestSourcePurgeManualCompletionPreservesReplacementContainerByName(t *testing.T) { + runner := &cleanupDockerRunner{ + fakeDockerRunner: &fakeDockerRunner{outputs: map[string][]byte{ + "inspect --type container web": []byte(`[{"Id":"replacement-id","Name":"/web","State":{"Running":true,"Status":"running"}}]`), + }}, + outputErrors: map[string]error{ + "volume inspect api-data": errors.New("Error response from daemon: get api-data: no such volume"), + "inspect --type container reviewed": errors.New("Error: No such container: reviewed"), + }, + } + client := &Client{Docker: runner} + result, err := client.PurgeSourceResources(context.Background(), SourcePurgeOptions{ + Containers: []SourcePurgeContainer{{ContainerID: "reviewed", ContainerName: "web"}}, + Volumes: []SourcePurgeVolume{{Name: "api-data", ExpectedAbsent: true}}, + }) + if err == nil || !strings.Contains(err.Error(), "now refers to container ID") { + t.Fatalf("expected replacement container to block absence verification, got result=%#v err=%v", result, err) + } + for _, args := range runner.outputArgs { + if strings.HasPrefix(strings.Join(args, " "), "rm -f ") { + t.Fatalf("replacement container was removed: %#v", runner.outputArgs) + } + } +} + +func TestPurgeSourceResourcesStopsBeforeNextMutationWhenProgressPersistenceFails(t *testing.T) { + runner := &fakeDockerRunner{outputs: map[string][]byte{ + "inspect --type container cid1": []byte(`[{"Id":"cid1","Name":"/one","State":{"Running":false,"Status":"exited"}}]`), + "rm -f cid1": []byte("cid1\n"), + "inspect --type container cid2": []byte(`[{"Id":"cid2","Name":"/two","State":{"Running":false,"Status":"exited"}}]`), + "rm -f cid2": []byte("cid2\n"), + }} + var durable SourcePurgeResult + client := &Client{Docker: runner} + result, err := client.PurgeSourceResources(context.Background(), SourcePurgeOptions{ + Containers: []SourcePurgeContainer{{ContainerID: "cid1", ContainerName: "one"}, {ContainerID: "cid2", ContainerName: "two"}}, + OnProgress: func(progress SourcePurgeResult) error { + latest := progress.Containers[len(progress.Containers)-1] + if latest.Ref == "cid2" && latest.Status == "started" { + return errors.New("backup unavailable") + } + durable = progress + return nil + }, + }) + if err == nil || !strings.Contains(err.Error(), "persist source purge progress") { + t.Fatalf("expected progress persistence failure, got result=%#v err=%v", result, err) + } + if len(durable.Containers) != 1 || durable.Containers[0].Status != "removed" { + t.Fatalf("expected first outcome to remain durable, got %#v", durable) + } + for _, args := range runner.outputArgs { + if strings.Join(args, " ") == "inspect --type container cid2" || strings.Join(args, " ") == "rm -f cid2" { + t.Fatalf("second resource mutated after persistence failure: %#v", runner.outputArgs) + } + } +} + +type stagedPurgeRunner struct { + fakeDockerRunner + volumeInspects int +} + +func (r *stagedPurgeRunner) Output(ctx context.Context, args ...string) ([]byte, error) { + if strings.Join(args, " ") == "volume inspect api-data" { + r.outputArgs = append(r.outputArgs, append([]string{}, args...)) + r.volumeInspects++ + if r.volumeInspects == 1 { + return nil, errors.New("Error response from daemon: get api-data: no such volume") + } + return []byte(`[{"Name":"api-data"}]`), nil + } + return r.fakeDockerRunner.Output(ctx, args...) +} + +func TestSourcePurgeAcceptsNamedVolumeThatRemainsAbsent(t *testing.T) { + runner := &cleanupDockerRunner{ + fakeDockerRunner: &fakeDockerRunner{}, + outputErrors: map[string]error{ + "volume inspect api-data": errors.New("Error response from daemon: get api-data: no such volume"), + }, + } + client := &Client{Docker: runner} + options, err := client.IdentifySourcePurgeResources(context.Background(), SourcePurgeOptions{ + Volumes: []SourcePurgeVolume{{Name: "api-data"}}, + }) + if err != nil { + t.Fatalf("IdentifySourcePurgeResources: %v", err) + } + result, err := client.PurgeSourceResources(context.Background(), options) + if err != nil { + t.Fatalf("PurgeSourceResources: %v", err) + } + if len(result.Volumes) != 1 || result.Volumes[0].Status != "skipped" || !strings.Contains(result.Volumes[0].Message, "remains absent") { + t.Fatalf("unexpected volume result: %#v", result.Volumes) + } +} + +type cleanupDockerRunner struct { + *fakeDockerRunner + outputErrors map[string]error +} + +func (r *cleanupDockerRunner) Output(ctx context.Context, args ...string) ([]byte, error) { + if err, ok := r.outputErrors[strings.Join(args, " ")]; ok { + return nil, err + } + return r.fakeDockerRunner.Output(ctx, args...) +} + +func TestDockerMissingResourceClassifierRejectsInfrastructureErrors(t *testing.T) { + if isDockerVolumeOrNetworkMissingErr(errors.New(`exec: "docker": executable file not found in $PATH`)) { + t.Fatal("missing Docker executable was classified as an absent resource") + } + for _, err := range []error{ + errors.New("Error response from daemon: remove api-data: no such volume"), + errors.New("Error response from daemon: network api-net not found"), + } { + if !isDockerVolumeOrNetworkMissingErr(err) { + t.Fatalf("expected missing Docker resource classification for %v", err) + } + } +} + +func TestProtectedSourcePurgeNetworkNamesAreCaseSensitive(t *testing.T) { + for _, name := range []string{"bridge", "host", "none", "ingress"} { + if !IsProtectedSourcePurgeNetwork(name) { + t.Fatalf("expected %q to be protected", name) + } + } + for _, name := range []string{"Bridge", "Proxy", "HOST", "coolify", "coolify-proxy", "proxy"} { + if IsProtectedSourcePurgeNetwork(name) { + t.Fatalf("expected distinct user network %q not to be protected", name) + } + } +} + +func TestPurgeSourceResourcesRejectsProtectedHostPath(t *testing.T) { + client := &Client{Docker: &fakeDockerRunner{}} + _, err := client.PurgeSourceResources(context.Background(), SourcePurgeOptions{Paths: []SourcePurgePath{{Path: "/data"}}}) + if err == nil || !strings.Contains(err.Error(), "protected host path") { + t.Fatalf("expected protected path error, got %v", err) + } +} + +func TestPathAbsentNoFollowRejectsAncestorSymlink(t *testing.T) { + if os.PathSeparator == '\\' { + t.Skip("windows refuses source path purge") + } + root := t.TempDir() + outside := t.TempDir() + outsideTarget := filepath.Join(outside, "target") + if err := os.Mkdir(outsideTarget, 0o700); err != nil { + t.Fatal(err) + } + sentinel := filepath.Join(outsideTarget, "sentinel") + if err := os.WriteFile(sentinel, []byte("keep"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.Symlink(outside, filepath.Join(root, "swapped")); err != nil { + t.Fatal(err) + } + + if _, err := pathAbsentNoFollow(filepath.Join(root, "swapped", "target")); err == nil { + t.Fatal("expected ancestor symlink to block absence validation") + } + if contents, err := os.ReadFile(sentinel); err != nil || string(contents) != "keep" { + t.Fatalf("outside path changed through ancestor symlink: contents=%q err=%v", contents, err) + } +} + +func TestPathAbsentNoFollowPreservesExistingDirectoryTree(t *testing.T) { + if os.PathSeparator == '\\' { + t.Skip("windows refuses source path purge") + } + root, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatal(err) + } + target := filepath.Join(root, "target") + if err := os.MkdirAll(filepath.Join(target, "nested"), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(target, "nested", "file"), []byte("remove"), 0o600); err != nil { + t.Fatal(err) + } + absent, err := pathAbsentNoFollow(target) + if err != nil { + t.Fatalf("check no-follow path absence: %v", err) + } + if absent { + t.Fatal("expected existing target tree not to be reported absent") + } + if contents, err := os.ReadFile(filepath.Join(target, "nested", "file")); err != nil || string(contents) != "remove" { + t.Fatalf("absence validation changed the target tree: contents=%q err=%v", contents, err) + } +} + func TestFindDokployPostgresContainerRequiresDokployPostgres(t *testing.T) { runner := &fakeDockerRunner{outputs: map[string][]byte{"ps --format {{.Names}}": []byte("postgres\n")}} if _, err := findDokployPostgresContainer(context.Background(), runner); err == nil { diff --git a/internal/target/dokploy/datamove_test.go b/internal/target/dokploy/datamove_test.go index 5763569..bc2b0a0 100644 --- a/internal/target/dokploy/datamove_test.go +++ b/internal/target/dokploy/datamove_test.go @@ -124,6 +124,8 @@ func (r *latePostDeployTargetRunner) Output(_ context.Context, args ...string) ( r.outputArgs = append(r.outputArgs, append([]string{}, args...)) key := strings.Join(args, " ") switch key { + case "ps --format {{.Names}}": + return []byte("dokploy-postgres.1.task\n"), nil case "ps -a --filter label=com.docker.compose.project=stack-1 --format {{.ID}}": r.psCalls++ if r.emptyFirst && r.psCalls == 1 { @@ -506,6 +508,9 @@ func TestApplyPushImageTagsMissingComposeImageFromSourceContainer(t *testing.T) "image inspect redis:7": []byte(`[{}]`), "inspect --type container src-worker": []byte(`[{"Id":"src-worker","Name":"/worker","Image":"sha256:worker-image","Config":{"Image":"example/worker:old"}}]`), "tag sha256:worker-image example/worker:local": []byte(""), + "ps --format {{.Names}}": []byte("dokploy-postgres.1.task\n"), + }, runOutputs: map[string][]byte{ + "exec -i dokploy-postgres.1.task psql -U dokploy -d dokploy -v ON_ERROR_STOP=1 -At": {}, }} client := &Client{BaseURL: server.URL, Token: "secret", HTTPClient: server.Client(), Docker: runner} app := preparer.AppPlan{Name: "api", Directory: "api"} From 64252a08177b1641f2c216bd1e506e676c6c613c Mon Sep 17 00:00:00 2001 From: Aikins Laryea Date: Sun, 2 Aug 2026 12:36:25 +0000 Subject: [PATCH 02/31] docs: clarify migration lifecycle (#9) * docs: clarify migration lifecycle * docs: simplify migration language --- README.md | 547 ++++++++++++++------------------------ docs/acceptance.md | 111 ++++++++ docs/cleanup-and-purge.md | 154 +++++++++++ docs/migration-guide.md | 207 +++++++++++++++ 4 files changed, 665 insertions(+), 354 deletions(-) create mode 100644 docs/acceptance.md create mode 100644 docs/cleanup-and-purge.md create mode 100644 docs/migration-guide.md diff --git a/README.md b/README.md index b1801e8..6a1c7c9 100644 --- a/README.md +++ b/README.md @@ -1,412 +1,251 @@ # bort

- move self-hosted apps between PaaS platforms without gambling on traffic, secrets, or data. + move self-hosted apps between platforms without gambling on traffic, secrets, or data.
- Coolify → Dokploy first. Dokploy → Coolify, cross-server moves, and more Docker-shaped platforms next. + Coolify → Dokploy on the same VPS first. Reverse and cross-server migrations later.
- About + Supported path · Quick start · - Safety model + Safety · - Roadmap + Documentation + · + Roadmap

## About -Bort is a migration cockpit for people running their own app platform on a VPS. -It turns a risky platform switch into a guided run: inventory what is actually -running, explain what needs attention, prepare the target privately, move state, -cut traffic over, and keep a rollback path until you accept the new home. - -The first product path is **same-VPS Coolify → Dokploy**. That is the migration -operators ask for when they already have real apps, domains, env, databases, -volumes, and platform leftovers on one box, and they do not want the answer to be -"delete everything and redeploy by hand." - -Bort is intentionally a small Go CLI today, but the user experience is product -shaped: - -- **app-first status** instead of a wall of Docker trivia. -- **copy-paste fixes** for missing env values and data-store decisions. -- **dry-run by default** so discovery, planning, validation, cleanup inventory, - rollback, and commit planning are safe to run repeatedly. -- **explicit live mode** only after gates are clear and target credentials exist. -- **private local artifacts** so migration bundles, state, and target API keys stay - on the server. -- **safe cleanup** that inventories leftovers and only applies narrow metadata - cleanup after a database backup, plus a separate confirmed source purge path - for post-acceptance Coolify leftovers. - -## Why Bort exists - -Self-hosted PaaS tools make deployment easy until you need to leave one. A real -move has more surface area than "copy the compose file": - -- the UI may not show the same truth as Docker on the host; -- env files and generated platform values are easy to mix up; -- databases and bind mounts need migration strategy, not vibes; -- domains need health checks, observation windows, and rollback; -- source-control integrations are platform credentials, not portable app config; -- old platform containers, volumes, networks, and proxy metadata should not be - pruned casually. - -Bort treats migration as an operation with gates. It should tell you what is safe, -what is blocked, and what to do next without making you reverse-engineer every -Traefik label, Docker network, bind mount, or platform database row yourself. - -## What works today - -Bort is at the local-first same-VPS migration stage. - -- Scan local Docker, the Coolify API, or the source server's Docker state. -- Export a private migration bundle with compose, env, routes, storage, topology, - source-control metadata, and a per-app runbook. -- Validate bundles for compose shape, portability risks, routes, and secret - handling. -- Show a linear status view with per-app health, attention items, and `fix:` - commands. -- Persist one explicitly selected current run in `.bort/state.json` so live apply, - acceptance, and cleanup operate on the run you reviewed instead of guessing by - file modification time. -- Record env answers and data-store strategies in `.bort/state.json` so resolved - issues disappear from the next run. -- Keep each source-created run self-contained under `.bort/runs/` with its - private manifest, bundle, plans, progress, and apply ledger. -- Prepare Dokploy resources in dry-run form, then create/reuse projects and compose - apps in explicit live mode. -- Upload sanitized env and deploy the captured raw compose/image snapshot without - requiring a Dokploy GitHub App. -- Move supported local state with logical dump/restore or stopped-copy paths. -- Cut traffic over with health checks, observation, rollback planning, and an - apply ledger so interrupted runs can resume or attach. -- Commit acceptance by stopping source app containers only after a successful live - apply. -- Run non-destructive cleanup inventory and, when explicitly applied, remove only - safe empty zero-domain Dokploy metadata after backing up the Dokploy database. -- Dry-run a source purge, then remove eligible source containers and networks - only with an explicit scope and confirmation phrase. When named volumes or host - paths are present, every listed source resource must instead be removed manually - before Bort verifies the completed purge. - -Not implemented yet: - -- Automatic creation or migration of Dokploy/Coolify source connections such as - GitHub Apps, deploy keys, and webhooks. -- Continuous volume delta sync. -- Database replication adapters beyond the current local dump/restore and - stopped-copy paths. -- Automatic source image pruning. `cleanup purge` intentionally leaves Docker - images alone because image tags/layers can be shared with target workloads. -- Reverse and third-party platform adapters listed in the roadmap below. - -## Quick start - -Install from Homebrew after the first tagged release: +Bort is a guided migration tool for people running their own app platform on a +VPS. It checks what is actually running, explains what needs attention, prepares +Dokploy, copies supported data, switches web traffic, and asks before any +destructive step. + +Bort organizes the migration by app rather than exposing a wall of Docker +details. It shows what is ready, provides copy-paste fixes, saves your answers, +and tells you what to do next. Discovery and planning only preview app changes. +The guided setup can offer to install Dokploy before live apply, but it explains +the server changes and asks first. Moving apps, data, and traffic begins only +when you explicitly run the live command. + +## Supported path + +The current product path is **Coolify → Dokploy on the same Linux VPS**. + +| Capability | Status | +| --- | --- | +| Guided Coolify → Dokploy planning | Implemented | +| Explicit live apply and resume | Implemented | +| Accepting Dokploy and stopping source containers | Implemented | +| Safe Dokploy database cleanup and separately confirmed source removal | Implemented | +| Automated end-to-end test with fake Docker and Dokploy | Implemented | +| End-to-end test on a disposable Linux host | Still required before relying on the live path in production | +| Executable rollback | Not implemented; Bort only prints the stored rollback plan | +| Dokploy → Coolify or cross-server migration | Not implemented | + +Bort publishes macOS, Linux, and Windows binaries, but a complete same-VPS move +depends on Linux, Docker, and access to protected host files. Use the non-Linux +builds to inspect migration files and help develop Bort, not to run a production +migration. + +Today Bort can: + +- discover local Coolify Docker state, local Docker, the Coolify API, or an + existing manifest; +- export private application bundles containing Compose/image details, + environment values, routes, storage, Docker networks, and source-control + details; +- record missing environment values and how each database or storage service + should be moved; +- prepare Dokploy projects and compose applications before explicit live apply; +- move supported local data by exporting and importing databases or copying data + while a service is stopped; +- switch web traffic after health checks and a waiting period, save progress for + retries, and store rollback instructions; +- accept the target only after successful live apply; +- list leftovers, remove only eligible unused records from Dokploy, and + separately remove eligible source containers and networks after confirmation. + +See the [migration guide](docs/migration-guide.md#current-workload-coverage) for +the workload-level coverage and limitations. + +## Before you begin + +Use a fresh VM or restorable snapshot while evaluating Bort. Do not test source +purge on a host containing data you intend to keep. + +Follow two workspace rules during the migration: + +1. Run every command from the **same working directory**. Bort stores its + workspace under `.bort` relative to that directory. +2. Use the **same OS user** throughout the migration. If the first command + uses `sudo`, keep using `sudo`; if the first command does not, do not add it + later. + +Bort keeps workspace directories and files private, but `.bort` contains target +credentials and application configuration. Do not commit or publish it. + +Read the [prerequisites and recovery guidance](docs/migration-guide.md) before a +live migration. In particular, Bort does not yet execute rollback. + +## Install + +Install from Homebrew: ```sh brew install aikins01/tap/bort ``` -Or install from source with Go: +Install from source with Go: ```sh go install github.com/aikins01/bort/cmd/bort@latest ``` -Build the CLI: +Or build the current checkout: ```sh -mkdir -p bin -go build -o bin/bort ./cmd/bort +make build ``` -The examples below use an installed `bort`; substitute `./bin/bort` when using -the local build. - -For a guided same-VPS Coolify → Dokploy migration, run Bort from the source -server in a terminal: - -```sh -sudo bort -``` +The examples below use an installed `bort`; substitute `./bin/bort` for a local +build. -`bort` is the normal migration cockpit. It starts guided setup when needed and -otherwise resumes the current run. Review each app, follow the shown `fix:` and -`next:` guidance, and rerun `sudo bort` until the cockpit shows `READY`. Target -setup is offered inline when credentials are needed. - -Use one privilege model for the whole lifecycle. Same-VPS discovery, state copy, -proxy cutover, source retirement, and purge need host-level Docker and filesystem -access, so the examples consistently use `sudo`. Bort keeps `.bort` private to -that OS identity (`0700` directories and `0600` files) and preserves the `sudo` -prefix in copy-paste guidance. If your regular account already has all required -Docker and source-path access, omit `sudo` from the first command and every later -lifecycle command instead; do not switch identities within one workspace. +## Quick start -The noninteractive start command performs discovery, exports a private bundle, -creates all run artifacts, and selects the new run in one step: +On the source VPS, create a dedicated workspace and start Bort: ```sh -sudo bort migrate --source coolify-local +mkdir -p ~/bort-migration +cd ~/bort-migration sudo bort -sudo bort migrate --live -sudo bort commit --apply -sudo bort cleanup -``` - -The lifecycle is deliberately explicit: - -1. `sudo bort` or `sudo bort migrate --source ...` starts or resumes a reviewed - dry-run. -2. `sudo bort` shows setup blockers and records interactive fixes. -3. `sudo bort migrate --live` applies only the selected reviewed run. -4. `sudo bort commit --apply` accepts the target and retires source app - containers. -5. `sudo bort cleanup` audits leftovers; optional apply and purge commands remain - separately gated. - -For an existing manifest, use the same consolidated path: - -```sh -sudo bort migrate --manifest manifest.json ``` -Source and manifest starts create a self-contained directory under -`.bort/runs/` containing the private manifest, exported bundle, reviewed -plans, progress, and apply ledger. `.bort/state.json` stores the current run along -with recorded env/data decisions and target credentials. Pass `--run ` to -select another run explicitly. Mutating commands never choose a run by mtime. +The guided screen starts setup when needed and otherwise resumes the current +run. Review each application, follow its generated `fix:` and `next:` guidance, +and rerun `sudo bort` until the run shows `READY`. -When Bort asks for missing values or data decisions, use the fix commands it -prints: +To start discovery directly without the setup questions: ```sh -sudo bort env demo-app API_TOKEN=secret DATABASE_URL=postgres://... -sudo bort data demo-app postgres --migrate +sudo bort migrate --source coolify-local sudo bort ``` -After you accept the target, retire the source app stack and audit leftovers: +The migration uses separate commands for each important step: -```sh -sudo bort commit --apply -sudo bort cleanup -sudo bort cleanup --apply -sudo bort cleanup purge -sudo bort cleanup purge --apply --app demo-app --confirm "purge " +```text +sudo bort # start or resume, then review and fix +sudo bort migrate --live # apply only the selected reviewed run +sudo bort rollback # inspect the stored manual rollback plan +sudo bort commit --apply # accept the target and retire source containers +sudo bort cleanup # audit leftovers without deleting source resources ``` -`cleanup` is the non-destructive cleanup surface. `cleanup --apply` is -metadata-only: it does not remove source containers, volumes, networks, -source-control credentials, or Dokploy target apps. - -`cleanup purge` is the destructive source cleanup surface. It is still dry-run by -default, and `--apply` refuses to run unless you pass an explicit scope -(`--app`, `--project`, or `--all-apps`), the run has a successful live apply -ledger, the target has been accepted with `sudo bort commit --apply`, and you provide -the confirmation phrase shown in the dry-run output. It writes a private -purge-plan backup before deleting eligible source containers and source networks -by their exact inspected Docker IDs. Bort never deletes named volumes or host -source paths. If either is included, the dry run requires manual removal of every -listed container, volume, network, and path; `--apply` only verifies that the -whole scope remains absent instead of combining absence checks with automatic -deletion. Verification-only purges do not set `PurgedAt`, because Bort cannot -lock external tools out of recreating an absent resource. `--all-apps` -covers non-platform apps by default; platform-role leftovers such as Coolify -source/proxy resources require `--include-platform`. It does not remove -source-control credentials, target Dokploy resources, or Docker images. A full -`--all-apps` purge marks the migration complete; app- or project-scoped purges do -not claim that the whole run is finished. - -## Acceptance trace - -The automated lifecycle trace uses a disposable workspace, fake Dokploy HTTP -API, and fake Docker executable. It creates a self-contained run, performs live -apply, verifies `TARGET LIVE`, commits, verifies `COMMITTED`, inventories cleanup, -applies a confirmed all-app purge, and verifies `COMPLETE`: +After reviewing the cleanup results, `cleanup --apply` can remove only eligible +unused records from the Dokploy database, and only after making a database +backup. Removing source resources remains a separate command that previews its +work first: ```sh -go test ./internal/cli -run '^TestLifecycleAcceptanceTraceWithFakeDokploy$' -v +sudo bort cleanup purge --all-apps ``` -Real container, volume, network, bind-path, and proxy behavior still needs a -throwaway same-VPS host. Never run this destructive acceptance pass on a host -containing data you intend to keep. On a fresh VM or restorable snapshot: - -1. Install Coolify and Dokploy, then deploy one stateless disposable Coolify app - with a route, source network, and non-secret test env value. -2. Record the source container, network, and target project names before migration. -3. Run the complete lifecycle under one identity: - - ```sh - run="bort-acceptance-$(date +%Y%m%d-%H%M%S)" - sudo bort migrate --source coolify-local --run "$run" - sudo bort - sudo bort migrate --live --run "$run" - sudo bort status --run "$run" - sudo bort commit --apply --run "$run" - sudo bort cleanup --run "$run" - sudo bort cleanup purge --run "$run" --all-apps - ``` - - Review the purge plan and verify the target still serves the disposable - workload before applying it: - - ```sh - sudo bort cleanup purge --apply --run "$run" --all-apps --confirm "purge $run" - sudo bort status --run "$run" - ``` - -4. Confirm the target app remains healthy, `run.json` contains all three - lifecycle timestamps, the selected source container and network are absent, - and target resources plus source-control credentials were not removed. -5. Restore the clean snapshot and repeat with a disposable app that has a named - volume and bind mount under a disposable directory. After the purge dry run, - manually remove every listed source resource. Confirm `--apply` performs only - absence verification, leaves the run committed rather than complete, and does - not delete any replacement resource introduced before verification. -6. Destroy the VM or restore the snapshot after saving only the redacted command - output and lifecycle metadata needed for release evidence. +Review its output and use the exact apply command and confirmation phrase +Bort generates. Named volumes and host source paths are never deleted +automatically. Read the [cleanup and purge guide](docs/cleanup-and-purge.md) +before applying cleanup or purge. ## Safety model -Bort is designed for production boxes where the safest default is "look first." - -- **dry-run first:** source setup, plan, validate, prepare, sync, cutover, - rollback, commit, cleanup, and purge planning are non-mutating by default. -- **explicit live path:** target creation and traffic movement happen only through - `sudo bort migrate --live` after setup blockers are clear. Live mode never creates a - run implicitly and rejects source/planning flags. -- **explicit run ownership:** `.bort/state.json` names the current reviewed run. - `--run` overrides it explicitly, and mutating commands never infer a destructive - target from whichever `run.json` has the newest mtime. -- **immutable live plans:** after live execution begins, Bort will not regenerate - that run's reviewed plan or reorder its apply steps. Create a new run when the - plan itself must change. -- **serialized run mutation:** planning, live apply, target acceptance, cleanup - apply, and purge apply share a per-run operation lock. Live apply also writes a - separate attachable apply lock and `.bort/runs//applied.json`, so reruns - can attach or resume without racing another mutation or replaying completed - steps blindly. -- **persisted lifecycle:** `run.json` records when the target went live, when the - target was accepted, and when a full source purge completed. Older runs remain - compatible through apply-ledger inference. -- **private files:** migration bundles, state, env, and target credentials are local - artifacts with private permissions. -- **no token flags:** Coolify API tokens are read from `BORT_COOLIFY_TOKEN`, not from - command-line arguments. -- **sanitized target input:** Bort strips source-platform values that should not be - replayed in Dokploy, including `COOLIFY_*`, `SOURCE_COMMIT`, Coolify labels, - Traefik labels, and Caddy labels. -- **reviewed service magic:** `SERVICE_URL_*`, `SERVICE_FQDN_*`, and - `SERVICE_NAME_*` are preserved because some Coolify service stacks depend on - them, but they are surfaced for review. -- **metadata-only source control:** Bort records repository, branch, provider, source - type, source ID, and deploy-key hints, but it does not copy GitHub App, deploy-key, - or webhook credentials into the target platform. -- **narrow cleanup:** `cleanup --apply` backs up the Dokploy database and only deletes - stale Dokploy platform metadata records named `coolify-proxy`, `proxy`, or - `source` when the project is empty and still has zero domains. -- **separate purge:** destructive source deletion lives under `cleanup purge`, - never under regular app entrypoints or metadata cleanup. `--apply` requires a - selected `--app`, `--project`, or `--all-apps` scope, a successful live apply - ledger, a committed target, and the exact `purge ` confirmation - phrase. Bort removes identified containers and networks only when the scope has - no absence-only prerequisite. A scope containing named volumes or host paths - requires manual removal of every listed source resource, which Bort verifies - remains absent without performing destructive operations or recording durable - lifecycle completion. -- **rollback stays inspectable:** `sudo bort rollback` prints the selected run's stored - rollback plan. Executable rollback is not implemented yet. - -## Product surface - -The primary product follows one lifecycle: - -```text -sudo bort # start or resume the current run and review fixes -sudo bort migrate --live # apply only the reviewed current run -sudo bort rollback # inspect the current run's source rollback plan -sudo bort commit --apply # accept the target and retire source containers -sudo bort cleanup # audit leftovers; --apply removes safe metadata only -sudo bort cleanup purge # review source purge and manual prerequisites -``` - -Setup and automation commands remain available for noninteractive use and future -source/target adapters: - -```text -sudo bort migrate --source # scan, export, and create a current run -sudo bort migrate --manifest # create a current run from a manifest -sudo bort migrate --bundle # create or refresh an advanced bundle run -sudo bort env KEY=value ... # record env values noninteractively -sudo bort data --migrate|--recreate|--managed # record a data-store strategy -sudo bort init-target dokploy # bootstrap target credentials explicitly -sudo bort status # compatibility alias for the current cockpit -sudo bort next # compatibility helper for one next action -``` - -The artifact pipeline is also retained for inspection, scripting, compatibility, -and adapter development. Every command below is local and dry-run only, so it can -run unprivileged when its input and output paths are accessible: - -```text -bort scan # extract source platform state -bort plan # classify apps and topology from a manifest -bort export # write an inspectable private migration bundle -bort validate # validate compose, env, routes, storage, and portability -bort prepare # plan target resources before creating anything -bort sync # plan state copy or replication work -bort cutover # plan traffic movement and rollback windows -``` - -## Roadmap and status - -Bort should become the migration layer for self-hosted PaaS users, not just a -one-way script. The practical roadmap starts with platforms that expose enough -Docker, Compose, Swarm, CLI, or API state to make a trustworthy migration plan. - -| Lane | Status | Why it is possible | Notes | -| --- | --- | --- | --- | -| Coolify → Dokploy, same VPS | In progress now | Both platforms run Docker apps with env, domains, volumes, databases, and Traefik-facing routes; local Docker state can verify what the UI/API omits. | Current primary path. | -| Dokploy → Coolify, same VPS | Next | Dokploy apps are Docker/Compose/Swarm-shaped and can be inventoried from the host plus Dokploy metadata; Coolify can recreate apps, databases, env, and domains. | Needs a Dokploy source scanner and Coolify target applier. | -| Coolify → Dokploy, cross-server | Planned | The bundle already separates source inventory from target plans; remote transfer can build on database dumps, stopped copies, rsync, and registry/image pulls. | Intended for moving from an old Coolify VPS to a fresh Dokploy VPS. Needs transfer orchestration and stronger rollback boundaries. | -| Dokploy → Coolify, cross-server | Planned | The same bundle/transfer shape can run in reverse once Dokploy scanning and Coolify apply exist. | Intended for teams standardizing back on Coolify or moving workloads between VPS providers. | -| Other cross-server PaaS moves | Research track | Docker images, compose files, env, domains, volumes, and database dumps are portable when the source platform exposes enough state. | Bort should only graduate these when it can preserve rollback and explain unsupported resources clearly. | -| Docker Compose / plain Docker → Coolify or Dokploy | Planned | Bort already scans Docker resources and exports compose-shaped bundles. | Best when compose files, labels, env files, and named volumes are discoverable. | -| CapRover ↔ Coolify/Dokploy | Research track | CapRover is Docker Swarm-based with app env, domains, volumes, and a CLI/API surface. | Simple apps are plausible; `captain-definition`, one-click apps, and weaker Compose fidelity make complex stacks harder. | -| Dokku ↔ Coolify/Dokploy | Research track | Dokku has CLI-accessible apps, env, domains, proxy config, plugins, and database services, and many apps are buildpack or Dockerfile based. | Plugin diversity means Bort will need per-service confidence levels. | -| Easypanel / Portainer stacks → Coolify/Dokploy | Research track | These tools manage Docker or Compose-like stacks and usually expose app/stack metadata. | Feasibility depends on API coverage for env, domains, volumes, and secrets. | -| Kubernetes-backed PaaS and hosted platforms | Later | Containers, env, services, ingress, and persistent volumes can be translated in principle. | Higher variance; not the first target for safe same-VPS migrations. | - -Near-term product priorities: - -1. polish the Coolify → Dokploy same-VPS path until the default `bort` flow feels - boring and safe; -2. add Dokploy source scanning and Coolify target creation for the reverse path; -3. promote the bundle format into a stable adapter contract; -4. add transfer adapters for cross-server moves; -5. graduate third-party adapters only when Bort can classify what is safe, - blocked, or manual without hiding risk. - -## Release process - -Releases are automated by GitHub Actions and GoReleaser: push a `v*` tag and the -release workflow builds macOS, Linux, and Windows artifacts, Linux -`.deb`/`.rpm` packages, checksums, and updates the `aikins01/homebrew-tap` formula. +Bort is designed for production boxes where the safest default is “look first.” + +- **Preview first:** discovery, planning, validation, rollback inspection, + acceptance planning, cleanup, and purge planning do not change the server. +- **Confirmed Dokploy installation:** the guided setup may offer to install + Dokploy in Bort's side-by-side same-VPS layout before live apply. It explains + the changes and asks for confirmation first. +- **Explicit live apply:** creating target apps, copying data, and moving traffic + only happen through `bort migrate --live` for an existing reviewed run. +- **Known current run:** `.bort/state.json` identifies the current run. Commands + that make changes do not guess based on which file was modified most recently. +- **Plans are locked during live work:** once live execution begins, changing + the reviewed plan requires a new run. +- **One change at a time:** Bort prevents two commands from changing the same run + at once, saves live progress for retries, and keeps `status` available. +- **Private files:** bundles, state, environment values, live progress, and + target credentials stay in the local workspace with private permissions. +- **Separate acceptance:** `commit --apply` retires source application containers + only after successful live apply and is required before destructive source + purge. +- **Separate destructive purge:** purge requires selected apps or projects, a + successful live apply, an accepted target, the exact confirmation phrase, a + recheck of Docker IDs, and a private backup. +- **Manual rollback only:** `bort rollback` prints the stored plan. Bort does not + currently execute recovery actions. + +## Documentation + +- [Migration guide](docs/migration-guide.md): requirements, workspace and user + rules, review, live apply, validation, acceptance, and recovery. +- [Cleanup and purge](docs/cleanup-and-purge.md): safe Dokploy cleanup, source + removal, manual steps, completion, and recovery from a partial failure. +- [End-to-end acceptance](docs/acceptance.md): automated test, disposable-host + test cases, destructive test procedure, and safe evidence handling. +- `bort help`: main migration commands. +- `bort help --advanced`: setup, automation, and migration-file commands. + +## Current limitations + +- Bort shows rollback instructions but does not run them. +- Bort does not keep copying new volume changes while applications remain live. +- Database moves currently use local export/import or copy data while the + database is stopped. +- GitHub Apps, deploy keys, webhooks, and equivalent source connections are + recorded as details but are not copied, recreated, or revoked. +- Bort does not automatically delete source Docker images because the target may + share their layers and tags. +- The full real-host acceptance matrix remains to be recorded on a disposable + Linux VPS. + +## Roadmap + +Near-term work remains focused on making the same-VPS Coolify → Dokploy path +boring and safe before adding more platforms: + +1. complete the disposable-host end-to-end tests; +2. make workspace selection, requirement checks, and failure recovery clearer + in the guided screen; +3. decide how automated rollback should work; +4. add Dokploy source scanning and Coolify target creation; +5. add cross-server transfers after the same-VPS steps are proven. + +Other Docker-, Compose-, and Swarm-based platforms remain possible future +targets. Bort should only support one when it can clearly say which resources +are safe to move, which are blocked, and which need manual work. + +## Development and releases + +Run the repository checks with: ```sh -git tag v0.1.0 -git push origin v0.1.0 +make test +make vet +make build ``` -The Homebrew tap update uses the `TAP_GITHUB_TOKEN` repository secret. +The fake end-to-end trace and destructive disposable-host checklist are in the +[acceptance guide](docs/acceptance.md). + +Releases are automated by GitHub Actions and GoReleaser. A new `v*` tag builds +macOS, Linux, and Windows archives, Linux packages, checksums, and the Homebrew +cask update. Use the intended release version rather than copying an old tag +from this README. ## Name `bort` is named after Bortianor, a coastal town in Accra. The project is about -safe passage: moving apps from one harbor to another without dropping traffic -into the sea. +safe passage: moving applications from one harbor to another without dropping +traffic into the sea. diff --git a/docs/acceptance.md b/docs/acceptance.md new file mode 100644 index 0000000..56d14b0 --- /dev/null +++ b/docs/acceptance.md @@ -0,0 +1,111 @@ +# End-to-end acceptance testing + +This guide is for maintainers testing Bort releases. It includes an automated +test with fake services and a destructive checklist for a disposable host. The +automated test does not prove that real Docker, proxy, volume, or filesystem +behavior works. + +Do not claim production-host support until a redacted result from the +disposable-host checklist has been recorded for the version being released. + +## Automated end-to-end test + +The automated test uses a temporary workspace, fake Dokploy HTTP API, and fake +Docker command. It creates a self-contained run, performs live apply, verifies +`TARGET LIVE`, commits, verifies `COMMITTED`, inventories cleanup, applies a +confirmed all-app purge, and verifies `COMPLETE`: + +```sh +go test ./internal/cli -run '^TestLifecycleAcceptanceTraceWithFakeDokploy$' -v +``` + +Run the complete repository checks separately: + +```sh +make test +make vet +make build +``` + +## Disposable-host test cases + +Use a fresh Linux VM or a restorable snapshot containing no data you intend to +keep. Exercise at least: + +| Scenario | Required evidence | +| --- | --- | +| Stateless app | Domain remains healthy after live apply, acceptance, and source purge. | +| Database | Representative data survives the selected migration strategy and remains readable and writable on the target. | +| Named volume | Dry run requires manual completion; apply verifies absence without deleting resources automatically or recording completion. | +| Bind mount | Bort requires manual removal of the host path and refuses protected paths. | +| Interrupted live apply | Rerunning the command continues from saved progress without repeating completed work. | +| Two live commands | The second process shows the active progress while `status` remains readable. | +| Automatic purge | Bort removes the exact source containers and networks while keeping target resources and credentials. | +| Same-name replacement | Bort does not mistake a newly created resource for the one that was reviewed. | + +## Disposable-host procedure + +1. Install Coolify, then deploy one stateless disposable Coolify app with a + route, source network, and non-secret test environment value. Do not perform a + normal Dokploy installation alongside the active Coolify proxy. Use Bort's + guided `same-VPS shadow mode`, its side-by-side Dokploy layout, so Dokploy's + proxy is prepared without taking ports 80/443 before Bort switches web + traffic. +2. Record the source container, network, and target project names before + migration. +3. In a dedicated working directory, run the complete migration as one OS user: + + ```sh + run="bort-acceptance-$(date +%Y%m%d-%H%M%S)" + sudo bort migrate --source coolify-local --run "$run" + sudo bort + sudo bort migrate --live --run "$run" + sudo bort status --run "$run" + sudo bort commit --apply --run "$run" + sudo bort cleanup --run "$run" + sudo bort cleanup purge --run "$run" --all-apps + ``` + +4. Review the purge plan and verify the target still serves the disposable + workload before applying it: + + ```sh + sudo bort cleanup purge --apply --run "$run" --all-apps --confirm "purge $run" + sudo bort status --run "$run" + ``` + +5. Confirm that the target application remains healthy, `run.json` records live + apply and acceptance, the selected non-platform source container and network + are absent, and target resources plus source-control credentials remain. A + real `coolify-local` run includes internal Coolify components that `--all-apps` + excludes unless `--include-platform` is supplied, so this purge must not + record `PurgedAt` or claim that the whole run is complete. +6. Restore the clean snapshot and repeat with a disposable app containing a + named volume and a bind mount under a disposable directory. After the purge + dry run, manually remove every listed source resource. Confirm that `--apply` + performs only absence verification, leaves the run accepted rather than + complete, and does not delete a replacement resource introduced before + verification. +7. Repeat once with an interrupted live apply and once with two concurrent live + invocations. Confirm that resume, attachment, and read-only status behave as + documented. +8. Destroy the VM or restore its snapshot after retaining only the redacted + evidence needed for the release. + +## Evidence handling + +Retain: + +- Bort version and release candidate commit; +- host OS, architecture, Docker version, Coolify version, and Dokploy version; +- redacted command output and migration status; +- the test-case results; +- confirmation that the VM was destroyed or restored. + +Do not retain or publish: + +- `.bort/state.json`; +- exported environment values; +- Coolify or Dokploy tokens; +- database dumps or application data; +- unredacted manifests, bundles, apply ledgers, or purge backups. diff --git a/docs/cleanup-and-purge.md b/docs/cleanup-and-purge.md new file mode 100644 index 0000000..4594769 --- /dev/null +++ b/docs/cleanup-and-purge.md @@ -0,0 +1,154 @@ +# Cleanup and purge + +Cleanup normally happens after live apply succeeds and you accept Dokploy. Bort +keeps two commands separate: + +- `cleanup` lists leftovers and can remove a small set of eligible unused + records from the Dokploy database; +- `cleanup purge` removes source resources only after you select the apps or + projects and confirm the command. + +Neither command removes target Dokploy applications or source-control +credentials. + +## Accept the target first + +Validate the target and wait through the `rollback window` recorded in the plan +before running: + +```sh +sudo bort commit --apply +``` + +This confirms that Dokploy is now the active platform and stops the source +application containers. Bort does not currently provide automatic rollback, so +do not run it until you have independently checked the target. Bort does not +measure the rollback window or block this command when the window has not passed. + +## Ordinary cleanup + +Inventory leftovers with: + +```sh +sudo bort cleanup +``` + +This command only shows what it found. It reports source containers, volumes, +networks, source-control details, resources created in Dokploy, and eligible +unused records in the Dokploy database. + +Remove only those eligible Dokploy records with: + +```sh +sudo bort cleanup --apply +``` + +Before changing the Dokploy database, Bort backs it up. The command only removes +eligible empty records with no domains from Dokploy projects named +`coolify-proxy`, `proxy`, or `source`. It does not remove source containers, +volumes, networks, paths, credentials, Docker images, or target applications. + +## Plan a source purge + +Start with a preview, which Bort calls a dry run: + +```sh +sudo bort cleanup purge +``` + +Select the apps or projects you intend to remove before applying: + +```sh +sudo bort cleanup purge --app demo-app +sudo bort cleanup purge --project demo-project +sudo bort cleanup purge --all-apps +``` + +`--all-apps` covers normal applications by default. Internal Coolify components, +such as its source and proxy services, additionally require +`--include-platform`. + +Review every listed resource and use the exact apply command and confirmation +phrase emitted by the dry run. A named example is: + +```sh +run="bort-acceptance-20260802-120000" +sudo bort cleanup purge --run "$run" --all-apps +sudo bort cleanup purge --apply --run "$run" --all-apps --confirm "purge $run" +``` + +Purge apply refuses to run unless: + +- `--app`, `--project`, or `--all-apps` selects what to remove; +- Bort's saved record shows that live apply succeeded; +- the target was accepted with `commit --apply`; +- the exact `purge ` confirmation is supplied. + +Bort writes a private purge-plan backup under `.bort/backups` before destructive +work begins. + +## Automatic removal and manual-removal checks + +Bort behaves differently depending on the selected resources. + +### Eligible containers and networks + +When none of the selected resources requires manual removal, Bort can remove +eligible source containers and networks. It records and rechecks their exact +Docker IDs so it will not delete a different resource that happens to reuse a +name. + +### Named volumes or host paths + +Bort never automatically deletes named volumes or host source paths. If the +selection contains either one, the preview requires you to remove every listed +source container, volume, network, and path yourself. + +After those resources have been removed manually, `cleanup purge --apply` only +verifies that everything you selected remains absent. It does not mix manual +removal with automatic deletion. + +Verification-only purge does not set the run's `PurgedAt` timestamp because Bort +cannot prevent an external tool from recreating a resource after the absence +check. + +## When the migration is marked complete + +A successful automatic purge covering all run applications can mark the +migration complete. Removing one app or project does not claim completion for +the whole run. A manual-removal check also leaves the run accepted rather than +marking the whole migration complete. + +Check the resulting state with: + +```sh +sudo bort status +``` + +## What purge preserves + +Source purge does not remove: + +- target Dokploy projects, compose applications, domains, or volumes; +- GitHub Apps, deploy keys, webhooks, or other source-control credentials; +- Docker images, because image layers and tags can be shared with target + workloads; +- named volumes or host paths automatically. + +## Recover from a partial purge + +A destructive operation can stop after some earlier resources have already been +removed. If purge reports an incomplete result: + +1. Stop issuing cleanup commands. +2. Save the complete output without publishing secrets. +3. Inspect the purge results and the private backup path printed by Bort. +4. Verify the target still serves the application. +5. Compare every selected source resource with the recorded result before + retrying the generated scoped command. + +Do not recreate a source resource under the same name and assume it is the same +object. Bort checks Docker IDs and refuses a different replacement resource. + +Use the destructive scenarios in the [acceptance guide](acceptance.md) only on a +fresh VM or restorable snapshot. diff --git a/docs/migration-guide.md b/docs/migration-guide.md new file mode 100644 index 0000000..8d10b48 --- /dev/null +++ b/docs/migration-guide.md @@ -0,0 +1,207 @@ +# Migration guide + +This guide covers Bort's current supported migration: moving applications from +Coolify to Dokploy on the same Linux VPS. + +The live migration still needs a recorded test on a disposable Linux host before +it should be relied on in production. Use a fresh VM or a restorable snapshot +for testing, and keep an independent backup of any data you need to retain. + +## Prerequisites + +Before starting, confirm that: + +- Coolify is running on the source Linux VPS and its applications are healthy; +- Docker is available to the OS user that will run Bort; +- `bash`, `python3`, `openssl`, `curl`, and the standard Linux networking tools + used by Bort's side-by-side Dokploy installer are available; if Docker + live-restore is enabled, the host must use systemd and provide a working + `systemctl` so the installer can reload Docker for Swarm mode; +- the host has enough free space for migration copies, database dumps, and + backups; +- you have a terminal on the source VPS and can keep using the same working + directory and OS user throughout the migration; +- Dokploy will run on this same VPS and use Bort's side-by-side layout, which the + CLI calls `same-VPS shadow mode`; either it is already prepared in that layout + or it will be installed through Bort's guided setup. This layout prevents + Dokploy's proxy from taking ports 80/443 before Bort switches traffic; +- the VPS has a current provider snapshot or another independently tested + recovery path. + +Bort publishes binaries for multiple operating systems, but a complete same-VPS +move depends on Linux, Docker, and access to protected host files. Use the macOS +and Windows builds to inspect migration files and help develop Bort, not to run a +production migration. + +## Keep one workspace and one identity + +Bort stores the current run, Dokploy credentials, your answers, plans, and live +progress under `.bort` in the current working directory. Running Bort from +another directory creates or reads a different workspace. + +Create a dedicated directory and return to it for every migration command: + +```sh +mkdir -p ~/bort-migration +cd ~/bort-migration +sudo bort +``` + +Use the same OS user for the whole migration. Finding apps, copying data, +switching web traffic, stopping source containers, and removing leftovers need +Docker and protected-file access, so this guide consistently uses `sudo`. Bort's +side-by-side Dokploy installer also requires root. Running without `sudo` is only +appropriate when Dokploy is already prepared and the regular account has every +required Docker and source-path permission. Choose before the first command and +do not switch users in one workspace. + +The `.bort` directory contains credentials and application configuration. Bort +uses private file permissions, but you should still keep the directory out of +source control, support tickets, and shared archives. + +## Start or resume + +The main command opens the guided status screen: + +```sh +sudo bort +``` + +It starts setup when the workspace has no current run and resumes the selected +run otherwise. To start Coolify discovery directly without the setup questions: + +```sh +sudo bort migrate --source coolify-local +sudo bort +``` + +For an existing manifest: + +```sh +sudo bort migrate --manifest manifest.json +``` + +Each source or manifest start creates a self-contained run under +`.bort/runs/`. Use `--run ` only when intentionally selecting a run +other than the workspace default. + +## Review and fix + +The guided screen shows whether each application is ready, what must be fixed, +and the next safe action. Follow the generated `fix:` commands and rerun Bort +until it shows `READY`. + +Use the guided screen for secret values so they are not placed in command +arguments, shell history, or process and sudo audit records. The direct command +is appropriate for non-secret values and database or storage choices: + +```sh +sudo bort env demo-app APP_MODE=production +sudo bort data demo-app postgres --migrate +sudo bort +``` + +Review application routes, environment values, database and storage choices, +named volumes, bind mounts, and source-control details. `READY` means the plan +has been reviewed; the server has not been changed yet. + +## Apply the reviewed run + +Live apply is a separate explicit command: + +```sh +sudo bort migrate --live +``` + +Live mode applies only the selected existing run. It does not create or refresh +a plan. Once live execution begins, that plan cannot be changed. Create a new +run if the plan itself must change. + +If live apply is interrupted, Bort saves its progress. Rerun the same command to +resume: + +```sh +sudo bort migrate --live +``` + +If another process is already applying the same run, a second `migrate --live` +command joins it and shows its progress. You can check status separately: + +```sh +sudo bort status +``` + +Do not remove lock files when another change is running. Wait for it to finish, +or use the guided screen and `status` to decide whether live apply should be +resumed. + +## Validate and accept the target + +After live apply, validate the target application before accepting it: + +- exercise every migrated domain and health URL; +- verify expected environment-dependent behavior; +- read and write representative database data; +- verify uploaded or persistent files; +- inspect Dokploy deployment and proxy health; +- keep observing for at least the `rollback window` recorded in the plan. + +The rollback window is guidance for the operator, not an enforced timer. Bort +does not measure how long you wait or block `commit --apply` when the window has +not passed. + +You can inspect the stored rollback plan with: + +```sh +sudo bort rollback +``` + +Bort does not currently execute rollback. The command prints the stored plan; +any recovery action remains manual. Do not accept the target unless that manual +recovery limitation is understood. + +Accepting the target retires source application containers: + +```sh +sudo bort commit --apply +``` + +This command stops the source application containers. Run it only after checking +the target and waiting through the planned rollback window. + +## Audit and clean up + +After acceptance, list the remaining Dokploy records and source resources: + +```sh +sudo bort cleanup +``` + +Ordinary cleanup and destructive source purge are separate operations. See the +[cleanup and purge guide](cleanup-and-purge.md) before applying either one. + +## Recovery guide + +| Situation | Safe next action | +| --- | --- | +| Bort cannot find the expected run | Return to the original working directory and original OS user. Do not create a replacement workspace accidentally. | +| Live apply was interrupted | Run `sudo bort status`, then rerun `sudo bort migrate --live` to resume from the saved progress. | +| Another change is running | Keep `status` open if useful and wait. A second live command joins an active live apply; other commands that make changes must wait. | +| The plan needs to change after live execution began | Keep the existing run as a record and create a new named run. A plan cannot change after live work starts. | +| Target validation fails before acceptance | Keep source resources available, inspect `sudo bort rollback`, and perform the required recovery manually. Automated rollback is not implemented. | +| Cleanup or purge stops partway through | Stop and inspect the command output and private backup before retrying. Follow the recovery guidance in the cleanup and purge guide. | +| `.bort` cannot be read | Do not change ownership or permissions blindly. Confirm the original working directory and OS user first, then restore the workspace from backup if necessary. | + +## Current workload coverage + +| Surface | Current behavior | +| --- | --- | +| Discovery | Local Coolify Docker state, local Docker, Coolify API, or an existing manifest. | +| App setup | Exports Compose or image-based app details and prepares Dokploy projects and compose applications. Unsupported Compose features remain review items. | +| Environment | Captures application values, removes Coolify-specific values that should not be copied, and keeps generated files private. | +| Routes | Plans supported routes with health checks, a waiting period, and rollback information. Missing or unsupported proxy details block the migration or require review. | +| Databases and persistent data | Supports local database dump/restore and copying while a service is stopped. Continuous syncing and other replication methods are not implemented. | +| Named volumes and bind mounts | Lists them and asks how they should be handled when manual review is required. Source purge never automatically deletes named volumes or host paths. | +| Source control | Records repository details and deploy-key hints, but does not copy, revoke, or recreate GitHub Apps, deploy keys, webhooks, or equivalent credentials. | + +For release-level validation, use the [acceptance guide](acceptance.md). From 14748b861bb18204188b136ab9bb4d64ca4ad0ef Mon Sep 17 00:00:00 2001 From: Aikins Laryea Date: Sun, 2 Aug 2026 13:26:07 +0000 Subject: [PATCH 03/31] migration: harden apply and purge safety (#10) Bracket every Dokploy compose mutation with active-patch checks and fail loudly if concurrent state appears. Dokploy does not expose a shared CAS or lock, so this detects but cannot prevent the residual race. Keep unresolved containers out of purge execution, persist resource errors, isolate Docker-dependent tests, and correct operator guidance. --- README.md | 6 +- docs/acceptance.md | 17 +++-- internal/cli/cleanup.go | 3 + internal/cli/cleanup_test.go | 17 ++++- internal/cli/wizard.go | 2 +- internal/target/dokploy/apply.go | 57 +++++++++++----- internal/target/dokploy/apply_test.go | 85 ++++++++++++++++++++++++ internal/target/dokploy/cleanup.go | 3 + internal/target/dokploy/cleanup_test.go | 3 + internal/target/dokploy/datamove.go | 2 +- internal/target/dokploy/datamove_test.go | 38 +++++++++++ 11 files changed, 207 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 6a1c7c9..3bdb938 100644 --- a/README.md +++ b/README.md @@ -165,8 +165,10 @@ Bort is designed for production boxes where the safest default is “look first. - **Preview first:** discovery, planning, validation, rollback inspection, acceptance planning, cleanup, and purge planning do not change the server. - **Confirmed Dokploy installation:** the guided setup may offer to install - Dokploy in Bort's side-by-side same-VPS layout before live apply. It explains - the changes and asks for confirmation first. + Dokploy in Bort's side-by-side same-VPS layout before live apply. It warns that + this writes system configuration, creates Docker resources, initializes Swarm + when needed, and may disable Docker live-restore and reload Docker before + asking for confirmation. - **Explicit live apply:** creating target apps, copying data, and moving traffic only happen through `bort migrate --live` for an existing reviewed run. - **Known current run:** `.bort/state.json` identifies the current run. Commands diff --git a/docs/acceptance.md b/docs/acceptance.md index 56d14b0..53947d7 100644 --- a/docs/acceptance.md +++ b/docs/acceptance.md @@ -61,12 +61,19 @@ keep. Exercise at least: sudo bort sudo bort migrate --live --run "$run" sudo bort status --run "$run" + ``` + +4. Verify that the target serves the workload and wait through the rollback + window recorded in the plan. Only then accept the target and plan source + removal: + + ```sh sudo bort commit --apply --run "$run" sudo bort cleanup --run "$run" sudo bort cleanup purge --run "$run" --all-apps ``` -4. Review the purge plan and verify the target still serves the disposable +5. Review the purge plan and verify the target still serves the disposable workload before applying it: ```sh @@ -74,22 +81,22 @@ keep. Exercise at least: sudo bort status --run "$run" ``` -5. Confirm that the target application remains healthy, `run.json` records live +6. Confirm that the target application remains healthy, `run.json` records live apply and acceptance, the selected non-platform source container and network are absent, and target resources plus source-control credentials remain. A real `coolify-local` run includes internal Coolify components that `--all-apps` excludes unless `--include-platform` is supplied, so this purge must not record `PurgedAt` or claim that the whole run is complete. -6. Restore the clean snapshot and repeat with a disposable app containing a +7. Restore the clean snapshot and repeat with a disposable app containing a named volume and a bind mount under a disposable directory. After the purge dry run, manually remove every listed source resource. Confirm that `--apply` performs only absence verification, leaves the run accepted rather than complete, and does not delete a replacement resource introduced before verification. -7. Repeat once with an interrupted live apply and once with two concurrent live +8. Repeat once with an interrupted live apply and once with two concurrent live invocations. Confirm that resume, attachment, and read-only status behave as documented. -8. Destroy the VM or restore its snapshot after retaining only the redacted +9. Destroy the VM or restore its snapshot after retaining only the redacted evidence needed for the release. ## Evidence handling diff --git a/internal/cli/cleanup.go b/internal/cli/cleanup.go index 042fbbc..344c8ec 100644 --- a/internal/cli/cleanup.go +++ b/internal/cli/cleanup.go @@ -469,6 +469,9 @@ func planCleanupPurge(run loadedMigrationRun, target string, filters cleanupPurg return cleanupPurgeResult{}, err } for _, container := range containers { + if strings.TrimSpace(container.ContainerID) == "" { + continue + } if sharedWith := cleanupContainerOwnersOutsideSelected(owners, container, selectedNames); len(sharedWith) > 0 { result.CompletesLifecycle = false result.Warnings = append(result.Warnings, fmt.Sprintf("source container %s for %s is also referenced by unselected app(s): %s", firstCleanupValue(container.ContainerName, container.ContainerID), app.Name, strings.Join(sharedWith, ", "))) diff --git a/internal/cli/cleanup_test.go b/internal/cli/cleanup_test.go index 9bdbe1d..27c4841 100644 --- a/internal/cli/cleanup_test.go +++ b/internal/cli/cleanup_test.go @@ -292,6 +292,16 @@ func TestRunCleanupPurgeInventoriesDestructiveSourceResources(t *testing.T) { func TestRunCleanupPurgeApplyRequiresExplicitScopeAndConfirmation(t *testing.T) { workDir := t.TempDir() t.Chdir(workDir) + binDir := filepath.Join(workDir, "bin") + if err := os.MkdirAll(binDir, 0o700); err != nil { + t.Fatal(err) + } + dockerPath := filepath.Join(binDir, "docker") + dockerStub := "#!/bin/sh\nif [ \"$1\" = inspect ]; then\n echo 'Error: No such object' >&2\n exit 1\nfi\nif [ \"$1\" = volume ] && [ \"$2\" = inspect ]; then\n echo 'Error response from daemon: no such volume' >&2\n exit 1\nfi\nexit 0\n" + if err := os.WriteFile(dockerPath, []byte(dockerStub), 0o700); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", binDir+string(os.PathListSeparator)+os.Getenv("PATH")) bundleDir := filepath.Join(workDir, "bort-bundle") writeTestBundle(t, bundleDir, manifest.Manifest{ Source: manifest.Source{Platform: "coolify-local"}, @@ -477,12 +487,15 @@ func TestCleanupPurgeFailsClosedWhenTopologyIsUnavailable(t *testing.T) { if err := os.WriteFile(topologyPath, topology, 0o600); err != nil { t.Fatal(err) } - run.Prepare.Apps[0].Resources.SourceServices = []preparer.SourceServiceRef{{ServiceName: "api"}} + run.Prepare.Apps[0].Resources.SourceServices = []preparer.SourceServiceRef{ + {ServiceName: "api", ContainerName: "api"}, + {ServiceName: "worker", ContainerID: "worker-id", ContainerName: "worker"}, + } incomplete, err := planCleanupPurge(run, "dokploy", cleanupPurgeFilters{AllApps: true}, nil) if err != nil { t.Fatal(err) } - if incomplete.CompletesLifecycle || !strings.Contains(strings.Join(incomplete.Warnings, "\n"), "has no stable container ID") { + if incomplete.CompletesLifecycle || !strings.Contains(strings.Join(incomplete.Warnings, "\n"), "has no stable container ID") || len(incomplete.SourceContainers) != 1 || incomplete.SourceContainers[0].ContainerID != "worker-id" { t.Fatalf("expected unresolved source coordinates to keep lifecycle incomplete, got %#v", incomplete) } } diff --git a/internal/cli/wizard.go b/internal/cli/wizard.go index 2a8f3c7..4ca2fc0 100644 --- a/internal/cli/wizard.go +++ b/internal/cli/wizard.go @@ -224,7 +224,7 @@ func promptInstallAndBootstrapDokploy(ctx context.Context, stdin io.Reader, stdo if strings.TrimSpace(defaultURL) == "" { defaultURL = "http://127.0.0.1:3030" } - description := fmt.Sprintf("Bort will set up Dokploy in same-VPS shadow mode at %s and keep Coolify on :80/:443 until cutover. You may be asked to choose a Coolify admin and enter its password.", defaultURL) + description := fmt.Sprintf("Bort will set up Dokploy in same-VPS shadow mode at %s. This writes system configuration, creates Docker resources, initializes Swarm when needed, and may disable Docker live-restore and reload Docker. Coolify keeps :80/:443 until Bort switches traffic. You may be asked to choose a Coolify admin and enter its password.", defaultURL) form := huh.NewForm(huh.NewGroup( huh.NewConfirm(). Title("Install Dokploy now?"). diff --git a/internal/target/dokploy/apply.go b/internal/target/dokploy/apply.go index 2c78cd0..3bacf19 100644 --- a/internal/target/dokploy/apply.go +++ b/internal/target/dokploy/apply.go @@ -267,6 +267,8 @@ func validatePlanReadyForLiveApply(plan Plan) error { continue } switch app.Readiness { + // acknowledged decisions remain in the immutable plan; the CLI blocks + // unacknowledged decisions before it calls the target client. case "", preparer.ReadinessReadyToCreate, preparer.ReadinessNeedsDecision: default: return fmt.Errorf("live apply blocked by prepare readiness for app %s: %s", app.Name, app.Readiness) @@ -303,6 +305,23 @@ func (c *Client) validateNoActiveBortOverrides(ctx context.Context, composeID st if len(patches) == 0 { return nil } + details := activeBortOverridePatchDetails(patches) + return fmt.Errorf("active Bort-owned Dokploy patch(es) override compose app(s): %s; merge the patch contents into the repo or disable the patch before live apply so the reviewed compose file remains the source of truth", strings.Join(details, "; ")) +} + +func (c *Client) validateNoActiveBortOverridesAfterMutation(ctx context.Context, composeID string) error { + patches, err := c.activeBortOverridePatches(ctx, composeID) + if err != nil { + return fmt.Errorf("recheck active Dokploy patches after changing compose %s; the compose was already changed and must be verified manually: %w", composeID, err) + } + if len(patches) == 0 { + return nil + } + details := activeBortOverridePatchDetails(patches) + return fmt.Errorf("active Bort-owned Dokploy patch(es) appeared while Bort was changing compose %s: %s; stop and reconcile the compose file and patch before resuming", composeID, strings.Join(details, "; ")) +} + +func activeBortOverridePatchDetails(patches []activeBortOverridePatch) []string { details := make([]string, 0, len(patches)) for _, patch := range patches { repo := strings.Trim(strings.TrimSpace(patch.Repository)+"@"+strings.TrimSpace(patch.Branch), "@") @@ -312,7 +331,7 @@ func (c *Client) validateNoActiveBortOverrides(ctx context.Context, composeID st details = append(details, fmt.Sprintf("%s on %s (%s, %s)", patch.PatchID, patch.ComposeName, patch.FilePath, repo)) } sort.Strings(details) - return fmt.Errorf("active Bort-owned Dokploy patch(es) override Git-backed compose app(s): %s; merge the patch contents into the repo or disable the patch before live apply so Git remains the source of truth", strings.Join(details, "; ")) + return details } func (c *Client) activeBortOverridePatches(ctx context.Context, composeID string) ([]activeBortOverridePatch, error) { @@ -343,7 +362,6 @@ join compose c on c."composeId" = p."composeId" where p.enabled = true and p."patchId" like 'bort-%%' and c."composeId" = %s - and coalesce(c."sourceType"::text, '') not in ('', 'raw') order by c.name, p."patchId"; `, sqlStringLiteral(composeID)) } @@ -698,10 +716,7 @@ func (c *Client) applyUploadEnv(ctx context.Context, actx *applyContext, step St if err != nil { return err } - if err := c.validateNoActiveBortOverrides(ctx, entry.ComposeID); err != nil { - return err - } - return c.UpdateCompose(ctx, entry.ComposeID, composeFile, envContent) + return c.updateComposeWithPatchGuard(ctx, entry.ComposeID, composeFile, envContent) } func (c *Client) applyPushImage(ctx context.Context, actx *applyContext, step Step) error { @@ -716,10 +731,7 @@ func (c *Client) applyPushImage(ctx context.Context, actx *applyContext, step St if err := ensureComposeImagesAvailable(ctx, c.dockerRunner(), actx.plan, step.App, composeFile); err != nil { return err } - if err := c.validateNoActiveBortOverrides(ctx, entry.ComposeID); err != nil { - return err - } - if err := c.DeployCompose(ctx, entry.ComposeID, deployComposeTitle(actx.plan)); err != nil { + if err := c.deployComposeWithPatchGuard(ctx, entry.ComposeID, deployComposeTitle(actx.plan)); err != nil { if safetyErr := c.validateMigratedVolumeMountsAfterDeploy(ctx, actx, step.App); safetyErr != nil && isUnsafeTargetResumeError(safetyErr) { return fmt.Errorf("deploy dokploy compose: %w; post-deploy safety check: %v", err, safetyErr) } @@ -731,6 +743,24 @@ func (c *Client) applyPushImage(ctx context.Context, actx *applyContext, step St return c.pauseTargetWritersForState(ctx, c.dockerRunner(), actx, step.App) } +func (c *Client) updateComposeWithPatchGuard(ctx context.Context, composeID, composeFile, envContent string) error { + if err := c.validateNoActiveBortOverrides(ctx, composeID); err != nil { + return err + } + mutationErr := c.UpdateCompose(ctx, composeID, composeFile, envContent) + patchErr := c.validateNoActiveBortOverridesAfterMutation(ctx, composeID) + return errors.Join(mutationErr, patchErr) +} + +func (c *Client) deployComposeWithPatchGuard(ctx context.Context, composeID, title string) error { + if err := c.validateNoActiveBortOverrides(ctx, composeID); err != nil { + return err + } + mutationErr := c.DeployCompose(ctx, composeID, title) + patchErr := c.validateNoActiveBortOverridesAfterMutation(ctx, composeID) + return errors.Join(mutationErr, patchErr) +} + func deployComposeTitle(plan Plan) string { title := "bort-migrate" if plan.RunName != "" { @@ -913,10 +943,7 @@ func (c *Client) applyActivateRoutes(ctx context.Context, actx *applyContext, st if err != nil { return err } - if err := c.validateNoActiveBortOverrides(ctx, entry.ComposeID); err != nil { - return err - } - if err := c.UpdateCompose(ctx, entry.ComposeID, composeFile, envContent); err != nil { + if err := c.updateComposeWithPatchGuard(ctx, entry.ComposeID, composeFile, envContent); err != nil { return err } for _, route := range cutoverRoutesForApp(actx.plan.Cutover, step.App) { @@ -928,7 +955,7 @@ func (c *Client) applyActivateRoutes(ctx context.Context, actx *applyContext, st return err } } - if err := c.DeployCompose(ctx, entry.ComposeID, deployComposeTitle(actx.plan)); err != nil { + if err := c.deployComposeWithPatchGuard(ctx, entry.ComposeID, deployComposeTitle(actx.plan)); err != nil { if safetyErr := c.validateMigratedVolumeMountsAfterDeploy(ctx, actx, step.App); safetyErr != nil && isUnsafeTargetResumeError(safetyErr) { return fmt.Errorf("deploy dokploy compose: %w; post-deploy safety check: %v", err, safetyErr) } diff --git a/internal/target/dokploy/apply_test.go b/internal/target/dokploy/apply_test.go index 9bc1c28..d5af979 100644 --- a/internal/target/dokploy/apply_test.go +++ b/internal/target/dokploy/apply_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "errors" + "io" "net/http" "net/http/httptest" "os" @@ -251,6 +252,90 @@ func TestActivePatchGuardBlocksGitBackedCompose(t *testing.T) { t.Fatalf("expected patch guard sql to contain %q, got:\n%s", want, sql) } } + if strings.Contains(sql, `coalesce(c."sourceType"::text, '') not in`) { + t.Fatalf("expected active Bort patches to block mutations regardless of current compose source type, got:\n%s", sql) + } +} + +type stagedBortPatchRunner struct { + fakeDockerRunner + checks int +} + +func (r *stagedBortPatchRunner) Run(ctx context.Context, stdin io.Reader, stdout io.Writer, args ...string) error { + key := strings.Join(args, " ") + if strings.Contains(key, " psql ") { + r.checks++ + if r.checks == 2 { + r.runOutputs[key] = []byte(`{"patchId":"bort-api-compose","filePath":"docker|compose.yml","composeName":"api","sourceType":"github","repository":"owner/repo","branch":"main"}` + "\n") + } else { + r.runOutputs[key] = nil + } + } + return r.fakeDockerRunner.Run(ctx, stdin, stdout, args...) +} + +func TestComposeMutationPatchGuardsDetectConcurrentPatch(t *testing.T) { + for _, test := range []struct { + name string + path string + mutate func(*Client) error + }{ + {name: "update", path: "/api/compose.update", mutate: func(client *Client) error { + return client.updateComposeWithPatchGuard(context.Background(), "compose-api", "services: {}\n", "") + }}, + {name: "deploy", path: "/api/compose.deploy", mutate: func(client *Client) error { + return client.deployComposeWithPatchGuard(context.Background(), "compose-api", "bort-migrate") + }}, + } { + t.Run(test.name, func(t *testing.T) { + mutations := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost || r.URL.Path != test.path { + http.NotFound(w, r) + return + } + mutations++ + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + runner := &stagedBortPatchRunner{fakeDockerRunner: fakeDockerRunner{ + outputs: map[string][]byte{"ps --format {{.Names}}": []byte("dokploy-postgres\n")}, + runOutputs: map[string][]byte{}, + }} + client := &Client{BaseURL: server.URL, Token: "secret", HTTPClient: server.Client(), Docker: runner} + err := test.mutate(client) + if err == nil || !strings.Contains(err.Error(), "appeared while Bort was changing compose") { + t.Fatalf("expected post-mutation patch detection, got %v", err) + } + if mutations != 1 || runner.checks != 2 { + t.Fatalf("expected one mutation between two patch checks, mutations=%d checks=%d", mutations, runner.checks) + } + }) + } +} + +func TestComposeMutationPatchGuardRechecksAfterHTTPError(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost || r.URL.Path != "/api/compose.update" { + http.NotFound(w, r) + return + } + http.Error(w, "mutation failed", http.StatusInternalServerError) + })) + defer server.Close() + runner := &stagedBortPatchRunner{fakeDockerRunner: fakeDockerRunner{ + outputs: map[string][]byte{"ps --format {{.Names}}": []byte("dokploy-postgres\n")}, + runOutputs: map[string][]byte{}, + }} + client := &Client{BaseURL: server.URL, Token: "secret", HTTPClient: server.Client(), Docker: runner} + err := client.updateComposeWithPatchGuard(context.Background(), "compose-api", "services: {}\n", "") + if err == nil || !strings.Contains(err.Error(), "mutation failed") || !strings.Contains(err.Error(), "appeared while Bort was changing compose") { + t.Fatalf("expected HTTP and post-mutation patch errors, got %v", err) + } + if runner.checks != 2 { + t.Fatalf("expected the failed HTTP mutation to remain bracketed by patch checks, got %d", runner.checks) + } } func TestApplyPushImageChecksActivePatchWithoutEnvOrRoutes(t *testing.T) { diff --git a/internal/target/dokploy/cleanup.go b/internal/target/dokploy/cleanup.go index 63e7fae..51bbb96 100644 --- a/internal/target/dokploy/cleanup.go +++ b/internal/target/dokploy/cleanup.go @@ -207,6 +207,9 @@ func runSourcePurgeResource(result *SourcePurgeResult, resources *[]SourcePurgeR return err } outcome, operationErr := operation() + if operationErr != nil && strings.TrimSpace(outcome.Message) == "" { + outcome.Message = operationErr.Error() + } (*resources)[len(*resources)-1] = outcome if err := publishSourcePurgeProgress(*result, onProgress); err != nil { if operationErr != nil { diff --git a/internal/target/dokploy/cleanup_test.go b/internal/target/dokploy/cleanup_test.go index 5c218f9..4c4af76 100644 --- a/internal/target/dokploy/cleanup_test.go +++ b/internal/target/dokploy/cleanup_test.go @@ -208,6 +208,9 @@ func TestPurgeSourceResourcesPreservesReplacementAfterReviewedContainerDisappear if err == nil || !strings.Contains(err.Error(), "now refers to container ID") { t.Fatalf("expected replacement container to block purge, got result=%#v err=%v", result, err) } + if len(result.Containers) != 1 || !strings.Contains(result.Containers[0].Message, "now refers to container ID") { + t.Fatalf("expected persisted resource result to include the operation error, got %#v", result.Containers) + } for _, args := range runner.outputArgs { if strings.HasPrefix(strings.Join(args, " "), "rm -f ") { t.Fatalf("replacement container was removed: %#v", runner.outputArgs) diff --git a/internal/target/dokploy/datamove.go b/internal/target/dokploy/datamove.go index c961acf..5fba3d6 100644 --- a/internal/target/dokploy/datamove.go +++ b/internal/target/dokploy/datamove.go @@ -1361,7 +1361,7 @@ func (c *Client) targetContainerForServiceWithRedeploy(ctx context.Context, runn } if allowRedeploy && len(containers) == 0 && entry.ComposeID != "" && !entry.DiscoveryRedeployAttempted { entry.DiscoveryRedeployAttempted = true - if err := c.DeployCompose(ctx, entry.ComposeID, deployComposeTitle(actx.plan)); err != nil { + if err := c.deployComposeWithPatchGuard(ctx, entry.ComposeID, deployComposeTitle(actx.plan)); err != nil { return dockerContainer{}, fmt.Errorf("redeploy dokploy compose project %q for target discovery: %w", entry.ComposeAppName, err) } } diff --git a/internal/target/dokploy/datamove_test.go b/internal/target/dokploy/datamove_test.go index bc2b0a0..a9e0cbd 100644 --- a/internal/target/dokploy/datamove_test.go +++ b/internal/target/dokploy/datamove_test.go @@ -1243,6 +1243,44 @@ func TestTargetContainerForServiceReportsComposeDeploymentError(t *testing.T) { } } +func TestTargetContainerForServiceBlocksActivePatchBeforeDiscoveryRedeploy(t *testing.T) { + deploys := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && r.URL.Path == "/api/compose.one": + _ = json.NewEncoder(w).Encode(Compose{ComposeID: "c1", AppName: "stack-1", ComposeStatus: "done"}) + case r.Method == http.MethodPost && r.URL.Path == "/api/compose.deploy": + deploys++ + w.WriteHeader(http.StatusOK) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + runner := &fakeDockerRunner{ + outputs: map[string][]byte{ + "ps -a --filter label=com.docker.compose.project=stack-1 --format {{.ID}}": []byte(""), + "ps --format {{.Names}}": []byte("dokploy-postgres\n"), + }, + runOutputs: map[string][]byte{ + "exec -i dokploy-postgres psql -U dokploy -d dokploy -v ON_ERROR_STOP=1 -At": []byte(`{"patchId":"bort-api-compose","filePath":"docker|compose.yml","composeName":"api","sourceType":"github","repository":"owner/repo","branch":"main"}` + "\n"), + }, + } + client := &Client{BaseURL: server.URL, Token: "secret", HTTPClient: server.Client(), Docker: runner} + actx := &applyContext{cache: map[string]*appCache{}} + entry := actx.entry("api") + entry.ComposeID = "c1" + entry.ComposeAppName = "stack-1" + + _, err := client.targetContainerForService(context.Background(), runner, actx, "api", "web") + if err == nil || !strings.Contains(err.Error(), "active Bort-owned Dokploy patch") { + t.Fatalf("expected active patch to block discovery redeploy, got %v", err) + } + if deploys != 0 { + t.Fatalf("active patch allowed discovery redeploy, deploys=%d", deploys) + } +} + func TestApplySyncVolumeRsyncsBindMount(t *testing.T) { app := preparer.AppPlan{Name: "api"} app.Resources.Volumes = []preparer.VolumeResource{{ From 9d18ee567b904e0825ef90dcce7c9c0c6cff790b Mon Sep 17 00:00:00 2001 From: Aikins Laryea Date: Sun, 2 Aug 2026 14:15:07 +0000 Subject: [PATCH 04/31] migration: pin reviewed bundle contents (#11) --- internal/cli/cockpit_test.go | 6 +- internal/cli/run.go | 105 ++++++++++++++--- internal/cli/run_test.go | 157 +++++++++++++++++++++++--- internal/target/dokploy/apply.go | 54 ++++++--- internal/target/dokploy/apply_test.go | 54 +++++++++ 5 files changed, 322 insertions(+), 54 deletions(-) diff --git a/internal/cli/cockpit_test.go b/internal/cli/cockpit_test.go index e17392d..b50d75d 100644 --- a/internal/cli/cockpit_test.go +++ b/internal/cli/cockpit_test.go @@ -184,7 +184,7 @@ func TestNextWizardDecisionsIncludesDownstreamReview(t *testing.T) { Items: []runDecisionItem{{ Stage: "cutover", App: "api", - Code: "cutover.review", + Code: "cutover.health_check_required", Readiness: preparer.ReadinessNeedsDecision, }}, }}, @@ -196,7 +196,7 @@ func TestNextWizardDecisionsIncludesDownstreamReview(t *testing.T) { } func TestMarkReviewDecisionDoneKeepsExcludedBlockersOpen(t *testing.T) { - reviewItem := runDecisionItem{Stage: "cutover", App: "api", Code: "cutover.review", Readiness: preparer.ReadinessNeedsDecision} + reviewItem := runDecisionItem{Stage: "cutover", App: "api", Code: "cutover.health_check_required", Readiness: preparer.ReadinessNeedsDecision} blockedItem := runDecisionItem{Stage: "cutover", App: "api", Code: "cutover.blocked", Readiness: preparer.ReadinessBlocked} run := loadedMigrationRun{ Decisions: runDecisions{ @@ -254,7 +254,7 @@ func TestCockpitShowsDownstreamDecisionsAsReviewOnly(t *testing.T) { Items: []runDecisionItem{{ Stage: "cutover", App: "api", - Code: "cutover.review", + Code: "cutover.health_check_required", Readiness: preparer.ReadinessNeedsDecision, }}, }}, diff --git a/internal/cli/run.go b/internal/cli/run.go index b6a1d27..de68838 100644 --- a/internal/cli/run.go +++ b/internal/cli/run.go @@ -3,6 +3,7 @@ package cli import ( "context" "crypto/sha256" + "encoding/hex" "encoding/json" "errors" "flag" @@ -45,6 +46,7 @@ type migrationRun struct { ApplyOutcomeRequired bool `json:"applyOutcomeRequired,omitempty"` Source string `json:"source,omitempty"` BundleDir string `json:"bundleDir"` + BundleDigest string `json:"bundleDigest,omitempty"` SourceBundleDir string `json:"sourceBundleDir,omitempty"` ManifestPath string `json:"manifest,omitempty"` Target string `json:"target"` @@ -405,15 +407,23 @@ func liveApplyBlockingDecisions(run loadedMigrationRun) []runDecision { if item.Readiness != preparer.ReadinessNeedsDecision { return false } - switch item.Stage { - case "cutover", "rollback", "commit": - return false - default: - return true - } + return !liveApplyReviewOnlyDecision(item) }) } +func liveApplyReviewOnlyDecision(item runDecisionItem) bool { + switch item.Stage { + case "cutover": + return item.Code == "cutover.sync_verification_required" || item.Code == "cutover.health_check_required" + case "rollback": + return item.Code == "rollback.trigger_required" || item.Code == "rollback.source_health_required" + case "commit": + return item.Code == "commit.target_acceptance_required" || item.Code == "commit.target_route_acceptance_required" || item.Code == "commit.rollback_window_closed" + default: + return false + } +} + func applyRunActive(runRef string) (bool, error) { runDir, err := existingRunDir(runRef) if err != nil { @@ -515,6 +525,10 @@ func applyLiveMigrationLocked(ctx context.Context, run loadedMigrationRun, stder return fmt.Errorf("lock live migration run: %w", err) } defer lock.Release() + bundleFiles, err := captureReviewedMigrationBundle(run) + if err != nil { + return err + } if err := validateLiveApplyReady(run); err != nil { return err } @@ -522,6 +536,7 @@ func applyLiveMigrationLocked(ctx context.Context, run loadedMigrationRun, stder if len(plan.Steps) == 0 { return fmt.Errorf("live migration plan has no executable steps") } + plan.BundleFiles = bundleFiles client, err := ensureDokployClient(ctx, run.Run.Target, os.Stdin, stderr, stderr) if err != nil { if err == errDokploySetupSkipped { @@ -583,6 +598,10 @@ func ensureSelfContainedLiveRunLocked(run loadedMigrationRun) (loadedMigrationRu if err != nil { return loadedMigrationRun{}, fmt.Errorf("snapshot legacy run bundle: %w", err) } + bundleDigest, err := digestMigrationBundle(bundleDir) + if err != nil { + return loadedMigrationRun{}, fmt.Errorf("digest legacy run bundle: %w", err) + } removeBundle := true defer func() { if removeBundle { @@ -605,6 +624,7 @@ func ensureSelfContainedLiveRunLocked(run loadedMigrationRun) (loadedMigrationRu run.Run.SourceBundleDir = run.Run.BundleDir } run.Run.BundleDir = bundleDir + run.Run.BundleDigest = hex.EncodeToString(bundleDigest[:]) run.Run.ApplyOutcomeRequired = true run.Run.UpdatedAt = now run.Run.Artifacts = artifacts @@ -1134,7 +1154,20 @@ func validateMigrationBundleSnapshot(sourceDir, snapshotDir string, sourceDigest } func digestMigrationBundle(root string) ([sha256.Size]byte, error) { + digest, _, err := readMigrationBundle(root, false) + return digest, err +} + +func captureMigrationBundle(root string) ([sha256.Size]byte, map[string][]byte, error) { + return readMigrationBundle(root, true) +} + +func readMigrationBundle(root string, capture bool) ([sha256.Size]byte, map[string][]byte, error) { digest := sha256.New() + var files map[string][]byte + if capture { + files = map[string][]byte{} + } err := filepath.WalkDir(root, func(path string, entry os.DirEntry, walkErr error) error { if walkErr != nil { return walkErr @@ -1169,6 +1202,9 @@ func digestMigrationBundle(root string) ([sha256.Size]byte, error) { if err != nil { return err } + if capture { + files[filepath.Clean(path)] = contents + } if err := writeMigrationBundleDigestField(digest, []byte("file")); err != nil { return err } @@ -1176,10 +1212,43 @@ func digestMigrationBundle(root string) ([sha256.Size]byte, error) { }) var result [sha256.Size]byte if err != nil { - return result, err + return result, nil, err } copy(result[:], digest.Sum(nil)) - return result, nil + return result, files, nil +} + +func verifyReviewedMigrationBundle(run loadedMigrationRun) error { + _, err := captureReviewedMigrationBundle(run) + return err +} + +func captureReviewedMigrationBundle(run loadedMigrationRun) (map[string][]byte, error) { + recovery := reviewedMigrationBundleRecovery(run) + var expected [sha256.Size]byte + if strings.TrimSpace(run.Run.BundleDigest) == "" { + return nil, fmt.Errorf("run %q predates reviewed bundle digests; %s", run.Run.Name, recovery) + } + expectedBytes, err := hex.DecodeString(run.Run.BundleDigest) + if err != nil || len(expectedBytes) != sha256.Size { + return nil, fmt.Errorf("reviewed migration bundle digest for run %q is invalid; %s", run.Run.Name, recovery) + } + copy(expected[:], expectedBytes) + actual, files, err := captureMigrationBundle(run.Run.BundleDir) + if err != nil { + return nil, fmt.Errorf("capture reviewed migration bundle for run %q: %w; %s", run.Run.Name, err, recovery) + } + if actual != expected { + return nil, fmt.Errorf("reviewed migration bundle for run %q changed after planning; %s", run.Run.Name, recovery) + } + return files, nil +} + +func reviewedMigrationBundleRecovery(run loadedMigrationRun) string { + if len(run.Applied.Steps) > 0 || run.Run.LiveAppliedAt != nil { + return "this run has started live execution and cannot be re-planned; reconcile any applied target changes, then create a new run" + } + return fmt.Sprintf("run `%s` to re-plan before live apply", runScopedCommand(run, "migrate")) } func writeMigrationBundleDigestField(w io.Writer, value []byte) error { @@ -1247,6 +1316,10 @@ func createMigrationRunLocked(opts migrationRunOptions, runDir, runName string, if _, err := applyStateEnvToBundle(state, bundleDir); err != nil { return loadedMigrationRun{}, err } + bundleDigest, err := digestMigrationBundle(bundleDir) + if err != nil { + return loadedMigrationRun{}, err + } preparePlan, err := preparer.Plan(preparer.Options{BundleDir: bundleDir, Target: opts.Target, AppName: opts.AppName}) if err != nil { @@ -1289,6 +1362,7 @@ func createMigrationRunLocked(opts migrationRunOptions, runDir, runName string, ApplyOutcomeRequired: true, Source: opts.Source, BundleDir: bundleDir, + BundleDigest: hex.EncodeToString(bundleDigest[:]), SourceBundleDir: sourceBundleDir, ManifestPath: opts.ManifestPath, Target: opts.Target, @@ -1681,21 +1755,16 @@ func openSetupDecisions(run loadedMigrationRun) []runDecision { func openReviewDecisions(run loadedMigrationRun) []runDecision { return openFilteredDecisions(run, func(item runDecisionItem) bool { - if item.Readiness != preparer.ReadinessNeedsDecision { - return false - } - switch item.Stage { - case "cutover", "rollback", "commit": - return true - default: - return false - } + return item.Readiness == preparer.ReadinessNeedsDecision && liveApplyReviewOnlyDecision(item) }) } func openDownstreamBlockingDecisions(run loadedMigrationRun) []runDecision { return openFilteredDecisions(run, func(item runDecisionItem) bool { - return item.Stage != "prepare" && (item.Readiness == preparer.ReadinessBlocked || item.Readiness == preparer.ReadinessNeedsInput) + if item.Stage == "prepare" { + return false + } + return item.Readiness == preparer.ReadinessBlocked || item.Readiness == preparer.ReadinessNeedsInput || (item.Readiness == preparer.ReadinessNeedsDecision && !liveApplyReviewOnlyDecision(item)) }) } diff --git a/internal/cli/run_test.go b/internal/cli/run_test.go index 32cf3c4..261b08a 100644 --- a/internal/cli/run_test.go +++ b/internal/cli/run_test.go @@ -3,6 +3,7 @@ package cli import ( "bytes" "context" + "encoding/hex" "errors" "io" "net/http" @@ -55,6 +56,9 @@ func TestRunMigrateCreatesLocalRunArtifactsAndSummary(t *testing.T) { if err := containedPath(runDir, run.BundleDir); err != nil { t.Fatalf("expected a self-contained reviewed bundle: %v", err) } + if run.BundleDigest == "" { + t.Fatal("expected the reviewed bundle digest to be recorded") + } prepareResult := readJSONFile[preparer.Result](t, filepath.Join(runDir, "prepare.json")) syncResult := readJSONFile[syncplan.Result](t, filepath.Join(runDir, "sync.json")) cutoverResult := readJSONFile[gateway.Result](t, filepath.Join(runDir, "cutover.json")) @@ -77,6 +81,9 @@ func TestRunMigrateCreatesLocalRunArtifactsAndSummary(t *testing.T) { if err := validateLiveApplyReady(loaded); err != nil { t.Fatalf("expected downstream review decision to remain visible without blocking live apply: %v", err) } + if err := verifyReviewedMigrationBundle(loaded); err != nil { + t.Fatalf("expected the recorded bundle digest to match: %v", err) + } output := stdout.String() for _, want := range []string{ @@ -200,6 +207,101 @@ func TestRunMigrateLivePreservesReviewedArtifactsAfterBundleChanges(t *testing.T } } +func TestApplyLiveMigrationRejectsChangedReviewedBundle(t *testing.T) { + for _, test := range []struct { + name string + partialApply bool + guidance string + }{ + {name: "before apply", guidance: "to re-plan before live apply"}, + {name: "partial apply resume", partialApply: true, guidance: "cannot be re-planned; reconcile any applied target changes, then create a new run"}, + } { + t.Run(test.name, func(t *testing.T) { + requests := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + requests++ + http.Error(w, "unexpected request", http.StatusInternalServerError) + })) + defer server.Close() + t.Setenv(dokploy.EnvBaseURL, server.URL) + t.Setenv(dokploy.EnvToken, "token") + + workDir := t.TempDir() + t.Chdir(workDir) + bundleDir := filepath.Join(workDir, "bort-bundle") + writeTestBundle(t, bundleDir, manifest.Manifest{ + Source: manifest.Source{Platform: "docker"}, + Apps: []manifest.App{{Name: "api", Services: []manifest.Service{{Name: "api", Image: "example/api:v1"}}}}, + }) + runCommand(t, runMigrate, []string{"--bundle", bundleDir, "--run", "changed-bundle"}) + run, err := loadMigrationRun("changed-bundle") + if err != nil { + t.Fatal(err) + } + if test.partialApply { + steps := dokploy.PlanFromArtifacts(run.Prepare, run.Sync, run.Cutover).Steps + if len(steps) == 0 { + t.Fatal("expected live apply steps") + } + applied := newRunApplied(run.Run) + applied.Steps = []appliedStep{{Index: 0, Kind: string(steps[0].Kind), App: steps[0].App, Ref: steps[0].Ref, Status: string(dokploy.StepStatusOK)}} + if err := writeRunApplied(runArtifactPath(run.Run.RunDir, run.Run.Artifacts.Applied), applied); err != nil { + t.Fatal(err) + } + run, err = loadMigrationRun("changed-bundle") + if err != nil { + t.Fatal(err) + } + } + composePath := filepath.Join(run.Run.BundleDir, filepath.FromSlash(run.Prepare.Apps[0].Directory), run.Prepare.Apps[0].Resources.App.ComposePath) + compose, err := os.ReadFile(composePath) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(composePath, append(compose, []byte("\nchanged: true\n")...), 0o600); err != nil { + t.Fatal(err) + } + + err = applyLiveMigrationLocked(context.Background(), run, io.Discard, nil) + if err == nil || !strings.Contains(err.Error(), "changed after planning") || !strings.Contains(err.Error(), test.guidance) { + t.Fatalf("expected changed reviewed bundle to block live apply with valid recovery guidance, got %v", err) + } + if requests != 0 { + t.Fatalf("changed reviewed bundle contacted dokploy %d time(s)", requests) + } + }) + } +} + +func TestLegacySelfContainedRunWithoutBundleDigestRemainsLoadable(t *testing.T) { + workDir := t.TempDir() + t.Chdir(workDir) + bundleDir := filepath.Join(workDir, "bort-bundle") + writeTestBundle(t, bundleDir, manifest.Manifest{ + Source: manifest.Source{Platform: "docker"}, + Apps: []manifest.App{{Name: "api", Services: []manifest.Service{{Name: "api", Image: "example/api:v1"}}}}, + }) + runCommand(t, runMigrate, []string{"--bundle", bundleDir, "--run", "legacy-contained"}) + run, err := loadMigrationRun("legacy-contained") + if err != nil { + t.Fatal(err) + } + run.Run.BundleDigest = "" + if err := writeJSONArtifact(filepath.Join(run.Run.RunDir, "run.json"), run.Run); err != nil { + t.Fatal(err) + } + loaded, err := loadMigrationRun("legacy-contained") + if err != nil { + t.Fatalf("legacy run without a bundle digest did not load: %v", err) + } + if loaded.Run.BundleDigest != "" { + t.Fatalf("legacy run unexpectedly acquired a digest while loading: %q", loaded.Run.BundleDigest) + } + if err := verifyReviewedMigrationBundle(loaded); err == nil || !strings.Contains(err.Error(), "predates reviewed bundle digests") || !strings.Contains(err.Error(), "to re-plan before live apply") { + t.Fatalf("legacy run without a bundle digest did not require a safe re-plan before live apply: %v", err) + } +} + func TestLoadMigrationRunDoesNotApplyLaterWorkspaceState(t *testing.T) { workDir := t.TempDir() t.Chdir(workDir) @@ -261,26 +363,39 @@ func TestValidateLiveApplyReadyBlocksEveryPrepareRequirement(t *testing.T) { t.Fatalf("expected prepare-stage needs_decision to block live apply with run-scoped review guidance, got %v", err) } - for _, stage := range []string{"cutover", "rollback", "commit"} { - run.Decisions.Decisions[0].Items[0].Stage = stage + reviewOnlyItems := []runDecisionItem{ + {Stage: "cutover", Code: "cutover.sync_verification_required"}, + {Stage: "cutover", Code: "cutover.health_check_required"}, + {Stage: "rollback", Code: "rollback.trigger_required"}, + {Stage: "rollback", Code: "rollback.source_health_required"}, + {Stage: "commit", Code: "commit.target_acceptance_required"}, + {Stage: "commit", Code: "commit.target_route_acceptance_required"}, + {Stage: "commit", Code: "commit.rollback_window_closed"}, + } + for _, item := range reviewOnlyItems { + item.Readiness = preparer.ReadinessNeedsDecision + run.Decisions.Decisions[0].Items[0] = item if err := validateLiveApplyReady(run); err != nil { - t.Fatalf("expected %s-stage review decision not to block live apply, got %v", stage, err) + t.Fatalf("expected %s not to block live apply, got %v", item.Code, err) } } - for _, kind := range []string{"data_stores", "database_review"} { - run.Decisions.Decisions[0].Kind = kind - if err := validateLiveApplyReady(run); err != nil { - t.Fatalf("expected downstream %s review decision not to block live apply, got %v", kind, err) + for _, stage := range []string{"cutover", "rollback", "commit"} { + run.Decisions.Decisions[0].Items[0] = runDecisionItem{Stage: stage, Code: stage + ".future_requirement", Readiness: preparer.ReadinessNeedsDecision} + if err := validateLiveApplyReady(run); err == nil { + t.Fatalf("expected unknown %s-stage needs_decision to block live apply", stage) + } + if decisions := openDownstreamBlockingDecisions(run); len(decisions) != 1 { + t.Fatalf("expected unknown %s-stage needs_decision to be surfaced as a downstream blocker, got %#v", stage, decisions) } } - run.Decisions.Decisions[0].Items[0].Stage = "" + run.Decisions.Decisions[0].Items[0] = runDecisionItem{Code: "cutover.sync_verification_required", Readiness: preparer.ReadinessNeedsDecision} if err := validateLiveApplyReady(run); err == nil { t.Fatal("expected needs_decision with an unknown stage to block live apply") } if decisions := openReviewDecisions(run); len(decisions) != 0 { t.Fatalf("expected needs_decision with an unknown stage to stay out of review-only decisions, got %#v", decisions) } - run.Decisions.Decisions[0].Items[0].Stage = "cutover" + run.Decisions.Decisions[0].Items[0] = reviewOnlyItems[0] for _, readiness := range []preparer.Readiness{preparer.ReadinessBlocked, preparer.ReadinessNeedsInput} { run.Decisions.Decisions[0].Readiness = readiness run.Decisions.Decisions[0].Items[0].Readiness = readiness @@ -315,14 +430,19 @@ func TestApplyLiveMigrationRejectsEmptyPlan(t *testing.T) { if err := os.Mkdir(bundleDir, 0o700); err != nil { t.Fatal(err) } + bundleDigest, err := digestMigrationBundle(bundleDir) + if err != nil { + t.Fatal(err) + } run := loadedMigrationRun{ Run: migrationRun{ - Name: "empty", - RunDir: runDir, - BundleDir: bundleDir, - Target: "dokploy", - DryRun: true, - Artifacts: defaultRunArtifacts(), + Name: "empty", + RunDir: runDir, + BundleDir: bundleDir, + BundleDigest: hex.EncodeToString(bundleDigest[:]), + Target: "dokploy", + DryRun: true, + Artifacts: defaultRunArtifacts(), }, Prepare: preparer.Result{BundleDir: bundleDir}, } @@ -1007,6 +1127,7 @@ func TestLegacyRunBundleIsSnapshottedBeforeLiveResume(t *testing.T) { legacy.Run.ApplyOutcomeRequired = false legacy.Run.SourceBundleDir = "" legacy.Run.BundleDir = bundleDir + legacy.Run.BundleDigest = "" legacy.Prepare.BundleDir = bundleDir legacy.Sync.BundleDir = bundleDir legacy.Cutover.BundleDir = bundleDir @@ -1060,6 +1181,12 @@ func TestLegacyRunBundleIsSnapshottedBeforeLiveResume(t *testing.T) { if !upgraded.Run.ApplyOutcomeRequired || upgraded.Run.SourceBundleDir != bundleDir || upgraded.Run.BundleDir == bundleDir { t.Fatalf("unexpected upgraded legacy metadata: %#v", upgraded.Run) } + if upgraded.Run.BundleDigest == "" { + t.Fatal("expected upgraded legacy run to record its reviewed bundle digest") + } + if err := verifyReviewedMigrationBundle(upgraded); err != nil { + t.Fatalf("expected upgraded legacy bundle digest to match: %v", err) + } if err := containedPath(upgraded.Run.RunDir, upgraded.Run.BundleDir); err != nil { t.Fatalf("legacy run bundle was not snapshotted into the run: %v", err) } diff --git a/internal/target/dokploy/apply.go b/internal/target/dokploy/apply.go index 3bacf19..f70c284 100644 --- a/internal/target/dokploy/apply.go +++ b/internal/target/dokploy/apply.go @@ -74,15 +74,16 @@ type StepProgress struct { } type Plan struct { - Steps []Step - Prepare preparer.Result - Sync syncplan.Result - Cutover gateway.Result - RunName string - RunDir string - ResumeFrom int - BeforeStep *func(StepProgress) error - OnProgress *func(StepProgress) + Steps []Step + Prepare preparer.Result + Sync syncplan.Result + Cutover gateway.Result + BundleFiles map[string][]byte + RunName string + RunDir string + ResumeFrom int + BeforeStep *func(StepProgress) error + OnProgress *func(StepProgress) } func PlanFromArtifacts(prepare preparer.Result, sync syncplan.Result, cutover gateway.Result) Plan { @@ -1268,11 +1269,13 @@ func readComposeFile(plan Plan, appName string) (string, error) { if err := safepath.ContainedPath(appDir, full); err != nil { return "", err } - contents, err := safepath.ReadFileNoFollow(full) + contents, err := readPlanBundleFile(plan, full) if err != nil { return "", fmt.Errorf("read compose file %s: %w", full, err) } - compose, err := inlineComposeEnvFilesWithFallbacks(string(contents), appDir, composeEnvFileFallbackPaths(app.TargetResources.Dokploy.EnvFiles)) + compose, err := inlineComposeEnvFilesWithFallbacksReader(string(contents), appDir, composeEnvFileFallbackPaths(app.TargetResources.Dokploy.EnvFiles), func(path string) ([]byte, error) { + return readPlanBundleFile(plan, path) + }) if err != nil { return "", fmt.Errorf("prepare compose file %s for dokploy: %w", full, err) } @@ -1331,6 +1334,10 @@ func inlineComposeEnvFiles(contents, appDir string) (string, error) { } func inlineComposeEnvFilesWithFallbacks(contents, appDir string, fallbackPaths []string) (string, error) { + return inlineComposeEnvFilesWithFallbacksReader(contents, appDir, fallbackPaths, safepath.ReadFileNoFollow) +} + +func inlineComposeEnvFilesWithFallbacksReader(contents, appDir string, fallbackPaths []string, readFile func(string) ([]byte, error)) (string, error) { var doc yaml.Node if err := yaml.Unmarshal([]byte(contents), &doc); err != nil { return "", err @@ -1373,7 +1380,7 @@ func inlineComposeEnvFilesWithFallbacks(contents, appDir string, fallbackPaths [ } inlinePaths = append(inlinePaths, path) } - envValues, err := readComposeEnvFilePathListValues(appDir, inlinePaths) + envValues, err := readComposeEnvFilePathListValues(appDir, inlinePaths, readFile) if err != nil { return "", err } @@ -1393,7 +1400,7 @@ func inlineComposeEnvFilesWithFallbacks(contents, appDir string, fallbackPaths [ changed = true } for _, path := range fallbacksByService[serviceName] { - envValues, err := readComposeEnvFilePathValues(appDir, path) + envValues, err := readComposeEnvFilePathValues(appDir, path, readFile) if err != nil { return "", err } @@ -1727,10 +1734,10 @@ func ensureServiceNetworkAlias(network *yaml.Node, alias string) bool { return true } -func readComposeEnvFilePathListValues(appDir string, paths []string) (map[string]string, error) { +func readComposeEnvFilePathListValues(appDir string, paths []string, readFile func(string) ([]byte, error)) (map[string]string, error) { values := map[string]string{} for _, envPath := range paths { - envValues, err := readComposeEnvFilePathValues(appDir, envPath) + envValues, err := readComposeEnvFilePathValues(appDir, envPath, readFile) if err != nil { if os.IsNotExist(err) { continue @@ -1775,12 +1782,12 @@ func normalizeComposeEnvFilePath(path string) string { return path } -func readComposeEnvFilePathValues(appDir, envPath string) (map[string]string, error) { +func readComposeEnvFilePathValues(appDir, envPath string, readFile func(string) ([]byte, error)) (map[string]string, error) { full := filepath.Join(appDir, filepath.FromSlash(envPath)) if err := safepath.ContainedPath(appDir, full); err != nil { return nil, err } - contents, err := safepath.ReadFileNoFollow(full) + contents, err := readFile(full) if err != nil { if os.IsNotExist(err) { return nil, err @@ -2014,7 +2021,7 @@ func readEnvContent(plan Plan, appName string) (string, error) { if err := safepath.ContainedPath(appDir, full); err != nil { return "", err } - contents, err := safepath.ReadFileNoFollow(full) + contents, err := readPlanBundleFile(plan, full) if err != nil { return "", fmt.Errorf("read env file %s: %w", full, err) } @@ -2046,6 +2053,17 @@ func readEnvContent(plan Plan, appName string) (string, error) { return b.String(), nil } +func readPlanBundleFile(plan Plan, path string) ([]byte, error) { + if plan.BundleFiles == nil { + return safepath.ReadFileNoFollow(path) + } + contents, ok := plan.BundleFiles[filepath.Clean(path)] + if !ok { + return nil, &os.PathError{Op: "open", Path: path, Err: os.ErrNotExist} + } + return contents, nil +} + func isSharedDokployEnvFile(path string) bool { switch normalizeComposeEnvFilePath(path) { case ".env", ".env.example": diff --git a/internal/target/dokploy/apply_test.go b/internal/target/dokploy/apply_test.go index d5af979..4e03653 100644 --- a/internal/target/dokploy/apply_test.go +++ b/internal/target/dokploy/apply_test.go @@ -45,6 +45,60 @@ func TestNewClientFromEnvRejectsRemoteHTTP(t *testing.T) { } } +func TestPlanBundleFilesPinComposeAndEnvContents(t *testing.T) { + bundleDir := t.TempDir() + appDir := filepath.Join(bundleDir, "api") + if err := os.MkdirAll(appDir, 0o700); err != nil { + t.Fatal(err) + } + composePath := filepath.Join(appDir, "compose.yaml") + envPath := filepath.Join(appDir, ".env") + reviewedCompose := []byte("services:\n api:\n image: example/api:v1\n") + reviewedEnv := []byte("TOKEN=reviewed\n") + if err := os.WriteFile(composePath, reviewedCompose, 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(envPath, reviewedEnv, 0o600); err != nil { + t.Fatal(err) + } + app := preparer.AppPlan{ + Name: "api", + Directory: "api", + TargetResources: &preparer.TargetResources{Dokploy: &preparer.DokployResources{ + ComposeApp: preparer.DokployComposeApp{ComposePath: "compose.yaml"}, + EnvFiles: []preparer.DokployEnvFile{{Path: ".env"}}, + }}, + } + plan := Plan{ + Prepare: preparer.Result{BundleDir: bundleDir, Apps: []preparer.AppPlan{app}}, + BundleFiles: map[string][]byte{ + filepath.Clean(composePath): reviewedCompose, + filepath.Clean(envPath): reviewedEnv, + }, + } + if err := os.WriteFile(composePath, []byte("services:\n api:\n image: example/api:v2\n"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(envPath, []byte("TOKEN=changed\n"), 0o600); err != nil { + t.Fatal(err) + } + + compose, err := readComposeFile(plan, "api") + if err != nil { + t.Fatal(err) + } + if !strings.Contains(compose, "example/api:v1") || strings.Contains(compose, "example/api:v2") { + t.Fatalf("compose did not use captured reviewed contents:\n%s", compose) + } + env, err := readEnvContent(plan, "api") + if err != nil { + t.Fatal(err) + } + if env != "TOKEN=reviewed\n" { + t.Fatalf("env did not use captured reviewed contents: %q", env) + } +} + func TestApplyDumpDataStoreNoopsForVolumeStrategyKinds(t *testing.T) { // mysql has no logical-dump implementation yet, so it migrates via // stopped-volume copy. the dump step in the plan must be a noop, not From d9b235beb6f5fab2d6f0223016c14e73855477a9 Mon Sep 17 00:00:00 2001 From: Aikins Laryea Date: Sun, 2 Aug 2026 14:51:03 +0000 Subject: [PATCH 05/31] migration: harden live apply completion (#12) --- internal/cli/commit.go | 1 + internal/cli/run.go | 31 ++++++++++-- internal/cli/run_test.go | 29 ++++++++++- internal/target/dokploy/apply.go | 71 +++++++++++++++++++++------ internal/target/dokploy/apply_test.go | 60 ++++++++++++++++++++++ 5 files changed, 173 insertions(+), 19 deletions(-) diff --git a/internal/cli/commit.go b/internal/cli/commit.go index ead601f..3ab94c6 100644 --- a/internal/cli/commit.go +++ b/internal/cli/commit.go @@ -190,6 +190,7 @@ func applyCommitFromArgs(ctx context.Context, runRef string, stderr io.Writer) e plan := dokploy.PlanForCommit(run.Prepare, run.Cutover) plan.RunName = run.Run.Name plan.RunDir = run.Run.RunDir + plan.ApprovedPrepareDecisions = approvedPrepareDecisions(run) fmt.Fprintf(stderr, "commit apply: run %s; planned %d step(s) to retire source\n", run.Run.Name, len(plan.Steps)) if err := client.Apply(ctx, plan); err != nil { return err diff --git a/internal/cli/run.go b/internal/cli/run.go index de68838..0d6a0fa 100644 --- a/internal/cli/run.go +++ b/internal/cli/run.go @@ -537,6 +537,7 @@ func applyLiveMigrationLocked(ctx context.Context, run loadedMigrationRun, stder return fmt.Errorf("live migration plan has no executable steps") } plan.BundleFiles = bundleFiles + plan.ApprovedPrepareDecisions = approvedPrepareDecisions(run) client, err := ensureDokployClient(ctx, run.Run.Target, os.Stdin, stderr, stderr) if err != nil { if err == errDokploySetupSkipped { @@ -586,6 +587,27 @@ func applyLiveMigrationLocked(ctx context.Context, run loadedMigrationRun, stder return markRunLiveAppliedLocked(run.Run) } +func approvedPrepareDecisions(run loadedMigrationRun) map[dokploy.PrepareDecision]struct{} { + approved := map[dokploy.PrepareDecision]struct{}{} + for _, decision := range run.Decisions.Decisions { + progress, ok := run.Progress.Decisions[decision.Kind] + if !ok { + continue + } + for _, item := range decision.Items { + if item.Stage != "prepare" || item.Readiness != preparer.ReadinessNeedsDecision || strings.TrimSpace(item.Code) == "" { + continue + } + itemProgress, ok := progress.Items[progressItemKey(item)] + if ok && (itemProgress.Status == progressStatusResolved || itemProgress.Status == progressStatusSkipped) { + decision := dokploy.NewPrepareDecision(item.App, item.Code, item.ResourceRef, item.Readiness, item.Message) + approved[decision] = struct{}{} + } + } + } + return approved +} + func ensureSelfContainedLiveRunLocked(run loadedMigrationRun) (loadedMigrationRun, error) { if err := containedPath(run.Run.RunDir, run.Prepare.BundleDir); err == nil { return run, nil @@ -736,7 +758,7 @@ func attachLiveMigration(ctx context.Context, run loadedMigrationRun, appliedPat startedAt := time.Now().UTC() var lastSeen time.Time initialApplied, initialAppliedOK := readRunApplied(appliedPath, run.Run) - if initialAppliedOK == nil && completedApplyPrefix(plan.Steps, initialApplied) >= len(plan.Steps) && len(plan.Steps) > 0 { + if initialAppliedOK == nil && initialApplied.SucceededAt != nil && completedApplyPrefix(plan.Steps, initialApplied) >= len(plan.Steps) && len(plan.Steps) > 0 { entries := attachProgressEntries(plan.Steps, initialApplied, time.Time{}) emitAttachProgress(onProgress, entries) if onProgress == nil { @@ -783,7 +805,7 @@ func attachLiveMigration(ctx context.Context, run loadedMigrationRun, appliedPat writeAttachTextProgress(stderr, latest.progress, len(plan.Steps), "recorded") } } - if completedApplyPrefix(plan.Steps, applied) >= len(plan.Steps) { + if applied.SucceededAt != nil && completedApplyPrefix(plan.Steps, applied) >= len(plan.Steps) { return nil } active, err := applyLockActive(lockPath) @@ -862,7 +884,10 @@ func writeAttachTextProgress(stderr io.Writer, progress dokploy.StepProgress, to func attachExitResult(run loadedMigrationRun, steps []dokploy.Step, applied runApplied, attachedAt time.Time) error { completed := completedApplyPrefix(steps, applied) if completed >= len(steps) { - return nil + if applied.SucceededAt != nil { + return nil + } + return fmt.Errorf("live migration recorded all %d step(s) but no successful live-apply outcome; rerun `%s` to continue", len(steps), liveApplyCommand(run)) } if failed, ok := latestAttachFailure(steps, applied, attachedAt); ok { return fmt.Errorf("live migration exited after %d/%d recorded step(s); latest failure: %s %s/%s: %s", completed, len(steps), failed.Kind, failed.App, failed.Ref, failed.Error) diff --git a/internal/cli/run_test.go b/internal/cli/run_test.go index 261b08a..8f0e414 100644 --- a/internal/cli/run_test.go +++ b/internal/cli/run_test.go @@ -409,6 +409,29 @@ func TestValidateLiveApplyReadyBlocksEveryPrepareRequirement(t *testing.T) { } } +func TestApprovedPrepareDecisionsRequireResolvedProgress(t *testing.T) { + item := runDecisionItem{ + Stage: "prepare", + App: "api", + Code: "prepare.review", + Message: "review the prepared app", + Readiness: preparer.ReadinessNeedsDecision, + } + decision := runDecision{ID: "prepare-review", Kind: "prepare-review", Items: []runDecisionItem{item}} + run := loadedMigrationRun{ + Decisions: runDecisions{APIVersion: decisionsAPIVersion, Decisions: []runDecision{decision}}, + Progress: runProgress{Decisions: map[string]decisionProgress{}}, + } + if approved := approvedPrepareDecisions(run); len(approved) != 0 { + t.Fatalf("unresolved prepare decision was approved: %#v", approved) + } + run.Progress = markDecisionDone(run.Progress, decision, progressStatusResolved, "reviewed", time.Now().UTC()) + approved := dokploy.NewPrepareDecision(item.App, item.Code, item.ResourceRef, item.Readiness, item.Message) + if _, ok := approvedPrepareDecisions(run)[approved]; !ok { + t.Fatalf("resolved prepare decision code was not approved") + } +} + func TestEnsureDokployClientShellQuotesRepairURL(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { http.Error(w, "unavailable", http.StatusServiceUnavailable) @@ -834,9 +857,11 @@ func TestFinalizeAttachedLiveMigrationRecordsLifecycle(t *testing.T) { if err != nil { t.Fatal(err) } - if err := applyLiveMigrationLocked(context.Background(), run, io.Discard, nil); err == nil || !strings.Contains(err.Error(), "no successful live-apply outcome") { + attachCtx, cancelAttach := context.WithCancel(context.Background()) + cancelAttach() + if err := applyLiveMigrationLocked(attachCtx, run, io.Discard, nil); !errors.Is(err, context.Canceled) { applyLock.Release() - t.Fatalf("expected raced attach without a successful outcome to remain incomplete, got %v", err) + t.Fatalf("expected raced attach to wait for the producer's successful outcome, got %v", err) } applyLock.Release() if err := finalizeAttachedLiveMigration(context.Background(), run.Run.RunDir); err == nil || !strings.Contains(err.Error(), "no successful live-apply outcome") { diff --git a/internal/target/dokploy/apply.go b/internal/target/dokploy/apply.go index f70c284..73c4b14 100644 --- a/internal/target/dokploy/apply.go +++ b/internal/target/dokploy/apply.go @@ -73,17 +73,30 @@ type StepProgress struct { Err error } +type PrepareDecision struct { + app string + code string + resourceRef string + readiness preparer.Readiness + message string +} + +func NewPrepareDecision(app, code, resourceRef string, readiness preparer.Readiness, message string) PrepareDecision { + return PrepareDecision{app: app, code: code, resourceRef: resourceRef, readiness: readiness, message: message} +} + type Plan struct { - Steps []Step - Prepare preparer.Result - Sync syncplan.Result - Cutover gateway.Result - BundleFiles map[string][]byte - RunName string - RunDir string - ResumeFrom int - BeforeStep *func(StepProgress) error - OnProgress *func(StepProgress) + Steps []Step + Prepare preparer.Result + Sync syncplan.Result + Cutover gateway.Result + BundleFiles map[string][]byte + ApprovedPrepareDecisions map[PrepareDecision]struct{} + RunName string + RunDir string + ResumeFrom int + BeforeStep *func(StepProgress) error + OnProgress *func(StepProgress) } func PlanFromArtifacts(prepare preparer.Result, sync syncplan.Result, cutover gateway.Result) Plan { @@ -268,15 +281,24 @@ func validatePlanReadyForLiveApply(plan Plan) error { continue } switch app.Readiness { - // acknowledged decisions remain in the immutable plan; the CLI blocks - // unacknowledged decisions before it calls the target client. - case "", preparer.ReadinessReadyToCreate, preparer.ReadinessNeedsDecision: + case "", preparer.ReadinessReadyToCreate: + case preparer.ReadinessNeedsDecision: + if !appDecisionCodesApproved(plan, app) { + return fmt.Errorf("live apply blocked by unapproved prepare decisions for app %s", app.Name) + } default: return fmt.Errorf("live apply blocked by prepare readiness for app %s: %s", app.Name, app.Readiness) } for _, gate := range app.Gates { switch gate.Readiness { - case preparer.ReadinessReadyToCreate, preparer.ReadinessNeedsDecision: + case preparer.ReadinessReadyToCreate: + case preparer.ReadinessNeedsDecision: + if strings.TrimSpace(gate.Code) == "" { + return fmt.Errorf("live apply blocked by unapproved prepare gate %s/%s: %s", app.Name, gate.Code, gate.Message) + } + if _, ok := plan.ApprovedPrepareDecisions[newPrepareDecision(app.Name, gate)]; !ok { + return fmt.Errorf("live apply blocked by unapproved prepare gate %s/%s: %s", app.Name, gate.Code, gate.Message) + } default: return fmt.Errorf("live apply blocked by prepare gate %s/%s: %s", app.Name, gate.Code, gate.Message) } @@ -285,6 +307,27 @@ func validatePlanReadyForLiveApply(plan Plan) error { return nil } +func appDecisionCodesApproved(plan Plan, app preparer.AppPlan) bool { + found := false + for _, gate := range app.Gates { + if gate.Readiness != preparer.ReadinessNeedsDecision { + continue + } + found = true + if strings.TrimSpace(gate.Code) == "" { + return false + } + if _, ok := plan.ApprovedPrepareDecisions[newPrepareDecision(app.Name, gate)]; !ok { + return false + } + } + return found +} + +func newPrepareDecision(app string, gate preparer.Gate) PrepareDecision { + return NewPrepareDecision(app, gate.Code, gate.ResourceRef, gate.Readiness, gate.Message) +} + type activeBortOverridePatch struct { PatchID string `json:"patchId"` FilePath string `json:"filePath"` diff --git a/internal/target/dokploy/apply_test.go b/internal/target/dokploy/apply_test.go index 4e03653..5991fd6 100644 --- a/internal/target/dokploy/apply_test.go +++ b/internal/target/dokploy/apply_test.go @@ -99,6 +99,66 @@ func TestPlanBundleFilesPinComposeAndEnvContents(t *testing.T) { } } +func TestValidatePlanReadyForLiveApplyRequiresApprovedDecisionCodes(t *testing.T) { + app := preparer.AppPlan{ + Name: "api", + Readiness: preparer.ReadinessNeedsDecision, + Gates: []preparer.Gate{{ + Code: "prepare.review", + Message: "review the prepared app", + Readiness: preparer.ReadinessNeedsDecision, + }}, + } + plan := Plan{Prepare: preparer.Result{Apps: []preparer.AppPlan{app}}} + if err := validatePlanReadyForLiveApply(plan); err == nil || !strings.Contains(err.Error(), "unapproved prepare decisions") { + t.Fatalf("expected an unapproved needs_decision app to block live apply, got %v", err) + } + approved := NewPrepareDecision("api", "prepare.review", "", preparer.ReadinessNeedsDecision, "review the prepared app") + plan.ApprovedPrepareDecisions = map[PrepareDecision]struct{}{approved: {}} + if err := validatePlanReadyForLiveApply(plan); err != nil { + t.Fatalf("expected the explicitly approved decision code to pass validation: %v", err) + } + plan.Prepare.Apps[0].Gates[0].Code = "prepare.future_requirement" + if err := validatePlanReadyForLiveApply(plan); err == nil || !strings.Contains(err.Error(), "unapproved prepare decisions") { + t.Fatalf("expected an unknown needs_decision code to fail closed, got %v", err) + } + plan.Prepare.Apps[0].Gates[0].Code = "" + blank := NewPrepareDecision("api", "", "", preparer.ReadinessNeedsDecision, "review the prepared app") + plan.ApprovedPrepareDecisions = map[PrepareDecision]struct{}{blank: {}} + if err := validatePlanReadyForLiveApply(plan); err == nil || !strings.Contains(err.Error(), "unapproved prepare decisions") { + t.Fatalf("expected a blank needs_decision code to fail closed, got %v", err) + } +} + +func TestValidatePlanReadyForLiveApplySeparatesDecisionIdentity(t *testing.T) { + gate := preparer.Gate{ + Code: "prepare.review", + ResourceRef: "service/api", + Message: "review the prepared app", + Readiness: preparer.ReadinessNeedsDecision, + } + plan := Plan{Prepare: preparer.Result{Apps: []preparer.AppPlan{ + {Name: "api", Readiness: preparer.ReadinessNeedsDecision, Gates: []preparer.Gate{gate}}, + {Name: "worker", Readiness: preparer.ReadinessNeedsDecision, Gates: []preparer.Gate{gate}}, + }}} + apiApproval := NewPrepareDecision("api", gate.Code, gate.ResourceRef, gate.Readiness, gate.Message) + plan.ApprovedPrepareDecisions = map[PrepareDecision]struct{}{apiApproval: {}} + if err := validatePlanReadyForLiveApply(plan); err == nil || !strings.Contains(err.Error(), "app worker") { + t.Fatalf("expected api approval not to authorize worker, got %v", err) + } + + workerApproval := NewPrepareDecision("worker", gate.Code, gate.ResourceRef, gate.Readiness, gate.Message) + plan.ApprovedPrepareDecisions[workerApproval] = struct{}{} + if err := validatePlanReadyForLiveApply(plan); err != nil { + t.Fatalf("expected exact approvals for both apps to pass validation: %v", err) + } + + plan.Prepare.Apps[1].Gates[0].ResourceRef = "service/worker" + if err := validatePlanReadyForLiveApply(plan); err == nil || !strings.Contains(err.Error(), "app worker") { + t.Fatalf("expected approval for a different resource not to authorize worker, got %v", err) + } +} + func TestApplyDumpDataStoreNoopsForVolumeStrategyKinds(t *testing.T) { // mysql has no logical-dump implementation yet, so it migrates via // stopped-volume copy. the dump step in the plan must be a noop, not From 68bfc611370c26644eb92bcd15eb67d1f7d91919 Mon Sep 17 00:00:00 2001 From: Aikins Laryea Date: Sun, 2 Aug 2026 16:23:55 +0000 Subject: [PATCH 06/31] migration: harden destructive command safety (#13) * migration: harden destructive command safety * test: use private backup fixture paths --- README.md | 4 +- internal/cli/cleanup.go | 60 +++++--- internal/cli/cleanup_test.go | 184 +++++++++++++++++++++--- internal/cli/rollback.go | 3 + internal/cli/rollback_test.go | 7 + internal/safepath/safepath.go | 145 ++++++++++++++++++- internal/safepath/safepath_other.go | 20 +++ internal/safepath/safepath_unix.go | 147 +++++++++++++++++++ internal/target/dokploy/cleanup.go | 25 +++- internal/target/dokploy/cleanup_test.go | 86 ++++++++++- 10 files changed, 635 insertions(+), 46 deletions(-) diff --git a/README.md b/README.md index 3bdb938..e460016 100644 --- a/README.md +++ b/README.md @@ -20,8 +20,8 @@ Bort is a guided migration tool for people running their own app platform on a VPS. It checks what is actually running, explains what needs attention, prepares -Dokploy, copies supported data, switches web traffic, and asks before any -destructive step. +Dokploy, copies supported data, switches web traffic, and requires an explicit +command before each destructive step. Bort organizes the migration by app rather than exposing a wall of Docker details. It shows what is ready, provides copy-paste fixes, saves your answers, diff --git a/internal/cli/cleanup.go b/internal/cli/cleanup.go index 344c8ec..69d41fc 100644 --- a/internal/cli/cleanup.go +++ b/internal/cli/cleanup.go @@ -353,12 +353,12 @@ func runCleanupPurge(ctx context.Context, args []string, stdin io.Reader, stdout if err := confirmCleanupPurgeApply(stdin, stderr, run.Run.Name, confirm); err != nil { return err } - backupPath, err := writeCleanupPurgeBackup(backupDir, result) + recorder, err := newCleanupPurgeBackupRecorder(backupDir, &result) if err != nil { return err } - result.BackupPath = backupPath - recorder := cleanupPurgeBackupRecorder{path: backupPath, result: &result, persist: updateCleanupPurgeBackup} + defer recorder.Close() + backupPath := recorder.path if err := recorder.Start(); err != nil { return fmt.Errorf("record source purge execution start in private backup %s: %w", backupPath, err) } @@ -877,30 +877,49 @@ func cleanupPurgeConfirmPhrase(runName string) string { } func writeCleanupPurgeBackup(backupDir string, result cleanupPurgeResult) (string, error) { + recorder, err := newCleanupPurgeBackupRecorder(backupDir, &result) + if err != nil { + return "", err + } + defer recorder.Close() + return recorder.path, nil +} + +func newCleanupPurgeBackupRecorder(backupDir string, result *cleanupPurgeResult) (*cleanupPurgeBackupRecorder, error) { backupDir = strings.TrimSpace(backupDir) if backupDir == "" { backupDir = filepath.Join(".bort", "backups") } - if err := os.MkdirAll(backupDir, 0o700); err != nil { - return "", err + dir, err := safepath.OpenPrivateDirNoFollow(backupDir) + if err != nil { + return nil, err } - if err := os.Chmod(backupDir, 0o700); err != nil { - return "", err + name := fmt.Sprintf("source-purge-%s-%s.json", cleanupBackupFilePart(result.RunName), time.Now().UTC().Format("20060102-150405.000000000")) + path := filepath.Join(backupDir, name) + result.BackupPath = path + recorder := &cleanupPurgeBackupRecorder{path: path, name: name, dir: dir, result: result} + if err := recorder.Persist(); err != nil { + _ = recorder.Close() + return nil, err } - path := filepath.Join(backupDir, fmt.Sprintf("source-purge-%s-%s.json", cleanupBackupFilePart(result.RunName), time.Now().UTC().Format("20060102-150405.000000000"))) - return path, updateCleanupPurgeBackup(path, result) + return recorder, nil } -func updateCleanupPurgeBackup(path string, result cleanupPurgeResult) error { - result.BackupPath = path - return writeJSONArtifact(path, result) +type cleanupPurgeBackupRecorder struct { + path string + name string + dir *safepath.PrivateDir + result *cleanupPurgeResult + err error } -type cleanupPurgeBackupRecorder struct { - path string - result *cleanupPurgeResult - persist func(string, cleanupPurgeResult) error - err error +func (r *cleanupPurgeBackupRecorder) Close() error { + if r.dir == nil { + return nil + } + err := r.dir.Close() + r.dir = nil + return err } func (r *cleanupPurgeBackupRecorder) Start() error { @@ -918,7 +937,12 @@ func (r *cleanupPurgeBackupRecorder) Persist() error { if r.err != nil { return r.err } - if err := r.persist(r.path, *r.result); err != nil { + contents, err := json.MarshalIndent(*r.result, "", " ") + if err == nil { + contents = append(contents, '\n') + err = r.dir.WriteFileAtomic(r.name, contents, 0o600) + } + if err != nil { r.err = err return err } diff --git a/internal/cli/cleanup_test.go b/internal/cli/cleanup_test.go index 27c4841..9f9e02c 100644 --- a/internal/cli/cleanup_test.go +++ b/internal/cli/cleanup_test.go @@ -51,7 +51,7 @@ func TestLifecycleAcceptanceTraceWithFakeDokploy(t *testing.T) { t.Setenv(dokploy.EnvBaseURL, server.URL) t.Setenv(dokploy.EnvToken, "test-token") - workDir := t.TempDir() + workDir := cleanupBackupTestDir(t) t.Chdir(workDir) binDir := filepath.Join(workDir, "bin") if err := os.MkdirAll(binDir, 0o700); err != nil { @@ -812,22 +812,22 @@ func TestCleanupPurgeApplyCommandRejectsUnscopedReview(t *testing.T) { } func TestUpdateCleanupPurgeBackupPersistsPartialResults(t *testing.T) { - backupDir := t.TempDir() + backupDir := cleanupBackupTestDir(t) result := cleanupPurgeResult{RunName: "partial", DryRun: false, PurgeResult: &dokploy.SourcePurgeResult{ Containers: []dokploy.SourcePurgeResourceResult{{Ref: "web", Status: "removed"}}, }} - path, err := writeCleanupPurgeBackup(backupDir, cleanupPurgeResult{RunName: result.RunName, DryRun: true}) + recorder, err := newCleanupPurgeBackupRecorder(backupDir, &result) if err != nil { t.Fatal(err) } - recorder := cleanupPurgeBackupRecorder{path: path, result: &result, persist: updateCleanupPurgeBackup} + defer recorder.Close() if err := recorder.Start(); err != nil { t.Fatal(err) } if err := recorder.Record(*result.PurgeResult); err != nil { t.Fatal(err) } - contents, err := os.ReadFile(path) + contents, err := os.ReadFile(recorder.path) if err != nil { t.Fatal(err) } @@ -840,24 +840,172 @@ func TestUpdateCleanupPurgeBackupPersistsPartialResults(t *testing.T) { } } +func TestWriteCleanupPurgeBackupRejectsPermissiveDirectoryWithoutChangingMode(t *testing.T) { + backupDir := filepath.Join(cleanupBackupTestDir(t), "shared") + if err := os.Mkdir(backupDir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.Chmod(backupDir, 0o755); err != nil { + t.Fatal(err) + } + if _, err := writeCleanupPurgeBackup(backupDir, cleanupPurgeResult{RunName: "mode"}); err == nil { + t.Fatal("expected permissive backup directory to be rejected") + } + dirInfo, err := os.Stat(backupDir) + if err != nil { + t.Fatal(err) + } + if dirInfo.Mode().Perm() != 0o755 { + t.Fatalf("existing backup directory mode changed to %o", dirInfo.Mode().Perm()) + } + entries, err := os.ReadDir(backupDir) + if err != nil { + t.Fatal(err) + } + if len(entries) != 0 { + t.Fatalf("permissive backup directory was modified: %#v", entries) + } +} + +func TestWriteCleanupPurgeBackupCreatesPrivateDirectory(t *testing.T) { + backupDir := filepath.Join(cleanupBackupTestDir(t), "new", "backups") + path, err := writeCleanupPurgeBackup(backupDir, cleanupPurgeResult{RunName: "private"}) + if err != nil { + t.Fatal(err) + } + info, err := os.Stat(backupDir) + if err != nil { + t.Fatal(err) + } + if info.Mode().Perm() != 0o700 { + t.Fatalf("created backup directory mode is %o, want 700", info.Mode().Perm()) + } + fileInfo, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + if fileInfo.Mode().Perm() != 0o600 { + t.Fatalf("backup file mode is %o, want 600", fileInfo.Mode().Perm()) + } +} + +func TestWriteCleanupPurgeBackupRejectsSymlinkDirectory(t *testing.T) { + root := cleanupBackupTestDir(t) + target := filepath.Join(root, "target") + if err := os.Mkdir(target, 0o755); err != nil { + t.Fatal(err) + } + link := filepath.Join(root, "backups") + if err := os.Symlink(target, link); err != nil { + t.Skipf("create directory symlink: %v", err) + } + if _, err := writeCleanupPurgeBackup(link, cleanupPurgeResult{RunName: "linked"}); err == nil { + t.Fatal("expected symlinked backup directory to be rejected") + } + entries, err := os.ReadDir(target) + if err != nil { + t.Fatal(err) + } + if len(entries) != 0 { + t.Fatalf("symlink target was modified: %#v", entries) + } +} + +func TestWriteCleanupPurgeBackupRejectsSymlinkAncestor(t *testing.T) { + root := cleanupBackupTestDir(t) + target := filepath.Join(root, "target") + if err := os.Mkdir(target, 0o755); err != nil { + t.Fatal(err) + } + link := filepath.Join(root, "linked-parent") + if err := os.Symlink(target, link); err != nil { + t.Skipf("create directory symlink: %v", err) + } + if _, err := writeCleanupPurgeBackup(filepath.Join(link, "backups"), cleanupPurgeResult{RunName: "linked"}); err == nil { + t.Fatal("expected symlinked backup directory ancestor to be rejected") + } + if _, err := os.Stat(filepath.Join(target, "backups")); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("symlink ancestor target was modified: %v", err) + } +} + +func TestCleanupPurgeBackupRecorderRetainsDirectoryIdentity(t *testing.T) { + root := cleanupBackupTestDir(t) + backupDir := filepath.Join(root, "backups") + result := cleanupPurgeResult{RunName: "identity"} + recorder, err := newCleanupPurgeBackupRecorder(backupDir, &result) + if err != nil { + t.Fatal(err) + } + defer recorder.Close() + heldDir := filepath.Join(root, "held") + if err := os.Rename(backupDir, heldDir); err != nil { + t.Fatal(err) + } + redirected := filepath.Join(root, "redirected") + if err := os.Mkdir(redirected, 0o755); err != nil { + t.Fatal(err) + } + if err := os.Symlink(redirected, backupDir); err != nil { + t.Skipf("replace backup directory with symlink: %v", err) + } + if err := recorder.Start(); err == nil || !strings.Contains(err.Error(), "private directory path changed") { + t.Fatalf("expected replaced backup directory path to abort persistence, got %v", err) + } + if _, err := os.Stat(filepath.Join(heldDir, filepath.Base(recorder.path))); err != nil { + t.Fatalf("held backup directory was not updated: %v", err) + } + entries, err := os.ReadDir(redirected) + if err != nil { + t.Fatal(err) + } + if len(entries) != 0 { + t.Fatalf("replacement symlink target was modified: %#v", entries) + } +} + +func cleanupBackupTestDir(t *testing.T) string { + t.Helper() + cacheDir, err := os.UserCacheDir() + if err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(cacheDir, 0o700); err != nil { + t.Fatal(err) + } + cacheDir, err = filepath.EvalSymlinks(cacheDir) + if err != nil { + t.Fatal(err) + } + dir, err := os.MkdirTemp(cacheDir, "bort-cleanup-test-") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := os.RemoveAll(dir); err != nil { + t.Errorf("remove cleanup backup test directory: %v", err) + } + }) + if err := os.Chmod(dir, 0o700); err != nil { + t.Fatal(err) + } + return dir +} + func TestCleanupPurgeBackupRecorderKeepsPersistenceFailureSticky(t *testing.T) { result := cleanupPurgeResult{RunName: "partial"} - writes := 0 - recorder := cleanupPurgeBackupRecorder{ - path: filepath.Join(t.TempDir(), "purge.json"), - result: &result, - persist: func(string, cleanupPurgeResult) error { - writes++ - return errors.New("disk unavailable") - }, + recorder, err := newCleanupPurgeBackupRecorder(cleanupBackupTestDir(t), &result) + if err != nil { + t.Fatal(err) } - if err := recorder.Start(); err == nil || !strings.Contains(err.Error(), "disk unavailable") { + if err := recorder.Close(); err != nil { + t.Fatal(err) + } + if err := recorder.Start(); err == nil || !strings.Contains(err.Error(), "private directory is closed") { t.Fatalf("expected initial persistence failure, got %v", err) } - if err := recorder.Record(dokploy.SourcePurgeResult{}); err == nil || !strings.Contains(err.Error(), "disk unavailable") { + firstErr := recorder.err + if err := recorder.Record(dokploy.SourcePurgeResult{}); err == nil || err != firstErr { t.Fatalf("expected sticky persistence failure, got %v", err) } - if writes != 1 { - t.Fatalf("expected persistence not to retry after failure, got %d writes", writes) - } } diff --git a/internal/cli/rollback.go b/internal/cli/rollback.go index 05acaa7..9a4526c 100644 --- a/internal/cli/rollback.go +++ b/internal/cli/rollback.go @@ -35,6 +35,9 @@ func runRollback(_ context.Context, args []string, stdout, stderr io.Writer) err if err := fs.Parse(args); err != nil { return err } + if fs.NArg() > 0 { + return fmt.Errorf("rollback does not accept positional argument %q", fs.Arg(0)) + } if err := checkOutputFormat("rollback", format); err != nil { return err } diff --git a/internal/cli/rollback_test.go b/internal/cli/rollback_test.go index 6bd00d3..06b1465 100644 --- a/internal/cli/rollback_test.go +++ b/internal/cli/rollback_test.go @@ -92,6 +92,13 @@ func TestRunRollbackWritesJSONPlan(t *testing.T) { } } +func TestRunRollbackRejectsPositionalArguments(t *testing.T) { + err := runRollback(context.Background(), []string{"typo", "--run", "recovery-run"}, io.Discard, io.Discard) + if err == nil || !strings.Contains(err.Error(), `rollback does not accept positional argument "typo"`) { + t.Fatalf("expected positional argument rejection, got %v", err) + } +} + func TestRunRollbackDefaultsToCurrentRun(t *testing.T) { workDir := t.TempDir() t.Chdir(workDir) diff --git a/internal/safepath/safepath.go b/internal/safepath/safepath.go index b183b55..4aff6f4 100644 --- a/internal/safepath/safepath.go +++ b/internal/safepath/safepath.go @@ -1,6 +1,8 @@ package safepath import ( + "crypto/rand" + "encoding/hex" "errors" "fmt" "io" @@ -28,6 +30,143 @@ func ContainedPath(root, path string) error { return nil } +type PrivateDir struct { + path string + dir *os.File +} + +func OpenPrivateDirNoFollow(path string) (*PrivateDir, error) { + if strings.TrimSpace(path) == "" { + return nil, fmt.Errorf("private directory path is empty") + } + abs := path + if !filepath.IsAbs(path) { + cwd, err := os.Getwd() + if err != nil { + return nil, err + } + cwd, err = filepath.EvalSymlinks(cwd) + if err != nil { + return nil, err + } + abs = filepath.Join(cwd, path) + } + abs = filepath.Clean(abs) + dir, err := openPrivateDirNoFollow(abs, true) + if err != nil { + return nil, err + } + info, err := dir.Stat() + if err != nil { + _ = dir.Close() + return nil, err + } + if info.Mode().Perm()&0o077 != 0 { + _ = dir.Close() + return nil, fmt.Errorf("private directory %s has permissions %o; remove group and other access before retrying", path, info.Mode().Perm()) + } + return &PrivateDir{path: abs, dir: dir}, nil +} + +func (d *PrivateDir) Close() error { + if d == nil || d.dir == nil { + return nil + } + err := d.dir.Close() + d.dir = nil + return err +} + +func (d *PrivateDir) CreateFile(name string, mode os.FileMode) (*os.File, error) { + if err := validatePrivateFileName(name); err != nil { + return nil, err + } + dir, err := d.openFile() + if err != nil { + return nil, err + } + return createPrivateFileNoFollow(dir, name, mode) +} + +func (d *PrivateDir) WriteFileAtomic(name string, data []byte, mode os.FileMode) error { + if err := validatePrivateFileName(name); err != nil { + return err + } + dir, err := d.openFile() + if err != nil { + return err + } + if err := writePrivateFileAtomicNoFollow(dir, name, data, mode); err != nil { + return err + } + return d.ValidatePath() +} + +func (d *PrivateDir) Remove(name string) error { + if err := validatePrivateFileName(name); err != nil { + return err + } + dir, err := d.openFile() + if err != nil { + return err + } + return removePrivateFileNoFollow(dir, name) +} + +func (d *PrivateDir) Sync() error { + dir, err := d.openFile() + if err != nil { + return err + } + return syncPrivateDir(dir) +} + +func (d *PrivateDir) ValidatePath() error { + dir, err := d.openFile() + if err != nil { + return err + } + current, err := openPrivateDirNoFollow(d.path, false) + if err != nil { + return fmt.Errorf("private directory path changed during operation: %w", err) + } + defer current.Close() + heldInfo, err := dir.Stat() + if err != nil { + return err + } + currentInfo, err := current.Stat() + if err != nil { + return err + } + if !os.SameFile(heldInfo, currentInfo) { + return fmt.Errorf("private directory path changed during operation") + } + return nil +} + +func (d *PrivateDir) openFile() (*os.File, error) { + if d == nil || d.dir == nil { + return nil, fmt.Errorf("private directory is closed") + } + return d.dir, nil +} + +func validatePrivateFileName(name string) error { + if strings.TrimSpace(name) == "" || name == "." || name == ".." || filepath.Base(name) != name { + return fmt.Errorf("private file name %q must be a base name", name) + } + return nil +} + +func privateTempName() (string, error) { + var suffix [16]byte + if _, err := rand.Read(suffix[:]); err != nil { + return "", err + } + return ".bort-tmp-" + hex.EncodeToString(suffix[:]), nil +} + func ReadFileNoFollow(path string) ([]byte, error) { f, err := openNoFollowRead(path) if err != nil { @@ -46,10 +185,14 @@ func WriteFileNoFollow(path string, data []byte, mode os.FileMode) error { f.Close() return err } + if err := f.Chmod(mode); err != nil { + f.Close() + return err + } if err := f.Close(); err != nil { return err } - return os.Chmod(path, mode) + return nil } func WriteFileAtomicNoFollow(path string, data []byte, mode os.FileMode) error { diff --git a/internal/safepath/safepath_other.go b/internal/safepath/safepath_other.go index c16cd6c..4af5f92 100644 --- a/internal/safepath/safepath_other.go +++ b/internal/safepath/safepath_other.go @@ -7,6 +7,26 @@ import ( "os" ) +func openPrivateDirNoFollow(path string, _ bool) (*os.File, error) { + return nil, fmt.Errorf("secure private directory access for %s is unavailable on Windows", path) +} + +func createPrivateFileNoFollow(*os.File, string, os.FileMode) (*os.File, error) { + return nil, fmt.Errorf("secure private file creation is unavailable on Windows") +} + +func writePrivateFileAtomicNoFollow(*os.File, string, []byte, os.FileMode) error { + return fmt.Errorf("secure private file replacement is unavailable on Windows") +} + +func removePrivateFileNoFollow(*os.File, string) error { + return fmt.Errorf("secure private file removal is unavailable on Windows") +} + +func syncPrivateDir(*os.File) error { + return fmt.Errorf("secure private directory sync is unavailable on Windows") +} + func openNoFollowRead(path string) (*os.File, error) { if err := rejectFinalSymlink(path); err != nil { return nil, err diff --git a/internal/safepath/safepath_unix.go b/internal/safepath/safepath_unix.go index 84cd65e..6b95cba 100644 --- a/internal/safepath/safepath_unix.go +++ b/internal/safepath/safepath_unix.go @@ -3,11 +3,158 @@ package safepath import ( + "errors" + "fmt" "os" "path/filepath" + "strconv" + "strings" "syscall" + + "golang.org/x/sys/unix" ) +func openPrivateDirNoFollow(path string, create bool) (*os.File, error) { + parts := strings.Split(strings.TrimPrefix(path, string(filepath.Separator)), string(filepath.Separator)) + parentFD, err := unix.Open(string(filepath.Separator), unix.O_RDONLY|unix.O_DIRECTORY|unix.O_CLOEXEC|unix.O_NOFOLLOW, 0) + if err != nil { + return nil, err + } + if err := validatePrivateDirDescriptor(parentFD, string(filepath.Separator)); err != nil { + _ = unix.Close(parentFD) + return nil, err + } + currentPath := string(filepath.Separator) + for _, part := range parts { + if part == "" { + continue + } + componentPath := filepath.Join(currentPath, part) + nextFD, err := unix.Openat(parentFD, part, unix.O_RDONLY|unix.O_DIRECTORY|unix.O_CLOEXEC|unix.O_NOFOLLOW, 0) + if create && errors.Is(err, unix.ENOENT) { + mkdirErr := unix.Mkdirat(parentFD, part, 0o700) + if mkdirErr != nil && !errors.Is(mkdirErr, unix.EEXIST) { + _ = unix.Close(parentFD) + return nil, fmt.Errorf("create private directory component %s: %w", componentPath, mkdirErr) + } + if mkdirErr == nil { + if syncErr := unix.Fsync(parentFD); syncErr != nil { + _ = unix.Close(parentFD) + return nil, fmt.Errorf("sync private directory parent after creating %s: %w", componentPath, syncErr) + } + } + nextFD, err = unix.Openat(parentFD, part, unix.O_RDONLY|unix.O_DIRECTORY|unix.O_CLOEXEC|unix.O_NOFOLLOW, 0) + } + if err != nil { + _ = unix.Close(parentFD) + return nil, fmt.Errorf("open private directory component %s without following links: %w", componentPath, err) + } + if err := validatePrivateDirDescriptor(nextFD, componentPath); err != nil { + _ = unix.Close(parentFD) + _ = unix.Close(nextFD) + return nil, err + } + if err := unix.Close(parentFD); err != nil { + _ = unix.Close(nextFD) + return nil, err + } + parentFD = nextFD + currentPath = componentPath + } + dir := os.NewFile(uintptr(parentFD), path) + if dir == nil { + _ = unix.Close(parentFD) + return nil, fmt.Errorf("open private directory %s", path) + } + return dir, nil +} + +func validatePrivateDirDescriptor(fd int, path string) error { + var stat unix.Stat_t + if err := unix.Fstat(fd, &stat); err != nil { + return err + } + if stat.Mode&0o022 != 0 { + return fmt.Errorf("private directory ancestor %s is writable by group or others", path) + } + if !trustedPrivateDirOwner(stat.Uid) { + return fmt.Errorf("private directory ancestor %s is owned by untrusted uid %d", path, stat.Uid) + } + return nil +} + +func trustedPrivateDirOwner(uid uint32) bool { + euid := uint32(os.Geteuid()) + if uid == 0 || uid == euid { + return true + } + if euid != 0 { + return false + } + sudoUID, err := strconv.ParseUint(strings.TrimSpace(os.Getenv("SUDO_UID")), 10, 32) + return err == nil && uid == uint32(sudoUID) +} + +func createPrivateFileNoFollow(dir *os.File, name string, mode os.FileMode) (*os.File, error) { + fd, err := unix.Openat(int(dir.Fd()), name, unix.O_WRONLY|unix.O_CREAT|unix.O_EXCL|unix.O_CLOEXEC|unix.O_NOFOLLOW, uint32(mode.Perm())) + if err != nil { + return nil, err + } + file := os.NewFile(uintptr(fd), filepath.Join(dir.Name(), name)) + if file == nil { + _ = unix.Close(fd) + return nil, fmt.Errorf("create private file %s", name) + } + return file, nil +} + +func writePrivateFileAtomicNoFollow(dir *os.File, name string, data []byte, mode os.FileMode) error { + tmpName, err := privateTempName() + if err != nil { + return err + } + tmp, err := createPrivateFileNoFollow(dir, tmpName, mode) + if err != nil { + return err + } + removeTmp := true + defer func() { + _ = tmp.Close() + if removeTmp { + _ = removePrivateFileNoFollow(dir, tmpName) + } + }() + if _, err := tmp.Write(data); err != nil { + return err + } + if err := tmp.Chmod(mode); err != nil { + return err + } + if err := tmp.Sync(); err != nil { + return err + } + if err := tmp.Close(); err != nil { + return err + } + if err := unix.Renameat(int(dir.Fd()), tmpName, int(dir.Fd()), name); err != nil { + return err + } + removeTmp = false + return unix.Fsync(int(dir.Fd())) +} + +func removePrivateFileNoFollow(dir *os.File, name string) error { + err := unix.Unlinkat(int(dir.Fd()), name, 0) + if errors.Is(err, unix.ENOENT) { + return nil + } + return err +} + +func syncPrivateDir(dir *os.File) error { + return unix.Fsync(int(dir.Fd())) +} + func openNoFollowRead(path string) (*os.File, error) { return os.OpenFile(path, os.O_RDONLY|syscall.O_NOFOLLOW, 0) } diff --git a/internal/target/dokploy/cleanup.go b/internal/target/dokploy/cleanup.go index 51bbb96..98fe847 100644 --- a/internal/target/dokploy/cleanup.go +++ b/internal/target/dokploy/cleanup.go @@ -11,6 +11,8 @@ import ( "sort" "strings" "time" + + "github.com/aikins01/bort/internal/safepath" ) type StalePlatformProject struct { @@ -921,28 +923,39 @@ func backupDokployDatabase(ctx context.Context, runner dockerRunner, pg, backupD if backupPrefix = strings.TrimSpace(backupPrefix); backupPrefix == "" { backupPrefix = "dokploy-cleanup" } - if err := os.MkdirAll(backupDir, 0o700); err != nil { - return "", err - } - if err := os.Chmod(backupDir, 0o700); err != nil { + dir, err := safepath.OpenPrivateDirNoFollow(backupDir) + if err != nil { return "", err } - path := filepath.Join(backupDir, fmt.Sprintf("%s-%s.sql", backupPrefix, time.Now().UTC().Format("20060102-150405.000000000"))) - file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) + defer dir.Close() + name := fmt.Sprintf("%s-%s.sql", backupPrefix, time.Now().UTC().Format("20060102-150405.000000000")) + path := filepath.Join(backupDir, name) + file, err := dir.CreateFile(name, 0o600) if err != nil { return "", err } if err := runner.Run(ctx, nil, file, "exec", pg, "pg_dump", "-U", "dokploy", "-d", "dokploy"); err != nil { _ = file.Close() + _ = dir.Remove(name) return "", fmt.Errorf("backup dokploy database to %s: %w", path, err) } if err := file.Chmod(0o600); err != nil { _ = file.Close() return "", err } + if err := file.Sync(); err != nil { + _ = file.Close() + return "", err + } if err := file.Close(); err != nil { return "", fmt.Errorf("close dokploy database backup %s: %w", path, err) } + if err := dir.Sync(); err != nil { + return "", fmt.Errorf("sync dokploy database backup directory for %s: %w", path, err) + } + if err := dir.ValidatePath(); err != nil { + return "", fmt.Errorf("verify dokploy database backup location %s: %w", path, err) + } return path, nil } diff --git a/internal/target/dokploy/cleanup_test.go b/internal/target/dokploy/cleanup_test.go index 4c4af76..70230b7 100644 --- a/internal/target/dokploy/cleanup_test.go +++ b/internal/target/dokploy/cleanup_test.go @@ -10,7 +10,10 @@ import ( ) func TestCleanupStalePlatformProjectsBacksUpThenDeletesMetadata(t *testing.T) { - backupDir := t.TempDir() + backupDir := dokployBackupTestDir(t) + if err := os.Chmod(backupDir, 0o700); err != nil { + t.Fatal(err) + } runner := &fakeDockerRunner{ outputs: map[string][]byte{ "ps --format {{.Names}}": []byte("dokploy-postgres.1.task\n"), @@ -42,6 +45,13 @@ func TestCleanupStalePlatformProjectsBacksUpThenDeletesMetadata(t *testing.T) { if string(backup) != "dump-bytes" { t.Fatalf("expected pg_dump backup contents, got %q", string(backup)) } + dirInfo, err := os.Stat(backupDir) + if err != nil { + t.Fatal(err) + } + if dirInfo.Mode().Perm() != 0o700 { + t.Fatalf("existing backup directory mode changed to %o", dirInfo.Mode().Perm()) + } if len(runner.runs) != 2 { t.Fatalf("expected pg_dump and psql runs, got %#v", runner.runs) } @@ -59,6 +69,80 @@ func TestCleanupStalePlatformProjectsBacksUpThenDeletesMetadata(t *testing.T) { } } +func TestBackupDokployDatabaseRejectsPermissiveDirectoryWithoutChangingMode(t *testing.T) { + backupDir := dokployBackupTestDir(t) + if err := os.Chmod(backupDir, 0o755); err != nil { + t.Fatal(err) + } + if _, err := backupDokployDatabase(context.Background(), &fakeDockerRunner{}, "dokploy-postgres", backupDir, "cleanup"); err == nil { + t.Fatal("expected permissive backup directory to be rejected") + } + info, err := os.Stat(backupDir) + if err != nil { + t.Fatal(err) + } + if info.Mode().Perm() != 0o755 { + t.Fatalf("existing backup directory mode changed to %o", info.Mode().Perm()) + } + entries, err := os.ReadDir(backupDir) + if err != nil { + t.Fatal(err) + } + if len(entries) != 0 { + t.Fatalf("permissive backup directory was modified: %#v", entries) + } +} + +func TestBackupDokployDatabaseRejectsSymlinkDirectory(t *testing.T) { + root := dokployBackupTestDir(t) + target := filepath.Join(root, "target") + if err := os.Mkdir(target, 0o755); err != nil { + t.Fatal(err) + } + link := filepath.Join(root, "backups") + if err := os.Symlink(target, link); err != nil { + t.Skipf("create directory symlink: %v", err) + } + if _, err := backupDokployDatabase(context.Background(), &fakeDockerRunner{}, "dokploy-postgres", link, "cleanup"); err == nil { + t.Fatal("expected symlinked backup directory to be rejected") + } + entries, err := os.ReadDir(target) + if err != nil { + t.Fatal(err) + } + if len(entries) != 0 { + t.Fatalf("symlink target was modified: %#v", entries) + } +} + +func dokployBackupTestDir(t *testing.T) string { + t.Helper() + cacheDir, err := os.UserCacheDir() + if err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(cacheDir, 0o700); err != nil { + t.Fatal(err) + } + cacheDir, err = filepath.EvalSymlinks(cacheDir) + if err != nil { + t.Fatal(err) + } + dir, err := os.MkdirTemp(cacheDir, "bort-dokploy-cleanup-test-") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := os.RemoveAll(dir); err != nil { + t.Errorf("remove Dokploy backup test directory: %v", err) + } + }) + if err := os.Chmod(dir, 0o700); err != nil { + t.Fatal(err) + } + return dir +} + func TestPurgeSourceResourcesRemovesDockerResources(t *testing.T) { runner := &fakeDockerRunner{outputs: map[string][]byte{ "inspect --type container cid123": []byte(`[{"Id":"cid123","Name":"/web","Config":{"Labels":{"coolify.managed":"true"}},"State":{"Running":false,"Status":"exited"}}]`), From fde3f2f07cb103c4011f04cbe5a754c5e65a2c35 Mon Sep 17 00:00:00 2001 From: Aikins Laryea Date: Sun, 2 Aug 2026 17:11:20 +0000 Subject: [PATCH 07/31] docs: keep migration guidance user-facing (#14) --- README.md | 68 +++++++++------------- docs/acceptance.md | 118 -------------------------------------- docs/cleanup-and-purge.md | 9 +-- docs/migration-guide.md | 22 ++++--- internal/cli/commit.go | 2 +- internal/cli/root.go | 4 +- internal/cli/run.go | 2 +- 7 files changed, 45 insertions(+), 180 deletions(-) delete mode 100644 docs/acceptance.md diff --git a/README.md b/README.md index e460016..faac5ec 100644 --- a/README.md +++ b/README.md @@ -26,9 +26,10 @@ command before each destructive step. Bort organizes the migration by app rather than exposing a wall of Docker details. It shows what is ready, provides copy-paste fixes, saves your answers, and tells you what to do next. Discovery and planning only preview app changes. -The guided setup can offer to install Dokploy before live apply, but it explains -the server changes and asks first. Moving apps, data, and traffic begins only -when you explicitly run the live command. +If Dokploy is not prepared, the `migrate --live` preflight can offer to install +it before application migration begins. Bort explains the server changes and +asks first. Moving apps, data, and traffic begins only when you explicitly run +the live command. ## Supported path @@ -40,8 +41,6 @@ The current product path is **Coolify → Dokploy on the same Linux VPS**. | Explicit live apply and resume | Implemented | | Accepting Dokploy and stopping source containers | Implemented | | Safe Dokploy database cleanup and separately confirmed source removal | Implemented | -| Automated end-to-end test with fake Docker and Dokploy | Implemented | -| End-to-end test on a disposable Linux host | Still required before relying on the live path in production | | Executable rollback | Not implemented; Bort only prints the stored rollback plan | | Dokploy → Coolify or cross-server migration | Not implemented | @@ -62,8 +61,8 @@ Today Bort can: - prepare Dokploy projects and compose applications before explicit live apply; - move supported local data by exporting and importing databases or copying data while a service is stopped; -- switch web traffic after health checks and a waiting period, save progress for - retries, and store rollback instructions; +- switch web traffic during explicit live apply, save progress for retries, and + store rollback instructions; - accept the target only after successful live apply; - list leftovers, remove only eligible unused records from Dokploy, and separately remove eligible source containers and networks after confirmation. @@ -95,16 +94,13 @@ live migration. In particular, Bort does not yet execute rollback. Install from Homebrew: ```sh -brew install aikins01/tap/bort +brew install --cask aikins01/tap/bort ``` -Install from source with Go: +Official archives and Linux packages are available from +[GitHub Releases](https://github.com/aikins01/bort/releases). -```sh -go install github.com/aikins01/bort/cmd/bort@latest -``` - -Or build the current checkout: +Build the current checkout from source with: ```sh make build @@ -125,7 +121,8 @@ sudo bort The guided screen starts setup when needed and otherwise resumes the current run. Review each application, follow its generated `fix:` and `next:` guidance, -and rerun `sudo bort` until the run shows `READY`. +and rerun `sudo bort` until the run shows `READY`. This means blocking inputs +are resolved; review any remaining non-blocking notes before live apply. To start discovery directly without the setup questions: @@ -138,7 +135,7 @@ The migration uses separate commands for each important step: ```text sudo bort # start or resume, then review and fix -sudo bort migrate --live # apply only the selected reviewed run +sudo bort migrate --live # apply only the selected planned run sudo bort rollback # inspect the stored manual rollback plan sudo bort commit --apply # accept the target and retire source containers sudo bort cleanup # audit leftovers without deleting source resources @@ -160,21 +157,21 @@ before applying cleanup or purge. ## Safety model -Bort is designed for production boxes where the safest default is “look first.” +Bort's safety model defaults to “look first.” - **Preview first:** discovery, planning, validation, rollback inspection, acceptance planning, cleanup, and purge planning do not change the server. -- **Confirmed Dokploy installation:** the guided setup may offer to install - Dokploy in Bort's side-by-side same-VPS layout before live apply. It warns that - this writes system configuration, creates Docker resources, initializes Swarm - when needed, and may disable Docker live-restore and reload Docker before - asking for confirmation. +- **Confirmed Dokploy installation:** the `migrate --live` preflight may offer to + install Dokploy in Bort's side-by-side same-VPS layout. It warns that this + writes system configuration, creates Docker resources, initializes Swarm when + needed, and may disable Docker live-restore and reload Docker before asking + for confirmation. Application migration begins only after confirmation. - **Explicit live apply:** creating target apps, copying data, and moving traffic - only happen through `bort migrate --live` for an existing reviewed run. + only happen through `bort migrate --live` for an existing planned run. - **Known current run:** `.bort/state.json` identifies the current run. Commands that make changes do not guess based on which file was modified most recently. - **Plans are locked during live work:** once live execution begins, changing - the reviewed plan requires a new run. + the selected plan requires a new run. - **One change at a time:** Bort prevents two commands from changing the same run at once, saves live progress for retries, and keeps `status` available. - **Private files:** bundles, state, environment values, live progress, and @@ -194,8 +191,6 @@ Bort is designed for production boxes where the safest default is “look first. rules, review, live apply, validation, acceptance, and recovery. - [Cleanup and purge](docs/cleanup-and-purge.md): safe Dokploy cleanup, source removal, manual steps, completion, and recovery from a partial failure. -- [End-to-end acceptance](docs/acceptance.md): automated test, disposable-host - test cases, destructive test procedure, and safe evidence handling. - `bort help`: main migration commands. - `bort help --advanced`: setup, automation, and migration-file commands. @@ -209,26 +204,23 @@ Bort is designed for production boxes where the safest default is “look first. recorded as details but are not copied, recreated, or revoked. - Bort does not automatically delete source Docker images because the target may share their layers and tags. -- The full real-host acceptance matrix remains to be recorded on a disposable - Linux VPS. ## Roadmap Near-term work remains focused on making the same-VPS Coolify → Dokploy path boring and safe before adding more platforms: -1. complete the disposable-host end-to-end tests; -2. make workspace selection, requirement checks, and failure recovery clearer +1. make workspace selection, requirement checks, and failure recovery clearer in the guided screen; -3. decide how automated rollback should work; -4. add Dokploy source scanning and Coolify target creation; -5. add cross-server transfers after the same-VPS steps are proven. +2. decide how automated rollback should work; +3. add Dokploy source scanning and Coolify target creation; +4. add cross-server transfers after the same-VPS steps are proven. Other Docker-, Compose-, and Swarm-based platforms remain possible future targets. Bort should only support one when it can clearly say which resources are safe to move, which are blocked, and which need manual work. -## Development and releases +## Development Run the repository checks with: @@ -238,14 +230,6 @@ make vet make build ``` -The fake end-to-end trace and destructive disposable-host checklist are in the -[acceptance guide](docs/acceptance.md). - -Releases are automated by GitHub Actions and GoReleaser. A new `v*` tag builds -macOS, Linux, and Windows archives, Linux packages, checksums, and the Homebrew -cask update. Use the intended release version rather than copying an old tag -from this README. - ## Name `bort` is named after Bortianor, a coastal town in Accra. The project is about diff --git a/docs/acceptance.md b/docs/acceptance.md deleted file mode 100644 index 53947d7..0000000 --- a/docs/acceptance.md +++ /dev/null @@ -1,118 +0,0 @@ -# End-to-end acceptance testing - -This guide is for maintainers testing Bort releases. It includes an automated -test with fake services and a destructive checklist for a disposable host. The -automated test does not prove that real Docker, proxy, volume, or filesystem -behavior works. - -Do not claim production-host support until a redacted result from the -disposable-host checklist has been recorded for the version being released. - -## Automated end-to-end test - -The automated test uses a temporary workspace, fake Dokploy HTTP API, and fake -Docker command. It creates a self-contained run, performs live apply, verifies -`TARGET LIVE`, commits, verifies `COMMITTED`, inventories cleanup, applies a -confirmed all-app purge, and verifies `COMPLETE`: - -```sh -go test ./internal/cli -run '^TestLifecycleAcceptanceTraceWithFakeDokploy$' -v -``` - -Run the complete repository checks separately: - -```sh -make test -make vet -make build -``` - -## Disposable-host test cases - -Use a fresh Linux VM or a restorable snapshot containing no data you intend to -keep. Exercise at least: - -| Scenario | Required evidence | -| --- | --- | -| Stateless app | Domain remains healthy after live apply, acceptance, and source purge. | -| Database | Representative data survives the selected migration strategy and remains readable and writable on the target. | -| Named volume | Dry run requires manual completion; apply verifies absence without deleting resources automatically or recording completion. | -| Bind mount | Bort requires manual removal of the host path and refuses protected paths. | -| Interrupted live apply | Rerunning the command continues from saved progress without repeating completed work. | -| Two live commands | The second process shows the active progress while `status` remains readable. | -| Automatic purge | Bort removes the exact source containers and networks while keeping target resources and credentials. | -| Same-name replacement | Bort does not mistake a newly created resource for the one that was reviewed. | - -## Disposable-host procedure - -1. Install Coolify, then deploy one stateless disposable Coolify app with a - route, source network, and non-secret test environment value. Do not perform a - normal Dokploy installation alongside the active Coolify proxy. Use Bort's - guided `same-VPS shadow mode`, its side-by-side Dokploy layout, so Dokploy's - proxy is prepared without taking ports 80/443 before Bort switches web - traffic. -2. Record the source container, network, and target project names before - migration. -3. In a dedicated working directory, run the complete migration as one OS user: - - ```sh - run="bort-acceptance-$(date +%Y%m%d-%H%M%S)" - sudo bort migrate --source coolify-local --run "$run" - sudo bort - sudo bort migrate --live --run "$run" - sudo bort status --run "$run" - ``` - -4. Verify that the target serves the workload and wait through the rollback - window recorded in the plan. Only then accept the target and plan source - removal: - - ```sh - sudo bort commit --apply --run "$run" - sudo bort cleanup --run "$run" - sudo bort cleanup purge --run "$run" --all-apps - ``` - -5. Review the purge plan and verify the target still serves the disposable - workload before applying it: - - ```sh - sudo bort cleanup purge --apply --run "$run" --all-apps --confirm "purge $run" - sudo bort status --run "$run" - ``` - -6. Confirm that the target application remains healthy, `run.json` records live - apply and acceptance, the selected non-platform source container and network - are absent, and target resources plus source-control credentials remain. A - real `coolify-local` run includes internal Coolify components that `--all-apps` - excludes unless `--include-platform` is supplied, so this purge must not - record `PurgedAt` or claim that the whole run is complete. -7. Restore the clean snapshot and repeat with a disposable app containing a - named volume and a bind mount under a disposable directory. After the purge - dry run, manually remove every listed source resource. Confirm that `--apply` - performs only absence verification, leaves the run accepted rather than - complete, and does not delete a replacement resource introduced before - verification. -8. Repeat once with an interrupted live apply and once with two concurrent live - invocations. Confirm that resume, attachment, and read-only status behave as - documented. -9. Destroy the VM or restore its snapshot after retaining only the redacted - evidence needed for the release. - -## Evidence handling - -Retain: - -- Bort version and release candidate commit; -- host OS, architecture, Docker version, Coolify version, and Dokploy version; -- redacted command output and migration status; -- the test-case results; -- confirmation that the VM was destroyed or restored. - -Do not retain or publish: - -- `.bort/state.json`; -- exported environment values; -- Coolify or Dokploy tokens; -- database dumps or application data; -- unredacted manifests, bundles, apply ledgers, or purge backups. diff --git a/docs/cleanup-and-purge.md b/docs/cleanup-and-purge.md index 4594769..ce2b19d 100644 --- a/docs/cleanup-and-purge.md +++ b/docs/cleanup-and-purge.md @@ -69,10 +69,11 @@ such as its source and proxy services, additionally require `--include-platform`. Review every listed resource and use the exact apply command and confirmation -phrase emitted by the dry run. A named example is: +phrase emitted by the dry run. To select a named run, replace `RUN_NAME` in this +example: ```sh -run="bort-acceptance-20260802-120000" +run="RUN_NAME" sudo bort cleanup purge --run "$run" --all-apps sudo bort cleanup purge --apply --run "$run" --all-apps --confirm "purge $run" ``` @@ -150,5 +151,5 @@ removed. If purge reports an incomplete result: Do not recreate a source resource under the same name and assume it is the same object. Bort checks Docker IDs and refuses a different replacement resource. -Use the destructive scenarios in the [acceptance guide](acceptance.md) only on a -fresh VM or restorable snapshot. +While evaluating Bort, exercise destructive cleanup only on a fresh VM or +restorable snapshot containing no data you intend to keep. diff --git a/docs/migration-guide.md b/docs/migration-guide.md index 8d10b48..6687475 100644 --- a/docs/migration-guide.md +++ b/docs/migration-guide.md @@ -3,9 +3,8 @@ This guide covers Bort's current supported migration: moving applications from Coolify to Dokploy on the same Linux VPS. -The live migration still needs a recorded test on a disposable Linux host before -it should be relied on in production. Use a fresh VM or a restorable snapshot -for testing, and keep an independent backup of any data you need to retain. +Start with a fresh VM or a restorable snapshot and keep an independent backup of +any data you need to retain. ## Prerequisites @@ -23,8 +22,8 @@ Before starting, confirm that: directory and OS user throughout the migration; - Dokploy will run on this same VPS and use Bort's side-by-side layout, which the CLI calls `same-VPS shadow mode`; either it is already prepared in that layout - or it will be installed through Bort's guided setup. This layout prevents - Dokploy's proxy from taking ports 80/443 before Bort switches traffic; + or the `migrate --live` preflight will offer to install it. This layout + prevents Dokploy's proxy from taking ports 80/443 before Bort switches traffic; - the VPS has a current provider snapshot or another independently tested recovery path. @@ -89,7 +88,8 @@ other than the workspace default. The guided screen shows whether each application is ready, what must be fixed, and the next safe action. Follow the generated `fix:` commands and rerun Bort -until it shows `READY`. +until it shows `READY`. This means the inputs that block live apply are resolved; +it does not clear non-blocking review items. Use the guided screen for secret values so they are not placed in command arguments, shell history, or process and sudo audit records. The direct command @@ -101,11 +101,11 @@ sudo bort data demo-app postgres --migrate sudo bort ``` -Review application routes, environment values, database and storage choices, -named volumes, bind mounts, and source-control details. `READY` means the plan -has been reviewed; the server has not been changed yet. +Before continuing, review application routes, environment values, database and +storage choices, named volumes, bind mounts, source-control details, and any +remaining `next:` notes. The server has not been changed yet. -## Apply the reviewed run +## Apply the selected run Live apply is a separate explicit command: @@ -203,5 +203,3 @@ Ordinary cleanup and destructive source purge are separate operations. See the | Databases and persistent data | Supports local database dump/restore and copying while a service is stopped. Continuous syncing and other replication methods are not implemented. | | Named volumes and bind mounts | Lists them and asks how they should be handled when manual review is required. Source purge never automatically deletes named volumes or host paths. | | Source control | Records repository details and deploy-key hints, but does not copy, revoke, or recreate GitHub Apps, deploy keys, webhooks, or equivalent credentials. | - -For release-level validation, use the [acceptance guide](acceptance.md). diff --git a/internal/cli/commit.go b/internal/cli/commit.go index 3ab94c6..0ee411f 100644 --- a/internal/cli/commit.go +++ b/internal/cli/commit.go @@ -46,7 +46,7 @@ func runCommit(ctx context.Context, args []string, stdout, stderr io.Writer) err if apply { for _, name := range []string{"app", "bundle", "format", "from-cutover", "output", "rollback-window", "target"} { if flagSet(fs, name) { - return fmt.Errorf("commit --apply does not accept --%s; select the reviewed run with --run", name) + return fmt.Errorf("commit --apply does not accept --%s; select the run with --run", name) } } return applyCommitFromArgs(ctx, runRef, stderr) diff --git a/internal/cli/root.go b/internal/cli/root.go index 502c796..7be69a1 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -85,7 +85,7 @@ func printHelp(w io.Writer, advanced bool) error { func writePrimaryHelp(w io.Writer, st *styler) { writeHelpSection(w, st, "Usage:", []helpLine{ {verb: bortCommand(""), desc: "start or resume the current migration"}, - {verb: bortCommand("migrate --live"), desc: "apply the reviewed current run to its target"}, + {verb: bortCommand("migrate --live"), desc: "apply the selected planned run to its target"}, {verb: bortCommand("rollback"), desc: "inspect the source rollback plan"}, {verb: bortCommand("commit --apply"), desc: "accept the target and retire source containers"}, {verb: bortCommand("cleanup"), desc: "audit leftovers; --apply removes safe metadata only"}, @@ -99,7 +99,7 @@ func writePrimaryHelp(w io.Writer, st *styler) { func writeAdvancedHelp(w io.Writer, st *styler) { writeHelpSection(w, st, "Setup and automation:", []helpLine{ - {verb: bortCommand("migrate --source "), desc: "scan, export, and create a reviewed run in one command"}, + {verb: bortCommand("migrate --source "), desc: "scan, export, and create a planned run in one command"}, {verb: bortCommand("migrate --manifest "), desc: "create a run from an existing manifest"}, {verb: bortCommand("env KEY=value ..."), desc: "record env values non-interactively"}, {verb: bortCommand("data --migrate|--recreate|--managed"), desc: "record a data strategy non-interactively"}, diff --git a/internal/cli/run.go b/internal/cli/run.go index 0d6a0fa..5b3ec53 100644 --- a/internal/cli/run.go +++ b/internal/cli/run.go @@ -208,7 +208,7 @@ func runMigrate(ctx context.Context, args []string, stdout, stderr io.Writer) er if live { for _, name := range []string{"app", "bundle", "manifest", "observation-window", "rollback-window", "source", "target"} { if flagSet(fs, name) { - return fmt.Errorf("--live applies an existing reviewed run and does not accept --%s; create or update the dry-run first, then apply it with --run or the current-run default", name) + return fmt.Errorf("--live applies an existing planned run and does not accept --%s; create or update the dry-run first, then apply it with --run or the current-run default", name) } } } From 21242c4d1b5463beb7e564e99feaefb4e19f7fdc Mon Sep 17 00:00:00 2001 From: Aikins Laryea Date: Sun, 2 Aug 2026 18:05:55 +0000 Subject: [PATCH 08/31] cleanup: preserve resources shared by same-name apps (#15) --- internal/cli/cleanup.go | 76 ++++++++++++++++++---- internal/cli/cleanup_test.go | 123 +++++++++++++++++++++++++++++++++++ 2 files changed, 185 insertions(+), 14 deletions(-) diff --git a/internal/cli/cleanup.go b/internal/cli/cleanup.go index 69d41fc..6dc5daa 100644 --- a/internal/cli/cleanup.go +++ b/internal/cli/cleanup.go @@ -436,11 +436,14 @@ func runCleanupPurge(ctx context.Context, args []string, stdin io.Reader, stdout } func planCleanupPurge(run loadedMigrationRun, target string, filters cleanupPurgeFilters, extraSourceDirs []string) (cleanupPurgeResult, error) { + if err := validateCleanupPurgeAppRefs(run.Prepare.Apps); err != nil { + return cleanupPurgeResult{}, err + } selected, err := cleanupPurgeSelectedApps(run, filters) if err != nil { return cleanupPurgeResult{}, err } - selectedNames := cleanupSelectedAppNames(selected) + selectedRefs := cleanupSelectedAppRefs(selected) owners, err := cleanupPurgeResourceOwners(run) if err != nil { return cleanupPurgeResult{}, err @@ -472,7 +475,7 @@ func planCleanupPurge(run loadedMigrationRun, target string, filters cleanupPurg if strings.TrimSpace(container.ContainerID) == "" { continue } - if sharedWith := cleanupContainerOwnersOutsideSelected(owners, container, selectedNames); len(sharedWith) > 0 { + if sharedWith := cleanupContainerOwnersOutsideSelected(owners, container, selectedRefs); len(sharedWith) > 0 { result.CompletesLifecycle = false result.Warnings = append(result.Warnings, fmt.Sprintf("source container %s for %s is also referenced by unselected app(s): %s", firstCleanupValue(container.ContainerName, container.ContainerID), app.Name, strings.Join(sharedWith, ", "))) continue @@ -487,7 +490,7 @@ func planCleanupPurge(run loadedMigrationRun, target string, filters cleanupPurg result.CompletesLifecycle = false volume.Action = "inspect_manually_before_purge" result.Warnings = append(result.Warnings, fmt.Sprintf("named volume for %s/%s has no recorded name and is not scheduled for purge", app.Name, volume.Service)) - } else if sharedWith := cleanupOwnersOutsideSelected(owners.NamedVolumes[volume.Name], selectedNames); len(sharedWith) > 0 { + } else if sharedWith := cleanupOwnersOutsideSelected(owners.NamedVolumes[volume.Name], selectedRefs); len(sharedWith) > 0 { result.CompletesLifecycle = false volume.Action = "preserve_shared_with_unselected_app" result.Warnings = append(result.Warnings, fmt.Sprintf("named volume %s for %s is also referenced by unselected app(s): %s", volume.Name, app.Name, strings.Join(sharedWith, ", "))) @@ -504,7 +507,7 @@ func planCleanupPurge(run loadedMigrationRun, target string, filters cleanupPurg } else if path == nil { result.CompletesLifecycle = false volume.Action = "inspect_manually_before_purge" - } else if sharedWith := cleanupPathOwnersOutsideSelected(owners.BindPaths, path.Path, selectedNames); len(sharedWith) > 0 { + } else if sharedWith := cleanupPathOwnersOutsideSelected(owners.BindPaths, path.Path, selectedRefs); len(sharedWith) > 0 { result.CompletesLifecycle = false volume.Action = "preserve_shared_with_unselected_app" result.Warnings = append(result.Warnings, fmt.Sprintf("bind mount source %s for %s is also referenced by unselected app(s): %s", path.Path, app.Name, strings.Join(sharedWith, ", "))) @@ -533,7 +536,7 @@ func planCleanupPurge(run loadedMigrationRun, target string, filters cleanupPurg result.Warnings = append(result.Warnings, fmt.Sprintf("source network %s for %s is a platform network and requires --include-platform for purge", network.Name, app.Name)) continue } - if sharedWith := cleanupOwnersOutsideSelected(owners.Networks[network.Name], selectedNames); len(sharedWith) > 0 { + if sharedWith := cleanupOwnersOutsideSelected(owners.Networks[network.Name], selectedRefs); len(sharedWith) > 0 { result.CompletesLifecycle = false result.Warnings = append(result.Warnings, fmt.Sprintf("source network %s for %s is also referenced by unselected app(s): %s", network.Name, app.Name, strings.Join(sharedWith, ", "))) continue @@ -551,7 +554,7 @@ func planCleanupPurge(run loadedMigrationRun, target string, filters cleanupPurg if err := dokploy.ValidateSourcePurgePath(cleaned, filters.IncludePlatform); err != nil { return cleanupPurgeResult{}, err } - if sharedWith := cleanupPathOwnersOutsideSelected(owners.BindPaths, cleaned, selectedNames); len(sharedWith) > 0 { + if sharedWith := cleanupPathOwnersOutsideSelected(owners.BindPaths, cleaned, selectedRefs); len(sharedWith) > 0 { return cleanupPurgeResult{}, fmt.Errorf("refusing source-dir %s because it overlaps bind mount(s) for unselected app(s): %s", cleaned, strings.Join(sharedWith, ", ")) } result.SourcePaths = append(result.SourcePaths, cleanupSourcePath{Path: cleaned, Source: "explicit", AllowPlatform: filters.IncludePlatform, Action: "require_absent_before_purge_apply"}) @@ -591,6 +594,41 @@ func cleanupSelectedAppNames(apps []preparer.AppPlan) map[string]struct{} { return names } +func cleanupSelectedAppRefs(apps []preparer.AppPlan) map[string]struct{} { + refs := map[string]struct{}{} + for _, app := range apps { + if ref := cleanupPurgeAppRef(app); ref != "" { + refs[ref] = struct{}{} + } + } + return refs +} + +func validateCleanupPurgeAppRefs(apps []preparer.AppPlan) error { + seen := map[string]struct{}{} + for index, app := range apps { + ref := cleanupPurgeAppRef(app) + if ref == "" { + return fmt.Errorf("refusing cleanup purge because app %d has no name or directory reference", index+1) + } + if _, exists := seen[ref]; exists { + return fmt.Errorf("refusing cleanup purge because app reference %q is not unique in the run", cleanupPurgeAppRefLabel(ref)) + } + seen[ref] = struct{}{} + } + return nil +} + +func cleanupPurgeAppRef(app preparer.AppPlan) string { + if directory := strings.TrimSpace(app.Directory); directory != "" { + return "directory:" + directory + } + if name := strings.TrimSpace(app.Name); name != "" { + return "name:" + name + } + return "" +} + func cleanupPurgeResourceOwners(run loadedMigrationRun) (cleanupPurgeOwners, error) { owners := cleanupPurgeOwners{ ContainerIDs: map[string][]string{}, @@ -600,8 +638,8 @@ func cleanupPurgeResourceOwners(run loadedMigrationRun) (cleanupPurgeOwners, err Networks: map[string][]string{}, } for _, app := range run.Prepare.Apps { - appName := strings.TrimSpace(app.Name) - if appName == "" { + appRef := cleanupPurgeAppRef(app) + if appRef == "" { continue } containers, err := cleanupContainersForApp(app) @@ -610,22 +648,22 @@ func cleanupPurgeResourceOwners(run loadedMigrationRun) (cleanupPurgeOwners, err } for _, container := range containers { if containerID := strings.TrimSpace(container.ContainerID); containerID != "" { - owners.ContainerIDs[containerID] = append(owners.ContainerIDs[containerID], appName) + owners.ContainerIDs[containerID] = append(owners.ContainerIDs[containerID], appRef) } if containerName := cleanupContainerName(container.ContainerName); containerName != "" { - owners.ContainerNames[containerName] = append(owners.ContainerNames[containerName], appName) + owners.ContainerNames[containerName] = append(owners.ContainerNames[containerName], appRef) } } for _, volume := range cleanupVolumesForApp(app) { switch volume.Type { case "volume": if strings.TrimSpace(volume.Name) != "" { - owners.NamedVolumes[volume.Name] = append(owners.NamedVolumes[volume.Name], appName) + owners.NamedVolumes[volume.Name] = append(owners.NamedVolumes[volume.Name], appRef) } case "bind": if strings.TrimSpace(volume.Source) != "" { path := filepath.Clean(volume.Source) - owners.BindPaths[path] = append(owners.BindPaths[path], appName) + owners.BindPaths[path] = append(owners.BindPaths[path], appRef) } } } @@ -635,7 +673,7 @@ func cleanupPurgeResourceOwners(run loadedMigrationRun) (cleanupPurgeOwners, err } for _, network := range networks { if strings.TrimSpace(network.Name) != "" { - owners.Networks[network.Name] = append(owners.Networks[network.Name], appName) + owners.Networks[network.Name] = append(owners.Networks[network.Name], appRef) } } } @@ -688,12 +726,22 @@ func cleanupOwnersOutsideSelected(owners []string, selected map[string]struct{}) continue } seen[owner] = struct{}{} - outside = append(outside, owner) + outside = append(outside, cleanupPurgeAppRefLabel(owner)) } sort.Strings(outside) return outside } +func cleanupPurgeAppRefLabel(ref string) string { + if strings.HasPrefix(ref, "directory:") { + return strings.TrimPrefix(ref, "directory:") + } + if strings.HasPrefix(ref, "name:") { + return strings.TrimPrefix(ref, "name:") + " (legacy name)" + } + return ref +} + func cleanupPathOwnersOutsideSelected(pathOwners map[string][]string, path string, selected map[string]struct{}) []string { owners := []string{} for ownedPath, pathOwners := range pathOwners { diff --git a/internal/cli/cleanup_test.go b/internal/cli/cleanup_test.go index 9f9e02c..51960de 100644 --- a/internal/cli/cleanup_test.go +++ b/internal/cli/cleanup_test.go @@ -452,6 +452,129 @@ func TestRunCleanupPurgePreservesResourcesSharedWithUnselectedApps(t *testing.T) } } +func TestPlanCleanupPurgePreservesResourceSharedWithUnselectedSameNameApp(t *testing.T) { + run := sameNameCleanupPurgeRun(t) + result, err := planCleanupPurge(run, "dokploy", cleanupPurgeFilters{Projects: []string{"project-a"}}, nil) + if err != nil { + t.Fatal(err) + } + if len(result.SourceContainers) != 0 { + t.Fatalf("expected the shared container to be preserved, got %#v", result.SourceContainers) + } + if len(result.SourceNetworks) != 0 || len(result.SourcePaths) != 0 { + t.Fatalf("expected shared networks and paths to be preserved, got networks=%#v paths=%#v", result.SourceNetworks, result.SourcePaths) + } + for _, volume := range result.SourceVolumes { + if volume.Action != "preserve_shared_with_unselected_app" { + t.Fatalf("expected shared volume to be preserved, got %#v", volume) + } + } + warnings := strings.Join(result.Warnings, "\n") + for _, want := range []string{ + "source container selected-id-name for api is also referenced by unselected app(s): api-2", + "source container shared-name for api is also referenced by unselected app(s): api-2", + "named volume shared-data for api is also referenced by unselected app(s): api-2", + "bind mount source /data/coolify/applications/shared/storage for api is also referenced by unselected app(s): api-2", + "source network shared-net for api is also referenced by unselected app(s): api-2", + } { + if !strings.Contains(warnings, want) { + t.Fatalf("expected warning %q for the unselected same-name app, got %q", want, warnings) + } + } +} + +func TestPlanCleanupPurgeRejectsSourceDirSharedWithUnselectedSameNameApp(t *testing.T) { + run := sameNameCleanupPurgeRun(t) + _, err := planCleanupPurge(run, "dokploy", cleanupPurgeFilters{Projects: []string{"project-a"}}, []string{"/data/coolify/applications/shared"}) + if err == nil || !strings.Contains(err.Error(), "refusing source-dir /data/coolify/applications/shared because it overlaps bind mount(s) for unselected app(s): api-2") { + t.Fatalf("expected the unselected same-name app to block the shared source directory, got %v", err) + } +} + +func sameNameCleanupPurgeRun(t *testing.T) loadedMigrationRun { + t.Helper() + bundleDir := t.TempDir() + for _, directory := range []string{"api", "api-2"} { + appDir := filepath.Join(bundleDir, directory) + if err := os.MkdirAll(appDir, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(appDir, "topology.json"), []byte(`{"networks":["shared-net"]}`), 0o600); err != nil { + t.Fatal(err) + } + } + return loadedMigrationRun{Prepare: preparer.Result{ + BundleDir: bundleDir, + Apps: []preparer.AppPlan{ + { + Name: "api", + Directory: "api", + ProjectGroup: &preparer.ProjectGroup{Name: "project-a"}, + Resources: preparer.ResourceSpecs{ + SourceServices: []preparer.SourceServiceRef{ + {ServiceName: "id-owner", ContainerID: "shared-id", ContainerName: "selected-id-name"}, + {ServiceName: "name-owner", ContainerID: "selected-name-id", ContainerName: "shared-name"}, + }, + Volumes: []preparer.VolumeResource{ + {Service: "web", Type: "volume", Name: "shared-data", Target: "/data"}, + {Service: "web", Type: "bind", Source: "/data/coolify/applications/shared/storage", Target: "/storage"}, + }, + }, + }, + { + Name: "api", + Directory: "api-2", + ProjectGroup: &preparer.ProjectGroup{Name: "project-b"}, + Resources: preparer.ResourceSpecs{ + SourceServices: []preparer.SourceServiceRef{ + {ServiceName: "id-owner", ContainerID: "shared-id", ContainerName: "unselected-id-name"}, + {ServiceName: "name-owner", ContainerID: "unselected-name-id", ContainerName: "shared-name"}, + }, + Volumes: []preparer.VolumeResource{ + {Service: "web", Type: "volume", Name: "shared-data", Target: "/data"}, + {Service: "web", Type: "bind", Source: "/data/coolify/applications/shared/storage", Target: "/storage"}, + }, + }, + }, + }, + }} +} + +func TestCleanupPurgeAppRefsSeparateDirectoriesFromLegacyNames(t *testing.T) { + selected := cleanupSelectedAppRefs([]preparer.AppPlan{{Name: "selected", Directory: "api"}}) + owners := []string{ + cleanupPurgeAppRef(preparer.AppPlan{Name: "selected", Directory: "api"}), + cleanupPurgeAppRef(preparer.AppPlan{Name: "api"}), + } + outside := cleanupOwnersOutsideSelected(owners, selected) + if len(outside) != 1 || outside[0] != "api (legacy name)" { + t.Fatalf("expected the legacy name owner to remain unselected, got %#v", outside) + } + outside = cleanupOwnersOutsideSelected(owners, nil) + if len(outside) != 2 || outside[0] != "api" || outside[1] != "api (legacy name)" { + t.Fatalf("expected directory and legacy name labels to remain distinct, got %#v", outside) + } +} + +func TestPlanCleanupPurgeRejectsDuplicateAppRefs(t *testing.T) { + run := sameNameCleanupPurgeRun(t) + run.Prepare.Apps[1].Directory = "api" + _, err := planCleanupPurge(run, "dokploy", cleanupPurgeFilters{AllApps: true}, nil) + if err == nil || !strings.Contains(err.Error(), `app reference "api" is not unique`) { + t.Fatalf("expected duplicate app references to fail closed, got %v", err) + } +} + +func TestPlanCleanupPurgeRejectsMissingAppRef(t *testing.T) { + run := sameNameCleanupPurgeRun(t) + run.Prepare.Apps[1].Name = "" + run.Prepare.Apps[1].Directory = "" + _, err := planCleanupPurge(run, "dokploy", cleanupPurgeFilters{AllApps: true}, nil) + if err == nil || !strings.Contains(err.Error(), "app 2 has no name or directory reference") { + t.Fatalf("expected a missing app reference to fail closed, got %v", err) + } +} + func TestCleanupPurgeFailsClosedWhenTopologyIsUnavailable(t *testing.T) { workDir := t.TempDir() t.Chdir(workDir) From a674cdf16471595543099f347ffb111b823e96c8 Mon Sep 17 00:00:00 2001 From: Aikins Laryea Date: Sun, 2 Aug 2026 18:30:21 +0000 Subject: [PATCH 09/31] migration: reject positional live-run arguments (#16) --- internal/cli/run.go | 3 +++ internal/cli/run_test.go | 48 +++++++++++++++++++++++++++++++--------- 2 files changed, 40 insertions(+), 11 deletions(-) diff --git a/internal/cli/run.go b/internal/cli/run.go index 5b3ec53..a147418 100644 --- a/internal/cli/run.go +++ b/internal/cli/run.go @@ -204,6 +204,9 @@ func runMigrate(ctx context.Context, args []string, stdout, stderr io.Writer) er if err := fs.Parse(args); err != nil { return err } + if fs.NArg() > 0 { + return fmt.Errorf("migrate does not accept positional argument %q", fs.Arg(0)) + } bundleFlagSet := flagSet(fs, "bundle") if live { for _, name := range []string{"app", "bundle", "manifest", "observation-window", "rollback-window", "source", "target"} { diff --git a/internal/cli/run_test.go b/internal/cli/run_test.go index 8f0e414..53bf61c 100644 --- a/internal/cli/run_test.go +++ b/internal/cli/run_test.go @@ -409,7 +409,7 @@ func TestValidateLiveApplyReadyBlocksEveryPrepareRequirement(t *testing.T) { } } -func TestApprovedPrepareDecisionsRequireResolvedProgress(t *testing.T) { +func TestApprovedPrepareDecisionsRequireResolvedOrSkippedProgress(t *testing.T) { item := runDecisionItem{ Stage: "prepare", App: "api", @@ -418,17 +418,30 @@ func TestApprovedPrepareDecisionsRequireResolvedProgress(t *testing.T) { Readiness: preparer.ReadinessNeedsDecision, } decision := runDecision{ID: "prepare-review", Kind: "prepare-review", Items: []runDecisionItem{item}} - run := loadedMigrationRun{ - Decisions: runDecisions{APIVersion: decisionsAPIVersion, Decisions: []runDecision{decision}}, - Progress: runProgress{Decisions: map[string]decisionProgress{}}, - } - if approved := approvedPrepareDecisions(run); len(approved) != 0 { - t.Fatalf("unresolved prepare decision was approved: %#v", approved) + tests := []struct { + name string + status string + want bool + }{ + {name: "unresolved"}, + {name: "resolved", status: progressStatusResolved, want: true}, + {name: "skipped", status: progressStatusSkipped, want: true}, } - run.Progress = markDecisionDone(run.Progress, decision, progressStatusResolved, "reviewed", time.Now().UTC()) - approved := dokploy.NewPrepareDecision(item.App, item.Code, item.ResourceRef, item.Readiness, item.Message) - if _, ok := approvedPrepareDecisions(run)[approved]; !ok { - t.Fatalf("resolved prepare decision code was not approved") + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + run := loadedMigrationRun{ + Decisions: runDecisions{APIVersion: decisionsAPIVersion, Decisions: []runDecision{decision}}, + Progress: runProgress{Decisions: map[string]decisionProgress{}}, + } + if tt.status != "" { + run.Progress = markDecisionDone(run.Progress, decision, tt.status, "reviewed", time.Now().UTC()) + } + approved := dokploy.NewPrepareDecision(item.App, item.Code, item.ResourceRef, item.Readiness, item.Message) + _, ok := approvedPrepareDecisions(run)[approved] + if ok != tt.want { + t.Fatalf("approved = %t, want %t", ok, tt.want) + } + }) } } @@ -566,6 +579,19 @@ func TestRunMigrateLiveRequiresReviewedRun(t *testing.T) { } } +func TestRunMigrateRejectsPositionalArgument(t *testing.T) { + workDir := t.TempDir() + t.Chdir(workDir) + + err := runMigrate(context.Background(), []string{"--live", "intended-run"}, io.Discard, io.Discard) + if err == nil || !strings.Contains(err.Error(), `migrate does not accept positional argument "intended-run"`) { + t.Fatalf("expected positional argument rejection, got %v", err) + } + if _, statErr := os.Stat(filepath.Join(workDir, ".bort")); !os.IsNotExist(statErr) { + t.Fatalf("positional argument rejection reached migration run setup: %v", statErr) + } +} + func TestRunMigrateLiveDoesNotUseMtimeFallback(t *testing.T) { workDir := t.TempDir() t.Chdir(workDir) From 919f1099deaaf0ca3c3e2f5a3f3d22a1523177e4 Mon Sep 17 00:00:00 2001 From: Aikins Laryea Date: Sun, 2 Aug 2026 19:03:10 +0000 Subject: [PATCH 10/31] migration: reject ambiguous run arguments (#17) --- internal/cli/run.go | 12 ++++++- internal/cli/run_test.go | 70 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+), 1 deletion(-) diff --git a/internal/cli/run.go b/internal/cli/run.go index a147418..e7e53e0 100644 --- a/internal/cli/run.go +++ b/internal/cli/run.go @@ -1540,7 +1540,17 @@ func loadRunFromArgs(command string, args []string, stderr io.Writer) (loadedMig if err := fs.Parse(args); err != nil { return loadedMigrationRun{}, err } - if runRef == "" && fs.NArg() == 1 { + runFlagSet := flagSet(fs, "run") + if runFlagSet && fs.NArg() > 0 { + return loadedMigrationRun{}, fmt.Errorf("%s does not accept positional argument %q with --run", command, fs.Arg(0)) + } + if runFlagSet && strings.TrimSpace(runRef) == "" { + return loadedMigrationRun{}, fmt.Errorf("%s requires a non-empty --run value", command) + } + if fs.NArg() > 1 { + return loadedMigrationRun{}, fmt.Errorf("%s does not accept positional argument %q after run reference %q", command, fs.Arg(1), fs.Arg(0)) + } + if fs.NArg() == 1 { runRef = fs.Arg(0) } resolved, err := resolveRunRef(runRef, false) diff --git a/internal/cli/run_test.go b/internal/cli/run_test.go index 53bf61c..1ce8069 100644 --- a/internal/cli/run_test.go +++ b/internal/cli/run_test.go @@ -1447,6 +1447,76 @@ func TestRunStatusAndNextReadExistingRun(t *testing.T) { } } +func TestRunStatusAndNextRejectAmbiguousRunArguments(t *testing.T) { + tests := []struct { + name string + run cliRunner + args []string + want string + }{ + { + name: "status multiple positional references", + run: runStatus, + args: []string{"intended-run", "typo"}, + want: `status does not accept positional argument "typo" after run reference "intended-run"`, + }, + { + name: "next multiple positional references", + run: runNext, + args: []string{"intended-run", "typo"}, + want: `next does not accept positional argument "typo" after run reference "intended-run"`, + }, + { + name: "status flag and positional reference", + run: runStatus, + args: []string{"--run", "intended-run", "typo"}, + want: `status does not accept positional argument "typo" with --run`, + }, + { + name: "next flag and positional reference", + run: runNext, + args: []string{"--run", "intended-run", "typo"}, + want: `next does not accept positional argument "typo" with --run`, + }, + { + name: "status empty flag and positional reference", + run: runStatus, + args: []string{"--run=", "intended-run"}, + want: `status does not accept positional argument "intended-run" with --run`, + }, + { + name: "next empty flag and positional reference", + run: runNext, + args: []string{"--run=", "intended-run"}, + want: `next does not accept positional argument "intended-run" with --run`, + }, + { + name: "status empty flag", + run: runStatus, + args: []string{"--run="}, + want: "status requires a non-empty --run value", + }, + { + name: "next empty flag", + run: runNext, + args: []string{"--run="}, + want: "next requires a non-empty --run value", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var stdout bytes.Buffer + err := tt.run(context.Background(), tt.args, &stdout, io.Discard) + if err == nil || !strings.Contains(err.Error(), tt.want) { + t.Fatalf("expected %q, got %v", tt.want, err) + } + if stdout.Len() != 0 { + t.Fatalf("ambiguous arguments produced output before rejection: %q", stdout.String()) + } + }) + } +} + func TestRunStatusUsesCurrentRunBeforeNewerMtime(t *testing.T) { workDir := t.TempDir() t.Chdir(workDir) From 260e695400b5ecb220e24270609028196a2cce0b Mon Sep 17 00:00:00 2001 From: Aikins Laryea Date: Mon, 3 Aug 2026 08:25:20 +0000 Subject: [PATCH 11/31] migration: stop target after guarded deploy failure (#18) --- internal/target/dokploy/apply.go | 127 ++++++++- internal/target/dokploy/apply_test.go | 335 +++++++++++++++++++++-- internal/target/dokploy/datamove.go | 7 +- internal/target/dokploy/datamove_test.go | 47 +++- 4 files changed, 471 insertions(+), 45 deletions(-) diff --git a/internal/target/dokploy/apply.go b/internal/target/dokploy/apply.go index 73c4b14..46bac19 100644 --- a/internal/target/dokploy/apply.go +++ b/internal/target/dokploy/apply.go @@ -3,6 +3,8 @@ package dokploy import ( "bytes" "context" + "crypto/rand" + "encoding/hex" "encoding/json" "errors" "fmt" @@ -11,6 +13,7 @@ import ( "sort" "strconv" "strings" + "time" "github.com/aikins01/bort/internal/gateway" "github.com/aikins01/bort/internal/preparer" @@ -337,6 +340,30 @@ type activeBortOverridePatch struct { Branch string `json:"branch"` } +type deployedComposeGuardError struct { + err error +} + +func (e deployedComposeGuardError) Error() string { + return e.err.Error() +} + +func (e deployedComposeGuardError) Unwrap() error { + return e.err +} + +type unsafeSourceResumeError struct { + err error +} + +func (e unsafeSourceResumeError) Error() string { + return e.err.Error() +} + +func (e unsafeSourceResumeError) Unwrap() error { + return e.err +} + func (c *Client) validateNoActiveBortOverrides(ctx context.Context, composeID string) error { composeID = strings.TrimSpace(composeID) if composeID == "" { @@ -476,7 +503,11 @@ func (c *Client) Apply(ctx context.Context, plan Plan) error { err := c.applyStep(ctx, actx, step) if err != nil { emitProgress(plan.OnProgress, StepProgress{Index: index, Total: total, Step: step, Status: StepStatusError, Err: err}) - c.bestEffortResume(ctx, actx, plan, total, pausedApps, coolifyProxyStopped, isUnsafeTargetResumeError(err)) + resumePausedApps := pausedApps + if isUnsafeSourceResumeError(err) { + resumePausedApps = nil + } + c.bestEffortResume(ctx, actx, plan, total, resumePausedApps, coolifyProxyStopped, isUnsafeTargetResumeError(err)) return fmt.Errorf("dokploy step %s for %s (%s): %w", step.Kind, step.App, step.Ref, err) } emitProgress(plan.OnProgress, StepProgress{Index: index, Total: total, Step: step, Status: StepStatusOK}) @@ -645,6 +676,11 @@ func isUnsafeTargetResumeError(err error) bool { return errors.As(err, &unsafe) } +func isUnsafeSourceResumeError(err error) bool { + var unsafe unsafeSourceResumeError + return errors.As(err, &unsafe) +} + func (c *Client) applyStep(ctx context.Context, actx *applyContext, step Step) error { switch step.Kind { case StepCreateProject: @@ -775,13 +811,7 @@ func (c *Client) applyPushImage(ctx context.Context, actx *applyContext, step St if err := ensureComposeImagesAvailable(ctx, c.dockerRunner(), actx.plan, step.App, composeFile); err != nil { return err } - if err := c.deployComposeWithPatchGuard(ctx, entry.ComposeID, deployComposeTitle(actx.plan)); err != nil { - if safetyErr := c.validateMigratedVolumeMountsAfterDeploy(ctx, actx, step.App); safetyErr != nil && isUnsafeTargetResumeError(safetyErr) { - return fmt.Errorf("deploy dokploy compose: %w; post-deploy safety check: %v", err, safetyErr) - } - return err - } - if err := c.validateMigratedVolumeMountsAfterDeploy(ctx, actx, step.App); err != nil { + if err := c.deployComposeForApply(ctx, actx, step.App); err != nil { return err } return c.pauseTargetWritersForState(ctx, c.dockerRunner(), actx, step.App) @@ -802,9 +832,82 @@ func (c *Client) deployComposeWithPatchGuard(ctx context.Context, composeID, tit } mutationErr := c.DeployCompose(ctx, composeID, title) patchErr := c.validateNoActiveBortOverridesAfterMutation(ctx, composeID) + if mutationErr == nil && patchErr != nil { + return deployedComposeGuardError{err: patchErr} + } return errors.Join(mutationErr, patchErr) } +func (c *Client) deployComposeForApply(ctx context.Context, actx *applyContext, appName string) error { + entry := actx.entry(appName) + title, err := deployComposeAttemptTitle(actx.plan) + if err != nil { + return err + } + err = c.deployComposeWithPatchGuard(ctx, entry.ComposeID, title) + if err == nil { + return c.validateMigratedVolumeMountsAfterDeploy(ctx, actx, appName) + } + var deployedGuardErr deployedComposeGuardError + if errors.As(err, &deployedGuardErr) { + stopCtx, cancelStop := context.WithTimeout(context.Background(), targetDiscoveryTimeout) + defer cancelStop() + stopErr := c.quiesceGuardedDeployment(stopCtx, actx, appName, title) + if stopErr != nil { + return unsafeSourceResumeError{err: unsafeTargetResumeError{err: fmt.Errorf("successful Dokploy deploy failed its post-deploy patch guard: %w; quiescence could not be proved, so Bort will leave any paused source applications stopped and skip target recovery: %v", err, stopErr)}} + } + safetyErr := c.validateMigratedVolumeMountsAfterDeploy(ctx, actx, appName) + combined := errors.Join(err, safetyErr) + return unsafeTargetResumeError{err: fmt.Errorf("successful Dokploy deploy failed its post-deploy patch guard; deployment quiesced and target compose containers were stopped: %w", combined)} + } + safetyErr := c.validateMigratedVolumeMountsAfterDeploy(ctx, actx, appName) + if safetyErr != nil && isUnsafeTargetResumeError(safetyErr) { + return fmt.Errorf("deploy dokploy compose: %w; post-deploy safety check: %v", err, safetyErr) + } + return err +} + +func deployComposeAttemptTitle(plan Plan) (string, error) { + var nonce [16]byte + if _, err := rand.Read(nonce[:]); err != nil { + return "", fmt.Errorf("generate Dokploy deployment identity: %w", err) + } + return deployComposeTitle(plan) + "-" + hex.EncodeToString(nonce[:]), nil +} + +func (c *Client) quiesceGuardedDeployment(ctx context.Context, actx *applyContext, appName, title string) error { + entry := actx.entry(appName) + for { + if err := c.stopTargetComposeContainers(ctx, actx, appName); err != nil { + return err + } + compose, err := c.GetCompose(ctx, entry.ComposeID) + if err != nil { + return err + } + attemptTerminal := false + allDeploymentsTerminal := true + for _, deployment := range compose.Deployments { + switch strings.ToLower(strings.TrimSpace(deployment.Status)) { + case "done", "error", "cancelled": + if deployment.Title == title { + attemptTerminal = true + } + default: + allDeploymentsTerminal = false + } + } + if attemptTerminal && allDeploymentsTerminal { + return c.stopTargetComposeContainers(ctx, actx, appName) + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(targetWriterDiscoveryDelay): + } + } +} + func deployComposeTitle(plan Plan) string { title := "bort-migrate" if plan.RunName != "" { @@ -999,13 +1102,7 @@ func (c *Client) applyActivateRoutes(ctx context.Context, actx *applyContext, st return err } } - if err := c.deployComposeWithPatchGuard(ctx, entry.ComposeID, deployComposeTitle(actx.plan)); err != nil { - if safetyErr := c.validateMigratedVolumeMountsAfterDeploy(ctx, actx, step.App); safetyErr != nil && isUnsafeTargetResumeError(safetyErr) { - return fmt.Errorf("deploy dokploy compose: %w; post-deploy safety check: %v", err, safetyErr) - } - return err - } - return c.validateMigratedVolumeMountsAfterDeploy(ctx, actx, step.App) + return c.deployComposeForApply(ctx, actx, step.App) } func (c *Client) ensureRouteDomain(ctx context.Context, composeID string, route gateway.Route) error { diff --git a/internal/target/dokploy/apply_test.go b/internal/target/dokploy/apply_test.go index 5991fd6..caf54b8 100644 --- a/internal/target/dokploy/apply_test.go +++ b/internal/target/dokploy/apply_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "errors" + "fmt" "io" "net/http" "net/http/httptest" @@ -11,6 +12,7 @@ import ( "path/filepath" "strings" "testing" + "time" "github.com/aikins01/bort/internal/gateway" "github.com/aikins01/bort/internal/preparer" @@ -373,14 +375,32 @@ func TestActivePatchGuardBlocksGitBackedCompose(t *testing.T) { type stagedBortPatchRunner struct { fakeDockerRunner - checks int + checks int + patchAt int + targetStopped bool +} + +func (r *stagedBortPatchRunner) Output(ctx context.Context, args ...string) ([]byte, error) { + key := strings.Join(args, " ") + if key == "inspect --type container target-id" && r.targetStopped { + r.outputArgs = append(r.outputArgs, append([]string{}, args...)) + return []byte(`[{"Id":"target-id","Name":"/target","State":{"Running":false,"Status":"exited"}}]`), nil + } + if key == "stop target-id" { + r.targetStopped = true + } + return r.fakeDockerRunner.Output(ctx, args...) } func (r *stagedBortPatchRunner) Run(ctx context.Context, stdin io.Reader, stdout io.Writer, args ...string) error { key := strings.Join(args, " ") if strings.Contains(key, " psql ") { r.checks++ - if r.checks == 2 { + patchAt := r.patchAt + if patchAt == 0 { + patchAt = 2 + } + if r.checks == patchAt { r.runOutputs[key] = []byte(`{"patchId":"bort-api-compose","filePath":"docker|compose.yml","composeName":"api","sourceType":"github","repository":"owner/repo","branch":"main"}` + "\n") } else { r.runOutputs[key] = nil @@ -389,16 +409,46 @@ func (r *stagedBortPatchRunner) Run(ctx context.Context, stdin io.Reader, stdout return r.fakeDockerRunner.Run(ctx, stdin, stdout, args...) } +type restartingTargetRunner struct { + inspectCalls int + stopCalls int +} + +func (r *restartingTargetRunner) Output(_ context.Context, args ...string) ([]byte, error) { + switch strings.Join(args, " ") { + case "ps -a --filter label=com.docker.compose.project=stack-1 --format {{.ID}}": + return []byte("target-id\n"), nil + case "inspect --type container target-id": + r.inspectCalls++ + running := r.inspectCalls <= 2 + status := "exited" + if running { + status = "running" + } + return []byte(fmt.Sprintf(`[{"Id":"target-id","Name":"/target","State":{"Running":%t,"Status":%q}}]`, running, status)), nil + case "stop target-id": + r.stopCalls++ + return []byte("target-id\n"), nil + default: + return nil, fmt.Errorf("docker output not stubbed: %s", strings.Join(args, " ")) + } +} + +func (r *restartingTargetRunner) Run(context.Context, io.Reader, io.Writer, ...string) error { + return nil +} + func TestComposeMutationPatchGuardsDetectConcurrentPatch(t *testing.T) { for _, test := range []struct { - name string - path string - mutate func(*Client) error + name string + path string + deployedBefore bool + mutate func(*Client) error }{ {name: "update", path: "/api/compose.update", mutate: func(client *Client) error { return client.updateComposeWithPatchGuard(context.Background(), "compose-api", "services: {}\n", "") }}, - {name: "deploy", path: "/api/compose.deploy", mutate: func(client *Client) error { + {name: "deploy", path: "/api/compose.deploy", deployedBefore: true, mutate: func(client *Client) error { return client.deployComposeWithPatchGuard(context.Background(), "compose-api", "bort-migrate") }}, } { @@ -422,6 +472,10 @@ func TestComposeMutationPatchGuardsDetectConcurrentPatch(t *testing.T) { if err == nil || !strings.Contains(err.Error(), "appeared while Bort was changing compose") { t.Fatalf("expected post-mutation patch detection, got %v", err) } + var deployedGuardErr deployedComposeGuardError + if got := errors.As(err, &deployedGuardErr); got != test.deployedBefore { + t.Fatalf("deployed guard error = %t, want %t", got, test.deployedBefore) + } if mutations != 1 || runner.checks != 2 { t.Fatalf("expected one mutation between two patch checks, mutations=%d checks=%d", mutations, runner.checks) } @@ -429,26 +483,275 @@ func TestComposeMutationPatchGuardsDetectConcurrentPatch(t *testing.T) { } } -func TestComposeMutationPatchGuardRechecksAfterHTTPError(t *testing.T) { +func TestPostDeployPatchGuardStopsTargetCompose(t *testing.T) { + var deploymentTitle string + var composeReads int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodPost || r.URL.Path != "/api/compose.update" { + switch { + case r.Method == http.MethodPost && r.URL.Path == "/api/compose.deploy": + var request deployComposeRequest + if err := json.NewDecoder(r.Body).Decode(&request); err != nil { + t.Fatal(err) + } + deploymentTitle = request.Title + w.WriteHeader(http.StatusOK) + case r.Method == http.MethodGet && r.URL.Path == "/api/compose.one": + composeReads++ + attemptStatus := "running" + if composeReads > 1 { + attemptStatus = "done" + } + otherStatus := "running" + if composeReads > 2 { + otherStatus = "done" + } + _ = json.NewEncoder(w).Encode(Compose{ComposeID: "compose-api", AppName: "stack-1", Deployments: []Deployment{ + {Title: deploymentTitle, Status: attemptStatus}, + {Title: "other-deployment", Status: otherStatus}, + }}) + default: http.NotFound(w, r) - return } - http.Error(w, "mutation failed", http.StatusInternalServerError) })) defer server.Close() runner := &stagedBortPatchRunner{fakeDockerRunner: fakeDockerRunner{ - outputs: map[string][]byte{"ps --format {{.Names}}": []byte("dokploy-postgres\n")}, + outputs: map[string][]byte{ + "ps --format {{.Names}}": []byte("dokploy-postgres\n"), + "ps -a --filter label=com.docker.compose.project=stack-1 --format {{.ID}}": []byte("target-id\n"), + "inspect --type container target-id": []byte(`[{"Id":"target-id","Name":"/target","State":{"Running":true,"Status":"running"}}]`), + "stop target-id": []byte("target-id\n"), + }, + runOutputs: map[string][]byte{}, + }} + client := &Client{BaseURL: server.URL, Token: "secret", HTTPClient: server.Client(), Docker: runner} + actx := &applyContext{plan: Plan{}, cache: map[string]*appCache{}} + entry := actx.entry("api") + entry.ComposeID = "compose-api" + entry.ComposeAppName = "stack-1" + + err := client.deployComposeForApply(context.Background(), actx, "api") + if err == nil || !strings.Contains(err.Error(), "deployment quiesced and target compose containers were stopped") || !isUnsafeTargetResumeError(err) { + t.Fatalf("expected guarded deploy failure to stop the target and prevent target recovery, got %v", err) + } + if isUnsafeSourceResumeError(err) { + t.Fatalf("successfully quiesced deployment should permit source recovery, got %v", err) + } + if composeReads < 3 { + t.Fatalf("quiescence ignored another nonterminal deployment, reads=%d", composeReads) + } + stopCalls := 0 + targetListCalls := 0 + for _, call := range runner.outputArgs { + if len(call) == 2 && call[0] == "stop" && call[1] == "target-id" { + stopCalls++ + } + if strings.Join(call, " ") == "ps -a --filter label=com.docker.compose.project=stack-1 --format {{.ID}}" { + targetListCalls++ + } + } + if stopCalls != 1 || targetListCalls < 3 { + t.Fatalf("expected target stop plus repeated and final verification, stops=%d lists=%d calls=%#v", stopCalls, targetListCalls, runner.outputArgs) + } + if fakeOutputCalled(&runner.fakeDockerRunner, "start", "target-id") { + t.Fatalf("successful deploy guard failure restarted target, calls=%#v", runner.outputArgs) + } +} + +func TestStopTargetComposeContainersRestopsRestartedTarget(t *testing.T) { + runner := &restartingTargetRunner{} + client := &Client{Docker: runner} + actx := &applyContext{cache: map[string]*appCache{"api": {ComposeAppName: "stack-1"}}} + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + if err := client.stopTargetComposeContainers(ctx, actx, "api"); err != nil { + t.Fatalf("stop restarted target: %v", err) + } + if runner.stopCalls != 2 { + t.Fatalf("expected restarted target to be stopped again, stops=%d", runner.stopCalls) + } +} + +func TestPostDeployPatchGuardQuiescenceFailurePreventsSourceRecovery(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodPost && r.URL.Path == "/api/compose.deploy": + w.WriteHeader(http.StatusOK) + case r.Method == http.MethodGet && r.URL.Path == "/api/compose.one": + http.Error(w, "deployment status unavailable", http.StatusInternalServerError) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + runner := &stagedBortPatchRunner{fakeDockerRunner: fakeDockerRunner{ + outputs: map[string][]byte{ + "ps --format {{.Names}}": []byte("dokploy-postgres\n"), + "ps -a --filter label=com.docker.compose.project=stack-1 --format {{.ID}}": {}, + }, runOutputs: map[string][]byte{}, }} client := &Client{BaseURL: server.URL, Token: "secret", HTTPClient: server.Client(), Docker: runner} - err := client.updateComposeWithPatchGuard(context.Background(), "compose-api", "services: {}\n", "") - if err == nil || !strings.Contains(err.Error(), "mutation failed") || !strings.Contains(err.Error(), "appeared while Bort was changing compose") { - t.Fatalf("expected HTTP and post-mutation patch errors, got %v", err) + actx := &applyContext{plan: Plan{}, cache: map[string]*appCache{}} + entry := actx.entry("api") + entry.ComposeID = "compose-api" + entry.ComposeAppName = "stack-1" + + err := client.deployComposeForApply(context.Background(), actx, "api") + if err == nil || !strings.Contains(err.Error(), "deployment status unavailable") { + t.Fatalf("expected deployment polling failure, got %v", err) + } + if !isUnsafeSourceResumeError(err) || !isUnsafeTargetResumeError(err) { + t.Fatalf("unproved deployment quiescence must prevent source and target recovery, got %v", err) + } +} + +func TestQuiesceGuardedDeploymentAcceptsTerminalStatuses(t *testing.T) { + for _, status := range []string{"done", "error", "cancelled"} { + t.Run(status, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet || r.URL.Path != "/api/compose.one" { + http.NotFound(w, r) + return + } + _ = json.NewEncoder(w).Encode(Compose{ComposeID: "compose-api", Deployments: []Deployment{{Title: "unique-title", Status: status}}}) + })) + defer server.Close() + runner := &fakeDockerRunner{outputs: map[string][]byte{ + "ps -a --filter label=com.docker.compose.project=stack-1 --format {{.ID}}": {}, + }} + client := &Client{BaseURL: server.URL, Token: "secret", HTTPClient: server.Client(), Docker: runner} + actx := &applyContext{cache: map[string]*appCache{"api": {ComposeID: "compose-api", ComposeAppName: "stack-1"}}} + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + if err := client.quiesceGuardedDeployment(ctx, actx, "api", "unique-title"); err != nil { + t.Fatalf("terminal status %q did not quiesce: %v", status, err) + } + }) } - if runner.checks != 2 { - t.Fatalf("expected the failed HTTP mutation to remain bracketed by patch checks, got %d", runner.checks) +} + +func TestDeployComposeAttemptTitleIsUniquePerAttempt(t *testing.T) { + plan := Plan{RunName: "run1"} + first, err := deployComposeAttemptTitle(plan) + if err != nil { + t.Fatal(err) + } + second, err := deployComposeAttemptTitle(plan) + if err != nil { + t.Fatal(err) + } + prefix := "bort-migrate-run1-" + if first == second { + t.Fatalf("deployment attempts reused title %q", first) + } + if !strings.HasPrefix(first, prefix) || len(strings.TrimPrefix(first, prefix)) != 32 { + t.Fatalf("deployment title does not contain a 128-bit identity: %q", first) + } + if !strings.HasPrefix(second, prefix) || len(strings.TrimPrefix(second, prefix)) != 32 { + t.Fatalf("deployment title does not contain a 128-bit identity: %q", second) + } +} + +func TestApplyLeavesPausedSourceStoppedWhenGuardedDeploymentCannotBeQuiesced(t *testing.T) { + bundleDir := t.TempDir() + appDir := filepath.Join(bundleDir, "api") + if err := os.MkdirAll(appDir, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(appDir, "compose.yaml"), []byte("services:\n web:\n image: example/api:latest\n"), 0o600); err != nil { + t.Fatal(err) + } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && r.URL.Path == "/api/project.all": + _ = json.NewEncoder(w).Encode([]Project{{ProjectID: "p1", Name: "api", Environments: []ProjectEnvironment{{EnvironmentID: "env1", Name: "production"}}}}) + case r.Method == http.MethodGet && r.URL.Path == "/api/compose.search": + _ = json.NewEncoder(w).Encode(composeSearchResponse{Items: []Compose{{ComposeID: "c1", Name: "api", AppName: "stack-1"}}, Total: 1}) + case r.Method == http.MethodPost && r.URL.Path == "/api/compose.update": + w.WriteHeader(http.StatusOK) + case r.Method == http.MethodPost && r.URL.Path == "/api/compose.deploy": + w.WriteHeader(http.StatusOK) + case r.Method == http.MethodGet && r.URL.Path == "/api/compose.one": + http.Error(w, "deployment status unavailable", http.StatusInternalServerError) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + runner := &stagedBortPatchRunner{ + patchAt: 4, + fakeDockerRunner: fakeDockerRunner{ + outputs: map[string][]byte{ + "ps --format {{.Names}}": []byte("dokploy-postgres\n"), + "image inspect example/api:latest": []byte("[]"), + "ps -a --filter label=com.docker.compose.project=stack-1 --format {{.ID}}": {}, + "inspect --type container source-id": []byte(`[{"Id":"source-id","Name":"/source","State":{"Running":false,"Status":"exited"}}]`), + "start source-id": []byte("source-id\n"), + }, + runOutputs: map[string][]byte{}, + }, + } + client := &Client{BaseURL: server.URL, Token: "secret", HTTPClient: server.Client(), Docker: runner} + app := preparer.AppPlan{Name: "api", Directory: "api"} + app.Resources.SourceServices = []preparer.SourceServiceRef{{ServiceName: "web", ContainerID: "source-id"}} + app.TargetResources = &preparer.TargetResources{Dokploy: &preparer.DokployResources{ComposeApp: preparer.DokployComposeApp{Name: "api", ComposePath: "compose.yaml"}}} + err := client.Apply(context.Background(), Plan{ + ResumeFrom: 3, + Steps: []Step{ + {Kind: StepCreateProject, App: "api", Ref: "api"}, + {Kind: StepCreateService, App: "api", Ref: "api"}, + {Kind: StepPauseSource, App: "api", Ref: "api"}, + {Kind: StepActivateRoutes, App: "api", Ref: "routes"}, + }, + Prepare: preparer.Result{BundleDir: bundleDir, Apps: []preparer.AppPlan{app}}, + }) + if err == nil || !strings.Contains(err.Error(), "leave any paused source applications stopped") { + t.Fatalf("expected fail-closed guarded deployment error, got %v", err) + } + if fakeOutputCalled(&runner.fakeDockerRunner, "start", "source-id") { + t.Fatalf("unsafe recovery restarted the paused source, calls=%#v", runner.outputArgs) + } +} + +func TestComposeMutationPatchGuardRechecksAfterHTTPError(t *testing.T) { + for _, test := range []struct { + name string + path string + mutate func(*Client) error + }{ + {name: "update", path: "/api/compose.update", mutate: func(client *Client) error { + return client.updateComposeWithPatchGuard(context.Background(), "compose-api", "services: {}\n", "") + }}, + {name: "deploy", path: "/api/compose.deploy", mutate: func(client *Client) error { + return client.deployComposeWithPatchGuard(context.Background(), "compose-api", "bort-migrate") + }}, + } { + t.Run(test.name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost || r.URL.Path != test.path { + http.NotFound(w, r) + return + } + http.Error(w, "mutation failed", http.StatusInternalServerError) + })) + defer server.Close() + runner := &stagedBortPatchRunner{fakeDockerRunner: fakeDockerRunner{ + outputs: map[string][]byte{"ps --format {{.Names}}": []byte("dokploy-postgres\n")}, + runOutputs: map[string][]byte{}, + }} + client := &Client{BaseURL: server.URL, Token: "secret", HTTPClient: server.Client(), Docker: runner} + err := test.mutate(client) + if err == nil || !strings.Contains(err.Error(), "mutation failed") || !strings.Contains(err.Error(), "appeared while Bort was changing compose") { + t.Fatalf("expected HTTP and post-mutation patch errors, got %v", err) + } + var deployedGuardErr deployedComposeGuardError + if errors.As(err, &deployedGuardErr) { + t.Fatalf("failed HTTP mutation was classified as a successful deploy: %v", err) + } + if runner.checks != 2 { + t.Fatalf("expected the failed HTTP mutation to remain bracketed by patch checks, got %d", runner.checks) + } + }) } } diff --git a/internal/target/dokploy/datamove.go b/internal/target/dokploy/datamove.go index 5fba3d6..895d2b8 100644 --- a/internal/target/dokploy/datamove.go +++ b/internal/target/dokploy/datamove.go @@ -565,7 +565,6 @@ func (c *Client) stopTargetComposeContainers(ctx context.Context, actx *applyCon runner := c.dockerRunner() var result error var quietSince time.Time - stoppedIDs := map[string]struct{}{} for { containers, err := listContainersByLabel(ctx, runner, "com.docker.compose.project="+entry.ComposeAppName) if err != nil { @@ -580,9 +579,6 @@ func (c *Client) stopTargetComposeContainers(ctx context.Context, actx *applyCon if !container.State.Running { continue } - if _, stopped := stoppedIDs[container.ID]; stopped { - continue - } sawRunning = true quietSince = time.Time{} if err := stopContainer(ctx, runner, container.ID); err != nil { @@ -594,7 +590,6 @@ func (c *Client) stopTargetComposeContainers(ctx context.Context, actx *applyCon result = fmt.Errorf("stop target container %s: %w", container.ID, err) continue } - stoppedIDs[container.ID] = struct{}{} } if hadStopError { return result @@ -1361,7 +1356,7 @@ func (c *Client) targetContainerForServiceWithRedeploy(ctx context.Context, runn } if allowRedeploy && len(containers) == 0 && entry.ComposeID != "" && !entry.DiscoveryRedeployAttempted { entry.DiscoveryRedeployAttempted = true - if err := c.deployComposeWithPatchGuard(ctx, entry.ComposeID, deployComposeTitle(actx.plan)); err != nil { + if err := c.deployComposeForApply(ctx, actx, appName); err != nil { return dockerContainer{}, fmt.Errorf("redeploy dokploy compose project %q for target discovery: %w", entry.ComposeAppName, err) } } diff --git a/internal/target/dokploy/datamove_test.go b/internal/target/dokploy/datamove_test.go index a9e0cbd..faa477b 100644 --- a/internal/target/dokploy/datamove_test.go +++ b/internal/target/dokploy/datamove_test.go @@ -1,6 +1,7 @@ package dokploy import ( + "bytes" "context" "encoding/json" "errors" @@ -83,6 +84,34 @@ func (f *fakeDockerRunner) Run(_ context.Context, stdin io.Reader, stdout io.Wri return f.runErr } +type statefulTargetRunner struct { + fakeDockerRunner + stopped map[string]bool +} + +func (r *statefulTargetRunner) Output(ctx context.Context, args ...string) ([]byte, error) { + if len(args) == 4 && args[0] == "inspect" && args[1] == "--type" && args[2] == "container" && r.stopped[args[3]] { + output, err := r.fakeDockerRunner.Output(ctx, args...) + if err != nil { + return nil, err + } + output = bytes.ReplaceAll(output, []byte(`"Running":true`), []byte(`"Running":false`)) + output = bytes.ReplaceAll(output, []byte(`"Status":"running"`), []byte(`"Status":"exited"`)) + return output, nil + } + output, err := r.fakeDockerRunner.Output(ctx, args...) + if err != nil || len(args) != 2 { + return output, err + } + switch args[0] { + case "stop": + r.stopped[args[1]] = true + case "start": + r.stopped[args[1]] = false + } + return output, nil +} + type stagedTargetWriterRunner struct { psCalls int outputArgs [][]string @@ -1365,15 +1394,16 @@ func TestApplyResumeTargetDetectsMigratedVolumeDrift(t *testing.T) { Name: "src-vol", Target: "/data", }} - runner := &fakeDockerRunner{ - outputs: map[string][]byte{ + runner := &statefulTargetRunner{ + stopped: map[string]bool{}, + fakeDockerRunner: fakeDockerRunner{outputs: map[string][]byte{ "ps -a --filter label=com.docker.compose.project=stack-1 --format {{.ID}}": []byte("dst-id\n"), "inspect --type container dst-id": []byte(`[{"Id":"dst-id","Name":"/dokploy-redis","Config":{"Labels":{"com.docker.compose.service":"redis","com.docker.compose.project":"stack-1"}},"State":{"Running":true,"Status":"running"},"Mounts":[{"Type":"volume","Name":"migrated-vol","Destination":"/data","RW":true}]}]`), "volume inspect src-vol": []byte(`[{"Name":"src-vol"}]`), "volume inspect migrated-vol": []byte(`[{"Name":"migrated-vol"}]`), "stop dst-id": []byte("dst-id\n"), "start dst-id": []byte("dst-id\n"), - }, + }}, } client := &Client{Docker: runner} actx := &applyContext{cache: map[string]*appCache{}} @@ -1392,10 +1422,10 @@ func TestApplyResumeTargetDetectsMigratedVolumeDrift(t *testing.T) { if err == nil || !strings.Contains(err.Error(), "changed from migrated volume migrated-vol to fresh-vol") { t.Fatalf("expected migrated volume drift error, got %v", err) } - if !fakeOutputCalled(runner, "stop", "dst-id") { + if !fakeOutputCalled(&runner.fakeDockerRunner, "stop", "dst-id") { t.Fatalf("expected unsafe target container to stop, calls=%#v", runner.outputArgs) } - if fakeOutputCalled(runner, "start", "writer-id") { + if fakeOutputCalled(&runner.fakeDockerRunner, "start", "writer-id") { t.Fatalf("target writer started before volume drift validation, calls=%#v", runner.outputArgs) } } @@ -1409,15 +1439,16 @@ func TestApplyResumeTargetLoadsPersistedMigratedVolumeState(t *testing.T) { Name: "src-vol", Target: "/data", }} - runner := &fakeDockerRunner{ - outputs: map[string][]byte{ + runner := &statefulTargetRunner{ + stopped: map[string]bool{}, + fakeDockerRunner: fakeDockerRunner{outputs: map[string][]byte{ "ps -a --filter label=com.docker.compose.project=stack-1 --format {{.ID}}": []byte("dst-id\n"), "inspect --type container dst-id": []byte(`[{"Id":"dst-id","Name":"/dokploy-redis","Config":{"Labels":{"com.docker.compose.service":"redis","com.docker.compose.project":"stack-1"}},"State":{"Running":true,"Status":"running"},"Mounts":[{"Type":"volume","Name":"migrated-vol","Destination":"/data","RW":true}]}]`), "volume inspect src-vol": []byte(`[{"Name":"src-vol"}]`), "volume inspect migrated-vol": []byte(`[{"Name":"migrated-vol"}]`), "stop dst-id": []byte("dst-id\n"), "start dst-id": []byte("dst-id\n"), - }, + }}, } client := &Client{Docker: runner} plan := Plan{Prepare: preparer.Result{Apps: []preparer.AppPlan{app}}, RunDir: runDir} From e0d994074e3a215599ccddf0fb85e1a4e94cae16 Mon Sep 17 00:00:00 2001 From: Aikins Laryea Date: Mon, 3 Aug 2026 08:47:27 +0000 Subject: [PATCH 12/31] migration: upgrade legacy reviewed bundles (#19) --- internal/cli/run.go | 31 ++++++++------ internal/cli/run_test.go | 91 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 109 insertions(+), 13 deletions(-) diff --git a/internal/cli/run.go b/internal/cli/run.go index e7e53e0..0c5c5b1 100644 --- a/internal/cli/run.go +++ b/internal/cli/run.go @@ -612,27 +612,34 @@ func approvedPrepareDecisions(run loadedMigrationRun) map[dokploy.PrepareDecisio } func ensureSelfContainedLiveRunLocked(run loadedMigrationRun) (loadedMigrationRun, error) { - if err := containedPath(run.Run.RunDir, run.Prepare.BundleDir); err == nil { + containedErr := containedPath(run.Run.RunDir, run.Prepare.BundleDir) + if containedErr == nil && strings.TrimSpace(run.Run.BundleDigest) != "" { return run, nil - } else if run.Run.ApplyOutcomeRequired { - return loadedMigrationRun{}, fmt.Errorf("run %q does not have a self-contained reviewed bundle; run `%s` to refresh it before live apply: %w", run.Run.Name, runScopedCommand(run, "migrate"), err) + } + if containedErr != nil && run.Run.ApplyOutcomeRequired { + return loadedMigrationRun{}, fmt.Errorf("run %q does not have a self-contained reviewed bundle; run `%s` to refresh it before live apply: %w", run.Run.Name, runScopedCommand(run, "migrate"), containedErr) } runDir := filepath.FromSlash(run.Run.RunDir) - bundleDir, err := snapshotMigrationBundle(run.Prepare.BundleDir, runDir) - if err != nil { - return loadedMigrationRun{}, fmt.Errorf("snapshot legacy run bundle: %w", err) - } - bundleDigest, err := digestMigrationBundle(bundleDir) - if err != nil { - return loadedMigrationRun{}, fmt.Errorf("digest legacy run bundle: %w", err) - } - removeBundle := true + bundleDir := run.Prepare.BundleDir + removeBundle := false defer func() { if removeBundle { _ = os.RemoveAll(bundleDir) } }() + if containedErr != nil { + var err error + bundleDir, err = snapshotMigrationBundle(run.Prepare.BundleDir, runDir) + if err != nil { + return loadedMigrationRun{}, fmt.Errorf("snapshot legacy run bundle: %w", err) + } + removeBundle = true + } + bundleDigest, err := digestMigrationBundle(bundleDir) + if err != nil { + return loadedMigrationRun{}, fmt.Errorf("digest legacy run bundle: %w", err) + } artifacts, artifactDir, err := nextRunArtifacts(runDir, run.Run) if err != nil { return loadedMigrationRun{}, err diff --git a/internal/cli/run_test.go b/internal/cli/run_test.go index 1ce8069..8b24991 100644 --- a/internal/cli/run_test.go +++ b/internal/cli/run_test.go @@ -273,7 +273,7 @@ func TestApplyLiveMigrationRejectsChangedReviewedBundle(t *testing.T) { } } -func TestLegacySelfContainedRunWithoutBundleDigestRemainsLoadable(t *testing.T) { +func TestLegacySelfContainedRunWithoutBundleDigestIsUpgradedBeforeLiveApply(t *testing.T) { workDir := t.TempDir() t.Chdir(workDir) bundleDir := filepath.Join(workDir, "bort-bundle") @@ -300,6 +300,95 @@ func TestLegacySelfContainedRunWithoutBundleDigestRemainsLoadable(t *testing.T) if err := verifyReviewedMigrationBundle(loaded); err == nil || !strings.Contains(err.Error(), "predates reviewed bundle digests") || !strings.Contains(err.Error(), "to re-plan before live apply") { t.Fatalf("legacy run without a bundle digest did not require a safe re-plan before live apply: %v", err) } + originalBundleDir := loaded.Run.BundleDir + originalArtifacts := loaded.Run.Artifacts + operationLock, err := acquireRunOperationLock(loaded.Run.RunDir) + if err != nil { + t.Fatal(err) + } + upgraded, upgradeErr := ensureSelfContainedLiveRunLocked(loaded) + operationLock.Release() + if upgradeErr != nil { + t.Fatal(upgradeErr) + } + if upgraded.Run.BundleDigest == "" { + t.Fatal("legacy self-contained run did not acquire a reviewed bundle digest") + } + if upgraded.Run.BundleDir != originalBundleDir { + t.Fatalf("legacy self-contained bundle was replaced: before=%q after=%q", originalBundleDir, upgraded.Run.BundleDir) + } + if upgraded.Run.Artifacts.Prepare == originalArtifacts.Prepare { + t.Fatalf("legacy digest upgrade did not publish a new artifact generation: %#v", upgraded.Run.Artifacts) + } + if err := verifyReviewedMigrationBundle(upgraded); err != nil { + t.Fatalf("upgraded reviewed bundle did not verify: %v", err) + } + reloaded, err := loadMigrationRun("legacy-contained") + if err != nil { + t.Fatal(err) + } + if reloaded.Run.BundleDigest != upgraded.Run.BundleDigest || reloaded.Run.Artifacts != upgraded.Run.Artifacts { + t.Fatalf("legacy digest upgrade was not persisted: upgraded=%#v reloaded=%#v", upgraded.Run, reloaded.Run) + } +} + +func TestLegacySelfContainedAppliedRunRetainsResumeStateDuringDigestUpgrade(t *testing.T) { + workDir := t.TempDir() + t.Chdir(workDir) + bundleDir := filepath.Join(workDir, "bort-bundle") + writeTestBundle(t, bundleDir, manifest.Manifest{ + Source: manifest.Source{Platform: "docker"}, + Apps: []manifest.App{{Name: "api", Services: []manifest.Service{{Name: "api", Image: "example/api:v1"}}}}, + }) + runCommand(t, runMigrate, []string{"--bundle", bundleDir, "--run", "legacy-applied"}) + run, err := loadMigrationRun("legacy-applied") + if err != nil { + t.Fatal(err) + } + steps := dokploy.PlanFromArtifacts(run.Prepare, run.Sync, run.Cutover).Steps + if len(steps) == 0 { + t.Fatal("expected legacy live steps") + } + run.Run.BundleDigest = "" + run.Applied.Steps = []appliedStep{{ + Index: 0, + Kind: string(steps[0].Kind), + App: steps[0].App, + Ref: steps[0].Ref, + Status: string(dokploy.StepStatusOK), + }} + if err := writeRunApplied(runArtifactPath(run.Run.RunDir, run.Run.Artifacts.Applied), run.Applied); err != nil { + t.Fatal(err) + } + if err := writeJSONArtifact(filepath.Join(run.Run.RunDir, "run.json"), run.Run); err != nil { + t.Fatal(err) + } + loaded, err := loadMigrationRun("legacy-applied") + if err != nil { + t.Fatal(err) + } + operationLock, err := acquireRunOperationLock(loaded.Run.RunDir) + if err != nil { + t.Fatal(err) + } + upgraded, upgradeErr := ensureSelfContainedLiveRunLocked(loaded) + operationLock.Release() + if upgradeErr != nil { + t.Fatal(upgradeErr) + } + if upgraded.Run.BundleDigest == "" || len(upgraded.Applied.Steps) != 1 || upgraded.Applied.Steps[0] != run.Applied.Steps[0] { + t.Fatalf("legacy digest upgrade lost applied resume state: run=%#v applied=%#v", upgraded.Run, upgraded.Applied) + } + if err := verifyReviewedMigrationBundle(upgraded); err != nil { + t.Fatalf("upgraded applied run bundle did not verify: %v", err) + } + reloaded, err := loadMigrationRun("legacy-applied") + if err != nil { + t.Fatal(err) + } + if reloaded.Run.BundleDigest != upgraded.Run.BundleDigest || len(reloaded.Applied.Steps) != 1 || reloaded.Applied.Steps[0] != run.Applied.Steps[0] { + t.Fatalf("persisted legacy digest upgrade lost applied resume state: run=%#v applied=%#v", reloaded.Run, reloaded.Applied) + } } func TestLoadMigrationRunDoesNotApplyLaterWorkspaceState(t *testing.T) { From 8b285366b533ebbde53040acb94d0e95a633e652 Mon Sep 17 00:00:00 2001 From: Aikins Laryea Date: Mon, 3 Aug 2026 09:19:44 +0000 Subject: [PATCH 13/31] migration: preserve operator run selection (#20) --- internal/cli/commit.go | 3 + internal/cli/commit_test.go | 20 +++++++ internal/cli/guide.go | 6 -- internal/cli/guide_test.go | 110 ++++++++++++++++++++++++++++++++++ internal/cli/rollback.go | 3 + internal/cli/rollback_test.go | 18 ++++++ 6 files changed, 154 insertions(+), 6 deletions(-) diff --git a/internal/cli/commit.go b/internal/cli/commit.go index 0ee411f..4e03099 100644 --- a/internal/cli/commit.go +++ b/internal/cli/commit.go @@ -43,6 +43,9 @@ func runCommit(ctx context.Context, args []string, stdout, stderr io.Writer) err if fs.NArg() > 0 { return fmt.Errorf("commit does not accept positional argument %q", fs.Arg(0)) } + if flagSet(fs, "run") && strings.TrimSpace(runRef) == "" { + return fmt.Errorf("commit requires a non-empty --run value") + } if apply { for _, name := range []string{"app", "bundle", "format", "from-cutover", "output", "rollback-window", "target"} { if flagSet(fs, name) { diff --git a/internal/cli/commit_test.go b/internal/cli/commit_test.go index 9ddcc38..21430ca 100644 --- a/internal/cli/commit_test.go +++ b/internal/cli/commit_test.go @@ -117,6 +117,26 @@ func TestRunCommitApplyRejectsPositionalArguments(t *testing.T) { } } +func TestRunCommitRejectsEmptyExplicitRun(t *testing.T) { + t.Chdir(t.TempDir()) + if err := mutateBortState(defaultStatePath(), func(state *bortState) bool { + state.CurrentRun = ".bort/runs/current" + return true + }); err != nil { + t.Fatal(err) + } + for _, args := range [][]string{{"--run="}, {"--apply", "--run="}} { + var stdout bytes.Buffer + err := runCommit(context.Background(), args, &stdout, io.Discard) + if err == nil || err.Error() != "commit requires a non-empty --run value" { + t.Fatalf("args=%v: expected empty run rejection, got %v", args, err) + } + if stdout.Len() != 0 { + t.Fatalf("args=%v: empty run produced output before rejection: %q", args, stdout.String()) + } + } +} + func TestRunCommitDefaultsToCurrentRun(t *testing.T) { workDir := t.TempDir() t.Chdir(workDir) diff --git a/internal/cli/guide.go b/internal/cli/guide.go index 60d1899..a5d2c47 100644 --- a/internal/cli/guide.go +++ b/internal/cli/guide.go @@ -57,9 +57,6 @@ func runGuide(ctx context.Context, stdin io.Reader, stdout, stderr io.Writer) er if err != nil { return err } - if err := rememberCurrentRun(run.Run); err != nil { - return err - } writeAppFirstCockpit(stdout, run) return nil } @@ -67,9 +64,6 @@ func runGuide(ctx context.Context, stdin io.Reader, stdout, stderr io.Writer) er if err != nil { return err } - if err := rememberCurrentRun(run.Run); err != nil { - return err - } if isRealTTY(stdin, stdout) { return runWizard(ctx, run, stdin, stdout, stderr) } diff --git a/internal/cli/guide_test.go b/internal/cli/guide_test.go index 854f78f..3caae5c 100644 --- a/internal/cli/guide_test.go +++ b/internal/cli/guide_test.go @@ -8,6 +8,7 @@ import ( "path/filepath" "strings" "testing" + "time" "github.com/aikins01/bort/internal/manifest" ) @@ -166,6 +167,115 @@ func TestRunWithoutArgsDoesNotPromoteOrRefreshMtimeFallback(t *testing.T) { } } +func TestRunWithoutArgsDoesNotRestoreStaleCurrentRunAfterRefresh(t *testing.T) { + workDir := t.TempDir() + t.Chdir(workDir) + oldBundle := filepath.Join(workDir, "old-bundle") + writeTestBundle(t, oldBundle, manifest.Manifest{ + Source: manifest.Source{Platform: "docker"}, + Apps: []manifest.App{{Name: "old", Services: []manifest.Service{{Name: "old", Image: "example/old:v1"}}}}, + }) + runCommand(t, runMigrate, []string{"--bundle", oldBundle, "--run", "old-current"}) + oldRun, err := loadMigrationRun("old-current") + if err != nil { + t.Fatal(err) + } + manifestPath, err := writeRunSourceManifest(oldRun.Run.RunDir, manifest.Manifest{ + Source: manifest.Source{Platform: "docker"}, + Apps: []manifest.App{{Name: "old", Services: []manifest.Service{{Name: "old", Image: "example/old:v1"}}}}, + }) + if err != nil { + t.Fatal(err) + } + oldRun.Run.Source = "docker" + oldRun.Run.ManifestPath = manifestPath + if err := writeJSONArtifact(filepath.Join(oldRun.Run.RunDir, "run.json"), oldRun.Run); err != nil { + t.Fatal(err) + } + runRef, currentSelected, found, err := guideRunRef() + if err != nil || !found || !currentSelected || runRef != oldRun.Run.RunDir { + t.Fatalf("unexpected initial guide selection: ref=%q current=%t found=%t err=%v", runRef, currentSelected, found, err) + } + applyActive, err := applyRunActive(runRef) + if err != nil || applyActive { + t.Fatalf("unexpected initial apply state: active=%t err=%v", applyActive, err) + } + if !shouldAutoRescanRun(oldRun.Run) { + t.Fatal("old current run is not eligible for source refresh") + } + binDir := t.TempDir() + dockerPath := filepath.Join(binDir, "docker") + t.Setenv("PATH", binDir+string(os.PathListSeparator)+os.Getenv("PATH")) + enteredPath := filepath.Join(workDir, "guide-entered") + releasePath := filepath.Join(workDir, "guide-release") + dockerScript := `#!/bin/sh +if [ "$1" = "ps" ]; then + touch "$BORT_GUIDE_ENTERED" + while [ ! -f "$BORT_GUIDE_RELEASE" ]; do sleep 0.01; done + printf 'reviewed-container\n' +elif [ "$1" = "inspect" ]; then + printf '%s\n' '[{"Id":"reviewed-container","Name":"/reviewed","Image":"sha256:reviewed","Config":{"Image":"example/reviewed:v2","Labels":{}},"State":{"Status":"running"},"Mounts":[],"NetworkSettings":{"Ports":{},"Networks":{}}}]' +elif [ "$1" = "image" ]; then + printf '%s\n' '[{"Id":"sha256:reviewed","RepoDigests":[]}]' +fi +` + if err := os.WriteFile(dockerPath, []byte(dockerScript), 0o700); err != nil { + t.Fatal(err) + } + t.Setenv("BORT_GUIDE_ENTERED", enteredPath) + t.Setenv("BORT_GUIDE_RELEASE", releasePath) + t.Cleanup(func() { _ = os.WriteFile(releasePath, nil, 0o600) }) + errCh := make(chan error, 1) + go func() { + errCh <- runGuide(context.Background(), strings.NewReader(""), io.Discard, io.Discard) + }() + deadline := time.Now().Add(5 * time.Second) + for { + select { + case err := <-errCh: + t.Fatalf("guide exited before source refresh: %v", err) + default: + } + if _, err := os.Stat(enteredPath); err == nil { + break + } else if !os.IsNotExist(err) { + t.Fatal(err) + } + if time.Now().After(deadline) { + t.Fatal("guide refresh did not start") + } + time.Sleep(10 * time.Millisecond) + } + newBundle := filepath.Join(workDir, "new-bundle") + writeTestBundle(t, newBundle, manifest.Manifest{ + Source: manifest.Source{Platform: "docker"}, + Apps: []manifest.App{{Name: "new", Services: []manifest.Service{{Name: "new", Image: "example/new:v1"}}}}, + }) + runCommand(t, runMigrate, []string{"--bundle", newBundle, "--run", "new-current"}) + newRun, err := loadMigrationRun("new-current") + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(releasePath, nil, 0o600); err != nil { + t.Fatal(err) + } + select { + case err := <-errCh: + if err != nil { + t.Fatalf("stale guide refresh failed: %v", err) + } + case <-time.After(5 * time.Second): + t.Fatal("stale guide refresh did not finish") + } + current, ok, err := currentRunRef() + if err != nil { + t.Fatal(err) + } + if !ok || current != filepath.ToSlash(filepath.Clean(newRun.Run.RunDir)) { + t.Fatalf("stale guide for %q replaced newer current run %q with %q", oldRun.Run.RunDir, newRun.Run.RunDir, current) + } +} + func TestRunWithoutArgsSurfacesApplyLockProbeError(t *testing.T) { workDir := t.TempDir() t.Chdir(workDir) diff --git a/internal/cli/rollback.go b/internal/cli/rollback.go index 9a4526c..df7df21 100644 --- a/internal/cli/rollback.go +++ b/internal/cli/rollback.go @@ -38,6 +38,9 @@ func runRollback(_ context.Context, args []string, stdout, stderr io.Writer) err if fs.NArg() > 0 { return fmt.Errorf("rollback does not accept positional argument %q", fs.Arg(0)) } + if flagSet(fs, "run") && strings.TrimSpace(runRef) == "" { + return fmt.Errorf("rollback requires a non-empty --run value") + } if err := checkOutputFormat("rollback", format); err != nil { return err } diff --git a/internal/cli/rollback_test.go b/internal/cli/rollback_test.go index 06b1465..8c06d73 100644 --- a/internal/cli/rollback_test.go +++ b/internal/cli/rollback_test.go @@ -99,6 +99,24 @@ func TestRunRollbackRejectsPositionalArguments(t *testing.T) { } } +func TestRunRollbackRejectsEmptyExplicitRun(t *testing.T) { + t.Chdir(t.TempDir()) + if err := mutateBortState(defaultStatePath(), func(state *bortState) bool { + state.CurrentRun = ".bort/runs/current" + return true + }); err != nil { + t.Fatal(err) + } + var stdout bytes.Buffer + err := runRollback(context.Background(), []string{"--run="}, &stdout, io.Discard) + if err == nil || err.Error() != "rollback requires a non-empty --run value" { + t.Fatalf("expected empty run rejection, got %v", err) + } + if stdout.Len() != 0 { + t.Fatalf("empty run produced output before rejection: %q", stdout.String()) + } +} + func TestRunRollbackDefaultsToCurrentRun(t *testing.T) { workDir := t.TempDir() t.Chdir(workDir) From 556042d79e0833ce4827804ad94ced514cdd6414 Mon Sep 17 00:00:00 2001 From: Aikins Laryea Date: Mon, 3 Aug 2026 12:38:41 +0000 Subject: [PATCH 14/31] cleanup: pin source network identity (#21) * cleanup: pin source network identity * tests: use private cleanup directories --- internal/cli/cleanup.go | 130 ++++++++++++-- internal/cli/cleanup_test.go | 131 +++++++++++++- internal/safepath/safepath.go | 72 ++++++++ internal/safepath/safepath_other.go | 144 +++++++++++++++ internal/safepath/safepath_unix.go | 29 +++ internal/source/localdocker/scanner.go | 4 +- internal/source/localdocker/scanner_test.go | 11 +- internal/target/dokploy/cleanup.go | 150 ++++++++++++++-- internal/target/dokploy/cleanup_test.go | 187 ++++++++++++++++++-- 9 files changed, 804 insertions(+), 54 deletions(-) diff --git a/internal/cli/cleanup.go b/internal/cli/cleanup.go index 6dc5daa..28f4abb 100644 --- a/internal/cli/cleanup.go +++ b/internal/cli/cleanup.go @@ -14,6 +14,7 @@ import ( "time" "github.com/aikins01/bort/internal/analyzer" + "github.com/aikins01/bort/internal/manifest" "github.com/aikins01/bort/internal/planutil" "github.com/aikins01/bort/internal/preparer" "github.com/aikins01/bort/internal/safepath" @@ -86,11 +87,12 @@ type cleanupSourceVolume struct { } type cleanupSourceNetwork struct { - App string `json:"app"` - Name string `json:"name"` - ExpectedIdentity string `json:"expectedIdentity,omitempty"` - ExpectedAbsent bool `json:"expectedAbsent,omitempty"` - Action string `json:"action"` + App string `json:"app"` + Name string `json:"name"` + DiscoveredIdentity string `json:"discoveredIdentity,omitempty"` + ExpectedIdentity string `json:"expectedIdentity,omitempty"` + ExpectedAbsent bool `json:"expectedAbsent,omitempty"` + Action string `json:"action"` } type cleanupTargetArtifact struct { @@ -444,7 +446,11 @@ func planCleanupPurge(run loadedMigrationRun, target string, filters cleanupPurg return cleanupPurgeResult{}, err } selectedRefs := cleanupSelectedAppRefs(selected) - owners, err := cleanupPurgeResourceOwners(run) + networkIdentities, err := cleanupManifestNetworkIdentities(run) + if err != nil { + return cleanupPurgeResult{}, err + } + owners, err := cleanupPurgeResourceOwners(run, networkIdentities) if err != nil { return cleanupPurgeResult{}, err } @@ -457,6 +463,7 @@ func planCleanupPurge(run loadedMigrationRun, target string, filters cleanupPurg CompletesLifecycle: cleanupPurgeCoversAllRunApps(run, filters), Filters: filters, } + manualNetworkCompletion := false for _, app := range selected { if control := cleanupSourceControlForApp(app); control != nil { @@ -522,7 +529,7 @@ func planCleanupPurge(run loadedMigrationRun, target string, filters cleanupPurg } result.SourceVolumes = append(result.SourceVolumes, volume) } - networks, err := cleanupNetworksForApp(run, app) + networks, err := cleanupNetworksForApp(run, app, networkIdentities) if err != nil { return cleanupPurgeResult{}, fmt.Errorf("inspect source networks for %s before purge: %w", app.Name, err) } @@ -541,7 +548,13 @@ func planCleanupPurge(run loadedMigrationRun, target string, filters cleanupPurg result.Warnings = append(result.Warnings, fmt.Sprintf("source network %s for %s is also referenced by unselected app(s): %s", network.Name, app.Name, strings.Join(sharedWith, ", "))) continue } - network.Action = "remove_on_purge_apply" + if identity := strings.TrimSpace(network.DiscoveredIdentity); len(identity) != 64 || !dokploy.ValidSourcePurgeNetworkID(identity) { + manualNetworkCompletion = true + network.Action = "require_absent_before_purge_apply" + result.Warnings = append(result.Warnings, fmt.Sprintf("source network %s for %s has no canonical recorded ID and must be removed manually before cleanup purge --apply; Bort will verify it remains absent", network.Name, app.Name)) + } else { + network.Action = "remove_on_purge_apply" + } result.SourceNetworks = append(result.SourceNetworks, network) } } @@ -563,7 +576,7 @@ func planCleanupPurge(run loadedMigrationRun, target string, filters cleanupPurg if filters.AllApps && !cleanupPurgeCoversAllRunApps(run, filters) { result.Warnings = append(result.Warnings, "--all-apps excludes platform-role apps unless --include-platform is set; this purge will not complete the migration lifecycle") } - if len(cleanupPurgeVolumeChecks(result.SourceVolumes)) > 0 || len(result.SourcePaths) > 0 { + if len(cleanupPurgeVolumeChecks(result.SourceVolumes)) > 0 || len(result.SourcePaths) > 0 || manualNetworkCompletion { setCleanupPurgeManualCompletion(&result) } if filters.AllApps && !result.CompletesLifecycle { @@ -629,7 +642,7 @@ func cleanupPurgeAppRef(app preparer.AppPlan) string { return "" } -func cleanupPurgeResourceOwners(run loadedMigrationRun) (cleanupPurgeOwners, error) { +func cleanupPurgeResourceOwners(run loadedMigrationRun, networkIdentities map[string]string) (cleanupPurgeOwners, error) { owners := cleanupPurgeOwners{ ContainerIDs: map[string][]string{}, ContainerNames: map[string][]string{}, @@ -667,7 +680,7 @@ func cleanupPurgeResourceOwners(run loadedMigrationRun) (cleanupPurgeOwners, err } } } - networks, err := cleanupNetworksForApp(run, app) + networks, err := cleanupNetworksForApp(run, app, networkIdentities) if err != nil { return cleanupPurgeOwners{}, fmt.Errorf("inspect source networks for %s before purge: %w", app.Name, err) } @@ -1032,7 +1045,7 @@ func cleanupPurgeOptions(result cleanupPurgeResult) dokploy.SourcePurgeOptions { opts.Volumes = append(opts.Volumes, dokploy.SourcePurgeVolume{App: volume.App, Service: volume.Service, Name: volume.Name, ExpectedAbsent: volume.ExpectedAbsent}) } for _, network := range result.SourceNetworks { - opts.Networks = append(opts.Networks, dokploy.SourcePurgeNetwork{App: network.App, Name: network.Name, ExpectedIdentity: network.ExpectedIdentity, ExpectedAbsent: network.ExpectedAbsent}) + opts.Networks = append(opts.Networks, dokploy.SourcePurgeNetwork{App: network.App, Name: network.Name, DiscoveredIdentity: network.DiscoveredIdentity, ExpectedIdentity: network.ExpectedIdentity, ExpectedAbsent: network.ExpectedAbsent}) } for _, path := range result.SourcePaths { opts.Paths = append(opts.Paths, dokploy.SourcePurgePath{App: path.App, Source: path.Source, Path: path.Path, AllowPlatform: path.AllowPlatform, ExpectedAbsent: path.ExpectedAbsent}) @@ -1300,6 +1313,10 @@ func planCleanup(ctx context.Context, run loadedMigrationRun, target string) cle DryRun: true, } result.StalePlatformRecords, result.Warnings = inspectStalePlatformRecords(ctx, target) + networkIdentities, networkIdentityErr := cleanupManifestNetworkIdentities(run) + if networkIdentityErr != nil { + result.Warnings = append(result.Warnings, fmt.Sprintf("inspect source network identities: %v", networkIdentityErr)) + } for _, record := range result.StalePlatformRecords { status := "planned" message := "remove Dokploy metadata only if the project still exists and has zero domains" @@ -1327,10 +1344,13 @@ func planCleanup(ctx context.Context, run loadedMigrationRun, target string) cle result.SourceContainers = append(result.SourceContainers, containers...) } result.SourceVolumes = append(result.SourceVolumes, cleanupVolumesForApp(app)...) - networks, err := cleanupNetworksForApp(run, app) + networks, err := cleanupNetworksForApp(run, app, networkIdentities) if err != nil { result.Warnings = append(result.Warnings, fmt.Sprintf("inspect source networks for %s: %v", app.Name, err)) } else { + for i := range networks { + networks[i].Action = "preserve_until_manual_purge_after_acceptance" + } result.SourceNetworks = append(result.SourceNetworks, networks...) } result.TargetArtifacts = append(result.TargetArtifacts, cleanupTargetArtifactsForApp(app)...) @@ -1570,7 +1590,7 @@ func cleanupVolumesForApp(app preparer.AppPlan) []cleanupSourceVolume { return volumes } -func cleanupNetworksForApp(run loadedMigrationRun, app preparer.AppPlan) ([]cleanupSourceNetwork, error) { +func cleanupNetworksForApp(run loadedMigrationRun, app preparer.AppPlan, identities map[string]string) ([]cleanupSourceNetwork, error) { topology, err := cleanupReadTopology(run, app) if err != nil { return nil, err @@ -1580,14 +1600,86 @@ func cleanupNetworksForApp(run loadedMigrationRun, app preparer.AppPlan) ([]clea } networks := make([]cleanupSourceNetwork, 0, len(topology.Networks)) for _, network := range topology.Networks { - if strings.TrimSpace(network) == "" { + network = strings.TrimSpace(network) + if network == "" { continue } - networks = append(networks, cleanupSourceNetwork{App: app.Name, Name: network, Action: "preserve_until_manual_purge_after_acceptance"}) + networks = append(networks, cleanupSourceNetwork{App: app.Name, Name: network, DiscoveredIdentity: identities[network]}) } return networks, nil } +func cleanupManifestNetworkIdentities(run loadedMigrationRun) (map[string]string, error) { + identities := map[string]string{} + manifestPath := strings.TrimSpace(run.Run.ManifestPath) + if manifestPath == "" { + return identities, nil + } + runDir := filepath.FromSlash(run.Run.RunDir) + path := filepath.FromSlash(manifestPath) + name := filepath.Base(path) + if path != name { + manifestAbsolute, err := filepath.Abs(path) + if err != nil { + return nil, err + } + expectedAbsolute, err := filepath.Abs(filepath.Join(runDir, name)) + if err != nil { + return nil, err + } + if manifestAbsolute != expectedAbsolute { + return nil, fmt.Errorf("source manifest path %s is outside migration run %s", manifestPath, run.Run.RunDir) + } + } + contents, err := safepath.ReadPrivateFile(runDir, name) + if err != nil { + return nil, fmt.Errorf("read source manifest network identities: %w", err) + } + var sourceManifest manifest.Manifest + if err := json.Unmarshal(contents, &sourceManifest); err != nil { + return nil, fmt.Errorf("decode source manifest network identities: %w", err) + } + conflicts := map[string]struct{}{} + record := func(name, identity string) { + name = strings.TrimSpace(name) + identity = strings.TrimSpace(identity) + if name == "" || identity == "" { + return + } + if !dokploy.ValidSourcePurgeNetworkID(identity) { + delete(identities, name) + conflicts[name] = struct{}{} + return + } + if _, conflict := conflicts[name]; conflict { + return + } + if current, ok := identities[name]; ok { + if current != identity && !dokploy.SourcePurgeNetworkIDsEquivalent(current, identity) { + delete(identities, name) + conflicts[name] = struct{}{} + return + } + if len(identity) > len(current) { + identities[name] = identity + } + return + } + identities[name] = identity + } + for _, network := range sourceManifest.Networks { + record(network.Name, network.ID) + } + for _, app := range sourceManifest.Apps { + for _, service := range app.Services { + for _, network := range service.Networks { + record(network.Name, network.NetworkID) + } + } + } + return identities, nil +} + func cleanupReadTopology(run loadedMigrationRun, app preparer.AppPlan) (analyzer.Topology, error) { bundleDir := run.Prepare.BundleDir appDir := filepath.Join(bundleDir, filepath.FromSlash(app.Directory)) @@ -1812,11 +1904,15 @@ func writeCleanupPurgeResourceResults(w io.Writer, label string, items []dokploy } fmt.Fprintf(w, " %s:\n", label) for _, item := range items { + identity := "" + if strings.TrimSpace(item.Identity) != "" { + identity = " [id: " + strings.TrimSpace(item.Identity) + "]" + } message := item.Message if message != "" { message = ": " + message } - fmt.Fprintf(w, " [%s] %s%s\n", item.Status, item.Ref, message) + fmt.Fprintf(w, " [%s] %s%s%s\n", item.Status, item.Ref, identity, message) } } diff --git a/internal/cli/cleanup_test.go b/internal/cli/cleanup_test.go index 51960de..3026160 100644 --- a/internal/cli/cleanup_test.go +++ b/internal/cli/cleanup_test.go @@ -10,6 +10,7 @@ import ( "net/http/httptest" "os" "path/filepath" + "runtime" "strings" "testing" "time" @@ -289,6 +290,127 @@ func TestRunCleanupPurgeInventoriesDestructiveSourceResources(t *testing.T) { } } +func TestPlanCleanupPurgePinsDiscoveredSourceNetworkIdentity(t *testing.T) { + workDir := cleanupBackupTestDir(t) + t.Chdir(workDir) + fullID := "123456789abc" + strings.Repeat("d", 52) + manifestPath := filepath.Join(workDir, "manifest.json") + if err := writeJSONArtifact(manifestPath, manifest.Manifest{ + Source: manifest.Source{Platform: "coolify-local"}, + Networks: []manifest.Network{{ID: fullID, Name: "api-net"}, {ID: "abcdef123456", Name: "legacy-net"}}, + Apps: []manifest.App{{ + Name: "api", + Services: []manifest.Service{{ + ID: "cid123", + Name: "web", + Image: "example/api:latest", + Networks: []manifest.ServiceNetwork{{Name: " api-net ", NetworkID: fullID}, {Name: "legacy-net", NetworkID: "abcdef123456"}}, + }}, + }}, + }); err != nil { + t.Fatal(err) + } + runCommand(t, runMigrate, []string{"--manifest", manifestPath, "--run", "network-identity"}) + run, err := loadMigrationRun("network-identity") + if err != nil { + t.Fatal(err) + } + result, err := planCleanupPurge(run, "dokploy", cleanupPurgeFilters{AllApps: true}, nil) + if err != nil { + t.Fatal(err) + } + if len(result.SourceNetworks) != 2 { + t.Fatalf("expected immutable source network identity in purge plan, got %#v", result.SourceNetworks) + } + networks := map[string]cleanupSourceNetwork{} + for _, network := range result.SourceNetworks { + networks[network.Name] = network + } + if networks["api-net"].DiscoveredIdentity != fullID || networks["api-net"].Action != "require_absent_before_purge_apply" { + t.Fatalf("expected manual completion to require canonical network absence too, got %#v", networks["api-net"]) + } + if networks["legacy-net"].DiscoveredIdentity != "abcdef123456" || networks["legacy-net"].Action != "require_absent_before_purge_apply" { + t.Fatalf("expected legacy network identity to require manual removal, got %#v", networks["legacy-net"]) + } + if !result.ManualCompletion || result.CompletesLifecycle { + t.Fatalf("expected legacy network identity to force manual completion, got %#v", result) + } +} + +func TestCleanupManifestNetworkIdentitiesRejectsPersistentConflict(t *testing.T) { + workDir := cleanupBackupTestDir(t) + t.Chdir(workDir) + runDir := filepath.Join(".bort", "runs", "network-conflict") + if err := os.MkdirAll(runDir, 0o700); err != nil { + t.Fatal(err) + } + manifestPath := filepath.Join(runDir, "manifest.json") + if err := writeJSONArtifact(manifestPath, manifest.Manifest{ + Networks: []manifest.Network{ + {ID: strings.Repeat("a", 64), Name: "api-net"}, + {ID: strings.Repeat("b", 64), Name: "api-net"}, + {ID: strings.Repeat("a", 64), Name: "api-net"}, + {ID: strings.Repeat("c", 64), Name: "stable-net"}, + {ID: strings.Repeat("d", 65), Name: "invalid-net"}, + {ID: strings.Repeat("d", 64), Name: "invalid-net"}, + {ID: "eeeeeeeeeeee", Name: "legacy-net"}, + {ID: strings.Repeat("e", 64), Name: "legacy-net"}, + {ID: "ffffffffffff", Name: "legacy-only-net"}, + }, + }); err != nil { + t.Fatal(err) + } + identities, err := cleanupManifestNetworkIdentities(loadedMigrationRun{Run: migrationRun{ + RunDir: filepath.ToSlash(runDir), + ManifestPath: filepath.ToSlash(manifestPath), + }}) + if err != nil { + t.Fatal(err) + } + if _, ok := identities["api-net"]; ok { + t.Fatalf("conflicting network identity was restored by a later duplicate: %#v", identities) + } + if identities["stable-net"] != strings.Repeat("c", 64) { + t.Fatalf("stable network identity was lost: %#v", identities) + } + if _, ok := identities["invalid-net"]; ok { + t.Fatalf("invalid network identity was restored by a later valid record: %#v", identities) + } + if _, ok := identities["legacy-net"]; ok { + t.Fatalf("legacy short network identity was restored by a later full record: %#v", identities) + } + if identities["legacy-only-net"] != "ffffffffffff" { + t.Fatalf("legacy short network identity was not preserved for manual cleanup: %#v", identities) + } +} + +func TestCleanupManifestNetworkIdentitiesRejectsManifestSymlink(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("symlink test") + } + workDir := cleanupBackupTestDir(t) + t.Chdir(workDir) + runDir := filepath.Join(".bort", "runs", "manifest-link") + if err := os.MkdirAll(runDir, 0o700); err != nil { + t.Fatal(err) + } + outsidePath := filepath.Join(workDir, "outside-manifest.json") + if err := writeJSONArtifact(outsidePath, manifest.Manifest{Networks: []manifest.Network{{ID: "aaaaaaaaaaaa", Name: "api-net"}}}); err != nil { + t.Fatal(err) + } + manifestPath := filepath.Join(runDir, "manifest.json") + if err := os.Symlink(outsidePath, manifestPath); err != nil { + t.Fatal(err) + } + _, err := cleanupManifestNetworkIdentities(loadedMigrationRun{Run: migrationRun{ + RunDir: filepath.ToSlash(runDir), + ManifestPath: filepath.ToSlash(manifestPath), + }}) + if err == nil { + t.Fatal("expected source manifest symlink to be rejected") + } +} + func TestRunCleanupPurgeApplyRequiresExplicitScopeAndConfirmation(t *testing.T) { workDir := t.TempDir() t.Chdir(workDir) @@ -704,8 +826,8 @@ func TestCleanupPurgePlatformNetworkRequiresExplicitInclusion(t *testing.T) { if err != nil { t.Fatal(err) } - if !withPlatform.CompletesLifecycle || len(withPlatform.SourceNetworks) != 1 || withPlatform.SourceNetworks[0].Name != "coolify" { - t.Fatalf("expected included platform network to be scheduled, got %#v", withPlatform) + if withPlatform.CompletesLifecycle || !withPlatform.ManualCompletion || len(withPlatform.SourceNetworks) != 1 || withPlatform.SourceNetworks[0].Name != "coolify" || withPlatform.SourceNetworks[0].Action != "require_absent_before_purge_apply" { + t.Fatalf("expected included platform network without a recorded identity to require manual absence, got %#v", withPlatform) } } @@ -891,6 +1013,7 @@ func TestCleanupRecoveryCommandsPreserveExternalRunAndPurgeScope(t *testing.T) { } func TestWriteCleanupPurgeTextWarnsAfterPartialApply(t *testing.T) { + networkID := strings.Repeat("a", 64) result := cleanupPurgeResult{ RunName: "partial", RunDir: filepath.Join(".bort", "runs", "partial"), @@ -898,11 +1021,11 @@ func TestWriteCleanupPurgeTextWarnsAfterPartialApply(t *testing.T) { PurgeResult: &dokploy.SourcePurgeResult{Containers: []dokploy.SourcePurgeResourceResult{ {Ref: "web", Status: "removed"}, {Ref: "worker", Status: "error"}, - }}, + }, Networks: []dokploy.SourcePurgeResourceResult{{Ref: "api-net", Identity: networkID, Status: "removed"}}}, } var output strings.Builder writeCleanupPurgeText(&output, result) - for _, want := range []string{"Mode: incomplete", "[removed] web", "[error] worker", "Earlier resources may already have been removed"} { + for _, want := range []string{"Mode: incomplete", "[removed] web", "[error] worker", "[removed] api-net [id: " + networkID + "]", "Earlier resources may already have been removed"} { if !strings.Contains(output.String(), want) { t.Fatalf("expected partial purge output to contain %q, got:\n%s", want, output.String()) } diff --git a/internal/safepath/safepath.go b/internal/safepath/safepath.go index 4aff6f4..d29f791 100644 --- a/internal/safepath/safepath.go +++ b/internal/safepath/safepath.go @@ -1,6 +1,7 @@ package safepath import ( + "bytes" "crypto/rand" "encoding/hex" "errors" @@ -88,6 +89,77 @@ func (d *PrivateDir) CreateFile(name string, mode os.FileMode) (*os.File, error) return createPrivateFileNoFollow(dir, name, mode) } +func (d *PrivateDir) ReadFile(name string) ([]byte, error) { + if err := validatePrivateFileName(name); err != nil { + return nil, err + } + dir, err := d.openFile() + if err != nil { + return nil, err + } + file, err := openPrivateFileNoFollow(dir, name) + if err != nil { + return nil, err + } + defer file.Close() + info, err := file.Stat() + if err != nil { + return nil, err + } + if !info.Mode().IsRegular() { + return nil, fmt.Errorf("private file %s is not regular", name) + } + contents, err := io.ReadAll(file) + if err != nil { + return nil, err + } + afterRead, err := file.Stat() + if err != nil { + return nil, err + } + if !samePrivateFileState(info, afterRead) { + return nil, fmt.Errorf("private file %s changed during read", name) + } + if _, err := file.Seek(0, io.SeekStart); err != nil { + return nil, err + } + verification, err := io.ReadAll(file) + if err != nil { + return nil, err + } + afterVerification, err := file.Stat() + if err != nil { + return nil, err + } + if !samePrivateFileState(afterRead, afterVerification) || !bytes.Equal(contents, verification) { + return nil, fmt.Errorf("private file %s changed during read", name) + } + current, err := openPrivateFileNoFollow(dir, name) + if err != nil { + return nil, err + } + defer current.Close() + currentInfo, err := current.Stat() + if err != nil { + return nil, err + } + if !currentInfo.Mode().IsRegular() || !samePrivateFileState(afterVerification, currentInfo) { + return nil, fmt.Errorf("private file %s changed during read", name) + } + return contents, nil +} + +func samePrivateFileState(a, b os.FileInfo) bool { + return os.SameFile(a, b) && a.Size() == b.Size() && a.Mode() == b.Mode() && a.ModTime().Equal(b.ModTime()) +} + +func ReadPrivateFile(path, name string) ([]byte, error) { + if err := validatePrivateFileName(name); err != nil { + return nil, err + } + return readPrivateFile(path, name) +} + func (d *PrivateDir) WriteFileAtomic(name string, data []byte, mode os.FileMode) error { if err := validatePrivateFileName(name); err != nil { return err diff --git a/internal/safepath/safepath_other.go b/internal/safepath/safepath_other.go index 4af5f92..7cab423 100644 --- a/internal/safepath/safepath_other.go +++ b/internal/safepath/safepath_other.go @@ -3,8 +3,12 @@ package safepath import ( + "bytes" "fmt" + "io" "os" + "path/filepath" + "strings" ) func openPrivateDirNoFollow(path string, _ bool) (*os.File, error) { @@ -15,6 +19,146 @@ func createPrivateFileNoFollow(*os.File, string, os.FileMode) (*os.File, error) return nil, fmt.Errorf("secure private file creation is unavailable on Windows") } +func openPrivateFileNoFollow(*os.File, string) (*os.File, error) { + return nil, fmt.Errorf("secure private file reading is unavailable on Windows") +} + +func readPrivateFile(path, name string) ([]byte, error) { + root, held, err := openPrivateRootNoFollow(path) + if err != nil { + return nil, err + } + defer root.Close() + before, err := root.Lstat(name) + if err != nil { + return nil, err + } + if !before.Mode().IsRegular() { + return nil, fmt.Errorf("private file %s is not regular", name) + } + file, err := root.Open(name) + if err != nil { + return nil, err + } + defer file.Close() + opened, err := file.Stat() + if err != nil { + return nil, err + } + if !opened.Mode().IsRegular() || !os.SameFile(before, opened) { + return nil, fmt.Errorf("private file %s changed during read", name) + } + contents, err := io.ReadAll(file) + if err != nil { + return nil, err + } + afterRead, err := file.Stat() + if err != nil { + return nil, err + } + if !samePrivateFileState(opened, afterRead) { + return nil, fmt.Errorf("private file %s changed during read", name) + } + if _, err := file.Seek(0, io.SeekStart); err != nil { + return nil, err + } + verification, err := io.ReadAll(file) + if err != nil { + return nil, err + } + afterVerification, err := file.Stat() + if err != nil { + return nil, err + } + if !samePrivateFileState(afterRead, afterVerification) || !bytes.Equal(contents, verification) { + return nil, fmt.Errorf("private file %s changed during read", name) + } + currentFile, err := root.Lstat(name) + if err != nil { + return nil, err + } + if !currentFile.Mode().IsRegular() || !samePrivateFileState(afterVerification, currentFile) { + return nil, fmt.Errorf("private file %s changed during read", name) + } + currentRoot, current, err := openPrivateRootNoFollow(path) + if err != nil { + return nil, err + } + defer currentRoot.Close() + if !os.SameFile(held, current) { + return nil, fmt.Errorf("private directory path changed during operation") + } + return contents, nil +} + +func openPrivateRootNoFollow(path string) (*os.Root, os.FileInfo, error) { + abs, err := filepath.Abs(path) + if err != nil { + return nil, nil, err + } + volume := filepath.VolumeName(abs) + if volume == "" { + return nil, nil, fmt.Errorf("private directory path %s has no volume", path) + } + rootPath := volume + string(os.PathSeparator) + rel, err := filepath.Rel(rootPath, abs) + if err != nil { + return nil, nil, err + } + if rel == ".." || strings.HasPrefix(rel, ".."+string(os.PathSeparator)) { + return nil, nil, fmt.Errorf("private directory path %s escapes volume root", path) + } + root, err := os.OpenRoot(rootPath) + if err != nil { + return nil, nil, err + } + held, err := root.Stat(".") + if err != nil { + root.Close() + return nil, nil, err + } + if rel == "." { + return root, held, nil + } + for _, part := range strings.Split(rel, string(os.PathSeparator)) { + if part == "" || part == "." { + continue + } + before, err := root.Lstat(part) + if err != nil { + root.Close() + return nil, nil, err + } + if !before.IsDir() || before.Mode()&os.ModeSymlink != 0 { + root.Close() + return nil, nil, fmt.Errorf("private directory component %s is not a regular directory", part) + } + next, err := root.OpenRoot(part) + if err != nil { + root.Close() + return nil, nil, err + } + opened, err := next.Stat(".") + if err != nil { + next.Close() + root.Close() + return nil, nil, err + } + if !os.SameFile(before, opened) { + next.Close() + root.Close() + return nil, nil, fmt.Errorf("private directory component %s changed while opening", part) + } + if err := root.Close(); err != nil { + next.Close() + return nil, nil, err + } + root = next + held = opened + } + return root, held, nil +} + func writePrivateFileAtomicNoFollow(*os.File, string, []byte, os.FileMode) error { return fmt.Errorf("secure private file replacement is unavailable on Windows") } diff --git a/internal/safepath/safepath_unix.go b/internal/safepath/safepath_unix.go index 6b95cba..3628e17 100644 --- a/internal/safepath/safepath_unix.go +++ b/internal/safepath/safepath_unix.go @@ -108,6 +108,35 @@ func createPrivateFileNoFollow(dir *os.File, name string, mode os.FileMode) (*os return file, nil } +func openPrivateFileNoFollow(dir *os.File, name string) (*os.File, error) { + fd, err := unix.Openat(int(dir.Fd()), name, unix.O_RDONLY|unix.O_CLOEXEC|unix.O_NOFOLLOW, 0) + if err != nil { + return nil, err + } + file := os.NewFile(uintptr(fd), filepath.Join(dir.Name(), name)) + if file == nil { + _ = unix.Close(fd) + return nil, fmt.Errorf("open private file %s", name) + } + return file, nil +} + +func readPrivateFile(path, name string) ([]byte, error) { + dir, err := OpenPrivateDirNoFollow(path) + if err != nil { + return nil, err + } + defer dir.Close() + contents, err := dir.ReadFile(name) + if err != nil { + return nil, err + } + if err := dir.ValidatePath(); err != nil { + return nil, err + } + return contents, nil +} + func writePrivateFileAtomicNoFollow(dir *os.File, name string, data []byte, mode os.FileMode) error { tmpName, err := privateTempName() if err != nil { diff --git a/internal/source/localdocker/scanner.go b/internal/source/localdocker/scanner.go index 7d1e729..6b8efd1 100644 --- a/internal/source/localdocker/scanner.go +++ b/internal/source/localdocker/scanner.go @@ -234,7 +234,7 @@ func (s *Scanner) inspectNetworks(ctx context.Context) ([]manifest.Network, erro networks := make([]manifest.Network, 0, len(inspected)) for _, network := range inspected { networks = append(networks, manifest.Network{ - ID: shortID(network.ID), + ID: network.ID, Name: network.Name, Driver: network.Driver, Scope: network.Scope, @@ -602,7 +602,7 @@ func ports(raw map[string][]portBinding) []manifest.Port { func networks(raw map[string]networkAttachInspect) []manifest.ServiceNetwork { networks := make([]manifest.ServiceNetwork, 0, len(raw)) for name, network := range raw { - networks = append(networks, manifest.ServiceNetwork{Name: name, NetworkID: shortID(network.NetworkID), IPAddress: network.IPAddress}) + networks = append(networks, manifest.ServiceNetwork{Name: name, NetworkID: network.NetworkID, IPAddress: network.IPAddress}) } sort.Slice(networks, func(i, j int) bool { return networks[i].Name < networks[j].Name }) return networks diff --git a/internal/source/localdocker/scanner_test.go b/internal/source/localdocker/scanner_test.go index 82e3946..91d8af6 100644 --- a/internal/source/localdocker/scanner_test.go +++ b/internal/source/localdocker/scanner_test.go @@ -231,6 +231,7 @@ func TestParseDuBytesAndInt64(t *testing.T) { } func TestScanPopulatesNewServiceFields(t *testing.T) { + networkID := strings.Repeat("a", 64) scanner := &Scanner{ Now: func() time.Time { return time.Unix(0, 0).UTC() }, runCommand: func(_ context.Context, args ...string) ([]byte, error) { @@ -243,7 +244,7 @@ func TestScanPopulatesNewServiceFields(t *testing.T) { "Config":{"Image":"app:1","Env":["PORT=80"],"Labels":{},"Healthcheck":{"Test":["CMD","ok"],"Interval":1000000000,"Retries":2}}, "State":{"Status":"running"}, "Mounts":[{"Type":"volume","Name":"data","Source":"/var/lib/docker/volumes/data/_data","Destination":"/data","RW":true}], - "NetworkSettings":{"Ports":{},"Networks":{}} + "NetworkSettings":{"Ports":{},"Networks":{"net":{"NetworkID":"` + networkID + `","IPAddress":"172.18.0.2"}}} }]`), nil case len(args) >= 2 && args[0] == "image" && args[1] == "inspect": return []byte(`[{"Id":"sha256:abc","RepoDigests":["registry/app@sha256:dead"]}]`), nil @@ -254,7 +255,7 @@ func TestScanPopulatesNewServiceFields(t *testing.T) { case len(args) == 3 && args[0] == "network" && args[1] == "ls" && args[2] == "-q": return []byte("n1\n"), nil case len(args) >= 2 && args[0] == "network" && args[1] == "inspect": - return []byte(`[{"Id":"n1","Name":"net","Driver":"bridge","Scope":"local"}]`), nil + return []byte(`[{"Id":"` + networkID + `","Name":"net","Driver":"bridge","Scope":"local"}]`), nil } return nil, fmt.Errorf("unexpected docker args: %v", args) }, @@ -283,6 +284,12 @@ func TestScanPopulatesNewServiceFields(t *testing.T) { if svc.Healthcheck == nil || svc.Healthcheck.Interval != "1s" || svc.Healthcheck.Retries != 2 { t.Fatalf("unexpected healthcheck: %+v", svc.Healthcheck) } + if len(svc.Networks) != 1 || svc.Networks[0].NetworkID != networkID { + t.Fatalf("expected full service network id %q, got %+v", networkID, svc.Networks) + } + if len(m.Networks) != 1 || m.Networks[0].ID != networkID { + t.Fatalf("expected full top-level network id %q, got %+v", networkID, m.Networks) + } if len(m.Volumes) != 1 { t.Fatalf("expected 1 volume, got %d", len(m.Volumes)) } diff --git a/internal/target/dokploy/cleanup.go b/internal/target/dokploy/cleanup.go index 98fe847..119da09 100644 --- a/internal/target/dokploy/cleanup.go +++ b/internal/target/dokploy/cleanup.go @@ -47,10 +47,11 @@ type SourcePurgeVolume struct { } type SourcePurgeNetwork struct { - App string `json:"app,omitempty"` - Name string `json:"name"` - ExpectedIdentity string `json:"expectedIdentity,omitempty"` - ExpectedAbsent bool `json:"expectedAbsent,omitempty"` + App string `json:"app,omitempty"` + Name string `json:"name"` + DiscoveredIdentity string `json:"discoveredIdentity,omitempty"` + ExpectedIdentity string `json:"expectedIdentity,omitempty"` + ExpectedAbsent bool `json:"expectedAbsent,omitempty"` } type SourcePurgePath struct { @@ -77,10 +78,11 @@ type SourcePurgeResult struct { } type SourcePurgeResourceResult struct { - App string `json:"app,omitempty"` - Ref string `json:"ref"` - Status string `json:"status"` - Message string `json:"message,omitempty"` + App string `json:"app,omitempty"` + Ref string `json:"ref"` + Identity string `json:"identity,omitempty"` + Status string `json:"status"` + Message string `json:"message,omitempty"` } func (c *Client) CleanupStalePlatformProjects(ctx context.Context, opts StalePlatformCleanupOptions) (StalePlatformCleanupResult, error) { @@ -114,7 +116,13 @@ func (c *Client) PurgeSourceResources(ctx context.Context, opts SourcePurgeOptio } volumes := cleanupSourcePurgeVolumes(opts.Volumes) paths := cleanupSourcePurgePaths(opts.Paths) - networks := cleanupSourcePurgeNetworks(opts.Networks) + networks, err := cleanupSourcePurgeNetworks(opts.Networks) + if err != nil { + return result, err + } + if err := validateSourcePurgeNetworksForExecution(networks); err != nil { + return result, err + } if sourcePurgeRequiresManualCompletion(volumes, paths, networks) { for _, volume := range volumes { if err := runSourcePurgeResource(&result, &result.Volumes, opts.OnProgress, SourcePurgeResourceResult{App: volume.App, Ref: volume.Name}, func() (SourcePurgeResourceResult, error) { @@ -138,7 +146,7 @@ func (c *Client) PurgeSourceResources(ctx context.Context, opts SourcePurgeOptio } } for _, network := range networks { - if err := runSourcePurgeResource(&result, &result.Networks, opts.OnProgress, SourcePurgeResourceResult{App: network.App, Ref: network.Name}, func() (SourcePurgeResourceResult, error) { + if err := runSourcePurgeResource(&result, &result.Networks, opts.OnProgress, SourcePurgeResourceResult{App: network.App, Ref: network.Name, Identity: network.ExpectedIdentity}, func() (SourcePurgeResourceResult, error) { return verifySourcePurgeNetworkAbsent(ctx, runner, network) }); err != nil { return result, err @@ -164,7 +172,7 @@ func (c *Client) PurgeSourceResources(ctx context.Context, opts SourcePurgeOptio if !network.ExpectedAbsent { continue } - if err := runSourcePurgeResource(&result, &result.Networks, opts.OnProgress, SourcePurgeResourceResult{App: network.App, Ref: network.Name}, func() (SourcePurgeResourceResult, error) { + if err := runSourcePurgeResource(&result, &result.Networks, opts.OnProgress, SourcePurgeResourceResult{App: network.App, Ref: network.Name, Identity: network.ExpectedIdentity}, func() (SourcePurgeResourceResult, error) { return purgeSourceNetwork(ctx, runner, network) }); err != nil { return result, err @@ -181,7 +189,7 @@ func (c *Client) PurgeSourceResources(ctx context.Context, opts SourcePurgeOptio if network.ExpectedAbsent { continue } - if err := runSourcePurgeResource(&result, &result.Networks, opts.OnProgress, SourcePurgeResourceResult{App: network.App, Ref: network.Name}, func() (SourcePurgeResourceResult, error) { + if err := runSourcePurgeResource(&result, &result.Networks, opts.OnProgress, SourcePurgeResourceResult{App: network.App, Ref: network.Name, Identity: network.ExpectedIdentity}, func() (SourcePurgeResourceResult, error) { return purgeSourceNetwork(ctx, runner, network) }); err != nil { return result, err @@ -264,11 +272,28 @@ func (c *Client) IdentifySourcePurgeResources(ctx context.Context, opts SourcePu identified.Volumes = append(identified.Volumes, volume) } identified.Networks = nil - for _, network := range cleanupSourcePurgeNetworks(opts.Networks) { + networks, err := cleanupSourcePurgeNetworks(opts.Networks) + if err != nil { + return SourcePurgeOptions{}, err + } + for _, network := range networks { identity, absent, err := inspectSourcePurgeNetworkIdentity(ctx, runner, network.Name) if err != nil { return SourcePurgeOptions{}, err } + discoveredIdentity := strings.TrimSpace(network.DiscoveredIdentity) + if !absent && discoveredIdentity == "" { + return SourcePurgeOptions{}, fmt.Errorf("source network %q has no stable ID from discovery; remove it manually before cleanup purge --apply", network.Name) + } + if !absent && !sourcePurgeCanonicalNetworkID(discoveredIdentity) { + return SourcePurgeOptions{}, fmt.Errorf("source network %q has non-canonical discovered ID %q; remove it manually before cleanup purge --apply", network.Name, discoveredIdentity) + } + if !absent && !sourcePurgeCanonicalNetworkID(identity) { + return SourcePurgeOptions{}, fmt.Errorf("docker returned non-canonical ID %q for source network %q", identity, network.Name) + } + if !absent && !SourcePurgeNetworkIDsEquivalent(identity, discoveredIdentity) { + return SourcePurgeOptions{}, fmt.Errorf("refusing source network %q because current ID %s does not match discovered ID %s", network.Name, identity, discoveredIdentity) + } network.ExpectedIdentity = identity network.ExpectedAbsent = absent identified.Networks = append(identified.Networks, network) @@ -349,6 +374,31 @@ func sourcePurgeContainerIDsEquivalent(a, b string) bool { return len(a) >= 12 && strings.HasPrefix(b, a) } +func ValidSourcePurgeNetworkID(identity string) bool { + identity = strings.TrimSpace(identity) + if len(identity) < 12 || len(identity) > 64 { + return false + } + for _, r := range identity { + if r < '0' || r > '9' { + if r < 'a' || r > 'f' { + return false + } + } + } + return true +} + +func sourcePurgeCanonicalNetworkID(identity string) bool { + return len(strings.TrimSpace(identity)) == 64 && ValidSourcePurgeNetworkID(identity) +} + +func SourcePurgeNetworkIDsEquivalent(a, b string) bool { + a = strings.TrimSpace(a) + b = strings.TrimSpace(b) + return sourcePurgeCanonicalNetworkID(a) && sourcePurgeCanonicalNetworkID(b) && a == b +} + func cleanupSourcePurgeContainers(containers []SourcePurgeContainer) ([]SourcePurgeContainer, error) { idsByName := map[string]string{} for _, container := range containers { @@ -434,22 +484,66 @@ func cleanupSourcePurgeVolumes(volumes []SourcePurgeVolume) []SourcePurgeVolume return cleaned } -func cleanupSourcePurgeNetworks(networks []SourcePurgeNetwork) []SourcePurgeNetwork { - seen := map[string]struct{}{} +func cleanupSourcePurgeNetworks(networks []SourcePurgeNetwork) ([]SourcePurgeNetwork, error) { + seen := map[string]int{} cleaned := []SourcePurgeNetwork{} for _, network := range networks { network.Name = strings.TrimSpace(network.Name) + network.DiscoveredIdentity = strings.TrimSpace(network.DiscoveredIdentity) + network.ExpectedIdentity = strings.TrimSpace(network.ExpectedIdentity) if network.Name == "" { continue } - if _, ok := seen[network.Name]; ok { + if network.ExpectedAbsent && network.ExpectedIdentity != "" { + return nil, fmt.Errorf("source network %q has conflicting expected identity and absence state", network.Name) + } + if index, ok := seen[network.Name]; ok { + current := &cleaned[index] + if current.DiscoveredIdentity != "" && network.DiscoveredIdentity != "" && current.DiscoveredIdentity != network.DiscoveredIdentity { + return nil, fmt.Errorf("source network %q has conflicting discovered identities", network.Name) + } + if current.ExpectedIdentity != "" && network.ExpectedIdentity != "" && current.ExpectedIdentity != network.ExpectedIdentity { + return nil, fmt.Errorf("source network %q has conflicting expected identities", network.Name) + } + if current.ExpectedAbsent != network.ExpectedAbsent { + return nil, fmt.Errorf("source network %q has conflicting expected absence state", network.Name) + } + if current.DiscoveredIdentity == "" { + current.DiscoveredIdentity = network.DiscoveredIdentity + } + if current.ExpectedIdentity == "" { + current.ExpectedIdentity = network.ExpectedIdentity + } continue } - seen[network.Name] = struct{}{} + seen[network.Name] = len(cleaned) cleaned = append(cleaned, network) } + for _, network := range cleaned { + if network.ExpectedIdentity != "" && !sourcePurgeCanonicalNetworkID(network.ExpectedIdentity) { + return nil, fmt.Errorf("source network %q has non-canonical confirmed ID %q", network.Name, network.ExpectedIdentity) + } + } sort.Slice(cleaned, func(i, j int) bool { return cleaned[i].Name < cleaned[j].Name }) - return cleaned + return cleaned, nil +} + +func validateSourcePurgeNetworksForExecution(networks []SourcePurgeNetwork) error { + for _, network := range networks { + if network.ExpectedAbsent { + continue + } + if !sourcePurgeCanonicalNetworkID(network.ExpectedIdentity) { + return fmt.Errorf("source network %q has no canonical confirmed ID", network.Name) + } + if !sourcePurgeCanonicalNetworkID(network.DiscoveredIdentity) { + return fmt.Errorf("source network %q has non-canonical discovered ID %q", network.Name, network.DiscoveredIdentity) + } + if !SourcePurgeNetworkIDsEquivalent(network.ExpectedIdentity, network.DiscoveredIdentity) { + return fmt.Errorf("source network %q has conflicting discovered and confirmed IDs", network.Name) + } + } + return nil } func cleanupSourcePurgePaths(paths []SourcePurgePath) []SourcePurgePath { @@ -614,6 +708,24 @@ func purgeSourceNetwork(ctx context.Context, runner dockerRunner, item SourcePur result.Status = "blocked" return result, fmt.Errorf("refusing to remove docker network %q without a pre-confirmation identity", name) } + if !sourcePurgeCanonicalNetworkID(identity) { + result.Status = "blocked" + return result, fmt.Errorf("refusing to remove docker network %q with non-canonical confirmed ID %q", name, identity) + } + discoveredIdentity := strings.TrimSpace(item.DiscoveredIdentity) + if discoveredIdentity == "" { + result.Status = "blocked" + return result, fmt.Errorf("refusing to remove docker network %q without a stable ID from discovery", name) + } + if !sourcePurgeCanonicalNetworkID(discoveredIdentity) { + result.Status = "blocked" + return result, fmt.Errorf("refusing to remove docker network %q with non-canonical discovered ID %q", name, discoveredIdentity) + } + if !SourcePurgeNetworkIDsEquivalent(identity, discoveredIdentity) { + result.Status = "blocked" + return result, fmt.Errorf("refusing to remove docker network %q because confirmed ID %s does not match discovered ID %s", name, identity, discoveredIdentity) + } + result.Identity = identity if _, err := runner.Output(ctx, "network", "rm", identity); err != nil { if isDockerVolumeOrNetworkMissingErr(err) { _, absent, inspectErr := inspectSourcePurgeNetworkIdentity(ctx, runner, name) @@ -638,7 +750,7 @@ func purgeSourceNetwork(ctx context.Context, runner dockerRunner, item SourcePur func verifySourcePurgeNetworkAbsent(ctx context.Context, runner dockerRunner, item SourcePurgeNetwork) (SourcePurgeResourceResult, error) { name := strings.TrimSpace(item.Name) - result := SourcePurgeResourceResult{App: item.App, Ref: name} + result := SourcePurgeResourceResult{App: item.App, Ref: name, Identity: strings.TrimSpace(item.ExpectedIdentity)} if err := validateDockerPurgeName("network", name); err != nil { result.Status = "blocked" return result, err diff --git a/internal/target/dokploy/cleanup_test.go b/internal/target/dokploy/cleanup_test.go index 70230b7..5bd3b51 100644 --- a/internal/target/dokploy/cleanup_test.go +++ b/internal/target/dokploy/cleanup_test.go @@ -144,17 +144,18 @@ func dokployBackupTestDir(t *testing.T) string { } func TestPurgeSourceResourcesRemovesDockerResources(t *testing.T) { + fullNetworkID := "123456789abc" + strings.Repeat("d", 52) runner := &fakeDockerRunner{outputs: map[string][]byte{ "inspect --type container cid123": []byte(`[{"Id":"cid123","Name":"/web","Config":{"Labels":{"coolify.managed":"true"}},"State":{"Running":false,"Status":"exited"}}]`), "rm -f cid123": []byte("cid123\n"), - "network inspect api-net": []byte(`[{"Id":"network-id","Name":"api-net"}]`), - "network rm network-id": []byte("network-id\n"), + "network inspect api-net": []byte(`[{"Id":"` + fullNetworkID + `","Name":"api-net"}]`), + "network rm " + fullNetworkID: []byte(fullNetworkID + "\n"), }} client := &Client{Docker: runner} options, err := client.IdentifySourcePurgeResources(context.Background(), SourcePurgeOptions{ Containers: []SourcePurgeContainer{{App: "api", Service: "web", ContainerID: "cid123", ContainerName: "web"}}, - Networks: []SourcePurgeNetwork{{App: "api", Name: "api-net"}}, + Networks: []SourcePurgeNetwork{{App: "api", Name: "api-net", DiscoveredIdentity: fullNetworkID}}, }) if err != nil { t.Fatalf("IdentifySourcePurgeResources: %v", err) @@ -169,11 +170,14 @@ func TestPurgeSourceResourcesRemovesDockerResources(t *testing.T) { if len(result.Networks) != 1 || result.Networks[0].Status != "removed" { t.Fatalf("unexpected network result: %#v", result.Networks) } + if result.Networks[0].Identity != fullNetworkID { + t.Fatalf("expected canonical network identity in result, got %#v", result.Networks[0]) + } gotArgs := []string{} for _, args := range runner.outputArgs { gotArgs = append(gotArgs, strings.Join(args, " ")) } - for _, want := range []string{"inspect --type container cid123", "rm -f cid123", "network inspect api-net", "network rm network-id"} { + for _, want := range []string{"inspect --type container cid123", "rm -f cid123", "network inspect api-net", "network rm " + fullNetworkID} { found := false for _, got := range gotArgs { if got == want { @@ -245,6 +249,51 @@ func TestCleanupSourcePurgeContainersDeduplicatesShortAndCanonicalIDs(t *testing } } +func TestCleanupSourcePurgeNetworksRejectsConflictingIdentityState(t *testing.T) { + for name, networks := range map[string][]SourcePurgeNetwork{ + "discovered identity": { + {Name: "api-net", DiscoveredIdentity: strings.Repeat("a", 64)}, + {Name: "api-net", DiscoveredIdentity: strings.Repeat("b", 64)}, + }, + "expected identity": { + {Name: "api-net", ExpectedIdentity: strings.Repeat("a", 64)}, + {Name: "api-net", ExpectedIdentity: strings.Repeat("b", 64)}, + }, + "expected absence": { + {Name: "api-net", ExpectedAbsent: true}, + {Name: "api-net"}, + }, + } { + t.Run(name, func(t *testing.T) { + if _, err := cleanupSourcePurgeNetworks(networks); err == nil || !strings.Contains(err.Error(), "conflicting") { + t.Fatalf("expected conflicting duplicate networks to fail closed, got %v", err) + } + }) + } + legacyID := "123456789abc" + networks, err := cleanupSourcePurgeNetworks([]SourcePurgeNetwork{ + {App: "api", Name: "api-net", DiscoveredIdentity: legacyID}, + {App: "worker", Name: "api-net", DiscoveredIdentity: legacyID}, + }) + if err != nil { + t.Fatal(err) + } + if len(networks) != 1 || networks[0].DiscoveredIdentity != legacyID { + t.Fatalf("identical legacy network IDs were not deduplicated: %#v", networks) + } + fullID := "123456789abc" + strings.Repeat("d", 52) + networks, err = cleanupSourcePurgeNetworks([]SourcePurgeNetwork{ + {App: "api", Name: "api-net", DiscoveredIdentity: fullID, ExpectedIdentity: fullID}, + {App: "worker", Name: "api-net", DiscoveredIdentity: fullID, ExpectedIdentity: fullID}, + }) + if err != nil { + t.Fatal(err) + } + if len(networks) != 1 || networks[0].DiscoveredIdentity != fullID || networks[0].ExpectedIdentity != fullID { + t.Fatalf("equivalent duplicate network IDs were not canonicalized: %#v", networks) + } +} + func TestPurgeSourceResourcesSkipsAbsentAllowedPath(t *testing.T) { client := &Client{Docker: &fakeDockerRunner{}} options, err := client.IdentifySourcePurgeResources(context.Background(), SourcePurgeOptions{Paths: []SourcePurgePath{{Path: "/data/coolify/applications/app-1"}}}) @@ -320,38 +369,134 @@ func TestPurgeSourceResourcesRejectsMismatchedInspectedContainerID(t *testing.T) } func TestPurgeSourceNetworkPreservesReplacementAfterReviewedIDDisappears(t *testing.T) { + reviewedID := "aaaaaaaaaaaa" + strings.Repeat("a", 52) + replacementID := strings.Repeat("b", 64) runner := &cleanupDockerRunner{ fakeDockerRunner: &fakeDockerRunner{outputs: map[string][]byte{ - "network inspect api-net": []byte(`[{"Id":"replacement-id","Name":"api-net"}]`), + "network inspect api-net": []byte(`[{"Id":"` + replacementID + `","Name":"api-net"}]`), }}, outputErrors: map[string]error{ - "network rm reviewed-id": errors.New("Error response from daemon: network reviewed-id not found"), + "network rm " + reviewedID: errors.New("Error response from daemon: network not found"), }, } client := &Client{Docker: runner} result, err := client.PurgeSourceResources(context.Background(), SourcePurgeOptions{ - Networks: []SourcePurgeNetwork{{Name: "api-net", ExpectedIdentity: "reviewed-id"}}, + Networks: []SourcePurgeNetwork{{Name: "api-net", DiscoveredIdentity: reviewedID, ExpectedIdentity: reviewedID}}, }) if err == nil || !strings.Contains(err.Error(), "appeared after confirmation") { t.Fatalf("expected replacement network to be preserved, got result=%#v err=%v", result, err) } for _, args := range runner.outputArgs { - if strings.Join(args, " ") == "network rm replacement-id" { + if strings.Join(args, " ") == "network rm "+replacementID { + t.Fatalf("replacement network was removed: %#v", runner.outputArgs) + } + } +} + +func TestIdentifySourcePurgeResourcesRejectsNetworkReplacementSinceDiscovery(t *testing.T) { + discoveredID := "aaaaaaaaaaaa" + strings.Repeat("a", 52) + replacementID := "aaaaaaaaaaaa" + strings.Repeat("b", 52) + runner := &fakeDockerRunner{outputs: map[string][]byte{ + "network inspect api-net": []byte(`[{"Id":"` + replacementID + `","Name":"api-net"}]`), + }} + client := &Client{Docker: runner} + _, err := client.IdentifySourcePurgeResources(context.Background(), SourcePurgeOptions{ + Networks: []SourcePurgeNetwork{{Name: "api-net", DiscoveredIdentity: discoveredID}}, + }) + if err == nil || !strings.Contains(err.Error(), "does not match discovered ID") { + t.Fatalf("expected replacement network to be rejected, got %v", err) + } + for _, args := range runner.outputArgs { + if len(args) > 1 && args[0] == "network" && args[1] == "rm" { t.Fatalf("replacement network was removed: %#v", runner.outputArgs) } } } +func TestIdentifySourcePurgeResourcesRequiresDiscoveredNetworkIdentity(t *testing.T) { + runner := &fakeDockerRunner{outputs: map[string][]byte{ + "network inspect api-net": []byte(`[{"Id":"` + strings.Repeat("a", 64) + `","Name":"api-net"}]`), + }} + client := &Client{Docker: runner} + for name, test := range map[string]struct { + identity string + want string + }{ + "missing": {want: "has no stable ID from discovery"}, + "legacy short": { + identity: "aaaaaaaaaaaa", + want: "has non-canonical discovered ID", + }, + } { + t.Run(name, func(t *testing.T) { + _, err := client.IdentifySourcePurgeResources(context.Background(), SourcePurgeOptions{ + Networks: []SourcePurgeNetwork{{Name: "api-net", DiscoveredIdentity: test.identity}}, + }) + if err == nil || !strings.Contains(err.Error(), test.want) || !strings.Contains(err.Error(), "remove it manually") { + t.Fatalf("expected discovered identity to require manual removal, got %v", err) + } + }) + } +} + +func TestPurgeSourceNetworkRejectsShortConfirmedIdentity(t *testing.T) { + client := &Client{Docker: &fakeDockerRunner{}} + progressCalls := 0 + result, err := client.PurgeSourceResources(context.Background(), SourcePurgeOptions{ + Networks: []SourcePurgeNetwork{{Name: "api-net", DiscoveredIdentity: "123456789abc", ExpectedIdentity: "123456789abc"}}, + OnProgress: func(SourcePurgeResult) error { + progressCalls++ + return nil + }, + }) + if err == nil || !strings.Contains(err.Error(), "non-canonical confirmed ID") { + t.Fatalf("expected short confirmed identity to block mutation, got result=%#v err=%v", result, err) + } + if progressCalls != 0 { + t.Fatalf("invalid confirmed identity was published before rejection: %d progress calls", progressCalls) + } +} + +func TestPurgeSourceNetworkRejectsInvalidIdentityTupleBeforeProgress(t *testing.T) { + confirmedID := strings.Repeat("a", 64) + for name, network := range map[string]SourcePurgeNetwork{ + "missing confirmed": {Name: "api-net", DiscoveredIdentity: confirmedID}, + "missing discovered": {Name: "api-net", ExpectedIdentity: confirmedID}, + "invalid discovered": {Name: "api-net", DiscoveredIdentity: "abcdef", ExpectedIdentity: confirmedID}, + "short discovered": {Name: "api-net", DiscoveredIdentity: confirmedID[:12], ExpectedIdentity: confirmedID}, + "mismatched IDs": {Name: "api-net", DiscoveredIdentity: strings.Repeat("b", 64), ExpectedIdentity: confirmedID}, + } { + t.Run(name, func(t *testing.T) { + runner := &fakeDockerRunner{} + progressCalls := 0 + result, err := (&Client{Docker: runner}).PurgeSourceResources(context.Background(), SourcePurgeOptions{ + Networks: []SourcePurgeNetwork{network}, + OnProgress: func(SourcePurgeResult) error { + progressCalls++ + return nil + }, + }) + if err == nil { + t.Fatalf("expected invalid identity tuple to fail, got %#v", result) + } + if progressCalls != 0 || len(runner.outputArgs) != 0 { + t.Fatalf("invalid identity tuple reached progress or Docker: progress=%d args=%#v", progressCalls, runner.outputArgs) + } + }) + } +} + func TestPurgeSourceResourcesReturnsPartialResults(t *testing.T) { + canonicalID := "aaaaaaaaaaaa" + strings.Repeat("a", 52) runner := &fakeDockerRunner{outputs: map[string][]byte{ "inspect --type container cid123": []byte(`[{"Id":"cid123","Name":"/web","State":{"Running":false,"Status":"exited"}}]`), "rm -f cid123": []byte("cid123\n"), - "network inspect api-net": []byte(`[{"Id":"network-id","Name":"api-net"}]`), + "network inspect api-net": []byte(`[{"Id":"` + canonicalID + `","Name":"api-net"}]`), }} client := &Client{Docker: runner} options, err := client.IdentifySourcePurgeResources(context.Background(), SourcePurgeOptions{ Containers: []SourcePurgeContainer{{App: "api", ContainerID: "cid123", ContainerName: "web"}}, - Networks: []SourcePurgeNetwork{{App: "api", Name: "api-net"}}, + Networks: []SourcePurgeNetwork{{App: "api", Name: "api-net", DiscoveredIdentity: canonicalID}}, }) if err != nil { t.Fatalf("IdentifySourcePurgeResources: %v", err) @@ -554,6 +699,28 @@ func TestSourcePurgeAcceptsNamedVolumeThatRemainsAbsent(t *testing.T) { } } +func TestSourcePurgeManualNetworkResultRetainsConfirmedIdentity(t *testing.T) { + canonicalID := strings.Repeat("a", 64) + runner := &cleanupDockerRunner{ + fakeDockerRunner: &fakeDockerRunner{}, + outputErrors: map[string]error{ + "volume inspect api-data": errors.New("Error response from daemon: get api-data: no such volume"), + "network inspect api-net": errors.New("Error response from daemon: network api-net not found"), + }, + } + client := &Client{Docker: runner} + result, err := client.PurgeSourceResources(context.Background(), SourcePurgeOptions{ + Volumes: []SourcePurgeVolume{{Name: "api-data", ExpectedAbsent: true}}, + Networks: []SourcePurgeNetwork{{Name: "api-net", DiscoveredIdentity: canonicalID, ExpectedIdentity: canonicalID}}, + }) + if err != nil { + t.Fatalf("PurgeSourceResources: %v", err) + } + if len(result.Networks) != 1 || result.Networks[0].Status != "skipped" || result.Networks[0].Identity != canonicalID { + t.Fatalf("confirmed network identity was lost from manual result: %#v", result.Networks) + } +} + type cleanupDockerRunner struct { *fakeDockerRunner outputErrors map[string]error From e585acc6debabc320f1dae8cf8379ed5aa34670b Mon Sep 17 00:00:00 2001 From: Aikins Laryea Date: Mon, 3 Aug 2026 12:59:16 +0000 Subject: [PATCH 15/31] commit: reject empty cutover artifact (#22) --- internal/cli/commit.go | 3 +++ internal/cli/commit_test.go | 16 ++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/internal/cli/commit.go b/internal/cli/commit.go index 4e03099..59ee13e 100644 --- a/internal/cli/commit.go +++ b/internal/cli/commit.go @@ -46,6 +46,9 @@ func runCommit(ctx context.Context, args []string, stdout, stderr io.Writer) err if flagSet(fs, "run") && strings.TrimSpace(runRef) == "" { return fmt.Errorf("commit requires a non-empty --run value") } + if !apply && flagSet(fs, "from-cutover") && strings.TrimSpace(cutoverPlanPath) == "" { + return fmt.Errorf("commit requires a non-empty --from-cutover value") + } if apply { for _, name := range []string{"app", "bundle", "format", "from-cutover", "output", "rollback-window", "target"} { if flagSet(fs, name) { diff --git a/internal/cli/commit_test.go b/internal/cli/commit_test.go index 21430ca..6145f85 100644 --- a/internal/cli/commit_test.go +++ b/internal/cli/commit_test.go @@ -137,6 +137,22 @@ func TestRunCommitRejectsEmptyExplicitRun(t *testing.T) { } } +func TestRunCommitRejectsEmptyCutoverArtifact(t *testing.T) { + var stdout bytes.Buffer + err := runCommit(context.Background(), []string{"--from-cutover="}, &stdout, io.Discard) + if err == nil || err.Error() != "commit requires a non-empty --from-cutover value" { + t.Fatalf("expected empty cutover artifact rejection, got %v", err) + } + if stdout.Len() != 0 { + t.Fatalf("empty cutover artifact produced output before rejection: %q", stdout.String()) + } + + err = runCommit(context.Background(), []string{"--apply", "--from-cutover="}, io.Discard, io.Discard) + if err == nil || err.Error() != "commit --apply does not accept --from-cutover; select the run with --run" { + t.Fatalf("expected apply mode to reject the cutover artifact flag, got %v", err) + } +} + func TestRunCommitDefaultsToCurrentRun(t *testing.T) { workDir := t.TempDir() t.Chdir(workDir) From 826afc56478bbd76586dfefece17485b72affc6c Mon Sep 17 00:00:00 2001 From: Aikins Laryea Date: Mon, 3 Aug 2026 15:00:41 +0000 Subject: [PATCH 16/31] migration: guard asynchronous Dokploy deploys (#23) --- internal/cli/cleanup_test.go | 14 +- internal/target/dokploy/apply.go | 152 ++++++- internal/target/dokploy/apply_test.go | 541 ++++++++++++++++++++--- internal/target/dokploy/client.go | 24 +- internal/target/dokploy/datamove.go | 15 +- internal/target/dokploy/datamove_test.go | 45 +- 6 files changed, 696 insertions(+), 95 deletions(-) diff --git a/internal/cli/cleanup_test.go b/internal/cli/cleanup_test.go index 3026160..857e9af 100644 --- a/internal/cli/cleanup_test.go +++ b/internal/cli/cleanup_test.go @@ -22,6 +22,7 @@ import ( func TestLifecycleAcceptanceTraceWithFakeDokploy(t *testing.T) { composeCreated := false + deploymentTitle := "" server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch { case r.Method == http.MethodGet && r.URL.Path == "/api/project.all": @@ -42,8 +43,19 @@ func TestLifecycleAcceptanceTraceWithFakeDokploy(t *testing.T) { case r.Method == http.MethodPost && r.URL.Path == "/api/compose.create": composeCreated = true _ = json.NewEncoder(w).Encode(dokploy.Compose{ComposeID: "compose-api", Name: "api", AppName: "compose-api", EnvironmentID: "environment-production"}) - case r.Method == http.MethodPost && (r.URL.Path == "/api/compose.update" || r.URL.Path == "/api/compose.deploy"): + case r.Method == http.MethodPost && r.URL.Path == "/api/compose.update": w.WriteHeader(http.StatusOK) + case r.Method == http.MethodPost && r.URL.Path == "/api/compose.deploy": + var request struct { + Title string `json:"title"` + } + if err := json.NewDecoder(r.Body).Decode(&request); err != nil { + t.Fatalf("decode compose.deploy: %v", err) + } + deploymentTitle = request.Title + w.WriteHeader(http.StatusOK) + case r.Method == http.MethodGet && r.URL.Path == "/api/compose.one": + _ = json.NewEncoder(w).Encode(dokploy.Compose{ComposeID: "compose-api", Name: "api", AppName: "compose-api", EnvironmentID: "environment-production", Deployments: []dokploy.Deployment{{Title: deploymentTitle, Status: "done"}}}) default: http.NotFound(w, r) } diff --git a/internal/target/dokploy/apply.go b/internal/target/dokploy/apply.go index 46bac19..4c521bc 100644 --- a/internal/target/dokploy/apply.go +++ b/internal/target/dokploy/apply.go @@ -352,6 +352,42 @@ func (e deployedComposeGuardError) Unwrap() error { return e.err } +type deployedComposeTerminalError struct { + err error +} + +func (e deployedComposeTerminalError) Error() string { + return e.err.Error() +} + +func (e deployedComposeTerminalError) Unwrap() error { + return e.err +} + +type deployedComposeMonitorError struct { + err error +} + +func (e deployedComposeMonitorError) Error() string { + return e.err.Error() +} + +func (e deployedComposeMonitorError) Unwrap() error { + return e.err +} + +type ambiguousDeployResponseError struct { + err error +} + +func (e ambiguousDeployResponseError) Error() string { + return e.err.Error() +} + +func (e ambiguousDeployResponseError) Unwrap() error { + return e.err +} + type unsafeSourceResumeError struct { err error } @@ -416,7 +452,18 @@ func (c *Client) activeBortOverridePatches(ctx context.Context, composeID string "exec", "-i", pg, "psql", "-U", "dokploy", "-d", "dokploy", "-v", "ON_ERROR_STOP=1", "-At"); err != nil { return nil, fmt.Errorf("inspect active Dokploy patches: %w", err) } - return parseActiveBortOverridePatches(out.String()) + patches, err := parseActiveBortOverridePatches(out.String()) + if err != nil { + return nil, err + } + active := make([]activeBortOverridePatch, 0, len(patches)) + for _, patch := range patches { + if strings.EqualFold(strings.TrimSpace(patch.SourceType), "raw") { + continue + } + active = append(active, patch) + } + return active, nil } func activeBortOverridePatchesSQL(composeID string) string { @@ -811,7 +858,11 @@ func (c *Client) applyPushImage(ctx context.Context, actx *applyContext, step St if err := ensureComposeImagesAvailable(ctx, c.dockerRunner(), actx.plan, step.App, composeFile); err != nil { return err } - if err := c.deployComposeForApply(ctx, actx, step.App); err != nil { + envContent, err := readEnvContent(actx.plan, step.App) + if err != nil { + return err + } + if err := c.deployComposeForApply(ctx, actx, step.App, composeFile, envContent); err != nil { return err } return c.pauseTargetWritersForState(ctx, c.dockerRunner(), actx, step.App) @@ -830,35 +881,73 @@ func (c *Client) deployComposeWithPatchGuard(ctx context.Context, composeID, tit if err := c.validateNoActiveBortOverrides(ctx, composeID); err != nil { return err } - mutationErr := c.DeployCompose(ctx, composeID, title) - patchErr := c.validateNoActiveBortOverridesAfterMutation(ctx, composeID) - if mutationErr == nil && patchErr != nil { - return deployedComposeGuardError{err: patchErr} + err := c.DeployCompose(ctx, composeID, title) + if err == nil { + return nil } - return errors.Join(mutationErr, patchErr) + var apiErr *APIError + if errors.As(err, &apiErr) { + return err + } + return ambiguousDeployResponseError{err: err} } -func (c *Client) deployComposeForApply(ctx context.Context, actx *applyContext, appName string) error { +const guardedDeploymentTimeout = 30 * time.Minute + +func (c *Client) deployComposeForApply(ctx context.Context, actx *applyContext, appName, composeFile, envContent string) error { entry := actx.entry(appName) + if err := c.updateComposeWithPatchGuard(ctx, entry.ComposeID, composeFile, envContent); err != nil { + return err + } title, err := deployComposeAttemptTitle(actx.plan) if err != nil { return err } err = c.deployComposeWithPatchGuard(ctx, entry.ComposeID, title) - if err == nil { - return c.validateMigratedVolumeMountsAfterDeploy(ctx, actx, appName) + var ambiguousErr ambiguousDeployResponseError + if err == nil || errors.As(err, &ambiguousErr) { + responseErr := err + deployment, guardErr := c.waitForGuardedDeployment(ctx, entry.ComposeID, title) + status := strings.ToLower(strings.TrimSpace(deployment.Status)) + if status != "" && status != "done" { + err = deployedComposeTerminalError{err: errors.Join(responseErr, fmt.Errorf("dokploy compose project %q deployment finished with status %q%s", entry.ComposeAppName, status, composeDeploymentDetails(&Compose{Deployments: []Deployment{deployment}})), guardErr)} + } else if guardErr != nil { + var monitorErr deployedComposeMonitorError + if errors.As(guardErr, &monitorErr) { + err = deployedComposeMonitorError{err: errors.Join(responseErr, guardErr)} + } else { + err = deployedComposeGuardError{err: errors.Join(responseErr, guardErr)} + } + } else { + return c.validateMigratedVolumeMountsAfterDeploy(ctx, actx, appName) + } } var deployedGuardErr deployedComposeGuardError - if errors.As(err, &deployedGuardErr) { - stopCtx, cancelStop := context.WithTimeout(context.Background(), targetDiscoveryTimeout) + var deployedTerminalErr deployedComposeTerminalError + var deployedMonitorErr deployedComposeMonitorError + if errors.As(err, &deployedGuardErr) || errors.As(err, &deployedTerminalErr) || errors.As(err, &deployedMonitorErr) { + failure := "Dokploy accepted the deploy request, but its post-deploy safety guard failed" + if errors.As(err, &deployedTerminalErr) { + failure = "Dokploy compose deployment did not complete successfully" + } else if errors.As(err, &deployedMonitorErr) { + failure = "Dokploy accepted the deploy request, but deployment monitoring could not prove safety" + if errors.As(err, &ambiguousErr) { + failure = "Dokploy deploy response was ambiguous and deployment monitoring could not prove safety" + } + } + quiescenceDeadline := time.Now().Add(targetDiscoveryTimeout) + if callerDeadline, ok := ctx.Deadline(); ok && callerDeadline.Before(quiescenceDeadline) { + quiescenceDeadline = callerDeadline + } + stopCtx, cancelStop := context.WithDeadline(context.Background(), quiescenceDeadline) defer cancelStop() stopErr := c.quiesceGuardedDeployment(stopCtx, actx, appName, title) if stopErr != nil { - return unsafeSourceResumeError{err: unsafeTargetResumeError{err: fmt.Errorf("successful Dokploy deploy failed its post-deploy patch guard: %w; quiescence could not be proved, so Bort will leave any paused source applications stopped and skip target recovery: %v", err, stopErr)}} + return unsafeSourceResumeError{err: unsafeTargetResumeError{err: fmt.Errorf("%s: %w; quiescence could not be proved, so Bort will leave any paused source applications stopped and skip target recovery: %v", failure, err, stopErr)}} } safetyErr := c.validateMigratedVolumeMountsAfterDeploy(ctx, actx, appName) combined := errors.Join(err, safetyErr) - return unsafeTargetResumeError{err: fmt.Errorf("successful Dokploy deploy failed its post-deploy patch guard; deployment quiesced and target compose containers were stopped: %w", combined)} + return unsafeTargetResumeError{err: fmt.Errorf("%s; deployment quiesced and target compose containers were stopped: %w", failure, combined)} } safetyErr := c.validateMigratedVolumeMountsAfterDeploy(ctx, actx, appName) if safetyErr != nil && isUnsafeTargetResumeError(safetyErr) { @@ -867,6 +956,34 @@ func (c *Client) deployComposeForApply(ctx context.Context, actx *applyContext, return err } +func (c *Client) waitForGuardedDeployment(ctx context.Context, composeID, title string) (Deployment, error) { + monitorCtx, cancel := context.WithTimeout(ctx, guardedDeploymentTimeout) + defer cancel() + for { + compose, err := c.GetCompose(monitorCtx, composeID) + if err != nil { + return Deployment{}, deployedComposeMonitorError{err: fmt.Errorf("monitor accepted Dokploy deployment %q: %w", title, err)} + } + for _, deployment := range compose.Deployments { + if deployment.Title != title { + continue + } + switch strings.ToLower(strings.TrimSpace(deployment.Status)) { + case "done", "error", "cancelled": + if err := c.validateNoActiveBortOverridesAfterMutation(monitorCtx, composeID); err != nil { + return deployment, err + } + return deployment, nil + } + } + select { + case <-monitorCtx.Done(): + return Deployment{}, deployedComposeMonitorError{err: fmt.Errorf("monitor accepted Dokploy deployment %q: %w", title, monitorCtx.Err())} + case <-time.After(targetDiscoveryDelay): + } + } +} + func deployComposeAttemptTitle(plan Plan) (string, error) { var nonce [16]byte if _, err := rand.Read(nonce[:]); err != nil { @@ -1059,6 +1176,9 @@ func (c *Client) applyInstallGateway(ctx context.Context, actx *applyContext, st if entry.ComposeID == "" { return fmt.Errorf("missing composeId for app %s; create_service must run first", step.App) } + if err := c.validateNoActiveBortOverrides(ctx, entry.ComposeID); err != nil { + return err + } route, ok := findCutoverRoute(actx.plan.Cutover, step.App, step.Ref) if !ok { return fmt.Errorf("route %s for app %s not found in cutover artifact", step.Ref, step.App) @@ -1090,7 +1210,7 @@ func (c *Client) applyActivateRoutes(ctx context.Context, actx *applyContext, st if err != nil { return err } - if err := c.updateComposeWithPatchGuard(ctx, entry.ComposeID, composeFile, envContent); err != nil { + if err := c.validateNoActiveBortOverrides(ctx, entry.ComposeID); err != nil { return err } for _, route := range cutoverRoutesForApp(actx.plan.Cutover, step.App) { @@ -1102,7 +1222,7 @@ func (c *Client) applyActivateRoutes(ctx context.Context, actx *applyContext, st return err } } - return c.deployComposeForApply(ctx, actx, step.App) + return c.deployComposeForApply(ctx, actx, step.App, composeFile, envContent) } func (c *Client) ensureRouteDomain(ctx context.Context, composeID string, route gateway.Route) error { diff --git a/internal/target/dokploy/apply_test.go b/internal/target/dokploy/apply_test.go index caf54b8..f0151bf 100644 --- a/internal/target/dokploy/apply_test.go +++ b/internal/target/dokploy/apply_test.go @@ -351,12 +351,14 @@ func TestActivePatchGuardBlocksGitBackedCompose(t *testing.T) { "ps --format {{.Names}}": []byte("dokploy-postgres.1.task\n"), }, runOutputs: map[string][]byte{ - "exec -i dokploy-postgres.1.task psql -U dokploy -d dokploy -v ON_ERROR_STOP=1 -At": []byte(`{"patchId":"bort-api-compose","filePath":"docker|compose.yml","composeName":"api","sourceType":"github","repository":"owner/repo","branch":"main"}` + "\n"), + "exec -i dokploy-postgres.1.task psql -U dokploy -d dokploy -v ON_ERROR_STOP=1 -At": []byte( + `{"patchId":"bort-api-compose","filePath":"docker|compose.yml","composeName":"api","sourceType":"github","repository":"owner/repo","branch":"main"}` + "\n" + + `{"patchId":"bort-raw-compose","filePath":"docker|compose.yml","composeName":"raw-api","sourceType":"raw","repository":"owner/repo","branch":"main"}` + "\n"), }, } client := &Client{Docker: runner} err := client.validateNoActiveBortOverrides(context.Background(), "compose-api") - if err == nil || !strings.Contains(err.Error(), "active Bort-owned Dokploy patch") || !strings.Contains(err.Error(), "bort-api-compose") || !strings.Contains(err.Error(), "docker|compose.yml") { + if err == nil || !strings.Contains(err.Error(), "active Bort-owned Dokploy patch") || !strings.Contains(err.Error(), "bort-api-compose") || !strings.Contains(err.Error(), "docker|compose.yml") || strings.Contains(err.Error(), "bort-raw-compose") { t.Fatalf("expected active patch guard, got %v", err) } if len(runner.runs) != 1 { @@ -368,9 +370,6 @@ func TestActivePatchGuardBlocksGitBackedCompose(t *testing.T) { t.Fatalf("expected patch guard sql to contain %q, got:\n%s", want, sql) } } - if strings.Contains(sql, `coalesce(c."sourceType"::text, '') not in`) { - t.Fatalf("expected active Bort patches to block mutations regardless of current compose source type, got:\n%s", sql) - } } type stagedBortPatchRunner struct { @@ -438,48 +437,28 @@ func (r *restartingTargetRunner) Run(context.Context, io.Reader, io.Writer, ...s return nil } -func TestComposeMutationPatchGuardsDetectConcurrentPatch(t *testing.T) { - for _, test := range []struct { - name string - path string - deployedBefore bool - mutate func(*Client) error - }{ - {name: "update", path: "/api/compose.update", mutate: func(client *Client) error { - return client.updateComposeWithPatchGuard(context.Background(), "compose-api", "services: {}\n", "") - }}, - {name: "deploy", path: "/api/compose.deploy", deployedBefore: true, mutate: func(client *Client) error { - return client.deployComposeWithPatchGuard(context.Background(), "compose-api", "bort-migrate") - }}, - } { - t.Run(test.name, func(t *testing.T) { - mutations := 0 - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodPost || r.URL.Path != test.path { - http.NotFound(w, r) - return - } - mutations++ - w.WriteHeader(http.StatusOK) - })) - defer server.Close() - runner := &stagedBortPatchRunner{fakeDockerRunner: fakeDockerRunner{ - outputs: map[string][]byte{"ps --format {{.Names}}": []byte("dokploy-postgres\n")}, - runOutputs: map[string][]byte{}, - }} - client := &Client{BaseURL: server.URL, Token: "secret", HTTPClient: server.Client(), Docker: runner} - err := test.mutate(client) - if err == nil || !strings.Contains(err.Error(), "appeared while Bort was changing compose") { - t.Fatalf("expected post-mutation patch detection, got %v", err) - } - var deployedGuardErr deployedComposeGuardError - if got := errors.As(err, &deployedGuardErr); got != test.deployedBefore { - t.Fatalf("deployed guard error = %t, want %t", got, test.deployedBefore) - } - if mutations != 1 || runner.checks != 2 { - t.Fatalf("expected one mutation between two patch checks, mutations=%d checks=%d", mutations, runner.checks) - } - }) +func TestUpdateComposePatchGuardDetectsConcurrentPatch(t *testing.T) { + updates := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost || r.URL.Path != "/api/compose.update" { + http.NotFound(w, r) + return + } + updates++ + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + runner := &stagedBortPatchRunner{fakeDockerRunner: fakeDockerRunner{ + outputs: map[string][]byte{"ps --format {{.Names}}": []byte("dokploy-postgres\n")}, + runOutputs: map[string][]byte{}, + }} + client := &Client{BaseURL: server.URL, Token: "secret", HTTPClient: server.Client(), Docker: runner} + err := client.updateComposeWithPatchGuard(context.Background(), "compose-api", "services: {}\n", "") + if err == nil || !strings.Contains(err.Error(), "appeared while Bort was changing compose") { + t.Fatalf("expected post-update patch detection, got %v", err) + } + if updates != 1 || runner.checks != 2 { + t.Fatalf("expected one update between two patch checks, updates=%d checks=%d", updates, runner.checks) } } @@ -488,6 +467,8 @@ func TestPostDeployPatchGuardStopsTargetCompose(t *testing.T) { var composeReads int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch { + case r.Method == http.MethodPost && r.URL.Path == "/api/compose.update": + w.WriteHeader(http.StatusOK) case r.Method == http.MethodPost && r.URL.Path == "/api/compose.deploy": var request deployComposeRequest if err := json.NewDecoder(r.Body).Decode(&request); err != nil { @@ -514,7 +495,7 @@ func TestPostDeployPatchGuardStopsTargetCompose(t *testing.T) { } })) defer server.Close() - runner := &stagedBortPatchRunner{fakeDockerRunner: fakeDockerRunner{ + runner := &stagedBortPatchRunner{patchAt: 4, fakeDockerRunner: fakeDockerRunner{ outputs: map[string][]byte{ "ps --format {{.Names}}": []byte("dokploy-postgres\n"), "ps -a --filter label=com.docker.compose.project=stack-1 --format {{.ID}}": []byte("target-id\n"), @@ -529,7 +510,7 @@ func TestPostDeployPatchGuardStopsTargetCompose(t *testing.T) { entry.ComposeID = "compose-api" entry.ComposeAppName = "stack-1" - err := client.deployComposeForApply(context.Background(), actx, "api") + err := client.deployComposeForApply(context.Background(), actx, "api", "services: {}\n", "") if err == nil || !strings.Contains(err.Error(), "deployment quiesced and target compose containers were stopped") || !isUnsafeTargetResumeError(err) { t.Fatalf("expected guarded deploy failure to stop the target and prevent target recovery, got %v", err) } @@ -557,6 +538,280 @@ func TestPostDeployPatchGuardStopsTargetCompose(t *testing.T) { } } +func TestDeployComposeForApplyResolvesAmbiguousResponseByTitle(t *testing.T) { + var deploymentTitle string + deploys := 0 + composeReads := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodPost && r.URL.Path == "/api/compose.update": + w.WriteHeader(http.StatusOK) + case r.Method == http.MethodPost && r.URL.Path == "/api/compose.deploy": + deploys++ + var request deployComposeRequest + if err := json.NewDecoder(r.Body).Decode(&request); err != nil { + t.Fatal(err) + } + deploymentTitle = request.Title + conn, _, err := w.(http.Hijacker).Hijack() + if err != nil { + t.Fatal(err) + } + _ = conn.Close() + case r.Method == http.MethodGet && r.URL.Path == "/api/compose.one": + composeReads++ + _ = json.NewEncoder(w).Encode(Compose{ComposeID: "compose-api", Deployments: []Deployment{{Title: deploymentTitle, Status: "done"}}}) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + runner := &stagedBortPatchRunner{patchAt: 100, fakeDockerRunner: fakeDockerRunner{ + outputs: map[string][]byte{"ps --format {{.Names}}": []byte("dokploy-postgres\n")}, + runOutputs: map[string][]byte{}, + }} + client := &Client{BaseURL: server.URL, Token: "secret", HTTPClient: server.Client(), Docker: runner} + actx := &applyContext{plan: Plan{}, cache: map[string]*appCache{"api": {ComposeID: "compose-api", ComposeAppName: "stack-1"}}} + + if err := client.deployComposeForApply(context.Background(), actx, "api", "services: {}\n", ""); err != nil { + t.Fatalf("expected titled terminal deployment to resolve ambiguous response, got %v", err) + } + if deploys != 1 || composeReads != 1 || deploymentTitle == "" { + t.Fatalf("expected one deploy resolved by one titled status read, deploys=%d reads=%d title=%q", deploys, composeReads, deploymentTitle) + } +} + +func TestDeployComposeForApplyFailsClosedWhenAmbiguousResponseCannotBeResolved(t *testing.T) { + var deploymentTitle string + composeReads := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodPost && r.URL.Path == "/api/compose.update": + w.WriteHeader(http.StatusOK) + case r.Method == http.MethodPost && r.URL.Path == "/api/compose.deploy": + var request deployComposeRequest + if err := json.NewDecoder(r.Body).Decode(&request); err != nil { + t.Fatal(err) + } + deploymentTitle = request.Title + conn, _, err := w.(http.Hijacker).Hijack() + if err != nil { + t.Fatal(err) + } + _ = conn.Close() + case r.Method == http.MethodGet && r.URL.Path == "/api/compose.one": + composeReads++ + http.Error(w, "deployment status unavailable", http.StatusInternalServerError) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + runner := &stagedBortPatchRunner{patchAt: 100, fakeDockerRunner: fakeDockerRunner{ + outputs: map[string][]byte{ + "ps --format {{.Names}}": []byte("dokploy-postgres\n"), + "ps -a --filter label=com.docker.compose.project=stack-1 --format {{.ID}}": {}, + }, + runOutputs: map[string][]byte{}, + }} + client := &Client{BaseURL: server.URL, Token: "secret", HTTPClient: server.Client(), Docker: runner} + actx := &applyContext{plan: Plan{}, cache: map[string]*appCache{"api": {ComposeID: "compose-api", ComposeAppName: "stack-1"}}} + + err := client.deployComposeForApply(context.Background(), actx, "api", "services: {}\n", "") + if err == nil || !strings.Contains(err.Error(), "response was ambiguous") || !strings.Contains(err.Error(), "deployment status unavailable") || !strings.Contains(err.Error(), "quiescence could not be proved") { + t.Fatalf("expected unresolved ambiguous response evidence, got %v", err) + } + if !isUnsafeSourceResumeError(err) || !isUnsafeTargetResumeError(err) { + t.Fatalf("unresolved ambiguous response must prevent source and target recovery, got %v", err) + } + if composeReads != 2 || deploymentTitle == "" { + t.Fatalf("expected monitoring and quiescence to inspect the titled attempt, reads=%d title=%q", composeReads, deploymentTitle) + } + if !fakeOutputCalled(&runner.fakeDockerRunner, "ps", "-a", "--filter", "label=com.docker.compose.project=stack-1", "--format", "{{.ID}}") { + t.Fatalf("unresolved ambiguous response did not attempt target quiescence, calls=%#v", runner.outputArgs) + } +} + +func TestDeployComposeForApplyDoesNotMonitorAuthoritativeAPIRejection(t *testing.T) { + for _, test := range []struct { + name string + status int + body bool + }{ + {name: "redirect", status: http.StatusFound}, + {name: "replay redirect", status: http.StatusTemporaryRedirect}, + {name: "truncated error", status: http.StatusConflict, body: true}, + } { + t.Run(test.name, func(t *testing.T) { + composeReads := 0 + redirects := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodPost && r.URL.Path == "/api/compose.update": + w.WriteHeader(http.StatusOK) + case r.Method == http.MethodPost && r.URL.Path == "/api/compose.deploy": + if test.status >= 300 && test.status < 400 { + w.Header().Set("Location", "/login") + } + if test.body { + w.Header().Set("Content-Length", "100") + } + w.WriteHeader(test.status) + if test.body { + _, _ = w.Write([]byte("deploy rejected")) + } + case r.URL.Path == "/login": + redirects++ + w.WriteHeader(http.StatusOK) + case r.Method == http.MethodGet && r.URL.Path == "/api/compose.one": + composeReads++ + http.NotFound(w, r) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + runner := &stagedBortPatchRunner{patchAt: 100, fakeDockerRunner: fakeDockerRunner{ + outputs: map[string][]byte{"ps --format {{.Names}}": []byte("dokploy-postgres\n")}, + runOutputs: map[string][]byte{}, + }} + client := &Client{BaseURL: server.URL, Token: "secret", HTTPClient: server.Client(), Docker: runner} + actx := &applyContext{plan: Plan{}, cache: map[string]*appCache{"api": {ComposeID: "compose-api", ComposeAppName: "stack-1"}}} + + err := client.deployComposeForApply(context.Background(), actx, "api", "services: {}\n", "") + var apiErr *APIError + if !errors.As(err, &apiErr) || apiErr.Status != test.status { + t.Fatalf("expected authoritative API rejection %d, got %v", test.status, err) + } + if composeReads != 0 || redirects != 0 || isUnsafeSourceResumeError(err) || isUnsafeTargetResumeError(err) { + t.Fatalf("authoritative rejection followed redirect or entered deployment recovery, redirects=%d reads=%d err=%v", redirects, composeReads, err) + } + }) + } +} + +func TestDeployComposeAcceptsCompletedSuccessWithTruncatedBody(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost || r.URL.Path != "/api/compose.deploy" { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Length", "100") + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("queued")) + })) + defer server.Close() + client := &Client{BaseURL: server.URL, Token: "secret", HTTPClient: server.Client()} + + if err := client.DeployCompose(context.Background(), "compose-api", "unique-title"); err != nil { + t.Fatalf("completed 2xx deploy response was not accepted: %v", err) + } +} + +func TestWaitForGuardedDeploymentRechecksPatchAfterTerminalStatus(t *testing.T) { + composeReads := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet || r.URL.Path != "/api/compose.one" { + http.NotFound(w, r) + return + } + composeReads++ + _ = json.NewEncoder(w).Encode(Compose{ComposeID: "compose-api", Deployments: []Deployment{{Title: "unique-title", Status: "done"}}}) + })) + defer server.Close() + runner := &stagedBortPatchRunner{ + patchAt: 1, + fakeDockerRunner: fakeDockerRunner{ + outputs: map[string][]byte{"ps --format {{.Names}}": []byte("dokploy-postgres\n")}, + runOutputs: map[string][]byte{}, + }, + } + client := &Client{BaseURL: server.URL, Token: "secret", HTTPClient: server.Client(), Docker: runner} + + deployment, err := client.waitForGuardedDeployment(context.Background(), "compose-api", "unique-title") + if err == nil || !strings.Contains(err.Error(), "appeared while Bort was changing compose") { + t.Fatalf("expected final patch check after terminal status, got %v", err) + } + if deployment.Title != "unique-title" || deployment.Status != "done" { + t.Fatalf("final patch check discarded terminal deployment: %+v", deployment) + } + if composeReads != 1 || runner.checks != 1 { + t.Fatalf("expected one terminal status read followed by a final patch check, reads=%d checks=%d", composeReads, runner.checks) + } +} + +func TestDeployComposeForApplyRejectsTerminalFailureStatuses(t *testing.T) { + for _, test := range []struct { + name string + status string + patchAt int + wantPatch bool + }{ + {name: "error", status: "error", patchAt: 100}, + {name: "cancelled", status: "cancelled", patchAt: 100}, + {name: "error with final patch conflict", status: "error", patchAt: 4, wantPatch: true}, + } { + t.Run(test.name, func(t *testing.T) { + var deploymentTitle string + rawUpdated := false + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodPost && r.URL.Path == "/api/compose.update": + var request updateComposeRequest + if err := json.NewDecoder(r.Body).Decode(&request); err != nil { + t.Fatal(err) + } + rawUpdated = request.SourceType == "raw" + w.WriteHeader(http.StatusOK) + case r.Method == http.MethodPost && r.URL.Path == "/api/compose.deploy": + if !rawUpdated { + t.Fatal("compose deploy was not preceded by a raw compose update") + } + var request deployComposeRequest + if err := json.NewDecoder(r.Body).Decode(&request); err != nil { + t.Fatal(err) + } + deploymentTitle = request.Title + w.WriteHeader(http.StatusOK) + case r.Method == http.MethodGet && r.URL.Path == "/api/compose.one": + _ = json.NewEncoder(w).Encode(Compose{ComposeID: "compose-api", Deployments: []Deployment{{Title: deploymentTitle, Status: test.status, ErrorMessage: "deploy failed", LogPath: "/tmp/deploy.log"}}}) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + runner := &stagedBortPatchRunner{ + patchAt: test.patchAt, + fakeDockerRunner: fakeDockerRunner{ + outputs: map[string][]byte{ + "ps --format {{.Names}}": []byte("dokploy-postgres\n"), + "ps -a --filter label=com.docker.compose.project=stack-1 --format {{.ID}}": []byte("target-id\n"), + "inspect --type container target-id": []byte(`[{"Id":"target-id","Name":"/target","State":{"Running":true,"Status":"running"}}]`), + "stop target-id": []byte("target-id\n"), + }, + runOutputs: map[string][]byte{}, + }, + } + client := &Client{BaseURL: server.URL, Token: "secret", HTTPClient: server.Client(), Docker: runner} + actx := &applyContext{plan: Plan{}, cache: map[string]*appCache{"api": {ComposeID: "compose-api", ComposeAppName: "stack-1"}}} + + err := client.deployComposeForApply(context.Background(), actx, "api", "services: {}\n", "") + if err == nil || !strings.Contains(err.Error(), fmt.Sprintf("status %q", test.status)) || !strings.Contains(err.Error(), "deploy failed") || !strings.Contains(err.Error(), "/tmp/deploy.log") || !strings.Contains(err.Error(), "Dokploy compose deployment did not complete successfully") || strings.Contains(err.Error(), "post-deploy safety guard failed") || !strings.Contains(err.Error(), "deployment quiesced and target compose containers were stopped") || !isUnsafeTargetResumeError(err) { + t.Fatalf("expected terminal %s deployment to fail, got %v", test.status, err) + } + if got := strings.Contains(err.Error(), "appeared while Bort was changing compose"); got != test.wantPatch { + t.Fatalf("final patch conflict = %t, want %t: %v", got, test.wantPatch, err) + } + if isUnsafeSourceResumeError(err) { + t.Fatalf("terminal %s deployment should permit source recovery after quiescence, got %v", test.status, err) + } + if !fakeOutputCalled(&runner.fakeDockerRunner, "stop", "target-id") { + t.Fatalf("terminal %s deployment did not stop target compose, calls=%#v", test.status, runner.outputArgs) + } + }) + } +} + func TestStopTargetComposeContainersRestopsRestartedTarget(t *testing.T) { runner := &restartingTargetRunner{} client := &Client{Docker: runner} @@ -571,9 +826,11 @@ func TestStopTargetComposeContainersRestopsRestartedTarget(t *testing.T) { } } -func TestPostDeployPatchGuardQuiescenceFailurePreventsSourceRecovery(t *testing.T) { +func TestDeploymentMonitoringQuiescenceFailurePreventsSourceRecovery(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch { + case r.Method == http.MethodPost && r.URL.Path == "/api/compose.update": + w.WriteHeader(http.StatusOK) case r.Method == http.MethodPost && r.URL.Path == "/api/compose.deploy": w.WriteHeader(http.StatusOK) case r.Method == http.MethodGet && r.URL.Path == "/api/compose.one": @@ -583,7 +840,7 @@ func TestPostDeployPatchGuardQuiescenceFailurePreventsSourceRecovery(t *testing. } })) defer server.Close() - runner := &stagedBortPatchRunner{fakeDockerRunner: fakeDockerRunner{ + runner := &stagedBortPatchRunner{patchAt: 100, fakeDockerRunner: fakeDockerRunner{ outputs: map[string][]byte{ "ps --format {{.Names}}": []byte("dokploy-postgres\n"), "ps -a --filter label=com.docker.compose.project=stack-1 --format {{.ID}}": {}, @@ -596,8 +853,8 @@ func TestPostDeployPatchGuardQuiescenceFailurePreventsSourceRecovery(t *testing. entry.ComposeID = "compose-api" entry.ComposeAppName = "stack-1" - err := client.deployComposeForApply(context.Background(), actx, "api") - if err == nil || !strings.Contains(err.Error(), "deployment status unavailable") { + err := client.deployComposeForApply(context.Background(), actx, "api", "services: {}\n", "") + if err == nil || !strings.Contains(err.Error(), "deployment status unavailable") || !strings.Contains(err.Error(), "deployment monitoring could not prove safety") || strings.Contains(err.Error(), "post-deploy safety guard failed") { t.Fatalf("expected deployment polling failure, got %v", err) } if !isUnsafeSourceResumeError(err) || !isUnsafeTargetResumeError(err) { @@ -605,6 +862,50 @@ func TestPostDeployPatchGuardQuiescenceFailurePreventsSourceRecovery(t *testing. } } +func TestDeploymentQuiescenceRespectsCallerDeadline(t *testing.T) { + composeReads := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodPost && r.URL.Path == "/api/compose.update": + w.WriteHeader(http.StatusOK) + case r.Method == http.MethodPost && r.URL.Path == "/api/compose.deploy": + w.WriteHeader(http.StatusOK) + case r.Method == http.MethodGet && r.URL.Path == "/api/compose.one": + composeReads++ + _ = json.NewEncoder(w).Encode(Compose{ComposeID: "compose-api"}) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + runner := &stagedBortPatchRunner{patchAt: 100, fakeDockerRunner: fakeDockerRunner{ + outputs: map[string][]byte{ + "ps --format {{.Names}}": []byte("dokploy-postgres\n"), + "ps -a --filter label=com.docker.compose.project=stack-1 --format {{.ID}}": {}, + }, + runOutputs: map[string][]byte{}, + }} + client := &Client{BaseURL: server.URL, Token: "secret", HTTPClient: server.Client(), Docker: runner} + actx := &applyContext{plan: Plan{}, cache: map[string]*appCache{"api": {ComposeID: "compose-api", ComposeAppName: "stack-1"}}} + ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) + defer cancel() + started := time.Now() + + err := client.deployComposeForApply(ctx, actx, "api", "services: {}\n", "") + if err == nil || !strings.Contains(err.Error(), "context deadline exceeded") || !strings.Contains(err.Error(), "quiescence could not be proved") { + t.Fatalf("expected deadline-bounded quiescence failure, got %v", err) + } + if !isUnsafeSourceResumeError(err) || !isUnsafeTargetResumeError(err) { + t.Fatalf("expired quiescence deadline must prevent source and target recovery, got %v", err) + } + if elapsed := time.Since(started); elapsed >= 2*time.Second { + t.Fatalf("quiescence exceeded caller deadline: %s", elapsed) + } + if composeReads == 0 { + t.Fatal("deployment monitor did not inspect the accepted attempt") + } +} + func TestQuiesceGuardedDeploymentAcceptsTerminalStatuses(t *testing.T) { for _, status := range []string{"done", "error", "cancelled"} { t.Run(status, func(t *testing.T) { @@ -679,7 +980,7 @@ func TestApplyLeavesPausedSourceStoppedWhenGuardedDeploymentCannotBeQuiesced(t * })) defer server.Close() runner := &stagedBortPatchRunner{ - patchAt: 4, + patchAt: 100, fakeDockerRunner: fakeDockerRunner{ outputs: map[string][]byte{ "ps --format {{.Names}}": []byte("dokploy-postgres\n"), @@ -715,14 +1016,16 @@ func TestApplyLeavesPausedSourceStoppedWhenGuardedDeploymentCannotBeQuiesced(t * func TestComposeMutationPatchGuardRechecksAfterHTTPError(t *testing.T) { for _, test := range []struct { - name string - path string - mutate func(*Client) error + name string + path string + wantPatch bool + wantChecks int + mutate func(*Client) error }{ - {name: "update", path: "/api/compose.update", mutate: func(client *Client) error { + {name: "update", path: "/api/compose.update", wantPatch: true, wantChecks: 2, mutate: func(client *Client) error { return client.updateComposeWithPatchGuard(context.Background(), "compose-api", "services: {}\n", "") }}, - {name: "deploy", path: "/api/compose.deploy", mutate: func(client *Client) error { + {name: "deploy", path: "/api/compose.deploy", wantChecks: 1, mutate: func(client *Client) error { return client.deployComposeWithPatchGuard(context.Background(), "compose-api", "bort-migrate") }}, } { @@ -741,15 +1044,18 @@ func TestComposeMutationPatchGuardRechecksAfterHTTPError(t *testing.T) { }} client := &Client{BaseURL: server.URL, Token: "secret", HTTPClient: server.Client(), Docker: runner} err := test.mutate(client) - if err == nil || !strings.Contains(err.Error(), "mutation failed") || !strings.Contains(err.Error(), "appeared while Bort was changing compose") { - t.Fatalf("expected HTTP and post-mutation patch errors, got %v", err) + if err == nil || !strings.Contains(err.Error(), "mutation failed") { + t.Fatalf("expected HTTP mutation error, got %v", err) + } + if got := strings.Contains(err.Error(), "appeared while Bort was changing compose"); got != test.wantPatch { + t.Fatalf("post-mutation patch error = %t, want %t: %v", got, test.wantPatch, err) } var deployedGuardErr deployedComposeGuardError if errors.As(err, &deployedGuardErr) { t.Fatalf("failed HTTP mutation was classified as a successful deploy: %v", err) } - if runner.checks != 2 { - t.Fatalf("expected the failed HTTP mutation to remain bracketed by patch checks, got %d", runner.checks) + if runner.checks != test.wantChecks { + t.Fatalf("patch checks = %d, want %d", runner.checks, test.wantChecks) } }) } @@ -782,6 +1088,99 @@ func TestApplyPushImageChecksActivePatchWithoutEnvOrRoutes(t *testing.T) { } } +func TestApplyActivateRoutesChecksPatchBeforeDomainMutation(t *testing.T) { + bundleDir := t.TempDir() + appDir := filepath.Join(bundleDir, "api") + if err := os.MkdirAll(appDir, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(appDir, "compose.yaml"), []byte("services:\n web:\n image: example/web\n expose:\n - \"8080\"\n"), 0o600); err != nil { + t.Fatal(err) + } + domainRequests := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasPrefix(r.URL.Path, "/api/domain.") { + domainRequests++ + } + http.NotFound(w, r) + })) + defer server.Close() + runner := &fakeDockerRunner{ + outputs: map[string][]byte{ + "image inspect example/web": []byte(`[{}]`), + "ps --format {{.Names}}": []byte("dokploy-postgres\n"), + }, + runOutputs: map[string][]byte{ + "exec -i dokploy-postgres psql -U dokploy -d dokploy -v ON_ERROR_STOP=1 -At": []byte(`{"patchId":"bort-api-compose","filePath":"docker|compose.yml","composeName":"api","sourceType":"github","repository":"owner/repo","branch":"main"}` + "\n"), + }, + } + client := &Client{BaseURL: server.URL, Token: "secret", HTTPClient: server.Client(), Docker: runner} + app := preparer.AppPlan{Name: "api", Directory: "api", TargetResources: &preparer.TargetResources{Dokploy: &preparer.DokployResources{ComposeApp: preparer.DokployComposeApp{ComposePath: "compose.yaml"}}}} + actx := &applyContext{ + plan: Plan{ + Prepare: preparer.Result{BundleDir: bundleDir, Apps: []preparer.AppPlan{app}}, + Cutover: gateway.Result{Apps: []gateway.AppPlan{{Name: "api", Routes: []gateway.Route{{Host: "api.example.com", ServiceName: "web", Port: "8080"}}}}}, + }, + cache: map[string]*appCache{"api": {ComposeID: "compose-api"}}, + } + + err := client.applyActivateRoutes(context.Background(), actx, Step{Kind: StepActivateRoutes, App: "api", Ref: "routes"}) + if err == nil || !strings.Contains(err.Error(), "active Bort-owned Dokploy patch") { + t.Fatalf("expected active patch to block route activation, got %v", err) + } + if domainRequests != 0 { + t.Fatalf("active patch allowed %d domain request(s) before rejection", domainRequests) + } +} + +func TestPlannedInstallGatewayChecksPatchBeforeDomainMutation(t *testing.T) { + bundleDir := t.TempDir() + appDir := filepath.Join(bundleDir, "api") + if err := os.MkdirAll(appDir, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(appDir, "compose.yaml"), []byte("services:\n web:\n image: example/web\n expose:\n - \"8080\"\n"), 0o600); err != nil { + t.Fatal(err) + } + app := preparer.AppPlan{Name: "api", Directory: "api", TargetResources: &preparer.TargetResources{Dokploy: &preparer.DokployResources{ComposeApp: preparer.DokployComposeApp{Name: "api", ComposePath: "compose.yaml"}}}} + cutover := gateway.Result{Apps: []gateway.AppPlan{{Name: "api", Routes: []gateway.Route{{Host: "api.example.com", ServiceName: "web", Port: "8080"}}}}} + plan := PlanFromArtifacts(preparer.Result{BundleDir: bundleDir, Apps: []preparer.AppPlan{app}}, syncplan.Result{}, cutover) + var firstRouteMutation Step + for _, step := range plan.Steps { + if step.Kind == StepInstallGateway || step.Kind == StepActivateRoutes { + firstRouteMutation = step + break + } + } + if firstRouteMutation.Kind != StepInstallGateway { + t.Fatalf("first planned route mutation = %s, want %s", firstRouteMutation.Kind, StepInstallGateway) + } + domainRequests := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasPrefix(r.URL.Path, "/api/domain.") { + domainRequests++ + } + http.NotFound(w, r) + })) + defer server.Close() + runner := &fakeDockerRunner{ + outputs: map[string][]byte{"ps --format {{.Names}}": []byte("dokploy-postgres\n")}, + runOutputs: map[string][]byte{ + "exec -i dokploy-postgres psql -U dokploy -d dokploy -v ON_ERROR_STOP=1 -At": []byte(`{"patchId":"bort-api-compose","filePath":"docker|compose.yml","composeName":"api","sourceType":"github","repository":"owner/repo","branch":"main"}` + "\n"), + }, + } + client := &Client{BaseURL: server.URL, Token: "secret", HTTPClient: server.Client(), Docker: runner} + actx := &applyContext{plan: plan, cache: map[string]*appCache{"api": {ComposeID: "compose-api"}}} + + err := client.applyStep(context.Background(), actx, firstRouteMutation) + if err == nil || !strings.Contains(err.Error(), "active Bort-owned Dokploy patch") { + t.Fatalf("expected planned install_gateway patch rejection, got %v", err) + } + if domainRequests != 0 { + t.Fatalf("planned install_gateway made %d domain request(s) before rejection", domainRequests) + } +} + func TestActivePatchGuardRejectsMissingComposeID(t *testing.T) { client := &Client{Docker: &fakeDockerRunner{}} if err := client.validateNoActiveBortOverrides(context.Background(), ""); err == nil || !strings.Contains(err.Error(), "missing composeId") { @@ -1279,6 +1678,7 @@ func TestApplyActivateRoutesUpdatesDomainsAndRedeploys(t *testing.T) { } var updatedDomain UpdateDomainRequest var updatedCompose updateComposeRequest + var deploymentTitle string deploys := 0 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch { @@ -1296,7 +1696,14 @@ func TestApplyActivateRoutesUpdatesDomainsAndRedeploys(t *testing.T) { _ = json.NewEncoder(w).Encode(Domain{DomainID: updatedDomain.DomainID, Host: updatedDomain.Host, DomainType: updatedDomain.DomainType, ComposeID: "c1", ServiceName: updatedDomain.ServiceName, Port: updatedDomain.Port}) case r.Method == http.MethodPost && r.URL.Path == "/api/compose.deploy": deploys++ + var request deployComposeRequest + if err := json.NewDecoder(r.Body).Decode(&request); err != nil { + t.Fatalf("decode compose.deploy: %v", err) + } + deploymentTitle = request.Title w.WriteHeader(http.StatusOK) + case r.Method == http.MethodGet && r.URL.Path == "/api/compose.one": + _ = json.NewEncoder(w).Encode(Compose{ComposeID: "c1", AppName: "stack-1", Deployments: []Deployment{{Title: deploymentTitle, Status: "done"}}}) default: http.NotFound(w, r) } @@ -1361,6 +1768,7 @@ func TestApplyActivateRoutesDetectsMigratedVolumeDriftAfterDeploy(t *testing.T) if err := os.WriteFile(filepath.Join(appDir, "compose.yaml"), []byte("services:\n web:\n image: example/web\n"), 0o600); err != nil { t.Fatalf("write compose: %v", err) } + var deploymentTitle string deploys := 0 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch { @@ -1368,7 +1776,14 @@ func TestApplyActivateRoutesDetectsMigratedVolumeDriftAfterDeploy(t *testing.T) w.WriteHeader(http.StatusOK) case r.Method == http.MethodPost && r.URL.Path == "/api/compose.deploy": deploys++ + var request deployComposeRequest + if err := json.NewDecoder(r.Body).Decode(&request); err != nil { + t.Fatalf("decode compose.deploy: %v", err) + } + deploymentTitle = request.Title w.WriteHeader(http.StatusOK) + case r.Method == http.MethodGet && r.URL.Path == "/api/compose.one": + _ = json.NewEncoder(w).Encode(Compose{ComposeID: "c1", AppName: "stack-1", Deployments: []Deployment{{Title: deploymentTitle, Status: "done"}}}) default: http.NotFound(w, r) } diff --git a/internal/target/dokploy/client.go b/internal/target/dokploy/client.go index 7f3bcd0..0bbd51d 100644 --- a/internal/target/dokploy/client.go +++ b/internal/target/dokploy/client.go @@ -108,24 +108,36 @@ func (c *Client) do(ctx context.Context, method, path string, query url.Values, req.Header.Set("Accept", "application/json") req.Header.Set("x-api-key", c.Token) - resp, err := c.HTTPClient.Do(req) + httpClient := *c.httpClient() + httpClient.CheckRedirect = func(*http.Request, []*http.Request) error { + return http.ErrUseLastResponse + } + resp, err := httpClient.Do(req) if err != nil { return fmt.Errorf("dokploy %s %s: %w", method, path, err) } defer resp.Body.Close() - data, err := io.ReadAll(resp.Body) - if err != nil { - return fmt.Errorf("read %s response: %w", path, err) - } - if resp.StatusCode >= 400 { + data, readErr := io.ReadAll(resp.Body) + if resp.StatusCode < 200 || resp.StatusCode >= 300 { apiErr := &APIError{Status: resp.StatusCode} _ = json.Unmarshal(data, apiErr) if apiErr.Message == "" { apiErr.Message = strings.TrimSpace(string(data)) } + if readErr != nil { + readFailure := fmt.Sprintf("read response: %v", readErr) + if apiErr.Message == "" { + apiErr.Message = readFailure + } else { + apiErr.Message += " (" + readFailure + ")" + } + } return apiErr } + if readErr != nil && out != nil { + return fmt.Errorf("read %s response: %w", path, readErr) + } if out != nil && len(data) > 0 { if err := json.Unmarshal(data, out); err != nil { return fmt.Errorf("decode %s response: %w", path, err) diff --git a/internal/target/dokploy/datamove.go b/internal/target/dokploy/datamove.go index 895d2b8..b2d9551 100644 --- a/internal/target/dokploy/datamove.go +++ b/internal/target/dokploy/datamove.go @@ -1356,8 +1356,19 @@ func (c *Client) targetContainerForServiceWithRedeploy(ctx context.Context, runn } if allowRedeploy && len(containers) == 0 && entry.ComposeID != "" && !entry.DiscoveryRedeployAttempted { entry.DiscoveryRedeployAttempted = true - if err := c.deployComposeForApply(ctx, actx, appName); err != nil { - return dockerContainer{}, fmt.Errorf("redeploy dokploy compose project %q for target discovery: %w", entry.ComposeAppName, err) + composeFile, err := c.composeFileForApply(ctx, actx, appName) + if err != nil { + return dockerContainer{}, err + } + envContent, err := readEnvContent(actx.plan, appName) + if err != nil { + return dockerContainer{}, err + } + redeployCtx, cancelRedeploy := context.WithDeadline(ctx, deadline) + redeployErr := c.deployComposeForApply(redeployCtx, actx, appName, composeFile, envContent) + cancelRedeploy() + if redeployErr != nil { + return dockerContainer{}, fmt.Errorf("redeploy dokploy compose project %q for target discovery: %w", entry.ComposeAppName, redeployErr) } } if time.Now().After(deadline) { diff --git a/internal/target/dokploy/datamove_test.go b/internal/target/dokploy/datamove_test.go index faa477b..a3ed229 100644 --- a/internal/target/dokploy/datamove_test.go +++ b/internal/target/dokploy/datamove_test.go @@ -525,12 +525,23 @@ func TestApplyPushImageTagsMissingComposeImageFromSourceContainer(t *testing.T) t.Fatalf("write compose: %v", err) } + var deploymentTitle string server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/api/compose.deploy" { + switch { + case r.Method == http.MethodPost && r.URL.Path == "/api/compose.update": + w.WriteHeader(http.StatusOK) + case r.Method == http.MethodPost && r.URL.Path == "/api/compose.deploy": + var request deployComposeRequest + if err := json.NewDecoder(r.Body).Decode(&request); err != nil { + t.Fatalf("decode compose.deploy: %v", err) + } + deploymentTitle = request.Title + w.WriteHeader(http.StatusOK) + case r.Method == http.MethodGet && r.URL.Path == "/api/compose.one": + _ = json.NewEncoder(w).Encode(Compose{ComposeID: "compose-1", AppName: "stack-1", Deployments: []Deployment{{Title: deploymentTitle, Status: "done"}}}) + default: http.NotFound(w, r) - return } - w.WriteHeader(http.StatusOK) })) defer server.Close() runner := &fakeDockerRunner{outputs: map[string][]byte{ @@ -565,14 +576,25 @@ func TestApplyPushImageDetectsMigratedVolumeDriftAfterDeploy(t *testing.T) { if err := os.WriteFile(filepath.Join(appDir, "compose.yaml"), []byte("services:\n web:\n image: example/web\n"), 0o600); err != nil { t.Fatalf("write compose: %v", err) } + var deploymentTitle string deploys := 0 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Method == http.MethodPost && r.URL.Path == "/api/compose.deploy" { + switch { + case r.Method == http.MethodPost && r.URL.Path == "/api/compose.update": + w.WriteHeader(http.StatusOK) + case r.Method == http.MethodPost && r.URL.Path == "/api/compose.deploy": deploys++ + var request deployComposeRequest + if err := json.NewDecoder(r.Body).Decode(&request); err != nil { + t.Fatalf("decode compose.deploy: %v", err) + } + deploymentTitle = request.Title w.WriteHeader(http.StatusOK) - return + case r.Method == http.MethodGet && r.URL.Path == "/api/compose.one": + _ = json.NewEncoder(w).Encode(Compose{ComposeID: "c1", AppName: "stack-1", Deployments: []Deployment{{Title: deploymentTitle, Status: "done"}}}) + default: + http.NotFound(w, r) } - http.NotFound(w, r) })) defer server.Close() runner := &latePostDeployTargetRunner{} @@ -1273,6 +1295,14 @@ func TestTargetContainerForServiceReportsComposeDeploymentError(t *testing.T) { } func TestTargetContainerForServiceBlocksActivePatchBeforeDiscoveryRedeploy(t *testing.T) { + bundleDir := t.TempDir() + appDir := filepath.Join(bundleDir, "api") + if err := os.MkdirAll(appDir, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(appDir, "compose.yaml"), []byte("services:\n web:\n image: example/web\n"), 0o600); err != nil { + t.Fatal(err) + } deploys := 0 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch { @@ -1296,7 +1326,8 @@ func TestTargetContainerForServiceBlocksActivePatchBeforeDiscoveryRedeploy(t *te }, } client := &Client{BaseURL: server.URL, Token: "secret", HTTPClient: server.Client(), Docker: runner} - actx := &applyContext{cache: map[string]*appCache{}} + app := preparer.AppPlan{Name: "api", Directory: "api", TargetResources: &preparer.TargetResources{Dokploy: &preparer.DokployResources{ComposeApp: preparer.DokployComposeApp{ComposePath: "compose.yaml"}}}} + actx := &applyContext{cache: map[string]*appCache{}, plan: Plan{Prepare: preparer.Result{BundleDir: bundleDir, Apps: []preparer.AppPlan{app}}}} entry := actx.entry("api") entry.ComposeID = "c1" entry.ComposeAppName = "stack-1" From 5516fbc572a59f677e0a9f0934192bbc3325f9c6 Mon Sep 17 00:00:00 2001 From: Aikins Laryea Date: Mon, 3 Aug 2026 15:47:49 +0000 Subject: [PATCH 17/31] migration: bound post-deploy safety checks (#24) --- internal/target/dokploy/datamove.go | 14 ++- internal/target/dokploy/datamove_test.go | 113 +++++++++++++++++++++++ internal/target/dokploy/docker.go | 37 ++++++-- 3 files changed, 154 insertions(+), 10 deletions(-) diff --git a/internal/target/dokploy/datamove.go b/internal/target/dokploy/datamove.go index b2d9551..1398d76 100644 --- a/internal/target/dokploy/datamove.go +++ b/internal/target/dokploy/datamove.go @@ -467,12 +467,20 @@ func (c *Client) applyResumeTarget(ctx context.Context, actx *applyContext, step return nil } -func (c *Client) validateMigratedVolumeMountsAfterDeploy(_ context.Context, actx *applyContext, appName string) error { - validationCtx, cancelValidation := context.WithTimeout(context.Background(), targetDiscoveryTimeout) +func (c *Client) validateMigratedVolumeMountsAfterDeploy(ctx context.Context, actx *applyContext, appName string) error { + operationDeadline := time.Now().Add(targetDiscoveryTimeout) + if callerDeadline, ok := ctx.Deadline(); ok && callerDeadline.Before(operationDeadline) { + operationDeadline = callerDeadline + } + validationDeadline := operationDeadline.Add(-dockerStopTimeout) + if validationDeadline.Before(time.Now()) { + validationDeadline = time.Now() + } + validationCtx, cancelValidation := context.WithDeadline(context.Background(), validationDeadline) defer cancelValidation() if err := c.validateMigratedVolumeMountsStable(validationCtx, actx, appName); err != nil { if isUnsafeTargetResumeError(err) { - stopCtx, cancelStop := context.WithTimeout(context.Background(), targetDiscoveryTimeout) + stopCtx, cancelStop := context.WithDeadline(context.Background(), operationDeadline) defer cancelStop() if stopErr := c.stopTargetComposeContainers(stopCtx, actx, appName); stopErr != nil { return fmt.Errorf("%w (also failed to stop unsafe target containers: %v)", err, stopErr) diff --git a/internal/target/dokploy/datamove_test.go b/internal/target/dokploy/datamove_test.go index a3ed229..8e47142 100644 --- a/internal/target/dokploy/datamove_test.go +++ b/internal/target/dokploy/datamove_test.go @@ -5,11 +5,13 @@ import ( "context" "encoding/json" "errors" + "fmt" "io" "net/http" "net/http/httptest" "os" "path/filepath" + "slices" "strings" "testing" "time" @@ -224,6 +226,20 @@ func TestStopContainerKillsAfterStopTimeout(t *testing.T) { } } +func TestStopPhaseContextPreservesDefaultDockerGrace(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), dockerStopTimeout) + defer cancel() + phaseCtx, cancelPhase := stopPhaseContext(ctx, dockerStopTimeout, 2) + defer cancelPhase() + deadline, ok := phaseCtx.Deadline() + if !ok { + t.Fatal("stop phase has no deadline") + } + if remaining := time.Until(deadline); remaining < 10*time.Second || remaining >= dockerStopTimeout { + t.Fatalf("initial stop phase = %s, want at least Docker's default grace within the total stop budget", remaining) + } +} + func TestStopContainerTreatsStoppedContainerAsSuccessAfterShortStopFailure(t *testing.T) { runner := &stopKillInspectRunner{} ctx, cancel := context.WithTimeout(context.Background(), time.Nanosecond) @@ -1644,6 +1660,103 @@ func TestPostDeployValidationIgnoresCanceledApplyContextAndStopsLateTarget(t *te } } +type deadlineBoundStopRunner struct { + calls []string + deadlines []time.Time + activeAtCall []bool + psCalls int +} + +func (r *deadlineBoundStopRunner) Output(ctx context.Context, args ...string) ([]byte, error) { + r.calls = append(r.calls, strings.Join(args, " ")) + r.activeAtCall = append(r.activeAtCall, ctx.Err() == nil) + if deadline, ok := ctx.Deadline(); ok { + r.deadlines = append(r.deadlines, deadline) + } + key := strings.Join(args, " ") + if key == "ps -a --filter label=com.docker.compose.project=stack-1 --format {{.ID}}" { + r.psCalls++ + if r.psCalls == 1 { + <-ctx.Done() + return nil, ctx.Err() + } + return []byte("web-id\n"), nil + } + if err := ctx.Err(); err != nil { + return nil, err + } + switch key { + case "inspect --type container web-id": + return []byte(`[{"Id":"web-id","Name":"/web","Config":{"Labels":{"com.docker.compose.service":"web","com.docker.compose.project":"stack-1"}},"State":{"Running":true,"Status":"running"},"Mounts":[{"Type":"volume","Name":"fresh-vol","Destination":"/data","RW":true}]}]`), nil + case "stop web-id", "stop -t 2 web-id", "kill web-id": + <-ctx.Done() + return nil, ctx.Err() + default: + return nil, fmt.Errorf("docker output not stubbed: %s", key) + } +} + +func (r *deadlineBoundStopRunner) Run(context.Context, io.Reader, io.Writer, ...string) error { + return nil +} + +func TestPostDeployValidationRespectsCallerDeadline(t *testing.T) { + runner := &deadlineBoundStopRunner{} + client := &Client{Docker: runner} + actx := &applyContext{cache: map[string]*appCache{}, plan: Plan{}} + entry := actx.entry("api") + entry.ComposeAppName = "stack-1" + entry.MigratedVolumeMounts = map[string]migratedVolumeMount{ + migratedMountKey("web", "/data"): {Service: "web", Target: "/data", VolumeName: "migrated-vol"}, + } + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + callerDeadline, ok := ctx.Deadline() + if !ok { + t.Fatal("test context has no deadline") + } + started := time.Now() + + err := client.validateMigratedVolumeMountsAfterDeploy(ctx, actx, "api") + if err == nil || !strings.Contains(err.Error(), "context deadline exceeded") || !isUnsafeTargetResumeError(err) { + t.Fatalf("expected caller deadline to bound post-deploy validation, got %v", err) + } + if elapsed := time.Since(started); elapsed >= time.Second { + t.Fatalf("post-deploy validation exceeded caller deadline: %s", elapsed) + } + wantCalls := []string{ + "ps -a --filter label=com.docker.compose.project=stack-1 --format {{.ID}}", + "ps -a --filter label=com.docker.compose.project=stack-1 --format {{.ID}}", + "inspect --type container web-id", + "stop web-id", + "stop -t 2 web-id", + "inspect --type container web-id", + "kill web-id", + "inspect --type container web-id", + } + if !slices.Equal(runner.calls, wantCalls) { + t.Fatalf("emergency stop calls = %v, want %v", runner.calls, wantCalls) + } + if len(runner.deadlines) != len(runner.calls) { + t.Fatalf("some validation or stop commands had no deadline: calls=%v deadlines=%v", runner.calls, runner.deadlines) + } + for _, deadline := range runner.deadlines { + if deadline.After(callerDeadline) { + t.Fatalf("validation or stop deadline %s exceeded caller deadline %s", deadline, callerDeadline) + } + } + for i := 3; i < len(runner.activeAtCall); i++ { + if !runner.activeAtCall[i] { + t.Fatalf("emergency stop phase %q started with an expired context", runner.calls[i]) + } + } + for i := 4; i < len(runner.deadlines); i++ { + if !runner.deadlines[i].After(runner.deadlines[i-1]) { + t.Fatalf("emergency stop phase deadlines did not increase: %v", runner.deadlines[3:]) + } + } +} + func TestPostDeployValidationDoesNotPassOnPreDeployContainer(t *testing.T) { runner := &postDeployRecreateRunner{} client := &Client{Docker: runner} diff --git a/internal/target/dokploy/docker.go b/internal/target/dokploy/docker.go index 265a3b2..394221f 100644 --- a/internal/target/dokploy/docker.go +++ b/internal/target/dokploy/docker.go @@ -77,29 +77,37 @@ const ( dockerStopTimeout = 30 * time.Second dockerStopFallbackGrace = "2" dockerStopFallbackWindow = 30 * time.Second + dockerStopInspectWindow = 10 * time.Second + dockerKillWindow = 15 * time.Second dockerStartTimeout = 3 * time.Minute ) func stopContainer(ctx context.Context, runner dockerRunner, id string) error { - stopCtx, cancel := context.WithTimeout(ctx, dockerStopTimeout) + stopCtx, cancel := stopPhaseContext(ctx, dockerStopTimeout, 2) defer cancel() if _, err := runner.Output(stopCtx, "stop", id); err != nil { if !errors.Is(stopCtx.Err(), context.DeadlineExceeded) { return err } - shortStopCtx, cancelShortStop := context.WithTimeout(context.Background(), dockerStopFallbackWindow) + var fallbackCtx context.Context = context.Background() + cancelFallback := func() {} + if deadline, ok := ctx.Deadline(); ok { + fallbackCtx, cancelFallback = context.WithDeadline(fallbackCtx, deadline) + } + defer cancelFallback() + shortStopCtx, cancelShortStop := stopPhaseContext(fallbackCtx, dockerStopFallbackWindow, 4) shortStopErr := func() error { defer cancelShortStop() _, shortStopErr := runner.Output(shortStopCtx, "stop", "-t", dockerStopFallbackGrace, id) return shortStopErr }() - if shortStopErr == nil || containerStoppedAfterStopFailure(runner, id) { + if shortStopErr == nil || containerStoppedAfterStopFailure(fallbackCtx, runner, id, 3) { return nil } - killCtx, cancelKill := context.WithTimeout(context.Background(), 15*time.Second) + killCtx, cancelKill := stopPhaseContext(fallbackCtx, dockerKillWindow, 2) defer cancelKill() if _, killErr := runner.Output(killCtx, "kill", id); killErr != nil { - if containerStoppedAfterStopFailure(runner, id) { + if containerStoppedAfterStopFailure(fallbackCtx, runner, id, 1) { return nil } return fmt.Errorf("docker stop %s timed out and kill failed: %w", id, killErr) @@ -108,8 +116,23 @@ func stopContainer(ctx context.Context, runner dockerRunner, id string) error { return nil } -func containerStoppedAfterStopFailure(runner dockerRunner, id string) bool { - inspectCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) +func stopPhaseContext(ctx context.Context, maximum time.Duration, remainingPhases int) (context.Context, context.CancelFunc) { + timeout := maximum + if deadline, ok := ctx.Deadline(); ok { + remaining := time.Until(deadline) + share := remaining / time.Duration(remainingPhases) + if share < timeout { + timeout = share + } + } + if timeout < 0 { + timeout = 0 + } + return context.WithTimeout(ctx, timeout) +} + +func containerStoppedAfterStopFailure(ctx context.Context, runner dockerRunner, id string, remainingPhases int) bool { + inspectCtx, cancel := stopPhaseContext(ctx, dockerStopInspectWindow, remainingPhases) defer cancel() container, err := inspectContainer(inspectCtx, runner, id) return err == nil && !container.State.Running From d3292826d80f6459d1fa4103cd25851f38f6baa0 Mon Sep 17 00:00:00 2001 From: Aikins Laryea Date: Mon, 3 Aug 2026 16:07:18 +0000 Subject: [PATCH 18/31] dokploy: avoid copying shared HTTP client (#25) --- internal/target/dokploy/client.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/internal/target/dokploy/client.go b/internal/target/dokploy/client.go index 0bbd51d..1be692a 100644 --- a/internal/target/dokploy/client.go +++ b/internal/target/dokploy/client.go @@ -108,9 +108,14 @@ func (c *Client) do(ctx context.Context, method, path string, query url.Values, req.Header.Set("Accept", "application/json") req.Header.Set("x-api-key", c.Token) - httpClient := *c.httpClient() - httpClient.CheckRedirect = func(*http.Request, []*http.Request) error { - return http.ErrUseLastResponse + configuredClient := c.httpClient() + httpClient := &http.Client{ + Transport: configuredClient.Transport, + CheckRedirect: func(*http.Request, []*http.Request) error { + return http.ErrUseLastResponse + }, + Jar: configuredClient.Jar, + Timeout: configuredClient.Timeout, } resp, err := httpClient.Do(req) if err != nil { From 9040d90a840f82d45647c7792d5752ef6ad9af35 Mon Sep 17 00:00:00 2001 From: Aikins Laryea Date: Mon, 3 Aug 2026 16:51:18 +0000 Subject: [PATCH 19/31] cleanup: remove incomplete database backups (#26) --- internal/target/dokploy/cleanup.go | 17 +++++++++---- internal/target/dokploy/cleanup_test.go | 32 +++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 5 deletions(-) diff --git a/internal/target/dokploy/cleanup.go b/internal/target/dokploy/cleanup.go index 119da09..fa14b17 100644 --- a/internal/target/dokploy/cleanup.go +++ b/internal/target/dokploy/cleanup.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" "os" pathpkg "path" @@ -1027,7 +1028,7 @@ func findDokployPostgresContainer(ctx context.Context, runner dockerRunner) (str return "", fmt.Errorf("dokploy postgres container was not found; cleanup must run on the Dokploy host") } -func backupDokployDatabase(ctx context.Context, runner dockerRunner, pg, backupDir, backupPrefix string) (string, error) { +func backupDokployDatabase(ctx context.Context, runner dockerRunner, pg, backupDir, backupPrefix string) (_ string, resultErr error) { backupDir = strings.TrimSpace(backupDir) if backupDir == "" { backupDir = filepath.Join(".bort", "backups") @@ -1046,17 +1047,22 @@ func backupDokployDatabase(ctx context.Context, runner dockerRunner, pg, backupD if err != nil { return "", err } + removeBackup := true + defer func() { + if removeBackup { + _ = file.Close() + if err := dir.Remove(name); err != nil { + resultErr = errors.Join(resultErr, fmt.Errorf("remove incomplete Dokploy database backup %s: %w", path, err)) + } + } + }() if err := runner.Run(ctx, nil, file, "exec", pg, "pg_dump", "-U", "dokploy", "-d", "dokploy"); err != nil { - _ = file.Close() - _ = dir.Remove(name) return "", fmt.Errorf("backup dokploy database to %s: %w", path, err) } if err := file.Chmod(0o600); err != nil { - _ = file.Close() return "", err } if err := file.Sync(); err != nil { - _ = file.Close() return "", err } if err := file.Close(); err != nil { @@ -1068,6 +1074,7 @@ func backupDokployDatabase(ctx context.Context, runner dockerRunner, pg, backupD if err := dir.ValidatePath(); err != nil { return "", fmt.Errorf("verify dokploy database backup location %s: %w", path, err) } + removeBackup = false return path, nil } diff --git a/internal/target/dokploy/cleanup_test.go b/internal/target/dokploy/cleanup_test.go index 5bd3b51..ff89e6a 100644 --- a/internal/target/dokploy/cleanup_test.go +++ b/internal/target/dokploy/cleanup_test.go @@ -3,6 +3,7 @@ package dokploy import ( "context" "errors" + "io" "os" "path/filepath" "strings" @@ -115,6 +116,37 @@ func TestBackupDokployDatabaseRejectsSymlinkDirectory(t *testing.T) { } } +type closingBackupRunner struct{} + +func (closingBackupRunner) Output(context.Context, ...string) ([]byte, error) { + return nil, errors.New("unexpected output call") +} + +func (closingBackupRunner) Run(_ context.Context, _ io.Reader, stdout io.Writer, _ ...string) error { + file, ok := stdout.(*os.File) + if !ok { + return errors.New("backup output is not a file") + } + if _, err := file.WriteString("incomplete"); err != nil { + return err + } + return file.Close() +} + +func TestBackupDokployDatabaseRemovesFileAfterFinalizationFailure(t *testing.T) { + backupDir := dokployBackupTestDir(t) + if _, err := backupDokployDatabase(context.Background(), closingBackupRunner{}, "dokploy-postgres", backupDir, "cleanup"); err == nil { + t.Fatal("expected backup finalization to fail") + } + entries, err := os.ReadDir(backupDir) + if err != nil { + t.Fatal(err) + } + if len(entries) != 0 { + t.Fatalf("incomplete backup remained visible: %#v", entries) + } +} + func dokployBackupTestDir(t *testing.T) string { t.Helper() cacheDir, err := os.UserCacheDir() From a6a9acdee5459ac3079a2b4670044b5a7fbff6d6 Mon Sep 17 00:00:00 2001 From: Aikins Laryea Date: Mon, 3 Aug 2026 17:05:29 +0000 Subject: [PATCH 20/31] cleanup: protect Docker gateway network (#27) --- internal/target/dokploy/cleanup.go | 2 +- internal/target/dokploy/cleanup_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/target/dokploy/cleanup.go b/internal/target/dokploy/cleanup.go index fa14b17..46abae2 100644 --- a/internal/target/dokploy/cleanup.go +++ b/internal/target/dokploy/cleanup.go @@ -856,7 +856,7 @@ func validateDockerPurgeName(kind, name string) error { func IsProtectedSourcePurgeNetwork(name string) bool { switch strings.TrimSpace(name) { - case "bridge", "host", "none", "ingress": + case "bridge", "host", "none", "ingress", "docker_gwbridge": return true default: return false diff --git a/internal/target/dokploy/cleanup_test.go b/internal/target/dokploy/cleanup_test.go index ff89e6a..6ce1980 100644 --- a/internal/target/dokploy/cleanup_test.go +++ b/internal/target/dokploy/cleanup_test.go @@ -780,7 +780,7 @@ func TestDockerMissingResourceClassifierRejectsInfrastructureErrors(t *testing.T } func TestProtectedSourcePurgeNetworkNamesAreCaseSensitive(t *testing.T) { - for _, name := range []string{"bridge", "host", "none", "ingress"} { + for _, name := range []string{"bridge", "host", "none", "ingress", "docker_gwbridge"} { if !IsProtectedSourcePurgeNetwork(name) { t.Fatalf("expected %q to be protected", name) } From ba5e811650b1d8ad92ff79562520b979c85044b7 Mon Sep 17 00:00:00 2001 From: Aikins Laryea Date: Mon, 3 Aug 2026 17:23:56 +0000 Subject: [PATCH 21/31] cleanup: reject conflicting purge paths (#28) --- internal/target/dokploy/cleanup.go | 28 ++++++++++++++++++------- internal/target/dokploy/cleanup_test.go | 21 +++++++++++++++++++ 2 files changed, 42 insertions(+), 7 deletions(-) diff --git a/internal/target/dokploy/cleanup.go b/internal/target/dokploy/cleanup.go index 46abae2..00d5bbf 100644 --- a/internal/target/dokploy/cleanup.go +++ b/internal/target/dokploy/cleanup.go @@ -116,7 +116,10 @@ func (c *Client) PurgeSourceResources(ctx context.Context, opts SourcePurgeOptio return result, err } volumes := cleanupSourcePurgeVolumes(opts.Volumes) - paths := cleanupSourcePurgePaths(opts.Paths) + paths, err := cleanupSourcePurgePaths(opts.Paths) + if err != nil { + return result, err + } networks, err := cleanupSourcePurgeNetworks(opts.Networks) if err != nil { return result, err @@ -300,7 +303,11 @@ func (c *Client) IdentifySourcePurgeResources(ctx context.Context, opts SourcePu identified.Networks = append(identified.Networks, network) } identified.Paths = nil - for _, path := range cleanupSourcePurgePaths(opts.Paths) { + paths, err := cleanupSourcePurgePaths(opts.Paths) + if err != nil { + return SourcePurgeOptions{}, err + } + for _, path := range paths { absent, err := sourcePurgePathAbsentNoFollow(path.Path, path.AllowPlatform) if err != nil { return SourcePurgeOptions{}, err @@ -547,22 +554,29 @@ func validateSourcePurgeNetworksForExecution(networks []SourcePurgeNetwork) erro return nil } -func cleanupSourcePurgePaths(paths []SourcePurgePath) []SourcePurgePath { - seen := map[string]struct{}{} +func cleanupSourcePurgePaths(paths []SourcePurgePath) ([]SourcePurgePath, error) { + seen := map[string]int{} cleaned := []SourcePurgePath{} for _, path := range paths { path.Path = pathpkg.Clean(strings.TrimSpace(path.Path)) if path.Path == "." || path.Path == "" { continue } - if _, ok := seen[path.Path]; ok { + if index, ok := seen[path.Path]; ok { + current := cleaned[index] + if current.AllowPlatform != path.AllowPlatform { + return nil, fmt.Errorf("source path %q has conflicting platform authorization", path.Path) + } + if current.ExpectedAbsent != path.ExpectedAbsent { + return nil, fmt.Errorf("source path %q has conflicting expected absence state", path.Path) + } continue } - seen[path.Path] = struct{}{} + seen[path.Path] = len(cleaned) cleaned = append(cleaned, path) } sort.Slice(cleaned, func(i, j int) bool { return cleaned[i].Path < cleaned[j].Path }) - return cleaned + return cleaned, nil } func purgeSourceContainer(ctx context.Context, runner dockerRunner, item SourcePurgeContainer) (SourcePurgeResourceResult, error) { diff --git a/internal/target/dokploy/cleanup_test.go b/internal/target/dokploy/cleanup_test.go index 6ce1980..f368146 100644 --- a/internal/target/dokploy/cleanup_test.go +++ b/internal/target/dokploy/cleanup_test.go @@ -326,6 +326,27 @@ func TestCleanupSourcePurgeNetworksRejectsConflictingIdentityState(t *testing.T) } } +func TestCleanupSourcePurgePathsRejectsConflictingState(t *testing.T) { + for name, paths := range map[string][]SourcePurgePath{ + "platform authorization": { + {Path: "/data/coolify", AllowPlatform: true}, + {Path: "/data/coolify"}, + }, + "expected absence": { + {Path: "/data/app", ExpectedAbsent: true}, + {Path: "/data/app"}, + }, + } { + t.Run(name, func(t *testing.T) { + for _, ordered := range [][]SourcePurgePath{paths, {paths[1], paths[0]}} { + if _, err := cleanupSourcePurgePaths(ordered); err == nil || !strings.Contains(err.Error(), "conflicting") { + t.Fatalf("expected conflicting duplicate paths to fail closed, got %v", err) + } + } + }) + } +} + func TestPurgeSourceResourcesSkipsAbsentAllowedPath(t *testing.T) { client := &Client{Docker: &fakeDockerRunner{}} options, err := client.IdentifySourcePurgeResources(context.Background(), SourcePurgeOptions{Paths: []SourcePurgePath{{Path: "/data/coolify/applications/app-1"}}}) From da3d3477d85aff740f429bb56c4e8a1ba10d452e Mon Sep 17 00:00:00 2001 From: Aikins Laryea Date: Mon, 3 Aug 2026 17:41:06 +0000 Subject: [PATCH 22/31] safepath: keep private reads non-mutating (#29) --- internal/safepath/safepath.go | 6 +++++- internal/safepath/safepath_test.go | 18 ++++++++++++++++++ internal/safepath/safepath_unix.go | 2 +- 3 files changed, 24 insertions(+), 2 deletions(-) create mode 100644 internal/safepath/safepath_test.go diff --git a/internal/safepath/safepath.go b/internal/safepath/safepath.go index d29f791..0427247 100644 --- a/internal/safepath/safepath.go +++ b/internal/safepath/safepath.go @@ -37,6 +37,10 @@ type PrivateDir struct { } func OpenPrivateDirNoFollow(path string) (*PrivateDir, error) { + return openPrivateDirPathNoFollow(path, true) +} + +func openPrivateDirPathNoFollow(path string, create bool) (*PrivateDir, error) { if strings.TrimSpace(path) == "" { return nil, fmt.Errorf("private directory path is empty") } @@ -53,7 +57,7 @@ func OpenPrivateDirNoFollow(path string) (*PrivateDir, error) { abs = filepath.Join(cwd, path) } abs = filepath.Clean(abs) - dir, err := openPrivateDirNoFollow(abs, true) + dir, err := openPrivateDirNoFollow(abs, create) if err != nil { return nil, err } diff --git a/internal/safepath/safepath_test.go b/internal/safepath/safepath_test.go new file mode 100644 index 0000000..8cf59e5 --- /dev/null +++ b/internal/safepath/safepath_test.go @@ -0,0 +1,18 @@ +package safepath + +import ( + "errors" + "os" + "path/filepath" + "testing" +) + +func TestReadPrivateFileDoesNotCreateMissingDirectory(t *testing.T) { + path := filepath.Join(t.TempDir(), "missing", "nested") + if _, err := ReadPrivateFile(path, "manifest.json"); err == nil { + t.Fatal("expected missing private file read to fail") + } + if _, err := os.Stat(path); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("private read created missing directory: %v", err) + } +} diff --git a/internal/safepath/safepath_unix.go b/internal/safepath/safepath_unix.go index 3628e17..1197a13 100644 --- a/internal/safepath/safepath_unix.go +++ b/internal/safepath/safepath_unix.go @@ -122,7 +122,7 @@ func openPrivateFileNoFollow(dir *os.File, name string) (*os.File, error) { } func readPrivateFile(path, name string) ([]byte, error) { - dir, err := OpenPrivateDirNoFollow(path) + dir, err := openPrivateDirPathNoFollow(path, false) if err != nil { return nil, err } From 73697c97bf1b1bbd620de8881c69c2ab2e3861fd Mon Sep 17 00:00:00 2001 From: Aikins Laryea Date: Mon, 3 Aug 2026 18:18:30 +0000 Subject: [PATCH 23/31] cleanup: reject conflicting purge volumes (#30) --- internal/target/dokploy/cleanup.go | 24 +++++++++++++++++------- internal/target/dokploy/cleanup_test.go | 16 ++++++++++++++++ 2 files changed, 33 insertions(+), 7 deletions(-) diff --git a/internal/target/dokploy/cleanup.go b/internal/target/dokploy/cleanup.go index 00d5bbf..3648ddf 100644 --- a/internal/target/dokploy/cleanup.go +++ b/internal/target/dokploy/cleanup.go @@ -115,7 +115,10 @@ func (c *Client) PurgeSourceResources(ctx context.Context, opts SourcePurgeOptio if err != nil { return result, err } - volumes := cleanupSourcePurgeVolumes(opts.Volumes) + volumes, err := cleanupSourcePurgeVolumes(opts.Volumes) + if err != nil { + return result, err + } paths, err := cleanupSourcePurgePaths(opts.Paths) if err != nil { return result, err @@ -264,7 +267,11 @@ func (c *Client) IdentifySourcePurgeResources(ctx context.Context, opts SourcePu } identified.Containers = containers identified.Volumes = nil - for _, volume := range cleanupSourcePurgeVolumes(opts.Volumes) { + volumes, err := cleanupSourcePurgeVolumes(opts.Volumes) + if err != nil { + return SourcePurgeOptions{}, err + } + for _, volume := range volumes { absent, err := inspectSourcePurgeVolumeAbsent(ctx, runner, volume.Name) if err != nil { return SourcePurgeOptions{}, err @@ -474,22 +481,25 @@ func cleanupSourcePurgeContainers(containers []SourcePurgeContainer) ([]SourcePu return cleaned, nil } -func cleanupSourcePurgeVolumes(volumes []SourcePurgeVolume) []SourcePurgeVolume { - seen := map[string]struct{}{} +func cleanupSourcePurgeVolumes(volumes []SourcePurgeVolume) ([]SourcePurgeVolume, error) { + seen := map[string]int{} cleaned := []SourcePurgeVolume{} for _, volume := range volumes { volume.Name = strings.TrimSpace(volume.Name) if volume.Name == "" { continue } - if _, ok := seen[volume.Name]; ok { + if index, ok := seen[volume.Name]; ok { + if cleaned[index].ExpectedAbsent != volume.ExpectedAbsent { + return nil, fmt.Errorf("source volume %q has conflicting expected absence state", volume.Name) + } continue } - seen[volume.Name] = struct{}{} + seen[volume.Name] = len(cleaned) cleaned = append(cleaned, volume) } sort.Slice(cleaned, func(i, j int) bool { return cleaned[i].Name < cleaned[j].Name }) - return cleaned + return cleaned, nil } func cleanupSourcePurgeNetworks(networks []SourcePurgeNetwork) ([]SourcePurgeNetwork, error) { diff --git a/internal/target/dokploy/cleanup_test.go b/internal/target/dokploy/cleanup_test.go index f368146..68d4ddc 100644 --- a/internal/target/dokploy/cleanup_test.go +++ b/internal/target/dokploy/cleanup_test.go @@ -326,6 +326,22 @@ func TestCleanupSourcePurgeNetworksRejectsConflictingIdentityState(t *testing.T) } } +func TestSourcePurgeRejectsConflictingVolumeAbsenceState(t *testing.T) { + client := &Client{Docker: &fakeDockerRunner{}} + volumes := []SourcePurgeVolume{ + {Name: "api-data", ExpectedAbsent: true}, + {Name: "api-data"}, + } + for _, ordered := range [][]SourcePurgeVolume{volumes, {volumes[1], volumes[0]}} { + if _, err := client.IdentifySourcePurgeResources(context.Background(), SourcePurgeOptions{Volumes: ordered}); err == nil || !strings.Contains(err.Error(), "conflicting") { + t.Fatalf("expected identification to reject conflicting duplicate volumes, got %v", err) + } + if _, err := client.PurgeSourceResources(context.Background(), SourcePurgeOptions{Volumes: ordered}); err == nil || !strings.Contains(err.Error(), "conflicting") { + t.Fatalf("expected apply to reject conflicting duplicate volumes, got %v", err) + } + } +} + func TestCleanupSourcePurgePathsRejectsConflictingState(t *testing.T) { for name, paths := range map[string][]SourcePurgePath{ "platform authorization": { From cb3169835a7918689b3a2a55cb799747661933ec Mon Sep 17 00:00:00 2001 From: Aikins Laryea Date: Mon, 3 Aug 2026 18:34:54 +0000 Subject: [PATCH 24/31] safepath: report temporary cleanup failures (#31) --- internal/safepath/safepath_unix.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/internal/safepath/safepath_unix.go b/internal/safepath/safepath_unix.go index 1197a13..084935d 100644 --- a/internal/safepath/safepath_unix.go +++ b/internal/safepath/safepath_unix.go @@ -137,7 +137,7 @@ func readPrivateFile(path, name string) ([]byte, error) { return contents, nil } -func writePrivateFileAtomicNoFollow(dir *os.File, name string, data []byte, mode os.FileMode) error { +func writePrivateFileAtomicNoFollow(dir *os.File, name string, data []byte, mode os.FileMode) (resultErr error) { tmpName, err := privateTempName() if err != nil { return err @@ -150,7 +150,9 @@ func writePrivateFileAtomicNoFollow(dir *os.File, name string, data []byte, mode defer func() { _ = tmp.Close() if removeTmp { - _ = removePrivateFileNoFollow(dir, tmpName) + if err := removePrivateFileNoFollow(dir, tmpName); err != nil { + resultErr = errors.Join(resultErr, fmt.Errorf("remove incomplete private file %s: %w", filepath.Join(dir.Name(), tmpName), err)) + } } }() if _, err := tmp.Write(data); err != nil { From 5323fc686bc6cb72117172dd5274cd716b155fe8 Mon Sep 17 00:00:00 2001 From: Aikins Laryea Date: Mon, 3 Aug 2026 19:10:10 +0000 Subject: [PATCH 25/31] cleanup: enforce destructive safety boundaries (#32) --- internal/cli/cleanup.go | 4 + internal/cli/cleanup_test.go | 14 +++ internal/target/dokploy/cleanup.go | 57 ++++++++---- internal/target/dokploy/cleanup_test.go | 110 ++++++++++++++++++++---- 4 files changed, 150 insertions(+), 35 deletions(-) diff --git a/internal/cli/cleanup.go b/internal/cli/cleanup.go index 28f4abb..82c2417 100644 --- a/internal/cli/cleanup.go +++ b/internal/cli/cleanup.go @@ -9,6 +9,7 @@ import ( "io" "os" "path/filepath" + "runtime" "sort" "strings" "time" @@ -355,6 +356,9 @@ func runCleanupPurge(ctx context.Context, args []string, stdin io.Reader, stdout if err := confirmCleanupPurgeApply(stdin, stderr, run.Run.Name, confirm); err != nil { return err } + if runtime.GOOS != "linux" { + return fmt.Errorf("cleanup purge --apply is unavailable on %s; rerun the same scoped and confirmed command on the Linux source host", runtime.GOOS) + } recorder, err := newCleanupPurgeBackupRecorder(backupDir, &result) if err != nil { return err diff --git a/internal/cli/cleanup_test.go b/internal/cli/cleanup_test.go index 857e9af..6f67af5 100644 --- a/internal/cli/cleanup_test.go +++ b/internal/cli/cleanup_test.go @@ -147,6 +147,20 @@ func TestLifecycleAcceptanceTraceWithFakeDokploy(t *testing.T) { if !strings.Contains(purgeOut.String(), "Source containers scheduled for purge") { t.Fatalf("expected purge inventory, got:\n%s", purgeOut.String()) } + if runtime.GOOS != "linux" { + err := runCleanup(context.Background(), []string{"purge", "--run", "acceptance", "--apply", "--all-apps", "--confirm", "purge acceptance"}, io.Discard, io.Discard) + if err == nil || !strings.Contains(err.Error(), "Linux source host") { + t.Fatalf("expected destructive purge to require Linux, got %v", err) + } + backups, err := filepath.Glob(filepath.Join(".bort", "backups", "source-purge-*")) + if err != nil { + t.Fatal(err) + } + if len(backups) != 0 { + t.Fatalf("platform rejection created purge backups: %#v", backups) + } + return + } if err := runCleanup(context.Background(), []string{"purge", "--run", "acceptance", "--apply", "--all-apps", "--confirm", "purge acceptance"}, io.Discard, io.Discard); err != nil { t.Fatal(err) } diff --git a/internal/target/dokploy/cleanup.go b/internal/target/dokploy/cleanup.go index 3648ddf..bf8a12e 100644 --- a/internal/target/dokploy/cleanup.go +++ b/internal/target/dokploy/cleanup.go @@ -9,6 +9,7 @@ import ( "os" pathpkg "path" "path/filepath" + "runtime" "sort" "strings" "time" @@ -87,6 +88,13 @@ type SourcePurgeResourceResult struct { } func (c *Client) CleanupStalePlatformProjects(ctx context.Context, opts StalePlatformCleanupOptions) (StalePlatformCleanupResult, error) { + if runtime.GOOS != "linux" { + return StalePlatformCleanupResult{}, fmt.Errorf("Dokploy metadata cleanup is unavailable on %s; run cleanup --apply on the Linux Dokploy host", runtime.GOOS) + } + return c.cleanupStalePlatformProjects(ctx, opts) +} + +func (c *Client) cleanupStalePlatformProjects(ctx context.Context, opts StalePlatformCleanupOptions) (StalePlatformCleanupResult, error) { names := cleanupProjectNames(opts.ProjectNames) if len(names) == 0 { return StalePlatformCleanupResult{}, fmt.Errorf("at least one project name is required") @@ -96,18 +104,29 @@ func (c *Client) CleanupStalePlatformProjects(ctx context.Context, opts StalePla if err != nil { return StalePlatformCleanupResult{}, err } - backupPath, err := backupDokployDatabase(ctx, runner, pg, opts.BackupDir, opts.BackupPrefix) + backup, err := backupDokployDatabase(ctx, runner, pg, opts.BackupDir, opts.BackupPrefix) if err != nil { return StalePlatformCleanupResult{}, err } + defer backup.dir.Close() + if err := backup.dir.ValidatePath(); err != nil { + return StalePlatformCleanupResult{}, fmt.Errorf("verify Dokploy database backup location %s before deleting metadata: %w", backup.path, err) + } deleted, err := deleteStalePlatformProjects(ctx, runner, pg, names, opts.ProjectIDs) if err != nil { - return StalePlatformCleanupResult{}, fmt.Errorf("delete stale Dokploy platform metadata after backup %s: %w", backupPath, err) + return StalePlatformCleanupResult{}, fmt.Errorf("delete stale Dokploy platform metadata after backup %s: %w", backup.path, err) } - return StalePlatformCleanupResult{BackupPath: backupPath, Deleted: deleted}, nil + return StalePlatformCleanupResult{BackupPath: backup.path, Deleted: deleted}, nil } func (c *Client) PurgeSourceResources(ctx context.Context, opts SourcePurgeOptions) (SourcePurgeResult, error) { + if runtime.GOOS != "linux" { + return SourcePurgeResult{}, fmt.Errorf("source resource purge is unavailable on %s; rerun this purge on the Linux source host", runtime.GOOS) + } + return c.purgeSourceResources(ctx, opts) +} + +func (c *Client) purgeSourceResources(ctx context.Context, opts SourcePurgeOptions) (SourcePurgeResult, error) { runner := c.dockerRunner() result := SourcePurgeResult{} @@ -1052,7 +1071,12 @@ func findDokployPostgresContainer(ctx context.Context, runner dockerRunner) (str return "", fmt.Errorf("dokploy postgres container was not found; cleanup must run on the Dokploy host") } -func backupDokployDatabase(ctx context.Context, runner dockerRunner, pg, backupDir, backupPrefix string) (_ string, resultErr error) { +type dokployDatabaseBackup struct { + path string + dir *safepath.PrivateDir +} + +func backupDokployDatabase(ctx context.Context, runner dockerRunner, pg, backupDir, backupPrefix string) (_ *dokployDatabaseBackup, resultErr error) { backupDir = strings.TrimSpace(backupDir) if backupDir == "" { backupDir = filepath.Join(".bort", "backups") @@ -1062,14 +1086,18 @@ func backupDokployDatabase(ctx context.Context, runner dockerRunner, pg, backupD } dir, err := safepath.OpenPrivateDirNoFollow(backupDir) if err != nil { - return "", err + return nil, err } - defer dir.Close() + defer func() { + if resultErr != nil { + _ = dir.Close() + } + }() name := fmt.Sprintf("%s-%s.sql", backupPrefix, time.Now().UTC().Format("20060102-150405.000000000")) path := filepath.Join(backupDir, name) file, err := dir.CreateFile(name, 0o600) if err != nil { - return "", err + return nil, err } removeBackup := true defer func() { @@ -1081,25 +1109,22 @@ func backupDokployDatabase(ctx context.Context, runner dockerRunner, pg, backupD } }() if err := runner.Run(ctx, nil, file, "exec", pg, "pg_dump", "-U", "dokploy", "-d", "dokploy"); err != nil { - return "", fmt.Errorf("backup dokploy database to %s: %w", path, err) + return nil, fmt.Errorf("backup dokploy database to %s: %w", path, err) } if err := file.Chmod(0o600); err != nil { - return "", err + return nil, err } if err := file.Sync(); err != nil { - return "", err + return nil, err } if err := file.Close(); err != nil { - return "", fmt.Errorf("close dokploy database backup %s: %w", path, err) + return nil, fmt.Errorf("close dokploy database backup %s: %w", path, err) } if err := dir.Sync(); err != nil { - return "", fmt.Errorf("sync dokploy database backup directory for %s: %w", path, err) - } - if err := dir.ValidatePath(); err != nil { - return "", fmt.Errorf("verify dokploy database backup location %s: %w", path, err) + return nil, fmt.Errorf("sync dokploy database backup directory for %s: %w", path, err) } removeBackup = false - return path, nil + return &dokployDatabaseBackup{path: path, dir: dir}, nil } func deleteStalePlatformProjects(ctx context.Context, runner dockerRunner, pg string, names []string, projectIDs map[string]string) ([]StalePlatformProject, error) { diff --git a/internal/target/dokploy/cleanup_test.go b/internal/target/dokploy/cleanup_test.go index 68d4ddc..92c8c99 100644 --- a/internal/target/dokploy/cleanup_test.go +++ b/internal/target/dokploy/cleanup_test.go @@ -6,10 +6,34 @@ import ( "io" "os" "path/filepath" + "runtime" "strings" "testing" ) +func TestDestructiveCleanupPlatformBoundary(t *testing.T) { + runner := &fakeDockerRunner{} + client := &Client{Docker: runner} + if runtime.GOOS == "linux" { + if _, err := client.CleanupStalePlatformProjects(context.Background(), StalePlatformCleanupOptions{}); err == nil || !strings.Contains(err.Error(), "at least one project name") { + t.Fatalf("expected metadata cleanup to reach input validation on Linux, got %v", err) + } + if _, err := client.PurgeSourceResources(context.Background(), SourcePurgeOptions{}); err != nil { + t.Fatalf("expected source purge to reach execution on Linux, got %v", err) + } + return + } + if _, err := client.CleanupStalePlatformProjects(context.Background(), StalePlatformCleanupOptions{}); err == nil || !strings.Contains(err.Error(), "Linux Dokploy host") { + t.Fatalf("expected metadata cleanup to require Linux, got %v", err) + } + if _, err := client.PurgeSourceResources(context.Background(), SourcePurgeOptions{}); err == nil || !strings.Contains(err.Error(), "Linux source host") { + t.Fatalf("expected source purge to require Linux, got %v", err) + } + if len(runner.outputArgs) != 0 || len(runner.runs) != 0 { + t.Fatalf("unsupported cleanup reached Docker: output=%#v runs=%#v", runner.outputArgs, runner.runs) + } +} + func TestCleanupStalePlatformProjectsBacksUpThenDeletesMetadata(t *testing.T) { backupDir := dokployBackupTestDir(t) if err := os.Chmod(backupDir, 0o700); err != nil { @@ -25,7 +49,7 @@ func TestCleanupStalePlatformProjectsBacksUpThenDeletesMetadata(t *testing.T) { } client := &Client{Docker: runner} - result, err := client.CleanupStalePlatformProjects(context.Background(), StalePlatformCleanupOptions{ + result, err := client.cleanupStalePlatformProjects(context.Background(), StalePlatformCleanupOptions{ ProjectNames: []string{"proxy", "source", "proxy"}, BackupDir: backupDir, BackupPrefix: "cleanup-test", @@ -70,6 +94,54 @@ func TestCleanupStalePlatformProjectsBacksUpThenDeletesMetadata(t *testing.T) { } } +type replacingBackupDirRunner struct { + fakeDockerRunner + backupDir string + deleteCalls int +} + +func (r *replacingBackupDirRunner) Run(ctx context.Context, stdin io.Reader, stdout io.Writer, args ...string) error { + if len(args) > 2 && args[2] == "pg_dump" { + if _, err := io.WriteString(stdout, "dump-bytes"); err != nil { + return err + } + if err := os.Rename(r.backupDir, r.backupDir+"-held"); err != nil { + return err + } + return os.Mkdir(r.backupDir, 0o700) + } + r.deleteCalls++ + return r.fakeDockerRunner.Run(ctx, stdin, stdout, args...) +} + +func TestCleanupStalePlatformProjectsRejectsReplacedBackupDirectoryBeforeDelete(t *testing.T) { + backupDir := dokployBackupTestDir(t) + runner := &replacingBackupDirRunner{ + fakeDockerRunner: fakeDockerRunner{outputs: map[string][]byte{ + "ps --format {{.Names}}": []byte("dokploy-postgres.1.task\n"), + }}, + backupDir: backupDir, + } + client := &Client{Docker: runner} + _, err := client.cleanupStalePlatformProjects(context.Background(), StalePlatformCleanupOptions{ + ProjectNames: []string{"proxy"}, + BackupDir: backupDir, + }) + if err == nil || !strings.Contains(err.Error(), "private directory path changed") { + t.Fatalf("expected replaced backup directory to block metadata deletion, got %v", err) + } + if runner.deleteCalls != 0 { + t.Fatalf("metadata deletion ran after backup directory replacement: %d", runner.deleteCalls) + } + entries, err := os.ReadDir(backupDir + "-held") + if err != nil { + t.Fatal(err) + } + if len(entries) != 1 { + t.Fatalf("expected completed backup in retained directory, got %#v", entries) + } +} + func TestBackupDokployDatabaseRejectsPermissiveDirectoryWithoutChangingMode(t *testing.T) { backupDir := dokployBackupTestDir(t) if err := os.Chmod(backupDir, 0o755); err != nil { @@ -192,7 +264,7 @@ func TestPurgeSourceResourcesRemovesDockerResources(t *testing.T) { if err != nil { t.Fatalf("IdentifySourcePurgeResources: %v", err) } - result, err := client.PurgeSourceResources(context.Background(), options) + result, err := client.purgeSourceResources(context.Background(), options) if err != nil { t.Fatalf("PurgeSourceResources: %v", err) } @@ -336,7 +408,7 @@ func TestSourcePurgeRejectsConflictingVolumeAbsenceState(t *testing.T) { if _, err := client.IdentifySourcePurgeResources(context.Background(), SourcePurgeOptions{Volumes: ordered}); err == nil || !strings.Contains(err.Error(), "conflicting") { t.Fatalf("expected identification to reject conflicting duplicate volumes, got %v", err) } - if _, err := client.PurgeSourceResources(context.Background(), SourcePurgeOptions{Volumes: ordered}); err == nil || !strings.Contains(err.Error(), "conflicting") { + if _, err := client.purgeSourceResources(context.Background(), SourcePurgeOptions{Volumes: ordered}); err == nil || !strings.Contains(err.Error(), "conflicting") { t.Fatalf("expected apply to reject conflicting duplicate volumes, got %v", err) } } @@ -369,7 +441,7 @@ func TestPurgeSourceResourcesSkipsAbsentAllowedPath(t *testing.T) { if err != nil { t.Fatalf("IdentifySourcePurgeResources: %v", err) } - result, err := client.PurgeSourceResources(context.Background(), options) + result, err := client.purgeSourceResources(context.Background(), options) if err != nil { t.Fatalf("PurgeSourceResources: %v", err) } @@ -385,7 +457,7 @@ func TestPurgeSourceResourcesDoesNotFallbackFromStaleIDToName(t *testing.T) { }} client := &Client{Docker: runner} - result, err := client.PurgeSourceResources(context.Background(), SourcePurgeOptions{Containers: []SourcePurgeContainer{{App: "api", ContainerID: "missing-id", ContainerName: "web"}}}) + result, err := client.purgeSourceResources(context.Background(), SourcePurgeOptions{Containers: []SourcePurgeContainer{{App: "api", ContainerID: "missing-id", ContainerName: "web"}}}) if err == nil || !strings.Contains(err.Error(), "docker inspect missing-id") { t.Fatalf("expected stale ID inspect error, got err=%v result=%#v", err, result) } @@ -406,7 +478,7 @@ func TestPurgeSourceResourcesPreservesReplacementAfterReviewedContainerDisappear }, } client := &Client{Docker: runner} - result, err := client.PurgeSourceResources(context.Background(), SourcePurgeOptions{Containers: []SourcePurgeContainer{{ContainerID: "reviewed-id", ContainerName: "web"}}}) + result, err := client.purgeSourceResources(context.Background(), SourcePurgeOptions{Containers: []SourcePurgeContainer{{ContainerID: "reviewed-id", ContainerName: "web"}}}) if err == nil || !strings.Contains(err.Error(), "now refers to container ID") { t.Fatalf("expected replacement container to block purge, got result=%#v err=%v", result, err) } @@ -426,7 +498,7 @@ func TestPurgeSourceResourcesRejectsMismatchedInspectedContainerID(t *testing.T) "rm -f replacement-id": []byte("replacement-id\n"), }} client := &Client{Docker: runner} - result, err := client.PurgeSourceResources(context.Background(), SourcePurgeOptions{Containers: []SourcePurgeContainer{{ContainerID: "reviewed-id", ContainerName: "web"}}}) + result, err := client.purgeSourceResources(context.Background(), SourcePurgeOptions{Containers: []SourcePurgeContainer{{ContainerID: "reviewed-id", ContainerName: "web"}}}) if err == nil || !strings.Contains(err.Error(), "not the reviewed ID") { t.Fatalf("expected mismatched inspected ID to block purge, got result=%#v err=%v", result, err) } @@ -449,7 +521,7 @@ func TestPurgeSourceNetworkPreservesReplacementAfterReviewedIDDisappears(t *test }, } client := &Client{Docker: runner} - result, err := client.PurgeSourceResources(context.Background(), SourcePurgeOptions{ + result, err := client.purgeSourceResources(context.Background(), SourcePurgeOptions{ Networks: []SourcePurgeNetwork{{Name: "api-net", DiscoveredIdentity: reviewedID, ExpectedIdentity: reviewedID}}, }) if err == nil || !strings.Contains(err.Error(), "appeared after confirmation") { @@ -511,7 +583,7 @@ func TestIdentifySourcePurgeResourcesRequiresDiscoveredNetworkIdentity(t *testin func TestPurgeSourceNetworkRejectsShortConfirmedIdentity(t *testing.T) { client := &Client{Docker: &fakeDockerRunner{}} progressCalls := 0 - result, err := client.PurgeSourceResources(context.Background(), SourcePurgeOptions{ + result, err := client.purgeSourceResources(context.Background(), SourcePurgeOptions{ Networks: []SourcePurgeNetwork{{Name: "api-net", DiscoveredIdentity: "123456789abc", ExpectedIdentity: "123456789abc"}}, OnProgress: func(SourcePurgeResult) error { progressCalls++ @@ -538,7 +610,7 @@ func TestPurgeSourceNetworkRejectsInvalidIdentityTupleBeforeProgress(t *testing. t.Run(name, func(t *testing.T) { runner := &fakeDockerRunner{} progressCalls := 0 - result, err := (&Client{Docker: runner}).PurgeSourceResources(context.Background(), SourcePurgeOptions{ + result, err := (&Client{Docker: runner}).purgeSourceResources(context.Background(), SourcePurgeOptions{ Networks: []SourcePurgeNetwork{network}, OnProgress: func(SourcePurgeResult) error { progressCalls++ @@ -570,7 +642,7 @@ func TestPurgeSourceResourcesReturnsPartialResults(t *testing.T) { if err != nil { t.Fatalf("IdentifySourcePurgeResources: %v", err) } - result, err := client.PurgeSourceResources(context.Background(), options) + result, err := client.purgeSourceResources(context.Background(), options) if err == nil { t.Fatal("expected purge to stop on the network failure") } @@ -587,7 +659,7 @@ func TestSourcePurgeRequiresNamedVolumeToRemainAbsent(t *testing.T) { if _, err := client.IdentifySourcePurgeResources(context.Background(), SourcePurgeOptions{Volumes: []SourcePurgeVolume{{Name: "api-data"}}}); err == nil || !strings.Contains(err.Error(), "remove it manually") { t.Fatalf("expected existing named volume to block before confirmation, got %v", err) } - result, err := client.PurgeSourceResources(context.Background(), SourcePurgeOptions{ + result, err := client.purgeSourceResources(context.Background(), SourcePurgeOptions{ Volumes: []SourcePurgeVolume{{Name: "api-data", ExpectedAbsent: true}}, }) if err == nil || !strings.Contains(err.Error(), "appeared after confirmation") { @@ -607,7 +679,7 @@ func TestSourcePurgeRechecksAbsenceBeforeRemovingContainers(t *testing.T) { "rm -f cid123": []byte("cid123\n"), }} client := &Client{Docker: runner} - result, err := client.PurgeSourceResources(context.Background(), SourcePurgeOptions{ + result, err := client.purgeSourceResources(context.Background(), SourcePurgeOptions{ Containers: []SourcePurgeContainer{{ContainerID: "cid123", ContainerName: "web"}}, Volumes: []SourcePurgeVolume{{Name: "api-data", ExpectedAbsent: true}}, }) @@ -627,7 +699,7 @@ func TestSourcePurgeWithAbsencePrerequisitesRequiresManualContainerRemoval(t *te "rm -f cid123": []byte("cid123\n"), }}} client := &Client{Docker: runner} - result, err := client.PurgeSourceResources(context.Background(), SourcePurgeOptions{ + result, err := client.purgeSourceResources(context.Background(), SourcePurgeOptions{ Containers: []SourcePurgeContainer{{ContainerID: "cid123", ContainerName: "web"}}, Volumes: []SourcePurgeVolume{{Name: "api-data", ExpectedAbsent: true}}, }) @@ -681,7 +753,7 @@ func TestSourcePurgeManualCompletionPreservesReplacementContainerByName(t *testi }, } client := &Client{Docker: runner} - result, err := client.PurgeSourceResources(context.Background(), SourcePurgeOptions{ + result, err := client.purgeSourceResources(context.Background(), SourcePurgeOptions{ Containers: []SourcePurgeContainer{{ContainerID: "reviewed", ContainerName: "web"}}, Volumes: []SourcePurgeVolume{{Name: "api-data", ExpectedAbsent: true}}, }) @@ -704,7 +776,7 @@ func TestPurgeSourceResourcesStopsBeforeNextMutationWhenProgressPersistenceFails }} var durable SourcePurgeResult client := &Client{Docker: runner} - result, err := client.PurgeSourceResources(context.Background(), SourcePurgeOptions{ + result, err := client.purgeSourceResources(context.Background(), SourcePurgeOptions{ Containers: []SourcePurgeContainer{{ContainerID: "cid1", ContainerName: "one"}, {ContainerID: "cid2", ContainerName: "two"}}, OnProgress: func(progress SourcePurgeResult) error { latest := progress.Containers[len(progress.Containers)-1] @@ -759,7 +831,7 @@ func TestSourcePurgeAcceptsNamedVolumeThatRemainsAbsent(t *testing.T) { if err != nil { t.Fatalf("IdentifySourcePurgeResources: %v", err) } - result, err := client.PurgeSourceResources(context.Background(), options) + result, err := client.purgeSourceResources(context.Background(), options) if err != nil { t.Fatalf("PurgeSourceResources: %v", err) } @@ -778,7 +850,7 @@ func TestSourcePurgeManualNetworkResultRetainsConfirmedIdentity(t *testing.T) { }, } client := &Client{Docker: runner} - result, err := client.PurgeSourceResources(context.Background(), SourcePurgeOptions{ + result, err := client.purgeSourceResources(context.Background(), SourcePurgeOptions{ Volumes: []SourcePurgeVolume{{Name: "api-data", ExpectedAbsent: true}}, Networks: []SourcePurgeNetwork{{Name: "api-net", DiscoveredIdentity: canonicalID, ExpectedIdentity: canonicalID}}, }) @@ -831,7 +903,7 @@ func TestProtectedSourcePurgeNetworkNamesAreCaseSensitive(t *testing.T) { func TestPurgeSourceResourcesRejectsProtectedHostPath(t *testing.T) { client := &Client{Docker: &fakeDockerRunner{}} - _, err := client.PurgeSourceResources(context.Background(), SourcePurgeOptions{Paths: []SourcePurgePath{{Path: "/data"}}}) + _, err := client.purgeSourceResources(context.Background(), SourcePurgeOptions{Paths: []SourcePurgePath{{Path: "/data"}}}) if err == nil || !strings.Contains(err.Error(), "protected host path") { t.Fatalf("expected protected path error, got %v", err) } From 725a1d59335974df5c0cb65855133ca2ad098e9e Mon Sep 17 00:00:00 2001 From: Aikins Laryea Date: Wed, 23 Sep 2026 16:17:26 +0000 Subject: [PATCH 26/31] cleanup: discard unpersisted purge progress (#33) --- docs/cleanup-and-purge.md | 5 +- internal/target/dokploy/cleanup.go | 2 + internal/target/dokploy/cleanup_test.go | 92 +++++++++++++++++++++++++ 3 files changed, 98 insertions(+), 1 deletion(-) diff --git a/docs/cleanup-and-purge.md b/docs/cleanup-and-purge.md index ce2b19d..2ca336f 100644 --- a/docs/cleanup-and-purge.md +++ b/docs/cleanup-and-purge.md @@ -86,7 +86,10 @@ Purge apply refuses to run unless: - the exact `purge ` confirmation is supplied. Bort writes a private purge-plan backup under `.bort/backups` before destructive -work begins. +work begins. It records each resource before and after removal and stops as soon +as recording fails. Printed results only show progress Bort confirmed was +recorded, so a resource reported as `started` may already have been removed: +compare the actual source state with both records before retrying. ## Automatic removal and manual-removal checks diff --git a/internal/target/dokploy/cleanup.go b/internal/target/dokploy/cleanup.go index bf8a12e..8671403 100644 --- a/internal/target/dokploy/cleanup.go +++ b/internal/target/dokploy/cleanup.go @@ -240,6 +240,7 @@ func runSourcePurgeResource(result *SourcePurgeResult, resources *[]SourcePurgeR started.Status = "started" *resources = append(*resources, started) if err := publishSourcePurgeProgress(*result, onProgress); err != nil { + *resources = (*resources)[:len(*resources)-1] return err } outcome, operationErr := operation() @@ -248,6 +249,7 @@ func runSourcePurgeResource(result *SourcePurgeResult, resources *[]SourcePurgeR } (*resources)[len(*resources)-1] = outcome if err := publishSourcePurgeProgress(*result, onProgress); err != nil { + (*resources)[len(*resources)-1] = started if operationErr != nil { return fmt.Errorf("%v; %w", operationErr, err) } diff --git a/internal/target/dokploy/cleanup_test.go b/internal/target/dokploy/cleanup_test.go index 92c8c99..2f1b0e0 100644 --- a/internal/target/dokploy/cleanup_test.go +++ b/internal/target/dokploy/cleanup_test.go @@ -793,6 +793,9 @@ func TestPurgeSourceResourcesStopsBeforeNextMutationWhenProgressPersistenceFails if len(durable.Containers) != 1 || durable.Containers[0].Status != "removed" { t.Fatalf("expected first outcome to remain durable, got %#v", durable) } + if len(result.Containers) != 1 || result.Containers[0].Status != "removed" { + t.Fatalf("expected returned progress to match the durable snapshot without the unpersisted entry, got %#v", result.Containers) + } for _, args := range runner.outputArgs { if strings.Join(args, " ") == "inspect --type container cid2" || strings.Join(args, " ") == "rm -f cid2" { t.Fatalf("second resource mutated after persistence failure: %#v", runner.outputArgs) @@ -800,6 +803,95 @@ func TestPurgeSourceResourcesStopsBeforeNextMutationWhenProgressPersistenceFails } } +func TestPurgeSourceResourcesDiscardsStartedProgressWhenFirstPersistenceFails(t *testing.T) { + runner := &fakeDockerRunner{outputs: map[string][]byte{ + "inspect --type container cid1": []byte(`[{"Id":"cid1","Name":"/one","State":{"Running":false,"Status":"exited"}}]`), + "rm -f cid1": []byte("cid1\n"), + }} + client := &Client{Docker: runner} + result, err := client.purgeSourceResources(context.Background(), SourcePurgeOptions{ + Containers: []SourcePurgeContainer{{ContainerID: "cid1", ContainerName: "one"}}, + OnProgress: func(SourcePurgeResult) error { + return errors.New("backup unavailable") + }, + }) + if err == nil || !strings.Contains(err.Error(), "persist source purge progress") { + t.Fatalf("expected progress persistence failure, got result=%#v err=%v", result, err) + } + if len(result.Containers) != 0 { + t.Fatalf("expected returned progress to drop the unpersisted started entry, got %#v", result.Containers) + } + if len(runner.outputArgs) != 0 { + t.Fatalf("expected no purge mutation after persistence failure, got %#v", runner.outputArgs) + } +} + +func TestPurgeSourceResourcesDiscardsUnpersistedOutcomeWhenProgressPersistenceFails(t *testing.T) { + runner := &fakeDockerRunner{outputs: map[string][]byte{ + "inspect --type container cid1": []byte(`[{"Id":"cid1","Name":"/one","State":{"Running":false,"Status":"exited"}}]`), + "rm -f cid1": []byte("cid1\n"), + }} + var durable SourcePurgeResult + client := &Client{Docker: runner} + result, err := client.purgeSourceResources(context.Background(), SourcePurgeOptions{ + Containers: []SourcePurgeContainer{{ContainerID: "cid1", ContainerName: "one"}}, + OnProgress: func(progress SourcePurgeResult) error { + latest := progress.Containers[len(progress.Containers)-1] + if latest.Status != "started" { + return errors.New("backup unavailable") + } + durable = progress + return nil + }, + }) + if err == nil || !strings.Contains(err.Error(), "persist source purge progress") { + t.Fatalf("expected progress persistence failure, got result=%#v err=%v", result, err) + } + removed := false + for _, args := range runner.outputArgs { + if strings.Join(args, " ") == "rm -f cid1" { + removed = true + } + } + if !removed { + t.Fatalf("expected the container removal to run before the failing progress publish, got %#v", runner.outputArgs) + } + if len(durable.Containers) != 1 || durable.Containers[0].Status != "started" { + t.Fatalf("expected the durable snapshot to keep the started entry, got %#v", durable) + } + if len(result.Containers) != 1 || result.Containers[0].Status != "started" { + t.Fatalf("expected returned progress to match the durable started entry instead of the unpersisted outcome, got %#v", result.Containers) + } +} + +func TestPurgeSourceResourcesReportsOperationErrorWhenOutcomePersistenceFails(t *testing.T) { + runner := &cleanupDockerRunner{ + fakeDockerRunner: &fakeDockerRunner{outputs: map[string][]byte{ + "inspect --type container cid1": []byte(`[{"Id":"cid1","Name":"/one","State":{"Running":false,"Status":"exited"}}]`), + }}, + outputErrors: map[string]error{ + "rm -f cid1": errors.New("docker daemon unreachable"), + }, + } + client := &Client{Docker: runner} + result, err := client.purgeSourceResources(context.Background(), SourcePurgeOptions{ + Containers: []SourcePurgeContainer{{ContainerID: "cid1", ContainerName: "one"}}, + OnProgress: func(progress SourcePurgeResult) error { + latest := progress.Containers[len(progress.Containers)-1] + if latest.Status != "started" { + return errors.New("backup unavailable") + } + return nil + }, + }) + if err == nil || !strings.Contains(err.Error(), "docker daemon unreachable") || !strings.Contains(err.Error(), "persist source purge progress") { + t.Fatalf("expected operation and persistence errors, got result=%#v err=%v", result, err) + } + if len(result.Containers) != 1 || result.Containers[0].Status != "started" { + t.Fatalf("expected returned progress to match the durable started entry, got %#v", result.Containers) + } +} + type stagedPurgeRunner struct { fakeDockerRunner volumeInspects int From b423fe202af54a43ae74912544432760dba1d770 Mon Sep 17 00:00:00 2001 From: Aikins Laryea Date: Wed, 23 Sep 2026 19:53:18 +0000 Subject: [PATCH 27/31] guide: clarify guided screens and failure recovery (#34) --- README.md | 8 +- internal/cli/cockpit.go | 3 + internal/cli/guide.go | 82 ++++++++++++++++-- internal/cli/guide_test.go | 95 ++++++++++++++++++++- internal/cli/main_test.go | 13 +++ internal/cli/run.go | 6 +- internal/cli/state.go | 3 + internal/cli/wizard.go | 5 +- internal/source/localdocker/scanner_test.go | 14 +-- 9 files changed, 207 insertions(+), 22 deletions(-) create mode 100644 internal/cli/main_test.go diff --git a/README.md b/README.md index faac5ec..d03faf9 100644 --- a/README.md +++ b/README.md @@ -210,11 +210,9 @@ Bort's safety model defaults to “look first.” Near-term work remains focused on making the same-VPS Coolify → Dokploy path boring and safe before adding more platforms: -1. make workspace selection, requirement checks, and failure recovery clearer - in the guided screen; -2. decide how automated rollback should work; -3. add Dokploy source scanning and Coolify target creation; -4. add cross-server transfers after the same-VPS steps are proven. +1. decide how automated rollback should work; +2. add Dokploy source scanning and Coolify target creation; +3. add cross-server transfers after the same-VPS steps are proven. Other Docker-, Compose-, and Swarm-based platforms remain possible future targets. Bort should only support one when it can clearly say which resources diff --git a/internal/cli/cockpit.go b/internal/cli/cockpit.go index c285c14..e3e1ccd 100644 --- a/internal/cli/cockpit.go +++ b/internal/cli/cockpit.go @@ -38,6 +38,9 @@ func writeAppFirstCockpit(w io.Writer, run loadedMigrationRun) { } header += " " + st.pill(migrationRunPhaseLabel(phase), severityForMigrationRunPhase(phase)) fmt.Fprintln(w, header) + if workspace := workspaceDir(); workspace != "" { + fmt.Fprintln(w, st.muted("workspace: "+workspace)) + } fmt.Fprintln(w) if len(apps) == 0 { diff --git a/internal/cli/guide.go b/internal/cli/guide.go index a5d2c47..ac26e01 100644 --- a/internal/cli/guide.go +++ b/internal/cli/guide.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "os" + "os/exec" "path/filepath" "sort" "strings" @@ -105,6 +106,9 @@ func runGuide(ctx context.Context, stdin io.Reader, stdout, stderr io.Writer) er run, err = runWizardScan(ctx, setup, stdout) } else { run, err = createGuidedMigrationRun(ctx, setup) + if err != nil { + return wrapGuideScanFailure(err, "") + } } if err != nil { return err @@ -150,13 +154,13 @@ func refreshGuideRun(ctx context.Context, runRef string, stdin io.Reader, stdout refreshedBundle, refreshedManifest, err = refreshRunSourceBundle(ctx, existing) } if err != nil { - return loadedMigrationRun{}, err + return loadedMigrationRun{}, wrapGuideRescanFailure(err) } defer os.RemoveAll(refreshedBundle) refreshed, err := refreshMigrationRunLockedWithInputs(runRef, refreshedBundle, refreshedManifest) if err != nil { _ = os.Remove(refreshedManifest) - return loadedMigrationRun{}, err + return loadedMigrationRun{}, wrapGuideRescanFailure(err) } return refreshed, nil } @@ -223,15 +227,28 @@ func isRealTTY(stdin io.Reader, stdout io.Writer) bool { return isInteractiveTerminal(inFile) && isInteractiveTerminal(outFile) } +func workspaceDir() string { + wd, err := os.Getwd() + if err != nil { + return "" + } + return wd +} + func promptGuidedSetupHuh(now time.Time) (guidedSetup, error) { source := "coolify-local" - manifestPath := "manifest.json" + manifestPath := "" + + description := "All actions are local dry-runs. bort runs on the same server as the source PaaS." + if workspace := workspaceDir(); workspace != "" { + description += "\nWorkspace: " + workspace + " — run every command from this directory with the same OS user." + } form := huh.NewForm( huh.NewGroup( huh.NewSelect[string](). Title("Where are your apps?"). - Description("All actions are local dry-runs. bort runs on the same server as the source PaaS."). + Description(description). Options( huh.NewOption("Coolify on this server (reads local Docker labels)", "coolify-local"), huh.NewOption("Coolify API (uses BORT_COOLIFY_URL/TOKEN)", "coolify"), @@ -270,6 +287,9 @@ func promptGuidedSetupWithReader(reader *bufio.Reader, stdout io.Writer, now tim st := newStyler(stdout) fmt.Fprintln(stdout, st.emph("Migration setup")) fmt.Fprintln(stdout, st.muted("All actions are local dry-runs. bort runs on the same server as the source PaaS.")) + if workspace := workspaceDir(); workspace != "" { + fmt.Fprintln(stdout, st.muted("Workspace: "+workspace+" — run every command from this directory with the same OS user.")) + } fmt.Fprintln(stdout) sourceName, err := promptChoice(reader, stdout, "Where are your apps?", []guideChoice{ @@ -365,7 +385,55 @@ func exportRunSourceBundle(runDir string, m manifest.Manifest, appName string) ( return bundleDir, nil } +func checkSourceRequirements(ctx context.Context, sourceName string) error { + switch sourceName { + case "docker", "local-docker", "coolify-local": + default: + return nil + } + if _, err := exec.LookPath("docker"); err != nil { + return fmt.Errorf("docker CLI not found: %w; install Docker on the source server first", err) + } + out, err := exec.CommandContext(ctx, "docker", "ps", "-q").CombinedOutput() + if err != nil { + detail := firstOutputLine(out) + if detail == "" { + detail = err.Error() + } + return fmt.Errorf("docker is not usable by this user: %s; Docker sources need Docker access, usually by running with sudo — if you switch to sudo, re-run with sudo and keep using sudo for every later command", detail) + } + return nil +} + +func firstOutputLine(out []byte) string { + for _, line := range strings.Split(string(out), "\n") { + trimmed := strings.TrimSpace(line) + if trimmed == "" || strings.HasPrefix(trimmed, "WARNING:") { + continue + } + return trimmed + } + return "" +} + +func wrapGuideScanFailure(err error, rerunArgs string) error { + if err == nil { + return nil + } + return fmt.Errorf("%w; discovery made no changes on the server — fix the issue and re-run `%s`", err, bortCommand(rerunArgs)) +} + +func wrapGuideRescanFailure(err error) error { + if err == nil { + return nil + } + return fmt.Errorf("%w; the existing run and its plan were not changed — fix the issue and re-run `%s`", err, bortCommand("")) +} + func guidedManifest(ctx context.Context, setup guidedSetup, runDir string) (manifest.Manifest, string, error) { + if err := checkSourceRequirements(ctx, setup.Source); err != nil { + return manifest.Manifest{}, "", err + } if setup.Source == "manifest" { m, err := readManifestFile(setup.ManifestPath) if err != nil { @@ -557,7 +625,7 @@ func guideRunRef() (string, bool, bool, error) { } if ok { if !migrationRunMetadataExists(current) { - return "", false, false, fmt.Errorf("current migration run %q no longer exists; run `%s` to select another run", current, bortCommand("migrate --run ")) + return "", false, false, fmt.Errorf("current migration run %q no longer exists in this workspace; if you moved directories, exit and run bort from the original migration directory — otherwise run `%s` with a run name from `.bort/runs`", current, bortCommand("migrate --run ")) } return current, true, true, nil } @@ -612,11 +680,15 @@ func writeGuideStart(w io.Writer) error { fmt.Fprintln(w, st.emph("bort")+" "+st.muted("— migrate self-hosted apps between PaaS platforms")) fmt.Fprintln(w) fmt.Fprintf(w, "%s\n", st.muted(fmt.Sprintf("No migration run was found. Run `%s` in a terminal for guided setup.", bortCommand("")))) + if workspace := workspaceDir(); workspace != "" { + fmt.Fprintf(w, "%s\n", st.muted("Working directory: "+workspace)) + } fmt.Fprintln(w) fmt.Fprintln(w, st.emph("For non-interactive setup:")) fmt.Fprintf(w, " %s\n", st.emph(bortCommand("migrate --source coolify-local"))) fmt.Fprintf(w, " %s\n", st.muted("or: "+bortCommand("migrate --manifest manifest.json"))) fmt.Fprintln(w) fmt.Fprintln(w, st.muted("This single command scans, exports a private bundle, and creates the current run.")) + fmt.Fprintln(w, st.muted("If you started this migration in a different directory, exit and run bort from there; each directory keeps its own .bort workspace.")) return nil } diff --git a/internal/cli/guide_test.go b/internal/cli/guide_test.go index 3caae5c..4c5d998 100644 --- a/internal/cli/guide_test.go +++ b/internal/cli/guide_test.go @@ -30,7 +30,7 @@ func TestRunWithoutArgsShowsGuidedStartWhenNoBundleOrRunExists(t *testing.T) { } output := stdout.String() - for _, want := range []string{"No migration run was found.", "bort migrate --source coolify-local", "bort migrate --manifest manifest.json"} { + for _, want := range []string{"No migration run was found.", "bort migrate --source coolify-local", "bort migrate --manifest manifest.json", "Working directory:", "different directory"} { if !strings.Contains(output, want) { t.Fatalf("expected guide output to contain %q, got:\n%s", want, output) } @@ -80,7 +80,7 @@ func TestRunWithoutArgsCreatesRunFromDefaultBundle(t *testing.T) { } output := stdout.String() - for _, want := range []string{"local bundle → dokploy", "api", "READY"} { + for _, want := range []string{"local bundle → dokploy", "api", "READY", "workspace:"} { if !strings.Contains(output, want) { t.Fatalf("expected guide output to contain %q, got:\n%s", want, output) } @@ -592,3 +592,94 @@ fi } return run } + +func TestCheckSourceRequirementsReportsMissingDockerCLI(t *testing.T) { + t.Setenv("PATH", t.TempDir()) + + err := checkSourceRequirements(context.Background(), "coolify-local") + if err == nil || !strings.Contains(err.Error(), "docker CLI not found") { + t.Fatalf("expected docker CLI requirement error, got %v", err) + } + if err := checkSourceRequirements(context.Background(), "manifest"); err != nil { + t.Fatalf("manifest source must not require docker: %v", err) + } + if err := checkSourceRequirements(context.Background(), "local-docker"); err == nil { + t.Fatal("local-docker alias must require docker like the docker source") + } +} + +func TestCheckSourceRequirementsSkipsDockerWarnings(t *testing.T) { + binDir := t.TempDir() + dockerStub := "#!/bin/sh\n" + + "echo 'WARNING: Error loading config file: open /root/.docker/config.json: permission denied'\n" + + "echo 'docker: Got permission denied while trying to connect to the Docker daemon socket at unix:///var/run/docker.sock'\n" + + "exit 1\n" + if err := os.WriteFile(filepath.Join(binDir, "docker"), []byte(dockerStub), 0o700); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", binDir) + + err := checkSourceRequirements(context.Background(), "docker") + if err == nil || !strings.Contains(err.Error(), "docker is not usable by this user: docker: Got permission denied") { + t.Fatalf("expected docker usability error with the real cause, got %v", err) + } + if strings.Contains(err.Error(), "WARNING") { + t.Fatalf("expected warning lines to be skipped, got %v", err) + } +} + +func TestGuidedSetupScanFailureIncludesRecoveryHint(t *testing.T) { + workDir := t.TempDir() + t.Chdir(workDir) + + input := testGuideInput{Reader: strings.NewReader("4\nmissing-manifest.json\n")} + var stdout bytes.Buffer + var stderr bytes.Buffer + err := RunWithInput(context.Background(), nil, input, &stdout, &stderr) + if err == nil { + t.Fatalf("expected guided setup to fail on a missing manifest, got:\n%s", stdout.String()) + } + if !strings.Contains(err.Error(), "discovery made no changes on the server") { + t.Fatalf("expected scan failure to include a recovery hint, got %v", err) + } +} + +func TestCurrentRunRefReportsWrongUserPermission(t *testing.T) { + if os.Getuid() == 0 { + t.Skip("permission bits are not enforced for root") + } + workDir := t.TempDir() + t.Chdir(workDir) + if err := os.MkdirAll(".bort", 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(".bort", "state.json"), []byte("{}\n"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.Chmod(filepath.Join(".bort", "state.json"), 0o000); err != nil { + t.Fatal(err) + } + _, _, err := currentRunRef() + if err == nil || !strings.Contains(err.Error(), "re-run as the OS user that owns it") { + t.Fatalf("expected wrong-user recovery hint, got %v", err) + } +} + +func TestMigrateSourceScanFailureIncludesRecoveryHint(t *testing.T) { + workDir := t.TempDir() + t.Chdir(workDir) + t.Setenv("PATH", t.TempDir()) + + var stdout bytes.Buffer + var stderr bytes.Buffer + err := runMigrate(context.Background(), []string{"--source", "coolify-local", "--run", "prod run"}, &stdout, &stderr) + if err == nil { + t.Fatalf("expected migrate --source to fail without docker, got:\n%s", stdout.String()) + } + if !strings.Contains(err.Error(), "discovery made no changes on the server") { + t.Fatalf("expected scan failure to include a recovery hint, got %v", err) + } + if !strings.Contains(err.Error(), "re-run `bort migrate --source coolify-local --run 'prod run'`") { + t.Fatalf("expected recovery hint to echo the invocation with quoted args, got %v", err) + } +} diff --git a/internal/cli/main_test.go b/internal/cli/main_test.go new file mode 100644 index 0000000..3271517 --- /dev/null +++ b/internal/cli/main_test.go @@ -0,0 +1,13 @@ +package cli + +import ( + "os" + "testing" +) + +// bortCommand output must stay deterministic in tests, even when go test is +// invoked via sudo. +func TestMain(m *testing.M) { + os.Setenv("SUDO_UID", "") + os.Exit(m.Run()) +} diff --git a/internal/cli/run.go b/internal/cli/run.go index 0c5c5b1..989b273 100644 --- a/internal/cli/run.go +++ b/internal/cli/run.go @@ -244,7 +244,11 @@ func runMigrate(ctx context.Context, args []string, stdout, stderr io.Writer) er ManifestPath: manifestPath, }, observationWindowSeconds, rollbackWindowSeconds) if err != nil { - return err + rerunArgs := []string{"migrate"} + for _, arg := range args { + rerunArgs = append(rerunArgs, shellQuote(arg)) + } + return wrapGuideScanFailure(err, strings.Join(rerunArgs, " ")) } if err := rememberCurrentRun(loadedRun.Run); err != nil { return err diff --git a/internal/cli/state.go b/internal/cli/state.go index 5da11f8..1b432b5 100644 --- a/internal/cli/state.go +++ b/internal/cli/state.go @@ -218,6 +218,9 @@ func rememberCurrentRun(run migrationRun) error { func currentRunRef() (string, bool, error) { state, err := readBortState(defaultStatePath()) if err != nil { + if errors.Is(err, os.ErrPermission) { + return "", false, fmt.Errorf("%w; this user cannot read the .bort workspace — re-run as the OS user that owns it, or fix its permissions", err) + } return "", false, err } ref := strings.TrimSpace(state.CurrentRun) diff --git a/internal/cli/wizard.go b/internal/cli/wizard.go index 4ca2fc0..b7d791e 100644 --- a/internal/cli/wizard.go +++ b/internal/cli/wizard.go @@ -365,12 +365,13 @@ func parseEnvBlock(input string) map[string]string { func runWizardScan(ctx context.Context, setup guidedSetup, stdout io.Writer) (loadedMigrationRun, error) { if !canAnimateStatus(stdout) { fmt.Fprintln(stdout, "~ Scanning source...") - return createGuidedMigrationRun(ctx, setup) + run, err := createGuidedMigrationRun(ctx, setup) + return run, wrapGuideScanFailure(err, "") } stopStatus := startScanStatus(stdout) run, err := createGuidedMigrationRun(ctx, setup) stopStatus(err) - return run, err + return run, wrapGuideScanFailure(err, "") } const scanStatusFrameCount = 3 diff --git a/internal/source/localdocker/scanner_test.go b/internal/source/localdocker/scanner_test.go index 91d8af6..89ca2a5 100644 --- a/internal/source/localdocker/scanner_test.go +++ b/internal/source/localdocker/scanner_test.go @@ -42,13 +42,13 @@ func TestRoutesFromLabelsExtractsHostsAndServicePort(t *testing.T) { func TestCaddyRoutesFromLabelsExtractsHostsAndPort(t *testing.T) { labels := map[string]string{ - "caddy_ingress_network": "coolify", - "caddy_0": "https://app.example.com", - "caddy_0.handle_path.0_reverse_proxy": "{{upstreams 3000}}", - "caddy_1": "https://api.example.com/v1", - "caddy_1.handle_path.1_reverse_proxy": "{{upstreams}}", - "traefik.http.routers.web.rule": "Host(`legacy.example.com`)", - "traefik.http.routers.web.service": "app-svc", + "caddy_ingress_network": "coolify", + "caddy_0": "https://app.example.com", + "caddy_0.handle_path.0_reverse_proxy": "{{upstreams 3000}}", + "caddy_1": "https://api.example.com/v1", + "caddy_1.handle_path.1_reverse_proxy": "{{upstreams}}", + "traefik.http.routers.web.rule": "Host(`legacy.example.com`)", + "traefik.http.routers.web.service": "app-svc", "traefik.http.services.app-svc.loadbalancer.server.port": "8080", } From 5273d860e2bc4f3030457d2baa47327d45ffb681 Mon Sep 17 00:00:00 2001 From: Aikins Laryea Date: Wed, 23 Sep 2026 23:34:30 +0000 Subject: [PATCH 28/31] rollback: restore source traffic after live migration (#35) --- README.md | 27 +- docs/cleanup-and-purge.md | 8 +- docs/migration-guide.md | 36 ++- internal/cli/cockpit.go | 14 +- internal/cli/commit.go | 9 + internal/cli/guide.go | 2 +- internal/cli/rollback.go | 135 ++++++++- internal/cli/rollback_test.go | 313 +++++++++++++++++++ internal/cli/root.go | 3 +- internal/cli/run.go | 50 +++- internal/cli/wizard.go | 4 +- internal/target/dokploy/apply.go | 55 ++-- internal/target/dokploy/docker.go | 28 +- internal/target/dokploy/rollback.go | 146 +++++++++ internal/target/dokploy/rollback_test.go | 366 +++++++++++++++++++++++ 15 files changed, 1151 insertions(+), 45 deletions(-) create mode 100644 internal/target/dokploy/rollback.go create mode 100644 internal/target/dokploy/rollback_test.go diff --git a/README.md b/README.md index d03faf9..a3e2263 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ The current product path is **Coolify → Dokploy on the same Linux VPS**. | Explicit live apply and resume | Implemented | | Accepting Dokploy and stopping source containers | Implemented | | Safe Dokploy database cleanup and separately confirmed source removal | Implemented | -| Executable rollback | Not implemented; Bort only prints the stored rollback plan | +| Executable rollback | Implemented (`bort rollback --live`) | | Dokploy → Coolify or cross-server migration | Not implemented | Bort publishes macOS, Linux, and Windows binaries, but a complete same-VPS move @@ -63,6 +63,8 @@ Today Bort can: while a service is stopped; - switch web traffic during explicit live apply, save progress for retries, and store rollback instructions; +- execute the reviewed rollback (`bort rollback --live`) to restart source + containers and return traffic to the source; - accept the target only after successful live apply; - list leftovers, remove only eligible unused records from Dokploy, and separately remove eligible source containers and networks after confirmation. @@ -87,7 +89,8 @@ Bort keeps workspace directories and files private, but `.bort` contains target credentials and application configuration. Do not commit or publish it. Read the [prerequisites and recovery guidance](docs/migration-guide.md) before a -live migration. In particular, Bort does not yet execute rollback. +live migration. In particular, review how `bort rollback --live` works before you +need it. ## Install @@ -136,7 +139,8 @@ The migration uses separate commands for each important step: ```text sudo bort # start or resume, then review and fix sudo bort migrate --live # apply only the selected planned run -sudo bort rollback # inspect the stored manual rollback plan +sudo bort rollback # inspect the stored rollback plan +sudo bort rollback --live # restart source containers and return traffic sudo bort commit --apply # accept the target and retire source containers sudo bort cleanup # audit leftovers without deleting source resources ``` @@ -182,8 +186,13 @@ Bort's safety model defaults to “look first.” - **Separate destructive purge:** purge requires selected apps or projects, a successful live apply, an accepted target, the exact confirmation phrase, a recheck of Docker IDs, and a private backup. -- **Manual rollback only:** `bort rollback` prints the stored plan. Bort does not - currently execute recovery actions. +- **Health-gated rollback:** `bort rollback --live` restarts the quiesced source + containers, verifies they are running and healthy before touching any traffic, + swaps the proxies back, and observes the result. It never deletes data; the + Dokploy target resources remain on the server for cleanup, and an interrupted + rollback can be re-run. Commit and live apply stay blocked once rollback starts. + Live rollback uses a five-second settling check per observed app, not the + stored observation window; continue monitoring for the reviewed period. ## Documentation @@ -196,7 +205,8 @@ Bort's safety model defaults to “look first.” ## Current limitations -- Bort shows rollback instructions but does not run them. +- Rollback returns traffic to the source but does not copy back data written to + the target after cutover. - Bort does not keep copying new volume changes while applications remain live. - Database moves currently use local export/import or copy data while the database is stopped. @@ -210,9 +220,8 @@ Bort's safety model defaults to “look first.” Near-term work remains focused on making the same-VPS Coolify → Dokploy path boring and safe before adding more platforms: -1. decide how automated rollback should work; -2. add Dokploy source scanning and Coolify target creation; -3. add cross-server transfers after the same-VPS steps are proven. +1. add Dokploy source scanning and Coolify target creation; +2. add cross-server transfers after the same-VPS steps are proven. Other Docker-, Compose-, and Swarm-based platforms remain possible future targets. Bort should only support one when it can clearly say which resources diff --git a/docs/cleanup-and-purge.md b/docs/cleanup-and-purge.md index 2ca336f..f684438 100644 --- a/docs/cleanup-and-purge.md +++ b/docs/cleanup-and-purge.md @@ -21,9 +21,11 @@ sudo bort commit --apply ``` This confirms that Dokploy is now the active platform and stops the source -application containers. Bort does not currently provide automatic rollback, so -do not run it until you have independently checked the target. Bort does not -measure the rollback window or block this command when the window has not passed. +application containers. Automated rollback is refused once source retirement +starts, even if commit is interrupted. Rerun `commit --apply` to finish an +interrupted acceptance. Do not start it until you have independently checked the +target. Bort does not measure the rollback window or block this command when the +window has not passed. ## Ordinary cleanup diff --git a/docs/migration-guide.md b/docs/migration-guide.md index 6687475..c7f1153 100644 --- a/docs/migration-guide.md +++ b/docs/migration-guide.md @@ -156,9 +156,32 @@ You can inspect the stored rollback plan with: sudo bort rollback ``` -Bort does not currently execute rollback. The command prints the stored plan; -any recovery action remains manual. Do not accept the target unless that manual -recovery limitation is understood. +After a successful live apply, if the target does not check out, return traffic +to the source: + +```sh +sudo bort rollback --live +``` + +The command restarts the source containers that live apply stopped, verifies +they are running (and healthy, when Docker tracks a healthcheck) before +touching any traffic, returns web traffic to the source by stopping the Dokploy +proxy and starting the source proxy, and re-checks the result after a five-second +settle per observed app. This does not enforce the observation window displayed +by `bort rollback`; continue monitoring the source for that reviewed period. +Every step inspects current state first, so an interrupted rollback can be +re-run. Once rollback starts, commit and live apply remain blocked, even if +rollback fails. Use `bort rollback --live` again to finish recovery. + +Rollback requires confirming the recorded rollback trigger first. If you skipped +that guided review before applying, inspect `bort rollback --run `, then +use `bort rollback --live --run --confirm 'rollback '` with the +actual run name. The command records that confirmation before starting recovery. + +Rollback never deletes resources: the Dokploy target resources remain on the +server for later cleanup, and data written to the target after cutover is not +copied back to the source. To migrate again, create a new named run with +`bort migrate --run ` and current `--source` or `--bundle` inputs. Accepting the target retires source application containers: @@ -167,7 +190,9 @@ sudo bort commit --apply ``` This command stops the source application containers. Run it only after checking -the target and waiting through the planned rollback window. +the target and waiting through the planned rollback window. Once source +retirement starts, automated rollback is no longer available, even if commit is +interrupted; rerun `commit --apply` to finish acceptance. ## Audit and clean up @@ -188,7 +213,8 @@ Ordinary cleanup and destructive source purge are separate operations. See the | Live apply was interrupted | Run `sudo bort status`, then rerun `sudo bort migrate --live` to resume from the saved progress. | | Another change is running | Keep `status` open if useful and wait. A second live command joins an active live apply; other commands that make changes must wait. | | The plan needs to change after live execution began | Keep the existing run as a record and create a new named run. A plan cannot change after live work starts. | -| Target validation fails before acceptance | Keep source resources available, inspect `sudo bort rollback`, and perform the required recovery manually. Automated rollback is not implemented. | +| Target validation fails after successful live apply, before acceptance | Run `sudo bort rollback --live` to restart and verify source containers and return traffic to the source. Target resources remain for cleanup; data written to the target after cutover is not copied back. | +| Rollback stops partway through | Inspect `sudo bort status`, then rerun `sudo bort rollback --live`. Commit and live apply are blocked because traffic may already have returned to the source. | | Cleanup or purge stops partway through | Stop and inspect the command output and private backup before retrying. Follow the recovery guidance in the cleanup and purge guide. | | `.bort` cannot be read | Do not change ownership or permissions blindly. Confirm the original working directory and OS user first, then restore the workspace from backup if necessary. | diff --git a/internal/cli/cockpit.go b/internal/cli/cockpit.go index e3e1ccd..d83582c 100644 --- a/internal/cli/cockpit.go +++ b/internal/cli/cockpit.go @@ -120,9 +120,15 @@ func writeAppFirstCockpit(w io.Writer, run loadedMigrationRun) { case "partial": fmt.Fprintf(w, "%s\n", st.muted(fmt.Sprintf("Live apply is incomplete. Run `%s` to resume safely.", liveApplyCommand(run)))) case "applied": - fmt.Fprintf(w, "%s\n", st.muted(fmt.Sprintf("Target is live. Verify it through the rollback window, then run `%s` to retire the source.", runScopedCommand(run, "commit --apply")))) + fmt.Fprintf(w, "%s\n", st.muted(fmt.Sprintf("Target is live. Verify it through the rollback window, then run `%s` to retire the source; if validation fails, run `%s` to roll back.", runScopedCommand(run, "commit --apply"), runScopedCommand(run, "rollback --live")))) case "committed": fmt.Fprintf(w, "%s\n", st.muted(fmt.Sprintf("Target accepted and source containers retired. Run `%s` to audit leftovers.", runScopedCommand(run, "cleanup")))) + case "committing": + fmt.Fprintf(w, "%s\n", st.muted(fmt.Sprintf("Source retirement started; rollback is no longer available. Run `%s` to finish acceptance.", runScopedCommand(run, "commit --apply")))) + case "rolled back": + fmt.Fprintf(w, "%s\n", st.muted("Rollback returned traffic to the source; the Dokploy target resources remain on the server. To migrate again, "+summary.Next.Action+".")) + case "rolling back": + fmt.Fprintf(w, "%s\n", st.muted(fmt.Sprintf("Rollback has not completed; traffic may already be on the source. Run `%s` to finish recovery.", runScopedCommand(run, "rollback --live")))) case "purged": fmt.Fprintln(w, st.muted("Migration complete. Target resources and source-control credentials were preserved.")) case "planning": @@ -149,6 +155,12 @@ func migrationRunPhase(run loadedMigrationRun) string { return "purged" case run.Run.CommittedAt != nil: return "committed" + case run.Run.RolledBackAt != nil: + return "rolled back" + case run.Run.RollbackStartedAt != nil: + return "rolling back" + case run.Run.CommitStartedAt != nil: + return "committing" case run.Run.LiveAppliedAt != nil || liveApplySucceeded(run): return "applied" case applyActiveErr != nil: diff --git a/internal/cli/commit.go b/internal/cli/commit.go index 59ee13e..6a9d866 100644 --- a/internal/cli/commit.go +++ b/internal/cli/commit.go @@ -186,6 +186,12 @@ func applyCommitFromArgs(ctx context.Context, runRef string, stderr io.Writer) e if run.Run.Target != "dokploy" { return fmt.Errorf("commit --apply is only supported for target dokploy, got %q", run.Run.Target) } + if run.Run.RolledBackAt != nil { + return fmt.Errorf("commit refused: run %q was rolled back; the source is serving traffic again", run.Run.Name) + } + if run.Run.RollbackStartedAt != nil { + return fmt.Errorf("commit refused: rollback started for run %q; run `%s` to finish recovery", run.Run.Name, runScopedCommand(run, "rollback --live")) + } if err := requireLiveApplySucceeded(run); err != nil { return err } @@ -197,6 +203,9 @@ func applyCommitFromArgs(ctx context.Context, runRef string, stderr io.Writer) e plan.RunName = run.Run.Name plan.RunDir = run.Run.RunDir plan.ApprovedPrepareDecisions = approvedPrepareDecisions(run) + if err := markRunCommitStartedLocked(run.Run); err != nil { + return fmt.Errorf("record source retirement start: %w", err) + } fmt.Fprintf(stderr, "commit apply: run %s; planned %d step(s) to retire source\n", run.Run.Name, len(plan.Steps)) if err := client.Apply(ctx, plan); err != nil { return err diff --git a/internal/cli/guide.go b/internal/cli/guide.go index ac26e01..cc58402 100644 --- a/internal/cli/guide.go +++ b/internal/cli/guide.go @@ -141,7 +141,7 @@ func refreshGuideRun(ctx context.Context, runRef string, stdin io.Reader, stdout if err != nil { return loadedMigrationRun{}, fmt.Errorf("read applied migration progress: %w", err) } - if existing.LiveAppliedAt != nil || existing.CommittedAt != nil || existing.PurgedAt != nil || hasAppliedSteps { + if existing.LiveAppliedAt != nil || existing.CommitStartedAt != nil || existing.CommittedAt != nil || existing.RollbackStartedAt != nil || existing.RolledBackAt != nil || existing.PurgedAt != nil || hasAppliedSteps { return loadMigrationRun(runRef) } if shouldAutoRescanRun(existing) { diff --git a/internal/cli/rollback.go b/internal/cli/rollback.go index df7df21..8ccf2ce 100644 --- a/internal/cli/rollback.go +++ b/internal/cli/rollback.go @@ -6,11 +6,13 @@ import ( "fmt" "io" "strings" + "time" rollbackplan "github.com/aikins01/bort/internal/rollback" + "github.com/aikins01/bort/internal/target/dokploy" ) -func runRollback(_ context.Context, args []string, stdout, stderr io.Writer) error { +func runRollback(ctx context.Context, args []string, stdout, stderr io.Writer) error { fs := flag.NewFlagSet("rollback", flag.ContinueOnError) fs.SetOutput(stderr) @@ -21,6 +23,8 @@ func runRollback(_ context.Context, args []string, stdout, stderr io.Writer) err var outputPath string var cutoverPlanPath string var runRef string + var live bool + var confirm string observationWindowSeconds := rollbackplan.DefaultObservationWindowSeconds fs.StringVar(&bundleDir, "bundle", "bort-bundle", "migration bundle directory") @@ -30,6 +34,8 @@ func runRollback(_ context.Context, args []string, stdout, stderr io.Writer) err fs.StringVar(&outputPath, "output", "-", "output path, or - for stdout") fs.StringVar(&cutoverPlanPath, "from-cutover", "", "read a prior cutover JSON plan artifact") fs.StringVar(&runRef, "run", "", "run name under .bort/runs, or a run directory path") + fs.BoolVar(&live, "live", false, "restart source containers and return traffic; performs a short settling check, not the stored observation window") + fs.StringVar(&confirm, "confirm", "", "confirm open rollback triggers with the exact phrase: rollback (requires --live)") fs.IntVar(&observationWindowSeconds, "observation-window", rollbackplan.DefaultObservationWindowSeconds, "observation window in seconds") if err := fs.Parse(args); err != nil { @@ -41,6 +47,17 @@ func runRollback(_ context.Context, args []string, stdout, stderr io.Writer) err if flagSet(fs, "run") && strings.TrimSpace(runRef) == "" { return fmt.Errorf("rollback requires a non-empty --run value") } + if flagSet(fs, "confirm") && !live { + return fmt.Errorf("rollback --confirm requires --live") + } + if live { + for _, name := range []string{"app", "bundle", "format", "from-cutover", "observation-window", "output", "target"} { + if flagSet(fs, name) { + return fmt.Errorf("rollback --live does not accept --%s; select the run with --run", name) + } + } + return applyRollbackFromArgs(ctx, runRef, confirm, stderr) + } if err := checkOutputFormat("rollback", format); err != nil { return err } @@ -146,3 +163,119 @@ func writeRollbackText(w io.Writer, result rollbackplan.Result) { } fmt.Fprintln(w, "Dry run only: no routes were changed and no rollback actions were executed.") } + +func applyRollbackFromArgs(ctx context.Context, runRef, confirm string, stderr io.Writer) error { + var err error + runRef, err = resolveRunRef(runRef, false) + if err != nil { + return err + } + operationLock, err := acquireRunOperationLock(runRef) + if err != nil { + return fmt.Errorf("rollback run %q: %w", runRef, err) + } + defer operationLock.Release() + run, err := loadMigrationRun(runRef) + if err != nil { + return err + } + if err := validateRollbackApplyReady(run); err != nil { + return err + } + phrase := "rollback " + run.Run.Name + if confirm != "" && confirm != phrase { + return fmt.Errorf("rollback confirmation must be exactly %q", phrase) + } + if decisions := rollbackTriggerDecisions(run); len(decisions) > 0 { + for _, decision := range decisions { + fmt.Fprintf(stderr, "%s\n%s\n", decisionAction(decision), decisionReason(decision)) + } + if confirm != phrase { + return fmt.Errorf("rollback is blocked by %d unconfirmed rollback trigger(s); review `%s`, then confirm with `%s --confirm %s`", len(decisions), runScopedCommand(run, "rollback"), runScopedCommand(run, "rollback --live"), shellQuote(phrase)) + } + for _, decision := range decisions { + run.Progress = markReviewDecisionDone(run, decision, time.Now().UTC()) + } + path, err := safeRunArtifactPath(run.Run.RunDir, run.Run.Artifacts.Progress) + if err != nil { + return err + } + if err := writeRunProgress(path, run.Progress); err != nil { + return err + } + } + plan := dokploy.PlanForRollback(run.Prepare, run.Cutover) + plan.RunName = run.Run.Name + plan.RunDir = run.Run.RunDir + plan.ApprovedPrepareDecisions = approvedPrepareDecisions(run) + onProgress := func(p dokploy.StepProgress) { + target := p.Step.App + if target == "" { + target = p.Step.Ref + } + line := fmt.Sprintf("rollback [%d/%d] %s %s: %s", p.Index+1, p.Total, p.Step.Kind, target, p.Status) + if p.Err != nil { + line += ": " + p.Err.Error() + } + fmt.Fprintln(stderr, line) + } + plan.OnProgress = &onProgress + if err := markRunRollbackStartedLocked(run.Run); err != nil { + return fmt.Errorf("record rollback start: %w", err) + } + fmt.Fprintf(stderr, "rollback live: run %s; planned %d step(s) to return traffic to the source\n", run.Run.Name, len(plan.Steps)) + client := &dokploy.Client{} + if err := client.Apply(ctx, plan); err != nil { + return err + } + if err := markRunRolledBackLocked(run.Run); err != nil { + return fmt.Errorf("rollback completed, but its outcome could not be recorded: %w", err) + } + writeRollbackAppliedSummary(stderr, run, plan) + return nil +} + +func validateRollbackApplyReady(run loadedMigrationRun) error { + if run.Run.Target != "dokploy" { + return fmt.Errorf("rollback --live is only supported for target dokploy, got %q", run.Run.Target) + } + if run.Run.PurgedAt != nil { + return fmt.Errorf("rollback refused: run %q was already purged", run.Run.Name) + } + if run.Run.CommittedAt != nil { + return fmt.Errorf("rollback refused: run %q was already committed; its source containers were retired", run.Run.Name) + } + if run.Run.CommitStartedAt != nil { + return fmt.Errorf("rollback refused: source retirement started for run %q; run `%s` to finish acceptance", run.Run.Name, runScopedCommand(run, "commit --apply")) + } + if run.Run.RolledBackAt != nil { + return fmt.Errorf("rollback refused: run %q was already rolled back; start a fresh run to migrate again", run.Run.Name) + } + if err := requireLiveApplySucceeded(run); err != nil { + return fmt.Errorf("rollback requires a successful live apply: %w", err) + } + active, err := applyRunActive(run.Run.RunDir) + if err != nil { + return fmt.Errorf("check live-apply lock: %w", err) + } + if active { + return fmt.Errorf("rollback refused: live apply is running for run %q; wait for it to finish before rolling back", run.Run.Name) + } + return nil +} + +func rollbackTriggerDecisions(run loadedMigrationRun) []runDecision { + return openFilteredDecisions(run, func(item runDecisionItem) bool { + return item.Stage == "rollback" && item.Code == "rollback.trigger_required" + }) +} + +func writeRollbackAppliedSummary(w io.Writer, run loadedMigrationRun, plan dokploy.Plan) { + fmt.Fprintln(w, "rollback complete: traffic is back on the source") + if len(plan.Steps) == 0 { + fmt.Fprintln(w, "this run stopped no source containers and changed no routes; the source was serving all along") + } + fmt.Fprintln(w, "the Dokploy target resources remain on the server; data written to the target after cutover was not copied back to the source") + fmt.Fprintln(w, "live rollback does not enforce the stored observation window; continue monitoring the source for the reviewed period") + fmt.Fprintln(w, "to migrate again, create a new named run with `bort migrate --run ` and current `--source` or `--bundle` inputs") +} diff --git a/internal/cli/rollback_test.go b/internal/cli/rollback_test.go index 8c06d73..62a8268 100644 --- a/internal/cli/rollback_test.go +++ b/internal/cli/rollback_test.go @@ -5,13 +5,17 @@ import ( "context" "encoding/json" "io" + "os" "path/filepath" "strings" "testing" + "time" "github.com/aikins01/bort/internal/exporter" "github.com/aikins01/bort/internal/manifest" + "github.com/aikins01/bort/internal/preparer" rollbackplan "github.com/aikins01/bort/internal/rollback" + "github.com/aikins01/bort/internal/target/dokploy" ) func TestRunRollbackWritesTextPlan(t *testing.T) { @@ -188,3 +192,312 @@ func TestRunRollbackUsesDefaultBundleWithoutCurrentRun(t *testing.T) { t.Fatalf("expected bare rollback to plan from the default bundle without a current run, got:\n%s", stdout.String()) } } + +func TestRunRollbackLiveRejectsConflictingFlags(t *testing.T) { + err := runRollback(context.Background(), []string{"--live", "--run", "r", "--format", "json"}, io.Discard, io.Discard) + if err == nil || !strings.Contains(err.Error(), "rollback --live does not accept --format") { + t.Fatalf("expected --live to reject --format, got %v", err) + } + err = runRollback(context.Background(), []string{"--live", "--run", "r", "--app", "api"}, io.Discard, io.Discard) + if err == nil || !strings.Contains(err.Error(), "rollback --live does not accept --app") { + t.Fatalf("expected --live to reject --app, got %v", err) + } + if err := runRollback(context.Background(), []string{"--confirm", "rollback r"}, io.Discard, io.Discard); err == nil || !strings.Contains(err.Error(), "requires --live") { + t.Fatalf("expected confirmation without live to be refused, got %v", err) + } +} + +func TestWizardLifecycleWithoutAppliedLedger(t *testing.T) { + now := time.Now().UTC() + for _, tc := range []struct { + name string + run migrationRun + want string + }{ + {name: "rollback started", run: migrationRun{RollbackStartedAt: &now}, want: "ROLLING BACK"}, + {name: "rolled back", run: migrationRun{RolledBackAt: &now}, want: "ROLLED BACK"}, + {name: "commit started", run: migrationRun{CommitStartedAt: &now}, want: "COMMITTING"}, + } { + t.Run(tc.name, func(t *testing.T) { + tc.run.RunDir = t.TempDir() + run := loadedMigrationRun{Run: tc.run, Prepare: preparer.Result{Apps: []preparer.AppPlan{{Name: "api"}}}} + var out bytes.Buffer + if err := runWizard(context.Background(), run, strings.NewReader(""), &out, io.Discard); err != nil { + t.Fatal(err) + } + if !strings.Contains(out.String(), tc.want) || strings.Contains(out.String(), "migrate --live") { + t.Fatalf("expected %s without live-apply recommendation, got:\n%s", tc.want, out.String()) + } + }) + } +} + +func TestInterruptedCommitRefusesRollbackAndResumesCommit(t *testing.T) { + t.Chdir(t.TempDir()) + t.Setenv("BORT_DOKPLOY_URL", "http://127.0.0.1:3030") + t.Setenv("BORT_DOKPLOY_TOKEN", "test-token") + binDir := t.TempDir() + stub := `#!/bin/sh +grep -q commitStartedAt .bort/runs/retirement/run.json || exit 9 +if [ -f fail-commit ]; then + echo 'injected inspection failure' >&2 + exit 1 +fi +echo '[{"Id":"source-id","State":{"Running":false}}]' +` + if err := os.WriteFile(filepath.Join(binDir, "docker"), []byte(stub), 0o700); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", binDir+string(os.PathListSeparator)+os.Getenv("PATH")) + writeTestBundle(t, "bundle", manifest.Manifest{ + Source: manifest.Source{Platform: "coolify-local"}, + Apps: []manifest.App{{Name: "api", Services: []manifest.Service{{ID: "source-id", Name: "web", Image: "example/api:latest"}}}}, + }) + runCommand(t, runMigrate, []string{"--bundle", "bundle", "--run", "retirement"}) + run, err := loadMigrationRun("retirement") + if err != nil { + t.Fatal(err) + } + for _, decision := range openSetupDecisions(run) { + if err := recordReviewDecision(run, decision, time.Now().UTC()); err != nil { + t.Fatal(err) + } + } + applied := newRunApplied(run.Run) + now := time.Now().UTC() + applied.SucceededAt = &now + for index, step := range dokploy.PlanFromArtifacts(run.Prepare, run.Sync, run.Cutover).Steps { + applied.Steps = append(applied.Steps, appliedStep{Index: index, Kind: string(step.Kind), App: step.App, Ref: step.Ref, Status: "ok"}) + } + if err := writeRunApplied(runArtifactPath(run.Run.RunDir, run.Run.Artifacts.Applied), applied); err != nil { + t.Fatal(err) + } + if err := os.WriteFile("fail-commit", nil, 0o600); err != nil { + t.Fatal(err) + } + if err := runCommit(context.Background(), []string{"--apply", "--run", "retirement"}, io.Discard, io.Discard); err == nil || !strings.Contains(err.Error(), "injected inspection failure") { + t.Fatalf("expected interrupted commit, got %v", err) + } + run, err = loadMigrationRun("retirement") + if err != nil { + t.Fatal(err) + } + if run.Run.CommitStartedAt == nil || run.Run.CommittedAt != nil { + t.Fatal("expected durable incomplete commit") + } + if err := runRollback(context.Background(), []string{"--live", "--run", "retirement"}, io.Discard, io.Discard); err == nil || !strings.Contains(err.Error(), "source retirement started") { + t.Fatalf("expected rollback refusal during retirement, got %v", err) + } + if err := os.Remove("fail-commit"); err != nil { + t.Fatal(err) + } + if err := runCommit(context.Background(), []string{"--apply", "--run", "retirement"}, io.Discard, io.Discard); err != nil { + t.Fatalf("expected commit recovery, got %v", err) + } + run, err = loadMigrationRun("retirement") + if err != nil || run.Run.CommittedAt == nil { + t.Fatalf("expected recorded commit completion, got %v", err) + } +} + +func TestValidateRollbackApplyReadyRefusals(t *testing.T) { + now := time.Now().UTC() + tests := []struct { + name string + run loadedMigrationRun + want string + }{ + {name: "purged", run: loadedMigrationRun{Run: migrationRun{Name: "r", Target: "dokploy", RunDir: t.TempDir(), PurgedAt: &now}}, want: "already purged"}, + {name: "committed", run: loadedMigrationRun{Run: migrationRun{Name: "r", Target: "dokploy", RunDir: t.TempDir(), CommittedAt: &now}}, want: "already committed"}, + {name: "rolled back", run: loadedMigrationRun{Run: migrationRun{Name: "r", Target: "dokploy", RunDir: t.TempDir(), LiveAppliedAt: &now, RolledBackAt: &now}}, want: "already rolled back"}, + {name: "no live work", run: loadedMigrationRun{Run: migrationRun{Name: "r", Target: "dokploy", RunDir: t.TempDir()}}, want: "requires a successful live apply"}, + {name: "metadata only", run: loadedMigrationRun{Run: migrationRun{Name: "r", Target: "dokploy", RunDir: t.TempDir(), LiveAppliedAt: &now}}, want: "requires a successful live apply"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := validateRollbackApplyReady(tt.run) + if err == nil || !strings.Contains(err.Error(), tt.want) { + t.Fatalf("expected refusal %q, got %v", tt.want, err) + } + }) + } + ready := loadedMigrationRun{ + Run: migrationRun{Name: "r", Target: "dokploy", RunDir: t.TempDir(), ApplyOutcomeRequired: true}, + Prepare: preparer.Result{Apps: []preparer.AppPlan{{Name: "api"}}}, + Applied: runApplied{SucceededAt: &now, Steps: []appliedStep{{Index: 0, Kind: "create_project", App: "api", Ref: "api", Status: "ok"}}}, + } + if err := validateRollbackApplyReady(ready); err != nil { + t.Fatalf("expected a live-applied run with no open triggers to be rollback-ready, got %v", err) + } + for _, status := range []string{"started", "error"} { + ready.Applied.Steps[0].Status = status + if err := validateRollbackApplyReady(ready); err == nil { + t.Fatalf("expected %s ledger to be refused", status) + } + } + ready.Applied.Steps[0].Status = "ok" + ready.Applied.SucceededAt = nil + if err := validateRollbackApplyReady(ready); err == nil { + t.Fatal("expected complete ledger without successful outcome to be refused") + } +} + +func TestRunRollbackLiveExecutesStoredRollbackAndGatesFollowUps(t *testing.T) { + workDir := t.TempDir() + t.Chdir(workDir) + binDir := filepath.Join(workDir, "bin") + if err := os.MkdirAll(binDir, 0o700); err != nil { + t.Fatal(err) + } + dockerStub := `#!/bin/sh +grep -q rollbackStartedAt .bort/runs/demo/run.json || exit 9 +echo "$*" >> docker-calls +if [ "$1" = inspect ] && [ "$2" = --type ]; then + running=true + case "$4" in + coolify-proxy) [ -f proxy-started ] || running=false ;; + dokploy-traefik) [ ! -f proxy-stopped ] || running=false ;; + cid123) if [ -f proxy-started ] && [ -f fail-observation ]; then running=false; fi ;; + esac + echo '[{"Id":"'"$4"'","Name":"/app","State":{"Running":'"$running"',"Status":"running"}}]' +elif [ "$1" = start ] && [ "$2" = coolify-proxy ]; then + touch proxy-started +elif [ "$1" = stop ]; then + touch proxy-stopped +fi +` + if err := os.WriteFile(filepath.Join(binDir, "docker"), []byte(dockerStub), 0o700); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", binDir+string(os.PathListSeparator)+os.Getenv("PATH")) + + bundleDir := filepath.Join(workDir, "bort-bundle") + writeTestBundle(t, bundleDir, manifest.Manifest{ + Source: manifest.Source{Platform: "coolify-local"}, + Apps: []manifest.App{{ + Name: "api", + Routes: []manifest.Route{{Host: "api.example.com", ServiceName: "web", Port: "3000"}}, + Services: []manifest.Service{{ + ID: "cid123", + Name: "web", + Image: "example/api:latest", + Mounts: []manifest.Mount{{Type: "volume", Name: "api-data", Target: "/data"}}, + }}, + }}, + }) + runCommand(t, runMigrate, []string{"--bundle", bundleDir, "--run", "demo", "--observation-window", "0", "--rollback-window", "0"}) + run, err := loadMigrationRun("demo") + if err != nil { + t.Fatal(err) + } + for _, decision := range openSetupDecisions(run) { + if err := recordReviewDecision(run, decision, run.Run.UpdatedAt.Add(2*time.Second)); err != nil { + t.Fatalf("resolve prepare decision: %v", err) + } + } + applied := newRunApplied(run.Run) + now := time.Now().UTC() + applied.SucceededAt = &now + for index, step := range dokploy.PlanFromArtifacts(run.Prepare, run.Sync, run.Cutover).Steps { + applied.Steps = append(applied.Steps, appliedStep{Index: index, Kind: string(step.Kind), App: step.App, Ref: step.Ref, Status: "ok"}) + } + if err := writeRunApplied(runArtifactPath(run.Run.RunDir, run.Run.Artifacts.Applied), applied); err != nil { + t.Fatal(err) + } + if err := markRunLiveAppliedLocked(run.Run); err != nil { + t.Fatal(err) + } + + var stdout bytes.Buffer + var stderr bytes.Buffer + for _, confirm := range []string{"", "rollback other"} { + err := runRollback(context.Background(), []string{"--live", "--run", "demo", "--confirm", confirm}, &stdout, &stderr) + if err == nil || (!strings.Contains(err.Error(), "unconfirmed rollback trigger") && !strings.Contains(err.Error(), "confirmation must be exactly")) { + t.Fatalf("expected refusal for confirmation %q, got %v", confirm, err) + } + } + if _, err := os.Stat("docker-calls"); !os.IsNotExist(err) { + t.Fatalf("expected no Docker calls before confirmation, got %v", err) + } + if err := os.WriteFile("fail-observation", nil, 0o600); err != nil { + t.Fatal(err) + } + if err := runRollback(context.Background(), []string{"--live", "--run", "demo", "--confirm", "rollback demo"}, &stdout, &stderr); err == nil || !strings.Contains(err.Error(), "stopped after rollback") { + t.Fatalf("expected failure after proxy swap, got %v", err) + } + interrupted, err := loadMigrationRun("demo") + if err != nil { + t.Fatal(err) + } + if interrupted.Run.RollbackStartedAt == nil || interrupted.Run.RolledBackAt != nil || len(rollbackTriggerDecisions(interrupted)) != 0 { + t.Fatal("expected durable rollback start and confirmation, but no completion") + } + if phase := migrationRunPhase(interrupted); phase != "rolling back" { + t.Fatalf("expected rolling back, got %q", phase) + } + if next := nextSafeStep(interrupted, nil); !strings.Contains(next.Action, "rollback --live") || strings.Contains(next.Action, "commit") { + t.Fatalf("expected rollback retry, got %#v", next) + } + if err := runCommit(context.Background(), []string{"--apply", "--run", "demo"}, io.Discard, io.Discard); err == nil || !strings.Contains(err.Error(), "rollback started") { + t.Fatalf("expected commit refusal after interrupted rollback, got %v", err) + } + if err := runMigrate(context.Background(), []string{"--live", "--run", "demo"}, io.Discard, io.Discard); err == nil || !strings.Contains(err.Error(), "rollback started") { + t.Fatalf("expected live apply refusal after interrupted rollback, got %v", err) + } + if err := os.Remove("fail-observation"); err != nil { + t.Fatal(err) + } + stderr.Reset() + if err := runRollback(context.Background(), []string{"--live", "--run", "demo"}, &stdout, &stderr); err != nil { + t.Fatalf("rollback --live failed: %v\nstderr:\n%s", err, stderr.String()) + } + if !strings.Contains(stderr.String(), "rollback live: run demo") { + t.Fatalf("expected rollback progress header, got:\n%s", stderr.String()) + } + for _, want := range []string{ + "rollback [1/5] resume_source api: started", + "rollback [1/5] resume_source api: ok", + "rollback [2/5] verify_source_health api: ok", + "rollback [5/5] observe_rollback api: ok", + } { + if !strings.Contains(stderr.String(), want) { + t.Fatalf("expected rollback progress %q, got:\n%s", want, stderr.String()) + } + } + if !strings.Contains(stderr.String(), "rollback complete: traffic is back on the source") { + t.Fatalf("expected rollback completion summary, got:\n%s", stderr.String()) + } + + rolled, err := loadMigrationRun("demo") + if err != nil { + t.Fatal(err) + } + if rolled.Run.RolledBackAt == nil { + t.Fatal("expected rolledBackAt to be recorded") + } + if phase := migrationRunPhase(rolled); phase != "rolled back" { + t.Fatalf("expected cockpit phase \"rolled back\", got %q", phase) + } + next := nextSafeStep(rolled, liveApplyBlockingDecisions(rolled)) + if !strings.Contains(next.Action, "create a new named migration run") { + t.Fatalf("expected next safe step to recommend a fresh run, got %#v", next) + } + stdout.Reset() + if err := runWizard(context.Background(), rolled, strings.NewReader(""), &stdout, io.Discard); err != nil { + t.Fatal(err) + } + if !strings.Contains(stdout.String(), "ROLLED BACK") || strings.Contains(stdout.String(), "migrate --live") { + t.Fatalf("expected rolled-back wizard to show status without suggesting live apply, got:\n%s", stdout.String()) + } + + stderr.Reset() + if err := runRollback(context.Background(), []string{"--live", "--run", "demo"}, io.Discard, &stderr); err == nil || !strings.Contains(err.Error(), "already rolled back") { + t.Fatalf("expected repeat rollback to be refused, got %v", err) + } + if err := runCommit(context.Background(), []string{"--apply", "--run", "demo"}, io.Discard, io.Discard); err == nil || !strings.Contains(err.Error(), "was rolled back") { + t.Fatalf("expected commit after rollback to be refused, got %v", err) + } + if err := runMigrate(context.Background(), []string{"--live", "--run", "demo"}, io.Discard, io.Discard); err == nil || !strings.Contains(err.Error(), "was rolled back") { + t.Fatalf("expected live apply after rollback to be refused, got %v", err) + } +} diff --git a/internal/cli/root.go b/internal/cli/root.go index 7be69a1..da8f9ac 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -86,7 +86,8 @@ func writePrimaryHelp(w io.Writer, st *styler) { writeHelpSection(w, st, "Usage:", []helpLine{ {verb: bortCommand(""), desc: "start or resume the current migration"}, {verb: bortCommand("migrate --live"), desc: "apply the selected planned run to its target"}, - {verb: bortCommand("rollback"), desc: "inspect the source rollback plan"}, + {verb: bortCommand("rollback"), desc: "inspect the stored rollback plan"}, + {verb: bortCommand("rollback --live"), desc: "restart source containers and return traffic to the source"}, {verb: bortCommand("commit --apply"), desc: "accept the target and retire source containers"}, {verb: bortCommand("cleanup"), desc: "audit leftovers; --apply removes safe metadata only"}, {verb: bortCommand("cleanup purge"), desc: "purge eligible source leftovers with confirmation"}, diff --git a/internal/cli/run.go b/internal/cli/run.go index 989b273..533aa99 100644 --- a/internal/cli/run.go +++ b/internal/cli/run.go @@ -41,7 +41,10 @@ type migrationRun struct { CreatedAt time.Time `json:"createdAt"` UpdatedAt time.Time `json:"updatedAt"` LiveAppliedAt *time.Time `json:"liveAppliedAt,omitempty"` + CommitStartedAt *time.Time `json:"commitStartedAt,omitempty"` CommittedAt *time.Time `json:"committedAt,omitempty"` + RollbackStartedAt *time.Time `json:"rollbackStartedAt,omitempty"` + RolledBackAt *time.Time `json:"rolledBackAt,omitempty"` PurgedAt *time.Time `json:"purgedAt,omitempty"` ApplyOutcomeRequired bool `json:"applyOutcomeRequired,omitempty"` Source string `json:"source,omitempty"` @@ -395,6 +398,15 @@ func refreshMigrationRunSafelyLocked(runRef string) (loadedMigrationRun, error) } func validateLiveApplyReady(run loadedMigrationRun) error { + if run.Run.RolledBackAt != nil { + return fmt.Errorf("live apply refused: run %q was rolled back; start a fresh migration run to migrate again", run.Run.Name) + } + if run.Run.RollbackStartedAt != nil { + return fmt.Errorf("live apply refused: rollback started for run %q; run `%s` to finish recovery", run.Run.Name, runScopedCommand(run, "rollback --live")) + } + if run.Run.CommitStartedAt != nil { + return fmt.Errorf("live apply refused: source retirement started for run %q; run `%s` to finish acceptance", run.Run.Name, runScopedCommand(run, "commit --apply")) + } if decisions := liveApplyBlockingDecisions(run); len(decisions) > 0 { decision := decisions[0] action := strings.TrimSpace(decision.Action) @@ -1055,7 +1067,7 @@ func existingMutableMigrationRun(runDir, runName string) (migrationRun, error) { } existing.RunDir = filepath.ToSlash(filepath.Clean(runDir)) existing.Artifacts = existing.Artifacts.withDefaults() - if existing.LiveAppliedAt != nil || existing.CommittedAt != nil || existing.PurgedAt != nil { + if existing.LiveAppliedAt != nil || existing.CommitStartedAt != nil || existing.CommittedAt != nil || existing.RollbackStartedAt != nil || existing.RolledBackAt != nil || existing.PurgedAt != nil { return migrationRun{}, immutableError() } appliedPath, err := safeRunArtifactPath(runDir, existing.Artifacts.Applied) @@ -1396,7 +1408,10 @@ func createMigrationRunLocked(opts migrationRunOptions, runDir, runName string, CreatedAt: createdAt, UpdatedAt: now, LiveAppliedAt: existingRun.LiveAppliedAt, + CommitStartedAt: existingRun.CommitStartedAt, CommittedAt: existingRun.CommittedAt, + RollbackStartedAt: existingRun.RollbackStartedAt, + RolledBackAt: existingRun.RolledBackAt, PurgedAt: existingRun.PurgedAt, ApplyOutcomeRequired: true, Source: opts.Source, @@ -1723,6 +1738,15 @@ func nextSafeStep(run loadedMigrationRun, decisions []runDecision) runNextStep { if run.Run.CommittedAt != nil { return runNextStep{Action: fmt.Sprintf("run `%s` to audit remaining metadata and source leftovers", runScopedCommand(run, "cleanup")), Reason: "the target is accepted and source app containers are retired"} } + if run.Run.RolledBackAt != nil { + return runNextStep{Action: "create a new named migration run with `bort migrate --run ` and current `--source` or `--bundle` inputs", Reason: "the rollback returned traffic to the source; the Dokploy target resources remain on the server"} + } + if run.Run.RollbackStartedAt != nil { + return runNextStep{Action: fmt.Sprintf("run `%s` to finish recovery", runScopedCommand(run, "rollback --live")), Reason: "rollback started but has not completed; traffic may already be on the source"} + } + if run.Run.CommitStartedAt != nil { + return runNextStep{Action: fmt.Sprintf("run `%s` to finish acceptance", runScopedCommand(run, "commit --apply")), Reason: "source retirement started; rollback is no longer available"} + } if run.Run.LiveAppliedAt != nil || liveApplySucceeded(run) { return runNextStep{Action: fmt.Sprintf("verify the target, then run `%s` after the rollback window", runScopedCommand(run, "commit --apply")), Reason: "the live apply completed and the source remains available for rollback"} } @@ -2482,6 +2506,14 @@ func markRunLiveAppliedLocked(run migrationRun) error { }) } +func markRunCommitStartedLocked(run migrationRun) error { + return updateRunLifecycleLocked(run, func(current *migrationRun, now time.Time) { + if current.CommitStartedAt == nil { + current.CommitStartedAt = &now + } + }) +} + func markRunCommittedLocked(run migrationRun) error { return updateRunLifecycleLocked(run, func(current *migrationRun, now time.Time) { if current.CommittedAt == nil { @@ -2490,6 +2522,22 @@ func markRunCommittedLocked(run migrationRun) error { }) } +func markRunRollbackStartedLocked(run migrationRun) error { + return updateRunLifecycleLocked(run, func(current *migrationRun, now time.Time) { + if current.RollbackStartedAt == nil { + current.RollbackStartedAt = &now + } + }) +} + +func markRunRolledBackLocked(run migrationRun) error { + return updateRunLifecycleLocked(run, func(current *migrationRun, now time.Time) { + if current.RolledBackAt == nil { + current.RolledBackAt = &now + } + }) +} + func markRunPurgedLocked(run migrationRun) error { return updateRunLifecycleLocked(run, func(current *migrationRun, now time.Time) { if current.PurgedAt == nil { diff --git a/internal/cli/wizard.go b/internal/cli/wizard.go index b7d791e..678501c 100644 --- a/internal/cli/wizard.go +++ b/internal/cli/wizard.go @@ -16,7 +16,7 @@ import ( func runWizard(ctx context.Context, run loadedMigrationRun, stdin io.Reader, stdout, stderr io.Writer) error { current := run - if len(current.Applied.Steps) > 0 { + if len(current.Applied.Steps) > 0 || current.Run.LiveAppliedAt != nil || current.Run.CommitStartedAt != nil || current.Run.CommittedAt != nil || current.Run.RollbackStartedAt != nil || current.Run.RolledBackAt != nil || current.Run.PurgedAt != nil { writeAppFirstCockpit(stdout, current) return nil } @@ -151,7 +151,7 @@ func promptReviewDecision(run loadedMigrationRun, decision runDecision, stdout i form := huh.NewForm(huh.NewGroup( huh.NewConfirm(). Title(decisionAction(decision)). - Description(decisionAction(decision) + "\n" + decisionReason(decision)). + Description(decisionReason(decision)). Affirmative("Reviewed"). Negative("Not yet"). Value(&confirm), diff --git a/internal/target/dokploy/apply.go b/internal/target/dokploy/apply.go index 4c521bc..30260ab 100644 --- a/internal/target/dokploy/apply.go +++ b/internal/target/dokploy/apply.go @@ -25,22 +25,26 @@ import ( type StepKind string const ( - StepCreateProject StepKind = "create_project" - StepCreateService StepKind = "create_service" - StepPushImage StepKind = "push_image" - StepUploadEnv StepKind = "upload_env" - StepCreateVolume StepKind = "create_volume" - StepPauseSource StepKind = "pause_source" - StepResumeSource StepKind = "resume_source" - StepResumeTarget StepKind = "resume_target" - StepSyncVolume StepKind = "sync_volume" - StepDumpDataStore StepKind = "dump_data_store" - StepRestoreDataStore StepKind = "restore_data_store" - StepInstallGateway StepKind = "install_gateway" - StepActivateRoutes StepKind = "activate_routes" - StepStopCoolifyProxy StepKind = "stop_coolify_proxy" - StepStartDokployProxy StepKind = "start_dokploy_proxy" - StepStopSourceApp StepKind = "stop_source_app" + StepCreateProject StepKind = "create_project" + StepCreateService StepKind = "create_service" + StepPushImage StepKind = "push_image" + StepUploadEnv StepKind = "upload_env" + StepCreateVolume StepKind = "create_volume" + StepPauseSource StepKind = "pause_source" + StepResumeSource StepKind = "resume_source" + StepResumeTarget StepKind = "resume_target" + StepSyncVolume StepKind = "sync_volume" + StepDumpDataStore StepKind = "dump_data_store" + StepRestoreDataStore StepKind = "restore_data_store" + StepInstallGateway StepKind = "install_gateway" + StepActivateRoutes StepKind = "activate_routes" + StepStopCoolifyProxy StepKind = "stop_coolify_proxy" + StepStartDokployProxy StepKind = "start_dokploy_proxy" + StepVerifySourceHealth StepKind = "verify_source_health" + StepStopDokployProxy StepKind = "stop_dokploy_proxy" + StepStartCoolifyProxy StepKind = "start_coolify_proxy" + StepObserveRollback StepKind = "observe_rollback" + StepStopSourceApp StepKind = "stop_source_app" ) // proxy container names assumed by the cutover swap. coolify-proxy is @@ -100,6 +104,7 @@ type Plan struct { ResumeFrom int BeforeStep *func(StepProgress) error OnProgress *func(StepProgress) + stepTimeout time.Duration } func PlanFromArtifacts(prepare preparer.Result, sync syncplan.Result, cutover gateway.Result) Plan { @@ -547,7 +552,15 @@ func (c *Client) Apply(ctx context.Context, plan Plan) error { if step.Kind == StepStopCoolifyProxy { coolifyProxyStopped = true } - err := c.applyStep(ctx, actx, step) + stepCtx := ctx + var cancel context.CancelFunc + if plan.stepTimeout > 0 { + stepCtx, cancel = context.WithTimeout(ctx, plan.stepTimeout) + } + err := c.applyStep(stepCtx, actx, step) + if cancel != nil { + cancel() + } if err != nil { emitProgress(plan.OnProgress, StepProgress{Index: index, Total: total, Step: step, Status: StepStatusError, Err: err}) resumePausedApps := pausedApps @@ -760,6 +773,14 @@ func (c *Client) applyStep(ctx context.Context, actx *applyContext, step Step) e return c.applyStopCoolifyProxy(ctx, actx, step) case StepStartDokployProxy: return c.applyStartDokployProxy(ctx, actx, step) + case StepVerifySourceHealth: + return c.applyVerifySourceHealth(ctx, actx, step) + case StepStopDokployProxy: + return c.applyStopDokployProxy(ctx, actx, step) + case StepStartCoolifyProxy: + return c.applyStartCoolifyProxy(ctx, actx, step) + case StepObserveRollback: + return c.applyObserveRollback(ctx, actx, step) case StepStopSourceApp: return c.applyStopSourceApp(ctx, actx, step) default: diff --git a/internal/target/dokploy/docker.go b/internal/target/dokploy/docker.go index 394221f..7653f7f 100644 --- a/internal/target/dokploy/docker.go +++ b/internal/target/dokploy/docker.go @@ -157,6 +157,20 @@ func containerRunningAfterStartFailure(runner dockerRunner, id string) bool { return err == nil && container.State.Running } +func waitContext(ctx context.Context, d time.Duration) error { + if d <= 0 { + return ctx.Err() + } + timer := time.NewTimer(d) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} + type dockerContainer struct { ID string Name string @@ -167,12 +181,17 @@ type dockerContainer struct { Labels map[string]string `json:"Labels"` } `json:"Config"` State struct { - Status string `json:"Status"` - Running bool `json:"Running"` + Status string `json:"Status"` + Running bool `json:"Running"` + Health *dockerHealthState `json:"Health"` } `json:"State"` Mounts []dockerMount `json:"Mounts"` } +type dockerHealthState struct { + Status string `json:"Status"` +} + type dockerMount struct { Type string `json:"Type"` Name string `json:"Name"` @@ -191,8 +210,9 @@ type dockerInspectRaw struct { Labels map[string]string `json:"Labels"` } `json:"Config"` State struct { - Status string `json:"Status"` - Running bool `json:"Running"` + Status string `json:"Status"` + Running bool `json:"Running"` + Health *dockerHealthState `json:"Health"` } `json:"State"` Mounts []dockerMount `json:"Mounts"` } diff --git a/internal/target/dokploy/rollback.go b/internal/target/dokploy/rollback.go new file mode 100644 index 0000000..17f699d --- /dev/null +++ b/internal/target/dokploy/rollback.go @@ -0,0 +1,146 @@ +package dokploy + +import ( + "context" + "fmt" + "time" + + "github.com/aikins01/bort/internal/gateway" + "github.com/aikins01/bort/internal/preparer" +) + +var rollbackObserveSettle = 5 * time.Second + +var sourceHealthPollInterval = 2 * time.Second + +// PlanForRollback builds the steps that return traffic to the source. +func PlanForRollback(prepare preparer.Result, cutover gateway.Result) Plan { + plan := Plan{Prepare: prepare, Cutover: cutover, stepTimeout: dockerStartTimeout} + quiescedApps := []preparer.AppPlan{} + for _, app := range prepare.Apps { + if len(sourceQuiesceTargetRefs(app)) == 0 { + continue + } + quiescedApps = append(quiescedApps, app) + } + for _, app := range quiescedApps { + plan.Steps = append(plan.Steps, Step{Kind: StepResumeSource, App: app.Name, Ref: app.Name}) + } + for _, app := range quiescedApps { + plan.Steps = append(plan.Steps, Step{Kind: StepVerifySourceHealth, App: app.Name, Ref: app.Name}) + } + if cutoverPlanHasRoutes(cutover) { + plan.Steps = append(plan.Steps, + Step{Kind: StepStopDokployProxy, Ref: dokployProxyContainer}, + Step{Kind: StepStartCoolifyProxy, Ref: coolifyProxyContainer}, + ) + for _, app := range quiescedApps { + plan.Steps = append(plan.Steps, Step{Kind: StepObserveRollback, App: app.Name, Ref: app.Name}) + } + } + return plan +} + +func (c *Client) applyVerifySourceHealth(ctx context.Context, actx *applyContext, step Step) error { + app, ok := findPrepareApp(actx.plan.Prepare, step.App) + if !ok { + return fmt.Errorf("app %s not found in prepare result", step.App) + } + runner := c.dockerRunner() + for _, ref := range sourceQuiesceTargetRefs(app) { + if err := verifySourceContainerHealthy(ctx, runner, ref); err != nil { + return err + } + } + return nil +} + +func verifySourceContainerHealthy(ctx context.Context, runner dockerRunner, ref commitTargetRef) error { + verifyCtx, cancel := context.WithTimeout(ctx, dockerStartTimeout) + defer cancel() + + container, err := sourceContainerForQuiesce(verifyCtx, runner, ref) + if err != nil { + return fmt.Errorf("inspect source container %s: %w", ref.label(), err) + } + if !container.State.Running { + return fmt.Errorf("source container %s is not running; not returning traffic to the source", ref.label()) + } + if container.State.Health == nil { + return nil + } + for { + switch container.State.Health.Status { + case "healthy": + return nil + case "unhealthy": + return fmt.Errorf("source container %s reports unhealthy; not returning traffic to the source", ref.label()) + } + if err := waitContext(verifyCtx, sourceHealthPollInterval); err != nil { + return fmt.Errorf("wait for source container %s health: %w", ref.label(), err) + } + container, err = sourceContainerForQuiesce(verifyCtx, runner, ref) + if err != nil { + return fmt.Errorf("inspect source container %s: %w", ref.label(), err) + } + if !container.State.Running { + return fmt.Errorf("source container %s is not running; not returning traffic to the source", ref.label()) + } + if container.State.Health == nil { + return fmt.Errorf("source container %s lost its healthcheck during verification", ref.label()) + } + } +} + +func (c *Client) applyStopDokployProxy(ctx context.Context, _ *applyContext, _ Step) error { + return stopProxyContainer(ctx, c.dockerRunner(), dokployProxyContainer) +} + +func (c *Client) applyStartCoolifyProxy(ctx context.Context, _ *applyContext, _ Step) error { + runner := c.dockerRunner() + if err := startProxyContainer(ctx, runner, coolifyProxyContainer); err != nil { + return err + } + container, err := inspectContainer(ctx, runner, coolifyProxyContainer) + if err != nil { + return fmt.Errorf("inspect proxy container %s: %w", coolifyProxyContainer, err) + } + if !container.State.Running { + return fmt.Errorf("%s did not stay running; source traffic is not restored", coolifyProxyContainer) + } + return nil +} + +func (c *Client) applyObserveRollback(ctx context.Context, actx *applyContext, step Step) error { + app, ok := findPrepareApp(actx.plan.Prepare, step.App) + if !ok { + return fmt.Errorf("app %s not found in prepare result", step.App) + } + if err := waitContext(ctx, rollbackObserveSettle); err != nil { + return fmt.Errorf("observe source rollback for %s: %w", step.App, err) + } + runner := c.dockerRunner() + for _, ref := range sourceQuiesceTargetRefs(app) { + container, err := sourceContainerForQuiesce(ctx, runner, ref) + if err != nil { + if isContainerMissingErr(err) { + return fmt.Errorf("source container %s disappeared after rollback; check the app before relying on it", ref.label()) + } + return fmt.Errorf("inspect source container %s: %w", ref.label(), err) + } + if !container.State.Running { + return fmt.Errorf("source container %s stopped after rollback; check the app before relying on it", ref.label()) + } + if container.State.Health != nil && container.State.Health.Status != "healthy" { + return fmt.Errorf("source container %s reports %s after rollback; check the app before relying on it", ref.label(), container.State.Health.Status) + } + } + proxy, err := inspectContainer(ctx, runner, coolifyProxyContainer) + if err != nil { + return fmt.Errorf("inspect proxy container %s: %w", coolifyProxyContainer, err) + } + if !proxy.State.Running { + return fmt.Errorf("%s stopped after rollback; source traffic is not being served", coolifyProxyContainer) + } + return nil +} diff --git a/internal/target/dokploy/rollback_test.go b/internal/target/dokploy/rollback_test.go new file mode 100644 index 0000000..4001eeb --- /dev/null +++ b/internal/target/dokploy/rollback_test.go @@ -0,0 +1,366 @@ +package dokploy + +import ( + "context" + "errors" + "fmt" + "io" + "strings" + "testing" + "time" + + "github.com/aikins01/bort/internal/gateway" + "github.com/aikins01/bort/internal/preparer" +) + +func rollbackPrepareFixture() preparer.Result { + api := preparer.AppPlan{Name: "api"} + api.Resources.SourceServices = []preparer.SourceServiceRef{ + {ServiceName: "web", ContainerID: "api-web-id"}, + } + return preparer.Result{Apps: []preparer.AppPlan{api, {Name: "assets"}}} +} + +func rollbackCutoverFixture() gateway.Result { + return gateway.Result{Apps: []gateway.AppPlan{{ + Name: "api", + Routes: []gateway.Route{{Host: "app.example.com"}}, + }}} +} + +func stepKinds(steps []Step) []StepKind { + kinds := make([]StepKind, 0, len(steps)) + for _, step := range steps { + kinds = append(kinds, step.Kind) + } + return kinds +} + +func TestPlanForRollbackOrdersSteps(t *testing.T) { + plan := PlanForRollback(rollbackPrepareFixture(), rollbackCutoverFixture()) + want := []StepKind{ + StepResumeSource, + StepVerifySourceHealth, + StepStopDokployProxy, + StepStartCoolifyProxy, + StepObserveRollback, + } + got := stepKinds(plan.Steps) + if len(got) != len(want) { + t.Fatalf("expected steps %v, got %v", want, got) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("expected steps %v, got %v", want, got) + } + } + for _, step := range plan.Steps { + if step.Kind == StepVerifySourceHealth || step.Kind == StepObserveRollback { + if step.App != "api" { + t.Fatalf("expected app-scoped step for api, got %v", step) + } + } + } +} + +func TestPlanForRollbackSkipsProxySwapWithoutRoutes(t *testing.T) { + plan := PlanForRollback(rollbackPrepareFixture(), gateway.Result{}) + for _, step := range plan.Steps { + switch step.Kind { + case StepStopDokployProxy, StepStartCoolifyProxy, StepObserveRollback: + t.Fatalf("did not expect proxy swap or observation without routes, got %v", plan.Steps) + } + } + got := stepKinds(plan.Steps) + if len(got) != 2 || got[0] != StepResumeSource || got[1] != StepVerifySourceHealth { + t.Fatalf("expected resume and verify only, got %v", got) + } +} + +type dependentSourceRunner struct { + fakeDockerRunner + running map[string]bool +} + +func (r *dependentSourceRunner) Output(_ context.Context, args ...string) ([]byte, error) { + if args[0] == "start" { + r.running[args[1]] = true + return nil, nil + } + id := args[len(args)-1] + health := "healthy" + if id == "api-web-id" && !r.running["worker-id"] { + health = "unhealthy" + } + return []byte(fmt.Sprintf(`[{"Id":%q,"State":{"Running":%t,"Health":{"Status":%q}}}]`, id, r.running[id], health)), nil +} + +func TestRollbackStartsDependenciesBeforeCheckingHealth(t *testing.T) { + prepare := rollbackPrepareFixture() + worker := preparer.AppPlan{Name: "worker"} + worker.Resources.SourceServices = []preparer.SourceServiceRef{{ServiceName: "worker", ContainerID: "worker-id"}} + prepare.Apps = append(prepare.Apps, worker) + runner := &dependentSourceRunner{running: map[string]bool{}} + client := &Client{Docker: runner} + if err := client.Apply(context.Background(), PlanForRollback(prepare, gateway.Result{})); err != nil { + t.Fatalf("expected dependent apps to resume before health checks: %v", err) + } + if !runner.running["api-web-id"] || !runner.running["worker-id"] { + t.Fatalf("expected both sources running, got %v", runner.running) + } +} + +func TestPlanForRollbackSkimsAppsWithoutQuiesceTargets(t *testing.T) { + prepare := preparer.Result{Apps: []preparer.AppPlan{{Name: "assets"}}} + plan := PlanForRollback(prepare, rollbackCutoverFixture()) + got := stepKinds(plan.Steps) + if len(got) != 2 || got[0] != StepStopDokployProxy || got[1] != StepStartCoolifyProxy { + t.Fatalf("expected proxy swap only for a run that stopped no source containers, got %v", got) + } +} + +type flappingHealthRunner struct { + fakeDockerRunner + calls int +} + +type deadlineInspectRunner struct { + t *testing.T + waitForCancellation bool + calls int +} + +func (r *deadlineInspectRunner) Output(ctx context.Context, _ ...string) ([]byte, error) { + r.calls++ + if _, ok := ctx.Deadline(); !ok { + r.t.Error("expected inspect context to have a deadline") + } + if r.waitForCancellation { + <-ctx.Done() + return nil, ctx.Err() + } + return []byte(`[{"Id":"api-web-id","Name":"/api-web","State":{"Running":true,"Status":"running"}}]`), nil +} + +func (r *deadlineInspectRunner) Run(context.Context, io.Reader, io.Writer, ...string) error { + return errors.New("unexpected docker run") +} + +func TestVerifySourceContainerHealthyBoundsInitialInspection(t *testing.T) { + runner := &deadlineInspectRunner{t: t} + err := verifySourceContainerHealthy(context.Background(), runner, commitTargetRef{id: "api-web-id"}) + if err != nil { + t.Fatalf("verifySourceContainerHealthy: %v", err) + } + if runner.calls != 1 { + t.Fatalf("expected one inspection, got %d", runner.calls) + } +} + +func TestVerifySourceContainerHealthyPropagatesCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + runner := &deadlineInspectRunner{t: t, waitForCancellation: true} + err := verifySourceContainerHealthy(ctx, runner, commitTargetRef{id: "api-web-id"}) + if !errors.Is(err, context.Canceled) { + t.Fatalf("expected cancellation, got %v", err) + } +} + +func TestRollbackBoundsEveryStepInspection(t *testing.T) { + oldSettle := rollbackObserveSettle + rollbackObserveSettle = 0 + t.Cleanup(func() { rollbackObserveSettle = oldSettle }) + plan := PlanForRollback(rollbackPrepareFixture(), rollbackCutoverFixture()) + if plan.stepTimeout != dockerStartTimeout { + t.Fatalf("expected bounded rollback steps, got %s", plan.stepTimeout) + } + for _, step := range plan.Steps { + t.Run(string(step.Kind), func(t *testing.T) { + runner := &deadlineInspectRunner{t: t, waitForCancellation: true} + client := &Client{Docker: runner} + stepPlan := plan + stepPlan.Steps = []Step{step} + stepPlan.stepTimeout = 10 * time.Millisecond + if err := client.Apply(context.Background(), stepPlan); !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("expected timed out inspection for %s, got %v", step.Kind, err) + } + if runner.calls == 0 { + t.Fatal("expected a Docker inspection") + } + }) + } +} + +func (f *flappingHealthRunner) Output(ctx context.Context, args ...string) ([]byte, error) { + f.calls++ + health := "starting" + if f.calls > 1 { + health = "healthy" + } + return []byte(`[{"Id":"api-web-id","Name":"/api-web","State":{"Running":true,"Status":"running","Health":{"Status":"` + health + `"}}}]`), nil +} + +func TestApplyVerifySourceHealthAcceptsRunningContainer(t *testing.T) { + runner := &fakeDockerRunner{ + outputs: map[string][]byte{ + "inspect --type container api-web-id": []byte(`[{"Id":"api-web-id","Name":"/api-web","State":{"Running":true,"Status":"running"}}]`), + }, + } + client := &Client{Docker: runner} + actx := &applyContext{cache: map[string]*appCache{}, plan: Plan{Prepare: rollbackPrepareFixture()}} + if err := client.applyVerifySourceHealth(context.Background(), actx, Step{Kind: StepVerifySourceHealth, App: "api"}); err != nil { + t.Fatalf("applyVerifySourceHealth: %v", err) + } +} + +func TestApplyVerifySourceHealthRefusesWhenNotRunning(t *testing.T) { + runner := &fakeDockerRunner{ + outputs: map[string][]byte{ + "inspect --type container api-web-id": []byte(`[{"Id":"api-web-id","Name":"/api-web","State":{"Running":false,"Status":"exited"}}]`), + }, + } + client := &Client{Docker: runner} + actx := &applyContext{cache: map[string]*appCache{}, plan: Plan{Prepare: rollbackPrepareFixture()}} + err := client.applyVerifySourceHealth(context.Background(), actx, Step{Kind: StepVerifySourceHealth, App: "api"}) + if err == nil || !strings.Contains(err.Error(), "not returning traffic to the source") { + t.Fatalf("expected refusal when the source did not restart, got %v", err) + } +} + +func TestApplyVerifySourceHealthRefusesUnhealthy(t *testing.T) { + runner := &fakeDockerRunner{ + outputs: map[string][]byte{ + "inspect --type container api-web-id": []byte(`[{"Id":"api-web-id","Name":"/api-web","State":{"Running":true,"Status":"running","Health":{"Status":"unhealthy"}}}]`), + }, + } + client := &Client{Docker: runner} + actx := &applyContext{cache: map[string]*appCache{}, plan: Plan{Prepare: rollbackPrepareFixture()}} + err := client.applyVerifySourceHealth(context.Background(), actx, Step{Kind: StepVerifySourceHealth, App: "api"}) + if err == nil || !strings.Contains(err.Error(), "reports unhealthy") { + t.Fatalf("expected refusal for an unhealthy container, got %v", err) + } +} + +func TestApplyVerifySourceHealthPollsUntilHealthy(t *testing.T) { + oldInterval := sourceHealthPollInterval + sourceHealthPollInterval = 0 + defer func() { sourceHealthPollInterval = oldInterval }() + + runner := &flappingHealthRunner{} + client := &Client{Docker: runner} + actx := &applyContext{cache: map[string]*appCache{}, plan: Plan{Prepare: rollbackPrepareFixture()}} + if err := client.applyVerifySourceHealth(context.Background(), actx, Step{Kind: StepVerifySourceHealth, App: "api"}); err != nil { + t.Fatalf("expected starting health to be polled until healthy, got %v", err) + } + if runner.calls < 2 { + t.Fatalf("expected at least two health polls, calls=%d", runner.calls) + } +} + +func TestApplyObserveRollbackPassesWhenAllRunning(t *testing.T) { + oldSettle := rollbackObserveSettle + rollbackObserveSettle = 0 + defer func() { rollbackObserveSettle = oldSettle }() + + runner := &fakeDockerRunner{ + outputs: map[string][]byte{ + "inspect --type container api-web-id": []byte(`[{"Id":"api-web-id","Name":"/api-web","State":{"Running":true,"Status":"running"}}]`), + "inspect --type container coolify-proxy": []byte(`[{"Id":"cp-id","Name":"/coolify-proxy","State":{"Running":true,"Status":"running"}}]`), + }, + } + client := &Client{Docker: runner} + actx := &applyContext{cache: map[string]*appCache{}, plan: Plan{Prepare: rollbackPrepareFixture()}} + if err := client.applyObserveRollback(context.Background(), actx, Step{Kind: StepObserveRollback, App: "api"}); err != nil { + t.Fatalf("applyObserveRollback: %v", err) + } +} + +func TestApplyObserveRollbackReportsStoppedContainer(t *testing.T) { + oldSettle := rollbackObserveSettle + rollbackObserveSettle = 0 + defer func() { rollbackObserveSettle = oldSettle }() + + runner := &fakeDockerRunner{ + outputs: map[string][]byte{ + "inspect --type container api-web-id": []byte(`[{"Id":"api-web-id","Name":"/api-web","State":{"Running":false,"Status":"exited"}}]`), + }, + } + client := &Client{Docker: runner} + actx := &applyContext{cache: map[string]*appCache{}, plan: Plan{Prepare: rollbackPrepareFixture()}} + err := client.applyObserveRollback(context.Background(), actx, Step{Kind: StepObserveRollback, App: "api"}) + if err == nil || !strings.Contains(err.Error(), "stopped after rollback") { + t.Fatalf("expected observation to report a stopped source container, got %v", err) + } +} + +func TestApplyObserveRollbackReportsStoppedProxy(t *testing.T) { + oldSettle := rollbackObserveSettle + rollbackObserveSettle = 0 + defer func() { rollbackObserveSettle = oldSettle }() + + runner := &fakeDockerRunner{ + outputs: map[string][]byte{ + "inspect --type container api-web-id": []byte(`[{"Id":"api-web-id","Name":"/api-web","State":{"Running":true,"Status":"running"}}]`), + "inspect --type container coolify-proxy": []byte(`[{"Id":"cp-id","Name":"/coolify-proxy","State":{"Running":false,"Status":"exited"}}]`), + }, + } + client := &Client{Docker: runner} + actx := &applyContext{cache: map[string]*appCache{}, plan: Plan{Prepare: rollbackPrepareFixture()}} + err := client.applyObserveRollback(context.Background(), actx, Step{Kind: StepObserveRollback, App: "api"}) + if err == nil || !strings.Contains(err.Error(), "source traffic is not being served") { + t.Fatalf("expected observation to report a stopped source proxy, got %v", err) + } +} + +func TestApplyObserveRollbackRejectsUnhealthyRunningSource(t *testing.T) { + oldSettle := rollbackObserveSettle + rollbackObserveSettle = 0 + t.Cleanup(func() { rollbackObserveSettle = oldSettle }) + runner := &fakeDockerRunner{outputs: map[string][]byte{ + "inspect --type container api-web-id": []byte(`[{"Id":"api-web-id","State":{"Running":true,"Health":{"Status":"healthy"}}}]`), + }} + client := &Client{Docker: runner} + actx := &applyContext{plan: Plan{Prepare: rollbackPrepareFixture()}} + if err := client.applyVerifySourceHealth(context.Background(), actx, Step{App: "api"}); err != nil { + t.Fatal(err) + } + runner.outputs["inspect --type container api-web-id"] = []byte(`[{"Id":"api-web-id","State":{"Running":true,"Health":{"Status":"unhealthy"}}}]`) + if err := client.applyObserveRollback(context.Background(), actx, Step{App: "api"}); err == nil || !strings.Contains(err.Error(), "unhealthy after rollback") { + t.Fatalf("expected post-swap health failure, got %v", err) + } +} + +func TestApplyStartCoolifyProxyStartsStoppedAndVerifiesRunning(t *testing.T) { + runner := &fakeDockerRunner{ + outputs: map[string][]byte{ + "inspect --type container coolify-proxy": []byte(`[{"Id":"cp-id","Name":"/coolify-proxy","State":{"Running":false,"Status":"exited"}}]`), + "start cp-id": []byte("cp-id\n"), + }, + } + client := &Client{Docker: runner} + err := client.applyStartCoolifyProxy(context.Background(), &applyContext{}, Step{Kind: StepStartCoolifyProxy, Ref: coolifyProxyContainer}) + if err == nil || !strings.Contains(err.Error(), "did not stay running") { + t.Fatalf("expected refusal when the proxy does not stay running, got %v", err) + } +} + +func TestApplyStartCoolifyProxyAcceptsRunningProxy(t *testing.T) { + runner := &fakeDockerRunner{ + outputs: map[string][]byte{ + "inspect --type container coolify-proxy": []byte(`[{"Id":"cp-id","Name":"/coolify-proxy","State":{"Running":true,"Status":"running"}}]`), + }, + } + client := &Client{Docker: runner} + if err := client.applyStartCoolifyProxy(context.Background(), &applyContext{}, Step{Kind: StepStartCoolifyProxy, Ref: coolifyProxyContainer}); err != nil { + t.Fatalf("applyStartCoolifyProxy: %v", err) + } +} + +func TestApplyStartCoolifyProxyErrorsWhenMissing(t *testing.T) { + client := &Client{Docker: &fakeDockerRunner{outputs: map[string][]byte{}}} + err := client.applyStartCoolifyProxy(context.Background(), &applyContext{}, Step{Kind: StepStartCoolifyProxy, Ref: coolifyProxyContainer}) + if err == nil { + t.Fatal("expected error when coolify-proxy is missing: bort cannot restore source traffic without it") + } +} From 988ca6f46b6ee55557b81fd13e27e10109d1f12d Mon Sep 17 00:00:00 2001 From: Aikins Laryea Date: Thu, 24 Sep 2026 00:58:26 +0000 Subject: [PATCH 29/31] cli: support Dokploy kernels without IPVS (#36) --- docs/migration-guide.md | 16 +++++ internal/cli/init_target.go | 34 ++++++++++ internal/cli/init_target_test.go | 105 +++++++++++++++++++++++++++++++ 3 files changed, 155 insertions(+) diff --git a/docs/migration-guide.md b/docs/migration-guide.md index c7f1153..c0b9c0c 100644 --- a/docs/migration-guide.md +++ b/docs/migration-guide.md @@ -32,6 +32,22 @@ move depends on Linux, Docker, and access to protected host files. Use the macOS and Windows builds to inspect migration files and help develop Bort, not to run a production migration. +### Kernels without IPVS + +Dokploy's control-plane services use Docker Swarm VIP networking by default. +If the kernel explicitly reports that IPVS is disabled, Bort stops before +installing anything. On these hosts, use Dokploy's documented DNS round-robin +mode when starting Bort, for example `sudo ENDPOINT_MODE=dnsrr bort migrate --live` +or `sudo ENDPOINT_MODE=dnsrr bort init-target --install`. + +This option applies to the three control-plane services that Bort creates, not +to existing services. Bort refuses a conflicting existing endpoint mode; review +and update those services before retrying. DNSRR requires host-mode published +ports, which Bort's shadow installer already uses. Bort migrates workloads as +Docker Compose containers, not Swarm services. Other applications deployed as +Swarm services can still require IPVS or direct task addressing; see +[Dokploy's networking guide](https://docs.dokploy.com/docs/core/troubleshooting/networking). + ## Keep one workspace and one identity Bort stores the current run, Dokploy credentials, your answers, plans, and live diff --git a/internal/cli/init_target.go b/internal/cli/init_target.go index ce4113a..206e5c0 100644 --- a/internal/cli/init_target.go +++ b/internal/cli/init_target.go @@ -685,6 +685,28 @@ ADDR_POOL="${ADDR_POOL:-auto}" VERSION_TAG="${DOKPLOY_VERSION:-latest}" ACME_EMAIL="${ACME_EMAIL:-admin@dokploy.local}" TRAEFIK_IMAGE="${DOKPLOY_TRAEFIK_IMAGE:-traefik:v3.6.7}" +ENDPOINT_MODE="${ENDPOINT_MODE:-vip}" + +validate_endpoint_mode() { + case "$1" in + vip) + if grep -Eq '^(# CONFIG_IP_VS is not set|CONFIG_IP_VS=n)$' <<<"$2"; then + echo "Docker Swarm VIP mode requires kernel IPVS support; use ENDPOINT_MODE=dnsrr for Dokploy's control-plane services or a kernel with IPVS support" >&2 + return 1 + fi + ;; + dnsrr) ;; + *) echo "ENDPOINT_MODE must be vip or dnsrr" >&2; return 1 ;; + esac +} + +kernel_config="" +if [ -r /proc/config.gz ]; then + kernel_config="$(gzip -dc /proc/config.gz 2>/dev/null || true)" +elif [ -r "/boot/config-$(uname -r)" ]; then + kernel_config="$(cat "/boot/config-$(uname -r)")" +fi +validate_endpoint_mode "$ENDPOINT_MODE" "$kernel_config" if ! [[ "$ACME_EMAIL" =~ ^[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}$ ]]; then echo "invalid ACME_EMAIL \"$ACME_EMAIL\"" >&2 @@ -714,6 +736,15 @@ if ! command -v python3 >/dev/null 2>&1; then exit 1 fi +for service in dokploy-postgres dokploy-redis dokploy; do + if mode="$(docker service inspect "$service" --format '{{.Spec.EndpointSpec.Mode}}' 2>/dev/null)"; then + if [ "$mode" != "$ENDPOINT_MODE" ]; then + echo "existing service $service uses endpoint mode $mode, not $ENDPOINT_MODE; review and update its endpoint mode before rerunning (existing services are not changed automatically)" >&2 + exit 1 + fi + fi +done + private_ip() { ip -o -4 addr show scope global 2>/dev/null | awk 'first == "" { split($4, a, "/"); first = a[1] } END { print first }' } @@ -900,6 +931,7 @@ log "Starting Dokploy Postgres" if ! docker service inspect dokploy-postgres >/dev/null 2>&1; then docker service create \ --name dokploy-postgres \ + --endpoint-mode "$ENDPOINT_MODE" \ --detach=true \ --constraint 'node.role==manager' \ --network dokploy-network \ @@ -915,6 +947,7 @@ log "Starting Dokploy Redis" if ! docker service inspect dokploy-redis >/dev/null 2>&1; then docker service create \ --name dokploy-redis \ + --endpoint-mode "$ENDPOINT_MODE" \ --detach=true \ --constraint 'node.role==manager' \ --network dokploy-network \ @@ -932,6 +965,7 @@ log "Starting Dokploy UI/API on port ${HOST_PORT}" if ! docker service inspect dokploy >/dev/null 2>&1; then docker service create \ --name dokploy \ + --endpoint-mode "$ENDPOINT_MODE" \ --detach=true \ --replicas 1 \ --network dokploy-network \ diff --git a/internal/cli/init_target_test.go b/internal/cli/init_target_test.go index 17fbbc9..dc5de24 100644 --- a/internal/cli/init_target_test.go +++ b/internal/cli/init_target_test.go @@ -9,6 +9,7 @@ import ( "net/http" "net/http/httptest" "os" + "os/exec" "path/filepath" "strings" "testing" @@ -663,6 +664,110 @@ func TestDokployShadowInstallScriptIsHardened(t *testing.T) { } } +func TestDokployShadowEndpointMode(t *testing.T) { + bash, err := exec.LookPath("bash") + if err != nil { + t.Skip("bash is unavailable") + } + start := strings.Index(dokployShadowInstallScript, "validate_endpoint_mode() {") + end := strings.Index(dokployShadowInstallScript, "\nkernel_config=") + if start < 0 || end < start { + t.Fatal("missing endpoint validation function") + } + for _, tc := range []struct { + mode, config, wantError string + }{ + {"vip", "CONFIG_IP_VS=y", ""}, + {"vip", "CONFIG_IP_VS=m", ""}, + {"vip", "", ""}, + {"vip", "# CONFIG_IP_VS is not set", "requires kernel IPVS"}, + {"vip", "CONFIG_IP_VS=n", "requires kernel IPVS"}, + {"dnsrr", "# CONFIG_IP_VS is not set", ""}, + {"dnsrr", "CONFIG_IP_VS=y", ""}, + {"invalid", "CONFIG_IP_VS=y", "must be vip or dnsrr"}, + } { + t.Run(tc.mode+"/"+tc.config, func(t *testing.T) { + cmd := exec.Command(bash, "-c", dokployShadowInstallScript[start:end]+"\nvalidate_endpoint_mode \"$1\" \"$2\"", "test", tc.mode, tc.config) + out, err := cmd.CombinedOutput() + if tc.wantError == "" { + if err != nil { + t.Fatalf("validation failed: %v: %s", err, out) + } + } else if err == nil || !strings.Contains(string(out), tc.wantError) { + t.Fatalf("wanted %q, got %v: %s", tc.wantError, err, out) + } + }) + } + for _, service := range []string{"dokploy-postgres", "dokploy-redis", "dokploy"} { + want := "--name " + service + " \\\n --endpoint-mode \"$ENDPOINT_MODE\"" + if !strings.Contains(dokployShadowInstallScript, want) { + t.Errorf("%s must use the selected endpoint mode", service) + } + } + if !strings.Contains(dokployShadowInstallScript, `--publish published="$HOST_PORT",target=3000,mode=host`) { + t.Fatal("DNSRR requires host-mode publication") + } +} + +func TestDokployShadowEndpointPreflight(t *testing.T) { + bash, err := exec.LookPath("bash") + if err != nil { + t.Skip("bash is unavailable") + } + configStart := strings.Index(dokployShadowInstallScript, "\nkernel_config=") + configEnd := strings.Index(dokployShadowInstallScript, "\nvalidate_endpoint_mode \"") + end := strings.Index(dokployShadowInstallScript, "\nprivate_ip() {") + if configStart < 0 || configEnd < configStart || end < configEnd { + t.Fatal("missing endpoint preflight boundaries") + } + preflight := dokployShadowInstallScript[:configStart] + "\nkernel_config=\"$2\"" + dokployShadowInstallScript[configEnd:end] + stub := ` +ENDPOINT_MODE="$1" +ACME_EMAIL=admin@dokploy.local +id() { echo 0; } +python3() { return 0; } +docker() { + if [ "$1 $2" != "service inspect" ]; then + echo unexpected-docker-command + return 1 + fi + case "$3" in + dokploy-postgres) mode="$POSTGRES_MODE" ;; + dokploy-redis) mode="$REDIS_MODE" ;; + dokploy) mode="$DOKPLOY_MODE" ;; + *) return 1 ;; + esac + [ "$mode" != absent ] || return 1 + printf '%s\n' "$mode" +} +POSTGRES_MODE="$3" +REDIS_MODE="$4" +DOKPLOY_MODE="$5" +` + for _, tc := range []struct { + name, mode, config, postgres, redis, dokploy, wantError string + }{ + {"matching", "dnsrr", "", "dnsrr", "dnsrr", "dnsrr", ""}, + {"absent", "dnsrr", "", "absent", "absent", "absent", ""}, + {"first conflict", "dnsrr", "", "vip", "dnsrr", "dnsrr", "existing service dokploy-postgres uses endpoint mode vip"}, + {"later conflict", "dnsrr", "", "dnsrr", "vip", "dnsrr", "existing service dokploy-redis uses endpoint mode vip"}, + {"last conflict", "dnsrr", "", "dnsrr", "dnsrr", "vip", "existing service dokploy uses endpoint mode vip"}, + {"disabled VIP", "vip", "# CONFIG_IP_VS is not set", "absent", "absent", "absent", "requires kernel IPVS"}, + } { + t.Run(tc.name, func(t *testing.T) { + cmd := exec.Command(bash, "-c", stub+preflight+"\necho preflight-complete", "test", tc.mode, tc.config, tc.postgres, tc.redis, tc.dokploy) + out, err := cmd.CombinedOutput() + if tc.wantError == "" { + if err != nil || !strings.Contains(string(out), "preflight-complete") { + t.Fatalf("preflight did not continue: %v: %s", err, out) + } + } else if err == nil || !strings.Contains(string(out), tc.wantError) || strings.Contains(string(out), "preflight-complete") { + t.Fatalf("wanted refusal %q before continuation, got %v: %s", tc.wantError, err, out) + } + }) + } +} + func TestValidateDokployBootstrapURLPolicy(t *testing.T) { t.Run("accepts http localhost case-insensitively", func(t *testing.T) { if err := validateDokployBootstrapURL("http://LOCALHOST:3030"); err != nil { From da1b1926f12079fe1fb795e810e292332264ec4c Mon Sep 17 00:00:00 2001 From: Aikins Laryea Date: Wed, 30 Sep 2026 12:54:30 +0000 Subject: [PATCH 30/31] migration: stage state and bind traffic authority before cutover (#37) * migration: stage state and bind traffic authority before cutover Live apply copied persistent data into an already deployed target, stateless rollback could not tell which run owned shared Dokploy traffic, and imported bundles or manifests could drive apply, cleanup --apply, and purge as if they were local Docker scans. Live apply now copies named volumes and Postgres dumps into Bort-owned staging volumes before the target deploys, records a durable Dokploy traffic owner and host-wide operation lock under /var/lib/bort, and gates route activation on every handed-off source app staying paused. Runs that reach the target through an imported bundle or manifest are inspection-only; commit --apply refuses Coolify sources because stopping containers cannot fence the orchestrator, so target authority is recorded through recover-authority after manual retirement. Cleanup --apply requires an owner-bound, locally scanned run and is audit-only otherwise. Platform detection prefers Coolify regardless of label order, and Postgres readiness checks use loopback TCP. * dokploy: refuse unstaged Postgres data dirs before pause * cli: route unpassable pause refusals to run release * safepath: allow trusted sticky ancestors Linux test paths commonly descend from root-owned /tmp. Accept sticky writable ancestors owned by a trusted UID while keeping the final private directory at mode 0700 and revalidating that boundary after it is opened. --- README.md | 173 +- docs/cleanup-and-purge.md | 45 +- docs/migration-guide.md | 855 ++++++++- internal/cli/applied.go | 416 ++++- internal/cli/applied_test.go | 510 ++++++ internal/cli/apply_lock.go | 22 + internal/cli/apply_lock_unix.go | 2 +- internal/cli/apply_lock_windows.go | 2 +- internal/cli/authority.go | 302 ++++ internal/cli/authority_test.go | 503 ++++++ internal/cli/cleanup.go | 170 +- internal/cli/cleanup_test.go | 315 +++- internal/cli/cockpit.go | 325 +++- internal/cli/cockpit_test.go | 959 ++++++++++- internal/cli/commit.go | 86 +- internal/cli/commit_test.go | 298 +++- internal/cli/dokploy_lock_unix.go | 251 +++ internal/cli/dokploy_lock_windows.go | 49 + internal/cli/dokploy_traffic_owner.go | 552 ++++++ internal/cli/dokploy_traffic_owner_test.go | 358 ++++ internal/cli/guide.go | 31 +- internal/cli/guide_test.go | 22 +- internal/cli/init_target.go | 1108 +++++++++++- internal/cli/init_target_test.go | 1447 +++++++++++++++- internal/cli/installer_process_unix.go | 196 +++ internal/cli/installer_process_unix_test.go | 484 ++++++ internal/cli/installer_process_windows.go | 24 + internal/cli/legacy_upgrade_test.go | 309 ++++ internal/cli/main_test.go | 73 +- internal/cli/rollback.go | 127 +- internal/cli/rollback_test.go | 485 +++++- internal/cli/root.go | 7 +- internal/cli/run.go | 1282 ++++++++++++-- internal/cli/run_test.go | 1272 +++++++++++++- internal/cli/state_apply_test.go | 9 +- internal/cli/style.go | 2 +- internal/cli/wizard.go | 51 +- internal/dockercli/environment.go | 40 + internal/dockercli/environment_test.go | 51 + internal/exporter/exporter.go | 22 +- internal/manifest/manifest.go | 5 +- internal/preparer/preparer.go | 24 +- internal/rollback/rollback.go | 14 +- internal/safepath/rename_noreplace_darwin.go | 9 + internal/safepath/rename_noreplace_linux.go | 9 + internal/safepath/rename_noreplace_other.go | 9 + internal/safepath/safepath.go | 49 + internal/safepath/safepath_other.go | 4 + internal/safepath/safepath_test.go | 29 + internal/safepath/safepath_unix.go | 62 +- internal/safepath/safepath_unix_test.go | 108 ++ internal/source/localdocker/scanner.go | 54 +- internal/source/localdocker/scanner_test.go | 89 +- internal/target/dokploy/apply.go | 783 +++++++-- internal/target/dokploy/apply_test.go | 705 +++++++- internal/target/dokploy/cleanup.go | 36 +- internal/target/dokploy/cleanup_test.go | 22 +- internal/target/dokploy/client.go | 514 +++++- internal/target/dokploy/client_host_test.go | 232 +++ internal/target/dokploy/datamove.go | 399 ++++- internal/target/dokploy/datamove_test.go | 187 +- internal/target/dokploy/docker.go | 151 +- internal/target/dokploy/proxy.go | 208 +++ internal/target/dokploy/proxy_test.go | 430 ++++- internal/target/dokploy/resume.go | 110 ++ internal/target/dokploy/rollback.go | 38 +- internal/target/dokploy/rollback_test.go | 126 +- internal/target/dokploy/source.go | 644 +++++-- internal/target/dokploy/source_test.go | 950 +++++++++- internal/target/dokploy/staging.go | 990 +++++++++++ internal/target/dokploy/staging_test.go | 1624 ++++++++++++++++++ 71 files changed, 20629 insertions(+), 1220 deletions(-) create mode 100644 internal/cli/apply_lock.go create mode 100644 internal/cli/authority.go create mode 100644 internal/cli/authority_test.go create mode 100644 internal/cli/dokploy_lock_unix.go create mode 100644 internal/cli/dokploy_lock_windows.go create mode 100644 internal/cli/dokploy_traffic_owner.go create mode 100644 internal/cli/dokploy_traffic_owner_test.go create mode 100644 internal/cli/installer_process_unix.go create mode 100644 internal/cli/installer_process_unix_test.go create mode 100644 internal/cli/installer_process_windows.go create mode 100644 internal/cli/legacy_upgrade_test.go create mode 100644 internal/dockercli/environment.go create mode 100644 internal/dockercli/environment_test.go create mode 100644 internal/safepath/rename_noreplace_darwin.go create mode 100644 internal/safepath/rename_noreplace_linux.go create mode 100644 internal/safepath/rename_noreplace_other.go create mode 100644 internal/safepath/safepath_unix_test.go create mode 100644 internal/target/dokploy/client_host_test.go create mode 100644 internal/target/dokploy/resume.go create mode 100644 internal/target/dokploy/staging.go create mode 100644 internal/target/dokploy/staging_test.go diff --git a/README.md b/README.md index a3e2263..2653f7c 100644 --- a/README.md +++ b/README.md @@ -20,16 +20,17 @@ Bort is a guided migration tool for people running their own app platform on a VPS. It checks what is actually running, explains what needs attention, prepares -Dokploy, copies supported data, switches web traffic, and requires an explicit -command before each destructive step. +Dokploy, moves named volumes and Postgres dumps into Bort-owned volumes before +the target deploys, switches web traffic, and requires an explicit command +before each destructive step. Bort organizes the migration by app rather than exposing a wall of Docker details. It shows what is ready, provides copy-paste fixes, saves your answers, and tells you what to do next. Discovery and planning only preview app changes. If Dokploy is not prepared, the `migrate --live` preflight can offer to install it before application migration begins. Bort explains the server changes and -asks first. Moving apps, data, and traffic begins only when you explicitly run -the live command. +asks first. Creating target apps, transferring persistent state, and moving +traffic begin only when you explicitly run the live command. ## Supported path @@ -39,9 +40,13 @@ The current product path is **Coolify → Dokploy on the same Linux VPS**. | --- | --- | | Guided Coolify → Dokploy planning | Implemented | | Explicit live apply and resume | Implemented | -| Accepting Dokploy and stopping source containers | Implemented | +| Accepting Dokploy and stopping source containers (sources without an orchestrator) | Implemented | +| Accepting Dokploy for a Coolify source | Manual source retirement, then recorded with the exact `bort recover-authority --run --authority target --source-retired --confirm '...'` command Bort prints | | Safe Dokploy database cleanup and separately confirmed source removal | Implemented | -| Executable rollback | Implemented (`bort rollback --live`) | +| Stateless traffic rollback | Implemented (`bort rollback --live`) | +| Stateful live apply (named volumes and Postgres dump/restore) | Implemented; state is staged into Bort-owned volumes before the target deploys | +| Bind-mount transfer for stateful apps | Not implemented; same-host bind mounts that keep their paths need no transfer | +| Automatic stateful rollback | Not available; manual authority recovery is required | | Dokploy → Coolify or cross-server migration | Not implemented | Bort publishes macOS, Linux, and Windows binaries, but a complete same-VPS move @@ -59,12 +64,13 @@ Today Bort can: - record missing environment values and how each database or storage service should be moved; - prepare Dokploy projects and compose applications before explicit live apply; -- move supported local data by exporting and importing databases or copying data - while a service is stopped; +- copy named volumes and Postgres dumps into Bort-owned staging volumes during + live apply, before the target deploys, and identify the state that must be + migrated outside Bort, such as bind-mount copies; - switch web traffic during explicit live apply, save progress for retries, and store rollback instructions; -- execute the reviewed rollback (`bort rollback --live`) to restart source - containers and return traffic to the source; +- execute a reviewed stateless rollback (`bort rollback --live`) to return web + traffic to the source; - accept the target only after successful live apply; - list leftovers, remove only eligible unused records from Dokploy, and separately remove eligible source containers and networks after confirmation. @@ -77,21 +83,35 @@ the workload-level coverage and limitations. Use a fresh VM or restorable snapshot while evaluating Bort. Do not test source purge on a host containing data you intend to keep. -Follow two workspace rules during the migration: +Follow two state-management rules during the migration: 1. Run every command from the **same working directory**. Bort stores its workspace under `.bort` relative to that directory. 2. Use the **same OS user** throughout the migration. If the first command uses `sudo`, keep using `sudo`; if the first command does not, do not add it - later. + later. Bort also stores its host-wide operation lock and durable Dokploy + traffic owner under `/var/lib/bort`. -Bort keeps workspace directories and files private, but `.bort` contains target -credentials and application configuration. Do not commit or publish it. +Bort keeps these directories and files private. `.bort` contains target +credentials and application configuration; `/var/lib/bort` binds shared +Dokploy traffic to one migration run. Preserve both, including the original +Dokploy credential, through any `cleanup --apply` you intend to run. Do not +commit or publish either directory. Read the [prerequisites and recovery guidance](docs/migration-guide.md) before a live migration. In particular, review how `bort rollback --live` works before you need it. +For a Dokploy install, prepare an absolute authentication-secret escrow path on +encrypted or off-host storage, inside a mode-0700 directory owned by root or +the invoking sudo user. No directory in the path may be a symbolic link. Every +parent must be owned by root or that user; a parent writable by group or others +must have the sticky bit set. Pass the path as `--auth-secret-backup` or as +`sudo BORT_DOKPLOY_AUTH_SECRET_BACKUP=/path bort ...` (sudo drops exported +variables by default), and retain it with the private provenance marker under +`/var/lib/bort`. Interrupted creation is retried only when the +retained escrow, private creation intent, and Docker secret label still match. + ## Install Install from Homebrew: @@ -124,8 +144,18 @@ sudo bort The guided screen starts setup when needed and otherwise resumes the current run. Review each application, follow its generated `fix:` and `next:` guidance, -and rerun `sudo bort` until the run shows `READY`. This means blocking inputs -are resolved; review any remaining non-blocking notes before live apply. +and rerun `sudo bort` until the run shows `READY` or reports `MANUAL STATE`. +`READY` means blocking inputs are resolved; review any remaining non-blocking +notes before live apply. `MANUAL STATE` means a historical stateful run was +applied with an older plan version that copied state into an already deployed +target. Bort cannot continue that run; follow the displayed manual recovery +guidance or create a new run. `PLAN BLOCKED` means live apply would refuse the +planned state transfer (for example a bind mount or a named volume shared by +two apps); choose `bort data --recreate` or `--managed` for a data +store volume and re-plan, or change the source compose and scan a new run. +`NEW RUN REQUIRED` means live apply refused this run at `pause_source` and the +run cannot be re-planned; follow the recovery `bort status` shows, then create +a new run. To start discovery directly without the setup questions: @@ -140,15 +170,21 @@ The migration uses separate commands for each important step: sudo bort # start or resume, then review and fix sudo bort migrate --live # apply only the selected planned run sudo bort rollback # inspect the stored rollback plan -sudo bort rollback --live # restart source containers and return traffic -sudo bort commit --apply # accept the target and retire source containers +sudo bort rollback --live # return stateless traffic when safe +sudo bort commit --apply # accept the target when no source orchestrator can recreate it sudo bort cleanup # audit leftovers without deleting source resources ``` -After reviewing the cleanup results, `cleanup --apply` can remove only eligible -unused records from the Dokploy database, and only after making a database -backup. Removing source resources remains a separate command that previews its -work first: +For a Coolify source, first disable future deployments and manually retire the +reviewed source apps and proxy. Then record permanent target authority with the +exact `recover-authority --authority target --source-retired` command Bort +prints; `commit --apply` refuses because stopping containers cannot fence +Coolify itself. + +For owner-bound current runs, `cleanup --apply` can remove only eligible unused +records from the bound local Dokploy database, and only after making a database +backup. Pre-upgrade runs without that durable binding are audit-only. Removing +source resources remains a separate command that previews its work first: ```sh sudo bort cleanup purge --all-apps @@ -170,29 +206,54 @@ Bort's safety model defaults to “look first.” writes system configuration, creates Docker resources, initializes Swarm when needed, and may disable Docker live-restore and reload Docker before asking for confirmation. Application migration begins only after confirmation. -- **Explicit live apply:** creating target apps, copying data, and moving traffic - only happen through `bort migrate --live` for an existing planned run. +- **Explicit live apply:** creating target apps and moving traffic only happen + through `bort migrate --live` for an existing planned run. +- **State moves before the target exists:** for apps with named volumes or a + Postgres database, Bort stops the source app, copies each volume and restores + each dump into volumes it owns and labels with the run name, verifies the + source stayed stopped and nothing else attached those volumes, and only then + deploys the Dokploy compose with those volumes declared as external. Bort + does not deploy the compose until state is staged and refuses the transfer + if the source restarts or any container attaches to a staging volume; do not + deploy the app manually in Dokploy during live apply. Routed apps stay + stopped until traffic moves; unrouted apps restart the source after the copy. - **Known current run:** `.bort/state.json` identifies the current run. Commands that make changes do not guess based on which file was modified most recently. - **Plans are locked during live work:** once live execution begins, changing the selected plan requires a new run. +- **Attested source host:** local discovery records the Docker Engine ID and + exact source container identities. Automated source mutation during live + apply, recovery, rollback, and commit refuses a different daemon, replaced + containers, imported manifests or bundles, and legacy runs without that + attestation. Those runs remain inspectable but require manual source handling + or a new local scan for live work. Runtime checks catch a replaced daemon or + container, not every stale detail in a reviewed configuration. - **One change at a time:** Bort prevents two commands from changing the same run at once, saves live progress for retries, and keeps `status` available. -- **Private files:** bundles, state, environment values, live progress, and - target credentials stay in the local workspace with private permissions. +- **Private files:** bundles, environment values, live progress, and target + credentials stay in the local workspace. The host-wide operation lock and + durable Dokploy traffic owner stay under `/var/lib/bort`. Both locations use + private permissions. - **Separate acceptance:** `commit --apply` retires source application containers - only after successful live apply and is required before destructive source - purge. + only when no external orchestrator can recreate them. It refuses Coolify + sources. Disable future Coolify deployments, manually retire the reviewed + source apps and proxy, verify they remain retired, then use confirmed + target-authority recovery with `--source-retired` to record acceptance. + Destructive source purge requires either accepted state. - **Separate destructive purge:** purge requires selected apps or projects, a - successful live apply, an accepted target, the exact confirmation phrase, a - recheck of Docker IDs, and a private backup. -- **Health-gated rollback:** `bort rollback --live` restarts the quiesced source - containers, verifies they are running and healthy before touching any traffic, - swaps the proxies back, and observes the result. It never deletes data; the - Dokploy target resources remain on the server for cleanup, and an interrupted - rollback can be re-run. Commit and live apply stay blocked once rollback starts. - Live rollback uses a five-second settling check per observed app, not the - stored observation window; continue monitoring for the reviewed period. + successful live apply or completed manual target-authority + recovery, an accepted target, the exact confirmation phrase, a recheck of + Docker IDs, and a private backup. +- **Fail-closed rollback:** `bort rollback --live` can swap traffic back only + when the run transferred no database or volume state. Dokploy v0.30.7 cannot + durably block queued or future deployments, so Bort refuses automatic + stateful rollback before changing either side. Preserve both sides and + establish writer and traffic authority manually if a stateful target fails + validation. When the durable host owner still belongs to the run, + `bort recover-authority` records the verified source or target decision and + finishes that run without abandoning its ownership record. + Rollback inspection reports `automaticAvailable` and `automaticBlocker` in + JSON and the equivalent reason in text before any mutation. ## Documentation @@ -205,11 +266,39 @@ Bort's safety model defaults to “look first.” ## Current limitations -- Rollback returns traffic to the source but does not copy back data written to - the target after cutover. -- Bort does not keep copying new volume changes while applications remain live. -- Database moves currently use local export/import or copy data while the - database is stopped. +- Stateful live apply transfers named volumes and Postgres dumps only. A + stateful app whose plan needs a bind-mount copy is refused; same-host bind + mounts that stay at their existing paths do not require a transfer. +- A named volume mounted by more than one compose service cannot be staged. + Live apply refuses it before changing Dokploy; choose `bort data + --recreate` or `--managed` for a data store volume and re-plan with + `bort migrate --run `, or change the source compose and scan a new run. +- A migrated Postgres store must keep its data directory (`PGDATA`) on a named + volume the service mounts, with no writable bind mount inside it. + Live apply refuses other layouts before the source pauses (a run already + past that app's `pause_source` when this check was added refuses at the + restore step and restarts the source during cleanup). When the service + mounts no named volume, sets `PGDATA` in its compose `environment` to a + literal path that breaks this rule, or interpolates a mount target, Bort + refuses before live apply starts (`PLAN BLOCKED`): choose `bort data + --recreate` or `--managed` and re-plan with `bort migrate --run + `, or change the source compose and scan a new run. When the image or + interpolation decides `PGDATA`, only the created staged container reveals + the directory, so the refusal comes at `pause_source` (`NEW RUN REQUIRED`) with + that app's source still running and none of its state transferred, and the + run cannot be re-planned: follow the recovery `bort status` shows (releasing + the run when no other app's source was paused or handed off), then choose a + strategy or change the source compose and create a new run. +- Runs created by older Bort versions whose plan copied state into an already + deployed target remain refused (`MANUAL STATE`). Dokploy v0.30.7 cannot + durably prevent queued or future deployments from restarting a target writer + during such a copy. Create a new run instead. +- Automatic rollback is limited to stateless traffic handoff. Stateful rollback + requires manual writer and traffic authority recovery. +- The temporary compose project Bort uses to restore a database writes its + `.env` in the format the running Dokploy release uses, read from + `settings.getDokployVersion` before the source is paused. Live apply refuses + staged restores when that version is not a `vMAJOR.MINOR.PATCH` release. - GitHub Apps, deploy keys, webhooks, and equivalent source connections are recorded as details but are not copied, recreated, or revoked. - Bort does not automatically delete source Docker images because the target may diff --git a/docs/cleanup-and-purge.md b/docs/cleanup-and-purge.md index f684438..2397d59 100644 --- a/docs/cleanup-and-purge.md +++ b/docs/cleanup-and-purge.md @@ -1,7 +1,9 @@ # Cleanup and purge -Cleanup normally happens after live apply succeeds and you accept Dokploy. Bort -keeps two commands separate: +Cleanup normally happens after live apply succeeds and you accept Dokploy. A +completed manual target-authority recovery can establish the same +prerequisite when automated live apply cannot prove the outcome. Bort keeps two +commands separate: - `cleanup` lists leftovers and can remove a small set of eligible unused records from the Dokploy database; @@ -13,19 +15,28 @@ credentials. ## Accept the target first -Validate the target and wait through the `rollback window` recorded in the plan -before running: +Validate the target and wait through the `rollback window` recorded in the plan. +For a source without an external orchestrator, then run: ```sh sudo bort commit --apply ``` -This confirms that Dokploy is now the active platform and stops the source -application containers. Automated rollback is refused once source retirement -starts, even if commit is interrupted. Rerun `commit --apply` to finish an -interrupted acceptance. Do not start it until you have independently checked the -target. Bort does not measure the rollback window or block this command when the -window has not passed. +This confirms that Dokploy is now the active platform and stops the exact +reviewed source application containers. Automated rollback is refused once +source retirement starts, even if commit is interrupted. Rerun `commit --apply` +to finish an interrupted acceptance. + +For a Coolify source, `commit --apply` refuses because Bort cannot disable +queued or future Coolify deployments. Disable future deployments in Coolify for +the reviewed apps, manually retire every reviewed source app (and the source +proxy when the cutover moved routes), and verify they remain retired. Then run +the exact +`recover-authority --authority target --source-retired --confirm ...` command +Bort prints. This records acceptance without mutating source resources. + +Do not accept the target until you have independently checked it. Bort does not +measure the rollback window or block acceptance when the window has not passed. ## Ordinary cleanup @@ -49,6 +60,14 @@ Before changing the Dokploy database, Bort backs it up. The command only removes eligible empty records with no domains from Dokploy projects named `coolify-proxy`, `proxy`, or `source`. It does not remove source containers, volumes, networks, paths, credentials, Docker images, or target applications. +It also requires a run created from a local Docker scan with its persisted +target origin, durable owner record, and original Dokploy credential identity. +Runs imported from a manifest or bundle stay audit-only. Preserve `.bort`, +`/var/lib/bort/dokploy-traffic-owner.json`, and the original credential through +this step. Complete any intended metadata `cleanup --apply` before another +migration performs its first Dokploy target mutation; that mutation replaces a +released owner record with the new run's ownership. Pre-upgrade runs without +the binding remain audit-only and do not offer an apply command. ## Plan a source purge @@ -83,8 +102,10 @@ sudo bort cleanup purge --apply --run "$run" --all-apps --confirm "purge $run" Purge apply refuses to run unless: - `--app`, `--project`, or `--all-apps` selects what to remove; -- Bort's saved record shows that live apply succeeded; -- the target was accepted with `commit --apply`; +- Bort's saved record shows that live apply succeeded or manual + target-authority recovery completed; +- the target was accepted with `commit --apply` or confirmed target-authority + recovery with `--source-retired`; - the exact `purge ` confirmation is supplied. Bort writes a private purge-plan backup under `.bort/backups` before destructive diff --git a/docs/migration-guide.md b/docs/migration-guide.md index c0b9c0c..807d9bd 100644 --- a/docs/migration-guide.md +++ b/docs/migration-guide.md @@ -11,7 +11,12 @@ any data you need to retain. Before starting, confirm that: - Coolify is running on the source Linux VPS and its applications are healthy; -- Docker is available to the OS user that will run Bort; +- Docker is available to the OS user that will run Bort through the system + socket at `/var/run/docker.sock`; unset `DOCKER_CONTEXT`, + `DOCKER_TLS_VERIFY`, and `DOCKER_CERT_PATH`, and leave `DOCKER_HOST` unset or + set it exactly to `unix:///var/run/docker.sock`; rootless, SSH, TCP, and other + Docker targets are refused so discovery and mutation cannot address different + daemons; - `bash`, `python3`, `openssl`, `curl`, and the standard Linux networking tools used by Bort's side-by-side Dokploy installer are available; if Docker live-restore is enabled, the host must use systemd and provide a working @@ -32,13 +37,519 @@ move depends on Linux, Docker, and access to protected host files. Use the macOS and Windows builds to inspect migration files and help develop Bort, not to run a production migration. +### Dokploy version and authentication secret + +Bort pins Dokploy v0.30.7 and its reviewed multi-platform image digest by +default. If you set `--dokploy-version`, use a release at v0.29.3 or newer and +include its independently reviewed registry digest as +`vX.Y.Z@sha256:<64 lowercase hex characters>`. Bare nondefault tags, floating +`latest` and `canary` tags, and unresolved or changed digests are refused +because a retry cannot prove which image receives the Docker socket and +secrets. Older releases cannot load the authentication key from the Docker +secret that Bort creates. + +Before a fresh install, pass `--auth-secret-backup` with an absolute path on +encrypted or off-host storage, or run +`sudo BORT_DOKPLOY_AUTH_SECRET_BACKUP=/path bort ...` (sudo drops exported +variables by default). Put it +inside a mode-0700 directory owned by root or the invoking sudo user. Every +ancestor must be a real directory owned by root or that user. An ancestor +writable by group or others must have the sticky bit set. Bort creates and +fsyncs the authentication key, opens it without following links, and supplies +that validated file descriptor to Docker. It then persists a private creation +intent and labels the new Docker secret with the matching nonce. After creation +it records the exact Docker secret ID and escrow digest in +`/var/lib/bort/dokploy-auth-secret.id`. A retry can finish the marker transition +after interruption only when the pending nonce, Docker label, and retained +escrow all match; otherwise it refuses the orphaned secret. +Retain both files. Bort mounts the key into Dokploy without putting it in the +service environment or migration workspace. Do not delete or rotate it +casually: Dokploy uses it for sessions and encrypted settings. If an existing +Dokploy service has no configured +`BETTER_AUTH_SECRET` or `BETTER_AUTH_SECRET_FILE`, Bort refuses to generate a +replacement implicitly. Recover it with the procedure below. Bort does not +print or store the secret in its workspace. Bort also refuses to create a key +when the Dokploy service is absent but an existing Dokploy database service or +volume remains. Restore the prior Dokploy service, original Bort-managed +`dokploy_auth_secret`, and `/var/lib/bort/dokploy-auth-secret.id` provenance +marker together from a consistent host backup before retrying. Bort rejects a +manually recreated secret even when its key bytes match because its Docker ID +and creation provenance differ. If no consistent backup exists, recover the +service manually outside Bort instead of rerunning the installer. Bort accepts +an existing explicit inline key only when it is exactly 64 lowercase +hexadecimal characters. Bort cannot prove how an existing key was generated. +Use the recovery procedure below for the published legacy value, and rotate any +other key whose provenance or entropy is unknown with a Dokploy-supported +procedure before retrying. + +The installer has a 20-minute overall deadline. A timeout terminates its shell +process group and leaves a durable installation-recovery marker after releasing +Bort's host-wide operation lock. That marker blocks live apply, commit, +rollback, cleanup mutations, and ordinary target initialization. Inspect the +durable install state, then rerun the same `init-target --install` command under +the recovery lock to reconcile or resume it. The private marker binds the +target URL, port, address pool, digest-qualified Dokploy version, ACME email, +and escrow path, and retains the exact shell-quoted recovery command. The +cockpit and `bort next` show that command; a retry with different installation +inputs is refused. The bound inputs include the admin display name and API-key +name used by post-install bootstrap. Bort removes the marker only after the +local Dokploy service answers and passes same-host verification. + +#### Recover an existing Dokploy service using the legacy secret + +Dokploy releases that have no explicit authentication secret use the legacy +value `better-auth-secret-123456789`. Dokploy also derives the key for encrypted +configuration from that value. Rotating the authentication secret without +preserving the derived key can leave stored configuration unreadable. + +Schedule a maintenance window first. Before taking the backup, deny every +inbound path to the Dokploy UI and API at the firewall or external load +balancer. Inspect the existing service with +`docker service inspect dokploy --format '{{json .Endpoint.Ports}}'`, then block +every published host port whose target port is 3000. Bort uses host port 3030 +by default; a custom `--install-port` uses a different host port. Block any +dashboard routes on ports 80/443 as well. Keep the local Dokploy task running +until the command below performs its first stop-first forced update. Verify from +every client network that both the UI and API are unreachable, and keep this +write fence in place through all service restarts. The command replaces the +legacy-secret task, waits for active database work to settle, and only then +captures the two-factor snapshot. + +With the write fence active, take recoverable disaster-recovery backups of the +complete Dokploy PostgreSQL database, `/etc/dokploy`, Docker service +configuration, and Docker secrets. Do not use the full database backup as the +routine recovery for an interrupted authentication-secret migration: Dokploy +workers can write unrelated rows while inbound HTTP is fenced, and restoring +the whole database can silently discard those writes. The procedure below +captures and compare-and-swap restores only `public.two_factor.secret` and +`public.two_factor.backup_codes`, the fields that Dokploy's migration changes. +Then preserve the legacy-derived encryption key: + +```sh +( +set -eu +if ! legacy_hmac="$(printf '%s' 'dokploy:db-encryption:v1' | + openssl dgst -sha256 -hmac 'better-auth-secret-123456789')"; then + echo 'failed to derive the legacy Dokploy encryption key' >&2 + exit 1 +fi +legacy_key="$(printf '%s\n' "$legacy_hmac" | awk '{print $NF}')" +test "${#legacy_key}" -eq 64 +printf '%s\n' "$legacy_key" | grep -Eq '^[0-9a-f]{64}$' +sudo install -d -m 700 /etc/dokploy +printf '%s\n' "$legacy_key" | sudo sh -eu -c ' +IFS= read -r key +[ "${#key}" -eq 64 ] +case "$key" in + *[!0-9a-f]*) echo "derived encryption key is malformed" >&2; exit 1 ;; +esac +file=/etc/dokploy/encryption.key +tmp=$(mktemp /etc/dokploy/.encryption.key.XXXXXX) +cleanup_key_tmp() { rm -f -- "${tmp:-}"; } +trap cleanup_key_tmp EXIT +trap "exit 129" HUP +trap "exit 130" INT +trap "exit 143" TERM +if [ -e "$file" ]; then + if [ -L "$file" ] || [ ! -f "$file" ]; then + echo "$file is not a regular file" >&2 + exit 1 + fi + if grep -Ev "^[0-9a-f]{64}$" "$file" | grep -q "^"; then + echo "$file contains an invalid key line" >&2 + exit 1 + fi + mode=$(stat -c "%a" "$file") + owner=$(stat -c "%u:%g" "$file") + while IFS= read -r line || [ -n "$line" ]; do + printf "%s\n" "$line" + done < "$file" > "$tmp" + chmod "$mode" "$tmp" + chown "$owner" "$tmp" +else + chmod 600 "$tmp" + chown root:root "$tmp" +fi +grep -Fxq "$key" "$tmp" || printf "%s\n" "$key" >> "$tmp" +sync -f "$tmp" +mv -f "$tmp" "$file" +sync -f "$(dirname "$file")" +tmp= +trap - HUP INT TERM +trap - EXIT +' +unset legacy_key +unset legacy_hmac +) +``` + +Each line in `/etc/dokploy/encryption.key` is a 32-byte key encoded as 64 +lowercase hexadecimal characters, not a raw authentication secret. The command +validates and preserves the existing key ring, then atomically adds the +legacy-derived key only if it is absent, and syncs the replacement before the +authentication-secret migration continues. Back up this file separately. +Dokploy v0.29.12 and newer web-server backups include the key ring by default +while **Include encryption key** remains enabled; confirm that option and keep +the independent copy as well. On v0.29.5 through v0.29.11, rely on the separate +key-file backup because that backup option is unavailable. + +Upgrade Dokploy to v0.29.5 or newer before running the migration. The migration +command does not exist before v0.29.3, and v0.29.5 fixes the migration process +not exiting when there are no two-factor records. Follow Dokploy's upgrade +procedure for your installed release, then inspect the running service and +confirm its image uses an explicit supported release tag: + +```sh +sudo docker service inspect dokploy \ + --format '{{.Spec.TaskTemplate.ContainerSpec.Image}}' +``` + +Do not use the hosted `0.29.3.sh` wrapper for this recovery. It treats a +120-second migration timeout as success and can rotate the service secret while +the database transaction is still uncommitted. Run the migration command +without that timeout, and update the service authentication setting only after +the command exits successfully. Before running it, set +`DOKPLOY_AUTH_SECRET_BACKUP` to a new absolute path on mounted encrypted or +off-host storage that is backed up independently of this host. The command +creates or reuses that mode-0600 escrow, syncs it, and verifies it byte for byte +before creating the Docker secret or changing the database. Also set +`DOKPLOY_TWO_FACTOR_BEFORE` to a new absolute mode-0600 snapshot path on the +same class of protected storage. Keep both files for as long as this Dokploy +installation uses the secret. + +The automated command below supports only Dokploy's default internal +`dokploy-postgres` service. It inspects the Dokploy service and refuses to run +when `DATABASE_URL` is configured or when `POSTGRES_HOST`, `POSTGRES_PORT`, +`POSTGRES_USER`, or `POSTGRES_DB` differs from the default internal connection. +For an external or otherwise custom PostgreSQL deployment, have the database +operator capture the same three-column snapshots and perform the same +compare-and-swap transaction against that exact database using its approved +connection and backup procedures. Do not substitute the local +`dokploy-postgres` container commands below. + +```sh +( + set -eu + if sudo docker secret inspect dokploy-auth-secret >/dev/null 2>&1; then + echo 'refusing to run migration while Docker secret dokploy-auth-secret already exists' >&2 + echo 'if an earlier attempt was interrupted before service rotation, keep the write fence active and use the two-factor compare-and-swap recovery below before removing the unused secret' >&2 + exit 1 + fi + : "${DOKPLOY_AUTH_SECRET_BACKUP:?set DOKPLOY_AUTH_SECRET_BACKUP to an absolute path on separately backed-up encrypted or off-host storage}" + case "$DOKPLOY_AUTH_SECRET_BACKUP" in + /*) ;; + *) echo 'DOKPLOY_AUTH_SECRET_BACKUP must be an absolute path' >&2; exit 1 ;; + esac + auth_secret_backup_dir="$(dirname -- "$DOKPLOY_AUTH_SECRET_BACKUP")" + test -d "$auth_secret_backup_dir" + test ! -L "$DOKPLOY_AUTH_SECRET_BACKUP" + : "${DOKPLOY_TWO_FACTOR_BEFORE:?set DOKPLOY_TWO_FACTOR_BEFORE to a new absolute path on separately backed-up encrypted or off-host storage}" + case "$DOKPLOY_TWO_FACTOR_BEFORE" in + /*) ;; + *) echo 'DOKPLOY_TWO_FACTOR_BEFORE must be an absolute path' >&2; exit 1 ;; + esac + two_factor_backup_dir="$(dirname -- "$DOKPLOY_TWO_FACTOR_BEFORE")" + test -d "$two_factor_backup_dir" + test ! -L "$DOKPLOY_TWO_FACTOR_BEFORE" + command -v shred >/dev/null 2>&1 + umask 077 + auth_secret_tmp="$(mktemp "${TMPDIR:-/tmp}/dokploy-auth-secret.XXXXXX")" + two_factor_tmp= + cleanup_migration_tmp() { + [ -z "${auth_secret_tmp:-}" ] || shred -u -- "$auth_secret_tmp" + [ -z "${two_factor_tmp:-}" ] || rm -f -- "$two_factor_tmp" + } + trap cleanup_migration_tmp EXIT + trap 'exit 129' HUP + trap 'exit 130' INT + trap 'exit 143' TERM + if ! DOKPLOY_MIGRATION_ENV="$(sudo docker service inspect dokploy --format '{{range .Spec.TaskTemplate.ContainerSpec.Env}}{{println .}}{{end}}' 2>/dev/null)"; then + echo 'could not inspect the Dokploy service database configuration; refusing authentication-secret migration' >&2 + exit 1 + fi + require_default_database_selector() { + selector="$1" + expected="$2" + count="$(printf '%s\n' "$DOKPLOY_MIGRATION_ENV" | grep -Ec "^${selector}=" || true)" + if [ "$count" -gt 1 ]; then + echo "Dokploy service has duplicate ${selector} settings; refusing authentication-secret migration" >&2 + exit 1 + fi + if [ "$count" -eq 1 ]; then + actual="$(printf '%s\n' "$DOKPLOY_MIGRATION_ENV" | sed -n "s/^${selector}=//p")" + if [ "$actual" != "$expected" ]; then + echo "this automated authentication-secret migration supports only Dokploy's default internal database; ${selector}=${actual} does not match ${expected}, so use a database-operator procedure that snapshots and compare-and-swap restores the same public.two_factor columns on the configured database" >&2 + exit 1 + fi + fi + } + if printf '%s\n' "$DOKPLOY_MIGRATION_ENV" | grep -Eq '^DATABASE_URL='; then + echo 'this automated authentication-secret migration supports only Dokploy with its default internal database; DATABASE_URL is configured, so use a database-operator procedure that snapshots and compare-and-swap restores the same public.two_factor columns on that external database' >&2 + exit 1 + fi + require_default_database_selector POSTGRES_HOST dokploy-postgres + require_default_database_selector POSTGRES_PORT 5432 + require_default_database_selector POSTGRES_USER dokploy + require_default_database_selector POSTGRES_DB dokploy + if [ -e "$DOKPLOY_AUTH_SECRET_BACKUP" ]; then + test -f "$DOKPLOY_AUTH_SECRET_BACKUP" + test "$(stat -c '%a' "$DOKPLOY_AUTH_SECRET_BACKUP")" = 600 + cat -- "$DOKPLOY_AUTH_SECRET_BACKUP" > "$auth_secret_tmp" + else + openssl rand -hex 32 > "$auth_secret_tmp" + (set -C; cat -- "$auth_secret_tmp" > "$DOKPLOY_AUTH_SECRET_BACKUP") + fi + chmod 600 "$auth_secret_tmp" + test "$(wc -c < "$auth_secret_tmp")" -eq 65 + grep -Eq '^[0-9a-f]{64}$' "$auth_secret_tmp" + sync -f "$DOKPLOY_AUTH_SECRET_BACKUP" + cmp -s -- "$auth_secret_tmp" "$DOKPLOY_AUTH_SECRET_BACKUP" + sync -f "$auth_secret_backup_dir" + sudo docker service update --force --update-order stop-first --detach=false dokploy + set -- $(sudo docker ps \ + --filter label=com.docker.swarm.service.name=dokploy \ + --format '{{.ID}}') + test "$#" -eq 1 + set -- $(sudo docker ps \ + --filter label=com.docker.swarm.service.name=dokploy-postgres \ + --format '{{.ID}}') + test "$#" -eq 1 + dokploy_postgres_container="$1" + database_settled=false + settle_attempt=0 + while [ "$settle_attempt" -lt 30 ]; do + active_sessions="$(sudo docker exec "$dokploy_postgres_container" \ + psql -U dokploy -d dokploy -X -v ON_ERROR_STOP=1 -Atc \ + "SELECT count(*) FROM pg_stat_activity WHERE datname = current_database() AND pid <> pg_backend_pid() AND state <> 'idle'")" + if [ "$active_sessions" = 0 ]; then + database_settled=true + break + fi + settle_attempt=$((settle_attempt + 1)) + sleep 1 + done + if [ "$database_settled" != true ]; then + echo 'Dokploy database sessions did not settle after the stop-first task replacement; refusing to snapshot two-factor state' >&2 + exit 1 + fi + two_factor_tmp="$(mktemp "$two_factor_backup_dir/.dokploy-two-factor-before.XXXXXX")" + sudo docker exec "$dokploy_postgres_container" \ + psql -U dokploy -d dokploy -X -v ON_ERROR_STOP=1 \ + -c "COPY (SELECT id, secret, backup_codes FROM public.two_factor ORDER BY id) TO STDOUT WITH (FORMAT csv, HEADER true, NULL '\N', FORCE_QUOTE *)" \ + > "$two_factor_tmp" + chmod 600 "$two_factor_tmp" + sync -f "$two_factor_tmp" + if [ -e "$DOKPLOY_TWO_FACTOR_BEFORE" ]; then + test -f "$DOKPLOY_TWO_FACTOR_BEFORE" + test "$(stat -c '%a' "$DOKPLOY_TWO_FACTOR_BEFORE")" = 600 + cmp -s -- "$two_factor_tmp" "$DOKPLOY_TWO_FACTOR_BEFORE" + rm -f -- "$two_factor_tmp" + else + ln -- "$two_factor_tmp" "$DOKPLOY_TWO_FACTOR_BEFORE" + sync -f "$two_factor_backup_dir" + rm -f -- "$two_factor_tmp" + fi + two_factor_tmp= + sudo docker secret create dokploy-auth-secret - < "$auth_secret_tmp" + sudo docker service update \ + --secret-add source=dokploy-auth-secret,target=/run/secrets/dokploy-auth-secret \ + --update-order stop-first \ + dokploy + set -- $(sudo docker ps \ + --filter label=com.docker.swarm.service.name=dokploy \ + --format '{{.ID}}') + test "$#" -eq 1 + dokploy_container="$1" + sudo docker exec \ + -e OLD_SECRET=better-auth-secret-123456789 \ + "$dokploy_container" \ + sh -c 'cd /app && NEW_SECRET="$(cat /run/secrets/dokploy-auth-secret)" pnpm run migrate-auth-secret' + sudo docker service update \ + --env-add BETTER_AUTH_SECRET_FILE=/run/secrets/dokploy-auth-secret \ + --env-rm BETTER_AUTH_SECRET \ + --update-order stop-first \ + dokploy + shred -u -- "$auth_secret_tmp" + auth_secret_tmp= + trap - HUP INT TERM + trap - EXIT + sudo docker service update --force --update-order stop-first dokploy + sudo docker service ps dokploy --no-trunc + sudo docker service logs --since 10m dokploy +) +``` + +Remove the maintenance fence only after the updated Dokploy task is healthy and +the authentication and stored-value checks below pass. + +Keep the write fence active after any interruption. If neither +`DOKPLOY_TWO_FACTOR_BEFORE` nor the Docker secret exists, the migration did not +reach its post-quiescence snapshot; rerun the exact command under the fence. If +the snapshot exists but the Docker secret does not, rerunning first replaces +and settles the legacy task again and requires the current snapshot to match. +Once the Docker secret exists, do not rerun. Use `docker top` to confirm that no +`migrate-auth-secret` process remains. If the Dokploy service already sets +`BETTER_AUTH_SECRET_FILE`, do not remove the secret or rewrite the database; +verify login, two-factor authentication, and the stored-value checks below. + +If the service still uses the legacy fallback, capture the immediate +post-interruption values to a new `DOKPLOY_TWO_FACTOR_AFTER` path. Then restore +only the two changed columns with a transaction that first proves the current +rows still equal that post-interruption snapshot. This compare-and-swap check +holds a writer-conflicting table lock from validation through restore and +refuses recovery if any two-factor row changed after the snapshot. Keep the +full database backup for disaster recovery; do not restore it for this +interruption. + +```sh +( +set -eu +: "${DOKPLOY_TWO_FACTOR_BEFORE:?retain the exact pre-migration snapshot}" +: "${DOKPLOY_TWO_FACTOR_AFTER:?set a new absolute path on encrypted or off-host storage}" +case "$DOKPLOY_TWO_FACTOR_AFTER" in + /*) ;; + *) echo 'DOKPLOY_TWO_FACTOR_AFTER must be an absolute path' >&2; exit 1 ;; +esac +test -f "$DOKPLOY_TWO_FACTOR_BEFORE" +test "$(stat -c '%a' "$DOKPLOY_TWO_FACTOR_BEFORE")" = 600 +test ! -e "$DOKPLOY_TWO_FACTOR_AFTER" +set -- $(sudo docker ps \ + --filter label=com.docker.swarm.service.name=dokploy-postgres \ + --format '{{.ID}}') +test "$#" -eq 1 +dokploy_postgres_container="$1" +umask 077 +after_tmp="$(mktemp "$(dirname -- "$DOKPLOY_TWO_FACTOR_AFTER")/.dokploy-two-factor-after.XXXXXX")" +cleanup_after_tmp() { rm -f -- "${after_tmp:-}"; } +trap cleanup_after_tmp EXIT +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM +sudo docker exec "$dokploy_postgres_container" \ + psql -U dokploy -d dokploy -X -v ON_ERROR_STOP=1 \ + -c "COPY (SELECT id, secret, backup_codes FROM public.two_factor ORDER BY id) TO STDOUT WITH (FORMAT csv, HEADER true, NULL '\N', FORCE_QUOTE *)" \ + > "$after_tmp" +chmod 600 "$after_tmp" +sync -f "$after_tmp" +ln -- "$after_tmp" "$DOKPLOY_TWO_FACTOR_AFTER" +sync -f "$(dirname -- "$DOKPLOY_TWO_FACTOR_AFTER")" +rm -f -- "$after_tmp" +after_tmp= +trap - HUP INT TERM +trap - EXIT + +( + set -eu + cleanup_two_factor_stage() { + status=$? + trap - EXIT + sudo docker exec "$dokploy_postgres_container" \ + rm -f /tmp/bort-two-factor-before.csv /tmp/bort-two-factor-after.csv || true + exit "$status" + } + trap cleanup_two_factor_stage EXIT + sudo docker exec "$dokploy_postgres_container" \ + sh -eu -c 'test ! -e /tmp/bort-two-factor-before.csv; test ! -e /tmp/bort-two-factor-after.csv' + sudo docker cp "$DOKPLOY_TWO_FACTOR_BEFORE" \ + "$dokploy_postgres_container:/tmp/bort-two-factor-before.csv" + sudo docker cp "$DOKPLOY_TWO_FACTOR_AFTER" \ + "$dokploy_postgres_container:/tmp/bort-two-factor-after.csv" + sudo docker exec -i "$dokploy_postgres_container" \ + psql -U dokploy -d dokploy -X -v ON_ERROR_STOP=1 <<'SQL' +BEGIN; +LOCK TABLE public.two_factor IN SHARE ROW EXCLUSIVE MODE; +CREATE TEMP TABLE bort_two_factor_before AS + SELECT id, secret, backup_codes FROM public.two_factor WITH NO DATA; +CREATE TEMP TABLE bort_two_factor_after AS + SELECT id, secret, backup_codes FROM public.two_factor WITH NO DATA; +\copy bort_two_factor_before FROM '/tmp/bort-two-factor-before.csv' WITH (FORMAT csv, HEADER true, NULL '\N') +\copy bort_two_factor_after FROM '/tmp/bort-two-factor-after.csv' WITH (FORMAT csv, HEADER true, NULL '\N') +DO $$ +BEGIN + IF EXISTS ( + SELECT 1 FROM ( + (SELECT id FROM bort_two_factor_before EXCEPT SELECT id FROM bort_two_factor_after) + UNION ALL + (SELECT id FROM bort_two_factor_after EXCEPT SELECT id FROM bort_two_factor_before) + ) AS id_drift + ) THEN + RAISE EXCEPTION 'two-factor row identities differ between before and after snapshots'; + END IF; + IF EXISTS ( + SELECT 1 FROM ( + (SELECT id, secret, backup_codes FROM public.two_factor + EXCEPT SELECT id, secret, backup_codes FROM bort_two_factor_after) + UNION ALL + (SELECT id, secret, backup_codes FROM bort_two_factor_after + EXCEPT SELECT id, secret, backup_codes FROM public.two_factor) + ) AS current_drift + ) THEN + RAISE EXCEPTION 'current two-factor values changed after the interruption snapshot'; + END IF; +END +$$; +UPDATE public.two_factor AS current +SET secret = original.secret, + backup_codes = original.backup_codes +FROM bort_two_factor_before AS original +WHERE current.id = original.id; +COMMIT; +SQL +) +) +``` + +Capture the three columns again and compare the result byte-for-byte with +`DOKPLOY_TWO_FACTOR_BEFORE`. Only after they match, remove the +`dokploy-auth-secret` mount from the still-legacy service, remove that Docker +secret, and rerun the full migration with both retained pre-migration files. +Never delete or overwrite either retained file during recovery. + +On Dokploy v0.29.12 and newer, the official migration re-encrypts only Better +Auth's two-factor secret and backup-code fields. It does not re-encrypt +application, Compose, project, environment, or database environment values, nor +application build arguments and build secrets. The preserved key remains +necessary to read those existing encrypted values. Before saving or deploying +anything, inspect representative values of each kind in Dokploy and confirm they +render as plaintext rather than `enc:v1:` ciphertext. Also verify login, +two-factor authentication if enabled, service health, and logs. If any check +fails, keep the write fence active and retain the backups unchanged while you +investigate. Do not restore the full PostgreSQL backup in place: unrelated +writes may have committed after it was taken. For selective recovery, restore +the backup to an isolated database, identify the affected rows, and use a +reviewed transaction that verifies current values before updating them. Reserve +an in-place full restore for disaster recovery after accounting for all +post-backup writes. Dokploy v0.29.5 through v0.29.11 keep these configuration +fields as plaintext. + +On v0.29.12 and newer, Dokploy supports `ENCRYPTION_KEY` and +`ENCRYPTION_KEY_FILE` for separating authentication and configuration +encryption. Configure one while the old secret is still active and retain the +old derived key in `/etc/dokploy/encryption.key` during the transition. Do not +use the derived hex key itself as `ENCRYPTION_KEY`; Dokploy derives another key +from that setting. Upgrade v0.29.5 through v0.29.11 before using either setting. + +### Keep the Dokploy API on the local Docker host + +Use the direct loopback HTTP endpoint, such as `http://127.0.0.1:3030`, for +standalone `bort init-target` and every live migration command. Include the +published service port and do not add a proxy path. Bort refuses HTTPS and +non-loopback URLs. Swarm membership cannot prove that an API on one manager +uses the local manager's Docker daemon, so a public or private network address +is not an acceptable same-host identity. This prevents a remote Dokploy API +from being combined with source-container or proxy changes on the local VPS. + ### Kernels without IPVS Dokploy's control-plane services use Docker Swarm VIP networking by default. If the kernel explicitly reports that IPVS is disabled, Bort stops before installing anything. On these hosts, use Dokploy's documented DNS round-robin -mode when starting Bort, for example `sudo ENDPOINT_MODE=dnsrr bort migrate --live` -or `sudo ENDPOINT_MODE=dnsrr bort init-target --install`. +mode when starting Bort, for example: + +```sh +sudo ENDPOINT_MODE=dnsrr bort migrate --live +sudo bort init-target --install --endpoint-mode dnsrr --auth-secret-backup /absolute/path/to/encrypted-or-off-host/dokploy-auth-secret +``` This option applies to the three control-plane services that Bort creates, not to existing services. Bort refuses a conflicting existing endpoint mode; review @@ -67,12 +578,76 @@ switching web traffic, stopping source containers, and removing leftovers need Docker and protected-file access, so this guide consistently uses `sudo`. Bort's side-by-side Dokploy installer also requires root. Running without `sudo` is only appropriate when Dokploy is already prepared and the regular account has every -required Docker and source-path permission. Choose before the first command and -do not switch users in one workspace. +required Docker and source-path permission. Before the first non-root command, +provision Bort's host-wide operation state for that account: + +```sh +sudo install -d -m 700 -o "$(id -u)" -g "$(id -g)" /var/lib/bort +``` + +Choose before the first command and do not switch users in one workspace or for +the `/var/lib/bort` owner. The `.bort` directory contains credentials and application configuration. Bort uses private file permissions, but you should still keep the directory out of -source control, support tickets, and shared archives. +source control, support tickets, and shared archives. `/var/lib/bort` contains a +transient operation lock, the fresh-install authentication-secret provenance +marker, and `dokploy-traffic-owner.json`, the durable record +that gives one migration run host-wide authority over Dokploy changes. Bort +claims it before the first live target mutation, including project or service +creation in a plan with no routes, and retains it until the run is committed or +rolled back. The record binds a one-way identifier for the organization-scoped +Dokploy API key; it never stores the key itself. Do not rotate or replace that +key during a migration or before metadata `cleanup --apply`. Preserve the owner +file with `.bort`, at the same path and under the same OS identity, through any +metadata cleanup you intend to apply. Do not restore one location without the +other. +The status screen reports `HOST OWNED`, the owning run, and its recorded +authority when a different run holds the host. New owner records also show the +absolute run path; older records may have only the run name. Open that run path +or return to its workspace and finish its commit or rollback; do not delete or +rewrite the owner file to bypass it. + +A local scan records the Docker Engine ID and the exact source container IDs and +names in the reviewed run. Live apply, interrupted source cleanup, rollback, +and commit re-read the system Docker socket and refuse mutation unless all of +those identities still match. Those checks catch a replaced daemon or container; +they do not re-verify every detail of the reviewed configuration. Unstarted runs +from imported manifests, imported bundles (`--bundle`, including bundles produced +by `bort export`), remote scans, or older Bort versions remain available for +inspection and planning only; replace them with a new local scan on the source +host before live work. +A run that already performed live work is immutable and cannot be refreshed. +If such a run lacks source attestation, Bort cannot safely choose containers to +restart, stop, retire, or purge: verify Docker Engine and container identities +manually, recover traffic and writers or retire the reviewed sources outside +Bort, and preserve the run and any host-owner record as audit evidence. macOS +and Windows inspection preserves the stored lifecycle phase and shows the +state-specific Linux handoff, but cannot perform live recovery or mutation. + +If a current run reports `AUTHORITY UNKNOWN` while its matching owner record is +still present, do not abandon the run or delete the owner. Inspect both sides, +manually establish source or target authority, then run the exact +`recover-authority` command shown by `bort status`. Bort binds the decision to +the owning run under both operation locks. A later retry finishes any +interrupted metadata or owner transition. + +If `dokploy-traffic-owner.json` is lost after any target mutation, current runs +fail closed: Bort cannot prove that another workspace did not change the target +or shared proxy. Preserve both source and target data, inspect target projects, +environments, Compose resources, the Coolify and Dokploy proxies, and routes, +establish writer and traffic authority manually, then start a fresh migration +run. Do not recreate the owner file by hand. + +Completed routed runs from before Bort persisted the immutable Dokploy origin +also appear as `AUTHORITY UNKNOWN`. Current credentials do not prove that they +address the original target, so automatic retry, commit, and rollback remain +disabled even when the old ledger says the run completed. Do not rebind the run +to the currently configured URL. Keep both copies of the data, identify the +original target from independent operator records, inspect its workloads and +routes alongside the source, and establish writer and traffic authority +manually. Retire or restore resources only after that inspection, then create a +fresh migration run for any remaining automated work. ## Start or resume @@ -104,8 +679,22 @@ other than the workspace default. The guided screen shows whether each application is ready, what must be fixed, and the next safe action. Follow the generated `fix:` commands and rerun Bort -until it shows `READY`. This means the inputs that block live apply are resolved; -it does not clear non-blocking review items. +until it shows `READY` or reports `MANUAL STATE`. `READY` means the inputs that +block live apply are resolved; it does not clear non-blocking review items. +`MANUAL STATE` means a historical stateful run was applied with an older plan +version that copied state into an already deployed target; Bort cannot continue +that run. Follow the displayed manual recovery guidance or create a new run. +`PLAN BLOCKED` means live apply would refuse the planned state transfer before +touching the target (for example a bind mount, a named volume shared by two +apps, or a compose file that cannot be staged). When the blocked volume belongs +to a data store, choose `bort data --recreate` or `--managed`, +then re-plan with `bort migrate --run `. Otherwise change the source +compose and scan a new run. Bort checks this before binding the run to a +Dokploy target, so the run stays editable. `NEW RUN REQUIRED` means live apply +refused this run at `pause_source` (for example a Postgres data directory that +only the created staged container revealed to be off its named volume) and the +run cannot be re-planned; follow the recovery `bort status` shows, then create +a new run. Use the guided screen for secret values so they are not placed in command arguments, shell history, or process and sudo audit records. The direct command @@ -134,12 +723,116 @@ a plan. Once live execution begins, that plan cannot be changed. Create a new run if the plan itself must change. If live apply is interrupted, Bort saves its progress. Rerun the same command to -resume: +resume when the saved boundary is safe: ```sh sudo bort migrate --live ``` +For an app with named volumes or a Postgres database, live apply moves the +state before the Dokploy compose is deployed. Dokploy v0.30.7 cannot durably +prevent a queued or future deployment from restarting a target writer, so Bort +never copies data into a deployed target. Instead it: + +1. Creates the Dokploy project, compose service, and environment, but does not + deploy. +2. Stops the reviewed source app containers. Logical-dump databases keep + running for `pg_dump`; volume-strategy stores stop with the app. +3. Copies each named volume into a Docker volume that Bort creates and labels + with the run name (`bort----`). It refuses a volume + with that name that another run owns. +4. Restores each Postgres dump by starting only the database service under a + temporary compose project (`bort-stage---`) with the + staging volume mounted, running `pg_restore`, and stopping it again. The + project files live under `.bort/runs//stage/`. +5. Re-inspects the source containers after each volume copy and refuses the + transfer if any started or ran during it. After every copy or restore it + also refuses if any container is still attached to the staging volume. +6. Deploys the compose with each transferred volume declared as + `external: true` pointing at the staging volume, then verifies that the + running target mounts those exact volumes and stops the target if it does + not. + +Routed apps stay stopped from step 2 until the proxy handoff moves traffic; +plan for that downtime. Unrouted apps restart the source after step 5, so the +source and target then run with diverging copies. + +A stateful app whose plan needs a bind-mount copy is refused. Same-host bind +mounts that keep their existing paths need no transfer. Do not remove real +storage from an app's manifest merely to bypass a refusal. + +A named volume mounted by more than one compose service cannot be staged, +because each service mount would become a separate staging volume. Bort refuses +it before changing Dokploy or stopping the source. When the volume belongs to a +data store, choose `bort data --recreate` or `--managed` and +re-plan the run with `bort migrate --run `. Otherwise change the source +compose so one service mounts the volume, then scan a new run. + +The temporary compose project runs with only `PATH` and `HOME`, as Dokploy +deploys do, and writes its `.env` in the format the running Dokploy release +uses: raw values before v0.30.0, every `$` escaped through v0.30.2, and +`${VAR}` references kept from v0.30.3. Bort reads the version from +`settings.getDokployVersion` before pausing the source and refuses staged +restores when it is not a `vMAJOR.MINOR.PATCH` release. Because the staged +service runs under a Bort project name, a compose file or `.env` that +interpolates `COMPOSE_PROJECT_NAME` for a staged data store is refused before +anything changes; reference the app name another way and re-plan the run. A +Dokploy compose app that has `.env` creation disabled or a custom deploy +command is refused before the source pauses, because its deploy would not +interpolate the compose file the way the staged restore did. The check runs +again right before each deploy of that app, so changing those settings in +Dokploy between a completed restore and a resumed deploy also refuses; restore +them and resume. + +The staged service mounts only its staging volumes plus any bind mounts under +`/docker-entrypoint-initdb.d`, read-only, so the restore runs the same init +scripts the deploy will. Those mounts must use absolute host paths; a relative +one is refused before the source pauses. Scripts that write into that +directory, or that read other bind mounts such as a `postgresql.conf`, do not +run the same way in staging; choose `--recreate` or `--managed` for such +stores. The restore waits for the final server to accept connections on +`127.0.0.1` inside the container, so the entrypoint's socket-only init server, +which runs those scripts, never receives the dump. A custom `listen_addresses` +must include that address; the official image's `*` default already does. + +A Postgres store's data directory (`PGDATA`; the official image defaults to +`/var/lib/postgresql/data` through Postgres 17 and to +`/var/lib/postgresql//docker` from 18, where the named volume normally +mounts at `/var/lib/postgresql`) must sit on a named volume that the service +mounts, with no writable bind mount inside it; otherwise the restore +would be lost when the staged container stops, or shadowed by the mount once +Dokploy starts the service. When the store service mounts no named volume, +sets `PGDATA` in its compose `environment` to a literal path that breaks this +rule, or interpolates a mount target, Bort refuses before live apply starts +(`PLAN BLOCKED`): choose `bort data --recreate` or `--managed` +and re-plan with `bort migrate --run `, or change the source compose and +scan a new run. When the image or interpolation decides `PGDATA`, or the run had already started live +execution before Bort learned to check the layout, Bort creates the staged +container without starting it right before `pause_source` and refuses then +(`NEW RUN REQUIRED`). That app's source keeps running and none of +its state was transferred, but the run has already started live execution and +cannot be re-planned. (A run that had already completed that app's +`pause_source` resumes past the check and refuses at the restore step instead; +Bort restarts that app's source during cleanup and `bort status` offers to +resume, and that resume stops at `pause_source` with the refusal described +here.) When no other app's source was paused or handed off, delete or +reconcile the Dokploy resources the run created, run the exact +`recover-authority --authority source` command `bort status` shows to release +host ownership, then choose a strategy or change the source compose and create +a new run. When an earlier app's source may already be paused or handed off, +`bort status` shows the manual path instead: inspect and preserve both sides, +restore the earlier apps' source writers and traffic manually, release host +ownership with the `recover-authority --authority source` command it shows, +then create a new run; target authority is unavailable because the refused +app has no target copy. + +Runs whose plan was created by an older Bort version copied state into an +already deployed target. Live apply refuses those runs (`MANUAL STATE`). If +such a run already started a volume copy or target database restore, it remains +fail-closed: Bort cannot prove whether the target later accepted unique writes, +so it refuses automatic retry or rollback rather than overwrite either side. +Follow the authority-recovery guidance below. + If another process is already applying the same run, a second `migrate --live` command joins it and shows its progress. You can check status separately: @@ -172,34 +865,112 @@ You can inspect the stored rollback plan with: sudo bort rollback ``` -After a successful live apply, if the target does not check out, return traffic -to the source: +The text and JSON inspection output state whether automatic rollback is +available and give the exact blocker when it is not. Treat unavailable as a +recovery boundary, not as a prompt to force the command. + +After a successful stateless live apply, if the target does not check out, +return traffic to the source: ```sh sudo bort rollback --live ``` -The command restarts the source containers that live apply stopped, verifies -they are running (and healthy, when Docker tracks a healthcheck) before -touching any traffic, returns web traffic to the source by stopping the Dokploy -proxy and starting the source proxy, and re-checks the result after a five-second -settle per observed app. This does not enforce the observation window displayed -by `bort rollback`; continue monitoring the source for that reviewed period. -Every step inspects current state first, so an interrupted rollback can be -re-run. Once rollback starts, commit and live apply remain blocked, even if -rollback fails. Use `bort rollback --live` again to finish recovery. - -Rollback requires confirming the recorded rollback trigger first. If you skipped -that guided review before applying, inspect `bort rollback --run `, then -use `bort rollback --live --run --confirm 'rollback '` with the -actual run name. The command records that confirmation before starting recovery. - -Rollback never deletes resources: the Dokploy target resources remain on the -server for later cleanup, and data written to the target after cutover is not -copied back to the source. To migrate again, create a new named run with +The command is available only when the run transferred no database or volume +state. In that case the source applications remained running, and rollback +stops the Dokploy proxy and starts the Coolify proxy. An interrupted stateless +rollback can be re-run. + +An interrupted stateful run with the current plan version resumes from its +saved progress like a stateless run. For a run applied with an older plan +version that stopped a source app, if its ledger proves that no database dump, +restore, volume copy, or traffic handoff may have started, re-running +`migrate --live` only restarts the source and records that cleanup; the run +itself remains refused. + +For a historical stateful run that already completed, `rollback --live` also +refuses before changing progress, Docker, or Dokploy. Dokploy v0.30.7 has no +durable application fence: +`compose.stop` does not cancel a queued deployment or block a future manual +deployment. Preserve both sides and establish writer and traffic authority +manually. Do not infer authority from a short period with no running target +containers. + +Recovery from an interrupted run whose writer authority is ambiguous is also +manual. In current runs this means a Dokploy target mutation (project or +service creation, env upload, compose deploy, gateway install, or route +activation) was interrupted or returned an ambiguous result. In ledgers written +by older Bort versions it also covers an in-place volume copy, database restore, +or handoff whose boundary cannot be proven. Bort refuses retry without mutating +either side. Staged copies and restores in current runs are not affected: rerun +`migrate --live` and the transfer restarts. + +When the durable Dokploy owner still identifies the same run, finish manual +recovery through that run: + +- To return to the source, first stop and fence every target writer, reverse + target traffic, and verify that the source is healthy and authoritative. Then + run: + + ```sh + sudo bort recover-authority --run --authority source \ + --confirm 'recover as source' + ``` + + Bort records the resolution and rollback lifecycle, changes the owner to + source authority, and releases host ownership. Target data is not copied back. + +- To retain the target, first stop and fence every source writer and verify the + target data and traffic. Then run: + + ```sh + sudo bort recover-authority --run --authority target \ + --confirm 'recover as target' + ``` + + When source attestation still passes, Bort records target authority. For a + source without an external orchestrator, `commit --apply` can then retire the + exact reviewed containers. For Coolify, disable future deployments for the + reviewed apps and manually retire every reviewed source app (and the source + proxy when the cutover moved routes) instead; `commit --apply` refuses + because Bort cannot durably fence Coolify. If the + reviewed source daemon or containers are no longer attestable, the recovery + command also refuses without recording the authority decision. In either + manual-retirement case, run the second command Bort prints with + `--source-retired` only after verifying the source remains retired. Its + stronger confirmation marks the run committed and releases host ownership + without mutating source resources. Automatic rollback remains unavailable + because Bort cannot verify the manual boundary. + +`recover-authority` records work that you already completed and verified. It +does not stop containers, move data, or switch traffic. It refuses a missing or +different owner and a mismatched confirmation phrase. The plain +`--authority source` and `--authority target` forms also refuse a run that has +no authority ambiguity, no unavailable automatic rollback, and no unreleased +host ownership after a definite live-apply failure. `--authority target +--source-retired` records permanent target acceptance after any successful live +apply, including stateless runs and runs whose plan never needed host +ownership. If a plan needed host ownership and the owner record is missing or +belongs to another run, keep both sides and complete recovery manually; start a +fresh run only after the old resources and traffic are safe. + +Stateless rollback does not enforce the observation window displayed by +`bort rollback`; continue monitoring the source for that reviewed period. Once +rollback starts, commit and live apply remain blocked, even if rollback fails. +Use `bort rollback --live` again to finish a stateless rollback. + +Stateless rollback requires confirming the recorded rollback trigger first. If +you skipped that guided review before applying, inspect +`bort rollback --run `, then use +`bort rollback --live --run --confirm 'rollback '` with the +actual run name. The command records that confirmation before changing traffic. + +Stateless rollback never deletes resources: the Dokploy target resources remain +on the server for later cleanup. To migrate again, create a new named run with `bort migrate --run ` and current `--source` or `--bundle` inputs. -Accepting the target retires source application containers: +For a source without an external orchestrator, accepting the target retires the +exact reviewed source application containers: ```sh sudo bort commit --apply @@ -210,6 +981,14 @@ the target and waiting through the planned rollback window. Once source retirement starts, automated rollback is no longer available, even if commit is interrupted; rerun `commit --apply` to finish acceptance. +For a Coolify source, `commit --apply` refuses before source mutation because a +queued or future deployment could replace a stopped container. Disable future +deployments in Coolify, manually retire every reviewed source app and the source +proxy, and verify they remain retired. Then run the exact +`recover-authority --authority target --source-retired --confirm ...` command +Bort prints. That command records permanent target acceptance without mutating +the source. + ## Audit and clean up After acceptance, list the remaining Dokploy records and source resources: @@ -226,11 +1005,19 @@ Ordinary cleanup and destructive source purge are separate operations. See the | Situation | Safe next action | | --- | --- | | Bort cannot find the expected run | Return to the original working directory and original OS user. Do not create a replacement workspace accidentally. | -| Live apply was interrupted | Run `sudo bort status`, then rerun `sudo bort migrate --live` to resume from the saved progress. | +| Stateless live apply was interrupted at a safe completed boundary | Run `sudo bort status`, then rerun `sudo bort migrate --live` to resume from the saved progress. | +| Live apply reports `MANUAL STATE` for a stateful run | The run was applied with an older plan version that copied state into a deployed target. The blocked run cannot continue. Complete recovery with the guidance shown by `bort status`, then create a new run. | +| A staged state transfer fails (source restarted, foreign-owned volume, attached staging volume) | That app's target was not deployed. Fix the reported cause and rerun `sudo bort migrate --live`; the transfer restarts from the beginning for that volume or database. | +| A stateful plan needs a bind-mount copy | Bort refuses before pausing the source. Re-plan with `bort migrate --run `; current plans keep same-host bind mounts at their existing paths, and the run stays editable because nothing live has started. | +| A named volume is mounted by more than one compose service | Bort refuses before changing Dokploy or pausing the source. For a data store volume, choose `bort data --recreate` or `--managed` and re-plan with `bort migrate --run `; otherwise change the source compose so one service mounts the volume and scan a new run. | +| A Postgres data directory is not on a named volume the service mounts, or a writable bind mount sits inside it | When the service mounts no named volume, sets `PGDATA` in its compose `environment` to a literal path that breaks this rule, or interpolates a mount target, Bort refuses before live apply (`PLAN BLOCKED`): choose `bort data --recreate` or `--managed` and re-plan with `bort migrate --run `, or change the source compose and scan a new run. When the image or interpolation decides `PGDATA`, only the created staged container reveals the directory, so Bort refuses at `pause_source` (`NEW RUN REQUIRED`) with that app's source still running and none of its state transferred; the run cannot be re-planned. If no other app's source was paused or handed off, delete or reconcile the Dokploy resources the run created, run the exact `recover-authority --authority source` command `bort status` shows to release host ownership, then choose a strategy or change the source compose and create a new run; otherwise restore the earlier apps' source writers and traffic manually, release host ownership with the source-authority command `bort status` shows, then create a new run. | +| A Dokploy target mutation (project or service creation, env upload, compose deploy, gateway install, route activation) was interrupted or ambiguous, or an older-plan ledger cannot prove writer authority after an in-place copy, restore, or handoff | Bort refuses automatic retry and rollback because Dokploy cannot durably fence the target. Inspect and preserve both sides. If the matching owner remains, establish authority manually and run the exact `recover-authority` command shown by `bort status`; otherwise complete recovery manually before starting a fresh run. | +| A deployed target fails its migrated-volume check and Bort cannot stop its containers | Bort leaves the paused source stopped and refuses automatic retry, because target containers may still be writing to the transferred state. Stop the target containers yourself, then run the exact `recover-authority` command shown by `bort status`. | | Another change is running | Keep `status` open if useful and wait. A second live command joins an active live apply; other commands that make changes must wait. | | The plan needs to change after live execution began | Keep the existing run as a record and create a new named run. A plan cannot change after live work starts. | -| Target validation fails after successful live apply, before acceptance | Run `sudo bort rollback --live` to restart and verify source containers and return traffic to the source. Target resources remain for cleanup; data written to the target after cutover is not copied back. | -| Rollback stops partway through | Inspect `sudo bort status`, then rerun `sudo bort rollback --live`. Commit and live apply are blocked because traffic may already have returned to the source. | +| A stateless target fails validation after successful live apply | Run `sudo bort rollback --live` to return traffic to the source. Target resources remain for cleanup. | +| A stateful target fails validation after successful live apply | Bort refuses automatic rollback because Dokploy cannot durably fence target writers. Preserve both sides, manually restore and verify source authority, then use the `recover-authority --authority source` command shown by `bort status`. | +| Stateless rollback stops partway through | Inspect `sudo bort status`, then rerun `sudo bort rollback --live`. Commit and live apply are blocked because traffic may already have returned to the source. | | Cleanup or purge stops partway through | Stop and inspect the command output and private backup before retrying. Follow the recovery guidance in the cleanup and purge guide. | | `.bort` cannot be read | Do not change ownership or permissions blindly. Confirm the original working directory and OS user first, then restore the workspace from backup if necessary. | @@ -242,6 +1029,6 @@ Ordinary cleanup and destructive source purge are separate operations. See the | App setup | Exports Compose or image-based app details and prepares Dokploy projects and compose applications. Unsupported Compose features remain review items. | | Environment | Captures application values, removes Coolify-specific values that should not be copied, and keeps generated files private. | | Routes | Plans supported routes with health checks, a waiting period, and rollback information. Missing or unsupported proxy details block the migration or require review. | -| Databases and persistent data | Supports local database dump/restore and copying while a service is stopped. Continuous syncing and other replication methods are not implemented. | +| Databases and persistent data | Dumps Postgres databases and copies named volumes into Bort-owned staging volumes before the target deploys, then hands those volumes to the Dokploy compose. Bind-mount copies are not implemented. | | Named volumes and bind mounts | Lists them and asks how they should be handled when manual review is required. Source purge never automatically deletes named volumes or host paths. | | Source control | Records repository details and deploy-key hints, but does not copy, revoke, or recreate GitHub Apps, deploy keys, webhooks, or equivalent credentials. | diff --git a/internal/cli/applied.go b/internal/cli/applied.go index c052905..657be3e 100644 --- a/internal/cli/applied.go +++ b/internal/cli/applied.go @@ -13,36 +13,50 @@ import ( "github.com/aikins01/bort/internal/target/dokploy" ) -const appliedAPIVersion = "bort.applied/v1alpha1" +const ( + appliedAPIVersion = "bort.applied/v1alpha2" + appliedLegacyAPIVersion = "bort.applied/v1alpha1" + appliedRecoveryProtocol = "target-authority-v1" + appliedPlanV1Alpha1 = "v1alpha1" + appliedPlanV1Alpha2 = "v1alpha2" + appliedPlanV1Alpha3 = "v1alpha3" + appliedPlanCurrent = appliedPlanV1Alpha3 +) // runApplied is the per-run audit ledger. it captures the outcome of every // step the live applier walks through so a partial run can be inspected and // safely retried. type runApplied struct { - APIVersion string `json:"apiVersion"` - RunName string `json:"runName"` - BundleDir string `json:"bundleDir,omitempty"` - Target string `json:"target,omitempty"` - UpdatedAt time.Time `json:"updatedAt"` - SucceededAt *time.Time `json:"succeededAt,omitempty"` - Steps []appliedStep `json:"steps,omitempty"` - Apps map[string]appliedApp `json:"apps,omitempty"` + APIVersion string `json:"apiVersion"` + RunName string `json:"runName"` + BundleDir string `json:"bundleDir,omitempty"` + Target string `json:"target,omitempty"` + TargetOrigin string `json:"targetOrigin,omitempty"` + UpdatedAt time.Time `json:"updatedAt"` + SucceededAt *time.Time `json:"succeededAt,omitempty"` + RecoveryProtocol string `json:"recoveryProtocol,omitempty"` + PlanVersion string `json:"planVersion,omitempty"` + Steps []appliedStep `json:"steps,omitempty"` + Apps map[string]appliedApp `json:"apps,omitempty"` } type appliedStep struct { - Index int `json:"index"` - Kind string `json:"kind"` - App string `json:"app"` - Ref string `json:"ref"` - Status string `json:"status"` - UpdatedAt time.Time `json:"updatedAt"` - Error string `json:"error,omitempty"` + Index int `json:"index"` + Kind string `json:"kind"` + App string `json:"app"` + Ref string `json:"ref"` + Status string `json:"status"` + UpdatedAt time.Time `json:"updatedAt"` + Error string `json:"error,omitempty"` + MutationAmbiguous *bool `json:"mutationAmbiguous,omitempty"` + RequiresNewRun bool `json:"requiresNewRun,omitempty"` } type appliedApp struct { - ProjectID string `json:"projectId,omitempty"` - EnvironmentID string `json:"environmentId,omitempty"` - ComposeID string `json:"composeId,omitempty"` + ProjectID string `json:"projectId,omitempty"` + EnvironmentID string `json:"environmentId,omitempty"` + ComposeID string `json:"composeId,omitempty"` + ComposeAppName string `json:"composeAppName,omitempty"` } func readRunApplied(path string, run migrationRun) (runApplied, error) { @@ -60,8 +74,13 @@ func readRunApplied(path string, run migrationRun) (runApplied, error) { if err := json.Unmarshal(contents, &applied); err != nil { return runApplied{}, fmt.Errorf("read %s: %w", path, err) } - if applied.APIVersion != "" && applied.APIVersion != appliedAPIVersion { - return runApplied{}, fmt.Errorf("%s has unsupported apiVersion %q (want %q)", path, applied.APIVersion, appliedAPIVersion) + if applied.APIVersion != "" && applied.APIVersion != appliedAPIVersion && applied.APIVersion != appliedLegacyAPIVersion { + return runApplied{}, fmt.Errorf("%s has unsupported apiVersion %q (want %q or %q)", path, applied.APIVersion, appliedAPIVersion, appliedLegacyAPIVersion) + } + switch applied.PlanVersion { + case "", appliedPlanV1Alpha1, appliedPlanV1Alpha2, appliedPlanV1Alpha3: + default: + return runApplied{}, fmt.Errorf("%s has unsupported planVersion %q", path, applied.PlanVersion) } if err := validateRunAppliedIdentity(path, applied, run); err != nil { return runApplied{}, err @@ -87,23 +106,36 @@ func validateRunAppliedIdentity(path string, applied runApplied, run migrationRu func newRunApplied(run migrationRun) runApplied { return runApplied{ - APIVersion: appliedAPIVersion, - RunName: run.Name, - BundleDir: run.BundleDir, - Target: run.Target, - UpdatedAt: time.Now().UTC(), - Apps: map[string]appliedApp{}, + APIVersion: appliedAPIVersion, + RunName: run.Name, + BundleDir: run.BundleDir, + Target: run.Target, + UpdatedAt: time.Now().UTC(), + RecoveryProtocol: appliedRecoveryProtocol, + PlanVersion: appliedPlanCurrent, + Apps: map[string]appliedApp{}, } } func writeRunApplied(path string, applied runApplied) error { - if applied.APIVersion == "" { - applied.APIVersion = appliedAPIVersion - } + applied = upgradeRunAppliedSchema(applied) applied.UpdatedAt = time.Now().UTC() return writeJSONArtifact(path, applied) } +func appliedHasHistory(applied runApplied) bool { + return len(applied.Steps) > 0 || len(applied.Apps) > 0 || applied.SucceededAt != nil +} + +// upgradeRunAppliedSchema pins a ledger with history to the plan order it +// was recorded against; step indexes only line up under that order. an +// empty ledger has nothing to preserve and moves to the current order. +func upgradeRunAppliedSchema(applied runApplied) runApplied { + applied.PlanVersion = appliedPlanVersion(applied) + applied.APIVersion = appliedAPIVersion + return applied +} + func recordAppliedStep(applied runApplied, progress dokploy.StepProgress) runApplied { step := appliedStep{ Index: progress.Index, @@ -115,6 +147,20 @@ func recordAppliedStep(applied runApplied, progress dokploy.StepProgress) runApp } if progress.Err != nil { step.Error = progress.Err.Error() + ambiguous := progress.MutationAmbiguous + step.MutationAmbiguous = &ambiguous + step.RequiresNewRun = progress.RequiresNewRun + } + if progress.Step.App != "" && progress.Target != nil { + if applied.Apps == nil { + applied.Apps = map[string]appliedApp{} + } + app := applied.Apps[progress.Step.App] + app.ProjectID = progress.Target.ProjectID + app.EnvironmentID = progress.Target.EnvironmentID + app.ComposeID = progress.Target.ComposeID + app.ComposeAppName = progress.Target.ComposeAppName + applied.Apps[progress.Step.App] = app } for i := range applied.Steps { if applied.Steps[i].Index == progress.Index { @@ -151,7 +197,7 @@ func (l *appliedLedger) Record(progress dokploy.StepProgress) error { if l.err != nil { return l.err } - l.state = recordAppliedStep(l.state, progress) + l.state = upgradeRunAppliedSchema(recordAppliedStep(l.state, progress)) if err := l.persist(l.path, l.state); err != nil { l.err = err return err @@ -159,6 +205,36 @@ func (l *appliedLedger) Record(progress dokploy.StepProgress) error { return nil } +func (l *appliedLedger) PrepareRetry(index int) error { + l.mu.Lock() + defer l.mu.Unlock() + if l.err != nil { + return l.err + } + next := l.state + next.Steps = make([]appliedStep, 0, len(l.state.Steps)) + for _, step := range l.state.Steps { + if step.Index < index { + next.Steps = append(next.Steps, step) + } + } + if index == 0 && !appliedHasHistory(next) { + next.PlanVersion = appliedPlanCurrent + } + if len(next.Steps) == len(l.state.Steps) && next.RecoveryProtocol == appliedRecoveryProtocol && next.APIVersion == appliedAPIVersion && next.PlanVersion == l.state.PlanVersion { + return nil + } + next.SucceededAt = nil + next.RecoveryProtocol = appliedRecoveryProtocol + next = upgradeRunAppliedSchema(next) + if err := l.persist(l.path, next); err != nil { + l.err = err + return err + } + l.state = next + return nil +} + func (l *appliedLedger) Err() error { l.mu.Lock() defer l.mu.Unlock() @@ -173,6 +249,7 @@ func (l *appliedLedger) MarkSucceeded() error { } now := time.Now().UTC() l.state.SucceededAt = &now + l.state = upgradeRunAppliedSchema(l.state) if err := l.persist(l.path, l.state); err != nil { l.err = err return err @@ -192,8 +269,266 @@ func (l *appliedLedger) Snapshot() runApplied { return clone } +func (l *appliedLedger) BindTargetOrigin(origin string) error { + l.mu.Lock() + defer l.mu.Unlock() + if l.err != nil { + return l.err + } + if err := validateAppliedTargetOrigin(l.state, origin); err != nil { + return err + } + if l.state.TargetOrigin == origin && l.state.APIVersion == appliedAPIVersion { + return nil + } + next := upgradeRunAppliedSchema(l.state) + next.TargetOrigin = origin + if err := l.persist(l.path, next); err != nil { + l.err = err + return err + } + l.state = next + return nil +} + +func (l *appliedLedger) ValidateTargetOrigin(origin string) error { + l.mu.Lock() + defer l.mu.Unlock() + if l.err != nil { + return l.err + } + return validateAppliedTargetOrigin(l.state, origin) +} + +func validateAppliedTargetOrigin(applied runApplied, origin string) error { + if applied.TargetOrigin != "" && applied.TargetOrigin != origin { + return fmt.Errorf("migration run is bound to Dokploy origin %s, not %s", applied.TargetOrigin, origin) + } + if applied.TargetOrigin == "" && (len(applied.Steps) > 0 || len(applied.Apps) > 0 || applied.SucceededAt != nil) { + return fmt.Errorf("migration run has prior live execution but no persisted Dokploy origin; refusing to bind it to %s because the original target cannot be proved; preserve the existing target and start a fresh migration run", origin) + } + return nil +} + func completedApplyPrefix(steps []dokploy.Step, applied runApplied) int { - return indexedCompletedApplyPrefix(steps, applied) + prefix := indexedCompletedApplyPrefix(steps, applied) + if prefix >= len(steps) { + return prefix + } + if applyMayHaveAmbiguousAuthority(applied) { + return prefix + } + if applied.RecoveryProtocol == appliedRecoveryProtocol { + return prefix + } + pauseByApp := map[string]int{} + transferredApps := map[string]struct{}{} + for index, step := range steps[:prefix] { + switch { + case step.Kind == dokploy.StepPauseSource: + pauseByApp[step.App] = index + delete(transferredApps, step.App) + case step.Kind == dokploy.StepDumpDataStore || step.Kind == dokploy.StepRestoreDataStore || step.Kind == dokploy.StepSyncVolume: + if _, paused := pauseByApp[step.App]; paused { + transferredApps[step.App] = struct{}{} + } + case step.Kind == dokploy.StepResumeSource: + delete(pauseByApp, step.App) + delete(transferredApps, step.App) + } + } + for app := range transferredApps { + if pauseIndex := pauseByApp[app]; pauseIndex < prefix { + prefix = pauseIndex + } + } + return prefix +} + +// appliedPlanVersion resolves the plan order a ledger's step indexes were +// recorded against. a ledger without history has nothing pinned and +// follows the current order. +func appliedPlanVersion(applied runApplied) string { + switch { + case !appliedHasHistory(applied): + return appliedPlanCurrent + case applied.APIVersion == appliedLegacyAPIVersion || applied.PlanVersion == appliedPlanV1Alpha1: + return appliedPlanV1Alpha1 + case applied.PlanVersion == "" && applied.APIVersion == appliedAPIVersion: + return appliedPlanV1Alpha2 + case applied.PlanVersion == "": + return appliedPlanCurrent + default: + return applied.PlanVersion + } +} + +func livePlanForApplied(run loadedMigrationRun, applied runApplied) dokploy.Plan { + switch appliedPlanVersion(applied) { + case appliedPlanV1Alpha1: + return dokploy.LegacyPlanFromArtifactsV1Alpha1(run.Prepare, run.Sync, run.Cutover) + case appliedPlanV1Alpha2: + return dokploy.PlanFromArtifactsV1Alpha2(run.Prepare, run.Sync, run.Cutover) + default: + return dokploy.PlanFromArtifacts(run.Prepare, run.Sync, run.Cutover) + } +} + +// appliedPlanTransfersStateInPlace reports whether the ledger is pinned to +// a plan order that deploys the target before copying state into its +// volumes. Bort no longer continues such runs past the stateful steps. +func appliedPlanTransfersStateInPlace(applied runApplied) bool { + switch appliedPlanVersion(applied) { + case appliedPlanV1Alpha1, appliedPlanV1Alpha2: + return true + default: + return false + } +} + +type applyAuthorityAmbiguity int + +const ( + applyAuthorityUnambiguous applyAuthorityAmbiguity = iota + applyTrafficAuthorityAmbiguous + applyWriterAuthorityAmbiguous +) + +func appliedAuthorityAmbiguity(applied runApplied) applyAuthorityAmbiguity { + if applied.SucceededAt != nil { + return applyAuthorityUnambiguous + } + if applied.APIVersion == appliedAPIVersion && applied.RecoveryProtocol == appliedRecoveryProtocol { + for _, recorded := range applied.Steps { + if appliedStepMutatesDokploy(recorded.Kind) && + appliedTargetMutationAmbiguous(recorded) { + return applyWriterAuthorityAmbiguous + } + } + return applyAuthorityUnambiguous + } + ambiguity := applyAuthorityUnambiguous + for _, recorded := range applied.Steps { + if recorded.Kind == string(dokploy.StepStartDokployProxy) && recorded.Ref == "dokploy-traefik" && + appliedStepMayHaveRun(recorded) { + ambiguity = applyTrafficAuthorityAmbiguous + } + if (recorded.Kind == string(dokploy.StepResumeTarget) || recorded.Kind == string(dokploy.StepActivateRoutes)) && + recorded.App != "" && appliedStepMayHaveRun(recorded) { + return applyWriterAuthorityAmbiguous + } + if (recorded.Kind == string(dokploy.StepSyncVolume) || recorded.Kind == string(dokploy.StepRestoreDataStore)) && + appliedStepMayHaveRun(recorded) { + return applyWriterAuthorityAmbiguous + } + if recorded.Kind == string(dokploy.StepPushImage) && appliedStepInterrupted(recorded) { + return applyWriterAuthorityAmbiguous + } + } + return ambiguity +} + +func appliedStepMutatesDokploy(kind string) bool { + switch dokploy.StepKind(kind) { + case dokploy.StepCreateProject, dokploy.StepCreateService, dokploy.StepUploadEnv, dokploy.StepPushImage, dokploy.StepInstallGateway, dokploy.StepActivateRoutes: + return true + default: + return false + } +} + +func applyMayHaveAmbiguousAuthority(applied runApplied) bool { + return appliedAuthorityAmbiguity(applied) != applyAuthorityUnambiguous +} + +func appliedStepInterrupted(step appliedStep) bool { + return step.Status == string(dokploy.StepStatusStarted) || step.Status == string(dokploy.StepStatusError) +} + +func appliedTargetMutationAmbiguous(step appliedStep) bool { + if step.Status == string(dokploy.StepStatusStarted) { + return true + } + if step.Status != string(dokploy.StepStatusError) { + return false + } + return step.MutationAmbiguous == nil || *step.MutationAmbiguous +} + +func appliedStepMayHaveRun(step appliedStep) bool { + return step.Status == string(dokploy.StepStatusStarted) || + step.Status == string(dokploy.StepStatusError) || + step.Status == string(dokploy.StepStatusOK) +} + +func appliedTargetIdentities(applied runApplied) map[string]dokploy.TargetIdentity { + identities := map[string]dokploy.TargetIdentity{} + for name, app := range applied.Apps { + if app.ProjectID == "" && app.EnvironmentID == "" && app.ComposeID == "" && app.ComposeAppName == "" { + continue + } + identities[name] = dokploy.TargetIdentity{ + ProjectID: app.ProjectID, + EnvironmentID: app.EnvironmentID, + ComposeID: app.ComposeID, + ComposeAppName: app.ComposeAppName, + } + } + return identities +} +func validateApplyResumeAuthority(applied runApplied) error { + switch appliedAuthorityAmbiguity(applied) { + case applyAuthorityUnambiguous: + return nil + case applyTrafficAuthorityAmbiguous: + return fmt.Errorf("live-apply state cannot prove traffic authority after the shared proxy handoff: inspect the Coolify and Dokploy proxies and routes and establish traffic authority manually") + default: + return fmt.Errorf("live-apply state cannot prove whether a target deployment was accepted or target writers changed state: inspect and preserve both sides and establish writer and traffic authority manually") + } +} + +func runMayHaveAmbiguousAuthority(run loadedMigrationRun) bool { + if run.Run.ResolvedAuthority != "" && run.Run.AuthorityResolvedAt != nil { + return false + } + if run.Run.CommittedAt != nil || run.Run.RolledBackAt != nil || run.Run.PurgedAt != nil { + return false + } + if completedLegacyRunMissingTargetOrigin(run) { + return true + } + missingOwner, ownerErr := targetMutationMissingDurableOwner(run) + if ownerErr != nil || missingOwner { + return true + } + if completedLegacyRun(run) { + return false + } + return applyMayHaveAmbiguousAuthority(run.Applied) +} + +func completedLegacyRunMissingTargetOrigin(run loadedMigrationRun) bool { + if !completedLegacyRun(run) { + return false + } + plan := dokploy.LegacyPlanFromArtifactsV1Alpha1(run.Prepare, run.Sync, run.Cutover) + for _, step := range plan.Steps { + if step.Kind == dokploy.StepDumpDataStore || step.Kind == dokploy.StepRestoreDataStore || step.Kind == dokploy.StepSyncVolume { + return true + } + } + return run.Applied.TargetOrigin == "" && trafficHandoffStepIndex(plan) >= 0 +} + +func completedLegacyRun(run loadedMigrationRun) bool { + if appliedPlanVersion(run.Applied) != appliedPlanV1Alpha1 { + return false + } + if run.Applied.SucceededAt != nil { + return true + } + plan := dokploy.LegacyPlanFromArtifactsV1Alpha1(run.Prepare, run.Sync, run.Cutover) + return len(plan.Steps) > 0 && completedApplyPrefix(plan.Steps, run.Applied) == len(plan.Steps) } func indexedCompletedApplyPrefix(steps []dokploy.Step, applied runApplied) int { @@ -206,13 +541,28 @@ func indexedCompletedApplyPrefix(steps []dokploy.Step, applied runApplied) int { } for index, step := range steps { recorded, ok := byIndex[index] - if !ok || !appliedStepCompleted(recorded) || !appliedStepMatches(recorded, step) { + if !ok || !appliedStepCompleted(recorded) { + return index + } + if !appliedStepMatches(recorded, step) && !legacyProxyCleanupPreservesStop(step, recorded, byIndex, index) { return index } } return len(steps) } +func legacyProxyCleanupPreservesStop(planned dokploy.Step, recorded appliedStep, byIndex map[int]appliedStep, index int) bool { + if planned.Kind != dokploy.StepStopCoolifyProxy || recorded.Kind != string(dokploy.StepStartCoolifyProxy) || recorded.Ref != planned.Ref { + return false + } + for laterIndex, later := range byIndex { + if laterIndex > index && appliedStepCompleted(later) { + return true + } + } + return false +} + func appliedStepCompleted(step appliedStep) bool { return step.Status == string(dokploy.StepStatusOK) || step.Status == string(dokploy.StepStatusSkipped) } diff --git a/internal/cli/applied_test.go b/internal/cli/applied_test.go index d8adbdc..44ab1d1 100644 --- a/internal/cli/applied_test.go +++ b/internal/cli/applied_test.go @@ -3,7 +3,9 @@ package cli import ( "errors" "path/filepath" + "strings" "testing" + "time" "github.com/aikins01/bort/internal/target/dokploy" ) @@ -75,6 +77,38 @@ func TestAppliedLedgerOverwritesByIndex(t *testing.T) { } } +func TestAppliedLedgerRewindRemovesStaleSuffixBeforeReplay(t *testing.T) { + steps := []dokploy.Step{ + {Kind: dokploy.StepPauseSource, App: "api", Ref: "api"}, + {Kind: dokploy.StepSyncVolume, App: "api", Ref: "volume:web -> /data"}, + {Kind: dokploy.StepResumeTarget, App: "api", Ref: "api"}, + } + applied := runApplied{APIVersion: appliedAPIVersion, RecoveryProtocol: appliedRecoveryProtocol} + for index, step := range steps { + applied = recordAppliedStep(applied, dokploy.StepProgress{Index: index, Step: step, Status: dokploy.StepStatusOK}) + } + applied = recordAppliedStep(applied, dokploy.StepProgress{ + Index: 0, + Step: dokploy.Step{Kind: dokploy.StepResumeSource, App: "api", Ref: "api"}, + Status: dokploy.StepStatusOK, + }) + ledger := &appliedLedger{state: applied, persist: func(string, runApplied) error { return nil }} + + resumeFrom := completedApplyPrefix(steps, ledger.Snapshot()) + if resumeFrom != 0 { + t.Fatalf("expected cleanup to rewind to source pause, got %d", resumeFrom) + } + if err := ledger.PrepareRetry(resumeFrom); err != nil { + t.Fatalf("rewind ledger: %v", err) + } + if err := ledger.Record(dokploy.StepProgress{Index: 0, Step: steps[0], Status: dokploy.StepStatusOK}); err != nil { + t.Fatalf("record replayed pause: %v", err) + } + if got := completedApplyPrefix(steps, ledger.Snapshot()); got != 1 { + t.Fatalf("expected a second retry to stop before the stale transfer suffix, got prefix %d", got) + } +} + func TestAppliedLedgerRetainsPersistenceFailure(t *testing.T) { ledger := &appliedLedger{ state: newRunApplied(migrationRun{Name: "run-failure"}), @@ -116,6 +150,111 @@ func TestAppliedLedgerPersistsSuccessfulOutcome(t *testing.T) { } } +func TestAppliedLedgerBindsTargetOrigin(t *testing.T) { + path := filepath.Join(t.TempDir(), "applied.json") + ledger, err := newAppliedLedger(path, migrationRun{Name: "origin", Target: "dokploy"}) + if err != nil { + t.Fatal(err) + } + if err := ledger.BindTargetOrigin("https://dokploy.example/api"); err != nil { + t.Fatal(err) + } + if got := ledger.Snapshot().TargetOrigin; got != "https://dokploy.example/api" { + t.Fatalf("unexpected normalized origin %q", got) + } + if err := ledger.BindTargetOrigin("https://dokploy.example/api"); err != nil { + t.Fatalf("equivalent origin was rejected: %v", err) + } + if err := ledger.BindTargetOrigin("https://dokploy.example/"); err == nil || !strings.Contains(err.Error(), "bound to Dokploy origin") { + t.Fatalf("expected path-routed target mismatch, got %v", err) + } + if err := ledger.BindTargetOrigin("https://other.example"); err == nil || !strings.Contains(err.Error(), "bound to Dokploy origin") { + t.Fatalf("expected target-origin mismatch, got %v", err) + } +} + +func TestAppliedLedgerRefusesToInventOriginForPriorExecution(t *testing.T) { + path := filepath.Join(t.TempDir(), "applied.json") + run := migrationRun{Name: "legacy-origin", Target: "dokploy"} + if err := writeJSONArtifact(path, runApplied{ + APIVersion: appliedLegacyAPIVersion, + RunName: run.Name, + Target: run.Target, + Steps: []appliedStep{{ + Index: 0, Kind: string(dokploy.StepCreateProject), App: "api", Ref: "api", Status: string(dokploy.StepStatusOK), + }}, + }); err != nil { + t.Fatal(err) + } + ledger, err := newAppliedLedger(path, run) + if err != nil { + t.Fatal(err) + } + if err := ledger.BindTargetOrigin("https://dokploy.example"); err == nil || !strings.Contains(err.Error(), "original target cannot be proved") { + t.Fatalf("expected missing legacy origin to fail closed, got %v", err) + } +} + +func TestLegacyProxyOnlyAmbiguityReportsTrafficRecovery(t *testing.T) { + applied := runApplied{ + APIVersion: appliedLegacyAPIVersion, + Steps: []appliedStep{{ + Kind: string(dokploy.StepStartDokployProxy), + Ref: "dokploy-traefik", + Status: string(dokploy.StepStatusStarted), + }}, + } + err := validateApplyResumeAuthority(applied) + if err == nil || !strings.Contains(err.Error(), "cannot prove traffic authority") || !strings.Contains(err.Error(), "inspect the Coolify and Dokploy proxies") { + t.Fatalf("expected proxy-specific recovery guidance, got %v", err) + } + if strings.Contains(err.Error(), "target data") || strings.Contains(err.Error(), "writer authority") { + t.Fatalf("proxy-only interruption reported writer or data ambiguity: %v", err) + } +} + +func TestWriteRunAppliedUpgradesLegacySchema(t *testing.T) { + path := filepath.Join(t.TempDir(), "applied.json") + legacy := runApplied{ + APIVersion: appliedLegacyAPIVersion, + Steps: []appliedStep{{ + Index: 0, Kind: string(dokploy.StepSyncVolume), App: "api", Status: string(dokploy.StepStatusError), + }}, + } + if err := writeRunApplied(path, legacy); err != nil { + t.Fatal(err) + } + upgraded, err := readRunApplied(path, migrationRun{}) + if err != nil { + t.Fatal(err) + } + if upgraded.APIVersion != appliedAPIVersion || upgraded.PlanVersion != appliedPlanV1Alpha1 { + t.Fatalf("legacy ledger did not retain its plan version during upgrade: %#v", upgraded) + } +} + +func TestPrepareRetryUsesCurrentPlanForUnexecutedLegacyLedger(t *testing.T) { + persisted := runApplied{} + ledger := &appliedLedger{ + state: runApplied{ + APIVersion: appliedAPIVersion, + PlanVersion: appliedPlanV1Alpha1, + RecoveryProtocol: appliedRecoveryProtocol, + Apps: map[string]appliedApp{}, + }, + persist: func(_ string, applied runApplied) error { + persisted = applied + return nil + }, + } + if err := ledger.PrepareRetry(0); err != nil { + t.Fatal(err) + } + if persisted.PlanVersion != appliedPlanCurrent || ledger.Snapshot().PlanVersion != appliedPlanCurrent { + t.Fatalf("unexecuted legacy ledger retained stale plan version: persisted=%#v snapshot=%#v", persisted, ledger.Snapshot()) + } +} + func TestReadRunAppliedRejectsUnknownAPIVersion(t *testing.T) { dir := t.TempDir() path := filepath.Join(dir, "applied.json") @@ -125,6 +264,12 @@ func TestReadRunAppliedRejectsUnknownAPIVersion(t *testing.T) { if _, err := readRunApplied(path, migrationRun{Name: "run-3"}); err == nil { t.Fatalf("expected error on unsupported apiVersion") } + if err := writeJSONArtifact(path, runApplied{APIVersion: appliedAPIVersion, PlanVersion: "v1alpha9", RunName: "run-3"}); err != nil { + t.Fatalf("write: %v", err) + } + if _, err := readRunApplied(path, migrationRun{Name: "run-3"}); err == nil || !strings.Contains(err.Error(), "planVersion") { + t.Fatalf("expected error on unsupported planVersion, got %v", err) + } } func TestReadRunAppliedRejectsMismatchedRunIdentity(t *testing.T) { @@ -176,3 +321,368 @@ func TestCompletedApplyPrefixRequiresContiguousIndexedSteps(t *testing.T) { t.Fatalf("expected prefix to stop at mismatched index 1, got %d", got) } } + +func TestCompletedApplyPrefixIgnoresStandaloneProxyCompensation(t *testing.T) { + steps := []dokploy.Step{ + {Kind: dokploy.StepStopCoolifyProxy, Ref: "coolify-proxy"}, + {Kind: dokploy.StepActivateRoutes, App: "api", Ref: "routes"}, + {Kind: dokploy.StepStartDokployProxy, Ref: "dokploy-traefik"}, + } + applied := runApplied{APIVersion: appliedAPIVersion, RecoveryProtocol: appliedRecoveryProtocol} + for index, step := range steps[:2] { + applied = recordAppliedStep(applied, dokploy.StepProgress{Index: index, Step: step, Status: dokploy.StepStatusOK}) + } + applied = recordAppliedStep(applied, dokploy.StepProgress{Index: 2, Step: steps[2], Status: dokploy.StepStatusError}) + applied = recordAppliedStep(applied, dokploy.StepProgress{ + Index: len(steps), + Step: dokploy.Step{Kind: dokploy.StepStartCoolifyProxy, Ref: "coolify-proxy"}, + Status: dokploy.StepStatusOK, + }) + if got := completedApplyPrefix(steps, applied); got != 2 { + t.Fatalf("proxy compensation rewound completed route activation: prefix=%d", got) + } +} + +func TestCompletedApplyPrefixAcceptsLegacyProxyCompensationWhenLaterProgressProvesStop(t *testing.T) { + steps := []dokploy.Step{ + {Kind: dokploy.StepStopCoolifyProxy, Ref: "coolify-proxy"}, + {Kind: dokploy.StepActivateRoutes, App: "api", Ref: "routes"}, + {Kind: dokploy.StepStartDokployProxy, Ref: "dokploy-traefik"}, + } + applied := runApplied{APIVersion: appliedAPIVersion, RecoveryProtocol: appliedRecoveryProtocol} + applied = recordAppliedStep(applied, dokploy.StepProgress{Index: 0, Step: steps[0], Status: dokploy.StepStatusOK}) + applied = recordAppliedStep(applied, dokploy.StepProgress{Index: 1, Step: steps[1], Status: dokploy.StepStatusOK}) + applied = recordAppliedStep(applied, dokploy.StepProgress{Index: 2, Step: steps[2], Status: dokploy.StepStatusError}) + applied = recordAppliedStep(applied, dokploy.StepProgress{ + Index: 0, + Step: dokploy.Step{Kind: dokploy.StepStartCoolifyProxy, Ref: "coolify-proxy"}, + Status: dokploy.StepStatusOK, + }) + if got := completedApplyPrefix(steps, applied); got != 2 { + t.Fatalf("legacy proxy compensation rewound completed route activation: prefix=%d", got) + } +} + +func TestCompletedApplyPrefixRejectsUnprovedLegacyProxyCompensation(t *testing.T) { + steps := []dokploy.Step{ + {Kind: dokploy.StepStopCoolifyProxy, Ref: "coolify-proxy"}, + {Kind: dokploy.StepActivateRoutes, App: "api", Ref: "routes"}, + } + applied := runApplied{APIVersion: appliedAPIVersion, RecoveryProtocol: appliedRecoveryProtocol} + applied = recordAppliedStep(applied, dokploy.StepProgress{ + Index: 0, + Step: dokploy.Step{Kind: dokploy.StepStartCoolifyProxy, Ref: "coolify-proxy"}, + Status: dokploy.StepStatusOK, + }) + if got := completedApplyPrefix(steps, applied); got != 0 { + t.Fatalf("unproved proxy stop was accepted: prefix=%d", got) + } +} + +func TestCompletedApplyPrefixReplaysStateTransferAfterSourceCleanup(t *testing.T) { + steps := []dokploy.Step{ + {Kind: dokploy.StepPauseSource, App: "api", Ref: "api"}, + {Kind: dokploy.StepSyncVolume, App: "api", Ref: "volume:web -> /data"}, + {Kind: dokploy.StepInstallGateway, App: "api", Ref: "api.example.com"}, + } + applied := runApplied{APIVersion: appliedAPIVersion, RecoveryProtocol: appliedRecoveryProtocol} + for index, step := range steps[:2] { + applied = recordAppliedStep(applied, dokploy.StepProgress{Index: index, Step: step, Status: dokploy.StepStatusOK}) + } + applied = recordAppliedStep(applied, dokploy.StepProgress{ + Index: 0, + Step: dokploy.Step{Kind: dokploy.StepResumeSource, App: "api", Ref: "api"}, + Status: dokploy.StepStatusStarted, + }) + + if got := completedApplyPrefix(steps, applied); got != 0 { + t.Fatalf("expected retry to replay from source pause after cleanup, got prefix %d", got) + } +} + +func TestCompletedApplyPrefixUsesCurrentCleanupMarkerBoundary(t *testing.T) { + steps := []dokploy.Step{ + {Kind: dokploy.StepPauseSource, App: "alpha", Ref: "alpha"}, + {Kind: dokploy.StepSyncVolume, App: "alpha", Ref: "volume:web -> /data"}, + {Kind: dokploy.StepPauseSource, App: "beta", Ref: "beta"}, + {Kind: dokploy.StepSyncVolume, App: "beta", Ref: "volume:web -> /data"}, + {Kind: dokploy.StepStopCoolifyProxy, Ref: "coolify-proxy"}, + {Kind: dokploy.StepResumeTarget, App: "alpha", Ref: "alpha"}, + } + applied := runApplied{APIVersion: appliedAPIVersion, RecoveryProtocol: appliedRecoveryProtocol} + for index, step := range steps[:5] { + applied = recordAppliedStep(applied, dokploy.StepProgress{Index: index, Step: step, Status: dokploy.StepStatusOK}) + } + applied = recordAppliedStep(applied, dokploy.StepProgress{Index: 5, Step: steps[5], Status: dokploy.StepStatusError}) + applied = recordAppliedStep(applied, dokploy.StepProgress{ + Index: 2, + Step: dokploy.Step{Kind: dokploy.StepResumeSource, App: "beta", Ref: "beta"}, + Status: dokploy.StepStatusOK, + }) + + if got := completedApplyPrefix(steps, applied); got != 2 { + t.Fatalf("expected beta's cleanup marker to be the replay boundary without rewinding alpha, got %d", got) + } +} + +func TestCompletedApplyPrefixPreservesAmbiguousLegacySourceCleanup(t *testing.T) { + steps := []dokploy.Step{ + {Kind: dokploy.StepCreateService, App: "api", Ref: "api"}, + {Kind: dokploy.StepPauseSource, App: "api", Ref: "api"}, + {Kind: dokploy.StepSyncVolume, App: "api", Ref: "volume:web -> /data"}, + {Kind: dokploy.StepInstallGateway, App: "api", Ref: "api.example.com"}, + } + applied := runApplied{} + for index, step := range steps[:3] { + applied = recordAppliedStep(applied, dokploy.StepProgress{Index: index, Step: step, Status: dokploy.StepStatusOK}) + } + applied = recordAppliedStep(applied, dokploy.StepProgress{Index: 3, Step: steps[3], Status: dokploy.StepStatusError, Err: errors.New("injected gateway failure")}) + applied = recordAppliedStep(applied, dokploy.StepProgress{ + Index: len(steps), + Step: dokploy.Step{Kind: dokploy.StepResumeSource, App: "api", Ref: "api"}, + Status: dokploy.StepStatusOK, + }) + + if !applyMayHaveAmbiguousAuthority(applied) { + t.Fatal("expected legacy cleanup after state transfer to preserve possible target writes") + } + if got := completedApplyPrefix(steps, applied); got != 3 { + t.Fatalf("expected ambiguous legacy cleanup to stop at the failed step without rewinding state, got prefix %d", got) + } +} + +func TestCompletedApplyPrefixPreservesIncompleteLegacyLogicalTransfer(t *testing.T) { + steps := []dokploy.Step{ + {Kind: dokploy.StepCreateService, App: "api", Ref: "api"}, + {Kind: dokploy.StepPauseSource, App: "api", Ref: "api"}, + {Kind: dokploy.StepDumpDataStore, App: "api", Ref: "data-store:db"}, + {Kind: dokploy.StepRestoreDataStore, App: "api", Ref: "data-store:db"}, + {Kind: dokploy.StepInstallGateway, App: "api", Ref: "api.example.com"}, + } + applied := runApplied{} + for index, step := range steps[:4] { + applied = recordAppliedStep(applied, dokploy.StepProgress{Index: index, Step: step, Status: dokploy.StepStatusOK}) + } + applied = recordAppliedStep(applied, dokploy.StepProgress{ + Index: 4, + Step: steps[4], + Status: dokploy.StepStatusError, + Err: errors.New("injected gateway failure"), + }) + + if !applyMayHaveAmbiguousAuthority(applied) { + t.Fatal("expected incomplete legacy restore to preserve possible target writes") + } + if got := completedApplyPrefix(steps, applied); got != 4 { + t.Fatalf("expected incomplete legacy logical transfer to stop at the failed step without rewinding state, got prefix %d", got) + } +} + +func TestCompletedApplyPrefixPreservesMultipleAmbiguousLegacyTransfers(t *testing.T) { + steps := []dokploy.Step{ + {Kind: dokploy.StepCreateService, App: "alpha", Ref: "alpha"}, + {Kind: dokploy.StepPauseSource, App: "alpha", Ref: "alpha"}, + {Kind: dokploy.StepSyncVolume, App: "alpha", Ref: "volume:web -> /data"}, + {Kind: dokploy.StepPauseSource, App: "beta", Ref: "beta"}, + {Kind: dokploy.StepSyncVolume, App: "beta", Ref: "volume:web -> /data"}, + {Kind: dokploy.StepInstallGateway, App: "alpha", Ref: "alpha.example.com"}, + } + applied := runApplied{} + for index, step := range steps[:5] { + applied = recordAppliedStep(applied, dokploy.StepProgress{Index: index, Step: step, Status: dokploy.StepStatusOK}) + } + applied = recordAppliedStep(applied, dokploy.StepProgress{Index: 5, Step: steps[5], Status: dokploy.StepStatusError}) + + if !applyMayHaveAmbiguousAuthority(applied) { + t.Fatal("expected incomplete legacy transfers to preserve possible target writes") + } + if got := completedApplyPrefix(steps, applied); got != 5 { + t.Fatalf("expected ambiguous legacy transfers to stop at the failed step without rewinding state, got %d", got) + } +} + +func TestCompletedApplyPrefixDoesNotReplayCurrentProtocolWithoutSourceCleanup(t *testing.T) { + steps := []dokploy.Step{ + {Kind: dokploy.StepPauseSource, App: "api", Ref: "api"}, + {Kind: dokploy.StepSyncVolume, App: "api", Ref: "volume:web -> /data"}, + {Kind: dokploy.StepStopCoolifyProxy, Ref: "coolify-proxy"}, + {Kind: dokploy.StepStartDokployProxy, Ref: "dokploy-traefik"}, + } + applied := runApplied{APIVersion: appliedAPIVersion, RecoveryProtocol: appliedRecoveryProtocol} + for index, step := range steps[:3] { + applied = recordAppliedStep(applied, dokploy.StepProgress{Index: index, Step: step, Status: dokploy.StepStatusOK}) + } + applied = recordAppliedStep(applied, dokploy.StepProgress{Index: 3, Step: steps[3], Status: dokploy.StepStatusStarted}) + + if got := completedApplyPrefix(steps, applied); got != 3 { + t.Fatalf("expected current recovery protocol to resume target handoff without replaying source state, got %d", got) + } +} + +func TestLegacyCutoverHandoffIsAmbiguous(t *testing.T) { + steps := []dokploy.Step{ + {Kind: dokploy.StepPauseSource, App: "api", Ref: "api"}, + {Kind: dokploy.StepStopCoolifyProxy, Ref: "coolify-proxy"}, + {Kind: dokploy.StepStartDokployProxy, Ref: "dokploy-traefik"}, + } + for _, tc := range []struct { + name string + api string + protocol string + status dokploy.StepStatus + want bool + }{ + {name: "legacy started", status: dokploy.StepStatusStarted, want: true}, + {name: "legacy error", status: dokploy.StepStatusError, want: true}, + {name: "current started", api: appliedAPIVersion, protocol: appliedRecoveryProtocol, status: dokploy.StepStatusStarted, want: false}, + } { + t.Run(tc.name, func(t *testing.T) { + applied := runApplied{APIVersion: tc.api, RecoveryProtocol: tc.protocol} + for index, step := range steps[:2] { + applied = recordAppliedStep(applied, dokploy.StepProgress{Index: index, Step: step, Status: dokploy.StepStatusOK}) + } + applied = recordAppliedStep(applied, dokploy.StepProgress{Index: 2, Step: steps[2], Status: tc.status}) + + if got := applyMayHaveAmbiguousAuthority(applied); got != tc.want { + t.Fatalf("applyMayHaveAmbiguousAuthority() = %t, want %t", got, tc.want) + } + if !tc.want { + return + } + if err := validateApplyResumeAuthority(applied); err == nil || !strings.Contains(err.Error(), "shared proxy handoff") || !strings.Contains(err.Error(), "establish traffic authority manually") { + t.Fatalf("expected actionable traffic-authority refusal, got %v", err) + } + if got := completedApplyPrefix(steps, applied); got != 2 { + t.Fatalf("expected ambiguous legacy handoff to keep the completed prefix, got %d", got) + } + }) + } +} + +func TestLegacyCutoverHandoffSurvivesCurrentPlanReordering(t *testing.T) { + applied := runApplied{Steps: []appliedStep{{ + Index: 1, + Kind: string(dokploy.StepStartDokployProxy), + Ref: "dokploy-traefik", + Status: string(dokploy.StepStatusStarted), + }}} + if !applyMayHaveAmbiguousAuthority(applied) { + t.Fatal("expected legacy proxy ambiguity to survive current plan reordering") + } +} + +func TestLegacyTargetResumeBeforeProxyHandoffIsAmbiguous(t *testing.T) { + applied := runApplied{Steps: []appliedStep{ + {Index: 0, Kind: string(dokploy.StepPauseSource), App: "api", Ref: "api", Status: string(dokploy.StepStatusOK)}, + {Index: 1, Kind: string(dokploy.StepResumeTarget), App: "api", Ref: "api", Status: string(dokploy.StepStatusOK)}, + }} + if !applyMayHaveAmbiguousAuthority(applied) { + t.Fatal("expected legacy target resume to require authority reconciliation") + } + if err := validateApplyResumeAuthority(applied); err == nil { + t.Fatal("expected legacy target resume to refuse automatic replay") + } +} + +func TestLegacyStateTransferWithoutTargetResumeIsAmbiguous(t *testing.T) { + applied := runApplied{Steps: []appliedStep{ + {Index: 0, Kind: string(dokploy.StepPauseSource), App: "api", Ref: "api", Status: string(dokploy.StepStatusOK)}, + {Index: 1, Kind: string(dokploy.StepSyncVolume), App: "api", Ref: "volume:web -> /data", Status: string(dokploy.StepStatusOK)}, + {Index: 2, Kind: string(dokploy.StepInstallGateway), App: "api", Ref: "api.example.com", Status: string(dokploy.StepStatusError)}, + }} + if !applyMayHaveAmbiguousAuthority(applied) { + t.Fatal("expected old failure cleanup after state transfer to require authority reconciliation") + } + if err := validateApplyResumeAuthority(applied); err == nil { + t.Fatal("expected legacy state transfer to refuse automatic replay") + } +} + +func TestCompletedLegacyApplyRetainsRollbackEligibility(t *testing.T) { + now := time.Now().UTC() + applied := runApplied{ + SucceededAt: &now, + Steps: []appliedStep{ + {Index: 0, Kind: string(dokploy.StepSyncVolume), App: "api", Ref: "volume:web -> /data", Status: string(dokploy.StepStatusOK)}, + {Index: 1, Kind: string(dokploy.StepResumeTarget), App: "api", Ref: "api", Status: string(dokploy.StepStatusOK)}, + {Index: 2, Kind: string(dokploy.StepStartDokployProxy), Ref: "dokploy-traefik", Status: string(dokploy.StepStatusOK)}, + }, + } + if applyMayHaveAmbiguousAuthority(applied) { + t.Fatal("completed legacy apply was classified as authority-ambiguous") + } + if err := validateApplyResumeAuthority(applied); err != nil { + t.Fatalf("completed legacy apply lost rollback eligibility: %v", err) + } +} + +func TestCurrentDeploymentErrorUsesPersistedAmbiguity(t *testing.T) { + retryable := false + ambiguous := true + for _, kind := range []dokploy.StepKind{ + dokploy.StepCreateProject, + dokploy.StepCreateService, + dokploy.StepUploadEnv, + dokploy.StepPushImage, + dokploy.StepInstallGateway, + dokploy.StepActivateRoutes, + } { + t.Run(string(kind), func(t *testing.T) { + base := runApplied{APIVersion: appliedAPIVersion, RecoveryProtocol: appliedRecoveryProtocol} + for _, test := range []struct { + name string + status dokploy.StepStatus + ambiguous *bool + wantError bool + }{ + {name: "interrupted before terminal record", status: dokploy.StepStatusStarted, wantError: true}, + {name: "old error without discriminator", status: dokploy.StepStatusError, wantError: true}, + {name: "ambiguous mutation response", status: dokploy.StepStatusError, ambiguous: &ambiguous, wantError: true}, + {name: "authoritative or pre-mutation error", status: dokploy.StepStatusError, ambiguous: &retryable}, + {name: "completed", status: dokploy.StepStatusOK}, + } { + t.Run(test.name, func(t *testing.T) { + applied := base + applied.Steps = []appliedStep{{ + Index: 0, Kind: string(kind), App: "api", Ref: "api", Status: string(test.status), MutationAmbiguous: test.ambiguous, + }} + err := validateApplyResumeAuthority(applied) + if test.wantError && (err == nil || !strings.Contains(err.Error(), "cannot prove whether a target deployment was accepted")) { + t.Fatalf("expected ambiguous deployment refusal, got %v", err) + } + if !test.wantError && err != nil { + t.Fatalf("retryable deployment failure remained ambiguous: %v", err) + } + }) + } + }) + } +} + +func TestRecordAppliedStepPersistsExplicitMutationAmbiguity(t *testing.T) { + for _, ambiguous := range []bool{false, true} { + applied := recordAppliedStep(runApplied{}, dokploy.StepProgress{ + Index: 1, + Step: dokploy.Step{Kind: dokploy.StepPushImage, App: "api", Ref: "api"}, + Status: dokploy.StepStatusError, + Err: errors.New("deploy failed"), + MutationAmbiguous: ambiguous, + }) + if len(applied.Steps) != 1 || applied.Steps[0].MutationAmbiguous == nil || *applied.Steps[0].MutationAmbiguous != ambiguous { + t.Fatalf("persisted mutation ambiguity = %#v, want %t", applied.Steps, ambiguous) + } + } +} + +func TestRecordAppliedStepPersistsTargetIdentity(t *testing.T) { + applied := recordAppliedStep(runApplied{}, dokploy.StepProgress{ + Index: 1, + Step: dokploy.Step{Kind: dokploy.StepCreateService, App: "api", Ref: "api"}, + Status: dokploy.StepStatusOK, + Target: &dokploy.TargetIdentity{ProjectID: "project-1", EnvironmentID: "environment-1", ComposeID: "compose-1", ComposeAppName: "stack-api"}, + }) + identity := applied.Apps["api"] + if identity.ProjectID != "project-1" || identity.EnvironmentID != "environment-1" || identity.ComposeID != "compose-1" || identity.ComposeAppName != "stack-api" { + t.Fatalf("unexpected persisted target identity: %#v", identity) + } +} diff --git a/internal/cli/apply_lock.go b/internal/cli/apply_lock.go new file mode 100644 index 0000000..46b7d36 --- /dev/null +++ b/internal/cli/apply_lock.go @@ -0,0 +1,22 @@ +package cli + +import ( + "errors" + "time" +) + +const ( + applyLockProbeGrace = 200 * time.Millisecond + applyLockProbeInterval = 10 * time.Millisecond +) + +func acquireApplyLock(path string) (*applyLock, error) { + deadline := time.Now().Add(applyLockProbeGrace) + for { + lock, err := tryAcquireApplyLock(path) + if !errors.Is(err, errApplyAlreadyRunning) || time.Now().After(deadline) { + return lock, err + } + time.Sleep(applyLockProbeInterval) + } +} diff --git a/internal/cli/apply_lock_unix.go b/internal/cli/apply_lock_unix.go index 9ef7162..19e6d22 100644 --- a/internal/cli/apply_lock_unix.go +++ b/internal/cli/apply_lock_unix.go @@ -17,7 +17,7 @@ type applyLock struct { file *os.File } -func acquireApplyLock(path string) (*applyLock, error) { +func tryAcquireApplyLock(path string) (*applyLock, error) { file, err := openLockFile(path, unix.O_CREAT|unix.O_RDWR) if err != nil { return nil, err diff --git a/internal/cli/apply_lock_windows.go b/internal/cli/apply_lock_windows.go index 9bcc3b8..1605ea3 100644 --- a/internal/cli/apply_lock_windows.go +++ b/internal/cli/apply_lock_windows.go @@ -17,7 +17,7 @@ type applyLock struct { overlapped windows.Overlapped } -func acquireApplyLock(path string) (*applyLock, error) { +func tryAcquireApplyLock(path string) (*applyLock, error) { file, err := openLockFile(path, windows.OPEN_ALWAYS) if err != nil { return nil, err diff --git a/internal/cli/authority.go b/internal/cli/authority.go new file mode 100644 index 0000000..4944670 --- /dev/null +++ b/internal/cli/authority.go @@ -0,0 +1,302 @@ +package cli + +import ( + "context" + "flag" + "fmt" + "io" + "strings" + + "github.com/aikins01/bort/internal/target/dokploy" +) + +func runRecoverAuthority(ctx context.Context, args []string, stdout, stderr io.Writer) error { + fs := flag.NewFlagSet("recover-authority", flag.ContinueOnError) + fs.SetOutput(stderr) + + var runRef string + var authority string + var confirm string + var sourceRetired bool + fs.StringVar(&runRef, "run", "", "migration run name or directory") + fs.StringVar(&authority, "authority", "", "manually verified authority: source or target") + fs.StringVar(&confirm, "confirm", "", "confirm with the exact phrase: recover as , or with --source-retired: recover as target with source retired") + fs.BoolVar(&sourceRetired, "source-retired", false, "confirm that source retirement was completed manually for target authority") + if err := fs.Parse(args); err != nil { + return err + } + if fs.NArg() > 0 { + return fmt.Errorf("recover-authority does not accept positional argument %q", fs.Arg(0)) + } + if !flagSet(fs, "run") || strings.TrimSpace(runRef) == "" { + return fmt.Errorf("recover-authority requires a non-empty --run value") + } + authority = strings.TrimSpace(authority) + if authority != dokployTrafficSource && authority != dokployTrafficTarget { + return fmt.Errorf("recover-authority requires --authority source or --authority target") + } + if sourceRetired && authority != dokployTrafficTarget { + return fmt.Errorf("recover-authority --source-retired requires --authority target") + } + + resolvedRun, err := resolveRunRef(runRef, false) + if err != nil { + return err + } + operationLock, err := acquireRunOperationLock(resolvedRun) + if err != nil { + return fmt.Errorf("recover authority for run %q: %w", runRef, err) + } + defer operationLock.Release() + run, err := loadMigrationRun(resolvedRun) + if err != nil { + return err + } + phrase := authorityRecoveryConfirmation(run.Run, authority) + if sourceRetired { + phrase = authorityRecoverySourceRetiredConfirmation(run.Run) + } + if confirm != phrase { + return fmt.Errorf("authority recovery confirmation must be exactly %q", phrase) + } + if run.Run.Target != "dokploy" { + return fmt.Errorf("recover-authority is only supported for target dokploy, got %q", run.Run.Target) + } + + targetLock, err := acquireDokployLiveOperationLock() + if err != nil { + return fmt.Errorf("lock Dokploy live operations: %w", err) + } + defer targetLock.Release() + run, err = loadMigrationRun(resolvedRun) + if err != nil { + return err + } + if err := validateAuthorityRecovery(run, authority, sourceRetired); err != nil { + return err + } + if sourceRetired && !planRequiresDokployHostOwner(run, dokploy.PlanForCommit(run.Prepare, run.Cutover)) { + return recordOwnerlessSourceRetirement(run, stdout) + } + owner, found, err := matchingAuthorityRecoveryOwner(run.Run, authority) + if err != nil { + return err + } + if !found { + if authority == dokployTrafficSource && run.Run.AuthorityFinalizedAt == nil { + if err := markRunAuthorityFinalizedLocked(run.Run); err != nil { + return fmt.Errorf("record source-authority finalization: %w", err) + } + } + fmt.Fprintf(stdout, "Authority recovery already complete for run %s: %s authority.\n", run.Run.Name, authority) + return nil + } + if authority == dokployTrafficTarget && !sourceRetired { + var sourceAttestationErr error + if err := validateLocalSourceAttestation(run); err != nil { + sourceAttestationErr = err + } else { + sourceAttestationErr = verifyLocalSourceRun(ctx, run) + } + if ctx.Err() != nil { + return ctx.Err() + } + if sourceAttestationErr != nil { + return fmt.Errorf("target authority was not recorded because source attestation failed: %v; after manually retiring the source, run `%s`", sourceAttestationErr, authorityRecoverySourceRetiredCommand(run)) + } + } + if err := ctx.Err(); err != nil { + return err + } + if err := markRunAuthorityResolvedLocked(run.Run, authority); err != nil { + return fmt.Errorf("record manual %s-authority resolution: %w", authority, err) + } + if authority == dokployTrafficSource { + if err := markRunRollbackStartedLocked(run.Run); err != nil { + return fmt.Errorf("record manual source recovery start: %w", err) + } + if owner.Authority != dokployTrafficSource { + if err := markDokployTrafficSource(run.Run); err != nil { + return fmt.Errorf("record durable source authority: %w", err) + } + } + if err := markRunRolledBackLocked(run.Run); err != nil { + return fmt.Errorf("record manual source recovery completion: %w", err) + } + if err := releaseDokployTrafficOwner(run.Run); err != nil { + return fmt.Errorf("release source-authority host ownership: %w", err) + } + if err := markRunAuthorityFinalizedLocked(run.Run); err != nil { + return fmt.Errorf("record source-authority finalization: %w", err) + } + fmt.Fprintf(stdout, "Authority recovery complete for run %s: source authority recorded, run rolled back, host ownership released.\n", run.Run.Name) + return nil + } + + if owner.Authority != dokployTrafficTarget { + if err := markDokployTrafficTarget(run.Run, owner.TargetOrigin); err != nil { + return fmt.Errorf("record durable target authority: %w", err) + } + } + if err := markRunLiveAppliedLocked(run.Run); err != nil { + return fmt.Errorf("record manual target recovery completion: %w", err) + } + if sourceRetired { + if err := markRunCommitStartedLocked(run.Run); err != nil { + return fmt.Errorf("record manual source retirement start: %w", err) + } + if err := markRunCommittedLocked(run.Run); err != nil { + return fmt.Errorf("record manual source retirement completion: %w", err) + } + if err := releaseDokployTargetOwner(run.Run); err != nil { + return fmt.Errorf("release target-authority host ownership: %w", err) + } + fmt.Fprintf(stdout, "Authority recovery complete for run %s: target authority and manual source retirement recorded, host ownership released. Bort did not mutate source resources.\n", run.Run.Name) + return nil + } + if coolifySourceRetirementRequired(run) { + fmt.Fprintf(stdout, "Authority recovery complete for run %s: target authority recorded. When ready to retire the source, %s.\n", run.Run.Name, manualCoolifySourceRetirementAction(run)) + return nil + } + fmt.Fprintf(stdout, "Authority recovery complete for run %s: target authority recorded. Run `%s` when ready to retire the source.\n", run.Run.Name, runScopedCommand(run, "commit --apply")) + return nil +} + +func recordOwnerlessSourceRetirement(run loadedMigrationRun, stdout io.Writer) error { + if err := markRunAuthorityResolvedLocked(run.Run, dokployTrafficTarget); err != nil { + return fmt.Errorf("record manual target-authority resolution: %w", err) + } + if err := markRunLiveAppliedLocked(run.Run); err != nil { + return fmt.Errorf("record manual target recovery completion: %w", err) + } + if err := markRunCommitStartedLocked(run.Run); err != nil { + return fmt.Errorf("record manual source retirement start: %w", err) + } + if err := markRunCommittedLocked(run.Run); err != nil { + return fmt.Errorf("record manual source retirement completion: %w", err) + } + fmt.Fprintf(stdout, "Authority recovery complete for run %s: target authority and manual source retirement recorded; this run never held host ownership. Bort did not mutate source resources.\n", run.Run.Name) + return nil +} + +func validateAuthorityRecovery(run loadedMigrationRun, authority string, sourceRetired bool) error { + if run.Run.PurgedAt != nil { + return fmt.Errorf("authority recovery refused: run %q is complete and purged", run.Run.Name) + } + if run.Run.CommittedAt != nil { + if run.Run.ResolvedAuthority == dokployTrafficTarget && authority == dokployTrafficTarget { + return nil + } + return fmt.Errorf("authority recovery refused: run %q already committed target authority", run.Run.Name) + } + if run.Run.RolledBackAt != nil { + if run.Run.ResolvedAuthority == dokployTrafficSource && authority == dokployTrafficSource { + return nil + } + return fmt.Errorf("authority recovery refused: run %q already rolled back to source authority", run.Run.Name) + } + if run.Run.CommitStartedAt != nil && run.Run.ResolvedAuthority == "" { + if authority == dokployTrafficTarget && sourceRetired { + return nil + } + return fmt.Errorf("authority recovery refused: source retirement already started for run %q; complete source retirement manually, then run `%s`", run.Run.Name, authorityRecoverySourceRetiredCommand(run)) + } + if run.Run.ResolvedAuthority != "" && run.Run.ResolvedAuthority != authority { + return fmt.Errorf("authority recovery refused: run %q already recorded %s authority", run.Run.Name, run.Run.ResolvedAuthority) + } + if run.Run.ResolvedAuthority != "" { + return nil + } + if runMayHaveAmbiguousAuthority(run) { + return nil + } + if sourceRetired && authority == dokployTrafficTarget && (run.Run.LiveAppliedAt != nil || liveApplySucceeded(run)) { + return nil + } + if run.Run.LiveAppliedAt != nil || liveApplySucceeded(run) || run.Run.RollbackStartedAt != nil { + if _, err := planAutomaticRollback(run); err != nil { + return nil + } + } + held, err := runHoldsPendingDokployOwner(run.Run) + if err != nil { + return fmt.Errorf("verify durable Dokploy host owner: %w", err) + } + if held && run.Applied.SucceededAt == nil { + return nil + } + return fmt.Errorf("authority recovery refused: run %q has no ambiguous authority, unavailable automatic rollback, or unreleased host ownership to resolve", run.Run.Name) +} + +func matchingAuthorityRecoveryOwner(run migrationRun, authority string) (dokployTrafficOwner, bool, error) { + owner, found, err := readDokployTrafficOwner() + if err != nil { + return dokployTrafficOwner{}, false, fmt.Errorf("verify durable Dokploy host owner: %w", err) + } + runID, err := dokployTrafficRunID(run) + if err != nil { + return dokployTrafficOwner{}, false, err + } + if !found || owner.RunID != runID { + released := !found || owner.Authority == dokployTrafficReleased || run.HostOwnerReleaseStartedAt != nil + if authority == dokployTrafficSource && run.ResolvedAuthority == authority && run.RolledBackAt != nil && released { + return owner, false, nil + } + if authority == dokployTrafficTarget && run.ResolvedAuthority == authority && run.CommittedAt != nil && released { + return owner, false, nil + } + if found { + return dokployTrafficOwner{}, false, fmt.Errorf("authority recovery refused: Dokploy host is owned by %s with %s authority", dokployOwnerRunLabel(owner), owner.Authority) + } + return dokployTrafficOwner{}, false, fmt.Errorf("authority recovery refused: run %q has no matching durable Dokploy host owner", run.Name) + } + if owner.Authority == dokployTrafficReleased { + if authority == dokployTrafficSource && run.ResolvedAuthority == authority && run.RolledBackAt != nil { + return owner, false, nil + } + if authority == dokployTrafficTarget && run.ResolvedAuthority == authority && run.CommittedAt != nil { + return owner, false, nil + } + return dokployTrafficOwner{}, false, fmt.Errorf("authority recovery refused: run %q released host ownership before recovery completed", run.Name) + } + if owner.Authority == dokployTrafficSource && authority != dokployTrafficSource { + return dokployTrafficOwner{}, false, fmt.Errorf("authority recovery refused: run %q already has durable source authority", run.Name) + } + return owner, true, nil +} + +func authorityRecoveryConfirmation(run migrationRun, authority string) string { + return fmt.Sprintf("recover %s as %s", run.Name, authority) +} + +func authorityRecoverySourceRetiredConfirmation(run migrationRun) string { + return fmt.Sprintf("recover %s as target with source retired", run.Name) +} + +func authorityRecoveryCommand(run loadedMigrationRun, authority string) string { + phrase := authorityRecoveryConfirmation(run.Run, authority) + return runScopedCommand(run, fmt.Sprintf("recover-authority --authority %s --confirm %s", authority, shellQuote(phrase))) +} + +func authorityRecoverySourceRetiredCommand(run loadedMigrationRun) string { + phrase := authorityRecoverySourceRetiredConfirmation(run.Run) + return runScopedCommand(run, fmt.Sprintf("recover-authority --authority target --source-retired --confirm %s", shellQuote(phrase))) +} + +// pendingAuthorityRecoveryCommand names the command that finishes a +// recorded but unfinalized authority decision. Ownerless target recovery +// records its decision before the lifecycle marks, so resuming it needs +// the same source-retired attestation that started it. +func pendingAuthorityRecoveryCommand(run loadedMigrationRun) string { + if run.Run.ResolvedAuthority == dokployTrafficTarget && !planRequiresDokployHostOwner(run, dokploy.PlanForCommit(run.Prepare, run.Cutover)) { + return authorityRecoverySourceRetiredCommand(run) + } + return authorityRecoveryCommand(run, run.Run.ResolvedAuthority) +} + +func authorityRecoveryInstruction(run loadedMigrationRun) string { + if authorityRecoveryAvailable(run) { + return fmt.Sprintf("inspect and preserve both sides, manually fence the other side and verify authority, then run `%s` for source or `%s` for target", authorityRecoveryCommand(run, dokployTrafficSource), authorityRecoveryCommand(run, dokployTrafficTarget)) + } + return "inspect and preserve both sides, establish writer and traffic authority manually, then start a fresh migration run" +} diff --git a/internal/cli/authority_test.go b/internal/cli/authority_test.go new file mode 100644 index 0000000..8c0f82b --- /dev/null +++ b/internal/cli/authority_test.go @@ -0,0 +1,503 @@ +package cli + +import ( + "bytes" + "context" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/aikins01/bort/internal/manifest" + "github.com/aikins01/bort/internal/target/dokploy" +) + +func TestRecoverAuthorityRequiresExactConfirmationWithoutMutation(t *testing.T) { + run := writeAmbiguousAuthorityRun(t, "confirm-run") + runPath := filepath.Join(run.Run.RunDir, "run.json") + ownerPath, err := dokployTrafficOwnerPath() + if err != nil { + t.Fatal(err) + } + runBefore, err := os.ReadFile(runPath) + if err != nil { + t.Fatal(err) + } + ownerBefore, err := os.ReadFile(ownerPath) + if err != nil { + t.Fatal(err) + } + + err = runRecoverAuthority(context.Background(), []string{"--run", "confirm-run", "--authority", "source", "--confirm", "recover the source"}, io.Discard, io.Discard) + if err == nil || !strings.Contains(err.Error(), `must be exactly "recover confirm-run as source"`) { + t.Fatalf("expected exact confirmation refusal, got %v", err) + } + runAfter, err := os.ReadFile(runPath) + if err != nil { + t.Fatal(err) + } + ownerAfter, err := os.ReadFile(ownerPath) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(runBefore, runAfter) || !bytes.Equal(ownerBefore, ownerAfter) { + t.Fatal("wrong confirmation mutated run or owner state") + } +} + +func TestRecoverAuthorityRefusesDifferentOwnerWithoutMutation(t *testing.T) { + run := writeAmbiguousAuthorityRun(t, "blocked-run") + other := migrationRun{Name: "other-run", RunDir: filepath.Join(t.TempDir(), "other-run"), CreatedAt: time.Now().UTC(), BundleDigest: "other"} + otherID, err := dokployTrafficRunID(other) + if err != nil { + t.Fatal(err) + } + if err := writeDokployTrafficOwner(dokployTrafficOwner{RunID: otherID, RunName: other.Name, RunDir: other.RunDir, TargetOrigin: "http://127.0.0.1:3030", Authority: dokployTrafficPending}); err != nil { + t.Fatal(err) + } + runPath := filepath.Join(run.Run.RunDir, "run.json") + runBefore, err := os.ReadFile(runPath) + if err != nil { + t.Fatal(err) + } + ownerPath, err := dokployTrafficOwnerPath() + if err != nil { + t.Fatal(err) + } + ownerBefore, err := os.ReadFile(ownerPath) + if err != nil { + t.Fatal(err) + } + + err = runRecoverAuthority(context.Background(), []string{"--run", "blocked-run", "--authority", "source", "--confirm", "recover blocked-run as source"}, io.Discard, io.Discard) + if err == nil || !strings.Contains(err.Error(), fmt.Sprintf("owned by migration run %q at %q", other.Name, other.RunDir)) { + t.Fatalf("expected different-owner refusal, got %v", err) + } + runAfter, err := os.ReadFile(runPath) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(runBefore, runAfter) { + t.Fatal("different-owner refusal mutated run state") + } + ownerAfter, err := os.ReadFile(ownerPath) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(ownerBefore, ownerAfter) { + t.Fatal("different-owner refusal mutated owner state") + } +} + +func TestRecoverAuthorityFinalizesSourceAndReleasesHost(t *testing.T) { + writeAmbiguousAuthorityRun(t, "source-run") + var output strings.Builder + args := []string{"--run", "source-run", "--authority", "source", "--confirm", "recover source-run as source"} + if err := runRecoverAuthority(context.Background(), args, &output, io.Discard); err != nil { + t.Fatal(err) + } + completed, err := loadMigrationRun("source-run") + if err != nil { + t.Fatal(err) + } + if completed.Run.ResolvedAuthority != dokployTrafficSource || completed.Run.AuthorityResolvedAt == nil || completed.Run.RollbackStartedAt == nil || completed.Run.RolledBackAt == nil || completed.Run.AuthorityFinalizedAt == nil { + t.Fatalf("source recovery lifecycle incomplete: %#v", completed.Run) + } + owner, found, err := readDokployTrafficOwner() + if err != nil || !found || owner.Authority != dokployTrafficReleased { + t.Fatalf("source recovery owner = %#v, found=%t err=%v", owner, found, err) + } + other := migrationRun{Name: "next-run", RunDir: filepath.Join(t.TempDir(), "next-run"), CreatedAt: time.Now().UTC(), BundleDigest: "next"} + plan := dokploy.Plan{Steps: []dokploy.Step{{Kind: dokploy.StepCreateProject, App: "api", Ref: "api"}}} + if err := ensureDokployTrafficRunAvailable(other, plan); err != nil { + t.Fatalf("released source recovery blocked a new run: %v", err) + } + if err := runRecoverAuthority(context.Background(), args, io.Discard, io.Discard); err != nil { + t.Fatalf("completed source recovery was not idempotent: %v", err) + } + if phase := migrationRunPhase(completed); phase != "rolled back" { + t.Fatalf("source recovery phase=%q, want rolled back", phase) + } +} + +func TestRecoverAuthorityFinalizesTargetForCommitAndRefusesAutomaticRollback(t *testing.T) { + writeAmbiguousAuthorityRun(t, "target-run") + args := []string{"--run", "target-run", "--authority", "target", "--confirm", "recover target-run as target"} + var output strings.Builder + if err := runRecoverAuthority(context.Background(), args, &output, io.Discard); err != nil { + t.Fatal(err) + } + if !strings.Contains(output.String(), "--source-retired") || strings.Contains(output.String(), "commit --apply") { + t.Fatalf("target recovery pointed a Coolify source at commit --apply instead of manual retirement: %q", output.String()) + } + completed, err := loadMigrationRun("target-run") + if err != nil { + t.Fatal(err) + } + if completed.Run.ResolvedAuthority != dokployTrafficTarget || completed.Run.AuthorityResolvedAt == nil || completed.Run.LiveAppliedAt == nil { + t.Fatalf("target recovery lifecycle incomplete: %#v", completed.Run) + } + if err := requireLiveApplySucceeded(completed); err != nil { + t.Fatalf("manual target authority did not enable commit: %v", err) + } + if _, err := planAutomaticRollback(completed); err == nil || !strings.Contains(err.Error(), "manual target-authority recovery") { + t.Fatalf("manual target authority allowed automatic rollback: %v", err) + } + owner, found, err := readDokployTrafficOwner() + if err != nil || !found || owner.Authority != dokployTrafficTarget { + t.Fatalf("target recovery owner = %#v, found=%t err=%v", owner, found, err) + } + if phase := migrationRunPhase(completed); phase != "applied" { + t.Fatalf("target recovery phase=%q, want applied", phase) + } + if next := nextSafeStep(completed, nil); !strings.Contains(next.Action, "--source-retired") || !strings.Contains(next.Action, "disable future Coolify deployments") || strings.Contains(next.Action, "commit --apply") || strings.Contains(next.Action, "rollback --live") { + t.Fatalf("target recovery next step is unsafe: %#v", next) + } + if err := runRecoverAuthority(context.Background(), args, io.Discard, io.Discard); err != nil { + t.Fatalf("completed target recovery was not idempotent: %v", err) + } +} + +func TestRecoverTargetAuthorityFinalizesManualRetirementWhenSourceIsStale(t *testing.T) { + writeAmbiguousAuthorityRun(t, "stale-source") + previous := verifyLocalSourceRun + verifyLocalSourceRun = func(context.Context, loadedMigrationRun) error { + return fmt.Errorf("reviewed source container was replaced") + } + t.Cleanup(func() { verifyLocalSourceRun = previous }) + var output strings.Builder + args := []string{"--run", "stale-source", "--authority", "target", "--confirm", "recover stale-source as target"} + err := runRecoverAuthority(context.Background(), args, &output, io.Discard) + if err == nil || !strings.Contains(err.Error(), "--source-retired") || !strings.Contains(err.Error(), "recover stale-source as target with source retired") { + t.Fatalf("stale source did not require explicit manual-retirement confirmation: %v", err) + } + unresolved, err := loadMigrationRun("stale-source") + if err != nil { + t.Fatal(err) + } + if unresolved.Run.ResolvedAuthority != "" || unresolved.Run.CommittedAt != nil { + t.Fatalf("failed source attestation mutated authority lifecycle: %#v", unresolved.Run) + } + args = []string{"--run", "stale-source", "--authority", "target", "--source-retired", "--confirm", "recover stale-source as target with source retired"} + if err := runRecoverAuthority(context.Background(), args, &output, io.Discard); err != nil { + t.Fatal(err) + } + completed, err := loadMigrationRun("stale-source") + if err != nil { + t.Fatal(err) + } + if completed.Run.ResolvedAuthority != dokployTrafficTarget || completed.Run.LiveAppliedAt == nil || completed.Run.CommitStartedAt == nil || completed.Run.CommittedAt == nil { + t.Fatalf("manual target finalization incomplete: %#v", completed.Run) + } + owner, found, err := readDokployTrafficOwner() + if err != nil || !found || owner.Authority != dokployTrafficReleased { + t.Fatalf("manual target finalization owner = %#v, found=%t err=%v", owner, found, err) + } + if !strings.Contains(output.String(), "manual source retirement recorded") || !strings.Contains(output.String(), "did not mutate source resources") { + t.Fatalf("manual target finalization output omitted its boundary: %s", output.String()) + } + if err := runRecoverAuthority(context.Background(), args, io.Discard, io.Discard); err != nil { + t.Fatalf("completed manual target finalization was not idempotent: %v", err) + } +} + +func TestRecoverTargetAuthorityResumesManualRetirementBoundaries(t *testing.T) { + for _, test := range []struct { + name string + interrupt func(migrationRun) error + }{ + {name: "after target authority"}, + {name: "after retirement start", interrupt: markRunCommitStartedLocked}, + {name: "after retirement completion", interrupt: func(run migrationRun) error { + if err := markRunCommitStartedLocked(run); err != nil { + return err + } + return markRunCommittedLocked(run) + }}, + } { + t.Run(test.name, func(t *testing.T) { + run := writeAmbiguousAuthorityRun(t, strings.ReplaceAll(test.name, " ", "-")) + if err := markRunAuthorityResolvedLocked(run.Run, dokployTrafficTarget); err != nil { + t.Fatal(err) + } + if err := markDokployTrafficTarget(run.Run, "http://127.0.0.1:3030"); err != nil { + t.Fatal(err) + } + if err := markRunLiveAppliedLocked(run.Run); err != nil { + t.Fatal(err) + } + if test.interrupt != nil { + if err := test.interrupt(run.Run); err != nil { + t.Fatal(err) + } + } + phrase := authorityRecoverySourceRetiredConfirmation(run.Run) + args := []string{"--run", run.Run.Name, "--authority", "target", "--source-retired", "--confirm", phrase} + if err := runRecoverAuthority(context.Background(), args, io.Discard, io.Discard); err != nil { + t.Fatal(err) + } + completed, err := loadMigrationRun(run.Run.Name) + if err != nil { + t.Fatal(err) + } + owner, found, err := readDokployTrafficOwner() + if err != nil || !found || owner.Authority != dokployTrafficReleased || completed.Run.CommittedAt == nil { + t.Fatalf("manual target recovery did not resume: owner=%#v found=%t err=%v run=%#v", owner, found, err, completed.Run) + } + }) + } +} + +func TestRecoverTargetAuthorityAcceptsRetiredSourceAfterInterruptedCommit(t *testing.T) { + run := writeAmbiguousAuthorityRun(t, "interrupted-commit") + if err := markDokployTrafficTarget(run.Run, "http://127.0.0.1:3030"); err != nil { + t.Fatal(err) + } + if err := markRunLiveAppliedLocked(run.Run); err != nil { + t.Fatal(err) + } + if err := markRunCommitStartedLocked(run.Run); err != nil { + t.Fatal(err) + } + sourceArgs := []string{"--run", "interrupted-commit", "--authority", "source", "--confirm", "recover interrupted-commit as source"} + err := runRecoverAuthority(context.Background(), sourceArgs, io.Discard, io.Discard) + if err == nil || !strings.Contains(err.Error(), "source retirement already started") || !strings.Contains(err.Error(), "--source-retired") { + t.Fatalf("interrupted retirement accepted source authority: %v", err) + } + targetArgs := []string{"--run", "interrupted-commit", "--authority", "target", "--confirm", "recover interrupted-commit as target"} + err = runRecoverAuthority(context.Background(), targetArgs, io.Discard, io.Discard) + if err == nil || !strings.Contains(err.Error(), "source retirement already started") { + t.Fatalf("interrupted retirement accepted target authority without a retirement attestation: %v", err) + } + unresolved, err := loadMigrationRun("interrupted-commit") + if err != nil { + t.Fatal(err) + } + if unresolved.Run.ResolvedAuthority != "" || unresolved.Run.CommittedAt != nil { + t.Fatalf("refused recovery mutated authority lifecycle: %#v", unresolved.Run) + } + retiredArgs := []string{"--run", "interrupted-commit", "--authority", "target", "--source-retired", "--confirm", authorityRecoverySourceRetiredConfirmation(run.Run)} + if err := runRecoverAuthority(context.Background(), retiredArgs, io.Discard, io.Discard); err != nil { + t.Fatalf("explicit retirement attestation was refused: %v", err) + } + completed, err := loadMigrationRun("interrupted-commit") + if err != nil { + t.Fatal(err) + } + owner, found, err := readDokployTrafficOwner() + if err != nil || !found || owner.Authority != dokployTrafficReleased || completed.Run.ResolvedAuthority != dokployTrafficTarget || completed.Run.CommittedAt == nil { + t.Fatalf("interrupted retirement was not finalized: owner=%#v found=%t err=%v run=%#v", owner, found, err, completed.Run) + } +} + +func TestRecoverAuthorityResumesAfterDurableResolution(t *testing.T) { + run := writeAmbiguousAuthorityRun(t, "resume-run") + if err := markRunAuthorityResolvedLocked(run.Run, dokployTrafficSource); err != nil { + t.Fatal(err) + } + interrupted, err := loadMigrationRun("resume-run") + if err != nil { + t.Fatal(err) + } + if phase := migrationRunPhase(interrupted); phase != "authority-finalizing" { + t.Fatalf("interrupted recovery phase=%q, want authority-finalizing", phase) + } + next := nextSafeStep(interrupted, nil) + if !strings.Contains(next.Action, "recover-authority") || !strings.Contains(next.Action, "recover resume-run as source") { + t.Fatalf("interrupted recovery did not expose its exact resume command: %#v", next) + } + if err := runRecoverAuthority(context.Background(), []string{"--run", "resume-run", "--authority", "source", "--confirm", "recover resume-run as source"}, io.Discard, io.Discard); err != nil { + t.Fatal(err) + } + completed, err := loadMigrationRun("resume-run") + if err != nil { + t.Fatal(err) + } + if completed.Run.RolledBackAt == nil { + t.Fatal("resumed authority recovery did not finish rollback lifecycle") + } +} + +func TestRecoverAuthorityResumesSourceOwnerRelease(t *testing.T) { + run := writeAmbiguousAuthorityRun(t, "release-run") + if err := markRunAuthorityResolvedLocked(run.Run, dokployTrafficSource); err != nil { + t.Fatal(err) + } + if err := markRunRollbackStartedLocked(run.Run); err != nil { + t.Fatal(err) + } + if err := markDokployTrafficSource(run.Run); err != nil { + t.Fatal(err) + } + if err := markRunRolledBackLocked(run.Run); err != nil { + t.Fatal(err) + } + interrupted, err := loadMigrationRun("release-run") + if err != nil { + t.Fatal(err) + } + if phase := migrationRunPhase(interrupted); phase != "authority-finalizing" { + t.Fatalf("unreleased source recovery phase=%q, want authority-finalizing", phase) + } + if err := runRecoverAuthority(context.Background(), []string{"--run", "release-run", "--authority", "source", "--confirm", "recover release-run as source"}, io.Discard, io.Discard); err != nil { + t.Fatal(err) + } + owner, found, err := readDokployTrafficOwner() + if err != nil || !found || owner.Authority != dokployTrafficReleased { + t.Fatalf("resumed source recovery owner = %#v, found=%t err=%v", owner, found, err) + } + completed, err := loadMigrationRun("release-run") + if err != nil { + t.Fatal(err) + } + if completed.Run.AuthorityFinalizedAt == nil { + t.Fatal("resumed source recovery did not record authority finalization") + } +} + +func TestSourceAuthorityRecoveryRequiresPersistedFinalizationWithoutHostState(t *testing.T) { + run := writeAmbiguousAuthorityRun(t, "portable-finalization") + if err := markRunAuthorityResolvedLocked(run.Run, dokployTrafficSource); err != nil { + t.Fatal(err) + } + if err := markRunRollbackStartedLocked(run.Run); err != nil { + t.Fatal(err) + } + if err := markRunRolledBackLocked(run.Run); err != nil { + t.Fatal(err) + } + ownerPath, err := dokployTrafficOwnerPath() + if err != nil { + t.Fatal(err) + } + if err := os.Remove(ownerPath); err != nil { + t.Fatal(err) + } + interrupted, err := loadMigrationRun("portable-finalization") + if err != nil { + t.Fatal(err) + } + if phase := migrationRunPhase(interrupted); phase != "authority-finalizing" { + t.Fatalf("missing finalization phase=%q, want authority-finalizing", phase) + } + args := []string{"--run", "portable-finalization", "--authority", "source", "--confirm", "recover portable-finalization as source"} + if err := runRecoverAuthority(context.Background(), args, io.Discard, io.Discard); err != nil { + t.Fatal(err) + } + completed, err := loadMigrationRun("portable-finalization") + if err != nil { + t.Fatal(err) + } + if completed.Run.AuthorityFinalizedAt == nil || migrationRunPhase(completed) != "rolled back" { + t.Fatalf("source authority finalization incomplete: %#v", completed.Run) + } +} + +func TestRecoverAuthorityGuidanceKeepsMatchingOwnerBoundRun(t *testing.T) { + run := writeAmbiguousAuthorityRun(t, "guided-run") + if phase := migrationRunPhase(run); phase != "authority-ambiguous" { + t.Fatalf("ambiguous owner-bound run phase=%q, want authority-ambiguous", phase) + } + var output strings.Builder + writeAppFirstCockpit(&output, run) + for _, want := range []string{"AUTHORITY UNKNOWN", "recover guided-run as source", "recover guided-run as target"} { + if !strings.Contains(output.String(), want) { + t.Fatalf("owner-bound authority guidance missing %q:\n%s", want, output.String()) + } + } + if strings.Contains(output.String(), "start a fresh migration run") { + t.Fatalf("owner-bound authority guidance stranded its durable owner:\n%s", output.String()) + } + next := nextSafeStep(run, nil) + if !strings.Contains(next.Action, "recover-authority") || strings.Contains(next.Action, "fresh migration run") { + t.Fatalf("owner-bound next step is not recoverable: %#v", next) + } +} + +func TestRunMetadataRejectsIncompleteAuthorityResolution(t *testing.T) { + run := writeAmbiguousAuthorityRun(t, "invalid-resolution") + run.Run.ResolvedAuthority = dokployTrafficSource + run.Run.AuthorityResolvedAt = nil + path := filepath.Join(run.Run.RunDir, "run.json") + if err := writeJSONArtifact(path, run.Run); err != nil { + t.Fatal(err) + } + if _, err := loadMigrationRun("invalid-resolution"); err == nil || !strings.Contains(err.Error(), "incomplete manual authority resolution") { + t.Fatalf("expected incomplete authority metadata refusal, got %v", err) + } +} + +func writeAmbiguousAuthorityRun(t *testing.T, name string) loadedMigrationRun { + t.Helper() + resetDokployTrafficOwner(t) + workDir := t.TempDir() + t.Chdir(workDir) + writeTestBundle(t, "bundle", manifest.Manifest{ + Source: manifest.Source{Platform: "coolify-local", DockerEngineID: "engine-reviewed"}, + Apps: []manifest.App{{ + Name: "api", + Services: []manifest.Service{{ID: "source-id", Name: "web", Image: "example/api:latest"}}, + Routes: []manifest.Route{{Host: "api.example.com", ServiceName: "web", Port: "3000"}}, + }}, + }) + runCommand(t, runMigrate, []string{"--bundle", "bundle", "--run", name, "--observation-window", "0", "--rollback-window", "0"}) + markRunLocallyScanned(t, name, "coolify-local") + run, err := loadMigrationRun(name) + if err != nil { + t.Fatal(err) + } + applied := newRunApplied(run.Run) + applied.Steps = []appliedStep{{Index: 0, Kind: string(dokploy.StepPushImage), App: "api", Ref: "example/api:latest", Status: string(dokploy.StepStatusError), UpdatedAt: time.Now().UTC(), Error: "outcome unknown"}} + if err := writeRunApplied(runArtifactPath(run.Run.RunDir, run.Run.Artifacts.Applied), applied); err != nil { + t.Fatal(err) + } + if err := claimDokployHostOwnership(run.Run, "http://127.0.0.1:3030", dokployCredentialID("test-token")); err != nil { + t.Fatal(err) + } + run, err = loadMigrationRun(name) + if err != nil { + t.Fatal(err) + } + return run +} + +func TestRecoverAuthorityAcceptsDefiniteFailureWhileHostOwnerIsPending(t *testing.T) { + run := writeAmbiguousAuthorityRun(t, "definite-run") + definite := false + applied := newRunApplied(run.Run) + applied.Steps = []appliedStep{{Index: 0, Kind: string(dokploy.StepPushImage), App: "api", Ref: "example/api:latest", Status: string(dokploy.StepStatusError), MutationAmbiguous: &definite, UpdatedAt: time.Now().UTC(), Error: "dokploy http 404"}} + if err := writeRunApplied(runArtifactPath(run.Run.RunDir, run.Run.Artifacts.Applied), applied); err != nil { + t.Fatal(err) + } + run, err := loadMigrationRun("definite-run") + if err != nil { + t.Fatal(err) + } + if runMayHaveAmbiguousAuthority(run) { + t.Fatal("fixture must be a definite, unambiguous failure") + } + if err := validateAuthorityRecovery(run, dokployTrafficSource, false); err != nil { + t.Fatalf("definite failure holding the pending host owner must stay recoverable: %v", err) + } + args := []string{"--run", "definite-run", "--authority", "source", "--confirm", "recover definite-run as source"} + if err := runRecoverAuthority(context.Background(), args, io.Discard, io.Discard); err != nil { + t.Fatal(err) + } + owner, found, err := readDokployTrafficOwner() + if err != nil || !found || owner.Authority != dokployTrafficReleased { + t.Fatalf("owner = %#v, found=%t err=%v", owner, found, err) + } + + resetDokployTrafficOwner(t) + run.Run.ResolvedAuthority = "" + run.Run.AuthorityResolvedAt = nil + run.Run.RollbackStartedAt = nil + run.Run.RolledBackAt = nil + run.Run.AuthorityFinalizedAt = nil + if err := validateAuthorityRecovery(run, dokployTrafficSource, false); err == nil || !strings.Contains(err.Error(), "authority recovery refused") { + t.Fatalf("definite failure without host ownership must still be refused, got %v", err) + } +} diff --git a/internal/cli/cleanup.go b/internal/cli/cleanup.go index 82c2417..e7c3f4b 100644 --- a/internal/cli/cleanup.go +++ b/internal/cli/cleanup.go @@ -39,6 +39,7 @@ type cleanupResult struct { Target string `json:"target"` DryRun bool `json:"dryRun"` Applied bool `json:"applied,omitempty"` + MetadataApplyBlocked string `json:"metadataApplyBlocked,omitempty"` BackupPath string `json:"backupPath,omitempty"` DeletedProjects []dokploy.StalePlatformProject `json:"deletedProjects,omitempty"` StalePlatformRecords []cleanupStalePlatformRecord `json:"stalePlatformRecords,omitempty"` @@ -200,18 +201,44 @@ func runCleanupWithInput(ctx context.Context, args []string, stdin io.Reader, st return err } var operationLock *applyLock + var targetLock *applyLock if apply { operationLock, err = acquireRunOperationLock(resolvedRunRef) if err != nil { return fmt.Errorf("cleanup run %q: %w", resolvedRunRef, err) } defer operationLock.Release() + targetLock, err = acquireDokployLiveOperationLock() + if err != nil { + return fmt.Errorf("lock Dokploy live operations: %w", err) + } + defer targetLock.Release() } run, err := loadMigrationRun(resolvedRunRef) if err != nil { return err } - result := planCleanup(ctx, run, target) + var client *dokploy.Client + if apply { + owner, conflict, err := conflictingDokployHostOwner(run.Run) + if err != nil { + return fmt.Errorf("verify Dokploy host ownership before cleanup: %w", err) + } + if conflict { + return fmt.Errorf("refusing cleanup --apply for run %q because Dokploy host mutations are owned by %s with %s authority", run.Run.Name, dokployOwnerRunLabel(owner), owner.Authority) + } + client, err = lookupDokployClient(target) + if err != nil { + return err + } + if err := validateCleanupDokployIdentity(run, client); err != nil { + return fmt.Errorf("refusing cleanup --apply without a bound local Dokploy installation: %w", err) + } + if err := verifyLocalDokployCleanupAPIHost(ctx, client); err != nil { + return fmt.Errorf("refusing cleanup --apply before Dokploy API access: %w", err) + } + } + result := planCleanupWithClient(ctx, run, target, client, !apply) result.DryRun = !apply if apply { if collisions := cleanupStaleProjectNameCollisions(run, defaultStaleDokployPlatformProjects); len(collisions) > 0 { @@ -219,14 +246,13 @@ func runCleanupWithInput(ctx context.Context, args []string, stdin io.Reader, st } applyNames := cleanupStaleProjectNamesForApply(result.StalePlatformRecords) if len(applyNames) == 0 { + if err := verifyLocalDokployCleanupHost(ctx, client); err != nil { + return fmt.Errorf("refusing cleanup --apply without a bound local Dokploy installation: %w", err) + } result.Applied = true result.Actions = append(result.Actions, cleanupAction{Kind: "dokploy_metadata", Ref: strings.Join(defaultStaleDokployPlatformProjects, ","), Safety: "metadata_only", Status: "noop", Message: "no empty zero-domain stale Dokploy platform project records are ready to delete"}) return writeFormattedOutput(stdout, outputPath, format, result, writeCleanupText) } - client, err := lookupDokployClient(target) - if err != nil { - return err - } applied, err := client.CleanupStalePlatformProjects(ctx, dokploy.StalePlatformCleanupOptions{ ProjectNames: applyNames, ProjectIDs: cleanupStaleProjectIDs(result.StalePlatformRecords), @@ -302,12 +328,20 @@ func runCleanupPurge(ctx context.Context, args []string, stdin io.Reader, stdout return err } var operationLock *applyLock + var targetLock *applyLock if apply { operationLock, err = acquireRunOperationLock(resolvedRunRef) if err != nil { return fmt.Errorf("cleanup purge run %q: %w", resolvedRunRef, err) } defer operationLock.Release() + if strings.TrimSpace(confirm) != "" { + targetLock, err = acquireDokployLiveOperationLock() + if err != nil { + return fmt.Errorf("lock Dokploy live operations: %w", err) + } + defer targetLock.Release() + } } run, err := loadMigrationRun(resolvedRunRef) if err != nil { @@ -345,20 +379,40 @@ func runCleanupPurge(ctx context.Context, args []string, stdin io.Reader, stdout return fmt.Errorf("refusing cleanup purge --apply before a successful live apply: %w", err) } if run.Run.CommittedAt == nil { + if coolifySourceRetirementRequired(run) { + return fmt.Errorf("refusing cleanup purge --apply before target acceptance; the source is still the rollback path: %s", manualCoolifySourceRetirementAction(run)) + } return fmt.Errorf("refusing cleanup purge --apply before `%s`; the source is still the rollback path", runScopedCommand(run, "commit --apply")) } - client := &dokploy.Client{} - identified, err := client.IdentifySourcePurgeResources(ctx, cleanupPurgeOptions(result)) - if err != nil { - return fmt.Errorf("identify selected source resources before confirmation: %w", err) - } - applyCleanupPurgeIdentities(&result, identified) if err := confirmCleanupPurgeApply(stdin, stderr, run.Run.Name, confirm); err != nil { return err } + if targetLock == nil { + targetLock, err = acquireDokployLiveOperationLock() + if err != nil { + return fmt.Errorf("lock Dokploy live operations: %w", err) + } + defer targetLock.Release() + } + owner, conflict, err := conflictingDokployHostOwner(run.Run) + if err != nil { + return fmt.Errorf("verify Dokploy host ownership before source purge: %w", err) + } + if conflict { + return fmt.Errorf("refusing cleanup purge --apply for run %q because Dokploy host mutations are owned by %s with %s authority", run.Run.Name, dokployOwnerRunLabel(owner), owner.Authority) + } if runtime.GOOS != "linux" { return fmt.Errorf("cleanup purge --apply is unavailable on %s; rerun the same scoped and confirmed command on the Linux source host", runtime.GOOS) } + if err := verifyLocalSourceEngine(ctx, run); err != nil { + return fmt.Errorf("refusing source purge on an unverified Docker source: %w", err) + } + client := &dokploy.Client{} + identified, err := client.IdentifySourcePurgeResources(ctx, cleanupPurgeOptions(result)) + if err != nil { + return fmt.Errorf("identify selected source resources under host lock: %w", err) + } + applyCleanupPurgeIdentities(&result, identified) recorder, err := newCleanupPurgeBackupRecorder(backupDir, &result) if err != nil { return err @@ -1309,6 +1363,10 @@ func cleanupStaleProjectNamesForApply(records []cleanupStalePlatformRecord) []st } func planCleanup(ctx context.Context, run loadedMigrationRun, target string) cleanupResult { + return planCleanupWithClient(ctx, run, target, nil, true) +} + +func planCleanupWithClient(ctx context.Context, run loadedMigrationRun, target string, client *dokploy.Client, assessMetadataApply bool) cleanupResult { result := cleanupResult{ APIVersion: cleanupAPIVersion, RunName: run.Run.Name, @@ -1316,7 +1374,20 @@ func planCleanup(ctx context.Context, run loadedMigrationRun, target string) cle Target: target, DryRun: true, } - result.StalePlatformRecords, result.Warnings = inspectStalePlatformRecords(ctx, target) + if client == nil { + var err error + if client, err = lookupDokployClient(target); err != nil { + result.MetadataApplyBlocked = err.Error() + } + } + if result.MetadataApplyBlocked == "" && assessMetadataApply { + result.MetadataApplyBlocked = cleanupMetadataApplyBlocker(ctx, run, client) + } + if result.MetadataApplyBlocked != "" { + result.StalePlatformRecords = stalePlatformRecordsWithStatus("blocked", "not inspected; metadata cleanup is unavailable for this run") + } else { + result.StalePlatformRecords, result.Warnings = inspectStalePlatformRecordsWithClient(ctx, client) + } networkIdentities, networkIdentityErr := cleanupManifestNetworkIdentities(run) if networkIdentityErr != nil { result.Warnings = append(result.Warnings, fmt.Sprintf("inspect source network identities: %v", networkIdentityErr)) @@ -1375,15 +1446,29 @@ func planCleanup(ctx context.Context, run loadedMigrationRun, target string) cle return result } -func inspectStalePlatformRecords(ctx context.Context, target string) ([]cleanupStalePlatformRecord, []string) { +func cleanupMetadataApplyBlocker(ctx context.Context, run loadedMigrationRun, client *dokploy.Client) string { + if err := validateCleanupDokployIdentity(run, client); err != nil { + return err.Error() + } + if !dokployLiveOperationsSupported() { + return "Dokploy metadata cleanup must run on the Linux Dokploy host" + } + if err := verifyLocalDokployCleanupHost(ctx, client); err != nil { + return fmt.Sprintf("the configured Dokploy API and database are not an eligible local installation: %v", err) + } + return "" +} + +func stalePlatformRecordsWithStatus(status, message string) []cleanupStalePlatformRecord { records := make([]cleanupStalePlatformRecord, 0, len(defaultStaleDokployPlatformProjects)) for _, name := range defaultStaleDokployPlatformProjects { - records = append(records, cleanupStalePlatformRecord{Name: name, Status: "unknown", Message: "will verify the project is empty and has zero domains against Dokploy DB before deleting"}) - } - client, err := lookupDokployClient(target) - if err != nil { - return records, []string{err.Error()} + records = append(records, cleanupStalePlatformRecord{Name: name, Status: status, Message: message}) } + return records +} + +func inspectStalePlatformRecordsWithClient(ctx context.Context, client *dokploy.Client) ([]cleanupStalePlatformRecord, []string) { + records := stalePlatformRecordsWithStatus("unknown", "will verify the project is empty and has zero domains against Dokploy DB before deleting") projects, err := client.ListProjects(ctx) if err != nil { return records, []string{fmt.Sprintf("inspect Dokploy projects: %v", err)} @@ -1443,6 +1528,50 @@ func inspectStalePlatformRecords(ctx context.Context, target string) ([]cleanupS return records, warnings } +var verifyLocalDokployCleanupHost = func(ctx context.Context, client *dokploy.Client) error { + return client.VerifySameDockerHostAndDatabase(ctx) +} + +var verifyLocalDokployCleanupAPIHost = func(ctx context.Context, client *dokploy.Client) error { + return client.VerifySameDockerHost(ctx) +} + +func validateCleanupDokployIdentity(run loadedMigrationRun, client *dokploy.Client) error { + if err := validateLocalSourceEngineAttestation(run); err != nil { + return err + } + if err := validateAppliedTargetOrigin(run.Applied, client.BaseURL); err != nil { + return err + } + if run.Applied.TargetOrigin == "" { + return fmt.Errorf("migration run %q has no persisted Dokploy target origin", run.Run.Name) + } + owner, found, err := readDokployTrafficOwner() + if err != nil { + return err + } + if !found { + return fmt.Errorf("migration run %q has no durable Dokploy host owner", run.Run.Name) + } + runID, err := dokployTrafficRunID(run.Run) + if err != nil { + return err + } + if owner.RunID != runID { + return fmt.Errorf("Dokploy host ownership belongs to %s, not %q", dokployOwnerRunLabel(owner), run.Run.Name) + } + if owner.TargetOrigin != client.BaseURL { + return fmt.Errorf("migration run %q owns Dokploy traffic for target %q, not %q", run.Run.Name, owner.TargetOrigin, client.BaseURL) + } + if owner.TargetCredentialID == "" { + return fmt.Errorf("migration run %q has no durable Dokploy credential identity", run.Run.Name) + } + if owner.TargetCredentialID != dokployCredentialID(client.Token) { + return fmt.Errorf("migration run %q owns Dokploy traffic with a different target credential identity; restore the exact Dokploy API key this run claimed the host with and retry", run.Run.Name) + } + return nil +} + func cleanupSourceControlForApp(app preparer.AppPlan) *cleanupSourceControl { if app.Resources.SourceControl == nil { return nil @@ -2003,6 +2132,11 @@ func writeCleanupText(w io.Writer, result cleanupResult) { fmt.Fprintln(w, "Applied only the safe Dokploy metadata cleanup. Source containers, volumes, networks, source-control credentials, and target apps were not removed.") return } + if result.MetadataApplyBlocked != "" { + fmt.Fprintf(w, "Audit only: `cleanup --apply` is unavailable because %s.\n", result.MetadataApplyBlocked) + fmt.Fprintln(w, "Source containers, volumes, networks, source-control credentials, and target apps are inventoried only and are preserved by this command.") + return + } run := loadedMigrationRun{Run: migrationRun{Name: result.RunName, RunDir: result.RunDir}} fmt.Fprintf(w, "Dry run only: run `%s` to remove only stale zero-domain Dokploy platform metadata after a DB backup.\n", runScopedCommand(run, "cleanup --apply")) fmt.Fprintln(w, "Source containers, volumes, networks, source-control credentials, and target apps are inventoried only and are preserved by this command.") diff --git a/internal/cli/cleanup_test.go b/internal/cli/cleanup_test.go index 6f67af5..b06b71e 100644 --- a/internal/cli/cleanup_test.go +++ b/internal/cli/cleanup_test.go @@ -12,6 +12,7 @@ import ( "path/filepath" "runtime" "strings" + "sync/atomic" "testing" "time" @@ -21,6 +22,7 @@ import ( ) func TestLifecycleAcceptanceTraceWithFakeDokploy(t *testing.T) { + resetDokployTrafficOwner(t) composeCreated := false deploymentTitle := "" server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -71,27 +73,59 @@ func TestLifecycleAcceptanceTraceWithFakeDokploy(t *testing.T) { t.Fatal(err) } dockerPath := filepath.Join(binDir, "docker") - dockerStub := "#!/bin/sh\nif [ \"$1\" = inspect ]; then\n echo 'Error: No such object' >&2\n exit 1\nfi\nif [ \"$1\" = ps ]; then\n echo dokploy-postgres\nfi\nexit 0\n" + dockerStub := `#!/bin/sh +if [ "$*" = "info --format {{.ID}}" ]; then + echo engine-reviewed + exit 0 +fi +if [ "$*" = "ps -aq" ]; then + echo source-api + exit 0 +fi +if [ "$*" = "inspect source-api" ]; then + printf '%s\n' '[{"Id":"source-api","Name":"/api","Image":"sha256:api","Config":{"Image":"example/api:latest","Labels":{}},"State":{"Status":"running"}}]' + exit 0 +fi +if [ "$*" = "inspect --type container source-api" ]; then + if [ -f source-stopped ]; then + printf '%s\n' '[{"Id":"source-api","Name":"/api","State":{"Running":false,"Status":"exited"}}]' + else + printf '%s\n' '[{"Id":"source-api","Name":"/api","State":{"Running":true,"Status":"running"}}]' + fi + exit 0 +fi +if [ "$*" = "stop source-api" ]; then + touch source-stopped + echo source-api + exit 0 +fi +if [ "$*" = "node inspect self --format {{.ID}}" ]; then + echo local-node + exit 0 +fi +if [ "$*" = "service ps --filter desired-state=running -q dokploy-postgres" ]; then + echo dokploy-postgres-task + exit 0 +fi +if [ "$*" = "inspect --type task dokploy-postgres-task" ]; then + printf '%s\n' '[{"NodeID":"local-node","Status":{"State":"running","ContainerStatus":{"ContainerID":"dokploy-postgres"}}}]' + exit 0 +fi +if [ "$1" = inspect ]; then + echo 'Error: No such object' >&2 + exit 1 +fi +if [ "$1" = ps ]; then + echo dokploy-postgres +fi +exit 0 +` if err := os.WriteFile(dockerPath, []byte(dockerStub), 0o700); err != nil { t.Fatal(err) } t.Setenv("PATH", binDir+string(os.PathListSeparator)+os.Getenv("PATH")) - manifestPath := filepath.Join(workDir, "manifest.json") - if err := writeJSONArtifact(manifestPath, manifest.Manifest{ - Source: manifest.Source{Platform: "coolify-local"}, - Apps: []manifest.App{{ - Name: "api", - Services: []manifest.Service{{ - ID: "source-api", - Name: "api", - Image: "example/api:latest", - }}, - }}, - }); err != nil { - t.Fatal(err) - } - runCommand(t, runMigrate, []string{"--manifest", manifestPath, "--run", "acceptance", "--observation-window", "0", "--rollback-window", "0"}) + runCommand(t, runMigrate, []string{"--source", "docker", "--run", "acceptance", "--observation-window", "0", "--rollback-window", "0"}) planned, err := loadMigrationRun("acceptance") if err != nil { t.Fatal(err) @@ -99,6 +133,9 @@ func TestLifecycleAcceptanceTraceWithFakeDokploy(t *testing.T) { if !strings.Contains(filepath.ToSlash(planned.Run.BundleDir), ".bort/runs/acceptance/bundle") { t.Fatalf("expected a self-contained bundle, got %s", planned.Run.BundleDir) } + if planned.Run.Source != "docker" || planned.Prepare.SourceDockerEngineID != "engine-reviewed" || len(planned.Prepare.Apps) != 1 || planned.Prepare.Apps[0].Name != "api" { + t.Fatalf("expected a locally scanned api run, got run=%#v prepare=%#v", planned.Run, planned.Prepare) + } for _, decision := range openSetupDecisions(planned) { if err := recordReviewDecision(planned, decision, planned.Run.UpdatedAt.Add(time.Second)); err != nil { t.Fatal(err) @@ -179,7 +216,10 @@ func TestLifecycleAcceptanceTraceWithFakeDokploy(t *testing.T) { } func TestRunCleanupInventoriesLeftoversAndSafeMetadata(t *testing.T) { + resetDokployTrafficOwner(t) + var requests atomic.Int32 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) switch { case r.Method == http.MethodGet && r.URL.Path == "/api/project.all": _ = json.NewEncoder(w).Encode([]dokploy.Project{{ProjectID: "p-proxy", Name: "proxy"}}) @@ -202,7 +242,7 @@ func TestRunCleanupInventoriesLeftoversAndSafeMetadata(t *testing.T) { t.Chdir(workDir) bundleDir := filepath.Join(workDir, "bort-bundle") writeTestBundle(t, bundleDir, manifest.Manifest{ - Source: manifest.Source{Platform: "coolify-local"}, + Source: manifest.Source{Platform: "coolify-local", DockerEngineID: "engine-reviewed"}, Apps: []manifest.App{{ Name: "api", Git: &manifest.GitSource{ @@ -226,9 +266,71 @@ func TestRunCleanupInventoriesLeftoversAndSafeMetadata(t *testing.T) { var stdout bytes.Buffer var stderr bytes.Buffer + if err := runCleanup(context.Background(), []string{"--run", "cleanup-run"}, &stdout, &stderr); err != nil { + t.Fatalf("imported cleanup failed: %v\nstderr:\n%s", err, stderr.String()) + } + if requests.Load() != 0 { + t.Fatalf("imported cleanup dry run sent %d Dokploy API request(s)", requests.Load()) + } + if !strings.Contains(stdout.String(), "Audit only: `cleanup --apply` is unavailable because run was created from an imported bundle, not a local Docker scan") { + t.Fatalf("expected imported-bundle cleanup to stay audit-only, got:\n%s", stdout.String()) + } + + markRunLocallyScanned(t, "cleanup-run", "coolify-local") + stdout.Reset() + stderr.Reset() + if err := runCleanup(context.Background(), []string{"--run", "cleanup-run"}, &stdout, &stderr); err != nil { + t.Fatalf("unbound cleanup failed: %v\nstderr:\n%s", err, stderr.String()) + } + if requests.Load() != 0 { + t.Fatalf("unbound cleanup dry run sent %d Dokploy API request(s) before verifying the target", requests.Load()) + } + for _, want := range []string{ + "[blocked] proxy: not inspected; metadata cleanup is unavailable for this run", + "Audit only: `cleanup --apply` is unavailable because migration run \"cleanup-run\" has no persisted Dokploy target origin.", + } { + if !strings.Contains(stdout.String(), want) { + t.Fatalf("expected unbound cleanup output to contain %q, got:\n%s", want, stdout.String()) + } + } + + run, err := loadMigrationRun("cleanup-run") + if err != nil { + t.Fatal(err) + } + applied := newRunApplied(run.Run) + applied.TargetOrigin = server.URL + if err := writeRunApplied(runArtifactPath(run.Run.RunDir, run.Run.Artifacts.Applied), applied); err != nil { + t.Fatal(err) + } + if err := claimDokployHostOwnership(run.Run, server.URL, dokployCredentialID("secret")); err != nil { + t.Fatal(err) + } + previousVerify := verifyLocalDokployCleanupHost + verifyLocalDokployCleanupHost = func(context.Context, *dokploy.Client) error { + return errors.New("DATABASE_URL selects an external database") + } + stdout.Reset() + stderr.Reset() + if err := runCleanup(context.Background(), []string{"--run", "cleanup-run"}, &stdout, &stderr); err != nil { + t.Fatalf("unverified-host cleanup failed: %v\nstderr:\n%s", err, stderr.String()) + } + verifyLocalDokployCleanupHost = previousVerify + if requests.Load() != 0 { + t.Fatalf("unverified-host cleanup dry run sent %d Dokploy API request(s)", requests.Load()) + } + if !strings.Contains(stdout.String(), "Audit only: `cleanup --apply` is unavailable because the configured Dokploy API and database are not an eligible local installation: DATABASE_URL selects an external database") { + t.Fatalf("expected unverified-host cleanup to report the blocker, got:\n%s", stdout.String()) + } + + stdout.Reset() + stderr.Reset() if err := runCleanup(context.Background(), []string{"--run", "cleanup-run"}, &stdout, &stderr); err != nil { t.Fatalf("cleanup failed: %v\nstderr:\n%s", err, stderr.String()) } + if requests.Load() == 0 { + t.Fatal("bound cleanup dry run did not inspect Dokploy") + } output := stdout.String() for _, want := range []string{ @@ -244,7 +346,7 @@ func TestRunCleanupInventoriesLeftoversAndSafeMetadata(t *testing.T) { "Source networks preserved:", "api api-net", "Target artifacts kept by default:", - "Dry run only: run `bort cleanup --apply --run cleanup-run`", + "Dry run only: run `bort cleanup --apply --run cleanup-run` to remove only stale zero-domain Dokploy platform metadata after a DB backup.", } { if !strings.Contains(output, want) { t.Fatalf("expected cleanup output to contain %q, got:\n%s", want, output) @@ -253,6 +355,34 @@ func TestRunCleanupInventoriesLeftoversAndSafeMetadata(t *testing.T) { if strings.Contains(output, "Source containers, volumes, networks, and target apps were removed") { t.Fatalf("cleanup dry run claimed destructive removal:\n%s", output) } + if strings.Contains(output, "Audit only") || strings.Contains(output, "not inspected") { + t.Fatalf("bound cleanup dry run reported an audit-only blocker:\n%s", output) + } +} + +func TestCleanupMetadataApplyRequiresCurrentDurableBinding(t *testing.T) { + resetDokployTrafficOwner(t) + runMeta := migrationRun{Name: "cleanup-current", RunDir: filepath.Join(t.TempDir(), "cleanup-current"), CreatedAt: time.Now().UTC(), BundleDigest: "cleanup-digest", Source: "docker"} + run := loadedMigrationRun{Run: runMeta, Applied: newRunApplied(runMeta), Prepare: preparer.Result{Source: "docker", SourceDockerEngineID: "engine-reviewed"}} + client := &dokploy.Client{BaseURL: "http://127.0.0.1:3030", Token: "original-token"} + if blocker := cleanupMetadataApplyBlocker(context.Background(), run, client); !strings.Contains(blocker, "no persisted Dokploy target origin") { + t.Fatalf("unbound cleanup blocker=%q", blocker) + } + run.Applied.TargetOrigin = client.BaseURL + if err := claimDokployHostOwnership(runMeta, client.BaseURL, dokployCredentialID(client.Token)); err != nil { + t.Fatal(err) + } + if blocker := cleanupMetadataApplyBlocker(context.Background(), run, client); blocker != "" { + t.Fatalf("current durable cleanup binding was blocked: %s", blocker) + } + previousVerify := verifyLocalDokployCleanupHost + verifyLocalDokployCleanupHost = func(context.Context, *dokploy.Client) error { + return errors.New("DATABASE_URL selects an external database") + } + t.Cleanup(func() { verifyLocalDokployCleanupHost = previousVerify }) + if blocker := cleanupMetadataApplyBlocker(context.Background(), run, client); !strings.Contains(blocker, "not an eligible local installation") || !strings.Contains(blocker, "external database") { + t.Fatalf("ineligible local cleanup blocker=%q", blocker) + } } func TestRunCleanupPurgeInventoriesDestructiveSourceResources(t *testing.T) { @@ -438,6 +568,7 @@ func TestCleanupManifestNetworkIdentitiesRejectsManifestSymlink(t *testing.T) { } func TestRunCleanupPurgeApplyRequiresExplicitScopeAndConfirmation(t *testing.T) { + resetDokployTrafficOwner(t) workDir := t.TempDir() t.Chdir(workDir) binDir := filepath.Join(workDir, "bin") @@ -496,12 +627,18 @@ func TestRunCleanupPurgeApplyRequiresExplicitScopeAndConfirmation(t *testing.T) if err := writeRunApplied(appliedPath, applied); err != nil { t.Fatal(err) } + if err := claimDokployHostOwnership(run.Run, "http://127.0.0.1:3030", dokployCredentialID("test-token")); err != nil { + t.Fatal(err) + } + if err := markDokployTrafficTarget(run.Run, "http://127.0.0.1:3030"); err != nil { + t.Fatal(err) + } stdout.Reset() stderr.Reset() err = runCleanup(context.Background(), []string{"purge", "--run", "purge-run", "--apply", "--app", "api"}, &stdout, &stderr) - if err == nil || !strings.Contains(err.Error(), "before `bort commit --apply --run purge-run`") { - t.Fatalf("expected target acceptance before confirmation, got err=%v stdout=%s stderr=%s", err, stdout.String(), stderr.String()) + if err == nil || !strings.Contains(err.Error(), "before target acceptance") || !strings.Contains(err.Error(), authorityRecoverySourceRetiredCommand(run)) { + t.Fatalf("expected Coolify source retirement before confirmation, got err=%v stdout=%s stderr=%s", err, stdout.String(), stderr.String()) } operationLock, err := acquireRunOperationLock(run.Run.RunDir) @@ -535,6 +672,144 @@ func TestRunCleanupRejectsMisplacedPurgeSubcommand(t *testing.T) { } } +func TestCleanupApplyCommandsShareDokployHostLock(t *testing.T) { + runDir := t.TempDir() + targetLock, err := acquireDokployLiveOperationLock() + if err != nil { + t.Fatal(err) + } + defer targetLock.Release() + for _, args := range [][]string{ + {"--run", runDir, "--apply"}, + {"purge", "--run", runDir, "--apply", "--all-apps", "--confirm", "purge fixture"}, + } { + err := runCleanup(context.Background(), args, io.Discard, io.Discard) + if !errors.Is(err, errDokployLiveOperationActive) { + t.Fatalf("cleanup apply %v did not honor host lock: %v", args, err) + } + } +} + +func TestCleanupApplyRefusesAnotherRunsHostOwnership(t *testing.T) { + resetDokployTrafficOwner(t) + workDir := cleanupBackupTestDir(t) + t.Chdir(workDir) + bundleDir := filepath.Join(workDir, "bundle") + writeTestBundle(t, bundleDir, manifest.Manifest{ + Source: manifest.Source{Platform: "docker"}, + Apps: []manifest.App{{Name: "api", Services: []manifest.Service{{Name: "api", Image: "example/api:v1"}}}}, + }) + runCommand(t, runMigrate, []string{"--bundle", bundleDir, "--run", "cleanup-run"}) + + ownerRun := migrationRun{Name: "owner-run", RunDir: filepath.Join(t.TempDir(), "owner-run"), CreatedAt: time.Now().UTC(), BundleDigest: "owner-digest"} + if err := claimDokployHostOwnership(ownerRun, "http://127.0.0.1:3030", dokployCredentialID("test-token")); err != nil { + t.Fatal(err) + } + if err := markDokployTrafficTarget(ownerRun, "http://127.0.0.1:3030"); err != nil { + t.Fatal(err) + } + ownerDir, err := dokployTrafficRunDir(ownerRun) + if err != nil { + t.Fatal(err) + } + + err = runCleanup(context.Background(), []string{"--run", "cleanup-run", "--apply"}, io.Discard, io.Discard) + if err == nil || !strings.Contains(err.Error(), `owned by migration run "owner-run" at "`+ownerDir+`" with target authority`) { + t.Fatalf("expected conflicting host owner refusal, got %v", err) + } +} + +func TestCleanupBindingRequiresRunOriginAndCredential(t *testing.T) { + resetDokployTrafficOwner(t) + runMeta := migrationRun{Name: "cleanup-bound", RunDir: filepath.Join(t.TempDir(), "cleanup-bound"), CreatedAt: time.Now().UTC(), BundleDigest: "cleanup-digest", Source: "docker"} + if err := claimDokployHostOwnership(runMeta, "http://127.0.0.1:3030", dokployCredentialID("original-token")); err != nil { + t.Fatal(err) + } + run := loadedMigrationRun{Run: runMeta, Applied: newRunApplied(runMeta), Prepare: preparer.Result{Source: "docker", SourceDockerEngineID: "engine-reviewed"}} + client := &dokploy.Client{BaseURL: "http://127.0.0.1:3030", Token: "original-token"} + if err := validateCleanupDokployIdentity(run, client); err == nil || !strings.Contains(err.Error(), "no persisted Dokploy target origin") { + t.Fatalf("expected cleanup without a run target binding to fail, got %v", err) + } + run.Applied.TargetOrigin = client.BaseURL + for source, want := range map[string]string{"": "imported bundle", "manifest": "imported manifest"} { + imported := run + imported.Run.Source = source + if err := validateCleanupDokployIdentity(imported, client); err == nil || !strings.Contains(err.Error(), want) { + t.Fatalf("expected cleanup for run source %q with a valid target binding to fail with %q, got %v", source, want, err) + } + } + if err := validateCleanupDokployIdentity(run, client); err != nil { + t.Fatalf("expected a locally scanned, bound run to be cleanup-eligible, got %v", err) + } + owner, found, err := readDokployTrafficOwner() + if err != nil || !found { + t.Fatalf("read cleanup owner: found=%t err=%v", found, err) + } + owner.TargetCredentialID = "" + if err := writeDokployTrafficOwner(owner); err != nil { + t.Fatal(err) + } + if err := validateCleanupDokployIdentity(run, client); err == nil || !strings.Contains(err.Error(), "no durable Dokploy credential identity") { + t.Fatalf("expected cleanup without durable credential identity to fail, got %v", err) + } + owner.TargetCredentialID = dokployCredentialID("original-token") + if err := writeDokployTrafficOwner(owner); err != nil { + t.Fatal(err) + } + client.Token = "replacement-token" + if err := validateCleanupDokployIdentity(run, client); err == nil || !strings.Contains(err.Error(), "different target credential identity") { + t.Fatalf("expected cleanup with replacement credential to fail, got %v", err) + } +} + +func TestCleanupPurgeApplyRefusesAnotherRunsHostOwnership(t *testing.T) { + resetDokployTrafficOwner(t) + workDir := t.TempDir() + t.Chdir(workDir) + writeTestBundle(t, "bundle", manifest.Manifest{ + Source: manifest.Source{Platform: "coolify-local"}, + Apps: []manifest.App{{ + Name: "api", + Services: []manifest.Service{{ + ID: "source-id", Name: "web", Image: "example/api:v1", + }}, + }}, + }) + runCommand(t, runMigrate, []string{"--bundle", "bundle", "--run", "purge-owner"}) + run, err := loadMigrationRun("purge-owner") + if err != nil { + t.Fatal(err) + } + applied := newRunApplied(run.Run) + for index, step := range dokploy.PlanFromArtifacts(run.Prepare, run.Sync, run.Cutover).Steps { + applied.Steps = append(applied.Steps, appliedStep{Index: index, Kind: string(step.Kind), App: step.App, Ref: step.Ref, Status: string(dokploy.StepStatusOK)}) + } + now := time.Now().UTC() + applied.SucceededAt = &now + if err := writeRunApplied(runArtifactPath(run.Run.RunDir, run.Run.Artifacts.Applied), applied); err != nil { + t.Fatal(err) + } + if err := markRunCommittedLocked(run.Run); err != nil { + t.Fatal(err) + } + ownerRun := migrationRun{Name: "newer-run", RunDir: filepath.Join(t.TempDir(), "newer-run"), CreatedAt: time.Now().UTC(), BundleDigest: "newer-digest"} + if err := claimDokployHostOwnership(ownerRun, "http://127.0.0.1:3030", dokployCredentialID("newer-token")); err != nil { + t.Fatal(err) + } + if err := markDokployTrafficTarget(ownerRun, "http://127.0.0.1:3030"); err != nil { + t.Fatal(err) + } + ownerDir, err := dokployTrafficRunDir(ownerRun) + if err != nil { + t.Fatal(err) + } + + err = runCleanup(context.Background(), []string{"purge", "--run", "purge-owner", "--apply", "--app", "api", "--confirm", "purge purge-owner"}, io.Discard, io.Discard) + if err == nil || !strings.Contains(err.Error(), `owned by migration run "newer-run" at "`+ownerDir+`" with target authority`) { + t.Fatalf("expected conflicting purge owner refusal, got %v", err) + } +} + func TestRunCleanupPurgePreservesResourcesSharedWithUnselectedApps(t *testing.T) { workDir := t.TempDir() t.Chdir(workDir) diff --git a/internal/cli/cockpit.go b/internal/cli/cockpit.go index d83582c..826b79e 100644 --- a/internal/cli/cockpit.go +++ b/internal/cli/cockpit.go @@ -1,9 +1,12 @@ package cli import ( + "context" + "errors" "fmt" "io" "os" + "runtime" "sort" "strconv" "strings" @@ -12,9 +15,17 @@ import ( ) func writeAppFirstCockpit(w io.Writer, run loadedMigrationRun) { + writeAppFirstCockpitContext(context.Background(), w, run) +} + +func writeAppFirstCockpitContext(ctx context.Context, w io.Writer, run loadedMigrationRun) { + summary := summarizeMigrationRunContext(ctx, run) + writeAppFirstCockpitSummary(w, run, summary) +} + +func writeAppFirstCockpitSummary(w io.Writer, run loadedMigrationRun, summary migrationRunSummary) { apps := appsFromRun(run) - summary := summarizeMigrationRun(run) - phase := migrationRunPhase(run) + phase := migrationRunPhaseWithNext(run, summary.Next) st := newStyler(w) ready, blocked := 0, 0 @@ -45,6 +56,8 @@ func writeAppFirstCockpit(w io.Writer, run loadedMigrationRun) { if len(apps) == 0 { fmt.Fprintln(w, "No apps in this run.") + fmt.Fprintln(w) + writeCockpitPhaseGuidance(w, st, run, phase, summary, apps) return } @@ -112,23 +125,98 @@ func writeAppFirstCockpit(w io.Writer, run loadedMigrationRun) { fmt.Fprintf(w, "%s\n", st.muted(fmt.Sprintf("Review-only decisions: %d open (non-blocking before live apply)", len(reviewDecisions)))) writeCockpitDecisions(w, st, reviewDecisions) } + writeCockpitPhaseGuidance(w, st, run, phase, summary, apps) +} + +func writeCockpitPhaseGuidance(w io.Writer, st *styler, run loadedMigrationRun, phase string, summary migrationRunSummary, apps []appView) { + if !dokployLiveOperationsSupported() { + switch phase { + case "empty", "committed", "rolled back", "purged": + default: + writeUnsupportedPlatformGuidance(w, st, run, phase) + return + } + } switch phase { case "lock-error": fmt.Fprintln(w, st.muted("Live apply lock state could not be verified. Inspect the run's apply.lock before retrying.")) + case "host-lock-error": + fmt.Fprintln(w, st.muted("The host-wide Dokploy operation lock could not be verified. Inspect /var/lib/bort/dokploy-live.lock before retrying.")) + case "install-recovery": + command, found, err := dokployInstallationRecoveryCommand() + switch { + case err != nil: + fmt.Fprintf(w, "%s\n", st.muted(fmt.Sprintf("A Dokploy installation was interrupted, but its recovery command could not be read: %v. Inspect /var/lib/bort/dokploy-live.lock.install-recovery-required before other host mutations.", err))) + case !found: + fmt.Fprintln(w, st.muted("A Dokploy installation was interrupted, but its recovery marker is no longer present. Re-check this run before other host mutations.")) + default: + fmt.Fprintf(w, "%s\n", st.muted(fmt.Sprintf("A Dokploy installation was interrupted after host mutation may have started. Run `%s` to reconcile it before other host mutations.", command))) + } + case "host-busy": + fmt.Fprintln(w, st.muted("Another process is changing this Dokploy host. Wait for that operation to finish, then check this run again.")) + case "empty": + fmt.Fprintln(w, st.muted("This run contains no migratable applications. Platform-role entries are excluded from live apply; create a new run that selects at least one non-platform application.")) + case "inspection-only": + if dokployLiveOperationsSupported() { + fmt.Fprintf(w, "%s\n", st.muted(fmt.Sprintf("This run is available for inspection but lacks local source attestation. %s. %s.", summary.Next.Reason, summary.Next.Action))) + } else { + fmt.Fprintf(w, "%s\n", st.muted(fmt.Sprintf("This run is ready for inspection, but Dokploy live actions are unavailable on %s. Continue on the Linux source host.", runtime.GOOS))) + } + case "source-attestation-error": + fmt.Fprintf(w, "%s\n", st.muted(fmt.Sprintf("The reviewed local Docker source could not be verified: %s. %s.", summary.Next.Reason, summary.Next.Action))) case "applying": - fmt.Fprintf(w, "%s\n", st.muted(fmt.Sprintf("Live apply is running. Run `%s` to attach to its current status.", liveApplyCommand(run)))) + fmt.Fprintf(w, "%s\n", st.muted(fmt.Sprintf("Live apply is running; %s.", summary.Next.Action))) + case "authority-finalizing": + fmt.Fprintf(w, "%s\n", st.muted(fmt.Sprintf("Manual %s authority is recorded, but lifecycle finalization is incomplete; %s.", run.Run.ResolvedAuthority, summary.Next.Action))) + case "authority-ambiguous": + writeAuthorityRecoveryGuidance(w, st, run, "The stored run cannot prove writer or traffic authority. Automatic retry, commit, and rollback are unavailable.") + case "authority-ambiguous-rollback": + writeAuthorityRecoveryGuidance(w, st, run, "An authority-ambiguous recovery is incomplete. Target fencing may be partial and source state may be unchanged.") + case "host-owned": + owner, _, _ := conflictingDokployHostOwner(run.Run) + fmt.Fprintf(w, "%s\n", st.muted(fmt.Sprintf("Dokploy host mutations are owned by %s with %s authority. Finish that run before applying this one.", dokployOwnerRunLabel(owner), owner.Authority))) + case "host-owner-error": + fmt.Fprintf(w, "%s\n", st.muted(fmt.Sprintf("Dokploy host ownership could not be verified: %s. %s.", summary.Next.Reason, summary.Next.Action))) + case "source-recovery": + fmt.Fprintf(w, "%s\n", st.muted(fmt.Sprintf("A historical source pause needs cleanup. Run `%s` to restart the source without transferring state; automatic state migration remains unavailable.", liveApplyCommand(run)))) + case "new-run-required": + fmt.Fprintf(w, "%s\n", st.muted(fmt.Sprintf("Live apply refused this run and no retry can pass it: %s. Next: %s.", summary.Next.Reason, summary.Next.Action))) case "partial": fmt.Fprintf(w, "%s\n", st.muted(fmt.Sprintf("Live apply is incomplete. Run `%s` to resume safely.", liveApplyCommand(run)))) + case "manual-state": + fmt.Fprintln(w, st.muted("Bort cannot continue this run: it was applied with an older plan version that copied state into the deployed target, and Dokploy cannot durably fence target writers during that copy. Complete target setup, state transfer, traffic cutover, and source retirement outside Bort, or create a new run (new runs stage state before the target is deployed).")) + case "plan-blocked": + fmt.Fprintf(w, "%s\n", st.muted(fmt.Sprintf("Live apply would refuse this plan before touching the target: %s. %s.", summary.Next.Reason, summary.Next.Action))) case "applied": - fmt.Fprintf(w, "%s\n", st.muted(fmt.Sprintf("Target is live. Verify it through the rollback window, then run `%s` to retire the source; if validation fails, run `%s` to roll back.", runScopedCommand(run, "commit --apply"), runScopedCommand(run, "rollback --live")))) + if coolifySourceRetirementRequired(run) { + if _, err := planAutomaticRollback(run); err != nil { + fmt.Fprintf(w, "%s\n", st.muted(fmt.Sprintf("Target is live. Automatic rollback is unavailable: %v. If validation fails, preserve both sides and recover authority manually; otherwise, after the rollback window, %s.", err, manualCoolifySourceRetirementAction(run)))) + writeManualRollbackRecoveryCommands(w, st, run) + } else { + fmt.Fprintf(w, "%s\n", st.muted(fmt.Sprintf("Target is live. Verify it through the rollback window, then %s; if validation fails, run `%s` to roll back stateless traffic.", manualCoolifySourceRetirementAction(run), runScopedCommand(run, "rollback --live")))) + } + } else if _, err := planAutomaticRollback(run); err != nil { + fmt.Fprintf(w, "%s\n", st.muted(fmt.Sprintf("Target is live. Automatic rollback is unavailable: %v. If validation fails, preserve both sides and recover authority manually; otherwise run `%s` after the rollback window.", err, runScopedCommand(run, "commit --apply")))) + writeManualRollbackRecoveryCommands(w, st, run) + } else { + fmt.Fprintf(w, "%s\n", st.muted(fmt.Sprintf("Target is live. Verify it through the rollback window, then run `%s` to retire the source; if validation fails, run `%s` to roll back stateless traffic.", runScopedCommand(run, "commit --apply"), runScopedCommand(run, "rollback --live")))) + } case "committed": fmt.Fprintf(w, "%s\n", st.muted(fmt.Sprintf("Target accepted and source containers retired. Run `%s` to audit leftovers.", runScopedCommand(run, "cleanup")))) case "committing": - fmt.Fprintf(w, "%s\n", st.muted(fmt.Sprintf("Source retirement started; rollback is no longer available. Run `%s` to finish acceptance.", runScopedCommand(run, "commit --apply")))) + if strings.HasPrefix(summary.Next.Action, "wait for the active Dokploy host operation") { + fmt.Fprintln(w, st.muted("Source retirement is in progress and rollback is no longer available. Another process holds the host-wide operation lock; wait for it to finish, then check this run again.")) + } else { + fmt.Fprintf(w, "%s\n", st.muted(fmt.Sprintf("Source retirement is in progress and rollback is no longer available; %s.", summary.Next.Action))) + } case "rolled back": fmt.Fprintf(w, "%s\n", st.muted("Rollback returned traffic to the source; the Dokploy target resources remain on the server. To migrate again, "+summary.Next.Action+".")) case "rolling back": - fmt.Fprintf(w, "%s\n", st.muted(fmt.Sprintf("Rollback has not completed; traffic may already be on the source. Run `%s` to finish recovery.", runScopedCommand(run, "rollback --live")))) + if strings.HasPrefix(summary.Next.Action, "wait for the active Dokploy host operation") { + fmt.Fprintln(w, st.muted("Rollback is in progress; traffic or source state may already have changed. Another process holds the host-wide operation lock; wait for it to finish, then check this run again.")) + } else { + fmt.Fprintf(w, "%s\n", st.muted(fmt.Sprintf("Rollback is in progress; traffic or source state may already have changed; %s.", summary.Next.Action))) + } case "purged": fmt.Fprintln(w, st.muted("Migration complete. Target resources and source-control credentials were preserved.")) case "planning": @@ -138,6 +226,54 @@ func writeAppFirstCockpit(w io.Writer, run loadedMigrationRun) { } } +func writeUnsupportedPlatformGuidance(w io.Writer, st *styler, run loadedMigrationRun, phase string) { + state := "This run is ready for inspection." + switch phase { + case "lock-error": + state = "Live apply lock state could not be verified; inspect the run's apply.lock before retrying." + case "host-lock-error": + state = "The host-wide Dokploy operation lock could not be verified; inspect /var/lib/bort/dokploy-live.lock before retrying." + case "host-owner-error": + state = "Dokploy host ownership could not be verified; inspect /var/lib/bort/dokploy-traffic-owner.json before live work." + case "empty": + fmt.Fprintln(w, st.muted("This run contains no migratable applications. Platform-role entries are excluded from live apply; create a new run that selects at least one non-platform application.")) + return + case "applying": + state = "Live apply is recorded as running; inspect the run and its lock before resuming it." + case "authority-finalizing": + state = fmt.Sprintf("Manual %s authority is recorded, but lifecycle finalization is incomplete.", run.Run.ResolvedAuthority) + case "authority-ambiguous", "authority-ambiguous-rollback": + state = "The stored run cannot prove writer or traffic authority; preserve both sides before recovery." + case "host-owned": + state = "Another migration run owns shared Dokploy host mutations." + case "source-recovery": + state = "A historical source pause still needs cleanup before this run can be retired." + case "new-run-required": + state = "Live apply refused the planned state transfer; this run cannot be re-planned and must be unwound before a new run." + case "partial": + state = "Live apply is incomplete and must be resumed from its durable ledger." + case "manual-state": + state = "This stateful run was applied with an older plan version and requires manual transfer and authority recovery." + case "plan-blocked": + state = "Live apply would refuse this plan's state transfer; choose a data store strategy and re-plan, or change the source compose and scan a new run." + case "applied": + state = "The target is recorded live; acceptance or rollback is still pending." + case "committing": + state = "Source retirement started and must be completed; rollback is no longer available." + case "rolling back": + state = "Rollback started and must be completed before other migration work." + case "committed": + state = "The target is accepted and source containers are retired." + case "rolled back": + state = "Traffic is recorded back on the source; target resources remain." + case "purged": + state = "Migration and source purge are complete." + case "planning": + state = "The run still has planning requirements." + } + fmt.Fprintf(w, "%s\n", st.muted(fmt.Sprintf("%s Dokploy live actions are unavailable on %s; continue any mutation or recovery on the Linux source host.", state, runtime.GOOS))) +} + func writeCockpitDecisions(w io.Writer, st *styler, decisions []runDecision) { limit := min(len(decisions), 3) for _, decision := range decisions[:limit] { @@ -149,32 +285,109 @@ func writeCockpitDecisions(w io.Writer, st *styler, decisions []runDecision) { } func migrationRunPhase(run loadedMigrationRun) string { - applyActive, applyActiveErr := applyRunActive(run.Run.RunDir) - switch { - case run.Run.PurgedAt != nil: + if !dokployLiveOperationsSupported() { + return migrationRunPhaseWithNext(run, runNextStep{}) + } + return migrationRunPhaseWithNext(run, nextSafeStep(run, nil)) +} + +func migrationRunPhaseWithNext(run loadedMigrationRun, next runNextStep) string { + applyActive, applyActiveErr := false, error(nil) + if run.Run.LiveAppliedAt == nil { + applyActive, applyActiveErr = applyRunActive(run.Run.RunDir) + } + ownerSupported := dokployLiveOperationsSupported() + if authorityRecoveryPending(run) { + return "authority-finalizing" + } + if ownerSupported { + ownerFinalizationPending, ownerFinalizationErr := completedDokployOwnerFinalization(run) + if ownerFinalizationErr != nil { + return "host-owner-error" + } + if ownerFinalizationPending { + if run.Run.RolledBackAt != nil { + return "rolling back" + } + return "committing" + } + } + if run.Run.PurgedAt != nil { return "purged" - case run.Run.CommittedAt != nil: + } + if run.Run.CommittedAt != nil { return "committed" - case run.Run.RolledBackAt != nil: + } + if run.Run.RolledBackAt != nil && !authorityRecoveryPending(run) { return "rolled back" - case run.Run.RollbackStartedAt != nil: + } + if next.Phase != "" { + return next.Phase + } + hostOperationActive, hostOperationErr := false, error(nil) + var hostOwned bool + var hostOwnerErr error + var trafficOwnerErr error + if ownerSupported { + hostOperationActive, hostOperationErr = dokployLiveOperationActive() + _, hostOwned, hostOwnerErr = conflictingDokployHostOwner(run.Run) + trafficOwnerErr = validateCurrentDokployTrafficOwner(run, rollbackInProgress(run)) + } + switch { + case rollbackInProgress(run) && runMayHaveAmbiguousAuthority(run): + return "authority-ambiguous-rollback" + case rollbackInProgress(run) && trafficOwnerErr != nil && hostOwnerErr == nil: + return "authority-ambiguous-rollback" + case rollbackInProgress(run): return "rolling back" + case run.Run.CommitStartedAt != nil && trafficOwnerErr != nil && hostOwnerErr == nil: + return "authority-ambiguous" case run.Run.CommitStartedAt != nil: return "committing" - case run.Run.LiveAppliedAt != nil || liveApplySucceeded(run): - return "applied" - case applyActiveErr != nil: - return "lock-error" case applyActive: return "applying" + case hostOwnerErr != nil: + return "host-owner-error" + case applyActiveErr != nil: + return "lock-error" + case errors.Is(hostOperationErr, errDokployInstallRecoveryRequired): + return "install-recovery" + case hostOperationErr != nil: + return "host-lock-error" + case hostOperationActive: + return "host-busy" + case manualTargetAuthorityRecorded(run) && trafficOwnerErr != nil: + return "host-owner-error" + case manualTargetAuthorityRecorded(run): + return "applied" + case runMayHaveAmbiguousAuthority(run): + return "authority-ambiguous" + case (run.Run.LiveAppliedAt != nil || liveApplySucceeded(run)) && trafficOwnerErr != nil: + return "authority-ambiguous" + case run.Run.LiveAppliedAt != nil || liveApplySucceeded(run): + return "applied" + case hostOwned: + return "host-owned" + case validateStatefulLiveApply(run) != nil && len(interruptedStatefulSourceCleanup(run)) > 0: + return "source-recovery" + case validateStatefulLiveApply(run) != nil: + return "manual-state" + case appliedRequiresNewRun(run.Applied): + return "new-run-required" case len(run.Applied.Steps) > 0: return "partial" + case stagedTransferRefusal(run) != nil: + return "plan-blocked" + case !hasMigratableRunApps(run): + return "empty" case len(openSetupDecisions(run)) > 0: return "planning" - case len(liveApplyBlockingDecisions(run)) == 0: - return "ready" - default: + case len(liveApplyBlockingDecisions(run)) > 0: return "planning" + case !ownerSupported: + return "inspection-only" + default: + return "ready" } } @@ -184,8 +397,32 @@ func migrationRunPhaseLabel(phase string) string { return "TARGET LIVE" case "purged": return "COMPLETE" - case "lock-error": + case "lock-error", "host-lock-error", "host-owner-error": return "LOCK ERROR" + case "install-recovery": + return "INSTALL RECOVERY" + case "host-busy": + return "HOST BUSY" + case "host-owned": + return "HOST OWNED" + case "empty": + return "NO APPS" + case "inspection-only": + return "INSPECTION ONLY" + case "source-attestation-error": + return "SOURCE CHANGED" + case "authority-ambiguous", "authority-ambiguous-rollback": + return "AUTHORITY UNKNOWN" + case "authority-finalizing": + return "RECOVERY PENDING" + case "manual-state": + return "MANUAL STATE" + case "plan-blocked": + return "PLAN BLOCKED" + case "source-recovery": + return "SOURCE RECOVERY" + case "new-run-required": + return "NEW RUN REQUIRED" default: return strings.ToUpper(phase) } @@ -195,13 +432,59 @@ func severityForMigrationRunPhase(phase string) severity { switch phase { case "ready", "applied", "committed", "purged": return sevGood - case "partial", "lock-error": + case "partial", "new-run-required", "lock-error", "host-lock-error", "host-owner-error", "install-recovery", "host-busy", "host-owned", "authority-ambiguous", "authority-ambiguous-rollback", "authority-finalizing", "source-attestation-error": return sevBad default: return sevWarn } } +func manualTargetAuthorityRecorded(run loadedMigrationRun) bool { + return run.Run.ResolvedAuthority == dokployTrafficTarget && run.Run.LiveAppliedAt != nil +} + +func rollbackInProgress(run loadedMigrationRun) bool { + return run.Run.RollbackStartedAt != nil && !manualTargetAuthorityRecorded(run) +} + +func authorityRecoveryPending(run loadedMigrationRun) bool { + switch run.Run.ResolvedAuthority { + case dokployTrafficSource: + return run.Run.AuthorityFinalizedAt == nil + case dokployTrafficTarget: + return run.Run.LiveAppliedAt == nil + default: + return false + } +} + +func authorityRecoveryAvailable(run loadedMigrationRun) bool { + owner, found, err := readDokployTrafficOwner() + if err != nil || !found || owner.Authority == dokployTrafficReleased { + return false + } + runID, err := dokployTrafficRunID(run.Run) + return err == nil && owner.RunID == runID +} + +func writeAuthorityRecoveryGuidance(w io.Writer, st *styler, run loadedMigrationRun, prefix string) { + if !authorityRecoveryAvailable(run) { + fmt.Fprintln(w, st.muted(prefix+" Inspect and preserve both sides, establish authority manually, and start a fresh migration run.")) + return + } + fmt.Fprintln(w, st.muted(prefix+" After manually fencing the other side and verifying the chosen writer and traffic authority, finish this owner-bound run with one of:")) + fmt.Fprintf(w, "%s\n", st.muted(" source: `"+authorityRecoveryCommand(run, dokployTrafficSource)+"`")) + fmt.Fprintf(w, "%s\n", st.muted(" target: `"+authorityRecoveryCommand(run, dokployTrafficTarget)+"`")) +} + +func writeManualRollbackRecoveryCommands(w io.Writer, st *styler, run loadedMigrationRun) { + if run.Run.ResolvedAuthority != "" || !authorityRecoveryAvailable(run) { + return + } + fmt.Fprintln(w, st.muted(" After manually restoring and verifying source authority, record it with:")) + fmt.Fprintf(w, "%s\n", st.muted(" source: `"+authorityRecoveryCommand(run, dokployTrafficSource)+"`")) +} + func issueActionFooter(apps []appView) string { hasFix, hasNext := false, false for _, app := range apps { diff --git a/internal/cli/cockpit_test.go b/internal/cli/cockpit_test.go index b50d75d..645c79c 100644 --- a/internal/cli/cockpit_test.go +++ b/internal/cli/cockpit_test.go @@ -1,6 +1,9 @@ package cli import ( + "context" + "errors" + "io" "os" "path/filepath" "strings" @@ -9,6 +12,8 @@ import ( "github.com/aikins01/bort/internal/manifest" "github.com/aikins01/bort/internal/preparer" + syncplan "github.com/aikins01/bort/internal/sync" + "github.com/aikins01/bort/internal/target/dokploy" ) func TestSafeRunArtifactPathRejectsEscapes(t *testing.T) { @@ -239,11 +244,668 @@ func TestAppliedFooterCountsOkAndError(t *testing.T) { } } -func TestCockpitShowsDownstreamDecisionsAsReviewOnly(t *testing.T) { +func TestCockpitLabelsAmbiguousLegacyHandoffAndScopesRollback(t *testing.T) { + workDir := t.TempDir() + t.Chdir(workDir) + externalDir := filepath.Join(t.TempDir(), "legacy-run") + if err := os.MkdirAll(externalDir, 0o700); err != nil { + t.Fatal(err) + } + run := ambiguousLegacyRunForTest("legacy-run", externalDir) + if err := validateApplyResumeAuthority(run.Applied); err == nil || !strings.Contains(err.Error(), "after the shared proxy handoff") { + t.Fatalf("legacy fixture is not ambiguous through the proxy handoff alone: %v", err) + } + + if phase := migrationRunPhase(run); phase != "authority-ambiguous" { + t.Fatalf("expected authority-ambiguous phase, got %q", phase) + } + var output strings.Builder + writeAppFirstCockpit(&output, run) + for _, want := range []string{"AUTHORITY UNKNOWN", "Inspect and preserve both sides", "Automatic retry, commit, and rollback are unavailable", "establish authority manually"} { + if !strings.Contains(output.String(), want) { + t.Fatalf("expected cockpit to contain %q, got:\n%s", want, output.String()) + } + } + now := time.Now().UTC() + run.Run.RollbackStartedAt = &now + if phase := migrationRunPhase(run); phase != "authority-ambiguous-rollback" { + t.Fatalf("expected interrupted ambiguous rollback phase, got %q", phase) + } + output.Reset() + writeAppFirstCockpit(&output, run) + for _, want := range []string{"AUTHORITY UNKNOWN", "Target fencing may be partial", "source state may be unchanged", "Inspect and preserve both sides"} { + if !strings.Contains(output.String(), want) { + t.Fatalf("expected interrupted ambiguous rollback cockpit to contain %q, got:\n%s", want, output.String()) + } + } + next := nextSafeStep(run, nil) + if !strings.Contains(next.Action, "inspect and preserve both source and target data") || !strings.Contains(next.Reason, "fencing may be partial") { + t.Fatalf("expected interrupted ambiguous rollback guidance, got %#v", next) + } +} + +func TestCockpitPrioritizesActiveApplyOverAmbiguousStep(t *testing.T) { + run := writeAmbiguousAuthorityRun(t, "active-ambiguous") + lock, err := acquireApplyLock(filepath.Join(run.Run.RunDir, "apply.lock")) + if err != nil { + t.Fatal(err) + } + defer lock.Release() + if phase := migrationRunPhase(run); phase != "applying" { + t.Fatalf("active ambiguous run phase=%q, want applying", phase) + } + next := nextSafeStep(run, nil) + if !strings.Contains(next.Action, "view the active apply") || strings.Contains(next.Action, "recover-authority") { + t.Fatalf("active apply next step is unsafe: %#v", next) + } +} + +func TestCockpitShowsConflictingHostOwner(t *testing.T) { + resetDokployTrafficOwner(t) + ownerRun := migrationRun{Name: "owner-run", RunDir: filepath.Join(t.TempDir(), "owner-run"), CreatedAt: time.Now().UTC(), BundleDigest: "owner-digest"} + if err := claimDokployHostOwnership(ownerRun, "http://127.0.0.1:3030", dokployCredentialID("test-token")); err != nil { + t.Fatal(err) + } + if err := markDokployTrafficTarget(ownerRun, "http://127.0.0.1:3030"); err != nil { + t.Fatal(err) + } run := loadedMigrationRun{ - Run: migrationRun{Name: "reviewed", RunDir: t.TempDir(), Target: "dokploy"}, + Run: migrationRun{Name: "waiting-run", RunDir: t.TempDir(), Target: "dokploy", CreatedAt: time.Now().UTC(), BundleDigest: "waiting-digest"}, + Prepare: preparer.Result{Apps: []preparer.AppPlan{{Name: "api"}}}, + } + if phase := migrationRunPhase(run); phase != "host-owned" { + t.Fatalf("expected host-owned phase, got %q", phase) + } + var output strings.Builder + writeAppFirstCockpit(&output, run) + for _, want := range []string{"HOST OWNED", `owned by migration run "owner-run"`, ownerRun.RunDir, "target authority"} { + if !strings.Contains(output.String(), want) { + t.Fatalf("expected cockpit to contain %q, got:\n%s", want, output.String()) + } + } + next := nextSafeStep(run, nil) + if !strings.Contains(next.Action, `finish migration run "owner-run"`) || !strings.Contains(next.Action, ownerRun.RunDir) || !strings.Contains(next.Reason, "target authority") { + t.Fatalf("expected owning-run guidance, got %#v", next) + } +} + +func TestCockpitShowsHostOwnerGuidanceForEmptyRun(t *testing.T) { + resetDokployTrafficOwner(t) + ownerRun := migrationRun{Name: "owner-run", RunDir: filepath.Join(t.TempDir(), "owner-run"), CreatedAt: time.Now().UTC(), BundleDigest: "owner-digest"} + if err := claimDokployHostOwnership(ownerRun, "http://127.0.0.1:3030", dokployCredentialID("owner-token")); err != nil { + t.Fatal(err) + } + run := loadedMigrationRun{Run: migrationRun{Name: "empty-run", RunDir: t.TempDir(), Target: "dokploy", CreatedAt: time.Now().UTC(), BundleDigest: "empty-digest"}} + var output strings.Builder + writeAppFirstCockpit(&output, run) + for _, want := range []string{"No apps in this run", "HOST OWNED", `migration run "owner-run"`, ownerRun.RunDir} { + if !strings.Contains(output.String(), want) { + t.Fatalf("empty-run cockpit missing %q:\n%s", want, output.String()) + } + } +} + +func TestCockpitRefusesPlatformOnlyLiveApply(t *testing.T) { + resetDokployTrafficOwner(t) + run := loadedMigrationRun{ + Run: migrationRun{Name: "platform-only", RunDir: t.TempDir(), Target: "dokploy"}, Prepare: preparer.Result{Apps: []preparer.AppPlan{{ - Name: "api", + Name: "coolify-proxy", + Role: "platform", + }}}, + } + if phase := migrationRunPhase(run); phase != "empty" { + t.Fatalf("platform-only phase=%q, want empty", phase) + } + var output strings.Builder + writeAppFirstCockpit(&output, run) + if !strings.Contains(output.String(), "NO APPS") || !strings.Contains(output.String(), "no migratable applications") || strings.Contains(output.String(), "migrate --live") { + t.Fatalf("platform-only cockpit offered live apply:\n%s", output.String()) + } + next := nextSafeStep(run, nil) + if !strings.Contains(next.Action, "at least one non-platform application") || strings.Contains(next.Action, "migrate --live") { + t.Fatalf("platform-only next step offered live apply: %#v", next) + } + if err := validateLiveApplyReady(run); err == nil || !strings.Contains(err.Error(), "no migratable applications") { + t.Fatalf("platform-only run passed live validation: %v", err) + } +} + +func TestCockpitKeepsStaticallyUnattestedSourcesInspectionOnly(t *testing.T) { + newRun := func() loadedMigrationRun { + return loadedMigrationRun{ + Run: migrationRun{Name: "unattested", RunDir: t.TempDir(), Target: "dokploy", Source: "docker"}, + Prepare: preparer.Result{ + Source: "docker", + SourceDockerEngineID: "engine-reviewed", + Apps: []preparer.AppPlan{{ + Name: "api", + Resources: preparer.ResourceSpecs{SourceServices: []preparer.SourceServiceRef{{ContainerID: "0123456789ab", ContainerName: "api"}}}, + }}, + }, + } + } + for _, test := range []struct { + name string + configure func(*loadedMigrationRun) + wantReason string + }{ + {name: "imported manifest", configure: func(run *loadedMigrationRun) { run.Run.Source = "manifest" }, wantReason: "imported manifest"}, + {name: "imported bundle", configure: func(run *loadedMigrationRun) { run.Run.Source = "" }, wantReason: "imported bundle"}, + {name: "missing engine", configure: func(run *loadedMigrationRun) { run.Prepare.SourceDockerEngineID = "" }, wantReason: "no reviewed Docker engine identity"}, + {name: "missing container", configure: func(run *loadedMigrationRun) { run.Prepare.Apps[0].Resources.SourceServices[0].ContainerID = "" }, wantReason: "no stable container ID"}, + } { + t.Run(test.name, func(t *testing.T) { + resetDokployTrafficOwner(t) + run := newRun() + test.configure(&run) + next := nextSafeStep(run, nil) + if next.Phase != "inspection-only" || !strings.Contains(next.Action, "new named migration run") || !strings.Contains(next.Reason, test.wantReason) || strings.Contains(next.Action, "migrate --live") { + t.Fatalf("unattested source received unsafe guidance: %#v", next) + } + if phase := migrationRunPhase(run); phase != "inspection-only" { + t.Fatalf("phase=%q, want inspection-only", phase) + } + var output strings.Builder + writeAppFirstCockpit(&output, run) + if !strings.Contains(output.String(), "INSPECTION ONLY") || !strings.Contains(output.String(), test.wantReason) || strings.Contains(output.String(), "migrate --live") { + t.Fatalf("cockpit hid static source-attestation failure:\n%s", output.String()) + } + }) + } +} + +func TestCockpitBlocksReadyRunWhenRuntimeSourceAttestationFails(t *testing.T) { + resetDokployTrafficOwner(t) + previous := verifyLocalSourceRun + verifyLocalSourceRun = func(context.Context, loadedMigrationRun) error { return context.DeadlineExceeded } + t.Cleanup(func() { verifyLocalSourceRun = previous }) + run := loadedMigrationRun{ + Run: migrationRun{Name: "source-changed", RunDir: t.TempDir(), Target: "dokploy", Source: "docker"}, + Prepare: preparer.Result{ + Source: "docker", + SourceDockerEngineID: "engine-reviewed", + Apps: []preparer.AppPlan{{ + Name: "api", + Resources: preparer.ResourceSpecs{SourceServices: []preparer.SourceServiceRef{{ContainerID: "0123456789ab", ContainerName: "api"}}}, + }}, + }, + } + next := nextSafeStep(run, nil) + if next.Phase != "source-attestation-error" || !strings.Contains(next.Reason, context.DeadlineExceeded.Error()) || strings.Contains(next.Action, "migrate --live") { + t.Fatalf("runtime source-attestation failure received unsafe guidance: %#v", next) + } + if phase := migrationRunPhase(run); phase != "source-attestation-error" { + t.Fatalf("phase=%q, want source-attestation-error", phase) + } + var output strings.Builder + writeAppFirstCockpit(&output, run) + if !strings.Contains(output.String(), "SOURCE CHANGED") || !strings.Contains(output.String(), context.DeadlineExceeded.Error()) || strings.Contains(output.String(), "migrate --live") { + t.Fatalf("cockpit hid runtime source-attestation failure:\n%s", output.String()) + } +} + +func TestMigrationSummaryPropagatesCancellationToSourceAttestation(t *testing.T) { + resetDokployTrafficOwner(t) + previous := verifyLocalSourceRun + verifyLocalSourceRun = func(ctx context.Context, _ loadedMigrationRun) error { return ctx.Err() } + t.Cleanup(func() { verifyLocalSourceRun = previous }) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + run := loadedMigrationRun{ + Run: migrationRun{Name: "canceled", RunDir: t.TempDir(), Target: "dokploy", Source: "docker"}, + Prepare: preparer.Result{ + Source: "docker", + SourceDockerEngineID: "engine-reviewed", + Apps: []preparer.AppPlan{{ + Name: "api", + Resources: preparer.ResourceSpecs{SourceServices: []preparer.SourceServiceRef{{ContainerID: "0123456789ab", ContainerName: "api"}}}, + }}, + }, + } + + summary := summarizeMigrationRunContext(ctx, run) + if summary.Next.Phase != "source-attestation-error" || !strings.Contains(summary.Next.Reason, context.Canceled.Error()) { + t.Fatalf("canceled source attestation produced unsafe guidance: %#v", summary.Next) + } +} + +func TestLiveMigrationSummaryDefersRuntimeSourceAttestation(t *testing.T) { + resetDokployTrafficOwner(t) + previous := verifyLocalSourceRun + calls := 0 + verifyLocalSourceRun = func(context.Context, loadedMigrationRun) error { + calls++ + return nil + } + t.Cleanup(func() { verifyLocalSourceRun = previous }) + run := loadedMigrationRun{ + Run: migrationRun{Name: "live", RunDir: t.TempDir(), Target: "dokploy", Source: "docker"}, + Prepare: preparer.Result{ + Source: "docker", + SourceDockerEngineID: "engine-reviewed", + Apps: []preparer.AppPlan{{ + Name: "api", + Resources: preparer.ResourceSpecs{SourceServices: []preparer.SourceServiceRef{{ContainerID: "0123456789ab", ContainerName: "api"}}}, + }}, + }, + } + + summary := summarizeMigrationRunForLive(context.Background(), run) + if calls != 0 || !strings.Contains(summary.Next.Action, "migrate --live") { + t.Fatalf("live summary probed the source or hid the ready action: calls=%d next=%#v", calls, summary.Next) + } + summarizeMigrationRunContext(context.Background(), run) + if calls != 1 { + t.Fatalf("read-only summary source probes=%d, want 1", calls) + } +} + +func TestCockpitBlocksMutationGuidanceWhenStartedRunSourceIsUnattested(t *testing.T) { + now := time.Now().UTC() + for _, test := range []struct { + name string + configure func(*loadedMigrationRun) + }{ + { + name: "partial apply", + configure: func(run *loadedMigrationRun) { + run.Applied.Steps = []appliedStep{{Index: 0, Kind: string(dokploy.StepCreateProject), App: "api", Ref: "api", Status: string(dokploy.StepStatusOK)}} + }, + }, + { + name: "target live", + configure: func(run *loadedMigrationRun) { + run.Run.LiveAppliedAt = &now + }, + }, + { + name: "stateful target live", + configure: func(run *loadedMigrationRun) { + run.Run.LiveAppliedAt = &now + run.Sync = syncplan.Result{Apps: []syncplan.AppPlan{{Name: "api", Steps: []syncplan.Step{{ + ResourceType: "volume", + ResourceRef: "volume:web -> /data", + Strategy: syncplan.StrategyDockerVolumeArchive, + }}}}} + }, + }, + } { + t.Run(test.name, func(t *testing.T) { + resetDokployTrafficOwner(t) + run := loadedMigrationRun{ + Run: migrationRun{Name: "started", RunDir: t.TempDir(), Target: "dokploy", Source: "docker"}, + Prepare: preparer.Result{ + Source: "docker", + SourceDockerEngineID: "engine-reviewed", + Apps: []preparer.AppPlan{{ + Name: "api", + Resources: preparer.ResourceSpecs{SourceServices: []preparer.SourceServiceRef{{ContainerID: "0123456789ab"}}}, + }}, + }, + } + test.configure(&run) + next := nextSafeStep(run, nil) + if next.Phase != "source-attestation-error" || !strings.Contains(next.Reason, "no reviewed container name") || !strings.Contains(next.Action, "do not replay") { + t.Fatalf("started unattested run received unsafe guidance: %#v", next) + } + for _, command := range []string{"migrate --live", "commit --apply", "rollback --live"} { + if strings.Contains(next.Action, command) { + t.Fatalf("started unattested run offered %q: %#v", command, next) + } + } + if phase := migrationRunPhase(run); phase != "source-attestation-error" { + t.Fatalf("phase=%q, want source-attestation-error", phase) + } + var output strings.Builder + writeAppFirstCockpit(&output, run) + if !strings.Contains(output.String(), "SOURCE CHANGED") || !strings.Contains(output.String(), "do not replay") { + t.Fatalf("cockpit hid started source-attestation failure:\n%s", output.String()) + } + for _, command := range []string{"migrate --live", "commit --apply", "rollback --live"} { + if strings.Contains(output.String(), command) { + t.Fatalf("cockpit offered %q for started unattested run:\n%s", command, output.String()) + } + } + }) + } + + run := loadedMigrationRun{ + Run: migrationRun{Name: "active", RunDir: t.TempDir(), Target: "dokploy", Source: "docker"}, + Prepare: preparer.Result{ + Source: "docker", + SourceDockerEngineID: "engine-reviewed", + Apps: []preparer.AppPlan{{ + Name: "api", + Resources: preparer.ResourceSpecs{SourceServices: []preparer.SourceServiceRef{{ContainerID: "0123456789ab"}}}, + }}, + }, + } + lock, err := acquireApplyLock(filepath.Join(run.Run.RunDir, "apply.lock")) + if err != nil { + t.Fatal(err) + } + defer lock.Release() + next := nextSafeStep(run, nil) + if next.Phase == "" || !strings.Contains(next.Reason, "no reviewed container name") || strings.Contains(next.Action, "migrate --live") { + t.Fatalf("active unattested run received unsafe attach guidance: %#v", next) + } +} + +func TestCockpitReportsUnreadableHostOwner(t *testing.T) { + resetDokployTrafficOwner(t) + path, err := dokployTrafficOwnerPath() + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte("{invalid"), 0o600); err != nil { + t.Fatal(err) + } + run := loadedMigrationRun{Run: migrationRun{Name: "blocked", RunDir: t.TempDir(), Target: "dokploy"}} + var output strings.Builder + writeAppFirstCockpit(&output, run) + if !strings.Contains(output.String(), "LOCK ERROR") || !strings.Contains(output.String(), "decode Dokploy traffic owner") || strings.Contains(output.String(), "fresh migration run") { + t.Fatalf("cockpit hid the durable-owner read failure:\n%s", output.String()) + } + next := nextSafeStep(run, nil) + if !strings.Contains(next.Action, "dokploy-traffic-owner.json") || !strings.Contains(next.Reason, "decode Dokploy traffic owner") { + t.Fatalf("next step hid the durable-owner read failure: %#v", next) + } +} + +func TestCockpitPreservesActiveLifecycleWhenHostOwnerIsUnreadable(t *testing.T) { + resetDokployTrafficOwner(t) + path, err := dokployTrafficOwnerPath() + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte("{invalid"), 0o600); err != nil { + t.Fatal(err) + } + now := time.Now().UTC() + for _, test := range []struct { + name string + configure func(*migrationRun) + phase string + status string + context string + }{ + { + name: "commit", + configure: func(run *migrationRun) { run.CommitStartedAt = &now }, + phase: "committing", + status: "COMMITTING", + context: "rollback is no longer available", + }, + { + name: "rollback", + configure: func(run *migrationRun) { run.RollbackStartedAt = &now }, + phase: "rolling back", + status: "ROLLING BACK", + context: "traffic or source state may already have changed", + }, + { + name: "authority finalization", + configure: func(run *migrationRun) { + run.ResolvedAuthority = dokployTrafficSource + run.AuthorityResolvedAt = &now + }, + phase: "authority-finalizing", + status: "RECOVERY PENDING", + context: "lifecycle finalization is incomplete", + }, + } { + t.Run(test.name, func(t *testing.T) { + meta := migrationRun{Name: test.name, RunDir: t.TempDir(), Target: "dokploy", LiveAppliedAt: &now} + test.configure(&meta) + run := loadedMigrationRun{Run: meta} + if phase := migrationRunPhase(run); phase != test.phase { + t.Fatalf("phase=%q, want %q", phase, test.phase) + } + next := nextSafeStep(run, nil) + if !strings.Contains(next.Action, "dokploy-traffic-owner.json") || !strings.Contains(next.Reason, "decode Dokploy traffic owner") { + t.Fatalf("next step hid lifecycle or owner failure: %#v", next) + } + var output strings.Builder + writeAppFirstCockpit(&output, run) + if !strings.Contains(output.String(), test.status) || !strings.Contains(output.String(), test.context) || !strings.Contains(output.String(), "dokploy-traffic-owner.json") || strings.Contains(output.String(), "LOCK ERROR") { + t.Fatalf("cockpit hid lifecycle behind owner failure:\n%s", output.String()) + } + }) + } +} + +func TestCockpitFinalizesCompletedDurableOwnerBeforeTerminalGuidance(t *testing.T) { + for _, test := range []struct { + name string + configureRun func(*migrationRun, *time.Time) + configureOwner func(migrationRun) error + phase string + resumeCommand string + releaseOwner func(migrationRun) error + terminalPhase string + }{ + { + name: "commit", + configureRun: func(run *migrationRun, now *time.Time) { + run.CommitStartedAt = now + run.CommittedAt = now + }, + configureOwner: func(run migrationRun) error { return markDokployTrafficTarget(run, "http://127.0.0.1:3030") }, + phase: "committing", + resumeCommand: "commit --apply", + releaseOwner: releaseDokployTargetOwner, + terminalPhase: "committed", + }, + { + name: "rollback", + configureRun: func(run *migrationRun, now *time.Time) { + run.RollbackStartedAt = now + run.RolledBackAt = now + }, + configureOwner: func(run migrationRun) error { return markDokployTrafficSource(run) }, + phase: "rolling back", + resumeCommand: "rollback --live", + releaseOwner: releaseDokployTrafficOwner, + terminalPhase: "rolled back", + }, + } { + t.Run(test.name, func(t *testing.T) { + resetDokployTrafficOwner(t) + now := time.Now().UTC() + meta := migrationRun{Name: test.name, RunDir: filepath.Join(t.TempDir(), test.name), Target: "dokploy", CreatedAt: now, BundleDigest: test.name + "-digest"} + test.configureRun(&meta, &now) + meta = persistRunFixture(t, meta) + if err := claimDokployHostOwnership(meta, "http://127.0.0.1:3030", dokployCredentialID("test-token")); err != nil { + t.Fatal(err) + } + if err := test.configureOwner(meta); err != nil { + t.Fatal(err) + } + run := loadedMigrationRun{Run: meta, Applied: newRunApplied(meta)} + if phase := migrationRunPhase(run); phase != test.phase { + t.Fatalf("phase=%q, want %q", phase, test.phase) + } + next := nextSafeStep(run, nil) + if !strings.Contains(next.Action, test.resumeCommand) || !strings.Contains(next.Reason, "not released") { + t.Fatalf("next step did not expose owner finalization: %#v", next) + } + if err := test.releaseOwner(meta); err != nil { + t.Fatal(err) + } + if phase := migrationRunPhase(run); phase != test.terminalPhase { + t.Fatalf("released phase=%q, want %q", phase, test.terminalPhase) + } + }) + } +} + +func TestCockpitManualTargetAuthorityDefersToHostOperationLock(t *testing.T) { + resetDokployTrafficOwner(t) + previous := verifyLocalSourceRun + verifyLocalSourceRun = func(context.Context, loadedMigrationRun) error { return nil } + t.Cleanup(func() { verifyLocalSourceRun = previous }) + now := time.Now().UTC() + rollbackStarted := now.Add(-time.Minute) + meta := migrationRun{Name: "manual-target", RunDir: filepath.Join(t.TempDir(), "manual-target"), Target: "dokploy", Source: "docker", CreatedAt: now, BundleDigest: "manual-digest", RollbackStartedAt: &rollbackStarted, LiveAppliedAt: &now, ResolvedAuthority: dokployTrafficTarget, AuthorityResolvedAt: &now} + if err := claimDokployHostOwnership(meta, "http://127.0.0.1:3030", dokployCredentialID("token")); err != nil { + t.Fatal(err) + } + if err := markDokployTrafficTarget(meta, "http://127.0.0.1:3030"); err != nil { + t.Fatal(err) + } + run := loadedMigrationRun{ + Run: meta, + Prepare: preparer.Result{Source: "docker", SourceDockerEngineID: "engine-reviewed", Apps: []preparer.AppPlan{{Name: "api", Resources: preparer.ResourceSpecs{SourceServices: []preparer.SourceServiceRef{{ContainerID: "0123456789ab", ContainerName: "api"}}}}}}, + Applied: newRunApplied(meta), + } + next := nextSafeStep(run, nil) + if !strings.Contains(next.Action, "commit --apply") || strings.Contains(next.Action, "recover-authority") { + t.Fatalf("next step treated the resolved run as an unfinished rollback: %#v", next) + } + if phase := migrationRunPhase(run); phase != "applied" { + t.Fatalf("phase=%q, want applied before the host lock is held", phase) + } + hostLock, err := acquireDokployLiveOperationLock() + if err != nil { + t.Fatal(err) + } + defer hostLock.Release() + next = nextSafeStep(run, nil) + if !strings.Contains(next.Action, "wait for the active Dokploy host operation") || strings.Contains(next.Action, "commit --apply") { + t.Fatalf("next step ignored the host-wide operation lock: %#v", next) + } + if phase := migrationRunPhase(run); phase != "host-busy" { + t.Fatalf("phase=%q disagrees with next step %q", phase, next.Action) + } +} + +func TestCockpitBlocksCompletedOrRecoveredRunWithoutProvableOwner(t *testing.T) { + for _, test := range []struct { + name string + configure func(*testing.T, migrationRun) + wantReason string + manualTarget bool + }{ + {name: "missing", configure: func(*testing.T, migrationRun) {}, wantReason: "no durable Dokploy host owner"}, + {name: "different", configure: func(t *testing.T, _ migrationRun) { + other := migrationRun{Name: "other", RunDir: filepath.Join(t.TempDir(), "other"), CreatedAt: time.Now().UTC(), BundleDigest: "other-digest"} + if err := claimDokployHostOwnership(other, "http://127.0.0.1:3030", dokployCredentialID("other-token")); err != nil { + t.Fatal(err) + } + if err := markDokployTrafficTarget(other, "http://127.0.0.1:3030"); err != nil { + t.Fatal(err) + } + }, wantReason: "ownership belongs to"}, + {name: "unreadable", configure: func(t *testing.T, _ migrationRun) { + path, err := dokployTrafficOwnerPath() + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte("{invalid"), 0o600); err != nil { + t.Fatal(err) + } + }, wantReason: "decode Dokploy traffic owner"}, + {name: "manual target missing", configure: func(*testing.T, migrationRun) {}, wantReason: "no durable host-wide Dokploy traffic owner", manualTarget: true}, + } { + t.Run(test.name, func(t *testing.T) { + resetDokployTrafficOwner(t) + now := time.Now().UTC() + meta := migrationRun{Name: test.name, RunDir: filepath.Join(t.TempDir(), test.name), Target: "dokploy", CreatedAt: now, BundleDigest: test.name + "-digest", CommitStartedAt: &now, CommittedAt: &now} + if test.manualTarget { + meta.CommitStartedAt = nil + meta.CommittedAt = nil + meta.ResolvedAuthority = dokployTrafficTarget + meta.AuthorityResolvedAt = &now + meta.LiveAppliedAt = &now + } + test.configure(t, meta) + run := loadedMigrationRun{Run: meta, Applied: newRunApplied(meta)} + if phase := migrationRunPhase(run); phase != "host-owner-error" { + t.Fatalf("phase=%q, want host-owner-error", phase) + } + next := nextSafeStep(run, nil) + if !strings.Contains(next.Action, "dokploy-traffic-owner.json") || !strings.Contains(next.Reason, test.wantReason) || strings.Contains(next.Action, "commit --apply") || strings.Contains(next.Action, "cleanup") { + t.Fatalf("next step hid owner finalization failure: %#v", next) + } + }) + } +} + +func TestCockpitRefusesAppliedActionsWithoutMatchingCurrentOwner(t *testing.T) { + for _, tc := range []struct { + name string + installOwner func(*testing.T) + }{ + {name: "missing owner", installOwner: func(t *testing.T) {}}, + {name: "different owner", installOwner: func(t *testing.T) { + ownerRun := migrationRun{Name: "other-run", RunDir: filepath.Join(t.TempDir(), "other-run"), CreatedAt: time.Now().UTC(), BundleDigest: "other-digest"} + if err := claimDokployHostOwnership(ownerRun, "http://127.0.0.1:3030", dokployCredentialID("other-token")); err != nil { + t.Fatal(err) + } + if err := markDokployTrafficTarget(ownerRun, "http://127.0.0.1:3030"); err != nil { + t.Fatal(err) + } + }}, + } { + t.Run(tc.name, func(t *testing.T) { + resetDokployTrafficOwner(t) + runMeta := migrationRun{Name: "applied-run", RunDir: filepath.Join(t.TempDir(), "applied-run"), Target: "dokploy", CreatedAt: time.Now().UTC(), BundleDigest: "applied-digest"} + now := time.Now().UTC() + run := loadedMigrationRun{ + Run: runMeta, + Prepare: preparer.Result{Apps: []preparer.AppPlan{{Name: "api"}}}, + Applied: newRunApplied(runMeta), + } + run.Run.LiveAppliedAt = &now + run.Applied.SucceededAt = &now + tc.installOwner(t) + + if phase := migrationRunPhase(run); phase != "authority-ambiguous" { + t.Fatalf("applied run without matching owner phase=%q, want authority-ambiguous", phase) + } + var output strings.Builder + writeAppFirstCockpit(&output, run) + if !strings.Contains(output.String(), "AUTHORITY UNKNOWN") || !strings.Contains(output.String(), "establish authority manually") || strings.Contains(output.String(), "commit --apply") || strings.Contains(output.String(), "rollback --live") { + t.Fatalf("cockpit offered actions without a matching owner:\n%s", output.String()) + } + next := nextSafeStep(run, nil) + if !strings.Contains(next.Action, "establish writer and traffic authority manually") || strings.Contains(next.Action, "commit --apply") || strings.Contains(next.Action, "rollback --live") { + t.Fatalf("next offered actions without a matching owner: %#v", next) + } + }) + } +} + +func TestCockpitRefusesResumeAfterSuccessfulTargetMutationLosesOwner(t *testing.T) { + resetDokployTrafficOwner(t) + runMeta := migrationRun{Name: "owner-lost", RunDir: t.TempDir(), Target: "dokploy", CreatedAt: time.Now().UTC(), BundleDigest: "owner-lost-digest"} + run := loadedMigrationRun{ + Run: runMeta, + Prepare: preparer.Result{Apps: []preparer.AppPlan{{Name: "api"}}}, + Applied: newRunApplied(runMeta), + } + plan := dokploy.PlanFromArtifacts(run.Prepare, run.Sync, run.Cutover) + if len(plan.Steps) == 0 { + t.Fatal("expected target mutation step") + } + step := plan.Steps[0] + run.Applied.Steps = []appliedStep{{Index: 0, Kind: string(step.Kind), App: step.App, Ref: step.Ref, Status: string(dokploy.StepStatusOK)}} + if phase := migrationRunPhase(run); phase != "authority-ambiguous" { + t.Fatalf("owner-lost partial run phase=%q, want authority-ambiguous", phase) + } + next := nextSafeStep(run, nil) + if !strings.Contains(next.Action, "establish writer and traffic authority manually") || strings.Contains(next.Action, "migrate --live") { + t.Fatalf("owner-lost partial run offered unsafe resume: %#v", next) + } +} + +func TestCockpitShowsDownstreamDecisionsAsReviewOnly(t *testing.T) { + run := loadedMigrationRun{ + Run: migrationRun{Name: "reviewed", RunDir: t.TempDir(), Target: "dokploy", Source: "docker"}, + Prepare: preparer.Result{Source: "docker", SourceDockerEngineID: "engine-reviewed", Apps: []preparer.AppPlan{{ + Name: "api", + Resources: preparer.ResourceSpecs{SourceServices: []preparer.SourceServiceRef{{ContainerID: "0123456789ab", ContainerName: "api"}}}, }}}, Decisions: runDecisions{ APIVersion: decisionsAPIVersion, @@ -273,6 +935,297 @@ func TestCockpitShowsDownstreamDecisionsAsReviewOnly(t *testing.T) { } } +func TestCockpitDirectsStatefulPlanToManualMigration(t *testing.T) { + run := loadedMigrationRun{ + Run: migrationRun{Name: "stateful", RunDir: t.TempDir(), Target: "dokploy", Source: "docker"}, + Prepare: preparer.Result{ + Source: "docker", + SourceDockerEngineID: "engine-reviewed", + Apps: []preparer.AppPlan{{ + Name: "api", + Resources: preparer.ResourceSpecs{SourceServices: []preparer.SourceServiceRef{{ContainerID: "0123456789ab", ContainerName: "api"}}}, + }}, + }, + Sync: syncplan.Result{Apps: []syncplan.AppPlan{{Name: "api", Steps: []syncplan.Step{{ + ResourceType: "volume", + ResourceRef: "volume:web -> /data", + Strategy: syncplan.StrategyDockerVolumeArchive, + }}}}}, + } + if phase := migrationRunPhase(run); phase != "ready" { + t.Fatalf("staged stateful plan phase = %q, want ready", phase) + } + if next := nextSafeStep(run, nil); !strings.Contains(next.Action, "migrate --live") { + t.Fatalf("staged stateful run did not offer live apply: %#v", next) + } + run.Applied = runApplied{APIVersion: appliedAPIVersion, PlanVersion: appliedPlanV1Alpha2, Apps: map[string]appliedApp{"api": {}}} + if phase := migrationRunPhase(run); phase != "manual-state" { + t.Fatalf("in-place stateful ledger phase = %q, want manual-state", phase) + } + var output strings.Builder + writeAppFirstCockpit(&output, run) + if !strings.Contains(output.String(), "MANUAL STATE") || !strings.Contains(output.String(), "cannot continue this run") || !strings.Contains(output.String(), "outside Bort") || strings.Contains(output.String(), "bort migrate --live") { + t.Fatalf("stateful cockpit offered an unusable live action:\n%s", output.String()) + } + next := nextSafeStep(run, nil) + if !strings.Contains(next.Action, "do not rerun this blocked run") || !strings.Contains(next.Action, "outside Bort") || strings.Contains(next.Action, "migrate --live") { + t.Fatalf("stateful next step offered an unusable live action: %#v", next) + } + run.Applied.Steps = []appliedStep{{Index: 0, Kind: string(dokploy.StepCreateProject), App: "api", Ref: "api", Status: string(dokploy.StepStatusOK)}} + if phase := migrationRunPhase(run); phase != "manual-state" { + t.Fatalf("pre-pause stateful prefix phase = %q, want manual-state", phase) + } + if next := nextSafeStep(run, nil); strings.Contains(next.Action, "migrate --live") { + t.Fatalf("pre-pause stateful prefix offered an unusable live action: %#v", next) + } + run.Applied.Steps = append(run.Applied.Steps, appliedStep{Index: 1, Kind: string(dokploy.StepPauseSource), App: "api", Ref: "api", Status: string(dokploy.StepStatusOK)}) + if phase := migrationRunPhase(run); phase != "source-recovery" { + t.Fatalf("paused stateful prefix phase = %q, want source-recovery", phase) + } + if next := nextSafeStep(run, nil); !strings.Contains(next.Action, "migrate --live") || !strings.Contains(next.Reason, "no state transfer") { + t.Fatalf("paused stateful prefix did not offer cleanup-only recovery: %#v", next) + } + run.Applied.Steps[1].Kind = string(dokploy.StepResumeSource) + if phase := migrationRunPhase(run); phase != "manual-state" { + t.Fatalf("cleaned stateful prefix phase = %q, want manual-state", phase) + } +} + +func TestCockpitBlocksPlanLiveApplyWouldRefuse(t *testing.T) { + app := preparer.AppPlan{ + Name: "api", + Resources: preparer.ResourceSpecs{SourceServices: []preparer.SourceServiceRef{{ContainerID: "0123456789ab", ContainerName: "api"}}}, + TargetResources: &preparer.TargetResources{Dokploy: &preparer.DokployResources{ComposeApp: preparer.DokployComposeApp{Name: "api"}}}, + } + app.Resources.Volumes = []preparer.VolumeResource{{Service: "web", Type: "bind", Source: "/srv/data", Target: "/data", SourceContainerID: "0123456789ab", SourceContainerName: "api"}} + run := loadedMigrationRun{ + Run: migrationRun{Name: "bind", RunDir: t.TempDir(), Target: "dokploy", Source: "docker"}, + Prepare: preparer.Result{ + Source: "docker", + SourceDockerEngineID: "engine-reviewed", + Apps: []preparer.AppPlan{app}, + }, + Sync: syncplan.Result{Apps: []syncplan.AppPlan{{Name: "api", Steps: []syncplan.Step{{ + ResourceType: "volume", + ResourceRef: "volume:web -> /data", + Strategy: syncplan.StrategyDockerVolumeArchive, + }}}}}, + } + if phase := migrationRunPhase(run); phase != "plan-blocked" { + t.Fatalf("bind-mount staged plan phase = %q, want plan-blocked", phase) + } + next := nextSafeStep(run, nil) + if !strings.Contains(next.Reason, "only named volumes are transferred before deploy") || !strings.Contains(next.Action, "re-plan") || strings.Contains(next.Action, "migrate --live") { + t.Fatalf("bind-mount staged plan offered an unusable next step: %#v", next) + } + var output strings.Builder + writeAppFirstCockpit(&output, run) + if !strings.Contains(output.String(), "PLAN BLOCKED") || strings.Contains(output.String(), "READY") || strings.Contains(output.String(), "migrate --live") { + t.Fatalf("bind-mount staged plan cockpit offered live apply:\n%s", output.String()) + } + run.Applied = runApplied{APIVersion: appliedAPIVersion, PlanVersion: appliedPlanCurrent, TargetOrigin: "http://127.0.0.1:3000"} + if phase := migrationRunPhase(run); phase != "plan-blocked" { + t.Fatalf("bound zero-step blocked plan phase = %q, want plan-blocked", phase) + } + next = nextSafeStep(run, nil) + if !strings.Contains(next.Action, "cannot be re-planned") || strings.Contains(next.Action, "bort migrate") { + t.Fatalf("bound zero-step blocked plan offered re-planning: %#v", next) + } + run.Applied.Steps = []appliedStep{{Index: 0, Kind: string(dokploy.StepCreateProject), App: "api", Ref: "api", Status: string(dokploy.StepStatusOK)}} + if phase := migrationRunPhase(run); phase != "partial" { + t.Fatalf("partial run with a currently unstageable bundle phase = %q, want partial", phase) + } + if next := nextSafeStep(run, nil); !strings.Contains(next.Action, "resume the interrupted apply") || strings.Contains(next.Action, "re-plan") { + t.Fatalf("partial run with a currently unstageable bundle lost its resume step: %#v", next) + } + run.Applied.Steps = append(run.Applied.Steps, appliedStep{Index: 1, Kind: string(dokploy.StepPauseSource), App: "api", Ref: "api", Status: string(dokploy.StepStatusError), Error: "staged restore preflight of db for app api failed, so pause_source did not stop the source", RequiresNewRun: true}) + next = nextSafeStep(run, nil) + if strings.Contains(next.Action, "resume the interrupted apply") || !strings.Contains(next.Action, "recover-authority --authority source") || !strings.Contains(next.Action, "no state was transferred") || !strings.Contains(next.Reason, "did not stop the source") { + t.Fatalf("pause_source preflight refusal offered resume instead of releasing the run: %#v", next) + } + if phase := migrationRunPhase(run); phase != "new-run-required" { + t.Fatalf("pause_source preflight refusal phase = %q, want new-run-required", phase) + } + output.Reset() + writeAppFirstCockpit(&output, run) + if !strings.Contains(output.String(), "NEW RUN REQUIRED") || strings.Contains(output.String(), "migrate --live") || strings.Contains(output.String(), "resume") || !strings.Contains(output.String(), "recover-authority --authority source") { + t.Fatalf("pause_source preflight refusal cockpit still offered resume:\n%s", output.String()) + } + refusal := run.Applied.Steps + refusedProgress := dokploy.StepProgress{Index: 1, Step: dokploy.Step{Kind: dokploy.StepPauseSource, App: "api", Ref: "api"}, Status: dokploy.StepStatusError, Err: errors.New("staged restore preflight of db for app api failed, so pause_source did not stop the source"), RequiresNewRun: true} + resumeProgress := dokploy.StepProgress{Index: 1, Step: dokploy.Step{Kind: dokploy.StepResumeSource, App: "api", Ref: "api"}} + replayed := runApplied{APIVersion: appliedAPIVersion, PlanVersion: appliedPlanCurrent, TargetOrigin: "http://127.0.0.1:3000"} + replayed = recordAppliedStep(replayed, dokploy.StepProgress{Index: 0, Step: dokploy.Step{Kind: dokploy.StepCreateProject, App: "api", Ref: "api"}, Status: dokploy.StepStatusOK}) + replayed = recordAppliedStep(replayed, refusedProgress) + resumeProgress.Status = dokploy.StepStatusOK + replayed = recordAppliedStep(replayed, resumeProgress) + replayed = recordAppliedStep(replayed, refusedProgress) + run.Applied = replayed + if next := nextSafeStep(run, nil); !strings.Contains(next.Action, "no state was transferred") || !strings.Contains(next.Action, "recover-authority --authority source") { + t.Fatalf("refusal re-recorded after restoring an earlier partial pause did not release the run: %#v", next) + } + replayed = recordAppliedStep(replayed, refusedProgress) + resumeProgress.Status, resumeProgress.Err = dokploy.StepStatusError, errors.New("start source container web-id: timeout") + run.Applied = recordAppliedStep(replayed, resumeProgress) + if phase := migrationRunPhase(run); phase != "partial" { + t.Fatalf("failed cleanup resume after a refusal phase = %q, want partial so the retry restores the source", phase) + } + if next := nextSafeStep(run, nil); !strings.Contains(next.Action, "resume the interrupted apply") { + t.Fatalf("failed cleanup resume after a refusal must retry to restore the source: %#v", next) + } + run.Applied.Steps = append([]appliedStep{ + {Index: 0, Kind: string(dokploy.StepPauseSource), App: "web", Ref: "web", Status: string(dokploy.StepStatusOK)}, + {Index: 0, Kind: string(dokploy.StepResumeSource), App: "web", Ref: "web", Status: string(dokploy.StepStatusOK)}, + }, refusal...) + if next := nextSafeStep(run, nil); !strings.Contains(next.Action, "no state was transferred") || !strings.Contains(next.Action, "recover-authority --authority source") { + t.Fatalf("refusal after another app's restored pause did not release the run: %#v", next) + } + run.Applied.Steps = append([]appliedStep{ + {Index: 0, Kind: string(dokploy.StepPauseSource), App: "web", Ref: "web", Status: string(dokploy.StepStatusOK)}, + {Index: 1, Kind: string(dokploy.StepPushImage), App: "web", Ref: "web", Status: string(dokploy.StepStatusOK)}, + }, refusal...) + if phase := migrationRunPhase(run); phase != "new-run-required" { + t.Fatalf("refusal after another app's handoff phase = %q, want new-run-required", phase) + } + next = nextSafeStep(run, nil) + if strings.Contains(next.Action, "resume the interrupted apply") || strings.Contains(next.Action, "no state was transferred") || strings.Contains(next.Action, "--authority") || !strings.Contains(next.Action, "restore the earlier apps' source writers and traffic manually") || !strings.Contains(next.Action, "create a fresh migration run") { + t.Fatalf("refusal after another app's handoff released or resumed the run: %#v", next) + } + if !strings.Contains(next.Reason, "earlier app's source may already be paused or handed off") || !strings.Contains(next.Reason, "did not stop the source") { + t.Fatalf("refusal after another app's handoff lost its reason: %#v", next) + } + output.Reset() + writeAppFirstCockpit(&output, run) + if !strings.Contains(output.String(), "NEW RUN REQUIRED") || strings.Contains(output.String(), "migrate --live") || strings.Contains(output.String(), "resume") { + t.Fatalf("refusal after another app's handoff cockpit offered resume:\n%s", output.String()) + } + resetDokployTrafficOwner(t) + if err := claimDokployHostOwnership(run.Run, "http://127.0.0.1:3000", "cred-1"); err != nil { + t.Fatal(err) + } + next = nextSafeStep(run, nil) + if !strings.Contains(next.Action, "--authority source") || strings.Contains(next.Action, "--authority target") || !strings.Contains(next.Action, "create a fresh migration run") { + t.Fatalf("refusal after another app's handoff with a claimed host must offer only source authority before a new run: %#v", next) + } + run.Applied.RecoveryProtocol = appliedRecoveryProtocol + for name, moved := range map[string][]dokploy.StepKind{ + "staged restore": {dokploy.StepDumpDataStore, dokploy.StepRestoreDataStore}, + "staged sync": {dokploy.StepSyncVolume}, + } { + earlier := []appliedStep{{Index: 0, Kind: string(dokploy.StepPauseSource), App: "web", Ref: "web", Status: string(dokploy.StepStatusOK)}} + for _, kind := range append(moved, dokploy.StepResumeSource, dokploy.StepPushImage) { + earlier = append(earlier, appliedStep{Index: len(earlier), Kind: string(kind), App: "web", Ref: "web", Status: string(dokploy.StepStatusOK)}) + } + refusedAPI := refusal[len(refusal)-1] + refusedAPI.Index = len(earlier) + run.Applied.Steps = append(earlier, refusedAPI) + if phase := migrationRunPhase(run); phase != "new-run-required" { + t.Fatalf("refusal after another app's %s and resumed source phase = %q, want new-run-required", name, phase) + } + next = nextSafeStep(run, nil) + if strings.Contains(next.Action, "no state was transferred") || !strings.Contains(next.Action, "restore the earlier apps' source writers and traffic manually") || !strings.Contains(next.Action, "--authority source") || !strings.Contains(next.Reason, "earlier app's source may already be paused or handed off") { + t.Fatalf("refusal after another app's %s and resumed source released the run: %#v", name, next) + } + } +} + +func TestStagedTransferRefusalGatesPlannedPostgresDataDirOnApplyHistory(t *testing.T) { + bundleDir := t.TempDir() + appDir := filepath.Join(bundleDir, "api") + if err := os.MkdirAll(appDir, 0o700); err != nil { + t.Fatal(err) + } + compose := "services:\n db:\n image: postgres:16\n environment:\n PGDATA: /pg/data\n volumes:\n - pgdata:/var/lib/postgresql/data\nvolumes:\n pgdata:\n" + if err := os.WriteFile(filepath.Join(appDir, "compose.yaml"), []byte(compose), 0o600); err != nil { + t.Fatal(err) + } + app := preparer.AppPlan{ + Name: "api", + Directory: "api", + Resources: preparer.ResourceSpecs{SourceServices: []preparer.SourceServiceRef{{ContainerID: "0123456789ab", ContainerName: "api"}}}, + TargetResources: &preparer.TargetResources{Dokploy: &preparer.DokployResources{ComposeApp: preparer.DokployComposeApp{Name: "api", ComposePath: "compose.yaml"}}}, + } + app.Resources.DataStores = []preparer.DataStoreResource{{Kind: "postgres", Service: "db", Strategy: "migrate"}} + app.Resources.Volumes = []preparer.VolumeResource{{Service: "db", Type: "volume", Name: "src-pgdata", Target: "/var/lib/postgresql/data", SourceContainerID: "0123456789ab", SourceContainerName: "api"}} + run := loadedMigrationRun{ + Run: migrationRun{Name: "pgdata", RunDir: t.TempDir(), Target: "dokploy", Source: "docker"}, + Prepare: preparer.Result{ + Source: "docker", + SourceDockerEngineID: "engine-reviewed", + BundleDir: bundleDir, + Apps: []preparer.AppPlan{app}, + }, + Sync: syncplan.Result{Apps: []syncplan.AppPlan{{Name: "api", Steps: []syncplan.Step{{ResourceType: "data_store", ResourceRef: "data-store:db"}}}}}, + } + err := stagedTransferRefusal(run) + if !errors.Is(err, dokploy.ErrNotImplemented) || !strings.Contains(err.Error(), "/pg/data") { + t.Fatalf("a run without apply history must be refused before binding for a literal PGDATA off the staged volume, got %v", err) + } + if phase := migrationRunPhase(run); phase != "plan-blocked" { + t.Fatalf("unstarted run with an unstaged PGDATA phase = %q, want plan-blocked", phase) + } + run.Applied = runApplied{APIVersion: appliedAPIVersion, PlanVersion: appliedPlanCurrent, TargetOrigin: "http://127.0.0.1:3000", Steps: []appliedStep{{Index: 0, Kind: string(dokploy.StepCreateProject), App: "api", Ref: "api", Status: string(dokploy.StepStatusOK)}}} + if err := stagedTransferRefusal(run); err != nil { + t.Fatalf("a run with apply history must reach Apply so its pause_source preflight can restart an owned source, got %v", err) + } + if phase := migrationRunPhase(run); phase != "partial" { + t.Fatalf("started run with an unstaged PGDATA phase = %q, want partial", phase) + } +} + +func TestWizardDoesNotRecommendLiveApplyForInPlaceStatefulRun(t *testing.T) { + run := loadedMigrationRun{ + Run: migrationRun{Name: "stateful", RunDir: t.TempDir(), Target: "dokploy"}, + Prepare: preparer.Result{Apps: []preparer.AppPlan{{Name: "api"}}}, + Sync: syncplan.Result{Apps: []syncplan.AppPlan{{Name: "api", Steps: []syncplan.Step{{ + ResourceType: "volume", + ResourceRef: "volume:web -> /data", + Strategy: syncplan.StrategyDockerVolumeArchive, + }}}}}, + Applied: runApplied{APIVersion: appliedAPIVersion, PlanVersion: appliedPlanV1Alpha2, Apps: map[string]appliedApp{"api": {}}}, + } + var output strings.Builder + if err := runWizard(context.Background(), run, strings.NewReader(""), &output, io.Discard); err != nil { + t.Fatal(err) + } + if !strings.Contains(output.String(), "MANUAL STATE") || strings.Contains(output.String(), "migrate --live") { + t.Fatalf("stateful wizard offered an unusable live action:\n%s", output.String()) + } +} + +func TestWizardDoesNotRecommendLiveApplyWhileHostIsOwned(t *testing.T) { + resetDokployTrafficOwner(t) + run := loadedMigrationRun{ + Run: migrationRun{Name: "waiting-run", RunDir: t.TempDir(), Target: "dokploy", Source: "docker", CreatedAt: time.Now().UTC(), BundleDigest: "waiting-digest"}, + Prepare: preparer.Result{ + Source: "docker", + SourceDockerEngineID: "engine-reviewed", + Apps: []preparer.AppPlan{{ + Name: "api", + Resources: preparer.ResourceSpecs{SourceServices: []preparer.SourceServiceRef{{ContainerID: "0123456789ab", ContainerName: "api"}}}, + }}, + }, + } + var unowned strings.Builder + if err := runWizard(context.Background(), run, strings.NewReader(""), &unowned, io.Discard); err != nil { + t.Fatal(err) + } + if !strings.Contains(unowned.String(), "Live apply is explicit") { + t.Fatalf("wizard did not offer live apply for an unowned host:\n%s", unowned.String()) + } + ownerRun := migrationRun{Name: "owner-run", RunDir: filepath.Join(t.TempDir(), "owner-run"), CreatedAt: time.Now().UTC(), BundleDigest: "owner-digest"} + if err := claimDokployHostOwnership(ownerRun, "http://127.0.0.1:3030", dokployCredentialID("owner-token")); err != nil { + t.Fatal(err) + } + var output strings.Builder + if err := runWizard(context.Background(), run, strings.NewReader(""), &output, io.Discard); err != nil { + t.Fatal(err) + } + if !strings.Contains(output.String(), "HOST OWNED") || strings.Contains(output.String(), "Live apply is explicit") { + t.Fatalf("wizard offered live apply while another run owned the host:\n%s", output.String()) + } +} + func TestCockpitShowsDownstreamBlockersAsBlocking(t *testing.T) { run := loadedMigrationRun{ Run: migrationRun{Name: "blocked", RunDir: t.TempDir(), Target: "dokploy"}, diff --git a/internal/cli/commit.go b/internal/cli/commit.go index 6a9d866..a4d43b4 100644 --- a/internal/cli/commit.go +++ b/internal/cli/commit.go @@ -5,7 +5,6 @@ import ( "flag" "fmt" "io" - "os" "strings" commitplan "github.com/aikins01/bort/internal/commit" @@ -33,7 +32,7 @@ func runCommit(ctx context.Context, args []string, stdout, stderr io.Writer) err fs.StringVar(&format, "format", "text", "output format: text, json") fs.StringVar(&outputPath, "output", "-", "output path, or - for stdout") fs.StringVar(&cutoverPlanPath, "from-cutover", "", "read a prior cutover JSON plan artifact") - fs.BoolVar(&apply, "apply", false, "execute commit cleanup (stop source containers and coolify-proxy)") + fs.BoolVar(&apply, "apply", false, "retire the exact reviewed source containers when no source orchestrator can recreate them; Coolify sources need manual retirement and recover-authority --source-retired") fs.StringVar(&runRef, "run", "", "run name under .bort/runs, or a run directory path") fs.IntVar(&rollbackWindowSeconds, "rollback-window", commitplan.DefaultRollbackWindowSeconds, "rollback window in seconds") @@ -189,17 +188,45 @@ func applyCommitFromArgs(ctx context.Context, runRef string, stderr io.Writer) e if run.Run.RolledBackAt != nil { return fmt.Errorf("commit refused: run %q was rolled back; the source is serving traffic again", run.Run.Name) } - if run.Run.RollbackStartedAt != nil { - return fmt.Errorf("commit refused: rollback started for run %q; run `%s` to finish recovery", run.Run.Name, runScopedCommand(run, "rollback --live")) + if authorityRecoveryPending(run) { + return fmt.Errorf("commit refused: manual %s-authority recovery is incomplete; run `%s` to finish it", run.Run.ResolvedAuthority, pendingAuthorityRecoveryCommand(run)) + } + if run.Run.RollbackStartedAt != nil && run.Run.ResolvedAuthority != dokployTrafficTarget { + return fmt.Errorf("commit refused: rollback started for run %q; %s", run.Run.Name, incompleteRollbackRecovery(run)) } if err := requireLiveApplySucceeded(run); err != nil { return err } - client, err := resolveDokployClient(ctx, run.Run.Target, os.Stdin, stderr) + if run.Run.CommittedAt == nil && coolifySourceRetirementRequired(run) { + return fmt.Errorf("commit --apply refused: Coolify can replace a stopped source container through a queued or future deployment, and Bort cannot durably fence that orchestrator; %s", manualCoolifySourceRetirementAction(run)) + } + targetLock, err := acquireDokployLiveOperationLock() if err != nil { - return err + return fmt.Errorf("lock Dokploy live operations: %w", err) } + defer targetLock.Release() plan := dokploy.PlanForCommit(run.Prepare, run.Cutover) + requiresHostOwner := planRequiresDokployHostOwner(run, plan) + if run.Run.CommittedAt != nil { + if err := markRunCommittedLocked(run.Run); err != nil { + return fmt.Errorf("reconfirm completed commit metadata durability: %w", err) + } + if requiresHostOwner { + if err := releaseDokployTargetOwner(run.Run); err != nil { + return fmt.Errorf("release completed commit host ownership: %w", err) + } + } + return nil + } + if err := verifyLocalSourceRun(ctx, run); err != nil { + return fmt.Errorf("commit refused on an unverified Docker source: %w", err) + } + if requiresHostOwner { + if err := ensureDokployTrafficTargetOwner(ctx, run, nil, false); err != nil { + return fmt.Errorf("commit refused: %w", err) + } + } + client := &dokploy.Client{} plan.RunName = run.Run.Name plan.RunDir = run.Run.RunDir plan.ApprovedPrepareDecisions = approvedPrepareDecisions(run) @@ -213,9 +240,48 @@ func applyCommitFromArgs(ctx context.Context, runRef string, stderr io.Writer) e if err := markRunCommittedLocked(run.Run); err != nil { return fmt.Errorf("source retirement completed, but migration commit metadata could not be recorded: %w", err) } + if requiresHostOwner { + if err := releaseDokployTargetOwner(run.Run); err != nil { + return fmt.Errorf("commit was recorded, but its Dokploy host ownership could not be released: %w", err) + } + } return nil } +func coolifySourceRetirementRequired(run loadedMigrationRun) bool { + switch strings.ToLower(strings.TrimSpace(run.Prepare.Source)) { + case "coolify-local", "coolify-local-traefik", "coolify-local-caddy": + return true + } + for _, app := range run.Prepare.Apps { + if strings.EqualFold(strings.TrimSpace(app.Platform), "coolify") { + return true + } + } + return false +} + +func manualCoolifySourceRetirementAction(run loadedMigrationRun) string { + targets := "every reviewed source app" + for _, step := range dokploy.PlanForCommit(run.Prepare, run.Cutover).Steps { + if step.Kind == dokploy.StepStopCoolifyProxy { + targets += " and the source proxy" + break + } + } + return fmt.Sprintf("manually disable future Coolify deployments for the reviewed apps, retire %s, verify they remain retired, then run `%s`", targets, authorityRecoverySourceRetiredCommand(run)) +} + +func finishStartedAcceptanceAction(run loadedMigrationRun) string { + if authorityRecoveryPending(run) { + return fmt.Sprintf("run `%s` to finish the recorded manual %s-authority recovery", pendingAuthorityRecoveryCommand(run), run.Run.ResolvedAuthority) + } + if coolifySourceRetirementRequired(run) { + return manualCoolifySourceRetirementAction(run) + } + return fmt.Sprintf("run `%s` to finish acceptance", runScopedCommand(run, "commit --apply")) +} + func requireLiveApplySucceeded(run loadedMigrationRun) error { return requireLiveApplySucceededForAppsSkipping(run, nil, nil) } @@ -229,10 +295,13 @@ func requireLiveApplySucceededForApps(run loadedMigrationRun, apps map[string]st } func requireLiveApplySucceededForAppsSkipping(run loadedMigrationRun, apps map[string]struct{}, skipKinds map[dokploy.StepKind]struct{}) error { + if run.Run.ResolvedAuthority == dokployTrafficTarget && run.Run.AuthorityResolvedAt != nil && run.Run.LiveAppliedAt != nil { + return nil + } if run.Run.ApplyOutcomeRequired && run.Applied.SucceededAt == nil { return fmt.Errorf("run %q has no successful live-apply outcome recorded; run `%s` first", run.Run.Name, liveApplyCommand(run)) } - live := dokploy.PlanFromArtifacts(run.Prepare, run.Sync, run.Cutover) + live := livePlanForApplied(run, run.Applied) if len(live.Steps) == 0 { return fmt.Errorf("run %q has no live apply steps; nothing to commit", run.Run.Name) } @@ -268,6 +337,9 @@ func requireLiveApplySucceededForAppsSkipping(run loadedMigrationRun, apps map[s if checked == 0 { return fmt.Errorf("run %q has no live apply steps for the selected purge scope", run.Run.Name) } + if runMayHaveAmbiguousAuthority(run) { + return fmt.Errorf("run %q cannot prove writer or traffic authority; %s", run.Run.Name, authorityRecoveryInstruction(run)) + } return nil } diff --git a/internal/cli/commit_test.go b/internal/cli/commit_test.go index 6145f85..34b8ee9 100644 --- a/internal/cli/commit_test.go +++ b/internal/cli/commit_test.go @@ -4,7 +4,9 @@ import ( "bytes" "context" "encoding/json" + "errors" "io" + "os" "path/filepath" "strings" "testing" @@ -240,6 +242,7 @@ func TestRunCommitUsesDefaultBundleWithoutCurrentRun(t *testing.T) { } func TestNewRunRequiresSuccessfulOutcomeBeforeCommit(t *testing.T) { + resetDokployTrafficOwner(t) workDir := t.TempDir() t.Chdir(workDir) bundleDir := filepath.Join(workDir, "bort-bundle") @@ -285,6 +288,245 @@ func TestNewRunRequiresSuccessfulOutcomeBeforeCommit(t *testing.T) { if err := applyCommitFromArgs(context.Background(), "missing-outcome", io.Discard); err == nil || !strings.Contains(err.Error(), "no successful live-apply outcome") { t.Fatalf("expected commit to reject complete steps without a durable outcome, got %v", err) } + succeededAt := time.Now().UTC() + applied.SucceededAt = &succeededAt + if err := writeRunApplied(appliedPath, applied); err != nil { + t.Fatal(err) + } + if err := claimDokployHostOwnership(run.Run, "http://127.0.0.1:3030", dokployCredentialID("test-token")); err != nil { + t.Fatal(err) + } + if err := markDokployTrafficTarget(run.Run, "http://127.0.0.1:3030"); err != nil { + t.Fatal(err) + } + t.Setenv(dokploy.EnvBaseURL, "http://127.0.0.1:3030") + t.Setenv(dokploy.EnvToken, "test-token") + targetLock, err := acquireDokployLiveOperationLock() + if err != nil { + t.Fatal(err) + } + err = applyCommitFromArgs(context.Background(), "missing-outcome", io.Discard) + if !errors.Is(err, errDokployLiveOperationActive) { + t.Fatalf("expected another Dokploy live operation to block commit, got %v", err) + } + targetLock.Release() + blocked, err := loadMigrationRun("missing-outcome") + if err != nil { + t.Fatal(err) + } + if blocked.Run.CommitStartedAt != nil { + t.Fatal("commit recorded a start before acquiring the host Dokploy lock") + } +} + +func TestCommitRefusesCoolifySourceBeforeRetirement(t *testing.T) { + resetDokployTrafficOwner(t) + workDir := t.TempDir() + t.Chdir(workDir) + binDir := filepath.Join(workDir, "bin") + if err := os.MkdirAll(binDir, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(binDir, "docker"), []byte("#!/bin/sh\necho \"$*\" >> docker-calls\nexit 90\n"), 0o700); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", binDir+string(os.PathListSeparator)+os.Getenv("PATH")) + writeTestBundle(t, "bundle", manifest.Manifest{ + Source: manifest.Source{Platform: "coolify-local", DockerEngineID: "engine-reviewed"}, + Apps: []manifest.App{{ + Name: "api", + Services: []manifest.Service{{ID: "source-id", Name: "web", Image: "example/api:latest"}}, + Routes: []manifest.Route{{Host: "api.example.com", ServiceName: "web", Port: "3000"}}, + }}, + }) + runCommand(t, runMigrate, []string{"--bundle", "bundle", "--run", "coolify-commit", "--observation-window", "0", "--rollback-window", "0"}) + run, err := loadMigrationRun("coolify-commit") + if err != nil { + t.Fatal(err) + } + applied := newRunApplied(run.Run) + now := time.Now().UTC() + applied.SucceededAt = &now + for index, step := range dokploy.PlanFromArtifacts(run.Prepare, run.Sync, run.Cutover).Steps { + applied.Steps = append(applied.Steps, appliedStep{Index: index, Kind: string(step.Kind), App: step.App, Ref: step.Ref, Status: string(dokploy.StepStatusOK)}) + } + if err := writeRunApplied(runArtifactPath(run.Run.RunDir, run.Run.Artifacts.Applied), applied); err != nil { + t.Fatal(err) + } + if err := markRunLiveAppliedLocked(run.Run); err != nil { + t.Fatal(err) + } + if err := claimDokployHostOwnership(run.Run, "http://127.0.0.1:3030", dokployCredentialID("test-token")); err != nil { + t.Fatal(err) + } + if err := markDokployTrafficTarget(run.Run, "http://127.0.0.1:3030"); err != nil { + t.Fatal(err) + } + + forbidLocalSourceVerification(t) + err = applyCommitFromArgs(context.Background(), "coolify-commit", io.Discard) + if err == nil || !strings.Contains(err.Error(), "Bort cannot durably fence that orchestrator") || !strings.Contains(err.Error(), authorityRecoverySourceRetiredCommand(run)) { + t.Fatalf("expected Coolify commit refusal with exact manual-retirement command, got %v", err) + } + blocked, err := loadMigrationRun("coolify-commit") + if err != nil { + t.Fatal(err) + } + if blocked.Run.CommitStartedAt != nil { + t.Fatal("Coolify commit refusal recorded source retirement") + } + if _, err := os.Stat(filepath.Join(run.Run.RunDir, "source-pause.json")); !os.IsNotExist(err) { + t.Fatalf("Coolify commit refusal created source pause state: %v", err) + } + if _, err := os.Stat("docker-calls"); !os.IsNotExist(err) { + t.Fatalf("Coolify commit refusal made Docker calls: %v", err) + } +} + +func TestCommitRefusesIncompleteSourceAuthorityRecoveryBeforeDocker(t *testing.T) { + resetDokployTrafficOwner(t) + workDir := t.TempDir() + t.Chdir(workDir) + binDir := filepath.Join(workDir, "bin") + if err := os.MkdirAll(binDir, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(binDir, "docker"), []byte("#!/bin/sh\necho \"$*\" >> docker-calls\nexit 90\n"), 0o700); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", binDir+string(os.PathListSeparator)+os.Getenv("PATH")) + writeTestBundle(t, "bundle", manifest.Manifest{ + Source: manifest.Source{Platform: "coolify-local"}, + Apps: []manifest.App{{ + Name: "api", + Routes: []manifest.Route{{Host: "api.example.com", ServiceName: "web", Port: "3000"}}, + Services: []manifest.Service{{ + ID: "source-id", + Name: "web", + Image: "example/api:latest", + Mounts: []manifest.Mount{{ + Type: "volume", Name: "api-data", Target: "/data", + }}, + }}, + }}, + }) + runCommand(t, runMigrate, []string{"--bundle", "bundle", "--run", "source-recovery", "--observation-window", "0", "--rollback-window", "0"}) + run, err := loadMigrationRun("source-recovery") + if err != nil { + t.Fatal(err) + } + applied := newRunApplied(run.Run) + now := time.Now().UTC() + applied.SucceededAt = &now + for index, step := range dokploy.PlanFromArtifacts(run.Prepare, run.Sync, run.Cutover).Steps { + applied.Steps = append(applied.Steps, appliedStep{Index: index, Kind: string(step.Kind), App: step.App, Ref: step.Ref, Status: string(dokploy.StepStatusOK)}) + } + if err := writeRunApplied(runArtifactPath(run.Run.RunDir, run.Run.Artifacts.Applied), applied); err != nil { + t.Fatal(err) + } + if err := markRunLiveAppliedLocked(run.Run); err != nil { + t.Fatal(err) + } + if err := claimDokployHostOwnership(run.Run, "http://127.0.0.1:3030", dokployCredentialID("test-token")); err != nil { + t.Fatal(err) + } + if err := markDokployTrafficTarget(run.Run, "http://127.0.0.1:3030"); err != nil { + t.Fatal(err) + } + if err := markRunAuthorityResolvedLocked(run.Run, dokployTrafficSource); err != nil { + t.Fatal(err) + } + + forbidLocalSourceVerification(t) + err = applyCommitFromArgs(context.Background(), "source-recovery", io.Discard) + if err == nil || !strings.Contains(err.Error(), "manual source-authority recovery is incomplete") || !strings.Contains(err.Error(), "recover source-recovery as source") { + t.Fatalf("expected incomplete source-authority recovery refusal, got %v", err) + } + blocked, err := loadMigrationRun("source-recovery") + if err != nil { + t.Fatal(err) + } + if blocked.Run.CommitStartedAt != nil { + t.Fatal("commit recorded source retirement during incomplete source-authority recovery") + } + if _, err := os.Stat("docker-calls"); !os.IsNotExist(err) { + t.Fatalf("blocked commit made Docker calls: %v", err) + } +} + +func TestCommitRetryRepublishesCompletedMetadataBeforeOwnerRelease(t *testing.T) { + run := writeAmbiguousAuthorityRun(t, "durable-commit") + if err := markRunAuthorityResolvedLocked(run.Run, dokployTrafficTarget); err != nil { + t.Fatal(err) + } + if err := markDokployTrafficTarget(run.Run, "http://127.0.0.1:3030"); err != nil { + t.Fatal(err) + } + if err := markRunLiveAppliedLocked(run.Run); err != nil { + t.Fatal(err) + } + if err := markRunCommitStartedLocked(run.Run); err != nil { + t.Fatal(err) + } + if err := markRunCommittedLocked(run.Run); err != nil { + t.Fatal(err) + } + if err := markRunHostOwnerReleaseStartedLocked(run.Run); err != nil { + t.Fatal(err) + } + path := filepath.Join(run.Run.RunDir, "run.json") + before, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + if err := applyCommitFromArgs(context.Background(), run.Run.Name, io.Discard); err != nil { + t.Fatal(err) + } + after, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + if os.SameFile(before, after) { + t.Fatal("completed commit metadata was not atomically republished before owner release") + } + owner, found, err := readDokployTrafficOwner() + if err != nil || !found || owner.Authority != dokployTrafficReleased { + t.Fatalf("completed commit owner = %#v, found=%t err=%v", owner, found, err) + } +} + +func TestCommitRetryKeepsOwnerWhenCompletedMetadataRepublishFails(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("root bypasses directory permissions") + } + run := writeAmbiguousAuthorityRun(t, "durable-commit-readonly") + if err := markRunAuthorityResolvedLocked(run.Run, dokployTrafficTarget); err != nil { + t.Fatal(err) + } + if err := markDokployTrafficTarget(run.Run, "http://127.0.0.1:3030"); err != nil { + t.Fatal(err) + } + if err := markRunLiveAppliedLocked(run.Run); err != nil { + t.Fatal(err) + } + if err := markRunCommitStartedLocked(run.Run); err != nil { + t.Fatal(err) + } + if err := markRunCommittedLocked(run.Run); err != nil { + t.Fatal(err) + } + if err := os.Chmod(run.Run.RunDir, 0o500); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chmod(run.Run.RunDir, 0o700) }) + err := applyCommitFromArgs(context.Background(), run.Run.Name, io.Discard) + if err == nil || !strings.Contains(err.Error(), "reconfirm completed commit metadata durability") { + t.Fatalf("expected metadata republish failure, got %v", err) + } + owner, found, err := readDokployTrafficOwner() + if err != nil || !found || owner.Authority != dokployTrafficTarget { + t.Fatalf("owner released before completed commit metadata was durable: %#v found=%t err=%v", owner, found, err) + } } func TestLegacyCompleteLedgerDoesNotRequireSuccessfulOutcomeMarker(t *testing.T) { @@ -293,7 +535,9 @@ func TestLegacyCompleteLedgerDoesNotRequireSuccessfulOutcomeMarker(t *testing.T) Prepare: preparer.Result{Apps: []preparer.AppPlan{{Name: "api"}}}, Cutover: gateway.Result{Apps: []gateway.AppPlan{{Name: "api", Routes: []gateway.Route{{Host: "api.example.com"}}}}}, } - steps := dokploy.PlanFromArtifacts(run.Prepare, run.Sync, run.Cutover).Steps + run.Applied.APIVersion = appliedLegacyAPIVersion + run.Applied.TargetOrigin = "http://127.0.0.1:3030" + steps := dokploy.LegacyPlanFromArtifactsV1Alpha1(run.Prepare, run.Sync, run.Cutover).Steps for index, step := range steps { run.Applied.Steps = append(run.Applied.Steps, appliedStep{ Index: index, @@ -464,3 +708,55 @@ func TestRequireLiveApplySucceededForAppsSkippingAllowsMissingSkippedKinds(t *te t.Fatalf("expected regular guard to reject missing %s step, got %v", skipKind, err) } } + +func TestCoolifySourceRetirementRequiredDetectsLabeledAppsInDockerScans(t *testing.T) { + for _, test := range []struct { + name string + source string + apps []preparer.AppPlan + want bool + }{ + {name: "coolify-local source", source: "coolify-local", want: true}, + {name: "coolify-local-traefik source", source: "coolify-local-traefik", want: true}, + {name: "docker scan without coolify apps", source: "docker", apps: []preparer.AppPlan{{Name: "api", Platform: "docker"}}, want: false}, + {name: "docker scan with coolify-labeled app", source: "docker", apps: []preparer.AppPlan{{Name: "api", Platform: "docker"}, {Name: "web", Platform: "coolify"}}, want: true}, + {name: "legacy bundle without app platform", source: "docker", apps: []preparer.AppPlan{{Name: "api"}}, want: false}, + } { + t.Run(test.name, func(t *testing.T) { + run := loadedMigrationRun{Prepare: preparer.Result{Source: test.source, Apps: test.apps}} + if got := coolifySourceRetirementRequired(run); got != test.want { + t.Fatalf("coolifySourceRetirementRequired = %t, want %t", got, test.want) + } + }) + } +} + +func forbidLocalSourceVerification(t *testing.T) { + t.Helper() + previousRun, previousEngine := verifyLocalSourceRun, verifyLocalSourceEngine + verifyLocalSourceRun = func(context.Context, loadedMigrationRun) error { + t.Error("commit refusal probed the local source run") + return nil + } + verifyLocalSourceEngine = func(context.Context, loadedMigrationRun) error { + t.Error("commit refusal probed the local source engine") + return nil + } + t.Cleanup(func() { + verifyLocalSourceRun, verifyLocalSourceEngine = previousRun, previousEngine + }) +} + +func TestManualCoolifySourceRetirementActionScopesProxyToRoutedCutovers(t *testing.T) { + run := loadedMigrationRun{ + Run: migrationRun{Name: "coolify-run"}, + Prepare: preparer.Result{Source: "coolify-local", Apps: []preparer.AppPlan{{Name: "api"}}}, + } + if action := manualCoolifySourceRetirementAction(run); strings.Contains(action, "source proxy") || !strings.Contains(action, "--source-retired") { + t.Fatalf("route-free retirement hint touched the shared proxy: %q", action) + } + run.Cutover = gateway.Result{Apps: []gateway.AppPlan{{Name: "api", Routes: []gateway.Route{{Host: "api.example.com"}}}}} + if action := manualCoolifySourceRetirementAction(run); !strings.Contains(action, "and the source proxy") { + t.Fatalf("routed retirement hint omitted the proxy handoff: %q", action) + } +} diff --git a/internal/cli/dokploy_lock_unix.go b/internal/cli/dokploy_lock_unix.go new file mode 100644 index 0000000..788a891 --- /dev/null +++ b/internal/cli/dokploy_lock_unix.go @@ -0,0 +1,251 @@ +//go:build !windows + +package cli + +import ( + "bytes" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "runtime" + "strings" + "syscall" +) + +var ( + dokployLiveOperationLockPath = defaultDokployLiveOperationLockPath + dokployTrafficOwnerPath = defaultDokployTrafficOwnerPath + dokployLiveOperationsSupported = func() bool { + return runtime.GOOS == "linux" + } +) + +func defaultDokployLiveOperationLockPath() (string, error) { + return filepath.Join("/var/lib/bort", "dokploy-live.lock"), nil +} + +func prepareDokployLiveOperationLockPath(path string) error { + return ensurePrivateHostDirectory(filepath.Dir(path), "state") +} + +func dokployInstallationRecoveryPath() (string, error) { + path, err := dokployLiveOperationLockPath() + if err != nil { + return "", err + } + return path + ".install-recovery-required", nil +} + +func markDokployInstallationRecoveryRequired(recovery dokployInstallationRecovery) error { + if err := validateDokployInstallationRecovery(recovery); err != nil { + return err + } + path, err := dokployInstallationRecoveryPath() + if err != nil { + return err + } + if err := prepareDokployLiveOperationLockPath(path); err != nil { + return err + } + existing, found, err := readDokployInstallationRecoveryMarker(path) + if err != nil { + return err + } + if found { + if existing.Identity != recovery.Identity { + return fmt.Errorf("interrupted Dokploy installation was started with different immutable options; rerun the exact original command: %s", existing.Command) + } + if sameDokployInstallationRecovery(existing, recovery) || existing.Phase == dokployInstallAPIKey && recovery.Phase == dokployInstallInstalling { + return nil + } + existingIntent := existing + existingIntent.APIKeyCreatedID = "" + recoveryIntent := recovery + recoveryIntent.APIKeyCreatedID = "" + updatingCreatedKey := existing.Phase == dokployInstallAPIKey && recovery.Phase == dokployInstallAPIKey && sameDokployInstallationRecovery(existingIntent, recoveryIntent) + if !updatingCreatedKey && (existing.Phase != dokployInstallInstalling || recovery.Phase != dokployInstallAPIKey) { + return fmt.Errorf("interrupted Dokploy installation recovery state cannot move from %q to %q", existing.Phase, recovery.Phase) + } + } + contents, err := json.Marshal(recovery) + if err != nil { + return err + } + return writeFileAtomic(path, append(contents, '\n'), 0o600) +} + +func readDokployInstallationRecovery() (dokployInstallationRecovery, bool, error) { + path, err := dokployInstallationRecoveryPath() + if err != nil { + return dokployInstallationRecovery{}, false, err + } + return readDokployInstallationRecoveryMarker(path) +} + +func dokployInstallationRecoveryRequired() (bool, error) { + path, err := dokployInstallationRecoveryPath() + if err != nil { + return false, err + } + _, found, err := readDokployInstallationRecoveryMarker(path) + return found, err +} + +func dokployInstallationRecoveryCommand() (string, bool, error) { + path, err := dokployInstallationRecoveryPath() + if err != nil { + return "", false, err + } + recovery, found, err := readDokployInstallationRecoveryMarker(path) + return recovery.Command, found, err +} + +func readDokployInstallationRecoveryMarker(path string) (dokployInstallationRecovery, bool, error) { + info, err := os.Lstat(path) + if errors.Is(err, os.ErrNotExist) { + return dokployInstallationRecovery{}, false, nil + } + if err != nil { + return dokployInstallationRecovery{}, false, err + } + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok || !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 || info.Mode().Perm() != 0o600 || stat.Uid != uint32(os.Geteuid()) || stat.Nlink != 1 { + return dokployInstallationRecovery{}, false, fmt.Errorf("Dokploy installation recovery marker %s must be a single-link mode 0600 regular file owned by uid %d", path, os.Geteuid()) + } + contents, err := readFileNoFollow(path) + if err != nil { + return dokployInstallationRecovery{}, false, err + } + var recovery dokployInstallationRecovery + if err := json.Unmarshal(contents, &recovery); err != nil { + return dokployInstallationRecovery{}, false, fmt.Errorf("Dokploy installation recovery marker %s is malformed", path) + } + canonical, err := json.Marshal(recovery) + if err != nil || !bytes.Equal(contents, append(canonical, '\n')) || validateDokployInstallationRecovery(recovery) != nil { + return dokployInstallationRecovery{}, false, fmt.Errorf("Dokploy installation recovery marker %s is malformed", path) + } + return recovery, true, nil +} + +func validateDokployInstallationRecovery(recovery dokployInstallationRecovery) error { + if recovery.APIVersion != dokployInstallRecoveryAPIVersion { + return fmt.Errorf("unsupported Dokploy installation recovery marker version") + } + if recovery.Phase != dokployInstallInstalling && recovery.Phase != dokployInstallAPIKey { + return fmt.Errorf("unsupported Dokploy installation recovery phase") + } + if recovery.CommandPrefix != "bort" && recovery.CommandPrefix != "sudo bort" { + return fmt.Errorf("unsupported Dokploy installation recovery command prefix") + } + if recovery.EndpointMode != "vip" && recovery.EndpointMode != "dnsrr" { + return fmt.Errorf("unsupported Dokploy installation endpoint mode") + } + if normalized, err := normalizeSwarmAddressPool(recovery.AddressPool); err != nil || normalized != recovery.AddressPool { + return fmt.Errorf("unsupported Dokploy installation address pool") + } + if err := validateDokployAPIKeyName(recovery.APIKeyName); err != nil { + return fmt.Errorf("unsupported Dokploy installation API-key name") + } + if recovery.Phase == dokployInstallInstalling && len(recovery.APIKeyBaselineIDs) != 0 { + return fmt.Errorf("Dokploy installation recovery baseline is only valid during API-key recovery") + } + if recovery.Phase == dokployInstallInstalling && recovery.APIKeyCreatedID != "" { + return fmt.Errorf("Dokploy installation recovery created key is only valid during API-key recovery") + } + if strings.ContainsAny(recovery.APIKeyCreatedID, "\r\n\x00") { + return fmt.Errorf("Dokploy installation recovery created API-key identity is malformed") + } + for index, id := range recovery.APIKeyBaselineIDs { + if id == "" || strings.ContainsAny(id, "\r\n\x00") || index > 0 && recovery.APIKeyBaselineIDs[index-1] >= id { + return fmt.Errorf("Dokploy installation recovery API-key baseline is malformed") + } + } + if decoded, err := hex.DecodeString(recovery.Identity); err != nil || len(decoded) != 32 || recovery.Identity != strings.ToLower(recovery.Identity) { + return fmt.Errorf("Dokploy installation recovery identity must be 64 lowercase hexadecimal characters") + } + if !sameDokployInstallationRecovery(recovery, canonicalDokployInstallationRecovery(recovery)) { + return fmt.Errorf("Dokploy installation recovery options are not canonical") + } + if recovery.Identity != dokployInstallRecoveryIdentity(recovery) { + return fmt.Errorf("Dokploy installation recovery identity does not match its immutable options") + } + if recovery.Command != dokployInstallRecoveryCommand(recovery) || recovery.Command == "" || recovery.Command != strings.TrimSpace(recovery.Command) || strings.ContainsAny(recovery.Command, "\r\n\x00") { + return fmt.Errorf("Dokploy installation recovery command is malformed") + } + return nil +} + +func clearDokployInstallationRecoveryRequired() error { + path, err := dokployInstallationRecoveryPath() + if err != nil { + return err + } + blocked, err := dokployInstallationRecoveryRequired() + if err != nil || !blocked { + return err + } + if err := os.Remove(path); err != nil { + return err + } + dir, err := os.Open(filepath.Dir(path)) + if err != nil { + return err + } + if err := dir.Sync(); err != nil { + _ = dir.Close() + return err + } + return dir.Close() +} + +func defaultDokployTrafficOwnerPath() (string, error) { + return filepath.Join("/var/lib/bort", "dokploy-traffic-owner.json"), nil +} + +func prepareDokployTrafficOwnerPath(path string) error { + return ensurePrivateHostDirectory(filepath.Dir(path), "state") +} + +func ensurePrivateHostDirectory(dir, kind string) error { + created := false + if err := os.Mkdir(dir, 0o700); err != nil { + if !os.IsExist(err) { + return fmt.Errorf("create host Dokploy %s directory %s: %w%s", kind, dir, err, hostDirectoryProvisioningHint(dir)) + } + } else { + created = true + } + info, err := os.Lstat(dir) + if err != nil { + return fmt.Errorf("inspect host Dokploy %s directory %s: %w", kind, dir, err) + } + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok || !info.IsDir() || info.Mode()&os.ModeSymlink != 0 || stat.Uid != uint32(os.Geteuid()) || info.Mode().Perm() != 0o700 { + return fmt.Errorf("host Dokploy %s directory %s must be a mode 0700 directory owned by uid %d%s", kind, dir, os.Geteuid(), hostDirectoryProvisioningHint(dir)) + } + if !created { + return nil + } + parent, err := os.Open(filepath.Dir(dir)) + if err != nil { + return fmt.Errorf("open parent of host Dokploy %s directory %s: %w", kind, dir, err) + } + if err := parent.Sync(); err != nil { + _ = parent.Close() + return fmt.Errorf("sync parent of host Dokploy %s directory %s: %w", kind, dir, err) + } + if err := parent.Close(); err != nil { + return fmt.Errorf("close parent of host Dokploy %s directory %s: %w", kind, dir, err) + } + return nil +} + +func hostDirectoryProvisioningHint(dir string) string { + if os.Geteuid() == 0 { + return "" + } + return fmt.Sprintf(`; provision it once with: sudo install -d -m 700 -o "$(id -u)" -g "$(id -g)" %s`, shellQuote(dir)) +} diff --git a/internal/cli/dokploy_lock_windows.go b/internal/cli/dokploy_lock_windows.go new file mode 100644 index 0000000..c0c818b --- /dev/null +++ b/internal/cli/dokploy_lock_windows.go @@ -0,0 +1,49 @@ +//go:build windows + +package cli + +import "fmt" + +var ( + dokployLiveOperationLockPath = defaultDokployLiveOperationLockPath + dokployTrafficOwnerPath = defaultDokployTrafficOwnerPath + dokployLiveOperationsSupported = func() bool { + return false + } +) + +func defaultDokployLiveOperationLockPath() (string, error) { + return "", fmt.Errorf("host-wide Dokploy live operations are unsupported on Windows") +} + +func prepareDokployLiveOperationLockPath(string) error { + return fmt.Errorf("host-wide Dokploy live operations are unsupported on Windows") +} + +func markDokployInstallationRecoveryRequired(dokployInstallationRecovery) error { + return fmt.Errorf("Dokploy installation recovery is unsupported on Windows") +} + +func readDokployInstallationRecovery() (dokployInstallationRecovery, bool, error) { + return dokployInstallationRecovery{}, false, fmt.Errorf("Dokploy installation recovery is unsupported on Windows") +} + +func dokployInstallationRecoveryRequired() (bool, error) { + return false, fmt.Errorf("Dokploy installation recovery is unsupported on Windows") +} + +func dokployInstallationRecoveryCommand() (string, bool, error) { + return "", false, fmt.Errorf("Dokploy installation recovery is unsupported on Windows") +} + +func clearDokployInstallationRecoveryRequired() error { + return fmt.Errorf("Dokploy installation recovery is unsupported on Windows") +} + +func defaultDokployTrafficOwnerPath() (string, error) { + return "", nil +} + +func prepareDokployTrafficOwnerPath(string) error { + return fmt.Errorf("host-wide Dokploy traffic ownership is unsupported on Windows") +} diff --git a/internal/cli/dokploy_traffic_owner.go b/internal/cli/dokploy_traffic_owner.go new file mode 100644 index 0000000..9bb79d8 --- /dev/null +++ b/internal/cli/dokploy_traffic_owner.go @@ -0,0 +1,552 @@ +package cli + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "time" + + "github.com/aikins01/bort/internal/target/dokploy" +) + +const dokployTrafficOwnerAPIVersion = "bort.dokploy-traffic-owner/v1alpha1" + +const ( + dokployTrafficPending = "pending" + dokployTrafficTarget = "target" + dokployTrafficSource = "source" + dokployTrafficReleased = "released" +) + +type dokployTrafficOwner struct { + APIVersion string `json:"apiVersion"` + RunID string `json:"runId"` + RunName string `json:"runName"` + RunDir string `json:"runDir,omitempty"` + TargetOrigin string `json:"targetOrigin"` + TargetCredentialID string `json:"targetCredentialId,omitempty"` + Authority string `json:"authority"` + UpdatedAt time.Time `json:"updatedAt"` +} + +func trafficHandoffStepIndex(plan dokploy.Plan) int { + for index, step := range plan.Steps { + if step.Kind == dokploy.StepStopCoolifyProxy { + return index + } + } + return -1 +} + +func dokployOwnershipStepIndex(plan dokploy.Plan) int { + if len(plan.Steps) == 0 { + return -1 + } + return 0 +} + +func planChangesDokployTraffic(plan dokploy.Plan) bool { + for _, step := range plan.Steps { + if step.Kind == dokploy.StepStopCoolifyProxy || step.Kind == dokploy.StepStopDokployProxy { + return true + } + } + return false +} + +func planRequiresDokployHostOwner(run loadedMigrationRun, plan dokploy.Plan) bool { + return planChangesDokployTraffic(plan) || run.Applied.RecoveryProtocol == appliedRecoveryProtocol +} + +func dokployTrafficRunID(run migrationRun) (string, error) { + runDir, err := dokployTrafficRunDir(run) + if err != nil { + return "", err + } + digest := sha256.Sum256([]byte(filepath.Clean(runDir) + "\n" + run.Name + "\n" + run.CreatedAt.UTC().Format(time.RFC3339Nano) + "\n" + run.BundleDigest)) + return hex.EncodeToString(digest[:]), nil +} + +func dokployTrafficRunDir(run migrationRun) (string, error) { + runDir, err := filepath.Abs(filepath.FromSlash(run.RunDir)) + if err != nil { + return "", fmt.Errorf("resolve migration run directory for Dokploy traffic ownership: %w", err) + } + if resolved, err := filepath.EvalSymlinks(runDir); err == nil { + runDir = resolved + } else if !os.IsNotExist(err) { + return "", fmt.Errorf("resolve migration run directory for Dokploy traffic ownership: %w", err) + } + return filepath.Clean(runDir), nil +} + +func dokployCredentialID(token string) string { + digest := sha256.Sum256([]byte(token)) + return hex.EncodeToString(digest[:]) +} + +func dokployOwnerRunLabel(owner dokployTrafficOwner) string { + if owner.RunDir != "" { + return fmt.Sprintf("migration run %q at %q", owner.RunName, owner.RunDir) + } + return fmt.Sprintf("migration run %q", owner.RunName) +} + +func readDokployTrafficOwner() (dokployTrafficOwner, bool, error) { + path, err := dokployTrafficOwnerPath() + if err != nil { + return dokployTrafficOwner{}, false, err + } + if path == "" { + return dokployTrafficOwner{}, false, nil + } + contents, err := readFileNoFollow(path) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return dokployTrafficOwner{}, false, nil + } + return dokployTrafficOwner{}, false, fmt.Errorf("read Dokploy traffic owner %s: %w", path, err) + } + var owner dokployTrafficOwner + if err := json.Unmarshal(contents, &owner); err != nil { + return dokployTrafficOwner{}, false, fmt.Errorf("decode Dokploy traffic owner %s: %w", path, err) + } + if owner.APIVersion != dokployTrafficOwnerAPIVersion || owner.RunID == "" || owner.RunName == "" { + return dokployTrafficOwner{}, false, fmt.Errorf("Dokploy traffic owner %s has invalid identity or API version %q", path, owner.APIVersion) + } + switch owner.Authority { + case dokployTrafficPending, dokployTrafficTarget, dokployTrafficSource, dokployTrafficReleased: + default: + return dokployTrafficOwner{}, false, fmt.Errorf("Dokploy traffic owner %s has invalid authority %q", path, owner.Authority) + } + if owner.Authority != dokployTrafficSource && owner.Authority != dokployTrafficReleased && owner.TargetOrigin == "" { + return dokployTrafficOwner{}, false, fmt.Errorf("Dokploy traffic owner %s has no target origin for %s authority", path, owner.Authority) + } + return owner, true, nil +} + +func writeDokployTrafficOwner(owner dokployTrafficOwner) error { + path, err := dokployTrafficOwnerPath() + if err != nil { + return err + } + if path == "" { + return fmt.Errorf("host-wide Dokploy traffic ownership is unsupported on this platform") + } + if err := prepareDokployTrafficOwnerPath(path); err != nil { + return err + } + owner.APIVersion = dokployTrafficOwnerAPIVersion + owner.UpdatedAt = time.Now().UTC() + if err := writeJSONArtifact(path, owner); err != nil { + return fmt.Errorf("write Dokploy traffic owner %s: %w", path, err) + } + return nil +} + +func ensureDokployTrafficRunAvailable(run migrationRun, plan dokploy.Plan) error { + if dokployOwnershipStepIndex(plan) < 0 { + return nil + } + owner, found, err := readDokployTrafficOwner() + if err != nil || !found || owner.Authority == dokployTrafficReleased { + return err + } + runID, err := dokployTrafficRunID(run) + if err != nil { + return err + } + if owner.RunID != runID { + return fmt.Errorf("Dokploy host is owned by %s; refusing run %q because shared target resources cannot be mutated by independent runs safely", dokployOwnerRunLabel(owner), run.Name) + } + return nil +} + +func ensureStandaloneDokployInitAvailable() error { + owner, found, err := readDokployTrafficOwner() + if err != nil || !found || owner.Authority == dokployTrafficReleased { + return err + } + return fmt.Errorf("Dokploy host is owned by %s with %s authority; refusing standalone init-target until that migration is committed or rolled back", dokployOwnerRunLabel(owner), owner.Authority) +} + +func conflictingDokployHostOwner(run migrationRun) (dokployTrafficOwner, bool, error) { + owner, found, err := readDokployTrafficOwner() + if err != nil || !found || owner.Authority == dokployTrafficReleased { + return owner, false, err + } + runID, err := dokployTrafficRunID(run) + if err != nil { + return dokployTrafficOwner{}, false, err + } + return owner, owner.RunID != runID, nil +} + +func runHoldsPendingDokployOwner(run migrationRun) (bool, error) { + owner, found, err := readDokployTrafficOwner() + if err != nil || !found || owner.Authority != dokployTrafficPending { + return false, err + } + runID, err := dokployTrafficRunID(run) + if err != nil { + return false, err + } + return owner.RunID == runID, nil +} + +func validateInlineDokploySetup(run migrationRun, applied runApplied, targetOrigin string) error { + if applied.TargetOrigin != "" || len(applied.Steps) > 0 || len(applied.Apps) > 0 || applied.SucceededAt != nil { + return fmt.Errorf("migration run %q already has a bound or in-progress Dokploy target; restore the exact original target credential and reachability instead of running inline setup", run.Name) + } + owner, found, err := readDokployTrafficOwner() + if err != nil || !found || owner.Authority == dokployTrafficReleased { + return err + } + runID, err := dokployTrafficRunID(run) + if err != nil { + return err + } + if owner.RunID != runID { + return fmt.Errorf("Dokploy host is owned by %s; refusing inline target setup for run %q", dokployOwnerRunLabel(owner), run.Name) + } + if targetOrigin != "" && owner.TargetOrigin != targetOrigin { + return fmt.Errorf("migration run %q owns Dokploy traffic for target %q, not %q", run.Name, owner.TargetOrigin, targetOrigin) + } + if owner.TargetCredentialID != "" { + return fmt.Errorf("migration run %q is bound to an existing Dokploy credential identity; restore the exact original credential instead of creating or storing a replacement", run.Name) + } + return nil +} + +func targetMutationMissingDurableOwner(run loadedMigrationRun) (bool, error) { + if run.Applied.APIVersion != appliedAPIVersion || run.Applied.RecoveryProtocol != appliedRecoveryProtocol { + return false, nil + } + plan := livePlanForApplied(run, run.Applied) + ownershipIndex := dokployOwnershipStepIndex(plan) + if ownershipIndex < 0 { + return false, nil + } + owner, found, err := readDokployTrafficOwner() + if err != nil { + return false, err + } + if found && owner.Authority != dokployTrafficReleased { + runID, err := dokployTrafficRunID(run.Run) + if err != nil { + return false, err + } + if owner.RunID == runID { + return false, nil + } + } + if completedApplyPrefix(plan.Steps, run.Applied) > ownershipIndex { + return true, nil + } + for _, recorded := range run.Applied.Steps { + if recorded.Index == ownershipIndex && appliedStepMatches(recorded, plan.Steps[ownershipIndex]) && appliedStepMayHaveRun(recorded) { + return true, nil + } + } + return false, nil +} + +func validateDokployTrafficResume(run migrationRun, plan dokploy.Plan, applied runApplied, targetOrigin, targetCredentialID string, resumeFrom int) error { + ownershipIndex := dokployOwnershipStepIndex(plan) + if ownershipIndex < 0 { + return nil + } + owner, found, err := readDokployTrafficOwner() + if err != nil { + return err + } + runID, err := dokployTrafficRunID(run) + if err != nil { + return err + } + if !found || owner.Authority == dokployTrafficReleased { + firstMutationAttempted := false + for _, recorded := range applied.Steps { + if recorded.Index == ownershipIndex && appliedStepMatches(recorded, plan.Steps[ownershipIndex]) && appliedStepMayHaveRun(recorded) { + firstMutationAttempted = true + break + } + } + if resumeFrom > ownershipIndex || firstMutationAttempted { + return fmt.Errorf("migration run %q changed Dokploy target resources without a matching durable host owner; refusing to infer authority", run.Name) + } + return nil + } + if owner.RunID != runID { + return fmt.Errorf("Dokploy host is owned by %s; refusing run %q because shared target resources cannot be mutated by independent runs safely", dokployOwnerRunLabel(owner), run.Name) + } + if owner.TargetOrigin != targetOrigin { + return fmt.Errorf("migration run %q owns Dokploy traffic for target %q, not %q", run.Name, owner.TargetOrigin, targetOrigin) + } + if owner.TargetCredentialID == "" { + for _, recorded := range applied.Steps { + if recorded.Index == ownershipIndex && appliedStepMatches(recorded, plan.Steps[ownershipIndex]) && appliedStepMayHaveRun(recorded) { + return fmt.Errorf("migration run %q attempted a Dokploy target mutation without a durable credential identity; refusing to infer target organization authority", run.Name) + } + } + } else if owner.TargetCredentialID != targetCredentialID { + return fmt.Errorf("migration run %q owns Dokploy traffic with a different target credential identity; restore the exact Dokploy API key this run claimed the host with and retry", run.Name) + } + return nil +} + +func claimDokployHostOwnership(run migrationRun, targetOrigin, targetCredentialID string) error { + if targetCredentialID == "" { + return fmt.Errorf("cannot claim Dokploy host ownership without a target credential identity") + } + runID, err := dokployTrafficRunID(run) + if err != nil { + return err + } + owner, found, err := readDokployTrafficOwner() + if err != nil { + return err + } + if found && owner.Authority != dokployTrafficReleased { + if owner.RunID != runID { + return fmt.Errorf("Dokploy host is owned by %s; refusing run %q", dokployOwnerRunLabel(owner), run.Name) + } + if owner.TargetOrigin != targetOrigin { + return fmt.Errorf("migration run %q owns Dokploy traffic for target %q, not %q", run.Name, owner.TargetOrigin, targetOrigin) + } + if owner.TargetCredentialID != "" && owner.TargetCredentialID != targetCredentialID { + return fmt.Errorf("migration run %q owns Dokploy traffic with a different target credential identity; restore the exact Dokploy API key this run claimed the host with and retry", run.Name) + } + } + runDir, err := dokployTrafficRunDir(run) + if err != nil { + return err + } + return writeDokployTrafficOwner(dokployTrafficOwner{RunID: runID, RunName: run.Name, RunDir: runDir, TargetOrigin: targetOrigin, TargetCredentialID: targetCredentialID, Authority: dokployTrafficPending}) +} + +func markDokployTrafficTarget(run migrationRun, targetOrigin string) error { + return updateDokployTrafficAuthority(run, targetOrigin, dokployTrafficTarget) +} + +func markDokployTrafficSource(run migrationRun) error { + return updateDokployTrafficAuthority(run, "", dokployTrafficSource) +} + +func releaseDokployTrafficOwner(run migrationRun) error { + return releaseDokployHostOwner(run, dokployTrafficSource) +} + +func releaseDokployTargetOwner(run migrationRun) error { + return releaseDokployHostOwner(run, dokployTrafficTarget) +} + +func releaseDokployHostOwner(run migrationRun, expectedAuthority string) error { + if run.HostOwnerReleaseStartedAt == nil { + if err := markRunHostOwnerReleaseStartedLocked(run); err != nil { + return fmt.Errorf("record Dokploy host-owner release start: %w", err) + } + } + owner, found, err := readDokployTrafficOwner() + if err != nil { + return err + } + if !found { + if run.HostOwnerReleaseStartedAt != nil { + return nil + } + return fmt.Errorf("migration run %q has no durable Dokploy host owner to release", run.Name) + } + runID, err := dokployTrafficRunID(run) + if err != nil { + return err + } + if owner.RunID != runID { + if run.HostOwnerReleaseStartedAt != nil { + return nil + } + return fmt.Errorf("migration run %q cannot release Dokploy host ownership held by %s", run.Name, dokployOwnerRunLabel(owner)) + } + if owner.Authority == dokployTrafficReleased { + return nil + } + if owner.Authority != expectedAuthority { + return fmt.Errorf("migration run %q cannot release Dokploy host ownership with %s authority", run.Name, owner.Authority) + } + owner.Authority = dokployTrafficReleased + return writeDokployTrafficOwner(owner) +} + +func ensureDokploySourceCleanupAvailable(run migrationRun) error { + owner, found, err := readDokployTrafficOwner() + if err != nil || !found { + return err + } + runID, err := dokployTrafficRunID(run) + if err != nil { + return err + } + if owner.RunID == runID && owner.Authority == dokployTrafficPending { + return nil + } + if owner.RunID == runID { + return fmt.Errorf("migration run %q cannot resume historical source writers after %s authority was established", run.Name, owner.Authority) + } + return fmt.Errorf("migration run %q cannot resume historical source writers because Dokploy host ownership belongs to %s with %s authority", run.Name, dokployOwnerRunLabel(owner), owner.Authority) +} + +func requireDokployTrafficTarget(run migrationRun, allowCompletedRollback bool) error { + owner, found, err := readDokployTrafficOwner() + if err != nil { + return err + } + runID, err := dokployTrafficRunID(run) + if err != nil { + return err + } + validAuthority := owner.Authority == dokployTrafficTarget || allowCompletedRollback && owner.Authority == dokployTrafficSource + if !found || owner.RunID != runID || !validAuthority { + if found { + return fmt.Errorf("migration run %q does not own target traffic; current durable owner is %s with %s authority", run.Name, dokployOwnerRunLabel(owner), owner.Authority) + } + return fmt.Errorf("migration run %q has no durable host-wide Dokploy traffic owner", run.Name) + } + return nil +} + +func validateCurrentDokployTrafficOwner(run loadedMigrationRun, allowSource bool) error { + if run.Applied.APIVersion != appliedAPIVersion || run.Applied.RecoveryProtocol != appliedRecoveryProtocol { + return nil + } + return requireDokployTrafficTarget(run.Run, allowSource) +} + +func completedDokployOwnerFinalization(run loadedMigrationRun) (bool, error) { + if run.Run.Target != "dokploy" || run.Applied.APIVersion != appliedAPIVersion || run.Applied.RecoveryProtocol != appliedRecoveryProtocol || run.Run.CommittedAt == nil && run.Run.RolledBackAt == nil { + return false, nil + } + if run.Run.ResolvedAuthority == dokployTrafficSource && run.Run.AuthorityFinalizedAt != nil { + return false, nil + } + owner, found, err := readDokployTrafficOwner() + if err != nil { + return false, fmt.Errorf("verify completed migration host ownership: %w", err) + } + releaseStarted := run.Run.HostOwnerReleaseStartedAt != nil + if !found { + if releaseStarted { + return false, nil + } + return false, fmt.Errorf("completed migration run %q has no durable Dokploy host owner", run.Run.Name) + } + runID, err := dokployTrafficRunID(run.Run) + if err != nil { + return false, err + } + if owner.RunID != runID { + if releaseStarted { + return false, nil + } + return false, fmt.Errorf("completed migration run %q cannot verify host-owner finalization because ownership belongs to %s with %s authority", run.Run.Name, dokployOwnerRunLabel(owner), owner.Authority) + } + if owner.Authority == dokployTrafficReleased { + return false, nil + } + expectedAuthority := dokployTrafficTarget + if run.Run.RolledBackAt != nil { + expectedAuthority = dokployTrafficSource + } + if owner.Authority != expectedAuthority { + return false, fmt.Errorf("completed migration run %q expected %s authority before host-owner release, but found %s authority", run.Run.Name, expectedAuthority, owner.Authority) + } + return true, nil +} + +func ensureDokployTrafficTargetOwner(ctx context.Context, run loadedMigrationRun, client *dokploy.Client, allowCompletedRollback bool) error { + owner, found, err := readDokployTrafficOwner() + if err != nil { + return err + } + if found && owner.Authority != dokployTrafficPending { + return requireDokployTrafficTarget(run.Run, allowCompletedRollback) + } + legacyCompleted := appliedPlanVersion(run.Applied) == appliedPlanV1Alpha1 && run.Applied.SucceededAt != nil + if !legacyCompleted { + return requireDokployTrafficTarget(run.Run, allowCompletedRollback) + } + if err := requireLiveApplySucceeded(run); err != nil { + return err + } + if run.Applied.TargetOrigin == "" { + return fmt.Errorf("completed legacy run %q has no persisted Dokploy origin; refusing to bind prior live execution to current credentials. preserve both sides and establish traffic authority manually", run.Run.Name) + } + if found { + runID, err := dokployTrafficRunID(run.Run) + if err != nil { + return err + } + if owner.RunID != runID { + return requireDokployTrafficTarget(run.Run, allowCompletedRollback) + } + } + if client == nil { + client, err = lookupDokployClient(run.Run.Target) + if err != nil { + return fmt.Errorf("load Dokploy credentials for legacy traffic-owner adoption: %w", err) + } + } + if client.BaseURL != run.Applied.TargetOrigin { + return fmt.Errorf("completed legacy run %q is bound to Dokploy origin %s, not %s", run.Run.Name, run.Applied.TargetOrigin, client.BaseURL) + } + if found { + if owner.TargetOrigin != client.BaseURL { + return fmt.Errorf("completed legacy run %q has pending Dokploy ownership for target %q, not %q", run.Run.Name, owner.TargetOrigin, client.BaseURL) + } + credentialID := dokployCredentialID(client.Token) + if owner.TargetCredentialID == "" || owner.TargetCredentialID != credentialID { + return fmt.Errorf("completed legacy run %q has pending Dokploy ownership with a missing or different target credential identity", run.Run.Name) + } + } + if err := client.VerifySameDockerHost(ctx); err != nil { + return fmt.Errorf("verify Dokploy target is on this Docker host before legacy traffic-owner adoption: %w", err) + } + if err := client.Ping(ctx); err != nil { + return fmt.Errorf("verify Dokploy target before legacy traffic-owner adoption: %w", err) + } + if err := client.AdoptTargetTrafficAuthority(ctx); err != nil { + return fmt.Errorf("legacy traffic-owner adoption refused: %w", err) + } + if !found { + if err := claimDokployHostOwnership(run.Run, client.BaseURL, dokployCredentialID(client.Token)); err != nil { + return err + } + } + if err := markDokployTrafficTarget(run.Run, client.BaseURL); err != nil { + return err + } + return nil +} + +func updateDokployTrafficAuthority(run migrationRun, targetOrigin, authority string) error { + owner, found, err := readDokployTrafficOwner() + if err != nil { + return err + } + runID, err := dokployTrafficRunID(run) + if err != nil { + return err + } + if !found || owner.RunID != runID { + return fmt.Errorf("migration run %q does not own the durable Dokploy traffic handoff", run.Name) + } + if targetOrigin != "" && owner.TargetOrigin != targetOrigin { + return fmt.Errorf("migration run %q owns Dokploy traffic for target %q, not %q", run.Name, owner.TargetOrigin, targetOrigin) + } + owner.Authority = authority + return writeDokployTrafficOwner(owner) +} diff --git a/internal/cli/dokploy_traffic_owner_test.go b/internal/cli/dokploy_traffic_owner_test.go new file mode 100644 index 0000000..06fc9ca --- /dev/null +++ b/internal/cli/dokploy_traffic_owner_test.go @@ -0,0 +1,358 @@ +package cli + +import ( + "os" + "path/filepath" + "runtime" + "strings" + "testing" + "time" + + "github.com/aikins01/bort/internal/preparer" + "github.com/aikins01/bort/internal/target/dokploy" +) + +func TestDokployTrafficRunIDIgnoresSymlinkedRunDirSpelling(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("symlink creation requires privileges on Windows") + } + base := t.TempDir() + realDir := filepath.Join(base, "real", "runs", "run-a") + if err := os.MkdirAll(realDir, 0o700); err != nil { + t.Fatal(err) + } + if err := os.Symlink(filepath.Join(base, "real"), filepath.Join(base, "alias")); err != nil { + t.Fatal(err) + } + now := time.Now().UTC() + viaReal := migrationRun{Name: "run-a", RunDir: realDir, CreatedAt: now, BundleDigest: "digest-a"} + viaAlias := migrationRun{Name: "run-a", RunDir: filepath.Join(base, "alias", "runs", "run-a"), CreatedAt: now, BundleDigest: "digest-a"} + realID, err := dokployTrafficRunID(viaReal) + if err != nil { + t.Fatal(err) + } + aliasID, err := dokployTrafficRunID(viaAlias) + if err != nil { + t.Fatal(err) + } + if realID != aliasID { + t.Fatalf("run ID depends on path spelling: real=%s alias=%s", realID, aliasID) + } +} + +func TestDokployTrafficOwnerSerializesRunsAcrossWorkspaces(t *testing.T) { + resetDokployTrafficOwner(t) + now := time.Now().UTC() + workspace := t.TempDir() + t.Chdir(workspace) + claimed := persistRunFixture(t, migrationRun{Name: "run-a", RunDir: filepath.Join(".bort", "runs", "run-a"), CreatedAt: now, BundleDigest: "digest-a", RolledBackAt: &now}) + runB := migrationRun{Name: "run-b", RunDir: filepath.Join(t.TempDir(), "run-b"), CreatedAt: time.Now().UTC(), BundleDigest: "digest-b"} + plan := dokploy.Plan{Steps: []dokploy.Step{{Kind: dokploy.StepCreateProject, App: "api", Ref: "api"}}} + + if err := ensureDokployTrafficRunAvailable(claimed, plan); err != nil { + t.Fatal(err) + } + if err := claimDokployHostOwnership(claimed, "http://127.0.0.1:3030", dokployCredentialID("test-token")); err != nil { + t.Fatal(err) + } + if err := markDokployTrafficTarget(claimed, "http://127.0.0.1:3030"); err != nil { + t.Fatal(err) + } + + runADir, err := dokployTrafficRunDir(claimed) + if err != nil { + t.Fatal(err) + } + runALabel := "migration run \"run-a\" at \"" + runADir + "\"" + runA := claimed + runA.RunDir = filepath.Join(workspace, ".bort", "runs", "run-a") + + t.Chdir(t.TempDir()) + if err := requireDokployTrafficTarget(runA, false); err != nil { + t.Fatalf("same run lost ownership after changing workspaces: %v", err) + } + if err := ensureDokployTrafficRunAvailable(runB, plan); err == nil || !strings.Contains(err.Error(), "owned by "+runALabel) { + t.Fatalf("expected second run target-mutation refusal, got %v", err) + } + if err := requireDokployTrafficTarget(runB, false); err == nil || !strings.Contains(err.Error(), "current durable owner is "+runALabel) { + t.Fatalf("expected second run rollback refusal, got %v", err) + } + if err := markDokployTrafficSource(runA); err != nil { + t.Fatal(err) + } + if err := ensureDokployTrafficRunAvailable(runB, plan); err == nil || !strings.Contains(err.Error(), "owned by "+runALabel) { + t.Fatalf("source authority released ownership before rollback finalization: %v", err) + } + if err := releaseDokployTrafficOwner(runA); err != nil { + t.Fatal(err) + } + if err := ensureDokployTrafficRunAvailable(runB, plan); err != nil { + t.Fatalf("finalized rollback did not release the proxy for a new run: %v", err) + } +} + +func TestDokployTrafficOwnerBindsCredentialWithoutPersistingToken(t *testing.T) { + resetDokployTrafficOwner(t) + run := migrationRun{Name: "bound", RunDir: filepath.Join(t.TempDir(), "bound"), CreatedAt: time.Now().UTC(), BundleDigest: "digest"} + const token = "organization-a-secret-token" + if err := claimDokployHostOwnership(run, "http://127.0.0.1:3030", dokployCredentialID(token)); err != nil { + t.Fatal(err) + } + owner, found, err := readDokployTrafficOwner() + if err != nil || !found { + t.Fatalf("read owner: found=%t err=%v", found, err) + } + wantDir, err := dokployTrafficRunDir(run) + if err != nil { + t.Fatal(err) + } + if owner.RunDir != wantDir || owner.TargetCredentialID != dokployCredentialID(token) { + t.Fatalf("owner did not preserve run path and credential identity: %#v", owner) + } + path, err := dokployTrafficOwnerPath() + if err != nil { + t.Fatal(err) + } + contents, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(contents), token) { + t.Fatal("durable host owner persisted the raw Dokploy token") + } + plan := dokploy.Plan{Steps: []dokploy.Step{{Kind: dokploy.StepCreateProject, App: "api", Ref: "api"}}} + err = validateDokployTrafficResume(run, plan, runApplied{}, "http://127.0.0.1:3030", dokployCredentialID("organization-b-secret-token"), 0) + if err == nil || !strings.Contains(err.Error(), "different target credential identity") { + t.Fatalf("expected swapped organization credential refusal, got %v", err) + } +} + +func TestInlineDokploySetupRefusesExistingRunBinding(t *testing.T) { + resetDokployTrafficOwner(t) + run := migrationRun{Name: "bound", RunDir: filepath.Join(t.TempDir(), "bound"), CreatedAt: time.Now().UTC(), BundleDigest: "digest"} + if err := validateInlineDokploySetup(run, runApplied{TargetOrigin: "http://127.0.0.1:3030"}, "http://127.0.0.1:3030"); err == nil || !strings.Contains(err.Error(), "restore the exact original target credential") { + t.Fatalf("expected persisted target binding to block inline setup, got %v", err) + } + if err := claimDokployHostOwnership(run, "http://127.0.0.1:3030", dokployCredentialID("original-token")); err != nil { + t.Fatal(err) + } + if err := validateInlineDokploySetup(run, runApplied{}, "http://127.0.0.1:3030"); err == nil || !strings.Contains(err.Error(), "bound to an existing Dokploy credential identity") { + t.Fatalf("expected durable credential identity to block replacement setup, got %v", err) + } +} + +func TestDokployTrafficOwnerRefusesMissingStateAfterHandoff(t *testing.T) { + resetDokployTrafficOwner(t) + run := migrationRun{Name: "resumed", RunDir: t.TempDir(), CreatedAt: time.Now().UTC(), BundleDigest: "digest"} + plan := dokploy.Plan{Steps: []dokploy.Step{ + {Kind: dokploy.StepStopCoolifyProxy, Ref: "coolify-proxy"}, + {Kind: dokploy.StepStartDokployProxy, Ref: "dokploy-traefik"}, + }} + if err := validateDokployTrafficResume(run, plan, runApplied{}, "http://127.0.0.1:3030", dokployCredentialID("test-token"), 1); err == nil || !strings.Contains(err.Error(), "without a matching durable host owner") { + t.Fatalf("expected missing post-handoff owner to fail closed, got %v", err) + } +} + +func TestDokployTrafficOwnerRefusesMissingStateAfterRouteMutation(t *testing.T) { + resetDokployTrafficOwner(t) + run := migrationRun{Name: "resumed", RunDir: t.TempDir(), CreatedAt: time.Now().UTC(), BundleDigest: "digest"} + plan := dokploy.Plan{Steps: []dokploy.Step{ + {Kind: dokploy.StepInstallGateway, App: "api", Ref: "api.example.com"}, + {Kind: dokploy.StepStopCoolifyProxy, Ref: "coolify-proxy"}, + {Kind: dokploy.StepStartDokployProxy, Ref: "dokploy-traefik"}, + }} + if index := dokployOwnershipStepIndex(plan); index != 0 { + t.Fatalf("Dokploy ownership index = %d, want 0", index) + } + if err := validateDokployTrafficResume(run, plan, runApplied{}, "http://127.0.0.1:3030", dokployCredentialID("test-token"), 1); err == nil || !strings.Contains(err.Error(), "changed Dokploy target resources without a matching durable host owner") { + t.Fatalf("expected missing post-route owner to fail closed, got %v", err) + } +} + +func TestDokployTrafficOwnerRefusesMissingStateAfterInterruptedRouteMutation(t *testing.T) { + resetDokployTrafficOwner(t) + run := migrationRun{Name: "resumed", RunDir: t.TempDir(), CreatedAt: time.Now().UTC(), BundleDigest: "digest"} + step := dokploy.Step{Kind: dokploy.StepInstallGateway, App: "api", Ref: "api.example.com"} + plan := dokploy.Plan{Steps: []dokploy.Step{step, {Kind: dokploy.StepStopCoolifyProxy, Ref: "coolify-proxy"}}} + applied := runApplied{Steps: []appliedStep{{ + Index: 0, Kind: string(step.Kind), App: step.App, Ref: step.Ref, Status: string(dokploy.StepStatusStarted), + }}} + + err := validateDokployTrafficResume(run, plan, applied, "http://127.0.0.1:3030", dokployCredentialID("test-token"), 0) + if err == nil || !strings.Contains(err.Error(), "without a matching durable host owner") { + t.Fatalf("expected interrupted route mutation with missing owner to fail closed, got %v", err) + } +} + +func TestTargetMutationDoesNotBorrowAnotherRunsDurableOwner(t *testing.T) { + resetDokployTrafficOwner(t) + runA := migrationRun{Name: "run-a", RunDir: filepath.Join(t.TempDir(), "run-a"), CreatedAt: time.Now().UTC(), BundleDigest: "digest-a"} + runB := migrationRun{Name: "run-b", RunDir: filepath.Join(t.TempDir(), "run-b"), CreatedAt: time.Now().UTC(), BundleDigest: "digest-b"} + if err := claimDokployHostOwnership(runB, "http://127.0.0.1:3030", dokployCredentialID("run-b-token")); err != nil { + t.Fatal(err) + } + loaded := loadedMigrationRun{ + Run: runA, + Prepare: preparer.Result{Apps: []preparer.AppPlan{{Name: "api"}}}, + Applied: runApplied{ + APIVersion: appliedAPIVersion, + RecoveryProtocol: appliedRecoveryProtocol, + Steps: []appliedStep{{ + Index: 0, Kind: string(dokploy.StepCreateProject), App: "api", Ref: "api", Status: string(dokploy.StepStatusOK), + }}, + }, + } + missing, err := targetMutationMissingDurableOwner(loaded) + if err != nil { + t.Fatal(err) + } + if !missing || !runMayHaveAmbiguousAuthority(loaded) { + t.Fatalf("run A borrowed run B's owner: missing=%t ambiguous=%t", missing, runMayHaveAmbiguousAuthority(loaded)) + } +} + +func TestDokploySourceCleanupRefusesEstablishedHostAuthority(t *testing.T) { + resetDokployTrafficOwner(t) + ownerRun := migrationRun{Name: "owner", RunDir: filepath.Join(t.TempDir(), "owner"), CreatedAt: time.Now().UTC(), BundleDigest: "owner"} + cleanupRun := migrationRun{Name: "cleanup", RunDir: filepath.Join(t.TempDir(), "cleanup"), CreatedAt: time.Now().UTC(), BundleDigest: "cleanup"} + if err := claimDokployHostOwnership(ownerRun, "http://127.0.0.1:3030", dokployCredentialID("test-token")); err != nil { + t.Fatal(err) + } + if err := markDokployTrafficTarget(ownerRun, "http://127.0.0.1:3030"); err != nil { + t.Fatal(err) + } + ownerDir, err := dokployTrafficRunDir(ownerRun) + if err != nil { + t.Fatal(err) + } + if err := ensureDokploySourceCleanupAvailable(cleanupRun); err == nil || !strings.Contains(err.Error(), "ownership belongs to migration run \"owner\" at \""+ownerDir+"\"") { + t.Fatalf("expected historical source cleanup to refuse established target authority, got %v", err) + } +} + +func TestDokployTrafficOwnerRejectsMalformedAndSymlinkState(t *testing.T) { + t.Run("malformed", func(t *testing.T) { + resetDokployTrafficOwner(t) + path, err := dokployTrafficOwnerPath() + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte("{not-json"), 0o600); err != nil { + t.Fatal(err) + } + if _, _, err := readDokployTrafficOwner(); err == nil || !strings.Contains(err.Error(), "decode Dokploy traffic owner") { + t.Fatalf("expected malformed owner refusal, got %v", err) + } + }) + + t.Run("symlink", func(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("symlink creation may require elevated privileges") + } + resetDokployTrafficOwner(t) + path, err := dokployTrafficOwnerPath() + if err != nil { + t.Fatal(err) + } + ownerRun := migrationRun{Name: "owner", RunDir: filepath.Join(t.TempDir(), "owner"), CreatedAt: time.Now().UTC(), BundleDigest: "owner"} + if err := claimDokployHostOwnership(ownerRun, "http://127.0.0.1:3030", dokployCredentialID("test-token")); err != nil { + t.Fatal(err) + } + if _, found, err := readDokployTrafficOwner(); err != nil || !found { + t.Fatalf("claimed owner record is not valid: found=%t err=%v", found, err) + } + target := filepath.Join(t.TempDir(), "owner.json") + if err := os.Rename(path, target); err != nil { + t.Fatal(err) + } + if err := os.Symlink(target, path); err != nil { + t.Fatal(err) + } + if _, _, err := readDokployTrafficOwner(); err == nil || !strings.Contains(err.Error(), "read Dokploy traffic owner") { + t.Fatalf("expected symlink owner state to fail closed on the no-follow read, got %v", err) + } + }) +} + +func TestDefaultDokployTrafficOwnerPathIsAbsoluteAcrossWorkspaces(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("host Dokploy state is unsupported on Windows") + } + first, err := defaultDokployTrafficOwnerPath() + if err != nil { + t.Fatal(err) + } + t.Chdir(t.TempDir()) + second, err := defaultDokployTrafficOwnerPath() + if err != nil { + t.Fatal(err) + } + if !filepath.IsAbs(first) || first != second { + t.Fatalf("host owner path must not depend on the working directory: %q vs %q", first, second) + } +} + +func TestReleasedHostOwnerSurvivesNextRunClaim(t *testing.T) { + resetDokployTrafficOwner(t) + now := time.Now().UTC() + runA := persistRunFixture(t, migrationRun{Name: "run-a", RunDir: filepath.Join(t.TempDir(), "run-a"), Target: "dokploy", CreatedAt: now, BundleDigest: "digest-a", CommitStartedAt: &now, CommittedAt: &now, ResolvedAuthority: dokployTrafficTarget, AuthorityResolvedAt: &now}) + runB := migrationRun{Name: "run-b", RunDir: filepath.Join(t.TempDir(), "run-b"), CreatedAt: now, BundleDigest: "digest-b"} + if err := claimDokployHostOwnership(runA, "http://127.0.0.1:3030", dokployCredentialID("token-a")); err != nil { + t.Fatal(err) + } + if err := markDokployTrafficTarget(runA, "http://127.0.0.1:3030"); err != nil { + t.Fatal(err) + } + if err := releaseDokployTargetOwner(runA); err != nil { + t.Fatal(err) + } + if err := claimDokployHostOwnership(runB, "http://127.0.0.1:3030", dokployCredentialID("token-b")); err != nil { + t.Fatal(err) + } + loaded := reloadRunFixture(t, runA) + if loaded.Run.HostOwnerReleaseStartedAt == nil { + t.Fatal("release did not persist its write-ahead marker in run.json") + } + if pending, err := completedDokployOwnerFinalization(loaded); err != nil || pending { + t.Fatalf("completed run lost its finalization after another run claimed the host: pending=%t err=%v", pending, err) + } + if err := releaseDokployTargetOwner(loaded.Run); err != nil { + t.Fatalf("repeat release after another run claimed the host: %v", err) + } + if _, found, err := matchingAuthorityRecoveryOwner(loaded.Run, dokployTrafficTarget); err != nil || found { + t.Fatalf("authority recovery no longer recognizes the released owner: found=%t err=%v", found, err) + } + owner, found, err := readDokployTrafficOwner() + if err != nil || !found || owner.RunName != "run-b" || owner.Authority != dokployTrafficPending { + t.Fatalf("completed run disturbed the next run's ownership: found=%t err=%v owner=%#v", found, err, owner) + } +} + +func TestHostOwnerReleaseMarkerKeepsUnreleasedOwnerPending(t *testing.T) { + resetDokployTrafficOwner(t) + now := time.Now().UTC() + run := persistRunFixture(t, migrationRun{Name: "run-a", RunDir: filepath.Join(t.TempDir(), "run-a"), Target: "dokploy", CreatedAt: now, BundleDigest: "digest-a", RollbackStartedAt: &now, RolledBackAt: &now}) + if err := claimDokployHostOwnership(run, "http://127.0.0.1:3030", dokployCredentialID("token-a")); err != nil { + t.Fatal(err) + } + if err := markDokployTrafficSource(run); err != nil { + t.Fatal(err) + } + if err := markRunHostOwnerReleaseStartedLocked(run); err != nil { + t.Fatal(err) + } + loaded := reloadRunFixture(t, run) + if pending, err := completedDokployOwnerFinalization(loaded); err != nil || !pending { + t.Fatalf("marker without a released owner must still require release: pending=%t err=%v", pending, err) + } + if err := releaseDokployTrafficOwner(loaded.Run); err != nil { + t.Fatal(err) + } + owner, _, err := readDokployTrafficOwner() + if err != nil || owner.Authority != dokployTrafficReleased { + t.Fatalf("expected released owner, got %#v err=%v", owner, err) + } + if pending, err := completedDokployOwnerFinalization(loaded); err != nil || pending { + t.Fatalf("release did not finalize: pending=%t err=%v", pending, err) + } +} diff --git a/internal/cli/guide.go b/internal/cli/guide.go index cc58402..23ab6a5 100644 --- a/internal/cli/guide.go +++ b/internal/cli/guide.go @@ -13,6 +13,7 @@ import ( "strings" "time" + "github.com/aikins01/bort/internal/dockercli" "github.com/aikins01/bort/internal/exporter" "github.com/aikins01/bort/internal/gateway" "github.com/aikins01/bort/internal/manifest" @@ -38,6 +39,10 @@ type guidePromptInput interface { } func runGuide(ctx context.Context, stdin io.Reader, stdout, stderr io.Writer) error { + handled, err := writeStandaloneDokployInstallationRecovery(stdout) + if err != nil || handled { + return err + } runRef, currentSelected, ok, err := guideRunRef() if err != nil { return err @@ -48,7 +53,7 @@ func runGuide(ctx context.Context, stdin io.Reader, stdout, stderr io.Writer) er if err != nil { return err } - writeAppFirstCockpit(stdout, run) + writeAppFirstCockpitContext(ctx, stdout, run) fmt.Fprintf(stdout, "This existing run is not selected for mutation. Run `%s` to make it current and refresh its dry-run.\n", bortCommand("migrate --run "+shellQuote(run.Run.Name))) return nil } @@ -58,7 +63,7 @@ func runGuide(ctx context.Context, stdin io.Reader, stdout, stderr io.Writer) er if err != nil { return err } - writeAppFirstCockpit(stdout, run) + writeAppFirstCockpitContext(ctx, stdout, run) return nil } run, err := refreshGuideRun(ctx, runRef, stdin, stdout) @@ -68,7 +73,7 @@ func runGuide(ctx context.Context, stdin io.Reader, stdout, stderr io.Writer) er if isRealTTY(stdin, stdout) { return runWizard(ctx, run, stdin, stdout, stderr) } - writeAppFirstCockpit(stdout, run) + writeAppFirstCockpitContext(ctx, stdout, run) return nil } @@ -85,7 +90,7 @@ func runGuide(ctx context.Context, stdin io.Reader, stdout, stderr io.Writer) er if err := rememberCurrentRun(run.Run); err != nil { return err } - writeAppFirstCockpit(stdout, run) + writeAppFirstCockpitContext(ctx, stdout, run) return nil } @@ -119,7 +124,7 @@ func runGuide(ctx context.Context, stdin io.Reader, stdout, stderr io.Writer) er if isRealTTY(stdin, stdout) { return runWizard(ctx, run, stdin, stdout, stderr) } - writeAppFirstCockpit(stdout, run) + writeAppFirstCockpitContext(ctx, stdout, run) return nil } @@ -137,11 +142,11 @@ func refreshGuideRun(ctx context.Context, runRef string, stdin io.Reader, stdout if err != nil { return loadedMigrationRun{}, err } - hasAppliedSteps, err := runHasAppliedSteps(existing) + liveExecutionStarted, err := runHasStartedLiveExecution(existing) if err != nil { return loadedMigrationRun{}, fmt.Errorf("read applied migration progress: %w", err) } - if existing.LiveAppliedAt != nil || existing.CommitStartedAt != nil || existing.CommittedAt != nil || existing.RollbackStartedAt != nil || existing.RolledBackAt != nil || existing.PurgedAt != nil || hasAppliedSteps { + if existing.LiveAppliedAt != nil || existing.CommitStartedAt != nil || existing.CommittedAt != nil || existing.RollbackStartedAt != nil || existing.RolledBackAt != nil || existing.PurgedAt != nil || liveExecutionStarted { return loadMigrationRun(runRef) } if shouldAutoRescanRun(existing) { @@ -180,7 +185,7 @@ func shouldAutoRescanRun(run migrationRun) bool { return containedPath(runDir, bundleDir) == nil } -func runHasAppliedSteps(run migrationRun) (bool, error) { +func runHasStartedLiveExecution(run migrationRun) (bool, error) { run.Artifacts = run.Artifacts.withDefaults() runDir := filepath.FromSlash(run.RunDir) if runDir == "" { @@ -194,7 +199,7 @@ func runHasAppliedSteps(run migrationRun) (bool, error) { if err != nil { return false, err } - return len(applied.Steps) > 0, nil + return applied.TargetOrigin != "" || len(applied.Steps) > 0, nil } func refreshRunSourceBundle(ctx context.Context, run migrationRun) (string, string, error) { @@ -394,7 +399,13 @@ func checkSourceRequirements(ctx context.Context, sourceName string) error { if _, err := exec.LookPath("docker"); err != nil { return fmt.Errorf("docker CLI not found: %w; install Docker on the source server first", err) } - out, err := exec.CommandContext(ctx, "docker", "ps", "-q").CombinedOutput() + env, err := dockercli.LocalEnvironment(os.Environ()) + if err != nil { + return err + } + cmd := exec.CommandContext(ctx, "docker", "ps", "-q") + cmd.Env = env + out, err := cmd.CombinedOutput() if err != nil { detail := firstOutputLine(out) if detail == "" { diff --git a/internal/cli/guide_test.go b/internal/cli/guide_test.go index 4c5d998..732057d 100644 --- a/internal/cli/guide_test.go +++ b/internal/cli/guide_test.go @@ -67,9 +67,9 @@ func TestRunWithoutArgsCreatesRunFromDefaultBundle(t *testing.T) { workDir := t.TempDir() t.Chdir(workDir) writeTestBundle(t, filepath.Join(workDir, "bort-bundle"), manifest.Manifest{ - Source: manifest.Source{Platform: "docker"}, + Source: manifest.Source{Platform: "docker", DockerEngineID: "engine-reviewed"}, Apps: []manifest.App{ - {Name: "api", Services: []manifest.Service{{Name: "api", Image: "example/api:latest"}}, Routes: []manifest.Route{{Host: "api.example.com", ServiceName: "api"}}}, + {Name: "api", Services: []manifest.Service{{ID: "0123456789ab", Name: "api", Image: "example/api:latest"}}, Routes: []manifest.Route{{Host: "api.example.com", ServiceName: "api"}}}, }, }) @@ -80,12 +80,12 @@ func TestRunWithoutArgsCreatesRunFromDefaultBundle(t *testing.T) { } output := stdout.String() - for _, want := range []string{"local bundle → dokploy", "api", "READY", "workspace:"} { + for _, want := range []string{"local bundle → dokploy", "api", "INSPECTION ONLY", "imported bundle", "workspace:"} { if !strings.Contains(output, want) { t.Fatalf("expected guide output to contain %q, got:\n%s", want, output) } } - for _, notWant := range []string{"No migration run found", "Migration run created:", "Artifacts:"} { + for _, notWant := range []string{"No migration run found", "Migration run created:", "Artifacts:", "migrate --live"} { if strings.Contains(output, notWant) { t.Fatalf("did not expect guide output to contain %q, got:\n%s", notWant, output) } @@ -100,9 +100,9 @@ func TestRunWithoutArgsResumesLatestRun(t *testing.T) { workDir := t.TempDir() t.Chdir(workDir) writeTestBundle(t, filepath.Join(workDir, "bort-bundle"), manifest.Manifest{ - Source: manifest.Source{Platform: "docker"}, + Source: manifest.Source{Platform: "docker", DockerEngineID: "engine-reviewed"}, Apps: []manifest.App{ - {Name: "api", Services: []manifest.Service{{Name: "api", Image: "example/api:latest"}}, Routes: []manifest.Route{{Host: "api.example.com", ServiceName: "api"}}}, + {Name: "api", Services: []manifest.Service{{ID: "0123456789ab", Name: "api", Image: "example/api:latest"}}, Routes: []manifest.Route{{Host: "api.example.com", ServiceName: "api"}}}, }, }) runCommand(t, runMigrate, []string{"--bundle", "bort-bundle", "--run", "demo-app", "--observation-window", "0", "--rollback-window", "0"}) @@ -114,7 +114,7 @@ func TestRunWithoutArgsResumesLatestRun(t *testing.T) { } output := stdout.String() - for _, want := range []string{"local bundle → dokploy", "READY"} { + for _, want := range []string{"local bundle → dokploy", "INSPECTION ONLY"} { if !strings.Contains(output, want) { t.Fatalf("expected resumed guide output to contain %q, got:\n%s", want, output) } @@ -209,7 +209,9 @@ func TestRunWithoutArgsDoesNotRestoreStaleCurrentRunAfterRefresh(t *testing.T) { enteredPath := filepath.Join(workDir, "guide-entered") releasePath := filepath.Join(workDir, "guide-release") dockerScript := `#!/bin/sh -if [ "$1" = "ps" ]; then +if [ "$1" = "info" ]; then + printf 'engine-guide\n' +elif [ "$1" = "ps" ]; then touch "$BORT_GUIDE_ENTERED" while [ ! -f "$BORT_GUIDE_RELEASE" ]; do sleep 0.01; done printf 'reviewed-container\n' @@ -532,7 +534,9 @@ func prepareScannableGuideRun(t *testing.T, workDir, runName string) loadedMigra binDir := t.TempDir() dockerPath := filepath.Join(binDir, "docker") dockerScript := `#!/bin/sh -if [ "$1" = "ps" ]; then +if [ "$1" = "info" ]; then + printf 'engine-guide\n' +elif [ "$1" = "ps" ]; then printf 'reviewed-container\n' elif [ "$1" = "inspect" ]; then printf '%s\n' '[{"Id":"reviewed-container","Name":"/reviewed","Image":"sha256:reviewed","Config":{"Image":"example/reviewed:v2","Labels":{}},"State":{"Status":"running"},"Mounts":[],"NetworkSettings":{"Ports":{},"Networks":{}}}]' diff --git a/internal/cli/init_target.go b/internal/cli/init_target.go index 206e5c0..0015e8a 100644 --- a/internal/cli/init_target.go +++ b/internal/cli/init_target.go @@ -4,6 +4,8 @@ import ( "bufio" "bytes" "context" + "crypto/sha256" + "encoding/hex" "errors" "flag" "fmt" @@ -11,27 +13,39 @@ import ( "net" "net/http" "net/mail" + "net/netip" "net/url" "os" "os/exec" "path/filepath" "regexp" + "slices" "strconv" "strings" "time" + "unicode/utf8" + "github.com/aikins01/bort/internal/dockercli" "github.com/aikins01/bort/internal/target/dokploy" "golang.org/x/crypto/bcrypt" "golang.org/x/term" ) const ( - defaultCoolifyEnvPath = "/data/coolify/source/.env" - defaultCoolifyDBContainer = "coolify-db" - defaultDokployAPIName = "bort-cli" - envCoolifyAdminPwd = "BORT_COOLIFY_ADMIN_PASSWORD" - installProgressPrefix = "BORT_INSTALL_STEP\t" - psqlAdminFieldSeparator = "\t" + defaultCoolifyEnvPath = "/data/coolify/source/.env" + defaultCoolifyDBContainer = "coolify-db" + defaultDokployAPIName = "bort-cli" + defaultDokployVersion = "v0.30.7" + defaultDokployDigest = "sha256:62e4354a49ee686ea749d3541d93113d494501befe81f5226c7f859f201e3a7c" + envCoolifyAdminPwd = "BORT_COOLIFY_ADMIN_PASSWORD" + envDokployAuthBackup = "BORT_DOKPLOY_AUTH_SECRET_BACKUP" + envDokployEndpointMode = "ENDPOINT_MODE" + installProgressPrefix = "BORT_INSTALL_STEP\t" + psqlAdminFieldSeparator = "\t" + dokployInstallTimeout = 20 * time.Minute + dokployInstallRecoveryAPIVersion = "bort.dokploy-install-recovery/v1alpha1" + dokployInstallInstalling = "installing" + dokployInstallAPIKey = "api-key-pending" ) type coolifyAdmin struct { @@ -45,32 +59,172 @@ type coolifyAdminLister interface { } type initTargetDeps struct { - lister coolifyAdminLister - installer dokployInstaller - newClient func(baseURL string) *dokploy.Client - statePath string + lister coolifyAdminLister + installer dokployInstaller + newClient func(baseURL string) *dokploy.Client + verifyLocalService func(context.Context, *dokploy.Client) error + statePath string + liveOperationLock *applyLock + retainLiveOperationLock **applyLock + revalidateAfterTargetLock func() error + installTimeout time.Duration } type dokployInstallOptions struct { - HostPort string - AddrPool string - Version string - ACMEEmail string + HostPort string + AddrPool string + Version string + EndpointMode string + ACMEEmail string + AuthSecretBackup string + AdminName string + APIKeyName string + operationLock *os.File +} + +type dokployInstallationRecovery struct { + APIVersion string `json:"apiVersion"` + Identity string `json:"identity"` + Phase string `json:"phase"` + CommandPrefix string `json:"commandPrefix"` + TargetURL string `json:"targetUrl"` + HostPort string `json:"hostPort"` + AddressPool string `json:"addressPool"` + DokployVersion string `json:"dokployVersion"` + EndpointMode string `json:"endpointMode"` + ACMEEmail string `json:"acmeEmail"` + AuthSecretBackup string `json:"authSecretBackup"` + AdminName string `json:"adminName"` + APIKeyName string `json:"apiKeyName"` + APIKeyBaselineIDs []string `json:"apiKeyBaselineIds"` + APIKeyCreatedID string `json:"apiKeyCreatedId,omitempty"` + Command string `json:"command"` +} + +func newDokployInstallationRecovery(baseURL string, opts dokployInstallOptions) dokployInstallationRecovery { + recovery := canonicalDokployInstallationRecovery(dokployInstallationRecovery{ + APIVersion: dokployInstallRecoveryAPIVersion, + Phase: dokployInstallInstalling, + CommandPrefix: bortCommand(""), + TargetURL: baseURL, + HostPort: opts.HostPort, + AddressPool: opts.AddrPool, + DokployVersion: opts.Version, + EndpointMode: opts.EndpointMode, + ACMEEmail: opts.ACMEEmail, + AuthSecretBackup: opts.AuthSecretBackup, + AdminName: opts.AdminName, + APIKeyName: opts.APIKeyName, + }) + recovery.Identity = dokployInstallRecoveryIdentity(recovery) + recovery.Command = dokployInstallRecoveryCommand(recovery) + return recovery +} + +func canonicalDokployInstallationRecovery(recovery dokployInstallationRecovery) dokployInstallationRecovery { + recovery.Phase = strings.TrimSpace(recovery.Phase) + recovery.CommandPrefix = strings.TrimSpace(recovery.CommandPrefix) + recovery.TargetURL = strings.TrimSpace(recovery.TargetURL) + recovery.HostPort = strings.TrimSpace(recovery.HostPort) + recovery.AddressPool = strings.TrimSpace(recovery.AddressPool) + if normalized, err := normalizeSwarmAddressPool(recovery.AddressPool); err == nil { + recovery.AddressPool = normalized + } + recovery.DokployVersion = strings.TrimSpace(recovery.DokployVersion) + recovery.EndpointMode = strings.TrimSpace(recovery.EndpointMode) + recovery.ACMEEmail = strings.TrimSpace(recovery.ACMEEmail) + recovery.AuthSecretBackup = strings.TrimSpace(recovery.AuthSecretBackup) + recovery.APIKeyName = strings.TrimSpace(recovery.APIKeyName) + recovery.APIKeyCreatedID = strings.TrimSpace(recovery.APIKeyCreatedID) + recovery.APIKeyBaselineIDs = append([]string(nil), recovery.APIKeyBaselineIDs...) + for index := range recovery.APIKeyBaselineIDs { + recovery.APIKeyBaselineIDs[index] = strings.TrimSpace(recovery.APIKeyBaselineIDs[index]) + } + slices.Sort(recovery.APIKeyBaselineIDs) + return recovery +} + +func sameDokployInstallationRecovery(a, b dokployInstallationRecovery) bool { + return a.APIVersion == b.APIVersion && + a.Identity == b.Identity && + a.Phase == b.Phase && + a.CommandPrefix == b.CommandPrefix && + a.TargetURL == b.TargetURL && + a.HostPort == b.HostPort && + a.AddressPool == b.AddressPool && + a.DokployVersion == b.DokployVersion && + a.EndpointMode == b.EndpointMode && + a.ACMEEmail == b.ACMEEmail && + a.AuthSecretBackup == b.AuthSecretBackup && + a.AdminName == b.AdminName && + a.APIKeyName == b.APIKeyName && + slices.Equal(a.APIKeyBaselineIDs, b.APIKeyBaselineIDs) && + a.APIKeyCreatedID == b.APIKeyCreatedID && + a.Command == b.Command +} + +func dokployInstallRecoveryCommand(recovery dokployInstallationRecovery) string { + return strings.Join([]string{ + recovery.CommandPrefix + " init-target --install", + "--dokploy-url " + shellQuote(recovery.TargetURL), + "--install-port " + shellQuote(recovery.HostPort), + "--swarm-addr-pool " + shellQuote(recovery.AddressPool), + "--dokploy-version " + shellQuote(recovery.DokployVersion), + "--endpoint-mode " + shellQuote(recovery.EndpointMode), + "--coolify-email " + shellQuote(recovery.ACMEEmail), + "--auth-secret-backup " + shellQuote(recovery.AuthSecretBackup), + "--name " + shellQuote(recovery.AdminName), + "--api-key-name " + shellQuote(recovery.APIKeyName), + }, " ") +} + +func dokployInstallRecoveryIdentity(recovery dokployInstallationRecovery) string { + canonical := strings.Join([]string{ + recovery.CommandPrefix, + recovery.TargetURL, + recovery.HostPort, + recovery.AddressPool, + recovery.DokployVersion, + recovery.EndpointMode, + recovery.ACMEEmail, + recovery.AuthSecretBackup, + recovery.AdminName, + recovery.APIKeyName, + }, "\x00") + digest := sha256.Sum256([]byte(canonical)) + return hex.EncodeToString(digest[:]) } type dokployInstaller interface { InstallDokploy(ctx context.Context, opts dokployInstallOptions, stdout, stderr io.Writer) error } +// the install script exits with this status only from its read-only +// foreign-service guard, before any host mutation, so the recovery +// marker can be cleared without inspecting the host. +const dokployInstallForeignServiceExitStatus = 75 + +func installerRefusedForeignService(err error) bool { + var exit interface{ ExitCode() int } + return errors.As(err, &exit) && exit.ExitCode() == dokployInstallForeignServiceExitStatus +} + type shellDokployInstaller struct{} func runInitTarget(ctx context.Context, args []string, stdin io.Reader, stdout, stderr io.Writer) error { - return runInitTargetWith(ctx, args, stdin, stdout, stderr, initTargetDeps{ + return runInitTargetWith(ctx, args, stdin, stdout, stderr, defaultInitTargetDeps(stderr)) +} + +func defaultInitTargetDeps(stderr io.Writer) initTargetDeps { + return initTargetDeps{ lister: &coolifyDBLister{envPath: defaultCoolifyEnvPath, container: defaultCoolifyDBContainer, stderr: stderr}, installer: shellDokployInstaller{}, newClient: defaultDokployClient, + verifyLocalService: func(ctx context.Context, client *dokploy.Client) error { + return client.VerifySameDockerHost(ctx) + }, statePath: defaultStatePath(), - }) + } } func defaultDokployClient(baseURL string) *dokploy.Client { @@ -104,7 +258,7 @@ func normalizedURLHost(u *url.URL) string { return net.JoinHostPort(host, port) } -func runInitTargetWith(ctx context.Context, args []string, stdin io.Reader, stdout, stderr io.Writer, deps initTargetDeps) error { +func runInitTargetWith(ctx context.Context, args []string, stdin io.Reader, stdout, stderr io.Writer, deps initTargetDeps) (err error) { fs := flag.NewFlagSet("init-target", flag.ContinueOnError) fs.SetOutput(stderr) @@ -118,6 +272,8 @@ func runInitTargetWith(ctx context.Context, args []string, stdin io.Reader, stdo installPort string addrPool string version string + endpointMode string + authBackup string ) fs.StringVar(&target, "target", "dokploy", "target platform to bootstrap (only dokploy is supported)") fs.StringVar(&email, "coolify-email", "", "coolify admin email to reuse (prompted if absent and multiple admins exist)") @@ -127,7 +283,13 @@ func runInitTargetWith(ctx context.Context, args []string, stdin io.Reader, stdo fs.BoolVar(&install, "install", false, "install dokploy on this VPS before bootstrapping credentials") fs.StringVar(&installPort, "install-port", "3030", "host port for dokploy UI/API when --install is used") fs.StringVar(&addrPool, "swarm-addr-pool", "auto", "docker swarm default address pool for --install (auto selects an unused private /16)") - fs.StringVar(&version, "dokploy-version", "latest", "dokploy image tag for --install") + fs.StringVar(&version, "dokploy-version", defaultDokployVersion, "dokploy release tag for --install; nondefault versions require @sha256:") + endpointMode = strings.TrimSpace(os.Getenv(envDokployEndpointMode)) + if endpointMode == "" { + endpointMode = "vip" + } + fs.StringVar(&endpointMode, "endpoint-mode", endpointMode, "Docker Swarm endpoint mode for Dokploy control-plane services: vip or dnsrr") + fs.StringVar(&authBackup, "auth-secret-backup", strings.TrimSpace(os.Getenv(envDokployAuthBackup)), "absolute path on encrypted or off-host storage for the Dokploy authentication-secret escrow") if len(args) > 0 && !strings.HasPrefix(args[0], "-") { target = args[0] @@ -139,11 +301,29 @@ func runInitTargetWith(ctx context.Context, args []string, stdin io.Reader, stdo if target != "dokploy" { return fmt.Errorf("init-target only supports --target=dokploy, got %q", target) } + apiKeyName = strings.TrimSpace(apiKeyName) + if err := validateDokployAPIKeyName(apiKeyName); err != nil { + return err + } if install { if err := validateInstallPort(installPort); err != nil { return err } - if err := validateDokployVersionTag(version); err != nil { + normalizedAddrPool, err := normalizeSwarmAddressPool(addrPool) + if err != nil { + return err + } + addrPool = normalizedAddrPool + normalizedVersion, err := normalizeDokployVersion(version) + if err != nil { + return err + } + version = normalizedVersion + endpointMode = strings.ToLower(strings.TrimSpace(endpointMode)) + if endpointMode != "vip" && endpointMode != "dnsrr" { + return fmt.Errorf("invalid --endpoint-mode %q: must be vip or dnsrr", endpointMode) + } + if err := validateAuthSecretBackupPath(authBackup); err != nil { return err } } @@ -159,6 +339,11 @@ func runInitTargetWith(ctx context.Context, args []string, stdin io.Reader, stdo if err := validateDokployBootstrapURL(dokployURL); err != nil { return err } + if install { + if err := validateInstallURLPort(dokployURL, installPort); err != nil { + return err + } + } admins, err := deps.lister.listAdmins(ctx) if err != nil { @@ -200,23 +385,124 @@ func runInitTargetWith(ctx context.Context, args []string, stdin io.Reader, stdo if displayName == "" { displayName = admin.Email } + acquiredTargetLock := false + if deps.liveOperationLock == nil { + var targetLock *applyLock + if install { + targetLock, err = acquireDokployInstallRecoveryLock() + } else { + targetLock, err = acquireDokployLiveOperationLock() + } + if err != nil { + return fmt.Errorf("lock Dokploy live operations: %w", err) + } + deps.liveOperationLock = targetLock + acquiredTargetLock = true + } + revalidate := deps.revalidateAfterTargetLock + if revalidate == nil { + revalidate = ensureStandaloneDokployInitAvailable + } + if err := revalidate(); err != nil { + if acquiredTargetLock { + deps.liveOperationLock.Release() + } + return err + } + if acquiredTargetLock { + if deps.retainLiveOperationLock != nil { + *deps.retainLiveOperationLock = deps.liveOperationLock + } else { + defer deps.liveOperationLock.Release() + } + } + installationRecoveryPending := false + var installationRecovery dokployInstallationRecovery + defer func() { + if err == nil || !installationRecoveryPending { + return + } + blocked, stateErr := dokployInstallationRecoveryRequired() + switch { + case stateErr != nil: + err = fmt.Errorf("%w; Dokploy installation recovery state is unreadable, so other host mutations may stay blocked: %v", err, stateErr) + case blocked: + err = fmt.Errorf("%w; the interrupted installation now blocks other host mutations: run `%s` to reconcile it", err, dokployInstallRecoveryCommand(installationRecovery)) + } + }() if install { - installer := deps.installer - if installer == nil { - installer = shellDokployInstaller{} + installOpts := dokployInstallOptions{ + HostPort: installPort, + AddrPool: addrPool, + Version: version, + EndpointMode: endpointMode, + ACMEEmail: admin.Email, + AuthSecretBackup: authBackup, + AdminName: displayName, + APIKeyName: apiKeyName, } - fmt.Fprintf(stdout, "Installing Dokploy in same-VPS shadow mode at %s\n", dokployURL) - if err := installer.InstallDokploy(ctx, dokployInstallOptions{HostPort: installPort, AddrPool: addrPool, Version: version, ACMEEmail: admin.Email}, stdout, stderr); err != nil { - return fmt.Errorf("install dokploy: %w", err) + installOpts.operationLock = deps.liveOperationLock.file + installationRecovery = newDokployInstallationRecovery(dokployURL, installOpts) + if err := markDokployInstallationRecoveryRequired(installationRecovery); err != nil { + return fmt.Errorf("record Dokploy installation recovery state: %w", err) } - fmt.Fprintf(stdout, "Waiting for Dokploy to answer at %s\n", dokployURL) - if err := waitForDokployHTTP(ctx, dokployURL, 2*time.Minute); err != nil { - return fmt.Errorf("wait for dokploy at %s: %w", dokployURL, err) + installationRecoveryPending = true + persistedRecovery, found, err := readDokployInstallationRecovery() + if err != nil { + return fmt.Errorf("read Dokploy installation recovery state: %w", err) + } + if !found { + return errors.New("Dokploy installation recovery state disappeared before installation") + } + installationRecovery = persistedRecovery + if installationRecovery.Phase != dokployInstallAPIKey { + installer := deps.installer + if installer == nil { + installer = shellDokployInstaller{} + } + installTimeout := deps.installTimeout + if installTimeout <= 0 { + installTimeout = dokployInstallTimeout + } + installCtx, cancelInstall := context.WithTimeout(ctx, installTimeout) + fmt.Fprintf(stdout, "Installing Dokploy in same-VPS shadow mode at %s\n", dokployURL) + installErr := installer.InstallDokploy(installCtx, installOpts, stdout, stderr) + installContextErr := installCtx.Err() + cancelInstall() + if installErr != nil { + if errors.Is(installContextErr, context.DeadlineExceeded) { + return fmt.Errorf("install dokploy timed out after %s: %w", installTimeout, installContextErr) + } + if installerRefusedForeignService(installErr) { + if err := clearDokployInstallationRecoveryRequired(); err != nil { + return fmt.Errorf("clear Dokploy installation recovery state after refusing a foreign service: %w", err) + } + return fmt.Errorf("install dokploy: the existing Dokploy service was not installed by Bort, so --install cannot adopt or repair it; bootstrap it without --install using the URL reported above: %s init-target --dokploy-url ", installationRecovery.CommandPrefix) + } + return fmt.Errorf("install dokploy: %w", installErr) + } + fmt.Fprintf(stdout, "Waiting for Dokploy to answer at %s\n", dokployURL) + if err := waitForDokployHTTP(ctx, dokployURL, 2*time.Minute); err != nil { + return fmt.Errorf("wait for dokploy at %s: %w", dokployURL, err) + } + persistedRecovery, found, err = readDokployInstallationRecovery() + if err != nil { + return fmt.Errorf("read Dokploy installation recovery state: %w", err) + } + if !found { + return errors.New("Dokploy installation recovery state disappeared before credential bootstrap") + } + installationRecovery = persistedRecovery } } client := deps.newClient(dokployURL) + if deps.verifyLocalService != nil { + if err := deps.verifyLocalService(ctx, client); err != nil { + return fmt.Errorf("verify local Dokploy service before bootstrap: %w", err) + } + } if err := client.SignUpAdmin(ctx, displayName, admin.Email, password); err != nil { return fmt.Errorf("dokploy sign-up: %w", err) } @@ -224,14 +510,39 @@ func runInitTargetWith(ctx context.Context, args []string, stdin io.Reader, stdo if err != nil { return fmt.Errorf("dokploy sign-in: %w", err) } - orgID, err := client.GetCurrentUserOrg(ctx, cookie) + var ( + orgID string + apiKeys []dokploy.APIKey + ) + if installationRecoveryPending { + orgID, apiKeys, err = client.GetCurrentUserOrgAndAPIKeys(ctx, cookie) + } else { + orgID, err = client.GetCurrentUserOrg(ctx, cookie) + } if err != nil { return fmt.Errorf("dokploy user.get: %w", err) } - apiKey, err := client.CreateAPIKey(ctx, cookie, apiKeyName, orgID) + if installationRecoveryPending { + installationRecovery, err = prepareDokployAPIKeyRecovery(ctx, client, cookie, installationRecovery, apiKeys) + if err != nil { + return err + } + } + apiKeyPrefix := "" + if installationRecoveryPending { + apiKeyPrefix = dokployInstallAPIKeyPrefix(installationRecovery) + } + apiKeyID, apiKey, err := client.CreateAPIKey(ctx, cookie, apiKeyName, apiKeyPrefix, orgID) if err != nil { return fmt.Errorf("dokploy createApiKey: %w", err) } + if installationRecoveryPending { + installationRecovery.APIKeyCreatedID = apiKeyID + installationRecovery = canonicalDokployInstallationRecovery(installationRecovery) + if err := markDokployInstallationRecoveryRequired(installationRecovery); err != nil { + return fmt.Errorf("record created Dokploy API-key identity: %w", err) + } + } creds := targetCredentials{ URL: client.BaseURL, @@ -246,37 +557,111 @@ func runInitTargetWith(ctx context.Context, args []string, stdin io.Reader, stdo }); err != nil { return err } + if installationRecoveryPending { + if err := clearDokployInstallationRecoveryRequired(); err != nil { + return fmt.Errorf("clear reconciled Dokploy installation recovery state: %w", err) + } + } fmt.Fprintf(stdout, "Dokploy setup complete for %s at %s\n", admin.Email, client.BaseURL) fmt.Fprintln(stdout, "Bort can now continue with this migration.") return nil } +func prepareDokployAPIKeyRecovery(ctx context.Context, client *dokploy.Client, cookie string, recovery dokployInstallationRecovery, keys []dokploy.APIKey) (dokployInstallationRecovery, error) { + if recovery.Phase == dokployInstallAPIKey { + baseline := make(map[string]struct{}, len(recovery.APIKeyBaselineIDs)) + for _, id := range recovery.APIKeyBaselineIDs { + baseline[id] = struct{}{} + } + matching := map[string]dokploy.APIKey{} + for _, key := range keys { + if key.Name != recovery.APIKeyName || key.Prefix != dokployInstallAPIKeyPrefix(recovery) { + continue + } + if _, existed := baseline[key.ID]; !existed { + matching[key.ID] = key + } + } + if recovery.APIKeyCreatedID == "" { + if len(matching) > 0 { + return recovery, fmt.Errorf("refusing to delete %d post-baseline Dokploy API key(s) named %q with recovery prefix %q because no created key ID was durably recorded; inspect them manually before retrying", len(matching), recovery.APIKeyName, dokployInstallAPIKeyPrefix(recovery)) + } + return recovery, nil + } + if key, found := matching[recovery.APIKeyCreatedID]; found { + if err := client.DeleteAPIKey(ctx, cookie, key.ID); err != nil { + return recovery, fmt.Errorf("delete orphaned Dokploy API key %s: %w", key.ID, err) + } + delete(matching, key.ID) + } else { + for _, key := range keys { + if key.ID == recovery.APIKeyCreatedID { + return recovery, fmt.Errorf("refusing to delete Dokploy API key %s because its name or prefix no longer matches the durable creation intent", key.ID) + } + } + } + recovery.APIKeyCreatedID = "" + recovery = canonicalDokployInstallationRecovery(recovery) + if err := markDokployInstallationRecoveryRequired(recovery); err != nil { + return recovery, fmt.Errorf("clear replaced Dokploy API-key identity: %w", err) + } + if len(matching) > 0 { + return recovery, fmt.Errorf("refusing to create another Dokploy API key while %d unowned post-baseline key(s) named %q with recovery prefix %q remain; inspect them manually before retrying", len(matching), recovery.APIKeyName, dokployInstallAPIKeyPrefix(recovery)) + } + return recovery, nil + } + recovery.Phase = dokployInstallAPIKey + recovery.APIKeyBaselineIDs = make([]string, 0, len(keys)) + for _, key := range keys { + recovery.APIKeyBaselineIDs = append(recovery.APIKeyBaselineIDs, key.ID) + } + recovery = canonicalDokployInstallationRecovery(recovery) + if err := markDokployInstallationRecoveryRequired(recovery); err != nil { + return recovery, fmt.Errorf("record Dokploy API-key recovery state: %w", err) + } + return recovery, nil +} + +func dokployInstallAPIKeyPrefix(recovery dokployInstallationRecovery) string { + return "bort_" + recovery.Identity[:12] +} + func validateDokployBootstrapURL(raw string) error { parsed, err := url.Parse(strings.TrimSpace(raw)) if err != nil || parsed.Scheme == "" || parsed.Host == "" { - return errors.New("invalid --dokploy-url: must be an absolute http or https URL with a host") + return errors.New("invalid --dokploy-url: must be an absolute URL with a host") } if parsed.User != nil { return fmt.Errorf("--dokploy-url must not embed credentials") } - if parsed.Scheme == "https" { - return nil + if parsed.Scheme != "http" && parsed.Scheme != "https" { + return fmt.Errorf("--dokploy-url must use direct HTTP on loopback for same-VPS bootstrap") + } + host := parsed.Hostname() + loopback := strings.EqualFold(host, "localhost") + if ip := net.ParseIP(host); ip != nil && ip.IsLoopback() { + loopback = true + } + if !loopback { + return fmt.Errorf("refusing same-VPS Dokploy bootstrap through non-loopback host %q; use http://127.0.0.1:", host) } if parsed.Scheme != "http" { - return fmt.Errorf("--dokploy-url must use https, or http on loopback for same-VPS bootstrap") + return fmt.Errorf("--dokploy-url must use the direct loopback HTTP endpoint, such as http://127.0.0.1:") } - host := parsed.Hostname() - if strings.EqualFold(host, "localhost") { - return nil + if parsed.Path != "" && parsed.Path != "/" || parsed.RawPath != "" || parsed.RawQuery != "" || parsed.ForceQuery || parsed.Fragment != "" { + return fmt.Errorf("--dokploy-url must be the root of the direct loopback HTTP endpoint") } - if ip := net.ParseIP(host); ip != nil && ip.IsLoopback() { - return nil + port, err := strconv.ParseUint(parsed.Port(), 10, 16) + if err != nil || port == 0 { + return fmt.Errorf("--dokploy-url must include the direct local Dokploy service port") } - return fmt.Errorf("refusing to send Coolify admin credentials to non-loopback http Dokploy host %q; use https or http://127.0.0.1:", host) + return nil } var dokployVersionTagPattern = regexp.MustCompile(`^[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}$`) +var dokployReleaseTagPattern = regexp.MustCompile(`^v([0-9]+)\.([0-9]+)\.([0-9]+)(?:-(.+))?$`) +var dockerSHA256DigestPattern = regexp.MustCompile(`^sha256:[0-9a-f]{64}$`) // keep the acme email inside a yaml-plain-safe character class: it is // interpolated unquoted into traefik.yml by the install script. @@ -291,13 +676,87 @@ func validateInstallPort(port string) error { return nil } +func validateInstallURLPort(rawURL, installPort string) error { + parsed, err := url.Parse(strings.TrimSpace(rawURL)) + if err != nil { + return err + } + urlPort := parsed.Port() + if urlPort != strings.TrimSpace(installPort) { + return fmt.Errorf("--dokploy-url port %s does not match --install-port %s; the install publishes Dokploy on --install-port, so use http://127.0.0.1:%s or pass --install-port %s", urlPort, installPort, installPort, urlPort) + } + return nil +} + +func normalizeSwarmAddressPool(raw string) (string, error) { + raw = strings.TrimSpace(raw) + if strings.EqualFold(raw, "auto") { + return "auto", nil + } + prefix, err := netip.ParsePrefix(raw) + if err != nil || !prefix.Addr().Is4() || prefix.Bits() > 24 { + return "", fmt.Errorf("invalid --swarm-addr-pool %q: must be auto or an IPv4 CIDR with a prefix length from 0 through 24", raw) + } + return prefix.Masked().String(), nil +} + +func validateDokployAPIKeyName(name string) error { + length := utf8.RuneCountInString(name) + if !utf8.ValidString(name) || length < 1 || length > 32 { + return fmt.Errorf("invalid --api-key-name: must contain 1 to 32 characters") + } + return nil +} + +func validateAuthSecretBackupPath(path string) error { + trimmed := strings.TrimSpace(path) + if trimmed == "" { + return fmt.Errorf("--auth-secret-backup is required with --install (or set %s); use an absolute path on encrypted or off-host storage and retain it after installation", envDokployAuthBackup) + } + if !filepath.IsAbs(trimmed) { + return fmt.Errorf("invalid --auth-secret-backup %q: use an absolute path on encrypted or off-host storage", path) + } + return nil +} + func validateDokployVersionTag(tag string) error { if !dokployVersionTagPattern.MatchString(tag) { return fmt.Errorf("invalid --dokploy-version %q: must be a docker tag matching [A-Za-z0-9_][A-Za-z0-9_.-]{0,127}", tag) } + matches := dokployReleaseTagPattern.FindStringSubmatch(tag) + if matches == nil { + return fmt.Errorf("unsupported --dokploy-version %q: use a release tag at v0.29.3 or newer so Dokploy can load its authentication key from a Docker secret", tag) + } + major, majorErr := strconv.ParseUint(matches[1], 10, 64) + minor, minorErr := strconv.ParseUint(matches[2], 10, 64) + patch, patchErr := strconv.ParseUint(matches[3], 10, 64) + if majorErr != nil || minorErr != nil || patchErr != nil { + return fmt.Errorf("invalid --dokploy-version %q: numeric version component is too large", tag) + } + if major == 0 && (minor < 29 || minor == 29 && (patch < 3 || patch == 3 && matches[4] != "")) { + return fmt.Errorf("unsupported --dokploy-version %q: Bort requires v0.29.3 or newer so Dokploy can load its authentication key from a Docker secret", tag) + } return nil } +func normalizeDokployVersion(version string) (string, error) { + version = strings.TrimSpace(version) + tag, digest, qualified := strings.Cut(version, "@") + if err := validateDokployVersionTag(tag); err != nil { + return "", err + } + if !qualified { + if tag == defaultDokployVersion { + return tag + "@" + defaultDokployDigest, nil + } + return "", fmt.Errorf("unsupported --dokploy-version %q: nondefault releases require an independently reviewed digest-qualified reference such as %s@sha256:<64 lowercase hex characters>", version, tag) + } + if !dockerSHA256DigestPattern.MatchString(digest) { + return "", fmt.Errorf("invalid --dokploy-version %q: digest must be sha256 followed by 64 lowercase hexadecimal characters", version) + } + return tag + "@" + digest, nil +} + func validateACMEEmail(email string) error { parsed, err := mail.ParseAddress(email) if err != nil || parsed.Address != email { @@ -310,12 +769,23 @@ func validateACMEEmail(email string) error { } func (shellDokployInstaller) InstallDokploy(ctx context.Context, opts dokployInstallOptions, stdout, stderr io.Writer) error { - cmd := exec.CommandContext(ctx, "bash", "-s") + env, err := dockercli.LocalEnvironment(os.Environ()) + if err != nil { + return err + } + authSecret, authSecretDigest, err := prepareDokployAuthSecretEscrow(opts.AuthSecretBackup) + if err != nil { + return err + } + defer authSecret.Close() + installCtx, stopSignals := dokployInstallerSignalContext(ctx) + defer stopSignals() + cmd := exec.CommandContext(installCtx, "bash", "-s") cmd.Stdin = strings.NewReader(dokployShadowInstallScript) + authSecretFD := configureDokployInstallerCommand(cmd, opts.operationLock, authSecret) progress := newInstallProgressWriter(stdout) cmd.Stdout = progress cmd.Stderr = stderr - env := os.Environ() if strings.TrimSpace(opts.HostPort) != "" { env = append(env, "HOST_PORT="+strings.TrimSpace(opts.HostPort)) } @@ -325,14 +795,27 @@ func (shellDokployInstaller) InstallDokploy(ctx context.Context, opts dokployIns if strings.TrimSpace(opts.Version) != "" { env = append(env, "DOKPLOY_VERSION="+strings.TrimSpace(opts.Version)) } + if strings.TrimSpace(opts.EndpointMode) != "" { + env = append(env, envDokployEndpointMode+"="+strings.TrimSpace(opts.EndpointMode)) + } if strings.TrimSpace(opts.ACMEEmail) != "" { env = append(env, "ACME_EMAIL="+strings.TrimSpace(opts.ACMEEmail)) } + if strings.TrimSpace(opts.AuthSecretBackup) != "" { + env = append(env, "DOKPLOY_AUTH_SECRET_BACKUP="+strings.TrimSpace(opts.AuthSecretBackup)) + } + env = append(env, + "DOKPLOY_AUTH_SECRET_FD="+strconv.Itoa(authSecretFD), + "DOKPLOY_AUTH_SECRET_DIGEST="+authSecretDigest, + ) cmd.Env = env - err := cmd.Run() + err = cmd.Run() if flushErr := progress.Flush(); flushErr != nil && err == nil { err = flushErr } + if err == nil { + err = verifyDokployAuthSecretEscrow(opts.AuthSecretBackup, authSecretDigest) + } return err } @@ -609,7 +1092,12 @@ func readCoolifyDBCredentials(env map[string]string, envPath string) (coolifyDBC } func runDockerCommand(ctx context.Context, name string, args ...string) ([]byte, error) { + env, err := dockercli.LocalEnvironment(os.Environ()) + if err != nil { + return nil, err + } cmd := exec.CommandContext(ctx, name, args...) + cmd.Env = env var stdout, stderr bytes.Buffer cmd.Stdout = &stdout cmd.Stderr = &stderr @@ -682,10 +1170,20 @@ set -euo pipefail HOST_PORT="${HOST_PORT:-3030}" ADDR_POOL="${ADDR_POOL:-auto}" -VERSION_TAG="${DOKPLOY_VERSION:-latest}" +VERSION_REF="${DOKPLOY_VERSION:?DOKPLOY_VERSION must be a digest-qualified release reference}" +VERSION_TAG="${VERSION_REF%@*}" +REQUESTED_DOKPLOY_IMAGE="dokploy/dokploy:${VERSION_REF}" ACME_EMAIL="${ACME_EMAIL:-admin@dokploy.local}" TRAEFIK_IMAGE="${DOKPLOY_TRAEFIK_IMAGE:-traefik:v3.6.7}" ENDPOINT_MODE="${ENDPOINT_MODE:-vip}" +AUTH_SECRET_BACKUP="${DOKPLOY_AUTH_SECRET_BACKUP:-}" +AUTH_SECRET_FD="${DOKPLOY_AUTH_SECRET_FD:?DOKPLOY_AUTH_SECRET_FD is required}" +AUTH_SECRET_DIGEST="${DOKPLOY_AUTH_SECRET_DIGEST:?DOKPLOY_AUTH_SECRET_DIGEST is required}" +AUTH_SECRET_FD_PATH="/dev/fd/$AUTH_SECRET_FD" +AUTH_SECRET_PROVENANCE=/var/lib/bort/dokploy-auth-secret.id +AUTH_SECRET_INTENT_LABEL=io.bort.auth-secret-intent +INSTALL_AUTH_SECRET_LABEL=io.bort.install-auth-secret-id +FOREIGN_SERVICE_EXIT_STATUS=75 validate_endpoint_mode() { case "$1" in @@ -731,10 +1229,33 @@ if ! command -v python3 >/dev/null 2>&1; then echo "python3 is required before installing Dokploy" >&2 exit 1 fi -if ! command -v python3 >/dev/null 2>&1; then - echo "python3 is required before installing Dokploy" >&2 +if [ -z "$AUTH_SECRET_BACKUP" ]; then + echo "DOKPLOY_AUTH_SECRET_BACKUP is required; set it to an absolute path on encrypted or off-host storage and retain it after installation" >&2 + exit 1 +fi +case "$AUTH_SECRET_BACKUP" in + /*) ;; + *) echo "DOKPLOY_AUTH_SECRET_BACKUP must be an absolute path" >&2; exit 1 ;; +esac +AUTH_SECRET_BACKUP_DIR="$(dirname -- "$AUTH_SECRET_BACKUP")" +if [ ! -d "$AUTH_SECRET_BACKUP_DIR" ]; then + echo "DOKPLOY_AUTH_SECRET_BACKUP parent directory does not exist: $AUTH_SECRET_BACKUP_DIR" >&2 + exit 1 +fi +if [ -L "$AUTH_SECRET_BACKUP" ]; then + echo "DOKPLOY_AUTH_SECRET_BACKUP must not be a symbolic link" >&2 + exit 1 +fi + +if ! SWARM_STATUS="$(docker info --format '{{.Swarm.LocalNodeState}} {{.Swarm.ControlAvailable}}' 2>/dev/null)"; then + echo "could not inspect Docker swarm state; refusing to continue" >&2 exit 1 fi +case "$SWARM_STATUS" in + "inactive false"|"active true") ;; + "active false") echo "this host is an active Docker swarm worker, not a manager; run Dokploy installation on a manager" >&2; exit 1 ;; + *) echo "unsupported Docker swarm state \"$SWARM_STATUS\"; refusing to continue" >&2; exit 1 ;; +esac for service in dokploy-postgres dokploy-redis dokploy; do if mode="$(docker service inspect "$service" --format '{{.Spec.EndpointSpec.Mode}}' 2>/dev/null)"; then @@ -745,6 +1266,430 @@ for service in dokploy-postgres dokploy-redis dokploy; do fi done +dokploy_image_supports_auth_secret_file() { + local image="${1%@*}" + local repository="${image%:*}" + local tag="${image##*:}" + case "$repository" in + dokploy/dokploy|docker.io/dokploy/dokploy|index.docker.io/dokploy/dokploy) ;; + *) return 1 ;; + esac + if ! [[ "$tag" =~ ^v([0-9]{1,9})\.([0-9]{1,9})\.([0-9]{1,9})(-(.+))?$ ]]; then + return 1 + fi + local major=$((10#${BASH_REMATCH[1]})) + local minor=$((10#${BASH_REMATCH[2]})) + local patch=$((10#${BASH_REMATCH[3]})) + if (( major > 0 || minor > 29 || minor == 29 && patch > 3 )); then + return 0 + fi + (( major == 0 && minor == 29 && patch == 3 )) && [ -z "${BASH_REMATCH[4]:-}" ] +} + +docker_object_exists() { + local object_kind="$1" + local object_name="$2" + local absent_pattern="$3" + local output + if output="$(docker "$object_kind" inspect "$object_name" 2>&1)"; then + return 0 + fi + if grep -Eqi "$absent_pattern" <<<"$output"; then + return 1 + fi + echo "could not determine whether Docker $object_kind $object_name exists; refusing to continue: $output" >&2 + return 2 +} + +validate_auth_secret_backup() { + if ! [[ "$AUTH_SECRET_FD" =~ ^[0-9]+$ ]]; then + echo "authentication-secret escrow descriptor is unavailable" >&2 + return 1 + fi + if ! : < "$AUTH_SECRET_FD_PATH" 2>/dev/null; then + echo "authentication-secret escrow descriptor is unavailable" >&2 + return 1 + fi + if ! grep -Eq '^[0-9a-f]{64}$' <<<"$AUTH_SECRET_DIGEST"; then + echo "authentication-secret escrow digest validated by Bort is malformed" >&2 + return 1 + fi +} + +prepare_auth_secret_backup() { + validate_auth_secret_backup +} + +current_auth_secret_id() { + local secret_id + if ! secret_id="$(docker secret inspect dokploy_auth_secret --format '{{.ID}}' 2>/dev/null)"; then + echo "could not inspect Docker secret dokploy_auth_secret identity; refusing to continue" >&2 + return 1 + fi + if ! grep -Eq '^[a-z0-9]{20,64}$' <<<"$secret_id"; then + echo "Docker returned an invalid identity for secret dokploy_auth_secret; refusing to continue" >&2 + return 1 + fi + printf '%s\n' "$secret_id" +} + +validate_auth_secret_provenance() { + local expected_secret_id="${1:-}" secret_id secret_intent backup_digest recorded_id recorded_digest recorded_intent extra + validate_auth_secret_backup + secret_id="$(current_auth_secret_id)" + if [ -n "$expected_secret_id" ] && [ "$secret_id" != "$expected_secret_id" ]; then + echo "Docker secret dokploy_auth_secret changed while Bort was creating it; refusing to continue" >&2 + return 1 + fi + backup_digest="$AUTH_SECRET_DIGEST" + if [ ! -f "$AUTH_SECRET_PROVENANCE" ] || [ -L "$AUTH_SECRET_PROVENANCE" ]; then + echo "orphaned Docker secret dokploy_auth_secret has no trusted Bort provenance marker at $AUTH_SECRET_PROVENANCE; refusing to reuse or replace it" >&2 + return 1 + fi + if [ "$(stat -c '%a' "$AUTH_SECRET_PROVENANCE")" != 600 ] || [ "$(stat -c '%u' "$AUTH_SECRET_PROVENANCE")" != "$(id -u)" ]; then + echo "Dokploy authentication-secret provenance marker must be mode 0600 and owned by root: $AUTH_SECRET_PROVENANCE" >&2 + return 1 + fi + IFS=' ' read -r recorded_id recorded_digest extra < "$AUTH_SECRET_PROVENANCE" + if [ "$(wc -l < "$AUTH_SECRET_PROVENANCE")" -ne 1 ] || [ -n "${extra:-}" ] || ! grep -Eq '^[0-9a-f]{64}$' <<<"$recorded_digest"; then + echo "Dokploy authentication-secret provenance marker is malformed: $AUTH_SECRET_PROVENANCE" >&2 + return 1 + fi + if [[ "$recorded_id" == pending:* ]]; then + recorded_intent="${recorded_id#pending:}" + if ! grep -Eq '^[0-9a-f]{32}$' <<<"$recorded_intent"; then + echo "Dokploy authentication-secret provenance marker is malformed: $AUTH_SECRET_PROVENANCE" >&2 + return 1 + fi + if ! secret_intent="$(docker secret inspect dokploy_auth_secret --format "{{ index .Spec.Annotations.Labels \"$AUTH_SECRET_INTENT_LABEL\" }}" 2>/dev/null)"; then + echo "could not inspect Docker secret dokploy_auth_secret creation intent; refusing to continue" >&2 + return 1 + fi + if [ "$recorded_digest" != "$backup_digest" ] || [ "$recorded_intent" != "$secret_intent" ]; then + echo "orphaned Docker secret dokploy_auth_secret or its retained escrow does not match the trusted Bort creation intent; refusing to continue" >&2 + return 1 + fi + write_auth_secret_provenance "$secret_id" + DOKPLOY_AUTH_SECRET_ID="$secret_id" + return + fi + if ! grep -Eq '^[a-z0-9]{20,64}$' <<<"$recorded_id"; then + echo "Dokploy authentication-secret provenance marker is malformed: $AUTH_SECRET_PROVENANCE" >&2 + return 1 + fi + if [ "$recorded_id" != "$secret_id" ] || [ "$recorded_digest" != "$backup_digest" ]; then + echo "orphaned Docker secret dokploy_auth_secret or its retained escrow does not match the trusted Bort provenance marker; refusing to continue" >&2 + return 1 + fi + DOKPLOY_AUTH_SECRET_ID="$secret_id" +} + +write_auth_secret_marker() { + local marker_id="$1" backup_digest="$2" provenance_dir provenance_tmp + provenance_dir="$(dirname -- "$AUTH_SECRET_PROVENANCE")" + if [ -L "$provenance_dir" ] || { [ -e "$provenance_dir" ] && [ ! -d "$provenance_dir" ]; }; then + echo "Dokploy provenance directory must be a real directory: $provenance_dir" >&2 + return 1 + fi + install -d -m 700 -o "$(id -u)" -g "$(id -g)" "$provenance_dir" + if [ "$(stat -c '%a' "$provenance_dir")" != 700 ] || [ "$(stat -c '%u' "$provenance_dir")" != "$(id -u)" ]; then + echo "Dokploy provenance directory must be mode 0700 and owned by root: $provenance_dir" >&2 + return 1 + fi + provenance_tmp="$(mktemp "$provenance_dir/.dokploy-auth-secret.id.XXXXXX")" + printf '%s %s\n' "$marker_id" "$backup_digest" > "$provenance_tmp" + chmod 600 "$provenance_tmp" + sync -f "$provenance_tmp" + mv -f -- "$provenance_tmp" "$AUTH_SECRET_PROVENANCE" + sync -f "$provenance_dir" +} + +write_auth_secret_intent() { + local creation_intent backup_digest + validate_auth_secret_backup + creation_intent="$(openssl rand -hex 16)" + if ! grep -Eq '^[0-9a-f]{32}$' <<<"$creation_intent"; then + echo "openssl did not produce the required authentication-secret creation intent" >&2 + return 1 + fi + backup_digest="$AUTH_SECRET_DIGEST" + write_auth_secret_marker "pending:$creation_intent" "$backup_digest" + printf '%s\n' "$creation_intent" +} + +write_auth_secret_provenance() { + local secret_id="$1" backup_digest + validate_auth_secret_backup + backup_digest="$AUTH_SECRET_DIGEST" + write_auth_secret_marker "$secret_id" "$backup_digest" +} + +validate_bort_install_service() { + local service_state="$1" service_image install_secret_id bound_secret_ids auth_file_count + if ! install_secret_id="$(docker service inspect dokploy --format "{{ index .Spec.Annotations.Labels \"$INSTALL_AUTH_SECRET_LABEL\" }}" 2>/dev/null)" || ! grep -Eq '^[a-z0-9]{20,64}$' <<<"$install_secret_id"; then + echo "the $service_state Dokploy service is not labeled with a valid Bort authentication-secret identity; refusing installation recovery" >&2 + return 1 + fi + validate_auth_secret_provenance + if [ "$install_secret_id" != "$DOKPLOY_AUTH_SECRET_ID" ]; then + echo "the $service_state Dokploy service is not bound to the reviewed Bort authentication-secret identity; refusing installation recovery" >&2 + return 1 + fi + if ! service_image="$(docker service inspect dokploy --format '{{.Spec.TaskTemplate.ContainerSpec.Image}}' 2>/dev/null)"; then + echo "could not inspect the $service_state Dokploy service image; refusing installation recovery" >&2 + return 1 + fi + case "$service_image" in + "$REQUESTED_DOKPLOY_IMAGE"|"docker.io/$REQUESTED_DOKPLOY_IMAGE"|"index.docker.io/$REQUESTED_DOKPLOY_IMAGE") ;; + *) + echo "the $service_state Dokploy service does not use the requested immutable image; refusing installation recovery" >&2 + return 1 + ;; + esac + auth_file_count="$(grep -Ec '^BETTER_AUTH_SECRET_FILE=' <<<"$DOKPLOY_AUTH_ENV" || true)" + if [ "$auth_file_count" -ne 1 ] || ! grep -Fqx 'BETTER_AUTH_SECRET_FILE=/run/secrets/dokploy_auth_secret' <<<"$DOKPLOY_AUTH_ENV"; then + echo "the $service_state Dokploy service does not use exactly Bort's authentication-secret mount path; refusing installation recovery" >&2 + return 1 + fi + if ! bound_secret_ids="$(docker service inspect dokploy --format '{{range .Spec.TaskTemplate.ContainerSpec.Secrets}}{{if eq .SecretName "dokploy_auth_secret"}}{{println .SecretID}}{{end}}{{end}}' 2>/dev/null)"; then + echo "could not verify the authentication secret bound to the $service_state Dokploy service; refusing installation recovery" >&2 + return 1 + fi + set -- $bound_secret_ids + if [ "$#" -ne 1 ] || [ "$1" != "$DOKPLOY_AUTH_SECRET_ID" ]; then + echo "the $service_state Dokploy service does not bind exactly the reviewed authentication-secret identity; refusing installation recovery" >&2 + return 1 + fi +} + +DOKPLOY_AUTH_SECRET_ID= +DOKPLOY_AUTH_SECRET_EXISTS=false +if docker_object_exists secret dokploy_auth_secret 'no such (secret|object)|not a swarm manager'; then + DOKPLOY_AUTH_SECRET_EXISTS=true +else + inspect_status=$? + if [ "$inspect_status" -ne 1 ]; then + exit 1 + fi +fi + +DOKPLOY_SERVICE_EXISTS=false +if docker_object_exists service dokploy 'no such (service|object)|not a swarm manager'; then + DOKPLOY_SERVICE_EXISTS=true + if ! DOKPLOY_AUTH_ENV="$(docker service inspect dokploy --format '{{range .Spec.TaskTemplate.ContainerSpec.Env}}{{println .}}{{end}}' 2>/dev/null)"; then + echo "could not inspect the existing Dokploy service authentication configuration; refusing to continue" >&2 + exit 1 + fi + DOKPLOY_AUTH_SECRET_REFUSAL= + DOKPLOY_INLINE_AUTH_COUNT="$(grep -Ec '^BETTER_AUTH_SECRET=' <<<"$DOKPLOY_AUTH_ENV" || true)" + if [ "$DOKPLOY_INLINE_AUTH_COUNT" -gt 0 ]; then + if [ "$DOKPLOY_INLINE_AUTH_COUNT" -ne 1 ]; then + echo "existing Dokploy service uses duplicate BETTER_AUTH_SECRET entries or a value that is shorter than Better Auth's 32-character minimum or is not a 64-lowercase-hex key; rotate it with a Dokploy-supported maintenance procedure and a cryptographically random 32-byte key before rerunning" >&2 + exit 1 + fi + if grep -Fqx 'BETTER_AUTH_SECRET=better-auth-secret-123456789' <<<"$DOKPLOY_AUTH_ENV"; then + DOKPLOY_AUTH_SECRET_REFUSAL="existing Dokploy service explicitly uses the published legacy authentication secret; refusing bootstrap until it is migrated because that value cannot securely sign authentication artifacts." + elif ! LC_ALL=C grep -Eq '^BETTER_AUTH_SECRET=[0-9a-f]{64}$' <<<"$DOKPLOY_AUTH_ENV"; then + echo "existing Dokploy service uses duplicate BETTER_AUTH_SECRET entries or a value that is shorter than Better Auth's 32-character minimum or is not a 64-lowercase-hex key; rotate it with a Dokploy-supported maintenance procedure and a cryptographically random 32-byte key before rerunning" >&2 + exit 1 + fi + elif grep -Eq '^BETTER_AUTH_SECRET_FILE=.+$' <<<"$DOKPLOY_AUTH_ENV"; then + if ! DOKPLOY_IMAGE="$(docker service inspect dokploy --format '{{.Spec.TaskTemplate.ContainerSpec.Image}}' 2>/dev/null)"; then + echo "could not inspect the existing Dokploy service image; refusing to continue" >&2 + exit 1 + fi + if ! dokploy_image_supports_auth_secret_file "$DOKPLOY_IMAGE"; then + echo "existing Dokploy service uses BETTER_AUTH_SECRET_FILE with image \"$DOKPLOY_IMAGE\", but Bort cannot verify file-backed authentication-secret support; upgrade to dokploy/dokploy:v0.29.3 or newer, then rerun" >&2 + exit 1 + fi + else + DOKPLOY_AUTH_SECRET_REFUSAL="existing Dokploy service has no authentication secret explicitly configured; refusing to generate a new key because changing the legacy fallback can make encrypted settings unreadable." + fi + if [ -n "$DOKPLOY_AUTH_SECRET_REFUSAL" ]; then + printf '%s\n' "$DOKPLOY_AUTH_SECRET_REFUSAL" >&2 + cat >&2 <<'EOF' +Before rotating it: schedule maintenance; inspect the service's published ports; fence every inbound Dokploy UI/API path at the firewall or external load balancer, including each published host port targeting container port 3000 (Bort defaults to host port 3030, while --install-port may differ) and dashboard routes on 80/443; verify those paths are unreachable; and keep the fence active through the service restart. Then back up the complete Dokploy PostgreSQL database and /etc/dokploy; validate and preserve every existing 64-lowercase-hex line in /etc/dokploy/encryption.key and atomically add HMAC-SHA256(key='better-auth-secret-123456789', data='dokploy:db-encryption:v1') only if absent; upgrade Dokploy to v0.29.5 or newer; and verify the running service image. Do not use the hosted 0.29.3.sh wrapper because it treats a 120-second migration timeout as success. Run this no-timeout sequence instead: + ( + set -eu + if sudo docker secret inspect dokploy-auth-secret >/dev/null 2>&1; then + echo 'refusing to run migration while Docker secret dokploy-auth-secret already exists; after an interrupted attempt, keep the write fence active and use the migration guide two-factor compare-and-swap recovery before removing the unused secret' >&2 + exit 1 + fi + : "${DOKPLOY_AUTH_SECRET_BACKUP:?set DOKPLOY_AUTH_SECRET_BACKUP to an absolute path on separately backed-up encrypted or off-host storage}" + case "$DOKPLOY_AUTH_SECRET_BACKUP" in + /*) ;; + *) echo 'DOKPLOY_AUTH_SECRET_BACKUP must be an absolute path' >&2; exit 1 ;; + esac + auth_secret_backup_dir="$(dirname -- "$DOKPLOY_AUTH_SECRET_BACKUP")" + test -d "$auth_secret_backup_dir" + test ! -L "$DOKPLOY_AUTH_SECRET_BACKUP" + : "${DOKPLOY_TWO_FACTOR_BEFORE:?set DOKPLOY_TWO_FACTOR_BEFORE to a new absolute snapshot path on separately backed-up encrypted or off-host storage}" + case "$DOKPLOY_TWO_FACTOR_BEFORE" in + /*) ;; + *) echo 'DOKPLOY_TWO_FACTOR_BEFORE must be an absolute path' >&2; exit 1 ;; + esac + two_factor_backup_dir="$(dirname -- "$DOKPLOY_TWO_FACTOR_BEFORE")" + test -d "$two_factor_backup_dir" + test ! -L "$DOKPLOY_TWO_FACTOR_BEFORE" + command -v shred >/dev/null 2>&1 + umask 077 + auth_secret_tmp="$(mktemp "${TMPDIR:-/tmp}/dokploy-auth-secret.XXXXXX")" + two_factor_tmp= + cleanup_migration_tmp() { + [ -z "${auth_secret_tmp:-}" ] || shred -u -- "$auth_secret_tmp" + [ -z "${two_factor_tmp:-}" ] || rm -f -- "$two_factor_tmp" + } + trap cleanup_migration_tmp EXIT + trap 'exit 129' HUP + trap 'exit 130' INT + trap 'exit 143' TERM + if ! DOKPLOY_MIGRATION_ENV="$(sudo docker service inspect dokploy --format '{{range .Spec.TaskTemplate.ContainerSpec.Env}}{{println .}}{{end}}' 2>/dev/null)"; then + echo 'could not inspect the Dokploy service database configuration; refusing authentication-secret migration' >&2 + exit 1 + fi + require_default_database_selector() { + selector="$1" + expected="$2" + count="$(printf '%s\n' "$DOKPLOY_MIGRATION_ENV" | grep -Ec "^${selector}=" || true)" + if [ "$count" -gt 1 ]; then + echo "Dokploy service has duplicate ${selector} settings; refusing authentication-secret migration" >&2 + exit 1 + fi + if [ "$count" -eq 1 ]; then + actual="$(printf '%s\n' "$DOKPLOY_MIGRATION_ENV" | sed -n "s/^${selector}=//p")" + if [ "$actual" != "$expected" ]; then + echo "this automated authentication-secret migration supports only Dokploy's default internal database; ${selector}=${actual} does not match ${expected}, so use a database-operator procedure that snapshots and compare-and-swap restores the same public.two_factor columns on the configured database" >&2 + exit 1 + fi + fi + } + if printf '%s\n' "$DOKPLOY_MIGRATION_ENV" | grep -Eq '^DATABASE_URL='; then + echo 'this automated authentication-secret migration supports only Dokploy with its default internal database; DATABASE_URL is configured, so use a database-operator procedure that snapshots and compare-and-swap restores the same public.two_factor columns on that external database' >&2 + exit 1 + fi + require_default_database_selector POSTGRES_HOST dokploy-postgres + require_default_database_selector POSTGRES_PORT 5432 + require_default_database_selector POSTGRES_USER dokploy + require_default_database_selector POSTGRES_DB dokploy + if [ -e "$DOKPLOY_AUTH_SECRET_BACKUP" ]; then + test -f "$DOKPLOY_AUTH_SECRET_BACKUP" + test "$(stat -c '%a' "$DOKPLOY_AUTH_SECRET_BACKUP")" = 600 + cat -- "$DOKPLOY_AUTH_SECRET_BACKUP" > "$auth_secret_tmp" + else + openssl rand -hex 32 > "$auth_secret_tmp" + (set -C; cat -- "$auth_secret_tmp" > "$DOKPLOY_AUTH_SECRET_BACKUP") + fi + chmod 600 "$auth_secret_tmp" + test "$(wc -c < "$auth_secret_tmp")" -eq 65 + grep -Eq '^[0-9a-f]{64}$' "$auth_secret_tmp" + sync -f "$DOKPLOY_AUTH_SECRET_BACKUP" + cmp -s -- "$auth_secret_tmp" "$DOKPLOY_AUTH_SECRET_BACKUP" + sync -f "$auth_secret_backup_dir" + sudo docker service update --force --update-order stop-first --detach=false dokploy + set -- $(sudo docker ps --filter label=com.docker.swarm.service.name=dokploy --format '{{.ID}}') + test "$#" -eq 1 + set -- $(sudo docker ps --filter label=com.docker.swarm.service.name=dokploy-postgres --format '{{.ID}}') + test "$#" -eq 1 + dokploy_postgres_container="$1" + database_settled=false + settle_attempt=0 + while [ "$settle_attempt" -lt 30 ]; do + active_sessions="$(sudo docker exec "$dokploy_postgres_container" psql -U dokploy -d dokploy -X -v ON_ERROR_STOP=1 -Atc "SELECT count(*) FROM pg_stat_activity WHERE datname = current_database() AND pid <> pg_backend_pid() AND state <> 'idle'")" + if [ "$active_sessions" = 0 ]; then + database_settled=true + break + fi + settle_attempt=$((settle_attempt + 1)) + sleep 1 + done + if [ "$database_settled" != true ]; then + echo 'Dokploy database sessions did not settle after the stop-first task replacement; refusing to snapshot two-factor state' >&2 + exit 1 + fi + two_factor_tmp="$(mktemp "$two_factor_backup_dir/.dokploy-two-factor-before.XXXXXX")" + sudo docker exec "$dokploy_postgres_container" psql -U dokploy -d dokploy -X -v ON_ERROR_STOP=1 -c "COPY (SELECT id, secret, backup_codes FROM public.two_factor ORDER BY id) TO STDOUT WITH (FORMAT csv, HEADER true, NULL '\N', FORCE_QUOTE *)" > "$two_factor_tmp" + chmod 600 "$two_factor_tmp" + sync -f "$two_factor_tmp" + if [ -e "$DOKPLOY_TWO_FACTOR_BEFORE" ]; then + test -f "$DOKPLOY_TWO_FACTOR_BEFORE" + test "$(stat -c '%a' "$DOKPLOY_TWO_FACTOR_BEFORE")" = 600 + cmp -s -- "$two_factor_tmp" "$DOKPLOY_TWO_FACTOR_BEFORE" + rm -f -- "$two_factor_tmp" + else + ln -- "$two_factor_tmp" "$DOKPLOY_TWO_FACTOR_BEFORE" + sync -f "$two_factor_backup_dir" + rm -f -- "$two_factor_tmp" + fi + two_factor_tmp= + sudo docker secret create dokploy-auth-secret - < "$auth_secret_tmp" + sudo docker service update --secret-add source=dokploy-auth-secret,target=/run/secrets/dokploy-auth-secret --update-order stop-first dokploy + set -- $(sudo docker ps --filter label=com.docker.swarm.service.name=dokploy --format '{{.ID}}') + test "$#" -eq 1 + dokploy_container="$1" + sudo docker exec -e OLD_SECRET=better-auth-secret-123456789 "$dokploy_container" sh -c 'cd /app && NEW_SECRET="$(cat /run/secrets/dokploy-auth-secret)" pnpm run migrate-auth-secret' + sudo docker service update --env-add BETTER_AUTH_SECRET_FILE=/run/secrets/dokploy-auth-secret --env-rm BETTER_AUTH_SECRET --update-order stop-first dokploy + shred -u -- "$auth_secret_tmp" + auth_secret_tmp= + trap - HUP INT TERM + trap - EXIT + ) +Retain DOKPLOY_AUTH_SECRET_BACKUP and DOKPLOY_TWO_FACTOR_BEFORE after success. Keep the inbound write fence active after an interruption. If neither the post-quiescence DOKPLOY_TWO_FACTOR_BEFORE snapshot nor the Docker secret exists, rerun the exact command under the fence. If the snapshot exists but the secret does not, rerunning replaces and settles the legacy task again and requires the snapshot to match. Once the Docker secret exists, do not rerun; confirm no migrate-auth-secret process remains. If Dokploy still uses the legacy fallback, capture an immediate post-interruption snapshot of only public.two_factor id, secret, and backup_codes, then use the compare-and-swap recovery in Bort's migration guide; do not restore the full PostgreSQL database because unrelated worker writes may have continued. If the service sets BETTER_AUTH_SECRET_FILE, do not remove the secret or rewrite the database; verify login, two-factor authentication, and stored values. +Restart Dokploy and verify representative application, Compose, project, environment, database, build-argument, and build-secret values render as plaintext before any deploy. The official migration re-encrypts only Better Auth two-factor secret and backup-code fields; it does not migrate those Dokploy configuration fields. See 'Recover an existing Dokploy service using the legacy secret' in Bort's migration guide for exact commands. +EOF + exit 1 + fi + if ! DOKPLOY_INSTALL_LABEL="$(docker service inspect dokploy --format "{{ index .Spec.Annotations.Labels \"$INSTALL_AUTH_SECRET_LABEL\" }}" 2>/dev/null)"; then + echo "could not inspect the existing Dokploy service labels; refusing to continue" >&2 + exit 1 + fi + if [ -z "$DOKPLOY_INSTALL_LABEL" ]; then + DOKPLOY_EXISTING_PORT="$(docker service inspect dokploy --format '{{range .Endpoint.Ports}}{{if eq .TargetPort 3000}}{{.PublishedPort}}{{end}}{{end}}' 2>/dev/null | head -n1)" + echo "existing Dokploy service was not installed by Bort, so --install cannot adopt or repair it; it answers at http://127.0.0.1:${DOKPLOY_EXISTING_PORT:-}" >&2 + exit "$FOREIGN_SERVICE_EXIT_STATUS" + fi +else + inspect_status=$? + if [ "$inspect_status" -ne 1 ]; then + exit 1 + fi +fi +if [ "$DOKPLOY_SERVICE_EXISTS" = false ] && [ "$DOKPLOY_AUTH_SECRET_EXISTS" = false ]; then + if [ "$SWARM_STATUS" = "active true" ]; then + if ! SWARM_MANAGER_COUNT="$(docker node ls --filter role=manager --format '{{.ID}}' | awk 'NF { count++ } END { print count + 0 }')"; then + echo "could not enumerate Docker swarm managers before checking for orphaned Dokploy database state; refusing to continue" >&2 + exit 1 + fi + if [ "$SWARM_MANAGER_COUNT" -ne 1 ]; then + echo "existing multi-manager Docker swarm can hide a node-local dokploy-postgres volume on another manager; automatic fresh install is unsupported unless the original Bort-managed dokploy_auth_secret and /var/lib/bort/dokploy-auth-secret.id provenance marker are restored together from a consistent host backup" >&2 + exit 1 + fi + fi + DOKPLOY_DATABASE_STATE_EXISTS=false + if docker_object_exists service dokploy-postgres 'no such (service|object)|not a swarm manager'; then + DOKPLOY_DATABASE_STATE_EXISTS=true + else + inspect_status=$? + if [ "$inspect_status" -ne 1 ]; then + exit 1 + fi + fi + if docker_object_exists volume dokploy-postgres 'no such (volume|object)'; then + DOKPLOY_DATABASE_STATE_EXISTS=true + else + inspect_status=$? + if [ "$inspect_status" -ne 1 ]; then + exit 1 + fi + fi + if [ "$DOKPLOY_DATABASE_STATE_EXISTS" = true ]; then + echo "existing Dokploy database state has no reusable dokploy_auth_secret; refusing to generate a new key because changing it can invalidate sessions and encrypted settings; restore the prior Dokploy service, original Bort-managed Docker secret, and /var/lib/bort/dokploy-auth-secret.id provenance marker together from a consistent host backup, then rerun; Bort does not accept a manually recreated secret even when its key bytes match" >&2 + exit 1 + fi + prepare_auth_secret_backup +elif [ "$DOKPLOY_SERVICE_EXISTS" = false ]; then + validate_auth_secret_provenance +fi + private_ip() { ip -o -4 addr show scope global 2>/dev/null | awk 'first == "" { split($4, a, "/"); first = a[1] } END { print first }' } @@ -927,6 +1872,25 @@ if ! docker secret inspect dokploy_postgres_password >/dev/null 2>&1; then echo "$POSTGRES_PASSWORD" | docker secret create dokploy_postgres_password - >/dev/null fi +if [ "$DOKPLOY_SERVICE_EXISTS" = false ]; then + log "Creating Dokploy authentication secret" + if ! docker secret inspect dokploy_auth_secret >/dev/null 2>&1; then + validate_auth_secret_backup + DOKPLOY_AUTH_SECRET_INTENT="$(write_auth_secret_intent)" + if ! DOKPLOY_AUTH_SECRET_CREATED_ID="$(docker secret create --label "$AUTH_SECRET_INTENT_LABEL=$DOKPLOY_AUTH_SECRET_INTENT" dokploy_auth_secret - < "$AUTH_SECRET_FD_PATH")"; then + echo "could not create Docker secret dokploy_auth_secret after recording Bort's creation intent; refusing to adopt any object that raced its creation" >&2 + exit 1 + fi + if ! grep -Eq '^[a-z0-9]{20,64}$' <<<"$DOKPLOY_AUTH_SECRET_CREATED_ID"; then + echo "Docker returned an invalid identity after creating secret dokploy_auth_secret; refusing to continue" >&2 + exit 1 + fi + validate_auth_secret_provenance "$DOKPLOY_AUTH_SECRET_CREATED_ID" + else + validate_auth_secret_provenance + fi +fi + log "Starting Dokploy Postgres" if ! docker service inspect dokploy-postgres >/dev/null 2>&1; then docker service create \ @@ -955,24 +1919,24 @@ if ! docker service inspect dokploy-redis >/dev/null 2>&1; then redis:7 >/dev/null fi -DOCKER_IMAGE="dokploy/dokploy:${VERSION_TAG}" release_tag_args=() -if [ "$VERSION_TAG" != "latest" ]; then - release_tag_args=(-e "RELEASE_TAG=$VERSION_TAG") -fi +release_tag_args=(-e "RELEASE_TAG=$VERSION_TAG") log "Starting Dokploy UI/API on port ${HOST_PORT}" +DOKPLOY_STAGED_SERVICE=false if ! docker service inspect dokploy >/dev/null 2>&1; then docker service create \ --name dokploy \ + --label "$INSTALL_AUTH_SECRET_LABEL=$DOKPLOY_AUTH_SECRET_ID" \ --endpoint-mode "$ENDPOINT_MODE" \ --detach=true \ - --replicas 1 \ + --replicas 0 \ --network dokploy-network \ --mount type=bind,source=/var/run/docker.sock,target=/var/run/docker.sock \ --mount type=bind,source=/etc/dokploy,target=/etc/dokploy \ --mount type=volume,source=dokploy,target=/root/.docker \ --secret source=dokploy_postgres_password,target=/run/secrets/postgres_password \ + --secret source=dokploy_auth_secret,target=/run/secrets/dokploy_auth_secret \ --publish published="$HOST_PORT",target=3000,mode=host \ --update-parallelism 1 \ --update-order stop-first \ @@ -980,7 +1944,47 @@ if ! docker service inspect dokploy >/dev/null 2>&1; then "${release_tag_args[@]}" \ -e ADVERTISE_ADDR="$ADVERTISE_ADDR" \ -e POSTGRES_PASSWORD_FILE=/run/secrets/postgres_password \ - "$DOCKER_IMAGE" >/dev/null + -e BETTER_AUTH_SECRET_FILE=/run/secrets/dokploy_auth_secret \ + "$REQUESTED_DOKPLOY_IMAGE" >/dev/null + DOKPLOY_STAGED_SERVICE=true + +elif ! DOKPLOY_SERVICE_REPLICAS="$(docker service inspect dokploy --format '{{.Spec.Mode.Replicated.Replicas}}' 2>/dev/null)"; then + echo "could not inspect the existing Dokploy service replica count; refusing installation recovery" >&2 + exit 1 +elif [ "$DOKPLOY_SERVICE_REPLICAS" != 0 ] && [ "$DOKPLOY_SERVICE_REPLICAS" != 1 ]; then + echo "the existing Dokploy service has $DOKPLOY_SERVICE_REPLICAS replicas, not Bort's staged or running replica count; refusing installation recovery" >&2 + exit 1 +else + DOKPLOY_INSTALL_SERVICE_STATE=running + if [ "$DOKPLOY_SERVICE_REPLICAS" = 0 ]; then + DOKPLOY_INSTALL_SERVICE_STATE=zero-replica + fi + validate_bort_install_service "$DOKPLOY_INSTALL_SERVICE_STATE" + if [ "$DOKPLOY_SERVICE_REPLICAS" = 0 ]; then + if ! DOKPLOY_STAGED_RUNNING_TASKS="$(docker service ps --filter desired-state=running -q dokploy 2>/dev/null)"; then + echo "could not inspect tasks for the zero-replica Dokploy service; refusing installation recovery" >&2 + exit 1 + fi + if [ -n "$DOKPLOY_STAGED_RUNNING_TASKS" ]; then + echo "the zero-replica Dokploy service still has a running task; refusing installation recovery" >&2 + exit 1 + fi + DOKPLOY_STAGED_SERVICE=true + fi +fi + +if [ "$DOKPLOY_STAGED_SERVICE" = true ]; then + if ! DOKPLOY_BOUND_AUTH_SECRET_IDS="$(docker service inspect dokploy --format '{{range .Spec.TaskTemplate.ContainerSpec.Secrets}}{{if eq .SecretName "dokploy_auth_secret"}}{{println .SecretID}}{{end}}{{end}}' 2>/dev/null)"; then + echo "could not verify the authentication secret bound to the new Dokploy service; leaving it at zero replicas" >&2 + exit 1 + fi + set -- $DOKPLOY_BOUND_AUTH_SECRET_IDS + if [ "$#" -ne 1 ] || [ "$1" != "$DOKPLOY_AUTH_SECRET_ID" ]; then + echo "the new zero-replica Dokploy service did not bind the reviewed authentication-secret identity; removing it and refusing to start a task" >&2 + docker service rm dokploy >/dev/null 2>&1 || true + exit 1 + fi + docker service scale dokploy=1 >/dev/null fi log "Preparing Dokploy edge proxy for cutover" diff --git a/internal/cli/init_target_test.go b/internal/cli/init_target_test.go index dc5de24..eedcd53 100644 --- a/internal/cli/init_target_test.go +++ b/internal/cli/init_target_test.go @@ -5,19 +5,32 @@ import ( "context" "encoding/json" "errors" + "fmt" "io" "net/http" "net/http/httptest" + "net/url" "os" "os/exec" "path/filepath" + "slices" "strings" "testing" + "time" "github.com/aikins01/bort/internal/target/dokploy" "golang.org/x/crypto/bcrypt" ) +func serverPort(t *testing.T, server *httptest.Server) string { + t.Helper() + parsed, err := url.Parse(server.URL) + if err != nil { + t.Fatal(err) + } + return parsed.Port() +} + type fakeAdminLister struct { admins []coolifyAdmin } @@ -37,6 +50,16 @@ func (f *fakeDokployInstaller) InstallDokploy(_ context.Context, opts dokployIns return nil } +type blockingDokployInstaller struct { + calls int +} + +func (i *blockingDokployInstaller) InstallDokploy(ctx context.Context, _ dokployInstallOptions, _, _ io.Writer) error { + i.calls++ + <-ctx.Done() + return ctx.Err() +} + func bcryptCoolifyHash(t *testing.T, password string) string { t.Helper() hash, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.MinCost) @@ -50,9 +73,14 @@ type dokployStub struct { signupCalls int signupExists bool signinCalls int + userGetCalls int createKeyCalls int + deleteKeyCalls int lastKeyName string + lastKeyPrefix string lastOrgID string + deletedKeyIDs []string + apiKeys []dokploy.APIKey } func newDokployStub(t *testing.T, stub *dokployStub) *httptest.Server { @@ -74,11 +102,18 @@ func newDokployStub(t *testing.T, stub *dokployStub) *httptest.Server { _ = json.NewEncoder(w).Encode(map[string]string{"id": "u1"}) }) mux.HandleFunc("/api/trpc/user.get", func(w http.ResponseWriter, r *http.Request) { + stub.userGetCalls++ if !strings.Contains(r.Header.Get("Cookie"), "better-auth.session_token=session-xyz") { http.Error(w, "no cookie", http.StatusUnauthorized) return } - _, _ = w.Write([]byte(`[{"result":{"data":{"json":{"organizationId":"org-1","role":"owner"}}}}]`)) + _ = json.NewEncoder(w).Encode([]any{map[string]any{ + "result": map[string]any{"data": map[string]any{"json": map[string]any{ + "organizationId": "org-1", + "role": "owner", + "user": map[string]any{"apiKeys": stub.apiKeys}, + }}}, + }}) }) mux.HandleFunc("/api/trpc/user.createApiKey", func(w http.ResponseWriter, r *http.Request) { stub.createKeyCalls++ @@ -93,10 +128,40 @@ func newDokployStub(t *testing.T, stub *dokployStub) *httptest.Server { } inner := payload["0"]["json"] stub.lastKeyName, _ = inner["name"].(string) + stub.lastKeyPrefix, _ = inner["prefix"].(string) if md, ok := inner["metadata"].(map[string]any); ok { stub.lastOrgID, _ = md["organizationId"].(string) } - _, _ = w.Write([]byte(`[{"result":{"data":{"json":{"key":"K-secret"}}}}]`)) + keyID := fmt.Sprintf("key-%d", stub.createKeyCalls) + stub.apiKeys = append(stub.apiKeys, dokploy.APIKey{ID: keyID, Name: stub.lastKeyName, Prefix: stub.lastKeyPrefix}) + key := "K-secret" + if stub.createKeyCalls > 1 { + key = fmt.Sprintf("K-secret-%d", stub.createKeyCalls) + } + _ = json.NewEncoder(w).Encode([]any{map[string]any{ + "result": map[string]any{"data": map[string]any{"json": map[string]string{"id": keyID, "key": key}}}, + }}) + }) + mux.HandleFunc("/api/trpc/user.deleteApiKey", func(w http.ResponseWriter, r *http.Request) { + stub.deleteKeyCalls++ + if !strings.Contains(r.Header.Get("Cookie"), "better-auth.session_token=session-xyz") { + http.Error(w, "no cookie", http.StatusUnauthorized) + return + } + var payload map[string]map[string]map[string]string + if err := json.NewDecoder(r.Body).Decode(&payload); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + apiKeyID := payload["0"]["json"]["apiKeyId"] + stub.deletedKeyIDs = append(stub.deletedKeyIDs, apiKeyID) + for index, key := range stub.apiKeys { + if key.ID == apiKeyID { + stub.apiKeys = append(stub.apiKeys[:index], stub.apiKeys[index+1:]...) + break + } + } + _, _ = w.Write([]byte(`[{"result":{"data":{"json":true}}}]`)) }) return httptest.NewServer(mux) } @@ -132,6 +197,236 @@ func TestInitTargetBcryptMismatchAbortsBeforeDokploy(t *testing.T) { } } +func TestInitTargetVerifiesLocalServiceBeforeSendingPassword(t *testing.T) { + stub := &dokployStub{} + server := newDokployStub(t, stub) + defer server.Close() + deps := initTargetDeps{ + lister: &fakeAdminLister{admins: []coolifyAdmin{ + {Email: "admin@example.com", Name: "Admin User", PasswordHash: bcryptCoolifyHash(t, "right-password")}, + }}, + newClient: func(string) *dokploy.Client { return defaultDokployClient(server.URL) }, + verifyLocalService: func(context.Context, *dokploy.Client) error { + return errors.New("no running local Dokploy task") + }, + statePath: filepath.Join(t.TempDir(), "state.json"), + } + t.Setenv(envCoolifyAdminPwd, "right-password") + err := runInitTargetWith(context.Background(), []string{ + "--coolify-email", "admin@example.com", + "--dokploy-url", server.URL, + }, strings.NewReader(""), io.Discard, io.Discard, deps) + if err == nil || !strings.Contains(err.Error(), "verify local Dokploy service before bootstrap") { + t.Fatalf("expected local service verification refusal, got %v", err) + } + if stub.signupCalls+stub.signinCalls+stub.createKeyCalls != 0 { + t.Fatalf("Dokploy credentials were sent before local service verification: %+v", stub) + } +} + +func TestInitTargetInstallHonorsHostSharedLiveOperationLock(t *testing.T) { + t.Setenv(envDokployAuthBackup, filepath.Join(t.TempDir(), "dokploy-auth-secret")) + targetLock, err := acquireDokployLiveOperationLock() + if err != nil { + t.Fatal(err) + } + defer targetLock.Release() + installer := &fakeDokployInstaller{} + deps := initTargetDeps{ + lister: &fakeAdminLister{admins: []coolifyAdmin{ + {Email: "admin@example.com", Name: "Admin User", PasswordHash: bcryptCoolifyHash(t, "right-password")}, + }}, + installer: installer, + newClient: func(string) *dokploy.Client { + t.Fatal("Dokploy client was created while another live operation held the host lock") + return nil + }, + statePath: filepath.Join(t.TempDir(), "state.json"), + } + t.Setenv(envCoolifyAdminPwd, "right-password") + err = runInitTargetWith(context.Background(), []string{ + "--install", + "--dokploy-url", "http://127.0.0.1:3030", + "--coolify-email", "admin@example.com", + }, strings.NewReader(""), io.Discard, io.Discard, deps) + if !errors.Is(err, errDokployLiveOperationActive) { + t.Fatalf("expected host-shared live-operation lock contention, got %v", err) + } + if installer.calls != 0 { + t.Fatalf("blocked init-target mutated Dokploy installation %d time(s)", installer.calls) + } + requireNoDokployInstallationRecovery(t) +} + +func TestInitTargetCollectsPasswordBeforeWaitingForHostLock(t *testing.T) { + t.Setenv(envDokployAuthBackup, filepath.Join(t.TempDir(), "dokploy-auth-secret")) + targetLock, err := acquireDokployLiveOperationLock() + if err != nil { + t.Fatal(err) + } + defer targetLock.Release() + installer := &fakeDokployInstaller{} + deps := initTargetDeps{ + lister: &fakeAdminLister{admins: []coolifyAdmin{ + {Email: "admin@example.com", Name: "Admin User", PasswordHash: bcryptCoolifyHash(t, "right-password")}, + }}, + installer: installer, + newClient: func(string) *dokploy.Client { + t.Fatal("Dokploy client was created while another live operation held the host lock") + return nil + }, + statePath: filepath.Join(t.TempDir(), "state.json"), + } + t.Setenv(envCoolifyAdminPwd, "") + var stdout bytes.Buffer + err = runInitTargetWith(context.Background(), []string{ + "--install", + "--dokploy-url", "http://127.0.0.1:3030", + "--coolify-email", "admin@example.com", + }, strings.NewReader("right-password\n"), &stdout, io.Discard, deps) + if !errors.Is(err, errDokployLiveOperationActive) { + t.Fatalf("expected host-shared live-operation lock contention, got %v", err) + } + if !strings.Contains(stdout.String(), "Password for admin@example.com:") { + t.Fatalf("password was not collected before lock acquisition: %q", stdout.String()) + } + if installer.calls != 0 { + t.Fatalf("blocked init-target mutated Dokploy installation %d time(s)", installer.calls) + } + requireNoDokployInstallationRecovery(t) +} + +func TestStandaloneInitTargetRefusesDurableHostOwnerBeforeMutation(t *testing.T) { + t.Setenv(envDokployAuthBackup, filepath.Join(t.TempDir(), "dokploy-auth-secret")) + resetDokployTrafficOwner(t) + run := migrationRun{ + Name: "owned-migration", + RunDir: filepath.Join(t.TempDir(), "owned-migration"), + CreatedAt: time.Now().UTC(), + BundleDigest: "digest", + } + if err := claimDokployHostOwnership(run, "http://127.0.0.1:3030", dokployCredentialID("migration-token")); err != nil { + t.Fatal(err) + } + installer := &fakeDokployInstaller{} + deps := initTargetDeps{ + lister: &fakeAdminLister{admins: []coolifyAdmin{ + {Email: "admin@example.com", Name: "Admin User", PasswordHash: bcryptCoolifyHash(t, "right-password")}, + }}, + installer: installer, + newClient: func(string) *dokploy.Client { + t.Fatal("standalone init-target created a client while the host had a durable owner") + return nil + }, + statePath: filepath.Join(t.TempDir(), "state.json"), + } + t.Setenv(envCoolifyAdminPwd, "right-password") + err := runInitTargetWith(context.Background(), []string{ + "--install", + "--dokploy-url", "http://127.0.0.1:3030", + "--coolify-email", "admin@example.com", + }, strings.NewReader(""), io.Discard, io.Discard, deps) + if err == nil || !strings.Contains(err.Error(), fmt.Sprintf("owned by migration run %q at %q", run.Name, run.RunDir)) { + t.Fatalf("expected durable owner refusal, got %v", err) + } + if installer.calls != 0 { + t.Fatalf("blocked init-target mutated Dokploy installation %d time(s)", installer.calls) + } + requireNoDokployInstallationRecovery(t) +} + +func TestInitTargetCanRetainAcquiredHostLockForLiveApply(t *testing.T) { + t.Setenv(envDokployAuthBackup, filepath.Join(t.TempDir(), "dokploy-auth-secret")) + stub := &dokployStub{} + server := newDokployStub(t, stub) + defer server.Close() + installer := &fakeDokployInstaller{} + var retained *applyLock + revalidated := false + deps := initTargetDeps{ + lister: &fakeAdminLister{admins: []coolifyAdmin{ + {Email: "admin@example.com", Name: "Admin User", PasswordHash: bcryptCoolifyHash(t, "right-password")}, + }}, + installer: installer, + newClient: func(string) *dokploy.Client { return defaultDokployClient(server.URL) }, + statePath: filepath.Join(t.TempDir(), "state.json"), + retainLiveOperationLock: &retained, + revalidateAfterTargetLock: func() error { + revalidated = true + second, err := acquireDokployLiveOperationLock() + if second != nil { + second.Release() + } + if !errors.Is(err, errDokployLiveOperationActive) { + return fmt.Errorf("expected revalidation to run under host lock, got %v", err) + } + return nil + }, + } + t.Setenv(envCoolifyAdminPwd, "right-password") + err := runInitTargetWith(context.Background(), []string{ + "--install", + "--dokploy-url", server.URL, + "--install-port", serverPort(t, server), + "--coolify-email", "admin@example.com", + }, strings.NewReader(""), io.Discard, io.Discard, deps) + if err != nil { + t.Fatalf("init-target with retained lock failed: %v", err) + } + if !revalidated || retained == nil { + t.Fatalf("expected revalidation and retained lock, revalidated=%t retained=%v", revalidated, retained) + } + second, err := acquireDokployLiveOperationLock() + if second != nil { + second.Release() + } + if !errors.Is(err, errDokployLiveOperationActive) { + t.Fatalf("setup released the retained host lock before live apply: %v", err) + } + retained.Release() + retained = nil + reacquired, err := acquireDokployLiveOperationLock() + if err != nil { + t.Fatalf("host lock was not released by the caller: %v", err) + } + reacquired.Release() +} + +func TestInitTargetReusesHeldLiveOperationLock(t *testing.T) { + t.Setenv(envDokployAuthBackup, filepath.Join(t.TempDir(), "dokploy-auth-secret")) + targetLock, err := acquireDokployLiveOperationLock() + if err != nil { + t.Fatal(err) + } + defer targetLock.Release() + stub := &dokployStub{} + server := newDokployStub(t, stub) + defer server.Close() + installer := &fakeDokployInstaller{} + deps := initTargetDeps{ + lister: &fakeAdminLister{admins: []coolifyAdmin{ + {Email: "admin@example.com", Name: "Admin User", PasswordHash: bcryptCoolifyHash(t, "right-password")}, + }}, + installer: installer, + newClient: func(string) *dokploy.Client { return defaultDokployClient(server.URL) }, + statePath: filepath.Join(t.TempDir(), "state.json"), + liveOperationLock: targetLock, + } + t.Setenv(envCoolifyAdminPwd, "right-password") + err = runInitTargetWith(context.Background(), []string{ + "--install", + "--dokploy-url", server.URL, + "--install-port", serverPort(t, server), + "--coolify-email", "admin@example.com", + }, strings.NewReader(""), io.Discard, io.Discard, deps) + if err != nil { + t.Fatalf("init-target under held live-operation lock failed: %v", err) + } + if installer.calls != 1 || stub.createKeyCalls != 1 { + t.Fatalf("expected setup to complete under inherited lock, installer=%d api_keys=%d", installer.calls, stub.createKeyCalls) + } +} + func TestInitTargetSuccessfulBootstrapPersistsState(t *testing.T) { stub := &dokployStub{} server := newDokployStub(t, stub) @@ -157,10 +452,10 @@ func TestInitTargetSuccessfulBootstrapPersistsState(t *testing.T) { if err := runInitTargetWith(context.Background(), args, strings.NewReader(""), stdout, stderr, deps); err != nil { t.Fatalf("unexpected error: %v", err) } - if stub.signupCalls != 1 || stub.signinCalls != 1 || stub.createKeyCalls != 1 { + if stub.signupCalls != 1 || stub.signinCalls != 1 || stub.userGetCalls != 1 || stub.createKeyCalls != 1 { t.Fatalf("unexpected call counts: %+v", stub) } - if stub.lastOrgID != "org-1" || stub.lastKeyName != defaultDokployAPIName { + if stub.lastOrgID != "org-1" || stub.lastKeyName != defaultDokployAPIName || stub.lastKeyPrefix != "" { t.Fatalf("unexpected createApiKey payload: %+v", stub) } if strings.Contains(stdout.String(), "K-secret") { @@ -187,6 +482,8 @@ func TestInitTargetSuccessfulBootstrapPersistsState(t *testing.T) { } func TestInitTargetInstallRunsInstallerBeforeBootstrap(t *testing.T) { + authBackup := filepath.Join(t.TempDir(), "dokploy-auth-secret") + t.Setenv(envDokployAuthBackup, authBackup) stub := &dokployStub{} server := newDokployStub(t, stub) defer server.Close() @@ -205,11 +502,14 @@ func TestInitTargetInstallRunsInstallerBeforeBootstrap(t *testing.T) { t.Setenv(envCoolifyAdminPwd, "right-password") args := []string{ "--install", - "--install-port", "3031", + "--install-port", serverPort(t, server), "--swarm-addr-pool", "172.29.0.0/16", - "--dokploy-version", "v0.26.6", + "--dokploy-version", "v0.30.7", + "--endpoint-mode", "dnsrr", "--coolify-email", "admin@example.com", "--dokploy-url", server.URL, + "--name", "Explicit Admin", + "--api-key-name", "migration key", } if err := runInitTargetWith(context.Background(), args, strings.NewReader(""), &bytes.Buffer{}, &bytes.Buffer{}, deps); err != nil { t.Fatalf("unexpected error: %v", err) @@ -217,15 +517,249 @@ func TestInitTargetInstallRunsInstallerBeforeBootstrap(t *testing.T) { if installer.calls != 1 { t.Fatalf("expected installer to run once, got %d", installer.calls) } - if installer.opts.HostPort != "3031" || installer.opts.AddrPool != "172.29.0.0/16" || installer.opts.Version != "v0.26.6" || installer.opts.ACMEEmail != "admin@example.com" { + wantVersion := defaultDokployVersion + "@" + defaultDokployDigest + if installer.opts.HostPort != serverPort(t, server) || installer.opts.AddrPool != "172.29.0.0/16" || installer.opts.Version != wantVersion || installer.opts.EndpointMode != "dnsrr" || installer.opts.ACMEEmail != "admin@example.com" || installer.opts.AuthSecretBackup != authBackup || installer.opts.AdminName != "Explicit Admin" || installer.opts.APIKeyName != "migration key" { t.Fatalf("unexpected install opts: %#v", installer.opts) } - if stub.signupCalls != 1 || stub.signinCalls != 1 || stub.createKeyCalls != 1 { + if stub.signupCalls != 1 || stub.signinCalls != 1 || stub.userGetCalls != 1 || stub.createKeyCalls != 1 { t.Fatalf("expected bootstrap after install, got %+v", stub) } } +type foreignServiceDokployInstaller struct { + calls int +} + +func (i *foreignServiceDokployInstaller) InstallDokploy(_ context.Context, _ dokployInstallOptions, _, stderr io.Writer) error { + i.calls++ + fmt.Fprintln(stderr, "existing Dokploy service was not installed by Bort, so --install cannot adopt or repair it; it answers at http://127.0.0.1:3000") + return exec.Command("sh", "-c", "exit 75").Run() +} + +func TestInitTargetInstallForeignServiceRefusalClearsRecoveryMarker(t *testing.T) { + t.Cleanup(func() { _ = clearDokployInstallationRecoveryRequired() }) + t.Setenv(envDokployAuthBackup, filepath.Join(t.TempDir(), "dokploy-auth-secret")) + stub := &dokployStub{} + server := newDokployStub(t, stub) + defer server.Close() + installer := &foreignServiceDokployInstaller{} + deps := initTargetDeps{ + lister: &fakeAdminLister{admins: []coolifyAdmin{ + {Email: "admin@example.com", Name: "Admin User", PasswordHash: bcryptCoolifyHash(t, "right-password")}, + }}, + installer: installer, + newClient: func(string) *dokploy.Client { return defaultDokployClient(server.URL) }, + statePath: filepath.Join(t.TempDir(), "state.json"), + } + t.Setenv(envCoolifyAdminPwd, "right-password") + t.Setenv("SUDO_UID", "1000") + err := runInitTargetWith(context.Background(), []string{ + "--install", + "--coolify-email", "admin@example.com", + "--dokploy-url", server.URL, + "--install-port", serverPort(t, server), + }, strings.NewReader(""), io.Discard, io.Discard, deps) + if err == nil || !strings.Contains(err.Error(), "not installed by Bort") || !strings.Contains(err.Error(), "sudo bort init-target --dokploy-url") || strings.Contains(err.Error(), "blocks other host mutations") { + t.Fatalf("expected foreign-service refusal with the recorded command prefix, got %v", err) + } + if installer.calls != 1 || stub.signupCalls != 0 { + t.Fatalf("unexpected calls after refusal: installer=%d signup=%d", installer.calls, stub.signupCalls) + } + if blocked, err := dokployInstallationRecoveryRequired(); err != nil || blocked { + t.Fatalf("refusing a foreign service left installation recovery required: blocked=%t err=%v", blocked, err) + } + if err := runInitTargetWith(context.Background(), []string{ + "--coolify-email", "admin@example.com", + "--dokploy-url", server.URL, + }, strings.NewReader(""), io.Discard, io.Discard, deps); err != nil { + t.Fatalf("suggested non-install bootstrap was blocked: %v", err) + } + if installer.calls != 1 || stub.signupCalls != 1 { + t.Fatalf("non-install bootstrap did not run against the existing service: installer=%d signup=%d", installer.calls, stub.signupCalls) + } +} + +func TestInitTargetInstallFailureKeepsRecoveryMarker(t *testing.T) { + t.Cleanup(func() { _ = clearDokployInstallationRecoveryRequired() }) + t.Setenv(envDokployAuthBackup, filepath.Join(t.TempDir(), "dokploy-auth-secret")) + stub := &dokployStub{} + server := newDokployStub(t, stub) + defer server.Close() + deps := initTargetDeps{ + lister: &fakeAdminLister{admins: []coolifyAdmin{ + {Email: "admin@example.com", Name: "Admin User", PasswordHash: bcryptCoolifyHash(t, "right-password")}, + }}, + installer: &failingDokployInstaller{err: exec.Command("sh", "-c", "exit 1").Run()}, + newClient: func(string) *dokploy.Client { return defaultDokployClient(server.URL) }, + statePath: filepath.Join(t.TempDir(), "state.json"), + } + t.Setenv(envCoolifyAdminPwd, "right-password") + err := runInitTargetWith(context.Background(), []string{ + "--install", + "--coolify-email", "admin@example.com", + "--dokploy-url", server.URL, + "--install-port", serverPort(t, server), + }, strings.NewReader(""), io.Discard, io.Discard, deps) + if err == nil || !strings.Contains(err.Error(), "install dokploy: exit status 1") { + t.Fatalf("expected generic installer failure, got %v", err) + } + if blocked, err := dokployInstallationRecoveryRequired(); err != nil || !blocked { + t.Fatalf("a mutating installer failure must keep recovery required: blocked=%t err=%v", blocked, err) + } +} + +type failingDokployInstaller struct { + err error +} + +func (i *failingDokployInstaller) InstallDokploy(context.Context, dokployInstallOptions, io.Writer, io.Writer) error { + return i.err +} + +func TestInitTargetInstallRetryReplacesOrphanedAPIKeyAfterStateWriteFailure(t *testing.T) { + t.Cleanup(func() { _ = clearDokployInstallationRecoveryRequired() }) + authBackup := filepath.Join(t.TempDir(), "dokploy-auth-secret") + t.Setenv(envDokployAuthBackup, authBackup) + stub := &dokployStub{apiKeys: []dokploy.APIKey{{ID: "unrelated-key", Name: "other tool"}}} + server := newDokployStub(t, stub) + defer server.Close() + installer := &fakeDokployInstaller{} + blockedParent := filepath.Join(t.TempDir(), "not-a-directory") + if err := os.WriteFile(blockedParent, []byte("block state writes"), 0o600); err != nil { + t.Fatal(err) + } + deps := initTargetDeps{ + lister: &fakeAdminLister{admins: []coolifyAdmin{ + {Email: "admin@example.com", Name: "Admin User", PasswordHash: bcryptCoolifyHash(t, "right-password")}, + }}, + installer: installer, + newClient: func(string) *dokploy.Client { return defaultDokployClient(server.URL) }, + statePath: filepath.Join(blockedParent, "state.json"), + } + t.Setenv(envCoolifyAdminPwd, "right-password") + args := []string{ + "--install", + "--coolify-email", "admin@example.com", + "--dokploy-url", server.URL, + "--install-port", serverPort(t, server), + } + if err := runInitTargetWith(context.Background(), args, strings.NewReader(""), io.Discard, io.Discard, deps); err == nil { + t.Fatal("expected state persistence failure after API-key creation") + } + recovery, found, err := readDokployInstallationRecovery() + if err != nil || !found { + t.Fatalf("read pending API-key recovery: found=%t err=%v", found, err) + } + if recovery.Phase != dokployInstallAPIKey || !slices.Equal(recovery.APIKeyBaselineIDs, []string{"unrelated-key"}) { + t.Fatalf("unexpected pending API-key recovery: %#v", recovery) + } + if stub.userGetCalls != 1 || stub.createKeyCalls != 1 || stub.deleteKeyCalls != 0 { + t.Fatalf("unexpected first-attempt API calls: user.get=%d create=%d delete=%d", stub.userGetCalls, stub.createKeyCalls, stub.deleteKeyCalls) + } + stub.apiKeys = append(stub.apiKeys, dokploy.APIKey{ID: "concurrent-key", Name: defaultDokployAPIName, Prefix: "manual_prefix"}) + + deps.statePath = filepath.Join(t.TempDir(), "state.json") + if err := runInitTargetWith(context.Background(), args, strings.NewReader(""), io.Discard, io.Discard, deps); err != nil { + t.Fatalf("retry interrupted install: %v", err) + } + if installer.calls != 1 || stub.userGetCalls != 2 || stub.createKeyCalls != 2 || stub.deleteKeyCalls != 1 || !slices.Equal(stub.deletedKeyIDs, []string{"key-1"}) { + t.Fatalf("retry did not replace only the orphaned key with one user.get per attempt: installer=%d user.get=%d create=%d delete=%d deleted=%v", installer.calls, stub.userGetCalls, stub.createKeyCalls, stub.deleteKeyCalls, stub.deletedKeyIDs) + } + if len(stub.apiKeys) != 3 || stub.apiKeys[0].ID != "unrelated-key" || stub.apiKeys[1].ID != "concurrent-key" || stub.apiKeys[2].ID != "key-2" || stub.apiKeys[2].Name != defaultDokployAPIName || stub.apiKeys[2].Prefix != dokployInstallAPIKeyPrefix(recovery) { + t.Fatalf("unexpected active API keys after retry: %#v", stub.apiKeys) + } + state, err := readBortState(deps.statePath) + if err != nil { + t.Fatal(err) + } + if got := state.Targets["dokploy"].Token; got != "K-secret-2" { + t.Fatalf("persisted API key=%q, want replacement key", got) + } + if _, found, err := readDokployInstallationRecovery(); err != nil || found { + t.Fatalf("API-key recovery marker survived successful persistence: found=%t err=%v", found, err) + } +} + +func TestPrepareDokployAPIKeyRecoveryPreservesUnownedMatchingKey(t *testing.T) { + stub := &dokployStub{} + server := newDokployStub(t, stub) + defer server.Close() + recovery := newDokployInstallationRecovery(server.URL, dokployInstallOptions{ + HostPort: "3030", + AddrPool: "auto", + Version: defaultDokployVersion + "@" + defaultDokployDigest, + EndpointMode: "dnsrr", + ACMEEmail: "admin@example.com", + AuthSecretBackup: filepath.Join(t.TempDir(), "auth-secret"), + AdminName: "Admin User", + APIKeyName: defaultDokployAPIName, + }) + recovery.Phase = dokployInstallAPIKey + recovery.APIKeyBaselineIDs = []string{"baseline-key"} + matching := dokploy.APIKey{ID: "operator-key", Name: recovery.APIKeyName, Prefix: dokployInstallAPIKeyPrefix(recovery)} + client := defaultDokployClient(server.URL) + _, err := prepareDokployAPIKeyRecovery(context.Background(), client, "session-xyz", recovery, []dokploy.APIKey{ + {ID: "baseline-key", Name: "existing"}, + matching, + }) + if err == nil || !strings.Contains(err.Error(), "no created key ID was durably recorded") { + t.Fatalf("expected unowned matching key refusal, got %v", err) + } + if stub.deleteKeyCalls != 0 { + t.Fatalf("unowned matching API key was deleted: %#v", stub.deletedKeyIDs) + } +} + +func TestPrepareDokployAPIKeyRecoveryClearsDeletedIdentityBeforeReplacement(t *testing.T) { + t.Cleanup(func() { _ = clearDokployInstallationRecoveryRequired() }) + stub := &dokployStub{} + server := newDokployStub(t, stub) + defer server.Close() + recovery := newDokployInstallationRecovery(server.URL, dokployInstallOptions{ + HostPort: "3030", + AddrPool: "auto", + Version: defaultDokployVersion + "@" + defaultDokployDigest, + EndpointMode: "dnsrr", + ACMEEmail: "admin@example.com", + AuthSecretBackup: filepath.Join(t.TempDir(), "auth-secret"), + AdminName: "Admin User", + APIKeyName: defaultDokployAPIName, + }) + recovery.Phase = dokployInstallAPIKey + recovery.APIKeyBaselineIDs = []string{"baseline-key"} + recovery.APIKeyCreatedID = "key-1" + if err := markDokployInstallationRecoveryRequired(recovery); err != nil { + t.Fatal(err) + } + client := defaultDokployClient(server.URL) + updated, err := prepareDokployAPIKeyRecovery(context.Background(), client, "session-xyz", recovery, []dokploy.APIKey{ + {ID: "baseline-key", Name: "existing"}, + {ID: "key-1", Name: recovery.APIKeyName, Prefix: dokployInstallAPIKeyPrefix(recovery)}, + }) + if err != nil { + t.Fatal(err) + } + if updated.APIKeyCreatedID != "" || !slices.Equal(stub.deletedKeyIDs, []string{"key-1"}) { + t.Fatalf("recorded key replacement was not reconciled: recovery=%#v deleted=%v", updated, stub.deletedKeyIDs) + } + persisted, found, err := readDokployInstallationRecovery() + if err != nil || !found || persisted.APIKeyCreatedID != "" { + t.Fatalf("cleared key identity was not durable: found=%t recovery=%#v err=%v", found, persisted, err) + } + _, err = prepareDokployAPIKeyRecovery(context.Background(), client, "session-xyz", updated, []dokploy.APIKey{ + {ID: "baseline-key", Name: "existing"}, + {ID: "key-2", Name: recovery.APIKeyName, Prefix: dokployInstallAPIKeyPrefix(recovery)}, + }) + if err == nil || !strings.Contains(err.Error(), "no created key ID was durably recorded") { + t.Fatalf("unrecorded replacement key did not block another creation: %v", err) + } + if !slices.Equal(stub.deletedKeyIDs, []string{"key-1"}) { + t.Fatalf("unowned replacement key was deleted: %v", stub.deletedKeyIDs) + } +} + func TestInitTargetInstallDefaultsSwarmAddrPoolToAuto(t *testing.T) { + t.Setenv(envDokployAuthBackup, filepath.Join(t.TempDir(), "dokploy-auth-secret")) stub := &dokployStub{} server := newDokployStub(t, stub) defer server.Close() @@ -246,6 +780,7 @@ func TestInitTargetInstallDefaultsSwarmAddrPoolToAuto(t *testing.T) { "--install", "--coolify-email", "admin@example.com", "--dokploy-url", server.URL, + "--install-port", serverPort(t, server), } if err := runInitTargetWith(context.Background(), args, strings.NewReader(""), &bytes.Buffer{}, &bytes.Buffer{}, deps); err != nil { t.Fatalf("unexpected error: %v", err) @@ -256,6 +791,7 @@ func TestInitTargetInstallDefaultsSwarmAddrPoolToAuto(t *testing.T) { } func TestInitTargetInstallPromptsAndVerifiesPasswordBeforeInstaller(t *testing.T) { + t.Setenv(envDokployAuthBackup, filepath.Join(t.TempDir(), "dokploy-auth-secret")) stub := &dokployStub{} server := newDokployStub(t, stub) defer server.Close() @@ -277,6 +813,7 @@ func TestInitTargetInstallPromptsAndVerifiesPasswordBeforeInstaller(t *testing.T "--install", "--coolify-email", "admin@example.com", "--dokploy-url", server.URL, + "--install-port", serverPort(t, server), } if err := runInitTargetWith(context.Background(), args, strings.NewReader("right-password\n"), stdout, &bytes.Buffer{}, deps); err != nil { t.Fatalf("unexpected error: %v", err) @@ -292,6 +829,7 @@ func TestInitTargetInstallPromptsAndVerifiesPasswordBeforeInstaller(t *testing.T } func TestInitTargetInstallPasswordMismatchSkipsInstaller(t *testing.T) { + t.Setenv(envDokployAuthBackup, filepath.Join(t.TempDir(), "dokploy-auth-secret")) stub := &dokployStub{} server := newDokployStub(t, stub) defer server.Close() @@ -311,6 +849,7 @@ func TestInitTargetInstallPasswordMismatchSkipsInstaller(t *testing.T) { "--install", "--coolify-email", "admin@example.com", "--dokploy-url", server.URL, + "--install-port", serverPort(t, server), } err := runInitTargetWith(context.Background(), args, strings.NewReader(""), &bytes.Buffer{}, &bytes.Buffer{}, deps) if err == nil || !strings.Contains(err.Error(), "verification failed") { @@ -532,8 +1071,41 @@ func TestValidateInstallPort(t *testing.T) { } } +func TestNormalizeSwarmAddressPool(t *testing.T) { + valid := map[string]string{ + "auto": "auto", + " AUTO ": "auto", + "172.29.4.8/16": "172.29.0.0/16", + "192.168.20.0/24": "192.168.20.0/24", + } + for input, want := range valid { + got, err := normalizeSwarmAddressPool(input) + if err != nil || got != want { + t.Fatalf("normalizeSwarmAddressPool(%q) = %q, %v; want %q", input, got, err, want) + } + } + for _, input := range []string{"", "not-a-cidr", "172.29.0.0", "172.29.0.0/25", "fd00::/64"} { + if _, err := normalizeSwarmAddressPool(input); err == nil { + t.Fatalf("expected address pool %q to be rejected", input) + } + } +} + +func TestValidateDokployAPIKeyName(t *testing.T) { + for _, name := range []string{"a", strings.Repeat("k", 32)} { + if err := validateDokployAPIKeyName(name); err != nil { + t.Fatalf("expected API-key name %q to be valid: %v", name, err) + } + } + for _, name := range []string{"", strings.Repeat("k", 33)} { + if err := validateDokployAPIKeyName(name); err == nil { + t.Fatalf("expected API-key name %q to be rejected", name) + } + } +} + func TestValidateDokployVersionTag(t *testing.T) { - valid := []string{"latest", "v0.26.6", "0.26.6-beta_1", "_rc2", strings.Repeat("a", 128)} + valid := []string{"v0.29.3", "v0.30.7-beta_1", "v1.0.0"} for _, tag := range valid { if err := validateDokployVersionTag(tag); err != nil { t.Fatalf("expected tag %q to be valid: %v", tag, err) @@ -548,6 +1120,14 @@ func TestValidateDokployVersionTag(t *testing.T) { ".leading-dot", "-leading-dash", "tag\nnext", + "feature", + "feature-amd64", + "latest", + "canary", + "_rc2", + "0.29.3", + strings.Repeat("a", 128), + "v" + strings.Repeat("9", 30) + ".0.0", strings.Repeat("a", 129), } for _, tag := range invalid { @@ -555,6 +1135,35 @@ func TestValidateDokployVersionTag(t *testing.T) { t.Fatalf("expected tag %q to be rejected", tag) } } + unsupported := []string{"v0.26.6", "0.29.2", "v0.29.2-rc.1", "v0.29.3-rc.1"} + for _, tag := range unsupported { + err := validateDokployVersionTag(tag) + if err == nil || !strings.Contains(err.Error(), "v0.29.3 or newer") { + t.Fatalf("expected tag %q to be rejected as unsupported, got %v", tag, err) + } + } +} + +func TestNormalizeDokployVersionRequiresReviewedDigest(t *testing.T) { + wantDefault := defaultDokployVersion + "@" + defaultDokployDigest + if got, err := normalizeDokployVersion(defaultDokployVersion); err != nil || got != wantDefault { + t.Fatalf("default version normalized to %q, %v; want %q", got, err, wantDefault) + } + digest := "sha256:" + strings.Repeat("a", 64) + if got, err := normalizeDokployVersion("v0.30.8@" + digest); err != nil || got != "v0.30.8@"+digest { + t.Fatalf("digest-qualified version normalized to %q, %v", got, err) + } + for _, version := range []string{ + "v0.30.8", + "v0.30.7@sha256:abcdef", + "v0.30.7@sha256:" + strings.Repeat("A", 64), + "v0.30.7@sha512:" + strings.Repeat("a", 64), + "v0.30.7@" + digest + "@" + digest, + } { + if _, err := normalizeDokployVersion(version); err == nil { + t.Fatalf("expected version %q to be rejected", version) + } + } } func TestValidateACMEEmail(t *testing.T) { @@ -599,6 +1208,27 @@ func TestInitTargetInstallRejectsInvalidPortBeforeBootstrap(t *testing.T) { } } +func TestInitTargetInstallRejectsURLPortMismatchBeforeRecoveryMarker(t *testing.T) { + installer := &fakeDokployInstaller{} + deps := initTargetDeps{ + lister: &fakeAdminLister{}, + installer: installer, + statePath: filepath.Join(t.TempDir(), "state.json"), + } + t.Setenv(envDokployAuthBackup, filepath.Join(t.TempDir(), "dokploy-auth-secret")) + args := []string{"--install", "--dokploy-url", "http://127.0.0.1:4040"} + err := runInitTargetWith(context.Background(), args, strings.NewReader(""), io.Discard, io.Discard, deps) + if err == nil || !strings.Contains(err.Error(), "port 4040 does not match --install-port 3030") { + t.Fatalf("expected install port mismatch refusal, got %v", err) + } + if installer.calls != 0 { + t.Fatalf("installer ran despite port mismatch") + } + if _, found, readErr := dokployInstallationRecoveryCommand(); readErr != nil || found { + t.Fatalf("recovery marker written before refusal: found=%v err=%v", found, readErr) + } +} + func TestInitTargetInstallRejectsInvalidVersionTag(t *testing.T) { installer := &fakeDokployInstaller{} deps := initTargetDeps{ @@ -616,7 +1246,37 @@ func TestInitTargetInstallRejectsInvalidVersionTag(t *testing.T) { } } +func TestInitTargetInstallRejectsInvalidEndpointMode(t *testing.T) { + installer := &fakeDokployInstaller{} + deps := initTargetDeps{ + lister: &fakeAdminLister{}, + installer: installer, + statePath: filepath.Join(t.TempDir(), "state.json"), + } + args := []string{"--install", "--endpoint-mode", "ingress", "--dokploy-url", "http://127.0.0.1:3030"} + err := runInitTargetWith(context.Background(), args, strings.NewReader(""), io.Discard, io.Discard, deps) + if err == nil || !strings.Contains(err.Error(), "endpoint-mode") { + t.Fatalf("expected endpoint-mode validation error, got %v", err) + } + if installer.calls != 0 { + t.Fatalf("installer ran despite invalid endpoint mode") + } +} + +func TestValidateAuthSecretBackupPath(t *testing.T) { + if err := validateAuthSecretBackupPath(""); err == nil || !strings.Contains(err.Error(), envDokployAuthBackup) { + t.Fatalf("expected missing escrow refusal, got %v", err) + } + if err := validateAuthSecretBackupPath("relative/secret"); err == nil || !strings.Contains(err.Error(), "absolute path") { + t.Fatalf("expected relative escrow refusal, got %v", err) + } + if err := validateAuthSecretBackupPath(filepath.Join(t.TempDir(), "dokploy-auth-secret")); err != nil { + t.Fatalf("expected absolute escrow path to be accepted, got %v", err) + } +} + func TestInitTargetInstallRejectsUnsafeACMEEmail(t *testing.T) { + t.Setenv(envDokployAuthBackup, filepath.Join(t.TempDir(), "dokploy-auth-secret")) stub := &dokployStub{} server := newDokployStub(t, stub) defer server.Close() @@ -629,7 +1289,7 @@ func TestInitTargetInstallRejectsUnsafeACMEEmail(t *testing.T) { newClient: func(string) *dokploy.Client { return defaultDokployClient(server.URL) }, statePath: filepath.Join(t.TempDir(), "state.json"), } - args := []string{"--install", "--coolify-email", "a#b@example.com", "--dokploy-url", server.URL} + args := []string{"--install", "--coolify-email", "a#b@example.com", "--dokploy-url", server.URL, "--install-port", serverPort(t, server)} err := runInitTargetWith(context.Background(), args, strings.NewReader(""), &bytes.Buffer{}, &bytes.Buffer{}, deps) if err == nil || !strings.Contains(err.Error(), "ACME") { t.Fatalf("expected ACME email validation error, got %v", err) @@ -664,6 +1324,402 @@ func TestDokployShadowInstallScriptIsHardened(t *testing.T) { } } +func TestDokployShadowInstallUsesStableAuthenticationSecret(t *testing.T) { + bash, err := exec.LookPath("bash") + if err != nil { + t.Skip("bash is unavailable") + } + syntax := exec.Command(bash, "-n") + syntax.Stdin = strings.NewReader(dokployShadowInstallScript) + if out, err := syntax.CombinedOutput(); err != nil { + t.Fatalf("install script syntax is invalid: %v: %s", err, out) + } + for _, want := range []string{ + `docker secret inspect dokploy_auth_secret >/dev/null 2>&1`, + `openssl rand -hex 32`, + `DOKPLOY_AUTH_SECRET_BACKUP`, + `AUTH_SECRET_PROVENANCE=/var/lib/bort/dokploy-auth-secret.id`, + `AUTH_SECRET_FD_PATH="/dev/fd/$AUTH_SECRET_FD"`, + `AUTH_SECRET_INTENT_LABEL=io.bort.auth-secret-intent`, + `INSTALL_AUTH_SECRET_LABEL=io.bort.install-auth-secret-id`, + `write_auth_secret_intent`, + `validate_auth_secret_provenance "$DOKPLOY_AUTH_SECRET_CREATED_ID"`, + `validate_auth_secret_provenance`, + `docker secret create --label "$AUTH_SECRET_INTENT_LABEL=$DOKPLOY_AUTH_SECRET_INTENT" dokploy_auth_secret -`, + `dokploy_auth_secret - < "$AUTH_SECRET_FD_PATH"`, + `--secret source=dokploy_auth_secret,target=/run/secrets/dokploy_auth_secret`, + `--replicas 0`, + `DOKPLOY_BOUND_AUTH_SECRET_IDS`, + `DOKPLOY_STAGED_SERVICE`, + `the $service_state Dokploy service is not bound to the reviewed Bort authentication-secret identity`, + `docker service scale dokploy=1`, + `-e BETTER_AUTH_SECRET_FILE=/run/secrets/dokploy_auth_secret`, + `BETTER_AUTH_SECRET=better-auth-secret-123456789`, + `^BETTER_AUTH_SECRET=[0-9a-f]{64}$`, + `cryptographically random 32-byte key`, + `^BETTER_AUTH_SECRET_FILE=.+$`, + `refusing to generate a new key because changing the legacy fallback can make encrypted settings unreadable`, + `better-auth-secret-123456789`, + `dokploy:db-encryption:v1`, + `/etc/dokploy/encryption.key`, + `fence every inbound Dokploy UI/API path`, + `published host port targeting container port 3000`, + `defaults to host port 3030`, + `validate and preserve every existing 64-lowercase-hex line`, + `atomically add HMAC-SHA256`, + `keep the fence active through the service restart`, + `hosted 0.29.3.sh wrapper`, + `upgrade Dokploy to v0.29.5 or newer`, + `verify the running service image`, + `120-second migration timeout as success`, + `--secret-add source=dokploy-auth-secret,target=/run/secrets/dokploy-auth-secret`, + `pnpm run migrate-auth-secret`, + `refusing to run migration while Docker secret dokploy-auth-secret already exists`, + `DOKPLOY_AUTH_SECRET_BACKUP`, + `mktemp "${TMPDIR:-/tmp}/dokploy-auth-secret.XXXXXX"`, + `sync -f "$DOKPLOY_AUTH_SECRET_BACKUP"`, + `cmp -s -- "$auth_secret_tmp" "$DOKPLOY_AUTH_SECRET_BACKUP"`, + `sudo docker service update --force --update-order stop-first --detach=false dokploy`, + `SELECT count(*) FROM pg_stat_activity`, + `sudo docker secret create dokploy-auth-secret - < "$auth_secret_tmp"`, + `shred -u -- "$auth_secret_tmp"`, + `Retain DOKPLOY_AUTH_SECRET_BACKUP and DOKPLOY_TWO_FACTOR_BEFORE after success`, + `DOKPLOY_TWO_FACTOR_BEFORE`, + `public.two_factor`, + `compare-and-swap recovery`, + `DATABASE_URL is configured`, + `use a database-operator procedure`, + `do not restore the full PostgreSQL database`, + `still uses the legacy fallback`, + `verify representative application, Compose, project, environment, database, build-argument, and build-secret values render as plaintext`, + `re-encrypts only Better Auth two-factor secret and backup-code fields`, + `dokploy_image_supports_auth_secret_file`, + `existing Dokploy database state has no reusable dokploy_auth_secret`, + `Bort does not accept a manually recreated secret even when its key bytes match`, + } { + if !strings.Contains(dokployShadowInstallScript, want) { + t.Fatalf("install script missing authentication-secret fragment %q", want) + } + } + if strings.Contains(dokployShadowInstallScript, `-e BETTER_AUTH_SECRET="$AUTH_SECRET"`) { + t.Fatal("authentication secret must not be stored in the service environment") + } + if strings.Contains(dokployShadowInstallScript, `dokploy_auth_secret - < "$AUTH_SECRET_BACKUP"`) { + t.Fatal("fresh authentication secret must be created from the validated descriptor, not a reopened path") + } + if strings.Contains(dokployShadowInstallScript, `curl -fsSL https://dokploy.com/security/0.29.3.sh`) { + t.Fatal("unsafe timeout-based authentication migration wrapper must not be recommended") + } + if strings.Contains(dokployShadowInstallScript, `openssl rand -hex 32 | sudo docker secret create dokploy-auth-secret -`) { + t.Fatal("legacy authentication migration creates the Docker secret before escrowing the generated value") + } + restart := strings.Index(dokployShadowInstallScript, `sudo docker service update --force --update-order stop-first --detach=false dokploy`) + snapshot := strings.Index(dokployShadowInstallScript, `COPY (SELECT id, secret, backup_codes FROM public.two_factor ORDER BY id)`) + if restart < 0 || snapshot < 0 || restart >= snapshot { + t.Fatal("legacy authentication migration must quiesce and settle Dokploy before its authoritative two-factor snapshot") + } + escrow := strings.Index(dokployShadowInstallScript, `sync -f "$DOKPLOY_AUTH_SECRET_BACKUP"`) + verified := strings.Index(dokployShadowInstallScript, `cmp -s -- "$auth_secret_tmp" "$DOKPLOY_AUTH_SECRET_BACKUP"`) + secretCreated := strings.Index(dokployShadowInstallScript, `sudo docker secret create dokploy-auth-secret - < "$auth_secret_tmp"`) + migrated := strings.Index(dokployShadowInstallScript, `pnpm run migrate-auth-secret`) + if escrow < 0 || verified <= escrow || secretCreated <= verified || migrated <= secretCreated { + t.Fatalf("legacy authentication migration must sync and verify escrow before Docker secret creation and database migration: escrow=%d verified=%d created=%d migrated=%d", escrow, verified, secretCreated, migrated) + } + zeroReplicas := strings.Index(dokployShadowInstallScript, `--replicas 0`) + boundIdentity := strings.Index(dokployShadowInstallScript, `DOKPLOY_BOUND_AUTH_SECRET_IDS=`) + startTask := strings.Index(dokployShadowInstallScript, `docker service scale dokploy=1`) + if zeroReplicas < 0 || boundIdentity <= zeroReplicas || startTask <= boundIdentity { + t.Fatalf("Dokploy must bind the named secret at zero replicas, verify its ID, then start a task: zero=%d bound=%d start=%d", zeroReplicas, boundIdentity, startTask) + } + + functionsStart := strings.Index(dokployShadowInstallScript, "validate_auth_secret_backup() {") + functionsEnd := strings.Index(dokployShadowInstallScript, "\nDOKPLOY_AUTH_SECRET_EXISTS=false") + setupStart := strings.LastIndex(dokployShadowInstallScript, `if [ "$DOKPLOY_SERVICE_EXISTS" = false ]; then`) + setupEnd := strings.Index(dokployShadowInstallScript[setupStart:], `log "Starting Dokploy Postgres"`) + if functionsStart < 0 || functionsEnd < functionsStart || setupStart < 0 || setupEnd < 0 { + t.Fatal("missing fresh authentication-secret functions or setup block") + } + functions := dokployShadowInstallScript[functionsStart:functionsEnd] + setup := dokployShadowInstallScript[setupStart : setupStart+setupEnd] + dir := t.TempDir() + statePath := filepath.Join(dir, "secret") + backupPath := filepath.Join(dir, "escrow") + provenancePath := filepath.Join(dir, "provenance", "dokploy-auth-secret.id") + createCountPath := filepath.Join(dir, "create-count") + randomCountPath := filepath.Join(dir, "random-count") + const secret = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + const secretID = "abcdefghijklmnopqrstuvwxy" + script := `set -euo pipefail +DOKPLOY_SERVICE_EXISTS=false +STATE_PATH="$1" +AUTH_SECRET_BACKUP="$2" +AUTH_SECRET_BACKUP_DIR="$(dirname -- "$AUTH_SECRET_BACKUP")" +AUTH_SECRET_PROVENANCE="$3" +CREATE_COUNT_PATH="$4" +RANDOM_COUNT_PATH="$5" +FAIL_AFTER_CREATE="$6" +COLLIDE_ON_CREATE="$7" +AUTH_SECRET_INTENT_LABEL=io.bort.auth-secret-intent +mkdir -p "$AUTH_SECRET_BACKUP_DIR" +if [ ! -e "$AUTH_SECRET_BACKUP" ]; then + printf '%s\n' '` + secret + `' > "$AUTH_SECRET_BACKUP" + chmod 600 "$AUTH_SECRET_BACKUP" +fi +exec 3< "$AUTH_SECRET_BACKUP" +AUTH_SECRET_FD=3 +AUTH_SECRET_FD_PATH="/dev/fd/$AUTH_SECRET_FD" +AUTH_SECRET_DIGEST='` + strings.Repeat("a", 64) + `' +log() { printf 'step: %s\n' "$*"; } +sync() { :; } +stat() { + case "$2" in + %a) if [ "$3" = "$(dirname -- "$AUTH_SECRET_PROVENANCE")" ]; then printf '700\n'; else printf '600\n'; fi ;; + %u) id -u ;; + *) return 1 ;; + esac +} +sha256sum() { printf '%s %s\n' '` + strings.Repeat("a", 64) + `' "$2"; } +docker() { + if [ "$1 $2 $3" = "secret inspect dokploy_auth_secret" ]; then + if [ ! -f "$STATE_PATH" ]; then + return 1 + fi + if [ "${4:-}" = "--format" ]; then + case "${5:-}" in + *Annotations.Labels*) printf '%s\n' '0123456789abcdef0123456789abcdef' ;; + *) printf '%s\n' '` + secretID + `' ;; + esac + fi + return + fi + if [ "$1 $2 $3" = "secret create --label" ] && [ "$4" = "$AUTH_SECRET_INTENT_LABEL=0123456789abcdef0123456789abcdef" ] && [ "$5 $6" = "dokploy_auth_secret -" ]; then + cat >"$STATE_PATH" + if [ "$COLLIDE_ON_CREATE" = 1 ]; then + return 1 + fi + printf 'create\n' >>"$CREATE_COUNT_PATH" + if [ "$FAIL_AFTER_CREATE" = 1 ]; then + kill -9 $$ + fi + printf '%s\n' '` + secretID + `' + return + fi + printf 'unexpected docker arguments' >&2 + return 1 +} +openssl() { + [ "$1 $2" = "rand -hex" ] || return 1 + case "$3" in + 32) printf 'random-secret\n' >>"$RANDOM_COUNT_PATH"; printf '%s\n' '` + secret + `' ;; + 16) printf 'random-intent\n' >>"$RANDOM_COUNT_PATH"; printf '%s\n' '0123456789abcdef0123456789abcdef' ;; + *) return 1 ;; + esac +} +` + functions + ` +prepare_auth_secret_backup +` + setup + collisionDir := filepath.Join(dir, "collision") + if err := os.Mkdir(collisionDir, 0o700); err != nil { + t.Fatal(err) + } + collisionState := filepath.Join(collisionDir, "secret") + collisionBackup := filepath.Join(collisionDir, "escrow") + collisionProvenance := filepath.Join(collisionDir, "provenance", "dokploy-auth-secret.id") + collisionCreates := filepath.Join(collisionDir, "create-count") + collisionRandom := filepath.Join(collisionDir, "random-count") + cmd := exec.Command(bash, "-c", script, "test", collisionState, collisionBackup, collisionProvenance, collisionCreates, collisionRandom, "0", "1") + out, err := cmd.CombinedOutput() + if err == nil || !strings.Contains(string(out), "refusing to adopt any object that raced its creation") { + t.Fatalf("expected colliding authentication-secret creation refusal, got %v: %s", err, out) + } + collisionMarker, err := os.ReadFile(collisionProvenance) + if err != nil { + t.Fatalf("read collision provenance marker: %v", err) + } + if !strings.HasPrefix(string(collisionMarker), "pending:") || strings.HasPrefix(string(collisionMarker), secretID+" ") { + t.Fatalf("colliding Docker secret was finalized as Bort provenance: %q", collisionMarker) + } + + cmd = exec.Command(bash, "-c", script, "test", statePath, backupPath, provenancePath, createCountPath, randomCountPath, "1", "0") + out, err = cmd.CombinedOutput() + if err == nil { + t.Fatal("expected interruption after Docker secret creation") + } + pendingMarker, err := os.ReadFile(provenancePath) + if err != nil { + t.Fatalf("read pending provenance marker: %v", err) + } + if !strings.HasPrefix(string(pendingMarker), "pending:0123456789abcdef0123456789abcdef ") { + t.Fatalf("creation intent was not durable before Docker secret creation: %q", pendingMarker) + } + if err := os.WriteFile(provenancePath, []byte("pending:fedcba9876543210fedcba9876543210 "+strings.Repeat("a", 64)+"\n"), 0o600); err != nil { + t.Fatal(err) + } + cmd = exec.Command(bash, "-c", script, "test", statePath, backupPath, provenancePath, createCountPath, randomCountPath, "0", "0") + out, err = cmd.CombinedOutput() + if err == nil || !strings.Contains(string(out), "does not match the trusted Bort creation intent") { + t.Fatalf("expected mismatched pending creation intent refusal, got %v: %s", err, out) + } + if err := os.WriteFile(provenancePath, pendingMarker, 0o600); err != nil { + t.Fatal(err) + } + cmd = exec.Command(bash, "-c", script, "test", statePath, backupPath, provenancePath, createCountPath, randomCountPath, "0", "0") + out, err = cmd.CombinedOutput() + if err != nil { + t.Fatalf("authentication-secret retry failed: %v: %s", err, out) + } + stored, err := os.ReadFile(statePath) + if err != nil { + t.Fatalf("read stored secret: %v", err) + } + if strings.TrimSpace(string(stored)) != secret { + t.Fatalf("unexpected stored secret: %q", stored) + } + created, err := os.ReadFile(createCountPath) + if err != nil { + t.Fatalf("read create count: %v", err) + } + if string(created) != "create\n" { + t.Fatalf("expected one secret creation, got %q", created) + } + random, err := os.ReadFile(randomCountPath) + if err != nil { + t.Fatalf("read random count: %v", err) + } + if string(random) != "random-intent\n" { + t.Fatalf("expected one creation-intent generation, got %q", random) + } + marker, err := os.ReadFile(provenancePath) + if err != nil { + t.Fatalf("read provenance marker: %v", err) + } + if !strings.HasPrefix(string(marker), secretID+" ") { + t.Fatalf("provenance marker does not bind the Docker secret ID: %q", marker) + } + if strings.Contains(string(out), secret) { + t.Fatalf("authentication secret leaked to output: %q", out) + } + if err := os.WriteFile(provenancePath, []byte("zyxwvutsrqponmlkjihgfedcb "+strings.Repeat("a", 64)+"\n"), 0o600); err != nil { + t.Fatal(err) + } + cmd = exec.Command(bash, "-c", script, "test", statePath, backupPath, provenancePath, createCountPath, randomCountPath, "0", "0") + out, err = cmd.CombinedOutput() + if err == nil || !strings.Contains(string(out), "does not match the trusted Bort provenance marker") { + t.Fatalf("expected mismatched Docker secret identity refusal, got %v: %s", err, out) + } +} + +func TestDokployShadowInstallRecoveryValidatesRunningService(t *testing.T) { + bash, err := exec.LookPath("bash") + if err != nil { + t.Skip("bash is unavailable") + } + start := strings.Index(dokployShadowInstallScript, "validate_bort_install_service() {") + end := strings.Index(dokployShadowInstallScript[start:], "\n}\n\nDOKPLOY_AUTH_SECRET_ID=") + if start < 0 || end < 0 { + t.Fatal("missing Bort-created Dokploy service recovery validator") + } + validator := dokployShadowInstallScript[start : start+end+3] + recoveryStart := strings.Index(dokployShadowInstallScript, `elif ! DOKPLOY_SERVICE_REPLICAS=`) + recoveryEnd := strings.Index(dokployShadowInstallScript[recoveryStart:], `if [ "$DOKPLOY_STAGED_SERVICE" = true ]; then`) + if recoveryStart < 0 || recoveryEnd < 0 { + t.Fatal("missing existing Dokploy service recovery path") + } + recovery := dokployShadowInstallScript[recoveryStart : recoveryStart+recoveryEnd] + if !strings.Contains(recovery, "validate_bort_install_service \"$DOKPLOY_INSTALL_SERVICE_STATE\"\n\tif [ \"$DOKPLOY_SERVICE_REPLICAS\" = 0 ]; then") { + t.Fatal("running Dokploy service recovery does not validate the durable Bort install identity before the zero-replica-only task check") + } + + const secretID = "abcdefghijklmnopqrstuvwxy" + const requestedImage = "dokploy/dokploy:v0.30.7@sha256:62e4354a49ee686ea749d3541d93113d494501befe81f5226c7f859f201e3a7c" + wrapper := `set -euo pipefail +INSTALL_AUTH_SECRET_LABEL=io.bort.install-auth-secret-id +REQUESTED_DOKPLOY_IMAGE=` + requestedImage + ` +INSTALL_LABEL="$1" +CURRENT_SECRET_ID="$2" +SERVICE_IMAGE="$3" +DOKPLOY_AUTH_ENV="$4" +BOUND_SECRET_IDS="$5" +validate_auth_secret_provenance() { DOKPLOY_AUTH_SECRET_ID="$CURRENT_SECRET_ID"; } +docker() { + [ "$1 $2 $3 $4" = "service inspect dokploy --format" ] || return 1 + case "$5" in + *Annotations.Labels*) printf '%s\n' "$INSTALL_LABEL" ;; + *ContainerSpec.Image*) printf '%s\n' "$SERVICE_IMAGE" ;; + *ContainerSpec.Secrets*) printf '%s\n' "$BOUND_SECRET_IDS" ;; + *) return 1 ;; + esac +} +` + validator + ` +validate_bort_install_service running +` + run := func(label, currentID, image, authEnv, boundIDs string) ([]byte, error) { + return exec.Command(bash, "-c", wrapper, "test", label, currentID, image, authEnv, boundIDs).CombinedOutput() + } + validEnv := "BETTER_AUTH_SECRET_FILE=/run/secrets/dokploy_auth_secret" + if out, err := run(secretID, secretID, requestedImage, validEnv, secretID); err != nil { + t.Fatalf("valid running service recovery was refused: %v: %s", err, out) + } + for name, tc := range map[string]struct { + label, currentID, image, authEnv, boundIDs, want string + }{ + "removed label": {"", secretID, requestedImage, validEnv, secretID, "not labeled"}, + "changed label": {"zyxwvutsrqponmlkjihgfedcb", secretID, requestedImage, validEnv, secretID, "not bound"}, + "changed named secret": {secretID, "zyxwvutsrqponmlkjihgfedcb", requestedImage, validEnv, secretID, "not bound"}, + "bare requested tag": {secretID, secretID, "dokploy/dokploy:v0.30.7", validEnv, secretID, "immutable image"}, + "changed digest": {secretID, secretID, "dokploy/dokploy:v0.30.7@sha256:" + strings.Repeat("b", 64), validEnv, secretID, "immutable image"}, + "changed mount path": {secretID, secretID, requestedImage, "BETTER_AUTH_SECRET_FILE=/run/secrets/other", secretID, "exactly Bort's"}, + "duplicate mount env": {secretID, secretID, requestedImage, validEnv + "\n" + validEnv, secretID, "exactly Bort's"}, + "changed binding": {secretID, secretID, requestedImage, validEnv, "zyxwvutsrqponmlkjihgfedcb", "does not bind exactly"}, + } { + t.Run(name, func(t *testing.T) { + out, err := run(tc.label, tc.currentID, tc.image, tc.authEnv, tc.boundIDs) + if err == nil || !strings.Contains(string(out), tc.want) { + t.Fatalf("expected %q refusal, got %v: %s", tc.want, err, out) + } + }) + } +} + +func TestDokployShadowForeignServiceGuardExitsWithRefusalStatus(t *testing.T) { + bash, err := exec.LookPath("bash") + if err != nil { + t.Skip("bash is unavailable") + } + statusLine := fmt.Sprintf("\nFOREIGN_SERVICE_EXIT_STATUS=%d\n", dokployInstallForeignServiceExitStatus) + if !strings.Contains(dokployShadowInstallScript, statusLine) { + t.Fatalf("install script does not define FOREIGN_SERVICE_EXIT_STATUS=%d", dokployInstallForeignServiceExitStatus) + } + const guardStart = " if [ -z \"$DOKPLOY_INSTALL_LABEL\" ]; then\n" + const guardEnd = "\n fi\n" + start := strings.Index(dokployShadowInstallScript, guardStart) + if start < 0 { + t.Fatal("missing foreign-service guard") + } + if authInspect := strings.Index(dokployShadowInstallScript, "DOKPLOY_AUTH_ENV=\"$(docker service inspect"); authInspect < 0 || authInspect > start { + t.Fatal("authentication-secret refusal must run before the foreign-service guard so a legacy-secret service is not routed to the unchecked non-install bootstrap") + } + end := strings.Index(dokployShadowInstallScript[start:], guardEnd) + if end < 0 { + t.Fatal("unterminated foreign-service guard") + } + guard := dokployShadowInstallScript[start : start+end+len(guardEnd)] + binDir := t.TempDir() + if err := os.WriteFile(filepath.Join(binDir, "docker"), []byte("#!/bin/sh\necho 3000\n"), 0o755); err != nil { + t.Fatal(err) + } + cmd := exec.Command(bash, "-c", statusLine+"DOKPLOY_INSTALL_LABEL=\n"+guard+"echo guard fell through\nexit 0\n") + cmd.Env = append(os.Environ(), "PATH="+binDir+string(os.PathListSeparator)+os.Getenv("PATH")) + out, err := cmd.CombinedOutput() + var exitErr *exec.ExitError + if !errors.As(err, &exitErr) || exitErr.ExitCode() != dokployInstallForeignServiceExitStatus { + t.Fatalf("expected exit status %d from the foreign-service guard, got %v: %s", dokployInstallForeignServiceExitStatus, err, out) + } + if !strings.Contains(string(out), "was not installed by Bort") || !strings.Contains(string(out), "http://127.0.0.1:3000") { + t.Fatalf("unexpected guard output: %s", out) + } +} + func TestDokployShadowEndpointMode(t *testing.T) { bash, err := exec.LookPath("bash") if err != nil { @@ -699,8 +1755,15 @@ func TestDokployShadowEndpointMode(t *testing.T) { }) } for _, service := range []string{"dokploy-postgres", "dokploy-redis", "dokploy"} { - want := "--name " + service + " \\\n --endpoint-mode \"$ENDPOINT_MODE\"" - if !strings.Contains(dokployShadowInstallScript, want) { + start := strings.Index(dokployShadowInstallScript, "--name "+service+" \\") + if start < 0 { + t.Fatalf("%s service create block not found", service) + } + block := dokployShadowInstallScript[start:] + if end := strings.Index(block, ">/dev/null"); end >= 0 { + block = block[:end] + } + if !strings.Contains(block, `--endpoint-mode "$ENDPOINT_MODE"`) { t.Errorf("%s must use the selected endpoint mode", service) } } @@ -709,6 +1772,146 @@ func TestDokployShadowEndpointMode(t *testing.T) { } } +func TestLegacyAuthSecretEscrowPreparation(t *testing.T) { + bash, err := exec.LookPath("bash") + if err != nil { + t.Skip("bash is unavailable") + } + start := strings.Index(dokployShadowInstallScript, `: "${DOKPLOY_AUTH_SECRET_BACKUP:?`) + if start < 0 { + t.Fatal("missing legacy authentication-secret escrow block") + } + endNeedle := `sudo docker secret create dokploy-auth-secret - < "$auth_secret_tmp"` + end := strings.Index(dokployShadowInstallScript[start:], endNeedle) + if end < 0 { + t.Fatal("missing legacy authentication-secret escrow block") + } + block := dokployShadowInstallScript[start : start+end+len(endNeedle)] + dir := t.TempDir() + backup := filepath.Join(dir, "off-host", "dokploy-auth-secret") + twoFactorBefore := filepath.Join(dir, "off-host", "dokploy-two-factor-before.csv") + if err := os.Mkdir(filepath.Dir(backup), 0o700); err != nil { + t.Fatal(err) + } + run := func(capture, databaseEnv string) ([]byte, error) { + wrapper := `set -euo pipefail +DOKPLOY_AUTH_SECRET_BACKUP="$1" +CAPTURE="$2" +TMPDIR="$3" +DOKPLOY_TWO_FACTOR_BEFORE="$4" +DATABASE_ENV="$5" +sync() { :; } +stat() { + [ "$1 $2" = "-c %a" ] || return 1 + printf '600\n' +} +shred() { + local last="${!#}" + printf 'destroyed\n' > "$last" + rm -f -- "$last" +} +sudo() { + if [ "$1 $2 $3" = "docker service inspect" ]; then + printf '%s\n' "$DATABASE_ENV" + return + fi + if [ "$1 $2 $3" = "docker service update" ]; then + return + fi + if [ "$1 $2" = "docker ps" ]; then + printf 'postgres-container\n' + return + fi + if [ "$1 $2 $3" = "docker exec postgres-container" ]; then + if [[ "$*" == *pg_stat_activity* ]]; then + printf '0\n' + return + fi + printf 'id,secret,backup_codes\n' + return + fi + if [ "$1 $2 $3 $4" = "docker secret create dokploy-auth-secret" ]; then + cat > "$CAPTURE" + return + fi + echo "unexpected sudo command: $*" >&2 + return 1 +} +` + block + cmd := exec.Command(bash, "-c", wrapper, "test", backup, capture, dir, twoFactorBefore, databaseEnv) + return cmd.CombinedOutput() + } + + for name, databaseEnv := range map[string]string{ + "database URL": "DATABASE_URL=postgresql://external.example/dokploy", + "host": "POSTGRES_HOST=external-db", + "port": "POSTGRES_PORT=6543", + "user": "POSTGRES_USER=operator", + "database": "POSTGRES_DB=other", + "duplicate host": "POSTGRES_HOST=dokploy-postgres\nPOSTGRES_HOST=external-db", + } { + t.Run(name, func(t *testing.T) { + externalCapture := filepath.Join(dir, "external-"+strings.ReplaceAll(name, " ", "-")) + if out, err := run(externalCapture, databaseEnv); err == nil || !strings.Contains(string(out), "refusing authentication-secret migration") && !strings.Contains(string(out), "database-operator procedure") { + t.Fatalf("expected custom database refusal, got %v: %s", err, out) + } + if _, err := os.Stat(backup); !os.IsNotExist(err) { + t.Fatalf("custom database refusal mutated the escrow: %v", err) + } + }) + } + + firstCapture := filepath.Join(dir, "first") + defaultDatabaseEnv := "POSTGRES_HOST=dokploy-postgres\nPOSTGRES_PORT=5432\nPOSTGRES_USER=dokploy\nPOSTGRES_DB=dokploy" + if out, err := run(firstCapture, defaultDatabaseEnv); err != nil { + t.Fatalf("initial escrow preparation failed: %v: %s", err, out) + } + escrowed, err := os.ReadFile(backup) + if err != nil { + t.Fatal(err) + } + created, err := os.ReadFile(firstCapture) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(escrowed, created) || len(escrowed) != 65 || escrowed[64] != '\n' { + t.Fatalf("Docker secret input differs from escrow: escrow=%q input=%q", escrowed, created) + } + info, err := os.Stat(backup) + if err != nil { + t.Fatal(err) + } + if info.Mode().Perm() != 0o600 { + t.Fatalf("escrow mode=%#o, want 0600", info.Mode().Perm()) + } + twoFactorSnapshot, err := os.ReadFile(twoFactorBefore) + if err != nil { + t.Fatalf("read retained two-factor snapshot: %v", err) + } + if string(twoFactorSnapshot) != "id,secret,backup_codes\n" { + t.Fatalf("retained two-factor snapshot was changed through its temporary hard link: %q", twoFactorSnapshot) + } + + secondCapture := filepath.Join(dir, "second") + if out, err := run(secondCapture, defaultDatabaseEnv); err != nil { + t.Fatalf("retained escrow reuse failed: %v: %s", err, out) + } + reused, err := os.ReadFile(secondCapture) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(escrowed, reused) { + t.Fatal("retry did not reuse the retained escrow") + } + temporaryFiles, err := filepath.Glob(filepath.Join(dir, "dokploy-auth-secret.*")) + if err != nil { + t.Fatal(err) + } + if len(temporaryFiles) != 0 { + t.Fatalf("temporary secret files were retained: %v", temporaryFiles) + } +} + func TestDokployShadowEndpointPreflight(t *testing.T) { bash, err := exec.LookPath("bash") if err != nil { @@ -721,41 +1924,184 @@ func TestDokployShadowEndpointPreflight(t *testing.T) { t.Fatal("missing endpoint preflight boundaries") } preflight := dokployShadowInstallScript[:configStart] + "\nkernel_config=\"$2\"" + dokployShadowInstallScript[configEnd:end] + authBackup := filepath.Join(t.TempDir(), "dokploy-auth-secret") stub := ` ENDPOINT_MODE="$1" ACME_EMAIL=admin@dokploy.local +DOKPLOY_VERSION=v0.30.7@sha256:62e4354a49ee686ea749d3541d93113d494501befe81f5226c7f859f201e3a7c +printf '%064d\n' 0 > "$DOKPLOY_AUTH_SECRET_BACKUP" +chmod 600 "$DOKPLOY_AUTH_SECRET_BACKUP" +exec 3< "$DOKPLOY_AUTH_SECRET_BACKUP" +DOKPLOY_AUTH_SECRET_FD=3 +DOKPLOY_AUTH_SECRET_DIGEST="$(printf '%064d' 0)" id() { echo 0; } python3() { return 0; } +sync() { :; } +stat() { + case "$2" in + %a) printf '600\n' ;; + %u) printf '0\n' ;; + *) return 1 ;; + esac +} +sha256sum() { printf '%064d %s\n' 0 "$2"; } docker() { - if [ "$1 $2" != "service inspect" ]; then - echo unexpected-docker-command + if [ "$1 $2" = "info --format" ]; then + printf '%s\n' "$SWARM_STATUS" + return + fi + if [ "$1 $2" = "node ls" ]; then + for ((i=0; i&2 + return 1 + fi + if [ "$mode" = error ]; then + echo 'docker daemon unavailable' >&2 + return 1 + fi + if [ "$mode" = not-manager ]; then + echo 'This node is not a swarm manager.' >&2 + return 1 + fi + if [[ "${5:-}" == *ContainerSpec.Env* ]]; then + printf '%s\n' "$DOKPLOY_AUTH" + return + fi + if [[ "${5:-}" == *ContainerSpec.Image* ]]; then + printf '%s\n' "$DOKPLOY_IMAGE" + return + fi + printf '%s\n' "$mode" + return + fi + if [ "$1 $2 $3" = "secret inspect dokploy_auth_secret" ]; then + if [ "$AUTH_SECRET_MODE" = present ]; then + if [ "${4:-}" = "--format" ]; then + printf 'abcdefghijklmnopqrstuvwxy\n' + fi + return + fi + if [ "$AUTH_SECRET_MODE" = error ]; then + echo 'docker daemon unavailable' >&2 + return 1 + fi + if [ "$AUTH_SECRET_MODE" = not-manager ]; then + echo 'This node is not a swarm manager.' >&2 + return 1 + fi + echo 'no such secret' >&2 return 1 fi - case "$3" in - dokploy-postgres) mode="$POSTGRES_MODE" ;; - dokploy-redis) mode="$REDIS_MODE" ;; - dokploy) mode="$DOKPLOY_MODE" ;; - *) return 1 ;; - esac - [ "$mode" != absent ] || return 1 - printf '%s\n' "$mode" + if [ "$1 $2 $3" = "volume inspect dokploy-postgres" ]; then + if [ "$POSTGRES_VOLUME_MODE" = present ]; then + return + fi + if [ "$POSTGRES_VOLUME_MODE" = error ]; then + echo 'docker daemon unavailable' >&2 + return 1 + fi + echo 'no such volume' >&2 + return 1 + fi + echo unexpected-docker-command + return 1 } POSTGRES_MODE="$3" REDIS_MODE="$4" DOKPLOY_MODE="$5" +DOKPLOY_AUTH="$6" +DOKPLOY_IMAGE="$7" +AUTH_SECRET_MODE="$8" +POSTGRES_VOLUME_MODE="$9" +SWARM_STATUS="${10}" +MANAGER_COUNT="${11:-1}" ` for _, tc := range []struct { - name, mode, config, postgres, redis, dokploy, wantError string + name, mode, config, postgres, redis, dokploy, auth, wantError string + }{ + {"matching file secret", "dnsrr", "", "dnsrr", "dnsrr", "dnsrr", "BETTER_AUTH_SECRET_FILE=/run/secrets/dokploy_auth_secret", ""}, + {"matching environment secret", "dnsrr", "", "dnsrr", "dnsrr", "dnsrr", "BETTER_AUTH_SECRET=0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", ""}, + {"duplicate environment secret", "dnsrr", "", "dnsrr", "dnsrr", "dnsrr", "BETTER_AUTH_SECRET=0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\nBETTER_AUTH_SECRET=0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", "duplicate BETTER_AUTH_SECRET entries"}, + {"custom and legacy environment secrets", "dnsrr", "", "dnsrr", "dnsrr", "dnsrr", "BETTER_AUTH_SECRET=0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef\nBETTER_AUTH_SECRET=better-auth-secret-123456789", "duplicate BETTER_AUTH_SECRET entries"}, + {"published legacy environment secret", "dnsrr", "", "dnsrr", "dnsrr", "dnsrr", "BETTER_AUTH_SECRET=better-auth-secret-123456789", "published legacy authentication secret"}, + {"short environment secret", "dnsrr", "", "dnsrr", "dnsrr", "dnsrr", "BETTER_AUTH_SECRET=too-short", "shorter than Better Auth's 32-character minimum"}, + {"absent", "dnsrr", "", "absent", "absent", "absent", "", ""}, + {"inactive fresh host", "dnsrr", "", "not-manager", "not-manager", "not-manager", "", ""}, + {"missing auth secret", "dnsrr", "", "dnsrr", "dnsrr", "dnsrr", "", "existing Dokploy service has no authentication secret"}, + {"first conflict", "dnsrr", "", "vip", "dnsrr", "dnsrr", "BETTER_AUTH_SECRET=configured", "existing service dokploy-postgres uses endpoint mode vip"}, + {"later conflict", "dnsrr", "", "dnsrr", "vip", "dnsrr", "BETTER_AUTH_SECRET=configured", "existing service dokploy-redis uses endpoint mode vip"}, + {"last conflict", "dnsrr", "", "dnsrr", "dnsrr", "vip", "BETTER_AUTH_SECRET=configured", "existing service dokploy uses endpoint mode vip"}, + {"disabled VIP", "vip", "# CONFIG_IP_VS is not set", "absent", "absent", "absent", "", "requires kernel IPVS"}, + } { + t.Run(tc.name, func(t *testing.T) { + image := "dokploy/dokploy:v0.30.7" + authSecret := "present" + if tc.dokploy == "absent" { + image = "" + } + if tc.name == "absent" { + authSecret = "absent" + } + swarmStatus := "active true" + if tc.postgres == "absent" && tc.redis == "absent" && tc.dokploy == "absent" { + swarmStatus = "inactive false" + } + if tc.name == "inactive fresh host" { + authSecret = "not-manager" + swarmStatus = "inactive false" + } + cmd := exec.Command(bash, "-c", stub+preflight+"\necho preflight-complete", "test", tc.mode, tc.config, tc.postgres, tc.redis, tc.dokploy, tc.auth, image, authSecret, "absent", swarmStatus) + cmd.Env = append(os.Environ(), "DOKPLOY_AUTH_SECRET_BACKUP="+authBackup) + out, err := cmd.CombinedOutput() + if tc.wantError == "" { + if err != nil || !strings.Contains(string(out), "preflight-complete") { + t.Fatalf("preflight did not continue: %v: %s", err, out) + } + } else if err == nil || !strings.Contains(string(out), tc.wantError) || strings.Contains(string(out), "preflight-complete") { + t.Fatalf("wanted refusal %q before continuation, got %v: %s", tc.wantError, err, out) + } + }) + } + for _, tc := range []struct { + name, postgres, dokploy, auth, image, authSecret, postgresVolume, wantError string }{ - {"matching", "dnsrr", "", "dnsrr", "dnsrr", "dnsrr", ""}, - {"absent", "dnsrr", "", "absent", "absent", "absent", ""}, - {"first conflict", "dnsrr", "", "vip", "dnsrr", "dnsrr", "existing service dokploy-postgres uses endpoint mode vip"}, - {"later conflict", "dnsrr", "", "dnsrr", "vip", "dnsrr", "existing service dokploy-redis uses endpoint mode vip"}, - {"last conflict", "dnsrr", "", "dnsrr", "dnsrr", "vip", "existing service dokploy uses endpoint mode vip"}, - {"disabled VIP", "vip", "# CONFIG_IP_VS is not set", "absent", "absent", "absent", "requires kernel IPVS"}, + {"minimum file-secret image", "dnsrr", "dnsrr", "BETTER_AUTH_SECRET_FILE=/run/secrets/dokploy_auth_secret", "docker.io/dokploy/dokploy:v0.29.3", "present", "present", ""}, + {"digest-qualified file-secret image", "dnsrr", "dnsrr", "BETTER_AUTH_SECRET_FILE=/run/secrets/dokploy_auth_secret", "dokploy/dokploy:v0.29.3@sha256:0123456789abcdef", "present", "present", ""}, + {"floating latest file-secret image", "dnsrr", "dnsrr", "BETTER_AUTH_SECRET_FILE=/run/secrets/dokploy_auth_secret", "dokploy/dokploy:latest@sha256:0123456789abcdef", "present", "present", "cannot verify file-backed authentication-secret support"}, + {"legacy file-secret image", "dnsrr", "dnsrr", "BETTER_AUTH_SECRET_FILE=/run/secrets/dokploy_auth_secret", "dokploy/dokploy:v0.29.2", "present", "present", "cannot verify file-backed authentication-secret support"}, + {"prerelease minimum file-secret image", "dnsrr", "dnsrr", "BETTER_AUTH_SECRET_FILE=/run/secrets/dokploy_auth_secret", "dokploy/dokploy:v0.29.3-rc.1", "present", "present", "cannot verify file-backed authentication-secret support"}, + {"unknown file-secret image", "dnsrr", "dnsrr", "BETTER_AUTH_SECRET_FILE=/run/secrets/dokploy_auth_secret", "dokploy/dokploy:feature", "present", "present", "cannot verify file-backed authentication-secret support"}, + {"partial database service without secret", "dnsrr", "absent", "", "", "absent", "absent", "existing Dokploy database state has no reusable dokploy_auth_secret"}, + {"partial database volume without secret", "absent", "absent", "", "", "absent", "present", "existing Dokploy database state has no reusable dokploy_auth_secret"}, + {"partial database with unprovenanced secret", "dnsrr", "absent", "", "", "present", "present", "no trusted Bort provenance marker"}, + {"database service inspection failure", "error", "absent", "", "", "absent", "absent", "could not determine whether Docker service dokploy-postgres exists"}, + {"database volume inspection failure", "absent", "absent", "", "", "absent", "error", "could not determine whether Docker volume dokploy-postgres exists"}, + {"Dokploy service inspection failure", "absent", "error", "", "", "absent", "absent", "could not determine whether Docker service dokploy exists"}, + {"auth-secret inspection failure", "absent", "absent", "", "", "error", "absent", "could not determine whether Docker secret dokploy_auth_secret exists"}, + {"active swarm worker", "absent", "absent", "", "", "absent", "absent", "active Docker swarm worker, not a manager"}, } { t.Run(tc.name, func(t *testing.T) { - cmd := exec.Command(bash, "-c", stub+preflight+"\necho preflight-complete", "test", tc.mode, tc.config, tc.postgres, tc.redis, tc.dokploy) + swarmStatus := "active true" + if tc.postgres == "absent" && tc.dokploy == "absent" { + swarmStatus = "inactive false" + } + if tc.name == "active swarm worker" { + swarmStatus = "active false" + } + cmd := exec.Command(bash, "-c", stub+preflight+"\necho preflight-complete", "test", "dnsrr", "", tc.postgres, "absent", tc.dokploy, tc.auth, tc.image, tc.authSecret, tc.postgresVolume, swarmStatus) + cmd.Env = append(os.Environ(), "DOKPLOY_AUTH_SECRET_BACKUP="+authBackup) out, err := cmd.CombinedOutput() if tc.wantError == "" { if err != nil || !strings.Contains(string(out), "preflight-complete") { @@ -766,6 +2112,15 @@ DOKPLOY_MODE="$5" } }) } + t.Run("multi-manager fresh state is ambiguous", func(t *testing.T) { + cmd := exec.Command(bash, "-c", stub+preflight+"\necho preflight-complete", "test", "dnsrr", "", "absent", "absent", "absent", "", "", "absent", "absent", "active true", "2") + cmd.Env = append(os.Environ(), "DOKPLOY_AUTH_SECRET_BACKUP="+authBackup) + out, err := cmd.CombinedOutput() + want := "automatic fresh install is unsupported unless the original Bort-managed dokploy_auth_secret and /var/lib/bort/dokploy-auth-secret.id provenance marker are restored together" + if err == nil || !strings.Contains(string(out), want) || strings.Contains(string(out), "preflight-complete") { + t.Fatalf("wanted refusal %q before continuation, got %v: %s", want, err, out) + } + }) } func TestValidateDokployBootstrapURLPolicy(t *testing.T) { @@ -775,9 +2130,27 @@ func TestValidateDokployBootstrapURLPolicy(t *testing.T) { } }) - t.Run("accepts remote https", func(t *testing.T) { - if err := validateDokployBootstrapURL("https://dokploy.example.com"); err != nil { - t.Fatalf("expected https URL to be accepted: %v", err) + t.Run("rejects remote https for same-VPS bootstrap", func(t *testing.T) { + if err := validateDokployBootstrapURL("https://dokploy.example.com"); err == nil || !strings.Contains(err.Error(), "non-loopback") { + t.Fatalf("expected remote bootstrap URL to be rejected: %v", err) + } + }) + + t.Run("rejects loopback https", func(t *testing.T) { + if err := validateDokployBootstrapURL("https://localhost:3030"); err == nil || !strings.Contains(err.Error(), "direct loopback HTTP endpoint") { + t.Fatalf("expected loopback HTTPS to be rejected: %v", err) + } + }) + + t.Run("rejects loopback proxy path", func(t *testing.T) { + if err := validateDokployBootstrapURL("http://127.0.0.1:3030/dokploy"); err == nil || !strings.Contains(err.Error(), "root of the direct loopback HTTP endpoint") { + t.Fatalf("expected loopback proxy path to be rejected: %v", err) + } + }) + + t.Run("requires the published service port", func(t *testing.T) { + if err := validateDokployBootstrapURL("http://127.0.0.1"); err == nil || !strings.Contains(err.Error(), "service port") { + t.Fatalf("expected missing local service port to be rejected: %v", err) } }) @@ -973,3 +2346,11 @@ func TestStageCoolifyPgpassCpFailureCleansContainer(t *testing.T) { t.Fatalf("expected container rm -f after failed cp, got %v", commands) } } + +func requireNoDokployInstallationRecovery(t *testing.T) { + t.Helper() + t.Cleanup(func() { _ = clearDokployInstallationRecoveryRequired() }) + if blocked, err := dokployInstallationRecoveryRequired(); err != nil || blocked { + t.Fatalf("blocked init-target left installation recovery required: blocked=%t err=%v", blocked, err) + } +} diff --git a/internal/cli/installer_process_unix.go b/internal/cli/installer_process_unix.go new file mode 100644 index 0000000..b711b89 --- /dev/null +++ b/internal/cli/installer_process_unix.go @@ -0,0 +1,196 @@ +//go:build !windows + +package cli + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "os" + "os/exec" + "os/signal" + "path/filepath" + "strings" + "syscall" + "time" + + "github.com/aikins01/bort/internal/safepath" +) + +func dokployInstallerSignalContext(parent context.Context) (context.Context, context.CancelFunc) { + return signal.NotifyContext(parent, os.Interrupt, syscall.SIGTERM) +} + +func configureDokployInstallerCommand(cmd *exec.Cmd, operationLock, authSecret *os.File) int { + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + nextFD := 3 + if operationLock != nil { + cmd.ExtraFiles = append(cmd.ExtraFiles, operationLock) + nextFD++ + } + authSecretFD := 0 + if authSecret != nil { + cmd.ExtraFiles = append(cmd.ExtraFiles, authSecret) + authSecretFD = nextFD + } + cmd.Cancel = func() error { + if cmd.Process == nil { + return os.ErrProcessDone + } + if err := syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL); err != nil { + if errors.Is(err, syscall.ESRCH) { + return os.ErrProcessDone + } + return err + } + return nil + } + cmd.WaitDelay = 5 * time.Second + return authSecretFD +} + +func prepareDokployAuthSecretEscrow(path string) (*os.File, string, error) { + dir, err := safepath.OpenExistingPrivateDirNoFollow(filepath.Dir(path)) + if err != nil { + return nil, "", fmt.Errorf("open authentication-secret escrow directory: %w", err) + } + defer dir.Close() + name := filepath.Base(path) + file, err := dir.OpenFile(name) + if errors.Is(err, os.ErrNotExist) { + var secret [32]byte + if _, err := rand.Read(secret[:]); err != nil { + return nil, "", fmt.Errorf("generate authentication-secret escrow: %w", err) + } + contents := append([]byte(hex.EncodeToString(secret[:])), '\n') + if err := dir.WriteFileAtomicNew(name, contents, 0o600); err != nil { + return nil, "", fmt.Errorf("create authentication-secret escrow: %w", err) + } + file, err = dir.OpenFile(name) + } + if err != nil { + return nil, "", fmt.Errorf("open authentication-secret escrow: %w", err) + } + valid := false + defer func() { + if !valid { + _ = file.Close() + } + }() + before, err := file.Stat() + if err != nil { + return nil, "", err + } + stat, ok := before.Sys().(*syscall.Stat_t) + if !ok || !before.Mode().IsRegular() || before.Mode().Perm() != 0o600 || stat.Uid != uint32(os.Geteuid()) || stat.Nlink != 1 { + return nil, "", fmt.Errorf("authentication-secret escrow must be a single-link mode 0600 regular file owned by uid %d", os.Geteuid()) + } + if before.Size() != 65 { + return nil, "", fmt.Errorf("authentication-secret escrow is %d bytes, not the 65 bytes of one 64-lowercase-hex key and newline", before.Size()) + } + contents, err := io.ReadAll(io.LimitReader(file, 66)) + if err != nil { + return nil, "", err + } + if len(contents) != 65 || contents[64] != '\n' || strings.ToLower(string(contents[:64])) != string(contents[:64]) { + return nil, "", fmt.Errorf("authentication-secret escrow must contain exactly one 64-lowercase-hex key and newline") + } + if decoded, err := hex.DecodeString(string(contents[:64])); err != nil || len(decoded) != 32 { + return nil, "", fmt.Errorf("authentication-secret escrow must contain exactly one 64-lowercase-hex key and newline") + } + after, err := file.Stat() + if err != nil { + return nil, "", err + } + if !os.SameFile(before, after) || before.Size() != after.Size() || before.Mode() != after.Mode() || !before.ModTime().Equal(after.ModTime()) { + return nil, "", fmt.Errorf("authentication-secret escrow changed during validation") + } + if err := dir.ValidatePath(); err != nil { + return nil, "", err + } + snapshot, writer, err := os.Pipe() + if err != nil { + return nil, "", err + } + written, writeErr := writer.Write(contents) + closeErr := writer.Close() + if writeErr != nil || written != len(contents) || closeErr != nil { + _ = snapshot.Close() + var shortWriteErr error + if written != len(contents) { + shortWriteErr = io.ErrShortWrite + } + return nil, "", errors.Join(writeErr, closeErr, shortWriteErr) + } + digest := sha256.Sum256(contents) + if err := file.Close(); err != nil { + _ = snapshot.Close() + return nil, "", err + } + valid = true + return snapshot, hex.EncodeToString(digest[:]), nil +} + +func verifyDokployAuthSecretEscrow(path, expectedDigest string) error { + dir, err := safepath.OpenExistingPrivateDirNoFollow(filepath.Dir(path)) + if err != nil { + return fmt.Errorf("reopen authentication-secret escrow directory after installation: %w", err) + } + defer dir.Close() + name := filepath.Base(path) + file, err := dir.OpenFile(name) + if err != nil { + return fmt.Errorf("reopen authentication-secret escrow after installation: %w", err) + } + defer file.Close() + before, err := file.Stat() + if err != nil { + return err + } + stat, ok := before.Sys().(*syscall.Stat_t) + if !ok || !before.Mode().IsRegular() || before.Mode().Perm() != 0o600 || stat.Uid != uint32(os.Geteuid()) || stat.Nlink != 1 || before.Size() != 65 { + return fmt.Errorf("authentication-secret escrow security properties changed during installation; installation recovery remains required") + } + contents, err := io.ReadAll(io.LimitReader(file, 66)) + if err != nil { + return err + } + if len(contents) != 65 { + return fmt.Errorf("authentication-secret escrow changed during post-install verification; installation recovery remains required") + } + after, err := file.Stat() + if err != nil { + return err + } + if !os.SameFile(before, after) || before.Size() != after.Size() || before.Mode() != after.Mode() || !before.ModTime().Equal(after.ModTime()) { + return fmt.Errorf("authentication-secret escrow changed during post-install verification; installation recovery remains required") + } + if err := dir.ValidatePath(); err != nil { + return err + } + current, err := dir.OpenFile(name) + if err != nil { + return fmt.Errorf("rebind authentication-secret escrow path after installation: %w", err) + } + defer current.Close() + currentInfo, err := current.Stat() + if err != nil { + return err + } + if !os.SameFile(after, currentInfo) { + return fmt.Errorf("authentication-secret escrow path changed during installation; installation recovery remains required") + } + currentStat, ok := currentInfo.Sys().(*syscall.Stat_t) + if !ok || currentInfo.Mode().Perm() != 0o600 || currentStat.Uid != uint32(os.Geteuid()) || currentStat.Nlink != 1 { + return fmt.Errorf("authentication-secret escrow security properties changed during installation; installation recovery remains required") + } + digest := sha256.Sum256(contents) + if hex.EncodeToString(digest[:]) != expectedDigest { + return fmt.Errorf("authentication-secret escrow changed during installation; installation recovery remains required") + } + return nil +} diff --git a/internal/cli/installer_process_unix_test.go b/internal/cli/installer_process_unix_test.go new file mode 100644 index 0000000..18298e5 --- /dev/null +++ b/internal/cli/installer_process_unix_test.go @@ -0,0 +1,484 @@ +//go:build !windows + +package cli + +import ( + "bufio" + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "io" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "syscall" + "testing" + "time" + + "github.com/aikins01/bort/internal/target/dokploy" +) + +func TestInstallerCancellationKillsDescendantProcessGroup(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cmd := exec.CommandContext(ctx, "sh", "-c", "sleep 30 & echo $!; wait") + configureDokployInstallerCommand(cmd, nil, nil) + stdout, err := cmd.StdoutPipe() + if err != nil { + t.Fatal(err) + } + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + defer syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) + scanner := bufio.NewScanner(stdout) + if !scanner.Scan() { + t.Fatalf("read installer child pid: %v", scanner.Err()) + } + childPID, err := strconv.Atoi(strings.TrimSpace(scanner.Text())) + if err != nil { + t.Fatal(err) + } + cancel() + done := make(chan error, 1) + go func() { done <- cmd.Wait() }() + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("installer process group did not stop after cancellation") + } + deadline := time.Now().Add(2 * time.Second) + for { + err := syscall.Kill(childPID, 0) + if errors.Is(err, syscall.ESRCH) { + break + } + if time.Now().After(deadline) { + t.Fatalf("installer descendant %d remained after cancellation: %v", childPID, err) + } + time.Sleep(10 * time.Millisecond) + } +} + +func TestInstallerDescendantRetainsHostLockAfterAbruptParentExit(t *testing.T) { + lockPath := filepath.Join(t.TempDir(), "host.lock") + lock, err := acquireApplyLock(lockPath) + if err != nil { + t.Fatal(err) + } + pidPath := filepath.Join(t.TempDir(), "child.pid") + cmd := exec.CommandContext(context.Background(), "sh", "-c", `sleep 30 >/dev/null 2>&1 & echo $! > "$1"`, "test", pidPath) + configureDokployInstallerCommand(cmd, lock.file, nil) + if err := cmd.Run(); err != nil { + lock.Release() + t.Fatal(err) + } + pidBytes, err := os.ReadFile(pidPath) + if err != nil { + lock.Release() + t.Fatal(err) + } + childPID, err := strconv.Atoi(strings.TrimSpace(string(pidBytes))) + if err != nil { + lock.Release() + t.Fatal(err) + } + if err := lock.file.Close(); err != nil { + t.Fatal(err) + } + lock.file = nil + defer syscall.Kill(childPID, syscall.SIGKILL) + active, err := applyLockActive(lockPath) + if err != nil { + t.Fatal(err) + } + if !active { + t.Fatal("installer descendant did not retain the host lock after the parent descriptor closed") + } + if err := syscall.Kill(childPID, syscall.SIGKILL); err != nil && !errors.Is(err, syscall.ESRCH) { + t.Fatal(err) + } + deadline := time.Now().Add(2 * time.Second) + for { + active, err = applyLockActive(lockPath) + if err != nil { + t.Fatal(err) + } + if !active { + break + } + if time.Now().After(deadline) { + t.Fatal("host lock remained held after the installer descendant exited") + } + time.Sleep(10 * time.Millisecond) + } +} + +func TestInitTargetInstallTimeoutBlocksHostMutationsUntilRecovery(t *testing.T) { + resetDokployTrafficOwner(t) + t.Cleanup(func() { _ = clearDokployInstallationRecoveryRequired() }) + t.Setenv("SUDO_UID", "501") + t.Setenv(envDokployAuthBackup, filepath.Join(t.TempDir(), "dokploy-auth-secret")) + installer := &blockingDokployInstaller{} + deps := initTargetDeps{ + lister: &fakeAdminLister{admins: []coolifyAdmin{ + {Email: "admin@example.com", Name: "Admin User", PasswordHash: bcryptCoolifyHash(t, "right-password")}, + }}, + installer: installer, + installTimeout: 10 * time.Millisecond, + newClient: func(string) *dokploy.Client { + t.Fatal("Dokploy client was created after installer timeout") + return nil + }, + statePath: filepath.Join(t.TempDir(), "state.json"), + } + t.Setenv(envCoolifyAdminPwd, "right-password") + err := runInitTargetWith(context.Background(), []string{ + "--install", + "--coolify-email", "admin@example.com", + "--dokploy-url", "http://127.0.0.1:3030", + "--name", "Recovery Admin", + "--api-key-name", "recovery key", + }, strings.NewReader(""), io.Discard, io.Discard, deps) + if err == nil || !strings.Contains(err.Error(), "install dokploy timed out after 10ms") || !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("expected bounded installer timeout, got %v", err) + } + initErr := err + if installer.calls != 1 { + t.Fatalf("installer calls=%d, want 1", installer.calls) + } + lock, lockErr := acquireDokployLiveOperationLock() + if lock != nil { + lock.Release() + } + recoveryCommand, found, err := dokployInstallationRecoveryCommand() + if err != nil || !found || !strings.Contains(recoveryCommand, "--install-port 3030") { + t.Fatalf("recovery command was not recorded: found=%t err=%v command=%q", found, err, recoveryCommand) + } + if !strings.Contains(initErr.Error(), "run `"+recoveryCommand+"` to reconcile it") { + t.Fatalf("installer timeout error did not name the recovery command: %v", initErr) + } + if !errors.Is(lockErr, errDokployInstallRecoveryRequired) || !strings.Contains(lockErr.Error(), "run `"+recoveryCommand+"`") { + t.Fatalf("installer timeout did not block later host mutations with the recovery command: %v", lockErr) + } + if active, err := dokployLiveOperationActive(); active || !errors.Is(err, errDokployInstallRecoveryRequired) || !strings.Contains(err.Error(), "run `"+recoveryCommand+"`") { + t.Fatalf("installer recovery state was not visible after timeout: active=%t err=%v", active, err) + } + t.Chdir(t.TempDir()) + for _, test := range []struct { + name string + run func(*strings.Builder) error + }{ + {name: "guide", run: func(output *strings.Builder) error { + return runGuide(context.Background(), strings.NewReader(""), output, io.Discard) + }}, + {name: "status", run: func(output *strings.Builder) error { + return runStatus(context.Background(), nil, output, io.Discard) + }}, + {name: "next", run: func(output *strings.Builder) error { + return runNext(context.Background(), nil, output, io.Discard) + }}, + } { + t.Run(test.name, func(t *testing.T) { + var output strings.Builder + if err := test.run(&output); err != nil { + t.Fatal(err) + } + if !strings.Contains(output.String(), "Dokploy installation recovery required") || !strings.Contains(output.String(), "sudo bort init-target --install") || strings.Contains(output.String(), "no current migration run") { + t.Fatalf("standalone recovery output omitted the exact retry command: %s", output.String()) + } + }) + } + blockedRun := loadedMigrationRun{Run: migrationRun{Name: "blocked", RunDir: t.TempDir(), Target: "dokploy"}} + if phase := migrationRunPhase(blockedRun); phase != "install-recovery" { + t.Fatalf("installer recovery marker produced phase %q", phase) + } + if next := nextSafeStep(blockedRun, nil); !strings.Contains(next.Action, "sudo bort init-target --install") || !strings.Contains(next.Action, "--dokploy-version v0.30.7@sha256:") || !strings.Contains(next.Action, "--auth-secret-backup") || !strings.Contains(next.Action, "--name 'Recovery Admin'") || !strings.Contains(next.Action, "--api-key-name 'recovery key'") { + t.Fatalf("installer recovery marker produced unsafe next step: %#v", next) + } + var cockpit strings.Builder + writeAppFirstCockpit(&cockpit, blockedRun) + if !strings.Contains(cockpit.String(), "sudo bort init-target --install") { + t.Fatalf("installer recovery cockpit omitted the persisted command: %s", cockpit.String()) + } + recoveryLock, err := acquireDokployInstallRecoveryLock() + if err != nil { + t.Fatalf("installer recovery could not reacquire the host lock: %v", err) + } + recoveryPath, err := dokployInstallationRecoveryPath() + if err != nil { + recoveryLock.Release() + t.Fatal(err) + } + recovery, found, err := readDokployInstallationRecoveryMarker(recoveryPath) + if err != nil || !found { + recoveryLock.Release() + t.Fatalf("read installer recovery identity: found=%t err=%v", found, err) + } + if recovery.CommandPrefix != "sudo bort" || recovery.TargetURL != "http://127.0.0.1:3030" || recovery.HostPort != "3030" || recovery.AddressPool != "auto" || recovery.DokployVersion != defaultDokployVersion+"@"+defaultDokployDigest || recovery.EndpointMode != "vip" || recovery.ACMEEmail != "admin@example.com" || recovery.AuthSecretBackup == "" || recovery.AdminName != "Recovery Admin" || recovery.APIKeyName != "recovery key" || !strings.Contains(recovery.Command, "--dokploy-url http://127.0.0.1:3030") { + recoveryLock.Release() + t.Fatalf("recovery marker omitted immutable install options: %#v", recovery) + } + if err := markDokployInstallationRecoveryRequired(recovery); err != nil { + recoveryLock.Release() + t.Fatalf("same installer options did not resume recovery: %v", err) + } + different := newDokployInstallationRecovery(recovery.TargetURL, dokployInstallOptions{ + HostPort: "3031", + AddrPool: recovery.AddressPool, + Version: recovery.DokployVersion, + EndpointMode: recovery.EndpointMode, + ACMEEmail: recovery.ACMEEmail, + AuthSecretBackup: recovery.AuthSecretBackup, + AdminName: recovery.AdminName, + APIKeyName: recovery.APIKeyName, + }) + if err := markDokployInstallationRecoveryRequired(different); err == nil || !strings.Contains(err.Error(), "exact original command: "+recovery.Command) { + recoveryLock.Release() + t.Fatalf("different installer options were allowed to replace recovery state: %v", err) + } + if err := clearDokployInstallationRecoveryRequired(); err != nil { + recoveryLock.Release() + t.Fatal(err) + } + recoveryLock.Release() + lock, lockErr = acquireDokployLiveOperationLock() + if lockErr != nil { + t.Fatalf("reconciled installer state did not release host mutations: %v", lockErr) + } + lock.Release() +} + +func TestDokployInstallationRecoveryCommandQuotesImmutableOptions(t *testing.T) { + t.Setenv("SUDO_UID", "501") + opts := dokployInstallOptions{ + HostPort: "3030", + AddrPool: "auto", + Version: defaultDokployVersion + "@" + defaultDokployDigest, + EndpointMode: "dnsrr", + ACMEEmail: "admin@example.com", + AuthSecretBackup: "/private/recovery backups/admin's secret", + AdminName: "Admin O'Neil", + APIKeyName: "bort recovery key", + } + recovery := newDokployInstallationRecovery("http://127.0.0.1:3030", opts) + + args := strings.TrimPrefix(recovery.Command, "sudo bort init-target --install") + command := exec.Command("/bin/sh", "-c", "set -- "+args+"; printf '%s\\n' \"$@\"") + output, err := command.Output() + if err != nil { + t.Fatalf("parse recovery command: %v", err) + } + want := strings.Join([]string{ + "--dokploy-url", "http://127.0.0.1:3030", + "--install-port", "3030", + "--swarm-addr-pool", "auto", + "--dokploy-version", defaultDokployVersion + "@" + defaultDokployDigest, + "--endpoint-mode", "dnsrr", + "--coolify-email", "admin@example.com", + "--auth-secret-backup", "/private/recovery backups/admin's secret", + "--name", "Admin O'Neil", + "--api-key-name", "bort recovery key", + "", + }, "\n") + if string(output) != want { + t.Fatalf("parsed recovery command arguments:\n%s\nwant:\n%s", output, want) + } + differentAdmin := opts + differentAdmin.AdminName = "Another Admin" + if recovery.Identity == newDokployInstallationRecovery(recovery.TargetURL, differentAdmin).Identity { + t.Fatal("recovery identity omitted the admin display name") + } + differentKey := opts + differentKey.APIKeyName = "another key" + if recovery.Identity == newDokployInstallationRecovery(recovery.TargetURL, differentKey).Identity { + t.Fatal("recovery identity omitted the API-key name") + } + differentMode := opts + differentMode.EndpointMode = "vip" + if recovery.Identity == newDokployInstallationRecovery(recovery.TargetURL, differentMode).Identity { + t.Fatal("recovery identity omitted the endpoint mode") + } + t.Setenv("SUDO_UID", "") + directRoot := newDokployInstallationRecovery(recovery.TargetURL, opts) + if directRoot.CommandPrefix != "bort" || !strings.HasPrefix(directRoot.Command, "bort init-target --install ") || strings.HasPrefix(directRoot.Command, "sudo ") { + t.Fatalf("direct-root recovery command changed invocation style: %#v", directRoot) + } + if directRoot.Identity == recovery.Identity { + t.Fatal("recovery identity omitted the invocation prefix") + } +} + +func TestPrepareDokployAuthSecretEscrowReturnsValidatedSnapshotAndDigest(t *testing.T) { + root, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatal(err) + } + dir := filepath.Join(root, "private") + if err := os.Mkdir(dir, 0o700); err != nil { + t.Fatal(err) + } + path := filepath.Join(dir, "dokploy-auth-secret") + file, digest, err := prepareDokployAuthSecretEscrow(path) + if err != nil { + t.Fatal(err) + } + defer file.Close() + original, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + wantDigest := sha256.Sum256(original) + if digest != hex.EncodeToString(wantDigest[:]) { + t.Fatalf("escrow digest=%q, want %x", digest, wantDigest) + } + if file.Name() == path { + t.Fatal("escrow returned the pathname-addressable file instead of a snapshot") + } + fromSnapshot, err := io.ReadAll(file) + if err != nil { + t.Fatal(err) + } + if len(original) != 65 || !bytes.Equal(fromSnapshot, original) { + t.Fatalf("snapshot=%q, want the 65-byte escrow contents %q", fromSnapshot, original) + } +} + +func TestPrepareDokployAuthSecretEscrowRejectsMissingDirectory(t *testing.T) { + root, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatal(err) + } + dir := filepath.Join(root, "missing") + path := filepath.Join(dir, "dokploy-auth-secret") + if _, _, err := prepareDokployAuthSecretEscrow(path); err == nil || !errors.Is(err, os.ErrNotExist) { + t.Fatalf("expected missing escrow directory refusal, got %v", err) + } + if _, err := os.Stat(dir); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("missing escrow directory was created: %v", err) + } +} + +func TestDokployAuthSecretValidationAcceptsPipeDescriptor(t *testing.T) { + bash, err := exec.LookPath("bash") + if err != nil { + t.Skip("bash is unavailable") + } + root, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatal(err) + } + dir := filepath.Join(root, "private") + if err := os.Mkdir(dir, 0o700); err != nil { + t.Fatal(err) + } + path := filepath.Join(dir, "dokploy-auth-secret") + descriptor, digest, err := prepareDokployAuthSecretEscrow(path) + if err != nil { + t.Fatal(err) + } + defer descriptor.Close() + start := strings.Index(dokployShadowInstallScript, "validate_auth_secret_backup() {") + if start < 0 { + t.Fatal("missing authentication-secret descriptor validator") + } + end := strings.Index(dokployShadowInstallScript[start:], "\n}\n\nprepare_auth_secret_backup()") + if end < 0 { + t.Fatal("missing authentication-secret descriptor validator") + } + validator := dokployShadowInstallScript[start : start+end+3] + cmd := exec.CommandContext(context.Background(), bash, "-c", "set -euo pipefail\nAUTH_SECRET_FD=3\nAUTH_SECRET_FD_PATH=/dev/fd/$AUTH_SECRET_FD\nAUTH_SECRET_DIGEST="+digest+"\n"+validator+"\nvalidate_auth_secret_backup\ncat \"$AUTH_SECRET_FD_PATH\"") + if fd := configureDokployInstallerCommand(cmd, nil, descriptor); fd != 3 { + t.Fatalf("authentication-secret descriptor fd=%d, want 3", fd) + } + output, err := cmd.Output() + if err != nil { + t.Fatalf("validate pipe-backed authentication-secret descriptor: %v", err) + } + want, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(output, want) { + t.Fatalf("validated descriptor bytes=%q, want %q", output, want) + } +} + +func TestPrepareDokployAuthSecretEscrowSnapshotsValidatedBytes(t *testing.T) { + root, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatal(err) + } + dir := filepath.Join(root, "private") + if err := os.Mkdir(dir, 0o700); err != nil { + t.Fatal(err) + } + path := filepath.Join(dir, "dokploy-auth-secret") + original := []byte(strings.Repeat("a", 64) + "\n") + if err := os.WriteFile(path, original, 0o600); err != nil { + t.Fatal(err) + } + file, digest, err := prepareDokployAuthSecretEscrow(path) + if err != nil { + t.Fatal(err) + } + defer file.Close() + if err := os.WriteFile(path, []byte(strings.Repeat("b", 64)+"\n"), 0o600); err != nil { + t.Fatal(err) + } + fromDescriptor, err := io.ReadAll(file) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(fromDescriptor, original) { + t.Fatalf("installer descriptor consumed mutable escrow bytes: %q", fromDescriptor) + } + if err := verifyDokployAuthSecretEscrow(path, digest); err == nil || !strings.Contains(err.Error(), "changed during installation") { + t.Fatalf("post-install escrow mutation was not detected: %v", err) + } +} + +func TestPrepareDokployAuthSecretEscrowRejectsHardLink(t *testing.T) { + root, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatal(err) + } + dir := filepath.Join(root, "private") + if err := os.Mkdir(dir, 0o700); err != nil { + t.Fatal(err) + } + path := filepath.Join(dir, "dokploy-auth-secret") + if err := os.WriteFile(path, []byte(strings.Repeat("a", 64)+"\n"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.Link(path, path+".link"); err != nil { + t.Fatal(err) + } + if _, _, err := prepareDokployAuthSecretEscrow(path); err == nil || !strings.Contains(err.Error(), "single-link") { + t.Fatalf("expected hard-linked escrow refusal, got %v", err) + } +} + +func TestPrepareDokployAuthSecretEscrowRejectsWrongSizeBeforeReading(t *testing.T) { + root, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatal(err) + } + dir := filepath.Join(root, "private") + if err := os.Mkdir(dir, 0o700); err != nil { + t.Fatal(err) + } + path := filepath.Join(dir, "dokploy-auth-secret") + if err := os.WriteFile(path, nil, 0o600); err != nil { + t.Fatal(err) + } + if err := os.Truncate(path, 1<<40); err != nil { + t.Fatal(err) + } + if _, _, err := prepareDokployAuthSecretEscrow(path); err == nil || !strings.Contains(err.Error(), "is 1099511627776 bytes, not the 65 bytes") { + t.Fatalf("expected size refusal before reading the escrow, got %v", err) + } +} diff --git a/internal/cli/installer_process_windows.go b/internal/cli/installer_process_windows.go new file mode 100644 index 0000000..c1e39e2 --- /dev/null +++ b/internal/cli/installer_process_windows.go @@ -0,0 +1,24 @@ +//go:build windows + +package cli + +import ( + "context" + "errors" + "os" + "os/exec" +) + +func dokployInstallerSignalContext(parent context.Context) (context.Context, context.CancelFunc) { + return context.WithCancel(parent) +} + +func configureDokployInstallerCommand(*exec.Cmd, *os.File, *os.File) int { return 0 } + +func prepareDokployAuthSecretEscrow(string) (*os.File, string, error) { + return nil, "", errors.New("Dokploy installation is unavailable on Windows") +} + +func verifyDokployAuthSecretEscrow(string, string) error { + return errors.New("Dokploy installation is unavailable on Windows") +} diff --git a/internal/cli/legacy_upgrade_test.go b/internal/cli/legacy_upgrade_test.go new file mode 100644 index 0000000..cd74fcb --- /dev/null +++ b/internal/cli/legacy_upgrade_test.go @@ -0,0 +1,309 @@ +package cli + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/aikins01/bort/internal/manifest" + "github.com/aikins01/bort/internal/target/dokploy" +) + +func TestCompletedLegacyStatefulRunRefusesCommitWithoutPersistedOrigin(t *testing.T) { + resetDokployTrafficOwner(t) + workDir := t.TempDir() + t.Chdir(workDir) + installLegacyUpgradeDockerStub(t, workDir) + installLegacyUpgradeDokployServer(t) + writeCompletedLegacyRun(t, "legacy-commit", true, true) + + if err := runCommit(context.Background(), []string{"--apply", "--run", "legacy-commit"}, io.Discard, io.Discard); err == nil || !strings.Contains(err.Error(), "cannot prove writer or traffic authority") { + t.Fatalf("expected legacy commit to fail closed, got %v", err) + } + run, err := loadMigrationRun("legacy-commit") + if err != nil { + t.Fatal(err) + } + if run.Run.CommitStartedAt != nil || run.Run.CommittedAt != nil { + t.Fatal("legacy commit refusal changed lifecycle state") + } + if run.Applied.APIVersion != appliedLegacyAPIVersion || run.Applied.TargetOrigin != "" { + t.Fatalf("legacy commit refusal rewrote the ledger: %#v", run.Applied) + } + if phase := migrationRunPhase(run); phase != "authority-ambiguous" { + t.Fatalf("completed legacy routed run phase=%q, want authority-ambiguous", phase) + } + next := nextSafeStep(run, nil) + if !strings.Contains(next.Action, "establish writer and traffic authority manually") || !strings.Contains(next.Reason, "cannot prove writer or traffic authority") { + t.Fatalf("completed legacy routed run gave unsafe next step: %#v", next) + } + _, found, err := readDokployTrafficOwner() + if err != nil { + t.Fatal(err) + } + if found { + t.Fatal("legacy commit refusal created a traffic owner") + } + if _, err := os.Stat("source-stopped"); !os.IsNotExist(err) { + t.Fatalf("legacy commit refusal changed the source: %v", err) + } +} + +func TestCompletedLegacyStatelessRunRefusesRollbackWithoutPersistedOrigin(t *testing.T) { + resetDokployTrafficOwner(t) + workDir := t.TempDir() + t.Chdir(workDir) + installLegacyUpgradeDockerStub(t, workDir) + installLegacyUpgradeDokployServer(t) + writeCompletedLegacyRun(t, "legacy-rollback", false, true) + + if err := runRollback(context.Background(), []string{"--live", "--run", "legacy-rollback", "--confirm", "rollback legacy-rollback"}, io.Discard, io.Discard); err == nil || !strings.Contains(err.Error(), "cannot prove writer or traffic authority") { + t.Fatalf("expected legacy rollback to fail closed, got %v", err) + } + run, err := loadMigrationRun("legacy-rollback") + if err != nil { + t.Fatal(err) + } + if run.Run.RollbackStartedAt != nil || run.Run.RolledBackAt != nil { + t.Fatal("legacy rollback refusal changed lifecycle state") + } + if run.Applied.APIVersion != appliedLegacyAPIVersion || run.Applied.TargetOrigin != "" { + t.Fatalf("legacy rollback refusal rewrote the ledger: %#v", run.Applied) + } + _, found, err := readDokployTrafficOwner() + if err != nil { + t.Fatal(err) + } + if found { + t.Fatal("legacy rollback refusal created a traffic owner") + } + for _, path := range []string{"target-proxy-stopped", "source-proxy-started"} { + if _, err := os.Stat(path); !os.IsNotExist(err) { + t.Fatalf("legacy rollback refusal changed %s: %v", path, err) + } + } +} + +func TestCompletedLegacyStatefulRunWithoutRoutesRefusesCommit(t *testing.T) { + resetDokployTrafficOwner(t) + workDir := t.TempDir() + t.Chdir(workDir) + installLegacyUpgradeDockerStub(t, workDir) + writeCompletedLegacyRun(t, "legacy-no-route", true, false) + + err := runCommit(context.Background(), []string{"--apply", "--run", "legacy-no-route"}, io.Discard, io.Discard) + if err == nil || !strings.Contains(err.Error(), "cannot prove writer or traffic authority") { + t.Fatalf("expected no-route legacy stateful commit to fail closed, got %v", err) + } + run, err := loadMigrationRun("legacy-no-route") + if err != nil { + t.Fatal(err) + } + if phase := migrationRunPhase(run); phase != "authority-ambiguous" { + t.Fatalf("no-route legacy stateful phase=%q, want authority-ambiguous", phase) + } + if run.Run.CommitStartedAt != nil || run.Run.CommittedAt != nil { + t.Fatal("no-route legacy stateful refusal changed commit lifecycle") + } + if _, err := os.Stat("source-stopped"); !os.IsNotExist(err) { + t.Fatalf("no-route legacy stateful refusal changed the source: %v", err) + } +} + +func TestCompletedLegacyStatelessCoolifyRunRecordsManualRetirementWithoutHostOwner(t *testing.T) { + resetDokployTrafficOwner(t) + workDir := t.TempDir() + t.Chdir(workDir) + installLegacyUpgradeDockerStub(t, workDir) + installLegacyUpgradeDokployServer(t) + writeCompletedLegacyRun(t, "legacy-coolify", false, false) + + commitErr := runCommit(context.Background(), []string{"--apply", "--run", "legacy-coolify"}, io.Discard, io.Discard) + if commitErr == nil || !strings.Contains(commitErr.Error(), "Coolify can replace a stopped source container") { + t.Fatalf("expected Coolify retirement refusal, got %v", commitErr) + } + run, err := loadMigrationRun("legacy-coolify") + if err != nil { + t.Fatal(err) + } + command := authorityRecoverySourceRetiredCommand(run) + if !strings.Contains(commitErr.Error(), command) { + t.Fatalf("commit refusal did not point at %q: %v", command, commitErr) + } + + var stdout strings.Builder + args := []string{"--run", "legacy-coolify", "--authority", "target", "--source-retired", "--confirm", authorityRecoverySourceRetiredConfirmation(run.Run)} + if err := runRecoverAuthority(context.Background(), args, &stdout, io.Discard); err != nil { + t.Fatalf("suggested recovery command failed: %v", err) + } + if !strings.Contains(stdout.String(), "never held host ownership") { + t.Fatalf("unexpected recovery output: %q", stdout.String()) + } + run, err = loadMigrationRun("legacy-coolify") + if err != nil { + t.Fatal(err) + } + if run.Run.ResolvedAuthority != dokployTrafficTarget || run.Run.CommitStartedAt == nil || run.Run.CommittedAt == nil { + t.Fatalf("manual retirement was not recorded: %#v", run.Run) + } + if _, found, err := readDokployTrafficOwner(); err != nil || found { + t.Fatalf("ownerless recovery touched the host owner: found=%t err=%v", found, err) + } + if _, err := os.Stat("source-stopped"); !os.IsNotExist(err) { + t.Fatalf("ownerless recovery changed the source: %v", err) + } + if err := runRecoverAuthority(context.Background(), args, io.Discard, io.Discard); err != nil { + t.Fatalf("repeated recovery is not idempotent: %v", err) + } + if err := runCommit(context.Background(), []string{"--apply", "--run", "legacy-coolify"}, io.Discard, io.Discard); err != nil { + t.Fatalf("commit after manual retirement failed: %v", err) + } +} + +func TestCompletedLegacyRunCannotPromotePendingOwnerWithDifferentCredential(t *testing.T) { + resetDokployTrafficOwner(t) + workDir := t.TempDir() + t.Chdir(workDir) + writeCompletedLegacyRun(t, "legacy-pending-owner", false, true) + run, err := loadMigrationRun("legacy-pending-owner") + if err != nil { + t.Fatal(err) + } + const origin = "http://127.0.0.1:3030" + run.Applied.TargetOrigin = origin + appliedPath, err := safeRunArtifactPath(run.Run.RunDir, run.Run.Artifacts.Applied) + if err != nil { + t.Fatal(err) + } + if err := writeJSONArtifact(appliedPath, run.Applied); err != nil { + t.Fatal(err) + } + if err := claimDokployHostOwnership(run.Run, origin, dokployCredentialID("original-token")); err != nil { + t.Fatal(err) + } + run, err = loadMigrationRun("legacy-pending-owner") + if err != nil { + t.Fatal(err) + } + err = ensureDokployTrafficTargetOwner(context.Background(), run, &dokploy.Client{BaseURL: origin, Token: "different-token"}, false) + if err == nil || !strings.Contains(err.Error(), "different target credential identity") { + t.Fatalf("expected pending ownership credential refusal, got %v", err) + } + owner, found, readErr := readDokployTrafficOwner() + if readErr != nil || !found { + t.Fatalf("read pending owner: found=%t err=%v", found, readErr) + } + if owner.Authority != dokployTrafficPending || owner.TargetCredentialID != dokployCredentialID("original-token") { + t.Fatalf("credential mismatch changed pending ownership: %#v", owner) + } +} + +func writeCompletedLegacyRun(t *testing.T, name string, stateful, routed bool) { + t.Helper() + service := manifest.Service{ID: "source-id", Name: "web", Image: "example/api:latest"} + if stateful { + service.Mounts = []manifest.Mount{{Type: "volume", Name: "api-data", Target: "/data"}} + } + app := manifest.App{ + Name: "api", + Services: []manifest.Service{service}, + } + if routed { + app.Routes = []manifest.Route{{Host: "api.example.com", ServiceName: "web", Port: "3000"}} + } + writeTestBundle(t, "bundle", manifest.Manifest{ + Source: manifest.Source{Platform: "coolify-local"}, + Apps: []manifest.App{app}, + }) + runCommand(t, runMigrate, []string{"--bundle", "bundle", "--run", name, "--observation-window", "0", "--rollback-window", "0"}) + run, err := loadMigrationRun(name) + if err != nil { + t.Fatal(err) + } + for _, decision := range openSetupDecisions(run) { + if err := recordReviewDecision(run, decision, run.Run.UpdatedAt.Add(2*time.Second)); err != nil { + t.Fatalf("resolve setup decision: %v", err) + } + } + now := time.Now().UTC() + applied := runApplied{ + APIVersion: appliedLegacyAPIVersion, + RunName: run.Run.Name, + BundleDir: run.Run.BundleDir, + Target: run.Run.Target, + SucceededAt: &now, + } + for index, step := range dokploy.LegacyPlanFromArtifactsV1Alpha1(run.Prepare, run.Sync, run.Cutover).Steps { + applied.Steps = append(applied.Steps, appliedStep{ + Index: index, + Kind: string(step.Kind), + App: step.App, + Ref: step.Ref, + Status: string(dokploy.StepStatusOK), + }) + } + appliedPath, err := safeRunArtifactPath(run.Run.RunDir, run.Run.Artifacts.Applied) + if err != nil { + t.Fatal(err) + } + if err := writeJSONArtifact(appliedPath, applied); err != nil { + t.Fatal(err) + } + if err := markRunLiveAppliedLocked(run.Run); err != nil { + t.Fatal(err) + } +} + +func installLegacyUpgradeDokployServer(t *testing.T) *httptest.Server { + t.Helper() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/api/project.all" { + http.NotFound(w, r) + return + } + _ = json.NewEncoder(w).Encode([]dokploy.Project{}) + })) + t.Cleanup(server.Close) + t.Setenv(dokploy.EnvBaseURL, server.URL) + t.Setenv(dokploy.EnvToken, "secret") + return server +} + +func installLegacyUpgradeDockerStub(t *testing.T, workDir string) { + t.Helper() + binDir := filepath.Join(workDir, "bin") + if err := os.MkdirAll(binDir, 0o700); err != nil { + t.Fatal(err) + } + stub := `#!/bin/sh +echo "$*" >> docker-calls +if [ "$1" = inspect ] && [ "$2" = --type ]; then + ref="$4" + running=true + case "$ref" in + coolify-proxy) [ -f source-proxy-started ] || running=false ;; + dokploy-traefik) [ ! -f target-proxy-stopped ] || running=false ;; + source-id) [ ! -f source-stopped ] || running=false ;; + esac + echo '[{"Id":"'"$ref"'","Name":"/'"$ref"'","State":{"Running":'"$running"',"Status":"running"},"HostConfig":{"RestartPolicy":{"Name":"always"}}}]' +elif [ "$1" = stop ]; then + case "$*" in + *dokploy-traefik) touch target-proxy-stopped ;; + *source-id) touch source-stopped ;; + esac +elif [ "$1" = start ] && [ "$2" = coolify-proxy ]; then + touch source-proxy-started +fi +` + if err := os.WriteFile(filepath.Join(binDir, "docker"), []byte(strings.TrimSpace(stub)+"\n"), 0o700); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", binDir+string(os.PathListSeparator)+os.Getenv("PATH")) +} diff --git a/internal/cli/main_test.go b/internal/cli/main_test.go index 3271517..2545f1e 100644 --- a/internal/cli/main_test.go +++ b/internal/cli/main_test.go @@ -1,13 +1,84 @@ package cli import ( + "context" "os" + "path/filepath" "testing" + + "github.com/aikins01/bort/internal/target/dokploy" ) // bortCommand output must stay deterministic in tests, even when go test is // invoked via sudo. func TestMain(m *testing.M) { os.Setenv("SUDO_UID", "") - os.Exit(m.Run()) + os.Setenv(envDokployEndpointMode, "") + dir, err := os.MkdirTemp("", "bort-cli-test-lock-") + if err != nil { + panic(err) + } + dokployLiveOperationLockPath = func() (string, error) { + return filepath.Join(dir, "dokploy-live.lock"), nil + } + dokployTrafficOwnerPath = func() (string, error) { + return filepath.Join(dir, "dokploy-traffic-owner.json"), nil + } + dokployLiveOperationsSupported = func() bool { return true } + verifyLocalSourceRun = func(context.Context, loadedMigrationRun) error { return nil } + verifyLocalSourceEngine = func(context.Context, loadedMigrationRun) error { return nil } + verifyLocalDokployClient = func(context.Context, *dokploy.Client) error { return nil } + verifyLocalDokployCleanupHost = func(context.Context, *dokploy.Client) error { return nil } + code := m.Run() + _ = os.RemoveAll(dir) + os.Exit(code) +} + +func markRunLocallyScanned(t *testing.T, runRef, source string) { + t.Helper() + run, err := loadMigrationRun(runRef) + if err != nil { + t.Fatal(err) + } + run.Run.Source = source + if err := writeJSONArtifact(filepath.Join(run.Run.RunDir, "run.json"), run.Run); err != nil { + t.Fatal(err) + } +} + +func resetDokployTrafficOwner(t *testing.T) { + t.Helper() + path, err := dokployTrafficOwnerPath() + if err != nil { + t.Fatal(err) + } + if err := os.Remove(path); err != nil && !os.IsNotExist(err) { + t.Fatal(err) + } + t.Cleanup(func() { + _ = os.Remove(path) + }) +} + +func persistRunFixture(t *testing.T, run migrationRun) migrationRun { + t.Helper() + run.APIVersion = runAPIVersion + run.DryRun = true + if err := os.MkdirAll(filepath.FromSlash(run.RunDir), 0o700); err != nil { + t.Fatal(err) + } + if err := writeJSONArtifact(filepath.Join(filepath.FromSlash(run.RunDir), "run.json"), run); err != nil { + t.Fatal(err) + } + return run +} + +func reloadRunFixture(t *testing.T, run migrationRun) loadedMigrationRun { + t.Helper() + current, err := readRunMetadata(filepath.Join(filepath.FromSlash(run.RunDir), "run.json")) + if err != nil { + t.Fatal(err) + } + current.RunDir = run.RunDir + return loadedMigrationRun{Run: current, Applied: newRunApplied(current)} } diff --git a/internal/cli/rollback.go b/internal/cli/rollback.go index 8ccf2ce..9376280 100644 --- a/internal/cli/rollback.go +++ b/internal/cli/rollback.go @@ -34,7 +34,7 @@ func runRollback(ctx context.Context, args []string, stdout, stderr io.Writer) e fs.StringVar(&outputPath, "output", "-", "output path, or - for stdout") fs.StringVar(&cutoverPlanPath, "from-cutover", "", "read a prior cutover JSON plan artifact") fs.StringVar(&runRef, "run", "", "run name under .bort/runs, or a run directory path") - fs.BoolVar(&live, "live", false, "restart source containers and return traffic; performs a short settling check, not the stored observation window") + fs.BoolVar(&live, "live", false, "return stateless traffic to a healthy source when safe; stateful authority recovery is manual") fs.StringVar(&confirm, "confirm", "", "confirm open rollback triggers with the exact phrase: rollback (requires --live)") fs.IntVar(&observationWindowSeconds, "observation-window", rollbackplan.DefaultObservationWindowSeconds, "observation window in seconds") @@ -86,6 +86,12 @@ func runRollback(ctx context.Context, args []string, stdout, stderr io.Writer) e return err } result = run.Rollback + blocker := automaticRollbackBlocker(ctx, run) + available := blocker == nil + if blocker != nil { + result.AutomaticBlocker = blocker.Error() + } + result.AutomaticAvailable = &available } else if cutoverPlanPath != "" { expect := artifactExpectations{AppName: appName} if flagWasSet(fs, "bundle") { @@ -120,7 +126,15 @@ func runRollback(ctx context.Context, args []string, stdout, stderr io.Writer) e func writeRollbackText(w io.Writer, result rollbackplan.Result) { fmt.Fprintf(w, "Rollback plan: %s -> %s\n", result.BundleDir, result.Target) - fmt.Fprintf(w, "Status: %s\n\n", result.Status) + fmt.Fprintf(w, "Status: %s\n", result.Status) + if result.AutomaticAvailable != nil { + if *result.AutomaticAvailable { + fmt.Fprintln(w, "Automatic rollback: available") + } else { + fmt.Fprintf(w, "Automatic rollback: unavailable: %s\n", result.AutomaticBlocker) + } + } + fmt.Fprintln(w) for _, app := range result.Apps { fmt.Fprintf(w, "[%s] %s\n", app.Status, app.Name) @@ -179,13 +193,47 @@ func applyRollbackFromArgs(ctx context.Context, runRef, confirm string, stderr i if err != nil { return err } + if run.Run.RolledBackAt != nil { + plan, planErr := dokploy.PlanForRollback(run.Prepare, run.Sync, run.Cutover) + if planErr != nil || planRequiresDokployHostOwner(run, plan) { + if err := releaseDokployTrafficOwner(run.Run); err != nil { + return fmt.Errorf("release completed rollback traffic ownership: %w", err) + } + } + fmt.Fprintf(stderr, "rollback already complete for run %s: traffic is back on the source; start a fresh run to migrate again\n", run.Run.Name) + return nil + } if err := validateRollbackApplyReady(run); err != nil { return err } + if err := validateApplyResumeAuthority(run.Applied); err != nil { + return fmt.Errorf("automatic rollback refused: %w; %s", err, authorityRecoveryInstruction(run)) + } + plan, err := planAutomaticRollback(run) + if err != nil { + return fmt.Errorf("automatic rollback refused: %w; %s", err, authorityRecoveryInstruction(run)) + } + if !dokployLiveOperationsSupported() { + return fmt.Errorf("automatic rollback is unavailable on this platform; continue on the Linux source host") + } phrase := "rollback " + run.Run.Name if confirm != "" && confirm != phrase { return fmt.Errorf("rollback confirmation must be exactly %q", phrase) } + if err := verifyLocalSourceRun(ctx, run); err != nil { + return fmt.Errorf("automatic rollback refused on an unverified Docker source: %w", err) + } + targetLock, err := acquireDokployLiveOperationLock() + if err != nil { + return fmt.Errorf("lock Dokploy live operations: %w", err) + } + defer targetLock.Release() + requiresHostOwner := planRequiresDokployHostOwner(run, plan) + if requiresHostOwner { + if err := ensureDokployTrafficTargetOwner(ctx, run, nil, run.Run.RollbackStartedAt != nil); err != nil { + return fmt.Errorf("automatic rollback refused: %w", err) + } + } if decisions := rollbackTriggerDecisions(run); len(decisions) > 0 { for _, decision := range decisions { fmt.Fprintf(stderr, "%s\n%s\n", decisionAction(decision), decisionReason(decision)) @@ -204,7 +252,7 @@ func applyRollbackFromArgs(ctx context.Context, runRef, confirm string, stderr i return err } } - plan := dokploy.PlanForRollback(run.Prepare, run.Cutover) + client := &dokploy.Client{} plan.RunName = run.Run.Name plan.RunDir = run.Run.RunDir plan.ApprovedPrepareDecisions = approvedPrepareDecisions(run) @@ -224,17 +272,80 @@ func applyRollbackFromArgs(ctx context.Context, runRef, confirm string, stderr i return fmt.Errorf("record rollback start: %w", err) } fmt.Fprintf(stderr, "rollback live: run %s; planned %d step(s) to return traffic to the source\n", run.Run.Name, len(plan.Steps)) - client := &dokploy.Client{} if err := client.Apply(ctx, plan); err != nil { return err } + if planChangesDokployTraffic(plan) { + if err := client.ReconcileSourceTrafficAuthority(ctx); err != nil { + return fmt.Errorf("verify source traffic authority: %w", err) + } + } + if requiresHostOwner { + if err := markDokployTrafficSource(run.Run); err != nil { + return fmt.Errorf("rollback completed, but source authority could not be recorded: %w", err) + } + } if err := markRunRolledBackLocked(run.Run); err != nil { return fmt.Errorf("rollback completed, but its outcome could not be recorded: %w", err) } + if requiresHostOwner { + if err := releaseDokployTrafficOwner(run.Run); err != nil { + return fmt.Errorf("rollback was recorded, but its Dokploy host ownership could not be released: %w", err) + } + } writeRollbackAppliedSummary(stderr, run, plan) return nil } +func planAutomaticRollback(run loadedMigrationRun) (dokploy.Plan, error) { + if run.Run.ResolvedAuthority != "" { + return dokploy.Plan{}, fmt.Errorf("run %q used manual %s-authority recovery; automatic rollback cannot infer the manually established boundary", run.Run.Name, run.Run.ResolvedAuthority) + } + return dokploy.PlanForRollback(run.Prepare, run.Sync, run.Cutover) +} + +func automaticRollbackBlocker(ctx context.Context, run loadedMigrationRun) error { + if err := validateRollbackApplyReady(run); err != nil { + return err + } + if err := validateApplyResumeAuthority(run.Applied); err != nil { + return err + } + plan, err := planAutomaticRollback(run) + if err != nil { + return err + } + if !dokployLiveOperationsSupported() { + return fmt.Errorf("automatic rollback is unavailable on this platform; continue on the Linux source host") + } + if err := verifyLocalSourceRun(ctx, run); err != nil { + return fmt.Errorf("source attestation failed: %w", err) + } + if planRequiresDokployHostOwner(run, plan) { + if err := validateCurrentDokployTrafficOwner(run, run.Run.RollbackStartedAt != nil); err != nil { + return err + } + } + hostOperationActive, err := dokployLiveOperationActive() + if err != nil { + return fmt.Errorf("host-wide Dokploy operation lock could not be verified: %w", err) + } + if hostOperationActive { + return fmt.Errorf("another process holds the host-wide Dokploy operation lock") + } + return nil +} + +func incompleteRollbackRecovery(run loadedMigrationRun) string { + if runMayHaveAmbiguousAuthority(run) { + return authorityRecoveryInstruction(run) + } + if _, err := planAutomaticRollback(run); err != nil { + return authorityRecoveryInstruction(run) + } + return fmt.Sprintf("run `%s` to finish recovery", runScopedCommand(run, "rollback --live")) +} + func validateRollbackApplyReady(run loadedMigrationRun) error { if run.Run.Target != "dokploy" { return fmt.Errorf("rollback --live is only supported for target dokploy, got %q", run.Run.Target) @@ -246,13 +357,15 @@ func validateRollbackApplyReady(run loadedMigrationRun) error { return fmt.Errorf("rollback refused: run %q was already committed; its source containers were retired", run.Run.Name) } if run.Run.CommitStartedAt != nil { - return fmt.Errorf("rollback refused: source retirement started for run %q; run `%s` to finish acceptance", run.Run.Name, runScopedCommand(run, "commit --apply")) + return fmt.Errorf("rollback refused: source retirement started for run %q; %s", run.Run.Name, finishStartedAcceptanceAction(run)) } if run.Run.RolledBackAt != nil { return fmt.Errorf("rollback refused: run %q was already rolled back; start a fresh run to migrate again", run.Run.Name) } - if err := requireLiveApplySucceeded(run); err != nil { - return fmt.Errorf("rollback requires a successful live apply: %w", err) + if !runMayHaveAmbiguousAuthority(run) { + if err := requireLiveApplySucceeded(run); err != nil { + return fmt.Errorf("rollback requires a successful live apply: %w", err) + } } active, err := applyRunActive(run.Run.RunDir) if err != nil { diff --git a/internal/cli/rollback_test.go b/internal/cli/rollback_test.go index 62a8268..11c0b5f 100644 --- a/internal/cli/rollback_test.go +++ b/internal/cli/rollback_test.go @@ -4,7 +4,10 @@ import ( "bytes" "context" "encoding/json" + "errors" "io" + "net/http" + "net/http/httptest" "os" "path/filepath" "strings" @@ -12,9 +15,11 @@ import ( "time" "github.com/aikins01/bort/internal/exporter" + "github.com/aikins01/bort/internal/gateway" "github.com/aikins01/bort/internal/manifest" "github.com/aikins01/bort/internal/preparer" rollbackplan "github.com/aikins01/bort/internal/rollback" + syncplan "github.com/aikins01/bort/internal/sync" "github.com/aikins01/bort/internal/target/dokploy" ) @@ -220,7 +225,15 @@ func TestWizardLifecycleWithoutAppliedLedger(t *testing.T) { } { t.Run(tc.name, func(t *testing.T) { tc.run.RunDir = t.TempDir() - run := loadedMigrationRun{Run: tc.run, Prepare: preparer.Result{Apps: []preparer.AppPlan{{Name: "api"}}}} + tc.run.Source = "docker" + run := loadedMigrationRun{Run: tc.run, Prepare: preparer.Result{ + Source: "docker", + SourceDockerEngineID: "engine-reviewed", + Apps: []preparer.AppPlan{{ + Name: "api", + Resources: preparer.ResourceSpecs{SourceServices: []preparer.SourceServiceRef{{ContainerID: "0123456789ab", ContainerName: "api"}}}, + }}, + }} var out bytes.Buffer if err := runWizard(context.Background(), run, strings.NewReader(""), &out, io.Discard); err != nil { t.Fatal(err) @@ -233,24 +246,47 @@ func TestWizardLifecycleWithoutAppliedLedger(t *testing.T) { } func TestInterruptedCommitRefusesRollbackAndResumesCommit(t *testing.T) { + resetDokployTrafficOwner(t) t.Chdir(t.TempDir()) t.Setenv("BORT_DOKPLOY_URL", "http://127.0.0.1:3030") t.Setenv("BORT_DOKPLOY_TOKEN", "test-token") binDir := t.TempDir() stub := `#!/bin/sh -grep -q commitStartedAt .bort/runs/retirement/run.json || exit 9 -if [ -f fail-commit ]; then - echo 'injected inspection failure' >&2 - exit 1 +if [ "$*" = "info --format {{.ID}}" ]; then + echo engine-reviewed + exit 0 fi -echo '[{"Id":"source-id","State":{"Running":false}}]' +if [ "$*" = "inspect --type container source-id" ]; then + count=0 + if [ -f inspect-count ]; then count=$(cat inspect-count); fi + count=$((count + 1)) + echo "$count" > inspect-count + if [ -f fail-commit ] && [ "$count" -gt 1 ]; then + grep -q commitStartedAt .bort/runs/retirement/run.json || exit 9 + echo 'injected inspection failure' >&2 + exit 1 + fi + if [ -f source-stopped ]; then + echo '[{"Id":"source-id","Name":"/web","State":{"Running":false,"Status":"exited"}}]' + else + echo '[{"Id":"source-id","Name":"/web","State":{"Running":true,"Status":"running"}}]' + fi + exit 0 +fi +if [ "$*" = "stop source-id" ]; then + grep -q commitStartedAt .bort/runs/retirement/run.json || exit 9 + touch source-stopped + echo source-id + exit 0 +fi +exit 0 ` if err := os.WriteFile(filepath.Join(binDir, "docker"), []byte(stub), 0o700); err != nil { t.Fatal(err) } t.Setenv("PATH", binDir+string(os.PathListSeparator)+os.Getenv("PATH")) writeTestBundle(t, "bundle", manifest.Manifest{ - Source: manifest.Source{Platform: "coolify-local"}, + Source: manifest.Source{Platform: "docker", DockerEngineID: "engine-reviewed"}, Apps: []manifest.App{{Name: "api", Services: []manifest.Service{{ID: "source-id", Name: "web", Image: "example/api:latest"}}}}, }) runCommand(t, runMigrate, []string{"--bundle", "bundle", "--run", "retirement"}) @@ -272,6 +308,12 @@ echo '[{"Id":"source-id","State":{"Running":false}}]' if err := writeRunApplied(runArtifactPath(run.Run.RunDir, run.Run.Artifacts.Applied), applied); err != nil { t.Fatal(err) } + if err := claimDokployHostOwnership(run.Run, "http://127.0.0.1:3030", dokployCredentialID("test-token")); err != nil { + t.Fatal(err) + } + if err := markDokployTrafficTarget(run.Run, "http://127.0.0.1:3030"); err != nil { + t.Fatal(err) + } if err := os.WriteFile("fail-commit", nil, 0o600); err != nil { t.Fatal(err) } @@ -298,6 +340,24 @@ echo '[{"Id":"source-id","State":{"Running":false}}]' if err != nil || run.Run.CommittedAt == nil { t.Fatalf("expected recorded commit completion, got %v", err) } + owner, found, err := readDokployTrafficOwner() + if err != nil { + t.Fatal(err) + } + if !found || owner.Authority != dokployTrafficReleased { + t.Fatalf("expected commit to release host ownership, got %#v found=%t", owner, found) + } + nextRun := migrationRun{Name: "next", RunDir: filepath.Join(t.TempDir(), "next"), CreatedAt: time.Now().UTC(), BundleDigest: "next"} + if err := ensureDokployTrafficRunAvailable(nextRun, dokploy.Plan{Steps: []dokploy.Step{{Kind: dokploy.StepCreateProject, App: "next", Ref: "next"}}}); err != nil { + t.Fatalf("completed commit blocked the next migration: %v", err) + } + if err := runCommit(context.Background(), []string{"--apply", "--run", "retirement"}, io.Discard, io.Discard); err != nil { + t.Fatalf("repeated commit did not preserve released ownership: %v", err) + } + owner, found, err = readDokployTrafficOwner() + if err != nil || !found || owner.Authority != dokployTrafficReleased { + t.Fatalf("repeated commit changed host ownership, got %#v found=%t err=%v", owner, found, err) + } } func TestValidateRollbackApplyReadyRefusals(t *testing.T) { @@ -342,7 +402,79 @@ func TestValidateRollbackApplyReadyRefusals(t *testing.T) { } } -func TestRunRollbackLiveExecutesStoredRollbackAndGatesFollowUps(t *testing.T) { +func TestAutomaticRollbackAvailabilityChecksExecutionAndHostOwnership(t *testing.T) { + resetDokployTrafficOwner(t) + if err := automaticRollbackBlocker(context.Background(), loadedMigrationRun{ + Run: migrationRun{Name: "unapplied", Target: "dokploy", RunDir: t.TempDir()}, + }); err == nil || !strings.Contains(err.Error(), "successful live apply") { + t.Fatalf("expected unapplied rollback to be unavailable, got %v", err) + } + + now := time.Now().UTC() + meta := migrationRun{ + Name: "missing-owner", + RunDir: t.TempDir(), + Target: "dokploy", + CreatedAt: now, + BundleDigest: strings.Repeat("a", 64), + ApplyOutcomeRequired: true, + } + prepare := preparer.Result{Apps: []preparer.AppPlan{{Name: "api"}}} + plan := dokploy.PlanFromArtifacts(prepare, syncplan.Result{}, gateway.Result{}) + applied := newRunApplied(meta) + for index, step := range plan.Steps { + applied = recordAppliedStep(applied, dokploy.StepProgress{Index: index, Step: step, Status: dokploy.StepStatusOK}) + } + applied.SucceededAt = &now + run := loadedMigrationRun{Run: meta, Prepare: prepare, Applied: applied} + if err := automaticRollbackBlocker(context.Background(), run); err == nil || !strings.Contains(err.Error(), "no durable host-wide Dokploy traffic owner") { + t.Fatalf("expected missing owner to make automatic rollback unavailable, got %v", err) + } + const targetOrigin = "http://127.0.0.1:3030" + if err := claimDokployHostOwnership(meta, targetOrigin, dokployCredentialID("test-token")); err != nil { + t.Fatal(err) + } + if err := markDokployTrafficTarget(meta, targetOrigin); err != nil { + t.Fatal(err) + } + + previousSupported := dokployLiveOperationsSupported + dokployLiveOperationsSupported = func() bool { return false } + if err := automaticRollbackBlocker(context.Background(), run); err == nil || !strings.Contains(err.Error(), "unavailable on this platform") { + t.Fatalf("expected unsupported platform to make automatic rollback unavailable, got %v", err) + } + dokployLiveOperationsSupported = previousSupported + t.Cleanup(func() { dokployLiveOperationsSupported = previousSupported }) + + hostLock, err := acquireDokployLiveOperationLock() + if err != nil { + t.Fatal(err) + } + defer hostLock.Release() + if err := automaticRollbackBlocker(context.Background(), run); err == nil || !strings.Contains(err.Error(), "holds the host-wide Dokploy operation lock") { + t.Fatalf("expected active host operation to make automatic rollback unavailable, got %v", err) + } +} + +func TestIncompleteStatefulRollbackRequiresManualAuthorityRecovery(t *testing.T) { + now := time.Now().UTC() + run := loadedMigrationRun{ + Run: migrationRun{Name: "stateful", RollbackStartedAt: &now}, + Sync: syncplan.Result{Apps: []syncplan.AppPlan{{Name: "api", Steps: []syncplan.Step{{ + ResourceType: "volume", + Strategy: syncplan.StrategyDockerVolumeArchive, + }}}}}, + } + want := "inspect and preserve both sides, establish writer and traffic authority manually, then start a fresh migration run" + if got := incompleteRollbackRecovery(run); got != want { + t.Fatalf("stateful rollback recovery = %q, want %q", got, want) + } + if err := validateLiveApplyReady(run); err == nil || !strings.Contains(err.Error(), want) || strings.Contains(err.Error(), "rollback --live") { + t.Fatalf("live apply reported an unusable rollback retry: %v", err) + } +} + +func TestRunRollbackLiveRefusesAmbiguousLegacyProxyHandoff(t *testing.T) { workDir := t.TempDir() t.Chdir(workDir) binDir := filepath.Join(workDir, "bin") @@ -350,20 +482,275 @@ func TestRunRollbackLiveExecutesStoredRollbackAndGatesFollowUps(t *testing.T) { t.Fatal(err) } dockerStub := `#!/bin/sh -grep -q rollbackStartedAt .bort/runs/demo/run.json || exit 9 echo "$*" >> docker-calls +if [ "$1" = ps ] && [ "$2" = -a ]; then + echo target-id +elif [ "$1" = inspect ] && [ "$2" = --type ]; then + running=true + case "$4" in + source-id) [ -f source-started ] || running=false ;; + target-id) [ -f target-stopped ] && running=false ;; + coolify-proxy) [ -f proxy-started ] || running=false ;; + esac + echo '[{"Id":"'"$4"'","Name":"/'"$4"'","State":{"Running":'"$running"',"Status":"running"}}]' +elif [ "$1" = start ] && [ "$2" = source-id ]; then + [ -f target-stopped ] || exit 23 + touch source-started +elif [ "$1" = start ] && [ "$2" = coolify-proxy ]; then + touch proxy-started +elif [ "$1" = stop ] && [ "$2" = target-id ]; then + touch target-stopped +fi +` + if err := os.WriteFile(filepath.Join(binDir, "docker"), []byte(dockerStub), 0o700); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", binDir+string(os.PathListSeparator)+os.Getenv("PATH")) + + writeTestBundle(t, "bundle", manifest.Manifest{ + Source: manifest.Source{Platform: "coolify-local"}, + Apps: []manifest.App{{ + Name: "api", + Routes: []manifest.Route{{Host: "api.example.com", ServiceName: "web", Port: "3000"}}, + Services: []manifest.Service{{ + ID: "source-id", + Name: "web", + Image: "example/api:latest", + }}, + }}, + }) + runCommand(t, runMigrate, []string{"--bundle", "bundle", "--run", "legacy", "--observation-window", "0", "--rollback-window", "0"}) + run, err := loadMigrationRun("legacy") + if err != nil { + t.Fatal(err) + } + for _, decision := range openSetupDecisions(run) { + if err := recordReviewDecision(run, decision, run.Run.UpdatedAt.Add(2*time.Second)); err != nil { + t.Fatalf("resolve prepare decision: %v", err) + } + } + run.Prepare.Apps[0].Resources.SourceServices = nil + run.Prepare.Apps[0].Resources.Volumes = nil + preparePath, err := safeRunArtifactPath(run.Run.RunDir, run.Run.Artifacts.Prepare) + if err != nil { + t.Fatal(err) + } + if err := writeJSONArtifact(preparePath, run.Prepare); err != nil { + t.Fatal(err) + } + livePlan := dokploy.PlanFromArtifacts(run.Prepare, run.Sync, run.Cutover) + applied := runApplied{APIVersion: appliedAPIVersion, RunName: run.Run.Name, BundleDir: run.Run.BundleDir, Target: run.Run.Target} + for index, step := range livePlan.Steps { + status := dokploy.StepStatusOK + switch step.Kind { + case dokploy.StepStartDokployProxy: + status = dokploy.StepStatusStarted + case dokploy.StepActivateRoutes, dokploy.StepResumeTarget: + status = dokploy.StepStatusSkipped + } + applied = recordAppliedStep(applied, dokploy.StepProgress{Index: index, Step: step, Status: status}) + if status == dokploy.StepStatusStarted { + break + } + } + if ambiguity := appliedAuthorityAmbiguity(applied); ambiguity != applyTrafficAuthorityAmbiguous { + t.Fatalf("fixture ambiguity = %d, want the proxy handoff alone (%d)", ambiguity, applyTrafficAuthorityAmbiguous) + } + if err := writeRunApplied(runArtifactPath(run.Run.RunDir, run.Run.Artifacts.Applied), applied); err != nil { + t.Fatal(err) + } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/project.all": + _ = json.NewEncoder(w).Encode([]dokploy.Project{{ + ProjectID: "project-1", + Name: "api", + Environments: []dokploy.ProjectEnvironment{{EnvironmentID: "environment-1", Name: "production"}}, + }}) + case "/api/compose.search": + _ = json.NewEncoder(w).Encode(map[string]any{ + "items": []dokploy.Compose{{ComposeID: "compose-1", Name: "api", AppName: "stack-api"}}, + "total": 1, + }) + case "/api/compose.one": + _ = json.NewEncoder(w).Encode(dokploy.Compose{ComposeID: "compose-1", Name: "api", AppName: "stack-api", EnvironmentID: "environment-1"}) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + t.Setenv(dokploy.EnvBaseURL, server.URL) + t.Setenv(dokploy.EnvToken, "secret") + + if err := runCommit(context.Background(), []string{"--apply", "--run", "legacy"}, io.Discard, io.Discard); err == nil || !strings.Contains(err.Error(), "no successful live-apply outcome") { + t.Fatalf("expected ambiguous legacy handoff to remain ineligible for commit, got %v", err) + } + if err := runRollback(context.Background(), []string{"--live", "--run", "legacy", "--confirm", "rollback legacy"}, io.Discard, io.Discard); err == nil || !strings.Contains(err.Error(), "cannot prove traffic authority after the shared proxy handoff") || !strings.Contains(err.Error(), "inspect the Coolify and Dokploy proxies") { + t.Fatalf("expected ambiguous legacy proxy handoff to fail rollback closed, got %v", err) + } + if _, err := os.Stat("docker-calls"); !os.IsNotExist(err) { + t.Fatalf("ambiguous legacy rollback made Docker calls: %v", err) + } + refused, err := loadMigrationRun("legacy") + if err != nil { + t.Fatal(err) + } + if refused.Run.RollbackStartedAt != nil || refused.Run.RolledBackAt != nil { + t.Fatal("ambiguous legacy refusal changed rollback lifecycle state") + } +} + +func TestRunRollbackLiveRefusesCompletedStatefulRunWithoutMutation(t *testing.T) { + resetDokployTrafficOwner(t) + workDir := t.TempDir() + t.Chdir(workDir) + binDir := filepath.Join(workDir, "bin") + if err := os.MkdirAll(binDir, 0o700); err != nil { + t.Fatal(err) + } + dockerStub := "#!/bin/sh\necho \"$*\" >> docker-calls\nexit 81\n" + if err := os.WriteFile(filepath.Join(binDir, "docker"), []byte(dockerStub), 0o700); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", binDir+string(os.PathListSeparator)+os.Getenv("PATH")) + + writeTestBundle(t, "bundle", manifest.Manifest{ + Source: manifest.Source{Platform: "coolify-local", DockerEngineID: "engine-reviewed"}, + Apps: []manifest.App{{ + Name: "api", + Routes: []manifest.Route{{Host: "api.example.com", ServiceName: "web", Port: "3000"}}, + Services: []manifest.Service{{ + ID: "source-id", + Name: "web", + Image: "example/api:latest", + Mounts: []manifest.Mount{{ + Type: "volume", Name: "api-data", Target: "/data", + }}, + }}, + }}, + }) + runCommand(t, runMigrate, []string{"--bundle", "bundle", "--run", "stateful", "--observation-window", "0", "--rollback-window", "0"}) + markRunLocallyScanned(t, "stateful", "coolify-local") + run, err := loadMigrationRun("stateful") + if err != nil { + t.Fatal(err) + } + for _, decision := range openSetupDecisions(run) { + if err := recordReviewDecision(run, decision, run.Run.UpdatedAt.Add(2*time.Second)); err != nil { + t.Fatalf("resolve prepare decision: %v", err) + } + } + livePlan := dokploy.PlanFromArtifacts(run.Prepare, run.Sync, run.Cutover) + foundPause := false + applied := newRunApplied(run.Run) + for index, step := range livePlan.Steps { + foundPause = foundPause || step.Kind == dokploy.StepPauseSource + applied = recordAppliedStep(applied, dokploy.StepProgress{Index: index, Step: step, Status: dokploy.StepStatusOK}) + } + if !foundPause { + t.Fatal("stateful fixture did not produce a source pause") + } + now := time.Now().UTC() + applied.SucceededAt = &now + if err := writeRunApplied(runArtifactPath(run.Run.RunDir, run.Run.Artifacts.Applied), applied); err != nil { + t.Fatal(err) + } + if err := claimDokployHostOwnership(run.Run, "http://127.0.0.1:3030", dokployCredentialID("test-token")); err != nil { + t.Fatal(err) + } + if err := markDokployTrafficTarget(run.Run, "http://127.0.0.1:3030"); err != nil { + t.Fatal(err) + } + if err := markRunLiveAppliedLocked(run.Run); err != nil { + t.Fatal(err) + } + run, err = loadMigrationRun("stateful") + if err != nil { + t.Fatal(err) + } + if phase := migrationRunPhase(run); phase != "applied" { + t.Fatalf("expected applied phase, got %q", phase) + } + next := nextSafeStep(run, nil) + if !strings.Contains(next.Action, "recover-authority") || !strings.Contains(next.Action, "as source") || strings.Contains(next.Action, "rollback --live") { + t.Fatalf("expected manual stateful recovery guidance, got %#v", next) + } + var cockpit strings.Builder + writeAppFirstCockpit(&cockpit, run) + for _, want := range []string{"TARGET LIVE", "Automatic rollback is unavailable: automatic rollback is unsafe for stateful app(s)", "recover authority manually", "source: `" + authorityRecoveryCommand(run, dokployTrafficSource) + "`"} { + if !strings.Contains(cockpit.String(), want) { + t.Fatalf("expected cockpit to contain %q, got:\n%s", want, cockpit.String()) + } + } + if strings.Contains(cockpit.String(), "rollback --live") { + t.Fatalf("stateful cockpit recommended automatic rollback:\n%s", cockpit.String()) + } + + runPath := filepath.Join(run.Run.RunDir, "run.json") + progressPath := runArtifactPath(run.Run.RunDir, run.Run.Artifacts.Progress) + runBefore, err := os.ReadFile(runPath) + if err != nil { + t.Fatal(err) + } + progressBefore, err := os.ReadFile(progressPath) + if err != nil { + t.Fatal(err) + } + err = runRollback(context.Background(), []string{"--live", "--run", "stateful", "--confirm", "rollback stateful"}, io.Discard, io.Discard) + if err == nil || !strings.Contains(err.Error(), "no durable application fence") || !strings.Contains(err.Error(), "establish writer and traffic authority manually") { + t.Fatalf("expected stateful rollback to fail closed, got %v", err) + } + if _, err := os.Stat("docker-calls"); !os.IsNotExist(err) { + t.Fatalf("stateful rollback refusal made Docker calls: %v", err) + } + runAfter, err := os.ReadFile(runPath) + if err != nil { + t.Fatal(err) + } + progressAfter, err := os.ReadFile(progressPath) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(runBefore, runAfter) || !bytes.Equal(progressBefore, progressAfter) { + t.Fatal("stateful rollback refusal changed run lifecycle or progress") + } + refused, err := loadMigrationRun("stateful") + if err != nil { + t.Fatal(err) + } + if refused.Run.RollbackStartedAt != nil || refused.Run.RolledBackAt != nil { + t.Fatal("stateful rollback refusal changed rollback lifecycle state") + } +} + +func TestRunRollbackLiveExecutesStoredStatelessRollbackAndGatesFollowUps(t *testing.T) { + resetDokployTrafficOwner(t) + workDir := t.TempDir() + t.Chdir(workDir) + binDir := filepath.Join(workDir, "bin") + if err := os.MkdirAll(binDir, 0o700); err != nil { + t.Fatal(err) + } + dockerStub := `#!/bin/sh +echo "$*" >> docker-calls +grep -q rollbackStartedAt .bort/runs/demo/run.json || exit 9 if [ "$1" = inspect ] && [ "$2" = --type ]; then running=true + policy=always + name=app case "$4" in + cid123) name=web ;; coolify-proxy) [ -f proxy-started ] || running=false ;; dokploy-traefik) [ ! -f proxy-stopped ] || running=false ;; - cid123) if [ -f proxy-started ] && [ -f fail-observation ]; then running=false; fi ;; esac - echo '[{"Id":"'"$4"'","Name":"/app","State":{"Running":'"$running"',"Status":"running"}}]' + [ ! -f "$4-unless" ] || policy=unless-stopped + echo '[{"Id":"'"$4"'","Name":"/'"$name"'","State":{"Running":'"$running"',"Status":"running"},"HostConfig":{"RestartPolicy":{"Name":"'"$policy"'"}}}]' elif [ "$1" = start ] && [ "$2" = coolify-proxy ]; then - touch proxy-started + [ -f fail-observation ] || touch proxy-started elif [ "$1" = stop ]; then touch proxy-stopped +elif [ "$1" = update ] && [ "$2" = --restart=unless-stopped ]; then + touch "$3-unless" fi ` if err := os.WriteFile(filepath.Join(binDir, "docker"), []byte(dockerStub), 0o700); err != nil { @@ -373,19 +760,19 @@ fi bundleDir := filepath.Join(workDir, "bort-bundle") writeTestBundle(t, bundleDir, manifest.Manifest{ - Source: manifest.Source{Platform: "coolify-local"}, + Source: manifest.Source{Platform: "coolify-local", DockerEngineID: "engine-reviewed"}, Apps: []manifest.App{{ Name: "api", Routes: []manifest.Route{{Host: "api.example.com", ServiceName: "web", Port: "3000"}}, Services: []manifest.Service{{ - ID: "cid123", - Name: "web", - Image: "example/api:latest", - Mounts: []manifest.Mount{{Type: "volume", Name: "api-data", Target: "/data"}}, + ID: "cid123", + Name: "web", + Image: "example/api:latest", }}, }}, }) runCommand(t, runMigrate, []string{"--bundle", bundleDir, "--run", "demo", "--observation-window", "0", "--rollback-window", "0"}) + markRunLocallyScanned(t, "demo", "coolify-local") run, err := loadMigrationRun("demo") if err != nil { t.Fatal(err) @@ -398,6 +785,7 @@ fi applied := newRunApplied(run.Run) now := time.Now().UTC() applied.SucceededAt = &now + applied.Apps["api"] = appliedApp{ProjectID: "project-1", EnvironmentID: "environment-1", ComposeID: "compose-1", ComposeAppName: "stack-api"} for index, step := range dokploy.PlanFromArtifacts(run.Prepare, run.Sync, run.Cutover).Steps { applied.Steps = append(applied.Steps, appliedStep{Index: index, Kind: string(step.Kind), App: step.App, Ref: step.Ref, Status: "ok"}) } @@ -407,6 +795,12 @@ fi if err := markRunLiveAppliedLocked(run.Run); err != nil { t.Fatal(err) } + if err := claimDokployHostOwnership(run.Run, "http://127.0.0.1:3000", dokployCredentialID("test-token")); err != nil { + t.Fatal(err) + } + if err := markDokployTrafficTarget(run.Run, "http://127.0.0.1:3000"); err != nil { + t.Fatal(err) + } var stdout bytes.Buffer var stderr bytes.Buffer @@ -419,10 +813,29 @@ fi if _, err := os.Stat("docker-calls"); !os.IsNotExist(err) { t.Fatalf("expected no Docker calls before confirmation, got %v", err) } + targetLock, err := acquireDokployLiveOperationLock() + if err != nil { + t.Fatal(err) + } + err = runRollback(context.Background(), []string{"--live", "--run", "demo", "--confirm", "rollback demo"}, &stdout, &stderr) + if !errors.Is(err, errDokployLiveOperationActive) { + t.Fatalf("expected another Dokploy live operation to block rollback, got %v", err) + } + targetLock.Release() + blocked, err := loadMigrationRun("demo") + if err != nil { + t.Fatal(err) + } + if blocked.Run.RollbackStartedAt != nil { + t.Fatal("rollback recorded a start before acquiring the host Dokploy lock") + } + if _, err := os.Stat("docker-calls"); !os.IsNotExist(err) { + t.Fatalf("blocked rollback made Docker calls: %v", err) + } if err := os.WriteFile("fail-observation", nil, 0o600); err != nil { t.Fatal(err) } - if err := runRollback(context.Background(), []string{"--live", "--run", "demo", "--confirm", "rollback demo"}, &stdout, &stderr); err == nil || !strings.Contains(err.Error(), "stopped after rollback") { + if err := runRollback(context.Background(), []string{"--live", "--run", "demo", "--confirm", "rollback demo"}, &stdout, &stderr); err == nil || !strings.Contains(err.Error(), "did not stay running") { t.Fatalf("expected failure after proxy swap, got %v", err) } interrupted, err := loadMigrationRun("demo") @@ -455,10 +868,10 @@ fi t.Fatalf("expected rollback progress header, got:\n%s", stderr.String()) } for _, want := range []string{ - "rollback [1/5] resume_source api: started", - "rollback [1/5] resume_source api: ok", - "rollback [2/5] verify_source_health api: ok", - "rollback [5/5] observe_rollback api: ok", + "rollback [1/4] verify_source_health api: ok", + "rollback [2/4] stop_dokploy_proxy dokploy-traefik: ok", + "rollback [3/4] start_coolify_proxy coolify-proxy: ok", + "rollback [4/4] observe_rollback api: ok", } { if !strings.Contains(stderr.String(), want) { t.Fatalf("expected rollback progress %q, got:\n%s", want, stderr.String()) @@ -475,6 +888,13 @@ fi if rolled.Run.RolledBackAt == nil { t.Fatal("expected rolledBackAt to be recorded") } + owner, found, err := readDokployTrafficOwner() + if err != nil { + t.Fatal(err) + } + if !found || owner.Authority != dokployTrafficReleased { + t.Fatalf("expected released traffic ownership after rollback, got %#v found=%t", owner, found) + } if phase := migrationRunPhase(rolled); phase != "rolled back" { t.Fatalf("expected cockpit phase \"rolled back\", got %q", phase) } @@ -490,9 +910,26 @@ fi t.Fatalf("expected rolled-back wizard to show status without suggesting live apply, got:\n%s", stdout.String()) } + owner.Authority = dokployTrafficSource + if err := writeDokployTrafficOwner(owner); err != nil { + t.Fatal(err) + } + if err := os.Remove("docker-calls"); err != nil { + t.Fatal(err) + } stderr.Reset() - if err := runRollback(context.Background(), []string{"--live", "--run", "demo"}, io.Discard, &stderr); err == nil || !strings.Contains(err.Error(), "already rolled back") { - t.Fatalf("expected repeat rollback to be refused, got %v", err) + if err := runRollback(context.Background(), []string{"--live", "--run", "demo"}, io.Discard, &stderr); err != nil { + t.Fatalf("repeat rollback after an unreleased owner failed: %v", err) + } + if !strings.Contains(stderr.String(), "rollback already complete") { + t.Fatalf("repeat rollback did not report completion, got:\n%s", stderr.String()) + } + if _, err := os.Stat("docker-calls"); !os.IsNotExist(err) { + t.Fatalf("repeat rollback made Docker calls: %v", err) + } + owner, found, err = readDokployTrafficOwner() + if err != nil || !found || owner.Authority != dokployTrafficReleased { + t.Fatalf("repeat rollback did not release host ownership, got %#v found=%t err=%v", owner, found, err) } if err := runCommit(context.Background(), []string{"--apply", "--run", "demo"}, io.Discard, io.Discard); err == nil || !strings.Contains(err.Error(), "was rolled back") { t.Fatalf("expected commit after rollback to be refused, got %v", err) diff --git a/internal/cli/root.go b/internal/cli/root.go index da8f9ac..8b651f5 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -65,6 +65,8 @@ func RunWithInput(ctx context.Context, args []string, stdin io.Reader, stdout, s return runNext(ctx, args[1:], stdout, stderr) case "init-target": return runInitTarget(ctx, args[1:], stdin, stdout, stderr) + case "recover-authority": + return runRecoverAuthority(ctx, args[1:], stdout, stderr) default: return fmt.Errorf("unknown command %q (run `bort help` for usage)", args[0]) } @@ -87,8 +89,8 @@ func writePrimaryHelp(w io.Writer, st *styler) { {verb: bortCommand(""), desc: "start or resume the current migration"}, {verb: bortCommand("migrate --live"), desc: "apply the selected planned run to its target"}, {verb: bortCommand("rollback"), desc: "inspect the stored rollback plan"}, - {verb: bortCommand("rollback --live"), desc: "restart source containers and return traffic to the source"}, - {verb: bortCommand("commit --apply"), desc: "accept the target and retire source containers"}, + {verb: bortCommand("rollback --live"), desc: "return stateless traffic to the source when safe"}, + {verb: bortCommand("commit --apply"), desc: "accept the target when no source orchestrator can recreate it"}, {verb: bortCommand("cleanup"), desc: "audit leftovers; --apply removes safe metadata only"}, {verb: bortCommand("cleanup purge"), desc: "purge eligible source leftovers with confirmation"}, }) @@ -105,6 +107,7 @@ func writeAdvancedHelp(w io.Writer, st *styler) { {verb: bortCommand("env KEY=value ..."), desc: "record env values non-interactively"}, {verb: bortCommand("data --migrate|--recreate|--managed"), desc: "record a data strategy non-interactively"}, {verb: bortCommand("init-target dokploy"), desc: "bootstrap target credentials before live execution"}, + {verb: bortCommand("recover-authority"), desc: "record manual authority recovery or source retirement shown by bort status"}, }) writeHelpSection(w, st, "Power-user pipeline (each step is local and dry-run only):", []helpLine{ {verb: "bort scan", desc: "discover local resources and write a migration manifest"}, diff --git a/internal/cli/run.go b/internal/cli/run.go index 533aa99..33c4b9a 100644 --- a/internal/cli/run.go +++ b/internal/cli/run.go @@ -10,6 +10,7 @@ import ( "fmt" "io" "net/http" + "net/url" "os" "path/filepath" "sort" @@ -23,6 +24,7 @@ import ( "github.com/aikins01/bort/internal/planutil" "github.com/aikins01/bort/internal/preparer" rollbackplan "github.com/aikins01/bort/internal/rollback" + "github.com/aikins01/bort/internal/source/localdocker" syncplan "github.com/aikins01/bort/internal/sync" "github.com/aikins01/bort/internal/target/dokploy" ) @@ -32,32 +34,40 @@ const ( decisionsAPIVersion = "bort.decisions/v1alpha1" ) -var errRunOperationActive = errors.New("migration run operation already in progress") +var ( + errRunOperationActive = errors.New("migration run operation already in progress") + errDokployLiveOperationActive = errors.New("another Dokploy live operation is already running on this host") + errDokployInstallRecoveryRequired = errors.New("an interrupted Dokploy installation requires recovery; rerun the same init-target --install command before other host mutations") +) type migrationRun struct { - APIVersion string `json:"apiVersion"` - Name string `json:"name"` - RunDir string `json:"runDir"` - CreatedAt time.Time `json:"createdAt"` - UpdatedAt time.Time `json:"updatedAt"` - LiveAppliedAt *time.Time `json:"liveAppliedAt,omitempty"` - CommitStartedAt *time.Time `json:"commitStartedAt,omitempty"` - CommittedAt *time.Time `json:"committedAt,omitempty"` - RollbackStartedAt *time.Time `json:"rollbackStartedAt,omitempty"` - RolledBackAt *time.Time `json:"rolledBackAt,omitempty"` - PurgedAt *time.Time `json:"purgedAt,omitempty"` - ApplyOutcomeRequired bool `json:"applyOutcomeRequired,omitempty"` - Source string `json:"source,omitempty"` - BundleDir string `json:"bundleDir"` - BundleDigest string `json:"bundleDigest,omitempty"` - SourceBundleDir string `json:"sourceBundleDir,omitempty"` - ManifestPath string `json:"manifest,omitempty"` - Target string `json:"target"` - AppName string `json:"app,omitempty"` - DryRun bool `json:"dryRun"` - ObservationWindowSeconds int `json:"observationWindowSeconds"` - RollbackWindowSeconds int `json:"rollbackWindowSeconds"` - Artifacts runArtifacts `json:"artifacts"` + APIVersion string `json:"apiVersion"` + Name string `json:"name"` + RunDir string `json:"runDir"` + CreatedAt time.Time `json:"createdAt"` + UpdatedAt time.Time `json:"updatedAt"` + LiveAppliedAt *time.Time `json:"liveAppliedAt,omitempty"` + CommitStartedAt *time.Time `json:"commitStartedAt,omitempty"` + CommittedAt *time.Time `json:"committedAt,omitempty"` + RollbackStartedAt *time.Time `json:"rollbackStartedAt,omitempty"` + RolledBackAt *time.Time `json:"rolledBackAt,omitempty"` + PurgedAt *time.Time `json:"purgedAt,omitempty"` + ResolvedAuthority string `json:"resolvedAuthority,omitempty"` + AuthorityResolvedAt *time.Time `json:"authorityResolvedAt,omitempty"` + AuthorityFinalizedAt *time.Time `json:"authorityFinalizedAt,omitempty"` + HostOwnerReleaseStartedAt *time.Time `json:"hostOwnerReleaseStartedAt,omitempty"` + ApplyOutcomeRequired bool `json:"applyOutcomeRequired,omitempty"` + Source string `json:"source,omitempty"` + BundleDir string `json:"bundleDir"` + BundleDigest string `json:"bundleDigest,omitempty"` + SourceBundleDir string `json:"sourceBundleDir,omitempty"` + ManifestPath string `json:"manifest,omitempty"` + Target string `json:"target"` + AppName string `json:"app,omitempty"` + DryRun bool `json:"dryRun"` + ObservationWindowSeconds int `json:"observationWindowSeconds"` + RollbackWindowSeconds int `json:"rollbackWindowSeconds"` + Artifacts runArtifacts `json:"artifacts"` } type runArtifacts struct { @@ -167,6 +177,7 @@ type runNextStep struct { Reason string Artifact string DecisionID string + Phase string } type migrationRunOptions struct { @@ -256,7 +267,7 @@ func runMigrate(ctx context.Context, args []string, stdout, stderr io.Writer) er if err := rememberCurrentRun(loadedRun.Run); err != nil { return err } - writeAppFirstCockpit(stdout, loadedRun) + writeAppFirstCockpitContext(ctx, stdout, loadedRun) return nil } @@ -281,7 +292,7 @@ func runMigrate(ctx context.Context, args []string, stdout, stderr io.Writer) er if err := rememberCurrentRun(loadedRun.Run); err != nil { return err } - summary := summarizeMigrationRun(loadedRun) + summary := summarizeMigrationRunForLive(ctx, loadedRun) writeLiveMigrationRunText(stdout, "Migration run loaded", summary) return attachLiveMigrationRun(ctx, loadedRun, stderr, nil) } @@ -303,7 +314,7 @@ func runMigrate(ctx context.Context, args []string, stdout, stderr io.Writer) er if loadErr != nil { return loadErr } - writeLiveMigrationRunText(stdout, "Migration run loaded", summarizeMigrationRun(loadedRun)) + writeLiveMigrationRunText(stdout, "Migration run loaded", summarizeMigrationRunForLive(ctx, loadedRun)) return attachLiveMigrationRun(ctx, loadedRun, stderr, nil) } return fmt.Errorf("start live migration for run %q: %w", resolved, err) @@ -316,7 +327,7 @@ func runMigrate(ctx context.Context, args []string, stdout, stderr io.Writer) er if err := rememberCurrentRun(loadedRun.Run); err != nil { return err } - writeLiveMigrationRunText(stdout, "Migration run loaded", summarizeMigrationRun(loadedRun)) + writeLiveMigrationRunText(stdout, "Migration run loaded", summarizeMigrationRunForLive(ctx, loadedRun)) if err := validateLiveApplyReady(loadedRun); err != nil { return err } @@ -331,7 +342,7 @@ func runMigrate(ctx context.Context, args []string, stdout, stderr io.Writer) er return err } - summary := summarizeMigrationRun(loadedRun) + summary := summarizeMigrationRunContext(ctx, loadedRun) writeMigrationRunText(stdout, "Migration run created", summary) return nil } @@ -387,11 +398,11 @@ func refreshMigrationRunSafelyLocked(runRef string) (loadedMigrationRun, error) if err != nil { return loadedMigrationRun{}, err } - hasAppliedSteps, err := runHasAppliedSteps(run) + liveExecutionStarted, err := runHasStartedLiveExecution(run) if err != nil { return loadedMigrationRun{}, fmt.Errorf("read applied migration progress: %w", err) } - if run.LiveAppliedAt != nil || run.CommittedAt != nil || run.PurgedAt != nil || hasAppliedSteps { + if run.LiveAppliedAt != nil || run.CommitStartedAt != nil || run.CommittedAt != nil || run.RollbackStartedAt != nil || run.RolledBackAt != nil || run.PurgedAt != nil || liveExecutionStarted { return loadMigrationRun(runRef) } return refreshMigrationRunLocked(runRef) @@ -402,10 +413,19 @@ func validateLiveApplyReady(run loadedMigrationRun) error { return fmt.Errorf("live apply refused: run %q was rolled back; start a fresh migration run to migrate again", run.Run.Name) } if run.Run.RollbackStartedAt != nil { - return fmt.Errorf("live apply refused: rollback started for run %q; run `%s` to finish recovery", run.Run.Name, runScopedCommand(run, "rollback --live")) + return fmt.Errorf("live apply refused: rollback started for run %q; %s", run.Run.Name, incompleteRollbackRecovery(run)) } if run.Run.CommitStartedAt != nil { - return fmt.Errorf("live apply refused: source retirement started for run %q; run `%s` to finish acceptance", run.Run.Name, runScopedCommand(run, "commit --apply")) + return fmt.Errorf("live apply refused: source retirement started for run %q; %s", run.Run.Name, finishStartedAcceptanceAction(run)) + } + if run.Run.ResolvedAuthority != "" { + return fmt.Errorf("live apply refused: run %q has a manual %s-authority resolution; finish that recovery instead of replaying the ambiguous apply", run.Run.Name, run.Run.ResolvedAuthority) + } + if err := validateStatefulLiveApply(run); err != nil { + if len(interruptedStatefulSourceCleanup(run)) > 0 { + return nil + } + return err } if decisions := liveApplyBlockingDecisions(run); len(decisions) > 0 { decision := decisions[0] @@ -415,9 +435,201 @@ func validateLiveApplyReady(run loadedMigrationRun) error { } return fmt.Errorf("live apply is blocked by %d unresolved requirement(s); next safe step: %s (run `%s` to review this run)", len(decisions), action, runScopedCommand(run, "status")) } + if hasOnlyPlatformRunApps(run) { + return fmt.Errorf("live apply refused: run %q has no migratable applications; platform-role entries are excluded from live apply", run.Run.Name) + } return nil } +func stagedTransferRefusal(run loadedMigrationRun) error { + return stagedTransferRefusalForPlan(livePlanForApplied(run, run.Applied), run.Applied) +} + +func stagedTransferRefusalForPlan(plan dokploy.Plan, applied runApplied) error { + if err := dokploy.ValidateStagedTransfer(plan); err != nil { + return err + } + if appliedHasHistory(applied) { + return nil + } + return dokploy.ValidatePlannedPostgresDataDirs(plan) +} + +// validateStatefulLiveApply refuses to continue a stateful run recorded +// under a plan order that copied state into the deployed target: Dokploy +// cannot durably keep a queued or future deployment from restarting +// target writers mid-copy. current plans transfer state into Bort-owned +// staging volumes before the target exists and need no such guard. +func validateStatefulLiveApply(run loadedMigrationRun) error { + if !appliedPlanTransfersStateInPlace(run.Applied) { + return nil + } + platformApps := map[string]struct{}{} + for _, app := range run.Prepare.Apps { + if strings.EqualFold(strings.TrimSpace(app.Role), "platform") { + platformApps[app.Name] = struct{}{} + } + } + apps := []string{} + seen := map[string]struct{}{} + for _, step := range livePlanForApplied(run, run.Applied).Steps { + if step.Kind != dokploy.StepPauseSource { + continue + } + if _, platform := platformApps[step.App]; platform { + continue + } + if _, exists := seen[step.App]; exists { + continue + } + seen[step.App] = struct{}{} + apps = append(apps, step.App) + } + if len(apps) == 0 { + return nil + } + return fmt.Errorf("live apply refused for stateful app(s) %s: this run was applied with an older plan version that copied state into the deployed target, and Dokploy cannot durably prevent a queued or future deployment from restarting target writers during that copy; Bort cannot continue this run, so complete target setup, state transfer, traffic cutover, and source retirement outside Bort, or create a new run (new runs stage state before the target is deployed)", strings.Join(apps, ", ")) +} + +type sourceCleanupStep struct { + app string + index int +} + +func interruptedStatefulSourceCleanup(run loadedMigrationRun) []sourceCleanupStep { + plan := livePlanForApplied(run, run.Applied) + byIndex := map[int]appliedStep{} + for _, recorded := range run.Applied.Steps { + if recorded.Index < 0 || recorded.Index >= len(plan.Steps) { + return nil + } + byIndex[recorded.Index] = recorded + } + pending := map[string]sourceCleanupStep{} + for index, step := range plan.Steps { + recorded, ok := byIndex[index] + if !ok { + continue + } + if recorded.Kind == string(dokploy.StepResumeSource) && step.Kind == dokploy.StepPauseSource && recorded.App == step.App && recorded.Ref == step.Ref { + if appliedStepCompleted(recorded) { + delete(pending, step.App) + } else if appliedStepMayHaveRun(recorded) { + pending[step.App] = sourceCleanupStep{app: step.App, index: index} + } + continue + } + if !appliedStepMatches(recorded, step) { + return nil + } + if !appliedStepMayHaveRun(recorded) { + continue + } + switch step.Kind { + case dokploy.StepPushImage, dokploy.StepRestoreDataStore, dokploy.StepSyncVolume, + dokploy.StepResumeTarget, dokploy.StepInstallGateway, dokploy.StepActivateRoutes, + dokploy.StepStopCoolifyProxy, dokploy.StepStartDokployProxy: + return nil + case dokploy.StepPauseSource: + pending[step.App] = sourceCleanupStep{app: step.App, index: index} + case dokploy.StepResumeSource: + if appliedStepCompleted(recorded) { + delete(pending, step.App) + } + } + } + cleanup := make([]sourceCleanupStep, 0, len(pending)) + for _, step := range pending { + cleanup = append(cleanup, step) + } + sort.Slice(cleanup, func(i, j int) bool { return cleanup[i].index < cleanup[j].index }) + return cleanup +} + +func recoverInterruptedStatefulSources(ctx context.Context, run loadedMigrationRun, stderr io.Writer, onProgress func(dokploy.StepProgress), refusal error) error { + cleanup := interruptedStatefulSourceCleanup(run) + if len(cleanup) == 0 { + return refusal + } + if err := verifyLocalSourceRun(ctx, run); err != nil { + return errors.Join(refusal, fmt.Errorf("refusing interrupted source cleanup on an unverified Docker host: %w", err)) + } + appliedPath, err := safeRunArtifactPath(run.Run.RunDir, run.Run.Artifacts.Applied) + if err != nil { + return errors.Join(refusal, err) + } + lockPath, err := safeRunArtifactPath(run.Run.RunDir, "apply.lock") + if err != nil { + return errors.Join(refusal, err) + } + lock, err := acquireApplyLock(lockPath) + if err != nil { + return errors.Join(refusal, fmt.Errorf("lock interrupted source cleanup: %w", err)) + } + defer lock.Release() + hostLock, err := acquireDokployLiveOperationLock() + if err != nil { + return errors.Join(refusal, fmt.Errorf("lock Dokploy live operations for interrupted source cleanup: %w", err)) + } + defer hostLock.Release() + if err := ensureDokploySourceCleanupAvailable(run.Run); err != nil { + return errors.Join(refusal, err) + } + ledger, err := newAppliedLedger(appliedPath, run.Run) + if err != nil { + return errors.Join(refusal, err) + } + apps := map[string]struct{}{} + appNames := make([]string, 0, len(cleanup)) + steps := make([]dokploy.Step, 0, len(cleanup)) + for _, item := range cleanup { + apps[item.app] = struct{}{} + appNames = append(appNames, item.app) + steps = append(steps, dokploy.Step{Kind: dokploy.StepResumeSource, App: item.app, Ref: item.app}) + } + prepare := run.Prepare + prepare.Apps = nil + for _, app := range run.Prepare.Apps { + if _, ok := apps[app.Name]; !ok { + continue + } + app.Readiness = "" + app.Gates = nil + prepare.Apps = append(prepare.Apps, app) + } + remap := func(progress dokploy.StepProgress) dokploy.StepProgress { + progress.Index = cleanup[progress.Index].index + return progress + } + beforeStep := func(progress dokploy.StepProgress) error { + return ledger.Record(remap(progress)) + } + progress := func(item dokploy.StepProgress) { + item = remap(item) + if item.Status == dokploy.StepStatusOK || item.Status == dokploy.StepStatusError || item.Status == dokploy.StepStatusSkipped { + if recordErr := ledger.Record(item); recordErr != nil { + fmt.Fprintf(stderr, "warning: failed to record source cleanup for %s: %v\n", item.Step.App, recordErr) + } + } + if onProgress != nil { + onProgress(item) + } + } + plan := dokploy.Plan{Steps: steps, Prepare: prepare, RunDir: run.Run.RunDir, BeforeStep: &beforeStep, OnProgress: &progress} + client := &dokploy.Client{} + if err := client.AdoptHistoricalSourcePause(ctx, plan, appNames); err != nil { + return errors.Join(refusal, fmt.Errorf("record historical source pause ownership: %w", err)) + } + if err := client.Apply(ctx, plan); err != nil { + return errors.Join(refusal, fmt.Errorf("resume source after interrupted historical pause: %w", err)) + } + if err := ledger.Err(); err != nil { + return errors.Join(refusal, fmt.Errorf("persist interrupted source cleanup: %w", err)) + } + fmt.Fprintln(stderr, "stateful recovery: resumed and recorded the source pause cleanup; no state transfer was attempted") + return fmt.Errorf("%w; the previously paused source was resumed and recorded, but state transfer remains disabled", refusal) +} + func liveApplyBlockingDecisions(run loadedMigrationRun) []runDecision { return openFilteredDecisions(run, func(item runDecisionItem) bool { if item.Stage == "prepare" || item.Readiness == preparer.ReadinessBlocked || item.Readiness == preparer.ReadinessNeedsInput { @@ -467,6 +679,67 @@ func acquireRunOperationLock(runRef string) (*applyLock, error) { return lock, err } +func acquireDokployLiveOperationLock() (*applyLock, error) { + return acquireDokployLiveOperationLockWithRecovery(false) +} + +func acquireDokployInstallRecoveryLock() (*applyLock, error) { + return acquireDokployLiveOperationLockWithRecovery(true) +} + +func acquireDokployLiveOperationLockWithRecovery(allowInstallRecovery bool) (*applyLock, error) { + if !dokployLiveOperationsSupported() { + return nil, fmt.Errorf("Dokploy live operations are unavailable on this platform; continue on the Linux source host") + } + lockPath, err := dokployLiveOperationLockPath() + if err != nil { + return nil, err + } + if err := prepareDokployLiveOperationLockPath(lockPath); err != nil { + return nil, err + } + lock, err := acquireApplyLock(lockPath) + if errors.Is(err, errApplyAlreadyRunning) { + return nil, errDokployLiveOperationActive + } + if err != nil { + return nil, err + } + if !allowInstallRecovery { + if err := dokployInstallationRecoveryBlocker(); err != nil { + lock.Release() + return nil, err + } + } + return lock, err +} + +func dokployInstallationRecoveryBlocker() error { + command, found, err := dokployInstallationRecoveryCommand() + if err != nil { + return fmt.Errorf("verify Dokploy installation recovery state: %w", err) + } + if !found { + return nil + } + return fmt.Errorf("%w: run `%s`", errDokployInstallRecoveryRequired, command) +} + +func dokployLiveOperationActive() (bool, error) { + if !dokployLiveOperationsSupported() { + return false, nil + } + lockPath, err := dokployLiveOperationLockPath() + if err != nil { + return false, err + } + active, err := applyLockActive(lockPath) + if err != nil || active { + return active, err + } + return false, dokployInstallationRecoveryBlocker() +} + func runOperationActive(runRef string) (bool, error) { runDir, err := existingRunDir(runRef) if err != nil { @@ -513,6 +786,9 @@ func applyLiveMigrationLocked(ctx context.Context, run loadedMigrationRun, stder if run.Run.Target != "dokploy" { return fmt.Errorf("--live is only supported for target dokploy, got %q", run.Run.Target) } + if err := validateStatefulLiveApply(run); err != nil { + return recoverInterruptedStatefulSources(ctx, run, stderr, onProgress, err) + } var err error run, err = ensureSelfContainedLiveRunLocked(run) if err != nil { @@ -551,28 +827,69 @@ func applyLiveMigrationLocked(ctx context.Context, run loadedMigrationRun, stder if err := validateLiveApplyReady(run); err != nil { return err } - plan := dokploy.PlanFromArtifacts(run.Prepare, run.Sync, run.Cutover) + plan := livePlanForApplied(run, run.Applied) if len(plan.Steps) == 0 { return fmt.Errorf("live migration plan has no executable steps") } + if err := verifyLocalSourceRun(ctx, run); err != nil { + return fmt.Errorf("refusing live migration on an unverified Docker source: %w", err) + } plan.BundleFiles = bundleFiles plan.ApprovedPrepareDecisions = approvedPrepareDecisions(run) - client, err := ensureDokployClient(ctx, run.Run.Target, os.Stdin, stderr, stderr) + if err := stagedTransferRefusalForPlan(plan, run.Applied); err != nil { + return fmt.Errorf("live apply refused before binding run %q to a Dokploy target: %w", run.Run.Name, err) + } + ledger, err := newAppliedLedger(appliedPath, run.Run) + if err != nil { + return err + } + plan.RunName = run.Run.Name + plan.RunDir = run.Run.RunDir + if plan.RunID, err = dokployTrafficRunID(run.Run); err != nil { + return err + } + applied := ledger.Snapshot() + if err := validateApplyResumeAuthority(applied); err != nil { + return fmt.Errorf("%w; %s", err, authorityRecoveryInstruction(run)) + } + var targetLock *applyLock + client, err := ensureBoundDokployClient(ctx, run.Run, run.Run.Target, os.Stdin, stderr, stderr, ledger, dokploySetupLockOptions{ + retain: &targetLock, + revalidate: func() error { + return ensureDokployTrafficRunAvailable(run.Run, plan) + }, + }) + if targetLock != nil { + defer targetLock.Release() + } if err != nil { if err == errDokploySetupSkipped { return nil } return err } - ledger, err := newAppliedLedger(appliedPath, run.Run) - if err != nil { + if targetLock == nil { + return fmt.Errorf("dokploy client became ready without retaining the host operation lock") + } + applied = ledger.Snapshot() + plan.TargetIdentities = appliedTargetIdentities(applied) + resumeFrom := completedApplyPrefix(plan.Steps, applied) + if err := validateDokployTrafficResume(run.Run, plan, applied, client.BaseURL, dokployCredentialID(client.Token), resumeFrom); err != nil { return err } - plan.RunName = run.Run.Name - plan.RunDir = run.Run.RunDir - resumeFrom := completedApplyPrefix(plan.Steps, ledger.Snapshot()) + if resumeFrom < len(plan.Steps) { + if err := ledger.PrepareRetry(resumeFrom); err != nil { + return fmt.Errorf("prepare applied ledger for retry: %w", err) + } + } plan.ResumeFrom = resumeFrom + ownershipIndex := dokployOwnershipStepIndex(plan) beforeStep := func(p dokploy.StepProgress) error { + if p.Index == ownershipIndex { + if err := claimDokployHostOwnership(run.Run, client.BaseURL, dokployCredentialID(client.Token)); err != nil { + return err + } + } return ledger.Record(p) } plan.BeforeStep = &beforeStep @@ -597,15 +914,247 @@ func applyLiveMigrationLocked(ctx context.Context, run loadedMigrationRun, stder if err := client.Apply(ctx, plan); err != nil { return err } + if trafficHandoffStepIndex(plan) >= 0 { + if err := client.ReconcileTargetTrafficAuthority(ctx); err != nil { + return fmt.Errorf("verify Dokploy target traffic authority: %w", err) + } + } + if err := markDokployTrafficTarget(run.Run, client.BaseURL); err != nil { + return fmt.Errorf("record Dokploy target authority: %w", err) + } if err := ledger.Err(); err != nil { return fmt.Errorf("live apply completed but applied ledger could not be persisted: %w", err) } + if err := client.FinalizeSourceResumes(plan); err != nil { + return fmt.Errorf("live apply completed but source resume ownership could not be cleared: %w", err) + } if err := ledger.MarkSucceeded(); err != nil { return fmt.Errorf("live apply completed but its successful outcome could not be persisted: %w", err) } return markRunLiveAppliedLocked(run.Run) } +func ensureBoundDokployClient(ctx context.Context, run migrationRun, target string, stdin io.Reader, stdout, stderr io.Writer, ledger *appliedLedger, lockOptions dokploySetupLockOptions) (*dokploy.Client, error) { + client, acquired, lookupErr := lookupDokployClientForApply(target, lockOptions) + if lookupErr == nil { + if err := ledger.ValidateTargetOrigin(client.BaseURL); err != nil { + if acquired != nil { + acquired.Release() + } + return nil, fmt.Errorf("bind migration run to Dokploy target: %w", err) + } + if err := verifyLocalDokployClient(ctx, client); err != nil { + if acquired != nil { + acquired.Release() + } + return nil, err + } + pingErr := client.Ping(ctx) + if pingErr == nil { + if err := ledger.BindTargetOrigin(client.BaseURL); err != nil { + if acquired != nil { + acquired.Release() + } + return nil, fmt.Errorf("bind migration run to Dokploy target: %w", err) + } + if acquired != nil { + *lockOptions.retain = acquired + } + return client, nil + } + if acquired != nil { + acquired.Release() + } + if err := validateInlineDokploySetup(run, ledger.Snapshot(), client.BaseURL); err != nil { + return nil, fmt.Errorf("dokploy is not reachable at %s: %w; %v", client.BaseURL, pingErr, err) + } + if target != "dokploy" || !stdinIsTerminal(stdin) { + return nil, fmt.Errorf("dokploy is not reachable at %s: %w. run `%s`", client.BaseURL, pingErr, dokployInstallRepairCommand(client.BaseURL)) + } + if err := promptInstallAndBootstrapDokploy(ctx, stdin, stdout, stderr, client.BaseURL, pingErr, lockOptions); err != nil { + return nil, err + } + } else { + var preflightErr *dokployLiveOperationPreflightError + if errors.As(lookupErr, &preflightErr) || target != "dokploy" || !stdinIsTerminal(stdin) { + return nil, lookupErr + } + if err := validateInlineDokploySetup(run, ledger.Snapshot(), ""); err != nil { + return nil, err + } + if err := promptInstallAndBootstrapDokploy(ctx, stdin, stdout, stderr, "", lookupErr, lockOptions); err != nil { + return nil, err + } + } + client, err := lookupDokployClient(target) + if err != nil { + return nil, err + } + if err := ledger.ValidateTargetOrigin(client.BaseURL); err != nil { + return nil, fmt.Errorf("bind migration run to Dokploy target: %w", err) + } + if err := verifyLocalDokployClient(ctx, client); err != nil { + return nil, err + } + if err := client.Ping(ctx); err != nil { + return nil, fmt.Errorf("dokploy ping failed after install/bootstrap: %w", err) + } + if err := ledger.BindTargetOrigin(client.BaseURL); err != nil { + return nil, fmt.Errorf("bind migration run to Dokploy target: %w", err) + } + return client, nil +} + +func dokployInstallRepairCommand(baseURL string) string { + command := "init-target dokploy --install --auth-secret-backup /absolute/path/to/encrypted-or-off-host/dokploy-auth-secret" + if parsed, err := url.Parse(baseURL); err == nil && parsed.Port() != "" { + command += " --install-port " + shellQuote(parsed.Port()) + } + return bortCommand(command + " --dokploy-url " + shellQuote(baseURL)) +} + +var verifyLocalDokployClient = defaultVerifyLocalDokployClient + +var verifyLocalSourceRun = defaultVerifyLocalSourceRun +var verifyLocalSourceEngine = defaultVerifyLocalSourceEngine + +func defaultVerifyLocalSourceRun(ctx context.Context, run loadedMigrationRun) error { + if err := defaultVerifyLocalSourceEngine(ctx, run); err != nil { + return err + } + verifyCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + return dokploy.VerifySourceContainers(verifyCtx, run.Prepare.Apps) +} + +func defaultVerifyLocalSourceEngine(ctx context.Context, run loadedMigrationRun) error { + if err := validateLocalSourceEngineAttestation(run); err != nil { + return err + } + expected := strings.TrimSpace(run.Prepare.SourceDockerEngineID) + verifyCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + actual, err := localdocker.DockerEngineID(verifyCtx) + if err != nil { + return err + } + if actual != expected { + return fmt.Errorf("reviewed Docker engine %q does not match local engine %q", expected, actual) + } + return nil +} + +func validateLocalSourceEngineAttestation(run loadedMigrationRun) error { + recovery := "refresh it from a local Docker scan before live work" + if sourceAttestationCannotRefresh(run) { + recovery = "this run already started live execution and cannot be refreshed; verify and recover or retire its reviewed sources manually" + } + switch strings.ToLower(strings.TrimSpace(run.Run.Source)) { + case "docker", "local-docker", "coolify-local": + case "manifest": + return fmt.Errorf("run was created from an imported manifest, not a local Docker scan; %s", recovery) + case "": + return fmt.Errorf("run was created from an imported bundle, not a local Docker scan; %s", recovery) + default: + return fmt.Errorf("run was created from source %q, not a local Docker scan; %s", run.Run.Source, recovery) + } + source := strings.ToLower(strings.TrimSpace(run.Prepare.Source)) + switch source { + case "docker", "coolify-local", "coolify-local-traefik", "coolify-local-caddy": + default: + return fmt.Errorf("run source %q is not a locally attested Docker scan; %s", run.Prepare.Source, recovery) + } + expected := strings.TrimSpace(run.Prepare.SourceDockerEngineID) + if expected == "" { + return fmt.Errorf("run has no reviewed Docker engine identity; %s", recovery) + } + return nil +} + +func validateLocalSourceAttestation(run loadedMigrationRun) error { + if err := validateLocalSourceEngineAttestation(run); err != nil { + return err + } + return dokploy.ValidateSourceContainerAttestations(run.Prepare.Apps) +} + +func sourceAttestationCannotRefresh(run loadedMigrationRun) bool { + return run.Applied.TargetOrigin != "" || len(run.Applied.Steps) > 0 || run.Run.LiveAppliedAt != nil || run.Run.CommitStartedAt != nil || run.Run.CommittedAt != nil || run.Run.RollbackStartedAt != nil || run.Run.RolledBackAt != nil || run.Run.PurgedAt != nil || run.Run.ResolvedAuthority != "" +} + +func sourceAttestationBlockerWithProbe(ctx context.Context, run loadedMigrationRun, probe bool) (runNextStep, bool) { + if err := validateLocalSourceAttestation(run); err != nil { + if sourceAttestationCannotRefresh(run) { + return runNextStep{ + Action: "inspect and preserve both source and target state, then recover or retire the reviewed sources manually; do not replay this run against replacement containers", + Reason: err.Error(), + Phase: "source-attestation-error", + }, true + } + return runNextStep{ + Action: "create a new named migration run from a local Docker scan on this source host", + Reason: err.Error(), + Phase: "inspection-only", + }, true + } + if !probe { + return runNextStep{}, false + } + if err := verifyLocalSourceRun(ctx, run); err != nil { + action := "restore the reviewed local Docker daemon and source containers, then check this run again; if they were replaced, create a new named migration run from a fresh local scan" + if sourceAttestationCannotRefresh(run) { + action = "inspect and preserve both source and target state, restore the reviewed Docker host and containers if safe, then recover or retire this run manually; do not replay it against replacement containers" + } + if run.Run.ResolvedAuthority == dokployTrafficTarget && run.Run.LiveAppliedAt != nil { + action = fmt.Sprintf("manually verify that the source is retired, then run `%s` to record manual retirement and release host ownership without mutating the unattested source", authorityRecoverySourceRetiredCommand(run)) + } + return runNextStep{ + Action: action, + Reason: "source attestation failed: " + err.Error(), + Phase: "source-attestation-error", + }, true + } + return runNextStep{}, false +} + +func defaultVerifyLocalDokployClient(ctx context.Context, client *dokploy.Client) error { + if err := client.VerifySameDockerHost(ctx); err != nil { + return fmt.Errorf("refusing same-VPS migration through %s: %w", client.BaseURL, err) + } + return nil +} + +type dokployLiveOperationPreflightError struct { + err error +} + +func (e *dokployLiveOperationPreflightError) Error() string { return e.err.Error() } + +func (e *dokployLiveOperationPreflightError) Unwrap() error { return e.err } + +func lookupDokployClientForApply(target string, lockOptions dokploySetupLockOptions) (*dokploy.Client, *applyLock, error) { + client, err := lookupDokployClient(target) + if err != nil || lockOptions.retain == nil || *lockOptions.retain != nil { + return client, nil, err + } + acquired, err := acquireDokployLiveOperationLock() + if err != nil { + return nil, nil, &dokployLiveOperationPreflightError{err: fmt.Errorf("lock Dokploy live operations: %w", err)} + } + if lockOptions.revalidate != nil { + if err := lockOptions.revalidate(); err != nil { + acquired.Release() + return nil, nil, &dokployLiveOperationPreflightError{err: err} + } + } + client, err = lookupDokployClient(target) + if err != nil { + acquired.Release() + return nil, nil, err + } + return client, acquired, nil +} + func approvedPrepareDecisions(run loadedMigrationRun) map[dokploy.PrepareDecision]struct{} { approved := map[dokploy.PrepareDecision]struct{}{} for _, decision := range run.Decisions.Decisions { @@ -779,11 +1328,11 @@ func finalizeAttachedLiveMigration(ctx context.Context, runRef string) error { const liveAttachPollInterval = 2 * time.Second func attachLiveMigration(ctx context.Context, run loadedMigrationRun, appliedPath, lockPath string, stderr io.Writer, onProgress func(dokploy.StepProgress)) error { - plan := dokploy.PlanFromArtifacts(run.Prepare, run.Sync, run.Cutover) pid := readApplyLockPID(lockPath) startedAt := time.Now().UTC() var lastSeen time.Time initialApplied, initialAppliedOK := readRunApplied(appliedPath, run.Run) + plan := livePlanForApplied(run, initialApplied) if initialAppliedOK == nil && initialApplied.SucceededAt != nil && completedApplyPrefix(plan.Steps, initialApplied) >= len(plan.Steps) && len(plan.Steps) > 0 { entries := attachProgressEntries(plan.Steps, initialApplied, time.Time{}) emitAttachProgress(onProgress, entries) @@ -803,7 +1352,7 @@ func attachLiveMigration(ctx context.Context, run loadedMigrationRun, appliedPat if latest, ok := latestAttachProgressEntry(entries); ok { lastSeen = latest.updatedAt if onProgress == nil { - writeAttachTextProgress(stderr, latest.progress, len(plan.Steps), "already recorded") + writeAttachTextProgress(stderr, latest.progress, latest.progress.Total, "already recorded") } } else if onProgress == nil { fmt.Fprintf(stderr, "live mode: 0/%d step(s) already recorded\n", len(plan.Steps)) @@ -822,13 +1371,14 @@ func attachLiveMigration(ctx context.Context, run loadedMigrationRun, appliedPat fmt.Fprintf(stderr, "warning: failed to read applied progress: %v\n", err) continue } + plan = livePlanForApplied(run, applied) entries := attachProgressEntries(plan.Steps, applied, lastSeen) if len(entries) > 0 { emitAttachProgress(onProgress, entries) latest, _ := latestAttachProgressEntry(entries) lastSeen = latest.updatedAt if onProgress == nil { - writeAttachTextProgress(stderr, latest.progress, len(plan.Steps), "recorded") + writeAttachTextProgress(stderr, latest.progress, latest.progress.Total, "recorded") } } if applied.SucceededAt != nil && completedApplyPrefix(plan.Steps, applied) >= len(plan.Steps) { @@ -866,14 +1416,29 @@ type attachProgressEntry struct { func attachProgressEntries(steps []dokploy.Step, applied runApplied, after time.Time) []attachProgressEntry { entries := []attachProgressEntry{} for _, recorded := range applied.Steps { - if recorded.Index < 0 || recorded.Index >= len(steps) || recorded.UpdatedAt.IsZero() || !recorded.UpdatedAt.After(after) { + if recorded.Index < 0 || recorded.UpdatedAt.IsZero() || !recorded.UpdatedAt.After(after) { + continue + } + if !attachStatusRecorded(recorded.Status) { continue } - step := steps[recorded.Index] - if !appliedStepMatches(recorded, step) || !attachStatusRecorded(recorded.Status) { + step := dokploy.Step{Kind: dokploy.StepKind(recorded.Kind), App: recorded.App, Ref: recorded.Ref} + if recorded.Index < len(steps) { + step = steps[recorded.Index] + if !appliedStepMatches(recorded, step) { + if !appliedCleanupMatches(recorded, step) { + continue + } + step = dokploy.Step{Kind: dokploy.StepKind(recorded.Kind), App: recorded.App, Ref: recorded.Ref} + } + } else if !standaloneAppliedCleanup(recorded, len(steps)) { continue } - progress := dokploy.StepProgress{Index: recorded.Index, Total: len(steps), Step: step, Status: dokploy.StepStatus(recorded.Status)} + total := len(steps) + if standaloneAppliedCleanup(recorded, len(steps)) { + total++ + } + progress := dokploy.StepProgress{Index: recorded.Index, Total: total, Step: step, Status: dokploy.StepStatus(recorded.Status)} if recorded.Error != "" { progress.Err = errors.New(recorded.Error) } @@ -887,6 +1452,17 @@ func attachStatusRecorded(status string) bool { return status == string(dokploy.StepStatusOK) || status == string(dokploy.StepStatusSkipped) || status == string(dokploy.StepStatusError) } +func appliedCleanupMatches(recorded appliedStep, planned dokploy.Step) bool { + sameIdentity := recorded.App == planned.App && recorded.Ref == planned.Ref + return sameIdentity && + (planned.Kind == dokploy.StepPauseSource && recorded.Kind == string(dokploy.StepResumeSource) || + planned.Kind == dokploy.StepStopCoolifyProxy && recorded.Kind == string(dokploy.StepStartCoolifyProxy)) +} + +func standaloneAppliedCleanup(recorded appliedStep, stepCount int) bool { + return recorded.Index == stepCount && recorded.Kind == string(dokploy.StepStartCoolifyProxy) && recorded.Ref == "coolify-proxy" +} + func emitAttachProgress(onProgress func(dokploy.StepProgress), entries []attachProgressEntry) { if onProgress == nil { return @@ -928,7 +1504,14 @@ func latestAttachFailure(steps []dokploy.Step, applied runApplied, attachedAt ti if step.Status != string(dokploy.StepStatusError) || step.UpdatedAt.Before(attachedAt.Add(-5*time.Second)) { continue } - if step.Index < 0 || step.Index >= len(steps) || !appliedStepMatches(step, steps[step.Index]) { + if step.Index < 0 || step.Index > len(steps) { + continue + } + if step.Index == len(steps) { + if !standaloneAppliedCleanup(step, len(steps)) { + continue + } + } else if !appliedStepMatches(step, steps[step.Index]) && !appliedCleanupMatches(step, steps[step.Index]) { continue } if !found || step.UpdatedAt.After(latest.UpdatedAt) { @@ -939,61 +1522,13 @@ func latestAttachFailure(steps []dokploy.Step, applied runApplied, attachedAt ti return latest, found } -func ensureDokployClient(ctx context.Context, target string, stdin io.Reader, stdout, stderr io.Writer) (*dokploy.Client, error) { - client, err := lookupDokployClient(target) - if err == nil { - if pingErr := client.Ping(ctx); pingErr == nil { - return client, nil - } else if target == "dokploy" && stdinIsTerminal(stdin) { - if err := promptInstallAndBootstrapDokploy(ctx, stdin, stdout, stderr, client.BaseURL, pingErr); err != nil { - return nil, err - } - return pingConfiguredDokployClient(ctx, target) - } else { - return nil, fmt.Errorf("dokploy is not reachable at %s: %w. run `%s`", client.BaseURL, pingErr, bortCommand("init-target dokploy --install --dokploy-url "+shellQuote(client.BaseURL))) - } - } - if target == "dokploy" && stdinIsTerminal(stdin) { - if err := promptInstallAndBootstrapDokploy(ctx, stdin, stdout, stderr, "", err); err != nil { - return nil, err - } - return pingConfiguredDokployClient(ctx, target) - } - return nil, err -} - -func pingConfiguredDokployClient(ctx context.Context, target string) (*dokploy.Client, error) { - client, err := lookupDokployClient(target) - if err != nil { - return nil, err - } - if err := client.Ping(ctx); err != nil { - return nil, fmt.Errorf("dokploy ping failed after install/bootstrap: %w", err) - } - return client, nil -} - func lookupDokployClient(target string) (*dokploy.Client, error) { - if client, err := dokploy.NewClientFromEnv(); err == nil { + client, envErr := dokploy.NewClientFromEnv() + if envErr == nil { return client, nil } - state, err := readBortState(defaultStatePath()) - if err != nil { - return nil, err - } - creds, ok := state.Targets[target] - if ok && creds.URL != "" && creds.Token != "" { - if err := dokploy.ValidateTokenBaseURL(creds.URL); err != nil { - return nil, err - } - return &dokploy.Client{BaseURL: creds.URL, Token: creds.Token, HTTPClient: &http.Client{Timeout: 30 * time.Second}}, nil - } - return nil, fmt.Errorf("no dokploy credentials available: set %s and %s, or run `%s`", dokploy.EnvBaseURL, dokploy.EnvToken, bortCommand("init-target dokploy --install")) -} - -func resolveDokployClient(ctx context.Context, target string, stdin io.Reader, stderr io.Writer) (*dokploy.Client, error) { - if client, err := lookupDokployClient(target); err == nil { - return client, nil + if strings.TrimSpace(os.Getenv(dokploy.EnvBaseURL)) != "" || strings.TrimSpace(os.Getenv(dokploy.EnvToken)) != "" { + return nil, envErr } state, err := readBortState(defaultStatePath()) if err != nil { @@ -1001,32 +1536,18 @@ func resolveDokployClient(ctx context.Context, target string, stdin io.Reader, s } creds, ok := state.Targets[target] if ok && creds.URL != "" && creds.Token != "" { - if err := dokploy.ValidateTokenBaseURL(creds.URL); err != nil { - return nil, err - } - return &dokploy.Client{BaseURL: creds.URL, Token: creds.Token, HTTPClient: &http.Client{Timeout: 30 * time.Second}}, nil - } - if target == "dokploy" && stdinIsTerminal(stdin) { - fmt.Fprintf(stderr, "no dokploy credentials found; running `%s` interactively\n", bortCommand("init-target dokploy")) - if err := runInitTarget(ctx, nil, stdin, stderr, stderr); err != nil { - return nil, err - } - state, err = readBortState(defaultStatePath()) + baseURL, err := dokploy.NormalizeTokenBaseURL(creds.URL) if err != nil { return nil, err } - creds = state.Targets[target] - if creds.URL != "" && creds.Token != "" { - if err := dokploy.ValidateTokenBaseURL(creds.URL); err != nil { - return nil, err - } - return &dokploy.Client{BaseURL: creds.URL, Token: creds.Token, HTTPClient: &http.Client{Timeout: 30 * time.Second}}, nil - } + return &dokploy.Client{BaseURL: baseURL, Token: creds.Token, HTTPClient: &http.Client{Timeout: 30 * time.Second}}, nil } - return nil, fmt.Errorf("no dokploy credentials available: set %s and %s, or run `%s` first", dokploy.EnvBaseURL, dokploy.EnvToken, bortCommand("init-target dokploy")) + return nil, fmt.Errorf("no dokploy credentials available: set %s and %s; for an existing Dokploy run `%s` (replace 3030 if it uses another published port), or if Dokploy is absent run `%s`", dokploy.EnvBaseURL, dokploy.EnvToken, bortCommand("init-target dokploy --dokploy-url http://127.0.0.1:3030"), bortCommand("init-target dokploy --install --auth-secret-backup /absolute/path/to/encrypted-or-off-host/dokploy-auth-secret")) } -func stdinIsTerminal(stdin io.Reader) bool { +var stdinIsTerminal = defaultStdinIsTerminal + +func defaultStdinIsTerminal(stdin io.Reader) bool { file, ok := stdin.(*os.File) if !ok { return false @@ -1055,7 +1576,7 @@ func existingMutableMigrationRun(runDir, runName string) (migrationRun, error) { if err != nil { return migrationRun{}, fmt.Errorf("refusing to rewrite run %q because its apply ledger cannot be verified: %w", runName, err) } - if len(applied.Steps) > 0 { + if applied.TargetOrigin != "" || len(applied.Steps) > 0 { return migrationRun{}, immutableError() } return migrationRun{}, nil @@ -1078,7 +1599,7 @@ func existingMutableMigrationRun(runDir, runName string) (migrationRun, error) { if err != nil { return migrationRun{}, fmt.Errorf("refusing to rewrite run %q because its apply ledger cannot be verified: %w", runName, err) } - if len(applied.Steps) > 0 { + if applied.TargetOrigin != "" || len(applied.Steps) > 0 { return migrationRun{}, immutableError() } return existing, nil @@ -1296,7 +1817,7 @@ func captureReviewedMigrationBundle(run loadedMigrationRun) (map[string][]byte, } func reviewedMigrationBundleRecovery(run loadedMigrationRun) string { - if len(run.Applied.Steps) > 0 || run.Run.LiveAppliedAt != nil { + if len(run.Applied.Steps) > 0 || run.Applied.TargetOrigin != "" || run.Run.LiveAppliedAt != nil { return "this run has started live execution and cannot be re-planned; reconcile any applied target changes, then create a new run" } return fmt.Sprintf("run `%s` to re-plan before live apply", runScopedCommand(run, "migrate")) @@ -1402,29 +1923,33 @@ func createMigrationRunLocked(opts migrationRunOptions, runDir, runName string, }() run := migrationRun{ - APIVersion: runAPIVersion, - Name: runName, - RunDir: filepath.ToSlash(filepath.Clean(runDir)), - CreatedAt: createdAt, - UpdatedAt: now, - LiveAppliedAt: existingRun.LiveAppliedAt, - CommitStartedAt: existingRun.CommitStartedAt, - CommittedAt: existingRun.CommittedAt, - RollbackStartedAt: existingRun.RollbackStartedAt, - RolledBackAt: existingRun.RolledBackAt, - PurgedAt: existingRun.PurgedAt, - ApplyOutcomeRequired: true, - Source: opts.Source, - BundleDir: bundleDir, - BundleDigest: hex.EncodeToString(bundleDigest[:]), - SourceBundleDir: sourceBundleDir, - ManifestPath: opts.ManifestPath, - Target: opts.Target, - AppName: opts.AppName, - DryRun: true, - ObservationWindowSeconds: opts.ObservationWindowSeconds, - RollbackWindowSeconds: opts.RollbackWindowSeconds, - Artifacts: artifacts, + APIVersion: runAPIVersion, + Name: runName, + RunDir: filepath.ToSlash(filepath.Clean(runDir)), + CreatedAt: createdAt, + UpdatedAt: now, + LiveAppliedAt: existingRun.LiveAppliedAt, + CommitStartedAt: existingRun.CommitStartedAt, + CommittedAt: existingRun.CommittedAt, + RollbackStartedAt: existingRun.RollbackStartedAt, + RolledBackAt: existingRun.RolledBackAt, + PurgedAt: existingRun.PurgedAt, + ResolvedAuthority: existingRun.ResolvedAuthority, + AuthorityResolvedAt: existingRun.AuthorityResolvedAt, + AuthorityFinalizedAt: existingRun.AuthorityFinalizedAt, + HostOwnerReleaseStartedAt: existingRun.HostOwnerReleaseStartedAt, + ApplyOutcomeRequired: true, + Source: opts.Source, + BundleDir: bundleDir, + BundleDigest: hex.EncodeToString(bundleDigest[:]), + SourceBundleDir: sourceBundleDir, + ManifestPath: opts.ManifestPath, + Target: opts.Target, + AppName: opts.AppName, + DryRun: true, + ObservationWindowSeconds: opts.ObservationWindowSeconds, + RollbackWindowSeconds: opts.RollbackWindowSeconds, + Artifacts: artifacts, } if err := writeJSONArtifact(runArtifactPath(runDir, run.Artifacts.Prepare), preparePlan); err != nil { @@ -1539,24 +2064,54 @@ func refreshMigrationRunLockedWithInputs(runRef, bundleOverride, manifestOverrid }, runDir, runNameFromDir(runDir), time.Now().UTC()) } -func runStatus(_ context.Context, args []string, stdout, stderr io.Writer) error { +func runStatus(ctx context.Context, args []string, stdout, stderr io.Writer) error { + if len(args) == 0 { + handled, err := writeStandaloneDokployInstallationRecovery(stdout) + if err != nil || handled { + return err + } + } run, err := loadRunFromArgs("status", args, stderr) if err != nil { return err } - writeAppFirstCockpit(stdout, run) + writeAppFirstCockpitContext(ctx, stdout, run) return nil } -func runNext(_ context.Context, args []string, stdout, stderr io.Writer) error { +func runNext(ctx context.Context, args []string, stdout, stderr io.Writer) error { + if len(args) == 0 { + handled, err := writeStandaloneDokployInstallationRecovery(stdout) + if err != nil || handled { + return err + } + } run, err := loadRunFromArgs("next", args, stderr) if err != nil { return err } - writeRunNextText(stdout, summarizeMigrationRun(run)) + writeRunNextText(stdout, summarizeMigrationRunContext(ctx, run)) return nil } +func writeStandaloneDokployInstallationRecovery(w io.Writer) (bool, error) { + if !dokployLiveOperationsSupported() { + return false, nil + } + command, found, err := dokployInstallationRecoveryCommand() + if err != nil { + return false, fmt.Errorf("read interrupted Dokploy installation recovery command: %w", err) + } + if !found { + return false, nil + } + st := newStyler(w) + fmt.Fprintln(w, st.glyph("Dokploy installation recovery required", sevBad)) + fmt.Fprintf(w, "Next safe step: run `%s` to reconcile the interrupted installation\n", command) + fmt.Fprintln(w, "Reason: the durable installation-recovery marker blocks other host mutations") + return true, nil +} + func loadRunFromArgs(command string, args []string, stderr io.Writer) (loadedMigrationRun, error) { fs := flag.NewFlagSet(command, flag.ContinueOnError) fs.SetOutput(stderr) @@ -1651,6 +2206,18 @@ func loadMigrationRun(runRef string) (loadedMigrationRun, error) { } func summarizeMigrationRun(run loadedMigrationRun) migrationRunSummary { + return summarizeMigrationRunContext(context.Background(), run) +} + +func summarizeMigrationRunContext(ctx context.Context, run loadedMigrationRun) migrationRunSummary { + return summarizeMigrationRunContextWithSourceProbe(ctx, run, true) +} + +func summarizeMigrationRunForLive(ctx context.Context, run loadedMigrationRun) migrationRunSummary { + return summarizeMigrationRunContextWithSourceProbe(ctx, run, false) +} + +func summarizeMigrationRunContextWithSourceProbe(ctx context.Context, run loadedMigrationRun, probeSource bool) migrationRunSummary { summary := migrationRunSummary{Run: run.Run, Readiness: preparer.ReadinessReadyToCreate, Status: preparer.StatusGreen} for _, app := range run.Commit.Apps { @@ -1714,7 +2281,7 @@ func summarizeMigrationRun(run loadedMigrationRun) migrationRunSummary { summary.Decisions = openRunDecisions(run) summary.Progress = progressSummary(run.Progress) summary.FirstGates = firstUnresolvedRunGates(run, 3) - summary.Next = nextSafeStep(run, blockingDecisions) + summary.Next = nextSafeStepContextWithSourceProbe(ctx, run, blockingDecisions, probeSource) return summary } @@ -1732,34 +2299,261 @@ func firstUnresolvedRunGates(run loadedMigrationRun, limit int) []runGateSummary } func nextSafeStep(run loadedMigrationRun, decisions []runDecision) runNextStep { + return nextSafeStepContext(context.Background(), run, decisions) +} + +func nextSafeStepContext(ctx context.Context, run loadedMigrationRun, decisions []runDecision) runNextStep { + return nextSafeStepContextWithSourceProbe(ctx, run, decisions, true) +} + +func nextSafeStepContextWithSourceProbe(ctx context.Context, run loadedMigrationRun, decisions []runDecision, probeSource bool) runNextStep { + sourceBlocker := func() (runNextStep, bool) { + return sourceAttestationBlockerWithProbe(ctx, run, probeSource) + } + if dokployLiveOperationsSupported() && !authorityRecoveryPending(run) { + ownerFinalizationPending, ownerFinalizationErr := completedDokployOwnerFinalization(run) + if ownerFinalizationErr != nil { + return runNextStep{Action: "inspect /var/lib/bort/dokploy-traffic-owner.json before treating this run as complete", Reason: ownerFinalizationErr.Error()} + } + if ownerFinalizationPending { + if run.Run.RolledBackAt != nil { + return runNextStep{Action: fmt.Sprintf("run `%s` to finish releasing rollback host ownership", runScopedCommand(run, "rollback --live")), Reason: "rollback is recorded, but its durable source-authority owner is not released"} + } + return runNextStep{Action: fmt.Sprintf("run `%s` to finish releasing commit host ownership", runScopedCommand(run, "commit --apply")), Reason: "commit is recorded, but its durable target-authority owner is not released"} + } + } if run.Run.PurgedAt != nil { return runNextStep{Action: "migration complete", Reason: "selected source leftovers were purged after target acceptance"} } if run.Run.CommittedAt != nil { return runNextStep{Action: fmt.Sprintf("run `%s` to audit remaining metadata and source leftovers", runScopedCommand(run, "cleanup")), Reason: "the target is accepted and source app containers are retired"} } - if run.Run.RolledBackAt != nil { + if run.Run.RolledBackAt != nil && !authorityRecoveryPending(run) { return runNextStep{Action: "create a new named migration run with `bort migrate --run ` and current `--source` or `--bundle` inputs", Reason: "the rollback returned traffic to the source; the Dokploy target resources remain on the server"} } - if run.Run.RollbackStartedAt != nil { + if !dokployLiveOperationsSupported() { + return unsupportedPlatformNextStep(run) + } + owner, hostOwned, hostOwnerErr := conflictingDokployHostOwner(run.Run) + applyActive, applyActiveErr := false, error(nil) + if run.Run.LiveAppliedAt == nil { + applyActive, applyActiveErr = applyRunActive(run.Run.RunDir) + } + hostOperationActive, hostOperationErr := dokployLiveOperationActive() + if authorityRecoveryPending(run) { + if hostOwnerErr != nil { + return runNextStep{Action: "inspect /var/lib/bort/dokploy-traffic-owner.json before finishing authority recovery", Reason: "manual authority is recorded but lifecycle finalization is incomplete: " + hostOwnerErr.Error()} + } + if applyActiveErr != nil { + return runNextStep{Action: fmt.Sprintf("inspect the live-apply lock for run %s before finishing authority recovery", shellQuote(run.Run.Name)), Reason: "manual authority is recorded but lifecycle finalization is incomplete: " + applyActiveErr.Error()} + } + if blocked, ok := dokployHostOperationBlocker(hostOperationActive, hostOperationErr); ok { + blocked.Reason = "manual authority is recorded but lifecycle finalization is incomplete: " + blocked.Reason + return blocked + } + return runNextStep{ + Action: fmt.Sprintf("run `%s` to finish the recorded manual %s-authority recovery", pendingAuthorityRecoveryCommand(run), run.Run.ResolvedAuthority), + Reason: "the authority decision is durable, but its lifecycle and host-owner finalization did not complete", + } + } + trafficOwnerErr := validateCurrentDokployTrafficOwner(run, rollbackInProgress(run)) + if rollbackInProgress(run) { + if hostOwnerErr != nil { + return runNextStep{Action: "inspect /var/lib/bort/dokploy-traffic-owner.json before continuing rollback", Reason: "rollback started and traffic or source state may already have changed: " + hostOwnerErr.Error()} + } + if applyActiveErr != nil { + return runNextStep{Action: fmt.Sprintf("inspect the live-apply lock for run %s before continuing rollback", shellQuote(run.Run.Name)), Reason: "rollback started and traffic or source state may already have changed: " + applyActiveErr.Error()} + } + if blocked, ok := dokployHostOperationBlocker(hostOperationActive, hostOperationErr); ok { + blocked.Reason = "rollback started and traffic or source state may already have changed: " + blocked.Reason + return blocked + } + if runMayHaveAmbiguousAuthority(run) { + if authorityRecoveryAvailable(run) { + return authorityRecoveryNextStep(run, "authority-ambiguous recovery did not complete; target fencing may be partial and source state may be unchanged") + } + return runNextStep{ + Action: "inspect and preserve both source and target data, establish writer and traffic authority manually, then create a fresh migration run", + Reason: "authority-ambiguous recovery did not complete; target fencing may be partial, source state may be unchanged, and Dokploy cannot durably fence later starts or deployments", + } + } + if trafficOwnerErr != nil { + if authorityRecoveryAvailable(run) { + return authorityRecoveryNextStep(run, "the durable owner remains bound to this run, but automatic rollback cannot prove authority") + } + return runNextStep{ + Action: "inspect and preserve both source and target data, establish writer and traffic authority manually, then create a fresh migration run", + Reason: "the durable Dokploy host owner no longer proves authority for rollback recovery: " + trafficOwnerErr.Error(), + } + } + if _, err := planAutomaticRollback(run); err != nil { + if authorityRecoveryAvailable(run) { + return authorityRecoveryNextStep(run, "automatic stateful rollback is unavailable because Dokploy cannot durably prevent later target deployments") + } + return runNextStep{ + Action: "inspect and preserve both source and target data, establish writer and traffic authority manually, then create a fresh migration run", + Reason: "automatic stateful rollback is unavailable because Dokploy cannot durably prevent later target deployments", + } + } + if blocked, ok := sourceBlocker(); ok { + return blocked + } return runNextStep{Action: fmt.Sprintf("run `%s` to finish recovery", runScopedCommand(run, "rollback --live")), Reason: "rollback started but has not completed; traffic may already be on the source"} } if run.Run.CommitStartedAt != nil { - return runNextStep{Action: fmt.Sprintf("run `%s` to finish acceptance", runScopedCommand(run, "commit --apply")), Reason: "source retirement started; rollback is no longer available"} + if hostOwnerErr != nil { + return runNextStep{Action: "inspect /var/lib/bort/dokploy-traffic-owner.json before continuing source retirement", Reason: "source retirement started and rollback is no longer available: " + hostOwnerErr.Error()} + } + if applyActiveErr != nil { + return runNextStep{Action: fmt.Sprintf("inspect the live-apply lock for run %s before continuing source retirement", shellQuote(run.Run.Name)), Reason: "source retirement started and rollback is no longer available: " + applyActiveErr.Error()} + } + if blocked, ok := dokployHostOperationBlocker(hostOperationActive, hostOperationErr); ok { + blocked.Reason = "source retirement started and rollback is no longer available: " + blocked.Reason + return blocked + } + if trafficOwnerErr != nil { + return runNextStep{ + Action: "inspect and preserve both source and target state, establish traffic authority manually, then complete source retirement manually", + Reason: "the durable Dokploy host owner no longer proves target authority for the interrupted commit: " + trafficOwnerErr.Error(), + } + } + if blocked, ok := sourceBlocker(); ok { + return blocked + } + return runNextStep{Action: finishStartedAcceptanceAction(run), Reason: "source retirement started; rollback is no longer available"} } - if run.Run.LiveAppliedAt != nil || liveApplySucceeded(run) { - return runNextStep{Action: fmt.Sprintf("verify the target, then run `%s` after the rollback window", runScopedCommand(run, "commit --apply")), Reason: "the live apply completed and the source remains available for rollback"} + if applyActive { + if hostOwnerErr != nil { + return runNextStep{Action: "inspect /var/lib/bort/dokploy-traffic-owner.json before retrying live apply", Reason: "live apply is recorded as active: " + hostOwnerErr.Error()} + } + if blocked, ok := dokployHostOperationBlocker(hostOperationActive, hostOperationErr); ok && !hostOperationActive { + blocked.Reason = "live apply is recorded as active: " + blocked.Reason + return blocked + } + if blocked, ok := sourceBlocker(); ok { + return blocked + } + return runNextStep{Action: fmt.Sprintf("run `%s` to view the active apply", liveApplyCommand(run)), Reason: "another process is applying this run"} + } + if hostOwnerErr != nil { + return runNextStep{ + Action: "inspect /var/lib/bort/dokploy-traffic-owner.json before further migration work", + Reason: hostOwnerErr.Error(), + } } - applyActive, applyActiveErr := applyRunActive(run.Run.RunDir) if applyActiveErr != nil { return runNextStep{Action: fmt.Sprintf("inspect the live-apply lock for run %s before retrying", shellQuote(run.Run.Name)), Reason: applyActiveErr.Error()} } - if applyActive { - return runNextStep{Action: fmt.Sprintf("run `%s` to view the active apply", liveApplyCommand(run)), Reason: "another process is applying this run"} + if blocked, ok := dokployHostOperationBlocker(hostOperationActive, hostOperationErr); ok { + return blocked + } + if manualTargetAuthorityRecorded(run) { + if trafficOwnerErr != nil { + return runNextStep{Action: "inspect /var/lib/bort/dokploy-traffic-owner.json before retiring the source", Reason: "manual target authority is recorded, but the durable Dokploy host owner no longer proves it: " + trafficOwnerErr.Error()} + } + if blocked, ok := sourceBlocker(); ok { + return blocked + } + if coolifySourceRetirementRequired(run) { + return runNextStep{Action: "verify the target, then " + manualCoolifySourceRetirementAction(run), Reason: "manual target authority is durable; automatic rollback remains unavailable"} + } + return runNextStep{Action: fmt.Sprintf("verify the target, then run `%s` to retire the source", runScopedCommand(run, "commit --apply")), Reason: "manual target authority is durable; automatic rollback remains unavailable"} + } + if runMayHaveAmbiguousAuthority(run) { + if authorityRecoveryAvailable(run) { + return authorityRecoveryNextStep(run, "the stored run cannot prove writer or traffic authority, and Dokploy cannot durably fence later target starts or deployments") + } + return runNextStep{ + Action: "inspect and preserve both source and target data, establish writer and traffic authority manually, then create a fresh migration run", + Reason: "the stored run cannot prove writer or traffic authority, and Dokploy cannot durably fence later target starts or deployments", + } + } + if run.Run.LiveAppliedAt != nil || liveApplySucceeded(run) { + if trafficOwnerErr != nil { + if authorityRecoveryAvailable(run) { + return authorityRecoveryNextStep(run, "the durable owner remains bound to this run, but automatic actions cannot prove target authority") + } + return runNextStep{ + Action: "inspect and preserve both source and target data, establish writer and traffic authority manually, then create a fresh migration run", + Reason: "the durable Dokploy host owner no longer proves target authority: " + trafficOwnerErr.Error(), + } + } + if blocked, ok := sourceBlocker(); ok { + return blocked + } + if _, err := planAutomaticRollback(run); err != nil { + if authorityRecoveryAvailable(run) { + return authorityRecoveryNextStep(run, "the live stateful migration completed, but Dokploy cannot durably fence target writers for automatic rollback") + } + if coolifySourceRetirementRequired(run) { + return runNextStep{ + Action: "verify the target; if validation fails, preserve both sides and recover authority manually; otherwise, after the rollback window, " + manualCoolifySourceRetirementAction(run), + Reason: "the live stateful migration completed, but Dokploy cannot durably fence target writers for automatic rollback", + } + } + return runNextStep{ + Action: fmt.Sprintf("verify the target; if validation fails, preserve both sides and recover authority manually; otherwise run `%s` after the rollback window", runScopedCommand(run, "commit --apply")), + Reason: "the live stateful migration completed, but Dokploy cannot durably fence target writers for automatic rollback", + } + } + if coolifySourceRetirementRequired(run) { + return runNextStep{Action: "verify the target through the rollback window, then " + manualCoolifySourceRetirementAction(run) + fmt.Sprintf("; if validation fails, run `%s` to roll back stateless traffic", runScopedCommand(run, "rollback --live")), Reason: "the live apply completed and the source remains available for rollback"} + } + return runNextStep{Action: fmt.Sprintf("verify the target, then run `%s` after the rollback window", runScopedCommand(run, "commit --apply")), Reason: "the live apply completed and the source remains available for rollback"} + } + if hostOwned { + return runNextStep{ + Action: fmt.Sprintf("finish %s before applying this run", dokployOwnerRunLabel(owner)), + Reason: fmt.Sprintf("that run owns Dokploy host mutations with %s authority", owner.Authority), + } + } + if err := validateStatefulLiveApply(run); err != nil { + if len(interruptedStatefulSourceCleanup(run)) > 0 { + if blocked, ok := sourceBlocker(); ok { + return blocked + } + return runNextStep{ + Action: fmt.Sprintf("run `%s` to restart the historically paused source without transferring state", liveApplyCommand(run)), + Reason: "the ledger proves no state transfer or traffic handoff started; automatic state migration remains unavailable after cleanup", + } + } + return runNextStep{ + Action: "do not rerun this blocked run; complete target setup, persistent-state transfer, traffic cutover, and source retirement outside Bort, or create a new run (new runs stage state before the target is deployed)", + Reason: err.Error(), + } + } + if step, otherMoved, ok := appliedNewRunRefusal(run.Applied); ok { + if blocked, ok := sourceBlocker(); ok { + return blocked + } + if otherMoved { + reason := fmt.Sprintf("no retry of this run can pass pause_source for app %s, and an earlier app's source may already be paused or handed off: %s", step.App, step.Error) + action := "inspect and preserve both source and target data, restore the earlier apps' source writers and traffic manually" + if authorityRecoveryAvailable(run) { + action += fmt.Sprintf(", run `%s` to release host ownership", authorityRecoveryCommand(run, dokployTrafficSource)) + } + return runNextStep{ + Action: action + ", then create a fresh migration run (target authority is unavailable because app " + step.App + " has no target copy)", + Reason: reason, + } + } + return runNextStep{ + Action: fmt.Sprintf("the source is still running and no state was transferred: delete or reconcile the Dokploy resources this run created, run `%s` to release host ownership, then choose `%s` or change the source compose, and scan a new run", authorityRecoveryCommand(run, dokployTrafficSource), bortCommand("data --recreate|--managed")), + Reason: step.Error, + } } if len(run.Applied.Steps) > 0 { + if blocked, ok := sourceBlocker(); ok { + return blocked + } return runNextStep{Action: fmt.Sprintf("run `%s` to resume the interrupted apply", liveApplyCommand(run)), Reason: "the apply ledger contains incomplete work"} } + if err := stagedTransferRefusal(run); err != nil { + return runNextStep{ + Action: fmt.Sprintf("for a data store volume, choose `%s`, then %s; otherwise change the source compose and scan a new run", bortCommand("data --recreate|--managed"), reviewedMigrationBundleRecovery(run)), + Reason: err.Error(), + } + } if decisions == nil { decisions = liveApplyBlockingDecisions(run) } @@ -1772,7 +2566,12 @@ func nextSafeStep(run loadedMigrationRun, decisions []runDecision) runNextStep { DecisionID: decision.ID, } } - + if !hasMigratableRunApps(run) { + return runNextStep{Action: "create a new migration run that selects at least one non-platform application", Reason: "this run contains no migratable applications; platform-role entries are excluded from live apply"} + } + if blocked, ok := sourceBlocker(); ok { + return blocked + } return runNextStep{ Action: fmt.Sprintf("run `%s` to apply the planned steps against the target", liveApplyCommand(run)), Reason: "all setup requirements are resolved; interactive target setup runs inline if needed", @@ -1780,6 +2579,119 @@ func nextSafeStep(run loadedMigrationRun, decisions []runDecision) runNextStep { } } +func dokployHostOperationBlocker(active bool, err error) (runNextStep, bool) { + if err != nil { + if errors.Is(err, errDokployInstallRecoveryRequired) { + command, found, recoveryErr := dokployInstallationRecoveryCommand() + if recoveryErr != nil || !found { + return runNextStep{Action: "inspect /var/lib/bort/dokploy-live.lock.install-recovery-required before further host mutation", Reason: errors.Join(err, recoveryErr).Error()}, true + } + return runNextStep{Action: fmt.Sprintf("run `%s` to reconcile the interrupted installation", command), Reason: errDokployInstallRecoveryRequired.Error()}, true + } + return runNextStep{Action: "inspect the host-wide Dokploy operation lock before changing this run", Reason: err.Error()}, true + } + if active { + return runNextStep{Action: "wait for the active Dokploy host operation to finish, then check this run again", Reason: "another process holds the host-wide Dokploy operation lock"}, true + } + return runNextStep{}, false +} + +func unsupportedPlatformNextStep(run loadedMigrationRun) runNextStep { + phase := migrationRunPhase(run) + action := "inspect this run here, then continue it on the Linux source host" + reason := "Dokploy live actions and their host-wide lock are available only on Linux" + switch phase { + case "empty": + action = "create a new migration run that selects at least one non-platform application" + reason = "this run contains only platform-role entries, which Bort excludes from live apply" + case "lock-error": + action = fmt.Sprintf("inspect the live-apply lock for run %s, then continue recovery on the Linux source host", shellQuote(run.Run.Name)) + reason = "the run's apply.lock state could not be verified" + case "host-lock-error": + action = "inspect /var/lib/bort/dokploy-live.lock, then continue recovery on the Linux source host" + reason = "the host-wide Dokploy operation lock could not be verified" + case "host-owner-error": + action = "inspect /var/lib/bort/dokploy-traffic-owner.json, then continue recovery on the Linux source host" + reason = "Dokploy host ownership could not be verified" + case "authority-finalizing": + action = fmt.Sprintf("continue the recorded manual %s-authority finalization on the Linux source host", run.Run.ResolvedAuthority) + reason = "the authority decision is durable, but lifecycle or host-owner finalization is incomplete" + case "authority-ambiguous", "authority-ambiguous-rollback": + action = "preserve both sides, then establish writer and traffic authority on the Linux source host" + reason = "the stored run cannot prove current writer or traffic authority" + case "applied": + action = "verify the live target here, then continue acceptance or rollback on the Linux source host" + reason = "the target is recorded live and the source remains pending acceptance or recovery" + case "committing": + action = "finish source retirement on the Linux source host" + reason = "commit started and rollback is no longer available" + case "rolling back": + action = "finish rollback recovery on the Linux source host" + reason = "rollback started and may already have changed traffic or source state" + case "source-recovery": + action = "resume the historically paused source on the Linux source host" + reason = "the durable ledger records an incomplete source cleanup" + case "manual-state": + action = "complete state transfer and authority recovery manually on the Linux source host" + reason = "automatic stateful migration is unavailable" + case "plan-blocked": + action = "choose a data store strategy and re-plan, or change the source compose and scan a new run, on the Linux source host" + reason = "live apply would refuse the planned state transfer" + case "new-run-required": + action = "recover this run's authority on the Linux source host, then choose a data store strategy or change the source compose and create a new run" + reason = "live apply refused the planned state transfer, and this run cannot be re-planned" + case "partial", "applying": + action = "inspect the durable apply ledger, then resume on the Linux source host" + reason = "live apply has incomplete work" + case "planning": + action = "resolve the recorded planning requirements before continuing on the Linux source host" + reason = "the run is not ready for live apply" + } + return runNextStep{Action: action, Reason: reason} +} + +// appliedNewRunRefusal reports a pause_source refusal that no retry can +// pass, and whether the ledger leaves another source paused or any state +// or traffic moved: the run can only be released when it did neither. +func appliedNewRunRefusal(applied runApplied) (refused appliedStep, otherMoved bool, found bool) { + pausedApps := map[string]struct{}{} + for _, step := range applied.Steps { + if step.RequiresNewRun && step.Status == string(dokploy.StepStatusError) { + refused, found = step, true + continue + } + if !appliedStepMayHaveRun(step) { + continue + } + switch dokploy.StepKind(step.Kind) { + case dokploy.StepPauseSource: + pausedApps[step.App] = struct{}{} + case dokploy.StepResumeSource: + if appliedStepCompleted(step) { + delete(pausedApps, step.App) + } else { + otherMoved = true + } + case dokploy.StepDumpDataStore, dokploy.StepRestoreDataStore, dokploy.StepSyncVolume, + dokploy.StepResumeTarget, dokploy.StepInstallGateway, dokploy.StepActivateRoutes, dokploy.StepStopCoolifyProxy, dokploy.StepStartDokployProxy: + otherMoved = true + } + } + return refused, otherMoved || len(pausedApps) > 0, found +} + +func appliedRequiresNewRun(applied runApplied) bool { + _, _, ok := appliedNewRunRefusal(applied) + return ok +} + +func authorityRecoveryNextStep(run loadedMigrationRun, reason string) runNextStep { + return runNextStep{ + Action: fmt.Sprintf("manually fence the other side and verify authority, then run `%s` for source or `%s` for target", authorityRecoveryCommand(run, dokployTrafficSource), authorityRecoveryCommand(run, dokployTrafficTarget)), + Reason: reason, + } +} + func liveApplyCommand(run loadedMigrationRun) string { return runScopedCommand(run, "migrate --live") } @@ -2337,6 +3249,19 @@ func isPlatformRunApp(role string) bool { return strings.EqualFold(strings.TrimSpace(role), "platform") } +func hasMigratableRunApps(run loadedMigrationRun) bool { + for _, app := range run.Prepare.Apps { + if !isPlatformRunApp(app.Role) { + return true + } + } + return false +} + +func hasOnlyPlatformRunApps(run loadedMigrationRun) bool { + return len(run.Prepare.Apps) > 0 && !hasMigratableRunApps(run) +} + func (counts *runStatusCounts) add(status preparer.Status) { switch status { case preparer.StatusRed: @@ -2495,6 +3420,18 @@ func readRunMetadata(path string) (migrationRun, error) { if !run.DryRun { return migrationRun{}, fmt.Errorf("%s is not a dry-run migration run", path) } + if (run.ResolvedAuthority == "") != (run.AuthorityResolvedAt == nil) { + return migrationRun{}, fmt.Errorf("%s has an incomplete manual authority resolution", path) + } + if run.ResolvedAuthority != "" && run.ResolvedAuthority != dokployTrafficSource && run.ResolvedAuthority != dokployTrafficTarget { + return migrationRun{}, fmt.Errorf("%s has invalid resolved authority %q", path, run.ResolvedAuthority) + } + if run.AuthorityFinalizedAt != nil && (run.ResolvedAuthority != dokployTrafficSource || run.RolledBackAt == nil) { + return migrationRun{}, fmt.Errorf("%s has invalid manual authority finalization", path) + } + if run.HostOwnerReleaseStartedAt != nil && run.CommittedAt == nil && run.RolledBackAt == nil { + return migrationRun{}, fmt.Errorf("%s records a host-owner release before commit or rollback", path) + } return run, nil } @@ -2506,6 +3443,31 @@ func markRunLiveAppliedLocked(run migrationRun) error { }) } +func markRunAuthorityResolvedLocked(run migrationRun, authority string) error { + return updateRunLifecycleLocked(run, func(current *migrationRun, now time.Time) { + if current.ResolvedAuthority == "" { + current.ResolvedAuthority = authority + current.AuthorityResolvedAt = &now + } + }) +} + +func markRunAuthorityFinalizedLocked(run migrationRun) error { + return updateRunLifecycleLocked(run, func(current *migrationRun, now time.Time) { + if current.AuthorityFinalizedAt == nil { + current.AuthorityFinalizedAt = &now + } + }) +} + +func markRunHostOwnerReleaseStartedLocked(run migrationRun) error { + return updateRunLifecycleLocked(run, func(current *migrationRun, now time.Time) { + if current.HostOwnerReleaseStartedAt == nil { + current.HostOwnerReleaseStartedAt = &now + } + }) +} + func markRunCommitStartedLocked(run migrationRun) error { return updateRunLifecycleLocked(run, func(current *migrationRun, now time.Time) { if current.CommitStartedAt == nil { diff --git a/internal/cli/run_test.go b/internal/cli/run_test.go index 8b24991..121454d 100644 --- a/internal/cli/run_test.go +++ b/internal/cli/run_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/hex" + "encoding/json" "errors" "io" "net/http" @@ -11,6 +12,7 @@ import ( "os" "path/filepath" "reflect" + "runtime" "strings" "testing" "time" @@ -30,9 +32,9 @@ func TestRunMigrateCreatesLocalRunArtifactsAndSummary(t *testing.T) { t.Chdir(workDir) bundleDir := filepath.Join(workDir, "bort-bundle") writeTestBundle(t, bundleDir, manifest.Manifest{ - Source: manifest.Source{Platform: "docker"}, + Source: manifest.Source{Platform: "docker", DockerEngineID: "engine-reviewed"}, Apps: []manifest.App{ - {Name: "api", Services: []manifest.Service{{Name: "api", Image: "example/api:latest"}}, Routes: []manifest.Route{{Host: "api.example.com", ServiceName: "api", Port: "3000"}}}, + {Name: "api", Services: []manifest.Service{{ID: "0123456789ab", Name: "api", Image: "example/api:latest"}}, Routes: []manifest.Route{{Host: "api.example.com", ServiceName: "api", Port: "3000"}}}, }, }) @@ -93,13 +95,16 @@ func TestRunMigrateCreatesLocalRunArtifactsAndSummary(t *testing.T) { "Routes: 1 cutover, 1 rollback, 1 commit", "Decisions: 3 open", "Open decisions:", - "Next safe step: run `bort migrate --live --run demo-app`", + "Next safe step: create a new named migration run from a local Docker scan on this source host", "Dry run only: no target resources, sync operations, route changes, ownership commits, or source cleanup were executed.", } { if !strings.Contains(output, want) { t.Fatalf("expected migrate output to contain %q, got:\n%s", want, output) } } + if strings.Contains(output, "migrate --live") { + t.Fatalf("imported bundle run must not recommend live apply, got:\n%s", output) + } } func TestRunMigrateLiveUsesCurrentRunInsteadOfDefaultBundle(t *testing.T) { @@ -132,6 +137,12 @@ func TestRunMigrateLiveUsesCurrentRunInsteadOfDefaultBundle(t *testing.T) { if !strings.Contains(err.Error(), "no dokploy credentials available") { t.Fatalf("expected the reviewed run to reach target setup, got err=%v stderr=%s stdout=%s", err, stderr.String(), stdout.String()) } + if !strings.Contains(err.Error(), "init-target dokploy --dokploy-url http://127.0.0.1:3030") || !strings.Contains(err.Error(), "replace 3030") { + t.Fatalf("generated credential guidance omitted existing-Dokploy bootstrap: %v", err) + } + if !strings.Contains(err.Error(), "--auth-secret-backup /absolute/path/to/encrypted-or-off-host/dokploy-auth-secret") { + t.Fatalf("generated install command omitted required authentication-secret escrow: %v", err) + } if !strings.Contains(stdout.String(), "Migration run loaded: .bort/runs/coolify-local") { t.Fatalf("expected live migrate to load the current run, got stdout=%s", stdout.String()) } @@ -191,6 +202,9 @@ func TestRunMigrateLivePreservesReviewedArtifactsAfterBundleChanges(t *testing.T if err == nil || !strings.Contains(err.Error(), "no dokploy credentials available") { t.Fatalf("expected live apply to stop at missing dokploy credentials, got %v", err) } + if !strings.Contains(err.Error(), "--auth-secret-backup /absolute/path/to/encrypted-or-off-host/dokploy-auth-secret") { + t.Fatalf("generated install command omitted required authentication-secret escrow: %v", err) + } after, err := os.ReadFile(preparePath) if err != nil { t.Fatal(err) @@ -534,25 +548,216 @@ func TestApprovedPrepareDecisionsRequireResolvedOrSkippedProgress(t *testing.T) } } -func TestEnsureDokployClientShellQuotesRepairURL(t *testing.T) { +func TestEnsureBoundDokployClientRejectsOriginBeforeCredentialedRequest(t *testing.T) { + requests := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + requests++ + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + t.Setenv(dokploy.EnvBaseURL, server.URL) + t.Setenv(dokploy.EnvToken, "secret") + path := filepath.Join(t.TempDir(), "applied.json") + ledger, err := newAppliedLedger(path, migrationRun{Name: "bound", Target: "dokploy"}) + if err != nil { + t.Fatal(err) + } + if err := ledger.BindTargetOrigin("http://127.0.0.1:1"); err != nil { + t.Fatal(err) + } + _, err = ensureBoundDokployClient(context.Background(), migrationRun{Name: "bound"}, "dokploy", strings.NewReader(""), io.Discard, io.Discard, ledger, dokploySetupLockOptions{}) + if err == nil || !strings.Contains(err.Error(), "bound to Dokploy origin") { + t.Fatalf("expected origin mismatch, got %v", err) + } + if requests != 0 { + t.Fatalf("origin mismatch sent credentials before refusal, requests=%d", requests) + } +} + +func TestEnsureBoundDokployClientRejectsNonLoopbackBeforeCredentialedRequest(t *testing.T) { + previousVerifier := verifyLocalDokployClient + verifyLocalDokployClient = defaultVerifyLocalDokployClient + t.Cleanup(func() { verifyLocalDokployClient = previousVerifier }) + requests := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + requests++ + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + nonLoopbackURL := strings.Replace(server.URL, "127.0.0.1", "0.0.0.0", 1) + if nonLoopbackURL == server.URL { + t.Fatalf("test server URL %q did not bind 127.0.0.1", server.URL) + } + t.Setenv(dokploy.EnvBaseURL, nonLoopbackURL) + t.Setenv(dokploy.EnvToken, "secret") + path := filepath.Join(t.TempDir(), "applied.json") + ledger, err := newAppliedLedger(path, migrationRun{Name: "non-loopback", Target: "dokploy"}) + if err != nil { + t.Fatal(err) + } + _, err = ensureBoundDokployClient(context.Background(), migrationRun{Name: "non-loopback"}, "dokploy", strings.NewReader(""), io.Discard, io.Discard, ledger, dokploySetupLockOptions{}) + if err == nil || !strings.Contains(err.Error(), "non-loopback http URL") { + t.Fatalf("expected non-loopback refusal, got %v", err) + } + if requests != 0 { + t.Fatalf("non-loopback target received %d credentialed request(s)", requests) + } + if origin := ledger.Snapshot().TargetOrigin; origin != "" { + t.Fatalf("non-loopback target permanently bound the run to %q", origin) + } +} + +func TestEnsureBoundDokployClientDoesNotBindUnreachableOrigin(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { http.Error(w, "unavailable", http.StatusServiceUnavailable) })) defer server.Close() - baseURL := server.URL + "?source=a&mode=b" + t.Setenv(dokploy.EnvBaseURL, server.URL) + t.Setenv(dokploy.EnvToken, "secret") + path := filepath.Join(t.TempDir(), "applied.json") + ledger, err := newAppliedLedger(path, migrationRun{Name: "unreachable", Target: "dokploy"}) + if err != nil { + t.Fatal(err) + } + _, err = ensureBoundDokployClient(context.Background(), migrationRun{Name: "unreachable"}, "dokploy", strings.NewReader(""), io.Discard, io.Discard, ledger, dokploySetupLockOptions{}) + if err == nil || !strings.Contains(err.Error(), "not reachable") { + t.Fatalf("expected reachability error, got %v", err) + } + if origin := ledger.Snapshot().TargetOrigin; origin != "" { + t.Fatalf("unreachable target permanently bound the run to %q", origin) + } +} + +func TestEnsureBoundDokployClientQuotesRepairURL(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + http.Error(w, "unavailable", http.StatusServiceUnavailable) + })) + defer server.Close() + baseURL := server.URL + "/a;b&c" t.Setenv(dokploy.EnvBaseURL, baseURL) t.Setenv(dokploy.EnvToken, "secret") + path := filepath.Join(t.TempDir(), "applied.json") + ledger, err := newAppliedLedger(path, migrationRun{Name: "quoted", Target: "dokploy"}) + if err != nil { + t.Fatal(err) + } + _, err = ensureBoundDokployClient(context.Background(), migrationRun{Name: "quoted"}, "dokploy", strings.NewReader(""), io.Discard, io.Discard, ledger, dokploySetupLockOptions{}) + if err == nil { + t.Fatal("expected reachability error") + } + if !strings.Contains(err.Error(), "--dokploy-url '"+baseURL+"'") { + t.Fatalf("repair command does not shell-quote the URL: %v", err) + } + port := strings.TrimPrefix(server.URL, "http://127.0.0.1:") + if !strings.Contains(err.Error(), "--install-port "+port+" ") { + t.Fatalf("repair command does not install on the URL's port %s: %v", port, err) + } +} - _, err := ensureDokployClient(context.Background(), "dokploy", strings.NewReader(""), io.Discard, io.Discard) - if err == nil || !strings.Contains(err.Error(), "--dokploy-url '"+baseURL+"'") { - t.Fatalf("expected shell-quoted repair URL, got %v", err) +func TestEnsureBoundDokployClientWithoutTerminalDoesNotOfferInstallForBoundRun(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + http.Error(w, "unavailable", http.StatusServiceUnavailable) + })) + defer server.Close() + t.Setenv(dokploy.EnvBaseURL, server.URL) + t.Setenv(dokploy.EnvToken, "secret") + path := filepath.Join(t.TempDir(), "applied.json") + ledger, err := newAppliedLedger(path, migrationRun{Name: "bound", Target: "dokploy"}) + if err != nil { + t.Fatal(err) + } + if err := ledger.BindTargetOrigin(server.URL); err != nil { + t.Fatal(err) + } + _, err = ensureBoundDokployClient(context.Background(), migrationRun{Name: "bound"}, "dokploy", strings.NewReader(""), io.Discard, io.Discard, ledger, dokploySetupLockOptions{}) + if err == nil || !strings.Contains(err.Error(), "not reachable") || !strings.Contains(err.Error(), "restore the exact original target credential") { + t.Fatalf("expected bound-run reachability error with credential-restore guidance, got %v", err) + } + if strings.Contains(err.Error(), "init-target") { + t.Fatalf("bound run was offered a fresh install: %v", err) + } +} + +func TestEnsureBoundDokployClientDoesNotOfferInstallWhileHostLockIsHeld(t *testing.T) { + resetDokployTrafficOwner(t) + stdinIsTerminal = func(io.Reader) bool { return true } + defer func() { stdinIsTerminal = defaultStdinIsTerminal }() + t.Setenv(dokploy.EnvBaseURL, "http://127.0.0.1:3030") + t.Setenv(dokploy.EnvToken, "test-token") + path := filepath.Join(t.TempDir(), "applied.json") + ledger, err := newAppliedLedger(path, migrationRun{Name: "lock-held", Target: "dokploy"}) + if err != nil { + t.Fatal(err) + } + if err := ledger.BindTargetOrigin("http://127.0.0.1:3030"); err != nil { + t.Fatal(err) + } + held, err := acquireDokployLiveOperationLock() + if err != nil { + t.Fatal(err) + } + defer held.Release() + var retained *applyLock + _, err = ensureBoundDokployClient(context.Background(), migrationRun{Name: "lock-held"}, "dokploy", strings.NewReader(""), io.Discard, io.Discard, ledger, dokploySetupLockOptions{retain: &retained}) + if !errors.Is(err, errDokployLiveOperationActive) { + t.Fatalf("expected the held host lock to surface directly, got %v", err) + } + if retained != nil { + retained.Release() + t.Fatal("lock-blocked apply retained a host lock") + } +} + +func TestEnsureBoundDokployClientReloadsCredentialsAfterHostLock(t *testing.T) { + resetDokployTrafficOwner(t) + firstRequests := 0 + first := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + firstRequests++ + w.WriteHeader(http.StatusOK) + })) + defer first.Close() + secondRequests := 0 + second := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + secondRequests++ + w.WriteHeader(http.StatusOK) + })) + defer second.Close() + t.Setenv(dokploy.EnvBaseURL, first.URL) + t.Setenv(dokploy.EnvToken, "first-token") + path := filepath.Join(t.TempDir(), "applied.json") + ledger, err := newAppliedLedger(path, migrationRun{Name: "credential-race", Target: "dokploy"}) + if err != nil { + t.Fatal(err) + } + var retained *applyLock + client, err := ensureBoundDokployClient(context.Background(), migrationRun{Name: "credential-race"}, "dokploy", strings.NewReader(""), io.Discard, io.Discard, ledger, dokploySetupLockOptions{ + retain: &retained, + revalidate: func() error { + t.Setenv(dokploy.EnvBaseURL, second.URL) + t.Setenv(dokploy.EnvToken, "second-token") + return nil + }, + }) + if retained != nil { + defer retained.Release() + } + if err != nil { + t.Fatal(err) + } + if client.BaseURL != second.URL || ledger.Snapshot().TargetOrigin != second.URL { + t.Fatalf("apply used credentials read before the host lock: client=%q origin=%q", client.BaseURL, ledger.Snapshot().TargetOrigin) + } + if firstRequests != 0 || secondRequests != 1 { + t.Fatalf("unexpected credentialed requests: first=%d second=%d", firstRequests, secondRequests) } } func TestApplyLiveMigrationRejectsEmptyPlan(t *testing.T) { - runDir := t.TempDir() + workDir := t.TempDir() + t.Chdir(workDir) + runDir := filepath.Join(workDir, ".bort", "runs", "empty") bundleDir := filepath.Join(runDir, "bundle") - if err := os.Mkdir(bundleDir, 0o700); err != nil { + if err := os.MkdirAll(bundleDir, 0o700); err != nil { t.Fatal(err) } bundleDigest, err := digestMigrationBundle(bundleDir) @@ -576,6 +781,53 @@ func TestApplyLiveMigrationRejectsEmptyPlan(t *testing.T) { } } +func TestApplyLiveMigrationRefusesUnstageablePlanBeforeBindingTarget(t *testing.T) { + workDir := t.TempDir() + t.Chdir(workDir) + runDir := filepath.Join(workDir, ".bort", "runs", "bind") + bundleDir := filepath.Join(runDir, "bundle") + if err := os.MkdirAll(bundleDir, 0o700); err != nil { + t.Fatal(err) + } + bundleDigest, err := digestMigrationBundle(bundleDir) + if err != nil { + t.Fatal(err) + } + previous := verifyLocalSourceRun + verifyLocalSourceRun = func(context.Context, loadedMigrationRun) error { return nil } + t.Cleanup(func() { verifyLocalSourceRun = previous }) + app := preparer.AppPlan{ + Name: "api", + Resources: preparer.ResourceSpecs{SourceServices: []preparer.SourceServiceRef{{ContainerID: "0123456789ab", ContainerName: "api"}}}, + TargetResources: &preparer.TargetResources{Dokploy: &preparer.DokployResources{ComposeApp: preparer.DokployComposeApp{Name: "api"}}}, + } + app.Resources.Volumes = []preparer.VolumeResource{{Service: "web", Type: "bind", Source: "/srv/data", Target: "/data", SourceContainerID: "0123456789ab"}} + run := loadedMigrationRun{ + Run: migrationRun{ + Name: "bind", + RunDir: runDir, + BundleDir: bundleDir, + BundleDigest: hex.EncodeToString(bundleDigest[:]), + Target: "dokploy", + Source: "docker", + Artifacts: defaultRunArtifacts(), + }, + Prepare: preparer.Result{BundleDir: bundleDir, Source: "docker", SourceDockerEngineID: "engine-reviewed", Apps: []preparer.AppPlan{app}}, + Sync: syncplan.Result{Apps: []syncplan.AppPlan{{Name: "api", Steps: []syncplan.Step{{ + ResourceType: "volume", + ResourceRef: "volume:web -> /data", + Strategy: syncplan.StrategyDockerVolumeArchive, + }}}}}, + } + err = applyLiveMigrationLocked(context.Background(), run, io.Discard, nil) + if err == nil || !strings.Contains(err.Error(), "before binding") || !strings.Contains(err.Error(), "only named volumes are transferred before deploy") { + t.Fatalf("expected staged-transfer refusal before target binding, got %v", err) + } + if _, statErr := os.Stat(filepath.Join(runDir, run.Run.Artifacts.Applied)); !errors.Is(statErr, os.ErrNotExist) { + t.Fatalf("refusal must not create the applied ledger, stat err=%v", statErr) + } +} + func TestOpenSetupDecisionsRecomputesFilteredMetadata(t *testing.T) { run := loadedMigrationRun{ Decisions: runDecisions{ @@ -618,7 +870,7 @@ func TestNextSafeStepScopesLifecycleCommandsToSelectedRun(t *testing.T) { }{ { name: "target live", - run: migrationRun{Name: "selected-run", LiveAppliedAt: &now}, + run: migrationRun{Name: "selected-run", Source: "docker", LiveAppliedAt: &now}, want: "bort commit --apply --run selected-run", }, { @@ -629,7 +881,18 @@ func TestNextSafeStepScopesLifecycleCommandsToSelectedRun(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - step := nextSafeStep(loadedMigrationRun{Run: tt.run}, nil) + run := loadedMigrationRun{Run: tt.run} + if tt.run.LiveAppliedAt != nil { + run.Prepare = preparer.Result{ + Source: "docker", + SourceDockerEngineID: "engine-reviewed", + Apps: []preparer.AppPlan{{ + Name: "api", + Resources: preparer.ResourceSpecs{SourceServices: []preparer.SourceServiceRef{{ContainerID: "0123456789ab", ContainerName: "api"}}}, + }}, + } + } + step := nextSafeStep(run, nil) if !strings.Contains(step.Action, tt.want) { t.Fatalf("expected next step to contain %q, got %q", tt.want, step.Action) } @@ -649,6 +912,64 @@ func TestRunScopedCommandPreservesExternalRunDirectory(t *testing.T) { } } +func TestAmbiguousLegacyHandoffRequiresManualAuthorityRecovery(t *testing.T) { + workDir := t.TempDir() + t.Chdir(workDir) + localDir := filepath.Join(workDir, ".bort", "runs", "named-run") + externalDir := filepath.Join(t.TempDir(), "external-run") + for _, runDir := range []string{localDir, externalDir} { + if err := os.MkdirAll(runDir, 0o700); err != nil { + t.Fatal(err) + } + } + + tests := []struct { + name string + run loadedMigrationRun + }{ + {name: "named", run: ambiguousLegacyRunForTest("named-run", localDir)}, + {name: "external", run: ambiguousLegacyRunForTest("external-run", externalDir)}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + next := nextSafeStep(test.run, nil) + if !strings.Contains(next.Action, "establish writer and traffic authority manually") || !strings.Contains(next.Action, "fresh migration run") || !strings.Contains(next.Reason, "cannot durably fence") { + t.Fatalf("expected manual authority recovery guidance, got %#v", next) + } + if strings.Contains(next.Action, "rollback") || strings.Contains(next.Action, "--run") { + t.Fatalf("ambiguous legacy guidance offered an unsafe automated recovery command: %#v", next) + } + }) + } +} + +func ambiguousLegacyRunForTest(name, runDir string) loadedMigrationRun { + prepare := preparer.Result{Apps: []preparer.AppPlan{{Name: "api"}}} + cutover := gateway.Result{Apps: []gateway.AppPlan{{Name: "api", Routes: []gateway.Route{{Host: "api.example.com"}}}}} + steps := dokploy.PlanFromArtifacts(prepare, syncplan.Result{}, cutover).Steps + applied := runApplied{RunName: name, Target: "dokploy"} + for index, step := range steps { + status := dokploy.StepStatusOK + switch step.Kind { + case dokploy.StepStartDokployProxy: + status = dokploy.StepStatusStarted + case dokploy.StepActivateRoutes, dokploy.StepResumeTarget: + status = dokploy.StepStatusSkipped + } + applied = recordAppliedStep(applied, dokploy.StepProgress{Index: index, Step: step, Status: status}) + if status == dokploy.StepStatusStarted { + break + } + } + return loadedMigrationRun{ + Run: migrationRun{Name: name, RunDir: runDir, Target: "dokploy"}, + Prepare: prepare, + Sync: syncplan.Result{}, + Cutover: cutover, + Applied: applied, + } +} + func TestRunMigrateLiveRequiresReviewedRun(t *testing.T) { workDir := t.TempDir() t.Chdir(workDir) @@ -734,6 +1055,51 @@ func TestApplyLockActiveSurfacesOpenErrors(t *testing.T) { } } +func TestAcquireApplyLockToleratesBriefProbeHold(t *testing.T) { + lockPath := filepath.Join(t.TempDir(), "apply.lock") + probe, err := tryAcquireApplyLock(lockPath) + if err != nil { + t.Fatal(err) + } + go func() { + time.Sleep(3 * applyLockProbeInterval) + probe.Release() + }() + lock, err := acquireApplyLock(lockPath) + if err != nil { + t.Fatalf("acquire during a brief probe hold: %v", err) + } + lock.Release() + held, err := tryAcquireApplyLock(lockPath) + if err != nil { + t.Fatal(err) + } + defer held.Release() + started := time.Now() + if _, err := acquireApplyLock(lockPath); !errors.Is(err, errApplyAlreadyRunning) { + t.Fatalf("expected a held lock to still be reported busy, got %v", err) + } + if time.Since(started) < applyLockProbeGrace { + t.Fatal("expected the acquirer to wait through the probe grace before reporting busy") + } +} + +func TestDokployLiveOperationProbeDoesNotCreateHostState(t *testing.T) { + original := dokployLiveOperationLockPath + t.Cleanup(func() { dokployLiveOperationLockPath = original }) + parent := filepath.Join(t.TempDir(), "missing") + dokployLiveOperationLockPath = func() (string, error) { + return filepath.Join(parent, "dokploy-live.lock"), nil + } + active, err := dokployLiveOperationActive() + if err != nil || active { + t.Fatalf("expected missing host state to report an inactive lock: active=%t err=%v", active, err) + } + if _, statErr := os.Stat(parent); !os.IsNotExist(statErr) { + t.Fatalf("read-only host lock probe created %s: %v", parent, statErr) + } +} + func TestApplyLockRejectsSymlinkWithoutChangingTarget(t *testing.T) { dir := t.TempDir() targetPath := filepath.Join(dir, "run.json") @@ -824,6 +1190,22 @@ func TestRunOperationLockTracksContentionAndRelease(t *testing.T) { reacquired.Release() } +func TestDokployLiveOperationLockSpansWorkspaces(t *testing.T) { + t.Chdir(t.TempDir()) + first, err := defaultDokployLiveOperationLockPath() + if err != nil { + t.Fatal(err) + } + t.Chdir(t.TempDir()) + second, err := defaultDokployLiveOperationLockPath() + if err != nil { + t.Fatal(err) + } + if !filepath.IsAbs(first) || first != second { + t.Fatalf("expected one absolute host lock path independent of the workspace, got %q and %q", first, second) + } +} + func TestBortStateMutationsSerializeReadModifyWrite(t *testing.T) { statePath := filepath.Join(t.TempDir(), ".bort", "state.json") firstEntered := make(chan struct{}) @@ -944,6 +1326,7 @@ func TestRunMigrateLiveAttachesToHeldApplyDespiteOperationLock(t *testing.T) { } func TestFinalizeAttachedLiveMigrationRecordsLifecycle(t *testing.T) { + resetDokployTrafficOwner(t) workDir := t.TempDir() t.Chdir(workDir) bundleDir := filepath.Join(workDir, "bort-bundle") @@ -994,6 +1377,32 @@ func TestFinalizeAttachedLiveMigrationRecordsLifecycle(t *testing.T) { if err := writeRunApplied(appliedPath, applied); err != nil { t.Fatal(err) } + ownerPath, err := dokployTrafficOwnerPath() + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(ownerPath, []byte("{malformed"), 0o600); err != nil { + t.Fatal(err) + } + if err := finalizeAttachedLiveMigration(context.Background(), run.Run.RunDir); err == nil || !strings.Contains(err.Error(), "cannot prove writer or traffic authority") { + t.Fatalf("expected malformed durable owner to block attached finalization, got %v", err) + } + withMalformedOwner, err := loadMigrationRun("attached-complete") + if err != nil { + t.Fatal(err) + } + if withMalformedOwner.Run.LiveAppliedAt != nil { + t.Fatal("malformed durable owner marked the attached run live") + } + if err := os.Remove(ownerPath); err != nil { + t.Fatal(err) + } + if err := claimDokployHostOwnership(run.Run, "http://127.0.0.1:3030", dokployCredentialID("test-token")); err != nil { + t.Fatal(err) + } + if err := markDokployTrafficTarget(run.Run, "http://127.0.0.1:3030"); err != nil { + t.Fatal(err) + } if err := finalizeAttachedLiveMigration(context.Background(), run.Run.RunDir); err != nil { t.Fatal(err) } @@ -1006,6 +1415,540 @@ func TestFinalizeAttachedLiveMigrationRecordsLifecycle(t *testing.T) { } } +func TestApplyLiveMigrationRefusesStatefulRunBeforeRecovery(t *testing.T) { + resetDokployTrafficOwner(t) + if runtime.GOOS == "windows" { + t.Skip("docker PATH shim requires a POSIX shell") + } + workDir := t.TempDir() + t.Chdir(workDir) + if err := os.Mkdir(filepath.Join(workDir, ".bort"), 0o700); err != nil { + t.Fatal(err) + } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet || r.URL.Path != "/api/project.all" { + http.NotFound(w, r) + return + } + _, _ = io.WriteString(w, "[]") + })) + defer server.Close() + t.Setenv(dokploy.EnvBaseURL, server.URL) + t.Setenv(dokploy.EnvToken, "secret") + + shimDir := t.TempDir() + dockerPath := filepath.Join(shimDir, "docker") + dockerShim := `#!/bin/sh +if [ "$*" = "inspect --type container source-id" ]; then + printf '%s\n' '[{"Id":"source-id","Name":"/source","State":{"Running":true,"Status":"running"}}]' + exit 0 +fi +printf 'unexpected docker command: %s\n' "$*" >&2 +exit 1 +` + if err := os.WriteFile(dockerPath, []byte(dockerShim), 0o700); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", shimDir+string(os.PathListSeparator)+os.Getenv("PATH")) + + runDir := t.TempDir() + bundleDir := filepath.Join(runDir, "bundle") + if err := os.Mkdir(bundleDir, 0o700); err != nil { + t.Fatal(err) + } + bundleDigest, err := digestMigrationBundle(bundleDir) + if err != nil { + t.Fatal(err) + } + app := preparer.AppPlan{Name: "api"} + app.Resources.SourceServices = []preparer.SourceServiceRef{{ServiceName: "web", ContainerID: "source-id"}} + run := loadedMigrationRun{ + Run: migrationRun{ + Name: "complete-without-outcome", + RunDir: runDir, + BundleDir: bundleDir, + BundleDigest: hex.EncodeToString(bundleDigest[:]), + Target: "dokploy", + DryRun: true, + Artifacts: defaultRunArtifacts(), + }, + Prepare: preparer.Result{BundleDir: bundleDir, Apps: []preparer.AppPlan{app}}, + Sync: syncplan.Result{Apps: []syncplan.AppPlan{{Name: "api", Steps: []syncplan.Step{{ + ResourceType: "volume", + ResourceRef: "volume:web -> /data", + Strategy: syncplan.StrategyDockerVolumeArchive, + }}}}}, + Cutover: gateway.Result{Apps: []gateway.AppPlan{{Name: "api", Routes: []gateway.Route{{Host: "api.example.com"}}}}}, + } + steps := dokploy.PlanFromArtifactsV1Alpha2(run.Prepare, run.Sync, run.Cutover).Steps + applied := newRunApplied(run.Run) + applied.PlanVersion = appliedPlanV1Alpha2 + applied.TargetOrigin = server.URL + for index, step := range steps { + applied = recordAppliedStep(applied, dokploy.StepProgress{Index: index, Step: step, Status: dokploy.StepStatusOK}) + } + run.Applied = applied + appliedPath := runArtifactPath(runDir, run.Run.Artifacts.Applied) + if err := writeRunApplied(appliedPath, applied); err != nil { + t.Fatal(err) + } + if err := claimDokployHostOwnership(run.Run, server.URL, dokployCredentialID("secret")); err != nil { + t.Fatal(err) + } + if err := markDokployTrafficTarget(run.Run, server.URL); err != nil { + t.Fatal(err) + } + + err = applyLiveMigrationLocked(context.Background(), run, io.Discard, nil) + if err == nil || !strings.Contains(err.Error(), "live apply refused for stateful app(s) api") { + t.Fatalf("expected stateful live apply refusal, got %v", err) + } + finalApplied := readJSONFile[runApplied](t, appliedPath) + if finalApplied.SucceededAt != nil { + t.Fatal("completed ledger with ambiguous source authority was marked successful") + } +} + +func TestApplyLiveMigrationResumesHistoricalPauseBeforeStatefulRefusal(t *testing.T) { + resetDokployTrafficOwner(t) + if runtime.GOOS == "windows" { + t.Skip("docker PATH shim requires a POSIX shell") + } + runDir := t.TempDir() + tracePath := filepath.Join(runDir, "docker.trace") + shimDir := t.TempDir() + dockerShim := `#!/bin/sh +printf '%s\n' "$*" >> "$TRACE" +if [ "$*" = "inspect --type container source-id" ]; then + if [ -f "$SOURCE_RUNNING" ]; then + printf '%s\n' '[{"Id":"source-id","Name":"/source","State":{"Running":true,"Status":"running"}}]' + else + printf '%s\n' '[{"Id":"source-id","Name":"/source","State":{"Running":false,"Status":"exited"}}]' + fi + exit 0 +fi +printf 'unexpected docker command: %s\n' "$*" >&2 +exit 1 +` + if err := os.WriteFile(filepath.Join(shimDir, "docker"), []byte(dockerShim), 0o700); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", shimDir+string(os.PathListSeparator)+os.Getenv("PATH")) + t.Setenv("TRACE", tracePath) + sourceRunningPath := filepath.Join(runDir, "source-running") + t.Setenv("SOURCE_RUNNING", sourceRunningPath) + app := preparer.AppPlan{Name: "api"} + app.Resources.SourceServices = []preparer.SourceServiceRef{{ServiceName: "web", ContainerID: "source-id"}} + run := loadedMigrationRun{ + Run: migrationRun{ + Name: "historical-pause", + RunDir: runDir, + BundleDir: filepath.Join(runDir, "bundle"), + Target: "dokploy", + Artifacts: defaultRunArtifacts(), + }, + Prepare: preparer.Result{Apps: []preparer.AppPlan{app}}, + Sync: syncplan.Result{Apps: []syncplan.AppPlan{{Name: "api", Steps: []syncplan.Step{{ + ResourceType: "volume", + ResourceRef: "volume:web -> /data", + Strategy: syncplan.StrategyDockerVolumeArchive, + }}}}}, + } + plan := dokploy.PlanFromArtifacts(run.Prepare, run.Sync, run.Cutover) + pauseIndex := -1 + applied := runApplied{ + APIVersion: appliedLegacyAPIVersion, + RunName: run.Run.Name, + BundleDir: run.Run.BundleDir, + Target: run.Run.Target, + Apps: map[string]appliedApp{}, + } + for index, step := range plan.Steps { + if step.Kind == dokploy.StepPauseSource { + pauseIndex = index + applied.Steps = append(applied.Steps, appliedStep{Index: index, Kind: string(step.Kind), App: step.App, Ref: step.Ref, Status: string(dokploy.StepStatusOK)}) + break + } + applied.Steps = append(applied.Steps, appliedStep{Index: index, Kind: string(step.Kind), App: step.App, Ref: step.Ref, Status: string(dokploy.StepStatusOK)}) + } + if pauseIndex < 0 { + t.Fatal("stateful fixture did not produce a source pause") + } + run.Applied = applied + appliedPath := runArtifactPath(runDir, run.Run.Artifacts.Applied) + if err := writeRunApplied(appliedPath, applied); err != nil { + t.Fatal(err) + } + if err := validateLiveApplyReady(run); err != nil { + t.Fatalf("safe source cleanup was blocked before recovery: %v", err) + } + var stderr strings.Builder + err := applyLiveMigrationLocked(context.Background(), run, &stderr, nil) + if err == nil || !strings.Contains(err.Error(), "live apply refused for stateful app(s) api") || !strings.Contains(err.Error(), "docker start source-id") || strings.Contains(err.Error(), "previously paused source was resumed") { + t.Fatalf("expected manual start instruction for a stopped source without pause ownership, got %v", err) + } + if _, err := os.Stat(filepath.Join(runDir, "source-pause.json")); !os.IsNotExist(err) { + t.Fatalf("pause ownership was recorded for a stopped source Bort cannot prove it stopped: %v", err) + } + if err := os.WriteFile(sourceRunningPath, nil, 0o600); err != nil { + t.Fatal(err) + } + stderr.Reset() + err = applyLiveMigrationLocked(context.Background(), run, &stderr, nil) + if err == nil || !strings.Contains(err.Error(), "live apply refused for stateful app(s) api") || !strings.Contains(err.Error(), "previously paused source was resumed") { + t.Fatalf("expected cleanup followed by stateful refusal, got %v", err) + } + if !strings.Contains(stderr.String(), "no state transfer was attempted") { + t.Fatalf("expected cleanup-only recovery status, got %q", stderr.String()) + } + trace, err := os.ReadFile(tracePath) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(trace), "start source-id") { + t.Fatalf("Bort started a source container it never proved it stopped:\n%s", trace) + } + finalApplied := readJSONFile[runApplied](t, appliedPath) + var cleanup appliedStep + for _, step := range finalApplied.Steps { + if step.Index == pauseIndex { + cleanup = step + break + } + } + if cleanup.Kind != string(dokploy.StepResumeSource) || cleanup.Status != string(dokploy.StepStatusOK) { + t.Fatalf("source cleanup was not durably recorded at pause index %d: %#v", pauseIndex, cleanup) + } +} + +func TestInterruptedStatefulSourceCleanupAllowsReadOnlyDump(t *testing.T) { + app := preparer.AppPlan{Name: "api"} + app.Resources.DataStores = []preparer.DataStoreResource{{Service: "db", Kind: "postgres", Strategy: "migrate"}} + run := loadedMigrationRun{ + Prepare: preparer.Result{Apps: []preparer.AppPlan{app}}, + Sync: syncplan.Result{Apps: []syncplan.AppPlan{{Name: "api", Steps: []syncplan.Step{{ + ResourceType: "data_store", + ResourceRef: "data-store:db", + }}}}}, + } + plan := dokploy.PlanFromArtifacts(run.Prepare, run.Sync, run.Cutover) + for _, dumpStatus := range []dokploy.StepStatus{dokploy.StepStatusStarted, dokploy.StepStatusOK} { + t.Run(string(dumpStatus), func(t *testing.T) { + applied := runApplied{APIVersion: appliedLegacyAPIVersion} + pauseIndex := -1 + for index, step := range plan.Steps { + if step.Kind == dokploy.StepPauseSource { + pauseIndex = index + } + status := dokploy.StepStatusOK + if step.Kind == dokploy.StepDumpDataStore { + status = dumpStatus + } + applied = recordAppliedStep(applied, dokploy.StepProgress{Index: index, Step: step, Status: status}) + if step.Kind == dokploy.StepDumpDataStore { + break + } + } + run.Applied = applied + cleanup := interruptedStatefulSourceCleanup(run) + if len(cleanup) != 1 || cleanup[0].app != "api" || cleanup[0].index != pauseIndex { + t.Fatalf("read-only dump blocked source recovery: %#v", cleanup) + } + }) + } +} + +func TestInterruptedStatefulSourceCleanupRefusesCompletedImagePush(t *testing.T) { + app := preparer.AppPlan{ + Name: "api", + TargetResources: &preparer.TargetResources{Dokploy: &preparer.DokployResources{ + ComposeApp: preparer.DokployComposeApp{Name: "api"}, + }}, + } + run := loadedMigrationRun{ + Prepare: preparer.Result{Apps: []preparer.AppPlan{app}}, + Sync: syncplan.Result{Apps: []syncplan.AppPlan{{Name: "api", Steps: []syncplan.Step{{ + ResourceType: "volume", + ResourceRef: "volume:web -> /data", + Strategy: syncplan.StrategyDockerVolumeArchive, + }}}}}, + } + run.Applied = runApplied{APIVersion: appliedAPIVersion, PlanVersion: appliedPlanV1Alpha2} + plan := dokploy.PlanFromArtifactsV1Alpha2(run.Prepare, run.Sync, run.Cutover) + pushCompleted := false + pauseCompleted := false + for index, step := range plan.Steps { + run.Applied = recordAppliedStep(run.Applied, dokploy.StepProgress{Index: index, Step: step, Status: dokploy.StepStatusOK}) + pushCompleted = pushCompleted || step.Kind == dokploy.StepPushImage + if step.Kind == dokploy.StepPauseSource { + pauseCompleted = true + break + } + } + if !pushCompleted || !pauseCompleted { + t.Fatalf("fixture did not complete image push and source pause: %v", plan.Steps) + } + if cleanup := interruptedStatefulSourceCleanup(run); len(cleanup) != 0 { + t.Fatalf("completed image push allowed automatic source recovery: %#v", cleanup) + } +} + +func TestCompletedPrefixReconcilesProxyBeforeTargetAuthority(t *testing.T) { + run, appliedPath := prepareCompletedPrefixProxyRun(t) + if err := applyLiveMigrationLocked(context.Background(), run, io.Discard, nil); err != nil { + t.Fatalf("completed-prefix recovery failed: %v", err) + } + for _, path := range []string{"coolify-stopped", "dokploy-started"} { + if _, err := os.Stat(path); err != nil { + t.Fatalf("proxy reconciliation did not create %s: %v", path, err) + } + } + owner, found, err := readDokployTrafficOwner() + if err != nil { + t.Fatal(err) + } + if !found || owner.Authority != dokployTrafficTarget { + t.Fatalf("target authority was not recorded after reconciliation: %#v found=%t", owner, found) + } + finalApplied := readJSONFile[runApplied](t, appliedPath) + if finalApplied.SucceededAt == nil { + t.Fatal("completed-prefix recovery did not record success") + } +} + +func TestCompletedPrefixKeepsAuthorityWhenProxyStartFails(t *testing.T) { + run, appliedPath := prepareCompletedPrefixProxyRun(t) + if err := os.WriteFile("fail-start", nil, 0o600); err != nil { + t.Fatal(err) + } + err := applyLiveMigrationLocked(context.Background(), run, io.Discard, nil) + if err == nil || !strings.Contains(err.Error(), "start proxy container dokploy-traefik") { + t.Fatalf("expected completed-prefix recovery to fail on the Dokploy proxy start, got %v", err) + } + if _, statErr := os.Stat("dokploy-started"); statErr == nil { + t.Fatal("docker stub recorded a proxy start despite failing") + } + owner, found, err := readDokployTrafficOwner() + if err != nil { + t.Fatal(err) + } + if found && owner.Authority == dokployTrafficTarget { + t.Fatalf("target authority was recorded although the Dokploy proxy never started: %#v", owner) + } + if finalApplied := readJSONFile[runApplied](t, appliedPath); finalApplied.SucceededAt != nil { + t.Fatal("completed-prefix recovery recorded success although the Dokploy proxy never started") + } +} + +func prepareCompletedPrefixProxyRun(t *testing.T) (loadedMigrationRun, string) { + t.Helper() + resetDokployTrafficOwner(t) + workDir := t.TempDir() + t.Chdir(workDir) + bundleDir := filepath.Join(workDir, "bundle") + writeTestBundle(t, bundleDir, manifest.Manifest{ + Source: manifest.Source{Platform: "coolify-local"}, + Apps: []manifest.App{{ + Name: "api", + Services: []manifest.Service{{ID: "source-id", Name: "web", Image: "example/api:latest"}}, + Routes: []manifest.Route{{Host: "api.example.com", ServiceName: "web", Port: "3000"}}, + }}, + }) + runCommand(t, runMigrate, []string{"--bundle", bundleDir, "--run", "proxy-reconcile", "--observation-window", "0", "--rollback-window", "0"}) + run, err := loadMigrationRun("proxy-reconcile") + if err != nil { + t.Fatal(err) + } + target := dokploy.TargetIdentity{ProjectID: "project-1", EnvironmentID: "environment-1", ComposeID: "compose-1", ComposeAppName: "stack-api"} + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + http.NotFound(w, r) + return + } + switch r.URL.Path { + case "/api/project.all": + _, _ = io.WriteString(w, "[]") + case "/api/project.one": + _ = json.NewEncoder(w).Encode(dokploy.Project{ProjectID: target.ProjectID, Name: "api", Environments: []dokploy.ProjectEnvironment{{EnvironmentID: target.EnvironmentID, Name: "production"}}}) + case "/api/compose.one": + _ = json.NewEncoder(w).Encode(dokploy.Compose{ComposeID: target.ComposeID, Name: "api", AppName: target.ComposeAppName, EnvironmentID: target.EnvironmentID}) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(server.Close) + t.Setenv(dokploy.EnvBaseURL, server.URL) + t.Setenv(dokploy.EnvToken, "secret") + binDir := t.TempDir() + dockerStub := `#!/bin/sh +if [ "$1" = inspect ] && [ "$2" = --type ]; then + running=false + policy=no + case "$4" in + coolify-proxy) [ ! -f coolify-stopped ] && running=true ;; + dokploy-traefik) [ -f dokploy-started ] && running=true ;; + esac + [ ! -f "$4-unless" ] || policy=unless-stopped + echo '[{"Id":"'"$4"'-id","Name":"/'"$4"'","State":{"Running":'"$running"',"Status":"running"},"HostConfig":{"RestartPolicy":{"Name":"'"$policy"'"}}}]' +elif [ "$1" = stop ]; then + touch coolify-stopped +elif [ "$1" = start ]; then + if [ -f fail-start ]; then + echo "Error response from daemon: driver failed programming external connectivity" >&2 + exit 1 + fi + touch dokploy-started +elif [ "$1" = update ] && [ "$2" = --restart=unless-stopped ]; then + touch "${3%-id}-unless" +else + echo "unexpected docker command: $*" >&2 + exit 1 +fi +` + if err := os.WriteFile(filepath.Join(binDir, "docker"), []byte(dockerStub), 0o700); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", binDir+string(os.PathListSeparator)+os.Getenv("PATH")) + + plan := dokploy.PlanFromArtifacts(run.Prepare, run.Sync, run.Cutover) + applied := newRunApplied(run.Run) + applied.TargetOrigin = server.URL + for index, step := range plan.Steps { + progress := dokploy.StepProgress{Index: index, Step: step, Status: dokploy.StepStatusOK} + if step.App != "" { + progress.Target = &target + } + applied = recordAppliedStep(applied, progress) + } + appliedPath := runArtifactPath(run.Run.RunDir, run.Run.Artifacts.Applied) + if err := writeRunApplied(appliedPath, applied); err != nil { + t.Fatal(err) + } + if err := claimDokployHostOwnership(run.Run, server.URL, dokployCredentialID("secret")); err != nil { + t.Fatal(err) + } + return run, appliedPath +} + +func TestValidateStatefulLiveApplyRefusesInPlaceTransferLedgersOnly(t *testing.T) { + run := loadedMigrationRun{ + Prepare: preparer.Result{Apps: []preparer.AppPlan{{Name: "api"}}}, + Sync: syncplan.Result{Apps: []syncplan.AppPlan{{Name: "api", Steps: []syncplan.Step{{ + ResourceType: "volume", + ResourceRef: "volume:web -> /data", + Strategy: syncplan.StrategyDockerVolumeArchive, + }}}}}, + } + for name, applied := range map[string]runApplied{ + "fresh": newRunApplied(migrationRun{}), + "legacy-api-unexecuted": {APIVersion: appliedLegacyAPIVersion}, + "v1alpha2-unexecuted": {APIVersion: appliedAPIVersion, PlanVersion: appliedPlanV1Alpha2}, + } { + run.Applied = applied + if err := validateStatefulLiveApply(run); err != nil { + t.Fatalf("%s ledger must stage state instead of refusing, got %v", name, err) + } + } + history := []appliedStep{{Index: 0, Kind: string(dokploy.StepCreateProject), App: "api", Ref: "api", Status: string(dokploy.StepStatusOK)}} + for name, applied := range map[string]runApplied{ + "legacy-api-history": {APIVersion: appliedLegacyAPIVersion, Steps: history}, + "v1alpha1-history": {APIVersion: appliedAPIVersion, PlanVersion: appliedPlanV1Alpha1, Steps: history}, + "v1alpha2-history": {APIVersion: appliedAPIVersion, PlanVersion: appliedPlanV1Alpha2, Steps: history}, + } { + run.Applied = applied + err := validateStatefulLiveApply(run) + if err == nil || !strings.Contains(err.Error(), "live apply refused for stateful app(s) api") || !strings.Contains(err.Error(), "older plan version") || !strings.Contains(err.Error(), "outside Bort") { + t.Fatalf("%s ledger: expected actionable stateful refusal, got %v", name, err) + } + } +} + +func TestAttachProgressIncludesSourceCleanupAtPauseIndex(t *testing.T) { + now := time.Now().UTC() + steps := []dokploy.Step{{Kind: dokploy.StepPauseSource, App: "api", Ref: "api"}} + applied := runApplied{Steps: []appliedStep{{ + Index: 0, + Kind: string(dokploy.StepResumeSource), + App: "api", + Ref: "api", + Status: string(dokploy.StepStatusError), + UpdatedAt: now, + Error: "source remains stopped", + }}} + + entries := attachProgressEntries(steps, applied, time.Time{}) + if len(entries) != 1 { + t.Fatalf("expected one attached cleanup entry, got %#v", entries) + } + progress := entries[0].progress + if progress.Index != 0 || progress.Step.Kind != dokploy.StepResumeSource || progress.Status != dokploy.StepStatusError || progress.Err == nil || !strings.Contains(progress.Err.Error(), "remains stopped") { + t.Fatalf("unexpected attached cleanup progress: %#v", progress) + } + failed, ok := latestAttachFailure(steps, applied, now) + if !ok || failed.Kind != string(dokploy.StepResumeSource) || !strings.Contains(failed.Error, "remains stopped") { + t.Fatalf("source cleanup failure was not visible to attach: %#v ok=%t", failed, ok) + } +} + +func TestAttachProgressIncludesStandaloneProxyCleanup(t *testing.T) { + now := time.Now().UTC() + steps := []dokploy.Step{{Kind: dokploy.StepStopCoolifyProxy, Ref: "coolify-proxy"}} + applied := runApplied{Steps: []appliedStep{{ + Index: len(steps), + Kind: string(dokploy.StepStartCoolifyProxy), + Ref: "coolify-proxy", + Status: string(dokploy.StepStatusError), + UpdatedAt: now, + Error: "proxy remains stopped", + }}} + + entries := attachProgressEntries(steps, applied, time.Time{}) + if len(entries) != 1 { + t.Fatalf("expected one attached proxy cleanup entry, got %#v", entries) + } + progress := entries[0].progress + if progress.Index != len(steps) || progress.Total != len(steps)+1 || progress.Step.Kind != dokploy.StepStartCoolifyProxy || progress.Status != dokploy.StepStatusError || progress.Err == nil || !strings.Contains(progress.Err.Error(), "remains stopped") { + t.Fatalf("unexpected attached proxy cleanup progress: %#v", progress) + } + var output bytes.Buffer + writeAttachTextProgress(&output, progress, progress.Total, "recorded") + if !strings.Contains(output.String(), "2/2 step(s) recorded") { + t.Fatalf("standalone cleanup rendered invalid progress: %q", output.String()) + } + failed, ok := latestAttachFailure(steps, applied, now) + if !ok || failed.Kind != string(dokploy.StepStartCoolifyProxy) || !strings.Contains(failed.Error, "remains stopped") { + t.Fatalf("proxy cleanup failure was not visible to attach: %#v ok=%t", failed, ok) + } +} + +func TestAttachUsesPersistedLegacyPlanVersion(t *testing.T) { + run := loadedMigrationRun{ + Run: migrationRun{Name: "legacy-attach", RunDir: t.TempDir(), Target: "dokploy"}, + Prepare: preparer.Result{Apps: []preparer.AppPlan{{Name: "api"}}}, + Cutover: gateway.Result{Apps: []gateway.AppPlan{{Name: "api", Routes: []gateway.Route{{Host: "api.example.com"}}}}}, + } + legacy := dokploy.LegacyPlanFromArtifactsV1Alpha1(run.Prepare, run.Sync, run.Cutover) + applied := runApplied{ + APIVersion: appliedAPIVersion, + PlanVersion: appliedPlanV1Alpha1, + RunName: run.Run.Name, + Target: run.Run.Target, + Apps: map[string]appliedApp{}, + } + for index, step := range legacy.Steps { + applied = recordAppliedStep(applied, dokploy.StepProgress{Index: index, Step: step, Status: dokploy.StepStatusOK}) + } + now := time.Now().UTC() + applied.SucceededAt = &now + appliedPath := filepath.Join(run.Run.RunDir, "applied.json") + if err := writeRunApplied(appliedPath, applied); err != nil { + t.Fatal(err) + } + if err := attachLiveMigration(context.Background(), run, appliedPath, filepath.Join(run.Run.RunDir, "apply.lock"), io.Discard, nil); err != nil { + t.Fatalf("attach rejected completed persisted legacy plan: %v", err) + } +} + func TestRunRefreshPublishesVersionedArtifactsTogether(t *testing.T) { workDir := t.TempDir() t.Chdir(workDir) @@ -1475,6 +2418,37 @@ func TestRunPlanBecomesImmutableAfterLiveStartLedger(t *testing.T) { } } +func TestRunPlanBecomesImmutableAfterTargetOriginBinding(t *testing.T) { + workDir := t.TempDir() + t.Chdir(workDir) + bundleDir := filepath.Join(workDir, "bort-bundle") + writeTestBundle(t, bundleDir, manifest.Manifest{ + Source: manifest.Source{Platform: "docker"}, + Apps: []manifest.App{{Name: "api", Services: []manifest.Service{{Name: "api", Image: "example/api:latest"}}}}, + }) + runCommand(t, runMigrate, []string{"--bundle", bundleDir, "--run", "target-bound"}) + run, err := loadMigrationRun("target-bound") + if err != nil { + t.Fatal(err) + } + applied := newRunApplied(run.Run) + applied.TargetOrigin = "http://127.0.0.1:3030" + appliedPath, err := safeRunArtifactPath(run.Run.RunDir, run.Run.Artifacts.Applied) + if err != nil { + t.Fatal(err) + } + if err := writeRunApplied(appliedPath, applied); err != nil { + t.Fatal(err) + } + + var stdout bytes.Buffer + var stderr bytes.Buffer + err = runMigrate(context.Background(), []string{"--bundle", bundleDir, "--run", "target-bound"}, &stdout, &stderr) + if err == nil || !strings.Contains(err.Error(), "reviewed plan is immutable") { + t.Fatalf("expected target-bound run plan to be immutable, got err=%v", err) + } +} + func TestRunPlanRefusesToOverwriteUnreadableMetadata(t *testing.T) { workDir := t.TempDir() t.Chdir(workDir) @@ -1506,19 +2480,20 @@ func TestRunStatusAndNextReadExistingRun(t *testing.T) { t.Chdir(workDir) bundleDir := filepath.Join(workDir, "bort-bundle") writeTestBundle(t, bundleDir, manifest.Manifest{ - Source: manifest.Source{Platform: "docker"}, + Source: manifest.Source{Platform: "docker", DockerEngineID: "engine-reviewed"}, Apps: []manifest.App{ - {Name: "api", Services: []manifest.Service{{Name: "api", Image: "example/api:latest"}}, Routes: []manifest.Route{{Host: "api.example.com", ServiceName: "api"}}}, + {Name: "api", Services: []manifest.Service{{ID: "0123456789ab", Name: "api", Image: "example/api:latest"}}, Routes: []manifest.Route{{Host: "api.example.com", ServiceName: "api"}}}, }, }) runCommand(t, runMigrate, []string{"--bundle", bundleDir, "--run", "demo-app", "--observation-window", "0", "--rollback-window", "0"}) + markRunLocallyScanned(t, "demo-app", "docker") var statusOut bytes.Buffer var statusErr bytes.Buffer if err := runStatus(context.Background(), []string{"--run", "demo-app"}, &statusOut, &statusErr); err != nil { t.Fatalf("status failed: %v\nstderr:\n%s", err, statusErr.String()) } - for _, want := range []string{"local bundle → dokploy", "api", "READY", "bort migrate --live --run demo-app"} { + for _, want := range []string{"Local Docker → dokploy", "api", "READY", "bort migrate --live --run demo-app"} { if !strings.Contains(statusOut.String(), want) { t.Fatalf("expected status output to contain %q, got:\n%s", want, statusOut.String()) } @@ -1534,6 +2509,251 @@ func TestRunStatusAndNextReadExistingRun(t *testing.T) { t.Fatalf("expected next output to contain %q, got:\n%s", want, nextOut.String()) } } + + hostLock, err := acquireDokployLiveOperationLock() + if err != nil { + t.Fatal(err) + } + defer hostLock.Release() + statusOut.Reset() + if err := runStatus(context.Background(), []string{"--run", "demo-app"}, &statusOut, io.Discard); err != nil { + t.Fatal(err) + } + if !strings.Contains(statusOut.String(), "HOST BUSY") || strings.Contains(statusOut.String(), " READY") { + t.Fatalf("status did not surface the host-wide operation lock:\n%s", statusOut.String()) + } + nextOut.Reset() + if err := runNext(context.Background(), []string{"demo-app"}, &nextOut, io.Discard); err != nil { + t.Fatal(err) + } + if !strings.Contains(nextOut.String(), "wait for the active Dokploy host operation") || strings.Contains(nextOut.String(), "migrate --live") { + t.Fatalf("next did not surface the host-wide operation lock:\n%s", nextOut.String()) + } +} + +func TestNonLinuxReadyRunIsInspectionOnly(t *testing.T) { + previous := dokployLiveOperationsSupported + dokployLiveOperationsSupported = func() bool { return false } + t.Cleanup(func() { dokployLiveOperationsSupported = previous }) + now := time.Now().UTC() + for _, tc := range []struct { + run loadedMigrationRun + phase string + }{ + {run: loadedMigrationRun{Run: migrationRun{Name: "ready", RunDir: t.TempDir(), Target: "dokploy"}}, phase: "empty"}, + {run: loadedMigrationRun{Run: migrationRun{Name: "applied", RunDir: t.TempDir(), Target: "dokploy", LiveAppliedAt: &now}}, phase: "applied"}, + {run: loadedMigrationRun{Run: migrationRun{Name: "rollback", RunDir: t.TempDir(), Target: "dokploy", LiveAppliedAt: &now, RollbackStartedAt: &now}}, phase: "rolling back"}, + } { + if phase := migrationRunPhase(tc.run); phase != tc.phase { + t.Fatalf("unsupported platform run %q phase=%q, want %q", tc.run.Run.Name, phase, tc.phase) + } + next := nextSafeStep(tc.run, nil) + wantAction := "Linux source host" + if tc.phase == "empty" { + wantAction = "at least one non-platform application" + } + if !strings.Contains(next.Action, wantAction) || strings.Contains(next.Action, "migrate --live") || strings.Contains(next.Action, "commit --apply") || strings.Contains(next.Action, "rollback --live") { + t.Fatalf("unsupported platform run %q offered a live action: %#v", tc.run.Run.Name, next) + } + } + migratable := loadedMigrationRun{ + Run: migrationRun{Name: "migratable", RunDir: t.TempDir(), Target: "dokploy", Source: "docker"}, + Prepare: preparer.Result{ + Source: "docker", + SourceDockerEngineID: "engine-reviewed", + Apps: []preparer.AppPlan{{ + Name: "api", + Resources: preparer.ResourceSpecs{SourceServices: []preparer.SourceServiceRef{{ContainerID: "0123456789ab", ContainerName: "api"}}}, + }}, + }, + } + if phase := migrationRunPhase(migratable); phase != "inspection-only" { + t.Fatalf("unsupported platform migratable run phase=%q, want inspection-only", phase) + } + if next := nextSafeStep(migratable, nil); !strings.Contains(next.Action, "Linux source host") || strings.Contains(next.Action, "migrate --live") { + t.Fatalf("unsupported platform migratable run offered a live action: %#v", next) + } + var migratableOutput strings.Builder + writeAppFirstCockpit(&migratableOutput, migratable) + if !strings.Contains(migratableOutput.String(), "INSPECTION ONLY") || !strings.Contains(migratableOutput.String(), "Dokploy live actions are unavailable on "+runtime.GOOS) || strings.Contains(migratableOutput.String(), "migrate --live") { + t.Fatalf("unsupported platform cockpit hid inspection-only guidance:\n%s", migratableOutput.String()) + } + for _, tc := range []struct { + run loadedMigrationRun + phase string + wantAction string + wantGuidance string + }{ + {run: loadedMigrationRun{Run: migrationRun{Name: "committed", RunDir: t.TempDir(), Target: "dokploy", CommittedAt: &now}}, phase: "committed", wantAction: "cleanup", wantGuidance: "audit leftovers"}, + {run: loadedMigrationRun{Run: migrationRun{Name: "rolled-back", RunDir: t.TempDir(), Target: "dokploy", RolledBackAt: &now}}, phase: "rolled back", wantAction: "new named migration run", wantGuidance: "To migrate again"}, + {run: loadedMigrationRun{Run: migrationRun{Name: "purged", RunDir: t.TempDir(), Target: "dokploy", PurgedAt: &now}}, phase: "purged", wantAction: "migration complete", wantGuidance: "Migration complete"}, + } { + if phase := migrationRunPhase(tc.run); phase != tc.phase { + t.Fatalf("unsupported platform terminal run %q phase=%q, want %q", tc.run.Run.Name, phase, tc.phase) + } + next := nextSafeStep(tc.run, nil) + if !strings.Contains(next.Action, tc.wantAction) { + t.Fatalf("unsupported platform terminal run %q hid safe next action: %#v", tc.run.Run.Name, next) + } + var output strings.Builder + writeAppFirstCockpit(&output, tc.run) + if !strings.Contains(output.String(), tc.wantGuidance) { + t.Fatalf("unsupported platform terminal run %q hid safe cockpit guidance:\n%s", tc.run.Run.Name, output.String()) + } + } + if _, err := acquireDokployLiveOperationLock(); err == nil || !strings.Contains(err.Error(), "unavailable on this platform") { + t.Fatalf("unsupported platform acquired live-operation lock: %v", err) + } + + runDir := t.TempDir() + lockTarget := filepath.Join(runDir, "lock-target") + if err := os.WriteFile(lockTarget, []byte("preserve\n"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.Symlink(lockTarget, filepath.Join(runDir, "apply.lock")); err != nil { + t.Skipf("symlink fixture is unavailable: %v", err) + } + lockErrorRun := loadedMigrationRun{Run: migrationRun{Name: "lock-error", RunDir: runDir, Target: "dokploy"}} + if phase := migrationRunPhase(lockErrorRun); phase != "lock-error" { + t.Fatalf("unsupported platform hid lock error behind phase %q", phase) + } + next := nextSafeStep(lockErrorRun, nil) + if !strings.Contains(next.Action, "inspect the live-apply lock") || !strings.Contains(next.Action, "Linux source host") { + t.Fatalf("unsupported platform hid apply-lock recovery: %#v", next) + } + var output strings.Builder + writeAppFirstCockpit(&output, lockErrorRun) + if !strings.Contains(output.String(), "LOCK ERROR") || !strings.Contains(output.String(), "apply.lock") || strings.Contains(output.String(), "ready for inspection") { + t.Fatalf("unsupported cockpit contradicted lock-error state:\n%s", output.String()) + } +} + +func TestDefaultVerifyLocalSourceRunAttestsEngineAndContainers(t *testing.T) { + binDir := t.TempDir() + dockerPath := filepath.Join(binDir, "docker") + script := `#!/bin/sh +if [ "$1" = info ]; then + printf '%s\n' "$ENGINE_ID" +elif [ "$1" = inspect ] && [ "$2" = --type ] && [ "$3" = container ]; then + printf '[{"Id":"%s","Name":"/%s","State":{"Running":true,"Status":"running"}}]\n' "$CONTAINER_ID" "$CONTAINER_NAME" +else + echo "unexpected docker command: $*" >&2 + exit 1 +fi +` + if err := os.WriteFile(dockerPath, []byte(script), 0o700); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", binDir+string(os.PathListSeparator)+os.Getenv("PATH")) + t.Setenv("ENGINE_ID", "engine-reviewed") + t.Setenv("CONTAINER_ID", "0123456789abcdef") + t.Setenv("CONTAINER_NAME", "reviewed-web") + run := loadedMigrationRun{Run: migrationRun{Source: "coolify-local"}, Prepare: preparer.Result{ + Source: "coolify-local", + SourceDockerEngineID: "engine-reviewed", + Apps: []preparer.AppPlan{{ + Name: "api", + Resources: preparer.ResourceSpecs{SourceServices: []preparer.SourceServiceRef{{ + ServiceName: "web", + ContainerID: "0123456789ab", + ContainerName: "reviewed-web", + }}}, + }}, + }} + if err := defaultVerifyLocalSourceRun(context.Background(), run); err != nil { + t.Fatalf("expected matching source attestation, got %v", err) + } + t.Setenv("ENGINE_ID", "engine-other") + if err := defaultVerifyLocalSourceRun(context.Background(), run); err == nil || !strings.Contains(err.Error(), "does not match") { + t.Fatalf("expected Docker engine mismatch refusal, got %v", err) + } + t.Setenv("ENGINE_ID", "engine-reviewed") + t.Setenv("CONTAINER_ID", "fedcba9876543210") + if err := defaultVerifyLocalSourceRun(context.Background(), run); err == nil || !strings.Contains(err.Error(), "resolved to ID") { + t.Fatalf("expected source container mismatch refusal, got %v", err) + } + run.Run.Source = "manifest" + t.Setenv("CONTAINER_ID", "0123456789abcdef") + if err := defaultVerifyLocalSourceRun(context.Background(), run); err == nil || !strings.Contains(err.Error(), "imported manifest") { + t.Fatalf("expected imported manifest refusal despite embedded local attestation, got %v", err) + } + run.Run.Source = "" + if err := defaultVerifyLocalSourceRun(context.Background(), run); err == nil || !strings.Contains(err.Error(), "imported bundle") { + t.Fatalf("expected imported bundle refusal despite embedded local attestation, got %v", err) + } + run.Run.Source = "coolify" + if err := defaultVerifyLocalSourceRun(context.Background(), run); err == nil || !strings.Contains(err.Error(), `source "coolify", not a local Docker scan`) { + t.Fatalf("expected remote Coolify source refusal, got %v", err) + } + run.Run.Source = "docker" + run.Prepare.Source = "imported" + if err := defaultVerifyLocalSourceRun(context.Background(), run); err == nil || !strings.Contains(err.Error(), "not a locally attested") { + t.Fatalf("expected imported source refusal, got %v", err) + } + run.Prepare.Source = "coolify-local-forged" + if err := defaultVerifyLocalSourceRun(context.Background(), run); err == nil || !strings.Contains(err.Error(), "not a locally attested") { + t.Fatalf("expected unknown Coolify source variant refusal, got %v", err) + } + run.Applied.TargetOrigin = "http://127.0.0.1:3030" + if err := defaultVerifyLocalSourceRun(context.Background(), run); err == nil || !strings.Contains(err.Error(), "already started live execution and cannot be refreshed") { + t.Fatalf("expected applied imported source to require manual recovery, got %v", err) + } +} + +func TestHostBusyPreservesRollbackLifecycleGuidance(t *testing.T) { + for _, test := range []struct { + name string + phase string + wantStatus string + wantText string + configure func(*migrationRun, time.Time) + }{ + { + name: "rollback", + phase: "rolling back", + wantStatus: "ROLLING BACK", + wantText: "Rollback is in progress", + configure: func(run *migrationRun, now time.Time) { run.RollbackStartedAt = &now }, + }, + { + name: "commit", + phase: "committing", + wantStatus: "COMMITTING", + wantText: "Source retirement is in progress", + configure: func(run *migrationRun, now time.Time) { run.CommitStartedAt = &now }, + }, + } { + t.Run(test.name, func(t *testing.T) { + resetDokployTrafficOwner(t) + now := time.Now().UTC() + meta := migrationRun{Name: test.name, RunDir: t.TempDir(), Target: "dokploy", CreatedAt: now, BundleDigest: test.name + "-digest", LiveAppliedAt: &now} + test.configure(&meta, now) + if err := claimDokployHostOwnership(meta, "http://127.0.0.1:3030", dokployCredentialID("test-token")); err != nil { + t.Fatal(err) + } + if err := markDokployTrafficTarget(meta, "http://127.0.0.1:3030"); err != nil { + t.Fatal(err) + } + run := loadedMigrationRun{Run: meta} + lock, err := acquireDokployLiveOperationLock() + if err != nil { + t.Fatal(err) + } + defer lock.Release() + if phase := migrationRunPhase(run); phase != test.phase { + t.Fatalf("busy host phase=%q, want %q", phase, test.phase) + } + next := nextSafeStep(run, nil) + if !strings.Contains(next.Action, "wait for the active Dokploy host operation") || strings.Contains(next.Action, "--live") || strings.Contains(next.Action, "commit --apply") { + t.Fatalf("busy host offered mutation recovery: %#v", next) + } + var output strings.Builder + writeAppFirstCockpit(&output, run) + if !strings.Contains(output.String(), test.wantStatus) || !strings.Contains(output.String(), test.wantText) || !strings.Contains(output.String(), "wait for it to finish") || strings.Contains(output.String(), "rollback --live") || strings.Contains(output.String(), "commit --apply") { + t.Fatalf("busy host hid lifecycle or offered a retry:\n%s", output.String()) + } + }) + } } func TestRunStatusAndNextRejectAmbiguousRunArguments(t *testing.T) { @@ -1671,14 +2891,16 @@ func TestRunStatusAndNextDoNotUseMtimeFallback(t *testing.T) { } func TestRunLifecycleTimestampsDriveCockpitLabels(t *testing.T) { + resetDokployTrafficOwner(t) workDir := t.TempDir() t.Chdir(workDir) bundleDir := filepath.Join(workDir, "bort-bundle") writeTestBundle(t, bundleDir, manifest.Manifest{ - Source: manifest.Source{Platform: "docker"}, - Apps: []manifest.App{{Name: "api", Services: []manifest.Service{{Name: "api", Image: "example/api:latest"}}}}, + Source: manifest.Source{Platform: "docker", DockerEngineID: "engine-reviewed"}, + Apps: []manifest.App{{Name: "api", Services: []manifest.Service{{ID: "0123456789ab", Name: "api", Image: "example/api:latest"}}}}, }) runCommand(t, runMigrate, []string{"--bundle", bundleDir, "--run", "lifecycle"}) + markRunLocallyScanned(t, "lifecycle", "docker") run, err := loadMigrationRun("lifecycle") if err != nil { t.Fatal(err) @@ -1689,6 +2911,12 @@ func TestRunLifecycleTimestampsDriveCockpitLabels(t *testing.T) { } defer operationLock.Release() + if err := claimDokployHostOwnership(run.Run, "http://127.0.0.1:3030", dokployCredentialID("test-token")); err != nil { + t.Fatal(err) + } + if err := markDokployTrafficTarget(run.Run, "http://127.0.0.1:3030"); err != nil { + t.Fatal(err) + } if err := markRunLiveAppliedLocked(run.Run); err != nil { t.Fatal(err) } @@ -1718,8 +2946,16 @@ func TestRunLifecycleTimestampsDriveCockpitLabels(t *testing.T) { } cockpit.Reset() writeAppFirstCockpit(&cockpit, run) + if !strings.Contains(cockpit.String(), "COMMITTING") || !strings.Contains(cockpit.String(), "finish releasing commit host ownership") { + t.Fatalf("expected commit-finalization cockpit, got:\n%s", cockpit.String()) + } + if err := releaseDokployTargetOwner(run.Run); err != nil { + t.Fatal(err) + } + cockpit.Reset() + writeAppFirstCockpit(&cockpit, run) if !strings.Contains(cockpit.String(), "COMMITTED") { - t.Fatalf("expected committed cockpit, got:\n%s", cockpit.String()) + t.Fatalf("expected committed cockpit after owner release, got:\n%s", cockpit.String()) } if err := markRunPurgedLocked(run.Run); err != nil { diff --git a/internal/cli/state_apply_test.go b/internal/cli/state_apply_test.go index 54c170d..482c7d0 100644 --- a/internal/cli/state_apply_test.go +++ b/internal/cli/state_apply_test.go @@ -380,11 +380,12 @@ func TestBortNoArgsRefreshesLatestRunAfterEnvCommand(t *testing.T) { t.Chdir(workDir) bundleDir := filepath.Join(workDir, "bort-bundle") writePrivateTestBundle(t, bundleDir, manifest.Manifest{ - Source: manifest.Source{Platform: "docker"}, + Source: manifest.Source{Platform: "docker", DockerEngineID: "engine-reviewed"}, Apps: []manifest.App{ { Name: "api", Services: []manifest.Service{{ + ID: "0123456789ab", Name: "api", Image: "example/api:latest", Environment: []manifest.EnvVar{ @@ -408,10 +409,10 @@ func TestBortNoArgsRefreshesLatestRunAfterEnvCommand(t *testing.T) { t.Fatalf("bort failed: %v\nstderr:\n%s", err, stderr.String()) } output := stdout.String() - if strings.Contains(output, "Fill environment values") { + if strings.Contains(output, "Fill environment values") || strings.Contains(output, "missing values") { t.Fatalf("expected no-arg bort to refresh the latest run after env state, got:\n%s", output) } - if !strings.Contains(output, "All app inputs ready") { - t.Fatalf("expected refreshed app setup to be ready, got:\n%s", output) + if !strings.Contains(output, "✓ Env 1 file(s)") { + t.Fatalf("expected refreshed env inputs to be ready, got:\n%s", output) } } diff --git a/internal/cli/style.go b/internal/cli/style.go index c0675c8..c00b795 100644 --- a/internal/cli/style.go +++ b/internal/cli/style.go @@ -88,7 +88,7 @@ func (s *styler) pill(label string, level severity) string { case sevWarn: style = style.Background(lipgloss.Color("214")).Foreground(lipgloss.Color("232")) case sevBad: - style = style.Background(lipgloss.Color("203")).Foreground(lipgloss.Color("231")) + style = style.Background(lipgloss.Color("203")).Foreground(lipgloss.Color("232")) } return style.Render(label) } diff --git a/internal/cli/wizard.go b/internal/cli/wizard.go index 678501c..01978d5 100644 --- a/internal/cli/wizard.go +++ b/internal/cli/wizard.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "io" + neturl "net/url" "os" "sort" "strings" @@ -17,7 +18,7 @@ import ( func runWizard(ctx context.Context, run loadedMigrationRun, stdin io.Reader, stdout, stderr io.Writer) error { current := run if len(current.Applied.Steps) > 0 || current.Run.LiveAppliedAt != nil || current.Run.CommitStartedAt != nil || current.Run.CommittedAt != nil || current.Run.RollbackStartedAt != nil || current.Run.RolledBackAt != nil || current.Run.PurgedAt != nil { - writeAppFirstCockpit(stdout, current) + writeAppFirstCockpitContext(ctx, stdout, current) return nil } for { @@ -51,8 +52,11 @@ func runWizard(ctx context.Context, run loadedMigrationRun, stdin io.Reader, std current = refreshed } - writeAppFirstCockpit(stdout, current) - fmt.Fprintf(stdout, "Live apply is explicit. Run `%s` when you are ready.\n", liveApplyCommand(current)) + summary := summarizeMigrationRunContext(ctx, current) + writeAppFirstCockpitSummary(stdout, current, summary) + if migrationRunPhaseWithNext(current, summary.Next) == "ready" { + fmt.Fprintf(stdout, "Live apply is explicit. Run `%s` when you are ready.\n", liveApplyCommand(current)) + } return nil } @@ -219,7 +223,12 @@ func markReviewDecisionDone(run loadedMigrationRun, decision runDecision, at tim var errDokploySetupSkipped = errors.New("dokploy setup skipped") -func promptInstallAndBootstrapDokploy(ctx context.Context, stdin io.Reader, stdout, stderr io.Writer, defaultURL string, reason error) error { +type dokploySetupLockOptions struct { + retain **applyLock + revalidate func() error +} + +func promptInstallAndBootstrapDokploy(ctx context.Context, stdin io.Reader, stdout, stderr io.Writer, defaultURL string, reason error, lockOptions dokploySetupLockOptions) error { install := false if strings.TrimSpace(defaultURL) == "" { defaultURL = "http://127.0.0.1:3030" @@ -240,7 +249,7 @@ func promptInstallAndBootstrapDokploy(ctx context.Context, stdin io.Reader, stdo fmt.Fprintf(stdout, "Skipped Dokploy setup. Run `%s` again when you're ready.\n", bortCommand("")) return errDokploySetupSkipped } - return promptInlineInitTargetWithOptions(ctx, stdin, stdout, stderr, inlineInitTargetOptions{Install: true, DefaultURL: defaultURL}) + return promptInlineInitTargetWithOptions(ctx, stdin, stdout, stderr, inlineInitTargetOptions{Install: true, DefaultURL: defaultURL, Lock: lockOptions}) } func promptInlineInitTarget(ctx context.Context, stdin io.Reader, stdout, stderr io.Writer) error { @@ -250,10 +259,11 @@ func promptInlineInitTarget(ctx context.Context, stdin io.Reader, stdout, stderr type inlineInitTargetOptions struct { Install bool DefaultURL string + Lock dokploySetupLockOptions } func promptInlineInitTargetWithOptions(ctx context.Context, stdin io.Reader, stdout, stderr io.Writer, opts inlineInitTargetOptions) error { - var url, email string + var url, email, authSecretBackup string if state, err := readBortState(defaultStatePath()); err == nil { if creds, ok := state.Targets["dokploy"]; ok { url = creds.URL @@ -263,10 +273,24 @@ func promptInlineInitTargetWithOptions(ctx context.Context, stdin io.Reader, std if strings.TrimSpace(opts.DefaultURL) != "" { url = opts.DefaultURL } - form := huh.NewForm(huh.NewGroup( + authSecretBackup = strings.TrimSpace(os.Getenv(envDokployAuthBackup)) + fields := []huh.Field{ huh.NewInput().Title("Dokploy base URL").Placeholder("http://127.0.0.1:3030").Value(&url), huh.NewInput().Title("Coolify admin email").Placeholder("press enter to choose from detected admins").Value(&email), - )) + } + if opts.Install { + fields = append(fields, huh.NewInput(). + Title("Authentication-secret backup path (encrypted/off-host; retain it)"). + Placeholder("/mnt/encrypted-backup/dokploy-auth-secret"). + Validate(func(path string) error { + if validateAuthSecretBackupPath(path) != nil { + return errors.New("Enter an absolute backup path") + } + return nil + }). + Value(&authSecretBackup)) + } + form := huh.NewForm(huh.NewGroup(fields...)) if err := form.Run(); err != nil { return err } @@ -276,12 +300,21 @@ func promptInlineInitTargetWithOptions(ctx context.Context, stdin io.Reader, std } if url != "" { args = append(args, "--dokploy-url", url) + if parsed, err := neturl.Parse(url); opts.Install && err == nil && parsed.Port() != "" { + args = append(args, "--install-port", parsed.Port()) + } } if email != "" { args = append(args, "--coolify-email", email) } + if authSecretBackup != "" { + args = append(args, "--auth-secret-backup", authSecretBackup) + } fmt.Fprintln(stdout, "Setting up Dokploy...") - return runInitTarget(ctx, args, stdin, stdout, stderr) + deps := defaultInitTargetDeps(stderr) + deps.retainLiveOperationLock = opts.Lock.retain + deps.revalidateAfterTargetLock = opts.Lock.revalidate + return runInitTargetWith(ctx, args, stdin, stdout, stderr, deps) } type appStorePair struct { diff --git a/internal/dockercli/environment.go b/internal/dockercli/environment.go new file mode 100644 index 0000000..57663e0 --- /dev/null +++ b/internal/dockercli/environment.go @@ -0,0 +1,40 @@ +package dockercli + +import ( + "fmt" + "runtime" + "strings" +) + +func LocalHost() string { + if runtime.GOOS == "windows" { + return "npipe:////./pipe/docker_engine" + } + return "unix:///var/run/docker.sock" +} + +func LocalEnvironment(env []string) ([]string, error) { + host := LocalHost() + filtered := make([]string, 0, len(env)+1) + for _, entry := range env { + key, value, _ := strings.Cut(entry, "=") + dockerKey := key + if runtime.GOOS == "windows" { + dockerKey = strings.ToUpper(key) + } + switch dockerKey { + case "DOCKER_HOST": + if strings.TrimSpace(value) != "" && value != host { + return nil, fmt.Errorf("Bort requires the local system Docker endpoint %s; unset DOCKER_HOST or set it to that exact value", host) + } + continue + case "DOCKER_CONTEXT", "DOCKER_TLS_VERIFY", "DOCKER_CERT_PATH": + if strings.TrimSpace(value) != "" { + return nil, fmt.Errorf("Bort requires the local system Docker endpoint %s; unset %s", host, dockerKey) + } + continue + } + filtered = append(filtered, entry) + } + return append(filtered, "DOCKER_HOST="+host), nil +} diff --git a/internal/dockercli/environment_test.go b/internal/dockercli/environment_test.go new file mode 100644 index 0000000..d5b3d4f --- /dev/null +++ b/internal/dockercli/environment_test.go @@ -0,0 +1,51 @@ +package dockercli + +import ( + "runtime" + "strings" + "testing" +) + +func TestLocalEnvironmentPinsLocalDaemon(t *testing.T) { + host := "unix:///var/run/docker.sock" + if runtime.GOOS == "windows" { + host = "npipe:////./pipe/docker_engine" + } + env, err := LocalEnvironment([]string{"PATH=/usr/bin", "DOCKER_HOST=" + host}) + if err != nil { + t.Fatal(err) + } + joined := strings.Join(env, "\n") + if !strings.Contains(joined, "DOCKER_HOST="+host) || !strings.Contains(joined, "PATH=/usr/bin") { + t.Fatalf("local Docker environment=%v", env) + } + for _, input := range [][]string{{"PATH=/usr/bin"}, {"PATH=/usr/bin", "DOCKER_HOST="}} { + env, err := LocalEnvironment(input) + if err != nil { + t.Fatal(err) + } + pinned := 0 + for _, entry := range env { + if strings.HasPrefix(entry, "DOCKER_HOST=") { + if entry != "DOCKER_HOST="+host { + t.Fatalf("local Docker environment for %v=%v", input, env) + } + pinned++ + } + } + if pinned != 1 { + t.Fatalf("local Docker environment for %v=%v", input, env) + } + } + for _, setting := range []string{"DOCKER_HOST=ssh://remote.example", "DOCKER_CONTEXT=production", "DOCKER_TLS_VERIFY=1", "DOCKER_CERT_PATH=/tmp/remote-certs"} { + _, err := LocalEnvironment([]string{setting}) + if err == nil || !strings.Contains(err.Error(), strings.SplitN(setting, "=", 2)[0]) { + t.Fatalf("expected %s refusal, got %v", setting, err) + } + } + if runtime.GOOS == "windows" { + if _, err := LocalEnvironment([]string{"docker_context=production"}); err == nil || !strings.Contains(err.Error(), "DOCKER_CONTEXT") { + t.Fatalf("expected lowercase Windows Docker context refusal, got %v", err) + } + } +} diff --git a/internal/exporter/exporter.go b/internal/exporter/exporter.go index c577f7d..fea751d 100644 --- a/internal/exporter/exporter.go +++ b/internal/exporter/exporter.go @@ -27,16 +27,18 @@ const ( ) type Summary struct { - OutputDir string `json:"outputDir"` - GeneratedAt time.Time `json:"generatedAt"` - Source string `json:"source"` - EnvMode string `json:"envMode,omitempty"` - Apps []AppSummary `json:"apps"` + OutputDir string `json:"outputDir"` + GeneratedAt time.Time `json:"generatedAt"` + Source string `json:"source"` + SourceDockerEngineID string `json:"sourceDockerEngineId,omitempty"` + EnvMode string `json:"envMode,omitempty"` + Apps []AppSummary `json:"apps"` } type AppSummary struct { Name string `json:"name"` Directory string `json:"directory"` + Platform string `json:"platform,omitempty"` Role string `json:"role,omitempty"` ProjectGroup *ProjectGroup `json:"projectGroup,omitempty"` PrivateEnvValues bool `json:"privateEnvValues,omitempty"` @@ -68,10 +70,11 @@ func Export(m manifest.Manifest, opts Options) (Summary, error) { } summary := Summary{ - OutputDir: opts.OutputDir, - GeneratedAt: time.Now().UTC(), - Source: m.Source.Platform, - EnvMode: exportEnvMode(opts), + OutputDir: opts.OutputDir, + GeneratedAt: time.Now().UTC(), + Source: m.Source.Platform, + SourceDockerEngineID: m.Source.DockerEngineID, + EnvMode: exportEnvMode(opts), } projectGroups := projectGroupsForApps(m, apps) @@ -92,6 +95,7 @@ func Export(m manifest.Manifest, opts Options) (Summary, error) { summary.Apps = append(summary.Apps, AppSummary{ Name: app.Name, Directory: filepath.ToSlash(dirName), + Platform: app.Platform, Role: migrationRole(app), ProjectGroup: projectGroups[appKey(app)], PrivateEnvValues: opts.IncludeEnvValues, diff --git a/internal/manifest/manifest.go b/internal/manifest/manifest.go index 48511a7..f140c12 100644 --- a/internal/manifest/manifest.go +++ b/internal/manifest/manifest.go @@ -23,8 +23,9 @@ type ProxyArtifact struct { } type Source struct { - Platform string `json:"platform"` - Hostname string `json:"hostname,omitempty"` + Platform string `json:"platform"` + Hostname string `json:"hostname,omitempty"` + DockerEngineID string `json:"dockerEngineId,omitempty"` } type App struct { diff --git a/internal/preparer/preparer.go b/internal/preparer/preparer.go index 7f59399..2f09a74 100644 --- a/internal/preparer/preparer.go +++ b/internal/preparer/preparer.go @@ -78,16 +78,19 @@ type Options struct { } type Result struct { - APIVersion string `json:"apiVersion"` - BundleDir string `json:"bundleDir"` - Target string `json:"target"` - Status Status `json:"status"` - Apps []AppPlan `json:"apps"` + APIVersion string `json:"apiVersion"` + BundleDir string `json:"bundleDir"` + Target string `json:"target"` + Source string `json:"source,omitempty"` + SourceDockerEngineID string `json:"sourceDockerEngineId,omitempty"` + Status Status `json:"status"` + Apps []AppPlan `json:"apps"` } type AppPlan struct { Name string `json:"name"` Directory string `json:"directory"` + Platform string `json:"platform,omitempty"` Role string `json:"role,omitempty"` ProjectGroup *ProjectGroup `json:"projectGroup,omitempty"` Status Status `json:"status"` @@ -236,7 +239,14 @@ func Plan(opts Options) (Result, error) { return Result{}, err } - result := Result{APIVersion: APIVersion, BundleDir: opts.BundleDir, Target: opts.Target, Status: StatusGreen} + result := Result{ + APIVersion: APIVersion, + BundleDir: opts.BundleDir, + Target: opts.Target, + Source: index.Source, + SourceDockerEngineID: index.SourceDockerEngineID, + Status: StatusGreen, + } for _, app := range index.Apps { if opts.AppName != "" && app.Name != opts.AppName && app.Directory != opts.AppName && planutil.Slug(app.Name) != planutil.Slug(opts.AppName) { continue @@ -267,7 +277,7 @@ func planApp(bundleDir, target string, app exporter.AppSummary) (AppPlan, error) return AppPlan{}, fmt.Errorf("read topology for %s: %w", app.Name, err) } - plan := AppPlan{Name: app.Name, Directory: app.Directory, Role: app.Role, ProjectGroup: projectGroup(app.ProjectGroup), Status: StatusGreen, Readiness: ReadinessReadyToCreate} + plan := AppPlan{Name: app.Name, Directory: app.Directory, Platform: app.Platform, Role: app.Role, ProjectGroup: projectGroup(app.ProjectGroup), Status: StatusGreen, Readiness: ReadinessReadyToCreate} plan.Resources = resourceSpecs(app, appDir, topology) addReadinessGates(&plan, topology) plan.add(SeverityInfo, "compose", fmt.Sprintf("would create %s compose app from compose.yaml", target)) diff --git a/internal/rollback/rollback.go b/internal/rollback/rollback.go index 24ccbcb..d6c3380 100644 --- a/internal/rollback/rollback.go +++ b/internal/rollback/rollback.go @@ -32,12 +32,14 @@ type Options struct { } type Result struct { - APIVersion string `json:"apiVersion"` - BundleDir string `json:"bundleDir"` - Target string `json:"target"` - DryRun bool `json:"dryRun"` - Status preparer.Status `json:"status"` - Apps []AppPlan `json:"apps"` + APIVersion string `json:"apiVersion"` + BundleDir string `json:"bundleDir"` + Target string `json:"target"` + DryRun bool `json:"dryRun"` + AutomaticAvailable *bool `json:"automaticAvailable,omitempty"` + AutomaticBlocker string `json:"automaticBlocker,omitempty"` + Status preparer.Status `json:"status"` + Apps []AppPlan `json:"apps"` } type AppPlan struct { diff --git a/internal/safepath/rename_noreplace_darwin.go b/internal/safepath/rename_noreplace_darwin.go new file mode 100644 index 0000000..fdbf346 --- /dev/null +++ b/internal/safepath/rename_noreplace_darwin.go @@ -0,0 +1,9 @@ +//go:build darwin + +package safepath + +import "golang.org/x/sys/unix" + +func renamePrivateFileNoReplace(dirFD int, from, to string) error { + return unix.RenameatxNp(dirFD, from, dirFD, to, unix.RENAME_EXCL) +} diff --git a/internal/safepath/rename_noreplace_linux.go b/internal/safepath/rename_noreplace_linux.go new file mode 100644 index 0000000..9d8908d --- /dev/null +++ b/internal/safepath/rename_noreplace_linux.go @@ -0,0 +1,9 @@ +//go:build linux + +package safepath + +import "golang.org/x/sys/unix" + +func renamePrivateFileNoReplace(dirFD int, from, to string) error { + return unix.Renameat2(dirFD, from, dirFD, to, unix.RENAME_NOREPLACE) +} diff --git a/internal/safepath/rename_noreplace_other.go b/internal/safepath/rename_noreplace_other.go new file mode 100644 index 0000000..2799bab --- /dev/null +++ b/internal/safepath/rename_noreplace_other.go @@ -0,0 +1,9 @@ +//go:build !linux && !darwin && !windows + +package safepath + +import "fmt" + +func renamePrivateFileNoReplace(int, string, string) error { + return fmt.Errorf("secure no-replace rename is unavailable on this platform") +} diff --git a/internal/safepath/safepath.go b/internal/safepath/safepath.go index 0427247..c324027 100644 --- a/internal/safepath/safepath.go +++ b/internal/safepath/safepath.go @@ -40,6 +40,10 @@ func OpenPrivateDirNoFollow(path string) (*PrivateDir, error) { return openPrivateDirPathNoFollow(path, true) } +func OpenExistingPrivateDirNoFollow(path string) (*PrivateDir, error) { + return openPrivateDirPathNoFollow(path, false) +} + func openPrivateDirPathNoFollow(path string, create bool) (*PrivateDir, error) { if strings.TrimSpace(path) == "" { return nil, fmt.Errorf("private directory path is empty") @@ -93,6 +97,34 @@ func (d *PrivateDir) CreateFile(name string, mode os.FileMode) (*os.File, error) return createPrivateFileNoFollow(dir, name, mode) } +func (d *PrivateDir) OpenFile(name string) (*os.File, error) { + if err := validatePrivateFileName(name); err != nil { + return nil, err + } + dir, err := d.openFile() + if err != nil { + return nil, err + } + file, err := openPrivateFileNoFollow(dir, name) + if err != nil { + return nil, err + } + info, err := file.Stat() + if err != nil { + _ = file.Close() + return nil, err + } + if !info.Mode().IsRegular() { + _ = file.Close() + return nil, fmt.Errorf("private file %s is not regular", name) + } + if err := d.ValidatePath(); err != nil { + _ = file.Close() + return nil, err + } + return file, nil +} + func (d *PrivateDir) ReadFile(name string) ([]byte, error) { if err := validatePrivateFileName(name); err != nil { return nil, err @@ -178,6 +210,20 @@ func (d *PrivateDir) WriteFileAtomic(name string, data []byte, mode os.FileMode) return d.ValidatePath() } +func (d *PrivateDir) WriteFileAtomicNew(name string, data []byte, mode os.FileMode) error { + if err := validatePrivateFileName(name); err != nil { + return err + } + dir, err := d.openFile() + if err != nil { + return err + } + if err := writePrivateFileAtomicNewNoFollow(dir, name, data, mode); err != nil { + return err + } + return d.ValidatePath() +} + func (d *PrivateDir) Remove(name string) error { if err := validatePrivateFileName(name); err != nil { return err @@ -218,6 +264,9 @@ func (d *PrivateDir) ValidatePath() error { if !os.SameFile(heldInfo, currentInfo) { return fmt.Errorf("private directory path changed during operation") } + if currentInfo.Mode().Perm()&0o077 != 0 { + return fmt.Errorf("private directory %s has permissions %o; remove group and other access before retrying", d.path, currentInfo.Mode().Perm()) + } return nil } diff --git a/internal/safepath/safepath_other.go b/internal/safepath/safepath_other.go index 7cab423..aacd7fc 100644 --- a/internal/safepath/safepath_other.go +++ b/internal/safepath/safepath_other.go @@ -163,6 +163,10 @@ func writePrivateFileAtomicNoFollow(*os.File, string, []byte, os.FileMode) error return fmt.Errorf("secure private file replacement is unavailable on Windows") } +func writePrivateFileAtomicNewNoFollow(*os.File, string, []byte, os.FileMode) error { + return fmt.Errorf("secure private file creation is unavailable on Windows") +} + func removePrivateFileNoFollow(*os.File, string) error { return fmt.Errorf("secure private file removal is unavailable on Windows") } diff --git a/internal/safepath/safepath_test.go b/internal/safepath/safepath_test.go index 8cf59e5..c17b69a 100644 --- a/internal/safepath/safepath_test.go +++ b/internal/safepath/safepath_test.go @@ -1,6 +1,7 @@ package safepath import ( + "bytes" "errors" "os" "path/filepath" @@ -16,3 +17,31 @@ func TestReadPrivateFileDoesNotCreateMissingDirectory(t *testing.T) { t.Fatalf("private read created missing directory: %v", err) } } + +func TestWriteFileAtomicNewDoesNotReplaceExistingFile(t *testing.T) { + dirPath, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatal(err) + } + if err := os.Chmod(dirPath, 0o700); err != nil { + t.Fatal(err) + } + dir, err := OpenPrivateDirNoFollow(dirPath) + if err != nil { + t.Fatal(err) + } + defer dir.Close() + if err := dir.WriteFileAtomicNew("secret", []byte("first"), 0o600); err != nil { + t.Fatal(err) + } + if err := dir.WriteFileAtomicNew("secret", []byte("second"), 0o600); err == nil { + t.Fatal("atomic new-file write replaced an existing file") + } + contents, err := os.ReadFile(filepath.Join(dirPath, "secret")) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(contents, []byte("first")) { + t.Fatalf("existing file changed after no-replace write: %q", contents) + } +} diff --git a/internal/safepath/safepath_unix.go b/internal/safepath/safepath_unix.go index 084935d..e4710da 100644 --- a/internal/safepath/safepath_unix.go +++ b/internal/safepath/safepath_unix.go @@ -20,12 +20,12 @@ func openPrivateDirNoFollow(path string, create bool) (*os.File, error) { if err != nil { return nil, err } - if err := validatePrivateDirDescriptor(parentFD, string(filepath.Separator)); err != nil { + if err := validatePrivateDirDescriptor(parentFD, string(filepath.Separator), true); err != nil { _ = unix.Close(parentFD) return nil, err } currentPath := string(filepath.Separator) - for _, part := range parts { + for index, part := range parts { if part == "" { continue } @@ -49,7 +49,7 @@ func openPrivateDirNoFollow(path string, create bool) (*os.File, error) { _ = unix.Close(parentFD) return nil, fmt.Errorf("open private directory component %s without following links: %w", componentPath, err) } - if err := validatePrivateDirDescriptor(nextFD, componentPath); err != nil { + if err := validatePrivateDirDescriptor(nextFD, componentPath, index < len(parts)-1); err != nil { _ = unix.Close(parentFD) _ = unix.Close(nextFD) return nil, err @@ -69,18 +69,25 @@ func openPrivateDirNoFollow(path string, create bool) (*os.File, error) { return dir, nil } -func validatePrivateDirDescriptor(fd int, path string) error { +func validatePrivateDirDescriptor(fd int, path string, allowSticky bool) error { var stat unix.Stat_t if err := unix.Fstat(fd, &stat); err != nil { return err } - if stat.Mode&0o022 != 0 { - return fmt.Errorf("private directory ancestor %s is writable by group or others", path) + return validatePrivateDirMetadata(uint32(stat.Mode), stat.Uid, path, allowSticky) +} + +func validatePrivateDirMetadata(mode, uid uint32, path string, allowSticky bool) error { + if !trustedPrivateDirOwner(uid) { + return fmt.Errorf("private directory component %s is owned by untrusted uid %d", path, uid) } - if !trustedPrivateDirOwner(stat.Uid) { - return fmt.Errorf("private directory ancestor %s is owned by untrusted uid %d", path, stat.Uid) + if mode&0o022 == 0 || (allowSticky && mode&unix.S_ISVTX != 0) { + return nil } - return nil + if allowSticky { + return fmt.Errorf("private directory ancestor %s is writable by group or others without the sticky bit; remove group and other write access or set the sticky bit before retrying", path) + } + return fmt.Errorf("private directory %s has permissions %o; remove group and other access before retrying", path, mode&0o777) } func trustedPrivateDirOwner(uid uint32) bool { @@ -174,6 +181,43 @@ func writePrivateFileAtomicNoFollow(dir *os.File, name string, data []byte, mode return unix.Fsync(int(dir.Fd())) } +func writePrivateFileAtomicNewNoFollow(dir *os.File, name string, data []byte, mode os.FileMode) (resultErr error) { + tmpName, err := privateTempName() + if err != nil { + return err + } + tmp, err := createPrivateFileNoFollow(dir, tmpName, mode) + if err != nil { + return err + } + removeTmp := true + defer func() { + _ = tmp.Close() + if removeTmp { + if err := removePrivateFileNoFollow(dir, tmpName); err != nil { + resultErr = errors.Join(resultErr, fmt.Errorf("remove incomplete private file %s: %w", filepath.Join(dir.Name(), tmpName), err)) + } + } + }() + if _, err := tmp.Write(data); err != nil { + return err + } + if err := tmp.Chmod(mode); err != nil { + return err + } + if err := tmp.Sync(); err != nil { + return err + } + if err := tmp.Close(); err != nil { + return err + } + if err := renamePrivateFileNoReplace(int(dir.Fd()), tmpName, name); err != nil { + return err + } + removeTmp = false + return unix.Fsync(int(dir.Fd())) +} + func removePrivateFileNoFollow(dir *os.File, name string) error { err := unix.Unlinkat(int(dir.Fd()), name, 0) if errors.Is(err, unix.ENOENT) { diff --git a/internal/safepath/safepath_unix_test.go b/internal/safepath/safepath_unix_test.go new file mode 100644 index 0000000..6161a19 --- /dev/null +++ b/internal/safepath/safepath_unix_test.go @@ -0,0 +1,108 @@ +//go:build !windows + +package safepath + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "golang.org/x/sys/unix" +) + +func TestOpenExistingPrivateDirNoFollowAcceptsOnlyProtectedWritableAncestors(t *testing.T) { + for _, test := range []struct { + name string + mode os.FileMode + wantErr string + }{ + {name: "sticky", mode: 0o0777 | os.ModeSticky}, + {name: "group-writable", mode: 0o0720, wantErr: "writable by group or others"}, + {name: "other-writable", mode: 0o0702, wantErr: "writable by group or others"}, + } { + t.Run(test.name, func(t *testing.T) { + root, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatal(err) + } + ancestor := filepath.Join(root, "writable") + if err := os.Mkdir(ancestor, 0o700); err != nil { + t.Fatal(err) + } + if err := os.Chmod(ancestor, test.mode); err != nil { + t.Fatal(err) + } + private := filepath.Join(ancestor, "private") + if err := os.Mkdir(private, 0o700); err != nil { + t.Fatal(err) + } + + dir, err := OpenExistingPrivateDirNoFollow(private) + if test.wantErr == "" { + if err != nil { + t.Fatalf("open private directory below sticky ancestor: %v", err) + } + if dir == nil { + t.Fatal("open private directory below sticky ancestor returned nil") + } + if err := dir.Close(); err != nil { + t.Fatal(err) + } + return + } + if dir != nil { + _ = dir.Close() + t.Fatal("opened private directory below non-sticky writable ancestor") + } + if err == nil || !strings.Contains(err.Error(), test.wantErr) { + t.Fatalf("open private directory below non-sticky writable ancestor: %v", err) + } + }) + } +} + +func TestValidatePrivateDirMetadataRejectsUntrustedStickyOwner(t *testing.T) { + t.Setenv("SUDO_UID", "") + untrustedUID := uint32(1) + if untrustedUID == uint32(os.Geteuid()) { + untrustedUID = 2 + } + err := validatePrivateDirMetadata(unix.S_ISVTX|0o0777, untrustedUID, "/sticky", true) + if err == nil || !strings.Contains(err.Error(), "owned by untrusted uid") { + t.Fatalf("validate untrusted sticky directory owner: %v", err) + } +} + +func TestPrivateDirValidatePathRejectsRelaxedPermissions(t *testing.T) { + for _, test := range []struct { + name string + mode os.FileMode + }{ + {name: "sticky-writable", mode: 0o0722 | os.ModeSticky}, + {name: "readable", mode: 0o0744}, + {name: "executable", mode: 0o0711}, + } { + t.Run(test.name, func(t *testing.T) { + root, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatal(err) + } + private := filepath.Join(root, "private") + if err := os.Mkdir(private, 0o700); err != nil { + t.Fatal(err) + } + dir, err := OpenExistingPrivateDirNoFollow(private) + if err != nil { + t.Fatal(err) + } + defer dir.Close() + if err := os.Chmod(private, test.mode); err != nil { + t.Fatal(err) + } + if err := dir.ValidatePath(); err == nil || !strings.Contains(err.Error(), "remove group and other access") { + t.Fatalf("validate private directory with relaxed permissions: %v", err) + } + }) + } +} diff --git a/internal/source/localdocker/scanner.go b/internal/source/localdocker/scanner.go index 6b8efd1..2d42fff 100644 --- a/internal/source/localdocker/scanner.go +++ b/internal/source/localdocker/scanner.go @@ -13,6 +13,7 @@ import ( "strings" "time" + "github.com/aikins01/bort/internal/dockercli" "github.com/aikins01/bort/internal/manifest" "github.com/aikins01/bort/internal/secrets" "github.com/aikins01/bort/internal/source" @@ -48,8 +49,12 @@ func (s *Scanner) Scan(ctx context.Context, opts source.ScanOptions) (manifest.M } } + engineID, err := s.engineID(ctx) + if err != nil { + return manifest.Manifest{}, err + } hostname, _ := os.Hostname() - result := manifest.New(manifest.Source{Platform: "docker", Hostname: hostname}, s.Now()) + result := manifest.New(manifest.Source{Platform: "docker", Hostname: hostname, DockerEngineID: engineID}, s.Now()) containers, err := s.inspectContainers(ctx) if err != nil { @@ -136,6 +141,22 @@ func (s *Scanner) Scan(ctx context.Context, opts source.ScanOptions) (manifest.M return result, nil } +func DockerEngineID(ctx context.Context) (string, error) { + return NewScanner().engineID(ctx) +} + +func (s *Scanner) engineID(ctx context.Context) (string, error) { + out, err := s.run(ctx, "info", "--format", "{{.ID}}") + if err != nil { + return "", fmt.Errorf("identify local Docker engine: %w", err) + } + id := strings.TrimSpace(string(out)) + if id == "" { + return "", fmt.Errorf("local Docker engine has no stable ID") + } + return id, nil +} + func (s *Scanner) inspectContainers(ctx context.Context) ([]containerInspect, error) { idsRaw, err := s.run(ctx, "ps", "-aq") if err != nil { @@ -345,7 +366,12 @@ func (s *Scanner) run(ctx context.Context, args ...string) ([]byte, error) { return s.runCommand(ctx, args...) } + env, err := dockercli.LocalEnvironment(os.Environ()) + if err != nil { + return nil, err + } cmd := exec.CommandContext(ctx, s.DockerPath, args...) + cmd.Env = env var stderr bytes.Buffer cmd.Stderr = &stderr out, err := cmd.Output() @@ -509,21 +535,29 @@ func isDokployTargetContainer(container containerInspect) bool { return true } swarmService := labels["com.docker.swarm.service.name"] - if swarmService == "dokploy" || strings.HasPrefix(swarmService, "dokploy-") { + switch swarmService { + case "dokploy", "dokploy-postgres", "dokploy-redis", "dokploy-traefik", "dokploy-monitoring", "dokploy-forward-auth": return true } name := strings.TrimPrefix(container.Name, "/") - return name == "dokploy" || strings.HasPrefix(name, "dokploy.") || strings.HasPrefix(name, "dokploy-") + switch name { + case "dokploy", "dokploy-traefik", "dokploy-monitoring": + return true + } + for _, prefix := range []string{"dokploy.", "dokploy-postgres.", "dokploy-redis.", "dokploy-traefik.", "dokploy-monitoring.", "dokploy-forward-auth."} { + if strings.HasPrefix(name, prefix) { + return true + } + } + return false } func detectPlatform(labels map[string]string) string { - for key, value := range labels { - combined := strings.ToLower(key + "=" + value) - switch { - case strings.Contains(combined, "coolify"): - return "coolify" - case strings.Contains(combined, "dokploy"): - return "dokploy" + for _, platform := range []string{"coolify", "dokploy"} { + for key, value := range labels { + if strings.Contains(strings.ToLower(key+"="+value), platform) { + return platform + } } } diff --git a/internal/source/localdocker/scanner_test.go b/internal/source/localdocker/scanner_test.go index 89ca2a5..5d09ad2 100644 --- a/internal/source/localdocker/scanner_test.go +++ b/internal/source/localdocker/scanner_test.go @@ -4,6 +4,9 @@ import ( "context" "encoding/json" "fmt" + "os" + "path/filepath" + "runtime" "strings" "testing" "time" @@ -12,6 +15,54 @@ import ( "github.com/aikins01/bort/internal/source" ) +func TestScannerDockerCommandRequiresSystemSocket(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("shell fixture is Unix-only") + } + bin := filepath.Join(t.TempDir(), "docker") + marker := filepath.Join(t.TempDir(), "executed") + script := "#!/bin/sh\ntouch " + marker + "\nprintf '%s' \"$DOCKER_HOST\"\n" + if err := os.WriteFile(bin, []byte(script), 0o700); err != nil { + t.Fatal(err) + } + + for _, setting := range []struct { + key string + value string + }{ + {key: "DOCKER_HOST", value: "ssh://remote.example"}, + {key: "DOCKER_CONTEXT", value: "production"}, + {key: "DOCKER_TLS_VERIFY", value: "1"}, + {key: "DOCKER_CERT_PATH", value: "/tmp/remote-certs"}, + } { + t.Run(setting.key, func(t *testing.T) { + for _, key := range []string{"DOCKER_HOST", "DOCKER_CONTEXT", "DOCKER_TLS_VERIFY", "DOCKER_CERT_PATH"} { + t.Setenv(key, "") + } + t.Setenv(setting.key, setting.value) + _, err := (&Scanner{DockerPath: bin}).run(context.Background(), "ps", "-aq") + if err == nil || !strings.Contains(err.Error(), setting.key) { + t.Fatalf("expected %s refusal, got %v", setting.key, err) + } + if _, err := os.Stat(marker); !os.IsNotExist(err) { + t.Fatalf("Docker command ran despite %s refusal: %v", setting.key, err) + } + }) + } + + for _, key := range []string{"DOCKER_CONTEXT", "DOCKER_TLS_VERIFY", "DOCKER_CERT_PATH"} { + t.Setenv(key, "") + } + t.Setenv("DOCKER_HOST", "") + out, err := (&Scanner{DockerPath: bin}).run(context.Background(), "ps", "-aq") + if err != nil { + t.Fatalf("system socket command failed: %v", err) + } + if string(out) != "unix:///var/run/docker.sock" { + t.Fatalf("Docker command saw DOCKER_HOST=%q", out) + } +} + func TestRoutesFromLabelsExtractsHostsAndServicePort(t *testing.T) { labels := map[string]string{ "traefik.http.routers.web.rule": "Host(`app.example.com`, `www.example.com`) || PathPrefix(`/`)", @@ -236,6 +287,8 @@ func TestScanPopulatesNewServiceFields(t *testing.T) { Now: func() time.Time { return time.Unix(0, 0).UTC() }, runCommand: func(_ context.Context, args ...string) ([]byte, error) { switch { + case len(args) == 3 && args[0] == "info" && args[1] == "--format" && args[2] == "{{.ID}}": + return []byte("engine-test\n"), nil case len(args) == 2 && args[0] == "ps" && args[1] == "-aq": return []byte("c1\n"), nil case len(args) > 0 && args[0] == "inspect": @@ -271,6 +324,9 @@ func TestScanPopulatesNewServiceFields(t *testing.T) { if err != nil { t.Fatal(err) } + if m.Source.DockerEngineID != "engine-test" { + t.Fatalf("expected Docker engine identity, got %q", m.Source.DockerEngineID) + } if len(m.Apps) != 1 || len(m.Apps[0].Services) != 1 { t.Fatalf("unexpected apps: %+v", m.Apps) } @@ -304,15 +360,21 @@ func TestScanIgnoresDokployTargetComposeStacks(t *testing.T) { Now: func() time.Time { return time.Unix(0, 0).UTC() }, runCommand: func(_ context.Context, args ...string) ([]byte, error) { switch { + case len(args) == 3 && args[0] == "info" && args[1] == "--format" && args[2] == "{{.ID}}": + return []byte("engine-test\n"), nil case len(args) == 2 && args[0] == "ps" && args[1] == "-aq": - return []byte("source\ntarget\ndokploy\ndokploy-db\ndokploy-traefik\n"), nil + return []byte("source\ntarget\ndokploy\ndokploy-db\ndokploy-traefik\ndokploy-traefik-swarm\ndokploy-monitoring\ndokploy-fwd\ndokploy-backup\n"), nil case len(args) > 0 && args[0] == "inspect": return []byte(`[ {"Id":"source","Name":"/source-api","Image":"sha256:source","Config":{"Image":"app:1","Labels":{"com.docker.compose.project":"source-api","com.docker.compose.project.working_dir":"/data/coolify/app/source-api"}},"State":{"Status":"running"},"Mounts":[],"NetworkSettings":{"Ports":{},"Networks":{}}}, {"Id":"target","Name":"/compose-noisy-source-api-1","Image":"sha256:target","Config":{"Image":"app:1","Labels":{"com.docker.compose.project":"compose-noisy","com.docker.compose.project.working_dir":"/etc/dokploy/compose/compose-noisy/code"}},"State":{"Status":"running"},"Mounts":[],"NetworkSettings":{"Ports":{},"Networks":{}}}, {"Id":"dokploy","Name":"/dokploy.1.task","Image":"sha256:dokploy","Config":{"Image":"dokploy/dokploy:latest","Labels":{"com.docker.swarm.service.name":"dokploy"}},"State":{"Status":"running"},"Mounts":[],"NetworkSettings":{"Ports":{},"Networks":{}}}, {"Id":"dokploy-db","Name":"/dokploy-postgres.1.task","Image":"sha256:db","Config":{"Image":"postgres:16","Labels":{"com.docker.swarm.service.name":"dokploy-postgres"}},"State":{"Status":"running"},"Mounts":[],"NetworkSettings":{"Ports":{},"Networks":{}}}, - {"Id":"dokploy-traefik","Name":"/dokploy-traefik","Image":"sha256:traefik","Config":{"Image":"traefik:v3.6.7","Labels":{}},"State":{"Status":"created"},"Mounts":[],"NetworkSettings":{"Ports":{},"Networks":{}}} + {"Id":"dokploy-traefik","Name":"/dokploy-traefik","Image":"sha256:traefik","Config":{"Image":"traefik:v3.6.7","Labels":{}},"State":{"Status":"created"},"Mounts":[],"NetworkSettings":{"Ports":{},"Networks":{}}}, + {"Id":"dokploy-traefik-swarm","Name":"/dokploy-traefik.1.task","Image":"sha256:traefik","Config":{"Image":"traefik:v3.6.7","Labels":{"com.docker.swarm.service.name":"dokploy-traefik"}},"State":{"Status":"running"},"Mounts":[],"NetworkSettings":{"Ports":{},"Networks":{}}}, + {"Id":"dokploy-monitoring","Name":"/dokploy-monitoring","Image":"sha256:monitoring","Config":{"Image":"dokploy/monitoring:latest","Labels":{}},"State":{"Status":"running"},"Mounts":[],"NetworkSettings":{"Ports":{},"Networks":{}}}, + {"Id":"dokploy-fwd","Name":"/dokploy-forward-auth.1.task","Image":"sha256:oauth2","Config":{"Image":"quay.io/oauth2-proxy/oauth2-proxy:v7.6.0","Labels":{"com.docker.swarm.service.name":"dokploy-forward-auth"}},"State":{"Status":"running"},"Mounts":[],"NetworkSettings":{"Ports":{},"Networks":{}}}, + {"Id":"dokploy-backup","Name":"/dokploy-backup","Image":"sha256:backup","Config":{"Image":"example/backup:1","Labels":{}},"State":{"Status":"running"},"Mounts":[],"NetworkSettings":{"Ports":{},"Networks":{}}} ]`), nil case len(args) >= 2 && args[0] == "image" && args[1] == "inspect": return []byte(`[]`), nil @@ -329,8 +391,12 @@ func TestScanIgnoresDokployTargetComposeStacks(t *testing.T) { if err != nil { t.Fatal(err) } - if len(m.Apps) != 1 || m.Apps[0].Name != "source-api" { - t.Fatalf("expected only source app, got %+v", m.Apps) + appNames := map[string]bool{} + for _, app := range m.Apps { + appNames[app.Name] = true + } + if len(m.Apps) != 2 || !appNames["source-api"] || !appNames["dokploy-backup"] { + t.Fatalf("expected source app and unrelated dokploy-prefix backup, got %+v", m.Apps) } } @@ -358,3 +424,18 @@ func inspectContainersJSON(t *testing.T, ids []string) []byte { } return encoded } + +func TestDetectPlatformPrefersCoolifyRegardlessOfLabelOrder(t *testing.T) { + labels := map[string]string{ + "com.docker.compose.project": "abc", + "traefik.http.routers.r.rule": "Host(`dokploy.example.com`)", + "com.docker.compose.service": "dokploy-exporter", + "coolify.managed": "true", + "com.docker.compose.project.working_dir": "/data/coolify/applications/abc", + } + for i := 0; i < 100; i++ { + if got := detectPlatform(labels); got != "coolify" { + t.Fatalf("iteration %d: detectPlatform=%q, want coolify", i, got) + } + } +} diff --git a/internal/target/dokploy/apply.go b/internal/target/dokploy/apply.go index 30260ab..54cb1a9 100644 --- a/internal/target/dokploy/apply.go +++ b/internal/target/dokploy/apply.go @@ -10,6 +10,7 @@ import ( "fmt" "os" "path/filepath" + "regexp" "sort" "strconv" "strings" @@ -72,12 +73,22 @@ const ( ) type StepProgress struct { - Index int - Total int - Step Step - Status StepStatus - Message string - Err error + Index int + Total int + Step Step + Status StepStatus + Message string + Err error + MutationAmbiguous bool + RequiresNewRun bool + Target *TargetIdentity +} + +type TargetIdentity struct { + ProjectID string + EnvironmentID string + ComposeID string + ComposeAppName string } type PrepareDecision struct { @@ -100,15 +111,86 @@ type Plan struct { BundleFiles map[string][]byte ApprovedPrepareDecisions map[PrepareDecision]struct{} RunName string + RunID string RunDir string ResumeFrom int + TargetIdentities map[string]TargetIdentity BeforeStep *func(StepProgress) error OnProgress *func(StepProgress) stepTimeout time.Duration } +type planStateOrder int + +const ( + // planStateStaged transfers state into Bort-owned staging volumes + // before the Dokploy deployment exists, then deploys a compose file + // that mounts those volumes. no target writer ever runs during a + // copy, so Dokploy needs no fence it cannot provide. + planStateStaged planStateOrder = iota + // planStateInPlaceV1Alpha2 and planStateInPlaceV1Alpha1 reproduce the + // historical orders (deploy first, pause target writers, copy into the + // deployed volumes) so recorded ledgers keep matching by index. + planStateInPlaceV1Alpha2 + planStateInPlaceV1Alpha1 +) + func PlanFromArtifacts(prepare preparer.Result, sync syncplan.Result, cutover gateway.Result) Plan { + return planFromArtifacts(prepare, sync, cutover, planStateStaged) +} + +func PlanFromArtifactsV1Alpha2(prepare preparer.Result, sync syncplan.Result, cutover gateway.Result) Plan { + return planFromArtifacts(prepare, sync, cutover, planStateInPlaceV1Alpha2) +} + +func LegacyPlanFromArtifactsV1Alpha1(prepare preparer.Result, sync syncplan.Result, cutover gateway.Result) Plan { + return planFromArtifacts(prepare, sync, cutover, planStateInPlaceV1Alpha1) +} + +func appStateSteps(prepare preparer.Result, app syncplan.AppPlan) (dataStoreSteps, volumeSteps []Step) { + prepApp, hasPrepApp := findPrepareApp(prepare, app.Name) + for _, step := range app.Steps { + switch step.ResourceType { + case "volume": + if step.Strategy == syncplan.StrategyNone || step.TargetAction == "preserve_vps_file_mount" { + continue + } + // volumes owned by logical-dump or skip-strategy stores must + // not get a raw sync step: logical owns migration, skip + // must leave the target empty. only "volume" migration + // kind allows raw copy of a data store volume. + if hasPrepApp { + if volume, ok := findPrepareVolume(prepApp, step.ResourceRef); ok && volumeOwnedByLogicalOrSkippedStore(prepApp, volume) { + continue + } + } + volumeSteps = append(volumeSteps, Step{Kind: StepSyncVolume, App: app.Name, Ref: step.ResourceRef}) + case "data_store": + if hasPrepApp { + if store, ok := findPrepareDataStore(prepApp, step.ResourceRef); ok && dataStoreMigrationKind(store) != dataStoreMigrationLogical { + continue + } + } + dataStoreSteps = append(dataStoreSteps, + Step{Kind: StepDumpDataStore, App: app.Name, Ref: step.ResourceRef}, + Step{Kind: StepRestoreDataStore, App: app.Name, Ref: step.ResourceRef}, + ) + } + } + return dataStoreSteps, volumeSteps +} + +func planFromArtifacts(prepare preparer.Result, sync syncplan.Result, cutover gateway.Result, order planStateOrder) Plan { plan := Plan{Prepare: prepare, Sync: sync, Cutover: cutover} + staged := order == planStateStaged + legacyOrder := order == planStateInPlaceV1Alpha1 + statefulApps := map[string]struct{}{} + for _, app := range sync.Apps { + if dataStoreSteps, volumeSteps := appStateSteps(prepare, app); len(dataStoreSteps) > 0 || len(volumeSteps) > 0 { + statefulApps[app.Name] = struct{}{} + } + } + deferredPushSteps := map[string]Step{} for _, app := range prepare.Apps { plan.Steps = append(plan.Steps, Step{Kind: StepCreateProject, App: app.Name, Ref: dokployProjectName(app)}) if app.TargetResources != nil && app.TargetResources.Dokploy != nil { @@ -117,16 +199,17 @@ func PlanFromArtifacts(prepare preparer.Result, sync syncplan.Result, cutover ga if len(app.TargetResources.Dokploy.EnvFiles) > 0 { plan.Steps = append(plan.Steps, Step{Kind: StepUploadEnv, App: app.Name, Ref: composeRef}) } - plan.Steps = append(plan.Steps, Step{Kind: StepPushImage, App: app.Name, Ref: composeRef}) + pushStep := Step{Kind: StepPushImage, App: app.Name, Ref: composeRef} + if _, stateful := statefulApps[app.Name]; staged && stateful { + deferredPushSteps[app.Name] = pushStep + } else { + plan.Steps = append(plan.Steps, pushStep) + } for _, volume := range app.TargetResources.Dokploy.Volumes { if volume.Type != "volume" { continue } - ref := volume.Name - if ref == "" { - ref = volume.Target - } - plan.Steps = append(plan.Steps, Step{Kind: StepCreateVolume, App: app.Name, Ref: ref}) + plan.Steps = append(plan.Steps, Step{Kind: StepCreateVolume, App: app.Name, Ref: createVolumeRef(volume.Name, volume.Target)}) } } } @@ -139,52 +222,37 @@ func PlanFromArtifacts(prepare preparer.Result, sync syncplan.Result, cutover ga routedAppNames[app.Name] = struct{}{} } } + routedTargetResumeSteps := []Step{} for _, app := range sync.Apps { - var dataStoreSteps, volumeSteps []Step - prepApp, hasPrepApp := findPrepareApp(prepare, app.Name) - for _, step := range app.Steps { - switch step.ResourceType { - case "volume": - if step.Strategy == syncplan.StrategyNone || step.TargetAction == "preserve_vps_file_mount" { - continue - } - // volumes owned by logical-dump or skip-strategy stores must - // not get a raw sync step: logical owns migration, skip - // must leave the target empty. only "volume" migration - // kind allows raw copy of a data store volume. - if hasPrepApp { - if volume, ok := findPrepareVolume(prepApp, step.ResourceRef); ok && volumeOwnedByLogicalOrSkippedStore(prepApp, volume) { - continue - } - } - volumeSteps = append(volumeSteps, Step{Kind: StepSyncVolume, App: app.Name, Ref: step.ResourceRef}) - case "data_store": - if hasPrepApp { - if store, ok := findPrepareDataStore(prepApp, step.ResourceRef); ok && dataStoreMigrationKind(store) != dataStoreMigrationLogical { - continue - } - } - dataStoreSteps = append(dataStoreSteps, - Step{Kind: StepDumpDataStore, App: app.Name, Ref: step.ResourceRef}, - Step{Kind: StepRestoreDataStore, App: app.Name, Ref: step.ResourceRef}, - ) - } + dataStoreSteps, volumeSteps := appStateSteps(prepare, app) + if len(dataStoreSteps) == 0 && len(volumeSteps) == 0 { + continue } // pause runs before any state work so app writers stop before // pg_dump captures its snapshot and before raw volume copies // see the on-disk format. logical-dump stores stay live (they // are excluded from quiesce targets); only the app's writers // and volume-strategy stores actually stop here. - if len(dataStoreSteps) > 0 || len(volumeSteps) > 0 { - plan.Steps = append(plan.Steps, Step{Kind: StepPauseSource, App: app.Name, Ref: app.Name}) - } + plan.Steps = append(plan.Steps, Step{Kind: StepPauseSource, App: app.Name, Ref: app.Name}) plan.Steps = append(plan.Steps, dataStoreSteps...) plan.Steps = append(plan.Steps, volumeSteps...) - if len(dataStoreSteps) > 0 || len(volumeSteps) > 0 { - if _, routed := routedAppNames[app.Name]; !routed { + _, routed := routedAppNames[app.Name] + if staged { + if !routed { plan.Steps = append(plan.Steps, Step{Kind: StepResumeSource, App: app.Name, Ref: app.Name}) } + if pushStep, ok := deferredPushSteps[app.Name]; ok { + plan.Steps = append(plan.Steps, pushStep) + } + continue + } + if !routed { + plan.Steps = append(plan.Steps, Step{Kind: StepResumeSource, App: app.Name, Ref: app.Name}) plan.Steps = append(plan.Steps, Step{Kind: StepResumeTarget, App: app.Name, Ref: app.Name}) + } else if legacyOrder { + plan.Steps = append(plan.Steps, Step{Kind: StepResumeTarget, App: app.Name, Ref: app.Name}) + } else { + routedTargetResumeSteps = append(routedTargetResumeSteps, Step{Kind: StepResumeTarget, App: app.Name, Ref: app.Name}) } } hasRoutes := false @@ -200,16 +268,22 @@ func PlanFromArtifacts(prepare preparer.Result, sync syncplan.Result, cutover ga } } } - for _, appName := range routedApps { - plan.Steps = append(plan.Steps, Step{Kind: StepActivateRoutes, App: appName, Ref: "routes"}) - } // only swap proxies when something actually depends on :80/:443 — // keeps no-route migrations from disturbing a healthy coolify host. if hasRoutes { - plan.Steps = append(plan.Steps, - Step{Kind: StepStopCoolifyProxy, Ref: coolifyProxyContainer}, - Step{Kind: StepStartDokployProxy, Ref: dokployProxyContainer}, - ) + if legacyOrder { + for _, appName := range routedApps { + plan.Steps = append(plan.Steps, Step{Kind: StepActivateRoutes, App: appName, Ref: "routes"}) + } + plan.Steps = append(plan.Steps, Step{Kind: StepStopCoolifyProxy, Ref: coolifyProxyContainer}) + } else { + plan.Steps = append(plan.Steps, Step{Kind: StepStopCoolifyProxy, Ref: coolifyProxyContainer}) + plan.Steps = append(plan.Steps, routedTargetResumeSteps...) + for _, appName := range routedApps { + plan.Steps = append(plan.Steps, Step{Kind: StepActivateRoutes, App: appName, Ref: "routes"}) + } + } + plan.Steps = append(plan.Steps, Step{Kind: StepStartDokployProxy, Ref: dokployProxyContainer}) } return plan } @@ -237,7 +311,7 @@ func dokployProjectName(app preparer.AppPlan) string { // re-route. containers are stopped, never removed: operators on a long // rollback window can docker start them back up to roll back manually. func PlanForCommit(prepare preparer.Result, commit gateway.Result) Plan { - plan := Plan{Prepare: prepare, Cutover: commit} + plan := Plan{Prepare: prepare, Cutover: commit, stepTimeout: dockerStartTimeout} for _, app := range prepare.Apps { plan.Steps = append(plan.Steps, Step{Kind: StepStopSourceApp, App: app.Name, Ref: app.Name}) } @@ -263,12 +337,15 @@ type appCache struct { ComposeAppName string DiscoveryRedeployAttempted bool TargetWritersStopped []dockerContainer + SourcePausedContainers []sourcePausedContainer + SourcePauseRecorded bool MigratedVolumeMounts map[string]migratedVolumeMount } type applyContext struct { - plan Plan - cache map[string]*appCache + plan Plan + cache map[string]*appCache + stagingEnvFormat stagingEnvFormat } func (a *applyContext) entry(app string) *appCache { @@ -312,9 +389,156 @@ func validatePlanReadyForLiveApply(plan Plan) error { } } } + return ValidateStagedTransfer(plan) +} + +func ValidateStagedTransfer(plan Plan) error { + for _, step := range plan.Steps { + if step.Kind != StepSyncVolume || shouldSkipApplyStep(plan, step) || !appStateIsStaged(plan, step.App) { + continue + } + app, ok := findPrepareApp(plan.Prepare, step.App) + if !ok { + continue + } + volume, ok := findPrepareVolume(app, step.Ref) + if !ok || volumeOwnedByLogicalOrSkippedStore(app, volume) { + continue + } + if _, ok := stagedVolumeFor(plan, step.App, volume); !ok { + return fmt.Errorf("%w: %s volume %s for app %s cannot be staged; only named volumes are transferred before deploy, so choose a recreate or managed data store strategy or change the source compose before live apply", ErrNotImplemented, volume.Type, volumeRefLabel(volume), step.App) + } + } + validated := map[string]struct{}{} + for _, step := range plan.Steps { + if step.Kind != StepCreateVolume || shouldSkipApplyStep(plan, step) { + continue + } + if _, done := validated[step.App]; done { + continue + } + validated[step.App] = struct{}{} + staged := stagedVolumesForApp(plan, step.App) + if len(staged) == 0 { + continue + } + composeFile, err := readComposeFile(plan, step.App) + if err != nil { + return err + } + if _, err := rewriteComposeStagedVolumes(composeFile, staged); err != nil { + return fmt.Errorf("staged volumes for app %s cannot be handed to Dokploy, so choose a recreate or managed data store strategy or change the source compose before live apply: %w", step.App, err) + } + } + type stagedSourceVolume struct { + app string + volume stagedVolume + } + bySourceName := map[string]stagedSourceVolume{} + for _, app := range plan.Prepare.Apps { + if isPlatformAppRole(app.Role) { + continue + } + for _, volume := range stagedVolumesForApp(plan, app.Name) { + name := strings.TrimSpace(volume.Source.Name) + if name == "" { + continue + } + if other, dup := bySourceName[name]; dup && other.volume.VolumeName != volume.VolumeName { + return fmt.Errorf("source volume %s is mounted by %s %s:%s and %s %s:%s, which would stage as separate volumes %s and %s; a shared named volume cannot be transferred before deploy, so choose a recreate or managed data store strategy or change the source compose before live apply", name, other.app, other.volume.Service, other.volume.Target, app.Name, volume.Service, volume.Target, other.volume.VolumeName, volume.VolumeName) + } + bySourceName[name] = stagedSourceVolume{app: app.Name, volume: volume} + } + } + return eachPlannedStagedRestore(plan, func(restore plannedStagedRestore) error { + envContent, err := readEnvContent(plan, restore.app.Name) + if err != nil { + return err + } + if stagingComposeProjectNamePattern.MatchString(restore.stagingCompose) || stagingComposeProjectNamePattern.MatchString(envContent) { + return fmt.Errorf("%w: data store service %s for app %s interpolates COMPOSE_PROJECT_NAME, which resolves to the Bort staging project instead of the Dokploy app; choose a recreate or managed data store strategy or change the source compose before live apply", ErrNotImplemented, restore.store.Service, restore.app.Name) + } + return nil + }) +} + +// ValidatePlannedPostgresDataDirs refuses, before a run starts live +// execution, staged Postgres restores whose planned layout cannot keep the +// data directory on a staged volume. A run that already has apply history +// must not be gated here: its pause_source preflight checks the created +// staging container and can restart any source it owns, whereas a +// refusal before Apply would leave that source stopped. +func ValidatePlannedPostgresDataDirs(plan Plan) error { + return eachPlannedStagedRestore(plan, func(restore plannedStagedRestore) error { + return requirePlannedPostgresDataDirStaged(restore.composeFile, restore.stagingCompose, restore.app, restore.store.Service, restore.stagedVolumes) + }) +} + +type plannedStagedRestore struct { + app preparer.AppPlan + store preparer.DataStoreResource + composeFile string + stagingCompose string + stagedVolumes []stagedVolume +} + +func eachPlannedStagedRestore(plan Plan, visit func(plannedStagedRestore) error) error { + for _, step := range plan.Steps { + if step.Kind != StepRestoreDataStore || shouldSkipApplyStep(plan, step) || !appStateIsStaged(plan, step.App) { + continue + } + app, ok := findPrepareApp(plan.Prepare, step.App) + if !ok { + continue + } + store, ok := findPrepareDataStore(app, step.Ref) + if !ok || dataStoreMigrationKind(store) != dataStoreMigrationLogical { + continue + } + composeFile, err := readComposeFile(plan, step.App) + if err != nil { + return err + } + stagedVolumes := stagedVolumesForService(plan, step.App, store.Service) + staged, err := stagingComposeFile(composeFile, store.Service, stagedVolumes) + if err != nil { + return fmt.Errorf("data store service %s for app %s cannot be staged, so choose a recreate or managed data store strategy or change the source compose before live apply: %w", store.Service, step.App, err) + } + if err := visit(plannedStagedRestore{app: app, store: store, composeFile: composeFile, stagingCompose: staged, stagedVolumes: stagedVolumes}); err != nil { + return err + } + } + return nil +} + +func appStagesDataStoreRestore(plan Plan, appName string) bool { + for _, step := range plan.Steps { + if step.Kind == StepRestoreDataStore && step.App == appName && !shouldSkipApplyStep(plan, step) && appStateIsStaged(plan, appName) { + return true + } + } + return false +} + +// requireStagingCompatibleCompose refuses target apps whose Dokploy deploy +// settings would interpolate the compose file differently from the staging +// run: a disabled .env or a custom deploy command. +func (c *Client) requireStagingCompatibleCompose(ctx context.Context, composeID, appName string) error { + compose, err := c.GetCompose(ctx, composeID) + if err != nil { + return fmt.Errorf("inspect Dokploy compose deploy settings for app %s: %w", appName, err) + } + if compose.CreateEnvFile != nil && !*compose.CreateEnvFile { + return fmt.Errorf("%w: Dokploy compose app %s has .env creation disabled, so its deploy would not see the environment the staged restore used; enable it in Dokploy and resume", ErrNotImplemented, appName) + } + if strings.TrimSpace(compose.Command) != "" { + return fmt.Errorf("%w: Dokploy compose app %s uses a custom deploy command, which bypasses the project name and .env the staged restore relies on; clear it in Dokploy and resume", ErrNotImplemented, appName) + } return nil } +var stagingComposeProjectNamePattern = regexp.MustCompile(`\$\{?COMPOSE_PROJECT_NAME\b`) + func appDecisionCodesApproved(plan Plan, app preparer.AppPlan) bool { found := false for _, gate := range app.Gates { @@ -393,6 +617,8 @@ func (e ambiguousDeployResponseError) Unwrap() error { return e.err } +func (e ambiguousDeployResponseError) mutationResponseAmbiguous() {} + type unsafeSourceResumeError struct { err error } @@ -446,6 +672,19 @@ func activeBortOverridePatchDetails(patches []activeBortOverridePatch) []string return details } +func findDokployPostgresContainer(ctx context.Context, runner dockerRunner) (string, error) { + const serviceLabel = "com.docker.swarm.service.name=dokploy-postgres" + out, err := runner.Output(ctx, "ps", "--filter", "label="+serviceLabel, "--filter", "status=running", "--format", "{{.ID}}") + if err != nil { + return "", fmt.Errorf("list local running dokploy-postgres containers: %w", err) + } + containerIDs := strings.Fields(string(out)) + if len(containerIDs) != 1 { + return "", fmt.Errorf("local Docker node has %d running dokploy-postgres service containers, want exactly 1", len(containerIDs)) + } + return containerIDs[0], nil +} + func (c *Client) activeBortOverridePatches(ctx context.Context, composeID string) ([]activeBortOverridePatch, error) { runner := c.dockerRunner() pg, err := findDokployPostgresContainer(ctx, runner) @@ -512,8 +751,28 @@ func (c *Client) Apply(ctx context.Context, plan Plan) error { if err := validatePlanReadyForLiveApply(plan); err != nil { return err } + for app, identity := range plan.TargetIdentities { + identity.ProjectID = strings.TrimSpace(identity.ProjectID) + identity.EnvironmentID = strings.TrimSpace(identity.EnvironmentID) + identity.ComposeID = strings.TrimSpace(identity.ComposeID) + identity.ComposeAppName = strings.TrimSpace(identity.ComposeAppName) + plan.TargetIdentities[app] = identity + } actx := &applyContext{plan: plan, cache: map[string]*appCache{}} - pausedApps := map[string]struct{}{} + if planStagesDataStoreRestore(plan) { + version, err := c.DokployVersion(ctx) + if err != nil { + return fmt.Errorf("resolve Dokploy version before staged state transfer: %w", err) + } + if actx.stagingEnvFormat, err = stagingEnvFormatForDokployVersion(version); err != nil { + return err + } + } + if err := actx.loadSourcePauseState(); err != nil { + return err + } + pausedApps := pausedSources{} + handedOff := map[string]struct{}{} coolifyProxyStopped := false total := len(plan.Steps) resumeFrom := plan.ResumeFrom @@ -523,9 +782,14 @@ func (c *Client) Apply(ctx context.Context, plan Plan) error { if resumeFrom > total { resumeFrom = total } - if resumeFrom > 0 && resumeFrom < total { - if err := c.primeResumeState(ctx, actx, plan.Steps[:resumeFrom], plan.Steps[resumeFrom], pausedApps, &coolifyProxyStopped); err != nil { - return err + if resumeFrom > 0 { + if err := c.primeResumeState(ctx, actx, plan.Steps[:resumeFrom], pausedApps, handedOff, &coolifyProxyStopped); err != nil { + resumePausedApps := pausedApps + if isUnsafeSourceResumeError(err) { + resumePausedApps = nil + } + cleanupErr := c.bestEffortResume(ctx, actx, plan, total, resumePausedApps, coolifyProxyStopped, isUnsafeTargetResumeError(err)) + return errors.Join(err, cleanupErr) } } for index := resumeFrom; index < len(plan.Steps); index++ { @@ -533,8 +797,8 @@ func (c *Client) Apply(ctx context.Context, plan Plan) error { started := StepProgress{Index: index, Total: total, Step: step, Status: StepStatusStarted} if plan.BeforeStep != nil { if err := (*plan.BeforeStep)(started); err != nil { - c.bestEffortResume(ctx, actx, plan, total, pausedApps, coolifyProxyStopped, false) - return fmt.Errorf("before dokploy step %s for %s (%s): %w", step.Kind, step.App, step.Ref, err) + stepErr := fmt.Errorf("before dokploy step %s for %s (%s): %w", step.Kind, step.App, step.Ref, err) + return errors.Join(stepErr, c.bestEffortResume(ctx, actx, plan, total, pausedApps, coolifyProxyStopped, false)) } } emitProgress(plan.OnProgress, started) @@ -542,35 +806,75 @@ func (c *Client) Apply(ctx context.Context, plan Plan) error { emitProgress(plan.OnProgress, StepProgress{Index: index, Total: total, Step: step, Status: StepStatusSkipped}) continue } - // mark eagerly: a partial pause that stops some containers and - // then errors must still be cleaned up by bestEffortResume. - // resume is stateless and only starts what is currently stopped, - // so marking before the step is safe. - if step.Kind == StepPauseSource { - pausedApps[step.App] = struct{}{} - } - if step.Kind == StepStopCoolifyProxy { - coolifyProxyStopped = true - } stepCtx := ctx var cancel context.CancelFunc if plan.stepTimeout > 0 { stepCtx, cancel = context.WithTimeout(ctx, plan.stepTimeout) } - err := c.applyStep(stepCtx, actx, step) + var err error + if step.Kind == StepPauseSource { + // an earlier attempt may have stopped some source containers + // before failing; a preflight refusal must still restart them. + if actx.entry(step.App).SourcePauseRecorded { + pausedApps[step.App] = false + } + err = c.preflightStagedRestores(stepCtx, actx, step.App) + } + if err == nil { + // mark eagerly: a partial pause that stops some containers and + // then errors must still be cleaned up by bestEffortResume. + // resume is stateless and only starts what is currently stopped, + // so marking before the step is safe. + if step.Kind == StepPauseSource { + pausedApps[step.App] = false + } + if step.Kind == StepStopCoolifyProxy { + coolifyProxyStopped = true + } + err = c.requireTransferredSourceStillPaused(stepCtx, actx, step, pausedApps, handedOff) + } + if err == nil { + err = c.applyStep(stepCtx, actx, step) + } if cancel != nil { cancel() } if err != nil { - emitProgress(plan.OnProgress, StepProgress{Index: index, Total: total, Step: step, Status: StepStatusError, Err: err}) + var preflight stagedRestorePreflightError + requiresNewRun := errors.As(err, &preflight) && preflight.requiresNewRun + _, sourceOwned := pausedApps[step.App] + // a refusal may only claim the run is releasable once any source + // containers Bort owns for this app are running again, and the + // cleanup resume reports at this same index, so the claim is + // recorded after a successful resume. + refused := StepProgress{Index: index, Total: total, Step: step, Status: StepStatusError, Err: err, MutationAmbiguous: mutationResponseMayHaveSucceeded(err), RequiresNewRun: requiresNewRun && !sourceOwned} + emitProgress(plan.OnProgress, refused) resumePausedApps := pausedApps if isUnsafeSourceResumeError(err) { resumePausedApps = nil } - c.bestEffortResume(ctx, actx, plan, total, resumePausedApps, coolifyProxyStopped, isUnsafeTargetResumeError(err)) - return fmt.Errorf("dokploy step %s for %s (%s): %w", step.Kind, step.App, step.Ref, err) + stepErr := fmt.Errorf("dokploy step %s for %s (%s): %w", step.Kind, step.App, step.Ref, err) + cleanupErr := c.bestEffortResume(ctx, actx, plan, total, resumePausedApps, coolifyProxyStopped, isUnsafeTargetResumeError(err)) + if requiresNewRun && sourceOwned && cleanupErr == nil { + refused.RequiresNewRun = true + emitProgress(plan.OnProgress, refused) + } + return errors.Join(stepErr, cleanupErr) + } + pausedApps.observeCompleted(step, handedOff) + completed := StepProgress{Index: index, Total: total, Step: step, Status: StepStatusOK} + if step.App != "" { + entry := actx.entry(step.App) + if entry.ProjectID != "" || entry.EnvironmentID != "" || entry.ComposeID != "" || entry.ComposeAppName != "" { + completed.Target = &TargetIdentity{ + ProjectID: entry.ProjectID, + EnvironmentID: entry.EnvironmentID, + ComposeID: entry.ComposeID, + ComposeAppName: entry.ComposeAppName, + } + } } - emitProgress(plan.OnProgress, StepProgress{Index: index, Total: total, Step: step, Status: StepStatusOK}) + emitProgress(plan.OnProgress, completed) } return nil } @@ -590,45 +894,140 @@ func isPlatformAppRole(role string) bool { return strings.EqualFold(strings.TrimSpace(role), "platform") } -func (c *Client) primeResumeState(ctx context.Context, actx *applyContext, completed []Step, next Step, pausedApps map[string]struct{}, coolifyProxyStopped *bool) error { +// pausedSources tracks the source apps this apply stopped. the value +// reports whether a state transfer completed after the pause, in which +// case restarting the source invalidates that transfer and must be +// recorded before the containers start. +type pausedSources map[string]bool + +func (p pausedSources) observeCompleted(step Step, handedOff map[string]struct{}) { + switch step.Kind { + case StepDumpDataStore, StepRestoreDataStore, StepSyncVolume: + if _, paused := p[step.App]; paused { + p[step.App] = true + } + case StepResumeSource: + delete(p, step.App) + case StepPushImage: + // a staged deployment now owns the transferred state; the source + // stays stopped until cutover retires it, so the handoff is the + // durable boundary and no retry may rewind past it. + if p[step.App] { + handedOff[step.App] = struct{}{} + } + delete(p, step.App) + } +} + +func (c *Client) requireTransferredSourceStillPaused(ctx context.Context, actx *applyContext, step Step, pausedApps pausedSources, handedOff map[string]struct{}) error { + switch step.Kind { + case StepPushImage: + if !pausedApps[step.App] { + return nil + } + ran, err := c.sourceRanSincePause(ctx, actx, step.App) + if err != nil { + return fmt.Errorf("verify source app %s stayed paused before deploying its transferred state: %w", step.App, err) + } + if ran { + return fmt.Errorf("source app %s ran after its state was transferred, which invalidated the transfer; retry the live apply to pause and transfer it again", step.App) + } + case StepActivateRoutes: + apps := make([]string, 0, len(handedOff)) + for app := range handedOff { + apps = append(apps, app) + } + sort.Strings(apps) + ran, err := c.firstSourceRanSincePause(ctx, actx, apps) + if err != nil { + return fmt.Errorf("verify handed-off source apps stayed paused before activating target routes: %w", err) + } + if ran != "" { + return unsafeSourceResumeError{err: staleStagedHandoffError(ran)} + } + } + return nil +} + +func staleStagedHandoffError(app string) error { + return fmt.Errorf("source app %s ran after its state was handed to the staged Dokploy deployment, so the staged data is stale; leaving the source untouched, start a new migration run to transfer it again", app) +} + +type sourcePauseHistory struct { + resumed bool + transferred bool + pushed bool +} + +func (c *Client) primeResumeState(ctx context.Context, actx *applyContext, completed []Step, pausedApps pausedSources, handedOff map[string]struct{}, coolifyProxyStopped *bool) error { if err := actx.loadMigratedVolumeMounts(); err != nil { return err } + if err := c.hydratePersistedTargetIdentities(ctx, actx); err != nil { + return fmt.Errorf("load persisted target identity for resume: %w", err) + } pushedApps := map[string]struct{}{} resumedTargetApps := map[string]struct{}{} + sourceHistory := map[string]sourcePauseHistory{} + for _, step := range completed { + history := sourceHistory[step.App] + switch step.Kind { + case StepPauseSource: + history = sourcePauseHistory{} + case StepResumeSource: + history.resumed = true + case StepDumpDataStore, StepRestoreDataStore, StepSyncVolume: + history.transferred = true + case StepPushImage: + history.pushed = true + } + sourceHistory[step.App] = history + } for _, step := range completed { if shouldSkipApplyStep(actx.plan, step) { continue } - replayNextApp := next.App != "" && step.App == next.App + pausedApps.observeCompleted(step, handedOff) switch step.Kind { case StepCreateProject: + if actx.plan.TargetIdentities[step.App].ProjectID != "" { + continue + } if err := c.applyCreateProject(ctx, actx, step); err != nil { return fmt.Errorf("refresh completed project for resume: %w", err) } case StepCreateService: - if err := c.applyCreateService(ctx, actx, step); err != nil { - return fmt.Errorf("refresh completed service for resume: %w", err) - } - case StepUploadEnv: - if !replayNextApp { + if actx.plan.TargetIdentities[step.App].ComposeID != "" { continue } - if err := c.applyUploadEnv(ctx, actx, step); err != nil { - return fmt.Errorf("refresh completed env for resume: %w", err) + if err := c.applyCreateService(ctx, actx, step); err != nil { + return fmt.Errorf("refresh completed service for resume: %w", err) } case StepPushImage: pushedApps[step.App] = struct{}{} - if !replayNextApp { + case StepPauseSource: + history := sourceHistory[step.App] + if history.resumed { continue } - if err := c.applyPushImage(ctx, actx, step); err != nil { - return fmt.Errorf("refresh completed deploy for resume: %w", err) + if !actx.entry(step.App).SourcePauseRecorded { + return unsafeSourceResumeError{err: fmt.Errorf("completed source pause for app %s has no durable container ownership state; refusing to infer which stopped containers Bort may restart", step.App)} } - case StepPauseSource: - pausedApps[step.App] = struct{}{} - case StepResumeSource: - delete(pausedApps, step.App) + ran, err := c.sourceRanSincePause(ctx, actx, step.App) + if err != nil { + return fmt.Errorf("verify completed source pause for resume app %s: %w", step.App, err) + } + if ran && history.pushed { + return unsafeSourceResumeError{err: staleStagedHandoffError(step.App)} + } + if ran && history.transferred { + pausedApps[step.App] = true + return fmt.Errorf("source app %s ran after its state was transferred, which invalidated the transfer; retry the live apply to pause and transfer it again", step.App) + } + if err := c.applyPauseSource(ctx, actx, step); err != nil { + return fmt.Errorf("restore completed source pause for resume app %s: %w", step.App, err) + } + pausedApps[step.App] = false case StepResumeTarget: resumedTargetApps[step.App] = struct{}{} actx.entry(step.App).TargetWritersStopped = nil @@ -646,52 +1045,88 @@ func (c *Client) primeResumeState(ctx context.Context, actx *applyContext, compl return fmt.Errorf("refresh target writer pause for resume app %s: %w", app, err) } } + if *coolifyProxyStopped { + reconcileCtx, cancel := context.WithTimeout(ctx, dockerStopTimeout) + err := stopProxyContainer(reconcileCtx, c.dockerRunner(), coolifyProxyContainer) + cancel() + if err != nil { + return fmt.Errorf("restore completed coolify proxy stop for resume: %w", err) + } + } return nil } -// bestEffortResume restarts source containers for every app that the -// failed run had paused. errors are swallowed so they don't mask the -// original failure; operators can still re-run apply or rollback. it -// uses a fresh context so a Ctrl-C that cancelled apply can still -// clean up — without that, source containers would stay stopped. -func (c *Client) bestEffortResume(_ context.Context, actx *applyContext, plan Plan, total int, pausedApps map[string]struct{}, coolifyProxyStopped bool, skipTargetWriters bool) { +func (c *Client) bestEffortResume(_ context.Context, actx *applyContext, plan Plan, total int, pausedApps pausedSources, coolifyProxyStopped bool, skipTargetWriters bool) error { if actx == nil { actx = &applyContext{} } actx.plan = plan if len(pausedApps) == 0 && !coolifyProxyStopped && (skipTargetWriters || !actx.hasStoppedTargetWriters()) { - return + return nil } + var cleanupErr error if !skipTargetWriters { - c.bestEffortResumeTargetWriters(actx, plan, total) + cleanupErr = errors.Join(cleanupErr, c.bestEffortResumeTargetWriters(actx, plan, total)) } - for app := range pausedApps { - cleanupCtx, cancel := context.WithTimeout(context.Background(), dockerStartTimeout) + sourcesRestored := true + for app, transferred := range pausedApps { resumeStep := Step{Kind: StepResumeSource, App: app, Ref: app} - emitProgress(plan.OnProgress, StepProgress{Index: total, Total: total, Step: resumeStep, Status: StepStatusStarted}) - if err := c.applyResumeSource(cleanupCtx, actx, resumeStep); err != nil { - cancel() - emitProgress(plan.OnProgress, StepProgress{Index: total, Total: total, Step: resumeStep, Status: StepStatusError, Err: err}) - continue + index, found := sourcePauseStepIndex(plan.Steps, app) + if !found { + index = total + } + started := StepProgress{Index: index, Total: total, Step: resumeStep, Status: StepStatusStarted} + if transferred && plan.BeforeStep != nil { + if err := (*plan.BeforeStep)(started); err != nil { + sourcesRestored = false + cleanupErr = errors.Join(cleanupErr, fmt.Errorf("source app %s stays stopped: its transferred state must be recorded as invalidated before its containers restart: %w", app, err)) + continue + } } + emitProgress(plan.OnProgress, started) + resumeCtx, cancel := context.WithTimeout(context.Background(), dockerStartTimeout) + err := c.applyResumeSource(resumeCtx, actx, resumeStep) cancel() - emitProgress(plan.OnProgress, StepProgress{Index: total, Total: total, Step: resumeStep, Status: StepStatusOK}) + if err != nil { + sourcesRestored = false + emitProgress(plan.OnProgress, StepProgress{Index: index, Total: total, Step: resumeStep, Status: StepStatusError, Err: err}) + cleanupErr = errors.Join(cleanupErr, fmt.Errorf("resume source app %s after failed apply: %w", app, err)) + continue + } + emitProgress(plan.OnProgress, StepProgress{Index: index, Total: total, Step: resumeStep, Status: StepStatusOK}) } // proxy resume runs last so coolify-proxy comes back online only // after source apps are restartable. it gets its own fresh // context so a slow source-resume loop can't starve the proxy // restart of its budget. - if coolifyProxyStopped { + if coolifyProxyStopped && sourcesRestored { proxyCtx, cancelProxy := context.WithTimeout(context.Background(), dockerStartTimeout) defer cancelProxy() - resumeProxyStep := Step{Kind: StepStopCoolifyProxy, Ref: coolifyProxyContainer} + resumeProxyStep := Step{Kind: StepStartCoolifyProxy, Ref: coolifyProxyContainer} emitProgress(plan.OnProgress, StepProgress{Index: total, Total: total, Step: resumeProxyStep, Status: StepStatusStarted}) if err := startProxyContainer(proxyCtx, c.dockerRunner(), coolifyProxyContainer); err != nil { emitProgress(plan.OnProgress, StepProgress{Index: total, Total: total, Step: resumeProxyStep, Status: StepStatusError, Err: err}) - return + cleanupErr = errors.Join(cleanupErr, fmt.Errorf("resume coolify proxy after failed apply: %w", err)) + return cleanupErr } emitProgress(plan.OnProgress, StepProgress{Index: total, Total: total, Step: resumeProxyStep, Status: StepStatusOK}) + } else if coolifyProxyStopped { + cleanupErr = errors.Join(cleanupErr, fmt.Errorf("coolify proxy remains stopped because at least one source app could not be resumed")) } + return cleanupErr +} + +func sourcePauseStepIndex(steps []Step, app string) (int, bool) { + return planStepIndex(steps, StepPauseSource, app) +} + +func planStepIndex(steps []Step, kind StepKind, app string) (int, bool) { + for index, step := range steps { + if step.Kind == kind && step.App == app { + return index, true + } + } + return 0, false } func (a *applyContext) hasStoppedTargetWriters() bool { @@ -706,22 +1141,32 @@ func (a *applyContext) hasStoppedTargetWriters() bool { return false } -func (c *Client) bestEffortResumeTargetWriters(actx *applyContext, plan Plan, total int) { +func (c *Client) bestEffortResumeTargetWriters(actx *applyContext, plan Plan, total int) error { if actx == nil { - return + return nil } + var cleanupErr error for app, entry := range actx.cache { if len(entry.TargetWritersStopped) == 0 { continue } resumeStep := Step{Kind: StepResumeTarget, App: app, Ref: app} - emitProgress(plan.OnProgress, StepProgress{Index: total, Total: total, Step: resumeStep, Status: StepStatusStarted}) - if err := c.applyResumeTarget(context.Background(), actx, resumeStep); err != nil { - emitProgress(plan.OnProgress, StepProgress{Index: total, Total: total, Step: resumeStep, Status: StepStatusError, Err: err}) + index, found := planStepIndex(plan.Steps, StepResumeTarget, app) + if !found { + index = total + } + emitProgress(plan.OnProgress, StepProgress{Index: index, Total: total, Step: resumeStep, Status: StepStatusStarted}) + resumeCtx, cancel := context.WithTimeout(context.Background(), dockerStartTimeout) + err := c.applyResumeTarget(resumeCtx, actx, resumeStep) + cancel() + if err != nil { + emitProgress(plan.OnProgress, StepProgress{Index: index, Total: total, Step: resumeStep, Status: StepStatusError, Err: err}) + cleanupErr = errors.Join(cleanupErr, fmt.Errorf("resume target writers for app %s after failed apply: %w", app, err)) continue } - emitProgress(plan.OnProgress, StepProgress{Index: total, Total: total, Step: resumeStep, Status: StepStatusOK}) + emitProgress(plan.OnProgress, StepProgress{Index: index, Total: total, Step: resumeStep, Status: StepStatusOK}) } + return cleanupErr } func emitProgress(fn *func(StepProgress), progress StepProgress) { @@ -752,7 +1197,7 @@ func (c *Client) applyStep(ctx context.Context, actx *applyContext, step Step) e case StepPushImage: return c.applyPushImage(ctx, actx, step) case StepCreateVolume: - return nil + return c.applyCreateVolume(ctx, actx, step) case StepInstallGateway: return c.applyInstallGateway(ctx, actx, step) case StepActivateRoutes: @@ -848,6 +1293,34 @@ func (c *Client) applyCreateService(ctx context.Context, actx *applyContext, ste } entry.ComposeID = compose.ComposeID entry.ComposeAppName = compose.AppName + if strings.TrimSpace(entry.ComposeAppName) == "" { + if err := c.refreshComposeAppName(ctx, entry); err != nil { + return err + } + } + if appStagesDataStoreRestore(actx.plan, step.App) { + return c.requireStagingCompatibleCompose(ctx, entry.ComposeID, step.App) + } + return nil +} + +func createVolumeRef(name, target string) string { + if name != "" { + return name + } + return target +} + +func (c *Client) applyCreateVolume(ctx context.Context, actx *applyContext, step Step) error { + runner := c.dockerRunner() + for _, volume := range stagedVolumesForApp(actx.plan, step.App) { + if createVolumeRef(volume.Source.Name, volume.Source.Target) != step.Ref { + continue + } + if err := ensureStagingVolume(ctx, runner, actx.plan, step.App, volume); err != nil { + return err + } + } return nil } @@ -872,6 +1345,9 @@ func (c *Client) applyPushImage(ctx context.Context, actx *applyContext, step St if entry.ComposeID == "" { return fmt.Errorf("missing composeId for app %s; create_service must run first", step.App) } + if err := requireStagedStateTransferred(actx, step.App); err != nil { + return err + } composeFile, err := c.composeFileForApply(ctx, actx, step.App) if err != nil { return err @@ -898,25 +1374,23 @@ func (c *Client) updateComposeWithPatchGuard(ctx context.Context, composeID, com return errors.Join(mutationErr, patchErr) } -func (c *Client) deployComposeWithPatchGuard(ctx context.Context, composeID, title string) error { - if err := c.validateNoActiveBortOverrides(ctx, composeID); err != nil { - return err - } +func (c *Client) deployComposeAfterPatchGuard(ctx context.Context, composeID, title string) error { err := c.DeployCompose(ctx, composeID, title) - if err == nil { - return nil - } - var apiErr *APIError - if errors.As(err, &apiErr) { - return err + if mutationResponseMayHaveSucceeded(err) { + return ambiguousDeployResponseError{err: err} } - return ambiguousDeployResponseError{err: err} + return err } const guardedDeploymentTimeout = 30 * time.Minute func (c *Client) deployComposeForApply(ctx context.Context, actx *applyContext, appName, composeFile, envContent string) error { entry := actx.entry(appName) + if appStagesDataStoreRestore(actx.plan, appName) { + if err := c.requireStagingCompatibleCompose(ctx, entry.ComposeID, appName); err != nil { + return err + } + } if err := c.updateComposeWithPatchGuard(ctx, entry.ComposeID, composeFile, envContent); err != nil { return err } @@ -924,7 +1398,10 @@ func (c *Client) deployComposeForApply(ctx context.Context, actx *applyContext, if err != nil { return err } - err = c.deployComposeWithPatchGuard(ctx, entry.ComposeID, title) + if err := c.validateNoActiveBortOverrides(ctx, entry.ComposeID); err != nil { + return err + } + err = c.deployComposeAfterPatchGuard(ctx, entry.ComposeID, title) var ambiguousErr ambiguousDeployResponseError if err == nil || errors.As(err, &ambiguousErr) { responseErr := err @@ -964,7 +1441,8 @@ func (c *Client) deployComposeForApply(ctx context.Context, actx *applyContext, defer cancelStop() stopErr := c.quiesceGuardedDeployment(stopCtx, actx, appName, title) if stopErr != nil { - return unsafeSourceResumeError{err: unsafeTargetResumeError{err: fmt.Errorf("%s: %w; quiescence could not be proved, so Bort will leave any paused source applications stopped and skip target recovery: %v", failure, err, stopErr)}} + unsafeErr := unsafeSourceResumeError{err: unsafeTargetResumeError{err: fmt.Errorf("%s: %w; quiescence could not be proved, so Bort will leave any paused source applications stopped and skip target recovery: %v", failure, err, stopErr)}} + return ambiguousMutationResponseError{err: unsafeErr} } safetyErr := c.validateMigratedVolumeMountsAfterDeploy(ctx, actx, appName) combined := errors.Join(err, safetyErr) @@ -972,7 +1450,7 @@ func (c *Client) deployComposeForApply(ctx context.Context, actx *applyContext, } safetyErr := c.validateMigratedVolumeMountsAfterDeploy(ctx, actx, appName) if safetyErr != nil && isUnsafeTargetResumeError(safetyErr) { - return fmt.Errorf("deploy dokploy compose: %w; post-deploy safety check: %v", err, safetyErr) + return fmt.Errorf("deploy dokploy compose: %w; post-deploy safety check: %w", err, safetyErr) } return err } @@ -989,8 +1467,7 @@ func (c *Client) waitForGuardedDeployment(ctx context.Context, composeID, title if deployment.Title != title { continue } - switch strings.ToLower(strings.TrimSpace(deployment.Status)) { - case "done", "error", "cancelled": + if deploymentTerminal(deployment) { if err := c.validateNoActiveBortOverridesAfterMutation(monitorCtx, composeID); err != nil { return deployment, err } @@ -1026,12 +1503,11 @@ func (c *Client) quiesceGuardedDeployment(ctx context.Context, actx *applyContex attemptTerminal := false allDeploymentsTerminal := true for _, deployment := range compose.Deployments { - switch strings.ToLower(strings.TrimSpace(deployment.Status)) { - case "done", "error", "cancelled": + if deploymentTerminal(deployment) { if deployment.Title == title { attemptTerminal = true } - default: + } else { allDeploymentsTerminal = false } } @@ -1046,6 +1522,15 @@ func (c *Client) quiesceGuardedDeployment(ctx context.Context, actx *applyContex } } +func deploymentTerminal(deployment Deployment) bool { + switch strings.ToLower(strings.TrimSpace(deployment.Status)) { + case "done", "error", "cancelled": + return true + default: + return false + } +} + func deployComposeTitle(plan Plan) string { title := "bort-migrate" if plan.RunName != "" { @@ -1063,9 +1548,31 @@ func (c *Client) composeFileForApply(ctx context.Context, actx *applyContext, ap if err != nil { return "", err } + staged := stagedVolumesForApp(actx.plan, appName) + if len(staged) > 0 && stagedVolumesRecorded(actx.entry(appName), staged) { + composeFile, err = rewriteComposeStagedVolumes(composeFile, staged) + if err != nil { + return "", err + } + } return composeFile, nil } +func requireStagedStateTransferred(actx *applyContext, appName string) error { + staged := stagedVolumesForApp(actx.plan, appName) + if len(staged) == 0 || stagedVolumesRecorded(actx.entry(appName), staged) { + return nil + } + missing := []string{} + for _, volume := range staged { + mount, ok := actx.entry(appName).MigratedVolumeMounts[migratedMountKey(volume.Service, volume.Target)] + if !ok || mount.VolumeName != volume.VolumeName { + missing = append(missing, volume.Service+":"+volume.Target) + } + } + return fmt.Errorf("staged state for app %s is incomplete (%s not transferred); refusing to deploy over fresh volumes", appName, strings.Join(missing, ", ")) +} + func (c *Client) replaceBuildOnlyServicesWithSourceImages(ctx context.Context, plan Plan, appName, composeFile string) (string, error) { app, ok := findPrepareApp(plan.Prepare, appName) if !ok { diff --git a/internal/target/dokploy/apply_test.go b/internal/target/dokploy/apply_test.go index f0151bf..ab894cd 100644 --- a/internal/target/dokploy/apply_test.go +++ b/internal/target/dokploy/apply_test.go @@ -10,7 +10,9 @@ import ( "net/http/httptest" "os" "path/filepath" + "slices" "strings" + "sync/atomic" "testing" "time" @@ -39,6 +41,182 @@ func TestNewClientFromEnvRequiresURLAndToken(t *testing.T) { } } +func TestLocalDockerRunnerRefusesExplicitRemoteDaemon(t *testing.T) { + t.Setenv("DOCKER_HOST", "ssh://remote.example") + t.Setenv("DOCKER_CONTEXT", "production") + t.Setenv("DOCKER_TLS_VERIFY", "1") + t.Setenv("DOCKER_CERT_PATH", "/tmp/remote-certs") + _, err := (localDockerRunner{Path: filepath.Join(t.TempDir(), "docker")}).Output(context.Background(), "version") + if err == nil || !strings.Contains(err.Error(), "DOCKER_HOST") { + t.Fatalf("expected explicit remote Docker target refusal, got %v", err) + } +} + +func TestPrimeResumeStateUsesPersistedComposeIdentity(t *testing.T) { + projectCatalogRequests := 0 + composeSearchRequests := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/project.one": + if r.URL.Query().Get("projectId") != "project-1" { + t.Fatalf("unexpected project identity: %s", r.URL.RawQuery) + } + _ = json.NewEncoder(w).Encode(Project{ProjectID: "project-1", Environments: []ProjectEnvironment{{EnvironmentID: "environment-1", Name: "production"}}}) + case "/api/compose.one": + if r.URL.Query().Get("composeId") != "compose-selected" { + t.Fatalf("unexpected compose identity: %s", r.URL.RawQuery) + } + _ = json.NewEncoder(w).Encode(Compose{ComposeID: "compose-selected", Name: "api", AppName: "stack-api", EnvironmentID: "environment-1"}) + case "/api/project.all": + projectCatalogRequests++ + http.Error(w, "name lookup must not run", http.StatusInternalServerError) + case "/api/compose.search": + composeSearchRequests++ + http.Error(w, "name lookup must not run", http.StatusInternalServerError) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + app := preparer.AppPlan{Name: "api"} + app.TargetResources = &preparer.TargetResources{Dokploy: &preparer.DokployResources{ + Project: preparer.DokployProject{Name: "api", Environment: "production"}, + ComposeApp: preparer.DokployComposeApp{Name: "api"}, + }} + plan := Plan{ + Prepare: preparer.Result{Apps: []preparer.AppPlan{app}}, + Steps: []Step{ + {Kind: StepCreateProject, App: "api", Ref: "api"}, + {Kind: StepCreateService, App: "api", Ref: "api"}, + {Kind: StepInstallGateway, App: "api", Ref: "api.example.com"}, + }, + TargetIdentities: map[string]TargetIdentity{"api": { + ProjectID: "project-1", EnvironmentID: "environment-1", ComposeID: "compose-selected", ComposeAppName: "stack-api", + }}, + } + client := &Client{BaseURL: server.URL, Token: "secret", HTTPClient: server.Client()} + actx := &applyContext{plan: plan, cache: map[string]*appCache{}} + pausedApps := pausedSources{} + coolifyProxyStopped := false + + if err := client.primeResumeState(context.Background(), actx, plan.Steps[:2], pausedApps, map[string]struct{}{}, &coolifyProxyStopped); err != nil { + t.Fatalf("primeResumeState: %v", err) + } + entry := actx.entry("api") + if entry.ProjectID != "project-1" || entry.EnvironmentID != "environment-1" || entry.ComposeID != "compose-selected" || entry.ComposeAppName != "stack-api" { + t.Fatalf("resume did not hydrate persisted target identity: %#v", entry) + } + if projectCatalogRequests != 0 || composeSearchRequests != 0 { + t.Fatalf("resume fell back to mutable names, project requests=%d compose searches=%d", projectCatalogRequests, composeSearchRequests) + } +} + +func TestHydratePersistedTargetIdentitiesLoadsSharedProjectOnce(t *testing.T) { + projectRequests := 0 + composeRequests := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/project.one": + projectRequests++ + _ = json.NewEncoder(w).Encode(Project{ProjectID: "project-1", Environments: []ProjectEnvironment{{EnvironmentID: "environment-1", Name: "production"}}}) + case "/api/compose.one": + composeRequests++ + id := r.URL.Query().Get("composeId") + _ = json.NewEncoder(w).Encode(Compose{ComposeID: id, AppName: "stack-" + id, EnvironmentID: "environment-1"}) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + apps := []preparer.AppPlan{ + {Name: "alpha", TargetResources: &preparer.TargetResources{Dokploy: &preparer.DokployResources{}}}, + {Name: "beta", TargetResources: &preparer.TargetResources{Dokploy: &preparer.DokployResources{}}}, + } + actx := &applyContext{plan: Plan{ + Prepare: preparer.Result{Apps: apps}, + TargetIdentities: map[string]TargetIdentity{ + "alpha": {ProjectID: "project-1", EnvironmentID: "environment-1", ComposeID: "compose-alpha"}, + "beta": {ProjectID: "project-1", EnvironmentID: "environment-1", ComposeID: "compose-beta"}, + }, + }, cache: map[string]*appCache{}} + client := &Client{BaseURL: server.URL, Token: "secret", HTTPClient: server.Client()} + + if err := client.hydratePersistedTargetIdentities(context.Background(), actx); err != nil { + t.Fatal(err) + } + if projectRequests != 1 || composeRequests != 2 { + t.Fatalf("expected one shared project lookup and two compose lookups, got projects=%d composes=%d", projectRequests, composeRequests) + } +} + +func TestHydratePersistedTargetIdentitiesRejectsEnvironmentOutsideProject(t *testing.T) { + composeRequests := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/project.one": + _ = json.NewEncoder(w).Encode(Project{ProjectID: "project-1", Environments: []ProjectEnvironment{{EnvironmentID: "other-environment"}}}) + case "/api/compose.one": + composeRequests++ + _ = json.NewEncoder(w).Encode(Compose{ComposeID: "compose-1", EnvironmentID: "environment-1"}) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + app := preparer.AppPlan{Name: "api", TargetResources: &preparer.TargetResources{Dokploy: &preparer.DokployResources{}}} + actx := &applyContext{plan: Plan{ + Prepare: preparer.Result{Apps: []preparer.AppPlan{app}}, + TargetIdentities: map[string]TargetIdentity{"api": { + ProjectID: "project-1", EnvironmentID: "environment-1", ComposeID: "compose-1", + }}, + }, cache: map[string]*appCache{}} + client := &Client{BaseURL: server.URL, Token: "secret", HTTPClient: server.Client()} + err := client.hydratePersistedTargetIdentities(context.Background(), actx) + if err == nil || !strings.Contains(err.Error(), "no longer belongs to project") { + t.Fatalf("expected environment membership refusal, got %v", err) + } + if composeRequests != 0 { + t.Fatalf("compose was loaded before project membership was verified, requests=%d", composeRequests) + } +} + +func TestLegacyPlanPreservesV1Alpha1RoutedStateOrder(t *testing.T) { + app := preparer.AppPlan{Name: "api"} + app.Resources.Volumes = []preparer.VolumeResource{{Service: "web", Type: "volume", Source: "data", Target: "/data"}} + prepare := preparer.Result{Apps: []preparer.AppPlan{app}} + sync := syncplan.Result{Apps: []syncplan.AppPlan{{Name: "api", Steps: []syncplan.Step{{ + ResourceType: "volume", + ResourceRef: "volume:web -> /data", + Strategy: syncplan.StrategyDockerVolumeArchive, + }}}}} + cutover := gateway.Result{Apps: []gateway.AppPlan{{Name: "api", Routes: []gateway.Route{{Host: "api.example.com"}}}}} + legacy := LegacyPlanFromArtifactsV1Alpha1(prepare, sync, cutover) + current := PlanFromArtifactsV1Alpha2(prepare, sync, cutover) + + indexOf := func(plan Plan, kind StepKind) int { + for index, step := range plan.Steps { + if step.Kind == kind { + return index + } + } + return -1 + } + legacyResume := indexOf(legacy, StepResumeTarget) + legacyActivate := indexOf(legacy, StepActivateRoutes) + legacyStop := indexOf(legacy, StepStopCoolifyProxy) + if legacyResume < 0 || legacyActivate <= legacyResume || legacyStop <= legacyActivate { + t.Fatalf("unexpected legacy routed state order: %v", stepKinds(legacy.Steps)) + } + currentStop := indexOf(current, StepStopCoolifyProxy) + currentResume := indexOf(current, StepResumeTarget) + currentActivate := indexOf(current, StepActivateRoutes) + if currentStop < 0 || currentResume <= currentStop || currentActivate <= currentResume { + t.Fatalf("unexpected current routed state order: %v", stepKinds(current.Steps)) + } +} + func TestNewClientFromEnvRejectsRemoteHTTP(t *testing.T) { t.Setenv(EnvBaseURL, "http://dokploy.example") t.Setenv(EnvToken, "secret") @@ -47,6 +225,25 @@ func TestNewClientFromEnvRejectsRemoteHTTP(t *testing.T) { } } +func TestNormalizeTokenBaseURLPreservesPathAndRejectsRequestComponents(t *testing.T) { + got, err := NormalizeTokenBaseURL("https://DOKPLOY.example:443/tenant-a/") + if err != nil { + t.Fatal(err) + } + if got != "https://dokploy.example/tenant-a" { + t.Fatalf("unexpected normalized URL %q", got) + } + for _, raw := range []string{ + "https://user@dokploy.example", + "https://dokploy.example?tenant=a", + "https://dokploy.example#tenant-a", + } { + if _, err := NormalizeTokenBaseURL(raw); err == nil { + t.Fatalf("expected credentialed URL %q to be rejected", raw) + } + } +} + func TestPlanBundleFilesPinComposeAndEnvContents(t *testing.T) { bundleDir := t.TempDir() appDir := filepath.Join(bundleDir, "api") @@ -253,6 +450,80 @@ func TestApplyResumeFromPrimesCompletedCreateSteps(t *testing.T) { } } +type partialSourcePauseRunner struct { + fakeDockerRunner + firstRunning bool +} + +func (r *partialSourcePauseRunner) Output(_ context.Context, args ...string) ([]byte, error) { + r.outputArgs = append(r.outputArgs, append([]string{}, args...)) + switch strings.Join(args, " ") { + case "inspect --type container source-a": + return []byte(fmt.Sprintf(`[{"Id":"source-a","Name":"/source-a","State":{"Running":%t,"Status":"running"}}]`, r.firstRunning)), nil + case "stop source-a": + r.firstRunning = false + return []byte("source-a\n"), nil + case "start source-a": + r.firstRunning = true + return []byte("source-a\n"), nil + case "inspect --type container source-b": + return []byte(`[{"Id":"source-b","Name":"/source-b","State":{"Running":true,"Status":"running"}}]`), nil + case "stop source-b": + return nil, errors.New("injected second source stop failure") + default: + return nil, fmt.Errorf("docker output not stubbed: %s", strings.Join(args, " ")) + } +} + +func TestApplyResumeFromReconcilesCompletedProxyStop(t *testing.T) { + runner := &fakeDockerRunner{outputs: map[string][]byte{ + "inspect --type container coolify-proxy": []byte(`[{"Id":"proxy-id","Name":"/coolify-proxy","State":{"Running":true,"Status":"running"}}]`), + "stop proxy-id": []byte("proxy-id\n"), + }} + client := &Client{Docker: runner} + plan := Plan{Steps: []Step{ + {Kind: StepStopCoolifyProxy, Ref: coolifyProxyContainer}, + {Kind: StepStartDokployProxy, Ref: dokployProxyContainer}, + }} + actx := &applyContext{plan: plan, cache: map[string]*appCache{}} + pausedApps := pausedSources{} + coolifyProxyStopped := false + + if err := client.primeResumeState(context.Background(), actx, plan.Steps[:1], pausedApps, map[string]struct{}{}, &coolifyProxyStopped); err != nil { + t.Fatalf("prime resume state: %v", err) + } + if !fakeOutputCalled(runner, "stop", "proxy-id") { + t.Fatalf("retry did not restore the durable proxy stop, calls=%#v", runner.outputArgs) + } +} + +func TestBestEffortProxyResumeUsesStandaloneProgressIndex(t *testing.T) { + runner := &fakeDockerRunner{outputs: map[string][]byte{ + "inspect --type container coolify-proxy": []byte(`[{"Id":"proxy-id","Name":"/coolify-proxy","State":{"Running":false,"Status":"exited"}}]`), + "start proxy-id": []byte("proxy-id\n"), + }} + client := &Client{Docker: runner} + progress := []StepProgress{} + onProgress := func(item StepProgress) { + progress = append(progress, item) + } + plan := Plan{ + Steps: []Step{{Kind: StepStopCoolifyProxy, Ref: coolifyProxyContainer}, {Kind: StepStartDokployProxy, Ref: dokployProxyContainer}}, + OnProgress: &onProgress, + } + if err := client.bestEffortResume(context.Background(), &applyContext{}, plan, len(plan.Steps), nil, true, true); err != nil { + t.Fatal(err) + } + if len(progress) != 2 { + t.Fatalf("expected proxy cleanup start and completion, got %#v", progress) + } + for _, item := range progress { + if item.Index != len(plan.Steps) || item.Step.Kind != StepStartCoolifyProxy { + t.Fatalf("proxy cleanup overwrote a forward step: %#v", item) + } + } +} + func TestApplySkipsPlatformAppSteps(t *testing.T) { client := &Client{} var progress []StepProgress @@ -308,17 +579,14 @@ func TestApplyStopsBeforeStepWhenPersistenceHookFails(t *testing.T) { } } -func TestApplyResumesPausedSourceWhenPersistenceFailsBetweenSteps(t *testing.T) { +func TestApplyResumesPausedSourceWhenNoStateTransferCompleted(t *testing.T) { app := preparer.AppPlan{Name: "api"} app.Resources.Volumes = []preparer.VolumeResource{{Service: "web", Type: "volume", SourceContainerID: "web-id"}} - runner := &fakeDockerRunner{outputs: map[string][]byte{ - "inspect --type container web-id": []byte(`[{"Id":"web-id","Name":"/web","State":{"Running":true,"Status":"running"}}]`), - "stop web-id": []byte("web-id\n"), - }} + runner := &sourceOwnershipRunner{running: map[string]bool{"web-id": true}} var beforeCalls int beforeStep := func(StepProgress) error { beforeCalls++ - if beforeCalls == 2 { + if beforeCalls >= 2 { return errors.New("persist failed") } return nil @@ -337,12 +605,21 @@ func TestApplyResumesPausedSourceWhenPersistenceFailsBetweenSteps(t *testing.T) if err == nil || !strings.Contains(err.Error(), "persist failed") { t.Fatalf("expected persistence failure, got %v", err) } + resumed := false for _, p := range progress { if p.Step.Kind == StepResumeSource && p.Status == StepStatusOK { - return + resumed = true } } - t.Fatalf("expected cleanup to resume the paused source, got %#v", progress) + if !resumed { + t.Fatalf("source did not resume after a pre-transfer failure: %#v", progress) + } + if !runner.running["web-id"] || !fakeOutputCalled(&runner.fakeDockerRunner, "start", "web-id") { + t.Fatalf("paused source was not restarted: running=%v calls=%v", runner.running, runner.outputArgs) + } + if beforeCalls != 2 { + t.Fatalf("unexpected cleanup persistence attempt before any state transfer, calls=%d", beforeCalls) + } } func TestActivePatchGuardBlocksGitBackedCompose(t *testing.T) { @@ -372,6 +649,31 @@ func TestActivePatchGuardBlocksGitBackedCompose(t *testing.T) { } } +func TestActivePatchGuardUsesCurrentLocalServiceTask(t *testing.T) { + runner := &fakeDockerRunner{ + outputs: map[string][]byte{ + "ps --format {{.Names}}": []byte("dokploy-postgres-backup\ndokploy-postgres.1.current\n"), + "ps --filter label=com.docker.swarm.service.name=dokploy-postgres --filter status=running --format {{.ID}}": []byte("current-container-id\n"), + }, + runOutputs: map[string][]byte{ + "exec -i dokploy-postgres-backup psql -U dokploy -d dokploy -v ON_ERROR_STOP=1 -At": {}, + "exec -i current-container-id psql -U dokploy -d dokploy -v ON_ERROR_STOP=1 -At": []byte(`{"patchId":"bort-current","filePath":"compose.yml","composeName":"api","sourceType":"github","repository":"owner/repo","branch":"main"}` + "\n"), + }, + } + client := &Client{Docker: runner} + + err := client.validateNoActiveBortOverrides(context.Background(), "compose-api") + if err == nil || !strings.Contains(err.Error(), "bort-current") { + t.Fatalf("expected current service task patch to block apply, got %v", err) + } + if len(runner.runs) != 1 || !slices.Contains(runner.runs[0].Args, "current-container-id") { + t.Fatalf("active patch guard queried the wrong database container: %#v", runner.runs) + } + if len(runner.outputArgs) != 1 || strings.Join(runner.outputArgs[0], " ") != "ps --filter label=com.docker.swarm.service.name=dokploy-postgres --filter status=running --format {{.ID}}" { + t.Fatalf("active patch guard did not use one exact service-label query: %#v", runner.outputArgs) + } +} + type stagedBortPatchRunner struct { fakeDockerRunner checks int @@ -462,6 +764,37 @@ func TestUpdateComposePatchGuardDetectsConcurrentPatch(t *testing.T) { } } +func TestDeployComposeForApplyChecksPatchBeforeDeploy(t *testing.T) { + deploys := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodPost && r.URL.Path == "/api/compose.update": + w.WriteHeader(http.StatusOK) + case r.Method == http.MethodPost && r.URL.Path == "/api/compose.deploy": + deploys++ + w.WriteHeader(http.StatusOK) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + runner := &stagedBortPatchRunner{patchAt: 3, fakeDockerRunner: fakeDockerRunner{ + outputs: map[string][]byte{"ps --format {{.Names}}": []byte("dokploy-postgres\n")}, + runOutputs: map[string][]byte{}, + }} + actx := &applyContext{ + cache: map[string]*appCache{"api": {ComposeID: "compose-api", ComposeAppName: "stack-1"}}, + } + client := &Client{BaseURL: server.URL, Token: "secret", HTTPClient: server.Client(), Docker: runner} + err := client.deployComposeForApply(context.Background(), actx, "api", "services: {}\n", "") + if err == nil || !strings.Contains(err.Error(), "active Bort-owned Dokploy patch") { + t.Fatalf("expected pre-deployment patch refusal, got %v", err) + } + if deploys != 0 { + t.Fatalf("pre-deployment refusal called compose.deploy %d time(s)", deploys) + } +} + func TestPostDeployPatchGuardStopsTargetCompose(t *testing.T) { var deploymentTitle string var composeReads int @@ -624,6 +957,9 @@ func TestDeployComposeForApplyFailsClosedWhenAmbiguousResponseCannotBeResolved(t if !isUnsafeSourceResumeError(err) || !isUnsafeTargetResumeError(err) { t.Fatalf("unresolved ambiguous response must prevent source and target recovery, got %v", err) } + if !mutationResponseMayHaveSucceeded(err) { + t.Fatalf("unresolved deploy response was not marked mutation-ambiguous: %v", err) + } if composeReads != 2 || deploymentTitle == "" { t.Fatalf("expected monitoring and quiescence to inspect the titled attempt, reads=%d title=%q", composeReads, deploymentTitle) } @@ -683,6 +1019,9 @@ func TestDeployComposeForApplyDoesNotMonitorAuthoritativeAPIRejection(t *testing if !errors.As(err, &apiErr) || apiErr.Status != test.status { t.Fatalf("expected authoritative API rejection %d, got %v", test.status, err) } + if mutationResponseMayHaveSucceeded(err) { + t.Fatalf("authoritative API rejection was marked mutation-ambiguous: %v", err) + } if composeReads != 0 || redirects != 0 || isUnsafeSourceResumeError(err) || isUnsafeTargetResumeError(err) { t.Fatalf("authoritative rejection followed redirect or entered deployment recovery, redirects=%d reads=%d err=%v", redirects, composeReads, err) } @@ -862,6 +1201,45 @@ func TestDeploymentMonitoringQuiescenceFailurePreventsSourceRecovery(t *testing. } } +func TestRejectedDeployKeepsPostDeploySafetyMarkers(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodPost && r.URL.Path == "/api/compose.update": + w.WriteHeader(http.StatusOK) + case r.Method == http.MethodPost && r.URL.Path == "/api/compose.deploy": + http.Error(w, "deploy rejected", http.StatusBadRequest) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + runner := &stagedBortPatchRunner{patchAt: 100, fakeDockerRunner: fakeDockerRunner{ + outputs: map[string][]byte{ + "ps --format {{.Names}}": []byte("dokploy-postgres\n"), + "ps -a --filter label=com.docker.compose.project=stack-1 --format {{.ID}}": []byte("web-id\n"), + "inspect --type container web-id": []byte(`[{"Id":"web-id","Name":"/web","Config":{"Labels":{"com.docker.compose.service":"web","com.docker.compose.project":"stack-1"}},"State":{"Running":true,"Status":"running"},"Mounts":[{"Type":"volume","Name":"fresh-vol","Destination":"/data","RW":true}]}]`), + }, + outputErrs: map[string]error{"stop web-id": errors.New("Error response from daemon: cannot stop container")}, + runOutputs: map[string][]byte{}, + }} + client := &Client{BaseURL: server.URL, Token: "secret", HTTPClient: server.Client(), Docker: runner} + actx := &applyContext{plan: Plan{}, cache: map[string]*appCache{}} + entry := actx.entry("api") + entry.ComposeID = "compose-api" + entry.ComposeAppName = "stack-1" + entry.MigratedVolumeMounts = map[string]migratedVolumeMount{ + migratedMountKey("web", "/data"): {Service: "web", Target: "/data", VolumeName: "migrated-vol"}, + } + + err := client.deployComposeForApply(context.Background(), actx, "api", "services: {}\n", "") + if err == nil || !strings.Contains(err.Error(), "deploy rejected") || !strings.Contains(err.Error(), "also failed to stop unsafe target containers") { + t.Fatalf("expected rejected deploy with failed safety stop, got %v", err) + } + if !isUnsafeSourceResumeError(err) || !isUnsafeTargetResumeError(err) || !mutationResponseMayHaveSucceeded(err) { + t.Fatalf("rejected deploy must keep the post-deploy safety markers, got %v", err) + } +} + func TestDeploymentQuiescenceRespectsCallerDeadline(t *testing.T) { composeReads := 0 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -996,8 +1374,15 @@ func TestApplyLeavesPausedSourceStoppedWhenGuardedDeploymentCannotBeQuiesced(t * app := preparer.AppPlan{Name: "api", Directory: "api"} app.Resources.SourceServices = []preparer.SourceServiceRef{{ServiceName: "web", ContainerID: "source-id"}} app.TargetResources = &preparer.TargetResources{Dokploy: &preparer.DokployResources{ComposeApp: preparer.DokployComposeApp{Name: "api", ComposePath: "compose.yaml"}}} + paused := &applyContext{plan: Plan{RunDir: bundleDir}, cache: map[string]*appCache{}} + paused.entry("api").SourcePauseRecorded = true + paused.entry("api").SourcePausedContainers = []sourcePausedContainer{{ID: "source-id", Stopped: true}} + if err := paused.persistSourcePauseState(); err != nil { + t.Fatal(err) + } err := client.Apply(context.Background(), Plan{ ResumeFrom: 3, + RunDir: bundleDir, Steps: []Step{ {Kind: StepCreateProject, App: "api", Ref: "api"}, {Kind: StepCreateService, App: "api", Ref: "api"}, @@ -1009,55 +1394,61 @@ func TestApplyLeavesPausedSourceStoppedWhenGuardedDeploymentCannotBeQuiesced(t * if err == nil || !strings.Contains(err.Error(), "leave any paused source applications stopped") { t.Fatalf("expected fail-closed guarded deployment error, got %v", err) } + if !mutationResponseMayHaveSucceeded(err) { + t.Fatalf("accepted deployment with unproven quiescence was not marked mutation-ambiguous: %v", err) + } if fakeOutputCalled(&runner.fakeDockerRunner, "start", "source-id") { t.Fatalf("unsafe recovery restarted the paused source, calls=%#v", runner.outputArgs) } } -func TestComposeMutationPatchGuardRechecksAfterHTTPError(t *testing.T) { - for _, test := range []struct { - name string - path string - wantPatch bool - wantChecks int - mutate func(*Client) error - }{ - {name: "update", path: "/api/compose.update", wantPatch: true, wantChecks: 2, mutate: func(client *Client) error { - return client.updateComposeWithPatchGuard(context.Background(), "compose-api", "services: {}\n", "") - }}, - {name: "deploy", path: "/api/compose.deploy", wantChecks: 1, mutate: func(client *Client) error { - return client.deployComposeWithPatchGuard(context.Background(), "compose-api", "bort-migrate") - }}, - } { - t.Run(test.name, func(t *testing.T) { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodPost || r.URL.Path != test.path { - http.NotFound(w, r) - return - } - http.Error(w, "mutation failed", http.StatusInternalServerError) - })) - defer server.Close() - runner := &stagedBortPatchRunner{fakeDockerRunner: fakeDockerRunner{ - outputs: map[string][]byte{"ps --format {{.Names}}": []byte("dokploy-postgres\n")}, - runOutputs: map[string][]byte{}, - }} - client := &Client{BaseURL: server.URL, Token: "secret", HTTPClient: server.Client(), Docker: runner} - err := test.mutate(client) - if err == nil || !strings.Contains(err.Error(), "mutation failed") { - t.Fatalf("expected HTTP mutation error, got %v", err) - } - if got := strings.Contains(err.Error(), "appeared while Bort was changing compose"); got != test.wantPatch { - t.Fatalf("post-mutation patch error = %t, want %t: %v", got, test.wantPatch, err) - } - var deployedGuardErr deployedComposeGuardError - if errors.As(err, &deployedGuardErr) { - t.Fatalf("failed HTTP mutation was classified as a successful deploy: %v", err) - } - if runner.checks != test.wantChecks { - t.Fatalf("patch checks = %d, want %d", runner.checks, test.wantChecks) - } - }) +func TestComposeUpdatePatchGuardRechecksAfterHTTPError(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost || r.URL.Path != "/api/compose.update" { + http.NotFound(w, r) + return + } + http.Error(w, "mutation failed", http.StatusInternalServerError) + })) + defer server.Close() + runner := &stagedBortPatchRunner{fakeDockerRunner: fakeDockerRunner{ + outputs: map[string][]byte{"ps --format {{.Names}}": []byte("dokploy-postgres\n")}, + runOutputs: map[string][]byte{}, + }} + client := &Client{BaseURL: server.URL, Token: "secret", HTTPClient: server.Client(), Docker: runner} + err := client.updateComposeWithPatchGuard(context.Background(), "compose-api", "services: {}\n", "") + if err == nil || !strings.Contains(err.Error(), "mutation failed") || !strings.Contains(err.Error(), "appeared while Bort was changing compose") { + t.Fatalf("expected HTTP mutation and post-mutation patch errors, got %v", err) + } + if !mutationResponseMayHaveSucceeded(err) { + t.Fatalf("HTTP 500 compose update was not marked mutation-ambiguous: %v", err) + } + if runner.checks != 2 { + t.Fatalf("patch checks = %d, want 2", runner.checks) + } +} + +func TestComposeUpdateTransportFailureIsMutationAmbiguous(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost || r.URL.Path != "/api/compose.update" { + http.NotFound(w, r) + return + } + conn, _, err := w.(http.Hijacker).Hijack() + if err != nil { + t.Fatal(err) + } + _ = conn.Close() + })) + defer server.Close() + runner := &stagedBortPatchRunner{patchAt: 100, fakeDockerRunner: fakeDockerRunner{ + outputs: map[string][]byte{"ps --format {{.Names}}": []byte("dokploy-postgres\n")}, + runOutputs: map[string][]byte{}, + }} + client := &Client{BaseURL: server.URL, Token: "secret", HTTPClient: server.Client(), Docker: runner} + err := client.updateComposeWithPatchGuard(context.Background(), "compose-api", "services: {}\n", "") + if err == nil || !mutationResponseMayHaveSucceeded(err) { + t.Fatalf("expected ambiguous compose update response, got %v", err) } } @@ -1086,6 +1477,29 @@ func TestApplyPushImageChecksActivePatchWithoutEnvOrRoutes(t *testing.T) { if err == nil || !strings.Contains(err.Error(), "active Bort-owned Dokploy patch") { t.Fatalf("expected unconditional deployment path to block active patch, got %v", err) } + if mutationResponseMayHaveSucceeded(err) { + t.Fatalf("pre-mutation active patch rejection was marked ambiguous: %v", err) + } +} + +func TestApplyProgressMarksPreMutationFailureRetryable(t *testing.T) { + var terminal StepProgress + onProgress := func(progress StepProgress) { + if progress.Status == StepStatusError { + terminal = progress + } + } + client := &Client{Docker: &fakeDockerRunner{}} + err := client.Apply(context.Background(), Plan{ + Steps: []Step{{Kind: StepPushImage, App: "api", Ref: "api"}}, + OnProgress: &onProgress, + }) + if err == nil || !strings.Contains(err.Error(), "missing composeId") { + t.Fatalf("expected pre-mutation validation failure, got %v", err) + } + if terminal.Status != StepStatusError || terminal.MutationAmbiguous { + t.Fatalf("pre-mutation progress was not explicitly retryable: %#v", terminal) + } } func TestApplyActivateRoutesChecksPatchBeforeDomainMutation(t *testing.T) { @@ -1195,9 +1609,10 @@ func TestParseActiveBortOverridePatchesFailsClosedOnMalformedRow(t *testing.T) { } func TestActivePatchGuardFailsClosedWhenInspectionCannotFindDokploy(t *testing.T) { - client := &Client{Docker: &fakeDockerRunner{}} + const query = "ps --filter label=com.docker.swarm.service.name=dokploy-postgres --filter status=running --format {{.ID}}" + client := &Client{Docker: &fakeDockerRunner{outputErrs: map[string]error{query: errors.New("docker unavailable")}}} err := client.validateNoActiveBortOverrides(context.Background(), "compose-api") - if err == nil || !strings.Contains(err.Error(), "active patch inspection") || !strings.Contains(err.Error(), "docker output not stubbed") { + if err == nil || !strings.Contains(err.Error(), "active patch inspection") || !strings.Contains(err.Error(), "docker unavailable") { t.Fatalf("expected active patch inspection to fail closed, got %v", err) } } @@ -1208,7 +1623,7 @@ func TestActivePatchGuardFailsClosedWhenDokployDatabaseIsUnavailable(t *testing. }} client := &Client{Docker: runner} - if err := client.validateNoActiveBortOverrides(context.Background(), "compose-api"); err == nil || !strings.Contains(err.Error(), "dokploy postgres container was not found") { + if err := client.validateNoActiveBortOverrides(context.Background(), "compose-api"); err == nil || !strings.Contains(err.Error(), "has 0 running dokploy-postgres service containers") { t.Fatalf("expected missing Dokploy DB to fail closed, got %v", err) } if len(runner.runs) != 0 { @@ -1430,6 +1845,102 @@ func TestPingPropagatesAPIError(t *testing.T) { } } +func TestSearchComposeSelectsOnlyExactName(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet || r.URL.Path != "/api/compose.search" { + http.NotFound(w, r) + return + } + _ = json.NewEncoder(w).Encode(composeSearchResponse{Items: []Compose{ + {ComposeID: "near", Name: "api-old"}, + {ComposeID: "exact", Name: "api"}, + }, Total: 2}) + })) + defer server.Close() + client := &Client{BaseURL: server.URL, Token: "secret", HTTPClient: server.Client()} + + compose, err := client.SearchCompose(context.Background(), "api", "env1") + if err != nil { + t.Fatalf("SearchCompose: %v", err) + } + if compose == nil || compose.ComposeID != "exact" { + t.Fatalf("expected only exact-name compose, got %#v", compose) + } +} + +func TestSearchComposeConsumesEveryCandidatePage(t *testing.T) { + items := make([]Compose, 100) + for i := range items { + items[i] = Compose{ComposeID: fmt.Sprintf("near-%d", i), Name: fmt.Sprintf("api-%d", i)} + } + var offsets []string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet || r.URL.Path != "/api/compose.search" { + http.NotFound(w, r) + return + } + offsets = append(offsets, r.URL.Query().Get("offset")) + if r.URL.Query().Get("offset") == "100" { + _ = json.NewEncoder(w).Encode(composeSearchResponse{Items: []Compose{{ComposeID: "exact", Name: "api"}}, Total: 101}) + return + } + _ = json.NewEncoder(w).Encode(composeSearchResponse{Items: items, Total: 101}) + })) + defer server.Close() + client := &Client{BaseURL: server.URL, Token: "secret", HTTPClient: server.Client()} + + compose, err := client.SearchCompose(context.Background(), "api", "env1") + if err != nil { + t.Fatalf("SearchCompose: %v", err) + } + if compose == nil || compose.ComposeID != "exact" || !slices.Equal(offsets, []string{"0", "100"}) { + t.Fatalf("expected exact compose from second page, got compose=%#v offsets=%v", compose, offsets) + } +} + +func TestSearchComposeUsesOneDeadlineAcrossPages(t *testing.T) { + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + request := requests.Add(1) + time.Sleep(70 * time.Millisecond) + _ = json.NewEncoder(w).Encode(composeSearchResponse{Items: []Compose{{ComposeID: fmt.Sprintf("near-%d", request), Name: "api-old"}}, Total: 2}) + })) + defer server.Close() + client := &Client{BaseURL: server.URL, Token: "secret", HTTPClient: &http.Client{Timeout: 100 * time.Millisecond}} + + _, err := client.SearchCompose(context.Background(), "api", "env1") + if err == nil || !strings.Contains(err.Error(), "context deadline exceeded") { + t.Fatalf("expected operation-level search timeout, got %v", err) + } + if got := requests.Load(); got != 2 { + t.Fatalf("search made %d requests before its operation deadline, want 2", got) + } +} + +func TestSearchComposeRefusesDuplicateExactNames(t *testing.T) { + requests := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet || r.URL.Path != "/api/compose.search" { + http.NotFound(w, r) + return + } + requests++ + _ = json.NewEncoder(w).Encode(composeSearchResponse{Items: []Compose{ + {ComposeID: "first", Name: "api"}, + {ComposeID: "second", Name: "api"}, + }, Total: 200}) + })) + defer server.Close() + client := &Client{BaseURL: server.URL, Token: "secret", HTTPClient: server.Client()} + + if _, err := client.SearchCompose(context.Background(), "api", "env1"); err == nil || !strings.Contains(err.Error(), `multiple Dokploy compose apps named "api"`) { + t.Fatalf("expected duplicate exact-name refusal, got %v", err) + } + if requests != 1 { + t.Fatalf("duplicate exact names fetched unnecessary pages: requests=%d", requests) + } +} + func TestCreateComposeIsIdempotent(t *testing.T) { calls := map[string]int{} server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -1499,6 +2010,47 @@ func TestCreateComposeCreatesWhenMissing(t *testing.T) { } } +func TestApplyCreateServiceRefreshesMissingComposeAppName(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && r.URL.Path == "/api/compose.search": + _ = json.NewEncoder(w).Encode(composeSearchResponse{Items: []Compose{{ComposeID: "c1", Name: "api"}}, Total: 1}) + case r.Method == http.MethodGet && r.URL.Path == "/api/compose.one": + _ = json.NewEncoder(w).Encode(Compose{ComposeID: "c1", Name: "api", AppName: "stack-api"}) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + bundleDir := t.TempDir() + composePath := filepath.Join(bundleDir, "api", "compose.yaml") + if err := os.MkdirAll(filepath.Dir(composePath), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(composePath, []byte("services: {}\n"), 0o600); err != nil { + t.Fatal(err) + } + app := preparer.AppPlan{ + Name: "api", + Directory: "api", + TargetResources: &preparer.TargetResources{Dokploy: &preparer.DokployResources{ + ComposeApp: preparer.DokployComposeApp{Name: "api", ComposePath: "compose.yaml"}, + }}, + } + actx := &applyContext{ + plan: Plan{Prepare: preparer.Result{BundleDir: bundleDir, Apps: []preparer.AppPlan{app}}}, + cache: map[string]*appCache{"api": {EnvironmentID: "env1"}}, + } + client := &Client{BaseURL: server.URL, Token: "secret", HTTPClient: server.Client()} + if err := client.applyCreateService(context.Background(), actx, Step{Kind: StepCreateService, App: "api", Ref: "api"}); err != nil { + t.Fatal(err) + } + entry := actx.entry("api") + if entry.ComposeID != "c1" || entry.ComposeAppName != "stack-api" { + t.Fatalf("compose identity was not refreshed before create completed: %#v", entry) + } +} + func TestCreateDomainIsIdempotent(t *testing.T) { calls := map[string]int{} server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -1532,6 +2084,49 @@ func TestCreateDomainIsIdempotent(t *testing.T) { } } +func TestDomainMutationTransportFailuresAreAmbiguous(t *testing.T) { + for _, test := range []struct { + name string + existing []Domain + failedPath string + wantAmbiguous bool + }{ + {name: "lookup", failedPath: "/api/domain.byComposeId"}, + {name: "create", failedPath: "/api/domain.create", wantAmbiguous: true}, + {name: "update", existing: []Domain{{DomainID: "domain-1", Host: "api.example.com", ServiceName: "old", Port: 3000}}, failedPath: "/api/domain.update", wantAmbiguous: true}, + } { + t.Run(test.name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == test.failedPath { + conn, _, err := w.(http.Hijacker).Hijack() + if err != nil { + t.Fatal(err) + } + _ = conn.Close() + return + } + switch { + case r.Method == http.MethodGet && r.URL.Path == "/api/domain.byComposeId": + _ = json.NewEncoder(w).Encode(test.existing) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + client := &Client{BaseURL: server.URL, Token: "secret", HTTPClient: server.Client()} + _, err := client.CreateDomain(context.Background(), CreateDomainRequest{ + Host: "api.example.com", ComposeID: "compose-1", ServiceName: "web", Port: 8080, + }) + if err == nil { + t.Fatal("expected transport failure") + } + if got := mutationResponseMayHaveSucceeded(err); got != test.wantAmbiguous { + t.Fatalf("mutation ambiguity = %t, want %t: %v", got, test.wantAmbiguous, err) + } + }) + } +} + func TestCreateDomainUpdatesExistingWhenServiceChanged(t *testing.T) { var received UpdateDomainRequest server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { diff --git a/internal/target/dokploy/cleanup.go b/internal/target/dokploy/cleanup.go index 8671403..98ddd2d 100644 --- a/internal/target/dokploy/cleanup.go +++ b/internal/target/dokploy/cleanup.go @@ -91,20 +91,20 @@ func (c *Client) CleanupStalePlatformProjects(ctx context.Context, opts StalePla if runtime.GOOS != "linux" { return StalePlatformCleanupResult{}, fmt.Errorf("Dokploy metadata cleanup is unavailable on %s; run cleanup --apply on the Linux Dokploy host", runtime.GOOS) } - return c.cleanupStalePlatformProjects(ctx, opts) -} - -func (c *Client) cleanupStalePlatformProjects(ctx context.Context, opts StalePlatformCleanupOptions) (StalePlatformCleanupResult, error) { - names := cleanupProjectNames(opts.ProjectNames) - if len(names) == 0 { + opts.ProjectNames = cleanupProjectNames(opts.ProjectNames) + if len(opts.ProjectNames) == 0 { return StalePlatformCleanupResult{}, fmt.Errorf("at least one project name is required") } - runner := c.dockerRunner() - pg, err := findDokployPostgresContainer(ctx, runner) + postgresContainerID, err := c.verifySameDockerHostAndDatabase(ctx) if err != nil { - return StalePlatformCleanupResult{}, err + return StalePlatformCleanupResult{}, fmt.Errorf("verify cleanup API and database belong to the local Dokploy services: %w", err) } - backup, err := backupDokployDatabase(ctx, runner, pg, opts.BackupDir, opts.BackupPrefix) + return c.cleanupStalePlatformProjects(ctx, opts, postgresContainerID) +} + +func (c *Client) cleanupStalePlatformProjects(ctx context.Context, opts StalePlatformCleanupOptions, postgresContainerID string) (StalePlatformCleanupResult, error) { + runner := c.dockerRunner() + backup, err := backupDokployDatabase(ctx, runner, postgresContainerID, opts.BackupDir, opts.BackupPrefix) if err != nil { return StalePlatformCleanupResult{}, err } @@ -112,7 +112,7 @@ func (c *Client) cleanupStalePlatformProjects(ctx context.Context, opts StalePla if err := backup.dir.ValidatePath(); err != nil { return StalePlatformCleanupResult{}, fmt.Errorf("verify Dokploy database backup location %s before deleting metadata: %w", backup.path, err) } - deleted, err := deleteStalePlatformProjects(ctx, runner, pg, names, opts.ProjectIDs) + deleted, err := deleteStalePlatformProjects(ctx, runner, postgresContainerID, opts.ProjectNames, opts.ProjectIDs) if err != nil { return StalePlatformCleanupResult{}, fmt.Errorf("delete stale Dokploy platform metadata after backup %s: %w", backup.path, err) } @@ -1059,20 +1059,6 @@ func cleanupProjectNames(names []string) []string { return cleaned } -func findDokployPostgresContainer(ctx context.Context, runner dockerRunner) (string, error) { - out, err := runner.Output(ctx, "ps", "--format", "{{.Names}}") - if err != nil { - return "", fmt.Errorf("list docker containers for Dokploy postgres: %w", err) - } - for _, line := range strings.Split(string(out), "\n") { - name := strings.TrimSpace(line) - if name == "dokploy-postgres" || strings.HasPrefix(name, "dokploy-postgres.") || strings.HasPrefix(name, "dokploy-postgres-") { - return name, nil - } - } - return "", fmt.Errorf("dokploy postgres container was not found; cleanup must run on the Dokploy host") -} - type dokployDatabaseBackup struct { path string dir *safepath.PrivateDir diff --git a/internal/target/dokploy/cleanup_test.go b/internal/target/dokploy/cleanup_test.go index 2f1b0e0..8782b26 100644 --- a/internal/target/dokploy/cleanup_test.go +++ b/internal/target/dokploy/cleanup_test.go @@ -40,11 +40,8 @@ func TestCleanupStalePlatformProjectsBacksUpThenDeletesMetadata(t *testing.T) { t.Fatal(err) } runner := &fakeDockerRunner{ - outputs: map[string][]byte{ - "ps --format {{.Names}}": []byte("dokploy-postgres.1.task\n"), - }, runOutputs: map[string][]byte{ - "exec -i dokploy-postgres.1.task psql -U dokploy -d dokploy -v ON_ERROR_STOP=1 -At -F |": []byte("BEGIN\ndeleted|proxy|p1\nCOMMIT\n"), + "exec -i postgres-container-id psql -U dokploy -d dokploy -v ON_ERROR_STOP=1 -At -F |": []byte("BEGIN\ndeleted|proxy|p1\nCOMMIT\n"), }, } client := &Client{Docker: runner} @@ -53,7 +50,7 @@ func TestCleanupStalePlatformProjectsBacksUpThenDeletesMetadata(t *testing.T) { ProjectNames: []string{"proxy", "source", "proxy"}, BackupDir: backupDir, BackupPrefix: "cleanup-test", - }) + }, "postgres-container-id") if err != nil { t.Fatalf("CleanupStalePlatformProjects: %v", err) } @@ -117,16 +114,14 @@ func (r *replacingBackupDirRunner) Run(ctx context.Context, stdin io.Reader, std func TestCleanupStalePlatformProjectsRejectsReplacedBackupDirectoryBeforeDelete(t *testing.T) { backupDir := dokployBackupTestDir(t) runner := &replacingBackupDirRunner{ - fakeDockerRunner: fakeDockerRunner{outputs: map[string][]byte{ - "ps --format {{.Names}}": []byte("dokploy-postgres.1.task\n"), - }}, - backupDir: backupDir, + fakeDockerRunner: fakeDockerRunner{}, + backupDir: backupDir, } client := &Client{Docker: runner} _, err := client.cleanupStalePlatformProjects(context.Background(), StalePlatformCleanupOptions{ ProjectNames: []string{"proxy"}, BackupDir: backupDir, - }) + }, "postgres-container-id") if err == nil || !strings.Contains(err.Error(), "private directory path changed") { t.Fatalf("expected replaced backup directory to block metadata deletion, got %v", err) } @@ -1053,10 +1048,3 @@ func TestPathAbsentNoFollowPreservesExistingDirectoryTree(t *testing.T) { t.Fatalf("absence validation changed the target tree: contents=%q err=%v", contents, err) } } - -func TestFindDokployPostgresContainerRequiresDokployPostgres(t *testing.T) { - runner := &fakeDockerRunner{outputs: map[string][]byte{"ps --format {{.Names}}": []byte("postgres\n")}} - if _, err := findDokployPostgresContainer(context.Background(), runner); err == nil { - t.Fatal("expected missing dokploy postgres to error") - } -} diff --git a/internal/target/dokploy/client.go b/internal/target/dokploy/client.go index 1be692a..9c83e3f 100644 --- a/internal/target/dokploy/client.go +++ b/internal/target/dokploy/client.go @@ -11,6 +11,7 @@ import ( "net/http" "net/url" "os" + "strconv" "strings" "time" ) @@ -33,15 +34,16 @@ type Client struct { } func NewClientFromEnv() (*Client, error) { - baseURL := strings.TrimRight(strings.TrimSpace(os.Getenv(EnvBaseURL)), "/") + baseURL := os.Getenv(EnvBaseURL) token := strings.TrimSpace(os.Getenv(EnvToken)) - if baseURL == "" { + if strings.TrimSpace(baseURL) == "" { return nil, fmt.Errorf("%s is required for live mode", EnvBaseURL) } if token == "" { return nil, fmt.Errorf("%s is required for live mode", EnvToken) } - if err := ValidateTokenBaseURL(baseURL); err != nil { + baseURL, err := NormalizeTokenBaseURL(baseURL) + if err != nil { return nil, err } return &Client{ @@ -51,25 +53,54 @@ func NewClientFromEnv() (*Client, error) { }, nil } -func ValidateTokenBaseURL(raw string) error { +func NormalizeTokenBaseURL(raw string) (string, error) { parsed, err := url.Parse(strings.TrimSpace(raw)) if err != nil || parsed.Scheme == "" || parsed.Host == "" { - return fmt.Errorf("invalid Dokploy URL %q", raw) + return "", fmt.Errorf("invalid Dokploy URL %q", raw) } - if parsed.Scheme == "https" { - return nil + if parsed.User != nil || parsed.RawQuery != "" || parsed.ForceQuery || parsed.Fragment != "" { + return "", fmt.Errorf("Dokploy URL must not include user information, a query, or a fragment") } - if parsed.Scheme != "http" { - return fmt.Errorf("Dokploy URL must use https, or http on loopback") + if parsed.RawPath != "" { + return "", fmt.Errorf("Dokploy URL path must not use escaped separators") } - host := parsed.Hostname() - if host == "localhost" { - return nil + scheme := strings.ToLower(parsed.Scheme) + hostname := strings.ToLower(parsed.Hostname()) + if hostname == "" { + return "", fmt.Errorf("invalid Dokploy URL %q", raw) } - if ip := net.ParseIP(host); ip != nil && ip.IsLoopback() { - return nil + port := parsed.Port() + if port != "" { + portNumber, err := strconv.ParseUint(port, 10, 16) + if err != nil || portNumber == 0 { + return "", fmt.Errorf("invalid Dokploy URL port in %q", raw) + } + port = strconv.FormatUint(portNumber, 10) + } + if scheme == "https" && port == "443" { + port = "" + } + host := hostname + if port != "" { + host = net.JoinHostPort(hostname, port) + } else if strings.Contains(hostname, ":") { + host = "[" + hostname + "]" + } + path := strings.TrimRight(parsed.Path, "/") + normalized := (&url.URL{Scheme: scheme, Host: host, Path: path}).String() + if scheme == "https" { + return normalized, nil + } + if scheme != "http" { + return "", fmt.Errorf("Dokploy URL must use https, or http on loopback") + } + if hostname == "localhost" { + return normalized, nil + } + if ip := net.ParseIP(hostname); ip != nil && ip.IsLoopback() { + return normalized, nil } - return fmt.Errorf("refusing to send Dokploy API token to non-loopback http URL %q", raw) + return "", fmt.Errorf("refusing to send Dokploy API token to non-loopback http URL %q", raw) } type APIError struct { @@ -85,6 +116,39 @@ func (e *APIError) Error() string { return fmt.Sprintf("dokploy http %d: %s", e.Status, e.Message) } +type ambiguousMutationResponseError struct { + err error +} + +func (e ambiguousMutationResponseError) Error() string { + return e.err.Error() +} + +func (e ambiguousMutationResponseError) Unwrap() error { + return e.err +} + +func (e ambiguousMutationResponseError) mutationResponseAmbiguous() {} + +func markMutationResponseAmbiguous(err error) error { + if err == nil { + return nil + } + var apiErr *APIError + if errors.As(err, &apiErr) && apiErr.Status != http.StatusRequestTimeout && apiErr.Status < http.StatusInternalServerError { + return err + } + return ambiguousMutationResponseError{err: err} +} + +func mutationResponseMayHaveSucceeded(err error) bool { + var ambiguous interface { + error + mutationResponseAmbiguous() + } + return errors.As(err, &ambiguous) +} + func (c *Client) do(ctx context.Context, method, path string, query url.Values, body any, out any) error { endpoint := c.BaseURL + path if len(query) > 0 { @@ -155,6 +219,268 @@ func (c *Client) Ping(ctx context.Context) error { return c.do(ctx, http.MethodGet, "/api/project.all", nil, nil, nil) } +func (c *Client) DokployVersion(ctx context.Context) (string, error) { + var version string + if err := c.do(ctx, http.MethodGet, "/api/settings.getDokployVersion", nil, nil, &version); err != nil { + return "", err + } + return strings.TrimSpace(version), nil +} + +type dokployServiceInspect struct { + Spec struct { + EndpointSpec struct { + Mode string `json:"Mode"` + Ports []struct { + Protocol string `json:"Protocol"` + TargetPort uint32 `json:"TargetPort"` + PublishedPort uint32 `json:"PublishedPort"` + PublishMode string `json:"PublishMode"` + } `json:"Ports"` + } `json:"EndpointSpec"` + TaskTemplate struct { + ContainerSpec struct { + Env []string `json:"Env"` + } `json:"ContainerSpec"` + Networks []struct { + Target string `json:"Target"` + } `json:"Networks"` + } `json:"TaskTemplate"` + } `json:"Spec"` +} + +type dokployTaskInspect struct { + NodeID string `json:"NodeID"` + Status struct { + State string `json:"State"` + ContainerStatus struct { + ContainerID string `json:"ContainerID"` + } `json:"ContainerStatus"` + } `json:"Status"` +} + +func (c *Client) VerifySameDockerHost(ctx context.Context) error { + _, _, err := c.verifySameDockerHost(ctx) + return err +} + +const localEndpointRebindHint = "rebind this workspace as the same OS user with `bort init-target dokploy --dokploy-url http://127.0.0.1:` (or set both " + EnvBaseURL + " and " + EnvToken + "), using the host port the dokploy service publishes for container port 3000" + +func (c *Client) verifySameDockerHost(ctx context.Context) (dokployServiceInspect, string, error) { + verifyCtx, cancel := context.WithTimeout(ctx, dockerStopTimeout) + defer cancel() + parsed, err := url.Parse(c.BaseURL) + if err != nil { + return dokployServiceInspect{}, "", fmt.Errorf("parse Dokploy URL for local-host verification: %w", err) + } + hostname := parsed.Hostname() + loopback := strings.EqualFold(hostname, "localhost") + if ip := net.ParseIP(hostname); ip != nil && ip.IsLoopback() { + loopback = true + } + if !loopback { + return dokployServiceInspect{}, "", fmt.Errorf("Dokploy URL %q is not loopback; same-VPS migration requires a loopback Dokploy API URL: %s", c.BaseURL, localEndpointRebindHint) + } + if !strings.EqualFold(parsed.Scheme, "http") { + return dokployServiceInspect{}, "", fmt.Errorf("Dokploy URL %q is not the direct local HTTP endpoint; same-VPS migration does not accept a loopback proxy or tunnel: %s", c.BaseURL, localEndpointRebindHint) + } + if parsed.Path != "" && parsed.Path != "/" || parsed.RawPath != "" || parsed.RawQuery != "" || parsed.ForceQuery || parsed.Fragment != "" { + return dokployServiceInspect{}, "", fmt.Errorf("Dokploy URL %q must be the root of the direct local HTTP endpoint", c.BaseURL) + } + port, err := strconv.ParseUint(parsed.Port(), 10, 16) + if err != nil || port == 0 { + return dokployServiceInspect{}, "", fmt.Errorf("Dokploy URL %q must include the local Dokploy service's published port", c.BaseURL) + } + runner := c.dockerRunner() + service, err := inspectDokployService(verifyCtx, runner, "dokploy") + if err != nil { + return dokployServiceInspect{}, "", fmt.Errorf("verify local Dokploy service before API access: %w", err) + } + mode := strings.ToLower(service.Spec.EndpointSpec.Mode) + if mode != "vip" && mode != "dnsrr" { + return dokployServiceInspect{}, "", fmt.Errorf("local Dokploy service endpoint mode is %q, want vip or dnsrr", service.Spec.EndpointSpec.Mode) + } + portPublished := false + for _, published := range service.Spec.EndpointSpec.Ports { + if published.PublishedPort == uint32(port) && published.TargetPort == 3000 && strings.EqualFold(published.Protocol, "tcp") && strings.EqualFold(published.PublishMode, "host") { + portPublished = true + break + } + } + if !portPublished { + return dokployServiceInspect{}, "", fmt.Errorf("Dokploy URL %q does not match a host-mode TCP port published by the local Dokploy service to target port 3000", c.BaseURL) + } + nodeID, err := localDockerNodeID(verifyCtx, runner) + if err != nil { + return dokployServiceInspect{}, "", err + } + if err := verifyLocalDokployTask(verifyCtx, runner, nodeID); err != nil { + return dokployServiceInspect{}, "", err + } + return service, nodeID, nil +} + +func (c *Client) VerifySameDockerHostAndDatabase(ctx context.Context) error { + _, err := c.verifySameDockerHostAndDatabase(ctx) + return err +} + +func (c *Client) verifySameDockerHostAndDatabase(ctx context.Context) (string, error) { + verifyCtx, cancel := context.WithTimeout(ctx, dockerStopTimeout) + defer cancel() + app, nodeID, err := c.verifySameDockerHost(verifyCtx) + if err != nil { + return "", err + } + return c.verifyLocalDokployDatabaseWith(verifyCtx, app, nodeID) +} + +func (c *Client) verifyLocalDokployDatabaseWith(ctx context.Context, app dokployServiceInspect, nodeID string) (string, error) { + runner := c.dockerRunner() + if err := verifyDefaultDokployDatabaseEnvironment(app.Spec.TaskTemplate.ContainerSpec.Env); err != nil { + return "", err + } + postgres, err := inspectDokployService(ctx, runner, "dokploy-postgres") + if err != nil { + return "", fmt.Errorf("inspect local Dokploy postgres service: %w", err) + } + networks := map[string]struct{}{} + for _, network := range app.Spec.TaskTemplate.Networks { + if target := strings.TrimSpace(network.Target); target != "" { + networks[target] = struct{}{} + } + } + for _, network := range postgres.Spec.TaskTemplate.Networks { + if _, ok := networks[strings.TrimSpace(network.Target)]; ok { + containerID, err := localRunningServiceContainerIDOnNode(ctx, runner, "dokploy-postgres", nodeID) + if err != nil { + return "", fmt.Errorf("resolve local Dokploy postgres container: %w", err) + } + return containerID, nil + } + } + return "", fmt.Errorf("local Dokploy and dokploy-postgres services do not share a Docker network") +} + +func verifyDefaultDokployDatabaseEnvironment(env []string) error { + defaults := map[string]string{ + "POSTGRES_HOST": "dokploy-postgres", + "POSTGRES_PORT": "5432", + "POSTGRES_USER": "dokploy", + "POSTGRES_DB": "dokploy", + } + seen := map[string]bool{} + for _, entry := range env { + name, value, ok := strings.Cut(entry, "=") + if !ok { + continue + } + if name == "DATABASE_URL" { + return fmt.Errorf("Dokploy service configures DATABASE_URL; direct cleanup of the default internal dokploy-postgres database is refused") + } + expected, selected := defaults[name] + if !selected { + continue + } + if seen[name] { + return fmt.Errorf("Dokploy service has duplicate %s settings; direct database cleanup is refused", name) + } + seen[name] = true + if value != expected { + return fmt.Errorf("Dokploy service configures %s=%s, not the supported default %s; direct cleanup of the default internal dokploy-postgres database is refused", name, value, expected) + } + } + return nil +} + +func localDockerNodeID(ctx context.Context, runner dockerRunner) (string, error) { + nodeOut, err := runner.Output(ctx, "node", "inspect", "self", "--format", "{{.ID}}") + if err != nil { + return "", fmt.Errorf("identify local Docker node: %w", err) + } + nodeID := strings.TrimSpace(string(nodeOut)) + if nodeID == "" { + return "", fmt.Errorf("local Docker node has no ID") + } + return nodeID, nil +} + +func localRunningServiceContainerIDOnNode(ctx context.Context, runner dockerRunner, service, nodeID string) (string, error) { + taskOut, err := runner.Output(ctx, "service", "ps", "--filter", "desired-state=running", "-q", service) + if err != nil { + return "", fmt.Errorf("list running %s service tasks: %w", service, err) + } + taskIDs := strings.Fields(string(taskOut)) + if len(taskIDs) == 0 { + return "", fmt.Errorf("%s service has no running tasks", service) + } + args := append([]string{"inspect", "--type", "task"}, taskIDs...) + inspected, err := runner.Output(ctx, args...) + if err != nil { + return "", fmt.Errorf("inspect running %s service tasks: %w", service, err) + } + var tasks []dokployTaskInspect + if err := json.Unmarshal(inspected, &tasks); err != nil { + return "", fmt.Errorf("decode running %s service tasks: %w", service, err) + } + containerIDs := []string{} + for _, task := range tasks { + if task.NodeID != nodeID || !strings.EqualFold(task.Status.State, "running") { + continue + } + containerID := strings.TrimSpace(task.Status.ContainerStatus.ContainerID) + if containerID == "" { + return "", fmt.Errorf("running local %s service task has no container ID", service) + } + containerIDs = append(containerIDs, containerID) + } + if len(containerIDs) != 1 { + return "", fmt.Errorf("local Docker node %s has %d running %s service tasks, want exactly 1", nodeID, len(containerIDs), service) + } + return containerIDs[0], nil +} + +func verifyLocalDokployTask(ctx context.Context, runner dockerRunner, nodeID string) error { + taskOut, err := runner.Output(ctx, "service", "ps", "--filter", "desired-state=running", "-q", "dokploy") + if err != nil { + return fmt.Errorf("list running local Dokploy service tasks: %w", err) + } + taskIDs := strings.Fields(string(taskOut)) + if len(taskIDs) == 0 { + return fmt.Errorf("local Dokploy service has no running tasks") + } + args := append([]string{"inspect", "--type", "task"}, taskIDs...) + inspected, err := runner.Output(ctx, args...) + if err != nil { + return fmt.Errorf("inspect running local Dokploy service tasks: %w", err) + } + var tasks []dokployTaskInspect + if err := json.Unmarshal(inspected, &tasks); err != nil { + return fmt.Errorf("decode running local Dokploy service tasks: %w", err) + } + for _, task := range tasks { + if task.NodeID == nodeID && strings.EqualFold(task.Status.State, "running") { + return nil + } + } + return fmt.Errorf("local Docker node %s has no running Dokploy service task", nodeID) +} + +func inspectDokployService(ctx context.Context, runner dockerRunner, name string) (dokployServiceInspect, error) { + out, err := runner.Output(ctx, "service", "inspect", name) + if err != nil { + return dokployServiceInspect{}, fmt.Errorf("docker service inspect %s: %w", name, err) + } + var services []dokployServiceInspect + if err := json.Unmarshal(out, &services); err != nil { + return dokployServiceInspect{}, fmt.Errorf("decode docker service inspect %s: %w", name, err) + } + if len(services) != 1 { + return dokployServiceInspect{}, fmt.Errorf("docker service inspect %s returned %d services, want 1", name, len(services)) + } + return services[0], nil +} + func (c *Client) httpClient() *http.Client { if c.HTTPClient != nil { return c.HTTPClient @@ -235,69 +561,127 @@ func (c *Client) SignIn(ctx context.Context, email, password string) (string, er return "", fmt.Errorf("dokploy sign-in did not return better-auth.session_token cookie") } -func (c *Client) GetCurrentUserOrg(ctx context.Context, sessionCookie string) (string, error) { +type APIKey struct { + ID string `json:"id"` + Name string `json:"name"` + Prefix string `json:"prefix"` +} + +type currentUserData struct { + OrganizationID string `json:"organizationId"` + User *struct { + APIKeys []APIKey `json:"apiKeys"` + } `json:"user"` +} + +func (c *Client) getCurrentUser(ctx context.Context, sessionCookie string) (currentUserData, []byte, error) { headers := map[string]string{"Cookie": "better-auth.session_token=" + sessionCookie} resp, data, err := c.sessionRequest(ctx, http.MethodGet, "/api/trpc/user.get?batch=1", nil, headers) if err != nil { - return "", err + return currentUserData{}, nil, err } if resp.StatusCode >= 400 { - return "", &APIError{Status: resp.StatusCode, Message: strings.TrimSpace(string(data))} + return currentUserData{}, nil, &APIError{Status: resp.StatusCode, Message: strings.TrimSpace(string(data))} } var batch []struct { Result struct { Data struct { - JSON struct { - OrganizationID string `json:"organizationId"` - } `json:"json"` + JSON currentUserData `json:"json"` } `json:"data"` } `json:"result"` } if err := json.Unmarshal(data, &batch); err != nil { - return "", fmt.Errorf("decode user.get response: %w", err) + return currentUserData{}, nil, fmt.Errorf("decode user.get response: %w", err) + } + if len(batch) == 0 { + return currentUserData{}, data, fmt.Errorf("dokploy user.get response missing result data: %s", strings.TrimSpace(string(data))) } - if len(batch) == 0 || batch[0].Result.Data.JSON.OrganizationID == "" { + return batch[0].Result.Data.JSON, data, nil +} + +func (c *Client) GetCurrentUserOrg(ctx context.Context, sessionCookie string) (string, error) { + user, data, err := c.getCurrentUser(ctx, sessionCookie) + if err != nil { + return "", err + } + if user.OrganizationID == "" { return "", fmt.Errorf("dokploy user.get response missing organizationId: %s", strings.TrimSpace(string(data))) } - return batch[0].Result.Data.JSON.OrganizationID, nil + return user.OrganizationID, nil +} + +func (c *Client) GetCurrentUserOrgAndAPIKeys(ctx context.Context, sessionCookie string) (string, []APIKey, error) { + user, data, err := c.getCurrentUser(ctx, sessionCookie) + if err != nil { + return "", nil, err + } + if user.OrganizationID == "" { + return "", nil, fmt.Errorf("dokploy user.get response missing organizationId: %s", strings.TrimSpace(string(data))) + } + if user.User == nil { + return "", nil, fmt.Errorf("dokploy user.get response missing user data: %s", strings.TrimSpace(string(data))) + } + return user.OrganizationID, user.User.APIKeys, nil +} + +func (c *Client) DeleteAPIKey(ctx context.Context, sessionCookie, apiKeyID string) error { + headers := map[string]string{"Cookie": "better-auth.session_token=" + sessionCookie} + body := map[string]any{ + "0": map[string]any{ + "json": map[string]string{"apiKeyId": apiKeyID}, + }, + } + resp, data, err := c.sessionRequest(ctx, http.MethodPost, "/api/trpc/user.deleteApiKey?batch=1", body, headers) + if err != nil { + return err + } + if resp.StatusCode >= 400 { + return &APIError{Status: resp.StatusCode, Message: strings.TrimSpace(string(data))} + } + return nil } -func (c *Client) CreateAPIKey(ctx context.Context, sessionCookie, keyName, organizationID string) (string, error) { +func (c *Client) CreateAPIKey(ctx context.Context, sessionCookie, keyName, prefix, organizationID string) (string, string, error) { headers := map[string]string{"Cookie": "better-auth.session_token=" + sessionCookie} // better-auth's api-key plugin defaults to 10 requests / 24h, which is // far too low for a migration tool. disable rate limiting at creation. + input := map[string]any{ + "name": keyName, + "metadata": map[string]string{"organizationId": organizationID}, + "rateLimitEnabled": false, + } + if prefix != "" { + input["prefix"] = prefix + } body := map[string]any{ "0": map[string]any{ - "json": map[string]any{ - "name": keyName, - "metadata": map[string]string{"organizationId": organizationID}, - "rateLimitEnabled": false, - }, + "json": input, }, } resp, data, err := c.sessionRequest(ctx, http.MethodPost, "/api/trpc/user.createApiKey?batch=1", body, headers) if err != nil { - return "", err + return "", "", err } if resp.StatusCode >= 400 { - return "", &APIError{Status: resp.StatusCode, Message: strings.TrimSpace(string(data))} + return "", "", &APIError{Status: resp.StatusCode, Message: strings.TrimSpace(string(data))} } var batch []struct { Result struct { Data struct { JSON struct { + ID string `json:"id"` Key string `json:"key"` } `json:"json"` } `json:"data"` } `json:"result"` } if err := json.Unmarshal(data, &batch); err != nil { - return "", fmt.Errorf("decode createApiKey response: %w", err) + return "", "", fmt.Errorf("decode createApiKey response: %w", err) } - if len(batch) == 0 || batch[0].Result.Data.JSON.Key == "" { - return "", fmt.Errorf("dokploy createApiKey response missing key: %s", strings.TrimSpace(string(data))) + if len(batch) == 0 || batch[0].Result.Data.JSON.ID == "" || batch[0].Result.Data.JSON.Key == "" { + return "", "", fmt.Errorf("dokploy createApiKey response missing id or key: %s", strings.TrimSpace(string(data))) } - return batch[0].Result.Data.JSON.Key, nil + return batch[0].Result.Data.JSON.ID, batch[0].Result.Data.JSON.Key, nil } type Project struct { @@ -353,7 +737,7 @@ func (c *Client) CreateProject(ctx context.Context, name, description string) (* } var project Project if err := c.do(ctx, http.MethodPost, "/api/project.create", nil, createProjectRequest{Name: name, Description: description}, &project); err != nil { - return nil, err + return nil, markMutationResponseAmbiguous(err) } if project.ProjectID == "" { fresh, err := c.FindProjectByName(ctx, name) @@ -396,6 +780,8 @@ type Compose struct { AppName string `json:"appName,omitempty"` EnvironmentID string `json:"environmentId,omitempty"` ComposeStatus string `json:"composeStatus,omitempty"` + Command string `json:"command,omitempty"` + CreateEnvFile *bool `json:"createEnvFile,omitempty"` Deployments []Deployment `json:"deployments,omitempty"` } @@ -412,20 +798,40 @@ type composeSearchResponse struct { } func (c *Client) SearchCompose(ctx context.Context, name, environmentID string) (*Compose, error) { - q := url.Values{} - q.Set("name", name) - q.Set("environmentId", environmentID) - q.Set("limit", "20") - var resp composeSearchResponse - if err := c.do(ctx, http.MethodGet, "/api/compose.search", q, nil, &resp); err != nil { - return nil, err - } - for i := range resp.Items { - if resp.Items[i].Name == name { - return &resp.Items[i], nil + timeout := c.httpClient().Timeout + if timeout <= 0 { + timeout = defaultTimeout + } + searchCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + var match *Compose + for offset := 0; ; { + q := url.Values{} + q.Set("name", name) + q.Set("environmentId", environmentID) + q.Set("limit", "100") + q.Set("offset", strconv.Itoa(offset)) + var resp composeSearchResponse + if err := c.do(searchCtx, http.MethodGet, "/api/compose.search", q, nil, &resp); err != nil { + return nil, err + } + for i := range resp.Items { + if resp.Items[i].Name == name { + if match != nil { + return nil, fmt.Errorf("multiple Dokploy compose apps named %q exist in environment %s", name, environmentID) + } + item := resp.Items[i] + match = &item + } + } + offset += len(resp.Items) + if offset >= resp.Total { + return match, nil + } + if len(resp.Items) == 0 { + return nil, fmt.Errorf("Dokploy compose search for %q returned no candidates at offset %d of %d", name, offset, resp.Total) } } - return nil, nil } func (c *Client) GetCompose(ctx context.Context, composeID string) (*Compose, error) { @@ -460,7 +866,7 @@ func (c *Client) CreateCompose(ctx context.Context, req CreateComposeRequest) (* } var compose Compose if err := c.do(ctx, http.MethodPost, "/api/compose.create", nil, req, &compose); err != nil { - return nil, err + return nil, markMutationResponseAmbiguous(err) } if compose.ComposeID == "" { fresh, err := c.SearchCompose(ctx, req.Name, req.EnvironmentID) @@ -489,7 +895,7 @@ func (c *Client) UpdateCompose(ctx context.Context, composeID, composeFile, envC Env: envContent, SourceType: "raw", } - return c.do(ctx, http.MethodPost, "/api/compose.update", nil, body, nil) + return markMutationResponseAmbiguous(c.do(ctx, http.MethodPost, "/api/compose.update", nil, body, nil)) } type deployComposeRequest struct { @@ -499,7 +905,7 @@ type deployComposeRequest struct { func (c *Client) DeployCompose(ctx context.Context, composeID, title string) error { body := deployComposeRequest{ComposeID: composeID, Title: title} - return c.do(ctx, http.MethodPost, "/api/compose.deploy", nil, body, nil) + return markMutationResponseAmbiguous(c.do(ctx, http.MethodPost, "/api/compose.deploy", nil, body, nil)) } type Domain struct { @@ -595,7 +1001,7 @@ func (c *Client) CreateDomain(ctx context.Context, req CreateDomainRequest) (*Do } var domain Domain if err := c.do(ctx, http.MethodPost, "/api/domain.create", nil, req, &domain); err != nil { - return nil, err + return nil, markMutationResponseAmbiguous(err) } return &domain, nil } @@ -609,7 +1015,7 @@ func (c *Client) UpdateDomain(ctx context.Context, req UpdateDomainRequest) (*Do } var domain Domain if err := c.do(ctx, http.MethodPost, "/api/domain.update", nil, req, &domain); err != nil { - return nil, err + return nil, markMutationResponseAmbiguous(err) } if domain.DomainID == "" { domain = Domain{ diff --git a/internal/target/dokploy/client_host_test.go b/internal/target/dokploy/client_host_test.go new file mode 100644 index 0000000..d353062 --- /dev/null +++ b/internal/target/dokploy/client_host_test.go @@ -0,0 +1,232 @@ +package dokploy + +import ( + "context" + "strings" + "testing" +) + +func TestVerifySameDockerHostAcceptsPortPublishedByLocalDokployService(t *testing.T) { + for _, mode := range []string{"vip", "dnsrr"} { + t.Run(mode, func(t *testing.T) { + runner := &fakeDockerRunner{outputs: map[string][]byte{ + "service inspect dokploy": []byte(`[{"Spec":{"EndpointSpec":{"Mode":"` + mode + `","Ports":[{"Protocol":"tcp","TargetPort":3000,"PublishedPort":3030,"PublishMode":"host"}]}}}]`), + "node inspect self --format {{.ID}}": []byte("local-node\n"), + "service ps --filter desired-state=running -q dokploy": []byte("task-id\n"), + "inspect --type task task-id": []byte(`[{"NodeID":"local-node","Status":{"State":"running"}}]`), + }} + client := &Client{BaseURL: "http://127.0.0.1:3030", Docker: runner} + if err := client.VerifySameDockerHost(context.Background()); err != nil { + t.Fatal(err) + } + }) + } +} + +func TestNormalizeTokenBaseURLPreservesExplicitLoopbackHTTPPort(t *testing.T) { + baseURL, err := NormalizeTokenBaseURL("http://127.0.0.1:80/") + if err != nil { + t.Fatal(err) + } + if baseURL != "http://127.0.0.1:80" { + t.Fatalf("normalized URL = %q, want explicit loopback port", baseURL) + } + runner := &fakeDockerRunner{outputs: map[string][]byte{ + "service inspect dokploy": []byte(`[{"Spec":{"EndpointSpec":{"Mode":"dnsrr","Ports":[{"Protocol":"tcp","TargetPort":3000,"PublishedPort":80,"PublishMode":"host"}]}}}]`), + "node inspect self --format {{.ID}}": []byte("local-node\n"), + "service ps --filter desired-state=running -q dokploy": []byte("task-id\n"), + "inspect --type task task-id": []byte(`[{"NodeID":"local-node","Status":{"State":"running"}}]`), + }} + if err := (&Client{BaseURL: baseURL, Docker: runner}).VerifySameDockerHost(context.Background()); err != nil { + t.Fatalf("verify normalized explicit port: %v", err) + } +} + +func TestNormalizeTokenBaseURLCanonicalizesNumericPorts(t *testing.T) { + for _, test := range []struct { + raw string + want string + }{ + {raw: "http://127.0.0.1:03030/", want: "http://127.0.0.1:3030"}, + {raw: "https://dokploy.example:0443/", want: "https://dokploy.example"}, + } { + got, err := NormalizeTokenBaseURL(test.raw) + if err != nil { + t.Fatalf("NormalizeTokenBaseURL(%q): %v", test.raw, err) + } + if got != test.want { + t.Fatalf("NormalizeTokenBaseURL(%q)=%q, want %q", test.raw, got, test.want) + } + } +} + +func TestVerifySameDockerHostRequiresRunningTaskOnLocalNode(t *testing.T) { + runner := &fakeDockerRunner{outputs: map[string][]byte{ + "service inspect dokploy": []byte(`[{"Spec":{"EndpointSpec":{"Mode":"vip","Ports":[{"Protocol":"tcp","TargetPort":3000,"PublishedPort":3030,"PublishMode":"host"}]}}}]`), + "node inspect self --format {{.ID}}": []byte("local-node\n"), + "service ps --filter desired-state=running -q dokploy": []byte("task-id\n"), + "inspect --type task task-id": []byte(`[{"NodeID":"other-node","Status":{"State":"running"}}]`), + }} + client := &Client{BaseURL: "http://127.0.0.1:3030", Docker: runner} + err := client.VerifySameDockerHost(context.Background()) + if err == nil || !strings.Contains(err.Error(), "no running Dokploy service task") { + t.Fatalf("expected remote-only service task refusal, got %v", err) + } +} + +func TestVerifySameDockerHostRejectsNonLoopbackWithoutDockerProbe(t *testing.T) { + runner := &fakeDockerRunner{} + client := &Client{BaseURL: "https://dokploy.example", Docker: runner} + err := client.VerifySameDockerHost(context.Background()) + if err == nil || !strings.Contains(err.Error(), "requires a loopback Dokploy API URL") { + t.Fatalf("expected non-loopback refusal, got %v", err) + } + if len(runner.outputArgs) != 0 { + t.Fatalf("non-loopback refusal contacted Docker: %v", runner.outputArgs) + } +} + +func TestVerifySameDockerHostRejectsLoopbackPortNotPublishedByDokploy(t *testing.T) { + runner := &fakeDockerRunner{outputs: map[string][]byte{ + "service inspect dokploy": []byte(`[{"Spec":{"EndpointSpec":{"Mode":"dnsrr","Ports":[{"Protocol":"tcp","TargetPort":3000,"PublishedPort":3030,"PublishMode":"host"}]}}}]`), + }} + client := &Client{BaseURL: "http://127.0.0.1:4040", Docker: runner} + err := client.VerifySameDockerHost(context.Background()) + if err == nil || !strings.Contains(err.Error(), "does not match") { + t.Fatalf("expected unrelated loopback listener refusal, got %v", err) + } +} + +func TestVerifySameDockerHostRejectsLoopbackProxyPathBeforeDockerProbe(t *testing.T) { + runner := &fakeDockerRunner{} + client := &Client{BaseURL: "http://127.0.0.1:3030/dokploy", Docker: runner} + err := client.VerifySameDockerHost(context.Background()) + if err == nil || !strings.Contains(err.Error(), "root of the direct local HTTP endpoint") { + t.Fatalf("expected loopback proxy path refusal, got %v", err) + } + if len(runner.outputArgs) != 0 { + t.Fatalf("loopback proxy path refusal contacted Docker: %v", runner.outputArgs) + } +} + +func TestVerifyLocalDokployDatabaseRequiresSharedServiceNetwork(t *testing.T) { + for _, test := range []struct { + name string + postgresNet string + env string + wantError bool + }{ + {name: "shared network", postgresNet: "dokploy-network", env: `[]`}, + {name: "explicit default database", postgresNet: "dokploy-network", env: `["POSTGRES_HOST=dokploy-postgres","POSTGRES_PORT=5432","POSTGRES_USER=dokploy","POSTGRES_DB=dokploy"]`}, + {name: "different network", postgresNet: "other-network", env: `[]`, wantError: true}, + {name: "database URL", postgresNet: "dokploy-network", env: `["DATABASE_URL=postgresql://external.example/dokploy"]`, wantError: true}, + {name: "custom host", postgresNet: "dokploy-network", env: `["POSTGRES_HOST=external-db"]`, wantError: true}, + {name: "custom port", postgresNet: "dokploy-network", env: `["POSTGRES_PORT=6543"]`, wantError: true}, + {name: "custom user", postgresNet: "dokploy-network", env: `["POSTGRES_USER=operator"]`, wantError: true}, + {name: "custom database", postgresNet: "dokploy-network", env: `["POSTGRES_DB=other"]`, wantError: true}, + {name: "duplicate selector", postgresNet: "dokploy-network", env: `["POSTGRES_HOST=dokploy-postgres","POSTGRES_HOST=dokploy-postgres"]`, wantError: true}, + } { + t.Run(test.name, func(t *testing.T) { + runner := &fakeDockerRunner{outputs: map[string][]byte{ + "service inspect dokploy": []byte(`[{"Spec":{"TaskTemplate":{"ContainerSpec":{"Env":` + test.env + `},"Networks":[{"Target":"dokploy-network"}]}}}]`), + "service inspect dokploy-postgres": []byte(`[{"Spec":{"TaskTemplate":{"Networks":[{"Target":"` + test.postgresNet + `"}]}}}]`), + "service ps --filter desired-state=running -q dokploy-postgres": []byte("postgres-task\n"), + "inspect --type task postgres-task": []byte(`[{"NodeID":"local-node","Status":{"State":"running","ContainerStatus":{"ContainerID":"postgres-container-id"}}}]`), + }} + client := &Client{Docker: runner} + app, err := inspectDokployService(context.Background(), runner, "dokploy") + if err != nil { + t.Fatal(err) + } + _, err = client.verifyLocalDokployDatabaseWith(context.Background(), app, "local-node") + if (err != nil) != test.wantError { + t.Fatalf("verifyLocalDokployDatabaseWith() error=%v, wantError=%t", err, test.wantError) + } + }) + } +} + +func TestVerifySameDockerHostAndDatabaseReusesServiceAndNodeAttestation(t *testing.T) { + runner := &fakeDockerRunner{outputs: map[string][]byte{ + "service inspect dokploy": []byte(`[{"Spec":{"EndpointSpec":{"Mode":"dnsrr","Ports":[{"Protocol":"tcp","TargetPort":3000,"PublishedPort":3030,"PublishMode":"host"}]},"TaskTemplate":{"ContainerSpec":{"Env":["POSTGRES_HOST=dokploy-postgres"]},"Networks":[{"Target":"dokploy-network"}]}}}]`), + "service inspect dokploy-postgres": []byte(`[{"Spec":{"TaskTemplate":{"Networks":[{"Target":"dokploy-network"}]}}}]`), + "node inspect self --format {{.ID}}": []byte("local-node\n"), + "service ps --filter desired-state=running -q dokploy": []byte("dokploy-task\n"), + "inspect --type task dokploy-task": []byte(`[{"NodeID":"local-node","Status":{"State":"running"}}]`), + "service ps --filter desired-state=running -q dokploy-postgres": []byte("postgres-task\n"), + "inspect --type task postgres-task": []byte(`[{"NodeID":"local-node","Status":{"State":"running","ContainerStatus":{"ContainerID":"postgres-container-id"}}}]`), + }} + client := &Client{BaseURL: "http://127.0.0.1:3030", Docker: runner} + if err := client.VerifySameDockerHostAndDatabase(context.Background()); err != nil { + t.Fatal(err) + } + counts := map[string]int{} + for _, args := range runner.outputArgs { + counts[strings.Join(args, " ")]++ + } + for _, command := range []string{"service inspect dokploy", "node inspect self --format {{.ID}}"} { + if counts[command] != 1 { + t.Fatalf("combined host/database attestation ran %q %d times: %#v", command, counts[command], runner.outputArgs) + } + } +} + +func TestLocalRunningServiceContainerIDRequiresExactlyOneLocalTask(t *testing.T) { + for _, test := range []struct { + name string + taskIDs string + tasks string + want string + }{ + { + name: "no running tasks", + taskIDs: "", + want: "has no running tasks", + }, + { + name: "no local running task", + taskIDs: "remote-task\n", + tasks: `[{"NodeID":"remote-node","Status":{"State":"running","ContainerStatus":{"ContainerID":"remote-container"}}}]`, + want: "has 0 running dokploy-postgres service tasks, want exactly 1", + }, + { + name: "multiple local running tasks", + taskIDs: "task-one\ntask-two\n", + tasks: `[{"NodeID":"local-node","Status":{"State":"running","ContainerStatus":{"ContainerID":"container-one"}}},{"NodeID":"local-node","Status":{"State":"running","ContainerStatus":{"ContainerID":"container-two"}}}]`, + want: "has 2 running dokploy-postgres service tasks, want exactly 1", + }, + } { + t.Run(test.name, func(t *testing.T) { + outputs := map[string][]byte{ + "service ps --filter desired-state=running -q dokploy-postgres": []byte(test.taskIDs), + } + if test.tasks != "" { + outputs["inspect --type task "+strings.Join(strings.Fields(test.taskIDs), " ")] = []byte(test.tasks) + } + _, err := localRunningServiceContainerIDOnNode(context.Background(), &fakeDockerRunner{outputs: outputs}, "dokploy-postgres", "local-node") + if err == nil || !strings.Contains(err.Error(), test.want) { + t.Fatalf("localRunningServiceContainerIDOnNode() error=%v, want %q", err, test.want) + } + }) + } +} + +func TestLocalRunningServiceContainerIDIgnoresStaleNamedContainer(t *testing.T) { + runner := &fakeDockerRunner{outputs: map[string][]byte{ + "ps --format {{.Names}}": []byte("dokploy-postgres.1.stale\n"), + "service ps --filter desired-state=running -q dokploy-postgres": []byte("current-task\n"), + "inspect --type task current-task": []byte(`[{"NodeID":"local-node","Status":{"State":"running","ContainerStatus":{"ContainerID":"current-container-id"}}}]`), + }} + containerID, err := localRunningServiceContainerIDOnNode(context.Background(), runner, "dokploy-postgres", "local-node") + if err != nil { + t.Fatal(err) + } + if containerID != "current-container-id" { + t.Fatalf("resolved container ID %q, want current-container-id", containerID) + } + for _, args := range runner.outputArgs { + if strings.Join(args, " ") == "ps --format {{.Names}}" { + t.Fatalf("resolved Dokploy postgres through an unbound container name listing: %#v", runner.outputArgs) + } + } +} diff --git a/internal/target/dokploy/datamove.go b/internal/target/dokploy/datamove.go index 1398d76..8c837dd 100644 --- a/internal/target/dokploy/datamove.go +++ b/internal/target/dokploy/datamove.go @@ -39,6 +39,13 @@ func (c *Client) applySyncVolume(ctx context.Context, actx *applyContext, step S } runner := c.dockerRunner() + if appStateIsStaged(actx.plan, step.App) { + staged, ok := stagedVolumeFor(actx.plan, step.App, volume) + if !ok { + return fmt.Errorf("%w: %s volume %s for app %s cannot be staged; only named volumes are transferred before deploy", ErrNotImplemented, volume.Type, volumeRefLabel(volume), step.App) + } + return c.syncVolumeToStaging(ctx, runner, actx, step.App, app, staged) + } target, err := c.targetContainerForService(ctx, runner, actx, step.App, volume.Service) if err != nil { return err @@ -47,7 +54,9 @@ func (c *Client) applySyncVolume(ctx context.Context, actx *applyContext, step S if err != nil { return err } - if err := withTargetStopped(ctx, runner, target, copy.Run); err != nil { + if err := withTargetStopped(ctx, runner, target, func() error { + return copy.Run() + }); err != nil { return err } if copy.TargetVolumeName != "" { @@ -62,6 +71,52 @@ func (c *Client) applySyncVolume(ctx context.Context, actx *applyContext, step S return nil } +// syncVolumeToStaging copies a stopped source volume into the Bort-owned +// staging volume. Dokploy's compose does not reference that volume until +// push_image, so the source snapshot is the only writer to guard: the +// source container set is inspected before and after the copy and any +// restart in between invalidates the transfer. +func (c *Client) syncVolumeToStaging(ctx context.Context, runner dockerRunner, actx *applyContext, appName string, app preparer.AppPlan, staged stagedVolume) error { + if err := actx.forgetMigratedVolumeMount(appName, staged.Service, staged.Target); err != nil { + return err + } + if err := ensureStagingVolume(ctx, runner, actx.plan, appName, staged); err != nil { + return err + } + srcVolName, err := resolveSourceVolume(ctx, runner, staged.Source) + if err != nil { + return err + } + if err := requireStagingVolumeUnattached(ctx, runner, staged); err != nil { + return err + } + before, err := inspectSourceQuiesceTargets(ctx, runner, app) + if err != nil { + return err + } + if err := requireSourceQuiescent(before); err != nil { + return err + } + if err := copyNamedVolume(ctx, runner, srcVolName, staged.VolumeName); err != nil { + return err + } + after, err := inspectSourceQuiesceTargets(ctx, runner, app) + if err != nil { + return err + } + if err := requireSourceQuiesceUnchanged(before, after); err != nil { + return err + } + if err := requireStagingVolumeUnattached(ctx, runner, staged); err != nil { + return err + } + return actx.recordMigratedVolumeMount(appName, migratedVolumeMount{ + Service: staged.Service, + Target: staged.Target, + VolumeName: staged.VolumeName, + }) +} + type plannedVolumeCopy struct { Run func() error TargetVolumeName string @@ -122,6 +177,29 @@ func (a *applyContext) recordMigratedVolumeMount(appName string, mount migratedV return a.persistMigratedVolumeMounts() } +func (a *applyContext) forgetMigratedVolumeMount(appName, service, target string) error { + if a == nil { + return nil + } + entry := a.entry(appName) + key := migratedMountKey(service, target) + if _, ok := entry.MigratedVolumeMounts[key]; !ok { + return nil + } + delete(entry.MigratedVolumeMounts, key) + return a.persistMigratedVolumeMounts() +} + +func stagedVolumesRecorded(entry *appCache, staged []stagedVolume) bool { + for _, volume := range staged { + mount, ok := entry.MigratedVolumeMounts[migratedMountKey(volume.Service, volume.Target)] + if !ok || mount.VolumeName != volume.VolumeName { + return false + } + } + return true +} + func migratedMountKey(service, target string) string { return strings.TrimSpace(service) + "\x00" + strings.TrimSpace(target) } @@ -168,7 +246,7 @@ func (a *applyContext) persistMigratedVolumeMounts() error { return fmt.Errorf("encode migrated volume state: %w", err) } contents = append(contents, '\n') - if err := os.WriteFile(path, contents, 0o600); err != nil { + if err := safepath.WriteFileAtomicNoFollow(path, contents, 0o600); err != nil { return fmt.Errorf("write migrated volume state: %w", err) } return nil @@ -256,7 +334,13 @@ func (c *Client) primeTargetWritersForResume(ctx context.Context, runner dockerR return c.pauseTargetWritersWithMode(ctx, runner, actx, appName, targetWriterKeepServices(actx.plan, appName), true) } +// appHasStateWork reports whether target writers must be paused around +// state transfer. staged plans finish every copy before the target +// exists, so they never need the pause. func appHasStateWork(plan Plan, appName string) bool { + if appStateIsStaged(plan, appName) { + return false + } for _, step := range plan.Steps { if step.App == appName && step.Kind == StepPauseSource { return true @@ -286,14 +370,17 @@ func (c *Client) pauseTargetWriters(ctx context.Context, runner dockerRunner, ac func (c *Client) pauseTargetWritersWithMode(ctx context.Context, runner dockerRunner, actx *applyContext, appName string, keepServices map[string]struct{}, recordAlreadyStopped bool) error { entry := actx.entry(appName) - if len(entry.TargetWritersStopped) > 0 { + if len(entry.TargetWritersStopped) > 0 && !recordAlreadyStopped { return nil } - containers, err := c.targetWriterContainers(ctx, runner, actx, appName, keepServices, recordAlreadyStopped) + discoveryCtx, cancelDiscovery := context.WithTimeout(ctx, targetDiscoveryTimeout) + containers, err := c.targetWriterContainers(discoveryCtx, runner, actx, appName, keepServices, recordAlreadyStopped) + cancelDiscovery() if err != nil { return err } stopped := []dockerContainer{} + stoppedNow := []dockerContainer{} for _, container := range containers { service := container.Config.Labels[composeServiceLabel] if _, keep := keepServices[service]; keep { @@ -305,13 +392,17 @@ func (c *Client) pauseTargetWritersWithMode(ctx context.Context, runner dockerRu } continue } - if err := stopContainer(ctx, runner, container.ID); err != nil { - for _, stoppedContainer := range stopped { + stopCtx, cancelStop := context.WithTimeout(ctx, dockerStopTimeout) + err := stopContainer(stopCtx, runner, container.ID) + cancelStop() + if err != nil { + for _, stoppedContainer := range stoppedNow { _ = startContainer(context.Background(), runner, stoppedContainer.ID) } return fmt.Errorf("stop target writer container %s: %w", container.ID, err) } stopped = append(stopped, container) + stoppedNow = append(stoppedNow, container) } entry.TargetWritersStopped = stopped return nil @@ -331,7 +422,7 @@ func (c *Client) withTargetWritersStopped(ctx context.Context, runner dockerRunn } if err := c.applyResumeTarget(context.Background(), actx, Step{Kind: StepResumeTarget, App: appName, Ref: appName}); err != nil { if opErr != nil { - return fmt.Errorf("%w (target writers also failed to restart: %v)", opErr, err) + return fmt.Errorf("%w (target writers also failed to restart: %w)", opErr, err) } return err } @@ -453,7 +544,7 @@ func (c *Client) applyResumeTarget(ctx context.Context, actx *applyContext, step stopCtx, cancelStop := context.WithTimeout(context.Background(), targetDiscoveryTimeout) defer cancelStop() if stopErr := c.stopTargetComposeContainers(stopCtx, actx, step.App); stopErr != nil { - return fmt.Errorf("%w (also failed to stop unsafe target containers: %v)", err, stopErr) + return ambiguousMutationResponseError{err: unsafeSourceResumeError{err: fmt.Errorf("%w (also failed to stop unsafe target containers, so Bort will leave any paused source applications stopped: %v)", err, stopErr)}} } } return err @@ -483,7 +574,7 @@ func (c *Client) validateMigratedVolumeMountsAfterDeploy(ctx context.Context, ac stopCtx, cancelStop := context.WithDeadline(context.Background(), operationDeadline) defer cancelStop() if stopErr := c.stopTargetComposeContainers(stopCtx, actx, appName); stopErr != nil { - return fmt.Errorf("%w (also failed to stop unsafe target containers: %v)", err, stopErr) + return ambiguousMutationResponseError{err: unsafeSourceResumeError{err: fmt.Errorf("%w (also failed to stop unsafe target containers, so Bort will leave any paused source applications stopped: %v)", err, stopErr)}} } } return err @@ -596,7 +687,6 @@ func (c *Client) stopTargetComposeContainers(ctx context.Context, actx *applyCon continue } result = fmt.Errorf("stop target container %s: %w", container.ID, err) - continue } } if hadStopError { @@ -713,60 +803,267 @@ func (c *Client) applyRestoreDataStore(ctx context.Context, actx *applyContext, } runner := c.dockerRunner() + if appStateIsStaged(actx.plan, step.App) { + return c.restoreDataStoreToStaging(ctx, runner, actx, app, step, store) + } dst, err := c.targetContainerForService(ctx, runner, actx, step.App, store.Service) if err != nil { return err } - creds := postgresCredsFromEnv(envMap(dst.Config.Env)) - if err := c.withTargetWritersStopped(ctx, runner, actx, step.App, map[string]struct{}{store.Service: {}}, func() error { - if err := waitPostgresReady(ctx, runner, dst.ID, creds); err != nil { - return err + return pgRestoreIntoContainer(ctx, runner, actx.plan, step, dst) + }); err != nil { + return err + } + return recordMigratedStoreMounts(actx, step.App, app, store.Service, dst) +} + +// requireStagedRestoreLayout refuses layouts the reviewed plan itself +// rules out; callers run it before stagingRestoreProject so those +// refusals stay distinguishable from the repairable Dokploy settings +// that stagingRestoreProject also reports as ErrNotImplemented. +func requireStagedRestoreLayout(plan Plan, step Step, store preparer.DataStoreResource) ([]stagedVolume, error) { + staged := stagedVolumesForService(plan, step.App, store.Service) + if len(staged) == 0 { + return nil, fmt.Errorf("%w: data store %s for app %s has no named volume to stage", ErrNotImplemented, step.Ref, step.App) + } + composeFile, err := readComposeFile(plan, step.App) + if err != nil { + return nil, err + } + return staged, requireLiteralMountTargets(composeFile, step.App, store.Service) +} + +func (c *Client) stagingRestoreProject(ctx context.Context, actx *applyContext, step Step, store preparer.DataStoreResource, staged []stagedVolume) (stagingProject, error) { + entry := actx.entry(step.App) + if strings.TrimSpace(entry.ComposeAppName) == "" { + return stagingProject{}, fmt.Errorf("missing compose app name for app %s; create_service must run first", step.App) + } + if actx.stagingEnvFormat == stagingEnvFormatUnresolved { + return stagingProject{}, fmt.Errorf("Dokploy .env format was not resolved before staging %s for app %s", store.Service, step.App) + } + if err := c.requireStagingCompatibleCompose(ctx, entry.ComposeID, step.App); err != nil { + return stagingProject{}, err + } + composeFile, err := c.composeFileForApply(ctx, actx, step.App) + if err != nil { + return stagingProject{}, err + } + stagingCompose, err := stagingComposeFile(composeFile, store.Service, staged) + if err != nil { + return stagingProject{}, err + } + envContent, err := readEnvContent(actx.plan, step.App) + if err != nil { + return stagingProject{}, err + } + return writeStagingProject(actx.plan, step.App, store.Service, stagingCompose, stagingEnvFileContent(entry.ComposeAppName, envContent, actx.stagingEnvFormat)) +} + +// preflightStagedRestores creates (without starting) each staged data +// store container while the source still runs, so a PGDATA or mount +// layout that only compose interpolation or the image decides is refused +// before pause_source stops the source. +func (c *Client) preflightStagedRestores(ctx context.Context, actx *applyContext, appName string) error { + if !appStateIsStaged(actx.plan, appName) { + return nil + } + app, ok := findPrepareApp(actx.plan.Prepare, appName) + if !ok { + return nil + } + runner := c.dockerRunner() + for _, step := range actx.plan.Steps { + if step.Kind != StepRestoreDataStore || step.App != appName || shouldSkipApplyStep(actx.plan, step) { + continue + } + store, ok := findPrepareDataStore(app, step.Ref) + if !ok || dataStoreMigrationKind(store) != dataStoreMigrationLogical { + continue + } + if err := c.preflightStagedRestore(ctx, runner, actx, app, step, store); err != nil { + return fmt.Errorf("staged restore preflight of %s for app %s failed, so pause_source did not stop the source: %w", store.Service, appName, err) } + } + return nil +} - dumpPath, err := dataStoreDumpPath(actx.plan, step.App, step.Ref) - if err != nil { +// stagedRestorePreflightError marks a refusal whose layout comes from the +// plan itself, so no retry of this run can pass pause_source. +type stagedRestorePreflightError struct { + err error + requiresNewRun bool +} + +func (e stagedRestorePreflightError) Error() string { return e.err.Error() } + +func (e stagedRestorePreflightError) Unwrap() error { return e.err } + +func stagedRestoreLayoutRefusal(err error, app string) error { + if !errors.Is(err, ErrNotImplemented) { + return err + } + return stagedRestorePreflightError{ + err: fmt.Errorf("%w; no retry of this run can pass pause_source for app %s because its reviewed plan is immutable: follow the recovery `bort status` shows to release this run, then choose a recreate or managed data store strategy or change the source compose and create a new run", err, app), + requiresNewRun: true, + } +} + +func (c *Client) preflightStagedRestore(ctx context.Context, runner dockerRunner, actx *applyContext, app preparer.AppPlan, step Step, store preparer.DataStoreResource) error { + staged, err := requireStagedRestoreLayout(actx.plan, step, store) + if err != nil { + return stagedRestoreLayoutRefusal(err, step.App) + } + project, err := c.stagingRestoreProject(ctx, actx, step, store, staged) + if err != nil { + return err + } + if err := project.down(runner); err != nil { + return err + } + for _, volume := range staged { + if err := ensureStagingVolume(ctx, runner, actx.plan, step.App, volume); err != nil { return err } - restoreListPath, cleanupRestoreList, err := preparePgRestoreList(ctx, runner, dst.ID, dumpPath, step.App, step.Ref) + } + checkErr := func() error { + if err := runner.Run(ctx, nil, nil, project.args("create", "--no-build", store.Service)...); err != nil { + return fmt.Errorf("create staging data store %s: %w", store.Service, err) + } + dst, err := project.serviceContainer(ctx, runner, store.Service) if err != nil { return err } - defer cleanupRestoreList() + if err := requireStagedPostgresDataDir(dst, staged); err != nil { + return fmt.Errorf("%w: %v", ErrNotImplemented, err) + } + return requireSourceMountsStageDataDir(app, store.Service, postgresDataDir(dst), staged) + }() + checkErr = stagedRestoreLayoutRefusal(checkErr, step.App) + if downErr := project.down(runner); downErr != nil { + if checkErr != nil { + return fmt.Errorf("%w (also failed to stop staging project: %v)", checkErr, downErr) + } + return downErr + } + return checkErr +} - file, err := os.Open(dumpPath) +// restoreDataStoreToStaging runs the data store service alone under a +// Bort-owned compose project with only its staging volumes and read-only +// init-script mounts, restores the dump into it, and stops it again. the +// volumes are recreated first so a retry never restores on top of a +// partial earlier attempt. +func (c *Client) restoreDataStoreToStaging(ctx context.Context, runner dockerRunner, actx *applyContext, app preparer.AppPlan, step Step, store preparer.DataStoreResource) error { + staged, err := requireStagedRestoreLayout(actx.plan, step, store) + if err != nil { + return err + } + project, err := c.stagingRestoreProject(ctx, actx, step, store, staged) + if err != nil { + return err + } + for _, volume := range staged { + if err := actx.forgetMigratedVolumeMount(step.App, volume.Service, volume.Target); err != nil { + return err + } + } + if err := project.down(runner); err != nil { + return err + } + for _, volume := range staged { + if err := requireStagingVolumeUnattached(ctx, runner, volume); err != nil { + return err + } + if err := recreateStagingVolume(ctx, runner, actx.plan, step.App, volume); err != nil { + return err + } + } + if err := runner.Run(ctx, nil, nil, project.args("up", "-d", "--no-build", "--no-deps", store.Service)...); err != nil { + return fmt.Errorf("start staging data store %s: %w", store.Service, err) + } + restoreErr := func() error { + dst, err := project.serviceContainer(ctx, runner, store.Service) if err != nil { - return fmt.Errorf("open dump file %s: %w", dumpPath, err) + return err + } + if err := requireStagedPostgresDataDir(dst, staged); err != nil { + return err + } + if err := requireSourceMountsStageDataDir(app, store.Service, postgresDataDir(dst), staged); err != nil { + return err + } + return pgRestoreIntoContainer(ctx, runner, actx.plan, step, dst) + }() + if downErr := project.down(runner); downErr != nil { + if restoreErr != nil { + return fmt.Errorf("%w (also failed to stop staging project: %v)", restoreErr, downErr) + } + return downErr + } + if restoreErr != nil { + return restoreErr + } + for _, volume := range staged { + if err := requireStagingVolumeUnattached(ctx, runner, volume); err != nil { + return err + } + if err := actx.recordMigratedVolumeMount(step.App, migratedVolumeMount{ + Service: volume.Service, + Target: volume.Target, + VolumeName: volume.VolumeName, + }); err != nil { + return err } - defer file.Close() + } + return nil +} - args := []string{"exec", "-i"} - if creds.Password != "" { - pgpassPath, cleanupPgpass, err := stageContainerPgpass(ctx, runner, dst.ID, filepath.Dir(dumpPath), creds.Password) - if err != nil { - return err - } - defer func() { - if err := cleanupPgpass(); err != nil { - fmt.Fprintf(os.Stderr, "warning: %v; the file holds the postgres database password, remove it manually\n", err) - } - }() - args = append(args, "-e", "PGPASSFILE="+pgpassPath) - } - args = append(args, dst.ID, "pg_restore", "-w", - "-U", creds.User, - "-d", creds.Database, - "--clean", "--if-exists", "--no-owner", "--no-acl", - "--single-transaction", "--exit-on-error", - ) - if restoreListPath != "" { - args = append(args, "-L", restoreListPath) - } - return runner.Run(ctx, file, nil, args...) - }); err != nil { +func pgRestoreIntoContainer(ctx context.Context, runner dockerRunner, plan Plan, step Step, dst dockerContainer) error { + creds := postgresCredsFromEnv(envMap(dst.Config.Env)) + if err := waitPostgresReady(ctx, runner, dst.ID, creds); err != nil { return err } - return recordMigratedStoreMounts(actx, step.App, app, store.Service, dst) + + dumpPath, err := dataStoreDumpPath(plan, step.App, step.Ref) + if err != nil { + return err + } + restoreListPath, cleanupRestoreList, err := preparePgRestoreList(ctx, runner, dst.ID, dumpPath, step.App, step.Ref) + if err != nil { + return err + } + defer cleanupRestoreList() + + file, err := os.Open(dumpPath) + if err != nil { + return fmt.Errorf("open dump file %s: %w", dumpPath, err) + } + defer file.Close() + + args := []string{"exec", "-i"} + if creds.Password != "" { + pgpassPath, cleanupPgpass, err := stageContainerPgpass(ctx, runner, dst.ID, filepath.Dir(dumpPath), creds.Password) + if err != nil { + return err + } + defer func() { + if err := cleanupPgpass(); err != nil { + fmt.Fprintf(os.Stderr, "warning: %v; the file holds the postgres database password, remove it manually\n", err) + } + }() + args = append(args, "-e", "PGPASSFILE="+pgpassPath) + } + args = append(args, dst.ID, "pg_restore", "-w", + "-U", creds.User, + "-d", creds.Database, + "--clean", "--if-exists", "--no-owner", "--no-acl", + "--single-transaction", "--exit-on-error", + ) + if restoreListPath != "" { + args = append(args, "-L", restoreListPath) + } + return runner.Run(ctx, file, nil, args...) } func recordMigratedStoreMounts(actx *applyContext, appName string, app preparer.AppPlan, service string, container dockerContainer) error { @@ -1077,17 +1374,19 @@ func isSupabaseRealtimePublicationListLine(fields []string) bool { return len(fields) > 5 && fields[3] == "PUBLICATION" && fields[4] == "-" && fields[5] == "supabase_realtime" } -// waitPostgresReady polls pg_isready inside the target container so the -// restore step does not race compose.deploy bringing up the database. +// waitPostgresReady polls pg_isready over loopback TCP inside the target +// container. the postgres entrypoint runs init scripts against a +// socket-only temporary server, so a socket probe would let the restore +// race those scripts; only the final server accepts TCP. func waitPostgresReady(ctx context.Context, runner dockerRunner, containerID string, creds postgresCreds) error { deadline := time.Now().Add(targetDiscoveryTimeout) for { - args := []string{"exec", containerID, "pg_isready", "-q", "-U", creds.User, "-d", creds.Database} + args := []string{"exec", containerID, "pg_isready", "-q", "-h", "127.0.0.1", "-U", creds.User, "-d", creds.Database} if err := runner.Run(ctx, nil, nil, args...); err == nil { return nil } if time.Now().After(deadline) { - return fmt.Errorf("target postgres %s was not ready after %s", containerID, targetDiscoveryTimeout) + return fmt.Errorf("target postgres %s did not accept TCP connections on 127.0.0.1 after %s; the restore needs the final server, not the entrypoint's socket-only init server", containerID, targetDiscoveryTimeout) } select { case <-ctx.Done(): diff --git a/internal/target/dokploy/datamove_test.go b/internal/target/dokploy/datamove_test.go index 8e47142..4a013ce 100644 --- a/internal/target/dokploy/datamove_test.go +++ b/internal/target/dokploy/datamove_test.go @@ -23,6 +23,7 @@ import ( type fakeDockerRunner struct { outputs map[string][]byte + outputErrs map[string]error runOutputs map[string][]byte outputArgs [][]string runs []fakeDockerRun @@ -53,9 +54,37 @@ func (f *fakeDockerRunner) Output(_ context.Context, args ...string) ([]byte, er return []byte{}, nil } key := strings.Join(args, " ") + if err, ok := f.outputErrs[key]; ok { + return nil, err + } if data, ok := f.outputs[key]; ok { return data, nil } + if key == "ps --filter label=com.docker.swarm.service.name=dokploy-postgres --filter status=running --format {{.ID}}" { + for _, name := range strings.Fields(string(f.outputs["ps --format {{.Names}}"])) { + if name == "dokploy-postgres" || strings.HasPrefix(name, "dokploy-postgres.") || strings.HasPrefix(name, "dokploy-postgres-") { + return []byte(name + "\n"), nil + } + } + return nil, nil + } + postgresContainer := "" + for _, name := range strings.Fields(string(f.outputs["ps --format {{.Names}}"])) { + if name == "dokploy-postgres" || strings.HasPrefix(name, "dokploy-postgres.") || strings.HasPrefix(name, "dokploy-postgres-") { + postgresContainer = name + break + } + } + if postgresContainer != "" { + switch key { + case "node inspect self --format {{.ID}}": + return []byte("local-node\n"), nil + case "service ps --filter desired-state=running -q dokploy-postgres": + return []byte("dokploy-postgres-task\n"), nil + case "inspect --type task dokploy-postgres-task": + return []byte(fmt.Sprintf(`[{"NodeID":"local-node","Status":{"State":"running","ContainerStatus":{"ContainerID":%q}}}]`, postgresContainer)), nil + } + } for prefix, data := range f.outputs { if strings.HasPrefix(key, prefix) { return data, nil @@ -92,14 +121,20 @@ type statefulTargetRunner struct { } func (r *statefulTargetRunner) Output(ctx context.Context, args ...string) ([]byte, error) { - if len(args) == 4 && args[0] == "inspect" && args[1] == "--type" && args[2] == "container" && r.stopped[args[3]] { - output, err := r.fakeDockerRunner.Output(ctx, args...) - if err != nil { - return nil, err + if len(args) >= 4 && args[0] == "inspect" && args[1] == "--type" && args[2] == "container" { + allStopped := true + for _, id := range args[3:] { + allStopped = allStopped && r.stopped[id] + } + if allStopped { + output, err := r.fakeDockerRunner.Output(ctx, args...) + if err != nil { + return nil, err + } + output = bytes.ReplaceAll(output, []byte(`"Running":true`), []byte(`"Running":false`)) + output = bytes.ReplaceAll(output, []byte(`"Status":"running"`), []byte(`"Status":"exited"`)) + return output, nil } - output = bytes.ReplaceAll(output, []byte(`"Running":true`), []byte(`"Running":false`)) - output = bytes.ReplaceAll(output, []byte(`"Status":"running"`), []byte(`"Status":"exited"`)) - return output, nil } output, err := r.fakeDockerRunner.Output(ctx, args...) if err != nil || len(args) != 2 { @@ -155,7 +190,7 @@ func (r *latePostDeployTargetRunner) Output(_ context.Context, args ...string) ( r.outputArgs = append(r.outputArgs, append([]string{}, args...)) key := strings.Join(args, " ") switch key { - case "ps --format {{.Names}}": + case "ps --filter label=com.docker.swarm.service.name=dokploy-postgres --filter status=running --format {{.ID}}": return []byte("dokploy-postgres.1.task\n"), nil case "ps -a --filter label=com.docker.compose.project=stack-1 --format {{.ID}}": r.psCalls++ @@ -721,8 +756,8 @@ func TestApplyRestoreDataStoreFiltersEventTriggers(t *testing.T) { "exec -i dst-id pg_restore -l": []byte("; archive header\n20; 2615 16457 SCHEMA - auth supabase_admin\n7; 3079 16950 EXTENSION - supabase_vault \n239; 1259 16488 TABLE auth users supabase_auth_admin\n5332; 0 16458 TABLE DATA auth users supabase_auth_admin\n5337; 0 16496 TABLE DATA auth schema_migrations supabase_auth_admin\n260; 1259 16970 VIEW vault decrypted_secrets supabase_admin\n271; 1259 100 TABLE public widgets bob\n2; 0 0 EVENT TRIGGER - pgrst_drop_watch bob\n3; 0 0 COMMENT - EVENT TRIGGER pgrst_drop_watch bob\n4; 0 100 TABLE DATA public widgets bob\n"), }, } - client := &Client{Docker: runner} - actx := &applyContext{cache: map[string]*appCache{}, plan: plan} + client := stagingCompatibleClient(t, runner, true, "") + actx := &applyContext{cache: map[string]*appCache{}, plan: plan, stagingEnvFormat: stagingEnvFormatKeepInterpolation} actx.entry("api").ComposeAppName = "stack-1" step := Step{Kind: StepRestoreDataStore, App: "api", Ref: "data-store:db"} @@ -852,11 +887,15 @@ func TestApplySyncVolumeCopiesNamedVolume(t *testing.T) { } client := &Client{Docker: runner} - actx := &applyContext{cache: map[string]*appCache{}} + step := Step{Kind: StepSyncVolume, App: "api", Ref: "volume:web -> /data"} + actx := &applyContext{ + cache: map[string]*appCache{}, + plan: Plan{ + Prepare: preparer.Result{Apps: []preparer.AppPlan{app}}, + }, + } actx.entry("api").ComposeAppName = "stack-1" - actx.plan = Plan{Prepare: preparer.Result{Apps: []preparer.AppPlan{app}}} - step := Step{Kind: StepSyncVolume, App: "api", Ref: "volume:web -> /data"} if err := client.applySyncVolume(context.Background(), actx, step); err != nil { t.Fatalf("applySyncVolume: %v", err) } @@ -1508,9 +1547,9 @@ func TestApplyResumeTargetLoadsPersistedMigratedVolumeState(t *testing.T) { resumed := &applyContext{cache: map[string]*appCache{}, plan: plan} resumed.entry("api").ComposeAppName = "stack-1" - pausedApps := map[string]struct{}{} + pausedApps := pausedSources{} coolifyProxyStopped := false - if err := client.primeResumeState(context.Background(), resumed, nil, Step{Kind: StepResumeTarget, App: "api", Ref: "api"}, pausedApps, &coolifyProxyStopped); err != nil { + if err := client.primeResumeState(context.Background(), resumed, nil, pausedApps, map[string]struct{}{}, &coolifyProxyStopped); err != nil { t.Fatalf("primeResumeState: %v", err) } runner.outputs["inspect --type container dst-id"] = []byte(`[{"Id":"dst-id","Name":"/dokploy-redis","Config":{"Labels":{"com.docker.compose.service":"redis","com.docker.compose.project":"stack-1"}},"State":{"Running":true,"Status":"running"},"Mounts":[{"Type":"volume","Name":"fresh-vol","Destination":"/data","RW":true}]}]`) @@ -1553,8 +1592,10 @@ func TestBestEffortResumeSkipsTargetWritersAfterUnsafeResumeError(t *testing.T) client := &Client{Docker: runner} actx := &applyContext{cache: map[string]*appCache{}} actx.entry("api").TargetWritersStopped = []dockerContainer{{ID: "target-id"}} + actx.entry("source-app").SourcePauseRecorded = true + actx.entry("source-app").SourcePausedContainers = []sourcePausedContainer{{ID: "source-id", Stopped: true}} plan := Plan{Prepare: preparer.Result{Apps: []preparer.AppPlan{{Name: "source-app", Resources: preparer.ResourceSpecs{Volumes: []preparer.VolumeResource{{Service: "web", Type: "volume", Target: "/data", SourceContainerID: "source-id"}}}}}}} - client.bestEffortResume(context.Background(), actx, plan, 1, map[string]struct{}{"source-app": {}}, false, true) + client.bestEffortResume(context.Background(), actx, plan, 1, pausedSources{"source-app": false}, false, true) if fakeOutputCalled(runner, "start", "target-id") { t.Fatalf("target writer restarted after unsafe resume error, calls=%#v", runner.outputArgs) @@ -1639,6 +1680,31 @@ func TestStopTargetComposeContainersWaitsForLatePostDeployTarget(t *testing.T) { } } +func TestPostDeployValidationStopFailureKeepsSourcePausedAndMarksAmbiguous(t *testing.T) { + runner := &fakeDockerRunner{ + outputs: map[string][]byte{ + "ps -a --filter label=com.docker.compose.project=stack-1 --format {{.ID}}": []byte("web-id\n"), + "inspect --type container web-id": []byte(`[{"Id":"web-id","Name":"/web","Config":{"Labels":{"com.docker.compose.service":"web","com.docker.compose.project":"stack-1"}},"State":{"Running":true,"Status":"running"},"Mounts":[{"Type":"volume","Name":"fresh-vol","Destination":"/data","RW":true}]}]`), + }, + outputErrs: map[string]error{"stop web-id": errors.New("Error response from daemon: cannot stop container")}, + } + client := &Client{Docker: runner} + actx := &applyContext{cache: map[string]*appCache{}, plan: Plan{}} + entry := actx.entry("api") + entry.ComposeAppName = "stack-1" + entry.MigratedVolumeMounts = map[string]migratedVolumeMount{ + migratedMountKey("web", "/data"): {Service: "web", Target: "/data", VolumeName: "migrated-vol"}, + } + + err := client.validateMigratedVolumeMountsAfterDeploy(context.Background(), actx, "api") + if err == nil || !strings.Contains(err.Error(), "changed from migrated volume migrated-vol to fresh-vol") || !strings.Contains(err.Error(), "also failed to stop unsafe target containers") { + t.Fatalf("expected drift error with failed stop, got %v", err) + } + if !isUnsafeSourceResumeError(err) || !isUnsafeTargetResumeError(err) || !mutationResponseMayHaveSucceeded(err) { + t.Fatalf("a target left running on migrated state must block source restart and ordinary retry, got %v", err) + } +} + func TestPostDeployValidationIgnoresCanceledApplyContextAndStopsLateTarget(t *testing.T) { runner := &latePostDeployTargetRunner{emptyFirst: true} client := &Client{Docker: runner} @@ -1855,13 +1921,13 @@ func TestPrimeResumeStateRecordsAlreadyStoppedTargetWriters(t *testing.T) { Prepare: preparer.Result{BundleDir: bundleDir, Apps: []preparer.AppPlan{app}}, }} actx.entry("api").ComposeAppName = "stack-api" - pausedApps := map[string]struct{}{} + pausedApps := pausedSources{} coolifyProxyStopped := false err := client.primeResumeState(context.Background(), actx, []Step{{Kind: StepPushImage, App: "api", Ref: "api"}}, - Step{Kind: StepPauseSource, App: "other", Ref: "other"}, pausedApps, + map[string]struct{}{}, &coolifyProxyStopped, ) if err != nil { @@ -1879,6 +1945,89 @@ func TestPrimeResumeStateRecordsAlreadyStoppedTargetWriters(t *testing.T) { } } +func TestPrimeResumeStateDoesNotRedeployCompletedPush(t *testing.T) { + bundleDir := t.TempDir() + appDir := filepath.Join(bundleDir, "api") + if err := os.MkdirAll(appDir, 0o700); err != nil { + t.Fatalf("mkdir app dir: %v", err) + } + if err := os.WriteFile(filepath.Join(appDir, "compose.yaml"), []byte("services:\n web:\n image: example/web\n"), 0o600); err != nil { + t.Fatalf("write compose: %v", err) + } + var mutations []string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodPost { + mutations = append(mutations, r.URL.Path) + } + switch { + case r.Method == http.MethodGet && r.URL.Path == "/api/compose.one": + _ = json.NewEncoder(w).Encode(Compose{ComposeID: "compose-1", AppName: "stack-api"}) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + runner := &fakeDockerRunner{outputs: map[string][]byte{ + "ps -a --filter label=com.docker.compose.project=stack-api --format {{.ID}}": []byte("web-id\n"), + "inspect --type container web-id": []byte(`[{"Id":"web-id","Name":"/web","Config":{"Labels":{"com.docker.compose.service":"web","com.docker.compose.project":"stack-api"}},"State":{"Running":true,"Status":"running"}}]`), + "stop web-id": []byte("web-id\n"), + }} + client := &Client{BaseURL: server.URL, Token: "secret", HTTPClient: server.Client(), Docker: runner} + app := preparer.AppPlan{Name: "api", Directory: "api"} + app.Resources.Volumes = []preparer.VolumeResource{{Service: "web", Type: "volume", SourceContainerID: "source-id"}} + app.TargetResources = &preparer.TargetResources{Dokploy: &preparer.DokployResources{ComposeApp: preparer.DokployComposeApp{ComposePath: "compose.yaml"}}} + plan := Plan{ + Steps: []Step{ + {Kind: StepUploadEnv, App: "api", Ref: "api"}, + {Kind: StepPushImage, App: "api", Ref: "api"}, + {Kind: StepPauseSource, App: "api", Ref: "api"}, + }, + Prepare: preparer.Result{BundleDir: bundleDir, Apps: []preparer.AppPlan{app}}, + } + actx := &applyContext{plan: plan, cache: map[string]*appCache{"api": {ComposeID: "compose-1", ComposeAppName: "stack-api"}}} + pausedApps := pausedSources{} + coolifyProxyStopped := false + + if err := client.primeResumeState(context.Background(), actx, plan.Steps[:2], pausedApps, map[string]struct{}{}, &coolifyProxyStopped); err != nil { + t.Fatalf("primeResumeState: %v", err) + } + if len(mutations) != 0 { + t.Fatalf("expected priming to leave the completed push untouched, got Dokploy mutations %v", mutations) + } + if stopped := actx.entry("api").TargetWritersStopped; len(stopped) != 1 || stopped[0].ID != "web-id" { + t.Fatalf("expected completed push to reconstruct its stopped writer, got %#v", stopped) + } + if !fakeOutputCalled(runner, "stop", "web-id") { + t.Fatalf("expected running target writer to be stopped, calls=%#v", runner.outputArgs) + } +} + +func TestTargetWriterReconciliationDoesNotRestartPreviouslyStoppedWriterOnFailure(t *testing.T) { + runner := &fakeDockerRunner{outputs: map[string][]byte{ + "ps -a --filter label=com.docker.compose.project=stack-api --format {{.ID}}": []byte("web-id\nworker-id\n"), + "inspect --type container web-id worker-id": []byte(`[ + {"Id":"web-id","Name":"/web","Config":{"Labels":{"com.docker.compose.service":"web","com.docker.compose.project":"stack-api"}},"State":{"Running":false,"Status":"exited"}}, + {"Id":"worker-id","Name":"/worker","Config":{"Labels":{"com.docker.compose.service":"worker","com.docker.compose.project":"stack-api"}},"State":{"Running":true,"Status":"running"}} + ]`), + }} + client := &Client{Docker: runner} + actx := &applyContext{cache: map[string]*appCache{}, plan: Plan{ + Steps: []Step{{Kind: StepPauseSource, App: "api", Ref: "api"}}, + }} + entry := actx.entry("api") + entry.ComposeAppName = "stack-api" + entry.TargetWritersStopped = []dockerContainer{{ID: "web-id"}} + + err := client.primeTargetWritersForResume(context.Background(), runner, actx, "api") + if err == nil || !strings.Contains(err.Error(), "stop target writer container worker-id") { + t.Fatalf("expected worker stop failure, got %v", err) + } + if fakeOutputCalled(runner, "start", "web-id") { + t.Fatalf("previously stopped target writer was restarted after reconciliation failure, calls=%#v", runner.outputArgs) + } +} + func TestApplySyncVolumeBindMountFailsOnStaleSource(t *testing.T) { app := preparer.AppPlan{Name: "api"} app.Resources.Volumes = []preparer.VolumeResource{{ @@ -1897,10 +2046,10 @@ func TestApplySyncVolumeBindMountFailsOnStaleSource(t *testing.T) { }, } client := &Client{Docker: runner} + step := Step{Kind: StepSyncVolume, App: "api", Ref: "volume:web -> /data"} actx := &applyContext{cache: map[string]*appCache{}} actx.entry("api").ComposeAppName = "stack-1" actx.plan = Plan{Prepare: preparer.Result{Apps: []preparer.AppPlan{app}}} - step := Step{Kind: StepSyncVolume, App: "api", Ref: "volume:web -> /data"} err := client.applySyncVolume(context.Background(), actx, step) if err == nil || !strings.Contains(err.Error(), "rescan before retrying") { t.Fatalf("expected stale-source error, got %v", err) diff --git a/internal/target/dokploy/docker.go b/internal/target/dokploy/docker.go index 7653f7f..ac8c6ef 100644 --- a/internal/target/dokploy/docker.go +++ b/internal/target/dokploy/docker.go @@ -7,10 +7,14 @@ import ( "errors" "fmt" "io" + "os" "os/exec" "path/filepath" + "runtime" "strings" "time" + + "github.com/aikins01/bort/internal/dockercli" ) // dockerRunner is the small surface bort needs to drive the local docker @@ -31,8 +35,38 @@ func (l localDockerRunner) binary() string { return "docker" } -func (l localDockerRunner) Output(ctx context.Context, args ...string) ([]byte, error) { +// command mirrors the `env -i PATH HOME` Dokploy deploys compose projects +// under, so staged compose interpolation cannot read Bort's shell; the local +// daemon pin moves to --host because DOCKER_HOST would leak into that env. +func (l localDockerRunner) command(ctx context.Context, args []string) (*exec.Cmd, error) { + env, err := dockercli.LocalEnvironment(os.Environ()) + if err != nil { + return nil, err + } + if len(args) > 0 && args[0] == "compose" { + kept := make([]string, 0, 2) + for _, entry := range env { + key, _, _ := strings.Cut(entry, "=") + if runtime.GOOS == "windows" { + key = strings.ToUpper(key) + } + if key == "PATH" || key == "HOME" { + kept = append(kept, entry) + } + } + env = kept + args = append([]string{"--host", dockercli.LocalHost()}, args...) + } cmd := exec.CommandContext(ctx, l.binary(), args...) + cmd.Env = env + return cmd, nil +} + +func (l localDockerRunner) Output(ctx context.Context, args ...string) ([]byte, error) { + cmd, err := l.command(ctx, args) + if err != nil { + return nil, err + } var stdout, stderr bytes.Buffer cmd.Stdout = &stdout cmd.Stderr = &stderr @@ -47,7 +81,10 @@ func (l localDockerRunner) Output(ctx context.Context, args ...string) ([]byte, } func (l localDockerRunner) Run(ctx context.Context, stdin io.Reader, stdout io.Writer, args ...string) error { - cmd := exec.CommandContext(ctx, l.binary(), args...) + cmd, err := l.command(ctx, args) + if err != nil { + return err + } if stdin != nil { cmd.Stdin = stdin } @@ -181,13 +218,23 @@ type dockerContainer struct { Labels map[string]string `json:"Labels"` } `json:"Config"` State struct { - Status string `json:"Status"` - Running bool `json:"Running"` - Health *dockerHealthState `json:"Health"` + Status string `json:"Status"` + Running bool `json:"Running"` + StartedAt string `json:"StartedAt"` + FinishedAt string `json:"FinishedAt"` + Health *dockerHealthState `json:"Health"` } `json:"State"` + HostConfig struct { + RestartPolicy dockerRestartPolicy `json:"RestartPolicy"` + } `json:"HostConfig"` Mounts []dockerMount `json:"Mounts"` } +type dockerRestartPolicy struct { + Name string `json:"Name"` + MaximumRetryCount int `json:"MaximumRetryCount"` +} + type dockerHealthState struct { Status string `json:"Status"` } @@ -210,10 +257,15 @@ type dockerInspectRaw struct { Labels map[string]string `json:"Labels"` } `json:"Config"` State struct { - Status string `json:"Status"` - Running bool `json:"Running"` - Health *dockerHealthState `json:"Health"` + Status string `json:"Status"` + Running bool `json:"Running"` + StartedAt string `json:"StartedAt"` + FinishedAt string `json:"FinishedAt"` + Health *dockerHealthState `json:"Health"` } `json:"State"` + HostConfig struct { + RestartPolicy dockerRestartPolicy `json:"RestartPolicy"` + } `json:"HostConfig"` Mounts []dockerMount `json:"Mounts"` } @@ -234,15 +286,44 @@ func inspectContainer(ctx context.Context, runner dockerRunner, ref string) (doc } r := raw[0] return dockerContainer{ - ID: r.ID, - Name: strings.TrimPrefix(r.Name, "/"), - Image: r.Image, - Config: r.Config, - State: r.State, - Mounts: r.Mounts, + ID: r.ID, + Name: strings.TrimPrefix(r.Name, "/"), + Image: r.Image, + Config: r.Config, + State: r.State, + HostConfig: r.HostConfig, + Mounts: r.Mounts, }, nil } +func inspectContainers(ctx context.Context, runner dockerRunner, refs []string) ([]dockerContainer, error) { + if len(refs) == 0 { + return nil, nil + } + args := append([]string{"inspect", "--type", "container"}, refs...) + out, err := runner.Output(ctx, args...) + if err != nil { + return nil, fmt.Errorf("docker inspect ids=%v: %w", refs, err) + } + var decoded []dockerInspectRaw + if err := json.Unmarshal(out, &decoded); err != nil { + return nil, fmt.Errorf("decode docker inspect batch: %w", err) + } + containers := make([]dockerContainer, 0, len(decoded)) + for _, raw := range decoded { + containers = append(containers, dockerContainer{ + ID: raw.ID, + Name: strings.TrimPrefix(raw.Name, "/"), + Image: raw.Image, + Config: raw.Config, + State: raw.State, + HostConfig: raw.HostConfig, + Mounts: raw.Mounts, + }) + } + return containers, nil +} + func listContainersByLabel(ctx context.Context, runner dockerRunner, label string) ([]dockerContainer, error) { out, err := runner.Output(ctx, "ps", "-a", "--filter", "label="+label, "--format", "{{.ID}}") if err != nil { @@ -258,38 +339,22 @@ func listContainersByLabel(ctx context.Context, runner dockerRunner, label strin if len(ids) == 0 { return nil, nil } - args := append([]string{"inspect", "--type", "container"}, ids...) - raw, err := runner.Output(ctx, args...) - if err != nil { - return nil, fmt.Errorf("docker inspect ids=%v: %w", ids, err) - } - var decoded []dockerInspectRaw - if err := json.Unmarshal(raw, &decoded); err != nil { - return nil, fmt.Errorf("decode docker inspect batch: %w", err) - } - containers := make([]dockerContainer, 0, len(decoded)) - for _, r := range decoded { - containers = append(containers, dockerContainer{ - ID: r.ID, - Name: strings.TrimPrefix(r.Name, "/"), - Image: r.Image, - Config: r.Config, - State: r.State, - Mounts: r.Mounts, - }) - } - return containers, nil + return inspectContainers(ctx, runner, ids) } func sourceContainer(ctx context.Context, runner dockerRunner, id, name string) (dockerContainer, error) { if id != "" { container, err := inspectContainer(ctx, runner, id) - if err == nil { - return container, nil - } - if name == "" { + if err != nil { return dockerContainer{}, err } + if !sourceContainerIDMatches(id, container.ID) { + return dockerContainer{}, fmt.Errorf("source container %q resolved to ID %q, want reviewed ID %q", name, container.ID, id) + } + if name != "" && strings.TrimPrefix(container.Name, "/") != strings.TrimPrefix(name, "/") { + return dockerContainer{}, fmt.Errorf("source container %s has name %q, want reviewed name %q", id, container.Name, name) + } + return container, nil } if name == "" { return dockerContainer{}, fmt.Errorf("source container ref is empty") @@ -297,6 +362,12 @@ func sourceContainer(ctx context.Context, runner dockerRunner, id, name string) return inspectContainer(ctx, runner, name) } +func sourceContainerIDMatches(reviewed, inspected string) bool { + reviewed = strings.TrimSpace(reviewed) + inspected = strings.TrimSpace(inspected) + return reviewed != "" && inspected != "" && (reviewed == inspected || len(reviewed) >= 12 && strings.HasPrefix(inspected, reviewed)) +} + func findMountByTarget(c dockerContainer, target string) (dockerMount, bool) { for _, mount := range c.Mounts { if mount.Destination == target { @@ -343,7 +414,7 @@ func copyNamedVolume(ctx context.Context, runner dockerRunner, src, dst string) "-v", dst + ":/to", volumeCopyImage, "sh", "-c", - "rm -rf /to/* /to/.[!.]* /to/..?* 2>/dev/null; cd /from && tar cpf - . | tar xpf - -C /to", + "set -o pipefail; find /to -mindepth 1 -delete && cd /from && tar cpf - . | tar xpf - -C /to && find /to \\( -type f -o -type d \\) -print0 | xargs -0 fsync", } return runner.Run(ctx, nil, nil, args...) } diff --git a/internal/target/dokploy/proxy.go b/internal/target/dokploy/proxy.go index d45b16f..bba2700 100644 --- a/internal/target/dokploy/proxy.go +++ b/internal/target/dokploy/proxy.go @@ -2,6 +2,7 @@ package dokploy import ( "context" + "errors" "fmt" "strings" ) @@ -22,6 +23,207 @@ func (c *Client) applyStartDokployProxy(ctx context.Context, _ *applyContext, _ return startProxyContainer(ctx, c.dockerRunner(), dokployProxyContainer) } +func (c *Client) VerifyTargetTrafficAuthority(ctx context.Context) error { + verifyCtx, cancel := context.WithTimeout(ctx, dockerStartTimeout) + defer cancel() + runner := c.dockerRunner() + target, err := inspectContainer(verifyCtx, runner, dokployProxyContainer) + if err != nil { + return fmt.Errorf("inspect proxy container %s: %w", dokployProxyContainer, err) + } + if normalizedRestartPolicyName(target.HostConfig.RestartPolicy.Name) != "unless-stopped" { + return fmt.Errorf("proxy container %s restart policy is %q, want unless-stopped", dokployProxyContainer, normalizedRestartPolicyName(target.HostConfig.RestartPolicy.Name)) + } + if !target.State.Running { + return fmt.Errorf("proxy container %s is not running", dokployProxyContainer) + } + source, err := inspectContainer(verifyCtx, runner, coolifyProxyContainer) + if err != nil { + if isContainerMissingErr(err) { + return nil + } + return fmt.Errorf("inspect proxy container %s: %w", coolifyProxyContainer, err) + } + if normalizedRestartPolicyName(source.HostConfig.RestartPolicy.Name) != "unless-stopped" { + return fmt.Errorf("proxy container %s restart policy is %q, want unless-stopped", coolifyProxyContainer, normalizedRestartPolicyName(source.HostConfig.RestartPolicy.Name)) + } + if source.State.Running { + return fmt.Errorf("proxy container %s is still running", coolifyProxyContainer) + } + return nil +} + +func (c *Client) AdoptTargetTrafficAuthority(ctx context.Context) error { + runner := c.dockerRunner() + if _, _, err := prepareProxyForReconcile(ctx, runner, dokployProxyContainer, false); err != nil { + return err + } + if _, _, err := prepareProxyForReconcile(ctx, runner, coolifyProxyContainer, true); err != nil { + return err + } + return c.VerifyTargetTrafficAuthority(ctx) +} + +func (c *Client) ReconcileTargetTrafficAuthority(ctx context.Context) error { + runner := c.dockerRunner() + target, _, err := prepareProxyForReconcile(ctx, runner, dokployProxyContainer, false) + if err != nil { + return err + } + source, found, err := prepareProxyForReconcile(ctx, runner, coolifyProxyContainer, true) + if err != nil { + return err + } + if found && source.State.Running { + if err := applyPreparedProxyState(ctx, runner, source, false); err != nil { + return c.restoreProxyAfterFailedHandoff(err, runner, true, source, target, c.verifySourceTrafficAuthority) + } + } + if !target.State.Running { + if err := applyPreparedProxyState(ctx, runner, target, true); err != nil { + return c.restoreProxyAfterFailedHandoff(err, runner, found, source, target, c.verifySourceTrafficAuthority) + } + } + if err := c.VerifyTargetTrafficAuthority(ctx); err != nil { + return c.restoreProxyAfterFailedHandoff(err, runner, found, source, target, c.verifySourceTrafficAuthority) + } + return nil +} + +func (c *Client) ReconcileSourceTrafficAuthority(ctx context.Context) error { + runner := c.dockerRunner() + source, _, err := prepareProxyForReconcile(ctx, runner, coolifyProxyContainer, false) + if err != nil { + return err + } + target, found, err := prepareProxyForReconcile(ctx, runner, dokployProxyContainer, true) + if err != nil { + return err + } + if found && target.State.Running { + if err := applyPreparedProxyState(ctx, runner, target, false); err != nil { + return c.restoreProxyAfterFailedHandoff(err, runner, true, target, source, c.VerifyTargetTrafficAuthority) + } + } + if !source.State.Running { + if err := applyPreparedProxyState(ctx, runner, source, true); err != nil { + return c.restoreProxyAfterFailedHandoff(err, runner, found, target, source, c.VerifyTargetTrafficAuthority) + } + } + if err := c.verifySourceTrafficAuthority(ctx); err != nil { + return c.restoreProxyAfterFailedHandoff(err, runner, found, target, source, c.VerifyTargetTrafficAuthority) + } + return nil +} + +func (c *Client) restoreProxyAfterFailedHandoff(handoffErr error, runner dockerRunner, restore bool, original, destination dockerContainer, verify func(context.Context) error) error { + if !restore { + return handoffErr + } + restoreCtx, cancel := context.WithTimeout(context.Background(), dockerStartTimeout) + defer cancel() + var operationErr error + if err := applyPreparedProxyState(restoreCtx, runner, destination, false); err != nil { + operationErr = errors.Join(operationErr, err) + } + if err := applyPreparedProxyState(restoreCtx, runner, original, true); err != nil { + operationErr = errors.Join(operationErr, err) + } + if err := verify(restoreCtx); err != nil { + return errors.Join(handoffErr, fmt.Errorf("restore proxy traffic authority to %s: %w", original.Name, errors.Join(operationErr, err))) + } + return fmt.Errorf("%w; restored proxy traffic authority to %s", handoffErr, original.Name) +} + +func prepareProxyForReconcile(ctx context.Context, runner dockerRunner, name string, allowMissing bool) (dockerContainer, bool, error) { + inspectCtx, cancelInspect := context.WithTimeout(ctx, dockerStopTimeout) + container, err := inspectContainer(inspectCtx, runner, name) + cancelInspect() + if err != nil { + if allowMissing && isContainerMissingErr(err) { + return dockerContainer{}, false, nil + } + return dockerContainer{}, false, fmt.Errorf("inspect proxy container %s restart policy: %w", name, err) + } + if normalizedRestartPolicyName(container.HostConfig.RestartPolicy.Name) != "unless-stopped" { + updateCtx, cancelUpdate := context.WithTimeout(ctx, dockerStopTimeout) + err := updateContainerRestartPolicy(updateCtx, runner, container.ID, "unless-stopped") + cancelUpdate() + if err != nil { + return dockerContainer{}, false, fmt.Errorf("set proxy container %s restart policy: %w", name, err) + } + } + return container, true, nil +} + +func applyPreparedProxyState(ctx context.Context, runner dockerRunner, container dockerContainer, running bool) error { + if running { + startCtx, cancel := context.WithTimeout(ctx, dockerStartTimeout) + err := startContainer(startCtx, runner, container.ID) + cancel() + if err != nil { + return fmt.Errorf("start proxy container %s: %w", container.Name, err) + } + return nil + } + stopCtx, cancel := context.WithTimeout(ctx, dockerStopTimeout) + err := stopContainer(stopCtx, runner, container.ID) + cancel() + if err != nil && !isContainerMissingErr(err) { + return fmt.Errorf("stop proxy container %s: %w", container.Name, err) + } + return nil +} + +func updateAndVerifyProxyRestartPolicy(ctx context.Context, runner dockerRunner, name, id string) error { + updateCtx, cancelUpdate := context.WithTimeout(ctx, dockerStopTimeout) + err := updateContainerRestartPolicy(updateCtx, runner, id, "unless-stopped") + cancelUpdate() + if err != nil { + return fmt.Errorf("set proxy container %s restart policy: %w", name, err) + } + verifyCtx, cancelVerify := context.WithTimeout(ctx, dockerStopTimeout) + container, err := inspectContainer(verifyCtx, runner, name) + cancelVerify() + if err != nil { + return fmt.Errorf("verify proxy container %s restart policy: %w", name, err) + } + if normalizedRestartPolicyName(container.HostConfig.RestartPolicy.Name) != "unless-stopped" { + return fmt.Errorf("proxy container %s restart policy is %q after update, want unless-stopped", name, normalizedRestartPolicyName(container.HostConfig.RestartPolicy.Name)) + } + return nil +} + +func (c *Client) verifySourceTrafficAuthority(ctx context.Context) error { + verifyCtx, cancel := context.WithTimeout(ctx, dockerStartTimeout) + defer cancel() + target, err := inspectContainer(verifyCtx, c.dockerRunner(), dokployProxyContainer) + if err != nil { + if isContainerMissingErr(err) { + target = dockerContainer{} + } else { + return fmt.Errorf("inspect proxy container %s: %w", dokployProxyContainer, err) + } + } + if target.State.Running { + return fmt.Errorf("proxy container %s is still running", dokployProxyContainer) + } + if target.ID != "" && normalizedRestartPolicyName(target.HostConfig.RestartPolicy.Name) != "unless-stopped" { + return fmt.Errorf("proxy container %s restart policy is %q, want unless-stopped", dokployProxyContainer, normalizedRestartPolicyName(target.HostConfig.RestartPolicy.Name)) + } + source, err := inspectContainer(verifyCtx, c.dockerRunner(), coolifyProxyContainer) + if err != nil { + return fmt.Errorf("inspect proxy container %s: %w", coolifyProxyContainer, err) + } + if !source.State.Running { + return fmt.Errorf("proxy container %s is not running", coolifyProxyContainer) + } + if normalizedRestartPolicyName(source.HostConfig.RestartPolicy.Name) != "unless-stopped" { + return fmt.Errorf("proxy container %s restart policy is %q, want unless-stopped", coolifyProxyContainer, normalizedRestartPolicyName(source.HostConfig.RestartPolicy.Name)) + } + return nil +} + func stopProxyContainer(ctx context.Context, runner dockerRunner, name string) error { container, err := inspectContainer(ctx, runner, name) if err != nil { @@ -33,6 +235,12 @@ func stopProxyContainer(ctx context.Context, runner dockerRunner, name string) e } return fmt.Errorf("inspect proxy container %s: %w", name, err) } + policy := normalizedRestartPolicyName(container.HostConfig.RestartPolicy.Name) + if policy != "no" && policy != "unless-stopped" { + if err := updateAndVerifyProxyRestartPolicy(ctx, runner, name, container.ID); err != nil { + return err + } + } if !container.State.Running { return nil } diff --git a/internal/target/dokploy/proxy_test.go b/internal/target/dokploy/proxy_test.go index 09cf399..cf4ee2e 100644 --- a/internal/target/dokploy/proxy_test.go +++ b/internal/target/dokploy/proxy_test.go @@ -2,7 +2,9 @@ package dokploy import ( "context" + "encoding/json" "errors" + "fmt" "strings" "testing" @@ -27,7 +29,7 @@ func TestPlanFromArtifactsAddsProxySwapAfterRoutes(t *testing.T) { got = append(got, step.Kind) } } - want := []StepKind{StepInstallGateway, StepActivateRoutes, StepStopCoolifyProxy, StepStartDokployProxy} + want := []StepKind{StepInstallGateway, StepStopCoolifyProxy, StepActivateRoutes, StepStartDokployProxy} if len(got) != len(want) { t.Fatalf("expected %v, got %v (full=%v)", want, got, plan.Steps) } @@ -62,6 +64,89 @@ func TestApplyStopCoolifyProxyStopsRunning(t *testing.T) { } } +func TestApplyStopCoolifyProxyDurablyFencesAlwaysRestartPolicy(t *testing.T) { + source := dockerContainer{ID: "coolify-id", Name: coolifyProxyContainer} + source.State.Running = true + source.HostConfig.RestartPolicy.Name = "always" + runner := &proxyStateRunner{containers: map[string]dockerContainer{coolifyProxyContainer: source}} + client := &Client{Docker: runner} + if err := client.applyStopCoolifyProxy(context.Background(), &applyContext{}, Step{Kind: StepStopCoolifyProxy, Ref: coolifyProxyContainer}); err != nil { + t.Fatal(err) + } + container := runner.containers[coolifyProxyContainer] + if container.State.Running || container.HostConfig.RestartPolicy.Name != "unless-stopped" { + t.Fatalf("proxy stop was not durable across daemon restart: %#v", container) + } + updateIndex, stopIndex := -1, -1 + for index, call := range runner.calls { + switch call { + case "update --restart=unless-stopped coolify-id": + updateIndex = index + case "stop coolify-id": + stopIndex = index + } + } + if updateIndex < 0 || stopIndex <= updateIndex { + t.Fatalf("restart policy was not made durable before stop: %v", runner.calls) + } +} + +func TestAdoptTargetTrafficAuthorityRepairsLegacyAlwaysPolicies(t *testing.T) { + target := dockerContainer{ID: "dokploy-id", Name: dokployProxyContainer} + target.State.Running = true + target.HostConfig.RestartPolicy.Name = "always" + source := dockerContainer{ID: "coolify-id", Name: coolifyProxyContainer} + source.HostConfig.RestartPolicy.Name = "always" + runner := &proxyStateRunner{containers: map[string]dockerContainer{dokployProxyContainer: target, coolifyProxyContainer: source}} + client := &Client{Docker: runner} + if err := client.AdoptTargetTrafficAuthority(context.Background()); err != nil { + t.Fatal(err) + } + for _, name := range []string{dokployProxyContainer, coolifyProxyContainer} { + if got := runner.containers[name].HostConfig.RestartPolicy.Name; got != "unless-stopped" { + t.Fatalf("%s restart policy = %q after adoption, want unless-stopped", name, got) + } + } + if !runner.containers[dokployProxyContainer].State.Running || runner.containers[coolifyProxyContainer].State.Running { + t.Fatalf("adoption changed proxy running state: %#v", runner.containers) + } + updates := 0 + for _, call := range runner.calls { + switch { + case strings.HasPrefix(call, "start ") || strings.HasPrefix(call, "stop "): + t.Fatalf("adoption must not start or stop proxies, got %v", runner.calls) + case strings.HasPrefix(call, "update --restart=unless-stopped "): + updates++ + } + } + if updates != 2 { + t.Fatalf("expected both proxy policies repaired, got calls %v", runner.calls) + } +} + +func TestAdoptTargetTrafficAuthorityRefusesRunningSourceProxyWithoutStopping(t *testing.T) { + target := dockerContainer{ID: "dokploy-id", Name: dokployProxyContainer} + target.State.Running = true + target.HostConfig.RestartPolicy.Name = "unless-stopped" + source := dockerContainer{ID: "coolify-id", Name: coolifyProxyContainer} + source.State.Running = true + source.HostConfig.RestartPolicy.Name = "always" + runner := &proxyStateRunner{containers: map[string]dockerContainer{dokployProxyContainer: target, coolifyProxyContainer: source}} + client := &Client{Docker: runner} + err := client.AdoptTargetTrafficAuthority(context.Background()) + if err == nil || !strings.Contains(err.Error(), "coolify-proxy is still running") { + t.Fatalf("expected running source proxy to refuse adoption, got %v", err) + } + if !runner.containers[coolifyProxyContainer].State.Running { + t.Fatalf("adoption stopped the source proxy: %v", runner.calls) + } + for _, call := range runner.calls { + if strings.HasPrefix(call, "stop ") || strings.HasPrefix(call, "start ") { + t.Fatalf("adoption must not start or stop proxies, got %v", runner.calls) + } + } +} + func TestApplyStopCoolifyProxyNoopWhenAlreadyStopped(t *testing.T) { runner := &fakeDockerRunner{ outputs: map[string][]byte{ @@ -127,6 +212,302 @@ func TestApplyStartDokployProxyErrorsWhenMissing(t *testing.T) { } } +func TestReconcileTargetTrafficAuthorityRepairsAndVerifiesProxyState(t *testing.T) { + source := dockerContainer{ID: "coolify-id", Name: coolifyProxyContainer} + source.State.Running = true + target := dockerContainer{ID: "dokploy-id", Name: dokployProxyContainer} + runner := &proxyStateRunner{containers: map[string]dockerContainer{ + coolifyProxyContainer: source, + dokployProxyContainer: target, + }} + client := &Client{Docker: runner} + if err := client.ReconcileTargetTrafficAuthority(context.Background()); err != nil { + t.Fatal(err) + } + if runner.containers[coolifyProxyContainer].State.Running || !runner.containers[dokployProxyContainer].State.Running { + t.Fatalf("proxy authority was not reconciled: %#v", runner.containers) + } + for name, container := range runner.containers { + if container.HostConfig.RestartPolicy.Name != "unless-stopped" { + t.Fatalf("proxy %s has restart policy %q", name, container.HostConfig.RestartPolicy.Name) + } + } + for _, name := range []string{coolifyProxyContainer, dokployProxyContainer} { + inspects := 0 + for _, call := range runner.calls { + if call == "inspect --type container "+name { + inspects++ + } + } + if inspects != 2 { + t.Fatalf("proxy %s inspected %d times, want initial and final inspections: %v", name, inspects, runner.calls) + } + } +} + +func TestReconcileSourceTrafficAuthorityDurablyStopsTarget(t *testing.T) { + source := dockerContainer{ID: "coolify-id", Name: coolifyProxyContainer} + target := dockerContainer{ID: "dokploy-id", Name: dokployProxyContainer} + target.State.Running = true + runner := &proxyStateRunner{containers: map[string]dockerContainer{ + coolifyProxyContainer: source, + dokployProxyContainer: target, + }} + client := &Client{Docker: runner} + if err := client.ReconcileSourceTrafficAuthority(context.Background()); err != nil { + t.Fatal(err) + } + if !runner.containers[coolifyProxyContainer].State.Running || runner.containers[dokployProxyContainer].State.Running { + t.Fatalf("source authority was not reconciled: %#v", runner.containers) + } + if runner.containers[dokployProxyContainer].HostConfig.RestartPolicy.Name != "unless-stopped" { + t.Fatalf("stopped target proxy can restart after daemon restart: %#v", runner.containers[dokployProxyContainer].HostConfig.RestartPolicy) + } +} + +func TestReconcileSourceTrafficAuthorityAcceptsMissingTarget(t *testing.T) { + source := dockerContainer{ID: "coolify-id", Name: coolifyProxyContainer} + runner := &proxyStateRunner{containers: map[string]dockerContainer{coolifyProxyContainer: source}} + client := &Client{Docker: runner} + + if err := client.ReconcileSourceTrafficAuthority(context.Background()); err != nil { + t.Fatalf("missing stopped target should satisfy source authority: %v", err) + } + if !runner.containers[coolifyProxyContainer].State.Running { + t.Fatalf("source proxy was not started: %#v", runner.containers) + } +} + +func TestReconcileTargetTrafficAuthorityRestoresSourceWhenTargetStartFails(t *testing.T) { + source := dockerContainer{ID: "coolify-id", Name: coolifyProxyContainer} + source.State.Running = true + target := dockerContainer{ID: "dokploy-id", Name: dokployProxyContainer} + ctx, cancel := context.WithCancel(context.Background()) + runner := &proxyStateRunner{ + containers: map[string]dockerContainer{ + coolifyProxyContainer: source, + dokployProxyContainer: target, + }, + failAt: map[string]int{"start dokploy-id": 1}, + beforeFailure: func(call string) { + if call == "start dokploy-id" { + cancel() + } + }, + } + client := &Client{Docker: runner} + err := client.ReconcileTargetTrafficAuthority(ctx) + if err == nil || !strings.Contains(err.Error(), "restored proxy traffic authority to "+coolifyProxyContainer) { + t.Fatalf("expected failed handoff with restored source authority, got %v", err) + } + if !runner.containers[coolifyProxyContainer].State.Running || runner.containers[dokployProxyContainer].State.Running { + t.Fatalf("failed target handoff did not restore source authority: %#v", runner.containers) + } + assertProxyCallOrder(t, runner.calls, "stop coolify-id", "start dokploy-id", "stop dokploy-id", "start coolify-id") +} + +func TestReconcileTargetTrafficAuthorityRestoresSourceAfterAmbiguousStopFailure(t *testing.T) { + source := dockerContainer{ID: "coolify-id", Name: coolifyProxyContainer} + source.State.Running = true + target := dockerContainer{ID: "dokploy-id", Name: dokployProxyContainer} + runner := &proxyStateRunner{ + containers: map[string]dockerContainer{ + coolifyProxyContainer: source, + dokployProxyContainer: target, + }, + failAt: map[string]int{"stop coolify-id": 1}, + } + runner.beforeFailure = func(call string) { + if call == "stop coolify-id" { + container := runner.containers[coolifyProxyContainer] + container.State.Running = false + runner.containers[coolifyProxyContainer] = container + } + } + client := &Client{Docker: runner} + err := client.ReconcileTargetTrafficAuthority(context.Background()) + if err == nil || !strings.Contains(err.Error(), "restored proxy traffic authority to "+coolifyProxyContainer) { + t.Fatalf("expected ambiguous stop failure with restored source authority, got %v", err) + } + if !runner.containers[coolifyProxyContainer].State.Running || runner.containers[dokployProxyContainer].State.Running { + t.Fatalf("ambiguous source stop left traffic without source authority: %#v", runner.containers) + } + assertProxyCallOrder(t, runner.calls, "stop coolify-id", "stop dokploy-id", "start coolify-id") +} + +func TestReconcileSourceTrafficAuthorityRestoresTargetAfterAmbiguousStopFailure(t *testing.T) { + source := dockerContainer{ID: "coolify-id", Name: coolifyProxyContainer} + target := dockerContainer{ID: "dokploy-id", Name: dokployProxyContainer} + target.State.Running = true + runner := &proxyStateRunner{ + containers: map[string]dockerContainer{ + coolifyProxyContainer: source, + dokployProxyContainer: target, + }, + failAt: map[string]int{"stop dokploy-id": 1}, + } + runner.beforeFailure = func(call string) { + if call == "stop dokploy-id" { + container := runner.containers[dokployProxyContainer] + container.State.Running = false + runner.containers[dokployProxyContainer] = container + } + } + client := &Client{Docker: runner} + err := client.ReconcileSourceTrafficAuthority(context.Background()) + if err == nil || !strings.Contains(err.Error(), "restored proxy traffic authority to "+dokployProxyContainer) { + t.Fatalf("expected ambiguous stop failure with restored target authority, got %v", err) + } + if runner.containers[coolifyProxyContainer].State.Running || !runner.containers[dokployProxyContainer].State.Running { + t.Fatalf("ambiguous target stop left traffic without target authority: %#v", runner.containers) + } + assertProxyCallOrder(t, runner.calls, "stop dokploy-id", "stop coolify-id", "start dokploy-id") +} + +func TestReconcileTargetTrafficAuthorityRestoresPreviouslyStoppedSourceWhenTargetStartFails(t *testing.T) { + source := dockerContainer{ID: "coolify-id", Name: coolifyProxyContainer} + target := dockerContainer{ID: "dokploy-id", Name: dokployProxyContainer} + runner := &proxyStateRunner{ + containers: map[string]dockerContainer{ + coolifyProxyContainer: source, + dokployProxyContainer: target, + }, + failAt: map[string]int{"start dokploy-id": 1}, + } + client := &Client{Docker: runner} + err := client.ReconcileTargetTrafficAuthority(context.Background()) + if err == nil || !strings.Contains(err.Error(), "restored proxy traffic authority to "+coolifyProxyContainer) { + t.Fatalf("expected failed handoff with restored source authority, got %v", err) + } + if !runner.containers[coolifyProxyContainer].State.Running || runner.containers[dokployProxyContainer].State.Running { + t.Fatalf("failed target handoff left both proxies stopped: %#v", runner.containers) + } + assertProxyCallOrder(t, runner.calls, "start dokploy-id", "stop dokploy-id", "start coolify-id") +} + +func TestReconcileSourceTrafficAuthorityRestoresTargetWhenFinalVerificationFails(t *testing.T) { + source := dockerContainer{ID: "coolify-id", Name: coolifyProxyContainer} + target := dockerContainer{ID: "dokploy-id", Name: dokployProxyContainer} + target.State.Running = true + ctx, cancel := context.WithCancel(context.Background()) + runner := &proxyStateRunner{ + containers: map[string]dockerContainer{ + coolifyProxyContainer: source, + dokployProxyContainer: target, + }, + failAt: map[string]int{"inspect --type container coolify-proxy": 2}, + beforeFailure: func(call string) { + if call == "inspect --type container coolify-proxy" { + cancel() + } + }, + } + client := &Client{Docker: runner} + err := client.ReconcileSourceTrafficAuthority(ctx) + if err == nil || !strings.Contains(err.Error(), "restored proxy traffic authority to "+dokployProxyContainer) { + t.Fatalf("expected failed verification with restored target authority, got %v", err) + } + if runner.containers[coolifyProxyContainer].State.Running || !runner.containers[dokployProxyContainer].State.Running { + t.Fatalf("failed source handoff did not restore target authority: %#v", runner.containers) + } + assertProxyCallOrder(t, runner.calls, "stop dokploy-id", "start coolify-id", "stop coolify-id", "start dokploy-id") +} + +func TestReconcileSourceTrafficAuthorityRestoresPreviouslyStoppedTargetWhenFinalVerificationFails(t *testing.T) { + source := dockerContainer{ID: "coolify-id", Name: coolifyProxyContainer} + target := dockerContainer{ID: "dokploy-id", Name: dokployProxyContainer} + runner := &proxyStateRunner{ + containers: map[string]dockerContainer{ + coolifyProxyContainer: source, + dokployProxyContainer: target, + }, + failAt: map[string]int{"inspect --type container coolify-proxy": 2}, + } + client := &Client{Docker: runner} + err := client.ReconcileSourceTrafficAuthority(context.Background()) + if err == nil || !strings.Contains(err.Error(), "restored proxy traffic authority to "+dokployProxyContainer) { + t.Fatalf("expected failed verification with restored target authority, got %v", err) + } + if runner.containers[coolifyProxyContainer].State.Running || !runner.containers[dokployProxyContainer].State.Running { + t.Fatalf("failed source handoff left both proxies stopped: %#v", runner.containers) + } + assertProxyCallOrder(t, runner.calls, "start coolify-id", "stop coolify-id", "start dokploy-id") +} + +func assertProxyCallOrder(t *testing.T, calls []string, expected ...string) { + t.Helper() + previous := -1 + for _, want := range expected { + found := -1 + for index := previous + 1; index < len(calls); index++ { + if calls[index] == want { + found = index + break + } + } + if found < 0 { + t.Fatalf("proxy call %q did not appear in order %v: %v", want, expected, calls) + } + previous = found + } +} + +type proxyStateRunner struct { + fakeDockerRunner + containers map[string]dockerContainer + calls []string + callCounts map[string]int + failAt map[string]int + beforeFailure func(string) +} + +func (r *proxyStateRunner) Output(ctx context.Context, args ...string) ([]byte, error) { + call := strings.Join(args, " ") + r.calls = append(r.calls, call) + if err := ctx.Err(); err != nil { + return nil, err + } + if r.callCounts == nil { + r.callCounts = map[string]int{} + } + r.callCounts[call]++ + if r.failAt[call] == r.callCounts[call] { + if r.beforeFailure != nil { + r.beforeFailure(call) + } + return nil, errors.New("injected proxy command failure") + } + if len(args) == 4 && args[0] == "inspect" { + container, ok := r.containers[args[3]] + if !ok { + return nil, fmt.Errorf("no such container: %s", args[3]) + } + return json.Marshal([]dockerContainer{container}) + } + if len(args) >= 2 && (args[0] == "start" || args[0] == "stop") { + ref := args[len(args)-1] + for name, container := range r.containers { + if container.ID != ref && name != ref { + continue + } + container.State.Running = args[0] == "start" + r.containers[name] = container + return []byte(ref + "\n"), nil + } + } + if len(args) == 3 && args[0] == "update" && args[1] == "--restart=unless-stopped" { + for name, container := range r.containers { + if container.ID != args[2] { + continue + } + container.HostConfig.RestartPolicy.Name = "unless-stopped" + r.containers[name] = container + return []byte(args[2] + "\n"), nil + } + } + return nil, fmt.Errorf("unexpected docker command: %s", strings.Join(args, " ")) +} + func TestPlanForCommitEmitsStopPerAppPlusProxyWhenRoutesExist(t *testing.T) { prepare := preparer.Result{Apps: []preparer.AppPlan{{Name: "api"}, {Name: "web"}}} cutover := gateway.Result{Apps: []gateway.AppPlan{{ @@ -170,13 +551,17 @@ func TestApplyStopSourceAppStopsAllSourceContainers(t *testing.T) { app.Resources.DataStores = []preparer.DataStoreResource{ {Service: "db", Kind: "postgres", Strategy: "migrate", SourceContainerID: "db-id"}, } - runner := &fakeDockerRunner{ - outputs: map[string][]byte{ - "inspect --type container web-id": []byte(`[{"Id":"web-id","Name":"/web","State":{"Running":true,"Status":"running"}}]`), - "inspect --type container db-id": []byte(`[{"Id":"db-id","Name":"/db","State":{"Running":true,"Status":"running"}}]`), - "stop web-id": []byte("web-id\n"), - "stop db-id": []byte("db-id\n"), + runner := &statefulTargetRunner{ + fakeDockerRunner: fakeDockerRunner{ + outputs: map[string][]byte{ + "inspect --type container web-id db-id": []byte(`[{"Id":"web-id","Name":"/web","State":{"Running":true,"Status":"running"}},{"Id":"db-id","Name":"/db","State":{"Running":true,"Status":"running"}}]`), + "inspect --type container web-id": []byte(`[{"Id":"web-id","Name":"/web","State":{"Running":true,"Status":"running"}}]`), + "inspect --type container db-id": []byte(`[{"Id":"db-id","Name":"/db","State":{"Running":true,"Status":"running"}}]`), + "stop web-id": []byte("web-id\n"), + "stop db-id": []byte("db-id\n"), + }, }, + stopped: map[string]bool{}, } client := &Client{Docker: runner} actx := &applyContext{cache: map[string]*appCache{}, plan: Plan{Prepare: preparer.Result{Apps: []preparer.AppPlan{app}}}} @@ -185,43 +570,42 @@ func TestApplyStopSourceAppStopsAllSourceContainers(t *testing.T) { } } -func TestApplyStopSourceAppToleratesMissingContainers(t *testing.T) { +func TestApplyStopSourceAppRefusesMissingReviewedContainer(t *testing.T) { app := preparer.AppPlan{Name: "api"} app.Resources.SourceServices = []preparer.SourceServiceRef{ {ServiceName: "web", ContainerID: "web-id"}, } - // inspect for web-id is unstubbed → fakeDockerRunner errors with - // "docker output not stubbed"; that must NOT match the missing- - // container substring set, so the apply errors out as expected. - runner := &fakeDockerRunner{outputs: map[string][]byte{}} + runner := &fakeDockerRunner{ + outputs: map[string][]byte{}, + outputErrs: map[string]error{"inspect --type container web-id": errors.New("Error: No such object: web-id")}, + } client := &Client{Docker: runner} actx := &applyContext{cache: map[string]*appCache{}, plan: Plan{Prepare: preparer.Result{Apps: []preparer.AppPlan{app}}}} err := client.applyStopSourceApp(context.Background(), actx, Step{Kind: StepStopSourceApp, App: "api"}) - if err == nil { - t.Fatal("expected unstubbed inspect error to bubble up") + if err == nil || !strings.Contains(err.Error(), "web-id") { + t.Fatalf("expected missing reviewed source identity refusal, got %v", err) } } -func TestApplyStopSourceAppFallsBackToContainerName(t *testing.T) { - // the recorded ID is stale (container was recreated since scan), but - // the recorded name still resolves; sourceContainer must use the - // name fallback instead of treating the missing-id as success. +func TestApplyStopSourceAppRefusesStaleContainerID(t *testing.T) { app := preparer.AppPlan{Name: "api"} app.Resources.SourceServices = []preparer.SourceServiceRef{ {ServiceName: "web", ContainerID: "stale-id", ContainerName: "coolify-web"}, } - // stale-id is intentionally NOT stubbed — fakeDockerRunner returns - // "docker output not stubbed", which sourceContainer treats as an - // id-inspect failure and falls back to the recorded name. runner := &fakeDockerRunner{ outputs: map[string][]byte{ "inspect --type container coolify-web": []byte(`[{"Id":"fresh-id","Name":"/coolify-web","State":{"Running":true,"Status":"running"}}]`), "stop fresh-id": []byte("fresh-id\n"), }, + outputErrs: map[string]error{"inspect --type container stale-id": errors.New("Error: No such object: stale-id")}, } client := &Client{Docker: runner} actx := &applyContext{cache: map[string]*appCache{}, plan: Plan{Prepare: preparer.Result{Apps: []preparer.AppPlan{app}}}} - if err := client.applyStopSourceApp(context.Background(), actx, Step{Kind: StepStopSourceApp, App: "api"}); err != nil { - t.Fatalf("applyStopSourceApp: %v", err) + err := client.applyStopSourceApp(context.Background(), actx, Step{Kind: StepStopSourceApp, App: "api"}) + if err == nil || !strings.Contains(err.Error(), "stale-id") { + t.Fatalf("expected stale source identity refusal, got %v", err) + } + if fakeOutputCalled(runner, "stop", "fresh-id") { + t.Fatalf("stale identity stopped a replacement container, calls=%v", runner.outputArgs) } } diff --git a/internal/target/dokploy/resume.go b/internal/target/dokploy/resume.go new file mode 100644 index 0000000..4d10d5b --- /dev/null +++ b/internal/target/dokploy/resume.go @@ -0,0 +1,110 @@ +package dokploy + +import ( + "context" + "fmt" + "strings" + + "github.com/aikins01/bort/internal/preparer" +) + +func (c *Client) loadPersistedTargetCompose(ctx context.Context, actx *applyContext, app preparer.AppPlan, identity TargetIdentity) error { + compose, err := c.GetCompose(ctx, identity.ComposeID) + if err != nil { + return fmt.Errorf("load persisted target compose %s for app %s: %w", identity.ComposeID, app.Name, err) + } + if compose.ComposeID != identity.ComposeID { + return fmt.Errorf("persisted target compose %s for app %s resolved to unexpected compose %s", identity.ComposeID, app.Name, compose.ComposeID) + } + if identity.EnvironmentID != "" && compose.EnvironmentID != "" && compose.EnvironmentID != identity.EnvironmentID { + return fmt.Errorf("persisted target compose %s for app %s moved from environment %s to %s", identity.ComposeID, app.Name, identity.EnvironmentID, compose.EnvironmentID) + } + if identity.ComposeAppName != "" && compose.AppName != "" && compose.AppName != identity.ComposeAppName { + return fmt.Errorf("persisted target compose %s for app %s changed Docker project identity from %s to %s", identity.ComposeID, app.Name, identity.ComposeAppName, compose.AppName) + } + composeAppName := strings.TrimSpace(compose.AppName) + if composeAppName == "" { + composeAppName = identity.ComposeAppName + } + if composeAppName == "" { + return fmt.Errorf("persisted target compose %s for app %s has no Docker project identity", identity.ComposeID, app.Name) + } + entry := actx.entry(app.Name) + entry.ProjectID = identity.ProjectID + entry.EnvironmentID = identity.EnvironmentID + entry.ComposeID = identity.ComposeID + entry.ComposeAppName = composeAppName + return nil +} + +func (c *Client) hydratePersistedTargetIdentities(ctx context.Context, actx *applyContext) error { + projects := map[string]*Project{} + for _, app := range actx.plan.Prepare.Apps { + identity := actx.plan.TargetIdentities[app.Name] + if identity.ProjectID == "" { + if identity.ComposeID != "" || identity.EnvironmentID != "" || identity.ComposeAppName != "" { + return fmt.Errorf("persisted target identity for app %s has no project ID", app.Name) + } + continue + } + project := projects[identity.ProjectID] + if project == nil { + var err error + project, err = c.verifyPersistedProjectEnvironment(ctx, app.Name, identity) + if err != nil { + return err + } + projects[identity.ProjectID] = project + } else if err := verifyPersistedProjectEnvironment(app.Name, identity, project); err != nil { + return err + } + if identity.ComposeID != "" { + if app.TargetResources == nil || app.TargetResources.Dokploy == nil { + return fmt.Errorf("target app %s has no reviewed Dokploy identity", app.Name) + } + if err := c.loadPersistedTargetCompose(ctx, actx, app, identity); err != nil { + return err + } + continue + } + entry := actx.entry(app.Name) + entry.ProjectID = identity.ProjectID + if identity.EnvironmentID != "" { + entry.EnvironmentID = identity.EnvironmentID + continue + } + _, environmentName := dokployProjectSelection(actx.plan, app.Name, app.Name) + environment := FindEnvironmentInProject(project, environmentName) + if environment == nil { + return fmt.Errorf("persisted target project %s for app %s has no environments", identity.ProjectID, app.Name) + } + entry.EnvironmentID = environment.EnvironmentID + } + return nil +} + +func (c *Client) verifyPersistedProjectEnvironment(ctx context.Context, appName string, identity TargetIdentity) (*Project, error) { + project, err := c.GetProject(ctx, identity.ProjectID) + if err != nil { + return nil, fmt.Errorf("load persisted target project %s for app %s: %w", identity.ProjectID, appName, err) + } + if err := verifyPersistedProjectEnvironment(appName, identity, project); err != nil { + return nil, err + } + return project, nil +} + +func verifyPersistedProjectEnvironment(appName string, identity TargetIdentity, project *Project) error { + if project.ProjectID != identity.ProjectID { + return fmt.Errorf("persisted target project %s for app %s resolved to unexpected project %s", identity.ProjectID, appName, project.ProjectID) + } + if identity.EnvironmentID == "" { + return nil + } + for _, environment := range project.Environments { + if environment.EnvironmentID == identity.EnvironmentID { + return nil + } + } + return fmt.Errorf("persisted target environment %s for app %s no longer belongs to project %s", identity.EnvironmentID, appName, identity.ProjectID) +} diff --git a/internal/target/dokploy/rollback.go b/internal/target/dokploy/rollback.go index 17f699d..352eee7 100644 --- a/internal/target/dokploy/rollback.go +++ b/internal/target/dokploy/rollback.go @@ -3,10 +3,12 @@ package dokploy import ( "context" "fmt" + "strings" "time" "github.com/aikins01/bort/internal/gateway" "github.com/aikins01/bort/internal/preparer" + syncplan "github.com/aikins01/bort/internal/sync" ) var rollbackObserveSettle = 5 * time.Second @@ -14,31 +16,43 @@ var rollbackObserveSettle = 5 * time.Second var sourceHealthPollInterval = 2 * time.Second // PlanForRollback builds the steps that return traffic to the source. -func PlanForRollback(prepare preparer.Result, cutover gateway.Result) Plan { +func PlanForRollback(prepare preparer.Result, sync syncplan.Result, cutover gateway.Result) (Plan, error) { plan := Plan{Prepare: prepare, Cutover: cutover, stepTimeout: dockerStartTimeout} - quiescedApps := []preparer.AppPlan{} - for _, app := range prepare.Apps { - if len(sourceQuiesceTargetRefs(app)) == 0 { + statefulApps := []string{} + seen := map[string]struct{}{} + livePlan := PlanFromArtifacts(prepare, sync, cutover) + for _, step := range livePlan.Steps { + if step.Kind != StepPauseSource || shouldSkipApplyStep(livePlan, step) { continue } - quiescedApps = append(quiescedApps, app) - } - for _, app := range quiescedApps { - plan.Steps = append(plan.Steps, Step{Kind: StepResumeSource, App: app.Name, Ref: app.Name}) + if _, exists := seen[step.App]; exists { + continue + } + seen[step.App] = struct{}{} + statefulApps = append(statefulApps, step.App) } - for _, app := range quiescedApps { - plan.Steps = append(plan.Steps, Step{Kind: StepVerifySourceHealth, App: app.Name, Ref: app.Name}) + if len(statefulApps) > 0 { + return Plan{}, fmt.Errorf("automatic rollback is unsafe for stateful app(s) %s: Dokploy has no durable application fence that prevents a queued or future deployment from restarting target writers after source writers resume; preserve both sides and establish writer and traffic authority manually", strings.Join(statefulApps, ", ")) } if cutoverPlanHasRoutes(cutover) { + statelessApps := []preparer.AppPlan{} + for _, app := range prepare.Apps { + if len(sourceQuiesceTargetRefs(app)) > 0 { + statelessApps = append(statelessApps, app) + } + } + for _, app := range statelessApps { + plan.Steps = append(plan.Steps, Step{Kind: StepVerifySourceHealth, App: app.Name, Ref: app.Name}) + } plan.Steps = append(plan.Steps, Step{Kind: StepStopDokployProxy, Ref: dokployProxyContainer}, Step{Kind: StepStartCoolifyProxy, Ref: coolifyProxyContainer}, ) - for _, app := range quiescedApps { + for _, app := range statelessApps { plan.Steps = append(plan.Steps, Step{Kind: StepObserveRollback, App: app.Name, Ref: app.Name}) } } - return plan + return plan, nil } func (c *Client) applyVerifySourceHealth(ctx context.Context, actx *applyContext, step Step) error { diff --git a/internal/target/dokploy/rollback_test.go b/internal/target/dokploy/rollback_test.go index 4001eeb..50fb064 100644 --- a/internal/target/dokploy/rollback_test.go +++ b/internal/target/dokploy/rollback_test.go @@ -3,7 +3,6 @@ package dokploy import ( "context" "errors" - "fmt" "io" "strings" "testing" @@ -11,6 +10,7 @@ import ( "github.com/aikins01/bort/internal/gateway" "github.com/aikins01/bort/internal/preparer" + syncplan "github.com/aikins01/bort/internal/sync" ) func rollbackPrepareFixture() preparer.Result { @@ -36,86 +36,75 @@ func stepKinds(steps []Step) []StepKind { return kinds } -func TestPlanForRollbackOrdersSteps(t *testing.T) { - plan := PlanForRollback(rollbackPrepareFixture(), rollbackCutoverFixture()) - want := []StepKind{ - StepResumeSource, - StepVerifySourceHealth, - StepStopDokployProxy, - StepStartCoolifyProxy, - StepObserveRollback, - } - got := stepKinds(plan.Steps) - if len(got) != len(want) { - t.Fatalf("expected steps %v, got %v", want, got) - } - for i := range want { - if got[i] != want[i] { - t.Fatalf("expected steps %v, got %v", want, got) - } +func TestPlanForRollbackRefusesStatefulAppsWithoutDurableTargetFence(t *testing.T) { + sync := syncplan.Result{Apps: []syncplan.AppPlan{{ + Name: "api", + Steps: []syncplan.Step{{ + ResourceType: "volume", + ResourceRef: "volume:web -> /data", + Strategy: syncplan.StrategyDockerVolumeArchive, + }}, + }}} + plan, err := PlanForRollback(rollbackPrepareFixture(), sync, rollbackCutoverFixture()) + if err == nil || !strings.Contains(err.Error(), "no durable application fence") || !strings.Contains(err.Error(), "api") { + t.Fatalf("expected stateful rollback refusal, got plan=%#v err=%v", plan, err) } - for _, step := range plan.Steps { - if step.Kind == StepVerifySourceHealth || step.Kind == StepObserveRollback { - if step.App != "api" { - t.Fatalf("expected app-scoped step for api, got %v", step) - } - } + if len(plan.Steps) != 0 { + t.Fatalf("unsafe stateful rollback returned executable steps: %#v", plan.Steps) } } -func TestPlanForRollbackSkipsProxySwapWithoutRoutes(t *testing.T) { - plan := PlanForRollback(rollbackPrepareFixture(), gateway.Result{}) - for _, step := range plan.Steps { - switch step.Kind { - case StepStopDokployProxy, StepStartCoolifyProxy, StepObserveRollback: - t.Fatalf("did not expect proxy swap or observation without routes, got %v", plan.Steps) - } +func TestPlanForRollbackReturnsTrafficOnlyForStatelessApps(t *testing.T) { + plan, err := PlanForRollback(rollbackPrepareFixture(), syncplan.Result{}, rollbackCutoverFixture()) + if err != nil { + t.Fatal(err) } got := stepKinds(plan.Steps) - if len(got) != 2 || got[0] != StepResumeSource || got[1] != StepVerifySourceHealth { - t.Fatalf("expected resume and verify only, got %v", got) - } -} - -type dependentSourceRunner struct { - fakeDockerRunner - running map[string]bool -} - -func (r *dependentSourceRunner) Output(_ context.Context, args ...string) ([]byte, error) { - if args[0] == "start" { - r.running[args[1]] = true - return nil, nil + want := []StepKind{StepVerifySourceHealth, StepStopDokployProxy, StepStartCoolifyProxy, StepObserveRollback} + if len(got) != len(want) { + t.Fatalf("expected stateless traffic rollback %v, got %v", want, got) } - id := args[len(args)-1] - health := "healthy" - if id == "api-web-id" && !r.running["worker-id"] { - health = "unhealthy" + for index := range want { + if got[index] != want[index] { + t.Fatalf("expected stateless traffic rollback %v, got %v", want, got) + } } - return []byte(fmt.Sprintf(`[{"Id":%q,"State":{"Running":%t,"Health":{"Status":%q}}}]`, id, r.running[id], health)), nil } -func TestRollbackStartsDependenciesBeforeCheckingHealth(t *testing.T) { +func TestPlanForRollbackIgnoresSkippedPlatformState(t *testing.T) { prepare := rollbackPrepareFixture() - worker := preparer.AppPlan{Name: "worker"} - worker.Resources.SourceServices = []preparer.SourceServiceRef{{ServiceName: "worker", ContainerID: "worker-id"}} - prepare.Apps = append(prepare.Apps, worker) - runner := &dependentSourceRunner{running: map[string]bool{}} - client := &Client{Docker: runner} - if err := client.Apply(context.Background(), PlanForRollback(prepare, gateway.Result{})); err != nil { - t.Fatalf("expected dependent apps to resume before health checks: %v", err) + prepare.Apps = append(prepare.Apps, preparer.AppPlan{Name: "coolify-proxy", Role: "platform"}) + sync := syncplan.Result{Apps: []syncplan.AppPlan{{ + Name: "coolify-proxy", + Steps: []syncplan.Step{{ + ResourceType: "volume", + ResourceRef: "volume:proxy -> /data", + Strategy: syncplan.StrategyDockerVolumeArchive, + }}, + }}} + plan, err := PlanForRollback(prepare, sync, rollbackCutoverFixture()) + if err != nil { + t.Fatal(err) } - if !runner.running["api-web-id"] || !runner.running["worker-id"] { - t.Fatalf("expected both sources running, got %v", runner.running) + got := stepKinds(plan.Steps) + want := []StepKind{StepVerifySourceHealth, StepStopDokployProxy, StepStartCoolifyProxy, StepObserveRollback} + if len(got) != len(want) { + t.Fatalf("expected skipped platform state to retain stateless rollback %v, got %v", want, got) + } + for index := range want { + if got[index] != want[index] { + t.Fatalf("expected skipped platform state to retain stateless rollback %v, got %v", want, got) + } } } -func TestPlanForRollbackSkimsAppsWithoutQuiesceTargets(t *testing.T) { - prepare := preparer.Result{Apps: []preparer.AppPlan{{Name: "assets"}}} - plan := PlanForRollback(prepare, rollbackCutoverFixture()) - got := stepKinds(plan.Steps) - if len(got) != 2 || got[0] != StepStopDokployProxy || got[1] != StepStartCoolifyProxy { - t.Fatalf("expected proxy swap only for a run that stopped no source containers, got %v", got) +func TestPlanForRollbackSkipsProxySwapWithoutRoutes(t *testing.T) { + plan, err := PlanForRollback(rollbackPrepareFixture(), syncplan.Result{}, gateway.Result{}) + if err != nil { + t.Fatal(err) + } + if len(plan.Steps) != 0 { + t.Fatalf("expected no rollback steps without state transfer or routes, got %v", plan.Steps) } } @@ -171,7 +160,10 @@ func TestRollbackBoundsEveryStepInspection(t *testing.T) { oldSettle := rollbackObserveSettle rollbackObserveSettle = 0 t.Cleanup(func() { rollbackObserveSettle = oldSettle }) - plan := PlanForRollback(rollbackPrepareFixture(), rollbackCutoverFixture()) + plan, err := PlanForRollback(rollbackPrepareFixture(), syncplan.Result{}, rollbackCutoverFixture()) + if err != nil { + t.Fatal(err) + } if plan.stepTimeout != dockerStartTimeout { t.Fatalf("expected bounded rollback steps, got %s", plan.stepTimeout) } diff --git a/internal/target/dokploy/source.go b/internal/target/dokploy/source.go index c4d306a..4a775de 100644 --- a/internal/target/dokploy/source.go +++ b/internal/target/dokploy/source.go @@ -2,78 +2,383 @@ package dokploy import ( "context" + "encoding/json" "fmt" + "os" + "path/filepath" + "sort" "strings" "github.com/aikins01/bort/internal/preparer" + "github.com/aikins01/bort/internal/safepath" ) -// applyPauseSource stops the source containers belonging to non-data-store -// services of the app so the upcoming volume/bind sync sees a quiesced -// filesystem. data-store containers stay running here; logical dumps -// already ran in earlier steps, and pausing the database service is left -// to a later step (or to source cleanup on commit). func (c *Client) applyPauseSource(ctx context.Context, actx *applyContext, step Step) error { app, ok := findPrepareApp(actx.plan.Prepare, step.App) if !ok { return fmt.Errorf("app %s not found in prepare result", step.App) } - refs := sourceQuiesceTargetRefs(app) - if len(refs) == 0 { - return nil - } runner := c.dockerRunner() - for _, ref := range refs { - container, err := sourceContainerForQuiesce(ctx, runner, ref) + containers, err := inspectSourceQuiesceTargets(ctx, runner, app) + if err != nil { + return err + } + entry := actx.entry(step.App) + recorded, err := entry.pausedSourceContainers(step.App, containers) + if err != nil { + return err + } + toStop := []string{} + changed := !entry.SourcePauseRecorded + for _, container := range containers { + if !container.State.Running { + continue + } + toStop = append(toStop, container.ID) + for i := range recorded { + if recorded[i].ID == container.ID && !recorded[i].Stopped { + recorded[i].Stopped = true + changed = true + } + } + } + if changed { + previous, previouslyRecorded := entry.SourcePausedContainers, entry.SourcePauseRecorded + entry.SourcePausedContainers = recorded + entry.SourcePauseRecorded = true + if err := actx.persistSourcePauseState(); err != nil { + entry.SourcePausedContainers = previous + entry.SourcePauseRecorded = previouslyRecorded + return unsafeSourceResumeError{err: fmt.Errorf("record source container ownership before pause: %w", err)} + } + } + for _, id := range toStop { + if err := stopContainer(ctx, runner, id); err != nil { + return fmt.Errorf("stop source container %s: %w", id, err) + } + } + if len(toStop) > 0 { + containers, err = inspectSourceQuiesceTargets(ctx, runner, app) if err != nil { - return fmt.Errorf("inspect source container %s: %w", ref.label(), err) + return fmt.Errorf("verify paused source containers for app %s: %w", step.App, err) } - if !container.State.Running { + } + byID := make(map[string]dockerContainer, len(containers)) + for _, container := range containers { + byID[container.ID] = container + } + changed = false + for i, paused := range entry.SourcePausedContainers { + container, found := byID[paused.ID] + if !found { + return unsafeSourceResumeError{err: fmt.Errorf("source container %s for app %s disappeared while pausing; refusing to record its stopped state", paused.ID, step.App)} + } + if paused.StartedAt == container.State.StartedAt && paused.FinishedAt == container.State.FinishedAt { continue } - if err := stopContainer(ctx, runner, container.ID); err != nil { - return fmt.Errorf("stop source container %s: %w", container.ID, err) + entry.SourcePausedContainers[i].StartedAt = container.State.StartedAt + entry.SourcePausedContainers[i].FinishedAt = container.State.FinishedAt + changed = true + } + if changed { + if err := actx.persistSourcePauseState(); err != nil { + return fmt.Errorf("record paused source container state for app %s: %w", step.App, err) } } return nil } -// applyResumeSource restarts the source containers that the quiesce step -// owns. it is stateless: rollback may run in a fresh process where the -// in-memory PausedContainers cache is empty, so we re-derive targets from -// the prepare result and only start what is currently stopped. +// pausedSourceContainers returns the durable pause records for the +// inspected quiesce targets: a fresh snapshot of every target on the +// first pause, or the recorded set once it exists. Any mismatch between +// the two means the evidence for a safe retry is gone. +func (e *appCache) pausedSourceContainers(appName string, containers []dockerContainer) ([]sourcePausedContainer, error) { + if !e.SourcePauseRecorded { + recorded := make([]sourcePausedContainer, 0, len(containers)) + for _, container := range containers { + recorded = append(recorded, sourcePausedContainer{ID: container.ID}) + } + sortSourcePausedContainers(recorded) + return recorded, nil + } + byID := make(map[string]struct{}, len(containers)) + for _, container := range containers { + byID[container.ID] = struct{}{} + } + recorded := make(map[string]struct{}, len(e.SourcePausedContainers)) + for _, paused := range e.SourcePausedContainers { + recorded[paused.ID] = struct{}{} + if _, found := byID[paused.ID]; !found { + return nil, unsafeSourceResumeError{err: fmt.Errorf("durably recorded source container %s for app %s no longer matches the reviewed source; refusing to infer pause ownership", paused.ID, appName)} + } + } + for _, container := range containers { + if _, found := recorded[container.ID]; !found { + return nil, unsafeSourceResumeError{err: fmt.Errorf("source container %s for app %s has no durable pause evidence; refusing to infer whether it ran since the pause", container.ID, appName)} + } + } + return append([]sourcePausedContainer(nil), e.SourcePausedContainers...), nil +} + +// sourceRanSincePause reports whether any quiesce target of the app ran +// after the recorded pause, which invalidates state transferred while +// the app was paused. +func (c *Client) sourceRanSincePause(ctx context.Context, actx *applyContext, appName string) (bool, error) { + ran, err := c.firstSourceRanSincePause(ctx, actx, []string{appName}) + return ran != "", err +} + +// firstSourceRanSincePause inspects the quiesce targets of all apps in one +// Docker call and returns the first app, in the given order, that ran. +func (c *Client) firstSourceRanSincePause(ctx context.Context, actx *applyContext, appNames []string) (string, error) { + var refs []commitTargetRef + counts := make([]int, len(appNames)) + for i, appName := range appNames { + app, ok := findPrepareApp(actx.plan.Prepare, appName) + if !ok { + return "", fmt.Errorf("app %s not found in prepare result", appName) + } + appRefs := sourceQuiesceTargetRefs(app) + counts[i] = len(appRefs) + refs = append(refs, appRefs...) + } + inspected, err := inspectSourceCommitTargets(ctx, c.dockerRunner(), refs) + if err != nil { + return "", err + } + offset := 0 + for i, appName := range appNames { + containers := inspected[offset : offset+counts[i]] + offset += counts[i] + recorded, err := actx.entry(appName).pausedSourceContainers(appName, containers) + if err != nil { + return "", err + } + byID := make(map[string]dockerContainer, len(containers)) + for _, container := range containers { + byID[container.ID] = container + } + for _, paused := range recorded { + container := byID[paused.ID] + if container.State.Running || paused.StartedAt != container.State.StartedAt || paused.FinishedAt != container.State.FinishedAt { + return appName, nil + } + } + } + return "", nil +} + func (c *Client) applyResumeSource(ctx context.Context, actx *applyContext, step Step) error { app, ok := findPrepareApp(actx.plan.Prepare, step.App) if !ok { return fmt.Errorf("app %s not found in prepare result", step.App) } - refs := sourceQuiesceTargetRefs(app) - if len(refs) == 0 { - return nil + entry := actx.entry(step.App) + if !entry.SourcePauseRecorded { + return unsafeSourceResumeError{err: fmt.Errorf("source pause ownership for app %s was not durably recorded; refusing to start stopped containers", step.App)} } runner := c.dockerRunner() - for _, ref := range refs { - container, err := sourceContainerForQuiesce(ctx, runner, ref) - if err != nil { - return fmt.Errorf("inspect source container %s: %w", ref.label(), err) + containers, err := inspectSourceQuiesceTargets(ctx, runner, app) + if err != nil { + return err + } + byID := make(map[string]dockerContainer, len(containers)) + for _, container := range containers { + byID[container.ID] = container + } + for _, paused := range entry.SourcePausedContainers { + if !paused.Stopped { + continue + } + container, found := byID[paused.ID] + if !found { + return unsafeSourceResumeError{err: fmt.Errorf("durably owned source container %s for app %s no longer matches the reviewed source; refusing to start it", paused.ID, step.App)} } if container.State.Running { continue } - if err := startContainer(ctx, runner, container.ID); err != nil { - return fmt.Errorf("start source container %s: %w", container.ID, err) + if err := startContainer(ctx, runner, paused.ID); err != nil { + return fmt.Errorf("start source container %s: %w", paused.ID, err) } } return nil } -// applyStopSourceApp is the commit-time stop: it shuts down every source -// container belonging to the app, including data stores excluded from the -// cutover quiesce step (logical-dump postgres, skip-strategy stores). the -// stop is idempotent — a missing or already-stopped container is a no-op -// — so commit can run safely even on a partially-cleaned host. containers -// are not removed: operators in a long rollback window can docker start -// them back up to roll back outside bort. +// AdoptHistoricalSourcePause records the reviewed quiesce targets of apps +// whose pause predates durable pause records. Bort cannot tell which of +// them it stopped, so it claims none: a stopped target must be started by +// the operator before the historical pause can be recorded as resumed. +// Apps with a record are left as is. +func (c *Client) AdoptHistoricalSourcePause(ctx context.Context, plan Plan, apps []string) error { + actx := &applyContext{plan: plan, cache: map[string]*appCache{}} + if err := actx.loadSourcePauseState(); err != nil { + return err + } + runner := c.dockerRunner() + changed := false + for _, name := range apps { + entry := actx.entry(name) + if entry.SourcePauseRecorded { + continue + } + app, ok := findPrepareApp(plan.Prepare, name) + if !ok { + return fmt.Errorf("app %s not found in prepare result", name) + } + containers, err := inspectSourceQuiesceTargets(ctx, runner, app) + if err != nil { + return err + } + recorded := make([]sourcePausedContainer, 0, len(containers)) + stopped := []string{} + for _, container := range containers { + if !container.State.Running { + stopped = append(stopped, container.ID) + } + recorded = append(recorded, sourcePausedContainer{ + ID: container.ID, + StartedAt: container.State.StartedAt, + FinishedAt: container.State.FinishedAt, + }) + } + if len(stopped) > 0 { + return unsafeSourceResumeError{err: fmt.Errorf("source container(s) %s for app %s are stopped, but this run predates durable pause records so Bort cannot prove it stopped them; verify they should run, start them with `docker start %s`, then retry", strings.Join(stopped, ", "), name, strings.Join(stopped, " "))} + } + sortSourcePausedContainers(recorded) + entry.SourcePausedContainers = recorded + entry.SourcePauseRecorded = true + changed = true + } + if !changed { + return nil + } + return actx.persistSourcePauseState() +} + +func (c *Client) FinalizeSourceResumes(plan Plan) error { + resumedApps := map[string]struct{}{} + for _, step := range plan.Steps { + if step.Kind == StepResumeSource { + resumedApps[step.App] = struct{}{} + } + } + if len(resumedApps) == 0 { + return nil + } + actx := &applyContext{plan: plan, cache: map[string]*appCache{}} + if err := actx.loadSourcePauseState(); err != nil { + return err + } + for app := range resumedApps { + entry := actx.entry(app) + entry.SourcePausedContainers = nil + entry.SourcePauseRecorded = false + } + return actx.persistSourcePauseState() +} + +func inspectSourceQuiesceTargets(ctx context.Context, runner dockerRunner, app preparer.AppPlan) ([]dockerContainer, error) { + return inspectSourceCommitTargets(ctx, runner, sourceQuiesceTargetRefs(app)) +} + +const sourcePauseStateArtifact = "source-pause.json" +const sourcePauseStateAPIVersion = "bort.source-pause/v1alpha1" + +type sourcePauseState struct { + APIVersion string `json:"apiVersion"` + Apps map[string][]sourcePausedContainer `json:"apps"` +} + +// sourcePausedContainer records a quiesce target of a paused app: whether +// Bort stopped it (and so may restart it) and the run timestamps observed +// after the pause, so a retry can tell whether it ran again in between. +type sourcePausedContainer struct { + ID string `json:"id"` + Stopped bool `json:"stopped,omitempty"` + StartedAt string `json:"startedAt,omitempty"` + FinishedAt string `json:"finishedAt,omitempty"` +} + +func sortSourcePausedContainers(containers []sourcePausedContainer) { + sort.Slice(containers, func(i, j int) bool { return containers[i].ID < containers[j].ID }) +} + +func (a *applyContext) persistSourcePauseState() error { + if a == nil || strings.TrimSpace(a.plan.RunDir) == "" { + return nil + } + state := sourcePauseState{APIVersion: sourcePauseStateAPIVersion, Apps: map[string][]sourcePausedContainer{}} + for app, entry := range a.cache { + if !entry.SourcePauseRecorded { + continue + } + containers := append([]sourcePausedContainer(nil), entry.SourcePausedContainers...) + sortSourcePausedContainers(containers) + state.Apps[app] = containers + } + contents, err := json.MarshalIndent(state, "", " ") + if err != nil { + return fmt.Errorf("encode source pause state: %w", err) + } + path, err := sourcePauseStatePath(a.plan.RunDir) + if err != nil { + return err + } + if err := safepath.WriteFileAtomicNoFollow(path, append(contents, '\n'), 0o600); err != nil { + return fmt.Errorf("write source pause state: %w", err) + } + return nil +} + +func (a *applyContext) loadSourcePauseState() error { + if a == nil || strings.TrimSpace(a.plan.RunDir) == "" { + return nil + } + path, err := sourcePauseStatePath(a.plan.RunDir) + if err != nil { + return err + } + contents, err := safepath.ReadFileNoFollow(path) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return fmt.Errorf("read source pause state: %w", err) + } + var state sourcePauseState + if err := json.Unmarshal(contents, &state); err != nil { + return fmt.Errorf("decode source pause state: %w", err) + } + if state.APIVersion != sourcePauseStateAPIVersion || state.Apps == nil { + return fmt.Errorf("%s has invalid source pause state", path) + } + for app, containers := range state.Apps { + seen := map[string]struct{}{} + for _, container := range containers { + if strings.TrimSpace(container.ID) == "" { + return fmt.Errorf("%s contains an empty source container identity for app %s", path, app) + } + if _, duplicate := seen[container.ID]; duplicate { + return fmt.Errorf("%s contains duplicate source container identity %s for app %s", path, container.ID, app) + } + seen[container.ID] = struct{}{} + } + entry := a.entry(app) + entry.SourcePausedContainers = append([]sourcePausedContainer(nil), containers...) + entry.SourcePauseRecorded = true + } + return nil +} + +func sourcePauseStatePath(runDir string) (string, error) { + path := filepath.Join(runDir, sourcePauseStateArtifact) + if err := safepath.ContainedPath(runDir, path); err != nil { + return "", err + } + return path, nil +} + +// Reviewed IDs never fall back to names; only legacy name-only refs tolerate absence. func (c *Client) applyStopSourceApp(ctx context.Context, actx *applyContext, step Step) error { app, ok := findPrepareApp(actx.plan.Prepare, step.App) if !ok { @@ -84,30 +389,94 @@ func (c *Client) applyStopSourceApp(ctx context.Context, actx *applyContext, ste return nil } runner := c.dockerRunner() - for _, ref := range refs { - container, err := sourceContainer(ctx, runner, ref.id, ref.name) - if err != nil { - // recreated source containers can change ID, so we already - // fell back to the recorded name above. anything still - // missing here is gone for real and counts as stopped. - if isContainerMissingErr(err) { - continue + containers, err := inspectSourceCommitTargets(ctx, runner, refs) + if err != nil { + return err + } + for _, container := range containers { + if normalizedRestartPolicyName(container.HostConfig.RestartPolicy.Name) != "no" { + if err := updateContainerRestartPolicy(ctx, runner, container.ID, "no"); err != nil { + return fmt.Errorf("fence retired source container %s restarts: %w", container.ID, err) } - return fmt.Errorf("inspect source container %s: %w", ref.label(), err) } if !container.State.Running { continue } if err := stopContainer(ctx, runner, container.ID); err != nil { - if isContainerMissingErr(err) { - continue - } return fmt.Errorf("stop source container %s: %w", container.ID, err) } } + containers, err = inspectSourceCommitTargets(ctx, runner, refs) + if err != nil { + return fmt.Errorf("verify retired source containers: %w", err) + } + for _, container := range containers { + if container.State.Running { + return fmt.Errorf("source container %s is still running after retirement", container.ID) + } + if normalizedRestartPolicyName(container.HostConfig.RestartPolicy.Name) != "no" { + return fmt.Errorf("source container %s restart policy is not fenced after retirement", container.ID) + } + } return nil } +func inspectSourceCommitTargets(ctx context.Context, runner dockerRunner, refs []commitTargetRef) ([]dockerContainer, error) { + labels := make([]string, 0, len(refs)) + for _, ref := range refs { + labels = append(labels, ref.label()) + } + inspected, err := inspectContainers(ctx, runner, labels) + if err != nil { + inspected = nil + for _, ref := range refs { + container, inspectErr := sourceContainer(ctx, runner, ref.id, ref.name) + if inspectErr != nil { + return nil, fmt.Errorf("inspect source container %s: %w", ref.label(), inspectErr) + } + inspected = append(inspected, container) + } + return inspected, nil + } + ordered := make([]dockerContainer, 0, len(refs)) + for _, ref := range refs { + found := false + for _, container := range inspected { + matches := ref.id != "" && sourceContainerIDMatches(ref.id, container.ID) + if ref.id == "" { + matches = strings.TrimPrefix(ref.name, "/") == strings.TrimPrefix(container.Name, "/") + } + if !matches { + continue + } + if ref.name != "" && strings.TrimPrefix(ref.name, "/") != strings.TrimPrefix(container.Name, "/") { + return nil, fmt.Errorf("source container %s has name %q, want reviewed name %q", ref.id, container.Name, ref.name) + } + ordered = append(ordered, container) + found = true + break + } + if !found { + return nil, fmt.Errorf("source container %s was absent from batch inspection", ref.label()) + } + } + return ordered, nil +} + +func normalizedRestartPolicyName(name string) string { + name = strings.ToLower(strings.TrimSpace(name)) + if name == "" { + return "no" + } + return name +} + +func updateContainerRestartPolicy(ctx context.Context, runner dockerRunner, containerID, policy string) error { + name := normalizedRestartPolicyName(policy) + _, err := runner.Output(ctx, "update", "--restart="+name, containerID) + return err +} + type commitTargetRef struct { id string name string @@ -125,29 +494,35 @@ func (r commitTargetRef) label() string { // app — workers, web services, and data stores all included. unlike // sourceQuiesceTargets which leaves logical-dump and skip-strategy stores // running so pg_dump can read them, commit cleanup must stop everything -// because the migration is over. id and name are kept separately so a -// recreated container (id changed since scan) can still be stopped via -// its stable name. +// because the migration is over. func sourceCommitTargets(app preparer.AppPlan) []commitTargetRef { seenID := map[string]struct{}{} seenName := map[string]struct{}{} refs := []commitTargetRef{} - add := func(id, name string) { - if id == "" && name == "" { - return - } + for _, ref := range sourceCommitTargetRefs(app) { + id, name := ref.id, ref.name if id != "" { if _, dup := seenID[id]; dup { - return + continue } seenID[id] = struct{}{} } else { if _, dup := seenName[name]; dup { - return + continue } seenName[name] = struct{}{} } - refs = append(refs, commitTargetRef{id: id, name: name}) + refs = append(refs, ref) + } + return refs +} + +func sourceCommitTargetRefs(app preparer.AppPlan) []commitTargetRef { + refs := []commitTargetRef{} + add := func(id, name string) { + if id != "" || name != "" { + refs = append(refs, commitTargetRef{id: id, name: name}) + } } for _, service := range app.Resources.SourceServices { add(service.ContainerID, service.ContainerName) @@ -162,77 +537,115 @@ func sourceCommitTargets(app preparer.AppPlan) []commitTargetRef { } func sourceContainerForQuiesce(ctx context.Context, runner dockerRunner, ref commitTargetRef) (dockerContainer, error) { - container, err := sourceContainer(ctx, runner, ref.id, ref.name) - if err == nil { - return container, nil - } - project := composeProjectFromCoolifyContainerName(ref.name, ref.service) - if project == "" { - return dockerContainer{}, err - } - container, ok, listErr := findComposeServiceContainerByProject(ctx, runner, project, ref.service) - if listErr != nil { - return dockerContainer{}, err - } - if ok { - return container, nil - } - return dockerContainer{}, err + return sourceContainer(ctx, runner, ref.id, ref.name) } -func findComposeServiceContainerByProject(ctx context.Context, runner dockerRunner, project, service string) (dockerContainer, bool, error) { - if project == "" || service == "" { - return dockerContainer{}, false, nil - } - out, err := runner.Output(ctx, "ps", "-a", - "--filter", "label=com.docker.compose.project="+project, - "--filter", "label="+composeServiceLabel+"="+service, - "--format", "{{.ID}}", - ) - if err != nil { - return dockerContainer{}, false, err - } - var fallback dockerContainer - haveFallback := false - for _, line := range strings.Split(string(out), "\n") { - id := strings.TrimSpace(line) - if id == "" { +func VerifySourceContainers(ctx context.Context, apps []preparer.AppPlan) error { + return verifySourceContainers(ctx, localDockerRunner{}, apps) +} + +type sourceContainerAttestation struct { + app string + ref commitTargetRef +} + +func collectSourceContainerAttestations(apps []preparer.AppPlan) ([]sourceContainerAttestation, error) { + seen := map[string]sourceContainerAttestation{} + attestations := []sourceContainerAttestation{} + + for _, app := range apps { + if isPlatformAppRole(app.Role) { continue } - container, err := inspectContainer(ctx, runner, id) - if err != nil { - if isContainerMissingErr(err) { + refs := sourceCommitTargetRefs(app) + if len(refs) == 0 { + return nil, fmt.Errorf("app %s has no reviewed source container identities; refresh the run from a local Docker scan before live work", app.Name) + } + for _, ref := range refs { + if ref.id == "" { + return nil, fmt.Errorf("source service %s for app %s has no stable container ID; refresh the run from a local Docker scan before live work", ref.label(), app.Name) + } + if ref.name == "" { + return nil, fmt.Errorf("source container %s for app %s has no reviewed container name; refresh the run from a local Docker scan before live work", ref.id, app.Name) + } + attestation := sourceContainerAttestation{app: app.Name, ref: ref} + if existing, ok := seen[ref.id]; ok { + if strings.TrimPrefix(existing.ref.name, "/") != strings.TrimPrefix(ref.name, "/") { + return nil, fmt.Errorf("reviewed source container %s has conflicting names %q and %q", ref.id, existing.ref.name, ref.name) + } continue } - return dockerContainer{}, false, err - } - if container.State.Running { - return container, true, nil + seen[ref.id] = attestation + attestations = append(attestations, attestation) } - if !haveFallback { - fallback = container - haveFallback = true + } + if len(attestations) == 0 { + return nil, fmt.Errorf("migration run has no stable source container identities; refresh it from a local Docker scan before live work") + } + return attestations, nil +} + +func ValidateSourceContainerAttestations(apps []preparer.AppPlan) error { + _, err := collectSourceContainerAttestations(apps) + return err +} + +func verifySourceContainers(ctx context.Context, runner dockerRunner, apps []preparer.AppPlan) error { + attestations, err := collectSourceContainerAttestations(apps) + if err != nil { + return err + } + const batchSize = 100 + for start := 0; start < len(attestations); start += batchSize { + end := min(start+batchSize, len(attestations)) + if err := verifySourceContainerAttestationBatch(ctx, runner, attestations[start:end]); err != nil { + return err } } - return fallback, haveFallback, nil + return nil } -func composeProjectFromCoolifyContainerName(name, service string) string { - name = strings.TrimPrefix(strings.TrimSpace(name), "/") - service = strings.TrimSpace(service) - if name == "" || service == "" { - return "" +func verifySourceContainerAttestationBatch(ctx context.Context, runner dockerRunner, attestations []sourceContainerAttestation) error { + if len(attestations) == 1 { + attestation := attestations[0] + if _, err := sourceContainer(ctx, runner, attestation.ref.id, attestation.ref.name); err != nil { + return fmt.Errorf("verify reviewed source container %s for app %s: %w", attestation.ref.label(), attestation.app, err) + } + return nil } - prefix := service + "-" - if !strings.HasPrefix(name, prefix) { - return "" + ids := make([]string, 0, len(attestations)) + for _, attestation := range attestations { + ids = append(ids, attestation.ref.id) + } + containers, err := inspectContainers(ctx, runner, ids) + if err == nil { + return validateSourceContainerAttestationBatch(attestations, containers) } - rest := strings.TrimPrefix(name, prefix) - lastDash := strings.LastIndex(rest, "-") - if lastDash <= 0 { - return "" + middle := len(attestations) / 2 + if err := verifySourceContainerAttestationBatch(ctx, runner, attestations[:middle]); err != nil { + return err } - return rest[:lastDash] + return verifySourceContainerAttestationBatch(ctx, runner, attestations[middle:]) +} + +func validateSourceContainerAttestationBatch(attestations []sourceContainerAttestation, containers []dockerContainer) error { + for _, attestation := range attestations { + matched := false + for _, container := range containers { + if !sourceContainerIDMatches(attestation.ref.id, container.ID) { + continue + } + if attestation.ref.name != "" && strings.TrimPrefix(container.Name, "/") != strings.TrimPrefix(attestation.ref.name, "/") { + return fmt.Errorf("verify reviewed source container %s for app %s: source container has name %q, want reviewed name %q", attestation.ref.label(), attestation.app, container.Name, attestation.ref.name) + } + matched = true + break + } + if !matched { + return fmt.Errorf("verify reviewed source container %s for app %s: reviewed ID was absent from batch inspection", attestation.ref.label(), attestation.app) + } + } + return nil } // sourceQuiesceTargets returns unique source container refs for every @@ -284,8 +697,7 @@ func sourceQuiesceTargetRefs(app preparer.AppPlan) []commitTargetRef { add(service.ServiceName, service.ContainerID, service.ContainerName) } // fall back to volume-owner discovery when source services were not - // captured (older bundles). use container name when id is missing so - // stale-id bundles still pause cleanly. + // captured in older bundles. for _, volume := range app.Resources.Volumes { add(volume.Service, volume.SourceContainerID, volume.SourceContainerName) } diff --git a/internal/target/dokploy/source_test.go b/internal/target/dokploy/source_test.go index e63f44c..b81a47b 100644 --- a/internal/target/dokploy/source_test.go +++ b/internal/target/dokploy/source_test.go @@ -2,6 +2,10 @@ package dokploy import ( "context" + "errors" + "fmt" + "slices" + "strings" "testing" "github.com/aikins01/bort/internal/gateway" @@ -9,6 +13,152 @@ import ( syncplan "github.com/aikins01/bort/internal/sync" ) +func TestVerifySourceContainersBatchesReviewedIDs(t *testing.T) { + webID := strings.Repeat("a", 64) + workerID := strings.Repeat("b", 64) + runner := &fakeDockerRunner{outputs: map[string][]byte{ + "inspect --type container aaaaaaaaaaaa bbbbbbbbbbbb": []byte(`[{"Id":"` + webID + `","Name":"/web"},{"Id":"` + workerID + `","Name":"/worker"}]`), + }} + apps := []preparer.AppPlan{{ + Name: "api", + Resources: preparer.ResourceSpecs{SourceServices: []preparer.SourceServiceRef{ + {ContainerID: "aaaaaaaaaaaa", ContainerName: "web"}, + {ContainerID: "bbbbbbbbbbbb", ContainerName: "worker"}, + }}, + }} + if err := verifySourceContainers(context.Background(), runner, apps); err != nil { + t.Fatal(err) + } + if len(runner.outputArgs) != 1 || strings.Join(runner.outputArgs[0], " ") != "inspect --type container aaaaaaaaaaaa bbbbbbbbbbbb" { + t.Fatalf("source attestation was not batched: %#v", runner.outputArgs) + } +} + +func TestVerifySourceContainersSkipsPlatformApps(t *testing.T) { + webID := strings.Repeat("a", 64) + runner := &fakeDockerRunner{outputs: map[string][]byte{ + "inspect --type container aaaaaaaaaaaa": []byte(`[{"Id":"` + webID + `","Name":"/web"}]`), + }} + apps := []preparer.AppPlan{ + { + Name: "api", + Resources: preparer.ResourceSpecs{SourceServices: []preparer.SourceServiceRef{ + {ContainerID: "aaaaaaaaaaaa", ContainerName: "web"}, + }}, + }, + { + Name: "coolify-proxy", + Role: "platform", + Resources: preparer.ResourceSpecs{SourceServices: []preparer.SourceServiceRef{ + {ContainerID: "stale-platform-id", ContainerName: "coolify-proxy"}, + }}, + }, + } + if err := verifySourceContainers(context.Background(), runner, apps); err != nil { + t.Fatal(err) + } + if len(runner.outputArgs) != 1 || strings.Join(runner.outputArgs[0], " ") != "inspect --type container aaaaaaaaaaaa" { + t.Fatalf("source attestation inspected an excluded platform app: %#v", runner.outputArgs) + } +} + +func TestValidateSourceContainerAttestationsRequiresStableSelectedIDs(t *testing.T) { + apps := []preparer.AppPlan{{ + Name: "api", + Resources: preparer.ResourceSpecs{SourceServices: []preparer.SourceServiceRef{ + {ContainerName: "web"}, + }}, + }} + if err := ValidateSourceContainerAttestations(apps); err == nil || !strings.Contains(err.Error(), "no stable container ID") { + t.Fatalf("expected missing reviewed ID refusal, got %v", err) + } + apps[0].Resources.SourceServices[0].ContainerID = "aaaaaaaaaaaa" + apps[0].Resources.SourceServices[0].ContainerName = "" + if err := ValidateSourceContainerAttestations(apps); err == nil || !strings.Contains(err.Error(), "no reviewed container name") { + t.Fatalf("expected missing reviewed name refusal, got %v", err) + } + apps[0].Resources.SourceServices[0].ContainerName = "web" + if err := ValidateSourceContainerAttestations(apps); err != nil { + t.Fatalf("expected stable reviewed identity to pass static validation, got %v", err) + } +} + +func TestValidateSourceContainerAttestationsRequiresEverySelectedApp(t *testing.T) { + apps := []preparer.AppPlan{ + { + Name: "api", + Resources: preparer.ResourceSpecs{SourceServices: []preparer.SourceServiceRef{ + {ContainerID: "aaaaaaaaaaaa", ContainerName: "web"}, + }}, + }, + {Name: "worker"}, + } + if err := ValidateSourceContainerAttestations(apps); err == nil || !strings.Contains(err.Error(), "app worker has no reviewed source container identities") { + t.Fatalf("expected unattested selected app refusal, got %v", err) + } +} + +func TestValidateSourceContainerAttestationsRejectsConflictingDuplicateID(t *testing.T) { + apps := []preparer.AppPlan{ + { + Name: "api", + Resources: preparer.ResourceSpecs{SourceServices: []preparer.SourceServiceRef{ + {ContainerID: "aaaaaaaaaaaa", ContainerName: "web"}, + }}, + }, + { + Name: "worker", + Resources: preparer.ResourceSpecs{SourceServices: []preparer.SourceServiceRef{ + {ContainerID: "aaaaaaaaaaaa", ContainerName: "worker"}, + }}, + }, + } + if err := ValidateSourceContainerAttestations(apps); err == nil || !strings.Contains(err.Error(), "conflicting names") { + t.Fatalf("expected conflicting duplicate source identity refusal, got %v", err) + } +} + +func TestValidateSourceContainerAttestationsRejectsSameAppConflictingDuplicateID(t *testing.T) { + apps := []preparer.AppPlan{{ + Name: "api", + Resources: preparer.ResourceSpecs{ + SourceServices: []preparer.SourceServiceRef{{ContainerID: "aaaaaaaaaaaa", ContainerName: "web"}}, + Volumes: []preparer.VolumeResource{{SourceContainerID: "aaaaaaaaaaaa", SourceContainerName: "replacement"}}, + }, + }} + if err := ValidateSourceContainerAttestations(apps); err == nil || !strings.Contains(err.Error(), "conflicting names") { + t.Fatalf("expected same-app conflicting source identity refusal, got %v", err) + } +} + +func TestVerifySourceContainersNarrowsFailedBatchWithAttribution(t *testing.T) { + webID := strings.Repeat("a", 64) + replacementID := strings.Repeat("c", 64) + runner := &fakeDockerRunner{ + outputs: map[string][]byte{ + "inspect --type container aaaaaaaaaaaa": []byte(`[{"Id":"` + webID + `","Name":"/web"}]`), + "inspect --type container bbbbbbbbbbbb": []byte(`[{"Id":"` + replacementID + `","Name":"/worker"}]`), + }, + outputErrs: map[string]error{ + "inspect --type container aaaaaaaaaaaa bbbbbbbbbbbb": errors.New("batch unavailable"), + }, + } + apps := []preparer.AppPlan{{ + Name: "api", + Resources: preparer.ResourceSpecs{SourceServices: []preparer.SourceServiceRef{ + {ContainerID: "aaaaaaaaaaaa", ContainerName: "web"}, + {ContainerID: "bbbbbbbbbbbb", ContainerName: "worker"}, + }}, + }} + err := verifySourceContainers(context.Background(), runner, apps) + if err == nil || !strings.Contains(err.Error(), "verify reviewed source container bbbbbbbbbbbb for app api") || !strings.Contains(err.Error(), "resolved to ID") { + t.Fatalf("batch isolation did not preserve container attribution: %v", err) + } + if len(runner.outputArgs) != 3 { + t.Fatalf("failed batch was not isolated by bisection: %#v", runner.outputArgs) + } +} + func TestSourceQuiesceTargetsSkipsLogicalDumpStores(t *testing.T) { app := preparer.AppPlan{Name: "api"} app.Resources.DataStores = []preparer.DataStoreResource{{Service: "db", Kind: "postgres", Strategy: "migrate"}} @@ -63,6 +213,7 @@ func TestPlanFromArtifactsPausesBeforeDumpAndVolumeSync(t *testing.T) { // on-disk format is consistent. app := preparer.AppPlan{Name: "api"} app.Resources.DataStores = []preparer.DataStoreResource{{Service: "db", Kind: "postgres", Strategy: "migrate"}} + app.TargetResources = &preparer.TargetResources{Dokploy: &preparer.DokployResources{ComposeApp: preparer.DokployComposeApp{Name: "api"}}} prepare := preparer.Result{Apps: []preparer.AppPlan{app}} syncResult := syncplan.Result{Apps: []syncplan.AppPlan{{ Name: "api", @@ -71,23 +222,63 @@ func TestPlanFromArtifactsPausesBeforeDumpAndVolumeSync(t *testing.T) { {ResourceType: "volume", ResourceRef: "volume:web -> /data", Strategy: syncplan.StrategyDockerVolumeArchive}, }, }}} - plan := PlanFromArtifacts(prepare, syncResult, gatewayResultEmpty()) - kinds := []StepKind{} - for _, step := range plan.Steps { - switch step.Kind { - case StepDumpDataStore, StepRestoreDataStore, StepPauseSource, StepSyncVolume, StepResumeSource: - kinds = append(kinds, step.Kind) + filter := func(plan Plan) []StepKind { + kinds := []StepKind{} + for _, step := range plan.Steps { + switch step.Kind { + case StepPushImage, StepDumpDataStore, StepRestoreDataStore, StepPauseSource, StepSyncVolume, StepResumeSource, StepResumeTarget: + kinds = append(kinds, step.Kind) + } } + return kinds } - want := []StepKind{StepPauseSource, StepDumpDataStore, StepRestoreDataStore, StepSyncVolume, StepResumeSource} - if len(kinds) != len(want) { - t.Fatalf("expected %v, got %v", want, kinds) + staged := filter(PlanFromArtifacts(prepare, syncResult, gatewayResultEmpty())) + wantStaged := []StepKind{StepPauseSource, StepDumpDataStore, StepRestoreDataStore, StepSyncVolume, StepResumeSource, StepPushImage} + if !slices.Equal(staged, wantStaged) { + t.Fatalf("staged: expected %v, got %v", wantStaged, staged) } - for i := range want { - if kinds[i] != want[i] { - t.Fatalf("step %d: expected %s, got %s (full=%v)", i, want[i], kinds[i], kinds) + inPlace := filter(PlanFromArtifactsV1Alpha2(prepare, syncResult, gatewayResultEmpty())) + wantInPlace := []StepKind{StepPushImage, StepPauseSource, StepDumpDataStore, StepRestoreDataStore, StepSyncVolume, StepResumeSource, StepResumeTarget} + if !slices.Equal(inPlace, wantInPlace) { + t.Fatalf("v1alpha2: expected %v, got %v", wantInPlace, inPlace) + } +} + +func TestPlanFromArtifactsDeploysStatelessAppsBeforeStateWork(t *testing.T) { + stateful := preparer.AppPlan{Name: "api"} + stateful.Resources.Volumes = []preparer.VolumeResource{{Service: "web", Type: "volume", Target: "/data"}} + stateful.TargetResources = &preparer.TargetResources{Dokploy: &preparer.DokployResources{ComposeApp: preparer.DokployComposeApp{Name: "api"}}} + stateless := preparer.AppPlan{Name: "docs"} + stateless.TargetResources = &preparer.TargetResources{Dokploy: &preparer.DokployResources{ComposeApp: preparer.DokployComposeApp{Name: "docs"}}} + plan := PlanFromArtifacts( + preparer.Result{Apps: []preparer.AppPlan{stateful, stateless}}, + syncplan.Result{Apps: []syncplan.AppPlan{ + {Name: "api", Steps: []syncplan.Step{{ResourceType: "volume", ResourceRef: "volume:web -> /data", Strategy: syncplan.StrategyDockerVolumeArchive}}}, + {Name: "docs"}, + }}, + gatewayResultEmpty(), + ) + pushIndex := map[string]int{} + syncIndex := -1 + for index, step := range plan.Steps { + switch step.Kind { + case StepPushImage: + pushIndex[step.App] = index + case StepSyncVolume: + syncIndex = index } } + docsPush, hasDocsPush := pushIndex["docs"] + apiPush, hasAPIPush := pushIndex["api"] + if !hasDocsPush || !hasAPIPush || syncIndex < 0 { + t.Fatalf("expected push_image for both apps and a sync_volume step, steps=%v", stepKinds(plan.Steps)) + } + if !(docsPush < syncIndex && syncIndex < apiPush) { + t.Fatalf("expected stateless push before state work and stateful push after it, steps=%v", stepKinds(plan.Steps)) + } + if !appStateIsStaged(plan, "api") || appStateIsStaged(plan, "docs") { + t.Fatalf("expected only the stateful app to be staged, steps=%v", stepKinds(plan.Steps)) + } } func TestPlanFromArtifactsOmitsPauseSourceWhenNoState(t *testing.T) { @@ -102,6 +293,61 @@ func TestPlanFromArtifactsOmitsPauseSourceWhenNoState(t *testing.T) { } } +func TestPlanFromArtifactsDeploysRoutedStatefulTargetAfterStateTransfer(t *testing.T) { + app := preparer.AppPlan{Name: "api"} + app.Resources.Volumes = []preparer.VolumeResource{{Service: "web", Type: "volume", Target: "/data"}} + app.TargetResources = &preparer.TargetResources{Dokploy: &preparer.DokployResources{ComposeApp: preparer.DokployComposeApp{Name: "api"}}} + plan := PlanFromArtifacts( + preparer.Result{Apps: []preparer.AppPlan{app}}, + syncplan.Result{Apps: []syncplan.AppPlan{{Name: "api", Steps: []syncplan.Step{{ResourceType: "volume", ResourceRef: "volume:web -> /data", Strategy: syncplan.StrategyDockerVolumeArchive}}}}}, + gateway.Result{Apps: []gateway.AppPlan{{Name: "api", Routes: []gateway.Route{{Host: "api.example.com"}}}}}, + ) + syncIndex, pushIndex, gatewayIndex := -1, -1, -1 + for index, step := range plan.Steps { + switch step.Kind { + case StepSyncVolume: + syncIndex = index + case StepPushImage: + pushIndex = index + case StepInstallGateway: + gatewayIndex = index + case StepResumeTarget, StepResumeSource: + t.Fatalf("routed staged app must not resume source or target writers, steps=%v", stepKinds(plan.Steps)) + } + } + if syncIndex < 0 || pushIndex <= syncIndex || gatewayIndex <= pushIndex { + t.Fatalf("expected sync_volume < push_image < install_gateway, steps=%v", stepKinds(plan.Steps)) + } +} + +func TestPlanFromArtifactsV1Alpha2StartsRoutedStatefulTargetDuringFinalProxyHandoff(t *testing.T) { + app := preparer.AppPlan{Name: "api"} + app.Resources.Volumes = []preparer.VolumeResource{{Service: "web", Type: "volume", Target: "/data"}} + plan := PlanFromArtifactsV1Alpha2( + preparer.Result{Apps: []preparer.AppPlan{app}}, + syncplan.Result{Apps: []syncplan.AppPlan{{Name: "api", Steps: []syncplan.Step{{ResourceType: "volume", ResourceRef: "volume:web -> /data", Strategy: syncplan.StrategyDockerVolumeArchive}}}}}, + gateway.Result{Apps: []gateway.AppPlan{{Name: "api", Routes: []gateway.Route{{Host: "api.example.com"}}}}}, + ) + stopProxyIndex, targetIndex, activateRoutesIndex, startProxyIndex := -1, -1, -1, -1 + for index, step := range plan.Steps { + if step.Kind == StepStopCoolifyProxy { + stopProxyIndex = index + } + if step.Kind == StepStartDokployProxy { + startProxyIndex = index + } + if step.Kind == StepResumeTarget && step.App == "api" { + targetIndex = index + } + if step.Kind == StepActivateRoutes && step.App == "api" { + activateRoutesIndex = index + } + } + if stopProxyIndex < 0 || targetIndex <= stopProxyIndex || activateRoutesIndex <= targetIndex || startProxyIndex <= activateRoutesIndex { + t.Fatalf("expected target writers to start after the source proxy stops and before route activation and target proxy start, steps=%v", plan.Steps) + } +} + func TestPlanFromArtifactsPreservesBindMountsWithoutStateWork(t *testing.T) { app := preparer.AppPlan{Name: "api"} app.TargetResources = &preparer.TargetResources{Dokploy: &preparer.DokployResources{Volumes: []preparer.DokployVolume{{Type: "bind", Source: "/srv/api/uploads", Target: "/uploads"}}}} @@ -142,7 +388,7 @@ func TestPlanFromArtifactsCopiesVolumeStrategyDataStoreVolumes(t *testing.T) { kinds := []StepKind{} for _, step := range plan.Steps { switch step.Kind { - case StepDumpDataStore, StepRestoreDataStore, StepPauseSource, StepSyncVolume, StepResumeSource: + case StepDumpDataStore, StepRestoreDataStore, StepPauseSource, StepSyncVolume, StepResumeSource, StepResumeTarget: kinds = append(kinds, step.Kind) } } @@ -190,10 +436,9 @@ func TestApplyPauseSourceStopsRunningContainers(t *testing.T) { } runner := &fakeDockerRunner{ outputs: map[string][]byte{ - "inspect --type container web-id": []byte(`[{"Id":"web-id","Name":"/web","State":{"Running":true,"Status":"running"}}]`), - "inspect --type container worker-id": []byte(`[{"Id":"worker-id","Name":"/worker","State":{"Running":true,"Status":"running"}}]`), - "stop web-id": []byte("web-id\n"), - "stop worker-id": []byte("worker-id\n"), + "inspect --type container web-id worker-id": []byte(`[{"Id":"web-id","Name":"/web","State":{"Running":true,"Status":"running"}},{"Id":"worker-id","Name":"/worker","State":{"Running":true,"Status":"running"}}]`), + "stop web-id": []byte("web-id\n"), + "stop worker-id": []byte("worker-id\n"), }, } client := &Client{Docker: runner} @@ -201,12 +446,17 @@ func TestApplyPauseSourceStopsRunningContainers(t *testing.T) { if err := client.applyPauseSource(context.Background(), actx, Step{Kind: StepPauseSource, App: "api"}); err != nil { t.Fatalf("applyPauseSource: %v", err) } + entry := actx.entry("api") + if !entry.SourcePauseRecorded || len(entry.SourcePausedContainers) != 2 || entry.SourcePausedContainers[0].ID != "web-id" || entry.SourcePausedContainers[1].ID != "worker-id" { + t.Fatalf("unexpected source pause ownership: %#v", entry) + } } -func TestApplyPauseSourceFallsBackToContainerNameForStaleID(t *testing.T) { +func TestApplyPauseSourceRefusesStaleContainerID(t *testing.T) { app := preparer.AppPlan{Name: "api"} app.Resources.SourceServices = []preparer.SourceServiceRef{{ServiceName: "web", ContainerID: "stale-id", ContainerName: "web-current"}} runner := &fakeDockerRunner{ + outputErrs: map[string]error{"inspect --type container stale-id": errors.New("Error: No such object: stale-id")}, outputs: map[string][]byte{ "inspect --type container web-current": []byte(`[{"Id":"current-id","Name":"/web-current","State":{"Running":true,"Status":"running"}}]`), "stop current-id": []byte("current-id\n"), @@ -214,18 +464,20 @@ func TestApplyPauseSourceFallsBackToContainerNameForStaleID(t *testing.T) { } client := &Client{Docker: runner} actx := &applyContext{cache: map[string]*appCache{}, plan: Plan{Prepare: preparer.Result{Apps: []preparer.AppPlan{app}}}} - if err := client.applyPauseSource(context.Background(), actx, Step{Kind: StepPauseSource, App: "api"}); err != nil { - t.Fatalf("applyPauseSource: %v", err) + err := client.applyPauseSource(context.Background(), actx, Step{Kind: StepPauseSource, App: "api"}) + if err == nil || !strings.Contains(err.Error(), "stale-id") { + t.Fatalf("expected stale source identity refusal, got %v", err) } - if !fakeOutputCalled(runner, "stop", "current-id") { - t.Fatalf("expected stale id fallback to stop current container, calls=%v", runner.outputArgs) + if fakeOutputCalled(runner, "stop", "current-id") { + t.Fatalf("stale identity stopped a replacement container, calls=%v", runner.outputArgs) } } -func TestApplyPauseSourceFallsBackToComposeServiceForRecreatedContainer(t *testing.T) { +func TestApplyPauseSourceRefusesRecreatedComposeContainer(t *testing.T) { app := preparer.AppPlan{Name: "api"} app.Resources.SourceServices = []preparer.SourceServiceRef{{ServiceName: "web", ContainerID: "stale-id", ContainerName: "web-proj-123"}} runner := &fakeDockerRunner{ + outputErrs: map[string]error{"inspect --type container stale-id": errors.New("Error: No such object: stale-id")}, outputs: map[string][]byte{ "ps -a --filter label=com.docker.compose.project=proj --filter label=com.docker.compose.service=web --format {{.ID}}": []byte("live-id\n"), "inspect --type container live-id": []byte(`[{"Id":"live-id","Name":"/web-proj-456","Config":{"Labels":{"com.docker.compose.service":"web"}},"State":{"Running":true,"Status":"running"}}]`), @@ -234,11 +486,12 @@ func TestApplyPauseSourceFallsBackToComposeServiceForRecreatedContainer(t *testi } client := &Client{Docker: runner} actx := &applyContext{cache: map[string]*appCache{}, plan: Plan{Prepare: preparer.Result{Apps: []preparer.AppPlan{app}}}} - if err := client.applyPauseSource(context.Background(), actx, Step{Kind: StepPauseSource, App: "api"}); err != nil { - t.Fatalf("applyPauseSource: %v", err) + err := client.applyPauseSource(context.Background(), actx, Step{Kind: StepPauseSource, App: "api"}) + if err == nil || !strings.Contains(err.Error(), "stale-id") { + t.Fatalf("expected recreated source identity refusal, got %v", err) } - if !fakeOutputCalled(runner, "stop", "live-id") { - t.Fatalf("expected compose service fallback to stop live container, calls=%v", runner.outputArgs) + if fakeOutputCalled(runner, "stop", "live-id") { + t.Fatalf("stale identity stopped a recreated Compose container, calls=%v", runner.outputArgs) } } @@ -259,9 +512,13 @@ func TestApplyPauseSourceSkipsAlreadyStopped(t *testing.T) { if err := client.applyPauseSource(context.Background(), actx, Step{Kind: StepPauseSource, App: "api"}); err != nil { t.Fatalf("applyPauseSource: %v", err) } + entry := actx.entry("api") + if !entry.SourcePauseRecorded || !slices.Equal(entry.SourcePausedContainers, []sourcePausedContainer{{ID: "web-id"}}) { + t.Fatalf("already-stopped source was claimed by Bort: %#v", entry) + } } -func TestApplyResumeSourceStartsStoppedContainers(t *testing.T) { +func TestApplyResumeSourceStartsOnlyOwnedStoppedContainers(t *testing.T) { app := preparer.AppPlan{Name: "api"} app.Resources.Volumes = []preparer.VolumeResource{ {Service: "web", Type: "volume", SourceContainerID: "web-id"}, @@ -269,19 +526,644 @@ func TestApplyResumeSourceStartsStoppedContainers(t *testing.T) { } runner := &fakeDockerRunner{ outputs: map[string][]byte{ - "inspect --type container web-id": []byte(`[{"Id":"web-id","Name":"/web","State":{"Running":false,"Status":"exited"}}]`), - "inspect --type container worker-id": []byte(`[{"Id":"worker-id","Name":"/worker","State":{"Running":true,"Status":"running"}}]`), - "start web-id": []byte("web-id\n"), + "inspect --type container web-id worker-id": []byte(`[{"Id":"web-id","Name":"/web","State":{"Running":false,"Status":"exited"}},{"Id":"worker-id","Name":"/worker","State":{"Running":false,"Status":"exited"}}]`), + "start web-id": []byte("web-id\n"), }, } client := &Client{Docker: runner} actx := &applyContext{cache: map[string]*appCache{}, plan: Plan{Prepare: preparer.Result{Apps: []preparer.AppPlan{app}}}} - // stub omits "start worker-id" on purpose: the running worker must not - // be restarted. if applyResumeSource calls it, fakeDockerRunner.Output - // returns an error because the key is unstubbed. + entry := actx.entry("api") + entry.SourcePauseRecorded = true + entry.SourcePausedContainers = []sourcePausedContainer{{ID: "web-id", Stopped: true}, {ID: "worker-id"}} if err := client.applyResumeSource(context.Background(), actx, Step{Kind: StepResumeSource, App: "api"}); err != nil { t.Fatalf("applyResumeSource: %v", err) } + if !fakeOutputCalled(runner, "start", "web-id") { + t.Fatalf("resume did not start the container Bort stopped: %#v", runner.outputArgs) + } + if fakeOutputCalled(runner, "start", "worker-id") { + t.Fatalf("resume started a container Bort did not stop: %#v", runner.outputArgs) + } + if !entry.SourcePauseRecorded || !slices.Equal(entry.SourcePausedContainers, []sourcePausedContainer{{ID: "web-id", Stopped: true}, {ID: "worker-id"}}) { + t.Fatalf("resume cleared source ownership before terminal progress was durable: %#v", entry) + } +} + +type sourceOwnershipRunner struct { + fakeDockerRunner + running map[string]bool + startedAt map[string]string + finishedAt map[string]string +} + +func (r *sourceOwnershipRunner) Output(_ context.Context, args ...string) ([]byte, error) { + r.outputArgs = append(r.outputArgs, append([]string{}, args...)) + if len(args) >= 4 && args[0] == "inspect" && args[1] == "--type" && args[2] == "container" { + entries := make([]string, 0, len(args)-3) + for _, id := range args[3:] { + entries = append(entries, fmt.Sprintf(`{"Id":%q,"Name":%q,"State":{"Running":%t,"Status":"exited","StartedAt":%q,"FinishedAt":%q}}`, id, "/"+id, r.running[id], r.startedAt[id], r.finishedAt[id])) + } + return []byte("[" + strings.Join(entries, ",") + "]"), nil + } + if len(args) == 2 && args[0] == "stop" { + r.running[args[1]] = false + if r.finishedAt != nil { + r.finishedAt[args[1]] = fmt.Sprintf("stopped-%d", len(r.outputArgs)) + } + return []byte(args[1] + "\n"), nil + } + if len(args) == 2 && args[0] == "start" { + r.running[args[1]] = true + if r.startedAt != nil { + r.startedAt[args[1]] = fmt.Sprintf("started-%d", len(r.outputArgs)) + } + return []byte(args[1] + "\n"), nil + } + return nil, fmt.Errorf("docker output not stubbed: %s", strings.Join(args, " ")) +} + +func TestApplyFailureResumesOnlySourceContainersItStopped(t *testing.T) { + app := preparer.AppPlan{Name: "api"} + app.Resources.Volumes = []preparer.VolumeResource{ + {Service: "web", Type: "volume", SourceContainerID: "web-id"}, + {Service: "worker", Type: "bind", SourceContainerID: "worker-id"}, + } + runner := &sourceOwnershipRunner{running: map[string]bool{"web-id": true, "worker-id": false}} + client := &Client{Docker: runner} + beforeCalls := 0 + beforeStep := func(StepProgress) error { + beforeCalls++ + if beforeCalls == 2 { + return errors.New("injected post-pause failure") + } + return nil + } + plan := Plan{ + RunDir: t.TempDir(), + Steps: []Step{{Kind: StepPauseSource, App: "api", Ref: "api"}, {Kind: StepCreateVolume, App: "api", Ref: "data"}}, + Prepare: preparer.Result{Apps: []preparer.AppPlan{app}}, + BeforeStep: &beforeStep, + } + err := client.Apply(context.Background(), plan) + if err == nil || !strings.Contains(err.Error(), "injected post-pause failure") { + t.Fatalf("expected injected failure, got %v", err) + } + if !runner.running["web-id"] { + t.Fatal("Bort-owned source container was not resumed") + } + if runner.running["worker-id"] || fakeOutputCalled(&runner.fakeDockerRunner, "start", "worker-id") { + t.Fatalf("pre-stopped source container was started: running=%t calls=%#v", runner.running["worker-id"], runner.outputArgs) + } + statePath, err := sourcePauseStatePath(plan.RunDir) + if err != nil { + t.Fatal(err) + } + loaded := &applyContext{plan: plan, cache: map[string]*appCache{}} + if err := loaded.loadSourcePauseState(); err != nil { + t.Fatalf("load final source pause state %s: %v", statePath, err) + } + want := []sourcePausedContainer{{ID: "web-id", Stopped: true, StartedAt: "", FinishedAt: runner.finishedAt["web-id"]}, {ID: "worker-id"}} + if !loaded.entry("api").SourcePauseRecorded || !slices.Equal(loaded.entry("api").SourcePausedContainers, want) { + t.Fatalf("source ownership was cleared before compensation progress became durable: %#v", loaded.entry("api")) + } +} + +func TestFinalizeSourceResumesClearsDurableOwnership(t *testing.T) { + runDir := t.TempDir() + plan := Plan{RunDir: runDir, Steps: []Step{{Kind: StepResumeSource, App: "api", Ref: "api"}}} + paused := &applyContext{plan: plan, cache: map[string]*appCache{}} + paused.entry("api").SourcePauseRecorded = true + paused.entry("api").SourcePausedContainers = []sourcePausedContainer{{ID: "web-id"}} + if err := paused.persistSourcePauseState(); err != nil { + t.Fatal(err) + } + client := &Client{} + if err := client.FinalizeSourceResumes(plan); err != nil { + t.Fatal(err) + } + loaded := &applyContext{plan: plan, cache: map[string]*appCache{}} + if err := loaded.loadSourcePauseState(); err != nil { + t.Fatal(err) + } + if loaded.entry("api").SourcePauseRecorded { + t.Fatalf("finalized source ownership remained durable: %#v", loaded.entry("api")) + } +} + +func TestApplyRetryRepausesOnlyDurablyOwnedSourceContainers(t *testing.T) { + app := preparer.AppPlan{Name: "api"} + app.Resources.Volumes = []preparer.VolumeResource{ + {Service: "web", Type: "volume", SourceContainerID: "web-id"}, + {Service: "worker", Type: "bind", SourceContainerID: "worker-id"}, + } + runDir := t.TempDir() + plan := Plan{ + RunDir: runDir, + ResumeFrom: 1, + Steps: []Step{{Kind: StepPauseSource, App: "api", Ref: "api"}, {Kind: StepCreateVolume, App: "api", Ref: "data"}}, + Prepare: preparer.Result{Apps: []preparer.AppPlan{app}}, + } + paused := &applyContext{plan: plan, cache: map[string]*appCache{}} + paused.entry("api").SourcePauseRecorded = true + paused.entry("api").SourcePausedContainers = []sourcePausedContainer{{ID: "web-id", Stopped: true}, {ID: "worker-id"}} + if err := paused.persistSourcePauseState(); err != nil { + t.Fatal(err) + } + runner := &sourceOwnershipRunner{running: map[string]bool{"web-id": true, "worker-id": true}, finishedAt: map[string]string{}} + client := &Client{Docker: runner} + if err := client.Apply(context.Background(), plan); err != nil { + t.Fatalf("retry paused source: %v", err) + } + if runner.running["web-id"] || !fakeOutputCalled(&runner.fakeDockerRunner, "stop", "web-id") { + t.Fatalf("durably owned source was not re-paused: running=%t calls=%#v", runner.running["web-id"], runner.outputArgs) + } + if runner.running["worker-id"] || !fakeOutputCalled(&runner.fakeDockerRunner, "stop", "worker-id") { + t.Fatalf("retry left a quiesce target that started since the pause running: running=%t calls=%#v", runner.running["worker-id"], runner.outputArgs) + } + loaded := &applyContext{plan: plan, cache: map[string]*appCache{}} + if err := loaded.loadSourcePauseState(); err != nil { + t.Fatal(err) + } + want := []sourcePausedContainer{ + {ID: "web-id", Stopped: true, FinishedAt: runner.finishedAt["web-id"]}, + {ID: "worker-id", Stopped: true, FinishedAt: runner.finishedAt["worker-id"]}, + } + if got := loaded.entry("api").SourcePausedContainers; !slices.Equal(got, want) { + t.Fatalf("retry did not adopt the container it stopped: got %#v, want %#v", got, want) + } +} + +func TestApplyRetryRefusesQuiesceTargetWithoutPauseEvidence(t *testing.T) { + app := preparer.AppPlan{Name: "api"} + app.Resources.Volumes = []preparer.VolumeResource{ + {Service: "web", Type: "volume", SourceContainerID: "web-id"}, + {Service: "worker", Type: "bind", SourceContainerID: "worker-id"}, + } + plan := Plan{ + RunDir: t.TempDir(), + ResumeFrom: 1, + Steps: []Step{{Kind: StepPauseSource, App: "api", Ref: "api"}, {Kind: StepCreateVolume, App: "api", Ref: "data"}}, + Prepare: preparer.Result{Apps: []preparer.AppPlan{app}}, + } + paused := &applyContext{plan: plan, cache: map[string]*appCache{}} + paused.entry("api").SourcePauseRecorded = true + paused.entry("api").SourcePausedContainers = []sourcePausedContainer{{ID: "web-id", Stopped: true}} + if err := paused.persistSourcePauseState(); err != nil { + t.Fatal(err) + } + runner := &sourceOwnershipRunner{running: map[string]bool{"web-id": false, "worker-id": false}} + client := &Client{Docker: runner} + err := client.Apply(context.Background(), plan) + if !isUnsafeSourceResumeError(err) || !strings.Contains(err.Error(), "worker-id") || !strings.Contains(err.Error(), "no durable pause evidence") { + t.Fatalf("expected missing pause evidence refusal, got %v", err) + } + if runner.running["web-id"] || fakeOutputCalled(&runner.fakeDockerRunner, "start", "web-id") { + t.Fatalf("source was restarted despite missing pause evidence: %#v", runner.outputArgs) + } +} + +func TestSourceRanSincePauseDetectsUnownedContainerThatRanAndStopped(t *testing.T) { + app := preparer.AppPlan{Name: "api"} + app.Resources.Volumes = []preparer.VolumeResource{ + {Service: "web", Type: "volume", SourceContainerID: "web-id"}, + {Service: "worker", Type: "bind", SourceContainerID: "worker-id"}, + } + actx := &applyContext{plan: Plan{Prepare: preparer.Result{Apps: []preparer.AppPlan{app}}}, cache: map[string]*appCache{}} + actx.entry("api").SourcePauseRecorded = true + actx.entry("api").SourcePausedContainers = []sourcePausedContainer{ + {ID: "web-id", Stopped: true, StartedAt: "started-1", FinishedAt: "stopped-2"}, + {ID: "worker-id", StartedAt: "started-0", FinishedAt: "stopped-0"}, + } + runner := &sourceOwnershipRunner{ + running: map[string]bool{"web-id": false, "worker-id": false}, + startedAt: map[string]string{"web-id": "started-1", "worker-id": "started-5"}, + finishedAt: map[string]string{"web-id": "stopped-2", "worker-id": "stopped-6"}, + } + client := &Client{Docker: runner} + ran, err := client.sourceRanSincePause(context.Background(), actx, "api") + if err != nil { + t.Fatal(err) + } + if !ran { + t.Fatal("a quiesce target Bort never stopped ran after the pause but was not detected") + } + runner.startedAt["worker-id"], runner.finishedAt["worker-id"] = "started-0", "stopped-0" + ran, err = client.sourceRanSincePause(context.Background(), actx, "api") + if err != nil { + t.Fatal(err) + } + if ran { + t.Fatal("unchanged quiesce targets were reported as having run") + } +} + +func TestApplyStopSourceAppRefusesReviewedContainerDisappearance(t *testing.T) { + app := preparer.AppPlan{Name: "api"} + app.Resources.SourceServices = []preparer.SourceServiceRef{{ServiceName: "web", ContainerID: "reviewed-id", ContainerName: "api-web"}} + runner := &fakeDockerRunner{ + outputs: map[string][]byte{ + "inspect --type container reviewed-id": []byte(`[{"Id":"reviewed-id","Name":"/api-web","HostConfig":{"RestartPolicy":{"Name":"always"}},"State":{"Running":true,"Status":"running"}}]`), + "update --restart=no reviewed-id": []byte("reviewed-id\n"), + }, + outputErrs: map[string]error{ + "stop reviewed-id": errors.New("Error response from daemon: No such container: reviewed-id"), + }, + } + client := &Client{Docker: runner} + actx := &applyContext{plan: Plan{Prepare: preparer.Result{Apps: []preparer.AppPlan{app}}}, cache: map[string]*appCache{}} + err := client.applyStopSourceApp(context.Background(), actx, Step{Kind: StepStopSourceApp, App: "api", Ref: "api"}) + if err == nil || !strings.Contains(err.Error(), "No such container") { + t.Fatalf("expected disappearing reviewed source to fail retirement, got %v", err) + } } func gatewayResultEmpty() gateway.Result { return gateway.Result{} } + +func TestPrimeResumeStateSkipsPauseReplayAfterCompletedResume(t *testing.T) { + app := preparer.AppPlan{Name: "api"} + app.Resources.Volumes = []preparer.VolumeResource{{Service: "web", Type: "volume", SourceContainerID: "web-id"}} + plan := Plan{ + RunDir: t.TempDir(), + Steps: []Step{ + {Kind: StepPauseSource, App: "api", Ref: "api"}, + {Kind: StepSyncVolume, App: "api", Ref: "volume:web -> /data"}, + {Kind: StepResumeSource, App: "api", Ref: "api"}, + {Kind: StepPushImage, App: "api", Ref: "api"}, + {Kind: StepStopCoolifyProxy, Ref: coolifyProxyContainer}, + }, + Prepare: preparer.Result{Apps: []preparer.AppPlan{app}}, + } + actx := &applyContext{plan: plan, cache: map[string]*appCache{}} + actx.entry("api").SourcePauseRecorded = true + actx.entry("api").SourcePausedContainers = []sourcePausedContainer{{ID: "web-id"}} + runner := &sourceOwnershipRunner{running: map[string]bool{"web-id": true}} + client := &Client{Docker: runner} + pausedApps := pausedSources{} + coolifyProxyStopped := false + if err := client.primeResumeState(context.Background(), actx, plan.Steps[:4], pausedApps, map[string]struct{}{}, &coolifyProxyStopped); err != nil { + t.Fatalf("prime resume state: %v", err) + } + if !runner.running["web-id"] || fakeOutputCalled(&runner.fakeDockerRunner, "stop", "web-id") { + t.Fatalf("a source resumed by a completed step was stopped again on retry: running=%t calls=%#v", runner.running["web-id"], runner.outputArgs) + } + if len(pausedApps) != 0 { + t.Fatalf("resumed source still tracked as paused: %#v", pausedApps) + } +} + +func TestPrimeResumeStateKeepsSourceStoppedAfterStagedDeploy(t *testing.T) { + app := preparer.AppPlan{Name: "api"} + app.Resources.Volumes = []preparer.VolumeResource{{Service: "web", Type: "volume", SourceContainerID: "web-id"}} + plan := Plan{ + RunDir: t.TempDir(), + Steps: []Step{ + {Kind: StepPauseSource, App: "api", Ref: "api"}, + {Kind: StepSyncVolume, App: "api", Ref: "volume:web -> /data"}, + {Kind: StepPushImage, App: "api", Ref: "api"}, + {Kind: StepStopCoolifyProxy, Ref: coolifyProxyContainer}, + }, + Prepare: preparer.Result{Apps: []preparer.AppPlan{app}}, + } + actx := &applyContext{plan: plan, cache: map[string]*appCache{}} + actx.entry("api").SourcePauseRecorded = true + actx.entry("api").SourcePausedContainers = []sourcePausedContainer{{ID: "web-id", Stopped: true}} + runner := &sourceOwnershipRunner{running: map[string]bool{"web-id": false}} + client := &Client{Docker: runner} + pausedApps := pausedSources{} + coolifyProxyStopped := false + if err := client.primeResumeState(context.Background(), actx, plan.Steps[:3], pausedApps, map[string]struct{}{}, &coolifyProxyStopped); err != nil { + t.Fatalf("prime resume state: %v", err) + } + if len(pausedApps) != 0 { + t.Fatalf("source handed to a staged deployment must not be restartable by cleanup: %#v", pausedApps) + } + if err := client.bestEffortResume(context.Background(), actx, plan, len(plan.Steps), pausedApps, false, true); err != nil { + t.Fatal(err) + } + if runner.running["web-id"] { + t.Fatal("cleanup restarted a source whose state was already handed to Dokploy") + } +} + +func TestApplyPauseSourceRecordsStoppedRunTimestamps(t *testing.T) { + app := preparer.AppPlan{Name: "api"} + app.Resources.Volumes = []preparer.VolumeResource{{Service: "web", Type: "volume", SourceContainerID: "web-id"}} + plan := Plan{RunDir: t.TempDir(), Prepare: preparer.Result{Apps: []preparer.AppPlan{app}}} + runner := &sourceOwnershipRunner{ + running: map[string]bool{"web-id": true}, + startedAt: map[string]string{"web-id": "started-0"}, + finishedAt: map[string]string{"web-id": "never"}, + } + client := &Client{Docker: runner} + actx := &applyContext{plan: plan, cache: map[string]*appCache{}} + if err := client.applyPauseSource(context.Background(), actx, Step{Kind: StepPauseSource, App: "api"}); err != nil { + t.Fatalf("applyPauseSource: %v", err) + } + loaded := &applyContext{plan: plan, cache: map[string]*appCache{}} + if err := loaded.loadSourcePauseState(); err != nil { + t.Fatal(err) + } + want := []sourcePausedContainer{{ID: "web-id", Stopped: true, StartedAt: "started-0", FinishedAt: runner.finishedAt["web-id"]}} + if got := loaded.entry("api").SourcePausedContainers; !slices.Equal(got, want) || want[0].FinishedAt == "never" { + t.Fatalf("durable pause state = %#v, want post-stop snapshot %#v", got, want) + } +} + +func TestPrimeResumeStateInvalidatesTransferWhenSourceRanAfterPause(t *testing.T) { + app := preparer.AppPlan{Name: "api"} + app.Resources.Volumes = []preparer.VolumeResource{{Service: "web", Type: "volume", SourceContainerID: "web-id"}} + var recorded []StepProgress + beforeStep := func(progress StepProgress) error { + recorded = append(recorded, progress) + return nil + } + plan := Plan{ + RunDir: t.TempDir(), + ResumeFrom: 2, + Steps: []Step{ + {Kind: StepPauseSource, App: "api", Ref: "api"}, + {Kind: StepSyncVolume, App: "api", Ref: "volume:web -> /data"}, + {Kind: StepCreateVolume, App: "api", Ref: "data"}, + }, + Prepare: preparer.Result{Apps: []preparer.AppPlan{app}}, + BeforeStep: &beforeStep, + } + paused := &applyContext{plan: plan, cache: map[string]*appCache{}} + paused.entry("api").SourcePauseRecorded = true + paused.entry("api").SourcePausedContainers = []sourcePausedContainer{{ID: "web-id", Stopped: true, StartedAt: "started-1", FinishedAt: "stopped-2"}} + if err := paused.persistSourcePauseState(); err != nil { + t.Fatal(err) + } + runner := &sourceOwnershipRunner{ + running: map[string]bool{"web-id": false}, + startedAt: map[string]string{"web-id": "started-9"}, + finishedAt: map[string]string{"web-id": "stopped-10"}, + } + client := &Client{Docker: runner} + err := client.Apply(context.Background(), plan) + if err == nil || !strings.Contains(err.Error(), "invalidated the transfer") { + t.Fatalf("expected invalidated transfer refusal, got %v", err) + } + if fakeOutputCalled(&runner.fakeDockerRunner, "stop", "web-id") { + t.Fatalf("a source that ran after transfer was silently re-paused: %#v", runner.outputArgs) + } + if !runner.running["web-id"] { + t.Fatal("source was not resumed after its transfer was invalidated") + } + if len(recorded) != 1 || recorded[0].Step.Kind != StepResumeSource || recorded[0].Index != 0 || recorded[0].Status != StepStatusStarted { + t.Fatalf("invalidating resume was not recorded at the pause index before restart: %#v", recorded) + } +} + +func TestPrimeResumeStateRefusesStaleStagedDeployWhenSourceRan(t *testing.T) { + app := preparer.AppPlan{Name: "api"} + app.Resources.Volumes = []preparer.VolumeResource{{Service: "web", Type: "volume", SourceContainerID: "web-id"}} + plan := Plan{ + RunDir: t.TempDir(), + Steps: []Step{ + {Kind: StepPauseSource, App: "api", Ref: "api"}, + {Kind: StepSyncVolume, App: "api", Ref: "volume:web -> /data"}, + {Kind: StepPushImage, App: "api", Ref: "api"}, + {Kind: StepStopCoolifyProxy, Ref: coolifyProxyContainer}, + }, + Prepare: preparer.Result{Apps: []preparer.AppPlan{app}}, + } + actx := &applyContext{plan: plan, cache: map[string]*appCache{}} + actx.entry("api").SourcePauseRecorded = true + actx.entry("api").SourcePausedContainers = []sourcePausedContainer{{ID: "web-id", StartedAt: "started-1", FinishedAt: "stopped-2"}} + runner := &sourceOwnershipRunner{running: map[string]bool{"web-id": true}, startedAt: map[string]string{"web-id": "started-7"}} + client := &Client{Docker: runner} + pausedApps := pausedSources{} + coolifyProxyStopped := false + err := client.primeResumeState(context.Background(), actx, plan.Steps[:3], pausedApps, map[string]struct{}{}, &coolifyProxyStopped) + if !isUnsafeSourceResumeError(err) || !strings.Contains(err.Error(), "staged data is stale") { + t.Fatalf("expected stale staged deployment refusal, got %v", err) + } + if fakeOutputCalled(&runner.fakeDockerRunner, "stop", "web-id") || !runner.running["web-id"] { + t.Fatalf("source that ran past the staged handoff was touched: %#v", runner.outputArgs) + } + if len(pausedApps) != 0 { + t.Fatalf("stale staged source must not be restartable by cleanup: %#v", pausedApps) + } +} + +func TestPausedSourcesObserveCompleted(t *testing.T) { + paused := pausedSources{"api": false, "other": false, "untransferred": false} + handedOff := map[string]struct{}{} + paused.observeCompleted(Step{Kind: StepSyncVolume, App: "api"}, handedOff) + paused.observeCompleted(Step{Kind: StepDumpDataStore, App: "stateless"}, handedOff) + if !paused["api"] || paused["other"] { + t.Fatalf("transfer must mark only the transferred app: %#v", paused) + } + if _, tracked := paused["stateless"]; tracked { + t.Fatalf("transfer for an unpaused app must not track it: %#v", paused) + } + paused.observeCompleted(Step{Kind: StepPushImage, App: "api"}, handedOff) + paused.observeCompleted(Step{Kind: StepPushImage, App: "untransferred"}, handedOff) + paused.observeCompleted(Step{Kind: StepResumeSource, App: "other"}, handedOff) + if len(paused) != 0 { + t.Fatalf("deploy and resume must release their apps: %#v", paused) + } + if _, ok := handedOff["api"]; !ok || len(handedOff) != 1 { + t.Fatalf("only a deploy over transferred state is a staged handoff: %#v", handedOff) + } +} + +func TestApplyRecordsCleanupResumeBeforeRestartingTransferredSource(t *testing.T) { + app := preparer.AppPlan{Name: "api"} + app.Resources.Volumes = []preparer.VolumeResource{{Service: "web", Type: "volume", SourceContainerID: "web-id"}} + app.Resources.DataStores = []preparer.DataStoreResource{{Kind: "redis", Service: "db", Strategy: "migrate"}} + runner := &sourceOwnershipRunner{running: map[string]bool{"web-id": true}} + client := &Client{Docker: runner} + var before []StepProgress + beforeStep := func(p StepProgress) error { + before = append(before, p) + if len(before) == 3 { + return errors.New("injected failure after transfer") + } + if p.Step.Kind == StepResumeSource { + if runner.running["web-id"] { + t.Fatal("cleanup started the source before recording the restart") + } + return errors.New("ledger unavailable") + } + return nil + } + plan := Plan{ + RunDir: t.TempDir(), + Steps: []Step{ + {Kind: StepPauseSource, App: "api", Ref: "api"}, + {Kind: StepDumpDataStore, App: "api", Ref: "data-store:db"}, + {Kind: StepCreateVolume, App: "api", Ref: "data"}, + }, + Prepare: preparer.Result{Apps: []preparer.AppPlan{app}}, + BeforeStep: &beforeStep, + } + err := client.Apply(context.Background(), plan) + if err == nil || !strings.Contains(err.Error(), "stays stopped") || !strings.Contains(err.Error(), "ledger unavailable") { + t.Fatalf("expected refused restart when the cleanup cannot be recorded, got %v", err) + } + if runner.running["web-id"] { + t.Fatal("source restarted although its completed transfer could not be invalidated in the ledger") + } + last := before[len(before)-1] + if last.Step.Kind != StepResumeSource || last.Index != 0 || last.Status != StepStatusStarted { + t.Fatalf("cleanup must be recorded at the pause index before restart, got %#v", last) + } +} + +func TestApplyFinalizationRetryRefusesStaleStagedDeployWhenSourceRan(t *testing.T) { + app := preparer.AppPlan{Name: "api"} + app.Resources.Volumes = []preparer.VolumeResource{{Service: "web", Type: "volume", SourceContainerID: "web-id"}} + plan := Plan{ + RunDir: t.TempDir(), + ResumeFrom: 3, + Steps: []Step{ + {Kind: StepPauseSource, App: "api", Ref: "api"}, + {Kind: StepSyncVolume, App: "api", Ref: "volume:web -> /data"}, + {Kind: StepPushImage, App: "api", Ref: "api"}, + }, + Prepare: preparer.Result{Apps: []preparer.AppPlan{app}}, + } + paused := &applyContext{plan: plan, cache: map[string]*appCache{}} + paused.entry("api").SourcePauseRecorded = true + paused.entry("api").SourcePausedContainers = []sourcePausedContainer{{ID: "web-id", Stopped: true, StartedAt: "started-1", FinishedAt: "stopped-2"}} + if err := paused.persistSourcePauseState(); err != nil { + t.Fatal(err) + } + runner := &sourceOwnershipRunner{ + running: map[string]bool{"web-id": false}, + startedAt: map[string]string{"web-id": "started-9"}, + finishedAt: map[string]string{"web-id": "stopped-10"}, + } + client := &Client{Docker: runner} + err := client.Apply(context.Background(), plan) + if !isUnsafeSourceResumeError(err) || !strings.Contains(err.Error(), "staged data is stale") { + t.Fatalf("expected stale staged deployment refusal on finalization retry, got %v", err) + } + if fakeOutputCalled(&runner.fakeDockerRunner, "stop", "web-id") || fakeOutputCalled(&runner.fakeDockerRunner, "start", "web-id") { + t.Fatalf("finalization retry touched the source: %#v", runner.outputArgs) + } +} + +func TestApplyRefusesStagedDeployWhenSourceRanAfterInProcessTransfer(t *testing.T) { + app := preparer.AppPlan{Name: "api"} + app.Resources.DataStores = []preparer.DataStoreResource{{Kind: "redis", Service: "cache", Strategy: "recreate"}} + app.Resources.Volumes = []preparer.VolumeResource{ + {Service: "web", Type: "volume", SourceContainerID: "web-id"}, + {Service: "cache", Type: "volume", Target: "/data", SourceContainerID: "cache-id"}, + } + runner := &sourceOwnershipRunner{ + running: map[string]bool{"web-id": true}, + startedAt: map[string]string{"web-id": "started-0"}, + finishedAt: map[string]string{"web-id": "never"}, + } + client := &Client{Docker: runner} + var recorded []StepProgress + beforeStep := func(progress StepProgress) error { + recorded = append(recorded, progress) + if progress.Step.Kind == StepPushImage { + runner.startedAt["web-id"], runner.finishedAt["web-id"] = "started-9", "stopped-10" + } + return nil + } + plan := Plan{ + RunDir: t.TempDir(), + Steps: []Step{ + {Kind: StepPauseSource, App: "api", Ref: "api"}, + {Kind: StepSyncVolume, App: "api", Ref: "volume:cache -> /data"}, + {Kind: StepPushImage, App: "api", Ref: "api"}, + }, + Prepare: preparer.Result{Apps: []preparer.AppPlan{app}}, + BeforeStep: &beforeStep, + } + err := client.Apply(context.Background(), plan) + if err == nil || !strings.Contains(err.Error(), "invalidated the transfer") { + t.Fatalf("expected invalidated transfer refusal before the staged deploy, got %v", err) + } + if !runner.running["web-id"] { + t.Fatal("source was not resumed after its transfer was invalidated") + } + last := recorded[len(recorded)-1] + if last.Step.Kind != StepResumeSource || last.Index != 0 || last.Status != StepStatusStarted { + t.Fatalf("invalidating resume was not recorded at the pause index before restart: %#v", recorded) + } +} + +func TestApplyRefusesRouteActivationWhenSourceRanAfterStagedHandoff(t *testing.T) { + app := preparer.AppPlan{Name: "api"} + app.Resources.Volumes = []preparer.VolumeResource{{Service: "web", Type: "volume", SourceContainerID: "web-id"}} + runner := &sourceOwnershipRunner{ + running: map[string]bool{"web-id": false, coolifyProxyContainer: true}, + startedAt: map[string]string{"web-id": "started-0"}, + finishedAt: map[string]string{"web-id": "stopped-1"}, + } + client := &Client{Docker: runner} + beforeStep := func(progress StepProgress) error { + if progress.Step.Kind == StepActivateRoutes { + runner.startedAt["web-id"], runner.finishedAt["web-id"] = "started-9", "stopped-10" + } + return nil + } + plan := Plan{ + ResumeFrom: 3, + RunDir: t.TempDir(), + Steps: []Step{ + {Kind: StepPauseSource, App: "api", Ref: "api"}, + {Kind: StepSyncVolume, App: "api", Ref: "volume:web -> /data"}, + {Kind: StepPushImage, App: "api", Ref: "api"}, + {Kind: StepStopCoolifyProxy, Ref: coolifyProxyContainer}, + {Kind: StepActivateRoutes, App: "api", Ref: "routes"}, + }, + Prepare: preparer.Result{Apps: []preparer.AppPlan{app}}, + BeforeStep: &beforeStep, + } + paused := &applyContext{plan: plan, cache: map[string]*appCache{}} + paused.entry("api").SourcePauseRecorded = true + paused.entry("api").SourcePausedContainers = []sourcePausedContainer{{ID: "web-id", Stopped: true, StartedAt: "started-0", FinishedAt: "stopped-1"}} + if err := paused.persistSourcePauseState(); err != nil { + t.Fatal(err) + } + err := client.Apply(context.Background(), plan) + if err == nil || !strings.Contains(err.Error(), "staged data is stale") { + t.Fatalf("expected stale staged handoff refusal before route activation, got %v", err) + } + if runner.running["web-id"] || fakeOutputCalled(&runner.fakeDockerRunner, "start", "web-id") { + t.Fatalf("source that ran past the staged handoff was restarted: %#v", runner.outputArgs) + } + if !runner.running[coolifyProxyContainer] { + t.Fatal("coolify proxy was left stopped after refusing route activation") + } +} + +func TestRouteActivationGateRechecksEveryHandedOffAppPerStep(t *testing.T) { + api := preparer.AppPlan{Name: "api"} + api.Resources.Volumes = []preparer.VolumeResource{{Service: "web", Type: "volume", SourceContainerID: "api-id"}} + worker := preparer.AppPlan{Name: "worker"} + worker.Resources.Volumes = []preparer.VolumeResource{{Service: "jobs", Type: "volume", SourceContainerID: "worker-id"}} + runner := &sourceOwnershipRunner{ + running: map[string]bool{"api-id": false, "worker-id": false}, + startedAt: map[string]string{"api-id": "started-0", "worker-id": "started-0"}, + finishedAt: map[string]string{"api-id": "stopped-1", "worker-id": "stopped-1"}, + } + client := &Client{Docker: runner} + actx := &applyContext{plan: Plan{Prepare: preparer.Result{Apps: []preparer.AppPlan{api, worker}}}, cache: map[string]*appCache{}} + for app, id := range map[string]string{"api": "api-id", "worker": "worker-id"} { + actx.entry(app).SourcePauseRecorded = true + actx.entry(app).SourcePausedContainers = []sourcePausedContainer{{ID: id, Stopped: true, StartedAt: "started-0", FinishedAt: "stopped-1"}} + } + handedOff := map[string]struct{}{"api": {}, "worker": {}} + first := Step{Kind: StepActivateRoutes, App: "api", Ref: "routes"} + if err := client.requireTransferredSourceStillPaused(context.Background(), actx, first, pausedSources{}, handedOff); err != nil { + t.Fatalf("first activation with quiet sources: %v", err) + } + if len(runner.outputArgs) != 1 || strings.Join(runner.outputArgs[0], " ") != "inspect --type container api-id worker-id" { + t.Fatalf("expected one batched inspect of every handed-off source, got %#v", runner.outputArgs) + } + runner.startedAt["worker-id"], runner.finishedAt["worker-id"] = "started-9", "stopped-10" + second := Step{Kind: StepActivateRoutes, App: "api", Ref: "routes"} + err := client.requireTransferredSourceStillPaused(context.Background(), actx, second, pausedSources{}, handedOff) + if err == nil || !strings.Contains(err.Error(), "source app worker ran after its state was handed") || !isUnsafeSourceResumeError(err) { + t.Fatalf("a handed-off source that ran must block every later activation, not only its own, got %v", err) + } +} diff --git a/internal/target/dokploy/staging.go b/internal/target/dokploy/staging.go new file mode 100644 index 0000000..a0e57a9 --- /dev/null +++ b/internal/target/dokploy/staging.go @@ -0,0 +1,990 @@ +package dokploy + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "os" + "path" + "path/filepath" + "regexp" + "strconv" + "strings" + "time" + + "github.com/aikins01/bort/internal/preparer" + "github.com/aikins01/bort/internal/safepath" + "gopkg.in/yaml.v3" +) + +const ( + stagingVolumeRunLabel = "bort.run" + stagingVolumeRunIDLabel = "bort.run-id" + stagingVolumeAppLabel = "bort.app" + stagingVolumeServiceLabel = "bort.service" + stagingVolumeTargetLabel = "bort.target" +) + +// stagingOwner identifies the run that owns staging volumes. run names +// repeat across directories and machines, so the CLI supplies a digest +// of the run's durable identity; the name alone is the fallback. +func stagingOwner(plan Plan) string { + if id := strings.TrimSpace(plan.RunID); id != "" { + return id + } + return plan.RunName +} + +type stagedVolume struct { + Service string + Target string + VolumeName string + Source preparer.VolumeResource +} + +// appStateIsStaged reports whether the app's persistent state is +// transferred before its Dokploy deployment exists. in that order the +// target never runs while Bort copies data, so no target writer pause is +// needed and the deployed compose must mount the staging volumes. +func appStateIsStaged(plan Plan, appName string) bool { + pushIndex, lastStateIndex := -1, -1 + for index, step := range plan.Steps { + if step.App != appName { + continue + } + switch step.Kind { + case StepPushImage: + pushIndex = index + case StepPauseSource, StepDumpDataStore, StepRestoreDataStore, StepSyncVolume: + lastStateIndex = index + } + } + return lastStateIndex >= 0 && pushIndex > lastStateIndex +} + +func stagedVolumesForApp(plan Plan, appName string) []stagedVolume { + if !appStateIsStaged(plan, appName) { + return nil + } + app, ok := findPrepareApp(plan.Prepare, appName) + if !ok { + return nil + } + seen := map[string]struct{}{} + staged := []stagedVolume{} + add := func(volume preparer.VolumeResource) { + if volume.Type != "volume" || strings.TrimSpace(volume.Service) == "" || strings.TrimSpace(volume.Target) == "" { + return + } + key := migratedMountKey(volume.Service, volume.Target) + if _, dup := seen[key]; dup { + return + } + seen[key] = struct{}{} + staged = append(staged, stagedVolume{ + Service: volume.Service, + Target: volume.Target, + VolumeName: stagingVolumeName(plan, appName, volume.Service, volume.Target), + Source: volume, + }) + } + for _, step := range plan.Steps { + if step.App != appName { + continue + } + switch step.Kind { + case StepSyncVolume: + if volume, ok := findPrepareVolume(app, step.Ref); ok { + add(volume) + } + case StepRestoreDataStore: + store, ok := findPrepareDataStore(app, step.Ref) + if !ok { + continue + } + for _, volume := range app.Resources.Volumes { + if volume.Service == store.Service { + add(volume) + } + } + } + } + return staged +} + +func stagedVolumesForService(plan Plan, appName, service string) []stagedVolume { + volumes := []stagedVolume{} + for _, volume := range stagedVolumesForApp(plan, appName) { + if volume.Service == service { + volumes = append(volumes, volume) + } + } + return volumes +} + +func stagedVolumeFor(plan Plan, appName string, volume preparer.VolumeResource) (stagedVolume, bool) { + for _, staged := range stagedVolumesForApp(plan, appName) { + if staged.Service == volume.Service && staged.Target == volume.Target { + return staged, true + } + } + return stagedVolume{}, false +} + +func stagingVolumeName(plan Plan, appName, service, target string) string { + return strings.Join([]string{ + "bort", + dockerNameSegment(plan.RunName, 24), + dockerNameSegment(appName, 24), + dockerNameSegment(service, 24), + stagingHash(stagingOwner(plan), appName, service, target), + }, "-") +} + +func stagingProjectName(plan Plan, appName, service string) string { + return "bort-stage-" + dockerNameSegment(appName, 24) + "-" + dockerNameSegment(service, 24) + "-" + stagingHash(stagingOwner(plan), appName, service) +} + +func stagingHash(parts ...string) string { + sum := sha256.Sum256([]byte(strings.Join(parts, "\x00"))) + return hex.EncodeToString(sum[:])[:8] +} + +func dockerNameSegment(value string, limit int) string { + var b strings.Builder + lastDash := true + for _, r := range strings.ToLower(strings.TrimSpace(value)) { + switch { + case r >= 'a' && r <= 'z', r >= '0' && r <= '9': + b.WriteRune(r) + lastDash = false + default: + if !lastDash { + b.WriteRune('-') + lastDash = true + } + } + if b.Len() >= limit { + break + } + } + segment := strings.Trim(b.String(), "-") + if segment == "" { + return "x" + } + return segment +} + +func stagingVolumeLabels(plan Plan, appName string, volume stagedVolume) []string { + return []string{ + stagingVolumeRunLabel + "=" + plan.RunName, + stagingVolumeRunIDLabel + "=" + stagingOwner(plan), + stagingVolumeAppLabel + "=" + appName, + stagingVolumeServiceLabel + "=" + volume.Service, + stagingVolumeTargetLabel + "=" + volume.Target, + } +} + +func stagingVolumeExists(ctx context.Context, runner dockerRunner, plan Plan, volume stagedVolume) (bool, error) { + out, err := runner.Output(ctx, "volume", "inspect", "--format", "{{index .Labels \""+stagingVolumeRunIDLabel+"\"}}", volume.VolumeName) + if err != nil { + if isDockerVolumeOrNetworkMissingErr(err) { + return false, nil + } + return false, fmt.Errorf("inspect staging volume %s: %w", volume.VolumeName, err) + } + if owner := strings.TrimSpace(string(out)); owner != stagingOwner(plan) { + return true, fmt.Errorf("docker volume %s already exists but is not owned by run %q (label %s=%q)", volume.VolumeName, plan.RunName, stagingVolumeRunIDLabel, owner) + } + return true, nil +} + +func ensureStagingVolume(ctx context.Context, runner dockerRunner, plan Plan, appName string, volume stagedVolume) error { + exists, err := stagingVolumeExists(ctx, runner, plan, volume) + if err != nil || exists { + return err + } + args := []string{"volume", "create"} + for _, label := range stagingVolumeLabels(plan, appName, volume) { + args = append(args, "--label", label) + } + args = append(args, volume.VolumeName) + if _, err := runner.Output(ctx, args...); err != nil { + return fmt.Errorf("create staging volume %s: %w", volume.VolumeName, err) + } + return nil +} + +func recreateStagingVolume(ctx context.Context, runner dockerRunner, plan Plan, appName string, volume stagedVolume) error { + exists, err := stagingVolumeExists(ctx, runner, plan, volume) + if err != nil { + return err + } + if exists { + if _, err := runner.Output(ctx, "volume", "rm", "-f", volume.VolumeName); err != nil { + return fmt.Errorf("remove staging volume %s: %w", volume.VolumeName, err) + } + } + return ensureStagingVolume(ctx, runner, plan, appName, volume) +} + +func requireStagingVolumeUnattached(ctx context.Context, runner dockerRunner, volume stagedVolume) error { + out, err := runner.Output(ctx, "ps", "-a", "--filter", "volume="+volume.VolumeName, "-q") + if err != nil { + return fmt.Errorf("list containers using staging volume %s: %w", volume.VolumeName, err) + } + if ids := strings.Fields(string(out)); len(ids) > 0 { + return fmt.Errorf("staging volume %s is attached to container(s) %s before Bort handed it to Dokploy; refusing to treat the transferred state as authoritative", volume.VolumeName, strings.Join(ids, ", ")) + } + return nil +} + +const defaultPostgresDataDir = "/var/lib/postgresql/data" + +// requireStagedPostgresDataDir refuses a staging restore whose data +// directory is not backed by one of the staged volumes: the restore +// would land in the container layer and vanish with it. +func requireStagedPostgresDataDir(container dockerContainer, staged []stagedVolume) error { + dataDir := postgresDataDir(container) + var mount dockerMount + found := false + for _, candidate := range container.Mounts { + if !mountCoversPath(candidate.Destination, dataDir) { + continue + } + if !found || len(candidate.Destination) > len(mount.Destination) { + mount, found = candidate, true + } + } + if !found { + return fmt.Errorf("postgres data directory %s is not mounted from a staged volume; the restore would be lost when the staging container stops", dataDir) + } + for _, volume := range staged { + if mount.Type == "volume" && mount.Name == volume.VolumeName { + return nil + } + } + return fmt.Errorf("postgres data directory %s is mounted from %s %q, not a staged volume; the restore would be lost when the staging container stops", dataDir, mount.Type, firstNonEmpty(mount.Name, mount.Source)) +} + +// requirePlannedPostgresDataDirStaged refuses at plan time only the +// layouts requireStagedPostgresDataDir is certain to refuse after +// pause_source: a service with nothing to stage, or a literal PGDATA that +// no scanned named volume covers. Interpolated or image-provided PGDATA +// and image VOLUMEs are only known to the created staging container, so +// preflightStagedRestores checks those before the source is paused. +// The mount check reads the source container's mount list, so a mount +// target that compose interpolates could land elsewhere on the target and +// is refused outright. +func requirePlannedPostgresDataDirStaged(composeFile, stagingCompose string, app preparer.AppPlan, service string, staged []stagedVolume) error { + if len(staged) == 0 { + return fmt.Errorf("%w: data store service %s for app %s mounts no named volume, so its restore would be lost when the staging container stops; choose a recreate or managed data store strategy or change the source compose before live apply", ErrNotImplemented, service, app.Name) + } + if err := requireLiteralMountTargets(composeFile, app.Name, service); err != nil { + return fmt.Errorf("%w; choose a recreate or managed data store strategy or change the source compose before live apply", err) + } + var doc yaml.Node + if err := yaml.Unmarshal([]byte(stagingCompose), &doc); err != nil { + return err + } + root, err := composeRoot(&doc) + if err != nil { + return err + } + raw := strings.TrimSpace(composeServiceEnvValue(mappingValue(mappingValue(root, "services"), service), "PGDATA")) + if raw == "" || strings.Contains(raw, "$") { + return nil + } + if err := requireSourceMountsStageDataDir(app, service, path.Clean(raw), staged); err != nil { + return fmt.Errorf("%w; choose a recreate or managed data store strategy or change the source compose before live apply", err) + } + return nil +} + +func requireLiteralMountTargets(composeFile, appName, service string) error { + if target, ok := composeServiceInterpolatedMountTarget(composeFile, service); ok { + return fmt.Errorf("%w: data store service %s for app %s mounts %q at an interpolated path, so the postgres data directory cannot be checked", ErrNotImplemented, service, appName, target) + } + return nil +} + +func composeServiceInterpolatedMountTarget(composeFile, service string) (string, bool) { + var doc yaml.Node + if err := yaml.Unmarshal([]byte(composeFile), &doc); err != nil { + return "", false + } + root, err := composeRoot(&doc) + if err != nil { + return "", false + } + entry := mappingValue(mappingValue(root, "services"), service) + if entry == nil { + return "", false + } + if entry, err = selfContainedNode(entry, map[*yaml.Node]bool{}); err != nil { + return "", false + } + volumes := mappingValue(entry, "volumes") + if volumes == nil || volumes.Kind != yaml.SequenceNode { + return "", false + } + for _, item := range volumes.Content { + var target string + switch item.Kind { + case yaml.ScalarNode: + fields := splitComposeVolumeShortSyntax(item.Value) + target = fields[0] + if len(fields) > 1 { + target = fields[1] + } + case yaml.MappingNode: + if dest := mappingValue(item, "target"); dest != nil { + target = dest.Value + } + } + if strings.Contains(target, "$") { + return strings.TrimSpace(target), true + } + } + return "", false +} + +// splitComposeVolumeShortSyntax splits SOURCE:TARGET[:MODE] on colons that +// are not inside a ${...} reference, where ${VAR:-default} keeps its colon. +func splitComposeVolumeShortSyntax(value string) []string { + fields := []string{} + depth, start := 0, 0 + for i := 0; i < len(value); i++ { + switch { + case value[i] == '$' && i+1 < len(value) && value[i+1] == '{': + depth++ + i++ + case value[i] == '}' && depth > 0: + depth-- + case value[i] == ':' && depth == 0: + fields = append(fields, value[start:i]) + start = i + 1 + } + } + return append(fields, value[start:]) +} + +// requireSourceMountsStageDataDir checks the source service's own mount +// list, which the deployed compose keeps verbatim: a bind at, above, or +// inside the data directory would shadow the restored data once Dokploy +// starts the service, even though the staging container never mounts it. +func requireSourceMountsStageDataDir(app preparer.AppPlan, service, dataDir string, staged []stagedVolume) error { + var mount preparer.VolumeResource + found := false + for _, candidate := range app.Resources.Volumes { + if candidate.Service != service { + continue + } + if mountCoversPath(candidate.Target, dataDir) { + if !found || len(path.Clean(candidate.Target)) > len(path.Clean(mount.Target)) { + mount, found = candidate, true + } + continue + } + if mountCoversPath(dataDir, candidate.Target) && candidate.ReadWrite && !stagedVolumeMount(candidate, staged) { + return fmt.Errorf("%w: %s %q at %s for service %s in app %s sits inside postgres data directory %s and would shadow the restored data once Dokploy starts the service", ErrNotImplemented, candidate.Type, firstNonEmpty(candidate.Name, candidate.Source), candidate.Target, service, app.Name, dataDir) + } + } + if !found { + return fmt.Errorf("%w: postgres data directory %s for service %s in app %s is not mounted from a named volume, so the restore would be lost when the staging container stops", ErrNotImplemented, dataDir, service, app.Name) + } + if stagedVolumeMount(mount, staged) { + return nil + } + return fmt.Errorf("%w: postgres data directory %s for service %s in app %s is mounted from %s %q at %s, not a named volume, so the restore would be lost when the staging container stops", ErrNotImplemented, dataDir, service, app.Name, mount.Type, firstNonEmpty(mount.Name, mount.Source), mount.Target) +} + +func stagedVolumeMount(mount preparer.VolumeResource, staged []stagedVolume) bool { + if mount.Type != "volume" { + return false + } + for _, volume := range staged { + if path.Clean(volume.Target) == path.Clean(mount.Target) { + return true + } + } + return false +} + +func postgresDataDir(container dockerContainer) string { + if dataDir := strings.TrimSpace(envMap(container.Config.Env)["PGDATA"]); dataDir != "" { + return path.Clean(dataDir) + } + return defaultPostgresDataDir +} + +func mountCoversPath(destination, target string) bool { + destination, target = path.Clean(destination), path.Clean(target) + return destination == target || strings.HasPrefix(target, strings.TrimSuffix(destination, "/")+"/") +} + +func composeServiceEnvValue(service *yaml.Node, key string) string { + environment := mappingValue(service, "environment") + if environment == nil { + return "" + } + switch environment.Kind { + case yaml.MappingNode: + if value := mappingValue(environment, key); value != nil && value.Kind == yaml.ScalarNode && value.Tag != "!!null" { + return value.Value + } + case yaml.SequenceNode: + for _, item := range environment.Content { + if name, value, ok := strings.Cut(item.Value, "="); ok && item.Kind == yaml.ScalarNode && strings.TrimSpace(name) == key { + return value + } + } + } + return "" +} + +func requireSourceQuiescent(containers []dockerContainer) error { + for _, container := range containers { + if container.State.Running { + return fmt.Errorf("source container %s is running while its state is being copied; pause_source must stop it first", container.Name) + } + } + return nil +} + +func requireSourceQuiesceUnchanged(before, after []dockerContainer) error { + if len(before) != len(after) { + return fmt.Errorf("source container set changed during state copy (%d before, %d after)", len(before), len(after)) + } + for index := range before { + b, a := before[index], after[index] + switch { + case b.ID != a.ID: + return fmt.Errorf("source container %s was replaced by %s during state copy", b.Name, a.Name) + case a.State.Running: + return fmt.Errorf("source container %s started during state copy; the copied state may be inconsistent", a.Name) + case b.State.FinishedAt != a.State.FinishedAt || b.State.StartedAt != a.State.StartedAt: + return fmt.Errorf("source container %s ran during state copy (started %s, finished %s); the copied state may be inconsistent", a.Name, a.State.StartedAt, a.State.FinishedAt) + } + } + return nil +} + +func composeServiceVolumeKeys(root *yaml.Node, service string) map[string]string { + keys := map[string]string{} + services := mappingValue(root, "services") + entry := mappingValue(services, service) + volumes := mappingValue(entry, "volumes") + if volumes == nil || volumes.Kind != yaml.SequenceNode { + return keys + } + for _, item := range volumes.Content { + key, target, ok := composeVolumeEntry(item) + if ok { + keys[target] = key + } + } + return keys +} + +func composeVolumeEntry(item *yaml.Node) (key, target string, ok bool) { + switch item.Kind { + case yaml.ScalarNode: + parts := strings.SplitN(item.Value, ":", 3) + if len(parts) < 2 { + return "", "", false + } + key, target = strings.TrimSpace(parts[0]), strings.TrimSpace(parts[1]) + if key == "" || target == "" || strings.ContainsAny(key[:1], "/.~$") { + return "", "", false + } + return key, target, true + case yaml.MappingNode: + kind := mappingValue(item, "type") + if kind == nil || kind.Value != "volume" { + return "", "", false + } + source, dest := mappingValue(item, "source"), mappingValue(item, "target") + if source == nil || dest == nil || strings.TrimSpace(source.Value) == "" || strings.TrimSpace(dest.Value) == "" { + return "", "", false + } + return strings.TrimSpace(source.Value), strings.TrimSpace(dest.Value), true + default: + return "", "", false + } +} + +const composeInitScriptsDir = "/docker-entrypoint-initdb.d" + +func composeServiceInitBindMounts(entry *yaml.Node) ([]*yaml.Node, error) { + volumes := mappingValue(entry, "volumes") + if volumes == nil || volumes.Kind != yaml.SequenceNode { + return nil, nil + } + mounts := []*yaml.Node{} + for _, item := range volumes.Content { + var source, target string + var options []string + switch item.Kind { + case yaml.ScalarNode: + parts := strings.SplitN(item.Value, ":", 3) + if len(parts) < 2 { + continue + } + source, target = strings.TrimSpace(parts[0]), strings.TrimSpace(parts[1]) + if source == "" || !strings.ContainsAny(source[:1], "/.~$") { + continue + } + if len(parts) == 3 { + options = strings.Split(parts[2], ",") + } + case yaml.MappingNode: + if kind := mappingValue(item, "type"); kind == nil || kind.Value != "bind" { + continue + } + src, dest := mappingValue(item, "source"), mappingValue(item, "target") + if src == nil || dest == nil { + continue + } + source, target = strings.TrimSpace(src.Value), strings.TrimSpace(dest.Value) + default: + continue + } + cleaned := path.Clean(target) + if cleaned != composeInitScriptsDir && !strings.HasPrefix(cleaned, composeInitScriptsDir+"/") { + continue + } + if !strings.HasPrefix(source, "/") { + return nil, fmt.Errorf("init script mount %s -> %s must use an absolute host path so the staged restore runs the same scripts as the deploy", source, target) + } + if item.Kind == yaml.MappingNode { + ensureMappingBool(item, "read_only", true) + mounts = append(mounts, item) + continue + } + mounts = append(mounts, stringNode(source+":"+target+":"+strings.Join(append(withoutAccessMode(options), "ro"), ","))) + } + return mounts, nil +} + +func withoutAccessMode(options []string) []string { + kept := options[:0:0] + for _, option := range options { + if option := strings.TrimSpace(option); option != "" && option != "ro" && option != "rw" { + kept = append(kept, option) + } + } + return kept +} + +func externalVolumeNode(name string) *yaml.Node { + return &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map", Content: []*yaml.Node{ + stringNode("name"), stringNode(name), + stringNode("external"), {Kind: yaml.ScalarNode, Tag: "!!bool", Value: "true"}, + }} +} + +func composeRoot(doc *yaml.Node) (*yaml.Node, error) { + root := doc + if doc.Kind == yaml.DocumentNode && len(doc.Content) > 0 { + root = doc.Content[0] + } + if root.Kind != yaml.MappingNode { + return nil, fmt.Errorf("compose file is not a mapping") + } + return root, nil +} + +// rewriteComposeStagedVolumes points the compose file's top-level volume +// entries at the Bort-owned staging volumes so Dokploy's deploy mounts +// the transferred state instead of creating fresh project volumes. +func rewriteComposeStagedVolumes(composeFile string, staged []stagedVolume) (string, error) { + if len(staged) == 0 { + return composeFile, nil + } + var doc yaml.Node + if err := yaml.Unmarshal([]byte(composeFile), &doc); err != nil { + return "", err + } + root, err := composeRoot(&doc) + if err != nil { + return "", err + } + topLevel := mappingValue(root, "volumes") + if topLevel == nil || topLevel.Kind != yaml.MappingNode { + topLevel = &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map"} + setMappingNode(root, "volumes", topLevel) + } + assigned := map[string]stagedVolume{} + for _, volume := range staged { + key, ok := composeServiceVolumeKeys(root, volume.Service)[volume.Target] + if !ok { + return "", fmt.Errorf("compose service %s has no named volume mounted at %s; cannot hand staged volume %s to Dokploy", volume.Service, volume.Target, volume.VolumeName) + } + if other, dup := assigned[key]; dup && other.VolumeName != volume.VolumeName { + return "", fmt.Errorf("compose volume %s is mounted by %s:%s and %s:%s, which would stage as separate volumes %s and %s; a shared named volume cannot be transferred before deploy", key, other.Service, other.Target, volume.Service, volume.Target, other.VolumeName, volume.VolumeName) + } + assigned[key] = volume + setMappingNode(topLevel, key, externalVolumeNode(volume.VolumeName)) + } + out, err := yaml.Marshal(&doc) + if err != nil { + return "", err + } + return string(out), nil +} + +func stagingComposeFile(composeFile, service string, staged []stagedVolume) (string, error) { + var doc yaml.Node + if err := yaml.Unmarshal([]byte(composeFile), &doc); err != nil { + return "", err + } + root, err := composeRoot(&doc) + if err != nil { + return "", err + } + services := mappingValue(root, "services") + entry := mappingValue(services, service) + if entry != nil { + if entry, err = selfContainedNode(entry, map[*yaml.Node]bool{}); err != nil { + return "", fmt.Errorf("compose service %s: %w", service, err) + } + } + if entry == nil || entry.Kind != yaml.MappingNode { + return "", fmt.Errorf("compose service %s not found", service) + } + if mappingValue(entry, "extends") != nil { + return "", fmt.Errorf("compose service %s uses extends, which Bort cannot resolve for staging", service) + } + setMappingNode(services, service, entry) + keys := composeServiceVolumeKeys(root, service) + mounts := &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"} + topLevel := &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map"} + for _, volume := range staged { + key, ok := keys[volume.Target] + if !ok { + return "", fmt.Errorf("compose service %s has no named volume mounted at %s; cannot stage volume %s", service, volume.Target, volume.VolumeName) + } + mounts.Content = append(mounts.Content, stringNode(key+":"+volume.Target)) + setMappingNode(topLevel, key, externalVolumeNode(volume.VolumeName)) + } + initMounts, err := composeServiceInitBindMounts(entry) + if err != nil { + return "", fmt.Errorf("compose service %s: %w", service, err) + } + mounts.Content = append(mounts.Content, initMounts...) + for _, key := range []string{"ports", "depends_on", "container_name", "networks", "network_mode", "secrets", "configs", "links", "profiles", "build"} { + removeMappingKey(entry, key) + } + setMappingNode(entry, "volumes", mounts) + setMappingScalar(entry, "restart", "no") + root.Content = []*yaml.Node{ + stringNode("services"), {Kind: yaml.MappingNode, Tag: "!!map", Content: []*yaml.Node{stringNode(service), entry}}, + stringNode("volumes"), topLevel, + } + out, err := yaml.Marshal(&doc) + if err != nil { + return "", err + } + return string(out), nil +} + +// selfContainedNode copies a subtree with every alias expanded and every +// merge key applied, so it stays valid after the anchors it referenced are +// pruned from the document. +func selfContainedNode(node *yaml.Node, expanding map[*yaml.Node]bool) (*yaml.Node, error) { + if node.Kind == yaml.AliasNode { + if node.Alias == nil || expanding[node.Alias] { + return nil, fmt.Errorf("alias *%s cannot be expanded", node.Value) + } + expanding[node.Alias] = true + defer delete(expanding, node.Alias) + return selfContainedNode(node.Alias, expanding) + } + copied := *node + copied.Anchor = "" + copied.Content = nil + for _, child := range node.Content { + resolved, err := selfContainedNode(child, expanding) + if err != nil { + return nil, err + } + copied.Content = append(copied.Content, resolved) + } + if copied.Kind == yaml.MappingNode { + applyMergeKeys(&copied) + } + return &copied, nil +} + +func applyMergeKeys(mapping *yaml.Node) { + var content, merged []*yaml.Node + for i := 0; i+1 < len(mapping.Content); i += 2 { + key, value := mapping.Content[i], mapping.Content[i+1] + if key.Tag != "!!merge" { + content = append(content, key, value) + continue + } + if value.Kind == yaml.SequenceNode { + merged = append(merged, value.Content...) + } else { + merged = append(merged, value) + } + } + mapping.Content = content + for _, source := range merged { + for i := 0; i+1 < len(source.Content); i += 2 { + if mappingValue(mapping, source.Content[i].Value) == nil { + mapping.Content = append(mapping.Content, source.Content[i], source.Content[i+1]) + } + } + } +} + +type stagingEnvFormat int + +const ( + stagingEnvFormatUnresolved stagingEnvFormat = iota + stagingEnvFormatRaw + stagingEnvFormatEscapeEveryDollar + stagingEnvFormatKeepInterpolation +) + +var dokployReleaseVersionPattern = regexp.MustCompile(`^v?(\d{1,6})\.(\d{1,6})\.(\d{1,6})$`) + +// stagingEnvFormatForDokployVersion picks the .env writer Dokploy's own +// compose deploy uses at that version: raw KEY=value before v0.30.0, quoted +// with every $ escaped through v0.30.2, quoted with ${VAR} kept from v0.30.3. +func stagingEnvFormatForDokployVersion(version string) (stagingEnvFormat, error) { + match := dokployReleaseVersionPattern.FindStringSubmatch(strings.TrimSpace(version)) + if match == nil { + return stagingEnvFormatUnresolved, fmt.Errorf("Dokploy reported version %q, want vMAJOR.MINOR.PATCH; staged state transfer needs a release version to write the same .env Dokploy will", version) + } + major, _ := strconv.Atoi(match[1]) + minor, _ := strconv.Atoi(match[2]) + patch, _ := strconv.Atoi(match[3]) + switch { + case major > 0 || minor > 30 || minor == 30 && patch >= 3: + return stagingEnvFormatKeepInterpolation, nil + case minor == 30: + return stagingEnvFormatEscapeEveryDollar, nil + default: + return stagingEnvFormatRaw, nil + } +} + +func planStagesDataStoreRestore(plan Plan) bool { + for _, step := range plan.Steps { + if step.Kind == StepRestoreDataStore && appStateIsStaged(plan, step.App) { + return true + } + } + return false +} + +// stagingEnvFileContent mirrors the .env Dokploy writes next to a compose +// deployment so interpolation in the staged service resolves identically. +func stagingEnvFileContent(composeAppName, envContent string, format stagingEnvFormat) string { + content := "APP_NAME=" + composeAppName + "\nCOMPOSE_PROJECT_NAME=" + composeAppName + "\n" + envContent + if !strings.Contains(content, "DOCKER_CONFIG") { + content += "\nDOCKER_CONFIG=/root/.docker" + } + lines := []string{} + for _, raw := range strings.Split(content, "\n") { + key, value, ok := parseDotenvLine(raw) + if !ok { + continue + } + switch format { + case stagingEnvFormatRaw: + lines = append(lines, key+"="+value) + case stagingEnvFormatEscapeEveryDollar: + lines = append(lines, key+"=\""+escapeDotenvValue(value, false)+"\"") + default: + lines = append(lines, key+"=\""+escapeDotenvValue(value, true)+"\"") + } + } + return strings.Join(lines, "\n") + "\n" +} + +func parseDotenvLine(raw string) (string, string, bool) { + line := strings.TrimSpace(raw) + if line == "" || strings.HasPrefix(line, "#") { + return "", "", false + } + line = strings.TrimSpace(strings.TrimPrefix(line, "export ")) + key, value, ok := strings.Cut(line, "=") + key = strings.TrimSpace(key) + if !ok || !dotenvKey(key) { + return "", "", false + } + value = strings.TrimSpace(value) + if inner, ok := dotenvQuotedValue(value); ok { + return key, inner, true + } + if comment := strings.IndexByte(value, '#'); comment >= 0 { + value = value[:comment] + } + return key, strings.TrimSpace(value), true +} + +func dotenvKey(key string) bool { + if key == "" { + return false + } + for _, r := range key { + switch { + case r == '_', r == '.', r == '-', r >= 'A' && r <= 'Z', r >= 'a' && r <= 'z', r >= '0' && r <= '9': + default: + return false + } + } + return true +} + +func dotenvQuotedValue(value string) (string, bool) { + if len(value) < 2 { + return "", false + } + quote := value[0] + if quote != '"' && quote != '\'' && quote != '`' { + return "", false + } + end := -1 + for index := 1; index < len(value); index++ { + if value[index] == '\\' && index+1 < len(value) && value[index+1] == quote { + index++ + continue + } + if value[index] == quote { + end = index + break + } + } + if end < 0 { + return "", false + } + if rest := strings.TrimSpace(value[end+1:]); rest != "" && !strings.HasPrefix(rest, "#") { + return "", false + } + inner := value[1:end] + if quote == '"' { + inner = strings.ReplaceAll(inner, `\n`, "\n") + inner = strings.ReplaceAll(inner, `\r`, "\r") + } + return inner, true +} + +func escapeDotenvValue(value string, keepInterpolation bool) string { + value = strings.ReplaceAll(value, `\`, `\\`) + value = strings.ReplaceAll(value, `"`, `\"`) + var b strings.Builder + for index := 0; index < len(value); index++ { + if value[index] == '$' && !(keepInterpolation && dotenvInterpolationAt(value[index:])) { + b.WriteString(`\$`) + continue + } + b.WriteByte(value[index]) + } + return b.String() +} + +func dotenvInterpolationAt(value string) bool { + if len(value) < 3 || value[1] != '{' || !dotenvNameStart(value[2]) { + return false + } + index := 3 + for index < len(value) && (dotenvNameStart(value[index]) || value[index] >= '0' && value[index] <= '9') { + index++ + } + if index < len(value) && value[index] == '}' { + return true + } + if index < len(value) && value[index] == ':' { + index++ + } + if index >= len(value) || !strings.ContainsRune("-+?", rune(value[index])) { + return false + } + for index++; index < len(value); index++ { + switch value[index] { + case '}': + return true + case '{': + return false + } + } + return false +} + +func dotenvNameStart(c byte) bool { + return c == '_' || c >= 'A' && c <= 'Z' || c >= 'a' && c <= 'z' +} + +type stagingProject struct { + name string + composePath string + envPath string +} + +func (p stagingProject) args(extra ...string) []string { + return append([]string{"compose", "-p", p.name, "--env-file", p.envPath, "-f", p.composePath}, extra...) +} + +func (p stagingProject) down(runner dockerRunner) error { + ctx, cancel := context.WithTimeout(context.Background(), dockerStopTimeout+dockerStartTimeout) + defer cancel() + if err := runner.Run(ctx, nil, nil, p.args("down", "--remove-orphans")...); err != nil { + return fmt.Errorf("stop staging compose project %s: %w", p.name, err) + } + return nil +} + +func writeStagingProject(plan Plan, appName, service, composeFile, envContent string) (stagingProject, error) { + if strings.TrimSpace(plan.RunDir) == "" { + return stagingProject{}, fmt.Errorf("plan.RunDir is empty; cannot stage compose project for %s/%s", appName, service) + } + dir := filepath.Join(plan.RunDir, "stage", safeDataPathSegment(appName), safeDataPathSegment(service)) + if err := safepath.ContainedPath(plan.RunDir, dir); err != nil { + return stagingProject{}, err + } + if err := os.MkdirAll(dir, 0o700); err != nil { + return stagingProject{}, fmt.Errorf("prepare staging dir: %w", err) + } + project := stagingProject{ + name: stagingProjectName(plan, appName, service), + composePath: filepath.Join(dir, "compose.yaml"), + envPath: filepath.Join(dir, ".env"), + } + if err := os.WriteFile(project.composePath, []byte(composeFile), 0o600); err != nil { + return stagingProject{}, fmt.Errorf("write staging compose: %w", err) + } + if err := os.WriteFile(project.envPath, []byte(envContent), 0o600); err != nil { + return stagingProject{}, fmt.Errorf("write staging env: %w", err) + } + return project, nil +} + +func (p stagingProject) serviceContainer(ctx context.Context, runner dockerRunner, service string) (dockerContainer, error) { + deadline := time.Now().Add(targetDiscoveryTimeout) + for { + out, err := runner.Output(ctx, p.args("ps", "-a", "-q", service)...) + if err != nil { + return dockerContainer{}, fmt.Errorf("find staging container for %s: %w", service, err) + } + if ids := strings.Fields(string(out)); len(ids) == 1 { + return inspectContainer(ctx, runner, ids[0]) + } else if len(ids) > 1 { + return dockerContainer{}, fmt.Errorf("staging compose project %s has %d containers for service %s", p.name, len(ids), service) + } + if time.Now().After(deadline) { + return dockerContainer{}, fmt.Errorf("staging compose project %s has no container for service %s after %s", p.name, service, targetDiscoveryTimeout) + } + select { + case <-ctx.Done(): + return dockerContainer{}, ctx.Err() + case <-time.After(targetDiscoveryDelay): + } + } +} diff --git a/internal/target/dokploy/staging_test.go b/internal/target/dokploy/staging_test.go new file mode 100644 index 0000000..2cb606a --- /dev/null +++ b/internal/target/dokploy/staging_test.go @@ -0,0 +1,1624 @@ +package dokploy + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "runtime" + "slices" + "strings" + "testing" + + "github.com/aikins01/bort/internal/preparer" + "gopkg.in/yaml.v3" +) + +type sequencedOutputRunner struct { + fakeDockerRunner + sequences map[string][][]byte +} + +func (r *sequencedOutputRunner) Output(ctx context.Context, args ...string) ([]byte, error) { + key := strings.Join(args, " ") + if seq, ok := r.sequences[key]; ok && len(seq) > 0 { + out := seq[0] + if len(seq) > 1 { + r.sequences[key] = seq[1:] + } + r.outputArgs = append(r.outputArgs, append([]string{}, args...)) + return out, nil + } + return r.fakeDockerRunner.Output(ctx, args...) +} + +func stagedPlan(t *testing.T, app preparer.AppPlan, runDir string, steps ...Step) Plan { + t.Helper() + return Plan{ + RunName: "run1", + RunDir: runDir, + Prepare: preparer.Result{Apps: []preparer.AppPlan{app}}, + Steps: append(steps, Step{Kind: StepPushImage, App: app.Name}), + } +} + +func decodeCompose(t *testing.T, compose string) map[string]any { + t.Helper() + var doc map[string]any + if err := yaml.Unmarshal([]byte(compose), &doc); err != nil { + t.Fatalf("decode compose: %v\n%s", err, compose) + } + return doc +} + +func composeVolumeDef(t *testing.T, doc map[string]any, key string) map[string]any { + t.Helper() + volumes, _ := doc["volumes"].(map[string]any) + def, ok := volumes[key].(map[string]any) + if !ok { + t.Fatalf("top-level volume %q missing or not a mapping: %#v", key, doc["volumes"]) + } + return def +} + +func TestRewriteComposeStagedVolumesHandlesShortAndLongSyntax(t *testing.T) { + compose := "services:\n db:\n image: postgres:16\n volumes:\n - pgdata:/var/lib/postgresql/data\n - ./local:/host\n web:\n image: example/web\n volumes:\n - type: volume\n source: uploads\n target: /uploads\n - type: bind\n source: ./cfg\n target: /cfg\nvolumes:\n pgdata:\n uploads:\n driver: local\n" + staged := []stagedVolume{ + {Service: "db", Target: "/var/lib/postgresql/data", VolumeName: "bort-run1-app-db-aaaaaaaa"}, + {Service: "web", Target: "/uploads", VolumeName: "bort-run1-app-web-bbbbbbbb"}, + } + out, err := rewriteComposeStagedVolumes(compose, staged) + if err != nil { + t.Fatalf("rewriteComposeStagedVolumes: %v", err) + } + doc := decodeCompose(t, out) + for key, want := range map[string]string{"pgdata": "bort-run1-app-db-aaaaaaaa", "uploads": "bort-run1-app-web-bbbbbbbb"} { + def := composeVolumeDef(t, doc, key) + if def["name"] != want || def["external"] != true { + t.Fatalf("volume %s not rewritten to external staging volume: %#v", key, def) + } + } + services := doc["services"].(map[string]any) + dbVolumes := services["db"].(map[string]any)["volumes"].([]any) + if len(dbVolumes) != 2 || dbVolumes[0] != "pgdata:/var/lib/postgresql/data" { + t.Fatalf("service mounts must be left untouched, got %#v", dbVolumes) + } + + _, err = rewriteComposeStagedVolumes(compose, []stagedVolume{{Service: "web", Target: "/cfg", VolumeName: "bort-x"}}) + if err == nil || !strings.Contains(err.Error(), "no named volume mounted at /cfg") { + t.Fatalf("expected error for bind-mounted target, got %v", err) + } +} + +func TestStagingComposeFileIsolatesDataStoreService(t *testing.T) { + compose := "services:\n db:\n image: postgres:16\n container_name: prod-db\n restart: always\n ports:\n - \"5432:5432\"\n depends_on:\n - redis\n networks:\n - backend\n environment:\n POSTGRES_PASSWORD: ${DB_PASSWORD}\n volumes:\n - pgdata:/var/lib/postgresql/data\n - pglogs:/var/log/postgresql\n - /srv/db/init:/docker-entrypoint-initdb.d\n - /srv/db/conf:/etc/postgresql\n redis:\n image: redis:7\nvolumes:\n pgdata:\n pglogs:\nnetworks:\n backend:\n" + staged := []stagedVolume{{Service: "db", Target: "/var/lib/postgresql/data", VolumeName: "bort-run1-app-db-aaaaaaaa"}} + out, err := stagingComposeFile(compose, "db", staged) + if err != nil { + t.Fatalf("stagingComposeFile: %v", err) + } + doc := decodeCompose(t, out) + if _, ok := doc["networks"]; ok { + t.Fatalf("top-level networks must be dropped:\n%s", out) + } + services := doc["services"].(map[string]any) + if len(services) != 1 { + t.Fatalf("staging compose must contain only the data store service, got %v", services) + } + db := services["db"].(map[string]any) + for _, key := range []string{"container_name", "ports", "depends_on", "networks"} { + if _, ok := db[key]; ok { + t.Fatalf("staging service must not carry %s:\n%s", key, out) + } + } + if db["restart"] != "no" { + t.Fatalf("staging service restart policy must be \"no\", got %#v", db["restart"]) + } + if env := db["environment"].(map[string]any); env["POSTGRES_PASSWORD"] != "${DB_PASSWORD}" { + t.Fatalf("environment must be preserved for interpolation, got %#v", env) + } + mounts := db["volumes"].([]any) + if len(mounts) != 2 || mounts[0] != "pgdata:/var/lib/postgresql/data" || mounts[1] != "/srv/db/init:/docker-entrypoint-initdb.d:ro" { + t.Fatalf("staging must mount only staged named volumes plus read-only init scripts, got %#v", mounts) + } + volumes := doc["volumes"].(map[string]any) + if len(volumes) != 1 { + t.Fatalf("only staged volumes may be declared, got %#v", volumes) + } + def := composeVolumeDef(t, doc, "pgdata") + if def["name"] != "bort-run1-app-db-aaaaaaaa" || def["external"] != true { + t.Fatalf("pgdata must point at the external staging volume, got %#v", def) + } +} + +func TestStagingComposeFileKeepsNestedAndLongFormInitScriptMounts(t *testing.T) { + compose := "services:\n db:\n image: postgres:16\n volumes:\n - pgdata:/var/lib/postgresql/data\n - /srv/db/roles.sql:/docker-entrypoint-initdb.d/01-roles.sql:rw,z\n - type: bind\n source: /srv/db/init\n target: /docker-entrypoint-initdb.d/extra\n bind:\n create_host_path: false\n - /srv/db/seed:/opt/../docker-entrypoint-initdb.d/seed\n - type: bind\n source: /srv/db/conf\n target: /etc/postgresql\n - /srv/db/initdb.dump:/docker-entrypoint-initdb.dump\n - /srv/db/other:/docker-entrypoint-initdb.d/../other\nvolumes:\n pgdata:\n" + staged := []stagedVolume{{Service: "db", Target: "/var/lib/postgresql/data", VolumeName: "bort-run1-app-db-aaaaaaaa"}} + out, err := stagingComposeFile(compose, "db", staged) + if err != nil { + t.Fatalf("stagingComposeFile: %v", err) + } + doc := decodeCompose(t, out) + mounts := doc["services"].(map[string]any)["db"].(map[string]any)["volumes"].([]any) + want := []any{ + "pgdata:/var/lib/postgresql/data", + "/srv/db/roles.sql:/docker-entrypoint-initdb.d/01-roles.sql:z,ro", + map[string]any{"type": "bind", "source": "/srv/db/init", "target": "/docker-entrypoint-initdb.d/extra", "bind": map[string]any{"create_host_path": false}, "read_only": true}, + "/srv/db/seed:/opt/../docker-entrypoint-initdb.d/seed:ro", + } + if fmt.Sprint(mounts) != fmt.Sprint(want) { + t.Fatalf("staging mounts = %#v, want %#v", mounts, want) + } +} + +func TestStagingComposeFileRefusesRelativeInitScriptMount(t *testing.T) { + compose := "services:\n db:\n image: postgres:16\n volumes:\n - pgdata:/var/lib/postgresql/data\n - ./init:/docker-entrypoint-initdb.d\nvolumes:\n pgdata:\n" + staged := []stagedVolume{{Service: "db", Target: "/var/lib/postgresql/data", VolumeName: "bort-run1-app-db-aaaaaaaa"}} + _, err := stagingComposeFile(compose, "db", staged) + if err == nil || !strings.Contains(err.Error(), "init script mount ./init -> /docker-entrypoint-initdb.d must use an absolute host path") { + t.Fatalf("expected relative init script mount refusal, got %v", err) + } +} + +func TestStagingComposeFileExpandsAnchorsAndMergeKeysFromPrunedSections(t *testing.T) { + compose := "x-pg-env: &pg-env\n POSTGRES_USER: bob\n POSTGRES_DB: shared\nx-common: &common\n restart: always\n logging: &log\n driver: json-file\nservices:\n db:\n <<: *common\n image: postgres:16\n environment:\n <<: *pg-env\n POSTGRES_DB: app\n volumes:\n - pgdata:/var/lib/postgresql/data\n web:\n image: example/web\n logging: *log\nvolumes:\n pgdata:\n" + staged := []stagedVolume{{Service: "db", Target: "/var/lib/postgresql/data", VolumeName: "bort-run1-app-db-aaaaaaaa"}} + out, err := stagingComposeFile(compose, "db", staged) + if err != nil { + t.Fatalf("stagingComposeFile: %v", err) + } + if strings.Contains(out, "*") || strings.Contains(out, "&") || strings.Contains(out, "<<") { + t.Fatalf("staging compose must not reference anchors or merge keys:\n%s", out) + } + db := decodeCompose(t, out)["services"].(map[string]any)["db"].(map[string]any) + env := db["environment"].(map[string]any) + if env["POSTGRES_USER"] != "bob" || env["POSTGRES_DB"] != "app" { + t.Fatalf("merged environment must keep explicit keys over anchored ones, got %#v", env) + } + if db["logging"].(map[string]any)["driver"] != "json-file" || db["restart"] != "no" { + t.Fatalf("service-level merge must apply and restart must still be overridden, got %#v", db) + } + + _, err = stagingComposeFile("services:\n db:\n extends:\n file: base.yaml\n service: pg\n volumes:\n - pgdata:/var/lib/postgresql/data\nvolumes:\n pgdata:\n", "db", staged) + if err == nil || !strings.Contains(err.Error(), "extends") { + t.Fatalf("extends must be refused rather than dropped, got %v", err) + } +} + +func TestStagingEnvFileContentQuotesAndEscapesLikeDokploy(t *testing.T) { + env := "# comment\nexport PLAIN=hello world\nPRICE=cost is $5 and ${PLAIN:-x} and $$\nQUOTED=\"say \\\"hi\\\"\"\n\nBROKEN_LINE\nPOSTGRES_USER=app # db user\nP=abc#123\nQ=\"a#b\" # c\nR=${X?must-be-set}\nS=${F+--v}\nT=\"unterminated # x\nU='single' trailing\nV=\"line\\nbreak\"\n" + out := stagingEnvFileContent("stack-1", env, stagingEnvFormatKeepInterpolation) + want := "APP_NAME=\"stack-1\"\nCOMPOSE_PROJECT_NAME=\"stack-1\"\nPLAIN=\"hello world\"\nPRICE=\"cost is \\$5 and ${PLAIN:-x} and \\$\\$\"\nQUOTED=\"say \\\\\\\"hi\\\\\\\"\"\nPOSTGRES_USER=\"app\"\nP=\"abc\"\nQ=\"a#b\"\nR=\"${X?must-be-set}\"\nS=\"${F+--v}\"\nT=\"\\\"unterminated\"\nU=\"'single' trailing\"\nV=\"line\nbreak\"\nDOCKER_CONFIG=\"/root/.docker\"\n" + if out != want { + t.Fatalf("env content mismatch\n got: %q\nwant: %q", out, want) + } + if got := stagingEnvFileContent("s", "DOCKER_CONFIG=/custom\n", stagingEnvFormatKeepInterpolation); strings.Count(got, "DOCKER_CONFIG=") != 1 || !strings.Contains(got, "DOCKER_CONFIG=\"/custom\"") { + t.Fatalf("existing DOCKER_CONFIG must not be overridden, got %q", got) + } + mixed := "PRICE=cost is $5 and ${PLAIN:-x}\nQUOTED=\"say \\\"hi\\\"\"\n" + if got, want := stagingEnvFileContent("s", mixed, stagingEnvFormatEscapeEveryDollar), "APP_NAME=\"s\"\nCOMPOSE_PROJECT_NAME=\"s\"\nPRICE=\"cost is \\$5 and \\${PLAIN:-x}\"\nQUOTED=\"say \\\\\\\"hi\\\\\\\"\"\nDOCKER_CONFIG=\"/root/.docker\"\n"; got != want { + t.Fatalf("v0.30.0 format mismatch\n got: %q\nwant: %q", got, want) + } + if got, want := stagingEnvFileContent("s", mixed, stagingEnvFormatRaw), "APP_NAME=s\nCOMPOSE_PROJECT_NAME=s\nPRICE=cost is $5 and ${PLAIN:-x}\nQUOTED=say \\\"hi\\\"\nDOCKER_CONFIG=/root/.docker\n"; got != want { + t.Fatalf("v0.29 format mismatch\n got: %q\nwant: %q", got, want) + } +} + +func TestStagingEnvFormatForDokployVersion(t *testing.T) { + for version, want := range map[string]stagingEnvFormat{ + "v0.29.3": stagingEnvFormatRaw, + "v0.30.0": stagingEnvFormatEscapeEveryDollar, + "v0.30.2": stagingEnvFormatEscapeEveryDollar, + "v0.30.3": stagingEnvFormatKeepInterpolation, + "v0.30.7": stagingEnvFormatKeepInterpolation, + "v0.31.0": stagingEnvFormatKeepInterpolation, + "1.0.0": stagingEnvFormatKeepInterpolation, + " v0.30.3": stagingEnvFormatKeepInterpolation, + } { + got, err := stagingEnvFormatForDokployVersion(version) + if err != nil || got != want { + t.Fatalf("version %q: got format %d err %v, want %d", version, got, err, want) + } + } + for _, version := range []string{"", "latest", "v0.30", "v0.30.3-canary", "canary"} { + if got, err := stagingEnvFormatForDokployVersion(version); err == nil || got != stagingEnvFormatUnresolved { + t.Fatalf("version %q must fail closed, got format %d err %v", version, got, err) + } + } +} + +func TestApplyResolvesDokployVersionBeforeStagedRestore(t *testing.T) { + plan, _, _ := stagedRestoreFixture(t) + for name, tc := range map[string]struct { + version string + status int + want string + }{ + "unparseable version": {version: `"latest"`, status: http.StatusOK, want: `Dokploy reported version "latest"`}, + "request failure": {version: `{"message":"Unauthorized"}`, status: http.StatusUnauthorized, want: "resolve Dokploy version before staged state transfer"}, + } { + t.Run(name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/api/settings.getDokployVersion" || r.Header.Get("x-api-key") != "secret" { + http.NotFound(w, r) + return + } + w.WriteHeader(tc.status) + _, _ = w.Write([]byte(tc.version)) + })) + defer server.Close() + runner := &fakeDockerRunner{} + client := &Client{BaseURL: server.URL, Token: "secret", HTTPClient: server.Client(), Docker: runner} + err := client.Apply(context.Background(), plan) + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("expected %q, got %v", tc.want, err) + } + if len(runner.outputArgs) != 0 || len(runner.runs) != 0 { + t.Fatalf("source was touched before the version refusal: outputs=%v runs=%v", runner.outputArgs, runner.runs) + } + }) + } +} + +func TestLocalDockerRunnerIsolatesComposeEnvironment(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("shell shim") + } + shim := filepath.Join(t.TempDir(), "docker") + if err := os.WriteFile(shim, []byte("#!/bin/sh\necho \"args: $*\"\nenv | sort\n"), 0o700); err != nil { + t.Fatal(err) + } + t.Setenv("BORT_SHELL_LEAK", "1") + runner := localDockerRunner{Path: shim} + compose, err := runner.Output(context.Background(), "compose", "-p", "stage", "config") + if err != nil { + t.Fatalf("compose shim: %v", err) + } + if !strings.HasPrefix(string(compose), "args: --host unix:///var/run/docker.sock compose -p stage config\n") { + t.Fatalf("compose must pin the local daemon through --host, got %s", compose) + } + if strings.Contains(string(compose), "BORT_SHELL_LEAK=") || strings.Contains(string(compose), "DOCKER_HOST=") || !strings.Contains(string(compose), "\nPATH=") || !strings.Contains(string(compose), "HOME=") { + t.Fatalf("compose must run with only PATH and HOME, got %s", compose) + } + other, err := runner.Output(context.Background(), "ps") + if err != nil { + t.Fatalf("ps shim: %v", err) + } + if !strings.HasPrefix(string(other), "args: ps\n") || !strings.Contains(string(other), "BORT_SHELL_LEAK=1") || !strings.Contains(string(other), "DOCKER_HOST=unix:///var/run/docker.sock") { + t.Fatalf("non-compose commands must keep the local environment and DOCKER_HOST pin, got %s", other) + } +} + +func TestValidatePlanReadyForLiveApplyRefusesComposeProjectNameInStagedDataStore(t *testing.T) { + plan, _, _ := stagedRestoreFixture(t) + composePath := filepath.Join(plan.Prepare.BundleDir, "api", "compose.yaml") + original := string(plan.BundleFiles[filepath.Clean(composePath)]) + for name, tc := range map[string]struct { + compose string + refuse bool + }{ + "braced reference in db": {compose: strings.Replace(original, "${DB_PASSWORD}", "${COMPOSE_PROJECT_NAME}_db", 1), refuse: true}, + "bare reference in db": {compose: strings.Replace(original, "${DB_PASSWORD}", "$COMPOSE_PROJECT_NAME", 1), refuse: true}, + "longer variable name in db": {compose: strings.Replace(original, "${DB_PASSWORD}", "${COMPOSE_PROJECT_NAME_SUFFIX}", 1), refuse: false}, + "reference in other service": {compose: strings.Replace(original, "image: example/web\n", "image: example/web\n environment:\n APP: ${COMPOSE_PROJECT_NAME}\n", 1), refuse: false}, + } { + t.Run(name, func(t *testing.T) { + if tc.compose == original { + t.Fatal("fixture did not change") + } + plan.BundleFiles[filepath.Clean(composePath)] = []byte(tc.compose) + err := validatePlanReadyForLiveApply(plan) + if tc.refuse && (!errors.Is(err, ErrNotImplemented) || !strings.Contains(err.Error(), "COMPOSE_PROJECT_NAME")) { + t.Fatalf("expected COMPOSE_PROJECT_NAME refusal, got %v", err) + } + if !tc.refuse && err != nil { + t.Fatalf("unexpected refusal: %v", err) + } + }) + } + plan.BundleFiles[filepath.Clean(composePath)] = []byte(original) + plan.BundleFiles[filepath.Clean(filepath.Join(plan.Prepare.BundleDir, "api", ".env"))] = []byte("DB_PASSWORD=s3cret\nPGDATA=/var/lib/postgresql/data/${COMPOSE_PROJECT_NAME}\n") + if err := validatePlanReadyForLiveApply(plan); !errors.Is(err, ErrNotImplemented) || !strings.Contains(err.Error(), "COMPOSE_PROJECT_NAME") { + t.Fatalf("expected refusal for a COMPOSE_PROJECT_NAME reference in the env file, got %v", err) + } +} + +func TestValidatePlanReadyForLiveApplyRefusesRelativeInitScriptMountBeforePause(t *testing.T) { + plan, _, _ := stagedRestoreFixture(t) + composePath := filepath.Join(plan.Prepare.BundleDir, "api", "compose.yaml") + original := string(plan.BundleFiles[filepath.Clean(composePath)]) + withInit := strings.Replace(original, " - pgdata:/var/lib/postgresql/data\n", " - pgdata:/var/lib/postgresql/data\n - ./init:/docker-entrypoint-initdb.d\n", 1) + if withInit == original { + t.Fatal("fixture did not change") + } + plan.BundleFiles[filepath.Clean(composePath)] = []byte(withInit) + err := validatePlanReadyForLiveApply(plan) + if err == nil || !strings.Contains(err.Error(), "init script mount ./init -> /docker-entrypoint-initdb.d must use an absolute host path") || !strings.Contains(err.Error(), "choose a recreate or managed data store strategy") { + t.Fatalf("expected relative init script mount refusal before live apply, got %v", err) + } + plan.BundleFiles[filepath.Clean(composePath)] = []byte(strings.Replace(withInit, "./init:", "/srv/db/init:", 1)) + if err := validatePlanReadyForLiveApply(plan); err != nil { + t.Fatalf("absolute init script mount must stage: %v", err) + } +} + +func stagingCompatibleClient(t *testing.T, runner dockerRunner, createEnvFile bool, command string) *Client { + t.Helper() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet || r.URL.Path != "/api/compose.one" || r.Header.Get("x-api-key") != "secret" { + http.NotFound(w, r) + return + } + _ = json.NewEncoder(w).Encode(Compose{ComposeID: r.URL.Query().Get("composeId"), AppName: "stack-1", CreateEnvFile: &createEnvFile, Command: command}) + })) + t.Cleanup(server.Close) + return &Client{BaseURL: server.URL, Token: "secret", HTTPClient: server.Client(), Docker: runner} +} + +func TestStagedRestoreRefusesIncompatibleDokployComposeSettings(t *testing.T) { + plan, step, _ := stagedRestoreFixture(t) + for name, tc := range map[string]struct { + createEnvFile bool + command string + want string + }{ + "env file disabled": {createEnvFile: false, want: ".env creation disabled"}, + "custom command": {createEnvFile: true, command: "docker compose -f compose.yaml up -d", want: "custom deploy command"}, + } { + t.Run(name, func(t *testing.T) { + runner := &fakeDockerRunner{} + client := stagingCompatibleClient(t, runner, tc.createEnvFile, tc.command) + actx := &applyContext{cache: map[string]*appCache{}, plan: plan, stagingEnvFormat: stagingEnvFormatKeepInterpolation} + actx.entry("api").ComposeID = "c1" + actx.entry("api").ComposeAppName = "stack-1" + err := client.applyRestoreDataStore(context.Background(), actx, step) + if !errors.Is(err, ErrNotImplemented) || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("expected %q refusal, got %v", tc.want, err) + } + if len(runner.outputArgs) != 0 || len(runner.runs) != 0 { + t.Fatalf("staging volumes were touched before the refusal: outputs=%v runs=%v", runner.outputArgs, runner.runs) + } + }) + } +} + +func TestApplyCreateServiceRefusesAdoptedComposeWithoutEnvFile(t *testing.T) { + plan, _, _ := stagedRestoreFixture(t) + disabled := false + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && r.URL.Path == "/api/compose.search": + _ = json.NewEncoder(w).Encode(composeSearchResponse{Items: []Compose{{ComposeID: "c1", Name: "api", AppName: "stack-1"}}, Total: 1}) + case r.Method == http.MethodGet && r.URL.Path == "/api/compose.one": + _ = json.NewEncoder(w).Encode(Compose{ComposeID: "c1", Name: "api", AppName: "stack-1", CreateEnvFile: &disabled}) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + client := &Client{BaseURL: server.URL, Token: "secret", HTTPClient: server.Client(), Docker: &fakeDockerRunner{}} + actx := &applyContext{plan: plan, cache: map[string]*appCache{"api": {EnvironmentID: "env1"}}} + err := client.applyCreateService(context.Background(), actx, Step{Kind: StepCreateService, App: "api", Ref: "api"}) + if !errors.Is(err, ErrNotImplemented) || !strings.Contains(err.Error(), ".env creation disabled") { + t.Fatalf("expected adopted compose refusal before the source pause, got %v", err) + } +} + +func TestDeployComposeForApplyRefusesIncompatibleSettingsAfterStagedRestore(t *testing.T) { + plan, _, _ := stagedRestoreFixture(t) + disabled := false + var writes []string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && r.URL.Path == "/api/compose.one": + _ = json.NewEncoder(w).Encode(Compose{ComposeID: "c1", Name: "api", AppName: "stack-1", CreateEnvFile: &disabled}) + default: + writes = append(writes, r.URL.Path) + _ = json.NewEncoder(w).Encode(map[string]any{}) + } + })) + defer server.Close() + client := &Client{BaseURL: server.URL, Token: "secret", HTTPClient: server.Client(), Docker: &fakeDockerRunner{}} + actx := &applyContext{plan: plan, cache: map[string]*appCache{}} + actx.entry("api").ComposeID = "c1" + actx.entry("api").ComposeAppName = "stack-1" + err := client.deployComposeForApply(context.Background(), actx, "api", "services: {}\n", "DB_PASSWORD=s3cret\n") + if !errors.Is(err, ErrNotImplemented) || !strings.Contains(err.Error(), ".env creation disabled") { + t.Fatalf("expected deploy-time refusal, got %v", err) + } + if len(writes) != 0 { + t.Fatalf("compose was written or deployed despite the refusal: %v", writes) + } +} + +func stagedSyncFixture(t *testing.T) (preparer.AppPlan, Plan, Step, stagedVolume) { + t.Helper() + app := preparer.AppPlan{Name: "api"} + app.Resources.SourceServices = []preparer.SourceServiceRef{{ServiceName: "web", ContainerID: "src-id", ContainerName: "coolify-web"}} + app.Resources.Volumes = []preparer.VolumeResource{{ + Service: "web", + Type: "volume", + Name: "src-vol", + Target: "/data", + SourceContainerID: "src-id", + SourceContainerName: "coolify-web", + }} + step := Step{Kind: StepSyncVolume, App: "api", Ref: "volume:web -> /data"} + plan := stagedPlan(t, app, t.TempDir(), Step{Kind: StepPauseSource, App: "api"}, step) + staged, ok := stagedVolumeFor(plan, "api", app.Resources.Volumes[0]) + if !ok { + t.Fatalf("volume not staged in plan %v", stepKinds(plan.Steps)) + } + return app, plan, step, staged +} + +func stoppedSourceInspect(startedAt, finishedAt string) []byte { + return []byte(`[{"Id":"src-id","Name":"/coolify-web","State":{"Running":false,"StartedAt":"` + startedAt + `","FinishedAt":"` + finishedAt + `"}}]`) +} + +func TestApplyCreateVolumeCreatesOnlyItsOwnStagedVolume(t *testing.T) { + app := preparer.AppPlan{Name: "api"} + app.Resources.Volumes = []preparer.VolumeResource{ + {Service: "web", Type: "volume", Name: "src-data", Target: "/data"}, + {Service: "web", Type: "volume", Name: "", Target: "/cache"}, + } + plan := stagedPlan(t, app, t.TempDir(), + Step{Kind: StepCreateVolume, App: "api", Ref: "src-data"}, + Step{Kind: StepCreateVolume, App: "api", Ref: "/cache"}, + Step{Kind: StepPauseSource, App: "api"}, + Step{Kind: StepSyncVolume, App: "api", Ref: "volume:web -> /data"}, + Step{Kind: StepSyncVolume, App: "api", Ref: "volume:web -> /cache"}, + ) + staged := stagedVolumesForApp(plan, "api") + if len(staged) != 2 { + t.Fatalf("expected two staged volumes, got %#v", staged) + } + runner := &fakeDockerRunner{outputs: map[string][]byte{"volume create": []byte("created\n")}, outputErrs: map[string]error{}} + for _, volume := range staged { + runner.outputErrs["volume inspect --format {{index .Labels \"bort.run-id\"}} "+volume.VolumeName] = errors.New("Error response from daemon: get " + volume.VolumeName + ": no such volume") + } + client := &Client{Docker: runner} + actx := &applyContext{cache: map[string]*appCache{}, plan: plan} + if err := client.applyCreateVolume(context.Background(), actx, plan.Steps[1]); err != nil { + t.Fatalf("applyCreateVolume: %v", err) + } + var created []string + for _, args := range runner.outputArgs { + if len(args) > 2 && args[0] == "volume" && args[1] == "create" { + created = append(created, args[len(args)-1]) + } + } + if len(created) != 1 || created[0] != staged[1].VolumeName { + t.Fatalf("expected only %s to be created for the /cache step, got %v", staged[1].VolumeName, created) + } +} + +func TestSyncVolumeToStagingCreatesOwnedVolumeAndRecordsMount(t *testing.T) { + app, plan, step, _ := stagedSyncFixture(t) + plan.RunID = "run-digest" + staged, ok := stagedVolumeFor(plan, "api", app.Resources.Volumes[0]) + if !ok { + t.Fatal("volume not staged after setting the run ID") + } + runner := &fakeDockerRunner{ + outputs: map[string][]byte{ + "volume inspect src-vol": []byte(`[{"Name":"src-vol"}]`), + "volume inspect " + staged.VolumeName: []byte(`[{"Name":"` + staged.VolumeName + `"}]`), + "volume create": []byte(staged.VolumeName + "\n"), + "inspect --type container src-id": stoppedSourceInspect("2026-01-01T00:00:00Z", "2026-01-02T00:00:00Z"), + "ps -a --filter volume=" + staged.VolumeName: []byte(""), + }, + outputErrs: map[string]error{ + "volume inspect --format {{index .Labels \"bort.run-id\"}} " + staged.VolumeName: errors.New("Error response from daemon: get " + staged.VolumeName + ": no such volume"), + }, + } + client := &Client{Docker: runner} + actx := &applyContext{cache: map[string]*appCache{}, plan: plan} + + if err := client.applySyncVolume(context.Background(), actx, step); err != nil { + t.Fatalf("applySyncVolume: %v", err) + } + var created []string + for _, args := range runner.outputArgs { + if len(args) > 2 && args[0] == "volume" && args[1] == "create" { + created = args + } + } + joined := strings.Join(created, " ") + if created == nil || !strings.Contains(joined, "--label bort.run=run1") || !strings.Contains(joined, "--label bort.run-id=run-digest") || !strings.HasSuffix(joined, staged.VolumeName) { + t.Fatalf("expected labeled staging volume creation, got %v", created) + } + if len(runner.runs) != 1 || !strings.Contains(strings.Join(runner.runs[0].Args, " "), "src-vol:/from:ro -v "+staged.VolumeName+":/to") { + t.Fatalf("expected one copy into the staging volume, got %#v", runner.runs) + } + mount, ok := actx.entry("api").MigratedVolumeMounts[migratedMountKey("web", "/data")] + if !ok || mount.VolumeName != staged.VolumeName { + t.Fatalf("expected staging mount recorded, got %#v", actx.entry("api").MigratedVolumeMounts) + } +} + +func TestSyncVolumeToStagingRefusesCopyWhoseFlushFails(t *testing.T) { + app, plan, step, _ := stagedSyncFixture(t) + plan.RunID = "run-digest" + staged, ok := stagedVolumeFor(plan, "api", app.Resources.Volumes[0]) + if !ok { + t.Fatal("volume not staged after setting the run ID") + } + runner := &fakeDockerRunner{ + outputs: map[string][]byte{ + "volume inspect src-vol": []byte(`[{"Name":"src-vol"}]`), + "volume inspect " + staged.VolumeName: []byte(`[{"Name":"` + staged.VolumeName + `"}]`), + "volume create": []byte(staged.VolumeName + "\n"), + "inspect --type container src-id": stoppedSourceInspect("2026-01-01T00:00:00Z", "2026-01-02T00:00:00Z"), + "ps -a --filter volume=" + staged.VolumeName: []byte(""), + }, + outputErrs: map[string]error{ + "volume inspect --format {{index .Labels \"bort.run-id\"}} " + staged.VolumeName: errors.New("Error response from daemon: get " + staged.VolumeName + ": no such volume"), + }, + runErr: errors.New("fsync: /to/data/db.sqlite: Input/output error"), + } + client := &Client{Docker: runner} + actx := &applyContext{cache: map[string]*appCache{}, plan: plan} + + err := client.applySyncVolume(context.Background(), actx, step) + if err == nil || !strings.Contains(err.Error(), "Input/output error") { + t.Fatalf("expected the flush failure to fail the copy, got %v", err) + } + if len(runner.runs) != 1 { + t.Fatalf("expected one copy attempt, got %#v", runner.runs) + } + script := runner.runs[0].Args[len(runner.runs[0].Args)-1] + if !strings.HasPrefix(script, "set -o pipefail; find /to -mindepth 1 -delete && ") || !strings.Contains(script, "tar xpf - -C /to && find /to") || !strings.HasSuffix(script, "-print0 | xargs -0 fsync") { + t.Fatalf("copy must clear /to without shell globbing, fsync every file and directory under /to, and propagate any batch failure before succeeding, got %q", script) + } + if _, ok := actx.entry("api").MigratedVolumeMounts[migratedMountKey("web", "/data")]; ok { + t.Fatalf("an unflushed copy must not be recorded as transferred: %#v", actx.entry("api").MigratedVolumeMounts) + } +} + +func TestSyncVolumeToStagingRefusesForeignOwnedVolume(t *testing.T) { + _, plan, step, staged := stagedSyncFixture(t) + runner := &fakeDockerRunner{ + outputs: map[string][]byte{ + "volume inspect --format {{index .Labels \"bort.run-id\"}} " + staged.VolumeName: []byte("other-run\n"), + }, + } + client := &Client{Docker: runner} + actx := &applyContext{cache: map[string]*appCache{}, plan: plan} + err := client.applySyncVolume(context.Background(), actx, step) + if err == nil || !strings.Contains(err.Error(), "not owned by run \"run1\"") { + t.Fatalf("expected ownership error, got %v", err) + } + if len(runner.runs) != 0 { + t.Fatalf("must not copy into a foreign volume, got %#v", runner.runs) + } +} + +func TestSyncVolumeToStagingRejectsSourceRestartDuringCopy(t *testing.T) { + _, plan, step, staged := stagedSyncFixture(t) + runner := &sequencedOutputRunner{ + fakeDockerRunner: fakeDockerRunner{ + outputs: map[string][]byte{ + "volume inspect src-vol": []byte(`[{"Name":"src-vol"}]`), + "volume inspect " + staged.VolumeName: []byte(`[{"Name":"` + staged.VolumeName + `"}]`), + "volume inspect --format {{index .Labels \"bort.run-id\"}} " + staged.VolumeName: []byte("run1\n"), + "ps -a --filter volume=" + staged.VolumeName: []byte(""), + }, + }, + sequences: map[string][][]byte{ + "inspect --type container src-id": { + stoppedSourceInspect("2026-01-01T00:00:00Z", "2026-01-02T00:00:00Z"), + stoppedSourceInspect("2026-01-03T00:00:00Z", "2026-01-03T00:00:05Z"), + }, + }, + } + client := &Client{Docker: runner} + actx := &applyContext{cache: map[string]*appCache{}, plan: plan} + actx.entry("api").MigratedVolumeMounts = map[string]migratedVolumeMount{ + migratedMountKey("web", "/data"): {Service: "web", Target: "/data", VolumeName: staged.VolumeName}, + } + + err := client.applySyncVolume(context.Background(), actx, step) + if err == nil || !strings.Contains(err.Error(), "ran during state copy") { + t.Fatalf("expected restart detection, got %v", err) + } + if len(runner.runs) != 1 { + t.Fatalf("copy must have run once before the post-check, got %#v", runner.runs) + } + if _, ok := actx.entry("api").MigratedVolumeMounts[migratedMountKey("web", "/data")]; ok { + t.Fatalf("stale mount record must be forgotten before a failed copy") + } +} + +func TestSyncVolumeToStagingRefusesRunningSource(t *testing.T) { + _, plan, step, staged := stagedSyncFixture(t) + runner := &fakeDockerRunner{ + outputs: map[string][]byte{ + "volume inspect src-vol": []byte(`[{"Name":"src-vol"}]`), + "volume inspect " + staged.VolumeName: []byte(`[{"Name":"` + staged.VolumeName + `"}]`), + "volume inspect --format {{index .Labels \"bort.run-id\"}} " + staged.VolumeName: []byte("run1\n"), + "ps -a --filter volume=" + staged.VolumeName: []byte(""), + "inspect --type container src-id": []byte(`[{"Id":"src-id","Name":"/coolify-web","State":{"Running":true}}]`), + }, + } + client := &Client{Docker: runner} + actx := &applyContext{cache: map[string]*appCache{}, plan: plan} + err := client.applySyncVolume(context.Background(), actx, step) + if err == nil || !strings.Contains(err.Error(), "is running while its state is being copied") { + t.Fatalf("expected running-source refusal, got %v", err) + } + if len(runner.runs) != 0 { + t.Fatalf("must not copy from a running source, got %#v", runner.runs) + } +} + +func TestApplySyncVolumeRefusesBindMountInStagedApp(t *testing.T) { + app := preparer.AppPlan{Name: "api"} + app.Resources.Volumes = []preparer.VolumeResource{{Service: "web", Type: "bind", Source: "/srv/data", Target: "/data"}} + step := Step{Kind: StepSyncVolume, App: "api", Ref: "volume:web -> /data"} + plan := stagedPlan(t, app, t.TempDir(), step) + runner := &fakeDockerRunner{} + client := &Client{Docker: runner} + actx := &applyContext{cache: map[string]*appCache{}, plan: plan} + err := client.applySyncVolume(context.Background(), actx, step) + if err == nil || !strings.Contains(err.Error(), "only named volumes are transferred before deploy") { + t.Fatalf("expected bind mount refusal, got %v", err) + } + if len(runner.outputArgs) != 0 || len(runner.runs) != 0 { + t.Fatalf("must not touch docker, got outputs=%v runs=%v", runner.outputArgs, runner.runs) + } +} + +func TestApplyRefusesStagedBindMountBeforePausingSource(t *testing.T) { + app := preparer.AppPlan{Name: "api"} + app.Resources.Volumes = []preparer.VolumeResource{ + {Service: "web", Type: "volume", Source: "src-vol", Target: "/named", SourceContainerID: "src-id"}, + {Service: "web", Type: "bind", Source: "/srv/data", Target: "/data", SourceContainerID: "src-id"}, + } + plan := stagedPlan(t, app, t.TempDir(), + Step{Kind: StepPauseSource, App: "api", Ref: "api"}, + Step{Kind: StepSyncVolume, App: "api", Ref: "volume:web -> /named"}, + Step{Kind: StepSyncVolume, App: "api", Ref: "volume:web -> /data"}, + ) + runner := &fakeDockerRunner{} + client := &Client{Docker: runner} + err := client.Apply(context.Background(), plan) + if !errors.Is(err, ErrNotImplemented) || !strings.Contains(err.Error(), "choose a recreate or managed data store strategy or change the source compose before live apply") { + t.Fatalf("expected pre-live bind mount refusal, got %v", err) + } + if len(runner.outputArgs) != 0 || len(runner.runs) != 0 { + t.Fatalf("source was touched before the bind mount refusal: outputs=%v runs=%v", runner.outputArgs, runner.runs) + } +} + +func stagedRestoreFixture(t *testing.T) (Plan, Step, stagedVolume) { + t.Helper() + runDir := t.TempDir() + bundleDir := t.TempDir() + appDir := filepath.Join(bundleDir, "api") + if err := os.MkdirAll(appDir, 0o700); err != nil { + t.Fatal(err) + } + compose := "services:\n db:\n image: postgres:16\n ports:\n - \"5432:5432\"\n environment:\n POSTGRES_PASSWORD: ${DB_PASSWORD}\n volumes:\n - pgdata:/var/lib/postgresql/data\n web:\n image: example/web\nvolumes:\n pgdata:\n" + env := "DB_PASSWORD=s3cret\n" + composePath := filepath.Join(appDir, "compose.yaml") + envPath := filepath.Join(appDir, ".env") + if err := os.WriteFile(composePath, []byte(compose), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(envPath, []byte(env), 0o600); err != nil { + t.Fatal(err) + } + app := preparer.AppPlan{ + Name: "api", + Directory: "api", + TargetResources: &preparer.TargetResources{Dokploy: &preparer.DokployResources{ + ComposeApp: preparer.DokployComposeApp{ComposePath: "compose.yaml"}, + EnvFiles: []preparer.DokployEnvFile{{Path: ".env"}}, + }}, + } + app.Resources.DataStores = []preparer.DataStoreResource{{Kind: "postgres", Service: "db", Strategy: "migrate"}} + app.Resources.Volumes = []preparer.VolumeResource{{Service: "db", Type: "volume", Name: "src-pgdata", Target: "/var/lib/postgresql/data"}} + step := Step{Kind: StepRestoreDataStore, App: "api", Ref: "data-store:db"} + plan := stagedPlan(t, app, runDir, Step{Kind: StepPauseSource, App: "api"}, Step{Kind: StepDumpDataStore, App: "api", Ref: step.Ref}, step) + plan.Prepare.BundleDir = bundleDir + plan.BundleFiles = map[string][]byte{filepath.Clean(composePath): []byte(compose), filepath.Clean(envPath): []byte(env)} + + dumpPath, err := dataStoreDumpPath(plan, "api", step.Ref) + if err != nil { + t.Fatalf("dataStoreDumpPath: %v", err) + } + if err := os.MkdirAll(filepath.Dir(dumpPath), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(dumpPath, []byte("pg-dump"), 0o600); err != nil { + t.Fatal(err) + } + staged := stagedVolumesForService(plan, "api", "db") + if len(staged) != 1 { + t.Fatalf("expected one staged volume for db, got %#v", staged) + } + return plan, step, staged[0] +} + +type stagingVolumeStateRunner struct { + *fakeDockerRunner + volumeName string + present bool +} + +func (r *stagingVolumeStateRunner) Output(ctx context.Context, args ...string) ([]byte, error) { + out, err := r.fakeDockerRunner.Output(ctx, args...) + key := strings.Join(args, " ") + switch { + case key == "volume rm -f "+r.volumeName: + r.present = false + case args[0] == "volume" && args[1] == "create" && args[len(args)-1] == r.volumeName: + r.present = true + case strings.HasPrefix(key, "volume inspect --format") && strings.HasSuffix(key, " "+r.volumeName) && !r.present: + return nil, errors.New("Error response from daemon: get " + r.volumeName + ": no such volume") + } + return out, err +} + +func TestRestoreDataStoreToStagingRunsIsolatedComposeProject(t *testing.T) { + plan, step, staged := stagedRestoreFixture(t) + project := stagingProjectName(plan, "api", "db") + fake := &fakeDockerRunner{ + outputs: map[string][]byte{ + "volume inspect --format {{index .Labels \"bort.run-id\"}} " + staged.VolumeName: []byte("run1\n"), + "volume rm -f " + staged.VolumeName: []byte(staged.VolumeName + "\n"), + "volume create": []byte(staged.VolumeName + "\n"), + "compose -p " + project: []byte("stg-id\n"), + "inspect --type container stg-id": []byte(`[{"Id":"stg-id","Name":"/` + project + `-db-1","Config":{"Env":["POSTGRES_USER=bob","POSTGRES_PASSWORD=s3cret","POSTGRES_DB=app"]},"State":{"Running":true,"Status":"running"},"Mounts":[{"Type":"volume","Name":"` + staged.VolumeName + `","Destination":"/var/lib/postgresql/data","RW":true}]}]`), + "ps -a --filter volume=" + staged.VolumeName: []byte(""), + "exec stg-id rm -f": []byte(""), + }, + runOutputs: map[string][]byte{ + "exec -i stg-id pg_restore -l": []byte("271; 1259 100 TABLE public widgets bob\n"), + }, + } + runner := &stagingVolumeStateRunner{fakeDockerRunner: fake, volumeName: staged.VolumeName, present: true} + client := stagingCompatibleClient(t, runner, true, "") + actx := &applyContext{cache: map[string]*appCache{}, plan: plan, stagingEnvFormat: stagingEnvFormatKeepInterpolation} + actx.entry("api").ComposeAppName = "stack-1" + + if err := client.applyRestoreDataStore(context.Background(), actx, step); err != nil { + t.Fatalf("applyRestoreDataStore: %v", err) + } + + stageDir := filepath.Join(plan.RunDir, "stage", "api", "db") + composeOut, err := os.ReadFile(filepath.Join(stageDir, "compose.yaml")) + if err != nil { + t.Fatalf("staging compose not written: %v", err) + } + doc := decodeCompose(t, string(composeOut)) + if services := doc["services"].(map[string]any); len(services) != 1 || services["db"] == nil { + t.Fatalf("staging compose must contain only db, got %#v", services) + } + if def := composeVolumeDef(t, doc, "pgdata"); def["name"] != staged.VolumeName { + t.Fatalf("staging compose must mount the staging volume, got %#v", def) + } + envOut, err := os.ReadFile(filepath.Join(stageDir, ".env")) + if err != nil { + t.Fatalf("staging env not written: %v", err) + } + for _, want := range []string{"COMPOSE_PROJECT_NAME=\"stack-1\"", "DB_PASSWORD=\"s3cret\""} { + if !strings.Contains(string(envOut), want) { + t.Fatalf("staging env missing %s:\n%s", want, envOut) + } + } + + composePath := filepath.Join(stageDir, "compose.yaml") + var composeRuns []string + upIndex, restoreIndex, lastDownIndex := -1, -1, -1 + for index, run := range runner.runs { + joined := strings.Join(run.Args, " ") + if run.Args[0] != "compose" { + if strings.Contains(joined, " pg_restore -w ") { + restoreIndex = index + } + continue + } + prefix := "compose -p " + project + " --env-file " + filepath.Join(stageDir, ".env") + " -f " + composePath + " " + if !strings.HasPrefix(joined, prefix) { + t.Fatalf("compose must target the Bort staging project with its env file, got %v", run.Args) + } + subcommand := strings.TrimPrefix(joined, prefix) + composeRuns = append(composeRuns, subcommand) + if strings.HasPrefix(subcommand, "up ") { + upIndex = index + } else if strings.HasPrefix(subcommand, "down") { + lastDownIndex = index + } + } + wantCompose := []string{"down --remove-orphans", "up -d --no-build --no-deps db", "down --remove-orphans"} + if strings.Join(composeRuns, "|") != strings.Join(wantCompose, "|") { + t.Fatalf("compose lifecycle mismatch\n got: %v\nwant: %v", composeRuns, wantCompose) + } + if restoreIndex < 0 || !(upIndex < restoreIndex && restoreIndex < lastDownIndex) { + t.Fatalf("pg_restore must run between compose up and the final down, runs=%#v", runner.runs) + } + + removed, recreated := false, false + recreateIndex, lookupIndex := -1, -1 + for index, args := range runner.outputArgs { + joined := strings.Join(args, " ") + if joined == "volume rm -f "+staged.VolumeName { + removed = true + } + if removed && !recreated && strings.HasPrefix(joined, "volume create ") && strings.HasSuffix(joined, " "+staged.VolumeName) { + if !strings.Contains(joined, " --label bort.run-id="+stagingOwner(plan)+" ") { + t.Fatalf("recreated staging volume must carry the run label, got %v", args) + } + recreated = true + recreateIndex = index + } + if lookupIndex < 0 && strings.HasPrefix(joined, "compose -p "+project+" ") { + lookupIndex = index + } + } + if !removed || !recreated { + t.Fatalf("staging volume must be recreated before restore, outputs=%v", runner.outputArgs) + } + if lookupIndex < 0 || lookupIndex < recreateIndex { + t.Fatalf("staging container lookup must follow the volume recreate (recreate=%d lookup=%d), outputs=%v", recreateIndex, lookupIndex, runner.outputArgs) + } + mount, ok := actx.entry("api").MigratedVolumeMounts[migratedMountKey("db", "/var/lib/postgresql/data")] + if !ok || mount.VolumeName != staged.VolumeName { + t.Fatalf("expected staging mount recorded after restore, got %#v", actx.entry("api").MigratedVolumeMounts) + } +} + +func TestRestoreDataStoreToStagingRefusesToRemoveForeignVolume(t *testing.T) { + plan, step, staged := stagedRestoreFixture(t) + runner := &fakeDockerRunner{ + outputs: map[string][]byte{ + "volume inspect --format {{index .Labels \"bort.run-id\"}} " + staged.VolumeName: []byte("other-run\n"), + "volume rm -f " + staged.VolumeName: []byte(staged.VolumeName + "\n"), + "ps -a --filter volume=" + staged.VolumeName: []byte(""), + }, + } + client := stagingCompatibleClient(t, runner, true, "") + actx := &applyContext{cache: map[string]*appCache{}, plan: plan, stagingEnvFormat: stagingEnvFormatKeepInterpolation} + actx.entry("api").ComposeAppName = "stack-1" + + err := client.applyRestoreDataStore(context.Background(), actx, step) + if err == nil || !strings.Contains(err.Error(), "not owned by run") { + t.Fatalf("expected foreign volume refusal, got %v", err) + } + if fakeOutputCalled(runner, "volume", "rm", "-f", staged.VolumeName) { + t.Fatalf("restore removed a volume it does not own: %#v", runner.outputArgs) + } +} + +func TestRestoreDataStoreToStagingStopsProjectAndSkipsRecordOnFailure(t *testing.T) { + plan, step, staged := stagedRestoreFixture(t) + project := stagingProjectName(plan, "api", "db") + runner := &fakeDockerRunner{ + outputs: map[string][]byte{ + "volume inspect --format {{index .Labels \"bort.run-id\"}} " + staged.VolumeName: []byte("run1\n"), + "volume rm -f " + staged.VolumeName: []byte(staged.VolumeName + "\n"), + "ps -a --filter volume=" + staged.VolumeName: []byte(""), + "compose -p " + project: []byte("stg-a\nstg-b\n"), + }, + } + client := stagingCompatibleClient(t, runner, true, "") + actx := &applyContext{cache: map[string]*appCache{}, plan: plan, stagingEnvFormat: stagingEnvFormatKeepInterpolation} + actx.entry("api").ComposeAppName = "stack-1" + + err := client.applyRestoreDataStore(context.Background(), actx, step) + if err == nil || !strings.Contains(err.Error(), "has 2 containers for service db") { + t.Fatalf("expected ambiguous container error, got %v", err) + } + last := runner.runs[len(runner.runs)-1].Args + if last[0] != "compose" || !strings.Contains(strings.Join(last, " "), " down --remove-orphans") { + t.Fatalf("staging project must be stopped after a failed restore, last run %v", last) + } + if len(actx.entry("api").MigratedVolumeMounts) != 0 { + t.Fatalf("failed restore must not record mounts, got %#v", actx.entry("api").MigratedVolumeMounts) + } +} + +func TestComposeFileForApplyRewritesVolumesOnlyAfterStagedTransfer(t *testing.T) { + plan, _, staged := stagedRestoreFixture(t) + client := &Client{Docker: &fakeDockerRunner{}} + actx := &applyContext{cache: map[string]*appCache{}, plan: plan} + + before, err := client.composeFileForApply(context.Background(), actx, "api") + if err != nil { + t.Fatalf("composeFileForApply: %v", err) + } + if strings.Contains(before, staged.VolumeName) { + t.Fatalf("compose must not reference staging volumes before transfer:\n%s", before) + } + err = requireStagedStateTransferred(actx, "api") + if err == nil || !strings.Contains(err.Error(), "db:/var/lib/postgresql/data not transferred") { + t.Fatalf("expected incomplete-state refusal, got %v", err) + } + + if err := actx.recordMigratedVolumeMount("api", migratedVolumeMount{Service: "db", Target: "/var/lib/postgresql/data", VolumeName: "wrong-volume"}); err != nil { + t.Fatal(err) + } + if err := requireStagedStateTransferred(actx, "api"); err == nil { + t.Fatalf("a mount recorded under a different volume name must not count as transferred") + } + + if err := actx.recordMigratedVolumeMount("api", migratedVolumeMount{Service: "db", Target: "/var/lib/postgresql/data", VolumeName: staged.VolumeName}); err != nil { + t.Fatal(err) + } + if err := requireStagedStateTransferred(actx, "api"); err != nil { + t.Fatalf("requireStagedStateTransferred after transfer: %v", err) + } + after, err := client.composeFileForApply(context.Background(), actx, "api") + if err != nil { + t.Fatalf("composeFileForApply: %v", err) + } + def := composeVolumeDef(t, decodeCompose(t, after), "pgdata") + if def["name"] != staged.VolumeName || def["external"] != true { + t.Fatalf("compose must hand the staging volume to Dokploy after transfer, got %#v", def) + } +} + +func TestAppStateIsStagedRequiresPushAfterStateSteps(t *testing.T) { + steps := []Step{ + {Kind: StepPushImage, App: "legacy"}, + {Kind: StepSyncVolume, App: "legacy", Ref: "volume:web -> /data"}, + {Kind: StepSyncVolume, App: "staged", Ref: "volume:web -> /data"}, + {Kind: StepPushImage, App: "staged"}, + {Kind: StepPushImage, App: "stateless"}, + } + plan := Plan{Steps: steps} + if appStateIsStaged(plan, "legacy") { + t.Fatalf("push before state work is in-place, not staged") + } + if !appStateIsStaged(plan, "staged") { + t.Fatalf("state work before push must be staged") + } + if appStateIsStaged(plan, "stateless") { + t.Fatalf("apps without state work are not staged") + } +} + +func TestSyncVolumeToStagingRefusesAttachedVolumeBeforeCopy(t *testing.T) { + _, plan, step, staged := stagedSyncFixture(t) + runner := &fakeDockerRunner{ + outputs: map[string][]byte{ + "volume inspect src-vol": []byte(`[{"Name":"src-vol"}]`), + "volume inspect " + staged.VolumeName: []byte(`[{"Name":"` + staged.VolumeName + `"}]`), + "volume inspect --format {{index .Labels \"bort.run-id\"}} " + staged.VolumeName: []byte("run1\n"), + "ps -a --filter volume=" + staged.VolumeName: []byte("deployed-id\n"), + "inspect --type container src-id": stoppedSourceInspect("2026-01-01T00:00:00Z", "2026-01-02T00:00:00Z"), + }, + } + client := &Client{Docker: runner} + actx := &applyContext{cache: map[string]*appCache{}, plan: plan} + err := client.applySyncVolume(context.Background(), actx, step) + if err == nil || !strings.Contains(err.Error(), "attached to container(s) deployed-id") { + t.Fatalf("expected attached-volume refusal, got %v", err) + } + if len(runner.runs) != 0 { + t.Fatalf("must not copy into an attached staging volume, got %#v", runner.runs) + } +} + +func TestStagingOwnershipUsesRunIDOverRunName(t *testing.T) { + _, plan, step, _ := stagedSyncFixture(t) + plan.RunID = "run-digest" + staged, ok := stagedVolumeFor(plan, "api", plan.Prepare.Apps[0].Resources.Volumes[0]) + if !ok { + t.Fatal("volume not staged") + } + sameName := stagedPlan(t, plan.Prepare.Apps[0], plan.RunDir, Step{Kind: StepPauseSource, App: "api"}, step) + if other, _ := stagedVolumeFor(sameName, "api", plan.Prepare.Apps[0].Resources.Volumes[0]); other.VolumeName == staged.VolumeName { + t.Fatalf("runs with the same name but different identity must not share staging volume %s", staged.VolumeName) + } + runner := &fakeDockerRunner{ + outputs: map[string][]byte{ + "volume inspect --format {{index .Labels \"bort.run-id\"}} " + staged.VolumeName: []byte("run1\n"), + }, + } + client := &Client{Docker: runner} + actx := &applyContext{cache: map[string]*appCache{}, plan: plan} + err := client.applySyncVolume(context.Background(), actx, step) + if err == nil || !strings.Contains(err.Error(), "bort.run-id=\"run1\"") { + t.Fatalf("expected run-id ownership refusal, got %v", err) + } +} + +func TestRestoreDataStoreToStagingRefusesUnstagedDataDir(t *testing.T) { + plan, step, staged := stagedRestoreFixture(t) + project := stagingProjectName(plan, "api", "db") + for name, tc := range map[string]struct { + inspect string + volumes []preparer.VolumeResource + want string + }{ + "bind at PGDATA": { + inspect: `"Config":{"Env":["POSTGRES_USER=bob","POSTGRES_DB=app","PGDATA=/srv/pg"]},"Mounts":[{"Type":"bind","Source":"/host/pg","Destination":"/srv/pg","RW":true},{"Type":"volume","Name":"` + staged.VolumeName + `","Destination":"/var/lib/postgresql/data","RW":true}]`, + want: "not a staged volume", + }, + "nothing at PGDATA": { + inspect: `"Config":{"Env":["POSTGRES_USER=bob","POSTGRES_DB=app"]},"Mounts":[{"Type":"volume","Name":"` + staged.VolumeName + `","Destination":"/backups","RW":true}]`, + want: "not mounted from a staged volume", + }, + "source bind inside image PGDATA": { + inspect: `"Config":{"Env":["POSTGRES_USER=bob","POSTGRES_DB=app"]},"Mounts":[{"Type":"volume","Name":"` + staged.VolumeName + `","Destination":"/var/lib/postgresql/data","RW":true}]`, + volumes: []preparer.VolumeResource{{Service: "db", Type: "bind", Source: "/srv/wal", Target: "/var/lib/postgresql/data/pg_wal", ReadWrite: true}}, + want: "sits inside postgres data directory", + }, + } { + t.Run(name, func(t *testing.T) { + plan := plan + plan.Prepare.Apps = []preparer.AppPlan{plan.Prepare.Apps[0]} + plan.Prepare.Apps[0].Resources.Volumes = append(append([]preparer.VolumeResource{}, plan.Prepare.Apps[0].Resources.Volumes...), tc.volumes...) + inspect := tc.inspect + runner := &fakeDockerRunner{ + outputs: map[string][]byte{ + "volume inspect --format {{index .Labels \"bort.run-id\"}} " + staged.VolumeName: []byte("run1\n"), + "volume rm -f " + staged.VolumeName: []byte(staged.VolumeName + "\n"), + "ps -a --filter volume=" + staged.VolumeName: []byte(""), + "compose -p " + project: []byte("stg-id\n"), + "inspect --type container stg-id": []byte(`[{"Id":"stg-id","Name":"/` + project + `-db-1",` + inspect + `,"State":{"Running":true,"Status":"running"}}]`), + }, + } + client := stagingCompatibleClient(t, runner, true, "") + actx := &applyContext{cache: map[string]*appCache{}, plan: plan, stagingEnvFormat: stagingEnvFormatKeepInterpolation} + actx.entry("api").ComposeAppName = "stack-1" + err := client.applyRestoreDataStore(context.Background(), actx, step) + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("expected data dir refusal mentioning %q, got %v", tc.want, err) + } + for _, run := range runner.runs { + if strings.Contains(strings.Join(run.Args, " "), "pg_restore") { + t.Fatalf("pg_restore must not run into an unstaged data dir: %v", run.Args) + } + } + if len(actx.entry("api").MigratedVolumeMounts) != 0 { + t.Fatalf("refused restore must not record mounts, got %#v", actx.entry("api").MigratedVolumeMounts) + } + }) + } +} + +func TestRequireStagedPostgresDataDirAcceptsParentVolumeMount(t *testing.T) { + var container dockerContainer + container.Config.Env = []string{"PGDATA=/var/lib/postgresql/data/pgdata"} + container.Mounts = []dockerMount{{Type: "volume", Name: "staged-vol", Destination: "/var/lib/postgresql/data"}} + if err := requireStagedPostgresDataDir(container, []stagedVolume{{VolumeName: "staged-vol"}}); err != nil { + t.Fatalf("parent volume mount must satisfy PGDATA: %v", err) + } + container.Mounts = append(container.Mounts, dockerMount{Type: "bind", Source: "/host", Destination: "/var/lib/postgresql/data/pgdata"}) + if err := requireStagedPostgresDataDir(container, []stagedVolume{{VolumeName: "staged-vol"}}); err == nil { + t.Fatal("the deepest mount at PGDATA wins; a bind there must be refused") + } +} + +func TestRewriteComposeStagedVolumesRefusesSharedKey(t *testing.T) { + compose := "services:\n web:\n volumes:\n - shared:/data\n worker:\n volumes:\n - shared:/work\nvolumes:\n shared:\n" + staged := []stagedVolume{ + {Service: "web", Target: "/data", VolumeName: "bort-web"}, + {Service: "worker", Target: "/work", VolumeName: "bort-worker"}, + } + _, err := rewriteComposeStagedVolumes(compose, staged) + if err == nil || !strings.Contains(err.Error(), "compose volume shared is mounted by web:/data and worker:/work") { + t.Fatalf("expected shared key refusal, got %v", err) + } +} + +func TestValidatePlanReadyForLiveApplyRefusesSharedKeyBeforeAnyMutation(t *testing.T) { + runDir := t.TempDir() + bundleDir := t.TempDir() + appDir := filepath.Join(bundleDir, "api") + if err := os.MkdirAll(appDir, 0o700); err != nil { + t.Fatal(err) + } + compose := "services:\n web:\n image: example/web\n volumes:\n - shared:/data\n worker:\n image: example/worker\n volumes:\n - shared:/work\nvolumes:\n shared:\n" + composePath := filepath.Join(appDir, "compose.yaml") + if err := os.WriteFile(composePath, []byte(compose), 0o600); err != nil { + t.Fatal(err) + } + app := preparer.AppPlan{ + Name: "api", + Directory: "api", + TargetResources: &preparer.TargetResources{Dokploy: &preparer.DokployResources{ComposeApp: preparer.DokployComposeApp{ComposePath: "compose.yaml"}}}, + } + app.Resources.Volumes = []preparer.VolumeResource{ + {Service: "web", Type: "volume", Name: "src-shared", Target: "/data"}, + {Service: "worker", Type: "volume", Name: "src-shared", Target: "/work"}, + } + plan := stagedPlan(t, app, runDir, + Step{Kind: StepCreateVolume, App: "api", Ref: "shared"}, + Step{Kind: StepPauseSource, App: "api"}, + Step{Kind: StepSyncVolume, App: "api", Ref: "volume:web -> /data"}, + Step{Kind: StepSyncVolume, App: "api", Ref: "volume:worker -> /work"}, + ) + plan.Prepare.BundleDir = bundleDir + plan.BundleFiles = map[string][]byte{filepath.Clean(composePath): []byte(compose)} + err := validatePlanReadyForLiveApply(plan) + if err == nil || !strings.Contains(err.Error(), "compose volume shared is mounted by") || !strings.Contains(err.Error(), "choose a recreate or managed data store strategy or change the source compose before live apply") { + t.Fatalf("expected shared key refusal before live apply starts, got %v", err) + } +} + +func TestValidatePlanReadyForLiveApplyRefusesSourceVolumeSharedAcrossApps(t *testing.T) { + runDir := t.TempDir() + bundleDir := t.TempDir() + apps := []preparer.AppPlan{} + steps := []Step{} + bundleFiles := map[string][]byte{} + for _, name := range []string{"api", "worker"} { + appDir := filepath.Join(bundleDir, name) + if err := os.MkdirAll(appDir, 0o700); err != nil { + t.Fatal(err) + } + compose := "services:\n main:\n image: example/" + name + "\n volumes:\n - data:/data\nvolumes:\n data:\n name: src-shared\n" + composePath := filepath.Join(appDir, "compose.yaml") + if err := os.WriteFile(composePath, []byte(compose), 0o600); err != nil { + t.Fatal(err) + } + bundleFiles[filepath.Clean(composePath)] = []byte(compose) + app := preparer.AppPlan{ + Name: name, + Directory: name, + TargetResources: &preparer.TargetResources{Dokploy: &preparer.DokployResources{ComposeApp: preparer.DokployComposeApp{ComposePath: "compose.yaml"}}}, + } + app.Resources.Volumes = []preparer.VolumeResource{{Service: "main", Type: "volume", Name: "src-shared", Target: "/data"}} + apps = append(apps, app) + steps = append(steps, + Step{Kind: StepCreateVolume, App: name, Ref: "data"}, + Step{Kind: StepPauseSource, App: name}, + Step{Kind: StepSyncVolume, App: name, Ref: "volume:main -> /data"}, + Step{Kind: StepPushImage, App: name}, + ) + } + plan := Plan{ + RunName: "run1", + RunDir: runDir, + Prepare: preparer.Result{BundleDir: bundleDir, Apps: apps}, + Steps: steps, + BundleFiles: bundleFiles, + } + err := validatePlanReadyForLiveApply(plan) + if err == nil || !strings.Contains(err.Error(), "source volume src-shared is mounted by api main:/data and worker main:/data") || !strings.Contains(err.Error(), "choose a recreate or managed data store strategy or change the source compose before live apply") { + t.Fatalf("expected cross-app shared source volume refusal before live apply starts, got %v", err) + } + plan.Prepare.Apps[1].Resources.Volumes[0].Name = "src-worker" + if err := validatePlanReadyForLiveApply(plan); err != nil { + t.Fatalf("distinct source volumes were refused: %v", err) + } +} + +func TestValidatePlannedPostgresDataDirsRefusesUnstagedLayoutsBeforeLiveApply(t *testing.T) { + for name, tc := range map[string]struct { + compose string + volumes []preparer.VolumeResource + env string + want string + }{ + "bind mount data dir": { + compose: "services:\n db:\n image: postgres:16\n volumes:\n - ./pg:/var/lib/postgresql/data\nvolumes: {}\n", + volumes: []preparer.VolumeResource{{Service: "db", Type: "bind", Source: "/srv/pg", Target: "/var/lib/postgresql/data"}}, + want: "mounts no named volume", + }, + "no data dir mount": { + compose: "services:\n db:\n image: postgres:16\n", + want: "mounts no named volume", + }, + "bind data dir beside unrelated named volume": { + compose: "services:\n db:\n image: postgres:16\n environment:\n PGDATA: /pg/data\n volumes:\n - backups:/backups\n - ./pg:/pg/data\nvolumes:\n backups:\n", + volumes: []preparer.VolumeResource{{Service: "db", Type: "volume", Name: "src-backups", Target: "/backups"}, {Service: "db", Type: "bind", Source: "/srv/pg", Target: "/pg/data"}}, + want: "bind \"/srv/pg\" at /pg/data", + }, + "null PGDATA defers to preflight": { + compose: "services:\n db:\n image: postgres:16\n environment:\n PGDATA: null\n volumes:\n - pgdata:/var/lib/postgresql/data\nvolumes:\n pgdata:\n", + volumes: []preparer.VolumeResource{{Service: "db", Type: "volume", Name: "src-pgdata", Target: "/var/lib/postgresql/data"}}, + }, + "list-form PGDATA beside named volume": { + compose: "services:\n db:\n image: postgres:16\n environment:\n - PGDATA=/pg/data\n volumes:\n - pgdata:/var/lib/postgresql/data\nvolumes:\n pgdata:\n", + volumes: []preparer.VolumeResource{{Service: "db", Type: "volume", Name: "src-pgdata", Target: "/var/lib/postgresql/data"}}, + want: "/pg/data", + }, + "bind inside PGDATA": { + compose: "services:\n db:\n image: postgres:16\n environment:\n PGDATA: /var/lib/postgresql/data\n volumes:\n - pgdata:/var/lib/postgresql/data\n - ./wal:/var/lib/postgresql/data/pg_wal\nvolumes:\n pgdata:\n", + volumes: []preparer.VolumeResource{{Service: "db", Type: "volume", Name: "src-pgdata", Target: "/var/lib/postgresql/data"}, {Service: "db", Type: "bind", Source: "/srv/wal", Target: "/var/lib/postgresql/data/pg_wal", ReadWrite: true}}, + want: "bind \"/srv/wal\" at /var/lib/postgresql/data/pg_wal", + }, + "read-only config file inside PGDATA": { + compose: "services:\n db:\n image: postgres:16\n environment:\n PGDATA: /var/lib/postgresql/data\n volumes:\n - pgdata:/var/lib/postgresql/data\n - ./pg_hba.conf:/var/lib/postgresql/data/pg_hba.conf:ro\nvolumes:\n pgdata:\n", + volumes: []preparer.VolumeResource{{Service: "db", Type: "volume", Name: "src-pgdata", Target: "/var/lib/postgresql/data"}, {Service: "db", Type: "bind", Source: "/srv/pg_hba.conf", Target: "/var/lib/postgresql/data/pg_hba.conf"}}, + }, + "writable config file inside PGDATA": { + compose: "services:\n db:\n image: postgres:16\n environment:\n PGDATA: /var/lib/postgresql/data\n volumes:\n - pgdata:/var/lib/postgresql/data\n - ./postgresql.conf:/var/lib/postgresql/data/postgresql.conf\nvolumes:\n pgdata:\n", + volumes: []preparer.VolumeResource{{Service: "db", Type: "volume", Name: "src-pgdata", Target: "/var/lib/postgresql/data"}, {Service: "db", Type: "bind", Source: "/srv/postgresql.conf", Target: "/var/lib/postgresql/data/postgresql.conf", ReadWrite: true}}, + want: "bind \"/srv/postgresql.conf\" at /var/lib/postgresql/data/postgresql.conf", + }, + "named volume inside PGDATA": { + compose: "services:\n db:\n image: postgres:16\n environment:\n PGDATA: /var/lib/postgresql/data\n volumes:\n - pgdata:/var/lib/postgresql/data\n - wal:/var/lib/postgresql/data/pg_wal\nvolumes:\n pgdata:\n wal:\n", + volumes: []preparer.VolumeResource{{Service: "db", Type: "volume", Name: "src-pgdata", Target: "/var/lib/postgresql/data"}, {Service: "db", Type: "volume", Name: "src-wal", Target: "/var/lib/postgresql/data/pg_wal"}}, + }, + "named volume beside custom PGDATA": { + compose: "services:\n db:\n image: postgres:16\n environment:\n PGDATA: /pg/data\n volumes:\n - pgdata:/var/lib/postgresql/data\nvolumes:\n pgdata:\n", + volumes: []preparer.VolumeResource{{Service: "db", Type: "volume", Name: "src-pgdata", Target: "/var/lib/postgresql/data"}}, + want: "/pg/data", + }, + "custom PGDATA from env file": { + compose: "services:\n db:\n image: postgres:16\n environment:\n - PGDATA=${PG_DIR}\n volumes:\n - pgdata:/var/lib/postgresql/data\nvolumes:\n pgdata:\n", + volumes: []preparer.VolumeResource{{Service: "db", Type: "volume", Name: "src-pgdata", Target: "/var/lib/postgresql/data"}}, + env: "PG_DIR=/pg/data\n", + }, + "interpolated PGDATA that cleans to a literal": { + compose: "services:\n db:\n image: postgres:16\n environment:\n PGDATA: ${PGROOT}/../data\n volumes:\n - pgdata:/var/lib/postgresql/data\nvolumes:\n pgdata:\n", + volumes: []preparer.VolumeResource{{Service: "db", Type: "volume", Name: "src-pgdata", Target: "/var/lib/postgresql/data"}}, + env: "PGROOT=/var/lib/postgresql/child\n", + }, + "interpolated bind target": { + compose: "services:\n db:\n image: postgres:16\n volumes:\n - pgdata:/var/lib/postgresql/data\n - ./other:${BIND_TARGET}\nvolumes:\n pgdata:\n", + volumes: []preparer.VolumeResource{{Service: "db", Type: "volume", Name: "src-pgdata", Target: "/var/lib/postgresql/data"}, {Service: "db", Type: "bind", Source: "/srv/other", Target: "/backups"}}, + env: "BIND_TARGET=/backups\n", + want: "\"${BIND_TARGET}\" at an interpolated path", + }, + "interpolated long-form bind target": { + compose: "services:\n db:\n image: postgres:16\n volumes:\n - pgdata:/var/lib/postgresql/data\n - type: bind\n source: ./other\n target: $BIND_TARGET\nvolumes:\n pgdata:\n", + volumes: []preparer.VolumeResource{{Service: "db", Type: "volume", Name: "src-pgdata", Target: "/var/lib/postgresql/data"}, {Service: "db", Type: "bind", Source: "/srv/other", Target: "/backups"}}, + env: "BIND_TARGET=/backups\n", + want: "\"$BIND_TARGET\" at an interpolated path", + }, + "interpolated bind source with literal target": { + compose: "services:\n db:\n image: postgres:16\n volumes:\n - pgdata:/var/lib/postgresql/data\n - ${BACKUP_DIR}:/backups\nvolumes:\n pgdata:\n", + volumes: []preparer.VolumeResource{{Service: "db", Type: "volume", Name: "src-pgdata", Target: "/var/lib/postgresql/data"}, {Service: "db", Type: "bind", Source: "/srv/other", Target: "/backups"}}, + env: "BACKUP_DIR=/srv/other\n", + }, + "defaulted bind source with interpolated target": { + compose: "services:\n db:\n image: postgres:16\n volumes:\n - pgdata:/var/lib/postgresql/data\n - ${BACKUP_DIR:-./backups}:${BACKUP_TARGET}\nvolumes:\n pgdata:\n", + volumes: []preparer.VolumeResource{{Service: "db", Type: "volume", Name: "src-pgdata", Target: "/var/lib/postgresql/data"}, {Service: "db", Type: "bind", Source: "/srv/other", Target: "/backups"}}, + env: "BACKUP_TARGET=/backups\n", + want: "at an interpolated path", + }, + "defaulted bind source with literal target": { + compose: "services:\n db:\n image: postgres:16\n volumes:\n - pgdata:/var/lib/postgresql/data\n - ${BACKUP_DIR:-$PWD/backups}:/backups\nvolumes:\n pgdata:\n", + volumes: []preparer.VolumeResource{{Service: "db", Type: "volume", Name: "src-pgdata", Target: "/var/lib/postgresql/data"}, {Service: "db", Type: "bind", Source: "/srv/other", Target: "/backups"}}, + }, + "custom PGDATA under named volume": { + compose: "services:\n db:\n image: postgres:16\n environment:\n PGDATA: /var/lib/postgresql/data/pgdata\n volumes:\n - pgdata:/var/lib/postgresql/data\nvolumes:\n pgdata:\n", + volumes: []preparer.VolumeResource{{Service: "db", Type: "volume", Name: "src-pgdata", Target: "/var/lib/postgresql/data"}}, + }, + "bind mount shadows named volume": { + compose: "services:\n db:\n image: postgres:16\n volumes:\n - pgdata:/var/lib/postgresql\n - ./pg:/var/lib/postgresql/data\nvolumes:\n pgdata:\n", + volumes: []preparer.VolumeResource{{Service: "db", Type: "volume", Name: "src-pg", Target: "/var/lib/postgresql"}, {Service: "db", Type: "bind", Source: "/srv/pg", Target: "/var/lib/postgresql/data"}}, + }, + "custom PGDATA through merge key": { + compose: "x-pg: &pg\n image: postgres:16\n environment:\n PGDATA: /pg/data\nservices:\n db:\n <<: *pg\n volumes:\n - pgdata:/var/lib/postgresql/data\nvolumes:\n pgdata:\n", + volumes: []preparer.VolumeResource{{Service: "db", Type: "volume", Name: "src-pgdata", Target: "/var/lib/postgresql/data"}}, + want: "/pg/data", + }, + "custom PGDATA through aliased environment": { + compose: "x-env: &env\n PGDATA: /pg/data\nservices:\n db:\n image: postgres:16\n environment: *env\n volumes:\n - pgdata:/var/lib/postgresql/data\nvolumes:\n pgdata:\n", + volumes: []preparer.VolumeResource{{Service: "db", Type: "volume", Name: "src-pgdata", Target: "/var/lib/postgresql/data"}}, + want: "/pg/data", + }, + "custom PGDATA through default env_file": { + compose: "services:\n db:\n image: postgres:16\n env_file: .env\n volumes:\n - pgdata:/var/lib/postgresql/data\nvolumes:\n pgdata:\n", + volumes: []preparer.VolumeResource{{Service: "db", Type: "volume", Name: "src-pgdata", Target: "/var/lib/postgresql/data"}}, + env: "PGDATA=/pg/data\n", + }, + "environment wins over env_file": { + compose: "services:\n db:\n image: postgres:16\n env_file: .env\n environment:\n PGDATA: /var/lib/postgresql/data/pgdata\n volumes:\n - pgdata:/var/lib/postgresql/data\nvolumes:\n pgdata:\n", + volumes: []preparer.VolumeResource{{Service: "db", Type: "volume", Name: "src-pgdata", Target: "/var/lib/postgresql/data"}}, + env: "PGDATA=/pg/data\n", + }, + "trailing slash on named volume target": { + compose: "services:\n db:\n image: postgres:16\n environment:\n PGDATA: /var/lib/postgresql/data\n volumes:\n - pgdata:/var/lib/postgresql/data/\nvolumes:\n pgdata:\n", + volumes: []preparer.VolumeResource{{Service: "db", Type: "volume", Name: "src-pgdata", Target: "/var/lib/postgresql/data/"}}, + }, + } { + t.Run(name, func(t *testing.T) { + plan, _, _ := stagedRestoreFixture(t) + composePath := filepath.Clean(filepath.Join(plan.Prepare.BundleDir, "api", "compose.yaml")) + plan.BundleFiles[composePath] = []byte(tc.compose) + if tc.env != "" { + plan.BundleFiles[filepath.Clean(filepath.Join(plan.Prepare.BundleDir, "api", ".env"))] = []byte(tc.env) + } + plan.Prepare.Apps[0].Resources.Volumes = tc.volumes + if err := validatePlanReadyForLiveApply(plan); err != nil { + t.Fatalf("the planned data dir gate must not block Apply of a started run: %v", err) + } + err := ValidatePlannedPostgresDataDirs(plan) + if tc.want == "" { + if err != nil { + t.Fatalf("unexpected refusal: %v", err) + } + return + } + if err == nil || !errors.Is(err, ErrNotImplemented) || !strings.Contains(err.Error(), tc.want) || !strings.Contains(err.Error(), "before live apply") { + t.Fatalf("expected data dir refusal mentioning %q, got %v", tc.want, err) + } + }) + } +} + +func TestPauseSourcePreflightsStagedRestoreOnCreatedContainer(t *testing.T) { + for name, tc := range map[string]struct { + env string + volumes []preparer.VolumeResource + want string + }{ + "image PGDATA outside staged volume": {env: `"PGDATA=/pg/data"`, want: "/pg/data"}, + "source bind nested under staged volume": { + env: `"PGDATA=/var/lib/postgresql/data/pgdata"`, + volumes: []preparer.VolumeResource{{Service: "db", Type: "bind", Source: "/srv/pg", Target: "/var/lib/postgresql/data/pgdata", ReadWrite: true}}, + want: "bind \"/srv/pg\" at /var/lib/postgresql/data/pgdata", + }, + "data dir on staged volume": {env: `"POSTGRES_USER=bob"`}, + } { + t.Run(name, func(t *testing.T) { + plan, _, staged := stagedRestoreFixture(t) + plan.Prepare.Apps[0].Resources.Volumes = append(append(plan.Prepare.Apps[0].Resources.Volumes, tc.volumes...), preparer.VolumeResource{Service: "web", Type: "volume", SourceContainerID: "web-id"}) + project := stagingProjectName(plan, "api", "db") + runner := &fakeDockerRunner{ + outputs: map[string][]byte{ + "volume inspect --format {{index .Labels \"bort.run-id\"}} " + staged.VolumeName: []byte("run1\n"), + "compose -p " + project: []byte("stg-id\n"), + "inspect --type container stg-id": []byte(`[{"Id":"stg-id","Name":"/` + project + `-db-1","Config":{"Env":[` + tc.env + `]},"State":{"Running":false,"Status":"created"},"Mounts":[{"Type":"volume","Name":"` + staged.VolumeName + `","Destination":"/var/lib/postgresql/data","RW":true}]}]`), + "inspect --type container web-id": []byte(`[{"Id":"web-id","Name":"/web","State":{"Running":true,"Status":"running"}}]`), + "stop web-id": []byte("web-id\n"), + }, + } + client := stagingCompatibleClient(t, runner, true, "") + actx := &applyContext{cache: map[string]*appCache{}, plan: plan, stagingEnvFormat: stagingEnvFormatKeepInterpolation} + actx.entry("api").ComposeAppName = "stack-1" + + err := client.preflightStagedRestores(context.Background(), actx, "api") + if err == nil { + err = client.applyStep(context.Background(), actx, Step{Kind: StepPauseSource, App: "api"}) + } + + stageDir := filepath.Join(plan.RunDir, "stage", "api", "db") + prefix := "compose -p " + project + " --env-file " + filepath.Join(stageDir, ".env") + " -f " + filepath.Join(stageDir, "compose.yaml") + " " + var composeRuns []string + for _, run := range runner.runs { + joined := strings.Join(run.Args, " ") + if run.Args[0] != "compose" { + continue + } + if !strings.HasPrefix(joined, prefix) { + t.Fatalf("compose must target the Bort staging project with its env file, got %v", run.Args) + } + composeRuns = append(composeRuns, strings.TrimPrefix(joined, prefix)) + } + wantCompose := []string{"down --remove-orphans", "create --no-build db", "down --remove-orphans"} + if strings.Join(composeRuns, "|") != strings.Join(wantCompose, "|") { + t.Fatalf("compose lifecycle mismatch\n got: %v\nwant: %v", composeRuns, wantCompose) + } + if fakeOutputCalled(runner, "volume", "rm", "-f", staged.VolumeName) { + t.Fatalf("preflight must not recreate the staging volume, calls=%v", runner.outputArgs) + } + if tc.want == "" { + if err != nil { + t.Fatalf("applyStep(pause_source): %v", err) + } + if !fakeOutputCalled(runner, "stop", "web-id") { + t.Fatalf("pause must stop the source after a clean preflight, calls=%v", runner.outputArgs) + } + return + } + if err == nil || !strings.Contains(err.Error(), tc.want) || !strings.Contains(err.Error(), "did not stop the source") || !strings.Contains(err.Error(), "create a new run") { + t.Fatalf("expected preflight refusal mentioning %q, got %v", tc.want, err) + } + var refusal stagedRestorePreflightError + if !errors.As(err, &refusal) || !refusal.requiresNewRun { + t.Fatalf("a layout refusal must require a new run, got %v", err) + } + if fakeOutputCalled(runner, "stop", "web-id") { + t.Fatalf("preflight refusal must not stop the source, calls=%v", runner.outputArgs) + } + if actx.entry("api").SourcePauseRecorded { + t.Fatalf("preflight refusal must not record a source pause: %#v", actx.entry("api")) + } + }) + } +} + +type failingCreateRunner struct { + *fakeDockerRunner +} + +func (r *failingCreateRunner) Run(ctx context.Context, stdin io.Reader, stdout io.Writer, args ...string) error { + if err := r.fakeDockerRunner.Run(ctx, stdin, stdout, args...); err != nil { + return err + } + if slices.Contains(args, "create") { + return errors.New("image pull failed") + } + return nil +} + +func TestPauseSourcePreflightStopsStagingProjectAfterFailedCreate(t *testing.T) { + plan, _, staged := stagedRestoreFixture(t) + plan.Prepare.Apps[0].Resources.Volumes = append(plan.Prepare.Apps[0].Resources.Volumes, preparer.VolumeResource{Service: "web", Type: "volume", SourceContainerID: "web-id"}) + runner := &failingCreateRunner{fakeDockerRunner: &fakeDockerRunner{ + outputs: map[string][]byte{ + "volume inspect --format {{index .Labels \"bort.run-id\"}} " + staged.VolumeName: []byte("run1\n"), + }, + }} + client := stagingCompatibleClient(t, runner, true, "") + actx := &applyContext{cache: map[string]*appCache{}, plan: plan, stagingEnvFormat: stagingEnvFormatKeepInterpolation} + actx.entry("api").ComposeAppName = "stack-1" + + err := client.preflightStagedRestores(context.Background(), actx, "api") + if err == nil || !strings.Contains(err.Error(), "image pull failed") || !strings.Contains(err.Error(), "did not stop the source") { + t.Fatalf("expected create failure to abort the pause, got %v", err) + } + var refusal stagedRestorePreflightError + if errors.As(err, &refusal) { + t.Fatalf("a transient create failure must stay retryable, got %v", err) + } + var composeRuns []string + for _, run := range runner.runs { + if run.Args[0] == "compose" { + composeRuns = append(composeRuns, strings.Join(run.Args[len(run.Args)-2:], " ")) + } + } + if last := composeRuns[len(composeRuns)-1]; last != "down --remove-orphans" || len(composeRuns) != 3 { + t.Fatalf("a failed create must still stop the staging project, got %v", composeRuns) + } + if fakeOutputCalled(runner.fakeDockerRunner, "stop", "web-id") { + t.Fatalf("create failure must not stop the source, calls=%v", runner.outputArgs) + } +} + +func TestApplyPreflightRefusalSkipsSourceResumeWhenNothingWasPaused(t *testing.T) { + for name, tc := range map[string]applyPreflightRefusalCase{ + "layout refusal": {want: "/pg/data", requiresNewRun: true}, + "transient create failure": {transient: true, want: "image pull failed"}, + "repairable Dokploy setting": {composeCommand: "docker compose up -d", want: "clear it in Dokploy and resume"}, + } { + t.Run(name, func(t *testing.T) { + testApplyPreflightRefusal(t, tc) + }) + } +} + +func TestApplyPreflightRefusalResumesEarlierPartialPause(t *testing.T) { + for name, tc := range map[string]applyPreflightRefusalCase{ + "resume succeeds": {want: "/pg/data", requiresNewRun: true, earlierPartialPause: true}, + "resume fails": {want: "/pg/data", requiresNewRun: true, earlierPartialPause: true, resumeFails: true}, + "no named volume": {want: "has no named volume to stage", requiresNewRun: true, earlierPartialPause: true, plan: func(plan *Plan) { + plan.Prepare.Apps[0].Resources.Volumes[0].Type = "bind" + }}, + "interpolated mount target outside the data dir": {want: "\"${WAL_TARGET:-/backups}\" at an interpolated path", requiresNewRun: true, earlierPartialPause: true, plan: func(plan *Plan) { + app := &plan.Prepare.Apps[0] + app.Resources.Volumes = append(app.Resources.Volumes, preparer.VolumeResource{Service: "db", Type: "bind", Source: "/srv/wal", Target: "/backups", ReadWrite: true}) + for path, contents := range plan.BundleFiles { + if filepath.Base(path) == "compose.yaml" { + plan.BundleFiles[path] = []byte(strings.Replace(string(contents), " - pgdata:/var/lib/postgresql/data\n", " - pgdata:/var/lib/postgresql/data\n - /srv/wal:${WAL_TARGET:-/backups}\n", 1)) + } + } + }}, + } { + t.Run(name, func(t *testing.T) { + testApplyPreflightRefusal(t, tc) + }) + } +} + +type applyPreflightRefusalCase struct { + transient bool + want string + requiresNewRun bool + earlierPartialPause bool + resumeFails bool + composeCommand string + plan func(*Plan) +} + +func testApplyPreflightRefusal(t *testing.T, tc applyPreflightRefusalCase) { + t.Helper() + transient, want, requiresNewRun := tc.transient, tc.want, tc.requiresNewRun + plan, _, staged := stagedRestoreFixture(t) + plan.Steps = append([]Step{{Kind: StepCreateProject, App: "api"}, {Kind: StepCreateService, App: "api"}}, plan.Steps...) + plan.ResumeFrom = 2 + plan.TargetIdentities = map[string]TargetIdentity{"api": {ProjectID: "proj-1", EnvironmentID: "env-1", ComposeID: "comp-1", ComposeAppName: "stack-1"}} + plan.Prepare.Apps[0].Resources.Volumes = append(plan.Prepare.Apps[0].Resources.Volumes, preparer.VolumeResource{Service: "web", Type: "volume", SourceContainerID: "web-id"}) + if tc.plan != nil { + tc.plan(&plan) + } + project := stagingProjectName(plan, "api", "db") + runner := &fakeDockerRunner{ + outputs: map[string][]byte{ + "volume inspect --format {{index .Labels \"bort.run-id\"}} " + staged.VolumeName: []byte("run1\n"), + "compose -p " + project: []byte("stg-id\n"), + "inspect --type container stg-id": []byte(`[{"Id":"stg-id","Name":"/` + project + `-db-1","Config":{"Env":["PGDATA=/pg/data"]},"State":{"Running":false,"Status":"created"},"Mounts":[{"Type":"volume","Name":"` + staged.VolumeName + `","Destination":"/var/lib/postgresql/data","RW":true}]}]`), + "inspect --type container web-id": []byte(`[{"Id":"web-id","Name":"/web","State":{"Running":true,"Status":"running"}}]`), + }, + } + if tc.earlierPartialPause { + runner.outputs["inspect --type container web-id"] = []byte(`[{"Id":"web-id","Name":"/web","State":{"Running":false,"Status":"exited"}}]`) + runner.outputs["start web-id"] = []byte("web-id\n") + if tc.resumeFails { + runner.outputErrs = map[string]error{"start web-id": errors.New("daemon unreachable")} + } + paused := &applyContext{plan: plan, cache: map[string]*appCache{}} + paused.entry("api").SourcePauseRecorded = true + paused.entry("api").SourcePausedContainers = []sourcePausedContainer{{ID: "web-id", Stopped: true}} + if err := paused.persistSourcePauseState(); err != nil { + t.Fatal(err) + } + } + createEnvFile := true + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("x-api-key") != "secret" { + http.NotFound(w, r) + return + } + switch r.URL.Path { + case "/api/settings.getDokployVersion": + _, _ = w.Write([]byte(`"v0.31.0"`)) + case "/api/project.one": + _ = json.NewEncoder(w).Encode(Project{ProjectID: "proj-1", Environments: []ProjectEnvironment{{EnvironmentID: "env-1", Name: "production"}}}) + case "/api/compose.one": + _ = json.NewEncoder(w).Encode(Compose{ComposeID: r.URL.Query().Get("composeId"), AppName: "stack-1", EnvironmentID: "env-1", CreateEnvFile: &createEnvFile, Command: tc.composeCommand}) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + var progress []StepProgress + onProgress := func(p StepProgress) { progress = append(progress, p) } + plan.OnProgress = &onProgress + var docker dockerRunner = runner + if transient { + docker = &failingCreateRunner{fakeDockerRunner: runner} + } + client := &Client{BaseURL: server.URL, Token: "secret", HTTPClient: server.Client(), Docker: docker} + + err := client.Apply(context.Background(), plan) + if err == nil || !strings.Contains(err.Error(), want) || !strings.Contains(err.Error(), "did not stop the source") { + t.Fatalf("expected preflight refusal, got %v", err) + } + if errors.As(err, new(stagedRestorePreflightError)) != requiresNewRun { + t.Fatalf("preflight error classified as layout refusal = %v, want %v: %v", !requiresNewRun, requiresNewRun, err) + } + if strings.Contains(err.Error(), "resume source app") != tc.resumeFails || strings.Contains(err.Error(), "was not durably recorded") { + t.Fatalf("resume failure reported = %v, want %v: %v", strings.Contains(err.Error(), "resume source app"), tc.resumeFails, err) + } + var first, last *StepProgress + resumed := false + for i, p := range progress { + if p.Step.Kind == StepResumeSource { + if !tc.earlierPartialPause { + t.Fatalf("unexpected resume progress after a preflight refusal: %#v", progress) + } + resumed = resumed || p.Status == StepStatusOK + } + if p.Index != 2 || p.Status == StepStatusStarted { + continue + } + if first == nil { + first = &progress[i] + } + last = &progress[i] + } + if first == nil || first.Step.Kind != StepPauseSource || first.Status != StepStatusError || first.RequiresNewRun != (requiresNewRun && !tc.earlierPartialPause) { + t.Fatalf("the refusal must not claim a new run while owned source containers may still be stopped, got %#v", first) + } + if tc.resumeFails { + if last == nil || last.Step.Kind != StepResumeSource || last.Status != StepStatusError || last.RequiresNewRun { + t.Fatalf("a failed cleanup resume must stay the last record at the pause index so status keeps retrying the resume, got %#v", last) + } + } else if last == nil || last.Step.Kind != StepPauseSource || last.Status != StepStatusError || last.RequiresNewRun != requiresNewRun { + t.Fatalf("the last record at the pause index must be the refusal (ledgers keep one record per index), got %#v", last) + } + if fakeOutputCalled(runner, "stop", "web-id") { + t.Fatalf("preflight refusal must not stop the source, calls=%v", runner.outputArgs) + } + wantResumed := tc.earlierPartialPause && !tc.resumeFails + if fakeOutputCalled(runner, "start", "web-id") != tc.earlierPartialPause || resumed != wantResumed { + t.Fatalf("source restart after refusal = %v (resume progress %v), want %v/%v: calls=%v", fakeOutputCalled(runner, "start", "web-id"), resumed, tc.earlierPartialPause, wantResumed, runner.outputArgs) + } +} From ad1166170226db757b79f6e52437c9f79a86f895 Mon Sep 17 00:00:00 2001 From: Aikins Laryea Date: Sat, 3 Oct 2026 07:34:18 +0000 Subject: [PATCH 31/31] migration: handle Docker 29, Coolify names, and interrupted applies (#38) * migration: handle Docker 29 and Coolify resource names Docker 29 reports absent volumes as volume not found. Preserve infrastructure failures while accepting that form, use reviewed Coolify source coordinates before route heuristics, and exclude platform apps from cleanup name collisions. * migration: tighten resource classification Prefer reviewed Coolify service mappings when their target keys exist, then fall back to runtime names for generated Compose. Match Docker missing-resource errors without swallowing contextual plugin failures. * migration: preserve compose service provenance Persist whether each bundle uses raw or generated Compose so route resolution can prefer logical service mappings for raw Compose and runtime names for generated Compose. Preserve compatible fallbacks for older bundles and prepared runs. * migration: lock reviewed route services * migration: align generated route services * migration: finalize route validation * migration: clarify route recovery paths * migration: fence Coolify and harden staging recovery * migration: close Coolify fence and finalization gaps * migration: fence in-flight Coolify deployments * migration: refuse only active Coolify deployment helpers * migration: clarify Coolify fence scope and commands * migration: name retired Coolify containers and resume completed handoffs * migration: keep pin loss on manual recovery and retire reviewed IDs * migration: align staged recovery guidance with checks * migration: refuse target recovery for unfinished staged transfers * migration: check durable transfer evidence before target recovery * migration: validate target staging before recording acceptance * migration: require handoff evidence for pinless target acceptance * migration: refuse unverified Postgres volumes and sharpen recovery commands * migration: leave Postgres layout recovery advice to callers * migration: deduplicate handed-off apps in linear time * dokploy: expand shared Compose aliases in linear time * cli: pin generated Docker commands to the local daemon * migration: keep the recorded pre-fence Coolify restart policy * dokploy: cover the running-container preference in migrated volume validation * migration: keep the pre-fence Coolify policy across interrupted live applies * dokploy: keep the staged project-name guard exact around escaped dollars * dokploy: pause routed stateless sources until retirement --- .gitignore | 1 + README.md | 79 +- docs/cleanup-and-purge.md | 21 +- docs/migration-guide.md | 180 +- internal/cli/applied.go | 87 +- internal/cli/applied_test.go | 97 ++ internal/cli/authority.go | 104 +- internal/cli/authority_test.go | 236 ++- internal/cli/cleanup.go | 3 + internal/cli/cleanup_test.go | 1 + internal/cli/cockpit.go | 21 +- internal/cli/cockpit_test.go | 45 +- internal/cli/commit.go | 33 +- internal/cli/commit_test.go | 137 +- internal/cli/issues.go | 20 + internal/cli/issues_test.go | 44 + internal/cli/main_test.go | 2 + internal/cli/rollback.go | 19 +- internal/cli/rollback_test.go | 9 +- internal/cli/run.go | 25 +- internal/cli/run_test.go | 11 +- internal/cli/style.go | 9 + internal/exporter/exporter.go | 125 +- internal/exporter/exporter_test.go | 149 +- internal/preparer/preparer.go | 140 +- internal/preparer/preparer_test.go | 319 +++- internal/target/dokploy/apply.go | 302 +++- internal/target/dokploy/apply_test.go | 65 +- internal/target/dokploy/cleanup.go | 56 +- internal/target/dokploy/cleanup_test.go | 47 +- internal/target/dokploy/datamove.go | 326 +++- internal/target/dokploy/datamove_test.go | 428 ++++- internal/target/dokploy/docker.go | 13 +- internal/target/dokploy/rollback.go | 22 +- internal/target/dokploy/rollback_test.go | 54 +- internal/target/dokploy/source.go | 150 +- internal/target/dokploy/source_test.go | 246 ++- internal/target/dokploy/staging.go | 775 ++++++++- internal/target/dokploy/staging_test.go | 1892 ++++++++++++++++++++-- 39 files changed, 5778 insertions(+), 515 deletions(-) diff --git a/.gitignore b/.gitignore index 9666228..2f44a0b 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ *.sqlite3 *.db *.bak +*.profraw diff --git a/README.md b/README.md index 2653f7c..f19a786 100644 --- a/README.md +++ b/README.md @@ -175,11 +175,16 @@ sudo bort commit --apply # accept the target when no source orchestrator can re sudo bort cleanup # audit leftovers without deleting source resources ``` -For a Coolify source, first disable future deployments and manually retire the -reviewed source apps and proxy. Then record permanent target authority with the -exact `recover-authority --authority target --source-retired` command Bort -prints; `commit --apply` refuses because stopping containers cannot fence -Coolify itself. +For a Coolify source, keep the Coolify control plane stopped, remove the +reviewed source app and proxy containers with Docker, and record permanent +target authority with the exact +`recover-authority --authority target --source-retired` command Bort prints. +Stopping Coolify pauses it for every app on the host. Leave it stopped +afterwards when nothing else needs it, because starting it again can recreate +its proxy and redeploy the removed apps; otherwise restore its recorded restart +policy, start it, and immediately delete the migrated apps in Coolify. +`commit --apply` refuses because stopping containers cannot fence Coolify +itself. For owner-bound current runs, `cleanup --apply` can remove only eligible unused records from the bound local Dokploy database, and only after making a database @@ -211,12 +216,33 @@ Bort's safety model defaults to “look first.” - **State moves before the target exists:** for apps with named volumes or a Postgres database, Bort stops the source app, copies each volume and restores each dump into volumes it owns and labels with the run name, verifies the - source stayed stopped and nothing else attached those volumes, and only then - deploys the Dokploy compose with those volumes declared as external. Bort - does not deploy the compose until state is staged and refuses the transfer - if the source restarts or any container attaches to a staging volume; do not - deploy the app manually in Dokploy during live apply. Routed apps stay - stopped until traffic moves; unrouted apps restart the source after the copy. + source stayed stopped, and keeps one read-only `bort-pin-*` container attached + to those volumes until the Dokploy target mounts them exactly. Bort does not + deploy the compose until state is staged and refuses unexpected attachments; + do not deploy the app manually in Dokploy during live apply. If handoff is + unsafe, Bort keeps the source stopped, reports whether it reverified the pin, + and directs recovery through `bort status`. A retryable failure before + handoff can restart the source, but keeps the pin for the next attempt. Do not + remove a `bort-pin-*` container manually. Source recovery checks that no + target container is attached, even if a pin is already absent. Final target + acceptance requires a durable record that each transfer into staging + completed and removes a remaining pin only when the target holds exactly the + recorded attachments. + The source stays stopped while the target becomes the writer, including for + unrouted apps. +- **Coolify stays fenced during stateful moves:** before a stateful live apply + from Coolify, let in-progress Coolify deployments finish, record the + `coolify` container's restart policy with + `sudo docker --host unix:///var/run/docker.sock inspect --format '{{.HostConfig.RestartPolicy.Name}}' coolify`, + then run + `sudo docker --host unix:///var/run/docker.sock update --restart=no coolify && sudo docker --host unix:///var/run/docker.sock stop coolify` so + Coolify cannot redeploy a paused source app. Bort refuses to start, pause a + source, deploy a target, or move routes while the `coolify` container is + running or would restart, or while a Coolify deployment helper is still + running deployment commands. Stopping it pauses Coolify for every app on the + host. Restart Coolify only after source authority is finalized; after target + acceptance, follow the restart guidance in the + [migration guide](docs/migration-guide.md#apply-the-selected-run). - **Known current run:** `.bort/state.json` identifies the current run. Commands that make changes do not guess based on which file was modified most recently. - **Plans are locked during live work:** once live execution begins, changing @@ -236,9 +262,11 @@ Bort's safety model defaults to “look first.” private permissions. - **Separate acceptance:** `commit --apply` retires source application containers only when no external orchestrator can recreate them. It refuses Coolify - sources. Disable future Coolify deployments, manually retire the reviewed - source apps and proxy, verify they remain retired, then use confirmed + sources. With the Coolify control plane stopped, remove the reviewed source + app and proxy containers, verify they stay removed, then use confirmed target-authority recovery with `--source-retired` to record acceptance. + Afterwards, restart Coolify with its recorded restart policy only if other + apps need it, and then delete the migrated apps in Coolify immediately. Destructive source purge requires either accepted state. - **Separate destructive purge:** purge requires selected apps or projects, a successful live apply or completed manual target-authority @@ -274,21 +302,24 @@ Bort's safety model defaults to “look first.” --recreate` or `--managed` for a data store volume and re-plan with `bort migrate --run `, or change the source compose and scan a new run. - A migrated Postgres store must keep its data directory (`PGDATA`) on a named - volume the service mounts, with no writable bind mount inside it. + volume the service mounts, with no writable bind mount inside it and no named + volume outside it. A logical dump cannot restore an auxiliary volume such as + `/backups`. Live apply refuses other layouts before the source pauses (a run already past that app's `pause_source` when this check was added refuses at the restore step and restarts the source during cleanup). When the service mounts no named volume, sets `PGDATA` in its compose `environment` to a - literal path that breaks this rule, or interpolates a mount target, Bort - refuses before live apply starts (`PLAN BLOCKED`): choose `bort data - --recreate` or `--managed` and re-plan with `bort migrate --run - `, or change the source compose and scan a new run. When the image or - interpolation decides `PGDATA`, only the created staged container reveals - the directory, so the refusal comes at `pause_source` (`NEW RUN REQUIRED`) with - that app's source still running and none of its state transferred, and the - run cannot be re-planned: follow the recovery `bort status` shows (releasing - the run when no other app's source was paused or handed off), then choose a - strategy or change the source compose and create a new run. + literal path that breaks this rule, interpolates a mount target, or declares + Compose `secrets` or `configs`, Bort refuses before live apply starts (`PLAN + BLOCKED`): choose `bort data --recreate` or `--managed` and + re-plan with `bort migrate --run `, or change the source compose and + scan a new run. When the image or interpolation decides `PGDATA`, only the + created staged container reveals the directory, so the refusal comes at + `pause_source` (`NEW RUN REQUIRED`) with that app's source still running and + none of its state transferred, and the run cannot be re-planned: follow the + recovery `bort status` shows (releasing the run when no other app's source + was paused or handed off), then choose a strategy or change the source + compose and create a new run. - Runs created by older Bort versions whose plan copied state into an already deployed target remain refused (`MANUAL STATE`). Dokploy v0.30.7 cannot durably prevent queued or future deployments from restarting a target writer diff --git a/docs/cleanup-and-purge.md b/docs/cleanup-and-purge.md index 2397d59..3f5616b 100644 --- a/docs/cleanup-and-purge.md +++ b/docs/cleanup-and-purge.md @@ -28,12 +28,23 @@ source retirement starts, even if commit is interrupted. Rerun `commit --apply` to finish an interrupted acceptance. For a Coolify source, `commit --apply` refuses because Bort cannot disable -queued or future Coolify deployments. Disable future deployments in Coolify for -the reviewed apps, manually retire every reviewed source app (and the source -proxy when the cutover moved routes), and verify they remain retired. Then run -the exact +queued or future Coolify deployments. If live apply already fenced Coolify for +a stateful move, keep the restart policy you recorded before that fence: the +container's policy now reads `no`, so do not record it again. Otherwise, record +the Coolify control plane's restart policy with +`sudo docker --host unix:///var/run/docker.sock inspect --format '{{.HostConfig.RestartPolicy.Name}}' coolify`, +then run +`sudo docker --host unix:///var/run/docker.sock update --restart=no coolify && sudo docker --host unix:///var/run/docker.sock stop coolify` +even if it is already stopped. Remove the reviewed source app containers (and the source proxy +container when the cutover moved routes) with the exact `sudo docker rm -f ...` +command Bort prints, and verify they stay removed. +Then run the exact `recover-authority --authority target --source-retired --confirm ...` command -Bort prints. This records acceptance without mutating source resources. +Bort prints. This records acceptance without mutating source resources. Leave +Coolify stopped afterwards when nothing else on the host needs it: starting it +again can recreate its proxy and redeploy the removed apps. If other apps still +need Coolify, restore the restart policy you recorded before fencing, start +it, and immediately delete the migrated apps in Coolify. Do not accept the target until you have independently checked it. Bort does not measure the rollback window or block acceptance when the window has not passed. diff --git a/docs/migration-guide.md b/docs/migration-guide.md index 807d9bd..f2dfce8 100644 --- a/docs/migration-guide.md +++ b/docs/migration-guide.md @@ -729,6 +729,51 @@ resume when the saved boundary is safe: sudo bort migrate --live ``` +Before a stateful live apply from a Coolify source, stop Coolify's control +plane so it cannot redeploy a source app while Bort copies its state. Bort does +not stop it for you. First wait until no deployment is queued or running in the +Coolify dashboard; a cancelled deployment can keep running, so confirm it has +ended. Then record the current restart policy so you can restore it later, and +stop the control plane: + +```sh +sudo docker --host unix:///var/run/docker.sock inspect --format '{{.HostConfig.RestartPolicy.Name}}' coolify +sudo docker --host unix:///var/run/docker.sock update --restart=no coolify && sudo docker --host unix:///var/run/docker.sock stop coolify +``` + +Live apply refuses to start unless the `coolify` container is stopped with +restart policy `no` (or is absent) and no `coolify-helper` container is still +running deployment commands. An application deployment that began before you +stopped Coolify keeps running in its helper container; wait until it finishes, +then retry. An idle helper left behind by a stopped deployment does not block +the run. Bort checks the fence again before pausing each source app, deploying +each target, and moving routes. Stopping the control plane does not stop +Coolify's proxy or your apps, but it pauses Coolify's management of every app +on the host, including apps outside this run. + +Bort cannot detect every in-progress deployment. Coolify runs Service and +database starts directly on the host rather than in a helper, and Bort +recognizes a custom helper image only through the `coolify` container's +`HELPER_IMAGE` setting. Draining deployments before +you stop Coolify is what keeps those from restarting a source app during the +transfer. + +Keep the control plane stopped until the run finishes. If you recover source +authority, restore the recorded restart policy and start Coolify after the +`recover-authority --authority source` command succeeds, replacing `` +with the value you recorded: + +```sh +sudo docker --host unix:///var/run/docker.sock update --restart= coolify && sudo docker --host unix:///var/run/docker.sock start coolify +``` + +If you accept the target, leave Coolify stopped when nothing else on the host +needs it: starting it again can recreate its proxy and redeploy the migrated +apps. If other apps +still need Coolify, restore its recorded restart policy and start it only after +acceptance, then immediately delete the migrated apps in Coolify; until you do, a Git push or scheduled deployment can +restart them from their old source data. + For an app with named volumes or a Postgres database, live apply moves the state before the Dokploy compose is deployed. Dokploy v0.30.7 cannot durably prevent a queued or future deployment from restarting a target writer, so Bort @@ -746,16 +791,35 @@ never copies data into a deployed target. Instead it: staging volume mounted, running `pg_restore`, and stopping it again. The project files live under `.bort/runs//stage/`. 5. Re-inspects the source containers after each volume copy and refuses the - transfer if any started or ran during it. After every copy or restore it - also refuses if any container is still attached to the staging volume. + transfer if any started or ran during it. One running, read-only + `bort-pin-*` container holds all staging volumes for the app from the first + copy or restore until target handoff; any other attachment is refused. 6. Deploys the compose with each transferred volume declared as `external: true` pointing at the staging volume, then verifies that the - running target mounts those exact volumes and stops the target if it does - not. + running target mounts those exact volumes. Bort then removes the pin. If + target attachment is unsafe or cannot be proved, Bort keeps the source + stopped and reports whether it reverified the pin. A retryable failure before + handoff can restart the source, but preserves the pin for the next attempt. + Do not remove the pin manually. Follow `bort status`. Before Bort releases + host ownership, source recovery checks every transferred app, even one + whose pin is already absent, and requires that no target container is + attached. Target acceptance (`commit --apply` or + `recover-authority --authority target --source-retired`) requires a durable + record that each transfer into staging completed, and removes a remaining + pin only when the target holds exactly the recorded attachments. When no pin + remains and the run's ledger records a completed target handoff, Bort does + not recheck target attachments; verify them yourself before accepting. + Target-authority recovery after an interrupted handoff still verifies that + the target mounts the migrated volumes before finalizing. + Plain `recover-authority --authority target` records target authority and + keeps the pins. Every `--authority target` form refuses when a staged + transfer did not finish, because the target cannot hold the complete source + state; only source recovery is available then. Host ownership remains held + if validation or removal fails, so the command can be retried. Routed apps stay stopped from step 2 until the proxy handoff moves traffic; -plan for that downtime. Unrouted apps restart the source after step 5, so the -source and target then run with diverging copies. +plan for that downtime. Unrouted apps also keep the source stopped while target +activation establishes target writer authority. A stateful app whose plan needs a bind-mount copy is refused. Same-host bind mounts that keep their existing paths need no transfer. Do not remove real @@ -768,6 +832,15 @@ data store, choose `bort data --recreate` or `--managed` and re-plan the run with `bort migrate --run `. Otherwise change the source compose so one service mounts the volume, then scan a new run. +A Postgres store migrated by logical dump cannot have a named volume outside +`PGDATA`, such as an auxiliary `/backups` volume, because the dump cannot +restore that volume's contents. When the compose file sets a literal `PGDATA`, +Bort blocks the plan (`PLAN BLOCKED`): choose a recreate or managed strategy and +re-plan with `bort migrate --run `, or change the source compose and scan +a new run. When the image decides `PGDATA`, Bort refuses at `pause_source` +(`NEW RUN REQUIRED`) with the source still running; follow `bort status` to +release the run, then create a new one. + The temporary compose project runs with only `PATH` and `HOME`, as Dokploy deploys do, and writes its `.env` in the format the running Dokploy release uses: raw values before v0.30.0, every `$` escaped through v0.30.2, and @@ -803,15 +876,16 @@ mounts, with no writable bind mount inside it; otherwise the restore would be lost when the staged container stops, or shadowed by the mount once Dokploy starts the service. When the store service mounts no named volume, sets `PGDATA` in its compose `environment` to a literal path that breaks this -rule, or interpolates a mount target, Bort refuses before live apply starts -(`PLAN BLOCKED`): choose `bort data --recreate` or `--managed` -and re-plan with `bort migrate --run `, or change the source compose and -scan a new run. When the image or interpolation decides `PGDATA`, or the run had already started live -execution before Bort learned to check the layout, Bort creates the staged -container without starting it right before `pause_source` and refuses then -(`NEW RUN REQUIRED`). That app's source keeps running and none of -its state was transferred, but the run has already started live execution and -cannot be re-planned. (A run that had already completed that app's +rule, interpolates a mount target, or declares Compose `secrets` or `configs`, +Bort refuses before live apply starts (`PLAN BLOCKED`): choose `bort data + --recreate` or `--managed` and re-plan with `bort migrate --run +`, or change the source compose and scan a new run. When the image or +interpolation decides `PGDATA`, or the run had already started live execution +before Bort learned to check the layout, Bort creates the staged container +without starting it right before `pause_source` and refuses then (`NEW RUN +REQUIRED`). That app's source keeps running and none of its state was +transferred, but the run has already started live execution and cannot be +re-planned. (A run that had already completed that app's `pause_source` resumes past the check and refuses at the restore step instead; Bort restarts that app's source during cleanup and `bort status` offers to resume, and that resume stops at `pause_source` with the refusal described @@ -877,8 +951,9 @@ sudo bort rollback --live ``` The command is available only when the run transferred no database or volume -state. In that case the source applications remained running, and rollback -stops the Dokploy proxy and starts the Coolify proxy. An interrupted stateless +state. Routed source applications are paused when the cutover takes traffic +and stay stopped until retirement, so rollback restarts them, then stops the +Dokploy proxy and starts the Coolify proxy. An interrupted stateless rollback can be re-run. An interrupted stateful run with the current plan version resumes from its @@ -901,16 +976,24 @@ manual. In current runs this means a Dokploy target mutation (project or service creation, env upload, compose deploy, gateway install, or route activation) was interrupted or returned an ambiguous result. In ledgers written by older Bort versions it also covers an in-place volume copy, database restore, -or handoff whose boundary cannot be proven. Bort refuses retry without mutating -either side. Staged copies and restores in current runs are not affected: rerun -`migrate --live` and the transfer restarts. +or handoff whose boundary cannot be proven, and any staged copy or restore that +may have started under the previous staged-transfer version, which had no +`bort-pin-*` containers. Bort refuses retry without mutating +either side. A staged copy or restore in a current run whose `bort-pin-*` +container is still present can be retried: rerun `migrate --live` and the +transfer restarts. If the pin is missing after state transfer started, +including after an interrupted target handoff, Bort cannot prove the staging +volumes stayed protected and requires the `recover-authority` command +`bort status` shows. When the durable Dokploy owner still identifies the same run, finish manual recovery through that run: -- To return to the source, first stop and fence every target writer, reverse - target traffic, and verify that the source is healthy and authoritative. Then - run: +- To return to the source, reverse target traffic and remove every target app + container attached to a transferred staging volume. Stopping a container is + not enough because the stopped container remains attached. Before recreating + the target, change it to use fresh volumes or no transferred volumes. Verify + that the source is healthy and authoritative, then run: ```sh sudo bort recover-authority --run --authority source \ @@ -928,12 +1011,16 @@ recovery through that run: --confirm 'recover as target' ``` - When source attestation still passes, Bort records target authority. For a + If `bort status` reports that a staged state transfer did not finish, only + source recovery is available: Bort refuses every `--authority target` form. + Otherwise, when source attestation still passes, Bort records target + authority. For a source without an external orchestrator, `commit --apply` can then retire the - exact reviewed containers. For Coolify, disable future deployments for the - reviewed apps and manually retire every reviewed source app (and the source - proxy when the cutover moved routes) instead; `commit --apply` refuses - because Bort cannot durably fence Coolify. If the + exact reviewed containers. For Coolify, keep the control plane stopped and + remove the reviewed source containers with the `docker rm -f` command Bort + prints instead; `commit --apply` refuses + because Bort cannot durably fence Coolify. Keep Coolify stopped at least until + the `--source-retired` command below succeeds. If the reviewed source daemon or containers are no longer attestable, the recovery command also refuses without recording the authority decision. In either manual-retirement case, run the second command Bort prints with @@ -943,8 +1030,10 @@ recovery through that run: because Bort cannot verify the manual boundary. `recover-authority` records work that you already completed and verified. It -does not stop containers, move data, or switch traffic. It refuses a missing or -different owner and a mismatched confirmation phrase. The plain +does not stop or remove application containers, move data, or switch traffic. +During successful finalization it can remove only the validated Bort-owned +read-only pin containers that protect transferred staging volumes. It refuses a +missing or different owner and a mismatched confirmation phrase. The plain `--authority source` and `--authority target` forms also refuse a run that has no authority ambiguity, no unavailable automatic rollback, and no unreleased host ownership after a definite live-apply failure. `--authority target @@ -982,12 +1071,24 @@ retirement starts, automated rollback is no longer available, even if commit is interrupted; rerun `commit --apply` to finish acceptance. For a Coolify source, `commit --apply` refuses before source mutation because a -queued or future deployment could replace a stopped container. Disable future -deployments in Coolify, manually retire every reviewed source app and the source -proxy, and verify they remain retired. Then run the exact +queued or future deployment could replace a stopped container. If live apply +already fenced Coolify for a stateful move, keep the restart policy you +recorded before that fence: the container's policy now reads `no`, so do not +record it again. Otherwise, record the Coolify control plane's restart policy +with +`sudo docker --host unix:///var/run/docker.sock inspect --format '{{.HostConfig.RestartPolicy.Name}}' coolify`, +then run +`sudo docker --host unix:///var/run/docker.sock update --restart=no coolify && sudo docker --host unix:///var/run/docker.sock stop coolify` +even if it is already stopped. Remove +the reviewed source app containers and the source proxy container with the +exact `sudo docker rm -f ...` command Bort prints, and verify they stay +removed. Then run the exact `recover-authority --authority target --source-retired --confirm ...` command Bort prints. That command records permanent target acceptance without mutating -the source. +the source. Leave Coolify stopped afterwards when nothing else on the host needs +it: starting it again can recreate its proxy and redeploy the removed apps. If +other apps still need Coolify, restore the restart policy you recorded before +fencing, start it, and immediately delete the migrated apps in Coolify. ## Audit and clean up @@ -1007,16 +1108,17 @@ Ordinary cleanup and destructive source purge are separate operations. See the | Bort cannot find the expected run | Return to the original working directory and original OS user. Do not create a replacement workspace accidentally. | | Stateless live apply was interrupted at a safe completed boundary | Run `sudo bort status`, then rerun `sudo bort migrate --live` to resume from the saved progress. | | Live apply reports `MANUAL STATE` for a stateful run | The run was applied with an older plan version that copied state into a deployed target. The blocked run cannot continue. Complete recovery with the guidance shown by `bort status`, then create a new run. | -| A staged state transfer fails (source restarted, foreign-owned volume, attached staging volume) | That app's target was not deployed. Fix the reported cause and rerun `sudo bort migrate --live`; the transfer restarts from the beginning for that volume or database. | +| Stateful live apply refuses because the Coolify control plane is running or would restart, or a Coolify deployment helper is still running deployment commands | Wait until no Coolify deployment is queued or running, record the `coolify` container's restart policy, run `sudo docker --host unix:///var/run/docker.sock update --restart=no coolify && sudo docker --host unix:///var/run/docker.sock stop coolify`, wait until no `coolify-helper` container is still deploying, then rerun `sudo bort migrate --live`. Restore the policy and start Coolify only after source-authority recovery succeeds; after target acceptance, restore the policy and start it only if other apps need it, then immediately delete the migrated apps in Coolify. | +| A staged state transfer fails (source restarted, foreign-owned volume, or unexpected attachment) | Run `sudo bort status` and follow its recovery. If it offers `migrate --live`, rerun it; the transfer restarts and reuses the preserved pin. If it requires authority recovery and reports that the staged transfer did not finish, only source recovery is available. Do not remove a `bort-pin-*` container manually. Bort may restart the source after a retryable pre-handoff failure, but it keeps the pin for the next attempt. If handoff may have started, Bort keeps the source stopped and reports whether it reverified the pin. Source recovery checks that no target container is attached, even if the pin is absent. Final target acceptance requires a durable record that each transfer into staging completed and removes a remaining pin only when the target holds exactly the recorded attachments; when no pin remains and the run's ledger records a completed target handoff, Bort does not recheck target attachments, so verify the target's attachments yourself before accepting — after an interrupted handoff, target-authority recovery still verifies them. Host ownership is released only after these checks pass. | | A stateful plan needs a bind-mount copy | Bort refuses before pausing the source. Re-plan with `bort migrate --run `; current plans keep same-host bind mounts at their existing paths, and the run stays editable because nothing live has started. | | A named volume is mounted by more than one compose service | Bort refuses before changing Dokploy or pausing the source. For a data store volume, choose `bort data --recreate` or `--managed` and re-plan with `bort migrate --run `; otherwise change the source compose so one service mounts the volume and scan a new run. | -| A Postgres data directory is not on a named volume the service mounts, or a writable bind mount sits inside it | When the service mounts no named volume, sets `PGDATA` in its compose `environment` to a literal path that breaks this rule, or interpolates a mount target, Bort refuses before live apply (`PLAN BLOCKED`): choose `bort data --recreate` or `--managed` and re-plan with `bort migrate --run `, or change the source compose and scan a new run. When the image or interpolation decides `PGDATA`, only the created staged container reveals the directory, so Bort refuses at `pause_source` (`NEW RUN REQUIRED`) with that app's source still running and none of its state transferred; the run cannot be re-planned. If no other app's source was paused or handed off, delete or reconcile the Dokploy resources the run created, run the exact `recover-authority --authority source` command `bort status` shows to release host ownership, then choose a strategy or change the source compose and create a new run; otherwise restore the earlier apps' source writers and traffic manually, release host ownership with the source-authority command `bort status` shows, then create a new run. | -| A Dokploy target mutation (project or service creation, env upload, compose deploy, gateway install, route activation) was interrupted or ambiguous, or an older-plan ledger cannot prove writer authority after an in-place copy, restore, or handoff | Bort refuses automatic retry and rollback because Dokploy cannot durably fence the target. Inspect and preserve both sides. If the matching owner remains, establish authority manually and run the exact `recover-authority` command shown by `bort status`; otherwise complete recovery manually before starting a fresh run. | -| A deployed target fails its migrated-volume check and Bort cannot stop its containers | Bort leaves the paused source stopped and refuses automatic retry, because target containers may still be writing to the transferred state. Stop the target containers yourself, then run the exact `recover-authority` command shown by `bort status`. | +| A Postgres data directory is not on a named volume the service mounts, a writable bind mount sits inside it, or the service declares Compose secrets/configs | When the service mounts no named volume, sets `PGDATA` in its compose `environment` to a literal path that breaks this rule, interpolates a mount target, or declares Compose `secrets` or `configs`, Bort refuses before live apply (`PLAN BLOCKED`): choose `bort data --recreate` or `--managed` and re-plan with `bort migrate --run `, or change the source compose and scan a new run. When the image or interpolation decides `PGDATA`, only the created staged container reveals the directory, so Bort refuses at `pause_source` (`NEW RUN REQUIRED`) with that app's source still running and none of its state transferred; the run cannot be re-planned. If no other app's source was paused or handed off, delete or reconcile the Dokploy resources the run created, run the exact `recover-authority --authority source` command `bort status` shows to release host ownership, then choose a strategy or change the source compose and create a new run; otherwise restore the earlier apps' source writers and traffic manually, release host ownership with the source-authority command `bort status` shows, then create a new run. | +| A Dokploy target mutation (project or service creation, env upload, compose deploy, gateway install, route activation) was interrupted or ambiguous, or an older-plan ledger cannot prove writer authority after an in-place copy, restore, or handoff, or after a staged copy or restore from the previous staged-transfer version without `bort-pin-*` containers | Bort refuses automatic retry and rollback because Dokploy cannot durably fence the target. Inspect and preserve both sides. If the matching owner remains, establish authority manually and run the exact `recover-authority` command shown by `bort status`; otherwise complete recovery manually before starting a fresh run. | +| A deployed target fails its migrated-volume check and Bort cannot stop its containers | Bort leaves the paused source stopped and refuses automatic retry, because target containers may still be writing to the transferred state. Stop and remove the target containers yourself, because a stopped container still holds its volume mounts, then run the exact `recover-authority` command shown by `bort status`. | | Another change is running | Keep `status` open if useful and wait. A second live command joins an active live apply; other commands that make changes must wait. | | The plan needs to change after live execution began | Keep the existing run as a record and create a new named run. A plan cannot change after live work starts. | | A stateless target fails validation after successful live apply | Run `sudo bort rollback --live` to return traffic to the source. Target resources remain for cleanup. | -| A stateful target fails validation after successful live apply | Bort refuses automatic rollback because Dokploy cannot durably fence target writers. Preserve both sides, manually restore and verify source authority, then use the `recover-authority --authority source` command shown by `bort status`. | +| A stateful target fails validation after successful live apply | Bort refuses automatic rollback because Dokploy cannot durably fence target writers. Move traffic back to the source, then remove (not just stop) every target container attached to the transferred `bort-*` staging volumes; the volumes and their data remain. Restore and verify source authority, then use the `recover-authority --authority source` command shown by `bort status`. | | Stateless rollback stops partway through | Inspect `sudo bort status`, then rerun `sudo bort rollback --live`. Commit and live apply are blocked because traffic may already have returned to the source. | | Cleanup or purge stops partway through | Stop and inspect the command output and private backup before retrying. Follow the recovery guidance in the cleanup and purge guide. | | `.bort` cannot be read | Do not change ownership or permissions blindly. Confirm the original working directory and OS user first, then restore the workspace from backup if necessary. | diff --git a/internal/cli/applied.go b/internal/cli/applied.go index 657be3e..75d92b2 100644 --- a/internal/cli/applied.go +++ b/internal/cli/applied.go @@ -20,7 +20,8 @@ const ( appliedPlanV1Alpha1 = "v1alpha1" appliedPlanV1Alpha2 = "v1alpha2" appliedPlanV1Alpha3 = "v1alpha3" - appliedPlanCurrent = appliedPlanV1Alpha3 + appliedPlanV1Alpha4 = "v1alpha4" + appliedPlanCurrent = appliedPlanV1Alpha4 ) // runApplied is the per-run audit ledger. it captures the outcome of every @@ -41,15 +42,16 @@ type runApplied struct { } type appliedStep struct { - Index int `json:"index"` - Kind string `json:"kind"` - App string `json:"app"` - Ref string `json:"ref"` - Status string `json:"status"` - UpdatedAt time.Time `json:"updatedAt"` - Error string `json:"error,omitempty"` - MutationAmbiguous *bool `json:"mutationAmbiguous,omitempty"` - RequiresNewRun bool `json:"requiresNewRun,omitempty"` + Index int `json:"index"` + Kind string `json:"kind"` + App string `json:"app"` + Ref string `json:"ref"` + Status string `json:"status"` + UpdatedAt time.Time `json:"updatedAt"` + Error string `json:"error,omitempty"` + MutationAmbiguous *bool `json:"mutationAmbiguous,omitempty"` + AuthorityRecoveryRequired bool `json:"authorityRecoveryRequired,omitempty"` + RequiresNewRun bool `json:"requiresNewRun,omitempty"` } type appliedApp struct { @@ -78,7 +80,7 @@ func readRunApplied(path string, run migrationRun) (runApplied, error) { return runApplied{}, fmt.Errorf("%s has unsupported apiVersion %q (want %q or %q)", path, applied.APIVersion, appliedAPIVersion, appliedLegacyAPIVersion) } switch applied.PlanVersion { - case "", appliedPlanV1Alpha1, appliedPlanV1Alpha2, appliedPlanV1Alpha3: + case "", appliedPlanV1Alpha1, appliedPlanV1Alpha2, appliedPlanV1Alpha3, appliedPlanV1Alpha4: default: return runApplied{}, fmt.Errorf("%s has unsupported planVersion %q", path, applied.PlanVersion) } @@ -149,6 +151,7 @@ func recordAppliedStep(applied runApplied, progress dokploy.StepProgress) runApp step.Error = progress.Err.Error() ambiguous := progress.MutationAmbiguous step.MutationAmbiguous = &ambiguous + step.AuthorityRecoveryRequired = progress.AuthorityRecoveryRequired step.RequiresNewRun = progress.RequiresNewRun } if progress.Step.App != "" && progress.Target != nil { @@ -218,7 +221,7 @@ func (l *appliedLedger) PrepareRetry(index int) error { next.Steps = append(next.Steps, step) } } - if index == 0 && !appliedHasHistory(next) { + if (index == 0 && !appliedHasHistory(next)) || appliedV1Alpha3UsesCurrentPlan(l.state) { next.PlanVersion = appliedPlanCurrent } if len(next.Steps) == len(l.state.Steps) && next.RecoveryProtocol == appliedRecoveryProtocol && next.APIVersion == appliedAPIVersion && next.PlanVersion == l.state.PlanVersion { @@ -369,11 +372,22 @@ func livePlanForApplied(run loadedMigrationRun, applied runApplied) dokploy.Plan return dokploy.LegacyPlanFromArtifactsV1Alpha1(run.Prepare, run.Sync, run.Cutover) case appliedPlanV1Alpha2: return dokploy.PlanFromArtifactsV1Alpha2(run.Prepare, run.Sync, run.Cutover) + case appliedPlanV1Alpha3: + if appliedV1Alpha3UsesCurrentPlan(applied) { + return dokploy.PlanFromArtifacts(run.Prepare, run.Sync, run.Cutover) + } + return dokploy.LegacyPlanFromArtifactsV1Alpha3(run.Prepare, run.Sync, run.Cutover) default: return dokploy.PlanFromArtifacts(run.Prepare, run.Sync, run.Cutover) } } +func appliedV1Alpha3UsesCurrentPlan(applied runApplied) bool { + return appliedPlanVersion(applied) == appliedPlanV1Alpha3 && + applied.SucceededAt == nil && + !appliedV1Alpha3StateTransferMayHaveRun(applied) +} + // appliedPlanTransfersStateInPlace reports whether the ledger is pinned to // a plan order that deploys the target before copying state into its // volumes. Bort no longer continues such runs past the stateful steps. @@ -398,6 +412,14 @@ func appliedAuthorityAmbiguity(applied runApplied) applyAuthorityAmbiguity { if applied.SucceededAt != nil { return applyAuthorityUnambiguous } + for _, recorded := range applied.Steps { + if recorded.AuthorityRecoveryRequired { + return applyWriterAuthorityAmbiguous + } + } + if appliedPlanVersion(applied) == appliedPlanV1Alpha3 && appliedV1Alpha3StateTransferMayHaveRun(applied) { + return applyWriterAuthorityAmbiguous + } if applied.APIVersion == appliedAPIVersion && applied.RecoveryProtocol == appliedRecoveryProtocol { for _, recorded := range applied.Steps { if appliedStepMutatesDokploy(recorded.Kind) && @@ -428,6 +450,47 @@ func appliedAuthorityAmbiguity(applied runApplied) applyAuthorityAmbiguity { return ambiguity } +func appliedV1Alpha3StateTransferMayHaveRun(applied runApplied) bool { + for _, recorded := range applied.Steps { + kind := dokploy.StepKind(recorded.Kind) + if (kind == dokploy.StepRestoreDataStore || kind == dokploy.StepSyncVolume) && appliedStepMayHaveRun(recorded) { + return true + } + } + return false +} + +func appliedStagingTransferApps(applied runApplied) []string { + apps := map[string]struct{}{} + for _, recorded := range applied.Steps { + kind := dokploy.StepKind(recorded.Kind) + if recorded.App != "" && (kind == dokploy.StepRestoreDataStore || kind == dokploy.StepSyncVolume) && appliedStepMayHaveRun(recorded) { + apps[recorded.App] = struct{}{} + } + } + result := make([]string, 0, len(apps)) + for app := range apps { + result = append(result, app) + } + sort.Strings(result) + return result +} + +func appliedHandedOffApps(applied runApplied) []string { + apps := map[string]struct{}{} + for _, recorded := range applied.Steps { + if recorded.App != "" && dokploy.StepKind(recorded.Kind) == dokploy.StepPushImage && appliedStepCompleted(recorded) { + apps[recorded.App] = struct{}{} + } + } + result := make([]string, 0, len(apps)) + for app := range apps { + result = append(result, app) + } + sort.Strings(result) + return result +} + func appliedStepMutatesDokploy(kind string) bool { switch dokploy.StepKind(kind) { case dokploy.StepCreateProject, dokploy.StepCreateService, dokploy.StepUploadEnv, dokploy.StepPushImage, dokploy.StepInstallGateway, dokploy.StepActivateRoutes: diff --git a/internal/cli/applied_test.go b/internal/cli/applied_test.go index 44ab1d1..6618baf 100644 --- a/internal/cli/applied_test.go +++ b/internal/cli/applied_test.go @@ -3,10 +3,14 @@ package cli import ( "errors" "path/filepath" + "slices" "strings" "testing" "time" + "github.com/aikins01/bort/internal/gateway" + "github.com/aikins01/bort/internal/preparer" + syncplan "github.com/aikins01/bort/internal/sync" "github.com/aikins01/bort/internal/target/dokploy" ) @@ -686,3 +690,96 @@ func TestRecordAppliedStepPersistsTargetIdentity(t *testing.T) { t.Fatalf("unexpected persisted target identity: %#v", identity) } } + +func TestPersistedV1Alpha3StateTransferUsesLegacyPlanAndRequiresAuthorityRecovery(t *testing.T) { + app := preparer.AppPlan{ + Name: "api", + TargetResources: &preparer.TargetResources{Dokploy: &preparer.DokployResources{ + ComposeApp: preparer.DokployComposeApp{Name: "api"}, + }}, + } + app.Resources.Volumes = []preparer.VolumeResource{{Service: "web", Type: "volume", Name: "data", Target: "/data"}} + run := loadedMigrationRun{ + Run: migrationRun{Name: "legacy-staged", RunDir: t.TempDir(), Target: "dokploy"}, + Prepare: preparer.Result{Apps: []preparer.AppPlan{app}}, + Sync: syncplan.Result{Apps: []syncplan.AppPlan{{Name: "api", Steps: []syncplan.Step{{ + ResourceType: "volume", + ResourceRef: "volume:web -> /data", + Strategy: syncplan.StrategyDockerVolumeArchive, + }}}}}, + Cutover: gateway.Result{}, + } + legacy := dokploy.LegacyPlanFromArtifactsV1Alpha3(run.Prepare, run.Sync, run.Cutover) + preTransfer := runApplied{APIVersion: appliedAPIVersion, RecoveryProtocol: appliedRecoveryProtocol, PlanVersion: appliedPlanV1Alpha3, RunName: run.Run.Name, Target: run.Run.Target} + for index, step := range legacy.Steps { + preTransfer = recordAppliedStep(preTransfer, dokploy.StepProgress{Index: index, Step: step, Status: dokploy.StepStatusOK}) + if step.Kind == dokploy.StepPauseSource { + break + } + } + safePlan := livePlanForApplied(run, preTransfer) + if slices.ContainsFunc(safePlan.Steps, func(step dokploy.Step) bool { return step.Kind == dokploy.StepResumeSource }) { + t.Fatalf("pre-transfer v1alpha3 run kept the unsafe legacy source resume: %#v", safePlan.Steps) + } + safePrefix := completedApplyPrefix(safePlan.Steps, preTransfer) + if safePrefix >= len(safePlan.Steps) || safePlan.Steps[safePrefix].Kind != dokploy.StepSyncVolume { + t.Fatalf("pre-transfer v1alpha3 run did not retain its completed prefix under the safe plan: prefix=%d steps=%#v", safePrefix, safePlan.Steps) + } + path := filepath.Join(run.Run.RunDir, "applied.json") + if err := writeRunApplied(path, preTransfer); err != nil { + t.Fatal(err) + } + ledger, err := newAppliedLedger(path, run.Run) + if err != nil { + t.Fatal(err) + } + if err := ledger.PrepareRetry(safePrefix); err != nil { + t.Fatal(err) + } + if got := ledger.Snapshot().PlanVersion; got != appliedPlanCurrent { + t.Fatalf("prepared legacy ledger planVersion=%q, want %q", got, appliedPlanCurrent) + } + if err := ledger.Record(dokploy.StepProgress{Index: safePrefix, Step: safePlan.Steps[safePrefix], Status: dokploy.StepStatusStarted}); err != nil { + t.Fatal(err) + } + restarted, err := readRunApplied(path, run.Run) + if err != nil { + t.Fatal(err) + } + if restarted.PlanVersion != appliedPlanCurrent || len(restarted.Steps) != len(preTransfer.Steps)+1 || restarted.Steps[len(restarted.Steps)-1].Kind != string(dokploy.StepSyncVolume) { + t.Fatalf("restarted ledger lost its current-plan transfer: %#v", restarted) + } + restartedPlan := livePlanForApplied(run, restarted) + if slices.ContainsFunc(restartedPlan.Steps, func(step dokploy.Step) bool { return step.Kind == dokploy.StepResumeSource }) { + t.Fatalf("repinned ledger reverted to the legacy source resume after restart: %#v", restartedPlan.Steps) + } + if err := validateApplyResumeAuthority(restarted); err != nil { + t.Fatalf("repinned current-plan transfer became ambiguous after restart: %v", err) + } + + applied := runApplied{APIVersion: appliedAPIVersion, RecoveryProtocol: appliedRecoveryProtocol, PlanVersion: appliedPlanV1Alpha3, RunName: run.Run.Name, Target: run.Run.Target} + for index, step := range legacy.Steps { + applied = recordAppliedStep(applied, dokploy.StepProgress{Index: index, Step: step, Status: dokploy.StepStatusOK}) + if step.Kind == dokploy.StepSyncVolume { + break + } + } + if err := writeRunApplied(path, applied); err != nil { + t.Fatal(err) + } + loaded, err := readRunApplied(path, run.Run) + if err != nil { + t.Fatal(err) + } + plan := livePlanForApplied(run, loaded) + if !slices.Equal(plan.Steps, legacy.Steps) { + t.Fatalf("v1alpha3 ledger did not reconstruct its historical plan: got %#v want %#v", plan.Steps, legacy.Steps) + } + prefix := completedApplyPrefix(plan.Steps, loaded) + if prefix >= len(plan.Steps) || plan.Steps[prefix].Kind != dokploy.StepResumeSource { + t.Fatalf("v1alpha3 resume lost its legacy step indexes: prefix=%d steps=%#v", prefix, plan.Steps) + } + if err := validateApplyResumeAuthority(loaded); err == nil || !strings.Contains(err.Error(), "establish writer and traffic authority manually") { + t.Fatalf("v1alpha3 transfer without a durable pin was allowed to resume: %v", err) + } +} diff --git a/internal/cli/authority.go b/internal/cli/authority.go index 4944670..0e3b7d3 100644 --- a/internal/cli/authority.go +++ b/internal/cli/authority.go @@ -10,6 +10,44 @@ import ( "github.com/aikins01/bort/internal/target/dokploy" ) +var releaseAuthorityStagingVolumePins = func(ctx context.Context, run loadedMigrationRun, plan dokploy.Plan, targetAuthority bool) error { + client, err := authorityRecoveryDokployClient(run, plan, targetAuthority) + if err != nil { + return err + } + return client.ReleaseStagingVolumePins(ctx, plan, targetAuthority) +} + +var validateAuthorityStagingVolumePins = func(ctx context.Context, run loadedMigrationRun, plan dokploy.Plan, targetAuthority bool) error { + client, err := authorityRecoveryDokployClient(run, plan, targetAuthority) + if err != nil { + return err + } + return client.ValidateStagingVolumePins(ctx, plan, targetAuthority) +} + +func validateRecoveredTargetAuthority(ctx context.Context, run loadedMigrationRun) error { + plan, err := authorityRecoveryPlan(run) + if err != nil { + return err + } + return validateAuthorityStagingVolumePins(ctx, run, plan, true) +} + +func authorityRecoveryDokployClient(run loadedMigrationRun, plan dokploy.Plan, targetAuthority bool) (*dokploy.Client, error) { + if !targetAuthority || !dokploy.PlanStagesVolumes(plan) { + return &dokploy.Client{}, nil + } + client, err := lookupDokployClient(run.Run.Target) + if err != nil { + return nil, fmt.Errorf("load Dokploy credentials to verify target attachments: %w", err) + } + if err := validateAppliedTargetOrigin(run.Applied, client.BaseURL); err != nil { + return nil, fmt.Errorf("verify target attachments: %w", err) + } + return client, nil +} + func runRecoverAuthority(ctx context.Context, args []string, stdout, stderr io.Writer) error { fs := flag.NewFlagSet("recover-authority", flag.ContinueOnError) fs.SetOutput(stderr) @@ -84,13 +122,33 @@ func runRecoverAuthority(ctx context.Context, args []string, stdout, stderr io.W } if !found { if authority == dokployTrafficSource && run.Run.AuthorityFinalizedAt == nil { + if err := releaseRecoveredAuthorityStagingVolumePins(ctx, run, false); err != nil { + return fmt.Errorf("remove source-authority staging-volume pins before finalization: %w", err) + } if err := markRunAuthorityFinalizedLocked(run.Run); err != nil { return fmt.Errorf("record source-authority finalization: %w", err) } } + if authority == dokployTrafficTarget && run.Run.CommittedAt != nil { + if err := releaseRecoveredAuthorityStagingVolumePins(ctx, run, true); err != nil { + return fmt.Errorf("remove target-authority staging-volume pins after source retirement: %w", err) + } + } fmt.Fprintf(stdout, "Authority recovery already complete for run %s: %s authority.\n", run.Run.Name, authority) return nil } + if authority == dokployTrafficTarget { + incomplete, err := incompleteStagingTransferApps(run) + if err != nil { + return fmt.Errorf("check staged transfers before recording target authority: %w", err) + } + if len(incomplete) > 0 { + return fmt.Errorf("target authority was not recorded: the staged state transfer for app(s) %s did not finish, so the target cannot hold the complete source state; restore the source and run `%s`", strings.Join(incomplete, ", "), authorityRecoveryCommand(run, dokployTrafficSource)) + } + if err := validateRecoveredTargetAuthority(ctx, run); err != nil { + return fmt.Errorf("target authority was not recorded because the transferred staging volumes do not match the target: %w", err) + } + } if authority == dokployTrafficTarget && !sourceRetired { var sourceAttestationErr error if err := validateLocalSourceAttestation(run); err != nil { @@ -123,6 +181,9 @@ func runRecoverAuthority(ctx context.Context, args []string, stdout, stderr io.W if err := markRunRolledBackLocked(run.Run); err != nil { return fmt.Errorf("record manual source recovery completion: %w", err) } + if err := releaseRecoveredAuthorityStagingVolumePins(ctx, run, false); err != nil { + return fmt.Errorf("source authority was recorded, but staging-volume validation or pin removal failed; host ownership remains held so this recovery can be retried: %w", err) + } if err := releaseDokployTrafficOwner(run.Run); err != nil { return fmt.Errorf("release source-authority host ownership: %w", err) } @@ -148,6 +209,9 @@ func runRecoverAuthority(ctx context.Context, args []string, stdout, stderr io.W if err := markRunCommittedLocked(run.Run); err != nil { return fmt.Errorf("record manual source retirement completion: %w", err) } + if err := releaseRecoveredAuthorityStagingVolumePins(ctx, run, true); err != nil { + return fmt.Errorf("target authority and source retirement were recorded, but staging-volume validation or pin removal failed; host ownership remains held so this recovery can be retried: %w", err) + } if err := releaseDokployTargetOwner(run.Run); err != nil { return fmt.Errorf("release target-authority host ownership: %w", err) } @@ -162,6 +226,37 @@ func runRecoverAuthority(ctx context.Context, args []string, stdout, stderr io.W return nil } +func releaseRecoveredAuthorityStagingVolumePins(ctx context.Context, run loadedMigrationRun, targetAuthority bool) error { + plan, err := authorityRecoveryPlan(run) + if err != nil { + return err + } + return releaseAuthorityStagingVolumePins(ctx, run, plan, targetAuthority) +} + +func authorityRecoveryPlan(run loadedMigrationRun) (dokploy.Plan, error) { + plan := livePlanForApplied(run, run.Applied) + plan.RunName = run.Run.Name + plan.RunDir = run.Run.RunDir + runID, err := dokployTrafficRunID(run.Run) + if err != nil { + return dokploy.Plan{}, err + } + plan.RunID = runID + plan.TargetIdentities = appliedTargetIdentities(run.Applied) + plan.StagingTransferApps = appliedStagingTransferApps(run.Applied) + plan.HandedOffApps = appliedHandedOffApps(run.Applied) + return plan, nil +} + +func incompleteStagingTransferApps(run loadedMigrationRun) ([]string, error) { + plan, err := authorityRecoveryPlan(run) + if err != nil { + return nil, err + } + return dokploy.IncompleteStagingTransfers(plan) +} + func recordOwnerlessSourceRetirement(run loadedMigrationRun, stdout io.Writer) error { if err := markRunAuthorityResolvedLocked(run.Run, dokployTrafficTarget); err != nil { return fmt.Errorf("record manual target-authority resolution: %w", err) @@ -296,7 +391,14 @@ func pendingAuthorityRecoveryCommand(run loadedMigrationRun) string { func authorityRecoveryInstruction(run loadedMigrationRun) string { if authorityRecoveryAvailable(run) { - return fmt.Sprintf("inspect and preserve both sides, manually fence the other side and verify authority, then run `%s` for source or `%s` for target", authorityRecoveryCommand(run, dokployTrafficSource), authorityRecoveryCommand(run, dokployTrafficTarget)) + return "inspect and preserve both sides, " + authorityRecoveryChoice(run) } return "inspect and preserve both sides, establish writer and traffic authority manually, then start a fresh migration run" } + +func authorityRecoveryChoice(run loadedMigrationRun) string { + if incomplete, err := incompleteStagingTransferApps(run); err == nil && len(incomplete) > 0 { + return fmt.Sprintf("restore and verify the source (the staged state transfer for %s did not finish, so target recovery is unavailable; remove, not just stop, any target container attached to the bort staging volumes), then run `%s`", strings.Join(incomplete, ", "), authorityRecoveryCommand(run, dokployTrafficSource)) + } + return fmt.Sprintf("manually fence the other side and verify authority, then run `%s` for source or `%s` for target", authorityRecoveryCommand(run, dokployTrafficSource), authorityRecoveryCommand(run, dokployTrafficTarget)) +} diff --git a/internal/cli/authority_test.go b/internal/cli/authority_test.go index 8c0f82b..a940fc1 100644 --- a/internal/cli/authority_test.go +++ b/internal/cli/authority_test.go @@ -3,6 +3,7 @@ package cli import ( "bytes" "context" + "errors" "fmt" "io" "os" @@ -12,6 +13,7 @@ import ( "time" "github.com/aikins01/bort/internal/manifest" + "github.com/aikins01/bort/internal/preparer" "github.com/aikins01/bort/internal/target/dokploy" ) @@ -123,6 +125,87 @@ func TestRecoverAuthorityFinalizesSourceAndReleasesHost(t *testing.T) { } } +func TestRecoverAuthorityKeepsHostOwnershipWhenPinCleanupFails(t *testing.T) { + for _, authority := range []string{dokployTrafficSource, dokployTrafficTarget} { + t.Run(authority, func(t *testing.T) { + run := writeAmbiguousAuthorityRun(t, "pin-cleanup-"+authority) + previous := releaseAuthorityStagingVolumePins + calls := 0 + releaseAuthorityStagingVolumePins = func(_ context.Context, _ loadedMigrationRun, plan dokploy.Plan, targetAuthority bool) error { + calls++ + if plan.RunName != run.Run.Name || plan.RunDir != run.Run.RunDir || plan.RunID == "" || targetAuthority != (authority == dokployTrafficTarget) { + t.Fatalf("pin cleanup received incomplete recovery identity: plan=%#v target=%t", plan, targetAuthority) + } + if calls == 1 { + return errors.New("pin cleanup failed") + } + return nil + } + t.Cleanup(func() { releaseAuthorityStagingVolumePins = previous }) + + args := []string{"--run", run.Run.Name, "--authority", authority, "--confirm", authorityRecoveryConfirmation(run.Run, authority)} + if authority == dokployTrafficTarget { + args = []string{"--run", run.Run.Name, "--authority", authority, "--source-retired", "--confirm", authorityRecoverySourceRetiredConfirmation(run.Run)} + } + err := runRecoverAuthority(context.Background(), args, io.Discard, io.Discard) + if err == nil || !strings.Contains(err.Error(), "host ownership remains held") || !strings.Contains(err.Error(), "pin cleanup failed") { + t.Fatalf("expected pin cleanup failure to retain host ownership, got %v", err) + } + owner, found, err := readDokployTrafficOwner() + if err != nil || !found || owner.Authority != authority { + t.Fatalf("pin cleanup failure released or changed owner: owner=%#v found=%t err=%v", owner, found, err) + } + interrupted, err := loadMigrationRun(run.Run.Name) + if err != nil { + t.Fatal(err) + } + if authority == dokployTrafficSource && (interrupted.Run.RolledBackAt == nil || interrupted.Run.AuthorityFinalizedAt != nil) { + t.Fatalf("source recovery did not stop at pin cleanup boundary: %#v", interrupted.Run) + } + if authority == dokployTrafficTarget && interrupted.Run.CommittedAt == nil { + t.Fatalf("target recovery did not record retirement before pin cleanup: %#v", interrupted.Run) + } + + if err := runRecoverAuthority(context.Background(), args, io.Discard, io.Discard); err != nil { + t.Fatalf("pin cleanup retry did not finish recovery: %v", err) + } + owner, found, err = readDokployTrafficOwner() + if err != nil || !found || owner.Authority != dokployTrafficReleased || calls != 2 { + t.Fatalf("recovery retry did not release owner: owner=%#v found=%t calls=%d err=%v", owner, found, calls, err) + } + }) + } +} + +func TestAuthorityRecoveryCarriesAppliedTransferEvidenceIntoPinCleanup(t *testing.T) { + run := writeAmbiguousAuthorityRun(t, "transfer-evidence") + run.Applied.PlanVersion = appliedPlanV1Alpha3 + run.Applied.Steps = []appliedStep{{ + Index: 1, + Kind: string(dokploy.StepRestoreDataStore), + App: "api", + Ref: "postgres:db", + Status: string(dokploy.StepStatusError), + }} + previous := releaseAuthorityStagingVolumePins + called := false + releaseAuthorityStagingVolumePins = func(_ context.Context, _ loadedMigrationRun, plan dokploy.Plan, targetAuthority bool) error { + called = true + if !targetAuthority || len(plan.StagingTransferApps) != 1 || plan.StagingTransferApps[0] != "api" { + t.Fatalf("pin cleanup did not receive transfer evidence: plan=%#v target=%t", plan, targetAuthority) + } + return nil + } + t.Cleanup(func() { releaseAuthorityStagingVolumePins = previous }) + + if err := releaseRecoveredAuthorityStagingVolumePins(context.Background(), run, true); err != nil { + t.Fatal(err) + } + if !called { + t.Fatal("pin cleanup was not invoked") + } +} + func TestRecoverAuthorityFinalizesTargetForCommitAndRefusesAutomaticRollback(t *testing.T) { writeAmbiguousAuthorityRun(t, "target-run") args := []string{"--run", "target-run", "--authority", "target", "--confirm", "recover target-run as target"} @@ -153,7 +236,7 @@ func TestRecoverAuthorityFinalizesTargetForCommitAndRefusesAutomaticRollback(t * if phase := migrationRunPhase(completed); phase != "applied" { t.Fatalf("target recovery phase=%q, want applied", phase) } - if next := nextSafeStep(completed, nil); !strings.Contains(next.Action, "--source-retired") || !strings.Contains(next.Action, "disable future Coolify deployments") || strings.Contains(next.Action, "commit --apply") || strings.Contains(next.Action, "rollback --live") { + if next := nextSafeStep(completed, nil); !strings.Contains(next.Action, "--source-retired") || !strings.Contains(next.Action, "even if it is already stopped") || strings.Contains(next.Action, "commit --apply") || strings.Contains(next.Action, "rollback --live") { t.Fatalf("target recovery next step is unsafe: %#v", next) } if err := runRecoverAuthority(context.Background(), args, io.Discard, io.Discard); err != nil { @@ -501,3 +584,154 @@ func TestRecoverAuthorityAcceptsDefiniteFailureWhileHostOwnerIsPending(t *testin t.Fatalf("definite failure without host ownership must still be refused, got %v", err) } } + +func TestAuthorityRecoveryClientBindsTargetCredentialsToRunOrigin(t *testing.T) { + origin, err := dokploy.NormalizeTokenBaseURL("http://127.0.0.1:3030") + if err != nil { + t.Fatal(err) + } + t.Setenv(dokploy.EnvBaseURL, "http://127.0.0.1:3030") + t.Setenv(dokploy.EnvToken, "token-1") + + source, err := authorityRecoveryDokployClient(loadedMigrationRun{}, dokploy.Plan{}, false) + if err != nil || source.BaseURL != "" { + t.Fatalf("source recovery should not need Dokploy credentials: client=%#v err=%v", source, err) + } + stateless, err := authorityRecoveryDokployClient(loadedMigrationRun{}, dokploy.Plan{}, true) + if err != nil || stateless.BaseURL != "" { + t.Fatalf("target recovery without staged volumes should not need Dokploy credentials: client=%#v err=%v", stateless, err) + } + app := preparer.AppPlan{Name: "api"} + app.Resources.Volumes = []preparer.VolumeResource{{Service: "web", Type: "volume", Name: "api-data", Target: "/data"}} + staged := dokploy.Plan{ + Prepare: preparer.Result{Apps: []preparer.AppPlan{app}}, + Steps: []dokploy.Step{{Kind: dokploy.StepSyncVolume, App: "api", Ref: "volume:web -> /data"}, {Kind: dokploy.StepPushImage, App: "api"}}, + } + target, err := authorityRecoveryDokployClient(loadedMigrationRun{Applied: runApplied{TargetOrigin: origin}}, staged, true) + if err != nil { + t.Fatal(err) + } + if target.BaseURL != origin || target.Token != "token-1" { + t.Fatalf("target recovery client is not configured for the run origin: %#v", target) + } + if _, err := authorityRecoveryDokployClient(loadedMigrationRun{Applied: runApplied{TargetOrigin: "http://127.0.0.1:4040"}}, staged, true); err == nil || !strings.Contains(err.Error(), "bound to Dokploy origin") { + t.Fatalf("target recovery accepted credentials for another Dokploy origin: %v", err) + } +} + +func TestRecoverAuthorityRefusesTargetForUnfinishedStagedTransfer(t *testing.T) { + resetDokployTrafficOwner(t) + t.Chdir(t.TempDir()) + writeTestBundle(t, "bundle", manifest.Manifest{ + Source: manifest.Source{Platform: "coolify-local", DockerEngineID: "engine-reviewed"}, + Apps: []manifest.App{{ + Name: "api", + Services: []manifest.Service{{ + ID: "source-id", Name: "web", Image: "example/api:latest", + Mounts: []manifest.Mount{{Type: "volume", Name: "api-data", Target: "/data"}}, + }}, + }}, + }) + runCommand(t, runMigrate, []string{"--bundle", "bundle", "--run", "unfinished", "--observation-window", "0", "--rollback-window", "0"}) + markRunLocallyScanned(t, "unfinished", "coolify-local") + run, err := loadMigrationRun("unfinished") + if err != nil { + t.Fatal(err) + } + applied := newRunApplied(run.Run) + applied.Steps = []appliedStep{{Index: 1, Kind: string(dokploy.StepSyncVolume), App: "api", Ref: "volume:web -> /data", Status: string(dokploy.StepStatusError), UpdatedAt: time.Now().UTC(), Error: "copy interrupted"}} + if err := writeRunApplied(runArtifactPath(run.Run.RunDir, run.Run.Artifacts.Applied), applied); err != nil { + t.Fatal(err) + } + if err := claimDokployHostOwnership(run.Run, "http://127.0.0.1:3030", dokployCredentialID("test-token")); err != nil { + t.Fatal(err) + } + if run, err = loadMigrationRun("unfinished"); err != nil { + t.Fatal(err) + } + if incomplete, err := incompleteStagingTransferApps(run); err != nil || len(incomplete) != 1 || incomplete[0] != "api" { + t.Fatalf("unfinished staged transfer was not detected: %v %v", incomplete, err) + } + + err = runRecoverAuthority(context.Background(), []string{"--run", "unfinished", "--authority", "target", "--confirm", authorityRecoveryConfirmation(run.Run, dokployTrafficTarget)}, io.Discard, io.Discard) + if err == nil || !strings.Contains(err.Error(), "did not finish") || !strings.Contains(err.Error(), "--authority source") { + t.Fatalf("target recovery accepted an unfinished staged transfer: %v", err) + } + after, err := loadMigrationRun("unfinished") + if err != nil { + t.Fatal(err) + } + if after.Run.ResolvedAuthority != "" || after.Run.CommittedAt != nil { + t.Fatalf("refused target recovery recorded authority: %#v", after.Run) + } + if guidance := authorityRecoveryInstruction(after) + authorityRecoveryNextStep(after, "").Action; strings.Contains(guidance, "--authority target") || !strings.Contains(guidance, "--authority source") { + t.Fatalf("recovery guidance offered target recovery for an unfinished transfer: %q", guidance) + } + var status bytes.Buffer + writeAuthorityRecoveryGuidance(&status, newStyler(&status), after, "") + if strings.Contains(status.String(), "--authority target") || !strings.Contains(status.String(), "--authority source") { + t.Fatalf("status offered target recovery for an unfinished transfer:\n%s", status.String()) + } + + retried := newRunApplied(after.Run) + if err := writeRunApplied(runArtifactPath(after.Run.RunDir, after.Run.Artifacts.Applied), retried); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(after.Run.RunDir, "migrated-volumes.json"), []byte(`{"apiVersion":"bort.migrated-volumes/v1alpha2","startedApps":["api"]}`), 0o600); err != nil { + t.Fatal(err) + } + if after, err = loadMigrationRun("unfinished"); err != nil { + t.Fatal(err) + } + for _, args := range [][]string{ + {"--run", "unfinished", "--authority", "target", "--confirm", authorityRecoveryConfirmation(after.Run, dokployTrafficTarget)}, + {"--run", "unfinished", "--authority", "target", "--source-retired", "--confirm", authorityRecoverySourceRetiredConfirmation(after.Run)}, + } { + err := runRecoverAuthority(context.Background(), args, io.Discard, io.Discard) + if err == nil || !strings.Contains(err.Error(), "did not finish") { + t.Fatalf("target recovery ignored durable transfer evidence after the ledger was trimmed (%v): %v", args, err) + } + } + final, err := loadMigrationRun("unfinished") + if err != nil { + t.Fatal(err) + } + owner, found, err := readDokployTrafficOwner() + if final.Run.ResolvedAuthority != "" || final.Run.CommittedAt != nil || err != nil || !found || owner.Authority != dokployTrafficPending { + t.Fatalf("refused target recovery changed run or owner state: run=%#v owner=%#v err=%v", final.Run, owner, err) + } +} + +func TestTargetRecoveryValidatesStagingVolumesBeforeRecordingAuthority(t *testing.T) { + for _, sourceRetired := range []bool{false, true} { + t.Run(fmt.Sprintf("source-retired=%t", sourceRetired), func(t *testing.T) { + run := writeAmbiguousAuthorityRun(t, fmt.Sprintf("target-validate-%t", sourceRetired)) + previous := validateAuthorityStagingVolumePins + validateAuthorityStagingVolumePins = func(_ context.Context, _ loadedMigrationRun, _ dokploy.Plan, targetAuthority bool) error { + if !targetAuthority { + t.Fatal("target recovery validated source authority") + } + return errors.New("migrated volume bort-v has no target container") + } + t.Cleanup(func() { validateAuthorityStagingVolumePins = previous }) + args := []string{"--run", run.Run.Name, "--authority", "target", "--confirm", authorityRecoveryConfirmation(run.Run, dokployTrafficTarget)} + if sourceRetired { + args = []string{"--run", run.Run.Name, "--authority", "target", "--source-retired", "--confirm", authorityRecoverySourceRetiredConfirmation(run.Run)} + } + err := runRecoverAuthority(context.Background(), args, io.Discard, io.Discard) + if err == nil || !strings.Contains(err.Error(), "has no target container") { + t.Fatalf("target recovery did not refuse on staging validation: %v", err) + } + after, err := loadMigrationRun(run.Run.Name) + if err != nil { + t.Fatal(err) + } + if after.Run.ResolvedAuthority != "" || after.Run.CommittedAt != nil || after.Run.LiveAppliedAt != nil { + t.Fatalf("refused target recovery recorded state: %#v", after.Run) + } + if _, err := os.Stat(filepath.Join(after.Run.RunDir, "run.json")); err != nil { + t.Fatal(err) + } + }) + } +} diff --git a/internal/cli/cleanup.go b/internal/cli/cleanup.go index e7c3f4b..07fa6c6 100644 --- a/internal/cli/cleanup.go +++ b/internal/cli/cleanup.go @@ -1323,6 +1323,9 @@ func cleanupStaleProjectNameCollisions(run loadedMigrationRun, names []string) [ collisions := []string{} seen := map[string]struct{}{} for _, app := range run.Prepare.Apps { + if isPlatformRunApp(app.Role) { + continue + } if app.TargetResources == nil || app.TargetResources.Dokploy == nil { continue } diff --git a/internal/cli/cleanup_test.go b/internal/cli/cleanup_test.go index b06b71e..0b45b58 100644 --- a/internal/cli/cleanup_test.go +++ b/internal/cli/cleanup_test.go @@ -1198,6 +1198,7 @@ func TestCleanupStaleProjectNameCollisions(t *testing.T) { run := loadedMigrationRun{Prepare: preparer.Result{Apps: []preparer.AppPlan{ {Name: "api", TargetResources: &preparer.TargetResources{Dokploy: &preparer.DokployResources{Project: preparer.DokployProject{Name: "proxy"}}}}, {Name: "worker", TargetResources: &preparer.TargetResources{Dokploy: &preparer.DokployResources{Project: preparer.DokployProject{Name: "app"}}}}, + {Name: "internal-router", Role: "platform", TargetResources: &preparer.TargetResources{Dokploy: &preparer.DokployResources{Project: preparer.DokployProject{Name: "coolify-proxy"}}}}, }}} collisions := cleanupStaleProjectNameCollisions(run, []string{"coolify-proxy", "proxy", "source"}) if len(collisions) != 1 || collisions[0] != "proxy" { diff --git a/internal/cli/cockpit.go b/internal/cli/cockpit.go index 826b79e..9dabc60 100644 --- a/internal/cli/cockpit.go +++ b/internal/cli/cockpit.go @@ -92,7 +92,7 @@ func writeAppFirstCockpitSummary(w io.Writer, run loadedMigrationRun, summary mi } if fix := issue.FixCommand(app.Name); fix != "" { fmt.Fprintf(w, " %s\n", st.fix(fix)) - } else if next := issue.NextStep(); next != "" { + } else if next := issue.NextStepForRun(run); next != "" { fmt.Fprintf(w, " %s\n", st.muted("next: "+next)) } } @@ -472,15 +472,28 @@ func writeAuthorityRecoveryGuidance(w io.Writer, st *styler, run loadedMigration fmt.Fprintln(w, st.muted(prefix+" Inspect and preserve both sides, establish authority manually, and start a fresh migration run.")) return } - fmt.Fprintln(w, st.muted(prefix+" After manually fencing the other side and verifying the chosen writer and traffic authority, finish this owner-bound run with one of:")) + incomplete, incompleteErr := incompleteStagingTransferApps(run) + if incompleteErr == nil && len(incomplete) > 0 { + fmt.Fprintln(w, st.muted(prefix+" The staged state transfer for "+strings.Join(incomplete, ", ")+" did not finish, so only source recovery is available. After restoring the source writers and traffic, finish this owner-bound run with:")) + } else { + fmt.Fprintln(w, st.muted(prefix+" After manually fencing the other side and verifying the chosen writer and traffic authority, finish this owner-bound run with one of:")) + } + if len(appliedStagingTransferApps(run.Applied)) > 0 || len(incomplete) > 0 { + fmt.Fprintln(w, st.muted(" Before source recovery, remove (not just stop) every target container attached to the transferred bort staging volumes. Do not remove the bort-pin-* containers.")) + } fmt.Fprintf(w, "%s\n", st.muted(" source: `"+authorityRecoveryCommand(run, dokployTrafficSource)+"`")) - fmt.Fprintf(w, "%s\n", st.muted(" target: `"+authorityRecoveryCommand(run, dokployTrafficTarget)+"`")) + if incompleteErr != nil || len(incomplete) == 0 { + fmt.Fprintf(w, "%s\n", st.muted(" target: `"+authorityRecoveryCommand(run, dokployTrafficTarget)+"`")) + } } func writeManualRollbackRecoveryCommands(w io.Writer, st *styler, run loadedMigrationRun) { if run.Run.ResolvedAuthority != "" || !authorityRecoveryAvailable(run) { return } + if len(appliedStagingTransferApps(run.Applied)) > 0 { + fmt.Fprintln(w, st.muted(" Before source recovery, remove (not just stop) every target container attached to the transferred bort staging volumes; the volumes and their data remain.")) + } fmt.Fprintln(w, st.muted(" After manually restoring and verifying source authority, record it with:")) fmt.Fprintf(w, "%s\n", st.muted(" source: `"+authorityRecoveryCommand(run, dokployTrafficSource)+"`")) } @@ -521,7 +534,7 @@ func appliedFooter(applied runApplied) string { errs++ } } - parts := []string{fmt.Sprintf("Applied: %d step(s) recorded", len(applied.Steps))} + parts := []string{fmt.Sprintf("Applied: %s recorded", pluralize(len(applied.Steps), "step", "steps"))} if ok > 0 { parts = append(parts, fmt.Sprintf("%d ok", ok)) } diff --git a/internal/cli/cockpit_test.go b/internal/cli/cockpit_test.go index 645c79c..34307b2 100644 --- a/internal/cli/cockpit_test.go +++ b/internal/cli/cockpit_test.go @@ -239,7 +239,14 @@ func TestAppliedFooterCountsOkAndError(t *testing.T) { {Index: 2, Status: "skipped"}, }} got := appliedFooter(applied) - if got != "Applied: 3 step(s) recorded · 2 ok · 1 failed" { + if got != "Applied: 3 steps recorded · 2 ok · 1 failed" { + t.Fatalf("unexpected footer: %q", got) + } +} + +func TestAppliedFooterUsesSingularStep(t *testing.T) { + applied := runApplied{Steps: []appliedStep{{Index: 0, Status: "ok"}}} + if got := appliedFooter(applied); got != "Applied: 1 step recorded · 1 ok" { t.Fatalf("unexpected footer: %q", got) } } @@ -1129,6 +1136,42 @@ func TestCockpitBlocksPlanLiveApplyWouldRefuse(t *testing.T) { } } +func TestMissingStagingPinStatusRequiresAuthorityRecovery(t *testing.T) { + resetDokployTrafficOwner(t) + now := time.Now().UTC() + meta := persistRunFixture(t, migrationRun{ + Name: "missing-pin", + RunDir: filepath.Join(t.TempDir(), "missing-pin"), + Target: "dokploy", + CreatedAt: now, + BundleDigest: "missing-pin-digest", + }) + run := loadedMigrationRun{Run: meta, Applied: runApplied{ + APIVersion: appliedAPIVersion, + RecoveryProtocol: appliedRecoveryProtocol, + PlanVersion: appliedPlanCurrent, + Steps: []appliedStep{{ + Index: 2, + Kind: string(dokploy.StepRestoreDataStore), + App: "api", + Ref: "postgres", + Status: string(dokploy.StepStatusError), + Error: "required staging volume pin is missing after state transfer", + AuthorityRecoveryRequired: true, + }}, + }} + if err := claimDokployHostOwnership(run.Run, "http://127.0.0.1:3030", "cred-1"); err != nil { + t.Fatal(err) + } + if phase := migrationRunPhase(run); phase != "authority-ambiguous" { + t.Fatalf("missing-pin phase = %q, want authority-ambiguous", phase) + } + next := nextSafeStep(run, nil) + if !strings.Contains(next.Action, "recover-authority --authority source") || !strings.Contains(next.Action, "recover-authority --authority target") || strings.Contains(next.Action, "migrate --live") || strings.Contains(next.Action, "resume") { + t.Fatalf("missing-pin status offered an impossible retry: %#v", next) + } +} + func TestStagedTransferRefusalGatesPlannedPostgresDataDirOnApplyHistory(t *testing.T) { bundleDir := t.TempDir() appDir := filepath.Join(bundleDir, "api") diff --git a/internal/cli/commit.go b/internal/cli/commit.go index a4d43b4..14a9202 100644 --- a/internal/cli/commit.go +++ b/internal/cli/commit.go @@ -212,6 +212,9 @@ func applyCommitFromArgs(ctx context.Context, runRef string, stderr io.Writer) e return fmt.Errorf("reconfirm completed commit metadata durability: %w", err) } if requiresHostOwner { + if err := releaseRecoveredAuthorityStagingVolumePins(ctx, run, true); err != nil { + return fmt.Errorf("remove target-authority staging-volume pins after completed source retirement; host ownership remains held so this commit can be retried: %w", err) + } if err := releaseDokployTargetOwner(run.Run); err != nil { return fmt.Errorf("release completed commit host ownership: %w", err) } @@ -221,6 +224,11 @@ func applyCommitFromArgs(ctx context.Context, runRef string, stderr io.Writer) e if err := verifyLocalSourceRun(ctx, run); err != nil { return fmt.Errorf("commit refused on an unverified Docker source: %w", err) } + if requiresHostOwner { + if err := validateRecoveredTargetAuthority(ctx, run); err != nil { + return fmt.Errorf("commit refused before retiring the source because the transferred staging volumes do not match the target: %w", err) + } + } if requiresHostOwner { if err := ensureDokployTrafficTargetOwner(ctx, run, nil, false); err != nil { return fmt.Errorf("commit refused: %w", err) @@ -229,11 +237,12 @@ func applyCommitFromArgs(ctx context.Context, runRef string, stderr io.Writer) e client := &dokploy.Client{} plan.RunName = run.Run.Name plan.RunDir = run.Run.RunDir + plan.RecoveryCommand = runScopedCommand(run, "status") plan.ApprovedPrepareDecisions = approvedPrepareDecisions(run) if err := markRunCommitStartedLocked(run.Run); err != nil { return fmt.Errorf("record source retirement start: %w", err) } - fmt.Fprintf(stderr, "commit apply: run %s; planned %d step(s) to retire source\n", run.Run.Name, len(plan.Steps)) + fmt.Fprintf(stderr, "commit apply: run %s; planned %s to retire source\n", run.Run.Name, pluralize(len(plan.Steps), "step", "steps")) if err := client.Apply(ctx, plan); err != nil { return err } @@ -241,6 +250,9 @@ func applyCommitFromArgs(ctx context.Context, runRef string, stderr io.Writer) e return fmt.Errorf("source retirement completed, but migration commit metadata could not be recorded: %w", err) } if requiresHostOwner { + if err := releaseRecoveredAuthorityStagingVolumePins(ctx, run, true); err != nil { + return fmt.Errorf("commit was recorded, but target-authority staging-volume validation or pin removal failed; host ownership remains held so this commit can be retried: %w", err) + } if err := releaseDokployTargetOwner(run.Run); err != nil { return fmt.Errorf("commit was recorded, but its Dokploy host ownership could not be released: %w", err) } @@ -262,14 +274,21 @@ func coolifySourceRetirementRequired(run loadedMigrationRun) bool { } func manualCoolifySourceRetirementAction(run loadedMigrationRun) string { - targets := "every reviewed source app" - for _, step := range dokploy.PlanForCommit(run.Prepare, run.Cutover).Steps { - if step.Kind == dokploy.StepStopCoolifyProxy { - targets += " and the source proxy" - break + refs := dokploy.SourceRetirementContainers(dokploy.PlanForCommit(run.Prepare, run.Cutover)) + removal := "remove every reviewed source app container (and the source proxy container when routes moved) with Docker" + if len(refs) > 0 { + quoted := make([]string, 0, len(refs)) + for _, ref := range refs { + quoted = append(quoted, shellQuote(ref)) } + removal = fmt.Sprintf("remove the reviewed source containers with `%s`", dockerCommand("rm -f "+strings.Join(quoted, " "))) + } + fence := fmt.Sprintf("run `%s && %s` even if it is already stopped (this pauses Coolify for every app on the host)", dockerCommand("update --restart=no coolify"), dockerCommand("stop coolify")) + keepStopped := "leave Coolify stopped afterwards if you can, because starting it again can recreate its proxy and redeploy the removed apps" + if appliedHasHistory(run.Applied) && dokploy.RequiresCoolifyDeploymentFence(livePlanForApplied(run, run.Applied)) { + return fmt.Sprintf("retain the restart policy you recorded before fencing Coolify for live apply (its policy now reads `no`; do not record it again), %s, %s, verify they stay removed, then run `%s`; %s; if other apps need it, restore the restart policy you recorded before fencing, start it, and immediately delete the migrated apps in Coolify", fence, removal, authorityRecoverySourceRetiredCommand(run), keepStopped) } - return fmt.Sprintf("manually disable future Coolify deployments for the reviewed apps, retire %s, verify they remain retired, then run `%s`", targets, authorityRecoverySourceRetiredCommand(run)) + return fmt.Sprintf("record the Coolify control plane's restart policy with `%s`, then %s, %s, verify they stay removed, then run `%s`; %s; if other apps need it, restore the recorded restart policy, start it, and immediately delete the migrated apps in Coolify", dockerCommand("inspect --format '{{.HostConfig.RestartPolicy.Name}}' coolify"), fence, removal, authorityRecoverySourceRetiredCommand(run), keepStopped) } func finishStartedAcceptanceAction(run loadedMigrationRun) string { diff --git a/internal/cli/commit_test.go b/internal/cli/commit_test.go index 34b8ee9..d87f2f6 100644 --- a/internal/cli/commit_test.go +++ b/internal/cli/commit_test.go @@ -17,6 +17,7 @@ import ( "github.com/aikins01/bort/internal/gateway" "github.com/aikins01/bort/internal/manifest" "github.com/aikins01/bort/internal/preparer" + syncplan "github.com/aikins01/bort/internal/sync" "github.com/aikins01/bort/internal/target/dokploy" ) @@ -474,6 +475,16 @@ func TestCommitRetryRepublishesCompletedMetadataBeforeOwnerRelease(t *testing.T) if err := markRunHostOwnerReleaseStartedLocked(run.Run); err != nil { t.Fatal(err) } + previous := releaseAuthorityStagingVolumePins + cleanupCalls := 0 + releaseAuthorityStagingVolumePins = func(_ context.Context, _ loadedMigrationRun, plan dokploy.Plan, targetAuthority bool) error { + cleanupCalls++ + if plan.RunName != run.Run.Name || plan.RunDir != run.Run.RunDir || plan.RunID == "" || !targetAuthority { + t.Fatalf("commit pin cleanup received incomplete target identity: plan=%#v target=%t", plan, targetAuthority) + } + return nil + } + t.Cleanup(func() { releaseAuthorityStagingVolumePins = previous }) path := filepath.Join(run.Run.RunDir, "run.json") before, err := os.Stat(path) if err != nil { @@ -493,6 +504,42 @@ func TestCommitRetryRepublishesCompletedMetadataBeforeOwnerRelease(t *testing.T) if err != nil || !found || owner.Authority != dokployTrafficReleased { t.Fatalf("completed commit owner = %#v, found=%t err=%v", owner, found, err) } + if cleanupCalls != 1 { + t.Fatalf("completed commit pin cleanup calls = %d, want 1", cleanupCalls) + } +} + +func TestCommitRetryKeepsOwnerWhenPinCleanupFails(t *testing.T) { + run := writeAmbiguousAuthorityRun(t, "durable-commit-pin-cleanup") + if err := markRunAuthorityResolvedLocked(run.Run, dokployTrafficTarget); err != nil { + t.Fatal(err) + } + if err := markDokployTrafficTarget(run.Run, "http://127.0.0.1:3030"); err != nil { + t.Fatal(err) + } + if err := markRunLiveAppliedLocked(run.Run); err != nil { + t.Fatal(err) + } + if err := markRunCommitStartedLocked(run.Run); err != nil { + t.Fatal(err) + } + if err := markRunCommittedLocked(run.Run); err != nil { + t.Fatal(err) + } + previous := releaseAuthorityStagingVolumePins + releaseAuthorityStagingVolumePins = func(context.Context, loadedMigrationRun, dokploy.Plan, bool) error { + return errors.New("pin cleanup failed") + } + t.Cleanup(func() { releaseAuthorityStagingVolumePins = previous }) + + err := applyCommitFromArgs(context.Background(), run.Run.Name, io.Discard) + if err == nil || !strings.Contains(err.Error(), "host ownership remains held") || !strings.Contains(err.Error(), "pin cleanup failed") { + t.Fatalf("expected pin cleanup failure, got %v", err) + } + owner, found, err := readDokployTrafficOwner() + if err != nil || !found || owner.Authority != dokployTrafficTarget { + t.Fatalf("pin cleanup failure released owner: owner=%#v found=%t err=%v", owner, found, err) + } } func TestCommitRetryKeepsOwnerWhenCompletedMetadataRepublishFails(t *testing.T) { @@ -748,15 +795,97 @@ func forbidLocalSourceVerification(t *testing.T) { } func TestManualCoolifySourceRetirementActionScopesProxyToRoutedCutovers(t *testing.T) { + app := preparer.AppPlan{Name: "api"} + app.Resources.SourceServices = []preparer.SourceServiceRef{ + {ServiceName: "web", ContainerID: "web-id", ContainerName: "web-x1y2"}, + {ServiceName: "db", ContainerID: "db-id"}, + } run := loadedMigrationRun{ Run: migrationRun{Name: "coolify-run"}, - Prepare: preparer.Result{Source: "coolify-local", Apps: []preparer.AppPlan{{Name: "api"}}}, + Prepare: preparer.Result{Source: "coolify-local", Apps: []preparer.AppPlan{app}}, } - if action := manualCoolifySourceRetirementAction(run); strings.Contains(action, "source proxy") || !strings.Contains(action, "--source-retired") { - t.Fatalf("route-free retirement hint touched the shared proxy: %q", action) + if action := manualCoolifySourceRetirementAction(run); !strings.Contains(action, "`"+dockerCommand("rm -f web-id db-id")+"`") || !strings.Contains(action, "--source-retired") { + t.Fatalf("route-free retirement hint did not name exactly the reviewed containers: %q", action) } run.Cutover = gateway.Result{Apps: []gateway.AppPlan{{Name: "api", Routes: []gateway.Route{{Host: "api.example.com"}}}}} - if action := manualCoolifySourceRetirementAction(run); !strings.Contains(action, "and the source proxy") { + if action := manualCoolifySourceRetirementAction(run); !strings.Contains(action, "`"+dockerCommand("rm -f web-id db-id coolify-proxy")+"`") { t.Fatalf("routed retirement hint omitted the proxy handoff: %q", action) } + applied := time.Now().UTC() + run.Run.ResolvedAuthority = dokployTrafficTarget + run.Run.AuthorityResolvedAt = &applied + run.Run.LiveAppliedAt = &applied + run.Sync = syncplan.Result{Apps: []syncplan.AppPlan{{ + Name: "api", + Steps: []syncplan.Step{{ResourceType: "volume", ResourceRef: "data", Strategy: syncplan.StrategyVolumeSync}}, + }}} + run.Applied.Steps = []appliedStep{{Index: 0, Kind: string(dokploy.StepSyncVolume), App: "api", Ref: "data", Status: string(dokploy.StepStatusOK), UpdatedAt: applied}} + action := manualCoolifySourceRetirementAction(run) + if !strings.Contains(action, "do not record it again") || + !strings.Contains(action, "restore the restart policy you recorded before fencing") || + strings.Contains(action, "record the Coolify control plane's restart policy with") { + t.Fatalf("retirement hint after a live-apply fence did not keep the pre-fence policy record: %q", action) + } + run.Run.ResolvedAuthority = "" + run.Run.AuthorityResolvedAt = nil + run.Run.LiveAppliedAt = nil + run.Applied.Steps[0].Status = string(dokploy.StepStatusError) + action = manualCoolifySourceRetirementAction(run) + if !strings.Contains(action, "do not record it again") || + strings.Contains(action, "record the Coolify control plane's restart policy with") { + t.Fatalf("retirement hint after an interrupted fenced live apply re-recorded the fence policy: %q", action) + } +} + +func TestCommitValidatesStagingVolumesBeforeRetiringSource(t *testing.T) { + resetDokployTrafficOwner(t) + t.Chdir(t.TempDir()) + writeTestBundle(t, "bundle", manifest.Manifest{ + Source: manifest.Source{Platform: "docker", DockerEngineID: "engine-reviewed"}, + Apps: []manifest.App{{ + Name: "api", + Services: []manifest.Service{{ID: "source-id", Name: "web", Image: "example/api:latest"}}, + Routes: []manifest.Route{{Host: "api.example.com", ServiceName: "web", Port: "3000"}}, + }}, + }) + runCommand(t, runMigrate, []string{"--bundle", "bundle", "--run", "docker-commit", "--observation-window", "0", "--rollback-window", "0"}) + markRunLocallyScanned(t, "docker-commit", "docker") + run, err := loadMigrationRun("docker-commit") + if err != nil { + t.Fatal(err) + } + applied := newRunApplied(run.Run) + applied.Steps = []appliedStep{{Index: 0, Kind: string(dokploy.StepPushImage), App: "api", Ref: "example/api:latest", Status: string(dokploy.StepStatusError), UpdatedAt: time.Now().UTC(), Error: "outcome unknown"}} + if err := writeRunApplied(runArtifactPath(run.Run.RunDir, run.Run.Artifacts.Applied), applied); err != nil { + t.Fatal(err) + } + if err := claimDokployHostOwnership(run.Run, "http://127.0.0.1:3030", dokployCredentialID("test-token")); err != nil { + t.Fatal(err) + } + if err := markRunAuthorityResolvedLocked(run.Run, dokployTrafficTarget); err != nil { + t.Fatal(err) + } + if err := markDokployTrafficTarget(run.Run, "http://127.0.0.1:3030"); err != nil { + t.Fatal(err) + } + if err := markRunLiveAppliedLocked(run.Run); err != nil { + t.Fatal(err) + } + previous := validateAuthorityStagingVolumePins + validateAuthorityStagingVolumePins = func(context.Context, loadedMigrationRun, dokploy.Plan, bool) error { + return errors.New("migrated volume bort-v has no target container") + } + t.Cleanup(func() { validateAuthorityStagingVolumePins = previous }) + + err = applyCommitFromArgs(context.Background(), "docker-commit", io.Discard) + if err == nil || !strings.Contains(err.Error(), "before retiring the source") { + t.Fatalf("commit did not validate staging volumes before retiring the source: %v", err) + } + after, err := loadMigrationRun("docker-commit") + if err != nil { + t.Fatal(err) + } + if after.Run.CommitStartedAt != nil || after.Run.CommittedAt != nil { + t.Fatalf("refused commit recorded source retirement: %#v", after.Run) + } } diff --git a/internal/cli/issues.go b/internal/cli/issues.go index 55916b9..5f92522 100644 --- a/internal/cli/issues.go +++ b/internal/cli/issues.go @@ -95,8 +95,28 @@ func (i appIssue) FixCommand(app string) string { } func (i appIssue) NextStep() string { + return i.nextStep(bortCommand("status"), bortCommand("migrate --run "), "the run's bundle") +} + +func (i appIssue) NextStepForRun(run loadedMigrationRun) string { + return i.nextStep(runScopedCommand(run, "status"), runScopedCommand(run, "migrate"), "`"+replanBundleDir(run.Run)+"`") +} + +func replanBundleDir(run migrationRun) string { + if strings.TrimSpace(run.Source) == "" && strings.TrimSpace(run.SourceBundleDir) != "" { + return run.SourceBundleDir + } + return run.BundleDir +} + +func (i appIssue) nextStep(statusCommand, migrateCommand, bundle string) string { switch i.Kind { case issueKindRoute: + for _, item := range i.Items { + if item.Code == preparer.GateDomainServiceMissing || item.Code == preparer.GateDomainServiceNotInCompose { + return fmt.Sprintf("if live execution has started, run `%s`, complete the required recovery, then correct the source or input bundle and create a new run; otherwise correct the route-to-Compose-service mapping in the app's topology.json in %s, keep its routes.json consistent, and re-plan with `%s`", statusCommand, bundle, migrateCommand) + } + } return "confirm the route host and service in Dokploy before live apply" case issueKindLink: return "review the detected database/storage settings only if you expect this app to switch to a different service" diff --git a/internal/cli/issues_test.go b/internal/cli/issues_test.go index bd46ea3..93c432a 100644 --- a/internal/cli/issues_test.go +++ b/internal/cli/issues_test.go @@ -1,6 +1,7 @@ package cli import ( + "path/filepath" "strings" "testing" @@ -87,6 +88,45 @@ func TestFixCommandOnlyReturnsCommandsThatPersistState(t *testing.T) { } } +func TestRouteIssueNextStepDistinguishesBlockedMappings(t *testing.T) { + wantBlocked := "if live execution has started, run `" + bortCommand("status") + "`, complete the required recovery, then correct the source or input bundle and create a new run; otherwise correct the route-to-Compose-service mapping in the app's topology.json in the run's bundle, keep its routes.json consistent, and re-plan with `" + bortCommand("migrate --run ") + "`" + for _, blocked := range []appIssue{ + {Kind: issueKindRoute, Items: []runDecisionItem{{Code: preparer.GateRoutesNone}, {Code: preparer.GateDomainServiceMissing}}}, + {Kind: issueKindRoute, Items: []runDecisionItem{{Code: preparer.GateDomainServiceNotInCompose}}}, + } { + if got := blocked.NextStep(); got != wantBlocked { + t.Fatalf("unexpected blocked route guidance: %q", got) + } + } + review := appIssue{Kind: issueKindRoute, Items: []runDecisionItem{{Code: preparer.GateRoutesNone}}} + if got := review.NextStep(); got != "confirm the route host and service in Dokploy before live apply" { + t.Fatalf("unexpected review route guidance: %q", got) + } + run := loadedMigrationRun{Run: migrationRun{Name: "demo", RunDir: filepath.Join(t.TempDir(), "demo")}} + blocked := appIssue{Kind: issueKindRoute, Items: []runDecisionItem{{Code: preparer.GateDomainServiceMissing}}} + got := blocked.NextStepForRun(run) + if !strings.Contains(got, "`"+runScopedCommand(run, "status")+"`") || !strings.Contains(got, "`"+runScopedCommand(run, "migrate")+"`") { + t.Fatalf("run-specific route guidance lost its run context: %q", got) + } + + for _, tc := range []struct { + name string + run migrationRun + want string + }{ + {name: "source-created run edits its snapshot", run: migrationRun{Source: "coolify-local", BundleDir: "/runs/demo/bundle-2", SourceBundleDir: "/stale/input"}, want: "/runs/demo/bundle-2"}, + {name: "input-bundle run edits the original bundle", run: migrationRun{BundleDir: "/runs/demo/bundle-2", SourceBundleDir: "/work/bort-bundle"}, want: "/work/bort-bundle"}, + } { + t.Run(tc.name, func(t *testing.T) { + tc.run.Name, tc.run.RunDir = run.Run.Name, run.Run.RunDir + got := blocked.NextStepForRun(loadedMigrationRun{Run: tc.run}) + if !strings.Contains(got, "topology.json in `"+tc.want+"`") { + t.Fatalf("route guidance should name re-plan bundle %s: %q", tc.want, got) + } + }) + } +} + func TestFixCommandShellQuotesUntrustedNames(t *testing.T) { envIssue := appIssue{Kind: issueKindEnv} if got := envIssue.FixCommand("My App"); got != "bort env 'My App' KEY=value" { @@ -111,4 +151,8 @@ func TestFixAndLiveCommandsPreserveSudoLifecycle(t *testing.T) { if got := liveApplyCommand(loadedMigrationRun{Run: migrationRun{Name: "demo"}}); got != "sudo bort migrate --live --run demo" { t.Fatalf("expected sudo live command, got %q", got) } + route := appIssue{Kind: issueKindRoute, Items: []runDecisionItem{{Code: preparer.GateDomainServiceMissing}}} + if got := route.NextStep(); !strings.Contains(got, "`sudo bort status`") || !strings.Contains(got, "`sudo bort migrate --run `") { + t.Fatalf("expected sudo route recovery commands, got %q", got) + } } diff --git a/internal/cli/main_test.go b/internal/cli/main_test.go index 2545f1e..c3ff5e7 100644 --- a/internal/cli/main_test.go +++ b/internal/cli/main_test.go @@ -29,6 +29,8 @@ func TestMain(m *testing.M) { verifyLocalSourceEngine = func(context.Context, loadedMigrationRun) error { return nil } verifyLocalDokployClient = func(context.Context, *dokploy.Client) error { return nil } verifyLocalDokployCleanupHost = func(context.Context, *dokploy.Client) error { return nil } + releaseAuthorityStagingVolumePins = func(context.Context, loadedMigrationRun, dokploy.Plan, bool) error { return nil } + validateAuthorityStagingVolumePins = func(context.Context, loadedMigrationRun, dokploy.Plan, bool) error { return nil } code := m.Run() _ = os.RemoveAll(dir) os.Exit(code) diff --git a/internal/cli/rollback.go b/internal/cli/rollback.go index 9376280..cf51541 100644 --- a/internal/cli/rollback.go +++ b/internal/cli/rollback.go @@ -194,6 +194,19 @@ func applyRollbackFromArgs(ctx context.Context, runRef, confirm string, stderr i return err } if run.Run.RolledBackAt != nil { + if run.Run.ResolvedAuthority == dokployTrafficSource && run.Run.AuthorityFinalizedAt == nil { + if err := releaseRecoveredAuthorityStagingVolumePins(ctx, run, false); err != nil { + return fmt.Errorf("complete rolled-back source-authority staging-volume validation: %w", err) + } + if err := releaseDokployTrafficOwner(run.Run); err != nil { + return fmt.Errorf("release rolled-back source-authority host ownership: %w", err) + } + if err := markRunAuthorityFinalizedLocked(run.Run); err != nil { + return fmt.Errorf("record rolled-back source-authority finalization: %w", err) + } + fmt.Fprintf(stderr, "rollback recovery complete for run %s: staging-volume attachments validated and host ownership released\n", run.Run.Name) + return nil + } plan, planErr := dokploy.PlanForRollback(run.Prepare, run.Sync, run.Cutover) if planErr != nil || planRequiresDokployHostOwner(run, plan) { if err := releaseDokployTrafficOwner(run.Run); err != nil { @@ -255,6 +268,7 @@ func applyRollbackFromArgs(ctx context.Context, runRef, confirm string, stderr i client := &dokploy.Client{} plan.RunName = run.Run.Name plan.RunDir = run.Run.RunDir + plan.RecoveryCommand = runScopedCommand(run, "status") plan.ApprovedPrepareDecisions = approvedPrepareDecisions(run) onProgress := func(p dokploy.StepProgress) { target := p.Step.App @@ -271,10 +285,13 @@ func applyRollbackFromArgs(ctx context.Context, runRef, confirm string, stderr i if err := markRunRollbackStartedLocked(run.Run); err != nil { return fmt.Errorf("record rollback start: %w", err) } - fmt.Fprintf(stderr, "rollback live: run %s; planned %d step(s) to return traffic to the source\n", run.Run.Name, len(plan.Steps)) + fmt.Fprintf(stderr, "rollback live: run %s; planned %s to return traffic to the source\n", run.Run.Name, pluralize(len(plan.Steps), "step", "steps")) if err := client.Apply(ctx, plan); err != nil { return err } + if err := client.FinalizeSourceResumes(plan); err != nil { + return fmt.Errorf("rollback completed, but its source resume state could not be finalized: %w", err) + } if planChangesDokployTraffic(plan) { if err := client.ReconcileSourceTrafficAuthority(ctx); err != nil { return fmt.Errorf("verify source traffic authority: %w", err) diff --git a/internal/cli/rollback_test.go b/internal/cli/rollback_test.go index 11c0b5f..cf88055 100644 --- a/internal/cli/rollback_test.go +++ b/internal/cli/rollback_test.go @@ -868,10 +868,11 @@ fi t.Fatalf("expected rollback progress header, got:\n%s", stderr.String()) } for _, want := range []string{ - "rollback [1/4] verify_source_health api: ok", - "rollback [2/4] stop_dokploy_proxy dokploy-traefik: ok", - "rollback [3/4] start_coolify_proxy coolify-proxy: ok", - "rollback [4/4] observe_rollback api: ok", + "rollback [1/5] resume_source api: ok", + "rollback [2/5] verify_source_health api: ok", + "rollback [3/5] stop_dokploy_proxy dokploy-traefik: ok", + "rollback [4/5] start_coolify_proxy coolify-proxy: ok", + "rollback [5/5] observe_rollback api: ok", } { if !strings.Contains(stderr.String(), want) { t.Fatalf("expected rollback progress %q, got:\n%s", want, stderr.String()) diff --git a/internal/cli/run.go b/internal/cli/run.go index 33c4b9a..a594bce 100644 --- a/internal/cli/run.go +++ b/internal/cli/run.go @@ -615,7 +615,7 @@ func recoverInterruptedStatefulSources(ctx context.Context, run loadedMigrationR onProgress(item) } } - plan := dokploy.Plan{Steps: steps, Prepare: prepare, RunDir: run.Run.RunDir, BeforeStep: &beforeStep, OnProgress: &progress} + plan := dokploy.Plan{Steps: steps, Prepare: prepare, RunDir: run.Run.RunDir, RecoveryCommand: runScopedCommand(run, "status"), BeforeStep: &beforeStep, OnProgress: &progress} client := &dokploy.Client{} if err := client.AdoptHistoricalSourcePause(ctx, plan, appNames); err != nil { return errors.Join(refusal, fmt.Errorf("record historical source pause ownership: %w", err)) @@ -845,6 +845,7 @@ func applyLiveMigrationLocked(ctx context.Context, run loadedMigrationRun, stder } plan.RunName = run.Run.Name plan.RunDir = run.Run.RunDir + plan.RecoveryCommand = runScopedCommand(run, "status") if plan.RunID, err = dokployTrafficRunID(run.Run); err != nil { return err } @@ -904,12 +905,12 @@ func applyLiveMigrationLocked(ctx context.Context, run loadedMigrationRun, stder } } plan.OnProgress = &fn - fmt.Fprintf(stderr, "live mode: %s reachable; planned %d dokploy step(s)\n", client.BaseURL, len(plan.Steps)) + fmt.Fprintf(stderr, "live mode: %s reachable; planned %s\n", client.BaseURL, pluralize(len(plan.Steps), "Dokploy step", "Dokploy steps")) if resumeFrom > 0 && resumeFrom < len(plan.Steps) { - fmt.Fprintf(stderr, "live mode: resuming after %d completed step(s)\n", resumeFrom) + fmt.Fprintf(stderr, "live mode: resuming after %s completed\n", pluralize(resumeFrom, "step", "steps")) } if resumeFrom == len(plan.Steps) && len(plan.Steps) > 0 { - fmt.Fprintln(stderr, "live mode: all planned dokploy steps are already recorded as complete") + fmt.Fprintln(stderr, "live mode: all planned Dokploy steps are already recorded as complete") } if err := client.Apply(ctx, plan); err != nil { return err @@ -1337,7 +1338,7 @@ func attachLiveMigration(ctx context.Context, run loadedMigrationRun, appliedPat entries := attachProgressEntries(plan.Steps, initialApplied, time.Time{}) emitAttachProgress(onProgress, entries) if onProgress == nil { - fmt.Fprintf(stderr, "live mode: apply already complete (%d/%d step(s) recorded)\n", len(plan.Steps), len(plan.Steps)) + fmt.Fprintf(stderr, "live mode: apply already complete (%s recorded)\n", pluralize(len(plan.Steps), "step", "steps")) } return nil } @@ -1355,7 +1356,7 @@ func attachLiveMigration(ctx context.Context, run loadedMigrationRun, appliedPat writeAttachTextProgress(stderr, latest.progress, latest.progress.Total, "already recorded") } } else if onProgress == nil { - fmt.Fprintf(stderr, "live mode: 0/%d step(s) already recorded\n", len(plan.Steps)) + fmt.Fprintf(stderr, "live mode: 0 of %s already recorded\n", pluralize(len(plan.Steps), "step", "steps")) } } @@ -1480,7 +1481,7 @@ func latestAttachProgressEntry(entries []attachProgressEntry) (attachProgressEnt } func writeAttachTextProgress(stderr io.Writer, progress dokploy.StepProgress, total int, label string) { - fmt.Fprintf(stderr, "live mode: %d/%d step(s) %s; latest %s %s/%s %s\n", progress.Index+1, total, label, progress.Status, progress.Step.App, progress.Step.Ref, progress.Step.Kind) + fmt.Fprintf(stderr, "live mode: %d of %s %s; latest %s %s/%s %s\n", progress.Index+1, pluralize(total, "step", "steps"), label, progress.Status, progress.Step.App, progress.Step.Ref, progress.Step.Kind) } func attachExitResult(run loadedMigrationRun, steps []dokploy.Step, applied runApplied, attachedAt time.Time) error { @@ -1489,12 +1490,12 @@ func attachExitResult(run loadedMigrationRun, steps []dokploy.Step, applied runA if applied.SucceededAt != nil { return nil } - return fmt.Errorf("live migration recorded all %d step(s) but no successful live-apply outcome; rerun `%s` to continue", len(steps), liveApplyCommand(run)) + return fmt.Errorf("live migration recorded %s but no successful live-apply outcome; rerun `%s` to continue", pluralize(len(steps), "step", "steps"), liveApplyCommand(run)) } if failed, ok := latestAttachFailure(steps, applied, attachedAt); ok { - return fmt.Errorf("live migration exited after %d/%d recorded step(s); latest failure: %s %s/%s: %s", completed, len(steps), failed.Kind, failed.App, failed.Ref, failed.Error) + return fmt.Errorf("live migration exited after recording %d of %s; latest failure: %s %s/%s: %s", completed, pluralize(len(steps), "step", "steps"), failed.Kind, failed.App, failed.Ref, failed.Error) } - return fmt.Errorf("live migration exited after %d/%d recorded step(s); rerun `%s` to continue", completed, len(steps), liveApplyCommand(run)) + return fmt.Errorf("live migration exited after recording %d of %s; rerun `%s` to continue", completed, pluralize(len(steps), "step", "steps"), liveApplyCommand(run)) } func latestAttachFailure(steps []dokploy.Step, applied runApplied, attachedAt time.Time) (appliedStep, bool) { @@ -2687,7 +2688,7 @@ func appliedRequiresNewRun(applied runApplied) bool { func authorityRecoveryNextStep(run loadedMigrationRun, reason string) runNextStep { return runNextStep{ - Action: fmt.Sprintf("manually fence the other side and verify authority, then run `%s` for source or `%s` for target", authorityRecoveryCommand(run, dokployTrafficSource), authorityRecoveryCommand(run, dokployTrafficTarget)), + Action: authorityRecoveryChoice(run), Reason: reason, } } @@ -3177,7 +3178,7 @@ func writeMigrationRunTextWithFooter(w io.Writer, heading string, summary migrat fmt.Fprintf(w, "Platform/internal apps excluded: %d\n", summary.PlatformApps) } fmt.Fprintf(w, "Routes: %d cutover, %d rollback, %d commit\n", summary.CutoverRoutes, summary.RollbackRoutes, summary.CommitRoutes) - fmt.Fprintf(w, "State sync: %d resource step(s), %d pause/decision step(s)\n", summary.StateSteps, summary.PauseSteps) + fmt.Fprintf(w, "State sync: %s, %s\n", pluralize(summary.StateSteps, "resource step", "resource steps"), pluralize(summary.PauseSteps, "pause/decision step", "pause/decision steps")) fmt.Fprintf(w, "Gates: %d blocked, %d need input, %d need decision\n", summary.GateCounts.Blocked, summary.GateCounts.NeedsInput, summary.GateCounts.NeedsDecision) fmt.Fprintf(w, "Decisions: %d open\n", len(summary.Decisions)) if summary.Progress != "" { diff --git a/internal/cli/run_test.go b/internal/cli/run_test.go index 121454d..81f4f90 100644 --- a/internal/cli/run_test.go +++ b/internal/cli/run_test.go @@ -1759,6 +1759,10 @@ func prepareCompletedPrefixProxyRun(t *testing.T) (loadedMigrationRun, string) { if err != nil { t.Fatal(err) } + pauseState := []byte(`{"apiVersion":"bort.source-pause/v1alpha1","apps":{"api":[{"id":"source-id","stopped":true}]}}` + "\n") + if err := os.WriteFile(filepath.Join(run.Run.RunDir, "source-pause.json"), pauseState, 0o600); err != nil { + t.Fatal(err) + } target := dokploy.TargetIdentity{ProjectID: "project-1", EnvironmentID: "environment-1", ComposeID: "compose-1", ComposeAppName: "stack-api"} server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodGet { @@ -1784,12 +1788,15 @@ func prepareCompletedPrefixProxyRun(t *testing.T) (loadedMigrationRun, string) { if [ "$1" = inspect ] && [ "$2" = --type ]; then running=false policy=no + id="$4-id" + name="$4" case "$4" in coolify-proxy) [ ! -f coolify-stopped ] && running=true ;; dokploy-traefik) [ -f dokploy-started ] && running=true ;; + source-id) id="source-id" name="web" ;; esac [ ! -f "$4-unless" ] || policy=unless-stopped - echo '[{"Id":"'"$4"'-id","Name":"/'"$4"'","State":{"Running":'"$running"',"Status":"running"},"HostConfig":{"RestartPolicy":{"Name":"'"$policy"'"}}}]' + echo '[{"Id":"'"$id"'","Name":"/'"$name"'","State":{"Running":'"$running"',"Status":"running"},"HostConfig":{"RestartPolicy":{"Name":"'"$policy"'"}}}]' elif [ "$1" = stop ]; then touch coolify-stopped elif [ "$1" = start ]; then @@ -1912,7 +1919,7 @@ func TestAttachProgressIncludesStandaloneProxyCleanup(t *testing.T) { } var output bytes.Buffer writeAttachTextProgress(&output, progress, progress.Total, "recorded") - if !strings.Contains(output.String(), "2/2 step(s) recorded") { + if !strings.Contains(output.String(), "2 of 2 steps recorded") { t.Fatalf("standalone cleanup rendered invalid progress: %q", output.String()) } failed, ok := latestAttachFailure(steps, applied, now) diff --git a/internal/cli/style.go b/internal/cli/style.go index c00b795..683d472 100644 --- a/internal/cli/style.go +++ b/internal/cli/style.go @@ -5,6 +5,7 @@ import ( "os" "strings" + "github.com/aikins01/bort/internal/dockercli" "github.com/charmbracelet/lipgloss" ) @@ -19,6 +20,14 @@ func bortCommand(args string) string { return command + " " + args } +func dockerCommand(args string) string { + command := "docker --host " + dockercli.LocalHost() + if strings.TrimSpace(os.Getenv("SUDO_UID")) != "" { + command = "sudo " + command + } + return command + " " + args +} + type styler struct { color bool diff --git a/internal/exporter/exporter.go b/internal/exporter/exporter.go index fea751d..9359b06 100644 --- a/internal/exporter/exporter.go +++ b/internal/exporter/exporter.go @@ -22,8 +22,11 @@ type Options struct { } const ( - privateDirMode os.FileMode = 0o700 - privateFileMode os.FileMode = 0o600 + privateDirMode os.FileMode = 0o700 + privateFileMode os.FileMode = 0o600 + ComposeSourceRaw = "raw" + ComposeSourceGenerated = "generated" + generatedComposeWarning = "generated compose from discovered container metadata" ) type Summary struct { @@ -42,10 +45,27 @@ type AppSummary struct { Role string `json:"role,omitempty"` ProjectGroup *ProjectGroup `json:"projectGroup,omitempty"` PrivateEnvValues bool `json:"privateEnvValues,omitempty"` + ComposeSource string `json:"composeSource,omitempty"` Routes []string `json:"routes,omitempty"` Warnings []string `json:"warnings,omitempty"` } +func (a AppSummary) EffectiveComposeSource() (string, error) { + switch a.ComposeSource { + case ComposeSourceRaw, ComposeSourceGenerated: + return a.ComposeSource, nil + case "": + for _, warning := range a.Warnings { + if warning == generatedComposeWarning { + return ComposeSourceGenerated, nil + } + } + return ComposeSourceRaw, nil + default: + return "", fmt.Errorf("unsupported compose source %q", a.ComposeSource) + } +} + type ProjectGroup struct { Name string `json:"name"` Environment string `json:"environment,omitempty"` @@ -87,7 +107,7 @@ func Export(m manifest.Manifest, opts Options) (Summary, error) { } topology := analyzer.TopologyForAppInManifest(m, app) - warnings, err := exportApp(appDir, app, topology, opts) + composeSource, warnings, err := exportApp(appDir, app, topology, opts) if err != nil { return Summary{}, err } @@ -99,6 +119,7 @@ func Export(m manifest.Manifest, opts Options) (Summary, error) { Role: migrationRole(app), ProjectGroup: projectGroups[appKey(app)], PrivateEnvValues: opts.IncludeEnvValues, + ComposeSource: composeSource, Routes: routeHosts(app.Routes), Warnings: warnings, }) @@ -111,9 +132,10 @@ func Export(m manifest.Manifest, opts Options) (Summary, error) { return summary, nil } -func exportApp(appDir string, app manifest.App, topology analyzer.Topology, opts Options) ([]string, error) { +func exportApp(appDir string, app manifest.App, topology analyzer.Topology, opts Options) (string, []string, error) { warnings := []string{} - compose, composeWarnings, serviceEnvFiles := composeForApp(app, opts.IncludeEnvValues) + compose, composeSource, composeWarnings, serviceEnvFiles := composeForApp(app, opts.IncludeEnvValues) + topology.Routes = routesForCompose(topology.Routes, topology.SourceServices, composeSource, compose) warnings = append(warnings, composeWarnings...) if names := analyzer.CoolifyServiceMagicEnvNames(app); len(names) > 0 { warnings = append(warnings, "preserved Coolify service magic env vars for review: "+strings.Join(names, ", ")) @@ -123,7 +145,7 @@ func exportApp(appDir string, app manifest.App, topology analyzer.Topology, opts files := map[string][]byte{ "compose.yaml": []byte(compose), ".env.example": []byte(envExample(appEnvironment, false)), - "migration-report.md": []byte(report(app, warnings)), + "migration-report.md": []byte(report(app, topology.Routes, warnings)), "migration-runbook.md": []byte(runbook(app, topology, warnings)), } for _, envFile := range serviceEnvFilesForMode(app.Services, false) { @@ -140,21 +162,84 @@ func exportApp(appDir string, app manifest.App, topology analyzer.Topology, opts for name, contents := range files { if err := writePrivateFile(filepath.Join(appDir, name), contents); err != nil { - return nil, err + return "", nil, err } } - if err := writeJSON(filepath.Join(appDir, "routes.json"), app.Routes); err != nil { - return nil, err + if err := writeJSON(filepath.Join(appDir, "routes.json"), topology.Routes); err != nil { + return "", nil, err } if err := writeJSON(filepath.Join(appDir, "storages.json"), app.Storages); err != nil { - return nil, err + return "", nil, err } if err := writeJSON(filepath.Join(appDir, "topology.json"), topology); err != nil { - return nil, err + return "", nil, err } - return warnings, nil + return composeSource, warnings, nil +} + +func routesForCompose(routes []manifest.Route, sourceServices []analyzer.SourceService, composeSource, compose string) []manifest.Route { + composeServices := composeServiceNameSet(compose) + rawServiceNamesByContainer := make(map[string]string, len(sourceServices)) + for _, sourceService := range sourceServices { + containerName := strings.TrimSpace(sourceService.ContainerName) + serviceName := strings.TrimSpace(sourceService.ServiceName) + if containerName == "" { + continue + } + if _, exists := rawServiceNamesByContainer[containerName]; exists { + continue + } + if _, exists := composeServices[serviceName]; exists { + rawServiceNamesByContainer[containerName] = serviceName + } + } + + result := append([]manifest.Route(nil), routes...) + for index := range result { + serviceName := strings.TrimSpace(result[index].ServiceName) + if len(composeServices) == 1 && (serviceName == "" || composeSource == ComposeSourceRaw && result[index].Source == "fqdn") { + for name := range composeServices { + result[index].ServiceName = name + } + continue + } + if composeSource == ComposeSourceGenerated { + if _, exists := composeServices[serviceName]; exists { + result[index].ServiceName = serviceName + continue + } + generatedName := planutil.Slug(serviceName) + if generatedName == "" && serviceName != "" { + generatedName = "app" + } + if _, exists := composeServices[generatedName]; exists { + result[index].ServiceName = generatedName + } + continue + } + if mappedService, exists := rawServiceNamesByContainer[serviceName]; exists { + result[index].ServiceName = mappedService + } else { + result[index].ServiceName = serviceName + } + } + return result +} + +func composeServiceNameSet(compose string) map[string]struct{} { + var document struct { + Services map[string]any `yaml:"services"` + } + if err := yaml.Unmarshal([]byte(compose), &document); err != nil || document.Services == nil { + return nil + } + names := make(map[string]struct{}, len(document.Services)) + for name := range document.Services { + names[strings.TrimSpace(name)] = struct{}{} + } + return names } func exportEnvMode(opts Options) string { @@ -169,7 +254,7 @@ type envFile struct { Vars []manifest.EnvVar } -func composeForApp(app manifest.App, includePrivateValues bool) (string, []string, []envFile) { +func composeForApp(app manifest.App, includePrivateValues bool) (string, string, []string, []envFile) { warnings := []string{} if app.Compose != nil { if strings.TrimSpace(app.Compose.Raw) != "" { @@ -180,7 +265,7 @@ func composeForApp(app manifest.App, includePrivateValues bool) (string, []strin if names := composeCoolifyServiceMagicEnvNames(compose); len(names) > 0 { warnings = append(warnings, "preserved Coolify service magic env vars in raw compose for review: "+strings.Join(names, ", ")) } - return ensureTrailingNewline(compose), warnings, serviceEnvFilesForMode(app.Services, includePrivateValues) + return ensureTrailingNewline(compose), ComposeSourceRaw, warnings, serviceEnvFilesForMode(app.Services, includePrivateValues) } if strings.TrimSpace(app.Compose.Resolved) != "" { warnings = append(warnings, "skipped resolved compose because it may contain interpolated secret values") @@ -189,7 +274,7 @@ func composeForApp(app manifest.App, includePrivateValues bool) (string, []strin if len(app.Services) == 0 { warnings = append(warnings, "no services were present in the manifest") - return "services: {}\n", warnings, nil + return "services: {}\n", ComposeSourceGenerated, warnings, nil } var builder strings.Builder @@ -254,8 +339,8 @@ func composeForApp(app manifest.App, includePrivateValues bool) (string, []strin } } - warnings = append(warnings, "generated compose from discovered container metadata") - return builder.String(), warnings, sortedEnvFiles(serviceEnvFileMap) + warnings = append(warnings, generatedComposeWarning) + return builder.String(), ComposeSourceGenerated, warnings, sortedEnvFiles(serviceEnvFileMap) } func serviceEnvFilesForMode(services []manifest.Service, includePrivateValues bool) []envFile { @@ -567,7 +652,7 @@ func envExample(envs []manifest.EnvVar, includePrivateValues bool) string { return builder.String() } -func report(app manifest.App, warnings []string) string { +func report(app manifest.App, routes []manifest.Route, warnings []string) string { var builder strings.Builder builder.WriteString("# migration report\n\n") builder.WriteString(fmt.Sprintf("app: `%s`\n\n", app.Name)) @@ -587,10 +672,10 @@ func report(app manifest.App, warnings []string) string { } builder.WriteString("## routes\n\n") - if len(app.Routes) == 0 { + if len(routes) == 0 { builder.WriteString("no routes detected.\n\n") } else { - for _, route := range app.Routes { + for _, route := range routes { builder.WriteString(fmt.Sprintf("- `%s` -> `%s`", route.Host, planutil.Fallback(route.ServiceName, app.Name))) if route.Port != "" { builder.WriteString(fmt.Sprintf(" port `%s`", route.Port)) diff --git a/internal/exporter/exporter_test.go b/internal/exporter/exporter_test.go index fc42a47..c423f97 100644 --- a/internal/exporter/exporter_test.go +++ b/internal/exporter/exporter_test.go @@ -41,6 +41,9 @@ func TestExportWritesBundleForComposeApp(t *testing.T) { if len(summary.Apps) != 1 { t.Fatalf("expected one app summary, got %#v", summary.Apps) } + if summary.Apps[0].ComposeSource != ComposeSourceRaw { + t.Fatalf("expected raw compose provenance, got %#v", summary.Apps[0]) + } appDir := filepath.Join(dir, "ghost-blog") assertMode(t, appDir, privateDirMode) @@ -113,11 +116,28 @@ func TestExportSkipsResolvedCompose(t *testing.T) { if !strings.Contains(compose, "image: \"example/api\"") { t.Fatalf("expected generated compose, got:\n%s", compose) } - if len(summary.Apps) != 1 || !strings.Contains(strings.Join(summary.Apps[0].Warnings, "\n"), "skipped resolved compose") { + if len(summary.Apps) != 1 || summary.Apps[0].ComposeSource != ComposeSourceGenerated || !strings.Contains(strings.Join(summary.Apps[0].Warnings, "\n"), "skipped resolved compose") { t.Fatalf("expected resolved compose warning, got %#v", summary.Apps) } } +func TestAppSummaryEffectiveComposeSourceSupportsLegacyIndexes(t *testing.T) { + var generated AppSummary + if err := json.Unmarshal([]byte(`{"warnings":["generated compose from discovered container metadata"]}`), &generated); err != nil { + t.Fatal(err) + } + if got, err := generated.EffectiveComposeSource(); err != nil || got != ComposeSourceGenerated { + t.Fatalf("expected legacy generated compose provenance, got %q, error %v", got, err) + } + var raw AppSummary + if err := json.Unmarshal([]byte(`{"warnings":["preserved Coolify service magic env vars for review: SERVICE_URL_API"]}`), &raw); err != nil { + t.Fatal(err) + } + if got, err := raw.EffectiveComposeSource(); err != nil || got != ComposeSourceRaw { + t.Fatalf("expected legacy raw compose provenance, got %q, error %v", got, err) + } +} + func TestExportScopesServiceEnvFiles(t *testing.T) { dir := t.TempDir() m := manifest.Manifest{ @@ -489,6 +509,133 @@ func TestExportRunbookIncludesLinkedResources(t *testing.T) { } } +func TestExportUsesReviewedComposeServiceInRouteArtifacts(t *testing.T) { + dir := t.TempDir() + m := manifest.Manifest{Apps: []manifest.App{{ + Name: "api", + Compose: &manifest.ComposeSource{Raw: "services:\n apiworker:\n image: example/api\n"}, + Services: []manifest.Service{{ + Name: "project-apiworker-1", + Image: "example/api", + Labels: map[string]string{"com.docker.compose.service": "apiworker"}, + }}, + Routes: []manifest.Route{{Host: "api.example.com", ServiceName: "project-apiworker-1", Port: "8080"}}, + }}} + if _, err := Export(m, Options{OutputDir: dir}); err != nil { + t.Fatal(err) + } + + appDir := filepath.Join(dir, "api") + var routes []manifest.Route + if err := json.Unmarshal([]byte(readFile(t, filepath.Join(appDir, "routes.json"))), &routes); err != nil { + t.Fatal(err) + } + if len(routes) != 1 || routes[0].ServiceName != "apiworker" { + t.Fatalf("expected routes.json to use the reviewed Compose service, got %#v", routes) + } + var topology analyzer.Topology + if err := json.Unmarshal([]byte(readFile(t, filepath.Join(appDir, "topology.json"))), &topology); err != nil { + t.Fatal(err) + } + if len(topology.Routes) != 1 || topology.Routes[0].ServiceName != "apiworker" { + t.Fatalf("expected topology route to use the reviewed Compose service, got %#v", topology.Routes) + } + report := readFile(t, filepath.Join(appDir, "migration-report.md")) + if !strings.Contains(report, "-> `apiworker`") || strings.Contains(report, "-> `project-apiworker-1`") { + t.Fatalf("expected migration report to use the reviewed Compose service, got:\n%s", report) + } + runbook := readFile(t, filepath.Join(appDir, "migration-runbook.md")) + if !strings.Contains(runbook, "for service `apiworker`") || strings.Contains(runbook, "for service `project-apiworker-1`") { + t.Fatalf("expected runbook to use the reviewed Compose service, got:\n%s", runbook) + } +} + +func TestExportMapsCoolifyApplicationFQDNToSoleRawComposeService(t *testing.T) { + dir := t.TempDir() + m := manifest.Manifest{Apps: []manifest.App{{ + Name: "prod-api", + Compose: &manifest.ComposeSource{Raw: "services:\n web:\n image: example/api\n"}, + Services: []manifest.Service{{Name: "prod-api", Image: "example/api"}}, + Routes: []manifest.Route{{Host: "api.example.com", ServiceName: "prod-api", Source: "fqdn"}}, + }}} + if _, err := Export(m, Options{OutputDir: dir}); err != nil { + t.Fatal(err) + } + + appDir := filepath.Join(dir, "prod-api") + var topology analyzer.Topology + if err := json.Unmarshal([]byte(readFile(t, filepath.Join(appDir, "topology.json"))), &topology); err != nil { + t.Fatal(err) + } + if len(topology.Routes) != 1 || topology.Routes[0].ServiceName != "web" { + t.Fatalf("expected application-level fqdn to resolve to sole raw Compose service, got %#v", topology.Routes) + } +} + +func TestExportMapsRoutesThroughAliasedRawComposeServices(t *testing.T) { + for name, compose := range map[string]string{ + "direct alias": "x-services: &app-services\n web:\n image: example/api\nservices: *app-services\n", + "merge key": "x-services: &app-services\n web:\n image: example/api\nservices:\n <<: *app-services\n", + } { + t.Run(name, func(t *testing.T) { + dir := t.TempDir() + m := manifest.Manifest{Apps: []manifest.App{{ + Name: "prod-api", + Compose: &manifest.ComposeSource{Raw: compose}, + Services: []manifest.Service{{Name: "prod-api", Image: "example/api"}}, + Routes: []manifest.Route{{Host: "api.example.com", ServiceName: "prod-api", Source: "fqdn"}}, + }}} + if _, err := Export(m, Options{OutputDir: dir}); err != nil { + t.Fatal(err) + } + var topology analyzer.Topology + if err := json.Unmarshal([]byte(readFile(t, filepath.Join(dir, "prod-api", "topology.json"))), &topology); err != nil { + t.Fatal(err) + } + if len(topology.Routes) != 1 || topology.Routes[0].ServiceName != "web" { + t.Fatalf("expected alias-expanded sole Compose service web, got %#v", topology.Routes) + } + }) + } +} + +func TestExportUsesGeneratedComposeServiceInRouteArtifacts(t *testing.T) { + dir := t.TempDir() + m := manifest.Manifest{Apps: []manifest.App{{ + Name: "blog", + Services: []manifest.Service{{Name: "Ghost Blog", Image: "example/blog"}}, + Routes: []manifest.Route{{Host: "blog.example.com", ServiceName: "Ghost Blog", Port: "3000"}}, + }}} + if _, err := Export(m, Options{OutputDir: dir}); err != nil { + t.Fatal(err) + } + + appDir := filepath.Join(dir, "blog") + compose := readFile(t, filepath.Join(appDir, "compose.yaml")) + if !strings.Contains(compose, " ghost-blog:") { + t.Fatalf("expected generated Compose service ghost-blog, got:\n%s", compose) + } + var routes []manifest.Route + if err := json.Unmarshal([]byte(readFile(t, filepath.Join(appDir, "routes.json"))), &routes); err != nil { + t.Fatal(err) + } + var topology analyzer.Topology + if err := json.Unmarshal([]byte(readFile(t, filepath.Join(appDir, "topology.json"))), &topology); err != nil { + t.Fatal(err) + } + for name, artifactRoutes := range map[string][]manifest.Route{"routes.json": routes, "topology.json": topology.Routes} { + if len(artifactRoutes) != 1 || artifactRoutes[0].ServiceName != "ghost-blog" { + t.Fatalf("expected %s to use generated Compose service, got %#v", name, artifactRoutes) + } + } + for _, name := range []string{"migration-report.md", "migration-runbook.md"} { + contents := readFile(t, filepath.Join(appDir, name)) + if !strings.Contains(contents, "`ghost-blog`") || strings.Contains(contents, "`Ghost Blog`") { + t.Fatalf("expected %s to use generated Compose service, got:\n%s", name, contents) + } + } +} + func readFile(t *testing.T, path string) string { t.Helper() contents, err := os.ReadFile(path) diff --git a/internal/preparer/preparer.go b/internal/preparer/preparer.go index 2f09a74..33cdadb 100644 --- a/internal/preparer/preparer.go +++ b/internal/preparer/preparer.go @@ -12,12 +12,15 @@ import ( "github.com/aikins01/bort/internal/analyzer" "github.com/aikins01/bort/internal/exporter" "github.com/aikins01/bort/internal/planutil" + "gopkg.in/yaml.v3" ) type Status string const ( - APIVersion = "bort.prepare/v1alpha1" + APIVersion = "bort.prepare/v1alpha1" + ComposeSourceRaw = exporter.ComposeSourceRaw + ComposeSourceGenerated = exporter.ComposeSourceGenerated StatusGreen Status = "green" StatusYellow Status = "yellow" @@ -52,6 +55,7 @@ const ( GateEnvValuesRedacted = "env.values_redacted" GateDomainHostMissing = "domain.host_missing" GateDomainServiceMissing = "domain.service_missing" + GateDomainServiceNotInCompose = "domain.service_not_in_compose" GateRoutesNone = "routes.none" GateDataStoreManualReview = "data_store.manual_review" GateDataStorePrepareRequired = "data_store.prepare_required" @@ -146,6 +150,7 @@ type AppResource struct { Type string `json:"type"` Name string `json:"name"` ComposePath string `json:"composePath"` + ComposeSource string `json:"composeSource,omitempty"` ComposeMissing bool `json:"composeMissing,omitempty"` MissingInputs []string `json:"missingInputs,omitempty"` Readiness Readiness `json:"readiness"` @@ -276,9 +281,16 @@ func planApp(bundleDir, target string, app exporter.AppSummary) (AppPlan, error) if err != nil { return AppPlan{}, fmt.Errorf("read topology for %s: %w", app.Name, err) } + composeSource, err := app.EffectiveComposeSource() + if err != nil { + return AppPlan{}, fmt.Errorf("read compose source for %s: %w", app.Name, err) + } plan := AppPlan{Name: app.Name, Directory: app.Directory, Platform: app.Platform, Role: app.Role, ProjectGroup: projectGroup(app.ProjectGroup), Status: StatusGreen, Readiness: ReadinessReadyToCreate} - plan.Resources = resourceSpecs(app, appDir, topology) + plan.Resources, err = resourceSpecs(app, appDir, topology, composeSource) + if err != nil { + return AppPlan{}, fmt.Errorf("read compose services for %s: %w", app.Name, err) + } addReadinessGates(&plan, topology) plan.add(SeverityInfo, "compose", fmt.Sprintf("would create %s compose app from compose.yaml", target)) addSourceControlActions(&plan) @@ -309,23 +321,65 @@ func projectGroup(group *exporter.ProjectGroup) *ProjectGroup { return &ProjectGroup{Name: group.Name, Environment: group.Environment, Source: group.Source} } -func resourceSpecs(app exporter.AppSummary, appDir string, topology analyzer.Topology) ResourceSpecs { +func resourceSpecs(app exporter.AppSummary, appDir string, topology analyzer.Topology, composeSource string) (ResourceSpecs, error) { + composePath := filepath.Join(appDir, "compose.yaml") + composeContents, composeErr := os.ReadFile(composePath) resources := ResourceSpecs{ - App: appResource(app.Name, appDir), + App: appResource(app.Name, composeSource, composeContents, composeErr), EnvFiles: envFileResources(appDir, app.PrivateEnvValues), } resources.SourceControl = sourceControlResource(topology.SourceControl) + for _, service := range topology.SourceServices { + if service.ContainerID == "" && service.ContainerName == "" { + continue + } + resources.SourceServices = append(resources.SourceServices, SourceServiceRef{ + ServiceName: service.ServiceName, + ContainerID: service.ContainerID, + ContainerName: service.ContainerName, + }) + } + composeServices := map[string]struct{}{} + if !resources.App.ComposeMissing { + var err error + composeServices, err = composeServiceNames(composeContents) + if err != nil { + return ResourceSpecs{}, err + } + } + rawServiceNamesByContainer := make(map[string]string, len(resources.SourceServices)) + routesNameContainers := strings.TrimSpace(app.ComposeSource) == "" + for _, sourceService := range resources.SourceServices { + containerName := strings.TrimSpace(sourceService.ContainerName) + serviceName := strings.TrimSpace(sourceService.ServiceName) + if !routesNameContainers || containerName == "" { + continue + } + if _, exists := rawServiceNamesByContainer[containerName]; exists { + continue + } + if _, exists := composeServices[serviceName]; exists { + rawServiceNamesByContainer[containerName] = serviceName + } + } for _, route := range topology.Routes { + serviceName := strings.TrimSpace(route.ServiceName) + serviceExists := true + if !resources.App.ComposeMissing { + serviceName, serviceExists = preparedServiceName(serviceName, route.Source, rawServiceNamesByContainer, composeSource, composeServices) + } readiness := ReadinessReadyToCreate if strings.TrimSpace(route.Host) == "" { readiness = ReadinessNeedsInput - } else if strings.TrimSpace(route.ServiceName) == "" { - readiness = ReadinessNeedsDecision + } else if strings.TrimSpace(serviceName) == "" { + readiness = ReadinessBlocked + } else if !serviceExists { + readiness = ReadinessBlocked } resources.Domains = append(resources.Domains, DomainResource{ Host: route.Host, - ServiceName: route.ServiceName, + ServiceName: serviceName, Port: route.Port, Source: route.Source, Readiness: readiness, @@ -350,30 +404,64 @@ func resourceSpecs(app exporter.AppSummary, appDir string, topology analyzer.Top for _, link := range topology.LinkedResources { resources.LinkedResources = append(resources.LinkedResources, linkedResourceCandidate(link)) } - for _, service := range topology.SourceServices { - if service.ContainerID == "" && service.ContainerName == "" { - continue - } - resources.SourceServices = append(resources.SourceServices, SourceServiceRef{ - ServiceName: service.ServiceName, - ContainerID: service.ContainerID, - ContainerName: service.ContainerName, - }) + + return resources, nil +} + +func composeServiceNames(contents []byte) (map[string]struct{}, error) { + var document struct { + Services map[string]yaml.Node `yaml:"services"` + } + if err := yaml.Unmarshal(contents, &document); err != nil { + return nil, err } + names := make(map[string]struct{}, len(document.Services)) + for name := range document.Services { + names[name] = struct{}{} + } + return names, nil +} - return resources +func preparedServiceName(serviceName, routeSource string, rawServiceNamesByContainer map[string]string, composeSource string, composeServices map[string]struct{}) (string, bool) { + if len(composeServices) == 1 && (serviceName == "" || (composeSource == ComposeSourceRaw && routeSource == "fqdn")) { + for name := range composeServices { + return name, true + } + } + switch composeSource { + case ComposeSourceGenerated: + if _, ok := composeServices[serviceName]; ok { + return serviceName, true + } + generatedName := planutil.Slug(serviceName) + if generatedName == "" && serviceName != "" { + generatedName = "app" + } + if _, ok := composeServices[generatedName]; ok { + return generatedName, true + } + return serviceName, false + case ComposeSourceRaw: + if mappedService, ok := rawServiceNamesByContainer[serviceName]; ok { + return mappedService, true + } + _, ok := composeServices[serviceName] + return serviceName, ok + default: + return serviceName, false + } } -func appResource(name, appDir string) AppResource { +func appResource(name, composeSource string, contents []byte, composeErr error) AppResource { resource := AppResource{ - Type: "compose", - Name: name, - ComposePath: "compose.yaml", - Readiness: ReadinessReadyToCreate, + Type: "compose", + Name: name, + ComposePath: "compose.yaml", + ComposeSource: composeSource, + Readiness: ReadinessReadyToCreate, } - contents, err := os.ReadFile(filepath.Join(appDir, resource.ComposePath)) - if err != nil { + if composeErr != nil { resource.Readiness = ReadinessBlocked resource.ComposeMissing = true return resource @@ -551,7 +639,9 @@ func addReadinessGates(plan *AppPlan, topology analyzer.Topology) { if strings.TrimSpace(domain.Host) == "" { plan.addGate(ReadinessNeedsInput, SeverityWarn, GateDomainHostMissing, "domain route has no host and must be filled before deploy", resourceRef, nil) } else if strings.TrimSpace(domain.ServiceName) == "" { - plan.addGate(ReadinessNeedsDecision, SeverityWarn, GateDomainServiceMissing, fmt.Sprintf("domain %s has no service mapping; confirm target service manually", domain.Host), resourceRef, nil) + plan.addGate(ReadinessBlocked, SeverityError, GateDomainServiceMissing, fmt.Sprintf("domain %s has no service mapping and compose.yaml does not identify a unique service; rescan or correct the route before deploy", domain.Host), resourceRef, nil) + } else if domain.Readiness == ReadinessBlocked { + plan.addGate(ReadinessBlocked, SeverityError, GateDomainServiceNotInCompose, fmt.Sprintf("domain %s points at service %q, which is not present in compose.yaml; rescan or correct the route before deploy", domain.Host, domain.ServiceName), resourceRef, []string{domain.ServiceName}) } } if len(plan.Resources.Domains) == 0 { diff --git a/internal/preparer/preparer_test.go b/internal/preparer/preparer_test.go index dc97db5..28af558 100644 --- a/internal/preparer/preparer_test.go +++ b/internal/preparer/preparer_test.go @@ -1,10 +1,14 @@ package preparer import ( + "encoding/json" + "os" + "path/filepath" "slices" "strings" "testing" + "github.com/aikins01/bort/internal/analyzer" "github.com/aikins01/bort/internal/exporter" "github.com/aikins01/bort/internal/manifest" ) @@ -55,7 +59,7 @@ func TestPlanBuildsDryRunActionsFromTopology(t *testing.T) { if app.ProjectGroup == nil || app.ProjectGroup.Name != "web" || app.ProjectGroup.Environment != "production" { t.Fatalf("unexpected project group: %#v", app.ProjectGroup) } - if app.Resources.App.Type != "compose" || app.Resources.App.ComposePath != "compose.yaml" { + if app.Resources.App.Type != "compose" || app.Resources.App.ComposePath != "compose.yaml" || app.Resources.App.ComposeSource != ComposeSourceGenerated { t.Fatalf("unexpected app resource: %#v", app.Resources.App) } if len(app.Resources.Domains) != 1 || app.Resources.Domains[0].Host != "web.example.com" || app.Resources.Domains[0].ServiceName != "web" || app.Resources.Domains[0].Port != "3000" { @@ -180,6 +184,9 @@ func TestPlanSurfacesSourceControlAsNonBlockingAction(t *testing.T) { if app.Readiness != ReadinessReadyToCreate || app.Resources.SourceControl == nil || app.Resources.SourceControl.Auth != "coolify_github_app" { t.Fatalf("expected source control to be non-blocking, got %#v", app) } + if app.Resources.App.ComposeSource != ComposeSourceRaw { + t.Fatalf("expected raw compose provenance, got %#v", app.Resources.App) + } if app.TargetResources == nil || app.TargetResources.Dokploy == nil || app.TargetResources.Dokploy.SourceControl == nil || app.TargetResources.Dokploy.SourceControl.Action != "connect_dokploy_source_after_cutover_if_needed" { t.Fatalf("expected dokploy source-control action, got %#v", app.TargetResources) } @@ -191,6 +198,251 @@ func TestPlanSurfacesSourceControlAsNonBlockingAction(t *testing.T) { } } +func TestPlanPersistsReviewedRawComposeRouteService(t *testing.T) { + dir := t.TempDir() + m := manifest.Manifest{ + Source: manifest.Source{Platform: "coolify"}, + Apps: []manifest.App{{ + Name: "api", + Compose: &manifest.ComposeSource{Raw: "services:\n apiworker:\n image: example/api\n"}, + Services: []manifest.Service{{ + Name: "project-apiworker-1", + Image: "example/api", + Labels: map[string]string{"com.docker.compose.service": "apiworker"}, + }}, + Routes: []manifest.Route{{Host: "api.example.com", ServiceName: "project-apiworker-1", Port: "8080"}}, + }}, + } + + if _, err := exporter.Export(m, exporter.Options{OutputDir: dir}); err != nil { + t.Fatal(err) + } + result, err := Plan(Options{BundleDir: dir, Target: "dokploy"}) + if err != nil { + t.Fatal(err) + } + app := result.Apps[0] + if len(app.Resources.Domains) != 1 || app.Resources.Domains[0].ServiceName != "apiworker" { + t.Fatalf("expected reviewed route service in prepare output, got %#v", app.Resources.Domains) + } + if app.TargetResources == nil || app.TargetResources.Dokploy == nil || len(app.TargetResources.Dokploy.Domains) != 1 || app.TargetResources.Dokploy.Domains[0].ServiceName != "apiworker" { + t.Fatalf("expected reviewed route service in target output, got %#v", app.TargetResources) + } + assertAction(t, app, "route|would create dokploy domain api.example.com for service apiworker on port 8080") +} + +func TestPlanPersistsUniqueComposeServiceForEmptyRouteMapping(t *testing.T) { + dir := t.TempDir() + m := manifest.Manifest{Apps: []manifest.App{{ + Name: "api", + Compose: &manifest.ComposeSource{Raw: "services:\n api:\n image: example/api\n"}, + Services: []manifest.Service{{Name: "api", Image: "example/api"}}, + Routes: []manifest.Route{{Host: "api.example.com", Port: "8080"}}, + }}} + exportWithLegacyRoutes(t, dir, m) + result, err := Plan(Options{BundleDir: dir, Target: "dokploy"}) + if err != nil { + t.Fatal(err) + } + app := result.Apps[0] + if result.Status != StatusGreen || app.Readiness != ReadinessReadyToCreate || len(app.Resources.Domains) != 1 || app.Resources.Domains[0].ServiceName != "api" { + t.Fatalf("expected unique Compose service to be persisted, got %#v", app) + } + assertNoGate(t, app, GateDomainServiceMissing) +} + +func TestPlanMapsLegacyFQDNRouteToUniqueRawComposeService(t *testing.T) { + dir := t.TempDir() + m := manifest.Manifest{Apps: []manifest.App{{ + Name: "api", + Compose: &manifest.ComposeSource{Raw: "services:\n api:\n image: example/api\n"}, + Routes: []manifest.Route{{Host: "api.example.com", ServiceName: "coolify-fqdn-target", Port: "8080", Source: "fqdn"}}, + }}} + exportWithLegacyRoutes(t, dir, m) + + result, err := Plan(Options{BundleDir: dir, Target: "dokploy"}) + if err != nil { + t.Fatal(err) + } + app := result.Apps[0] + if result.Status != StatusGreen || app.Readiness != ReadinessReadyToCreate || len(app.Resources.Domains) != 1 || app.Resources.Domains[0].ServiceName != "api" { + t.Fatalf("legacy FQDN route was not mapped to the unique raw Compose service: %#v", app) + } + assertNoGate(t, app, GateDomainServiceNotInCompose) +} + +func TestPlanPersistsSluggedGeneratedComposeService(t *testing.T) { + dir := t.TempDir() + m := manifest.Manifest{Apps: []manifest.App{{ + Name: "blog", + Services: []manifest.Service{{Name: "Ghost Blog", Image: "example/blog"}}, + Routes: []manifest.Route{{Host: "blog.example.com", ServiceName: "Ghost Blog", Port: "3000"}}, + }}} + exportWithLegacyRoutes(t, dir, m) + result, err := Plan(Options{BundleDir: dir, Target: "dokploy"}) + if err != nil { + t.Fatal(err) + } + app := result.Apps[0] + if result.Status != StatusGreen || app.Readiness != ReadinessReadyToCreate || len(app.Resources.Domains) != 1 || app.Resources.Domains[0].ServiceName != "ghost-blog" { + t.Fatalf("expected generated Compose service name to be persisted, got %#v", app) + } + assertNoGate(t, app, GateDomainServiceNotInCompose) +} + +func exportWithLegacyRoutes(t *testing.T, dir string, m manifest.Manifest) { + t.Helper() + summary, err := exporter.Export(m, exporter.Options{OutputDir: dir}) + if err != nil { + t.Fatal(err) + } + appDir := filepath.Join(dir, summary.Apps[0].Directory) + topologyPath := filepath.Join(appDir, "topology.json") + contents, err := os.ReadFile(topologyPath) + if err != nil { + t.Fatal(err) + } + var topology analyzer.Topology + if err := json.Unmarshal(contents, &topology); err != nil { + t.Fatal(err) + } + topology.Routes = append([]manifest.Route(nil), m.Apps[0].Routes...) + contents, err = json.MarshalIndent(topology, "", " ") + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(topologyPath, append(contents, '\n'), 0o600); err != nil { + t.Fatal(err) + } + routes, err := json.MarshalIndent(m.Apps[0].Routes, "", " ") + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(appDir, "routes.json"), append(routes, '\n'), 0o600); err != nil { + t.Fatal(err) + } +} + +func TestPlanBlocksMismatchedGeneratedComposeService(t *testing.T) { + dir := t.TempDir() + m := manifest.Manifest{Apps: []manifest.App{{ + Name: "blog", + Services: []manifest.Service{{Name: "Ghost Blog", Image: "example/blog"}}, + Routes: []manifest.Route{{Host: "blog.example.com", ServiceName: "Wrong Blog", Port: "3000"}}, + }}} + if _, err := exporter.Export(m, exporter.Options{OutputDir: dir}); err != nil { + t.Fatal(err) + } + result, err := Plan(Options{BundleDir: dir, Target: "dokploy"}) + if err != nil { + t.Fatal(err) + } + app := result.Apps[0] + if result.Status != StatusRed || app.Readiness != ReadinessBlocked || len(app.Resources.Domains) != 1 || app.Resources.Domains[0].ServiceName != "Wrong Blog" { + t.Fatalf("expected mismatched generated Compose service to block preparation, got %#v", app) + } + assertGate(t, app, GateDomainServiceNotInCompose) +} + +func TestPlanReportsMalformedCompose(t *testing.T) { + dir := t.TempDir() + m := manifest.Manifest{Apps: []manifest.App{{ + Name: "api", + Compose: &manifest.ComposeSource{Raw: "services:\n api:\n image: example/api\n"}, + Services: []manifest.Service{{Name: "api", Image: "example/api"}}, + Routes: []manifest.Route{{Host: "api.example.com", ServiceName: "api", Port: "8080"}}, + }}} + summary, err := exporter.Export(m, exporter.Options{OutputDir: dir}) + if err != nil { + t.Fatal(err) + } + composePath := filepath.Join(dir, summary.Apps[0].Directory, "compose.yaml") + if err := os.WriteFile(composePath, []byte("services:\n api: [\n"), 0o600); err != nil { + t.Fatal(err) + } + if _, err := Plan(Options{BundleDir: dir, Target: "dokploy"}); err == nil || !strings.Contains(err.Error(), "read compose services for api") { + t.Fatalf("expected malformed Compose error, got %v", err) + } +} + +func TestPlanBlocksEmptyRouteMappingWithMultipleComposeServices(t *testing.T) { + dir := t.TempDir() + m := manifest.Manifest{Apps: []manifest.App{{ + Name: "api", + Compose: &manifest.ComposeSource{Raw: "services:\n web:\n image: example/web\n api:\n image: example/api\n"}, + Services: []manifest.Service{{Name: "web", Image: "example/web"}, {Name: "api", Image: "example/api"}}, + Routes: []manifest.Route{{Host: "api.example.com", Port: "8080"}}, + }}} + if _, err := exporter.Export(m, exporter.Options{OutputDir: dir}); err != nil { + t.Fatal(err) + } + result, err := Plan(Options{BundleDir: dir, Target: "dokploy"}) + if err != nil { + t.Fatal(err) + } + app := result.Apps[0] + if result.Status != StatusRed || app.Readiness != ReadinessBlocked || len(app.Resources.Domains) != 1 || app.Resources.Domains[0].Readiness != ReadinessBlocked { + t.Fatalf("expected ambiguous empty route mapping to block preparation, got %#v", app) + } + assertGate(t, app, GateDomainServiceMissing) +} + +func TestPlanBlocksRouteServiceMissingFromCompose(t *testing.T) { + dir := t.TempDir() + m := manifest.Manifest{Apps: []manifest.App{{ + Name: "api", + Compose: &manifest.ComposeSource{Raw: "services:\n api:\n image: example/api\n"}, + Services: []manifest.Service{{Name: "api", Image: "example/api"}}, + Routes: []manifest.Route{{Host: "api.example.com", ServiceName: "api-stale", Port: "8080"}}, + }}} + if _, err := exporter.Export(m, exporter.Options{OutputDir: dir}); err != nil { + t.Fatal(err) + } + result, err := Plan(Options{BundleDir: dir, Target: "dokploy"}) + if err != nil { + t.Fatal(err) + } + app := result.Apps[0] + if result.Status != StatusRed || app.Readiness != ReadinessBlocked || len(app.Resources.Domains) != 1 || app.Resources.Domains[0].Readiness != ReadinessBlocked { + t.Fatalf("expected missing Compose service to block preparation, got %#v", app) + } + assertGate(t, app, GateDomainServiceNotInCompose) + if app.TargetResources == nil || app.TargetResources.Dokploy == nil || len(app.TargetResources.Dokploy.Domains) != 1 || app.TargetResources.Dokploy.Domains[0].Readiness != ReadinessBlocked { + t.Fatalf("expected blocked route in target output, got %#v", app.TargetResources) + } +} + +func TestPlanRejectsUnknownComposeSource(t *testing.T) { + dir := t.TempDir() + m := manifest.Manifest{Apps: []manifest.App{{ + Name: "api", + Services: []manifest.Service{{Name: "api", Image: "example/api"}}, + }}} + if _, err := exporter.Export(m, exporter.Options{OutputDir: dir}); err != nil { + t.Fatal(err) + } + indexPath := filepath.Join(dir, "index.json") + contents, err := os.ReadFile(indexPath) + if err != nil { + t.Fatal(err) + } + var summary exporter.Summary + if err := json.Unmarshal(contents, &summary); err != nil { + t.Fatal(err) + } + summary.Apps[0].ComposeSource = "genrated" + contents, err = json.MarshalIndent(summary, "", " ") + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(indexPath, contents, 0o600); err != nil { + t.Fatal(err) + } + if _, err := Plan(Options{BundleDir: dir}); err == nil || !strings.Contains(err.Error(), `unsupported compose source "genrated"`) { + t.Fatalf("expected unknown compose source rejection, got %v", err) + } +} + func TestPlanBlocksIncompleteDeployArtifact(t *testing.T) { dir := t.TempDir() m := manifest.Manifest{ @@ -296,3 +548,68 @@ func assertNoGate(t *testing.T, app AppPlan, code string) { } } } + +func TestPlanKeepsExportedRawRouteServiceWhenAnotherContainerShadowsIt(t *testing.T) { + m := manifest.Manifest{ + Source: manifest.Source{Platform: "coolify"}, + Apps: []manifest.App{{ + Name: "shop", + Compose: &manifest.ComposeSource{Raw: "services:\n web:\n image: example/web\n api:\n image: example/api\n container_name: web\n"}, + Services: []manifest.Service{ + {Name: "proj-web-1", Image: "example/web", Labels: map[string]string{"com.docker.compose.service": "web"}}, + {Name: "web", Image: "example/api", Labels: map[string]string{"com.docker.compose.service": "api"}}, + }, + }}, + } + for _, tc := range []struct { + name string + route string + legacy bool + want string + }{ + {name: "exported route already names the Compose service", route: "proj-web-1", want: "web"}, + {name: "legacy bundle maps the api container name", route: "web", legacy: true, want: "api"}, + } { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + m.Apps[0].Routes = []manifest.Route{{Host: "shop.example.com", ServiceName: tc.route, Port: "8080"}} + if tc.legacy { + exportWithLegacyRoutes(t, dir, m) + indexPath := filepath.Join(dir, "index.json") + var index map[string]any + if err := json.Unmarshal([]byte(mustReadFile(t, indexPath)), &index); err != nil { + t.Fatal(err) + } + for _, app := range index["apps"].([]any) { + delete(app.(map[string]any), "composeSource") + } + contents, err := json.Marshal(index) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(indexPath, contents, 0o600); err != nil { + t.Fatal(err) + } + } else if _, err := exporter.Export(m, exporter.Options{OutputDir: dir}); err != nil { + t.Fatal(err) + } + result, err := Plan(Options{BundleDir: dir, Target: "dokploy"}) + if err != nil { + t.Fatal(err) + } + app := result.Apps[0] + if len(app.Resources.Domains) != 1 || app.Resources.Domains[0].ServiceName != tc.want { + t.Fatalf("route service = %#v, want %s", app.Resources.Domains, tc.want) + } + }) + } +} + +func mustReadFile(t *testing.T, path string) string { + t.Helper() + contents, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + return string(contents) +} diff --git a/internal/target/dokploy/apply.go b/internal/target/dokploy/apply.go index 54cb1a9..e2da5e4 100644 --- a/internal/target/dokploy/apply.go +++ b/internal/target/dokploy/apply.go @@ -73,15 +73,16 @@ const ( ) type StepProgress struct { - Index int - Total int - Step Step - Status StepStatus - Message string - Err error - MutationAmbiguous bool - RequiresNewRun bool - Target *TargetIdentity + Index int + Total int + Step Step + Status StepStatus + Message string + Err error + MutationAmbiguous bool + AuthorityRecoveryRequired bool + RequiresNewRun bool + Target *TargetIdentity } type TargetIdentity struct { @@ -113,8 +114,11 @@ type Plan struct { RunName string RunID string RunDir string + RecoveryCommand string ResumeFrom int TargetIdentities map[string]TargetIdentity + StagingTransferApps []string + HandedOffApps []string BeforeStep *func(StepProgress) error OnProgress *func(StepProgress) stepTimeout time.Duration @@ -123,11 +127,14 @@ type Plan struct { type planStateOrder int const ( - // planStateStaged transfers state into Bort-owned staging volumes + // planStateStagedV1Alpha4 transfers state into Bort-owned staging volumes // before the Dokploy deployment exists, then deploys a compose file // that mounts those volumes. no target writer ever runs during a // copy, so Dokploy needs no fence it cannot provide. - planStateStaged planStateOrder = iota + planStateStagedV1Alpha4 planStateOrder = iota + // planStateStagedV1Alpha3 reproduces the first staged-transfer order, + // which resumed unrouted source writers before deploying the target. + planStateStagedV1Alpha3 // planStateInPlaceV1Alpha2 and planStateInPlaceV1Alpha1 reproduce the // historical orders (deploy first, pause target writers, copy into the // deployed volumes) so recorded ledgers keep matching by index. @@ -136,7 +143,11 @@ const ( ) func PlanFromArtifacts(prepare preparer.Result, sync syncplan.Result, cutover gateway.Result) Plan { - return planFromArtifacts(prepare, sync, cutover, planStateStaged) + return planFromArtifacts(prepare, sync, cutover, planStateStagedV1Alpha4) +} + +func LegacyPlanFromArtifactsV1Alpha3(prepare preparer.Result, sync syncplan.Result, cutover gateway.Result) Plan { + return planFromArtifacts(prepare, sync, cutover, planStateStagedV1Alpha3) } func PlanFromArtifactsV1Alpha2(prepare preparer.Result, sync syncplan.Result, cutover gateway.Result) Plan { @@ -182,7 +193,8 @@ func appStateSteps(prepare preparer.Result, app syncplan.AppPlan) (dataStoreStep func planFromArtifacts(prepare preparer.Result, sync syncplan.Result, cutover gateway.Result, order planStateOrder) Plan { plan := Plan{Prepare: prepare, Sync: sync, Cutover: cutover} - staged := order == planStateStaged + staged := order == planStateStagedV1Alpha4 || order == planStateStagedV1Alpha3 + legacyStaged := order == planStateStagedV1Alpha3 legacyOrder := order == planStateInPlaceV1Alpha1 statefulApps := map[string]struct{}{} for _, app := range sync.Apps { @@ -238,7 +250,7 @@ func planFromArtifacts(prepare preparer.Result, sync syncplan.Result, cutover ga plan.Steps = append(plan.Steps, volumeSteps...) _, routed := routedAppNames[app.Name] if staged { - if !routed { + if legacyStaged && !routed { plan.Steps = append(plan.Steps, Step{Kind: StepResumeSource, App: app.Name, Ref: app.Name}) } if pushStep, ok := deferredPushSteps[app.Name]; ok { @@ -271,6 +283,20 @@ func planFromArtifacts(prepare preparer.Result, sync syncplan.Result, cutover ga // only swap proxies when something actually depends on :80/:443 — // keeps no-route migrations from disturbing a healthy coolify host. if hasRoutes { + // a routed source container keeps its Coolify Traefik labels, which + // collide with the activated target router on the same host; a stopped + // container's labels drop out of Traefik's docker provider, so routed + // stateless sources pause at the cutover boundary and stay stopped + // until retirement, like routed stateful ones. + for _, appName := range routedApps { + if _, routed := routedAppNames[appName]; !routed { + continue + } + if _, stateful := statefulApps[appName]; stateful { + continue + } + plan.Steps = append(plan.Steps, Step{Kind: StepPauseSource, App: appName, Ref: appName}) + } if legacyOrder { for _, appName := range routedApps { plan.Steps = append(plan.Steps, Step{Kind: StepActivateRoutes, App: appName, Ref: "routes"}) @@ -339,13 +365,15 @@ type appCache struct { TargetWritersStopped []dockerContainer SourcePausedContainers []sourcePausedContainer SourcePauseRecorded bool + StagingTransferStarted bool MigratedVolumeMounts map[string]migratedVolumeMount } type applyContext struct { - plan Plan - cache map[string]*appCache - stagingEnvFormat stagingEnvFormat + plan Plan + cache map[string]*appCache + stagingVolumePins map[string]stagingVolumePin + stagingEnvFormat stagingEnvFormat } func (a *applyContext) entry(app string) *appCache { @@ -455,7 +483,7 @@ func ValidateStagedTransfer(plan Plan) error { if err != nil { return err } - if stagingComposeProjectNamePattern.MatchString(restore.stagingCompose) || stagingComposeProjectNamePattern.MatchString(envContent) { + if hasUnescapedComposeProjectName(restore.stagingCompose) || hasUnescapedComposeProjectName(envContent) { return fmt.Errorf("%w: data store service %s for app %s interpolates COMPOSE_PROJECT_NAME, which resolves to the Bort staging project instead of the Dokploy app; choose a recreate or managed data store strategy or change the source compose before live apply", ErrNotImplemented, restore.store.Service, restore.app.Name) } return nil @@ -539,6 +567,10 @@ func (c *Client) requireStagingCompatibleCompose(ctx context.Context, composeID, var stagingComposeProjectNamePattern = regexp.MustCompile(`\$\{?COMPOSE_PROJECT_NAME\b`) +func hasUnescapedComposeProjectName(value string) bool { + return stagingComposeProjectNamePattern.MatchString(strings.ReplaceAll(value, "$$", "\x00")) +} + func appDecisionCodesApproved(plan Plan, app preparer.AppPlan) bool { found := false for _, gate := range app.Gates { @@ -605,6 +637,18 @@ func (e deployedComposeMonitorError) Unwrap() error { return e.err } +type completedDeploymentHandoffError struct { + err error +} + +func (e completedDeploymentHandoffError) Error() string { + return e.err.Error() +} + +func (e completedDeploymentHandoffError) Unwrap() error { + return e.err +} + type ambiguousDeployResponseError struct { err error } @@ -631,6 +675,18 @@ func (e unsafeSourceResumeError) Unwrap() error { return e.err } +type authorityRecoveryRequiredError struct { + err error +} + +func (e authorityRecoveryRequiredError) Error() string { + return e.err.Error() +} + +func (e authorityRecoveryRequiredError) Unwrap() error { + return e.err +} + func (c *Client) validateNoActiveBortOverrides(ctx context.Context, composeID string) error { composeID = strings.TrimSpace(composeID) if composeID == "" { @@ -751,6 +807,11 @@ func (c *Client) Apply(ctx context.Context, plan Plan) error { if err := validatePlanReadyForLiveApply(plan); err != nil { return err } + if RequiresCoolifyDeploymentFence(plan) { + if err := requireCoolifyDeploymentFence(ctx, c.dockerRunner()); err != nil { + return err + } + } for app, identity := range plan.TargetIdentities { identity.ProjectID = strings.TrimSpace(identity.ProjectID) identity.EnvironmentID = strings.TrimSpace(identity.EnvironmentID) @@ -784,6 +845,10 @@ func (c *Client) Apply(ctx context.Context, plan Plan) error { } if resumeFrom > 0 { if err := c.primeResumeState(ctx, actx, plan.Steps[:resumeFrom], pausedApps, handedOff, &coolifyProxyStopped); err != nil { + if requiresAuthorityRecovery(err) && len(plan.Steps) > 0 { + index := min(resumeFrom, len(plan.Steps)-1) + emitProgress(plan.OnProgress, StepProgress{Index: index, Total: total, Step: plan.Steps[index], Status: StepStatusError, Err: err, AuthorityRecoveryRequired: true}) + } resumePausedApps := pausedApps if isUnsafeSourceResumeError(err) { resumePausedApps = nil @@ -797,7 +862,7 @@ func (c *Client) Apply(ctx context.Context, plan Plan) error { started := StepProgress{Index: index, Total: total, Step: step, Status: StepStatusStarted} if plan.BeforeStep != nil { if err := (*plan.BeforeStep)(started); err != nil { - stepErr := fmt.Errorf("before dokploy step %s for %s (%s): %w", step.Kind, step.App, step.Ref, err) + stepErr := fmt.Errorf("before Dokploy step %s for %s (%s): %w", step.Kind, step.App, step.Ref, err) return errors.Join(stepErr, c.bestEffortResume(ctx, actx, plan, total, pausedApps, coolifyProxyStopped, false)) } } @@ -847,14 +912,19 @@ func (c *Client) Apply(ctx context.Context, plan Plan) error { // containers Bort owns for this app are running again, and the // cleanup resume reports at this same index, so the claim is // recorded after a successful resume. - refused := StepProgress{Index: index, Total: total, Step: step, Status: StepStatusError, Err: err, MutationAmbiguous: mutationResponseMayHaveSucceeded(err), RequiresNewRun: requiresNewRun && !sourceOwned} + refused := StepProgress{Index: index, Total: total, Step: step, Status: StepStatusError, Err: err, MutationAmbiguous: mutationResponseMayHaveSucceeded(err), AuthorityRecoveryRequired: requiresAuthorityRecovery(err), RequiresNewRun: requiresNewRun && !sourceOwned} emitProgress(plan.OnProgress, refused) resumePausedApps := pausedApps if isUnsafeSourceResumeError(err) { resumePausedApps = nil } - stepErr := fmt.Errorf("dokploy step %s for %s (%s): %w", step.Kind, step.App, step.Ref, err) + stepErr := fmt.Errorf("Dokploy step %s for %s (%s): %w", step.Kind, step.App, step.Ref, err) cleanupErr := c.bestEffortResume(ctx, actx, plan, total, resumePausedApps, coolifyProxyStopped, isUnsafeTargetResumeError(err)) + if requiresAuthorityRecovery(cleanupErr) { + refused.Err = errors.Join(err, cleanupErr) + refused.AuthorityRecoveryRequired = true + emitProgress(plan.OnProgress, refused) + } if requiresNewRun && sourceOwned && cleanupErr == nil { refused.RequiresNewRun = true emitProgress(plan.OnProgress, refused) @@ -920,6 +990,12 @@ func (p pausedSources) observeCompleted(step Step, handedOff map[string]struct{} } func (c *Client) requireTransferredSourceStillPaused(ctx context.Context, actx *applyContext, step Step, pausedApps pausedSources, handedOff map[string]struct{}) error { + if RequiresCoolifyDeploymentFence(actx.plan) && + (step.Kind == StepPauseSource || step.Kind == StepPushImage || step.Kind == StepActivateRoutes) { + if err := requireCoolifyDeploymentFence(ctx, c.dockerRunner()); err != nil { + return err + } + } switch step.Kind { case StepPushImage: if !pausedApps[step.App] { @@ -933,18 +1009,25 @@ func (c *Client) requireTransferredSourceStillPaused(ctx context.Context, actx * return fmt.Errorf("source app %s ran after its state was transferred, which invalidated the transfer; retry the live apply to pause and transfer it again", step.App) } case StepActivateRoutes: - apps := make([]string, 0, len(handedOff)) + apps := make([]string, 0, len(handedOff)+len(pausedApps)) for app := range handedOff { apps = append(apps, app) } + for app := range pausedApps { + apps = append(apps, app) + } sort.Strings(apps) ran, err := c.firstSourceRanSincePause(ctx, actx, apps) if err != nil { return fmt.Errorf("verify handed-off source apps stayed paused before activating target routes: %w", err) } - if ran != "" { + if ran == "" { + return nil + } + if _, off := handedOff[ran]; off || pausedApps[ran] { return unsafeSourceResumeError{err: staleStagedHandoffError(ran)} } + return unsafeSourceResumeError{err: fmt.Errorf("source app %s ran after its cutover pause, so its source routes would compete with the activated target routes; retry the live apply to pause it again before activating routes", ran)} } return nil } @@ -1079,10 +1162,18 @@ func (c *Client) bestEffortResume(_ context.Context, actx *applyContext, plan Pl if transferred && plan.BeforeStep != nil { if err := (*plan.BeforeStep)(started); err != nil { sourcesRestored = false - cleanupErr = errors.Join(cleanupErr, fmt.Errorf("source app %s stays stopped: its transferred state must be recorded as invalidated before its containers restart: %w", app, err)) + cleanupErr = errors.Join(cleanupErr, fmt.Errorf("source app %s remains stopped because its transferred state must be recorded as invalidated before its containers restart; follow `%s` for the required recovery: %w", app, recoveryStatusCommand(plan), err)) continue } } + pinCtx, cancelPin := context.WithTimeout(context.Background(), dockerStopTimeout) + pinErr := validateCachedStagingVolumePin(pinCtx, c.dockerRunner(), actx, app) + cancelPin() + if pinErr != nil { + sourcesRestored = false + cleanupErr = errors.Join(cleanupErr, fmt.Errorf("source app %s remains stopped because its staging-volume pin could not be verified before restart; follow `%s` for the required recovery: %w", app, recoveryStatusCommand(plan), pinErr)) + continue + } emitProgress(plan.OnProgress, started) resumeCtx, cancel := context.WithTimeout(context.Background(), dockerStartTimeout) err := c.applyResumeSource(resumeCtx, actx, resumeStep) @@ -1090,7 +1181,7 @@ func (c *Client) bestEffortResume(_ context.Context, actx *applyContext, plan Pl if err != nil { sourcesRestored = false emitProgress(plan.OnProgress, StepProgress{Index: index, Total: total, Step: resumeStep, Status: StepStatusError, Err: err}) - cleanupErr = errors.Join(cleanupErr, fmt.Errorf("resume source app %s after failed apply: %w", app, err)) + cleanupErr = errors.Join(cleanupErr, fmt.Errorf("source app %s remains stopped after failed apply; follow `%s` for the required recovery: %w", app, recoveryStatusCommand(plan), err)) continue } emitProgress(plan.OnProgress, StepProgress{Index: index, Total: total, Step: resumeStep, Status: StepStatusOK}) @@ -1111,7 +1202,7 @@ func (c *Client) bestEffortResume(_ context.Context, actx *applyContext, plan Pl } emitProgress(plan.OnProgress, StepProgress{Index: total, Total: total, Step: resumeProxyStep, Status: StepStatusOK}) } else if coolifyProxyStopped { - cleanupErr = errors.Join(cleanupErr, fmt.Errorf("coolify proxy remains stopped because at least one source app could not be resumed")) + cleanupErr = errors.Join(cleanupErr, fmt.Errorf("coolify proxy and at least one paused source application remain stopped; follow `%s` for the required recovery", recoveryStatusCommand(plan))) } return cleanupErr } @@ -1186,6 +1277,11 @@ func isUnsafeSourceResumeError(err error) bool { return errors.As(err, &unsafe) } +func requiresAuthorityRecovery(err error) bool { + var required authorityRecoveryRequiredError + return errors.As(err, &required) +} + func (c *Client) applyStep(ctx context.Context, actx *applyContext, step Step) error { switch step.Kind { case StepCreateProject: @@ -1359,10 +1455,78 @@ func (c *Client) applyPushImage(ctx context.Context, actx *applyContext, step St if err != nil { return err } - if err := c.deployComposeForApply(ctx, actx, step.App, composeFile, envContent); err != nil { + staged := stagedVolumesForApp(actx.plan, step.App) + if len(staged) == 0 { + if err := c.deployComposeForApply(ctx, actx, step.App, composeFile, envContent); err != nil { + return err + } + return c.pauseTargetWritersForState(ctx, c.dockerRunner(), actx, step.App) + } + runner := c.dockerRunner() + staged, pin, err := ensureAppStagingVolumePin(ctx, runner, actx, step.App, false) + if err != nil { return err } - return c.pauseTargetWritersForState(ctx, c.dockerRunner(), actx, step.App) + if err := requireStagingVolumeAttachments(ctx, runner, staged, []string{pin.containerID}); err != nil { + return failedStagingVolumeHandoff(runner, actx.plan, step.App, staged, pin, "a previous target attachment may have survived an interrupted deployment", err) + } + deployErr := c.deployComposeForApply(ctx, actx, step.App, composeFile, envContent) + if deployErr != nil { + var completed completedDeploymentHandoffError + if errors.As(deployErr, &completed) || isUnsafeSourceResumeError(deployErr) { + return failedStagingVolumeHandoff(runner, actx.plan, step.App, staged, pin, "target deployment completed or may still be active", deployErr) + } + if isUnsafeTargetResumeError(deployErr) { + cleanupCtx, cancelCleanup := context.WithTimeout(context.Background(), targetDiscoveryTimeout) + detachErr := c.removeStoppedTargetAttachments(cleanupCtx, actx, step.App, staged, pin.containerID) + cancelCleanup() + if detachErr != nil { + return failedStagingVolumeHandoff(runner, actx.plan, step.App, staged, pin, "failed to release stopped target containers before source recovery", errors.Join(deployErr, detachErr)) + } + } else if err := requireStagingVolumeAttachments(ctx, runner, staged, []string{pin.containerID}); err != nil { + return failedStagingVolumeHandoff(runner, actx.plan, step.App, staged, pin, "the failed deployment left an unverified target attachment", errors.Join(deployErr, err)) + } + return deployErr + } + if err := requireStagingVolumesOwned(ctx, runner, actx.plan, step.App, staged); err != nil { + return failedStagingVolumeHandoff(runner, actx.plan, step.App, staged, pin, "target deployment completed, but staging-volume ownership changed", err) + } + if err := requireStagingVolumePin(ctx, runner, actx.plan, staged, pin); err != nil { + return failedStagingVolumeHandoff(runner, actx.plan, step.App, staged, pin, "target deployment completed, but its volume pin changed", err) + } + targetIDsByVolume, err := c.migratedVolumeAttachmentIDs(ctx, actx, step.App) + if err != nil { + return failedStagingVolumeHandoff(runner, actx.plan, step.App, staged, pin, "target deployment completed, but target attachment validation failed", err) + } + allowedIDsByVolume := make(map[string][]string, len(staged)) + for _, volume := range staged { + targetIDs := targetIDsByVolume[volume.VolumeName] + if len(targetIDs) == 0 { + return failedStagingVolumeHandoff(runner, actx.plan, step.App, staged, pin, "target deployment completed, but the final attachment set was incomplete", fmt.Errorf("migrated volume %s has no target container", volume.VolumeName)) + } + allowedIDsByVolume[volume.VolumeName] = append([]string{pin.containerID}, targetIDs...) + } + if err := requireStagingVolumeAttachmentSets(ctx, runner, staged, allowedIDsByVolume); err != nil { + return failedStagingVolumeHandoff(runner, actx.plan, step.App, staged, pin, "target deployment completed, but the final attachment set was not exact", err) + } + if err := releaseStagingVolumePin(ctx, runner, actx.plan, staged, pin); err != nil { + return failedStagingVolumeHandoff(runner, actx.plan, step.App, staged, pin, "target deployment completed, but Bort could not release its staging-volume pin", err) + } + delete(actx.stagingVolumePins, step.App) + return c.pauseTargetWritersForState(ctx, runner, actx, step.App) +} + +func failedStagingVolumeHandoff(runner dockerRunner, plan Plan, appName string, volumes []stagedVolume, pin stagingVolumePin, reason string, err error) error { + ctx, cancel := context.WithTimeout(context.Background(), dockerStopTimeout) + defer cancel() + guardErr := requireStagingVolumesOwned(ctx, runner, plan, appName, volumes) + if guardErr == nil { + guardErr = requireStagingVolumePin(ctx, runner, plan, volumes, pin) + } + if guardErr == nil { + return ambiguousMutationResponseError{err: unsafeSourceResumeError{err: fmt.Errorf("%s; paused source applications remain stopped and Bort verified and preserved read-only staging-volume pin %s; follow `%s` for the required recovery: %w", reason, pin.name, recoveryStatusCommand(plan), err)}} + } + return ambiguousMutationResponseError{err: unsafeSourceResumeError{err: fmt.Errorf("%s; paused source applications remain stopped, but Bort could not verify the read-only staging-volume guard for pin %s; follow `%s` for the required recovery: %w", reason, pin.name, recoveryStatusCommand(plan), errors.Join(err, guardErr))}} } func (c *Client) updateComposeWithPatchGuard(ctx context.Context, composeID, composeFile, envContent string) error { @@ -1417,7 +1581,10 @@ func (c *Client) deployComposeForApply(ctx context.Context, actx *applyContext, err = deployedComposeGuardError{err: errors.Join(responseErr, guardErr)} } } else { - return c.validateMigratedVolumeMountsAfterDeploy(ctx, actx, appName) + if err := c.validateMigratedVolumeMountsAfterDeploy(ctx, actx, appName); err != nil { + return completedDeploymentHandoffError{err: err} + } + return nil } } var deployedGuardErr deployedComposeGuardError @@ -1441,7 +1608,7 @@ func (c *Client) deployComposeForApply(ctx context.Context, actx *applyContext, defer cancelStop() stopErr := c.quiesceGuardedDeployment(stopCtx, actx, appName, title) if stopErr != nil { - unsafeErr := unsafeSourceResumeError{err: unsafeTargetResumeError{err: fmt.Errorf("%s: %w; quiescence could not be proved, so Bort will leave any paused source applications stopped and skip target recovery: %v", failure, err, stopErr)}} + unsafeErr := unsafeSourceResumeError{err: unsafeTargetResumeError{err: fmt.Errorf("%s: %w; quiescence could not be proved, so paused source applications remain stopped and Bort skipped target recovery; follow `%s` for the required recovery: %v", failure, err, recoveryStatusCommand(actx.plan), stopErr)}} return ambiguousMutationResponseError{err: unsafeErr} } safetyErr := c.validateMigratedVolumeMountsAfterDeploy(ctx, actx, appName) @@ -1715,9 +1882,10 @@ func (c *Client) applyInstallGateway(ctx context.Context, actx *applyContext, st if err != nil { return err } - route, err = resolveRouteForCompose(route, composeFile) + sourceServices, composeSource := routeServiceContextForApp(actx.plan, step.App) + route, err = resolveRouteForCompose(route, composeFile, sourceServices, composeSource) if err != nil { - return err + return withRecoveryStatusCommand(err, actx.plan) } return c.ensureRouteDomain(ctx, entry.ComposeID, route) } @@ -1741,10 +1909,11 @@ func (c *Client) applyActivateRoutes(ctx context.Context, actx *applyContext, st if err := c.validateNoActiveBortOverrides(ctx, entry.ComposeID); err != nil { return err } + sourceServices, composeSource := routeServiceContextForApp(actx.plan, step.App) for _, route := range cutoverRoutesForApp(actx.plan.Cutover, step.App) { - resolved, err := resolveRouteForCompose(route, composeFile) + resolved, err := resolveRouteForCompose(route, composeFile, sourceServices, composeSource) if err != nil { - return err + return withRecoveryStatusCommand(err, actx.plan) } if err := c.ensureRouteDomain(ctx, entry.ComposeID, resolved); err != nil { return err @@ -1809,32 +1978,64 @@ type composeServiceSummary struct { Ports map[string]struct{} } -func resolveRouteForCompose(route gateway.Route, composeFile string) (gateway.Route, error) { +func routeServiceContextForApp(plan Plan, appName string) ([]preparer.SourceServiceRef, string) { + app, ok := findPrepareApp(plan.Prepare, appName) + if !ok { + return nil, "" + } + if app.Resources.App.ComposeSource != "" { + return nil, app.Resources.App.ComposeSource + } + return app.Resources.SourceServices, app.Resources.App.ComposeSource +} + +func withRecoveryStatusCommand(err error, plan Plan) error { + if !requiresAuthorityRecovery(err) { + return err + } + return authorityRecoveryRequiredError{err: fmt.Errorf("%w; run `%s` for the exact recovery commands", err, recoveryStatusCommand(plan))} +} + +func resolveRouteForCompose(route gateway.Route, composeFile string, sourceServices []preparer.SourceServiceRef, composeSource string) (gateway.Route, error) { services, err := composeServiceSummaries(composeFile) if err != nil { return gateway.Route{}, err } - if len(services) == 0 { - return route, nil + switch composeSource { + case preparer.ComposeSourceRaw, preparer.ComposeSourceGenerated: + serviceName := strings.TrimSpace(route.ServiceName) + if _, ok := services[serviceName]; ok { + route.ServiceName = serviceName + return route, nil + } + case "": + default: + return gateway.Route{}, fmt.Errorf("route %s has unsupported compose source %q", planutilFallback(route.Host, "unknown"), composeSource) } - if _, ok := services[route.ServiceName]; ok { + if len(services) == 0 && composeSource == "" { return route, nil } - if serviceName, ok := inferComposeServiceForRoute(route, services); ok { - route.ServiceName = serviceName - return route, nil + if composeSource == "" { + if serviceName, ok := inferComposeServiceForRoute(route, services, sourceServices); ok { + route.ServiceName = serviceName + return route, nil + } } available := make([]string, 0, len(services)) for name := range services { available = append(available, name) } sort.Strings(available) - return gateway.Route{}, fmt.Errorf("route %s points at service %q, but the dokploy compose has %s; rescan before retrying so bort can refresh the route mapping", - planutilFallback(route.Host, "unknown"), route.ServiceName, strings.Join(available, ", ")) + availableServices := strings.Join(available, ", ") + if availableServices == "" { + availableServices = "no services" + } + return gateway.Route{}, authorityRecoveryRequiredError{err: fmt.Errorf("route %s points at service %q, but the Dokploy Compose file has %s; no retry of this immutable run can pass this step, so follow the run status guidance (`bort status`) to record source or target authority with recover-authority, then correct the source configuration or the bundle used to create the next run and create a new run", + planutilFallback(route.Host, "unknown"), route.ServiceName, availableServices)} } -func inferComposeServiceForRoute(route gateway.Route, services map[string]composeServiceSummary) (string, bool) { - for _, candidate := range routeServiceNameCandidates(route) { +func inferComposeServiceForRoute(route gateway.Route, services map[string]composeServiceSummary, sourceServices []preparer.SourceServiceRef) (string, bool) { + for _, candidate := range routeServiceNameCandidates(route, sourceServices) { if _, ok := services[candidate]; ok { return candidate, true } @@ -1859,7 +2060,7 @@ func inferComposeServiceForRoute(route gateway.Route, services map[string]compos return "", false } -func routeServiceNameCandidates(route gateway.Route) []string { +func routeServiceNameCandidates(route gateway.Route, sourceServices []preparer.SourceServiceRef) []string { candidates := []string{} add := func(value string) { value = strings.TrimSpace(value) @@ -1874,6 +2075,11 @@ func routeServiceNameCandidates(route gateway.Route) []string { candidates = append(candidates, value) } add(route.ServiceName) + for _, sourceService := range sourceServices { + if sourceService.ContainerName == route.ServiceName { + add(sourceService.ServiceName) + } + } add(stripCoolifyGeneratedServiceSuffix(route.ServiceName)) if fromSource := serviceNameFromTraefikRouterSource(route.Source); fromSource != "" { add(fromSource) @@ -1956,6 +2162,10 @@ func composeServiceSummaries(contents string) (map[string]composeServiceSummary, if doc.Kind == yaml.DocumentNode && len(doc.Content) > 0 { root = doc.Content[0] } + root, err := selfContainedNode(root) + if err != nil { + return nil, err + } services := mappingValue(root, "services") if services == nil || services.Kind != yaml.MappingNode { return nil, nil diff --git a/internal/target/dokploy/apply_test.go b/internal/target/dokploy/apply_test.go index ab894cd..a192376 100644 --- a/internal/target/dokploy/apply_test.go +++ b/internal/target/dokploy/apply_test.go @@ -217,6 +217,35 @@ func TestLegacyPlanPreservesV1Alpha1RoutedStateOrder(t *testing.T) { } } +func TestLegacyPlanPreservesV1Alpha3UnroutedStateOrder(t *testing.T) { + app := preparer.AppPlan{ + Name: "api", + TargetResources: &preparer.TargetResources{Dokploy: &preparer.DokployResources{ + ComposeApp: preparer.DokployComposeApp{Name: "api"}, + }}, + } + app.Resources.Volumes = []preparer.VolumeResource{{Service: "web", Type: "volume", Name: "data", Target: "/data"}} + prepare := preparer.Result{Apps: []preparer.AppPlan{app}} + sync := syncplan.Result{Apps: []syncplan.AppPlan{{Name: "api", Steps: []syncplan.Step{{ + ResourceType: "volume", + ResourceRef: "volume:web -> /data", + Strategy: syncplan.StrategyDockerVolumeArchive, + }}}}} + legacy := LegacyPlanFromArtifactsV1Alpha3(prepare, sync, gateway.Result{}) + current := PlanFromArtifacts(prepare, sync, gateway.Result{}) + + legacyKinds := stepKinds(legacy.Steps) + syncIndex := slices.Index(legacyKinds, StepSyncVolume) + pushIndex := slices.Index(legacyKinds, StepPushImage) + resumeIndex := slices.Index(legacyKinds, StepResumeSource) + if syncIndex < 0 || resumeIndex != syncIndex+1 || pushIndex != resumeIndex+1 { + t.Fatalf("legacy v1alpha3 order changed: %v", legacyKinds) + } + if slices.Contains(stepKinds(current.Steps), StepResumeSource) { + t.Fatalf("current staged plan resumed an unrouted source writer: %v", stepKinds(current.Steps)) + } +} + func TestNewClientFromEnvRejectsRemoteHTTP(t *testing.T) { t.Setenv(EnvBaseURL, "http://dokploy.example") t.Setenv(EnvToken, "secret") @@ -1391,7 +1420,7 @@ func TestApplyLeavesPausedSourceStoppedWhenGuardedDeploymentCannotBeQuiesced(t * }, Prepare: preparer.Result{BundleDir: bundleDir, Apps: []preparer.AppPlan{app}}, }) - if err == nil || !strings.Contains(err.Error(), "leave any paused source applications stopped") { + if err == nil || !strings.Contains(err.Error(), "paused source applications remain stopped") || !strings.Contains(err.Error(), "bort status") { t.Fatalf("expected fail-closed guarded deployment error, got %v", err) } if !mutationResponseMayHaveSucceeded(err) { @@ -2408,3 +2437,37 @@ func TestApplyActivateRoutesDetectsMigratedVolumeDriftAfterDeploy(t *testing.T) t.Fatalf("expected drifted target container to stop, calls=%#v", runner.outputArgs) } } + +func TestResolveRouteForComposeExpandsServiceAliases(t *testing.T) { + for name, compose := range map[string]string{ + "direct alias": "x-services: &app-services\n web:\n image: example/api\n expose:\n - \"8080\"\nservices: *app-services\n", + "merge key": "x-services: &app-services\n web:\n image: example/api\n expose:\n - \"8080\"\nservices:\n <<: *app-services\n", + } { + t.Run(name, func(t *testing.T) { + route, err := resolveRouteForCompose(gateway.Route{Host: "api.example.com", ServiceName: "web", Port: "8080"}, compose, nil, preparer.ComposeSourceRaw) + if err != nil || route.ServiceName != "web" { + t.Fatalf("aliased Compose service was not resolved at apply time: route=%#v err=%v", route, err) + } + }) + } +} + +func TestComposeServiceSummariesExpandsSharedAliasChainsWithoutExponentialCost(t *testing.T) { + var compose strings.Builder + compose.WriteString("x-base: &level0\n image: example/api\n expose:\n - \"8080\"\n") + for i := 1; i < 30; i++ { + fmt.Fprintf(&compose, "x-level%d: &level%d\n <<: *level%d\n back: *level%d\n", i, i, i-1, i-1) + } + compose.WriteString("services:\n api:\n <<: *level29\n") + summaries, err := composeServiceSummaries(compose.String()) + if err != nil { + t.Fatalf("composeServiceSummaries: %v", err) + } + summary, ok := summaries["api"] + if !ok { + t.Fatalf("expected an api service summary, got %#v", summaries) + } + if _, ok := summary.Ports["8080"]; !ok { + t.Fatalf("port exposed through 30 levels of shared anchors was lost: %#v", summary.Ports) + } +} diff --git a/internal/target/dokploy/cleanup.go b/internal/target/dokploy/cleanup.go index 98ddd2d..bb2479f 100644 --- a/internal/target/dokploy/cleanup.go +++ b/internal/target/dokploy/cleanup.go @@ -773,7 +773,7 @@ func purgeSourceNetwork(ctx context.Context, runner dockerRunner, item SourcePur } result.Identity = identity if _, err := runner.Output(ctx, "network", "rm", identity); err != nil { - if isDockerVolumeOrNetworkMissingErr(err) { + if isDockerResourceMissingErr(err, "network", identity) { _, absent, inspectErr := inspectSourcePurgeNetworkIdentity(ctx, runner, name) if inspectErr != nil { result.Status = "error" @@ -856,7 +856,7 @@ type sourcePurgeNetworkIdentity struct { func inspectSourcePurgeVolumeAbsent(ctx context.Context, runner dockerRunner, name string) (bool, error) { out, err := runner.Output(ctx, "volume", "inspect", name) if err != nil { - if isDockerVolumeOrNetworkMissingErr(err) { + if isDockerResourceMissingErr(err, "volume", name) { return true, nil } return false, err @@ -874,7 +874,7 @@ func inspectSourcePurgeVolumeAbsent(ctx context.Context, runner dockerRunner, na func inspectSourcePurgeNetworkIdentity(ctx context.Context, runner dockerRunner, name string) (string, bool, error) { out, err := runner.Output(ctx, "network", "inspect", name) if err != nil { - if isDockerVolumeOrNetworkMissingErr(err) { + if isDockerResourceMissingErr(err, "network", name) { return "", true, nil } return "", false, err @@ -1031,14 +1031,54 @@ func firstNonEmpty(values ...string) string { return "" } -func isDockerVolumeOrNetworkMissingErr(err error) bool { +func asciiLower(s string) string { + b := []byte(s) + for i, c := range b { + if 'A' <= c && c <= 'Z' { + b[i] = c + ('a' - 'A') + } + } + return string(b) +} + +func isDockerResourceMissingErr(err error, resource, name string) bool { if err == nil { return false } - message := strings.ToLower(err.Error()) - return strings.Contains(message, "no such volume") || - strings.Contains(message, "no such network") || - (strings.Contains(message, "error response from daemon: network ") && strings.Contains(message, " not found")) + message := err.Error() + lower := asciiLower(message) + name = strings.TrimSpace(name) + legacyPrefix := "error: no such " + resource + ": " + if index := strings.LastIndex(lower, legacyPrefix); index >= 0 && dockerMessageNamesResource(strings.TrimSpace(message[index:]), legacyPrefix, name, "") { + return resource == "volume" || resource == "network" + } + marker := "error response from daemon: " + index := strings.LastIndex(lower, marker) + if index < 0 { + return false + } + detail := strings.TrimSpace(message[index+len(marker):]) + switch resource { + case "volume": + return dockerMessageNamesResource(detail, "get ", name, ": no such volume") || + dockerMessageNamesResource(detail, "remove ", name, ": no such volume") || + dockerMessageNamesResource(detail, "volume ", name, " not found") + case "network": + return strings.EqualFold(detail, "network not found") || + dockerMessageNamesResource(detail, "network ", name, " not found") || + dockerMessageNamesResource(detail, "no such network: ", name, "") + default: + return false + } +} + +func dockerMessageNamesResource(message, prefix, name, suffix string) bool { + if len(message) < len(prefix)+len(name)+len(suffix) || + !strings.EqualFold(message[:len(prefix)], prefix) || + !strings.EqualFold(message[len(message)-len(suffix):], suffix) { + return false + } + return message[len(prefix):len(message)-len(suffix)] == name } func cleanupProjectNames(names []string) []string { diff --git a/internal/target/dokploy/cleanup_test.go b/internal/target/dokploy/cleanup_test.go index 8782b26..74e5a0e 100644 --- a/internal/target/dokploy/cleanup_test.go +++ b/internal/target/dokploy/cleanup_test.go @@ -961,17 +961,42 @@ func (r *cleanupDockerRunner) Output(ctx context.Context, args ...string) ([]byt return r.fakeDockerRunner.Output(ctx, args...) } -func TestDockerMissingResourceClassifierRejectsInfrastructureErrors(t *testing.T) { - if isDockerVolumeOrNetworkMissingErr(errors.New(`exec: "docker": executable file not found in $PATH`)) { - t.Fatal("missing Docker executable was classified as an absent resource") - } - for _, err := range []error{ - errors.New("Error response from daemon: remove api-data: no such volume"), - errors.New("Error response from daemon: network api-net not found"), - } { - if !isDockerVolumeOrNetworkMissingErr(err) { - t.Fatalf("expected missing Docker resource classification for %v", err) - } +func TestDockerMissingResourceClassifier(t *testing.T) { + tests := []struct { + name string + resource string + ref string + err error + want bool + }{ + {name: "legacy volume", resource: "volume", ref: "api-data", err: errors.New("Error response from daemon: get api-data: no such volume"), want: true}, + {name: "removed volume", resource: "volume", ref: "api-data", err: errors.New("Error response from daemon: remove api-data: no such volume"), want: true}, + {name: "Docker 29 volume", resource: "volume", ref: "api-data", err: errors.New("Error response from daemon: volume api-data not found"), want: true}, + {name: "wrapped Docker 29 volume", resource: "volume", ref: "api-data", err: errors.New("exit status 1: Error response from daemon: volume api-data not found"), want: true}, + {name: "Docker 20 volume", resource: "volume", ref: "api-data", err: errors.New("exit status 1: Error: No such volume: api-data"), want: true}, + {name: "named network", resource: "network", ref: "api-net", err: errors.New("Error response from daemon: network api-net not found"), want: true}, + {name: "removed network", resource: "network", ref: strings.Repeat("a", 64), err: errors.New("Error response from daemon: network not found"), want: true}, + {name: "legacy network", resource: "network", ref: "api-net", err: errors.New("Error response from daemon: no such network: api-net"), want: true}, + {name: "Docker 20 network", resource: "network", ref: "api-net", err: errors.New("exit status 1: Error: No such network: api-net"), want: true}, + {name: "missing executable", resource: "volume", ref: "api-data", err: errors.New(`exec: "docker": executable file not found in $PATH`)}, + {name: "plugin", resource: "volume", ref: "api-data", err: errors.New("Error response from daemon: plugin local not found")}, + {name: "volume plugin", resource: "volume", ref: "api-data", err: errors.New("Error response from daemon: volume api-data: error looking up volume plugin local: plugin local not found")}, + {name: "network plugin", resource: "network", ref: "api-net", err: errors.New("Error response from daemon: network api-net: error looking up network plugin local: plugin local not found")}, + {name: "wrong volume", resource: "volume", ref: "api-data", err: errors.New("Error response from daemon: volume other-data not found")}, + {name: "wrong Docker 20 volume", resource: "volume", ref: "api-data", err: errors.New("Error: No such volume: other-data")}, + {name: "case-only Docker 29 volume mismatch", resource: "volume", ref: "ProdData", err: errors.New("Error response from daemon: volume proddata not found")}, + {name: "case-only Docker 20 network mismatch", resource: "network", ref: "ApiNet", err: errors.New("Error: No such network: apinet")}, + {name: "network error during volume inspect", resource: "volume", ref: "api-data", err: errors.New("Error response from daemon: network not found")}, + {name: "volume error during network inspect", resource: "network", ref: "api-net", err: errors.New("Error response from daemon: get api-data: no such volume")}, + {name: "unknown resource", resource: "secret", ref: "api-key", err: errors.New("Error response from daemon: secret api-key not found")}, + {name: "invalid UTF-8 before the daemon marker", resource: "volume", ref: "api-data", err: errors.New(strings.Repeat("\xff", 40) + " Error response from daemon: volume api-data not found"), want: true}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := isDockerResourceMissingErr(tc.err, tc.resource, tc.ref); got != tc.want { + t.Fatalf("isDockerResourceMissingErr(%q, %q, %v) = %t, want %t", tc.resource, tc.ref, tc.err, got, tc.want) + } + }) } } diff --git a/internal/target/dokploy/datamove.go b/internal/target/dokploy/datamove.go index 8c837dd..6a790ab 100644 --- a/internal/target/dokploy/datamove.go +++ b/internal/target/dokploy/datamove.go @@ -77,17 +77,23 @@ func (c *Client) applySyncVolume(ctx context.Context, actx *applyContext, step S // source container set is inspected before and after the copy and any // restart in between invalidates the transfer. func (c *Client) syncVolumeToStaging(ctx context.Context, runner dockerRunner, actx *applyContext, appName string, app preparer.AppPlan, staged stagedVolume) error { + entry := actx.entry(appName) + allowPinCreate := !stagingTransferStarted(entry) if err := actx.forgetMigratedVolumeMount(appName, staged.Service, staged.Target); err != nil { return err } - if err := ensureStagingVolume(ctx, runner, actx.plan, appName, staged); err != nil { + srcVolName, err := resolveSourceVolume(ctx, runner, staged.Source) + if err != nil { return err } - srcVolName, err := resolveSourceVolume(ctx, runner, staged.Source) + allStaged, pin, err := ensureAppStagingVolumePin(ctx, runner, actx, appName, allowPinCreate) if err != nil { return err } - if err := requireStagingVolumeUnattached(ctx, runner, staged); err != nil { + if err := requireStagingVolumeAttachments(ctx, runner, allStaged, []string{pin.containerID}); err != nil { + return err + } + if err := actx.markStagingTransferStarted(appName); err != nil { return err } before, err := inspectSourceQuiesceTargets(ctx, runner, app) @@ -107,7 +113,13 @@ func (c *Client) syncVolumeToStaging(ctx context.Context, runner dockerRunner, a if err := requireSourceQuiesceUnchanged(before, after); err != nil { return err } - if err := requireStagingVolumeUnattached(ctx, runner, staged); err != nil { + if err := requireStagingVolumesOwned(ctx, runner, actx.plan, appName, allStaged); err != nil { + return err + } + if err := requireStagingVolumePin(ctx, runner, actx.plan, allStaged, pin); err != nil { + return err + } + if err := requireStagingVolumeAttachments(ctx, runner, allStaged, []string{pin.containerID}); err != nil { return err } return actx.recordMigratedVolumeMount(appName, migratedVolumeMount{ @@ -207,11 +219,15 @@ func migratedMountKey(service, target string) string { const migratedVolumeMountsArtifact = "migrated-volumes.json" type migratedVolumeMountsState struct { - APIVersion string `json:"apiVersion"` - Apps map[string][]migratedVolumeMount `json:"apps,omitempty"` + APIVersion string `json:"apiVersion"` + Apps map[string][]migratedVolumeMount `json:"apps,omitempty"` + StartedApps []string `json:"startedApps,omitempty"` } -const migratedVolumeMountsAPIVersion = "bort.migrated-volumes/v1alpha1" +const ( + migratedVolumeMountsLegacyAPIVersion = "bort.migrated-volumes/v1alpha1" + migratedVolumeMountsAPIVersion = "bort.migrated-volumes/v1alpha2" +) func (a *applyContext) persistMigratedVolumeMounts() error { if a == nil || strings.TrimSpace(a.plan.RunDir) == "" { @@ -226,6 +242,9 @@ func (a *applyContext) persistMigratedVolumeMounts() error { Apps: map[string][]migratedVolumeMount{}, } for app, entry := range a.cache { + if stagingTransferStarted(entry) { + state.StartedApps = append(state.StartedApps, app) + } if len(entry.MigratedVolumeMounts) == 0 { continue } @@ -238,6 +257,7 @@ func (a *applyContext) persistMigratedVolumeMounts() error { }) state.Apps[app] = mounts } + sort.Strings(state.StartedApps) if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { return fmt.Errorf("prepare migrated volume state dir: %w", err) } @@ -271,11 +291,19 @@ func (a *applyContext) loadMigratedVolumeMounts() error { if err := json.Unmarshal(contents, &state); err != nil { return fmt.Errorf("decode migrated volume state: %w", err) } - if state.APIVersion != "" && state.APIVersion != migratedVolumeMountsAPIVersion { + if state.APIVersion != "" && state.APIVersion != migratedVolumeMountsLegacyAPIVersion && state.APIVersion != migratedVolumeMountsAPIVersion { return fmt.Errorf("%s has unsupported apiVersion %q (want %q)", path, state.APIVersion, migratedVolumeMountsAPIVersion) } + for _, app := range state.StartedApps { + if strings.TrimSpace(app) != "" { + a.entry(app).StagingTransferStarted = true + } + } for app, mounts := range state.Apps { entry := a.entry(app) + if len(mounts) > 0 { + entry.StagingTransferStarted = true + } if entry.MigratedVolumeMounts == nil { entry.MigratedVolumeMounts = map[string]migratedVolumeMount{} } @@ -289,6 +317,23 @@ func (a *applyContext) loadMigratedVolumeMounts() error { return nil } +func stagingTransferStarted(entry *appCache) bool { + return entry != nil && (entry.StagingTransferStarted || len(entry.MigratedVolumeMounts) > 0) +} + +func (a *applyContext) markStagingTransferStarted(appName string) error { + entry := a.entry(appName) + if entry.StagingTransferStarted { + return nil + } + entry.StagingTransferStarted = true + if err := a.persistMigratedVolumeMounts(); err != nil { + entry.StagingTransferStarted = false + return fmt.Errorf("record staging transfer start for app %s: %w", appName, err) + } + return nil +} + func migratedVolumeMountsPath(runDir string) (string, error) { path := filepath.Join(runDir, migratedVolumeMountsArtifact) if err := safepath.ContainedPath(runDir, path); err != nil { @@ -342,7 +387,11 @@ func appHasStateWork(plan Plan, appName string) bool { return false } for _, step := range plan.Steps { - if step.App == appName && step.Kind == StepPauseSource { + if step.App != appName || shouldSkipApplyStep(plan, step) { + continue + } + switch step.Kind { + case StepDumpDataStore, StepRestoreDataStore, StepSyncVolume: return true } } @@ -544,7 +593,7 @@ func (c *Client) applyResumeTarget(ctx context.Context, actx *applyContext, step stopCtx, cancelStop := context.WithTimeout(context.Background(), targetDiscoveryTimeout) defer cancelStop() if stopErr := c.stopTargetComposeContainers(stopCtx, actx, step.App); stopErr != nil { - return ambiguousMutationResponseError{err: unsafeSourceResumeError{err: fmt.Errorf("%w (also failed to stop unsafe target containers, so Bort will leave any paused source applications stopped: %v)", err, stopErr)}} + return ambiguousMutationResponseError{err: unsafeSourceResumeError{err: fmt.Errorf("%w (also failed to stop unsafe target containers, so paused source applications remain stopped; follow `%s` for the required recovery: %v)", err, recoveryStatusCommand(actx.plan), stopErr)}} } } return err @@ -574,7 +623,7 @@ func (c *Client) validateMigratedVolumeMountsAfterDeploy(ctx context.Context, ac stopCtx, cancelStop := context.WithDeadline(context.Background(), operationDeadline) defer cancelStop() if stopErr := c.stopTargetComposeContainers(stopCtx, actx, appName); stopErr != nil { - return ambiguousMutationResponseError{err: unsafeSourceResumeError{err: fmt.Errorf("%w (also failed to stop unsafe target containers, so Bort will leave any paused source applications stopped: %v)", err, stopErr)}} + return ambiguousMutationResponseError{err: unsafeSourceResumeError{err: fmt.Errorf("%w (also failed to stop unsafe target containers, so paused source applications remain stopped; follow `%s` for the required recovery: %v)", err, recoveryStatusCommand(actx.plan), stopErr)}} } } return err @@ -615,28 +664,116 @@ func (c *Client) validateMigratedVolumeMountsStable(ctx context.Context, actx *a } func (c *Client) validateMigratedVolumeMountsSnapshot(ctx context.Context, actx *applyContext, appName string) (string, error) { + signature, _, err := c.validatedMigratedVolumeMountsSnapshot(ctx, actx, appName) + return signature, err +} + +func (c *Client) migratedVolumeAttachmentIDs(ctx context.Context, actx *applyContext, appName string) (map[string][]string, error) { + _, idsByVolume, err := c.validatedMigratedVolumeMountsSnapshot(ctx, actx, appName) + return idsByVolume, err +} + +func (c *Client) validatedMigratedVolumeMountsSnapshot(ctx context.Context, actx *applyContext, appName string) (string, map[string][]string, error) { entry := actx.entry(appName) if len(entry.MigratedVolumeMounts) == 0 { - return "", nil + return "", map[string][]string{}, nil + } + services := map[string]struct{}{} + for _, expected := range entry.MigratedVolumeMounts { + services[expected.Service] = struct{}{} + } + containers, err := c.migratedServiceContainers(ctx, actx, appName, services) + if err != nil { + return "", nil, unsafeTargetResumeError{err: err} } - runner := c.dockerRunner() parts := make([]string, 0, len(entry.MigratedVolumeMounts)) + containerIDsByVolume := map[string]map[string]struct{}{} for _, expected := range entry.MigratedVolumeMounts { - container, err := c.targetContainerForServiceNoRedeploy(ctx, runner, actx, appName, expected.Service) - if err != nil { - return "", unsafeTargetResumeError{err: err} - } + container := containers[expected.Service] mount, ok := findMountByTarget(container, expected.Target) if !ok || mount.Type != "volume" || mount.Name == "" { - return "", unsafeTargetResumeError{err: fmt.Errorf("target service %s no longer has migrated volume mounted at %s", expected.Service, expected.Target)} + return "", nil, unsafeTargetResumeError{err: fmt.Errorf("target service %s no longer has migrated volume mounted at %s", expected.Service, expected.Target)} } if mount.Name != expected.VolumeName { - return "", unsafeTargetResumeError{err: fmt.Errorf("target service %s volume mount %s changed from migrated volume %s to %s; refusing to accept a deploy that may be using fresh state", expected.Service, expected.Target, expected.VolumeName, mount.Name)} + return "", nil, unsafeTargetResumeError{err: fmt.Errorf("target service %s volume mount %s changed from migrated volume %s to %s; refusing to accept a deploy that may be using fresh state", expected.Service, expected.Target, expected.VolumeName, mount.Name)} } parts = append(parts, migratedMountKey(expected.Service, expected.Target)+"\x00"+container.ID+"\x00"+mount.Name) + if containerIDsByVolume[mount.Name] == nil { + containerIDsByVolume[mount.Name] = map[string]struct{}{} + } + containerIDsByVolume[mount.Name][container.ID] = struct{}{} } sort.Strings(parts) - return strings.Join(parts, "\x1e"), nil + idsByVolume := make(map[string][]string, len(containerIDsByVolume)) + for volumeName, containerIDs := range containerIDsByVolume { + ids := make([]string, 0, len(containerIDs)) + for id := range containerIDs { + ids = append(ids, id) + } + sort.Strings(ids) + idsByVolume[volumeName] = ids + } + return strings.Join(parts, "\x1e"), idsByVolume, nil +} + +func (c *Client) migratedServiceContainers(ctx context.Context, actx *applyContext, appName string, services map[string]struct{}) (map[string]dockerContainer, error) { + entry := actx.entry(appName) + if entry.ComposeAppName == "" { + if err := c.refreshComposeAppName(ctx, entry); err != nil { + return nil, err + } + } + deadline := time.Now().Add(targetDiscoveryTimeout) + for { + containers, err := listContainersByLabel(ctx, c.dockerRunner(), "com.docker.compose.project="+entry.ComposeAppName) + if err != nil { + return nil, err + } + matchesByService := make(map[string][]dockerContainer, len(services)) + for _, container := range containers { + service := container.Config.Labels[composeServiceLabel] + if _, wanted := services[service]; wanted { + matchesByService[service] = append(matchesByService[service], container) + } + } + selected := make(map[string]dockerContainer, len(services)) + missing := false + for service := range services { + matches := matchesByService[service] + var running []dockerContainer + for _, container := range matches { + if container.State.Running { + running = append(running, container) + } + } + switch { + case len(running) > 1: + return nil, fmt.Errorf("target compose project %s has %d running containers for migrated service %s; refusing to validate only one writer", entry.ComposeAppName, len(running), service) + case len(running) == 1: + selected[service] = running[0] + case len(matches) == 1: + selected[service] = matches[0] + case len(matches) > 1: + return nil, fmt.Errorf("target compose project %s has %d stopped containers for migrated service %s; refusing ambiguous target identity", entry.ComposeAppName, len(matches), service) + default: + missing = true + } + } + if !missing { + return selected, nil + } + if err := c.composeDeploymentError(ctx, entry); err != nil { + return nil, err + } + if time.Now().After(deadline) { + return nil, fmt.Errorf("migrated target services not found in dokploy compose project %q after %s", entry.ComposeAppName, targetDiscoveryTimeout) + } + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(targetDiscoveryDelay): + } + } } type unsafeTargetResumeError struct { @@ -711,6 +848,68 @@ func (c *Client) stopTargetComposeContainers(ctx context.Context, actx *applyCon } } +func (c *Client) removeStoppedTargetAttachments(ctx context.Context, actx *applyContext, appName string, volumes []stagedVolume, pinID string) error { + attached := map[string]struct{}{} + if err := requireStagingVolumesOwned(ctx, c.dockerRunner(), actx.plan, appName, volumes); err != nil { + return err + } + attachedByVolume, err := stagingVolumeAttachmentSets(ctx, c.dockerRunner(), volumes) + if err != nil { + return err + } + for _, ids := range attachedByVolume { + for _, id := range ids { + if stagingContainerIDsMatch(id, pinID) { + continue + } + attached[id] = struct{}{} + } + } + if len(attached) == 0 { + return nil + } + entry := actx.entry(appName) + if entry.ComposeAppName == "" { + if err := c.refreshComposeAppName(ctx, entry); err != nil { + return err + } + } + containers, err := listContainersByLabel(ctx, c.dockerRunner(), "com.docker.compose.project="+entry.ComposeAppName) + if err != nil { + return err + } + remove := map[string]dockerContainer{} + for attachedID := range attached { + matched := false + for _, container := range containers { + if !sourceContainerIDMatches(attachedID, container.ID) { + continue + } + if container.State.Running { + return fmt.Errorf("target container %s for app %s restarted while Bort was releasing a failed deployment; refusing to resume the source", container.Name, appName) + } + remove[container.ID] = container + matched = true + break + } + if !matched { + return fmt.Errorf("staging volume for app %s is attached to container %s outside Dokploy Compose project %s", appName, attachedID, entry.ComposeAppName) + } + } + for _, container := range remove { + if _, err := c.dockerRunner().Output(ctx, "rm", container.ID); err != nil { + return fmt.Errorf("remove stopped target container %s after failed deployment: %w", container.Name, err) + } + } + if err := requireStagingVolumesOwned(ctx, c.dockerRunner(), actx.plan, appName, volumes); err != nil { + return err + } + if err := requireStagingVolumeAttachments(ctx, c.dockerRunner(), volumes, []string{pinID}); err != nil { + return err + } + return nil +} + func startStoppedTargetWriters(runner dockerRunner, containers []dockerContainer) error { var result error for _, container := range containers { @@ -831,7 +1030,7 @@ func requireStagedRestoreLayout(plan Plan, step Step, store preparer.DataStoreRe if err != nil { return nil, err } - return staged, requireLiteralMountTargets(composeFile, step.App, store.Service) + return staged, requireStagedComposeInputs(composeFile, step.App, store.Service) } func (c *Client) stagingRestoreProject(ctx context.Context, actx *applyContext, step Step, store preparer.DataStoreResource, staged []stagedVolume) (stagingProject, error) { @@ -899,12 +1098,12 @@ func (e stagedRestorePreflightError) Error() string { return e.err.Error() } func (e stagedRestorePreflightError) Unwrap() error { return e.err } -func stagedRestoreLayoutRefusal(err error, app string) error { +func stagedRestoreLayoutRefusal(err error, plan Plan, app string) error { if !errors.Is(err, ErrNotImplemented) { return err } return stagedRestorePreflightError{ - err: fmt.Errorf("%w; no retry of this run can pass pause_source for app %s because its reviewed plan is immutable: follow the recovery `bort status` shows to release this run, then choose a recreate or managed data store strategy or change the source compose and create a new run", err, app), + err: fmt.Errorf("%w; no retry of this run can pass pause_source for app %s because its reviewed plan is immutable: follow the recovery `%s` shows to release this run, then choose a recreate or managed data store strategy or change the source compose and create a new run", err, app, recoveryStatusCommand(plan)), requiresNewRun: true, } } @@ -912,7 +1111,7 @@ func stagedRestoreLayoutRefusal(err error, app string) error { func (c *Client) preflightStagedRestore(ctx context.Context, runner dockerRunner, actx *applyContext, app preparer.AppPlan, step Step, store preparer.DataStoreResource) error { staged, err := requireStagedRestoreLayout(actx.plan, step, store) if err != nil { - return stagedRestoreLayoutRefusal(err, step.App) + return stagedRestoreLayoutRefusal(err, actx.plan, step.App) } project, err := c.stagingRestoreProject(ctx, actx, step, store, staged) if err != nil { @@ -930,6 +1129,9 @@ func (c *Client) preflightStagedRestore(ctx context.Context, runner dockerRunner if err := runner.Run(ctx, nil, nil, project.args("create", "--no-build", store.Service)...); err != nil { return fmt.Errorf("create staging data store %s: %w", store.Service, err) } + if err := requireStagingVolumesOwned(ctx, runner, actx.plan, step.App, staged); err != nil { + return err + } dst, err := project.serviceContainer(ctx, runner, store.Service) if err != nil { return err @@ -939,7 +1141,7 @@ func (c *Client) preflightStagedRestore(ctx context.Context, runner dockerRunner } return requireSourceMountsStageDataDir(app, store.Service, postgresDataDir(dst), staged) }() - checkErr = stagedRestoreLayoutRefusal(checkErr, step.App) + checkErr = stagedRestoreLayoutRefusal(checkErr, actx.plan, step.App) if downErr := project.down(runner); downErr != nil { if checkErr != nil { return fmt.Errorf("%w (also failed to stop staging project: %v)", checkErr, downErr) @@ -951,9 +1153,9 @@ func (c *Client) preflightStagedRestore(ctx context.Context, runner dockerRunner // restoreDataStoreToStaging runs the data store service alone under a // Bort-owned compose project with only its staging volumes and read-only -// init-script mounts, restores the dump into it, and stops it again. the -// volumes are recreated first so a retry never restores on top of a -// partial earlier attempt. +// init-script mounts, restores the dump into it, and stops it again. each +// volume is pinned and cleared in place first so a retry cannot replace +// its identity or restore on top of a partial earlier attempt. func (c *Client) restoreDataStoreToStaging(ctx context.Context, runner dockerRunner, actx *applyContext, app preparer.AppPlan, step Step, store preparer.DataStoreResource) error { staged, err := requireStagedRestoreLayout(actx.plan, step, store) if err != nil { @@ -963,30 +1165,65 @@ func (c *Client) restoreDataStoreToStaging(ctx context.Context, runner dockerRun if err != nil { return err } + entry := actx.entry(step.App) + allowPinCreate := !stagingTransferStarted(entry) + if entry.MigratedVolumeMounts == nil { + entry.MigratedVolumeMounts = map[string]migratedVolumeMount{} + } for _, volume := range staged { - if err := actx.forgetMigratedVolumeMount(step.App, volume.Service, volume.Target); err != nil { - return err - } + delete(entry.MigratedVolumeMounts, migratedMountKey(volume.Service, volume.Target)) + } + if err := actx.persistMigratedVolumeMounts(); err != nil { + return err } if err := project.down(runner); err != nil { return err } + allStaged, pin, err := ensureAppStagingVolumePin(ctx, runner, actx, step.App, allowPinCreate) + if err != nil { + return err + } + if err := requireStagingVolumeAttachments(ctx, runner, allStaged, []string{pin.containerID}); err != nil { + return err + } + if err := actx.markStagingTransferStarted(step.App); err != nil { + return err + } for _, volume := range staged { - if err := requireStagingVolumeUnattached(ctx, runner, volume); err != nil { - return err - } - if err := recreateStagingVolume(ctx, runner, actx.plan, step.App, volume); err != nil { + if err := clearStagingVolume(ctx, runner, actx.plan, step.App, volume); err != nil { return err } } - if err := runner.Run(ctx, nil, nil, project.args("up", "-d", "--no-build", "--no-deps", store.Service)...); err != nil { - return fmt.Errorf("start staging data store %s: %w", store.Service, err) + if err := requireStagingVolumePin(ctx, runner, actx.plan, allStaged, pin); err != nil { + return err + } + if err := requireStagingVolumeAttachments(ctx, runner, allStaged, []string{pin.containerID}); err != nil { + return err + } + if startErr := runner.Run(ctx, nil, nil, project.args("up", "-d", "--no-build", "--no-deps", store.Service)...); startErr != nil { + if downErr := project.down(runner); downErr != nil { + return fmt.Errorf("start staging data store %s: %w (also failed to stop staging project: %v)", store.Service, startErr, downErr) + } + return fmt.Errorf("start staging data store %s: %w", store.Service, startErr) } restoreErr := func() error { + if err := requireStagingVolumesOwned(ctx, runner, actx.plan, step.App, staged); err != nil { + return err + } dst, err := project.serviceContainer(ctx, runner, store.Service) if err != nil { return err } + allowedIDsByVolume := make(map[string][]string, len(allStaged)) + for _, volume := range allStaged { + allowedIDsByVolume[volume.VolumeName] = []string{pin.containerID} + } + for _, volume := range staged { + allowedIDsByVolume[volume.VolumeName] = append(allowedIDsByVolume[volume.VolumeName], dst.ID) + } + if err := requireStagingVolumeAttachmentSets(ctx, runner, allStaged, allowedIDsByVolume); err != nil { + return err + } if err := requireStagedPostgresDataDir(dst, staged); err != nil { return err } @@ -1004,19 +1241,20 @@ func (c *Client) restoreDataStoreToStaging(ctx context.Context, runner dockerRun if restoreErr != nil { return restoreErr } + if err := requireStagingVolumePin(ctx, runner, actx.plan, allStaged, pin); err != nil { + return err + } + if err := requireStagingVolumeAttachments(ctx, runner, allStaged, []string{pin.containerID}); err != nil { + return err + } for _, volume := range staged { - if err := requireStagingVolumeUnattached(ctx, runner, volume); err != nil { - return err - } - if err := actx.recordMigratedVolumeMount(step.App, migratedVolumeMount{ + entry.MigratedVolumeMounts[migratedMountKey(volume.Service, volume.Target)] = migratedVolumeMount{ Service: volume.Service, Target: volume.Target, VolumeName: volume.VolumeName, - }); err != nil { - return err } } - return nil + return actx.persistMigratedVolumeMounts() } func pgRestoreIntoContainer(ctx context.Context, runner dockerRunner, plan Plan, step Step, dst dockerContainer) error { diff --git a/internal/target/dokploy/datamove_test.go b/internal/target/dokploy/datamove_test.go index 4a013ce..7b13655 100644 --- a/internal/target/dokploy/datamove_test.go +++ b/internal/target/dokploy/datamove_test.go @@ -22,13 +22,15 @@ import ( ) type fakeDockerRunner struct { - outputs map[string][]byte - outputErrs map[string]error - runOutputs map[string][]byte - outputArgs [][]string - runs []fakeDockerRun - copies []fakeDockerCopy - runErr error + outputs map[string][]byte + outputErrs map[string]error + runOutputs map[string][]byte + outputArgs [][]string + runs []fakeDockerRun + copies []fakeDockerCopy + runErr error + activePins map[string]dockerContainer + activeComposeProjects map[string][]string } type fakeDockerRun struct { @@ -43,6 +45,31 @@ type fakeDockerCopy struct { Contents string } +type detachAfterRemoveRunner struct { + fakeDockerRunner + volumeName string + targetID string + removed bool + attachmentChecks int +} + +func (r *detachAfterRemoveRunner) Output(ctx context.Context, args ...string) ([]byte, error) { + key := strings.Join(args, " ") + if key == "ps -a --filter volume="+r.volumeName+" --format {{.ID}}" { + r.outputArgs = append(r.outputArgs, append([]string{}, args...)) + r.attachmentChecks++ + if r.removed { + return []byte("pin-id\n"), nil + } + return []byte("pin-id\n" + r.targetID + "\n"), nil + } + out, err := r.fakeDockerRunner.Output(ctx, args...) + if err == nil && key == "rm "+r.targetID { + r.removed = true + } + return out, err +} + func (f *fakeDockerRunner) Output(_ context.Context, args ...string) ([]byte, error) { f.outputArgs = append(f.outputArgs, append([]string{}, args...)) if len(args) == 3 && args[0] == "cp" { @@ -57,9 +84,183 @@ func (f *fakeDockerRunner) Output(_ context.Context, args ...string) ([]byte, er if err, ok := f.outputErrs[key]; ok { return nil, err } + if len(args) >= 3 && args[0] == "volume" && args[1] == "create" { + ownerKey := "volume inspect --format {{index .Labels \"bort.run-id\"}} " + args[len(args)-1] + delete(f.outputErrs, ownerKey) + } + if len(args) > 1 && args[0] == "run" && args[1] == "-d" && slices.Contains(args, "--label") && slices.Contains(args, stagingVolumePinLabel+"=true") { + id := "pin-id" + if data, ok := f.outputs[key]; ok { + id = strings.TrimSpace(string(data)) + } + if f.activePins == nil { + f.activePins = map[string]dockerContainer{} + } + pin := dockerContainer{ID: id} + pin.State.Running = true + pin.State.Status = "running" + pin.HostConfig.RestartPolicy.Name = "unless-stopped" + for index := 1; index < len(args); index++ { + switch args[index] { + case "--name": + index++ + pin.Name = args[index] + case "--label": + index++ + name, value, _ := strings.Cut(args[index], "=") + if pin.Config.Labels == nil { + pin.Config.Labels = map[string]string{} + } + pin.Config.Labels[name] = value + case "-v": + index++ + parts := strings.Split(args[index], ":") + pin.Mounts = append(pin.Mounts, dockerMount{Type: "volume", Name: parts[0], Destination: parts[1], RW: len(parts) < 3 || parts[2] != "ro"}) + } + } + for _, existing := range f.activePins { + if existing.Name == pin.Name { + return nil, errors.New("Error response from daemon: Conflict. The container name is already in use") + } + } + f.activePins[id] = pin + return []byte(id + "\n"), nil + } + if len(args) == 2 && args[0] == "start" { + pin, ok := f.activePins[args[1]] + if ok { + pin.State.Running = true + pin.State.Status = "running" + f.activePins[args[1]] = pin + return []byte(args[1] + "\n"), nil + } + } + if len(args) == 3 && args[0] == "rm" && args[1] == "-f" { + if _, ok := f.activePins[args[2]]; ok { + delete(f.activePins, args[2]) + return []byte(args[2] + "\n"), nil + } + } + if len(args) >= 5 && args[0] == "ps" && args[1] == "-a" && args[len(args)-2] == "--format" && args[len(args)-1] == "{{.ID}}" && slices.Contains(args, "label="+stagingVolumePinLabel+"=true") { + labels := map[string]string{} + for index := 2; index+1 < len(args)-2; index += 2 { + name, value, _ := strings.Cut(strings.TrimPrefix(args[index+1], "label="), "=") + labels[name] = value + } + ids := make([]string, 0, len(f.activePins)) + for id, pin := range f.activePins { + matches := true + for name, value := range labels { + if pin.Config.Labels[name] != value { + matches = false + break + } + } + if matches { + ids = append(ids, id) + } + } + slices.Sort(ids) + return []byte(strings.Join(ids, "\n")), nil + } + if len(args) >= 4 && args[0] == "inspect" && args[1] == "--type" && args[2] == "container" { + if data, ok := f.outputs[key]; ok { + return data, nil + } + containers := make([]dockerContainer, 0, len(args)-3) + for _, id := range args[3:] { + pin, ok := f.activePins[id] + if ok { + containers = append(containers, pin) + continue + } + data, ok := f.outputs["inspect --type container "+id] + if !ok { + containers = nil + break + } + var inspected []dockerContainer + if err := json.Unmarshal(data, &inspected); err != nil || len(inspected) != 1 { + containers = nil + break + } + containers = append(containers, inspected[0]) + } + if len(containers) > 0 { + return json.Marshal(containers) + } + } + if len(args) >= 5 && args[0] == "ps" && args[1] == "-a" && args[len(args)-2] == "--format" && args[len(args)-1] == "{{.ID}}" { + volumes := map[string]struct{}{} + for index := 2; index+1 < len(args); index += 2 { + volumes[strings.TrimPrefix(args[index+1], "volume=")] = struct{}{} + } + ids := map[string]struct{}{} + for id, pin := range f.activePins { + if containerMountsAnyVolume(pin, volumes) { + ids[id] = struct{}{} + } + } + for _, projectIDs := range f.activeComposeProjects { + for _, id := range projectIDs { + data, ok := f.outputs["inspect --type container "+id] + if !ok { + continue + } + var containers []dockerContainer + if json.Unmarshal(data, &containers) == nil && len(containers) == 1 && containerMountsAnyVolume(containers[0], volumes) { + ids[id] = struct{}{} + } + } + } + for prefix, data := range f.outputs { + if strings.HasPrefix(key, prefix) { + for _, id := range strings.Fields(string(data)) { + ids[id] = struct{}{} + } + break + } + } + ordered := make([]string, 0, len(ids)) + for id := range ids { + ordered = append(ordered, id) + } + slices.Sort(ordered) + return []byte(strings.Join(ordered, "\n")), nil + } + if len(args) == 3 && args[0] == "volume" && args[1] == "inspect" { + ownerKey := "volume inspect --format {{index .Labels \"bort.run-id\"}} " + args[2] + if err, ok := f.outputErrs[ownerKey]; ok { + return nil, err + } + if owner, ok := f.outputs[ownerKey]; ok { + if data, exists := f.outputs[key]; exists && strings.Contains(string(data), `"Labels"`) { + return data, nil + } + return []byte(fmt.Sprintf(`[{"Name":%q,"Labels":{%q:%q}}]`, args[2], stagingVolumeRunIDLabel, strings.TrimSpace(string(owner)))), nil + } + } if data, ok := f.outputs[key]; ok { return data, nil } + if len(args) > 3 && args[0] == "volume" && args[1] == "inspect" { + states := []stagingVolumeState{} + for _, name := range args[2:] { + data, ok := f.outputs["volume inspect "+name] + if !ok { + return nil, errors.New("docker output not stubbed: " + key) + } + var decoded []stagingVolumeState + if err := json.Unmarshal(data, &decoded); err != nil { + return nil, err + } + states = append(states, decoded...) + } + return json.Marshal(states) + } + if key == "rm -f pin-id" { + return []byte("pin-id\n"), nil + } if key == "ps --filter label=com.docker.swarm.service.name=dokploy-postgres --filter status=running --format {{.ID}}" { for _, name := range strings.Fields(string(f.outputs["ps --format {{.Names}}"])) { if name == "dokploy-postgres" || strings.HasPrefix(name, "dokploy-postgres.") || strings.HasPrefix(name, "dokploy-postgres-") { @@ -112,6 +313,37 @@ func (f *fakeDockerRunner) Run(_ context.Context, stdin io.Reader, stdout io.Wri run.Output = string(output) } f.runs = append(f.runs, run) + if strings.Contains(key, "find /volume -mindepth 1 -delete && sync") { + volumeName := "" + for index := 0; index+1 < len(args); index++ { + if args[index] == "-v" && strings.HasSuffix(args[index+1], ":/volume") { + volumeName = strings.TrimSuffix(args[index+1], ":/volume") + break + } + } + pinned := false + for _, pin := range f.activePins { + if pin.State.Running && containerMountsAnyVolume(pin, map[string]struct{}{volumeName: {}}) { + pinned = true + break + } + } + if volumeName == "" || !pinned { + return errors.New("test rejected staging-volume clear without its running app-wide pin") + } + } + if f.runErr == nil && len(args) > 2 && args[0] == "compose" && args[1] == "-p" { + project := args[2] + switch { + case slices.Contains(args, "up"): + if f.activeComposeProjects == nil { + f.activeComposeProjects = map[string][]string{} + } + f.activeComposeProjects[project] = strings.Fields(string(f.outputs["compose -p "+project])) + case slices.Contains(args, "down"): + delete(f.activeComposeProjects, project) + } + } return f.runErr } @@ -1261,7 +1493,7 @@ networks: } } -func TestResolveRouteForComposeUsesCurrentServiceByPort(t *testing.T) { +func TestResolveRouteForComposeLegacyPlanUsesCurrentServiceByPort(t *testing.T) { compose := `services: web: image: example/web @@ -1278,7 +1510,7 @@ func TestResolveRouteForComposeUsesCurrentServiceByPort(t *testing.T) { Port: "8080", Source: "traefik.http.routers.https-0-stack-api.rule", } - resolved, err := resolveRouteForCompose(route, compose) + resolved, err := resolveRouteForCompose(route, compose, nil, "") if err != nil { t.Fatalf("resolveRouteForCompose: %v", err) } @@ -1287,7 +1519,7 @@ func TestResolveRouteForComposeUsesCurrentServiceByPort(t *testing.T) { } } -func TestResolveRouteForComposeStripsGeneratedCoolifyServiceSuffix(t *testing.T) { +func TestResolveRouteForComposeLegacyPlanStripsGeneratedCoolifyServiceSuffix(t *testing.T) { compose := `services: proxy: image: example/proxy @@ -1306,7 +1538,7 @@ func TestResolveRouteForComposeStripsGeneratedCoolifyServiceSuffix(t *testing.T) Host: "app.example.com", ServiceName: tc.name, } - resolved, err := resolveRouteForCompose(route, compose) + resolved, err := resolveRouteForCompose(route, compose, nil, "") if err != nil { t.Fatalf("resolveRouteForCompose(%s): %v", tc.name, err) } @@ -1316,6 +1548,85 @@ func TestResolveRouteForComposeStripsGeneratedCoolifyServiceSuffix(t *testing.T) } } +func TestResolveRouteForComposeKeepsReviewedService(t *testing.T) { + compose := "services:\n project-apiworker-1:\n image: example/runtime-name\n apiworker:\n image: example/worker\n" + for _, tc := range []struct { + name string + serviceName string + composeSource string + }{ + {name: "raw", serviceName: "apiworker", composeSource: preparer.ComposeSourceRaw}, + {name: "generated", serviceName: "project-apiworker-1", composeSource: preparer.ComposeSourceGenerated}, + } { + t.Run(tc.name, func(t *testing.T) { + route := gateway.Route{Host: "stateless.example.com", ServiceName: tc.serviceName} + resolved, err := resolveRouteForCompose(route, compose, nil, tc.composeSource) + if err != nil { + t.Fatalf("resolve reviewed service: %v", err) + } + if resolved.ServiceName != tc.serviceName { + t.Fatalf("reviewed service changed from %q to %q", tc.serviceName, resolved.ServiceName) + } + }) + } +} + +func TestResolveRouteForComposeRejectsReviewedServiceMismatch(t *testing.T) { + for _, tc := range []struct { + name string + serviceName string + compose string + composeSource string + }{ + {name: "raw", serviceName: "apiworker", compose: "services:\n project-apiworker-1:\n image: example/runtime-name\n", composeSource: preparer.ComposeSourceRaw}, + {name: "generated", serviceName: "project-apiworker-1", compose: "services:\n apiworker:\n image: example/worker\n", composeSource: preparer.ComposeSourceGenerated}, + } { + t.Run(tc.name, func(t *testing.T) { + route := gateway.Route{Host: "stateless.example.com", ServiceName: tc.serviceName, Port: "8080"} + tc.compose = strings.Replace(tc.compose, " image:", " expose:\n - \"8080\"\n image:", 1) + if _, err := resolveRouteForCompose(route, tc.compose, nil, tc.composeSource); err == nil || + !requiresAuthorityRecovery(err) || + !strings.Contains(err.Error(), "points at service") || + !strings.Contains(err.Error(), "run status guidance") || + !strings.Contains(err.Error(), "source configuration or the bundle used to create the next run") || + !strings.Contains(err.Error(), "create a new run") { + t.Fatalf("expected reviewed service mismatch, got %v", err) + } + }) + } +} + +func TestResolveRouteForComposeRejectsUnknownProvenance(t *testing.T) { + route := gateway.Route{Host: "stateless.example.com", ServiceName: "apiworker"} + if _, err := resolveRouteForCompose(route, "services:\n apiworker:\n image: example/worker\n", nil, "genrated"); err == nil || !strings.Contains(err.Error(), `unsupported compose source "genrated"`) { + t.Fatalf("expected unknown compose provenance rejection, got %v", err) + } +} + +func TestResolveRouteForComposeLegacyPlanUsesExactThenMappedService(t *testing.T) { + route := gateway.Route{Host: "stateless.example.com", ServiceName: "project-apiworker-1"} + sourceServices := []preparer.SourceServiceRef{{ + ServiceName: "apiworker", + ContainerName: "project-apiworker-1", + }} + compose := "services:\n project-apiworker-1:\n image: example/runtime-name\n apiworker:\n image: example/worker\n" + resolved, err := resolveRouteForCompose(route, compose, sourceServices, "") + if err != nil { + t.Fatalf("resolve exact legacy service: %v", err) + } + if resolved.ServiceName != "project-apiworker-1" { + t.Fatalf("expected exact service to win for legacy plan, got %#v", resolved) + } + + resolved, err = resolveRouteForCompose(route, "services:\n apiworker:\n image: example/worker\n", sourceServices, "") + if err != nil { + t.Fatalf("resolve mapped legacy service: %v", err) + } + if resolved.ServiceName != "apiworker" { + t.Fatalf("expected mapped service fallback for legacy plan, got %#v", resolved) + } +} + func TestTargetContainerForServiceReportsComposeDeploymentError(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path != "/api/compose.one" || r.URL.Query().Get("composeId") != "c1" { @@ -1663,6 +1974,63 @@ func TestValidateMigratedVolumeMountsDoesNotRedeployForDiscovery(t *testing.T) { } } +func TestValidateMigratedVolumeMountsRejectsTwoRunningContainersForService(t *testing.T) { + runner := &fakeDockerRunner{outputs: map[string][]byte{ + "ps -a --filter label=com.docker.compose.project=stack-1 --format {{.ID}}": []byte("old-web\nnew-web\n"), + "inspect --type container new-web old-web": []byte(`[{"Id":"new-web","Name":"/new-web","Config":{"Labels":{"com.docker.compose.service":"web","com.docker.compose.project":"stack-1"}},"State":{"Running":true,"Status":"running"},"Mounts":[{"Type":"volume","Name":"fresh-vol","Destination":"/data","RW":true}]},{"Id":"old-web","Name":"/old-web","Config":{"Labels":{"com.docker.compose.service":"web","com.docker.compose.project":"stack-1"}},"State":{"Running":true,"Status":"running"},"Mounts":[{"Type":"volume","Name":"migrated-vol","Destination":"/data","RW":true}]}]`), + }} + client := &Client{Docker: runner} + actx := &applyContext{cache: map[string]*appCache{}, plan: Plan{}} + entry := actx.entry("api") + entry.ComposeAppName = "stack-1" + entry.MigratedVolumeMounts = map[string]migratedVolumeMount{ + migratedMountKey("web", "/data"): {Service: "web", Target: "/data", VolumeName: "migrated-vol"}, + } + + err := client.validateMigratedVolumeMounts(context.Background(), actx, "api") + if err == nil || !isUnsafeTargetResumeError(err) || !strings.Contains(err.Error(), "2 running containers for migrated service web") { + t.Fatalf("expected both running replicas to block single-writer validation, got %v", err) + } +} + +func stoppedAndRunningTargetContainersFixture(stoppedVolume, runningVolume string) map[string][]byte { + return map[string][]byte{ + "ps -a --filter label=com.docker.compose.project=stack-1 --format {{.ID}}": []byte("old-web\nweb-id\n"), + "inspect --type container old-web web-id": []byte(`[{"Id":"old-web","Name":"/old-web","Config":{"Labels":{"com.docker.compose.service":"web","com.docker.compose.project":"stack-1"}},"State":{"Running":false,"Status":"exited"},"Mounts":[{"Type":"volume","Name":"` + stoppedVolume + `","Destination":"/data","RW":true}]},{"Id":"web-id","Name":"/web-id","Config":{"Labels":{"com.docker.compose.service":"web","com.docker.compose.project":"stack-1"}},"State":{"Running":true,"Status":"running"},"Mounts":[{"Type":"volume","Name":"` + runningVolume + `","Destination":"/data","RW":true}]}]`), + } +} + +func TestValidateMigratedVolumeMountsValidatesRunningReplacementNotStoppedPredecessor(t *testing.T) { + runner := &fakeDockerRunner{outputs: stoppedAndRunningTargetContainersFixture("migrated-vol", "fresh-vol")} + client := &Client{Docker: runner} + actx := &applyContext{cache: map[string]*appCache{}, plan: Plan{}} + entry := actx.entry("api") + entry.ComposeAppName = "stack-1" + entry.MigratedVolumeMounts = map[string]migratedVolumeMount{ + migratedMountKey("web", "/data"): {Service: "web", Target: "/data", VolumeName: "migrated-vol"}, + } + + err := client.validateMigratedVolumeMounts(context.Background(), actx, "api") + if err == nil || !isUnsafeTargetResumeError(err) || !strings.Contains(err.Error(), "changed from migrated volume migrated-vol to fresh-vol") { + t.Fatalf("expected the running replacement's fresh volume to fail validation, got %v", err) + } +} + +func TestValidateMigratedVolumeMountsAcceptsRunningReplacementWithDivergedStoppedPredecessor(t *testing.T) { + runner := &fakeDockerRunner{outputs: stoppedAndRunningTargetContainersFixture("fresh-vol", "migrated-vol")} + client := &Client{Docker: runner} + actx := &applyContext{cache: map[string]*appCache{}, plan: Plan{}} + entry := actx.entry("api") + entry.ComposeAppName = "stack-1" + entry.MigratedVolumeMounts = map[string]migratedVolumeMount{ + migratedMountKey("web", "/data"): {Service: "web", Target: "/data", VolumeName: "migrated-vol"}, + } + + if err := client.validateMigratedVolumeMounts(context.Background(), actx, "api"); err != nil { + t.Fatalf("expected the stopped predecessor's stale volume to be ignored, got %v", err) + } +} + func TestStopTargetComposeContainersWaitsForLatePostDeployTarget(t *testing.T) { runner := &latePostDeployTargetRunner{emptyFirst: true} client := &Client{Docker: runner} @@ -1680,6 +2048,38 @@ func TestStopTargetComposeContainersWaitsForLatePostDeployTarget(t *testing.T) { } } +func TestRemoveStoppedTargetAttachmentsDetachesFailedDeployment(t *testing.T) { + _, plan, _, staged := stagedSyncFixture(t) + const targetID = "abcdef123456" + runner := &detachAfterRemoveRunner{ + fakeDockerRunner: fakeDockerRunner{outputs: map[string][]byte{ + "volume inspect " + staged.VolumeName: ownedStagingVolumeInspect(plan, staged), + "ps -a --filter label=com.docker.compose.project=stack-1 --format {{.ID}}": []byte(targetID + "\n"), + "inspect --type container " + targetID: []byte(`[{"Id":"` + targetID + `","Name":"/web","Config":{"Labels":{"com.docker.compose.project":"stack-1"}},"State":{"Running":false,"Status":"exited"}}]`), + "rm " + targetID: []byte(targetID + "\n"), + }}, + volumeName: staged.VolumeName, + targetID: targetID, + } + client := &Client{Docker: runner} + actx := &applyContext{plan: plan, cache: map[string]*appCache{}} + actx.entry("api").ComposeAppName = "stack-1" + pin, err := acquireStagingVolumePin(context.Background(), runner, plan, "api", []stagedVolume{staged}, true) + if err != nil { + t.Fatalf("acquireStagingVolumePin: %v", err) + } + + if err := client.removeStoppedTargetAttachments(context.Background(), actx, "api", []stagedVolume{staged}, pin.containerID); err != nil { + t.Fatalf("removeStoppedTargetAttachments: %v", err) + } + if !fakeOutputCalled(&runner.fakeDockerRunner, "rm", targetID) { + t.Fatalf("expected stopped failed-deployment container removal, calls=%#v", runner.outputArgs) + } + if !runner.removed || runner.attachmentChecks != 2 { + t.Fatalf("post-removal attachment validation did not observe the removal: removed=%v checks=%d calls=%#v", runner.removed, runner.attachmentChecks, runner.outputArgs) + } +} + func TestPostDeployValidationStopFailureKeepsSourcePausedAndMarksAmbiguous(t *testing.T) { runner := &fakeDockerRunner{ outputs: map[string][]byte{ @@ -1909,6 +2309,7 @@ func TestPrimeResumeStateRecordsAlreadyStoppedTargetWriters(t *testing.T) { t.Fatalf("write compose: %v", err) } app := preparer.AppPlan{Name: "api", Directory: "api"} + app.Resources.DataStores = []preparer.DataStoreResource{{Kind: "postgres", Service: "db", Strategy: "migrate"}} app.TargetResources = &preparer.TargetResources{Dokploy: &preparer.DokployResources{ComposeApp: preparer.DokployComposeApp{ComposePath: "compose.yaml"}}} runner := &fakeDockerRunner{outputs: map[string][]byte{ "ps -a --filter label=com.docker.compose.project=stack-api --format {{.ID}}": []byte("web-id\n"), @@ -1917,7 +2318,7 @@ func TestPrimeResumeStateRecordsAlreadyStoppedTargetWriters(t *testing.T) { }} client := &Client{Docker: runner} actx := &applyContext{cache: map[string]*appCache{}, plan: Plan{ - Steps: []Step{{Kind: StepPauseSource, App: "api", Ref: "api"}, {Kind: StepResumeTarget, App: "api", Ref: "api"}}, + Steps: []Step{{Kind: StepPauseSource, App: "api", Ref: "api"}, {Kind: StepDumpDataStore, App: "api", Ref: "postgres:db"}, {Kind: StepRestoreDataStore, App: "api", Ref: "postgres:db"}, {Kind: StepResumeTarget, App: "api", Ref: "api"}}, Prepare: preparer.Result{BundleDir: bundleDir, Apps: []preparer.AppPlan{app}}, }} actx.entry("api").ComposeAppName = "stack-api" @@ -1982,6 +2383,7 @@ func TestPrimeResumeStateDoesNotRedeployCompletedPush(t *testing.T) { {Kind: StepUploadEnv, App: "api", Ref: "api"}, {Kind: StepPushImage, App: "api", Ref: "api"}, {Kind: StepPauseSource, App: "api", Ref: "api"}, + {Kind: StepSyncVolume, App: "api", Ref: "volume:web -> /data"}, }, Prepare: preparer.Result{BundleDir: bundleDir, Apps: []preparer.AppPlan{app}}, } @@ -2013,7 +2415,7 @@ func TestTargetWriterReconciliationDoesNotRestartPreviouslyStoppedWriterOnFailur }} client := &Client{Docker: runner} actx := &applyContext{cache: map[string]*appCache{}, plan: Plan{ - Steps: []Step{{Kind: StepPauseSource, App: "api", Ref: "api"}}, + Steps: []Step{{Kind: StepPauseSource, App: "api", Ref: "api"}, {Kind: StepSyncVolume, App: "api", Ref: "volume:web -> /data"}}, }} entry := actx.entry("api") entry.ComposeAppName = "stack-api" diff --git a/internal/target/dokploy/docker.go b/internal/target/dokploy/docker.go index ac8c6ef..e333dc4 100644 --- a/internal/target/dokploy/docker.go +++ b/internal/target/dokploy/docker.go @@ -325,9 +325,18 @@ func inspectContainers(ctx context.Context, runner dockerRunner, refs []string) } func listContainersByLabel(ctx context.Context, runner dockerRunner, label string) ([]dockerContainer, error) { - out, err := runner.Output(ctx, "ps", "-a", "--filter", "label="+label, "--format", "{{.ID}}") + return listContainersByLabels(ctx, runner, label) +} + +func listContainersByLabels(ctx context.Context, runner dockerRunner, labels ...string) ([]dockerContainer, error) { + args := []string{"ps", "-a"} + for _, label := range labels { + args = append(args, "--filter", "label="+label) + } + args = append(args, "--format", "{{.ID}}") + out, err := runner.Output(ctx, args...) if err != nil { - return nil, fmt.Errorf("docker ps label=%s: %w", label, err) + return nil, fmt.Errorf("docker ps labels=%s: %w", strings.Join(labels, ","), err) } ids := []string{} for _, line := range strings.Split(string(out), "\n") { diff --git a/internal/target/dokploy/rollback.go b/internal/target/dokploy/rollback.go index 352eee7..f9447ee 100644 --- a/internal/target/dokploy/rollback.go +++ b/internal/target/dokploy/rollback.go @@ -18,7 +18,14 @@ var sourceHealthPollInterval = 2 * time.Second // PlanForRollback builds the steps that return traffic to the source. func PlanForRollback(prepare preparer.Result, sync syncplan.Result, cutover gateway.Result) (Plan, error) { plan := Plan{Prepare: prepare, Cutover: cutover, stepTimeout: dockerStartTimeout} - statefulApps := []string{} + statefulApps := map[string]struct{}{} + for _, app := range sync.Apps { + if dataStoreSteps, volumeSteps := appStateSteps(prepare, app); len(dataStoreSteps) > 0 || len(volumeSteps) > 0 { + statefulApps[app.Name] = struct{}{} + } + } + statefulPauses := []string{} + pausedStateless := []string{} seen := map[string]struct{}{} livePlan := PlanFromArtifacts(prepare, sync, cutover) for _, step := range livePlan.Steps { @@ -29,12 +36,19 @@ func PlanForRollback(prepare preparer.Result, sync syncplan.Result, cutover gate continue } seen[step.App] = struct{}{} - statefulApps = append(statefulApps, step.App) + if _, stateful := statefulApps[step.App]; stateful { + statefulPauses = append(statefulPauses, step.App) + continue + } + pausedStateless = append(pausedStateless, step.App) } - if len(statefulApps) > 0 { - return Plan{}, fmt.Errorf("automatic rollback is unsafe for stateful app(s) %s: Dokploy has no durable application fence that prevents a queued or future deployment from restarting target writers after source writers resume; preserve both sides and establish writer and traffic authority manually", strings.Join(statefulApps, ", ")) + if len(statefulPauses) > 0 { + return Plan{}, fmt.Errorf("automatic rollback is unsafe for stateful app(s) %s: Dokploy has no durable application fence that prevents a queued or future deployment from restarting target writers after source writers resume; preserve both sides and establish writer and traffic authority manually", strings.Join(statefulPauses, ", ")) } if cutoverPlanHasRoutes(cutover) { + for _, app := range pausedStateless { + plan.Steps = append(plan.Steps, Step{Kind: StepResumeSource, App: app, Ref: app}) + } statelessApps := []preparer.AppPlan{} for _, app := range prepare.Apps { if len(sourceQuiesceTargetRefs(app)) > 0 { diff --git a/internal/target/dokploy/rollback_test.go b/internal/target/dokploy/rollback_test.go index 50fb064..43084cc 100644 --- a/internal/target/dokploy/rollback_test.go +++ b/internal/target/dokploy/rollback_test.go @@ -4,6 +4,7 @@ import ( "context" "errors" "io" + "slices" "strings" "testing" "time" @@ -60,7 +61,7 @@ func TestPlanForRollbackReturnsTrafficOnlyForStatelessApps(t *testing.T) { t.Fatal(err) } got := stepKinds(plan.Steps) - want := []StepKind{StepVerifySourceHealth, StepStopDokployProxy, StepStartCoolifyProxy, StepObserveRollback} + want := []StepKind{StepResumeSource, StepVerifySourceHealth, StepStopDokployProxy, StepStartCoolifyProxy, StepObserveRollback} if len(got) != len(want) { t.Fatalf("expected stateless traffic rollback %v, got %v", want, got) } @@ -87,7 +88,7 @@ func TestPlanForRollbackIgnoresSkippedPlatformState(t *testing.T) { t.Fatal(err) } got := stepKinds(plan.Steps) - want := []StepKind{StepVerifySourceHealth, StepStopDokployProxy, StepStartCoolifyProxy, StepObserveRollback} + want := []StepKind{StepResumeSource, StepVerifySourceHealth, StepStopDokployProxy, StepStartCoolifyProxy, StepObserveRollback} if len(got) != len(want) { t.Fatalf("expected skipped platform state to retain stateless rollback %v, got %v", want, got) } @@ -108,6 +109,55 @@ func TestPlanForRollbackSkipsProxySwapWithoutRoutes(t *testing.T) { } } +func TestPlanForRollbackResumesPausedRoutedStatelessSourcesBeforeVerifying(t *testing.T) { + web := preparer.AppPlan{Name: "web"} + web.Resources.SourceServices = []preparer.SourceServiceRef{{ServiceName: "web", ContainerID: "web-id"}} + api := preparer.AppPlan{Name: "api"} + api.Resources.SourceServices = []preparer.SourceServiceRef{{ServiceName: "api", ContainerID: "api-id"}} + prepare := preparer.Result{Apps: []preparer.AppPlan{web, api}} + cutover := gateway.Result{Apps: []gateway.AppPlan{ + {Name: "web", Routes: []gateway.Route{{Host: "web.example.com"}}}, + {Name: "api", Routes: []gateway.Route{{Host: "api.example.com"}}}, + }} + plan, err := PlanForRollback(prepare, syncplan.Result{}, cutover) + if err != nil { + t.Fatal(err) + } + got := stepKinds(plan.Steps) + want := []StepKind{StepResumeSource, StepResumeSource, StepVerifySourceHealth, StepVerifySourceHealth, StepStopDokployProxy, StepStartCoolifyProxy, StepObserveRollback, StepObserveRollback} + if !slices.Equal(got, want) { + t.Fatalf("expected rollback to resume both paused routed sources before verification, want %v got %v", want, got) + } + resumed := []string{plan.Steps[0].App, plan.Steps[1].App} + if !slices.Equal(resumed, []string{"web", "api"}) { + t.Fatalf("expected resumes in cutover order, got %v", resumed) + } +} + +func TestPlanForRollbackRefusesWhenAnyPausedSourceIsStateful(t *testing.T) { + prepare := rollbackPrepareFixture() + web := preparer.AppPlan{Name: "web"} + web.Resources.SourceServices = []preparer.SourceServiceRef{{ServiceName: "web", ContainerID: "web-id"}} + prepare.Apps = append(prepare.Apps, web) + cutover := rollbackCutoverFixture() + cutover.Apps = append(cutover.Apps, gateway.AppPlan{Name: "web", Routes: []gateway.Route{{Host: "web.example.com"}}}) + sync := syncplan.Result{Apps: []syncplan.AppPlan{{ + Name: "api", + Steps: []syncplan.Step{{ + ResourceType: "volume", + ResourceRef: "volume:web -> /data", + Strategy: syncplan.StrategyDockerVolumeArchive, + }}, + }}} + plan, err := PlanForRollback(prepare, sync, cutover) + if err == nil || !strings.Contains(err.Error(), "no durable application fence") || !strings.Contains(err.Error(), "api") { + t.Fatalf("expected stateful refusal with a routed stateless app present, got plan=%#v err=%v", plan, err) + } + if strings.Contains(err.Error(), "web") { + t.Fatalf("routed stateless pause must not block rollback, got %v", err) + } +} + type flappingHealthRunner struct { fakeDockerRunner calls int diff --git a/internal/target/dokploy/source.go b/internal/target/dokploy/source.go index 4a775de..8bc7b58 100644 --- a/internal/target/dokploy/source.go +++ b/internal/target/dokploy/source.go @@ -9,10 +9,113 @@ import ( "sort" "strings" + "github.com/aikins01/bort/internal/dockercli" "github.com/aikins01/bort/internal/preparer" "github.com/aikins01/bort/internal/safepath" ) +const coolifyControlPlaneContainer = "coolify" + +// RequiresCoolifyDeploymentFence reports whether applying plan needs the +// Coolify control plane durably stopped first. +func RequiresCoolifyDeploymentFence(plan Plan) bool { + source := strings.ToLower(strings.TrimSpace(plan.Prepare.Source)) + coolifySource := source == "coolify-local" || source == "coolify-local-traefik" || source == "coolify-local-caddy" + for _, step := range plan.Steps { + if (step.Kind != StepRestoreDataStore && step.Kind != StepSyncVolume) || shouldSkipApplyStep(plan, step) { + continue + } + if coolifySource { + return true + } + if app, ok := findPrepareApp(plan.Prepare, step.App); ok && strings.EqualFold(strings.TrimSpace(app.Platform), "coolify") { + return true + } + } + return false +} + +func requireCoolifyDeploymentFence(ctx context.Context, runner dockerRunner) error { + container, err := inspectContainer(ctx, runner, coolifyControlPlaneContainer) + if err != nil && !isContainerMissingErr(err) { + return fmt.Errorf("stateful live apply requires a durable Coolify deployment fence, but Bort could not inspect the %s control-plane container: %w; record its current restart policy, run `%s`, then retry", coolifyControlPlaneContainer, err, coolifyFenceCommand()) + } + if err == nil { + policy := normalizedRestartPolicyName(container.HostConfig.RestartPolicy.Name) + if container.State.Running || policy != "no" { + return fmt.Errorf("stateful live apply requires the Coolify control-plane container %s to be stopped with restart policy no (running=%t, restart=%s); wait until no Coolify deployment is queued or running (a cancelled deployment can keep running, so confirm it has ended), record its current restart policy, run `%s`, then retry; restart Coolify only after source authority is finalized; after target acceptance, restart it only if other apps need it and immediately delete the migrated apps in Coolify", coolifyControlPlaneContainer, container.State.Running, policy, coolifyFenceCommand()) + } + } + helperRepositories := []string{coolifyHelperImageRepository} + if err == nil { + if custom := imageRepository(envMap(container.Config.Env)["HELPER_IMAGE"]); custom != "" { + helperRepositories = append(helperRepositories, custom) + } + } + helpers, err := activeCoolifyDeploymentHelpers(ctx, runner, helperRepositories) + if err != nil { + return fmt.Errorf("stateful live apply requires a durable Coolify deployment fence, but Bort could not inspect Coolify deployment helpers: %w", err) + } + if len(helpers) > 0 { + return fmt.Errorf("stateful live apply requires in-flight Coolify deployments to finish, but helper container(s) %s are still running deployment commands that started before the control plane stopped; wait until they finish, then retry", strings.Join(helpers, ", ")) + } + return nil +} + +func coolifyFenceCommand() string { + docker := "docker --host " + dockercli.LocalHost() + if strings.TrimSpace(os.Getenv("SUDO_UID")) != "" { + docker = "sudo " + docker + } + return docker + " update --restart=no " + coolifyControlPlaneContainer + " && " + docker + " stop " + coolifyControlPlaneContainer +} + +const coolifyHelperImageRepository = "coollabsio/coolify-helper" + +func activeCoolifyDeploymentHelpers(ctx context.Context, runner dockerRunner, helperRepositories []string) ([]string, error) { + var out []byte + for attempt := 0; ; attempt++ { + listed, err := runner.Output(ctx, "ps", "-q", "--no-trunc") + if err != nil { + return nil, err + } + ids := strings.Fields(string(listed)) + if len(ids) == 0 { + return nil, nil + } + out, err = runner.Output(ctx, append([]string{"inspect", "--type", "container", "--format", "{{.Name}} {{.Config.Image}} {{len .ExecIDs}}"}, ids...)...) + if err == nil { + break + } + if attempt == 2 || !isContainerMissingErr(err) { + return nil, err + } + } + active := []string{} + for _, line := range strings.Split(string(out), "\n") { + fields := strings.Fields(line) + if len(fields) != 3 || fields[2] == "0" { + continue + } + repository := imageRepository(fields[1]) + for _, helper := range helperRepositories { + if repository == helper || strings.HasSuffix(repository, "/"+helper) || strings.HasSuffix(helper, "/"+repository) { + active = append(active, strings.TrimPrefix(fields[0], "/")) + break + } + } + } + return active, nil +} + +func imageRepository(image string) string { + repository, _, _ := strings.Cut(strings.TrimSpace(image), "@") + if colon := strings.LastIndex(repository, ":"); colon > strings.LastIndex(repository, "/") { + repository = repository[:colon] + } + return repository +} + func (c *Client) applyPauseSource(ctx context.Context, actx *applyContext, step Step) error { app, ok := findPrepareApp(actx.plan.Prepare, step.App) if !ok { @@ -173,15 +276,27 @@ func (c *Client) applyResumeSource(ctx context.Context, actx *applyContext, step if !ok { return fmt.Errorf("app %s not found in prepare result", step.App) } - entry := actx.entry(step.App) - if !entry.SourcePauseRecorded { - return unsafeSourceResumeError{err: fmt.Errorf("source pause ownership for app %s was not durably recorded; refusing to start stopped containers", step.App)} - } runner := c.dockerRunner() containers, err := inspectSourceQuiesceTargets(ctx, runner, app) if err != nil { return err } + entry := actx.entry(step.App) + if !entry.SourcePauseRecorded { + // a rollback of a run applied before cutover pauses existed, or an + // apply that failed before its pause step, has no recorded ownership; + // resuming is only a no-op when nothing needs starting. + stopped := []string{} + for _, container := range containers { + if !container.State.Running { + stopped = append(stopped, container.ID) + } + } + if len(stopped) > 0 { + return unsafeSourceResumeError{err: fmt.Errorf("source pause ownership for app %s was not durably recorded and source container(s) %s are stopped; verify they should run, start them with `docker start %s`, then retry", step.App, strings.Join(stopped, ", "), strings.Join(stopped, " "))} + } + return nil + } byID := make(map[string]dockerContainer, len(containers)) for _, container := range containers { byID[container.ID] = container @@ -517,6 +632,33 @@ func sourceCommitTargets(app preparer.AppPlan) []commitTargetRef { return refs } +func SourceRetirementContainers(plan Plan) []string { + refs := []string{} + seen := map[string]struct{}{} + add := func(ref string) { + if _, dup := seen[ref]; ref != "" && !dup { + seen[ref] = struct{}{} + refs = append(refs, ref) + } + } + for _, step := range plan.Steps { + if shouldSkipApplyStep(plan, step) { + continue + } + switch step.Kind { + case StepStopSourceApp: + if app, ok := findPrepareApp(plan.Prepare, step.App); ok { + for _, ref := range sourceCommitTargets(app) { + add(ref.label()) + } + } + case StepStopCoolifyProxy: + add(coolifyProxyContainer) + } + } + return refs +} + func sourceCommitTargetRefs(app preparer.AppPlan) []commitTargetRef { refs := []commitTargetRef{} add := func(id, name string) { diff --git a/internal/target/dokploy/source_test.go b/internal/target/dokploy/source_test.go index b81a47b..0c7d99a 100644 --- a/internal/target/dokploy/source_test.go +++ b/internal/target/dokploy/source_test.go @@ -8,11 +8,77 @@ import ( "strings" "testing" + "github.com/aikins01/bort/internal/dockercli" "github.com/aikins01/bort/internal/gateway" "github.com/aikins01/bort/internal/preparer" syncplan "github.com/aikins01/bort/internal/sync" ) +func TestCoolifyDeploymentFenceRequiresStoppedNoRestartControlPlane(t *testing.T) { + const inspectRunning = "inspect --type container --format {{.Name}} {{.Config.Image}} {{len .ExecIDs}} " + for _, tc := range []struct { + name string + missing bool + running bool + policy string + env string + listed string + want string + }{ + {name: "fenced", policy: "no", listed: "/dokploy-traefik traefik:v3.6 0\n/cache coollabsio/coolify-helper-cache:1 3\n"}, + {name: "control plane removed", missing: true}, + {name: "running", running: true, policy: "no", want: "docker --host " + dockercli.LocalHost() + " update --restart=no coolify"}, + {name: "restart enabled", policy: "always", want: "docker --host " + dockercli.LocalHost() + " update --restart=no coolify"}, + {name: "idle helper left by a stopped deployment", policy: "no", listed: "/x8k2 docker.io/coollabsio/coolify-helper:1.0.17 0\n"}, + {name: "helper still running deployment commands", policy: "no", listed: "/x8k2 docker.io/coollabsio/coolify-helper:1.0.17 1\n", want: "helper container(s) x8k2"}, + {name: "registry helper pinned by digest", policy: "no", listed: "/x9 registry.local:5000/coollabsio/coolify-helper@sha256:abc 2\n", want: "helper container(s) x9"}, + {name: "custom HELPER_IMAGE mirror", policy: "no", env: "HELPER_IMAGE=registry.internal/coolify-helper", listed: "/x7 registry.internal/coolify-helper:1.0.12 1\n", want: "helper container(s) x7"}, + } { + t.Run(tc.name, func(t *testing.T) { + runner := &fakeDockerRunner{outputs: map[string][]byte{}, outputErrs: map[string]error{}} + if tc.missing { + runner.outputErrs["inspect --type container coolify"] = errors.New("Error: No such object: coolify") + } else { + runner.outputs["inspect --type container coolify"] = []byte(fmt.Sprintf(`[{"Id":"coolify-id","Name":"/coolify","Config":{"Env":[%q]},"State":{"Running":%t,"Status":"exited"},"HostConfig":{"RestartPolicy":{"Name":"%s"}}}]`, tc.env, tc.running, tc.policy)) + } + ids := []string{} + for _, line := range strings.Split(strings.TrimSpace(tc.listed), "\n") { + if fields := strings.Fields(line); len(fields) == 3 { + ids = append(ids, strings.TrimPrefix(fields[0], "/")) + } + } + runner.outputs["ps -q --no-trunc"] = []byte(strings.Join(ids, "\n")) + runner.outputs[inspectRunning+strings.Join(ids, " ")] = []byte(tc.listed) + err := requireCoolifyDeploymentFence(context.Background(), runner) + if tc.want != "" && (err == nil || !strings.Contains(err.Error(), tc.want)) { + t.Fatalf("expected actionable fence refusal, got %v", err) + } + if tc.want == "" && err != nil { + t.Fatalf("valid Coolify fence was refused: %v", err) + } + }) + } +} + +func TestStatefulCoolifyPlanRequiresDeploymentFence(t *testing.T) { + plan := Plan{ + Prepare: preparer.Result{Source: "coolify-local"}, + Steps: []Step{{Kind: StepPauseSource, App: "api"}, {Kind: StepSyncVolume, App: "api"}, {Kind: StepPushImage, App: "api"}}, + } + if !RequiresCoolifyDeploymentFence(plan) { + t.Fatal("stateful Coolify plan did not require a deployment fence") + } + plan.Prepare.Source = "docker" + plan.Prepare.Apps = []preparer.AppPlan{{Name: "api", Platform: "docker"}, {Name: "worker", Platform: "coolify"}} + if RequiresCoolifyDeploymentFence(plan) { + t.Fatal("plain Docker source unexpectedly required the Coolify control-plane fence") + } + plan.Steps = append(plan.Steps, Step{Kind: StepSyncVolume, App: "worker"}) + if !RequiresCoolifyDeploymentFence(plan) { + t.Fatal("Docker-scanned Coolify app did not require the deployment fence") + } +} + func TestVerifySourceContainersBatchesReviewedIDs(t *testing.T) { webID := strings.Repeat("a", 64) workerID := strings.Repeat("b", 64) @@ -233,7 +299,7 @@ func TestPlanFromArtifactsPausesBeforeDumpAndVolumeSync(t *testing.T) { return kinds } staged := filter(PlanFromArtifacts(prepare, syncResult, gatewayResultEmpty())) - wantStaged := []StepKind{StepPauseSource, StepDumpDataStore, StepRestoreDataStore, StepSyncVolume, StepResumeSource, StepPushImage} + wantStaged := []StepKind{StepPauseSource, StepDumpDataStore, StepRestoreDataStore, StepSyncVolume, StepPushImage} if !slices.Equal(staged, wantStaged) { t.Fatalf("staged: expected %v, got %v", wantStaged, staged) } @@ -320,6 +386,81 @@ func TestPlanFromArtifactsDeploysRoutedStatefulTargetAfterStateTransfer(t *testi } } +func TestPlanFromArtifactsPausesRoutedStatelessSourceAtCutover(t *testing.T) { + routed := preparer.AppPlan{Name: "web"} + unrouted := preparer.AppPlan{Name: "worker"} + stateful := preparer.AppPlan{Name: "db"} + stateful.Resources.Volumes = []preparer.VolumeResource{{Service: "db", Type: "volume", Target: "/data"}} + prepare := preparer.Result{Apps: []preparer.AppPlan{routed, unrouted, stateful}} + sync := syncplan.Result{Apps: []syncplan.AppPlan{ + {Name: "web"}, + {Name: "worker"}, + {Name: "db", Steps: []syncplan.Step{{ResourceType: "volume", ResourceRef: "volume:db -> /data", Strategy: syncplan.StrategyDockerVolumeArchive}}}, + }} + cutover := gateway.Result{Apps: []gateway.AppPlan{ + {Name: "web", Routes: []gateway.Route{{Host: "web.example.com"}}}, + {Name: "db", Routes: []gateway.Route{{Host: "db.example.com"}}}, + }} + plan := PlanFromArtifacts(prepare, sync, cutover) + pauseCount := map[string]int{} + webPause, stopProxy, webActivate, startProxy := -1, -1, -1, -1 + for index, step := range plan.Steps { + if step.Kind == StepPauseSource { + pauseCount[step.App]++ + if step.App == "web" { + webPause = index + } + } + if step.Kind == StepResumeSource && step.App == "web" { + t.Fatalf("routed stateless source must stay paused until retirement, got plan=%v", stepKinds(plan.Steps)) + } + if step.Kind == StepStopCoolifyProxy { + stopProxy = index + } + if step.Kind == StepActivateRoutes && step.App == "web" { + webActivate = index + } + if step.Kind == StepStartDokployProxy { + startProxy = index + } + } + if pauseCount["web"] != 1 || pauseCount["db"] != 1 || pauseCount["worker"] != 0 { + t.Fatalf("expected one pause per routed app and none for the unrouted stateless app, got %v plan=%v", pauseCount, stepKinds(plan.Steps)) + } + if webPause < 0 || stopProxy < 0 || webActivate < 0 || startProxy < 0 || !(webPause < stopProxy && stopProxy < webActivate && webActivate < startProxy) { + t.Fatalf("expected pause < stop proxy < activate < start proxy, got plan=%v", stepKinds(plan.Steps)) + } +} + +func TestPlanFromArtifactsPausesRoutedStatelessSourceInEveryPlanOrder(t *testing.T) { + prepare := preparer.Result{Apps: []preparer.AppPlan{{Name: "web"}}} + sync := syncplan.Result{Apps: []syncplan.AppPlan{{Name: "web"}}} + cutover := gateway.Result{Apps: []gateway.AppPlan{{Name: "web", Routes: []gateway.Route{{Host: "web.example.com"}}}}} + builders := []struct { + name string + plan Plan + }{ + {"v1alpha4", PlanFromArtifacts(prepare, sync, cutover)}, + {"v1alpha3", LegacyPlanFromArtifactsV1Alpha3(prepare, sync, cutover)}, + {"v1alpha2", PlanFromArtifactsV1Alpha2(prepare, sync, cutover)}, + {"v1alpha1", LegacyPlanFromArtifactsV1Alpha1(prepare, sync, cutover)}, + } + for _, builder := range builders { + pause, activate := -1, -1 + for index, step := range builder.plan.Steps { + if step.Kind == StepPauseSource && step.App == "web" { + pause = index + } + if step.Kind == StepActivateRoutes && step.App == "web" { + activate = index + } + } + if pause < 0 || activate < 0 || pause > activate { + t.Fatalf("%s: expected routed stateless pause before route activation, got %v", builder.name, stepKinds(builder.plan.Steps)) + } + } +} + func TestPlanFromArtifactsV1Alpha2StartsRoutedStatefulTargetDuringFinalProxyHandoff(t *testing.T) { app := preparer.AppPlan{Name: "api"} app.Resources.Volumes = []preparer.VolumeResource{{Service: "web", Type: "volume", Target: "/data"}} @@ -392,7 +533,7 @@ func TestPlanFromArtifactsCopiesVolumeStrategyDataStoreVolumes(t *testing.T) { kinds = append(kinds, step.Kind) } } - want := []StepKind{StepPauseSource, StepSyncVolume, StepResumeSource} + want := []StepKind{StepPauseSource, StepSyncVolume} if len(kinds) != len(want) { t.Fatalf("expected %v, got %v", want, kinds) } @@ -549,6 +690,47 @@ func TestApplyResumeSourceStartsOnlyOwnedStoppedContainers(t *testing.T) { } } +func TestApplyResumeSourceWithoutRecordNoOpsWhenEverythingRuns(t *testing.T) { + app := preparer.AppPlan{Name: "api"} + app.Resources.Volumes = []preparer.VolumeResource{ + {Service: "web", Type: "volume", SourceContainerID: "web-id"}, + } + runner := &fakeDockerRunner{ + outputs: map[string][]byte{ + "inspect --type container web-id": []byte(`[{"Id":"web-id","Name":"/web","State":{"Running":true,"Status":"running"}}]`), + }, + } + client := &Client{Docker: runner} + actx := &applyContext{cache: map[string]*appCache{}, plan: Plan{Prepare: preparer.Result{Apps: []preparer.AppPlan{app}}}} + if err := client.applyResumeSource(context.Background(), actx, Step{Kind: StepResumeSource, App: "api"}); err != nil { + t.Fatalf("applyResumeSource: %v", err) + } + if fakeOutputCalled(runner, "start", "web-id") { + t.Fatalf("resume without recorded ownership must not start a running container, calls=%v", runner.outputArgs) + } +} + +func TestApplyResumeSourceWithoutRecordRefusesStoppedContainer(t *testing.T) { + app := preparer.AppPlan{Name: "api"} + app.Resources.Volumes = []preparer.VolumeResource{ + {Service: "web", Type: "volume", SourceContainerID: "web-id"}, + } + runner := &fakeDockerRunner{ + outputs: map[string][]byte{ + "inspect --type container web-id": []byte(`[{"Id":"web-id","Name":"/web","State":{"Running":false,"Status":"exited"}}]`), + }, + } + client := &Client{Docker: runner} + actx := &applyContext{cache: map[string]*appCache{}, plan: Plan{Prepare: preparer.Result{Apps: []preparer.AppPlan{app}}}} + err := client.applyResumeSource(context.Background(), actx, Step{Kind: StepResumeSource, App: "api"}) + if !isUnsafeSourceResumeError(err) || !strings.Contains(err.Error(), "was not durably recorded") { + t.Fatalf("expected ownership refusal for a stopped source without recorded ownership, got %v", err) + } + if fakeOutputCalled(runner, "start", "web-id") { + t.Fatalf("resume without recorded ownership started a stopped container, calls=%v", runner.outputArgs) + } +} + type sourceOwnershipRunner struct { fakeDockerRunner running map[string]bool @@ -1000,7 +1182,7 @@ func TestApplyRecordsCleanupResumeBeforeRestartingTransferredSource(t *testing.T BeforeStep: &beforeStep, } err := client.Apply(context.Background(), plan) - if err == nil || !strings.Contains(err.Error(), "stays stopped") || !strings.Contains(err.Error(), "ledger unavailable") { + if err == nil || !strings.Contains(err.Error(), "remains stopped") || !strings.Contains(err.Error(), "bort status") || !strings.Contains(err.Error(), "ledger unavailable") { t.Fatalf("expected refused restart when the cleanup cannot be recorded, got %v", err) } if runner.running["web-id"] { @@ -1167,3 +1349,61 @@ func TestRouteActivationGateRechecksEveryHandedOffAppPerStep(t *testing.T) { t.Fatalf("a handed-off source that ran must block every later activation, not only its own, got %v", err) } } + +func TestRouteActivationRefusesPausedStatelessSourceThatRan(t *testing.T) { + app := preparer.AppPlan{Name: "api"} + app.Resources.Volumes = []preparer.VolumeResource{ + {Service: "web", Type: "volume", SourceContainerID: "web-id"}, + } + runner := &sourceOwnershipRunner{ + running: map[string]bool{"web-id": true}, + startedAt: map[string]string{"web-id": "started-9"}, + finishedAt: map[string]string{"web-id": "stopped-10"}, + } + client := &Client{Docker: runner} + actx := &applyContext{ + plan: Plan{Prepare: preparer.Result{Apps: []preparer.AppPlan{app}}}, + cache: map[string]*appCache{}, + } + actx.entry("api").SourcePauseRecorded = true + actx.entry("api").SourcePausedContainers = []sourcePausedContainer{{ID: "web-id", Stopped: true, StartedAt: "started-0", FinishedAt: "stopped-1"}} + step := Step{Kind: StepActivateRoutes, App: "api", Ref: "routes"} + err := client.requireTransferredSourceStillPaused(context.Background(), actx, step, pausedSources{"api": false}, map[string]struct{}{}) + if !isUnsafeSourceResumeError(err) || !strings.Contains(err.Error(), "compete with the activated target routes") { + t.Fatalf("expected route-shadowing refusal for a restarted paused source, got %v", err) + } + if fakeOutputCalled(&runner.fakeDockerRunner, "start", "web-id") { + t.Fatalf("refusal restarted the source, calls=%v", runner.outputArgs) + } +} + +type vanishingContainerRunner struct { + *fakeDockerRunner + listed int +} + +func (r *vanishingContainerRunner) Output(ctx context.Context, args ...string) ([]byte, error) { + key := strings.Join(args, " ") + if key == "ps -q --no-trunc" { + r.listed++ + if r.listed == 1 { + return []byte("short-lived\nx8k2\n"), nil + } + return []byte("x8k2\n"), nil + } + if strings.HasSuffix(key, "short-lived x8k2") { + return nil, errors.New("Error: No such container: short-lived") + } + return r.fakeDockerRunner.Output(ctx, args...) +} + +func TestCoolifyDeploymentFenceRelistsWhenAnUnrelatedContainerExits(t *testing.T) { + runner := &vanishingContainerRunner{fakeDockerRunner: &fakeDockerRunner{outputs: map[string][]byte{ + "inspect --type container coolify": []byte(`[{"Id":"coolify-id","Name":"/coolify","State":{"Running":false,"Status":"exited"},"HostConfig":{"RestartPolicy":{"Name":"no"}}}]`), + "inspect --type container --format {{.Name}} {{.Config.Image}} {{len .ExecIDs}} x8k2": []byte("/x8k2 coollabsio/coolify-helper:1.0.17 1\n"), + }}} + err := requireCoolifyDeploymentFence(context.Background(), runner) + if err == nil || !strings.Contains(err.Error(), "helper container(s) x8k2") || runner.listed != 2 { + t.Fatalf("fence did not re-list after a container exited mid-check: listed=%d err=%v", runner.listed, err) + } +} diff --git a/internal/target/dokploy/staging.go b/internal/target/dokploy/staging.go index a0e57a9..b98bee8 100644 --- a/internal/target/dokploy/staging.go +++ b/internal/target/dokploy/staging.go @@ -4,11 +4,14 @@ import ( "context" "crypto/sha256" "encoding/hex" + "encoding/json" + "errors" "fmt" "os" "path" "path/filepath" "regexp" + "slices" "strconv" "strings" "time" @@ -24,6 +27,7 @@ const ( stagingVolumeAppLabel = "bort.app" stagingVolumeServiceLabel = "bort.service" stagingVolumeTargetLabel = "bort.target" + stagingVolumePinLabel = "bort.staging-pin" ) // stagingOwner identifies the run that owns staging volumes. run names @@ -186,22 +190,76 @@ func stagingVolumeLabels(plan Plan, appName string, volume stagedVolume) []strin } } -func stagingVolumeExists(ctx context.Context, runner dockerRunner, plan Plan, volume stagedVolume) (bool, error) { - out, err := runner.Output(ctx, "volume", "inspect", "--format", "{{index .Labels \""+stagingVolumeRunIDLabel+"\"}}", volume.VolumeName) +func stagingVolumeExists(ctx context.Context, runner dockerRunner, plan Plan, appName string, volume stagedVolume) (bool, error) { + out, err := runner.Output(ctx, "volume", "inspect", volume.VolumeName) if err != nil { - if isDockerVolumeOrNetworkMissingErr(err) { + if isDockerResourceMissingErr(err, "volume", volume.VolumeName) { return false, nil } return false, fmt.Errorf("inspect staging volume %s: %w", volume.VolumeName, err) } - if owner := strings.TrimSpace(string(out)); owner != stagingOwner(plan) { - return true, fmt.Errorf("docker volume %s already exists but is not owned by run %q (label %s=%q)", volume.VolumeName, plan.RunName, stagingVolumeRunIDLabel, owner) + return true, validateStagingVolumeOwnership(plan, appName, []stagedVolume{volume}, out) +} + +type stagingVolumeState struct { + Name string `json:"Name"` + Labels map[string]string `json:"Labels"` +} + +func requireStagingVolumeOwned(ctx context.Context, runner dockerRunner, plan Plan, appName string, volume stagedVolume) error { + return requireStagingVolumesOwned(ctx, runner, plan, appName, []stagedVolume{volume}) +} + +func requireStagingVolumesOwned(ctx context.Context, runner dockerRunner, plan Plan, appName string, volumes []stagedVolume) error { + if len(volumes) == 0 { + return nil + } + args := []string{"volume", "inspect"} + for _, volume := range volumes { + args = append(args, volume.VolumeName) + } + out, err := runner.Output(ctx, args...) + if err != nil { + return fmt.Errorf("inspect staging volume ownership: %w", err) } - return true, nil + return validateStagingVolumeOwnership(plan, appName, volumes, out) +} + +func validateStagingVolumeOwnership(plan Plan, appName string, volumes []stagedVolume, out []byte) error { + expected := make(map[string]stagedVolume, len(volumes)) + for _, volume := range volumes { + expected[volume.VolumeName] = volume + } + var states []stagingVolumeState + if err := json.Unmarshal(out, &states); err != nil { + return fmt.Errorf("decode staging volume ownership: %w", err) + } + if len(states) != len(expected) { + return fmt.Errorf("docker volume inspect returned %d resources, want %d", len(states), len(expected)) + } + seen := make(map[string]struct{}, len(states)) + for _, state := range states { + name := strings.TrimSpace(state.Name) + volume, ok := expected[name] + if !ok { + return fmt.Errorf("docker volume inspect returned unexpected resource %q", name) + } + if _, duplicate := seen[name]; duplicate { + return fmt.Errorf("docker volume inspect returned duplicate resource %q", name) + } + seen[name] = struct{}{} + for _, expectedLabel := range stagingVolumeLabels(plan, appName, volume) { + key, value, _ := strings.Cut(expectedLabel, "=") + if state.Labels[key] != value { + return fmt.Errorf("docker volume %s is not the staged %s:%s volume for app %s in run %q (label %s=%q, want %q)", name, volume.Service, volume.Target, appName, plan.RunName, key, state.Labels[key], value) + } + } + } + return nil } func ensureStagingVolume(ctx context.Context, runner dockerRunner, plan Plan, appName string, volume stagedVolume) error { - exists, err := stagingVolumeExists(ctx, runner, plan, volume) + exists, err := stagingVolumeExists(ctx, runner, plan, appName, volume) if err != nil || exists { return err } @@ -213,33 +271,606 @@ func ensureStagingVolume(ctx context.Context, runner dockerRunner, plan Plan, ap if _, err := runner.Output(ctx, args...); err != nil { return fmt.Errorf("create staging volume %s: %w", volume.VolumeName, err) } + return requireStagingVolumeOwned(ctx, runner, plan, appName, volume) +} + +type stagingVolumePin struct { + name string + containerID string +} + +func ensureAppStagingVolumePin(ctx context.Context, runner dockerRunner, actx *applyContext, appName string, allowCreate bool) ([]stagedVolume, stagingVolumePin, error) { + volumes := stagedVolumesForApp(actx.plan, appName) + if len(volumes) == 0 { + return nil, stagingVolumePin{}, fmt.Errorf("pin staging volumes for app %s: no volumes supplied", appName) + } + if pin, ok := actx.stagingVolumePins[appName]; ok { + if err := requireStagingVolumesOwned(ctx, runner, actx.plan, appName, volumes); err != nil { + return nil, stagingVolumePin{}, err + } + if err := requireStagingVolumePin(ctx, runner, actx.plan, volumes, pin); err != nil { + return nil, stagingVolumePin{}, err + } + return volumes, pin, nil + } + if allowCreate { + for _, volume := range volumes { + if err := ensureStagingVolume(ctx, runner, actx.plan, appName, volume); err != nil { + return nil, stagingVolumePin{}, err + } + } + } + pin, err := acquireStagingVolumePin(ctx, runner, actx.plan, appName, volumes, allowCreate) + if pin.containerID != "" { + if actx.stagingVolumePins == nil { + actx.stagingVolumePins = map[string]stagingVolumePin{} + } + actx.stagingVolumePins[appName] = pin + } + if err != nil { + return nil, stagingVolumePin{}, err + } + return volumes, pin, nil +} + +func acquireStagingVolumePin(ctx context.Context, runner dockerRunner, plan Plan, appName string, volumes []stagedVolume, allowCreate bool) (stagingVolumePin, error) { + if len(volumes) == 0 { + return stagingVolumePin{}, fmt.Errorf("pin staging volumes: no volumes supplied") + } + if err := requireStagingVolumesOwned(ctx, runner, plan, appName, volumes); err != nil { + return stagingVolumePin{}, err + } + name := stagingVolumePinName(plan, volumes) + finish := func(pin stagingVolumePin) (stagingVolumePin, error) { + if err := requireStagingVolumesOwned(ctx, runner, plan, appName, volumes); err != nil { + return pin, fmt.Errorf("verify staging volume ownership after pinning with %s: %w", pin.name, err) + } + if err := removeSupersededStagingVolumePins(ctx, runner, plan, appName, volumes, pin); err != nil { + return pin, err + } + return pin, nil + } + existing, err := reconcileStagingVolumePins(ctx, runner, plan, appName, volumes, name) + if err != nil { + return stagingVolumePin{}, err + } + if existing.containerID != "" { + return finish(existing) + } + if !allowCreate { + return stagingVolumePin{}, unsafeSourceResumeError{err: authorityRecoveryRequiredError{err: fmt.Errorf("required staging volume pin %s is missing after state transfer; refusing to recreate it because the volume identity was unprotected; paused source applications remain stopped; follow `%s` to recover source or target authority", name, recoveryStatusCommand(plan))}} + } + args := []string{ + "run", "-d", "--name", name, "--network", "none", "--read-only", "--restart", "unless-stopped", + "--label", stagingVolumePinLabel + "=true", + "--label", stagingVolumeRunIDLabel + "=" + stagingOwner(plan), + "--label", stagingVolumeAppLabel + "=" + appName, + } + for index, volume := range volumes { + args = append(args, "-v", volume.VolumeName+":/bort-volume/"+strconv.Itoa(index)+":ro") + } + args = append(args, volumeCopyImage, "sh", "-c", "while :; do sleep 2147483647; done") + out, err := runner.Output(ctx, args...) + if err != nil { + recovered, reconcileErr := reconcileStagingVolumePinsWithTimeout(runner, plan, appName, volumes, name) + if reconcileErr == nil && recovered.containerID != "" { + return finish(recovered) + } + return stagingVolumePin{}, errors.Join(fmt.Errorf("pin staging volumes: %w", err), reconcileErr) + } + pin := stagingVolumePin{name: name, containerID: strings.TrimSpace(string(out))} + if pin.containerID == "" { + recovered, reconcileErr := reconcileStagingVolumePinsWithTimeout(runner, plan, appName, volumes, name) + if reconcileErr == nil && recovered.containerID != "" { + return finish(recovered) + } + return stagingVolumePin{}, errors.Join(fmt.Errorf("pin staging volumes: docker returned an empty container ID"), reconcileErr) + } + if err := requireStagingVolumePin(ctx, runner, plan, volumes, pin); err != nil { + return stagingVolumePin{}, err + } + return finish(pin) +} + +func stagingVolumePinName(plan Plan, volumes []stagedVolume) string { + names := make([]string, 0, len(volumes)) + for _, volume := range volumes { + names = append(names, volume.VolumeName) + } + slices.Sort(names) + return "bort-pin-" + dockerNameSegment(plan.RunName, 24) + "-" + stagingHash(stagingOwner(plan), strings.Join(names, "\x00")) +} + +func releaseStagingVolumePin(ctx context.Context, runner dockerRunner, plan Plan, volumes []stagedVolume, pin stagingVolumePin) error { + ctx, cancel := context.WithTimeout(ctx, dockerStopTimeout) + defer cancel() + if _, err := runner.Output(ctx, "rm", "-f", pin.containerID); err != nil { + remaining, inspectErr := findStagingVolumePin(ctx, runner, plan, volumes, pin.name) + if inspectErr == nil && remaining.containerID == "" { + return nil + } + return errors.Join(fmt.Errorf("release staging volume pin %s: %w", pin.name, err), inspectErr) + } return nil } -func recreateStagingVolume(ctx context.Context, runner dockerRunner, plan Plan, appName string, volume stagedVolume) error { - exists, err := stagingVolumeExists(ctx, runner, plan, volume) +func validateCachedStagingVolumePin(ctx context.Context, runner dockerRunner, actx *applyContext, appName string) error { + pin, ok := actx.stagingVolumePins[appName] + if !ok { + if !stagingTransferStarted(actx.entry(appName)) { + return nil + } + _, reconciled, err := ensureAppStagingVolumePin(ctx, runner, actx, appName, false) + if err != nil { + return err + } + pin = reconciled + } + volumes := stagedVolumesForApp(actx.plan, appName) + if err := requireStagingVolumesOwned(ctx, runner, actx.plan, appName, volumes); err != nil { + return err + } + if err := requireStagingVolumePin(ctx, runner, actx.plan, volumes, pin); err != nil { + return err + } + return requireStagingVolumeAttachments(ctx, runner, volumes, []string{pin.containerID}) +} + +type releasableStagingPin struct { + pin stagingVolumePin + volumes []stagedVolume +} + +func (c *Client) ValidateStagingVolumePins(ctx context.Context, plan Plan, targetAuthority bool) error { + ctx, cancel := context.WithTimeout(ctx, targetDiscoveryTimeout) + defer cancel() + _, err := c.validatedStagingVolumePins(ctx, plan, targetAuthority, true) + return err +} + +func (c *Client) ReleaseStagingVolumePins(ctx context.Context, plan Plan, targetAuthority bool) error { + ctx, cancel := context.WithTimeout(ctx, targetDiscoveryTimeout) + defer cancel() + pins, err := c.validatedStagingVolumePins(ctx, plan, targetAuthority, false) if err != nil { return err } - if exists { - if _, err := runner.Output(ctx, "volume", "rm", "-f", volume.VolumeName); err != nil { - return fmt.Errorf("remove staging volume %s: %w", volume.VolumeName, err) + for _, item := range pins { + if err := releaseStagingVolumePin(ctx, c.dockerRunner(), plan, item.volumes, item.pin); err != nil { + return err } } - return ensureStagingVolume(ctx, runner, plan, appName, volume) + return nil } -func requireStagingVolumeUnattached(ctx context.Context, runner dockerRunner, volume stagedVolume) error { - out, err := runner.Output(ctx, "ps", "-a", "--filter", "volume="+volume.VolumeName, "-q") +func (c *Client) validatedStagingVolumePins(ctx context.Context, plan Plan, targetAuthority, requireHandoff bool) ([]releasableStagingPin, error) { + runner := c.dockerRunner() + containers, err := listContainersByLabels(ctx, runner, stagingVolumePinLabel+"=true", stagingVolumeRunIDLabel+"="+stagingOwner(plan)) if err != nil { - return fmt.Errorf("list containers using staging volume %s: %w", volume.VolumeName, err) + return nil, fmt.Errorf("find staging volume pins for recovery: %w", err) + } + + type expectedPin struct { + app string + volumes []stagedVolume + } + expected := map[string]expectedPin{} + for _, app := range plan.Prepare.Apps { + volumes := stagedVolumesForApp(plan, app.Name) + if len(volumes) == 0 { + continue + } + expected[stagingVolumePinName(plan, volumes)] = expectedPin{app: app.Name, volumes: volumes} } - if ids := strings.Fields(string(out)); len(ids) > 0 { - return fmt.Errorf("staging volume %s is attached to container(s) %s before Bort handed it to Dokploy; refusing to treat the transferred state as authoritative", volume.VolumeName, strings.Join(ids, ", ")) + + actx := &applyContext{plan: plan, cache: map[string]*appCache{}} + if err := actx.loadMigratedVolumeMounts(); err != nil { + return nil, err + } + for _, app := range plan.StagingTransferApps { + actx.entry(app).StagingTransferStarted = true + } + targetIdentitiesVerified := false + + discovered := map[string]stagingVolumePin{} + for _, container := range containers { + want, ok := expected[container.Name] + if !ok { + return nil, fmt.Errorf("run %q owns unexpected staging volume pin %s; refusing to remove an unverified container", plan.RunName, container.Name) + } + if _, duplicate := discovered[container.Name]; duplicate { + return nil, fmt.Errorf("run %q has more than one staging volume pin named %s", plan.RunName, container.Name) + } + pin := stagingVolumePin{name: container.Name, containerID: container.ID} + if err := validateStagingVolumePinIdentity(plan, want.volumes, pin, container); err != nil { + return nil, err + } + discovered[container.Name] = pin + } + + pins := make([]releasableStagingPin, 0, len(discovered)) + for name, want := range expected { + entry := actx.entry(want.app) + pin, hasPin := discovered[name] + if !hasPin && !stagingTransferStarted(entry) { + continue + } + if targetAuthority && !stagedVolumesRecorded(entry, want.volumes) { + return nil, fmt.Errorf("run %q has no complete durable migrated-volume record for app %s; refusing target-authority finalization", plan.RunName, want.app) + } + if targetAuthority && !hasPin && (!requireHandoff || slices.Contains(plan.HandedOffApps, want.app)) { + continue + } + if targetAuthority && !targetIdentitiesVerified { + if err := c.hydratePersistedTargetIdentities(ctx, actx); err != nil { + return nil, fmt.Errorf("verify persisted target identities before authority finalization: %w", err) + } + targetIdentitiesVerified = true + } + volumes := want.volumes + if !targetAuthority && !hasPin { + volumes = nil + for _, volume := range want.volumes { + exists, err := stagingVolumeExists(ctx, runner, plan, want.app, volume) + if err != nil { + return nil, fmt.Errorf("verify staging volume %s for app %s before source-authority finalization: %w", volume.VolumeName, want.app, err) + } + if exists { + volumes = append(volumes, volume) + } + } + if len(volumes) == 0 { + continue + } + } + if err := requireStagingVolumesOwned(ctx, runner, plan, want.app, volumes); err != nil { + return nil, fmt.Errorf("verify staging volumes for app %s before authority finalization: %w", want.app, err) + } + allowedIDsByVolume := make(map[string][]string, len(volumes)) + for _, volume := range volumes { + if hasPin { + allowedIDsByVolume[volume.VolumeName] = []string{pin.containerID} + } else { + allowedIDsByVolume[volume.VolumeName] = []string{} + } + } + if targetAuthority { + targetIDsByVolume, err := c.migratedVolumeAttachmentIDs(ctx, actx, want.app) + if err != nil { + return nil, fmt.Errorf("verify target attachments for app %s before authority finalization: %w", want.app, err) + } + for _, volume := range want.volumes { + targetIDs := targetIDsByVolume[volume.VolumeName] + if len(targetIDs) == 0 { + return nil, fmt.Errorf("verify target attachments for app %s before authority finalization: migrated volume %s has no target container", want.app, volume.VolumeName) + } + allowedIDsByVolume[volume.VolumeName] = append(allowedIDsByVolume[volume.VolumeName], targetIDs...) + } + } + if err := requireStagingVolumeAttachmentSets(ctx, runner, volumes, allowedIDsByVolume); err != nil { + if !targetAuthority { + return nil, fmt.Errorf("verify attachments for app %s before source-authority finalization: %w; remove every container attached to these staging volumes other than a bort-pin-* container (stopping is not enough because a stopped container keeps its mounts), then rerun this command", want.app, err) + } + return nil, fmt.Errorf("verify attachments for app %s before authority finalization: %w", want.app, err) + } + if hasPin { + pins = append(pins, releasableStagingPin{pin: pin, volumes: want.volumes}) + } + } + + return pins, nil +} + +func PlanStagesVolumes(plan Plan) bool { + for _, app := range plan.Prepare.Apps { + if len(stagedVolumesForApp(plan, app.Name)) > 0 { + return true + } + } + return false +} + +func IncompleteStagingTransfers(plan Plan) ([]string, error) { + actx := &applyContext{plan: plan, cache: map[string]*appCache{}} + if err := actx.loadMigratedVolumeMounts(); err != nil { + return nil, err + } + for _, app := range plan.StagingTransferApps { + actx.entry(app).StagingTransferStarted = true + } + incomplete := []string{} + for _, app := range plan.Prepare.Apps { + volumes := stagedVolumesForApp(plan, app.Name) + entry := actx.entry(app.Name) + if len(volumes) > 0 && stagingTransferStarted(entry) && !stagedVolumesRecorded(entry, volumes) { + incomplete = append(incomplete, app.Name) + } + } + return incomplete, nil +} + +func reconcileStagingVolumePinsWithTimeout(runner dockerRunner, plan Plan, appName string, volumes []stagedVolume, expectedName string) (stagingVolumePin, error) { + ctx, cancel := context.WithTimeout(context.Background(), dockerStopTimeout) + defer cancel() + return reconcileStagingVolumePins(ctx, runner, plan, appName, volumes, expectedName) +} + +func reconcileStagingVolumePins(ctx context.Context, runner dockerRunner, plan Plan, appName string, volumes []stagedVolume, expectedName string) (stagingVolumePin, error) { + containers, err := listContainersByLabels(ctx, runner, stagingVolumePinLabel+"=true", stagingVolumeRunIDLabel+"="+stagingOwner(plan), stagingVolumeAppLabel+"="+appName) + if err != nil { + return stagingVolumePin{}, fmt.Errorf("find staging volume pins: %w", err) + } + for _, container := range containers { + if container.Name != expectedName { + continue + } + pin := stagingVolumePin{name: container.Name, containerID: container.ID} + if err := validateStagingVolumePinIdentity(plan, volumes, pin, container); err != nil { + return stagingVolumePin{}, err + } + if container.State.Running { + return pin, nil + } + _, startErr := runner.Output(ctx, "start", container.ID) + started, inspectErr := inspectContainer(ctx, runner, container.ID) + if inspectErr == nil { + if validateErr := validateStagingVolumePin(plan, volumes, pin, started); validateErr == nil { + return pin, nil + } else { + inspectErr = validateErr + } + } + if startErr != nil { + startErr = fmt.Errorf("restart stopped staging volume pin %s: %w", container.Name, startErr) + } + return stagingVolumePin{}, errors.Join(startErr, inspectErr) + } + return stagingVolumePin{}, nil +} + +func removeSupersededStagingVolumePins(ctx context.Context, runner dockerRunner, plan Plan, appName string, volumes []stagedVolume, current stagingVolumePin) error { + containers, err := listContainersByLabels(ctx, runner, stagingVolumePinLabel+"=true", stagingVolumeRunIDLabel+"="+stagingOwner(plan), stagingVolumeAppLabel+"="+appName) + if err != nil { + return fmt.Errorf("find superseded staging volume pins: %w", err) + } + wanted := make(map[string]struct{}, len(volumes)) + for _, volume := range volumes { + wanted[volume.VolumeName] = struct{}{} + } + for _, container := range containers { + if stagingContainerIDsMatch(container.ID, current.containerID) || !containerMountsAnyVolume(container, wanted) { + continue + } + if _, err := runner.Output(ctx, "rm", "-f", container.ID); err != nil { + return fmt.Errorf("remove superseded staging volume pin %s: %w", container.Name, err) + } + } + return nil +} + +func findStagingVolumePin(ctx context.Context, runner dockerRunner, plan Plan, volumes []stagedVolume, expectedName string) (stagingVolumePin, error) { + containers, err := listContainersByLabels(ctx, runner, stagingVolumePinLabel+"=true", stagingVolumeRunIDLabel+"="+stagingOwner(plan)) + if err != nil { + return stagingVolumePin{}, err + } + for _, container := range containers { + if container.Name != expectedName || strings.TrimSpace(container.Config.Labels[stagingVolumeRunIDLabel]) != stagingOwner(plan) { + continue + } + pin := stagingVolumePin{name: container.Name, containerID: container.ID} + if err := validateStagingVolumePin(plan, volumes, pin, container); err != nil { + return stagingVolumePin{}, err + } + return pin, nil + } + return stagingVolumePin{}, nil +} + +func requireStagingVolumePin(ctx context.Context, runner dockerRunner, plan Plan, volumes []stagedVolume, pin stagingVolumePin) error { + container, err := inspectContainer(ctx, runner, pin.containerID) + if err != nil { + return err + } + return validateStagingVolumePin(plan, volumes, pin, container) +} + +func validateStagingVolumePin(plan Plan, volumes []stagedVolume, pin stagingVolumePin, container dockerContainer) error { + if err := validateStagingVolumePinIdentity(plan, volumes, pin, container); err != nil { + return err + } + if !container.State.Running { + return fmt.Errorf("staging volume pin %s is not running", pin.name) } return nil } +func validateStagingVolumePinIdentity(plan Plan, volumes []stagedVolume, pin stagingVolumePin, container dockerContainer) error { + if container.ID == "" || !stagingContainerIDsMatch(pin.containerID, container.ID) || container.Name != pin.name || container.Name != stagingVolumePinName(plan, volumes) { + return fmt.Errorf("staging volume pin %s resolved to an unexpected container", pin.name) + } + if container.Config.Labels[stagingVolumePinLabel] != "true" || strings.TrimSpace(container.Config.Labels[stagingVolumeRunIDLabel]) != stagingOwner(plan) { + return fmt.Errorf("staging volume pin %s is not owned by run %q", pin.name, plan.RunName) + } + if normalizedRestartPolicyName(container.HostConfig.RestartPolicy.Name) != "unless-stopped" { + return fmt.Errorf("staging volume pin %s does not have restart policy unless-stopped", pin.name) + } + wanted := make(map[string]struct{}, len(volumes)) + for _, volume := range volumes { + wanted[volume.VolumeName] = struct{}{} + } + seen := map[string]struct{}{} + for _, mount := range container.Mounts { + if mount.Type != "volume" { + continue + } + if _, ok := wanted[mount.Name]; !ok || mount.RW { + return fmt.Errorf("staging volume pin %s has unexpected or writable volume mount %s", pin.name, mount.Name) + } + seen[mount.Name] = struct{}{} + } + if len(seen) != len(wanted) { + return fmt.Errorf("staging volume pin %s mounts %d owned volumes, want %d", pin.name, len(seen), len(wanted)) + } + return nil +} + +func containerMountsAnyVolume(container dockerContainer, names map[string]struct{}) bool { + for _, mount := range container.Mounts { + if mount.Type == "volume" { + if _, ok := names[mount.Name]; ok { + return true + } + } + } + return false +} + +func clearStagingVolume(ctx context.Context, runner dockerRunner, plan Plan, appName string, volume stagedVolume) error { + if err := requireStagingVolumeOwned(ctx, runner, plan, appName, volume); err != nil { + return err + } + if err := runner.Run(ctx, nil, nil, + "run", "--rm", "--network", "none", + "-v", volume.VolumeName+":/volume", + volumeCopyImage, + "sh", "-c", "find /volume -mindepth 1 -delete && sync", + ); err != nil { + return fmt.Errorf("clear staging volume %s: %w", volume.VolumeName, err) + } + return nil +} + +func stagingVolumeAttachments(ctx context.Context, runner dockerRunner, volume stagedVolume) ([]string, error) { + args := []string{"ps", "-a", "--filter", "volume=" + volume.VolumeName, "--format", "{{.ID}}"} + out, err := runner.Output(ctx, args...) + if err != nil { + return nil, fmt.Errorf("list containers using staging volume %s: %w", volume.VolumeName, err) + } + return strings.Fields(string(out)), nil +} + +func requireStagingVolumeAttachments(ctx context.Context, runner dockerRunner, volumes []stagedVolume, allowedIDs []string) error { + allowedByVolume := make(map[string][]string, len(volumes)) + for _, volume := range volumes { + allowedByVolume[volume.VolumeName] = allowedIDs + } + return requireStagingVolumeAttachmentSets(ctx, runner, volumes, allowedByVolume) +} + +func requireStagingVolumeAttachmentSets(ctx context.Context, runner dockerRunner, volumes []stagedVolume, allowedByVolume map[string][]string) error { + attachedByVolume, err := stagingVolumeAttachmentSets(ctx, runner, volumes) + if err != nil { + return err + } + for _, volume := range volumes { + allowedIDs, ok := allowedByVolume[volume.VolumeName] + if !ok { + return fmt.Errorf("staging volume %s has no expected attachment set; refusing an incomplete handoff", volume.VolumeName) + } + if err := requireExactStagingVolumeAttachments(volume.VolumeName, attachedByVolume[volume.VolumeName], allowedIDs); err != nil { + return err + } + } + return nil +} + +func stagingVolumeAttachmentSets(ctx context.Context, runner dockerRunner, volumes []stagedVolume) (map[string][]string, error) { + attachedByVolume := make(map[string][]string, len(volumes)) + if len(volumes) == 0 { + return attachedByVolume, nil + } + if len(volumes) == 1 { + ids, err := stagingVolumeAttachments(ctx, runner, volumes[0]) + if err != nil { + return nil, err + } + attachedByVolume[volumes[0].VolumeName] = ids + return attachedByVolume, nil + } + wanted := make(map[string]struct{}, len(volumes)) + args := []string{"ps", "-a"} + for _, volume := range volumes { + wanted[volume.VolumeName] = struct{}{} + attachedByVolume[volume.VolumeName] = nil + args = append(args, "--filter", "volume="+volume.VolumeName) + } + args = append(args, "--format", "{{.ID}}") + out, err := runner.Output(ctx, args...) + if err != nil { + return nil, fmt.Errorf("list containers using staging volumes: %w", err) + } + ids := strings.Fields(string(out)) + if len(ids) == 0 { + return attachedByVolume, nil + } + containers, err := inspectContainers(ctx, runner, ids) + if err != nil { + return nil, err + } + seen := make(map[string]map[string]struct{}, len(volumes)) + for _, container := range containers { + for _, mount := range container.Mounts { + if mount.Type != "volume" { + continue + } + if _, ok := wanted[mount.Name]; !ok { + continue + } + if seen[mount.Name] == nil { + seen[mount.Name] = map[string]struct{}{} + } + if _, duplicate := seen[mount.Name][container.ID]; duplicate { + continue + } + seen[mount.Name][container.ID] = struct{}{} + attachedByVolume[mount.Name] = append(attachedByVolume[mount.Name], container.ID) + } + } + for volumeName := range attachedByVolume { + slices.Sort(attachedByVolume[volumeName]) + } + return attachedByVolume, nil +} + +func requireExactStagingVolumeAttachments(volumeName string, ids, allowedIDs []string) error { + for _, attachedID := range ids { + allowed := false + for _, id := range allowedIDs { + if stagingContainerIDsMatch(id, attachedID) { + allowed = true + break + } + } + if !allowed { + return fmt.Errorf("staging volume %s is attached to unexpected container %s; refusing a handoff with an unverified writer", volumeName, attachedID) + } + } + for _, allowedID := range allowedIDs { + attached := false + for _, id := range ids { + if stagingContainerIDsMatch(allowedID, id) { + attached = true + break + } + } + if !attached { + return fmt.Errorf("expected staging volume %s attachment to container %s is missing; refusing an incomplete handoff", volumeName, allowedID) + } + } + return nil +} + +func stagingContainerIDsMatch(first, second string) bool { + return sourceContainerIDMatches(first, second) || sourceContainerIDMatches(second, first) +} + +func recoveryStatusCommand(plan Plan) string { + if command := strings.TrimSpace(plan.RecoveryCommand); command != "" { + return command + } + return "bort status" +} + const defaultPostgresDataDir = "/var/lib/postgresql/data" // requireStagedPostgresDataDir refuses a staging restore whose data @@ -281,7 +912,7 @@ func requirePlannedPostgresDataDirStaged(composeFile, stagingCompose string, app if len(staged) == 0 { return fmt.Errorf("%w: data store service %s for app %s mounts no named volume, so its restore would be lost when the staging container stops; choose a recreate or managed data store strategy or change the source compose before live apply", ErrNotImplemented, service, app.Name) } - if err := requireLiteralMountTargets(composeFile, app.Name, service); err != nil { + if err := requireStagedComposeInputs(composeFile, app.Name, service); err != nil { return fmt.Errorf("%w; choose a recreate or managed data store strategy or change the source compose before live apply", err) } var doc yaml.Node @@ -293,7 +924,7 @@ func requirePlannedPostgresDataDirStaged(composeFile, stagingCompose string, app return err } raw := strings.TrimSpace(composeServiceEnvValue(mappingValue(mappingValue(root, "services"), service), "PGDATA")) - if raw == "" || strings.Contains(raw, "$") { + if raw == "" || hasUnescapedComposeDollar(raw) { return nil } if err := requireSourceMountsStageDataDir(app, service, path.Clean(raw), staged); err != nil { @@ -302,29 +933,56 @@ func requirePlannedPostgresDataDirStaged(composeFile, stagingCompose string, app return nil } -func requireLiteralMountTargets(composeFile, appName, service string) error { - if target, ok := composeServiceInterpolatedMountTarget(composeFile, service); ok { +func requireStagedComposeInputs(composeFile, appName, service string) error { + entry := resolvedComposeServiceNode(composeFile, service) + if target, ok := composeServiceInterpolatedMountTarget(entry); ok { return fmt.Errorf("%w: data store service %s for app %s mounts %q at an interpolated path, so the postgres data directory cannot be checked", ErrNotImplemented, service, appName, target) } + if input, ok := composeServiceExternalInput(entry); ok { + return fmt.Errorf("%w: data store service %s for app %s uses Compose %s, which Bort cannot preserve in its isolated staging service", ErrNotImplemented, service, appName, input) + } return nil } -func composeServiceInterpolatedMountTarget(composeFile, service string) (string, bool) { +func resolvedComposeServiceNode(composeFile, service string) *yaml.Node { var doc yaml.Node if err := yaml.Unmarshal([]byte(composeFile), &doc); err != nil { - return "", false + return nil } root, err := composeRoot(&doc) if err != nil { - return "", false + return nil } entry := mappingValue(mappingValue(root, "services"), service) if entry == nil { - return "", false + return nil } - if entry, err = selfContainedNode(entry, map[*yaml.Node]bool{}); err != nil { - return "", false + if entry, err = selfContainedNode(entry); err != nil { + return nil } + return entry +} + +func composeServiceExternalInput(entry *yaml.Node) (string, bool) { + for _, key := range []string{"secrets", "configs"} { + value := mappingValue(entry, key) + if value == nil { + continue + } + if value.Kind == yaml.SequenceNode || value.Kind == yaml.MappingNode { + if len(value.Content) > 0 { + return key, true + } + continue + } + if strings.TrimSpace(value.Value) != "" && value.Tag != "!!null" { + return key, true + } + } + return "", false +} + +func composeServiceInterpolatedMountTarget(entry *yaml.Node) (string, bool) { volumes := mappingValue(entry, "volumes") if volumes == nil || volumes.Kind != yaml.SequenceNode { return "", false @@ -343,13 +1001,17 @@ func composeServiceInterpolatedMountTarget(composeFile, service string) (string, target = dest.Value } } - if strings.Contains(target, "$") { + if hasUnescapedComposeDollar(target) { return strings.TrimSpace(target), true } } return "", false } +func hasUnescapedComposeDollar(value string) bool { + return strings.Contains(strings.ReplaceAll(value, "$$", ""), "$") +} + // splitComposeVolumeShortSyntax splits SOURCE:TARGET[:MODE] on colons that // are not inside a ${...} reference, where ${VAR:-default} keeps its colon. func splitComposeVolumeShortSyntax(value string) []string { @@ -394,10 +1056,16 @@ func requireSourceMountsStageDataDir(app preparer.AppPlan, service, dataDir stri if !found { return fmt.Errorf("%w: postgres data directory %s for service %s in app %s is not mounted from a named volume, so the restore would be lost when the staging container stops", ErrNotImplemented, dataDir, service, app.Name) } - if stagedVolumeMount(mount, staged) { - return nil + if !stagedVolumeMount(mount, staged) { + return fmt.Errorf("%w: postgres data directory %s for service %s in app %s is mounted from %s %q at %s, not a named volume, so the restore would be lost when the staging container stops", ErrNotImplemented, dataDir, service, app.Name, mount.Type, firstNonEmpty(mount.Name, mount.Source), mount.Target) + } + for _, candidate := range app.Resources.Volumes { + if candidate.Service != service || candidate.Type != "volume" || path.Clean(candidate.Target) == path.Clean(mount.Target) || mountCoversPath(dataDir, candidate.Target) { + continue + } + return fmt.Errorf("%w: named volume %q at %s for service %s in app %s is outside postgres data directory %s, so a logical restore cannot preserve its contents", ErrNotImplemented, firstNonEmpty(candidate.Name, candidate.Source), candidate.Target, service, app.Name, dataDir) } - return fmt.Errorf("%w: postgres data directory %s for service %s in app %s is mounted from %s %q at %s, not a named volume, so the restore would be lost when the staging container stops", ErrNotImplemented, dataDir, service, app.Name, mount.Type, firstNonEmpty(mount.Name, mount.Source), mount.Target) + return nil } func stagedVolumeMount(mount preparer.VolumeResource, staged []stagedVolume) bool { @@ -559,8 +1227,9 @@ func composeServiceInitBindMounts(entry *yaml.Node) ([]*yaml.Node, error) { return nil, fmt.Errorf("init script mount %s -> %s must use an absolute host path so the staged restore runs the same scripts as the deploy", source, target) } if item.Kind == yaml.MappingNode { - ensureMappingBool(item, "read_only", true) - mounts = append(mounts, item) + mount := cloneNode(item) + ensureMappingBool(mount, "read_only", true) + mounts = append(mounts, mount) continue } mounts = append(mounts, stringNode(source+":"+target+":"+strings.Join(append(withoutAccessMode(options), "ro"), ","))) @@ -568,6 +1237,15 @@ func composeServiceInitBindMounts(entry *yaml.Node) ([]*yaml.Node, error) { return mounts, nil } +func cloneNode(node *yaml.Node) *yaml.Node { + copied := *node + copied.Content = make([]*yaml.Node, len(node.Content)) + for i, child := range node.Content { + copied.Content[i] = cloneNode(child) + } + return &copied +} + func withoutAccessMode(options []string) []string { kept := options[:0:0] for _, option := range options { @@ -647,7 +1325,7 @@ func stagingComposeFile(composeFile, service string, staged []stagedVolume) (str services := mappingValue(root, "services") entry := mappingValue(services, service) if entry != nil { - if entry, err = selfContainedNode(entry, map[*yaml.Node]bool{}); err != nil { + if entry, err = selfContainedNode(entry); err != nil { return "", fmt.Errorf("compose service %s: %w", service, err) } } @@ -692,21 +1370,38 @@ func stagingComposeFile(composeFile, service string, staged []stagedVolume) (str // selfContainedNode copies a subtree with every alias expanded and every // merge key applied, so it stays valid after the anchors it referenced are -// pruned from the document. -func selfContainedNode(node *yaml.Node, expanding map[*yaml.Node]bool) (*yaml.Node, error) { +// pruned from the document. Expansions of one anchor are shared between its +// reference sites, so callers may edit the returned root but must treat +// nodes nested inside it as read-only. +func selfContainedNode(node *yaml.Node) (*yaml.Node, error) { + return expandSelfContained(node, map[*yaml.Node]bool{}, map[*yaml.Node]*yaml.Node{}) +} + +func expandSelfContained(node *yaml.Node, expanding map[*yaml.Node]bool, memo map[*yaml.Node]*yaml.Node) (*yaml.Node, error) { if node.Kind == yaml.AliasNode { - if node.Alias == nil || expanding[node.Alias] { + if node.Alias == nil { + return nil, fmt.Errorf("alias *%s cannot be expanded", node.Value) + } + if expanded, ok := memo[node.Alias]; ok { + return expanded, nil + } + if expanding[node.Alias] { return nil, fmt.Errorf("alias *%s cannot be expanded", node.Value) } expanding[node.Alias] = true defer delete(expanding, node.Alias) - return selfContainedNode(node.Alias, expanding) + expanded, err := expandSelfContained(node.Alias, expanding, memo) + if err != nil { + return nil, err + } + memo[node.Alias] = expanded + return expanded, nil } copied := *node copied.Anchor = "" copied.Content = nil for _, child := range node.Content { - resolved, err := selfContainedNode(child, expanding) + resolved, err := expandSelfContained(child, expanding, memo) if err != nil { return nil, err } diff --git a/internal/target/dokploy/staging_test.go b/internal/target/dokploy/staging_test.go index 2cb606a..10ae75b 100644 --- a/internal/target/dokploy/staging_test.go +++ b/internal/target/dokploy/staging_test.go @@ -10,10 +10,12 @@ import ( "net/http/httptest" "os" "path/filepath" + "reflect" "runtime" "slices" "strings" "testing" + "time" "github.com/aikins01/bort/internal/preparer" "gopkg.in/yaml.v3" @@ -24,6 +26,51 @@ type sequencedOutputRunner struct { sequences map[string][][]byte } +type deadlineRecordingRunner struct { + fakeDockerRunner + deadlines []time.Time +} + +type foreignVolumeAfterCreateRunner struct { + fakeDockerRunner + volumeName string + created bool +} + +func (r *deadlineRecordingRunner) Output(ctx context.Context, args ...string) ([]byte, error) { + deadline, ok := ctx.Deadline() + if !ok { + return nil, errors.New("docker call has no deadline") + } + r.deadlines = append(r.deadlines, deadline) + return r.fakeDockerRunner.Output(ctx, args...) +} + +func (r *deadlineRecordingRunner) Run(ctx context.Context, stdin io.Reader, stdout io.Writer, args ...string) error { + deadline, ok := ctx.Deadline() + if !ok { + return errors.New("docker call has no deadline") + } + r.deadlines = append(r.deadlines, deadline) + return r.fakeDockerRunner.Run(ctx, stdin, stdout, args...) +} + +func (r *foreignVolumeAfterCreateRunner) Output(ctx context.Context, args ...string) ([]byte, error) { + key := strings.Join(args, " ") + if key == "volume inspect "+r.volumeName { + r.outputArgs = append(r.outputArgs, append([]string{}, args...)) + if !r.created { + return nil, errors.New("Error response from daemon: volume " + r.volumeName + " not found") + } + return []byte(`[{"Name":"` + r.volumeName + `","Labels":{"bort.run-id":"other-run"}}]`), nil + } + out, err := r.fakeDockerRunner.Output(ctx, args...) + if err == nil && len(args) > 2 && args[0] == "volume" && args[1] == "create" && args[len(args)-1] == r.volumeName { + r.created = true + } + return out, err +} + func (r *sequencedOutputRunner) Output(ctx context.Context, args ...string) ([]byte, error) { key := strings.Join(args, " ") if seq, ok := r.sequences[key]; ok && len(seq) > 0 { @@ -85,9 +132,18 @@ func TestRewriteComposeStagedVolumesHandlesShortAndLongSyntax(t *testing.T) { } services := doc["services"].(map[string]any) dbVolumes := services["db"].(map[string]any)["volumes"].([]any) - if len(dbVolumes) != 2 || dbVolumes[0] != "pgdata:/var/lib/postgresql/data" { + wantDBVolumes := []any{"pgdata:/var/lib/postgresql/data", "./local:/host"} + if !reflect.DeepEqual(dbVolumes, wantDBVolumes) { t.Fatalf("service mounts must be left untouched, got %#v", dbVolumes) } + webVolumes := services["web"].(map[string]any)["volumes"].([]any) + wantWebVolumes := []any{ + map[string]any{"type": "volume", "source": "uploads", "target": "/uploads"}, + map[string]any{"type": "bind", "source": "./cfg", "target": "/cfg"}, + } + if !reflect.DeepEqual(webVolumes, wantWebVolumes) { + t.Fatalf("long-form service mounts must be left untouched, got %#v", webVolumes) + } _, err = rewriteComposeStagedVolumes(compose, []stagedVolume{{Service: "web", Target: "/cfg", VolumeName: "bort-x"}}) if err == nil || !strings.Contains(err.Error(), "no named volume mounted at /cfg") { @@ -275,6 +331,19 @@ func TestLocalDockerRunnerIsolatesComposeEnvironment(t *testing.T) { } t.Setenv("BORT_SHELL_LEAK", "1") runner := localDockerRunner{Path: shim} + command, err := runner.command(context.Background(), []string{"compose", "-p", "stage", "config"}) + if err != nil { + t.Fatalf("build compose command: %v", err) + } + keys := make([]string, 0, len(command.Env)) + for _, entry := range command.Env { + key, _, _ := strings.Cut(entry, "=") + keys = append(keys, key) + } + slices.Sort(keys) + if !slices.Equal(keys, []string{"HOME", "PATH"}) { + t.Fatalf("compose command environment keys must be exactly HOME and PATH, got %v", keys) + } compose, err := runner.Output(context.Background(), "compose", "-p", "stage", "config") if err != nil { t.Fatalf("compose shim: %v", err) @@ -302,10 +371,12 @@ func TestValidatePlanReadyForLiveApplyRefusesComposeProjectNameInStagedDataStore compose string refuse bool }{ - "braced reference in db": {compose: strings.Replace(original, "${DB_PASSWORD}", "${COMPOSE_PROJECT_NAME}_db", 1), refuse: true}, - "bare reference in db": {compose: strings.Replace(original, "${DB_PASSWORD}", "$COMPOSE_PROJECT_NAME", 1), refuse: true}, - "longer variable name in db": {compose: strings.Replace(original, "${DB_PASSWORD}", "${COMPOSE_PROJECT_NAME_SUFFIX}", 1), refuse: false}, - "reference in other service": {compose: strings.Replace(original, "image: example/web\n", "image: example/web\n environment:\n APP: ${COMPOSE_PROJECT_NAME}\n", 1), refuse: false}, + "braced reference in db": {compose: strings.Replace(original, "${DB_PASSWORD}", "${COMPOSE_PROJECT_NAME}_db", 1), refuse: true}, + "bare reference in db": {compose: strings.Replace(original, "${DB_PASSWORD}", "$COMPOSE_PROJECT_NAME", 1), refuse: true}, + "escaped reference in db": {compose: strings.Replace(original, "${DB_PASSWORD}", "$${COMPOSE_PROJECT_NAME}_db", 1), refuse: false}, + "reference before escaped dollar in db": {compose: strings.Replace(original, "${DB_PASSWORD}", "$COMPOSE_PROJECT_NAME$$archive", 1), refuse: true}, + "longer variable name in db": {compose: strings.Replace(original, "${DB_PASSWORD}", "${COMPOSE_PROJECT_NAME_SUFFIX}", 1), refuse: false}, + "reference in other service": {compose: strings.Replace(original, "image: example/web\n", "image: example/web\n environment:\n APP: ${COMPOSE_PROJECT_NAME}\n", 1), refuse: false}, } { t.Run(name, func(t *testing.T) { if tc.compose == original { @@ -347,6 +418,23 @@ func TestValidatePlanReadyForLiveApplyRefusesRelativeInitScriptMountBeforePause( } } +func TestValidatePlanReadyForLiveApplyRefusesStagedComposeSecretsAndConfigsBeforePause(t *testing.T) { + for _, input := range []string{"secrets", "configs"} { + t.Run(input, func(t *testing.T) { + plan, _, _ := stagedRestoreFixture(t) + composePath := filepath.Join(plan.Prepare.BundleDir, "api", "compose.yaml") + original := string(plan.BundleFiles[filepath.Clean(composePath)]) + compose := strings.Replace(original, " volumes:\n", " "+input+":\n - db_input\n volumes:\n", 1) + "\n" + input + ":\n db_input:\n file: /run/bort-db-input\n" + plan.BundleFiles[filepath.Clean(composePath)] = []byte(compose) + + err := ValidatePlannedPostgresDataDirs(plan) + if !errors.Is(err, ErrNotImplemented) || !strings.Contains(err.Error(), "uses Compose "+input) || !strings.Contains(err.Error(), "cannot preserve") { + t.Fatalf("expected staged %s refusal before live apply, got %v", input, err) + } + }) + } +} + func stagingCompatibleClient(t *testing.T, runner dockerRunner, createEnvFile bool, command string) *Client { t.Helper() server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -385,116 +473,1338 @@ func TestStagedRestoreRefusesIncompatibleDokployComposeSettings(t *testing.T) { } }) } -} +} + +func TestApplyCreateServiceRefusesAdoptedComposeWithoutEnvFile(t *testing.T) { + plan, _, _ := stagedRestoreFixture(t) + disabled := false + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && r.URL.Path == "/api/compose.search": + _ = json.NewEncoder(w).Encode(composeSearchResponse{Items: []Compose{{ComposeID: "c1", Name: "api", AppName: "stack-1"}}, Total: 1}) + case r.Method == http.MethodGet && r.URL.Path == "/api/compose.one": + _ = json.NewEncoder(w).Encode(Compose{ComposeID: "c1", Name: "api", AppName: "stack-1", CreateEnvFile: &disabled}) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + client := &Client{BaseURL: server.URL, Token: "secret", HTTPClient: server.Client(), Docker: &fakeDockerRunner{}} + actx := &applyContext{plan: plan, cache: map[string]*appCache{"api": {EnvironmentID: "env1"}}} + err := client.applyCreateService(context.Background(), actx, Step{Kind: StepCreateService, App: "api", Ref: "api"}) + if !errors.Is(err, ErrNotImplemented) || !strings.Contains(err.Error(), ".env creation disabled") { + t.Fatalf("expected adopted compose refusal before the source pause, got %v", err) + } +} + +func TestDeployComposeForApplyRefusesIncompatibleSettingsAfterStagedRestore(t *testing.T) { + plan, _, _ := stagedRestoreFixture(t) + disabled := false + var writes []string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && r.URL.Path == "/api/compose.one": + _ = json.NewEncoder(w).Encode(Compose{ComposeID: "c1", Name: "api", AppName: "stack-1", CreateEnvFile: &disabled}) + default: + writes = append(writes, r.URL.Path) + _ = json.NewEncoder(w).Encode(map[string]any{}) + } + })) + defer server.Close() + client := &Client{BaseURL: server.URL, Token: "secret", HTTPClient: server.Client(), Docker: &fakeDockerRunner{}} + actx := &applyContext{plan: plan, cache: map[string]*appCache{}} + actx.entry("api").ComposeID = "c1" + actx.entry("api").ComposeAppName = "stack-1" + err := client.deployComposeForApply(context.Background(), actx, "api", "services: {}\n", "DB_PASSWORD=s3cret\n") + if !errors.Is(err, ErrNotImplemented) || !strings.Contains(err.Error(), ".env creation disabled") { + t.Fatalf("expected deploy-time refusal, got %v", err) + } + if len(writes) != 0 { + t.Fatalf("compose was written or deployed despite the refusal: %v", writes) + } +} + +func stagedSyncFixture(t *testing.T) (preparer.AppPlan, Plan, Step, stagedVolume) { + t.Helper() + app := preparer.AppPlan{Name: "api"} + app.Resources.SourceServices = []preparer.SourceServiceRef{{ServiceName: "web", ContainerID: "src-id", ContainerName: "coolify-web"}} + app.Resources.Volumes = []preparer.VolumeResource{{ + Service: "web", + Type: "volume", + Name: "src-vol", + Target: "/data", + SourceContainerID: "src-id", + SourceContainerName: "coolify-web", + }} + step := Step{Kind: StepSyncVolume, App: "api", Ref: "volume:web -> /data"} + plan := stagedPlan(t, app, t.TempDir(), Step{Kind: StepPauseSource, App: "api"}, step) + staged, ok := stagedVolumeFor(plan, "api", app.Resources.Volumes[0]) + if !ok { + t.Fatalf("volume not staged in plan %v", stepKinds(plan.Steps)) + } + return app, plan, step, staged +} + +func stoppedSourceInspect(startedAt, finishedAt string) []byte { + return []byte(`[{"Id":"src-id","Name":"/coolify-web","State":{"Running":false,"StartedAt":"` + startedAt + `","FinishedAt":"` + finishedAt + `"}}]`) +} + +func ownedStagingVolumeInspect(plan Plan, volume stagedVolume) []byte { + owner := plan.RunID + if owner == "" { + owner = plan.RunName + } + return stagingVolumeInspect(plan, volume, owner) +} + +func stagingVolumeInspect(plan Plan, volume stagedVolume, owner string) []byte { + labels := map[string]string{ + "bort.run": plan.RunName, + "bort.run-id": owner, + "bort.app": "api", + "bort.service": volume.Service, + "bort.target": volume.Target, + } + state, _ := json.Marshal([]stagingVolumeState{{Name: volume.VolumeName, Labels: labels}}) + return state +} + +func targetAuthorityTestClient(t *testing.T, runner dockerRunner, plan *Plan) *Client { + t.Helper() + const projectID, environmentID, composeID, composeAppName = "project-1", "environment-1", "compose-1", "stack-1" + plan.TargetIdentities = map[string]TargetIdentity{"api": { + ProjectID: projectID, EnvironmentID: environmentID, ComposeID: composeID, ComposeAppName: composeAppName, + }} + plan.Prepare.Apps[0].TargetResources = &preparer.TargetResources{Dokploy: &preparer.DokployResources{}} + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/project.one": + _ = json.NewEncoder(w).Encode(Project{ProjectID: projectID, Environments: []ProjectEnvironment{{EnvironmentID: environmentID}}}) + case "/api/compose.one": + _ = json.NewEncoder(w).Encode(Compose{ComposeID: composeID, EnvironmentID: environmentID, AppName: composeAppName}) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(server.Close) + return &Client{BaseURL: server.URL, Token: "secret", HTTPClient: server.Client(), Docker: runner} +} + +func TestRequireStagingVolumeOwnedRejectsMismatchedIdentityLabel(t *testing.T) { + _, plan, _, staged := stagedSyncFixture(t) + for label, value := range map[string]string{ + "bort.run": "other-run", + "bort.app": "other-app", + "bort.service": "worker", + "bort.target": "/other", + } { + t.Run(label, func(t *testing.T) { + labels := map[string]string{ + "bort.run": plan.RunName, + "bort.run-id": plan.RunName, + "bort.app": "api", + "bort.service": staged.Service, + "bort.target": staged.Target, + } + labels[label] = value + state, err := json.Marshal([]stagingVolumeState{{Name: staged.VolumeName, Labels: labels}}) + if err != nil { + t.Fatal(err) + } + runner := &fakeDockerRunner{outputs: map[string][]byte{"volume inspect " + staged.VolumeName: state}} + err = requireStagingVolumeOwned(context.Background(), runner, plan, "api", staged) + if err == nil || !strings.Contains(err.Error(), "label "+label) { + t.Fatalf("expected %s mismatch refusal, got %v", label, err) + } + }) + } +} + +func TestApplyCreateVolumeCreatesOnlyItsOwnStagedVolume(t *testing.T) { + app := preparer.AppPlan{Name: "api"} + app.Resources.Volumes = []preparer.VolumeResource{ + {Service: "web", Type: "volume", Name: "src-data", Target: "/data"}, + {Service: "web", Type: "volume", Name: "", Target: "/cache"}, + } + plan := stagedPlan(t, app, t.TempDir(), + Step{Kind: StepCreateVolume, App: "api", Ref: "src-data"}, + Step{Kind: StepCreateVolume, App: "api", Ref: "/cache"}, + Step{Kind: StepPauseSource, App: "api"}, + Step{Kind: StepSyncVolume, App: "api", Ref: "volume:web -> /data"}, + Step{Kind: StepSyncVolume, App: "api", Ref: "volume:web -> /cache"}, + ) + staged := stagedVolumesForApp(plan, "api") + if len(staged) != 2 { + t.Fatalf("expected two staged volumes, got %#v", staged) + } + runner := &fakeDockerRunner{outputs: map[string][]byte{"volume create": []byte("created\n")}, outputErrs: map[string]error{}} + for _, volume := range staged { + runner.outputErrs["volume inspect --format {{index .Labels \"bort.run-id\"}} "+volume.VolumeName] = errors.New("Error response from daemon: volume " + volume.VolumeName + " not found") + runner.outputs["volume inspect "+volume.VolumeName] = ownedStagingVolumeInspect(plan, volume) + } + client := &Client{Docker: runner} + actx := &applyContext{cache: map[string]*appCache{}, plan: plan} + if err := client.applyCreateVolume(context.Background(), actx, plan.Steps[1]); err != nil { + t.Fatalf("applyCreateVolume: %v", err) + } + var created []string + for _, args := range runner.outputArgs { + if len(args) > 2 && args[0] == "volume" && args[1] == "create" { + created = append(created, args[len(args)-1]) + } + } + if len(created) != 1 || created[0] != staged[1].VolumeName { + t.Fatalf("expected only %s to be created for the /cache step, got %v", staged[1].VolumeName, created) + } +} + +func TestEnsureStagingVolumeRejectsConcurrentForeignCreate(t *testing.T) { + _, plan, _, staged := stagedSyncFixture(t) + runner := &foreignVolumeAfterCreateRunner{ + fakeDockerRunner: fakeDockerRunner{outputs: map[string][]byte{"volume create": []byte(staged.VolumeName + "\n")}}, + volumeName: staged.VolumeName, + } + if err := ensureStagingVolume(context.Background(), runner, plan, "api", staged); err == nil || !strings.Contains(err.Error(), "not the staged") { + t.Fatalf("expected concurrent foreign volume to be rejected, got %v", err) + } + if !runner.created { + t.Fatalf("foreign replacement was exposed before volume creation: calls=%v", runner.outputArgs) + } +} + +func TestAcquireStagingVolumePinVerifiesOwnershipAndKeepsRunning(t *testing.T) { + _, plan, _, staged := stagedSyncFixture(t) + runner := &fakeDockerRunner{outputs: map[string][]byte{ + "volume inspect " + staged.VolumeName: ownedStagingVolumeInspect(plan, staged), + }} + pin, err := acquireStagingVolumePin(context.Background(), runner, plan, "api", []stagedVolume{staged}, true) + if err != nil || pin.containerID != "pin-id" { + t.Fatalf("expected an owned staging volume pin, pin=%#v err=%v", pin, err) + } + if !runner.activePins[pin.containerID].State.Running || fakeOutputCalled(runner, "rm", "-f", pin.containerID) { + t.Fatalf("staging volume pin was not left running: active=%#v calls=%v", runner.activePins, runner.outputArgs) + } + pinRuns := 0 + for _, args := range runner.outputArgs { + if len(args) > 1 && args[0] == "run" && args[1] == "-d" { + pinRuns++ + if got := strings.Join(args, " "); !strings.HasSuffix(got, volumeCopyImage+" sh -c while :; do sleep 2147483647; done") { + t.Fatalf("pin does not use the long-running keepalive command: %s", got) + } + } + } + if pinRuns != 1 { + t.Fatalf("pin creation calls = %d, want 1: %v", pinRuns, runner.outputArgs) + } +} + +func TestAcquireStagingVolumePinRejectsReplacementBeforeOperation(t *testing.T) { + _, plan, _, staged := stagedSyncFixture(t) + runner := &fakeDockerRunner{outputs: map[string][]byte{ + "volume inspect " + staged.VolumeName: []byte(`[{"Name":"` + staged.VolumeName + `","Labels":{"bort.run-id":"other-run"}}]`), + }} + _, err := acquireStagingVolumePin(context.Background(), runner, plan, "api", []stagedVolume{staged}, true) + if err == nil || !strings.Contains(err.Error(), "not the staged") { + t.Fatalf("expected replacement volume rejection, got %v", err) + } +} + +type replaceStagingVolumeAfterPinRunner struct { + *fakeDockerRunner + volumeName string +} + +func (r *replaceStagingVolumeAfterPinRunner) Output(ctx context.Context, args ...string) ([]byte, error) { + out, err := r.fakeDockerRunner.Output(ctx, args...) + if err == nil && len(args) > 1 && args[0] == "run" && args[1] == "-d" && slices.Contains(args, stagingVolumePinLabel+"=true") { + r.outputs["volume inspect "+r.volumeName] = []byte(`[{"Name":"` + r.volumeName + `","Labels":{"bort.run":"run1","bort.run-id":"other-run","bort.app":"api","bort.service":"web","bort.target":"/data"}}]`) + } + return out, err +} + +func TestSyncVolumeToStagingRechecksOwnershipAfterPinCreation(t *testing.T) { + _, plan, step, staged := stagedSyncFixture(t) + runner := &replaceStagingVolumeAfterPinRunner{ + fakeDockerRunner: &fakeDockerRunner{outputs: map[string][]byte{ + "volume inspect src-vol": []byte(`[{"Name":"src-vol"}]`), + "volume inspect " + staged.VolumeName: ownedStagingVolumeInspect(plan, staged), + "volume inspect --format {{index .Labels \"bort.run-id\"}} " + staged.VolumeName: []byte("run1\n"), + }}, + volumeName: staged.VolumeName, + } + client := &Client{Docker: runner} + actx := &applyContext{plan: plan, cache: map[string]*appCache{}} + + err := client.applySyncVolume(context.Background(), actx, step) + if err == nil || !strings.Contains(err.Error(), "verify staging volume ownership after pinning") || !strings.Contains(err.Error(), "bort.run-id") { + t.Fatalf("expected post-pin ownership refusal, got %v", err) + } + if len(runner.runs) != 0 { + t.Fatalf("ownership drift after pin creation must abort before copy or clear, runs=%#v", runner.runs) + } + if pin := actx.stagingVolumePins["api"]; pin.containerID != "pin-id" { + t.Fatalf("known pin was not cached for cleanup after post-pin refusal: %#v", actx.stagingVolumePins) + } +} + +func TestApplyPushImageRefusesMissingPinAfterDurableTransfer(t *testing.T) { + app, plan, _, staged := stagedSyncFixture(t) + bundleDir := t.TempDir() + appDir := filepath.Join(bundleDir, "api") + if err := os.MkdirAll(appDir, 0o700); err != nil { + t.Fatal(err) + } + compose := "services:\n web:\n image: example/web\n volumes:\n - data:/data\nvolumes:\n data:\n" + composePath := filepath.Join(appDir, "compose.yaml") + if err := os.WriteFile(composePath, []byte(compose), 0o600); err != nil { + t.Fatal(err) + } + app.Directory = "api" + app.TargetResources = &preparer.TargetResources{Dokploy: &preparer.DokployResources{ComposeApp: preparer.DokployComposeApp{ComposePath: "compose.yaml"}}} + plan.Prepare = preparer.Result{BundleDir: bundleDir, Apps: []preparer.AppPlan{app}} + plan.BundleFiles = map[string][]byte{filepath.Clean(composePath): []byte(compose)} + runner := &fakeDockerRunner{outputs: map[string][]byte{ + "image inspect example/web": []byte(`[{}]`), + "volume inspect " + staged.VolumeName: ownedStagingVolumeInspect(plan, staged), + }} + client := &Client{Docker: runner} + actx := &applyContext{plan: plan, cache: map[string]*appCache{}} + entry := actx.entry("api") + entry.ComposeID = "compose-1" + entry.ComposeAppName = "stack-1" + entry.MigratedVolumeMounts = map[string]migratedVolumeMount{ + migratedMountKey("web", "/data"): {Service: "web", Target: "/data", VolumeName: staged.VolumeName}, + } + + err := client.applyPushImage(context.Background(), actx, Step{Kind: StepPushImage, App: "api", Ref: "api"}) + if err == nil || !strings.Contains(err.Error(), "missing after state transfer") || !strings.Contains(err.Error(), "identity was unprotected") { + t.Fatalf("expected missing transferred-state pin refusal, got %v", err) + } + for _, args := range runner.outputArgs { + if len(args) > 1 && args[0] == "run" && args[1] == "-d" { + t.Fatalf("push_image recreated a missing transferred-state pin: calls=%v", runner.outputArgs) + } + } +} + +func TestApplyKeepsSourceStoppedWhenTransferredPinIsMissing(t *testing.T) { + for _, failBeforePin := range []bool{false, true} { + name := "missing pin at acquisition" + if failBeforePin { + name = "failure before pin acquisition" + } + t.Run(name, func(t *testing.T) { + testApplyKeepsSourceStoppedWhenTransferredPinIsMissing(t, failBeforePin) + }) + } +} + +func testApplyKeepsSourceStoppedWhenTransferredPinIsMissing(t *testing.T, failBeforePin bool) { + t.Helper() + app, plan, step, staged := stagedSyncFixture(t) + bundleDir := t.TempDir() + appDir := filepath.Join(bundleDir, "api") + if err := os.MkdirAll(appDir, 0o700); err != nil { + t.Fatal(err) + } + compose := "services:\n web:\n image: example/web\n volumes:\n - data:/data\nvolumes:\n data:\n" + composePath := filepath.Join(appDir, "compose.yaml") + if err := os.WriteFile(composePath, []byte(compose), 0o600); err != nil { + t.Fatal(err) + } + app.Directory = "api" + app.TargetResources = &preparer.TargetResources{Dokploy: &preparer.DokployResources{ComposeApp: preparer.DokployComposeApp{ComposePath: "compose.yaml"}}} + plan.Prepare = preparer.Result{BundleDir: bundleDir, Apps: []preparer.AppPlan{app}} + plan.BundleFiles = map[string][]byte{filepath.Clean(composePath): []byte(compose)} + plan.Steps = []Step{{Kind: StepPauseSource, App: "api", Ref: "api"}, step, {Kind: StepPushImage, App: "api", Ref: "api"}} + plan.RecoveryCommand = "sudo bort status --run missing-pin" + plan.ResumeFrom = 2 + plan.TargetIdentities = map[string]TargetIdentity{"api": { + ProjectID: "project-1", + EnvironmentID: "environment-1", + ComposeID: "compose-1", + ComposeAppName: "stack-1", + }} + var failure StepProgress + onProgress := func(progress StepProgress) { + if progress.Status == StepStatusError { + failure = progress + } + } + plan.OnProgress = &onProgress + + seed := &applyContext{plan: plan, cache: map[string]*appCache{}} + seed.entry("api").SourcePauseRecorded = true + seed.entry("api").SourcePausedContainers = []sourcePausedContainer{{ + ID: "src-id", + Stopped: true, + StartedAt: "2026-01-01T00:00:00Z", + FinishedAt: "2026-01-02T00:00:00Z", + }} + if err := seed.persistSourcePauseState(); err != nil { + t.Fatal(err) + } + if err := seed.recordMigratedVolumeMount("api", migratedVolumeMount{Service: "web", Target: "/data", VolumeName: staged.VolumeName}); err != nil { + t.Fatal(err) + } + runner := &fakeDockerRunner{outputs: map[string][]byte{ + "image inspect example/web": []byte(`[{}]`), + "volume inspect " + staged.VolumeName: ownedStagingVolumeInspect(plan, staged), + "inspect --type container src-id": stoppedSourceInspect("2026-01-01T00:00:00Z", "2026-01-02T00:00:00Z"), + "ps -a --filter volume=" + staged.VolumeName: []byte(""), + }} + if failBeforePin { + runner.outputErrs = map[string]error{"image inspect example/web": errors.New("image unavailable")} + } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/project.one": + _ = json.NewEncoder(w).Encode(Project{ProjectID: "project-1", Environments: []ProjectEnvironment{{EnvironmentID: "environment-1"}}}) + case "/api/compose.one": + _ = json.NewEncoder(w).Encode(Compose{ComposeID: "compose-1", EnvironmentID: "environment-1", AppName: "stack-1"}) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + err := (&Client{BaseURL: server.URL, Token: "secret", HTTPClient: server.Client(), Docker: runner}).Apply(context.Background(), plan) + if err == nil || !isUnsafeSourceResumeError(err) || !strings.Contains(err.Error(), "missing after state transfer") || !strings.Contains(err.Error(), "`sudo bort status --run missing-pin`") { + t.Fatalf("expected a missing transferred-state pin to block source recovery, got %v", err) + } + if fakeOutputCalled(runner, "start", "src-id") { + t.Fatalf("source restarted after its transferred-state pin disappeared: calls=%v", runner.outputArgs) + } + if !failure.AuthorityRecoveryRequired { + t.Fatalf("missing-pin failure was not marked for authority recovery: %#v", failure) + } +} + +func TestSyncVolumeReplayRefusesMissingPinAfterDurableTransfer(t *testing.T) { + _, plan, step, staged := stagedSyncFixture(t) + seed := &applyContext{plan: plan, cache: map[string]*appCache{}} + if err := seed.recordMigratedVolumeMount("api", migratedVolumeMount{Service: staged.Service, Target: staged.Target, VolumeName: staged.VolumeName}); err != nil { + t.Fatal(err) + } + actx := &applyContext{plan: plan, cache: map[string]*appCache{}} + if err := actx.loadMigratedVolumeMounts(); err != nil { + t.Fatal(err) + } + runner := &fakeDockerRunner{outputs: map[string][]byte{ + "volume inspect src-vol": []byte(`[{"Name":"src-vol"}]`), + "volume inspect " + staged.VolumeName: ownedStagingVolumeInspect(plan, staged), + }} + + err := (&Client{Docker: runner}).applySyncVolume(context.Background(), actx, step) + if err == nil || !isUnsafeSourceResumeError(err) || !strings.Contains(err.Error(), "missing after state transfer") { + t.Fatalf("expected same-transfer replay to refuse pin recreation, got %v", err) + } + for _, args := range runner.outputArgs { + if len(args) > 1 && args[0] == "run" && args[1] == "-d" { + t.Fatalf("same-transfer replay recreated the missing pin: calls=%v", runner.outputArgs) + } + } +} + +func TestLaterStagedTransferRefusesMissingAppPin(t *testing.T) { + app, plan, firstStep, first := stagedSyncFixture(t) + secondSource := preparer.VolumeResource{ + Service: "worker", + Type: "volume", + Name: "src-cache", + Target: "/cache", + SourceContainerID: "src-id", + SourceContainerName: "coolify-web", + } + app.Resources.Volumes = append(app.Resources.Volumes, secondSource) + secondStep := Step{Kind: StepSyncVolume, App: "api", Ref: "volume:worker -> /cache"} + plan = stagedPlan(t, app, plan.RunDir, Step{Kind: StepPauseSource, App: "api"}, firstStep, secondStep) + second, ok := stagedVolumeFor(plan, "api", secondSource) + if !ok { + t.Fatal("second volume is not staged") + } + runner := &fakeDockerRunner{outputs: map[string][]byte{ + "volume inspect src-cache": []byte(`[{"Name":"src-cache"}]`), + "volume inspect " + first.VolumeName: ownedStagingVolumeInspect(plan, first), + "volume inspect " + second.VolumeName: ownedStagingVolumeInspect(plan, second), + }} + client := &Client{Docker: runner} + actx := &applyContext{plan: plan, cache: map[string]*appCache{}} + actx.entry("api").MigratedVolumeMounts = map[string]migratedVolumeMount{ + migratedMountKey(first.Service, first.Target): {Service: first.Service, Target: first.Target, VolumeName: first.VolumeName}, + } + + err := client.applySyncVolume(context.Background(), actx, secondStep) + if err == nil || !strings.Contains(err.Error(), "missing after state transfer") { + t.Fatalf("expected later transfer to refuse a missing app pin, got %v", err) + } + if len(runner.runs) != 0 { + t.Fatalf("later transfer copied or cleared data without the original pin: runs=%#v", runner.runs) + } + for _, args := range runner.outputArgs { + if len(args) > 1 && args[0] == "run" && args[1] == "-d" { + t.Fatalf("later transfer recreated the missing app pin: calls=%v", runner.outputArgs) + } + } +} + +func TestAcquireStagingVolumePinHoldsEveryVolumeUntilExplicitRelease(t *testing.T) { + _, plan, _, first := stagedSyncFixture(t) + second := first + second.Service = "worker" + second.Target = "/work" + second.VolumeName += "-worker" + runner := &fakeDockerRunner{outputs: map[string][]byte{ + "volume inspect " + first.VolumeName: ownedStagingVolumeInspect(plan, first), + "volume inspect " + second.VolumeName: ownedStagingVolumeInspect(plan, second), + }} + volumes := []stagedVolume{first, second} + pin, err := acquireStagingVolumePin(context.Background(), runner, plan, "api", volumes, true) + if err != nil { + t.Fatalf("acquireStagingVolumePin: %v", err) + } + var runCalls []string + for _, args := range runner.outputArgs { + if len(args) > 1 && args[0] == "run" && args[1] == "-d" { + runCalls = append(runCalls, strings.Join(args, " ")) + } + } + if len(runCalls) != 1 || !strings.Contains(runCalls[0], "--restart unless-stopped") || !strings.Contains(runCalls[0], "-v "+first.VolumeName+":/bort-volume/0:ro") || !strings.Contains(runCalls[0], "-v "+second.VolumeName+":/bort-volume/1:ro") { + t.Fatalf("expected one persistent read-only helper mounting every staging volume, got %v", runCalls) + } + if err := releaseStagingVolumePin(context.Background(), runner, plan, volumes, pin); err != nil { + t.Fatalf("releaseStagingVolumePin: %v", err) + } + if !fakeOutputCalled(runner, "rm", "-f", pin.containerID) { + t.Fatalf("expected explicit staging volume pin release, calls=%v", runner.outputArgs) + } +} + +func TestRequireStagingVolumeAttachmentSetsBatchesMultipleVolumes(t *testing.T) { + volumes := []stagedVolume{{VolumeName: "stage-web"}, {VolumeName: "stage-worker"}} + runner := &fakeDockerRunner{outputs: map[string][]byte{ + "ps -a --filter volume=stage-web --filter volume=stage-worker --format {{.ID}}": []byte("pin-id\ntarget-id\n"), + "inspect --type container pin-id target-id": []byte(`[ + {"Id":"pin-id","Mounts":[{"Type":"volume","Name":"stage-web"},{"Type":"volume","Name":"stage-worker"}]}, + {"Id":"target-id","Mounts":[{"Type":"volume","Name":"stage-web"}]} + ]`), + }} + allowed := map[string][]string{ + "stage-web": {"pin-id", "target-id"}, + "stage-worker": {"pin-id"}, + } + if err := requireStagingVolumeAttachmentSets(context.Background(), runner, volumes, allowed); err != nil { + t.Fatal(err) + } + if !fakeOutputCalled(runner, "ps", "-a", "--filter", "volume=stage-web", "--filter", "volume=stage-worker", "--format", "{{.ID}}") { + t.Fatalf("multiple volume attachments were not discovered in one Docker query: %v", runner.outputArgs) + } + for _, volume := range volumes { + if fakeOutputCalled(runner, "ps", "-a", "--filter", "volume="+volume.VolumeName, "--format", "{{.ID}}") { + t.Fatalf("volume %s was queried separately: %v", volume.VolumeName, runner.outputArgs) + } + } +} + +func TestReleaseStagingVolumePinsRequiresAuthorityAttachmentSet(t *testing.T) { + for _, targetAuthority := range []bool{false, true} { + name := "source" + if targetAuthority { + name = "target" + } + t.Run(name, func(t *testing.T) { + _, plan, _, staged := stagedSyncFixture(t) + plan.RunID = "run-digest" + staged, _ = stagedVolumeFor(plan, "api", plan.Prepare.Apps[0].Resources.Volumes[0]) + runner := &fakeDockerRunner{outputs: map[string][]byte{ + "volume inspect " + staged.VolumeName: ownedStagingVolumeInspect(plan, staged), + }} + pin, err := acquireStagingVolumePin(context.Background(), runner, plan, "api", []stagedVolume{staged}, true) + if err != nil { + t.Fatal(err) + } + client := &Client{Docker: runner} + if targetAuthority { + const targetID = "target-id" + client = targetAuthorityTestClient(t, runner, &plan) + runner.outputs["ps -a --filter label=com.docker.compose.project=stack-1 --format {{.ID}}"] = []byte(targetID + "\n") + runner.outputs["inspect --type container "+targetID] = []byte(`[{"Id":"` + targetID + `","Name":"/web","Config":{"Labels":{"com.docker.compose.service":"web","com.docker.compose.project":"stack-1"}},"State":{"Running":true,"Status":"running"},"Mounts":[{"Type":"volume","Name":"` + staged.VolumeName + `","Destination":"/data","RW":true}]}]`) + runner.activeComposeProjects = map[string][]string{"stack-1": {targetID}} + state := &applyContext{plan: plan, cache: map[string]*appCache{}} + if err := state.recordMigratedVolumeMount("api", migratedVolumeMount{Service: "web", Target: "/data", VolumeName: staged.VolumeName}); err != nil { + t.Fatal(err) + } + } + + if err := client.ReleaseStagingVolumePins(context.Background(), plan, targetAuthority); err != nil { + t.Fatalf("release pins under %s authority: %v", name, err) + } + if _, ok := runner.activePins[pin.containerID]; ok { + t.Fatalf("verified pin remained after %s-authority cleanup: %#v", name, runner.activePins) + } + if err := client.ReleaseStagingVolumePins(context.Background(), plan, targetAuthority); err != nil { + t.Fatalf("repeated %s-authority cleanup was not idempotent: %v", name, err) + } + }) + } +} + +func TestReleaseStagingVolumePinsBoundsDockerCallsByCallerDeadline(t *testing.T) { + _, plan, _, staged := stagedSyncFixture(t) + base := fakeDockerRunner{outputs: map[string][]byte{ + "volume inspect " + staged.VolumeName: ownedStagingVolumeInspect(plan, staged), + }} + if _, err := acquireStagingVolumePin(context.Background(), &base, plan, "api", []stagedVolume{staged}, true); err != nil { + t.Fatal(err) + } + base.outputArgs = nil + runner := &deadlineRecordingRunner{fakeDockerRunner: base} + deadline := time.Now().Add(5 * time.Second) + ctx, cancel := context.WithDeadline(context.Background(), deadline) + defer cancel() + if err := (&Client{Docker: runner}).ReleaseStagingVolumePins(ctx, plan, false); err != nil { + t.Fatal(err) + } + if len(runner.deadlines) == 0 { + t.Fatal("pin release made no Docker validation call") + } + for _, got := range runner.deadlines { + if !got.Equal(deadline) { + t.Fatalf("Docker deadline = %s, want caller deadline %s", got, deadline) + } + } + for _, want := range [][]string{ + {"ps", "-a", "--filter", "label=bort.staging-pin=true", "--filter", "label=bort.run-id=run1", "--format", "{{.ID}}"}, + {"inspect", "--type", "container", "pin-id"}, + {"volume", "inspect", staged.VolumeName}, + {"ps", "-a", "--filter", "volume=" + staged.VolumeName, "--format", "{{.ID}}"}, + } { + if !fakeOutputCalled(&runner.fakeDockerRunner, want...) { + t.Fatalf("deadline test did not reach Docker call %v: %v", want, runner.outputArgs) + } + } +} + +func TestReleaseTargetAuthorityPinRequiresDurableRecordAndTargetAttachment(t *testing.T) { + for _, tc := range []struct { + name string + recordMount bool + targetMounts string + want string + }{ + {name: "missing durable record", targetMounts: `[{"Type":"volume","Name":"%s","Destination":"/data","RW":true}]`, want: "no complete durable migrated-volume record"}, + {name: "missing target attachment", recordMount: true, targetMounts: `[]`, want: "no longer has migrated volume mounted"}, + } { + t.Run(tc.name, func(t *testing.T) { + _, plan, _, staged := stagedSyncFixture(t) + plan.RunID = "run-digest" + staged, _ = stagedVolumeFor(plan, "api", plan.Prepare.Apps[0].Resources.Volumes[0]) + const targetID = "target-id" + targetMounts := tc.targetMounts + if strings.Contains(targetMounts, "%s") { + targetMounts = fmt.Sprintf(targetMounts, staged.VolumeName) + } + runner := &fakeDockerRunner{outputs: map[string][]byte{ + "volume inspect " + staged.VolumeName: ownedStagingVolumeInspect(plan, staged), + "ps -a --filter label=com.docker.compose.project=stack-1 --format {{.ID}}": []byte(targetID + "\n"), + "inspect --type container " + targetID: []byte(fmt.Sprintf(`[{"Id":"%s","Name":"/web","Config":{"Labels":{"com.docker.compose.service":"web","com.docker.compose.project":"stack-1"}},"State":{"Running":true,"Status":"running"},"Mounts":%s}]`, targetID, targetMounts)), + }, activeComposeProjects: map[string][]string{"stack-1": {targetID}}} + client := targetAuthorityTestClient(t, runner, &plan) + pin, err := acquireStagingVolumePin(context.Background(), runner, plan, "api", []stagedVolume{staged}, true) + if err != nil { + t.Fatal(err) + } + if tc.recordMount { + state := &applyContext{plan: plan, cache: map[string]*appCache{}} + if err := state.recordMigratedVolumeMount("api", migratedVolumeMount{Service: "web", Target: "/data", VolumeName: staged.VolumeName}); err != nil { + t.Fatal(err) + } + } + + err = client.ReleaseStagingVolumePins(context.Background(), plan, true) + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("expected %q rejection, got %v", tc.want, err) + } + if _, ok := runner.activePins[pin.containerID]; !ok { + t.Fatalf("target-authority validation removed the pin after %s", tc.name) + } + }) + } +} + +func TestReleaseTargetAuthorityPinRejectsCrossMountedStagingVolume(t *testing.T) { + app, plan, firstStep, first := stagedSyncFixture(t) + app.Resources.Volumes = append(app.Resources.Volumes, preparer.VolumeResource{Service: "worker", Type: "volume", Name: "worker-data", Target: "/work"}) + secondStep := Step{Kind: StepSyncVolume, App: "api", Ref: "volume:worker -> /work"} + plan = stagedPlan(t, app, plan.RunDir, Step{Kind: StepPauseSource, App: "api"}, firstStep, secondStep) + plan.RunID = "run-digest" + volumes := stagedVolumesForApp(plan, "api") + if len(volumes) != 2 { + t.Fatalf("staged volumes = %#v, want two", volumes) + } + first, _ = stagedVolumeFor(plan, "api", app.Resources.Volumes[0]) + second, _ := stagedVolumeFor(plan, "api", app.Resources.Volumes[1]) + const webID, workerID = "web-id", "worker-id" + webInspect := fmt.Sprintf(`{"Id":"%s","Name":"/web","Config":{"Labels":{"com.docker.compose.service":"web","com.docker.compose.project":"stack-1"}},"State":{"Running":true,"Status":"running"},"Mounts":[{"Type":"volume","Name":"%s","Destination":"/data","RW":true},{"Type":"volume","Name":"%s","Destination":"/stolen","RW":true}]}`, webID, first.VolumeName, second.VolumeName) + workerInspect := fmt.Sprintf(`{"Id":"%s","Name":"/worker","Config":{"Labels":{"com.docker.compose.service":"worker","com.docker.compose.project":"stack-1"}},"State":{"Running":true,"Status":"running"},"Mounts":[{"Type":"volume","Name":"%s","Destination":"/work","RW":true}]}`, workerID, second.VolumeName) + runner := &fakeDockerRunner{outputs: map[string][]byte{ + "volume inspect " + first.VolumeName: ownedStagingVolumeInspect(plan, first), + "volume inspect " + second.VolumeName: ownedStagingVolumeInspect(plan, second), + "ps -a --filter label=com.docker.compose.project=stack-1 --format {{.ID}}": []byte(webID + "\n" + workerID + "\n"), + "inspect --type container " + webID: []byte("[" + webInspect + "]"), + "inspect --type container " + workerID: []byte("[" + workerInspect + "]"), + "inspect --type container " + webID + " " + workerID: []byte("[" + webInspect + "," + workerInspect + "]"), + }, activeComposeProjects: map[string][]string{"stack-1": {webID, workerID}}} + client := targetAuthorityTestClient(t, runner, &plan) + pin, err := acquireStagingVolumePin(context.Background(), runner, plan, "api", volumes, true) + if err != nil { + t.Fatal(err) + } + state := &applyContext{plan: plan, cache: map[string]*appCache{}} + for _, mount := range []migratedVolumeMount{ + {Service: "web", Target: "/data", VolumeName: first.VolumeName}, + {Service: "worker", Target: "/work", VolumeName: second.VolumeName}, + } { + if err := state.recordMigratedVolumeMount("api", mount); err != nil { + t.Fatal(err) + } + } + + err = client.ReleaseStagingVolumePins(context.Background(), plan, true) + if err == nil || !strings.Contains(err.Error(), "unexpected container "+webID) || !strings.Contains(err.Error(), second.VolumeName) { + t.Fatalf("cross-mounted writer was not rejected per volume: %v", err) + } + if _, ok := runner.activePins[pin.containerID]; !ok { + t.Fatal("cross-volume validation released the pin") + } +} + +func TestReleaseSourceAuthorityPinRefusesTargetAttachment(t *testing.T) { + _, plan, _, staged := stagedSyncFixture(t) + runner := &fakeDockerRunner{outputs: map[string][]byte{ + "volume inspect " + staged.VolumeName: ownedStagingVolumeInspect(plan, staged), + }} + pin, err := acquireStagingVolumePin(context.Background(), runner, plan, "api", []stagedVolume{staged}, true) + if err != nil { + t.Fatal(err) + } + const targetID = "target-id" + runner.outputs["inspect --type container "+targetID] = []byte(`[{"Id":"` + targetID + `","Name":"/web","Mounts":[{"Type":"volume","Name":"` + staged.VolumeName + `","Destination":"/data","RW":true}]}]`) + runner.activeComposeProjects = map[string][]string{"stack-1": {targetID}} + + err = (&Client{Docker: runner}).ReleaseStagingVolumePins(context.Background(), plan, false) + if err == nil || !strings.Contains(err.Error(), "unexpected container "+targetID) { + t.Fatalf("source authority removed a pin while the target remained attached: %v", err) + } + if _, ok := runner.activePins[pin.containerID]; !ok { + t.Fatal("source-authority cleanup removed the pin despite a target attachment") + } +} + +func TestReleaseSourceAuthorityRefusesPinMountDrift(t *testing.T) { + _, plan, _, staged := stagedSyncFixture(t) + runner := &fakeDockerRunner{outputs: map[string][]byte{ + "volume inspect " + staged.VolumeName: ownedStagingVolumeInspect(plan, staged), + }} + pin, err := acquireStagingVolumePin(context.Background(), runner, plan, "api", []stagedVolume{staged}, true) + if err != nil { + t.Fatal(err) + } + container := runner.activePins[pin.containerID] + container.Mounts[0].RW = true + runner.activePins[pin.containerID] = container + + err = (&Client{Docker: runner}).ReleaseStagingVolumePins(context.Background(), plan, false) + if err == nil || !strings.Contains(err.Error(), "unexpected or writable volume mount") { + t.Fatalf("pin mount drift was not rejected: %v", err) + } + if _, ok := runner.activePins[pin.containerID]; !ok { + t.Fatal("drifted pin was removed") + } +} + +func TestReleaseSourceAuthorityWithoutPinStillRefusesTargetAttachment(t *testing.T) { + _, plan, _, staged := stagedSyncFixture(t) + plan.StagingTransferApps = []string{"api"} + const targetID = "target-id" + runner := &fakeDockerRunner{ + outputs: map[string][]byte{ + "volume inspect " + staged.VolumeName: ownedStagingVolumeInspect(plan, staged), + "inspect --type container " + targetID: []byte(`[{"Id":"` + targetID + `","Name":"/web","Mounts":[{"Type":"volume","Name":"` + staged.VolumeName + `","Destination":"/data","RW":true}]}]`), + }, + activeComposeProjects: map[string][]string{"stack-1": {targetID}}, + } + + err := (&Client{Docker: runner}).ReleaseStagingVolumePins(context.Background(), plan, false) + if err == nil || !strings.Contains(err.Error(), "unexpected container "+targetID) { + t.Fatalf("source authority bypassed attachment validation because the pin was absent: %v", err) + } +} + +func TestReleaseTargetAuthorityWithoutPinRequiresDurableTransferRecord(t *testing.T) { + _, plan, _, staged := stagedSyncFixture(t) + plan.StagingTransferApps = []string{"api"} + runner := &fakeDockerRunner{outputs: map[string][]byte{ + "volume inspect " + staged.VolumeName: ownedStagingVolumeInspect(plan, staged), + }} + + err := (&Client{Docker: runner}).ReleaseStagingVolumePins(context.Background(), plan, true) + if err == nil || !strings.Contains(err.Error(), "no complete durable migrated-volume record") { + t.Fatalf("target authority bypassed durable transfer validation because the pin was absent: %v", err) + } +} + +func TestReleaseSourceAuthoritySkipsUncreatedVolumesAfterPartialTransfer(t *testing.T) { + app, plan, firstStep, first := stagedSyncFixture(t) + app.Resources.Volumes = append(app.Resources.Volumes, preparer.VolumeResource{Service: "worker", Type: "volume", Name: "worker-data", Target: "/work"}) + plan = stagedPlan(t, app, plan.RunDir, Step{Kind: StepPauseSource, App: "api"}, firstStep, Step{Kind: StepSyncVolume, App: "api", Ref: "volume:worker -> /work"}) + plan.StagingTransferApps = []string{"api"} + volumes := stagedVolumesForApp(plan, "api") + first, _ = stagedVolumeFor(plan, "api", app.Resources.Volumes[0]) + second, _ := stagedVolumeFor(plan, "api", app.Resources.Volumes[1]) + runner := &fakeDockerRunner{ + outputs: map[string][]byte{"volume inspect " + first.VolumeName: ownedStagingVolumeInspect(plan, first)}, + outputErrs: map[string]error{"volume inspect " + second.VolumeName: errors.New("Error response from daemon: volume " + second.VolumeName + " not found")}, + } + + if err := (&Client{Docker: runner}).ReleaseStagingVolumePins(context.Background(), plan, false); err != nil { + t.Fatalf("source recovery dead-ended on an uncreated later volume: %v (planned=%#v)", err, volumes) + } + if !fakeOutputCalled(runner, "ps", "-a", "--filter", "volume="+first.VolumeName, "--format", "{{.ID}}") { + t.Fatalf("source recovery skipped attachment validation for the created volume: %v", runner.outputArgs) + } + for _, args := range runner.outputArgs { + if strings.Contains(strings.Join(args, " "), "volume="+second.VolumeName) { + t.Fatalf("source recovery queried attachments for the uncreated volume: %v", args) + } + } +} + +func TestReleaseSourceAuthorityHonorsLegacyTransferStartEvidence(t *testing.T) { + _, plan, _, staged := stagedSyncFixture(t) + path, err := migratedVolumeMountsPath(plan.RunDir) + if err != nil { + t.Fatal(err) + } + contents, err := json.Marshal(migratedVolumeMountsState{APIVersion: migratedVolumeMountsLegacyAPIVersion, StartedApps: []string{"api"}}) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, contents, 0o600); err != nil { + t.Fatal(err) + } + const targetID = "target-id" + runner := &fakeDockerRunner{ + outputs: map[string][]byte{ + "volume inspect " + staged.VolumeName: ownedStagingVolumeInspect(plan, staged), + "inspect --type container " + targetID: []byte(`[{"Id":"` + targetID + `","Name":"/web","Mounts":[{"Type":"volume","Name":"` + staged.VolumeName + `","Destination":"/data","RW":true}]}]`), + }, + activeComposeProjects: map[string][]string{"stack-1": {targetID}}, + } + + err = (&Client{Docker: runner}).ReleaseStagingVolumePins(context.Background(), plan, false) + if err == nil || !strings.Contains(err.Error(), "unexpected container "+targetID) { + t.Fatalf("legacy transfer-start evidence did not trigger attachment validation: %v", err) + } +} + +func TestExactStagingVolumeAttachmentsAcceptsShortAndFullContainerIDs(t *testing.T) { + shortID := "123456789abc" + fullID := shortID + strings.Repeat("d", 52) + for _, ids := range [][2]string{{shortID, fullID}, {fullID, shortID}} { + if err := requireExactStagingVolumeAttachments("data", []string{ids[0]}, []string{ids[1]}); err != nil { + t.Fatalf("equivalent Docker IDs %q and %q did not match: %v", ids[0], ids[1], err) + } + } +} + +func TestStagingVolumePinRejectsUnexpectedAttachment(t *testing.T) { + _, plan, _, staged := stagedSyncFixture(t) + attachmentCommand := "ps -a --filter volume=" + staged.VolumeName + " --format {{.ID}}" + runner := &sequencedOutputRunner{ + fakeDockerRunner: fakeDockerRunner{outputs: map[string][]byte{ + "volume inspect " + staged.VolumeName: ownedStagingVolumeInspect(plan, staged), + }}, + sequences: map[string][][]byte{ + attachmentCommand: {[]byte("pin-id\nforeign-writer\n")}, + }, + } + pin, err := acquireStagingVolumePin(context.Background(), runner, plan, "api", []stagedVolume{staged}, true) + if err != nil { + t.Fatalf("acquireStagingVolumePin: %v", err) + } + err = requireStagingVolumeAttachments(context.Background(), runner, []stagedVolume{staged}, []string{pin.containerID}) + if err == nil || !strings.Contains(err.Error(), "unexpected container foreign-writer") { + t.Fatalf("expected a foreign attachment to block handoff, got %v", err) + } +} + +type ambiguousPinCreateRunner struct { + *fakeDockerRunner + failCreate bool +} + +func (r *ambiguousPinCreateRunner) Output(ctx context.Context, args ...string) ([]byte, error) { + if len(args) > 1 && args[0] == "run" && args[1] == "-d" && slices.Contains(args, stagingVolumePinLabel+"=true") && r.failCreate { + r.failCreate = false + _, _ = r.fakeDockerRunner.Output(ctx, args...) + return nil, errors.New("docker response lost after creating pin") + } + return r.fakeDockerRunner.Output(ctx, args...) +} + +func TestAcquireStagingVolumePinAdoptsAmbiguousCreateAndRetry(t *testing.T) { + _, plan, _, staged := stagedSyncFixture(t) + runner := &ambiguousPinCreateRunner{ + fakeDockerRunner: &fakeDockerRunner{outputs: map[string][]byte{ + "volume inspect " + staged.VolumeName: ownedStagingVolumeInspect(plan, staged), + }}, + failCreate: true, + } + pin, err := acquireStagingVolumePin(context.Background(), runner, plan, "api", []stagedVolume{staged}, true) + if err != nil || pin.containerID != "pin-id" { + t.Fatalf("expected ambiguous create to adopt the running pin, pin=%#v err=%v", pin, err) + } + if len(runner.activePins) != 1 || fakeOutputCalled(runner.fakeDockerRunner, "rm", "-f", "pin-id") { + t.Fatalf("adopted pin was not preserved: active=%v calls=%v", runner.activePins, runner.outputArgs) + } + second, err := acquireStagingVolumePin(context.Background(), runner, plan, "api", []stagedVolume{staged}, true) + if err != nil || second.containerID != pin.containerID { + t.Fatalf("adopted pin blocked ordinary retry: first=%#v second=%#v err=%v", pin, second, err) + } + runCalls := 0 + for _, args := range runner.outputArgs { + if len(args) > 1 && args[0] == "run" && args[1] == "-d" { + runCalls++ + } + } + if runCalls != 1 { + t.Fatalf("ambiguous create and retry ran %d pin containers, want 1; calls=%v", runCalls, runner.outputArgs) + } +} + +func TestAcquireStagingVolumePinRestartsStoppedExactPinInPlace(t *testing.T) { + _, plan, _, staged := stagedSyncFixture(t) + runner := &fakeDockerRunner{outputs: map[string][]byte{ + "volume inspect " + staged.VolumeName: ownedStagingVolumeInspect(plan, staged), + }} + pin, err := acquireStagingVolumePin(context.Background(), runner, plan, "api", []stagedVolume{staged}, true) + if err != nil { + t.Fatalf("create staging volume pin: %v", err) + } + stopped := runner.activePins[pin.containerID] + stopped.State.Running = false + stopped.State.Status = "exited" + runner.activePins[pin.containerID] = stopped + runner.outputArgs = nil + + restarted, err := acquireStagingVolumePin(context.Background(), runner, plan, "api", []stagedVolume{staged}, true) + if err != nil || restarted != pin { + t.Fatalf("expected stopped exact pin to restart in place, pin=%#v restarted=%#v err=%v", pin, restarted, err) + } + if !runner.activePins[pin.containerID].State.Running || !fakeOutputCalled(runner, "start", pin.containerID) { + t.Fatalf("stopped pin was not restarted: active=%#v calls=%v", runner.activePins, runner.outputArgs) + } + for _, args := range runner.outputArgs { + if len(args) > 1 && args[0] == "run" && args[1] == "-d" || len(args) > 1 && args[0] == "rm" { + t.Fatalf("stopped exact pin was replaced instead of restarted: calls=%v", runner.outputArgs) + } + } +} + +func TestAcquireStagingVolumePinFailedRestartPreservesStoppedPin(t *testing.T) { + _, plan, _, staged := stagedSyncFixture(t) + runner := &fakeDockerRunner{outputs: map[string][]byte{ + "volume inspect " + staged.VolumeName: ownedStagingVolumeInspect(plan, staged), + }} + pin, err := acquireStagingVolumePin(context.Background(), runner, plan, "api", []stagedVolume{staged}, true) + if err != nil { + t.Fatalf("create staging volume pin: %v", err) + } + stopped := runner.activePins[pin.containerID] + stopped.State.Running = false + stopped.State.Status = "exited" + runner.activePins[pin.containerID] = stopped + runner.outputArgs = nil + runner.outputErrs = map[string]error{"start " + pin.containerID: errors.New("restart failed")} + + _, err = acquireStagingVolumePin(context.Background(), runner, plan, "api", []stagedVolume{staged}, true) + if err == nil || !strings.Contains(err.Error(), "restart stopped staging volume pin") { + t.Fatalf("expected stopped pin restart failure, got %v", err) + } + if active, ok := runner.activePins[pin.containerID]; !ok || active.State.Running { + t.Fatalf("failed restart did not preserve the stopped pin: active=%#v", runner.activePins) + } + for _, args := range runner.outputArgs { + if len(args) > 1 && args[0] == "run" && args[1] == "-d" || len(args) > 1 && args[0] == "rm" { + t.Fatalf("failed restart replaced or removed the stopped pin: calls=%v", runner.outputArgs) + } + } +} + +type sourceRecoveryPinRunner struct { + *fakeDockerRunner + sourceRunning bool +} + +func (r *sourceRecoveryPinRunner) Output(ctx context.Context, args ...string) ([]byte, error) { + if len(args) == 4 && args[0] == "inspect" && args[1] == "--type" && args[2] == "container" && args[3] == "src-id" { + r.outputArgs = append(r.outputArgs, append([]string{}, args...)) + return []byte(fmt.Sprintf(`[{"Id":"src-id","Name":"/coolify-web","State":{"Running":%t,"Status":"exited"}}]`, r.sourceRunning)), nil + } + if len(args) == 2 && args[0] == "start" && args[1] == "src-id" { + r.outputArgs = append(r.outputArgs, append([]string{}, args...)) + r.sourceRunning = true + return []byte("src-id\n"), nil + } + return r.fakeDockerRunner.Output(ctx, args...) +} + +func TestBestEffortResumePreservesVerifiedPreHandoffPin(t *testing.T) { + _, plan, _, staged := stagedSyncFixture(t) + runner := &sourceRecoveryPinRunner{fakeDockerRunner: &fakeDockerRunner{outputs: map[string][]byte{ + "volume inspect " + staged.VolumeName: ownedStagingVolumeInspect(plan, staged), + }}} + pin, err := acquireStagingVolumePin(context.Background(), runner, plan, "api", []stagedVolume{staged}, true) + if err != nil { + t.Fatal(err) + } + actx := &applyContext{plan: plan, cache: map[string]*appCache{}, stagingVolumePins: map[string]stagingVolumePin{"api": pin}} + entry := actx.entry("api") + entry.SourcePauseRecorded = true + entry.SourcePausedContainers = []sourcePausedContainer{{ID: "src-id", Stopped: true}} + client := &Client{Docker: runner} + + if err := client.bestEffortResume(context.Background(), actx, plan, len(plan.Steps), pausedSources{"api": true}, false, true); err != nil { + t.Fatalf("resume source with pre-handoff pin: %v", err) + } + if !runner.sourceRunning { + t.Fatal("source was not resumed") + } + if _, ok := runner.activePins[pin.containerID]; !ok { + t.Fatalf("verified pre-handoff pin was removed before a retry could reuse it: %#v", runner.activePins) + } + if cached, ok := actx.stagingVolumePins["api"]; !ok || cached != pin { + t.Fatalf("preserved pin was not cached for retry: %#v", actx.stagingVolumePins) + } + pinRuns := 0 + for _, args := range runner.outputArgs { + if len(args) > 1 && args[0] == "run" && args[1] == "-d" { + pinRuns++ + } + if len(args) > 1 && args[0] == "rm" && args[1] == "-f" { + t.Fatalf("source recovery removed the preserved pin: %v", runner.outputArgs) + } + } + if pinRuns != 1 { + t.Fatalf("source recovery recreated the preserved pin: run calls=%d all calls=%v", pinRuns, runner.outputArgs) + } +} + +func TestBestEffortResumeKeepsSourceStoppedWhenPinGuardIsUnverified(t *testing.T) { + _, plan, _, staged := stagedSyncFixture(t) + runner := &sourceRecoveryPinRunner{fakeDockerRunner: &fakeDockerRunner{outputs: map[string][]byte{ + "volume inspect " + staged.VolumeName: ownedStagingVolumeInspect(plan, staged), + }}} + pin, err := acquireStagingVolumePin(context.Background(), runner, plan, "api", []stagedVolume{staged}, true) + if err != nil { + t.Fatal(err) + } + const targetID = "unexpected-target" + runner.outputs["inspect --type container "+targetID] = []byte(`[{"Id":"` + targetID + `","Name":"/web","Mounts":[{"Type":"volume","Name":"` + staged.VolumeName + `","Destination":"/data","RW":true}]}]`) + runner.activeComposeProjects = map[string][]string{"stack-1": {targetID}} + actx := &applyContext{plan: plan, cache: map[string]*appCache{}, stagingVolumePins: map[string]stagingVolumePin{"api": pin}} + entry := actx.entry("api") + entry.SourcePauseRecorded = true + entry.SourcePausedContainers = []sourcePausedContainer{{ID: "src-id", Stopped: true}} + client := &Client{Docker: runner} + + err = client.bestEffortResume(context.Background(), actx, plan, len(plan.Steps), pausedSources{"api": true}, false, true) + if err == nil || !strings.Contains(err.Error(), "pin could not be verified before restart") { + t.Fatalf("expected unverified pin guard to block source restart, got %v", err) + } + if runner.sourceRunning || fakeOutputCalled(runner.fakeDockerRunner, "start", "src-id") { + t.Fatalf("source restarted with an unverified pin guard: running=%t calls=%v", runner.sourceRunning, runner.outputArgs) + } + if _, ok := runner.activePins[pin.containerID]; !ok { + t.Fatal("unverified pin guard was removed") + } +} + +func TestFailedStagingVolumeHandoffReportsGuardVerification(t *testing.T) { + _, plan, _, staged := stagedSyncFixture(t) + runner := &fakeDockerRunner{outputs: map[string][]byte{ + "volume inspect " + staged.VolumeName: ownedStagingVolumeInspect(plan, staged), + }} + pin, err := acquireStagingVolumePin(context.Background(), runner, plan, "api", []stagedVolume{staged}, true) + if err != nil { + t.Fatal(err) + } + verified := failedStagingVolumeHandoff(runner, plan, "api", []stagedVolume{staged}, pin, "handoff failed", errors.New("deploy error")) + if !strings.Contains(verified.Error(), "verified and preserved") { + t.Fatalf("verified guard was not reported accurately: %v", verified) + } + runner.outputs["volume inspect "+staged.VolumeName] = []byte(`[{"Name":"` + staged.VolumeName + `","Labels":{"bort.run-id":"other-run"}}]`) + unverified := failedStagingVolumeHandoff(runner, plan, "api", []stagedVolume{staged}, pin, "handoff failed", errors.New("deploy error")) + if !strings.Contains(unverified.Error(), "could not verify the read-only staging-volume guard") || strings.Contains(unverified.Error(), "verified and preserved") { + t.Fatalf("unverified guard was reported inaccurately: %v", unverified) + } +} + +type handoffPinRunner struct { + *fakeDockerRunner + pinCommand string + pinID string + targetID string + targetInspect string + pinned bool + sawTarget bool +} + +func (r *handoffPinRunner) Output(ctx context.Context, args ...string) ([]byte, error) { + key := strings.Join(args, " ") + if key == r.targetInspect { + if !r.pinned { + return nil, errors.New("target attached after staging volume pin was released") + } + r.sawTarget = true + } + if strings.HasPrefix(key, "ps -a --filter volume=") && r.sawTarget { + return []byte(r.pinID + "\n" + r.targetID + "\n"), nil + } + out, err := r.fakeDockerRunner.Output(ctx, args...) + if err != nil { + return nil, err + } + if key == r.pinCommand { + r.pinned = true + } + if key == "rm -f "+r.pinID { + if !r.sawTarget { + return nil, errors.New("staging volume pin released before target attachment was verified") + } + r.pinned = false + } + return out, nil +} + +type completedDriftHandoffRunner struct { + *fakeDockerRunner + targetID string + targetVisible bool + targetStopped bool +} + +func (r *completedDriftHandoffRunner) Output(ctx context.Context, args ...string) ([]byte, error) { + key := strings.Join(args, " ") + if strings.HasPrefix(key, "ps -a --filter volume=") { + out, err := r.fakeDockerRunner.Output(ctx, args...) + if err != nil || !r.targetVisible { + return out, err + } + return append(out, []byte("\n"+r.targetID+"\n")...), nil + } + out, err := r.fakeDockerRunner.Output(ctx, args...) + if err != nil { + return nil, err + } + if key == "stop "+r.targetID { + r.targetStopped = true + } + if key == "inspect --type container "+r.targetID && r.targetStopped { + out = []byte(strings.ReplaceAll(strings.ReplaceAll(string(out), `"Running":true`, `"Running":false`), `"Status":"running"`, `"Status":"exited"`)) + } + return out, nil +} + +func TestApplyPushImagePinsStagingVolumeThroughTargetValidation(t *testing.T) { + app, plan, _, staged := stagedSyncFixture(t) + bundleDir := t.TempDir() + appDir := filepath.Join(bundleDir, "api") + if err := os.MkdirAll(appDir, 0o700); err != nil { + t.Fatal(err) + } + compose := "services:\n web:\n image: example/web\n volumes:\n - data:/data\nvolumes:\n data:\n" + composePath := filepath.Join(appDir, "compose.yaml") + if err := os.WriteFile(composePath, []byte(compose), 0o600); err != nil { + t.Fatal(err) + } + app.Directory = "api" + app.TargetResources = &preparer.TargetResources{Dokploy: &preparer.DokployResources{ComposeApp: preparer.DokployComposeApp{ComposePath: "compose.yaml"}}} + plan.Prepare = preparer.Result{BundleDir: bundleDir, Apps: []preparer.AppPlan{app}} + plan.BundleFiles = map[string][]byte{filepath.Clean(composePath): []byte(compose)} + + var deploymentTitle string + deploys := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodPost && r.URL.Path == "/api/compose.update": + w.WriteHeader(http.StatusOK) + case r.Method == http.MethodPost && r.URL.Path == "/api/compose.deploy": + deploys++ + var request deployComposeRequest + if err := json.NewDecoder(r.Body).Decode(&request); err != nil { + t.Fatal(err) + } + deploymentTitle = request.Title + w.WriteHeader(http.StatusOK) + case r.Method == http.MethodGet && r.URL.Path == "/api/compose.one": + _ = json.NewEncoder(w).Encode(Compose{ComposeID: "c1", AppName: "stack-1", Deployments: []Deployment{{Title: deploymentTitle, Status: "done"}}}) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + const targetID = "abcdef123456" + pinCommand := "run -d --name " + stagingVolumePinName(plan, []stagedVolume{staged}) + " --network none --read-only --restart unless-stopped --label " + stagingVolumePinLabel + "=true --label " + stagingVolumeRunIDLabel + "=" + stagingOwner(plan) + " --label " + stagingVolumeAppLabel + "=api -v " + staged.VolumeName + ":/bort-volume/0:ro " + volumeCopyImage + " sh -c while :; do sleep 2147483647; done" + targetInspect := "inspect --type container " + targetID + runner := &handoffPinRunner{ + fakeDockerRunner: &fakeDockerRunner{ + outputs: map[string][]byte{ + "image inspect example/web": []byte(`[{}]`), + "ps --format {{.Names}}": []byte("dokploy-postgres\n"), + pinCommand: []byte("handoff-pin\n"), + "volume inspect --format {{index .Labels \"bort.run-id\"}} " + staged.VolumeName: []byte("run1\n"), + "volume inspect " + staged.VolumeName: ownedStagingVolumeInspect(plan, staged), + "rm -f handoff-pin": []byte("handoff-pin\n"), + "ps -a --filter label=com.docker.compose.project=stack-1 --format {{.ID}}": []byte(targetID + "\n"), + targetInspect: []byte(`[{"Id":"` + targetID + `","Name":"/web","Config":{"Labels":{"com.docker.compose.service":"web","com.docker.compose.project":"stack-1"}},"State":{"Running":true,"Status":"running"},"Mounts":[{"Type":"volume","Name":"` + staged.VolumeName + `","Destination":"/data","RW":true}]}]`), + }, + runOutputs: map[string][]byte{ + "exec -i dokploy-postgres psql -U dokploy -d dokploy -v ON_ERROR_STOP=1 -At": {}, + }, + }, + pinCommand: pinCommand, + pinID: "handoff-pin", + targetID: targetID, + targetInspect: targetInspect, + } + client := &Client{BaseURL: server.URL, Token: "secret", HTTPClient: server.Client(), Docker: runner} + actx := &applyContext{plan: plan, cache: map[string]*appCache{}} + entry := actx.entry("api") + entry.ComposeID = "c1" + entry.ComposeAppName = "stack-1" + entry.MigratedVolumeMounts = map[string]migratedVolumeMount{ + migratedMountKey("web", "/data"): {Service: "web", Target: "/data", VolumeName: staged.VolumeName}, + } + if _, err := acquireStagingVolumePin(context.Background(), runner, plan, "api", []stagedVolume{staged}, true); err != nil { + t.Fatalf("seed transferred-state pin: %v", err) + } + + if err := client.applyPushImage(context.Background(), actx, Step{Kind: StepPushImage, App: "api", Ref: "api"}); err != nil { + t.Fatalf("applyPushImage: %v", err) + } + if !runner.sawTarget || runner.pinned { + t.Fatalf("expected target validation while pinned followed by release, sawTarget=%t pinned=%t", runner.sawTarget, runner.pinned) + } + if _, err := acquireStagingVolumePin(context.Background(), runner, plan, "api", []stagedVolume{staged}, true); err != nil { + t.Fatalf("seed interrupted-handoff pin: %v", err) + } + runner.sawTarget = true + err := client.applyPushImage(context.Background(), actx, Step{Kind: StepPushImage, App: "api", Ref: "api"}) + if err == nil || !isUnsafeSourceResumeError(err) || !mutationResponseMayHaveSucceeded(err) || !strings.Contains(err.Error(), "previous target attachment may have survived") || !runner.pinned { + t.Fatalf("a resumed push with a surviving target attachment must preserve the pin and keep the source stopped, pinned=%t err=%v", runner.pinned, err) + } + if deploys != 1 { + t.Fatalf("a surviving target attachment must block another deployment, deploys=%d", deploys) + } + if err := releaseStagingVolumePin(context.Background(), runner, plan, []stagedVolume{staged}, stagingVolumePin{name: stagingVolumePinName(plan, []stagedVolume{staged}), containerID: runner.pinID}); err != nil { + t.Fatalf("release preserved test pin: %v", err) + } + delete(actx.stagingVolumePins, "api") + runner.sawTarget = false + if _, err := acquireStagingVolumePin(context.Background(), runner, plan, "api", []stagedVolume{staged}, true); err != nil { + t.Fatalf("seed transferred-state pin for release failure: %v", err) + } + runner.outputErrs = map[string]error{"rm -f handoff-pin": errors.New("docker response lost while releasing pin")} + err = client.applyPushImage(context.Background(), actx, Step{Kind: StepPushImage, App: "api", Ref: "api"}) + if err == nil || !isUnsafeSourceResumeError(err) || !mutationResponseMayHaveSucceeded(err) || !strings.Contains(err.Error(), "paused source applications remain stopped") || !strings.Contains(err.Error(), "bort status") { + t.Fatalf("a pin-release failure after target validation must keep the source stopped, got %v", err) + } +} + +func TestApplyPushImageKeepsSourceStoppedAfterCompletedDeploymentMountDrift(t *testing.T) { + app, plan, _, staged := stagedSyncFixture(t) + bundleDir := t.TempDir() + appDir := filepath.Join(bundleDir, "api") + if err := os.MkdirAll(appDir, 0o700); err != nil { + t.Fatal(err) + } + compose := "services:\n web:\n image: example/web\n volumes:\n - data:/data\nvolumes:\n data:\n" + composePath := filepath.Join(appDir, "compose.yaml") + if err := os.WriteFile(composePath, []byte(compose), 0o600); err != nil { + t.Fatal(err) + } + app.Directory = "api" + app.TargetResources = &preparer.TargetResources{Dokploy: &preparer.DokployResources{ComposeApp: preparer.DokployComposeApp{ComposePath: "compose.yaml"}}} + plan.Prepare = preparer.Result{BundleDir: bundleDir, Apps: []preparer.AppPlan{app}} + plan.BundleFiles = map[string][]byte{filepath.Clean(composePath): []byte(compose)} -func TestApplyCreateServiceRefusesAdoptedComposeWithoutEnvFile(t *testing.T) { - plan, _, _ := stagedRestoreFixture(t) - disabled := false + const targetID = "drifted-web" + runner := &completedDriftHandoffRunner{ + fakeDockerRunner: &fakeDockerRunner{ + outputs: map[string][]byte{ + "image inspect example/web": []byte(`[{}]`), + "ps --format {{.Names}}": []byte("dokploy-postgres\n"), + "volume inspect --format {{index .Labels \"bort.run-id\"}} " + staged.VolumeName: []byte("run1\n"), + "volume inspect " + staged.VolumeName: ownedStagingVolumeInspect(plan, staged), + "ps -a --filter label=com.docker.compose.project=stack-1 --format {{.ID}}": []byte(targetID + "\n"), + "inspect --type container " + targetID: []byte(`[{"Id":"` + targetID + `","Name":"/web","Config":{"Labels":{"com.docker.compose.service":"web","com.docker.compose.project":"stack-1"}},"State":{"Running":true,"Status":"running"},"Mounts":[{"Type":"volume","Name":"fresh-vol","Destination":"/data","RW":true}]}]`), + "stop " + targetID: []byte(targetID + "\n"), + }, + runOutputs: map[string][]byte{ + "exec -i dokploy-postgres psql -U dokploy -d dokploy -v ON_ERROR_STOP=1 -At": {}, + }, + }, + targetID: targetID, + } + var deploymentTitle string + deploys := 0 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch { - case r.Method == http.MethodGet && r.URL.Path == "/api/compose.search": - _ = json.NewEncoder(w).Encode(composeSearchResponse{Items: []Compose{{ComposeID: "c1", Name: "api", AppName: "stack-1"}}, Total: 1}) + case r.Method == http.MethodPost && r.URL.Path == "/api/compose.update": + w.WriteHeader(http.StatusOK) + case r.Method == http.MethodPost && r.URL.Path == "/api/compose.deploy": + deploys++ + var request deployComposeRequest + if err := json.NewDecoder(r.Body).Decode(&request); err != nil { + t.Fatal(err) + } + deploymentTitle = request.Title + runner.targetVisible = true + w.WriteHeader(http.StatusOK) case r.Method == http.MethodGet && r.URL.Path == "/api/compose.one": - _ = json.NewEncoder(w).Encode(Compose{ComposeID: "c1", Name: "api", AppName: "stack-1", CreateEnvFile: &disabled}) + _ = json.NewEncoder(w).Encode(Compose{ComposeID: "c1", AppName: "stack-1", Deployments: []Deployment{{Title: deploymentTitle, Status: "done"}}}) default: http.NotFound(w, r) } })) defer server.Close() - client := &Client{BaseURL: server.URL, Token: "secret", HTTPClient: server.Client(), Docker: &fakeDockerRunner{}} - actx := &applyContext{plan: plan, cache: map[string]*appCache{"api": {EnvironmentID: "env1"}}} - err := client.applyCreateService(context.Background(), actx, Step{Kind: StepCreateService, App: "api", Ref: "api"}) - if !errors.Is(err, ErrNotImplemented) || !strings.Contains(err.Error(), ".env creation disabled") { - t.Fatalf("expected adopted compose refusal before the source pause, got %v", err) - } -} -func TestDeployComposeForApplyRefusesIncompatibleSettingsAfterStagedRestore(t *testing.T) { - plan, _, _ := stagedRestoreFixture(t) - disabled := false - var writes []string - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - switch { - case r.Method == http.MethodGet && r.URL.Path == "/api/compose.one": - _ = json.NewEncoder(w).Encode(Compose{ComposeID: "c1", Name: "api", AppName: "stack-1", CreateEnvFile: &disabled}) - default: - writes = append(writes, r.URL.Path) - _ = json.NewEncoder(w).Encode(map[string]any{}) - } - })) - defer server.Close() - client := &Client{BaseURL: server.URL, Token: "secret", HTTPClient: server.Client(), Docker: &fakeDockerRunner{}} + client := &Client{BaseURL: server.URL, Token: "secret", HTTPClient: server.Client(), Docker: runner} actx := &applyContext{plan: plan, cache: map[string]*appCache{}} - actx.entry("api").ComposeID = "c1" - actx.entry("api").ComposeAppName = "stack-1" - err := client.deployComposeForApply(context.Background(), actx, "api", "services: {}\n", "DB_PASSWORD=s3cret\n") - if !errors.Is(err, ErrNotImplemented) || !strings.Contains(err.Error(), ".env creation disabled") { - t.Fatalf("expected deploy-time refusal, got %v", err) - } - if len(writes) != 0 { - t.Fatalf("compose was written or deployed despite the refusal: %v", writes) + entry := actx.entry("api") + entry.ComposeID = "c1" + entry.ComposeAppName = "stack-1" + entry.MigratedVolumeMounts = map[string]migratedVolumeMount{ + migratedMountKey("web", "/data"): {Service: "web", Target: "/data", VolumeName: staged.VolumeName}, } -} - -func stagedSyncFixture(t *testing.T) (preparer.AppPlan, Plan, Step, stagedVolume) { - t.Helper() - app := preparer.AppPlan{Name: "api"} - app.Resources.SourceServices = []preparer.SourceServiceRef{{ServiceName: "web", ContainerID: "src-id", ContainerName: "coolify-web"}} - app.Resources.Volumes = []preparer.VolumeResource{{ - Service: "web", - Type: "volume", - Name: "src-vol", - Target: "/data", - SourceContainerID: "src-id", - SourceContainerName: "coolify-web", - }} - step := Step{Kind: StepSyncVolume, App: "api", Ref: "volume:web -> /data"} - plan := stagedPlan(t, app, t.TempDir(), Step{Kind: StepPauseSource, App: "api"}, step) - staged, ok := stagedVolumeFor(plan, "api", app.Resources.Volumes[0]) - if !ok { - t.Fatalf("volume not staged in plan %v", stepKinds(plan.Steps)) + if _, err := acquireStagingVolumePin(context.Background(), runner, plan, "api", []stagedVolume{staged}, true); err != nil { + t.Fatalf("seed transferred-state pin: %v", err) } - return app, plan, step, staged -} - -func stoppedSourceInspect(startedAt, finishedAt string) []byte { - return []byte(`[{"Id":"src-id","Name":"/coolify-web","State":{"Running":false,"StartedAt":"` + startedAt + `","FinishedAt":"` + finishedAt + `"}}]`) -} -func TestApplyCreateVolumeCreatesOnlyItsOwnStagedVolume(t *testing.T) { - app := preparer.AppPlan{Name: "api"} - app.Resources.Volumes = []preparer.VolumeResource{ - {Service: "web", Type: "volume", Name: "src-data", Target: "/data"}, - {Service: "web", Type: "volume", Name: "", Target: "/cache"}, - } - plan := stagedPlan(t, app, t.TempDir(), - Step{Kind: StepCreateVolume, App: "api", Ref: "src-data"}, - Step{Kind: StepCreateVolume, App: "api", Ref: "/cache"}, - Step{Kind: StepPauseSource, App: "api"}, - Step{Kind: StepSyncVolume, App: "api", Ref: "volume:web -> /data"}, - Step{Kind: StepSyncVolume, App: "api", Ref: "volume:web -> /cache"}, - ) - staged := stagedVolumesForApp(plan, "api") - if len(staged) != 2 { - t.Fatalf("expected two staged volumes, got %#v", staged) - } - runner := &fakeDockerRunner{outputs: map[string][]byte{"volume create": []byte("created\n")}, outputErrs: map[string]error{}} - for _, volume := range staged { - runner.outputErrs["volume inspect --format {{index .Labels \"bort.run-id\"}} "+volume.VolumeName] = errors.New("Error response from daemon: get " + volume.VolumeName + ": no such volume") + err := client.applyPushImage(context.Background(), actx, Step{Kind: StepPushImage, App: "api", Ref: "api"}) + if err == nil || !isUnsafeSourceResumeError(err) || !mutationResponseMayHaveSucceeded(err) || !strings.Contains(err.Error(), "changed from migrated volume") { + t.Fatalf("a completed deployment with mount drift must keep the source stopped, got %v", err) } - client := &Client{Docker: runner} - actx := &applyContext{cache: map[string]*appCache{}, plan: plan} - if err := client.applyCreateVolume(context.Background(), actx, plan.Steps[1]); err != nil { - t.Fatalf("applyCreateVolume: %v", err) + if !runner.targetStopped { + t.Fatalf("the drifted target was not stopped, calls=%#v", runner.outputArgs) } - var created []string - for _, args := range runner.outputArgs { - if len(args) > 2 && args[0] == "volume" && args[1] == "create" { - created = append(created, args[len(args)-1]) - } + if len(runner.activePins) != 1 { + t.Fatalf("the staging-volume pin must remain active after an unsafe completed deployment, active=%#v", runner.activePins) } - if len(created) != 1 || created[0] != staged[1].VolumeName { - t.Fatalf("expected only %s to be created for the /cache step, got %v", staged[1].VolumeName, created) + if deploys != 1 { + t.Fatalf("mount drift test observed %d deploy requests, want 1", deploys) } } @@ -507,9 +1817,11 @@ func TestSyncVolumeToStagingCreatesOwnedVolumeAndRecordsMount(t *testing.T) { } runner := &fakeDockerRunner{ outputs: map[string][]byte{ - "volume inspect src-vol": []byte(`[{"Name":"src-vol"}]`), - "volume inspect " + staged.VolumeName: []byte(`[{"Name":"` + staged.VolumeName + `"}]`), - "volume create": []byte(staged.VolumeName + "\n"), + "volume inspect src-vol": []byte(`[{"Name":"src-vol"}]`), + "volume inspect " + staged.VolumeName: ownedStagingVolumeInspect(plan, staged), + "volume create": []byte(staged.VolumeName + "\n"), + "create --network none -v " + staged.VolumeName + ":/volume " + volumeCopyImage + " true": []byte("pin-id\n"), + "rm -f pin-id": []byte("pin-id\n"), "inspect --type container src-id": stoppedSourceInspect("2026-01-01T00:00:00Z", "2026-01-02T00:00:00Z"), "ps -a --filter volume=" + staged.VolumeName: []byte(""), }, @@ -540,6 +1852,9 @@ func TestSyncVolumeToStagingCreatesOwnedVolumeAndRecordsMount(t *testing.T) { if !ok || mount.VolumeName != staged.VolumeName { t.Fatalf("expected staging mount recorded, got %#v", actx.entry("api").MigratedVolumeMounts) } + if len(runner.activePins) != 1 { + t.Fatalf("sync must leave the app-wide staging-volume pin active for deployment handoff, active=%#v", runner.activePins) + } } func TestSyncVolumeToStagingRefusesCopyWhoseFlushFails(t *testing.T) { @@ -551,9 +1866,11 @@ func TestSyncVolumeToStagingRefusesCopyWhoseFlushFails(t *testing.T) { } runner := &fakeDockerRunner{ outputs: map[string][]byte{ - "volume inspect src-vol": []byte(`[{"Name":"src-vol"}]`), - "volume inspect " + staged.VolumeName: []byte(`[{"Name":"` + staged.VolumeName + `"}]`), - "volume create": []byte(staged.VolumeName + "\n"), + "volume inspect src-vol": []byte(`[{"Name":"src-vol"}]`), + "volume inspect " + staged.VolumeName: ownedStagingVolumeInspect(plan, staged), + "volume create": []byte(staged.VolumeName + "\n"), + "create --network none -v " + staged.VolumeName + ":/volume " + volumeCopyImage + " true": []byte("pin-id\n"), + "rm -f pin-id": []byte("pin-id\n"), "inspect --type container src-id": stoppedSourceInspect("2026-01-01T00:00:00Z", "2026-01-02T00:00:00Z"), "ps -a --filter volume=" + staged.VolumeName: []byte(""), }, @@ -585,13 +1902,13 @@ func TestSyncVolumeToStagingRefusesForeignOwnedVolume(t *testing.T) { _, plan, step, staged := stagedSyncFixture(t) runner := &fakeDockerRunner{ outputs: map[string][]byte{ - "volume inspect --format {{index .Labels \"bort.run-id\"}} " + staged.VolumeName: []byte("other-run\n"), + "volume inspect " + staged.VolumeName: stagingVolumeInspect(plan, staged, "other-run"), }, } client := &Client{Docker: runner} actx := &applyContext{cache: map[string]*appCache{}, plan: plan} err := client.applySyncVolume(context.Background(), actx, step) - if err == nil || !strings.Contains(err.Error(), "not owned by run \"run1\"") { + if err == nil || !strings.Contains(err.Error(), "label bort.run-id=\"other-run\", want \"run1\"") { t.Fatalf("expected ownership error, got %v", err) } if len(runner.runs) != 0 { @@ -605,7 +1922,7 @@ func TestSyncVolumeToStagingRejectsSourceRestartDuringCopy(t *testing.T) { fakeDockerRunner: fakeDockerRunner{ outputs: map[string][]byte{ "volume inspect src-vol": []byte(`[{"Name":"src-vol"}]`), - "volume inspect " + staged.VolumeName: []byte(`[{"Name":"` + staged.VolumeName + `"}]`), + "volume inspect " + staged.VolumeName: ownedStagingVolumeInspect(plan, staged), "volume inspect --format {{index .Labels \"bort.run-id\"}} " + staged.VolumeName: []byte("run1\n"), "ps -a --filter volume=" + staged.VolumeName: []byte(""), }, @@ -622,6 +1939,9 @@ func TestSyncVolumeToStagingRejectsSourceRestartDuringCopy(t *testing.T) { actx.entry("api").MigratedVolumeMounts = map[string]migratedVolumeMount{ migratedMountKey("web", "/data"): {Service: "web", Target: "/data", VolumeName: staged.VolumeName}, } + if _, err := acquireStagingVolumePin(context.Background(), runner, plan, "api", []stagedVolume{staged}, true); err != nil { + t.Fatalf("seed transferred-state pin: %v", err) + } err := client.applySyncVolume(context.Background(), actx, step) if err == nil || !strings.Contains(err.Error(), "ran during state copy") { @@ -640,7 +1960,7 @@ func TestSyncVolumeToStagingRefusesRunningSource(t *testing.T) { runner := &fakeDockerRunner{ outputs: map[string][]byte{ "volume inspect src-vol": []byte(`[{"Name":"src-vol"}]`), - "volume inspect " + staged.VolumeName: []byte(`[{"Name":"` + staged.VolumeName + `"}]`), + "volume inspect " + staged.VolumeName: ownedStagingVolumeInspect(plan, staged), "volume inspect --format {{index .Labels \"bort.run-id\"}} " + staged.VolumeName: []byte("run1\n"), "ps -a --filter volume=" + staged.VolumeName: []byte(""), "inspect --type container src-id": []byte(`[{"Id":"src-id","Name":"/coolify-web","State":{"Running":true}}]`), @@ -746,34 +2066,13 @@ func stagedRestoreFixture(t *testing.T) (Plan, Step, stagedVolume) { return plan, step, staged[0] } -type stagingVolumeStateRunner struct { - *fakeDockerRunner - volumeName string - present bool -} - -func (r *stagingVolumeStateRunner) Output(ctx context.Context, args ...string) ([]byte, error) { - out, err := r.fakeDockerRunner.Output(ctx, args...) - key := strings.Join(args, " ") - switch { - case key == "volume rm -f "+r.volumeName: - r.present = false - case args[0] == "volume" && args[1] == "create" && args[len(args)-1] == r.volumeName: - r.present = true - case strings.HasPrefix(key, "volume inspect --format") && strings.HasSuffix(key, " "+r.volumeName) && !r.present: - return nil, errors.New("Error response from daemon: get " + r.volumeName + ": no such volume") - } - return out, err -} - func TestRestoreDataStoreToStagingRunsIsolatedComposeProject(t *testing.T) { plan, step, staged := stagedRestoreFixture(t) project := stagingProjectName(plan, "api", "db") - fake := &fakeDockerRunner{ + runner := &fakeDockerRunner{ outputs: map[string][]byte{ "volume inspect --format {{index .Labels \"bort.run-id\"}} " + staged.VolumeName: []byte("run1\n"), - "volume rm -f " + staged.VolumeName: []byte(staged.VolumeName + "\n"), - "volume create": []byte(staged.VolumeName + "\n"), + "volume inspect " + staged.VolumeName: ownedStagingVolumeInspect(plan, staged), "compose -p " + project: []byte("stg-id\n"), "inspect --type container stg-id": []byte(`[{"Id":"stg-id","Name":"/` + project + `-db-1","Config":{"Env":["POSTGRES_USER=bob","POSTGRES_PASSWORD=s3cret","POSTGRES_DB=app"]},"State":{"Running":true,"Status":"running"},"Mounts":[{"Type":"volume","Name":"` + staged.VolumeName + `","Destination":"/var/lib/postgresql/data","RW":true}]}]`), "ps -a --filter volume=" + staged.VolumeName: []byte(""), @@ -783,7 +2082,6 @@ func TestRestoreDataStoreToStagingRunsIsolatedComposeProject(t *testing.T) { "exec -i stg-id pg_restore -l": []byte("271; 1259 100 TABLE public widgets bob\n"), }, } - runner := &stagingVolumeStateRunner{fakeDockerRunner: fake, volumeName: staged.VolumeName, present: true} client := stagingCompatibleClient(t, runner, true, "") actx := &applyContext{cache: map[string]*appCache{}, plan: plan, stagingEnvFormat: stagingEnvFormatKeepInterpolation} actx.entry("api").ComposeAppName = "stack-1" @@ -845,43 +2143,128 @@ func TestRestoreDataStoreToStagingRunsIsolatedComposeProject(t *testing.T) { t.Fatalf("pg_restore must run between compose up and the final down, runs=%#v", runner.runs) } - removed, recreated := false, false - recreateIndex, lookupIndex := -1, -1 - for index, args := range runner.outputArgs { - joined := strings.Join(args, " ") - if joined == "volume rm -f "+staged.VolumeName { - removed = true - } - if removed && !recreated && strings.HasPrefix(joined, "volume create ") && strings.HasSuffix(joined, " "+staged.VolumeName) { - if !strings.Contains(joined, " --label bort.run-id="+stagingOwner(plan)+" ") { - t.Fatalf("recreated staging volume must carry the run label, got %v", args) - } - recreated = true - recreateIndex = index - } - if lookupIndex < 0 && strings.HasPrefix(joined, "compose -p "+project+" ") { - lookupIndex = index + clearIndex := -1 + for index, run := range runner.runs { + if strings.Contains(strings.Join(run.Args, " "), "find /volume -mindepth 1 -delete && sync") { + clearIndex = index + break } } - if !removed || !recreated { - t.Fatalf("staging volume must be recreated before restore, outputs=%v", runner.outputArgs) + if clearIndex < 0 || upIndex < 0 || clearIndex >= upIndex { + t.Fatalf("staging volume must be cleared under its pin before restore, clear=%d up=%d runs=%#v", clearIndex, upIndex, runner.runs) } - if lookupIndex < 0 || lookupIndex < recreateIndex { - t.Fatalf("staging container lookup must follow the volume recreate (recreate=%d lookup=%d), outputs=%v", recreateIndex, lookupIndex, runner.outputArgs) + if fakeOutputCalled(runner, "volume", "rm", "-f", staged.VolumeName) { + t.Fatalf("restore must preserve the owned staging volume identity, outputs=%v", runner.outputArgs) } mount, ok := actx.entry("api").MigratedVolumeMounts[migratedMountKey("db", "/var/lib/postgresql/data")] if !ok || mount.VolumeName != staged.VolumeName { t.Fatalf("expected staging mount recorded after restore, got %#v", actx.entry("api").MigratedVolumeMounts) } + if len(runner.activePins) != 1 { + t.Fatalf("restore must leave the app-wide staging-volume pin active for deployment handoff, active=%#v", runner.activePins) + } +} + +func TestRestoreDataStoreToStagingLeavesUnrelatedAppVolumeAttachedOnlyToPin(t *testing.T) { + plan, step, databaseVolume := stagedRestoreFixture(t) + app := &plan.Prepare.Apps[0] + uploads := preparer.VolumeResource{Service: "web", Type: "volume", Name: "src-uploads", Target: "/uploads"} + app.Resources.Volumes = append(app.Resources.Volumes, uploads) + plan.Steps = slices.Insert(plan.Steps, len(plan.Steps)-1, Step{Kind: StepSyncVolume, App: "api", Ref: "volume:web -> /uploads"}) + uploadsVolume, ok := stagedVolumeFor(plan, "api", uploads) + if !ok { + t.Fatal("uploads volume is not staged") + } + project := stagingProjectName(plan, "api", "db") + runner := &fakeDockerRunner{ + outputs: map[string][]byte{ + "volume inspect " + databaseVolume.VolumeName: ownedStagingVolumeInspect(plan, databaseVolume), + "volume inspect " + uploadsVolume.VolumeName: ownedStagingVolumeInspect(plan, uploadsVolume), + "compose -p " + project: []byte("stg-id\n"), + "inspect --type container stg-id": []byte(`[{"Id":"stg-id","Name":"/` + project + `-db-1","Config":{"Env":["POSTGRES_USER=bob","POSTGRES_PASSWORD=[REDACTED:password]","POSTGRES_DB=app"]},"State":{"Running":true,"Status":"running"},"Mounts":[{"Type":"volume","Name":"` + databaseVolume.VolumeName + `","Destination":"/var/lib/postgresql/data","RW":true}]}]`), + "exec stg-id rm -f": []byte(""), + }, + runOutputs: map[string][]byte{ + "exec -i stg-id pg_restore -l": []byte("271; 1259 100 TABLE public widgets bob\n"), + }, + } + client := stagingCompatibleClient(t, runner, true, "") + actx := &applyContext{cache: map[string]*appCache{}, plan: plan, stagingEnvFormat: stagingEnvFormatKeepInterpolation} + actx.entry("api").ComposeAppName = "stack-1" + + if err := client.applyRestoreDataStore(context.Background(), actx, step); err != nil { + t.Fatalf("restore rejected the unrelated uploads volume: %v", err) + } + if len(runner.activePins) != 1 { + t.Fatalf("app-wide pin was not preserved across restore: %#v", runner.activePins) + } + var pin dockerContainer + for _, candidate := range runner.activePins { + pin = candidate + } + mounted := map[string]bool{} + for _, mount := range pin.Mounts { + if mount.Type == "volume" { + mounted[mount.Name] = !mount.RW + } + } + if !mounted[databaseVolume.VolumeName] || !mounted[uploadsVolume.VolumeName] || len(mounted) != 2 { + t.Fatalf("app-wide pin mounts = %#v, want both staging volumes read-only", pin.Mounts) + } + attachments, err := stagingVolumeAttachmentSets(context.Background(), runner, []stagedVolume{databaseVolume, uploadsVolume}) + if err != nil { + t.Fatal(err) + } + for _, volume := range []stagedVolume{databaseVolume, uploadsVolume} { + if got := attachments[volume.VolumeName]; len(got) != 1 || !stagingContainerIDsMatch(got[0], pin.ID) { + t.Fatalf("staging volume %s attachments = %v, want only pin %s", volume.VolumeName, got, pin.ID) + } + } +} + +func TestRestoreDataStoreReplayRefusesMissingPinAfterDurableTransfer(t *testing.T) { + plan, step, staged := stagedRestoreFixture(t) + seed := &applyContext{plan: plan, cache: map[string]*appCache{}} + if err := seed.recordMigratedVolumeMount("api", migratedVolumeMount{Service: staged.Service, Target: staged.Target, VolumeName: staged.VolumeName}); err != nil { + t.Fatal(err) + } + actx := &applyContext{plan: plan, cache: map[string]*appCache{}, stagingEnvFormat: stagingEnvFormatKeepInterpolation} + if err := actx.loadMigratedVolumeMounts(); err != nil { + t.Fatal(err) + } + actx.entry("api").ComposeID = "c1" + actx.entry("api").ComposeAppName = "stack-1" + runner := &fakeDockerRunner{outputs: map[string][]byte{ + "volume inspect " + staged.VolumeName: ownedStagingVolumeInspect(plan, staged), + }} + client := stagingCompatibleClient(t, runner, true, "") + + err := client.applyRestoreDataStore(context.Background(), actx, step) + if err == nil || !isUnsafeSourceResumeError(err) || !strings.Contains(err.Error(), "missing after state transfer") { + t.Fatalf("expected same-restore replay to refuse pin recreation, got %v", err) + } + for _, run := range runner.runs { + joined := strings.Join(run.Args, " ") + if strings.Contains(joined, "find /volume -mindepth 1 -delete") { + t.Fatalf("same-restore replay cleared data: runs=%#v", runner.runs) + } + } + for _, args := range runner.outputArgs { + if len(args) > 1 && args[0] == "run" && args[1] == "-d" { + t.Fatalf("same-restore replay recreated the missing pin: outputs=%#v", runner.outputArgs) + } + } + if len(runner.activePins) != 0 { + t.Fatalf("same-restore replay left an unexpected pin active: %#v", runner.activePins) + } } -func TestRestoreDataStoreToStagingRefusesToRemoveForeignVolume(t *testing.T) { +func TestRestoreDataStoreToStagingRefusesToClearForeignVolume(t *testing.T) { plan, step, staged := stagedRestoreFixture(t) runner := &fakeDockerRunner{ outputs: map[string][]byte{ - "volume inspect --format {{index .Labels \"bort.run-id\"}} " + staged.VolumeName: []byte("other-run\n"), - "volume rm -f " + staged.VolumeName: []byte(staged.VolumeName + "\n"), - "ps -a --filter volume=" + staged.VolumeName: []byte(""), + "volume inspect " + staged.VolumeName: stagingVolumeInspect(plan, staged, "other-run"), + "ps -a --filter volume=" + staged.VolumeName: []byte(""), }, } client := stagingCompatibleClient(t, runner, true, "") @@ -889,11 +2272,13 @@ func TestRestoreDataStoreToStagingRefusesToRemoveForeignVolume(t *testing.T) { actx.entry("api").ComposeAppName = "stack-1" err := client.applyRestoreDataStore(context.Background(), actx, step) - if err == nil || !strings.Contains(err.Error(), "not owned by run") { + if err == nil || !strings.Contains(err.Error(), "label bort.run-id=\"other-run\", want \"run1\"") { t.Fatalf("expected foreign volume refusal, got %v", err) } - if fakeOutputCalled(runner, "volume", "rm", "-f", staged.VolumeName) { - t.Fatalf("restore removed a volume it does not own: %#v", runner.outputArgs) + for _, run := range runner.runs { + if strings.Contains(strings.Join(run.Args, " "), "find /volume -mindepth 1 -delete") { + t.Fatalf("restore cleared a volume it does not own: %#v", runner.runs) + } } } @@ -903,7 +2288,7 @@ func TestRestoreDataStoreToStagingStopsProjectAndSkipsRecordOnFailure(t *testing runner := &fakeDockerRunner{ outputs: map[string][]byte{ "volume inspect --format {{index .Labels \"bort.run-id\"}} " + staged.VolumeName: []byte("run1\n"), - "volume rm -f " + staged.VolumeName: []byte(staged.VolumeName + "\n"), + "volume inspect " + staged.VolumeName: ownedStagingVolumeInspect(plan, staged), "ps -a --filter volume=" + staged.VolumeName: []byte(""), "compose -p " + project: []byte("stg-a\nstg-b\n"), }, @@ -925,6 +2310,51 @@ func TestRestoreDataStoreToStagingStopsProjectAndSkipsRecordOnFailure(t *testing } } +type failingStagingUpRunner struct { + *fakeDockerRunner +} + +func (r *failingStagingUpRunner) Run(ctx context.Context, stdin io.Reader, stdout io.Writer, args ...string) error { + if err := r.fakeDockerRunner.Run(ctx, stdin, stdout, args...); err != nil { + return err + } + if slices.Contains(args, "up") { + return errors.New("docker compose response lost after start") + } + return nil +} + +func TestRestoreDataStoreToStagingStopsProjectAfterAmbiguousStartFailure(t *testing.T) { + plan, step, staged := stagedRestoreFixture(t) + project := stagingProjectName(plan, "api", "db") + runner := &failingStagingUpRunner{fakeDockerRunner: &fakeDockerRunner{outputs: map[string][]byte{ + "volume inspect --format {{index .Labels \"bort.run-id\"}} " + staged.VolumeName: []byte("run1\n"), + "volume inspect " + staged.VolumeName: ownedStagingVolumeInspect(plan, staged), + "ps -a --filter volume=" + staged.VolumeName: []byte(""), + }}} + client := stagingCompatibleClient(t, runner, true, "") + actx := &applyContext{cache: map[string]*appCache{}, plan: plan, stagingEnvFormat: stagingEnvFormatKeepInterpolation} + actx.entry("api").ComposeAppName = "stack-1" + + err := client.applyRestoreDataStore(context.Background(), actx, step) + if err == nil || !strings.Contains(err.Error(), "docker compose response lost after start") { + t.Fatalf("expected staging start failure, got %v", err) + } + var composeRuns []string + for _, run := range runner.runs { + joined := strings.Join(run.Args, " ") + if strings.HasPrefix(joined, "compose -p "+project+" ") { + composeRuns = append(composeRuns, joined) + } + } + if len(composeRuns) != 3 || !strings.Contains(composeRuns[1], " up -d ") || !strings.HasSuffix(composeRuns[2], " down --remove-orphans") { + t.Fatalf("failed staging start must be followed by teardown, runs=%v", composeRuns) + } + if len(actx.entry("api").MigratedVolumeMounts) != 0 { + t.Fatalf("failed staging start must not record mounts, got %#v", actx.entry("api").MigratedVolumeMounts) + } +} + func TestComposeFileForApplyRewritesVolumesOnlyAfterStagedTransfer(t *testing.T) { plan, _, staged := stagedRestoreFixture(t) client := &Client{Docker: &fakeDockerRunner{}} @@ -990,7 +2420,7 @@ func TestSyncVolumeToStagingRefusesAttachedVolumeBeforeCopy(t *testing.T) { runner := &fakeDockerRunner{ outputs: map[string][]byte{ "volume inspect src-vol": []byte(`[{"Name":"src-vol"}]`), - "volume inspect " + staged.VolumeName: []byte(`[{"Name":"` + staged.VolumeName + `"}]`), + "volume inspect " + staged.VolumeName: ownedStagingVolumeInspect(plan, staged), "volume inspect --format {{index .Labels \"bort.run-id\"}} " + staged.VolumeName: []byte("run1\n"), "ps -a --filter volume=" + staged.VolumeName: []byte("deployed-id\n"), "inspect --type container src-id": stoppedSourceInspect("2026-01-01T00:00:00Z", "2026-01-02T00:00:00Z"), @@ -999,7 +2429,7 @@ func TestSyncVolumeToStagingRefusesAttachedVolumeBeforeCopy(t *testing.T) { client := &Client{Docker: runner} actx := &applyContext{cache: map[string]*appCache{}, plan: plan} err := client.applySyncVolume(context.Background(), actx, step) - if err == nil || !strings.Contains(err.Error(), "attached to container(s) deployed-id") { + if err == nil || !strings.Contains(err.Error(), "attached to unexpected container deployed-id") { t.Fatalf("expected attached-volume refusal, got %v", err) } if len(runner.runs) != 0 { @@ -1020,7 +2450,7 @@ func TestStagingOwnershipUsesRunIDOverRunName(t *testing.T) { } runner := &fakeDockerRunner{ outputs: map[string][]byte{ - "volume inspect --format {{index .Labels \"bort.run-id\"}} " + staged.VolumeName: []byte("run1\n"), + "volume inspect " + staged.VolumeName: stagingVolumeInspect(plan, staged, "run1"), }, } client := &Client{Docker: runner} @@ -1061,7 +2491,7 @@ func TestRestoreDataStoreToStagingRefusesUnstagedDataDir(t *testing.T) { runner := &fakeDockerRunner{ outputs: map[string][]byte{ "volume inspect --format {{index .Labels \"bort.run-id\"}} " + staged.VolumeName: []byte("run1\n"), - "volume rm -f " + staged.VolumeName: []byte(staged.VolumeName + "\n"), + "volume inspect " + staged.VolumeName: ownedStagingVolumeInspect(plan, staged), "ps -a --filter volume=" + staged.VolumeName: []byte(""), "compose -p " + project: []byte("stg-id\n"), "inspect --type container stg-id": []byte(`[{"Id":"stg-id","Name":"/` + project + `-db-1",` + inspect + `,"State":{"Running":true,"Status":"running"}}]`), @@ -1242,6 +2672,11 @@ func TestValidatePlannedPostgresDataDirsRefusesUnstagedLayoutsBeforeLiveApply(t compose: "services:\n db:\n image: postgres:16\n environment:\n PGDATA: /var/lib/postgresql/data\n volumes:\n - pgdata:/var/lib/postgresql/data\n - wal:/var/lib/postgresql/data/pg_wal\nvolumes:\n pgdata:\n wal:\n", volumes: []preparer.VolumeResource{{Service: "db", Type: "volume", Name: "src-pgdata", Target: "/var/lib/postgresql/data"}, {Service: "db", Type: "volume", Name: "src-wal", Target: "/var/lib/postgresql/data/pg_wal"}}, }, + "named volume outside PGDATA": { + compose: "services:\n db:\n image: postgres:16\n environment:\n PGDATA: /var/lib/postgresql/data\n volumes:\n - pgdata:/var/lib/postgresql/data\n - backups:/backups\nvolumes:\n pgdata:\n backups:\n", + volumes: []preparer.VolumeResource{{Service: "db", Type: "volume", Name: "src-pgdata", Target: "/var/lib/postgresql/data"}, {Service: "db", Type: "volume", Name: "src-backups", Target: "/backups"}}, + want: "named volume \"src-backups\" at /backups", + }, "named volume beside custom PGDATA": { compose: "services:\n db:\n image: postgres:16\n environment:\n PGDATA: /pg/data\n volumes:\n - pgdata:/var/lib/postgresql/data\nvolumes:\n pgdata:\n", volumes: []preparer.VolumeResource{{Service: "db", Type: "volume", Name: "src-pgdata", Target: "/var/lib/postgresql/data"}}, @@ -1363,10 +2798,11 @@ func TestPauseSourcePreflightsStagedRestoreOnCreatedContainer(t *testing.T) { runner := &fakeDockerRunner{ outputs: map[string][]byte{ "volume inspect --format {{index .Labels \"bort.run-id\"}} " + staged.VolumeName: []byte("run1\n"), - "compose -p " + project: []byte("stg-id\n"), - "inspect --type container stg-id": []byte(`[{"Id":"stg-id","Name":"/` + project + `-db-1","Config":{"Env":[` + tc.env + `]},"State":{"Running":false,"Status":"created"},"Mounts":[{"Type":"volume","Name":"` + staged.VolumeName + `","Destination":"/var/lib/postgresql/data","RW":true}]}]`), - "inspect --type container web-id": []byte(`[{"Id":"web-id","Name":"/web","State":{"Running":true,"Status":"running"}}]`), - "stop web-id": []byte("web-id\n"), + "volume inspect " + staged.VolumeName: ownedStagingVolumeInspect(plan, staged), + "compose -p " + project: []byte("stg-id\n"), + "inspect --type container stg-id": []byte(`[{"Id":"stg-id","Name":"/` + project + `-db-1","Config":{"Env":[` + tc.env + `]},"State":{"Running":false,"Status":"created"},"Mounts":[{"Type":"volume","Name":"` + staged.VolumeName + `","Destination":"/var/lib/postgresql/data","RW":true}]}]`), + "inspect --type container web-id": []byte(`[{"Id":"web-id","Name":"/web","State":{"Running":true,"Status":"running"}}]`), + "stop web-id": []byte("web-id\n"), }, } client := stagingCompatibleClient(t, runner, true, "") @@ -1444,6 +2880,7 @@ func TestPauseSourcePreflightStopsStagingProjectAfterFailedCreate(t *testing.T) runner := &failingCreateRunner{fakeDockerRunner: &fakeDockerRunner{ outputs: map[string][]byte{ "volume inspect --format {{index .Labels \"bort.run-id\"}} " + staged.VolumeName: []byte("run1\n"), + "volume inspect " + staged.VolumeName: ownedStagingVolumeInspect(plan, staged), }, }} client := stagingCompatibleClient(t, runner, true, "") @@ -1532,9 +2969,10 @@ func testApplyPreflightRefusal(t *testing.T, tc applyPreflightRefusalCase) { runner := &fakeDockerRunner{ outputs: map[string][]byte{ "volume inspect --format {{index .Labels \"bort.run-id\"}} " + staged.VolumeName: []byte("run1\n"), - "compose -p " + project: []byte("stg-id\n"), - "inspect --type container stg-id": []byte(`[{"Id":"stg-id","Name":"/` + project + `-db-1","Config":{"Env":["PGDATA=/pg/data"]},"State":{"Running":false,"Status":"created"},"Mounts":[{"Type":"volume","Name":"` + staged.VolumeName + `","Destination":"/var/lib/postgresql/data","RW":true}]}]`), - "inspect --type container web-id": []byte(`[{"Id":"web-id","Name":"/web","State":{"Running":true,"Status":"running"}}]`), + "volume inspect " + staged.VolumeName: ownedStagingVolumeInspect(plan, staged), + "compose -p " + project: []byte("stg-id\n"), + "inspect --type container stg-id": []byte(`[{"Id":"stg-id","Name":"/` + project + `-db-1","Config":{"Env":["PGDATA=/pg/data"]},"State":{"Running":false,"Status":"created"},"Mounts":[{"Type":"volume","Name":"` + staged.VolumeName + `","Destination":"/var/lib/postgresql/data","RW":true}]}]`), + "inspect --type container web-id": []byte(`[{"Id":"web-id","Name":"/web","State":{"Running":true,"Status":"running"}}]`), }, } if tc.earlierPartialPause { @@ -1584,8 +3022,8 @@ func testApplyPreflightRefusal(t *testing.T, tc applyPreflightRefusalCase) { if errors.As(err, new(stagedRestorePreflightError)) != requiresNewRun { t.Fatalf("preflight error classified as layout refusal = %v, want %v: %v", !requiresNewRun, requiresNewRun, err) } - if strings.Contains(err.Error(), "resume source app") != tc.resumeFails || strings.Contains(err.Error(), "was not durably recorded") { - t.Fatalf("resume failure reported = %v, want %v: %v", strings.Contains(err.Error(), "resume source app"), tc.resumeFails, err) + if strings.Contains(err.Error(), "remains stopped after failed apply") != tc.resumeFails || strings.Contains(err.Error(), "was not durably recorded") { + t.Fatalf("resume failure reported = %v, want %v: %v", strings.Contains(err.Error(), "remains stopped after failed apply"), tc.resumeFails, err) } var first, last *StepProgress resumed := false @@ -1622,3 +3060,111 @@ func testApplyPreflightRefusal(t *testing.T, tc applyPreflightRefusalCase) { t.Fatalf("source restart after refusal = %v (resume progress %v), want %v/%v: calls=%v", fakeOutputCalled(runner, "start", "web-id"), resumed, tc.earlierPartialPause, wantResumed, runner.outputArgs) } } + +func TestReleaseTargetAuthorityPinsSkipsDokployAPIWithoutStagedState(t *testing.T) { + _, plan, _, _ := stagedSyncFixture(t) + plan.TargetIdentities = map[string]TargetIdentity{"api": {ProjectID: "project-1", EnvironmentID: "env-1", ComposeID: "compose-1", ComposeAppName: "stack-1"}} + runner := &fakeDockerRunner{outputs: map[string][]byte{ + "ps -a --filter label=bort.staging-pin=true --filter label=bort.run-id=run1 --format {{.ID}}": nil, + }} + if err := (&Client{Docker: runner}).ReleaseStagingVolumePins(context.Background(), plan, true); err != nil { + t.Fatalf("target finalization without staged state needed the Dokploy API: %v", err) + } +} + +func TestRequireSourceMountsStageDataDirRefusesAuxiliaryVolumeWithDefaultPGDATA(t *testing.T) { + app := preparer.AppPlan{Name: "api"} + app.Resources.Volumes = []preparer.VolumeResource{ + {Service: "db", Type: "volume", Name: "pgdata", Target: "/var/lib/postgresql/data", ReadWrite: true}, + {Service: "db", Type: "volume", Name: "backups", Target: "/backups", ReadWrite: true}, + } + staged := []stagedVolume{ + {Service: "db", Target: "/var/lib/postgresql/data", VolumeName: "bort-pgdata"}, + {Service: "db", Target: "/backups", VolumeName: "bort-backups"}, + } + err := requireSourceMountsStageDataDir(app, "db", postgresDataDir(dockerContainer{}), staged) + if !errors.Is(err, ErrNotImplemented) || !strings.Contains(err.Error(), `"backups" at /backups`) { + t.Fatalf("default-PGDATA layout must refuse the auxiliary volume before pause_source, got %v", err) + } +} + +func TestReleaseTargetAuthorityWithoutPinSkipsLiveAttachmentChecks(t *testing.T) { + _, plan, _, staged := stagedSyncFixture(t) + plan.StagingTransferApps = []string{"api"} + plan.TargetIdentities = map[string]TargetIdentity{"api": {ProjectID: "project-1", EnvironmentID: "env-1", ComposeID: "compose-1", ComposeAppName: "stack-1"}} + state := &applyContext{plan: plan, cache: map[string]*appCache{}} + if err := state.recordMigratedVolumeMount("api", migratedVolumeMount{Service: "web", Target: "/data", VolumeName: staged.VolumeName}); err != nil { + t.Fatal(err) + } + runner := &fakeDockerRunner{outputs: map[string][]byte{ + "ps -a --filter label=bort.staging-pin=true --filter label=bort.run-id=run1 --format {{.ID}}": nil, + }} + + if err := (&Client{Docker: runner}).ReleaseStagingVolumePins(context.Background(), plan, true); err != nil { + t.Fatalf("target finalization with no pin left depended on live target state: %v", err) + } + for _, args := range runner.outputArgs { + if len(args) > 0 && args[0] != "ps" || strings.Contains(strings.Join(args, " "), "volume=") { + t.Fatalf("target finalization with no pin left inspected live attachments: %v", runner.outputArgs) + } + } +} + +func TestIncompleteStagingTransfersRequiresEveryStagedVolumeRecord(t *testing.T) { + app, plan, firstStep, _ := stagedSyncFixture(t) + app.Resources.Volumes = append(app.Resources.Volumes, preparer.VolumeResource{Service: "worker", Type: "volume", Name: "worker-data", Target: "/work"}) + plan = stagedPlan(t, app, plan.RunDir, Step{Kind: StepPauseSource, App: "api"}, firstStep, Step{Kind: StepSyncVolume, App: "api", Ref: "volume:worker -> /work"}) + plan.StagingTransferApps = []string{"api"} + first, _ := stagedVolumeFor(plan, "api", app.Resources.Volumes[0]) + second, _ := stagedVolumeFor(plan, "api", app.Resources.Volumes[1]) + actx := &applyContext{plan: plan, cache: map[string]*appCache{}} + for i, volume := range []stagedVolume{first, second} { + if incomplete, err := IncompleteStagingTransfers(plan); err != nil || len(incomplete) != 1 || incomplete[0] != "api" { + t.Fatalf("after %d of 2 volume records: incomplete = %v, %v; want [api]", i, incomplete, err) + } + if err := actx.recordMigratedVolumeMount("api", migratedVolumeMount{Service: volume.Service, Target: volume.Target, VolumeName: volume.VolumeName}); err != nil { + t.Fatal(err) + } + } + if incomplete, err := IncompleteStagingTransfers(plan); err != nil || len(incomplete) != 0 { + t.Fatalf("fully recorded transfer reported incomplete: %v, %v", incomplete, err) + } +} + +func TestValidateTargetAuthorityRequiresHandoffEvidenceWhenPinIsMissing(t *testing.T) { + _, plan, _, staged := stagedSyncFixture(t) + plan.StagingTransferApps = []string{"api"} + plan.RunID = "run-digest" + staged, _ = stagedVolumeFor(plan, "api", plan.Prepare.Apps[0].Resources.Volumes[0]) + const targetID = "target-id" + runner := &fakeDockerRunner{outputs: map[string][]byte{ + "volume inspect " + staged.VolumeName: ownedStagingVolumeInspect(plan, staged), + "ps -a --filter label=com.docker.compose.project=stack-1 --format {{.ID}}": []byte(targetID + "\n"), + "inspect --type container " + targetID: []byte(`[{"Id":"` + targetID + `","Name":"/web","Config":{"Labels":{"com.docker.compose.service":"web","com.docker.compose.project":"stack-1"}},"State":{"Running":true,"Status":"running"},"Mounts":[{"Type":"volume","Name":"fresh-data","Destination":"/data","RW":true}]}]`), + }, activeComposeProjects: map[string][]string{"stack-1": {targetID}}} + client := targetAuthorityTestClient(t, runner, &plan) + actx := &applyContext{plan: plan, cache: map[string]*appCache{}} + if err := actx.recordMigratedVolumeMount("api", migratedVolumeMount{Service: "web", Target: "/data", VolumeName: staged.VolumeName}); err != nil { + t.Fatal(err) + } + + if err := client.ValidateStagingVolumePins(context.Background(), plan, true); err == nil || !strings.Contains(err.Error(), "fresh-data") { + t.Fatalf("target validation accepted a missing pin without handoff evidence while the target mounts fresh state: %v", err) + } + plan.HandedOffApps = []string{"api"} + if err := client.ValidateStagingVolumePins(context.Background(), plan, true); err != nil { + t.Fatalf("target validation refused a recorded handoff whose pin was released: %v", err) + } +} + +func TestRequireSourceMountsStageDataDirRefusesUnmeasuredAnonymousVolume(t *testing.T) { + app := preparer.AppPlan{Name: "api"} + app.Resources.Volumes = []preparer.VolumeResource{ + {Service: "db", Type: "volume", Name: "pgdata", Target: "/pgdata", ReadWrite: true}, + {Service: "db", Type: "volume", Name: strings.Repeat("ab", 32), Target: "/var/lib/postgresql/data", ReadWrite: true}, + } + staged := []stagedVolume{{Service: "db", Target: "/pgdata", VolumeName: "bort-pgdata"}, {Service: "db", Target: "/var/lib/postgresql/data", VolumeName: "bort-anon"}} + if err := requireSourceMountsStageDataDir(app, "db", "/pgdata", staged); err == nil || !errors.Is(err, ErrNotImplemented) { + t.Fatalf("anonymous volume outside PGDATA with unknown contents was accepted: %v", err) + } +}