diff --git a/.github/workflows/clippy.yml b/.github/workflows/clippy.yml index b5afc1a3b..fafd81908 100644 --- a/.github/workflows/clippy.yml +++ b/.github/workflows/clippy.yml @@ -32,4 +32,11 @@ jobs: with: # Only pushes to main save, so other runs restore main's cache. save-if: ${{ github.ref == 'refs/heads/main' }} - - run: cargo clippy --all-targets -- -D warnings + # With the Cassandra and MongoDB crates' `test-hooks` features, so the + # hook modules (`src/test_hooks.rs`) and their call sites, which only + # the integration test binaries build otherwise, are linted too. + - run: cargo clippy --all-targets --features extenddb-storage-cassandra/test-hooks,extenddb-storage-mongodb/test-hooks -- -D warnings + # The backup job's kill-point pauses (`EXTENDDB_BACKUP_PAUSE`), which + # `devtools/backup-kill-points.sh` runs by hand on a build with the + # feature; no other job builds it. + - run: cargo clippy --all-targets --features extenddb-engine/backup-test-hooks -- -D warnings diff --git a/.github/workflows/integration-cassandra.yml b/.github/workflows/integration-cassandra.yml index 65f3d557d..8ec847ce2 100644 --- a/.github/workflows/integration-cassandra.yml +++ b/.github/workflows/integration-cassandra.yml @@ -113,8 +113,10 @@ jobs: - name: Direct storage-trait integration tests # Ported from the plug-in repo's tests/rust suite. Parallel-safe: # each test provisions its own account and keyspace; the shared - # control-plane setting test restores what it mutates. - run: cargo test -p extenddb-storage-cassandra --test direct_integration + # control-plane setting test restores what it mutates. The + # test-hooks feature compiles the tests that pause an operation or + # shorten a heartbeat (crates/storage-cassandra/src/test_hooks.rs). + run: cargo test -p extenddb-storage-cassandra --features test-hooks --test direct_integration env: EXTENDDB_TEST_CASSANDRA: required @@ -171,6 +173,10 @@ jobs: - name: Build release (cassandra backend) run: cargo build --release --no-default-features --features cassandra - name: Run Cassandra pytest suite + env: + # run-cassandra-tests initializes the server without [backup.store], + # and Cassandra takes the in-catalog create with one. + EXTENDDB_TEST_BACKUP_STORE: none run: devtools/run-cassandra-tests -- --pytest --comprehensive --parallel cassandra-rust: @@ -226,6 +232,10 @@ jobs: - name: Build release (cassandra backend) run: cargo build --release --no-default-features --features cassandra - name: Run Cassandra rust integration suite + env: + # run-cassandra-tests initializes the server without [backup.store]: + # the Rust backup tests restore the in-catalog create's backups. + EXTENDDB_TEST_BACKUP_STORE: none run: devtools/run-cassandra-tests -- --rust-integration cassandra-production-build: diff --git a/.github/workflows/integration-mongodb.yml b/.github/workflows/integration-mongodb.yml index 3bf940c58..2f439e0f1 100644 --- a/.github/workflows/integration-mongodb.yml +++ b/.github/workflows/integration-mongodb.yml @@ -53,6 +53,12 @@ jobs: - name: Build release (mongodb backend) run: cargo build --release --no-default-features --features mongodb-test-hooks - name: Run MongoDB pytest suite + env: + # run-mongodb-tests gives the server a filesystem [backup.store] + # under its output directory: CreateBackup is the store-backed job + # and the restore cells (tests/test_mongodb_backup_restore.py) + # rewrite prefixes under that root. + EXTENDDB_TEST_BACKUP_STORE: filesystem run: devtools/run-mongodb-tests -- --pytest --comprehensive --parallel # SnapshotEngine consistency check against a disposable replica set. The @@ -129,8 +135,61 @@ jobs: - name: Build release (mongodb backend) run: cargo build --release --no-default-features --features mongodb-test-hooks - name: Run MongoDB rust integration suite + env: + # run-mongodb-tests gives the server a filesystem [backup.store]: + # the Rust backup tests restore the store-backed job's backups. + EXTENDDB_TEST_BACKUP_STORE: filesystem run: devtools/run-mongodb-tests --rust-live -- --rust --rust-integration + # The store-backed CreateBackup on MongoDB: a server with a filesystem + # store, the backup selection of the pytest suite with + # EXTENDDB_TEST_BACKUP_STORE=filesystem (the fidelity suite's store branch, + # the ABAC and authorization backup cells), and the job suite + # (tests/test_backup_job.py), which starts servers of its own on a replica + # set of its own: the catalog's database names are fixed per replica set, + # and each of its tests destroys and initializes the catalog. + mongodb-backup-store: + runs-on: extenddb_ubuntu-2404_4-core + timeout-minutes: 40 + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master + with: + toolchain: stable + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 + with: + cache-on-failure: true + # Only pushes to main save, so other runs restore main's cache. + save-if: ${{ github.ref == 'refs/heads/main' }} + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.11" + - name: Install Python dependencies + run: pip install -r requirements.txt + - name: Build release (mongodb backend) + run: cargo build --release --no-default-features --features mongodb-test-hooks + - name: Run the backup selection against a filesystem store + env: + EXTENDDB_TEST_BACKUP_STORE: filesystem + run: | + devtools/run-mongodb-tests --mongo-port 27022 --port 18453 \ + --backup-store-root "$RUNNER_TEMP/backups" \ + -- --pytest --parallel --filter "backup" + - name: Start the job suite's replica set + # No pipe: a bootstrap failure must fail this step. + run: devtools/run-mongodb-tests --mongo-only --mongo-port 27023 + - name: Run the backup job suite + env: + EXTENDDB_BINARY: ./target/release/extenddb + EXTENDDB_TEST_MONGODB_URI: mongodb://127.0.0.1:27023/?replicaSet=rs0&directConnection=true + run: python3 -m pytest tests/test_backup_job.py -v -rP -p no:cacheprovider + - name: Stop the job suite's replica set + if: always() + run: docker rm -f extenddb-runtests-mongo-27023 + - name: Dump server syslog on failure + if: failure() + run: sudo journalctl -t extenddb --no-pager -n 500 || true + # The catalog migrations against released catalogs and released binaries, # each check on a mongo:7 replica set of its own that the script starts # with its ports on 127.0.0.1 and removes. The schema check uploads the @@ -191,12 +250,13 @@ jobs: mongodb-integration: runs-on: extenddb_ubuntu-2404_4-core timeout-minutes: 40 - needs: [mongodb-pytest, mongodb-rust, mongodb-catalog, mongodb-production-build, mongodb-snapshot] + needs: [mongodb-pytest, mongodb-rust, mongodb-backup-store, mongodb-catalog, mongodb-production-build, mongodb-snapshot] if: always() steps: - run: | if [ "${{ needs.mongodb-pytest.result }}" != "success" ] || \ [ "${{ needs.mongodb-rust.result }}" != "success" ] || \ + [ "${{ needs.mongodb-backup-store.result }}" != "success" ] || \ [ "${{ needs.mongodb-catalog.result }}" != "success" ] || \ [ "${{ needs.mongodb-production-build.result }}" != "success" ] || \ [ "${{ needs.mongodb-snapshot.result }}" != "success" ]; then diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 7fd08685e..124e64e50 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -28,7 +28,10 @@ jobs: # rather than "latest" because ubuntu-latest migrates to 26.04 in Oct–Nov # 2026 and this compiles Rust; move to 26 deliberately. runs-on: extenddb_ubuntu-2404_4-core - timeout-minutes: 40 + # The job's own suites run close to 40 minutes; the backup store and job + # suites it ends with (their own servers, the restore cells among them) + # add about ten, so the bound leaves room for a slow runner. + timeout-minutes: 55 services: postgres: # pgvector's own image, which is postgres:16 plus the extension. The @@ -71,19 +74,34 @@ jobs: password=$(echo "$output" | grep -oP 'Password: \K\S+') echo "admin_password=$password" >> "$GITHUB_OUTPUT" + # A filesystem backup store, so this job's CreateBackup is the + # store-backed job (PostgreSQL has no default root). The suite reads + # EXTENDDB_TEST_BACKUP_STORE=filesystem below to expect that path. + - name: Configure a filesystem backup store + run: | + mkdir -m 0700 "$RUNNER_TEMP/backups" + printf '\n[backup.store]\nkind = "filesystem"\n[backup.store.filesystem]\npath = "%s/backups"\n' \ + "$RUNNER_TEMP" >> extenddb.toml + - name: Start ExtendDB run: | # --write-pid-file so devtools/run-tests can restart the server with # 'extenddb stop' when it needs to apply a config change. + # The wait is for the backup store component to be healthy, not for + # the word "healthy" anywhere in the reply: a server whose store + # failed its check reports "unhealthy" (which that word matches) + # and the suite would then fail on a backup cell with no pointer to + # the cause. ./target/release/extenddb serve --config extenddb.toml --foreground --write-pid-file & for i in $(seq 1 30); do - if curl -sk https://127.0.0.1:18443/health | grep -q healthy; then - echo "Server ready" + if curl -sk https://127.0.0.1:18443/health | python3 -c 'import json, sys; sys.exit(0 if json.load(sys.stdin).get("components", {}).get("backup_store", {}).get("status") == "healthy" else 1)' 2>/dev/null; then + echo "Server ready with a healthy backup store" exit 0 fi sleep 1 done - echo "Server failed to start" + echo "Server failed to start, or its backup store did not become healthy" + curl -sk https://127.0.0.1:18443/health || true exit 1 - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 @@ -154,8 +172,37 @@ jobs: EXTENDDB_ADMIN_USER: admin EXTENDDB_ADMIN_PASSWORD: ${{ steps.init.outputs.admin_password }} EXTENDDB_TEST_PG_ADMIN_CONNECTION_STRING: postgresql://postgres:devpass@127.0.0.1:5432 + EXTENDDB_TEST_BACKUP_STORE: filesystem run: devtools/run-tests --extenddb --pytest --comprehensive --parallel --filter "not import_export" + # moto for the job suite's store-outage cell, the one cell that + # exercises `spool_path` on PostgreSQL (an S3 download into the spool, + # the retry under a cut, the spool entry removed); without the endpoint + # the cell skips. Same environment as the s3 job's. + - name: Start S3 mock + run: | + python3 -m venv "$RUNNER_TEMP/moto" + "$RUNNER_TEMP/moto/bin/pip" install -r requirements-moto.txt + nohup "$RUNNER_TEMP/moto/bin/moto_server" -p 9000 > moto.log 2>&1 & + for _ in $(seq 1 30); do + if curl -sf http://127.0.0.1:9000/ > /dev/null; then exit 0; fi + sleep 1 + done + echo "moto_server did not start"; cat moto.log; exit 1 + + # The backup store and job suites start servers of their own, so they + # are excluded from run-tests' line and run by name. + - name: Run the backup store and job suites + env: + EXTENDDB_TEST_PG_CONNECTION_STRING: postgresql://postgres:devpass@127.0.0.1:5432 + EXTENDDB_TEST_PG_ADMIN_CONNECTION_STRING: postgresql://postgres:devpass@127.0.0.1:5432 + EXTENDDB_TEST_S3_ENDPOINT: http://127.0.0.1:9000 + run: python3 -m pytest tests/test_backup_store_config.py tests/test_backup_job.py -v -rP -p no:cacheprovider + + - name: Print the S3 mock's log + if: failure() + run: cat moto.log || true + # run-tests restarts the server daemonized to apply the import/export # config, and a daemonized server logs to syslog, not the job log. Without # this dump, the storage error behind a client-visible 500 leaves no @@ -166,7 +213,10 @@ jobs: run-integration-sqlite: runs-on: extenddb_ubuntu-2404_4-core - timeout-minutes: 40 + # The job's own suites run close to 40 minutes; the backup store and job + # suites it ends with (their own servers, the restore cells among them) + # add about ten, so the bound leaves room for a slow runner. + timeout-minutes: 55 steps: - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 @@ -228,15 +278,21 @@ jobs: run: | # --write-pid-file so devtools/run-tests can restart the server with # 'extenddb stop' when it needs to apply a config change. + # The wait is for the backup store component to be healthy, not for + # the word "healthy" anywhere in the reply: a server whose store + # failed its check reports "unhealthy" (which that word matches) + # and the suite would then fail on a backup cell with no pointer to + # the cause. ./target/release/extenddb serve --config extenddb.toml --foreground --write-pid-file & for i in $(seq 1 30); do - if curl -sk https://127.0.0.1:18443/health | grep -q healthy; then - echo "Server ready" + if curl -sk https://127.0.0.1:18443/health | python3 -c 'import json, sys; sys.exit(0 if json.load(sys.stdin).get("components", {}).get("backup_store", {}).get("status") == "healthy" else 1)' 2>/dev/null; then + echo "Server ready with a healthy backup store" exit 0 fi sleep 1 done - echo "Server failed to start" + echo "Server failed to start, or its backup store did not become healthy" + curl -sk https://127.0.0.1:18443/health || true exit 1 - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 @@ -246,19 +302,171 @@ jobs: - name: Install Python dependencies run: pip install -r requirements.txt + # Local S3 endpoint for tests/test_backup_store_config.py's S3 cells + # (an S3 store's /health against an endpoint that refuses, stalls and + # recovers), which devtools/run-tests runs on a SQLite build after the + # main suite. Same moto as the unit-test workflow, pinned in + # requirements-moto.txt, installed into a virtualenv of its own: its + # boto3 and botocore pins differ from requirements.txt's, and the + # suite keeps running on the environment installed above. On port + # 9200, clear of the server's. + - name: Start S3 mock + run: | + python -m venv "$RUNNER_TEMP/moto-venv" + "$RUNNER_TEMP/moto-venv/bin/pip" install -r requirements-moto.txt + nohup "$RUNNER_TEMP/moto-venv/bin/moto_server" -p 9200 > moto.log 2>&1 & + for _ in $(seq 1 30); do + if curl -sf http://127.0.0.1:9200/ > /dev/null; then exit 0; fi + sleep 1 + done + echo "moto_server did not start"; cat moto.log; exit 1 + - name: Run integration tests env: EXTENDDB_TEST_ENDPOINT: https://127.0.0.1:18443 EXTENDDB_ADMIN_USER: admin EXTENDDB_ADMIN_PASSWORD: ${{ steps.init.outputs.admin_password }} + # The backup store wire tests' S3 cells; the file sets the AWS_* + # variables for the servers it starts, so none leaks here. + EXTENDDB_TEST_S3_ENDPOINT: http://127.0.0.1:9200 + # A file-backed SQLite server without [backup.store] writes to the + # defaulted root beside the database file. + EXTENDDB_TEST_BACKUP_STORE: filesystem run: devtools/run-tests --extenddb --pytest --comprehensive --parallel --filter "not import_export" + - name: Run the backup store and job suites + run: python3 -m pytest tests/test_backup_store_config.py tests/test_backup_job.py -v -rP -p no:cacheprovider + # Same rationale as the postgres job: the daemonized server logs to # syslog, so this dump is the only server-side evidence on failure. - name: Dump server syslog on failure if: failure() run: sudo journalctl -t extenddb --no-pager -n 500 || true + - name: Print the S3 mock's log + if: failure() + run: cat moto.log || true + + # The S3 leg: the SQLite build against an S3 backup store on moto. The + # backup tests of the main suite (the tests named "backup") run against it + # with EXTENDDB_TEST_BACKUP_STORE=s3, and the fidelity suite's health check + # confirms the server's store is the S3 one; the store configuration + # suite's S3 cells need the same endpoint. The job suite + # (tests/test_backup_job.py) starts servers of its own: most of its cells + # on the SQLite build's defaulted filesystem root, as on the SQLite job, + # and the store-outage cell on an S3 store at the moto endpoint, which it + # cuts mid-download and restores, so this job runs one restore from an S3 + # store. + run-integration-sqlite-s3: + runs-on: extenddb_ubuntu-2404_4-core + timeout-minutes: 40 + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + + - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master + with: + toolchain: stable + + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 + with: + cache-on-failure: true + # Only pushes to main save, so other runs restore main's cache. + save-if: ${{ github.ref == 'refs/heads/main' }} + + - name: Build release (SQLite backend) + run: cargo build --release -p extenddb --no-default-features --features sqlite + + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Install Python dependencies + run: pip install -r requirements.txt + + # moto is a pip package pinned in requirements-moto.txt (test.yml runs + # the same server for the store crate's conformance suite); its boto3 + # floor differs from requirements.txt's pin, so it gets an environment + # of its own. + - name: Start S3 mock + run: | + python3 -m venv "$RUNNER_TEMP/moto" + "$RUNNER_TEMP/moto/bin/pip" install -r requirements-moto.txt + nohup "$RUNNER_TEMP/moto/bin/moto_server" -p 9000 > moto.log 2>&1 & + for _ in $(seq 1 30); do + if curl -sf http://127.0.0.1:9000/ > /dev/null; then exit 0; fi + sleep 1 + done + echo "moto_server did not start"; cat moto.log; exit 1 + + - name: Initialize ExtendDB + id: init + env: + AWS_ACCESS_KEY_ID: testing + AWS_SECRET_ACCESS_KEY: testing + AWS_REGION: us-east-1 + run: | + output=$(./target/release/extenddb init --backend sqlite --config extenddb.toml 2>&1) + echo "$output" + password=$(echo "$output" | grep -oP 'Password: \K\S+') + echo "admin_password=$password" >> "$GITHUB_OUTPUT" + python3 -c 'import boto3; boto3.client("s3", endpoint_url="http://127.0.0.1:9000", region_name="us-east-1").create_bucket(Bucket="extenddb-ci-backups")' + printf '\n[backup.store]\nkind = "s3"\n[backup.store.s3]\nbucket = "extenddb-ci-backups"\nendpoint = "http://127.0.0.1:9000"\nforce_path_style = true\nregion = "us-east-1"\n' >> extenddb.toml + + - name: Start ExtendDB + env: + AWS_ACCESS_KEY_ID: testing + AWS_SECRET_ACCESS_KEY: testing + AWS_REGION: us-east-1 + AWS_EC2_METADATA_DISABLED: "true" + run: | + # An S3 store is checked after the listener is up and reports + # "checking" until its first check returns; a CreateBackup inside + # that window is refused, so the wait is for the component to be + # healthy, which also names a store that failed its check here + # rather than in a backup cell. + ./target/release/extenddb serve --config extenddb.toml --foreground --write-pid-file & + for i in $(seq 1 30); do + if curl -sk https://127.0.0.1:18443/health | python3 -c 'import json, sys; sys.exit(0 if json.load(sys.stdin).get("components", {}).get("backup_store", {}).get("status") == "healthy" else 1)' 2>/dev/null; then + echo "Server ready with a healthy backup store" + exit 0 + fi + sleep 1 + done + echo "Server failed to start, or its backup store did not become healthy" + curl -sk https://127.0.0.1:18443/health || true + exit 1 + + - name: Run the backup tests against the S3 store + env: + EXTENDDB_TEST_ENDPOINT: https://127.0.0.1:18443 + EXTENDDB_ADMIN_USER: admin + EXTENDDB_ADMIN_PASSWORD: ${{ steps.init.outputs.admin_password }} + EXTENDDB_TEST_BACKUP_STORE: s3 + # run-tests restarts the server daemonized, which needs the store's + # credentials in its environment too. + AWS_ACCESS_KEY_ID: testing + AWS_SECRET_ACCESS_KEY: testing + AWS_REGION: us-east-1 + AWS_EC2_METADATA_DISABLED: "true" + run: devtools/run-tests --extenddb --pytest --parallel --filter "backup" + + # The store configuration suite's S3 cells against moto; the job suite + # against the build's defaulted filesystem root, except the store-outage + # cell, which runs against moto through EXTENDDB_TEST_S3_ENDPOINT (the + # job comment above). + - name: Run the store configuration suite's S3 cells and the job suite + env: + EXTENDDB_TEST_S3_ENDPOINT: http://127.0.0.1:9000 + run: python3 -m pytest tests/test_backup_store_config.py tests/test_backup_job.py -v -rP -p no:cacheprovider + + - name: Dump server syslog on failure + if: failure() + run: sudo journalctl -t extenddb --no-pager -n 500 || true + + - name: Print the S3 mock's log + if: failure() + run: cat moto.log || true + run-integration-dev-mode: # dev-mode was shipped with no CI coverage at all, which is how the batch and # transaction authorization regression reached main: the build compiled, so @@ -400,6 +608,9 @@ jobs: # tests mandatory. The refusal suite moves to the novector job, which is # the only place those assertions still mean anything. EXTENDDB_EXPECT_VECTORS: "1" + # This server has no [backup.store]: CreateBackup is the in-catalog + # create and the backup tests restore its backups. + EXTENDDB_TEST_BACKUP_STORE: none run: devtools/run-tests --extenddb --rust-integration --release # The control plane for vector indexes is not reachable over the wire while @@ -500,6 +711,9 @@ jobs: # the backend reports, so without this it could skip every assertion # and still report green. EXTENDDB_EXPECT_VECTORS: "0" + # No [backup.store] on this server either; the filter runs no backup + # test, and the variable says so for whoever widens it. + EXTENDDB_TEST_BACKUP_STORE: none run: >- devtools/run-tests --extenddb --rust-integration --release --filter vector_index_unsupported @@ -585,6 +799,12 @@ jobs: # which is what would happen if the backend lost the capability. Pinning the # expectation turns that skip into a failure. EXTENDDB_EXPECT_VECTORS: "1" + # A file-backed SQLite server without [backup.store] has the defaulted + # filesystem root beside its database file, so CreateBackup is the + # asynchronous, store-backed one: the backup tests wait for AVAILABLE, + # delete what they create, and restore the store-backed backups + # (the round trip). + EXTENDDB_TEST_BACKUP_STORE: filesystem run: devtools/run-tests --extenddb --rust-integration --release # The daemonized server logs to syslog; dump it so server-side failures @@ -600,6 +820,7 @@ jobs: [ run-integration, run-integration-sqlite, + run-integration-sqlite-s3, run-integration-dev-mode, run-rust-integration, run-rust-integration-postgres-novector, @@ -610,6 +831,7 @@ jobs: - run: | if [ "${{ needs.run-integration.result }}" != "success" ] || \ [ "${{ needs.run-integration-sqlite.result }}" != "success" ] || \ + [ "${{ needs.run-integration-sqlite-s3.result }}" != "success" ] || \ [ "${{ needs.run-integration-dev-mode.result }}" != "success" ] || \ [ "${{ needs.run-rust-integration.result }}" != "success" ] || \ [ "${{ needs.run-rust-integration-postgres-novector.result }}" != "success" ] || \ diff --git a/.github/workflows/matrix-nightly.yml b/.github/workflows/matrix-nightly.yml new file mode 100644 index 000000000..acd126e3f --- /dev/null +++ b/.github/workflows/matrix-nightly.yml @@ -0,0 +1,481 @@ +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 +name: Cross-backend restore matrix (nightly) + +# The cross-backend restore matrix (design PR 9b): a backup taken on one +# backend restores, through the store, into another. Each cell starts a source +# and a target server from the per-backend binaries, copies the source's backup +# prefix under a root the target owns, adopts it into the target's catalog with +# `extenddb backup sync`, restores and compares; the --adopt-store cells point +# the target at the source's root instead and adopt the root with `backup sync +# --adopt-store`. The cells are heavy (two servers each, four backends, +# PostgreSQL + a MongoDB replica set + Cassandra) so they run nightly rather +# than on every pull request. + +on: + schedule: + - cron: "0 7 * * *" + workflow_dispatch: + +permissions: + contents: read + +# One run at a time: the cells drop and re-create fixed MongoDB database and +# Cassandra keyspace names, so two runs on one runner would destroy each +# other's catalogs. A second run queues behind the first. +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: false + +jobs: + matrix: + runs-on: extenddb_ubuntu-2404_4-core + timeout-minutes: 120 + services: + postgres: + image: pgvector/pgvector:pg16 + env: + POSTGRES_PASSWORD: devpass + ports: + - 127.0.0.1:55432:5432 + options: >- + --health-cmd "pg_isready -U postgres" + --health-interval 10s + --health-timeout 5s + --health-retries 10 + cassandra: + image: cassandra:4.1 + ports: + - 127.0.0.1:9042:9042 + env: + CASSANDRA_CLUSTER_NAME: extenddb-ci + HEAP_NEWSIZE: 128M + MAX_HEAP_SIZE: 1024M + options: >- + --health-cmd "cqlsh -e 'SELECT release_version FROM system.local'" + --health-interval 15s + --health-timeout 10s + --health-retries 20 + --health-start-period 60s + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master + with: + toolchain: stable + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 + with: + cache-on-failure: true + # Only runs on main save, so other runs restore main's cache; a + # scheduled run's ref is the default branch, so the nightly saves. + save-if: ${{ github.ref == 'refs/heads/main' }} + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.11" + - name: Install Python dependencies + # requirements-matrix.txt adds the MongoDB and Cassandra drivers the + # script's setup and teardown use on those backends. + run: pip install -r requirements.txt -r requirements-matrix.txt + + - name: Install the PostgreSQL client + # The matrix script tears down every PostgreSQL source and target + # through the host's psql; the runner image is not assumed to + # carry one. + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends postgresql-client + + - name: Check the host tools the matrix script calls + run: | + psql --version + python3 -c "import pymongo, cassandra; print(pymongo.version, cassandra.__version__)" + + - name: Configure Cassandra authentication + run: | + docker exec ${{ job.services.cassandra.id }} sed -i \ + 's/^authenticator: .*/authenticator: PasswordAuthenticator/' \ + /etc/cassandra/cassandra.yaml + docker restart ${{ job.services.cassandra.id }} + for i in $(seq 1 90); do + if docker exec ${{ job.services.cassandra.id }} \ + cqlsh -u cassandra -p cassandra \ + -e 'SELECT release_version FROM system.local' >/dev/null 2>&1; then + echo "Cassandra ready with PasswordAuthenticator after ${i}s" + exit 0 + fi + sleep 2 + done + echo "Cassandra did not become ready" >&2 + exit 1 + + - name: Start a MongoDB replica set + # GitHub `services:` cannot express a replica set (--replSet + initiate + # + wait-for-PRIMARY), so start it directly, as devtools/run-mongodb-tests + # does. Published on loopback only; no authentication. + run: | + docker run -d --name matrix-mongo -p 127.0.0.1:27117:27017 mongo:7 \ + mongod --replSet rs0 --bind_ip_all + for i in $(seq 1 30); do + docker exec matrix-mongo mongosh --quiet --eval 'db.runCommand({ping:1}).ok' 2>/dev/null \ + | grep -qx 1 && break + sleep 1 + done + docker exec matrix-mongo mongosh --quiet --eval \ + 'rs.initiate({_id:"rs0",members:[{_id:0,host:"localhost:27017"}]})' >/dev/null 2>&1 || true + for i in $(seq 1 30); do + state=$(docker exec matrix-mongo mongosh --quiet --eval 'rs.status().members[0].stateStr' 2>/dev/null | tail -1) + [ "$state" = PRIMARY ] && { echo "replica set PRIMARY after ${i}s"; exit 0; } + sleep 1 + done + echo "MongoDB replica set did not elect a PRIMARY" >&2 + exit 1 + + - name: Build the four backend binaries + # One target directory, four sequential builds (cargo rebuilds only + # the feature-dependent crates between them), each binary copied to + # the path the script's --bin defaults use + # (target/rel-/release/extenddb). Every `init` the script + # runs writes the TLS material under ~/.extenddb/tls when absent, so + # nothing has to generate it here. + run: | + for b in postgres sqlite mongodb cassandra; do + cargo build --release -p extenddb --no-default-features --features "$b" + mkdir -p "target/rel-$b/release" + cp target/release/extenddb "target/rel-$b/release/extenddb" + done + + - name: Run the matrix + run: | + set -o pipefail + mkdir -p matrix-logs + rc=0 + plain="postgres sqlite mongodb" + targets="postgres sqlite mongodb cassandra" + for s in $plain; do + for t in $targets; do + echo "=== $s -> $t ===" + python3 devtools/backup-restore-matrix --source "$s" --target "$t" \ + --work "matrix-logs/$s-$t" || rc=1 + done + done + # The duplicate-key cell into every target, from a PostgreSQL source. + for t in $targets; do + echo "=== postgres -> $t --duplicate-key ===" + python3 devtools/backup-restore-matrix --source postgres --target "$t" --duplicate-key \ + --work "matrix-logs/postgres-$t-dup" || rc=1 + done + # The vector cells: into PostgreSQL and SQLite the vector index is + # restored and answers; into MongoDB and Cassandra the manifest is + # refused before any table. + for s in postgres sqlite; do + for t in postgres sqlite mongodb cassandra; do + echo "=== $s -> $t --vector ===" + python3 devtools/backup-restore-matrix --source "$s" --target "$t" --vector \ + --work "matrix-logs/$s-$t-vector" || rc=1 + done + done + # The adoption of a root another catalog owns, both ways between + # SQLite and PostgreSQL. + for pair in "sqlite postgres" "postgres sqlite"; do + set -- $pair + echo "=== $1 -> $2 --adopt-store ===" + python3 devtools/backup-restore-matrix --source "$1" --target "$2" --adopt-store \ + --work "matrix-logs/$1-$2-adopt" || rc=1 + done + exit $rc + + - name: Upload the matrix logs on failure + if: failure() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: matrix-logs + path: matrix-logs + if-no-files-found: ignore + + - name: Stop the MongoDB replica set + if: always() + run: docker rm -f matrix-mongo >/dev/null 2>&1 || true + + # The restore pause points, driven through devtools/backup-kill-points.py + # (committed by the CreateBackup PR; the restore half of the harness): + # `restore --bin --work ` pauses a + # restore at the point, kills the server, restarts it and asserts the + # outcome. No other job builds `backup-test-hooks`, so the restore kill + # points have no CI coverage without these jobs, one per backend, each + # starting its backend as the matrix job does. The points a backend has are + # the script's `RESTORE_POINTS`; the two second-reach runs put the kill + # after an earlier batch's progress is recorded. Every outcome is ACTIVE + # with the exact base and GSI counts, except the tampered-file point + # (`restore_between_failed_and_removal`), whose target is removed. + restore-kill-points-sqlite: + runs-on: extenddb_ubuntu-2404_4-core + timeout-minutes: 90 + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master + with: + toolchain: stable + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 + with: + cache-on-failure: true + save-if: ${{ github.ref == 'refs/heads/main' }} + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.11" + - name: Install Python dependencies + run: pip install -r requirements.txt + - name: Build the SQLite binary with the backup test hooks + # The pause points inside the backends' restore writes compile only + # under `backup-test-hooks`; the harness drives this binary. + run: cargo build --release -p extenddb --no-default-features --features sqlite,extenddb-app/backup-test-hooks + - name: Run the SQLite restore kill points + run: | + set -o pipefail + rc=0 + for point in restore_after_download restore_after_file_commit \ + restore_sqlite_mid_batch restore_between_failed_and_removal sigterm; do + echo "=== sqlite restore kill point: $point ===" + python3 devtools/backup-kill-points.py restore sqlite "$point" \ + --bin target/release/extenddb --work "kill-point-logs/sqlite-$point" || rc=1 + done + exit $rc + - name: Upload the kill-point logs on failure + if: failure() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: kill-point-logs-sqlite + path: kill-point-logs + if-no-files-found: ignore + + restore-kill-points-postgres: + runs-on: extenddb_ubuntu-2404_4-core + timeout-minutes: 90 + services: + postgres: + image: pgvector/pgvector:pg16 + env: + POSTGRES_PASSWORD: devpass + ports: + - 127.0.0.1:55432:5432 + options: >- + --health-cmd "pg_isready -U postgres" + --health-interval 10s + --health-timeout 5s + --health-retries 10 + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master + with: + toolchain: stable + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 + with: + cache-on-failure: true + save-if: ${{ github.ref == 'refs/heads/main' }} + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.11" + - name: Install Python dependencies + run: pip install -r requirements.txt + - name: Install the PostgreSQL client + # The script creates, probes (`pg_terminate_backend`, the vector + # relation check) and drops its PostgreSQL databases through the + # host's psql. + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends postgresql-client + psql --version + - name: Build the PostgreSQL binary with the backup test hooks + run: cargo build --release -p extenddb --no-default-features --features postgres,extenddb-app/backup-test-hooks + - name: Run the PostgreSQL restore kill points + # `pg_terminate_backend` is not a pause point of its own: the claim + # session is terminated while the job is paused after its first + # file's commit. `--vector` on the tampered-file point asserts the + # removed target's vector relation gone (the service image carries + # pgvector). + run: | + set -o pipefail + rc=0 + for point in restore_after_download restore_after_file_commit \ + restore_postgres_before_progress_delete restore_between_failed_and_removal \ + restore_fenced_out pg_terminate_backend sigterm; do + echo "=== postgres restore kill point: $point ===" + python3 devtools/backup-kill-points.py restore postgres "$point" \ + --bin target/release/extenddb --work "kill-point-logs/postgres-$point" || rc=1 + done + echo "=== postgres restore kill point: restore_between_failed_and_removal --vector ===" + python3 devtools/backup-kill-points.py restore postgres restore_between_failed_and_removal \ + --vector --bin target/release/extenddb \ + --work kill-point-logs/postgres-restore_between_failed_and_removal-vector || rc=1 + exit $rc + - name: Upload the kill-point logs on failure + if: failure() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: kill-point-logs-postgres + path: kill-point-logs + if-no-files-found: ignore + + restore-kill-points-mongodb: + runs-on: extenddb_ubuntu-2404_4-core + timeout-minutes: 90 + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master + with: + toolchain: stable + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 + with: + cache-on-failure: true + save-if: ${{ github.ref == 'refs/heads/main' }} + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.11" + - name: Install Python dependencies + # requirements-matrix.txt adds pymongo, which the script's setup and + # teardown use to drop the fixed databases. + run: | + pip install -r requirements.txt -r requirements-matrix.txt + python3 -c "import pymongo; print(pymongo.version)" + - name: Start a MongoDB replica set + # As the matrix job does: `services:` cannot express a replica set. + # Published on loopback only; no authentication. + run: | + docker run -d --name kill-point-mongo -p 127.0.0.1:27117:27017 mongo:7 \ + mongod --replSet rs0 --bind_ip_all + for i in $(seq 1 30); do + docker exec kill-point-mongo mongosh --quiet --eval 'db.runCommand({ping:1}).ok' 2>/dev/null \ + | grep -qx 1 && break + sleep 1 + done + docker exec kill-point-mongo mongosh --quiet --eval \ + 'rs.initiate({_id:"rs0",members:[{_id:0,host:"localhost:27017"}]})' >/dev/null 2>&1 || true + for i in $(seq 1 30); do + state=$(docker exec kill-point-mongo mongosh --quiet --eval 'rs.status().members[0].stateStr' 2>/dev/null | tail -1) + [ "$state" = PRIMARY ] && { echo "replica set PRIMARY after ${i}s"; exit 0; } + sleep 1 + done + echo "MongoDB replica set did not elect a PRIMARY" >&2 + exit 1 + - name: Build the MongoDB binary with the backup test hooks + run: cargo build --release -p extenddb --no-default-features --features mongodb,extenddb-app/backup-test-hooks + - name: Run the MongoDB restore kill points + # The second reach of `restore_mongodb_after_insert_many` kills after + # an earlier batch's progress committed (24,000 items in three files). + run: | + set -o pipefail + rc=0 + for point in restore_after_download restore_after_file_commit \ + restore_mongodb_after_insert_many restore_mongodb_before_worker_flip \ + restore_between_failed_and_removal sigterm; do + echo "=== mongodb restore kill point: $point ===" + python3 devtools/backup-kill-points.py restore mongodb "$point" \ + --bin target/release/extenddb --work "kill-point-logs/mongodb-$point" || rc=1 + done + echo "=== mongodb restore kill point: restore_mongodb_after_insert_many --second-reach ===" + python3 devtools/backup-kill-points.py restore mongodb restore_mongodb_after_insert_many \ + --second-reach --items 24000 --bin target/release/extenddb \ + --work kill-point-logs/mongodb-restore_mongodb_after_insert_many-second || rc=1 + exit $rc + - name: Upload the kill-point logs on failure + if: failure() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: kill-point-logs-mongodb + path: kill-point-logs + if-no-files-found: ignore + - name: Stop the MongoDB replica set + if: always() + run: docker rm -f kill-point-mongo >/dev/null 2>&1 || true + + restore-kill-points-cassandra: + runs-on: extenddb_ubuntu-2404_4-core + timeout-minutes: 90 + services: + cassandra: + image: cassandra:4.1 + ports: + - 127.0.0.1:9042:9042 + env: + CASSANDRA_CLUSTER_NAME: extenddb-ci + HEAP_NEWSIZE: 128M + MAX_HEAP_SIZE: 1024M + options: >- + --health-cmd "cqlsh -e 'SELECT release_version FROM system.local'" + --health-interval 15s + --health-timeout 10s + --health-retries 20 + --health-start-period 60s + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master + with: + toolchain: stable + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 + with: + cache-on-failure: true + save-if: ${{ github.ref == 'refs/heads/main' }} + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.11" + - name: Install Python dependencies + # requirements-matrix.txt adds cassandra-driver, which the script's + # teardown uses to drop the keyspaces. + run: | + pip install -r requirements.txt -r requirements-matrix.txt + python3 -c "import cassandra; print(cassandra.__version__)" + - name: Configure Cassandra authentication + # The script's `init` and config use cassandra/cassandra, as the + # matrix job's do. + run: | + docker exec ${{ job.services.cassandra.id }} sed -i \ + 's/^authenticator: .*/authenticator: PasswordAuthenticator/' \ + /etc/cassandra/cassandra.yaml + docker restart ${{ job.services.cassandra.id }} + for i in $(seq 1 90); do + if docker exec ${{ job.services.cassandra.id }} \ + cqlsh -u cassandra -p cassandra \ + -e 'SELECT release_version FROM system.local' >/dev/null 2>&1; then + echo "Cassandra ready with PasswordAuthenticator after ${i}s" + exit 0 + fi + sleep 2 + done + echo "Cassandra did not become ready" >&2 + exit 1 + - name: Build the Cassandra hooks binary and the SQLite source binary + # Cassandra has no SnapshotEngine, so its CreateBackup is in-catalog + # and no backup taken on it restores through the store-backed job: the + # script takes each point's backup on a SQLite source server + # (`--source-bin`, a plain SQLite build) and adopts it into the + # Cassandra target's catalog, as the matrix does. One target + # directory, two sequential builds. + run: | + cargo build --release -p extenddb --no-default-features --features sqlite + mkdir -p target/rel-sqlite/release + cp target/release/extenddb target/rel-sqlite/release/extenddb + cargo build --release -p extenddb --no-default-features --features cassandra,extenddb-app/backup-test-hooks + - name: Run the Cassandra restore kill points + # The first reach of `restore_cassandra_before_progress` leaves the + # first batch's items with no progress recorded and resumes with + # upserts; the second reach kills after the first batch's progress. + run: | + set -o pipefail + rc=0 + for point in restore_after_download restore_after_file_commit \ + restore_cassandra_before_progress restore_between_failed_and_removal sigterm; do + echo "=== cassandra restore kill point: $point ===" + python3 devtools/backup-kill-points.py restore cassandra "$point" \ + --bin target/release/extenddb --source-bin target/rel-sqlite/release/extenddb \ + --work "kill-point-logs/cassandra-$point" || rc=1 + done + echo "=== cassandra restore kill point: restore_cassandra_before_progress --second-reach ===" + python3 devtools/backup-kill-points.py restore cassandra restore_cassandra_before_progress \ + --second-reach --bin target/release/extenddb --source-bin target/rel-sqlite/release/extenddb \ + --work kill-point-logs/cassandra-restore_cassandra_before_progress-second || rc=1 + exit $rc + - name: Upload the kill-point logs on failure + if: failure() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: kill-point-logs-cassandra + path: kill-point-logs + if-no-files-found: ignore diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b0d641a98..e5c071e16 100755 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -47,10 +47,28 @@ cargo build --workspace # Rust unit tests cargo test --workspace -# Integration tests (requires a running extenddb server) -devtools/run-tests --extenddb --pytest +# Integration tests (requires a running extenddb server; the variable says +# where that server writes its backups, see the developer test guide) +EXTENDDB_TEST_BACKUP_STORE=filesystem devtools/run-tests --extenddb --pytest + +# The backup store and job suites start servers of their own from the +# release build named by EXTENDDB_BINARY, so they run by name (the S3 cells +# need a moto server at EXTENDDB_TEST_S3_ENDPOINT; see the developer test +# guide for the PostgreSQL and moto variables) +EXTENDDB_BINARY=target/release/extenddb python3 -m pytest tests/test_backup_store_config.py tests/test_backup_job.py -v ``` +The MongoDB backup cells that read the store's root and the server's log +(`EXTENDDB_TEST_BACKUP_ROOT`, `EXTENDDB_TEST_SERVER_LOG`) run through +`devtools/run-mongodb-tests`, which exports both; its legacy LSI fallback +cell needs the runner's `--backup-store none`. The cross-backend restore +matrix is `devtools/backup-restore-matrix` (one cell per invocation, from the +four per-backend release binaries; `pip install -r requirements-matrix.txt` +for its MongoDB and Cassandra cells) and runs nightly from +`.github/workflows/matrix-nightly.yml`. The developer test guide +(`docs/manuals/06-developer-test-guide.md`, "Test Runner Script") has a +paragraph on each. + ### Code Style All contributions must pass: @@ -341,26 +359,27 @@ SQLite, PostgreSQL, Cassandra and MongoDB migrate through this framework. ### Example: a nullable column on `tables` -Migration 6, `table_note`, adds a nullable `note TEXT` column to the catalog's -`tables` table on SQLite and PostgreSQL. +Migration 8, `table_note` (8 being the next free id at the time of writing; +migration 7 is `restore_progress`), adds a nullable `note TEXT` column to +the catalog's `tables` table on SQLite and PostgreSQL. In `crates/storage/src/migrations/registry.rs`, at the end of the `migration_ids!` list: ```rust /// Adds a nullable `note` column to the `tables` catalog table. - TableNote = 6, "table_note", + TableNote = 8, "table_note", "add a nullable note column to the tables catalog"; ``` PostgreSQL has `ADD COLUMN IF NOT EXISTS`, so its body is a file, -`crates/storage-postgres/migrations/0006_table_note.sql`: +`crates/storage-postgres/migrations/0008_table_note.sql`: ```sql -- Copyright 2026 ExtendDB contributors -- SPDX-License-Identifier: Apache-2.0 -- --- Migration 6, table_note, PostgreSQL catalog database: a nullable note +-- Migration 8, table_note, PostgreSQL catalog database: a nullable note -- column on tables. IF NOT EXISTS makes a second run change nothing. ALTER TABLE tables ADD COLUMN IF NOT EXISTS note TEXT; @@ -375,7 +394,7 @@ In `crates/storage-postgres/src/migrations/mod.rs`, the arm in `migration()`: bodies: vec![file_body!( StoreKind::Catalog, "migrations", - "0006_table_note.sql" + "0008_table_note.sql" )], }, ``` @@ -387,19 +406,21 @@ and the id in the transactional arm of `runs_in()`: | MigrationId::ResourceTagConditionKeys | MigrationId::SortKeyDefinitions | MigrationId::DropContinuousBackups + | MigrationId::BackupCatalogIndex + | MigrationId::RestoreProgress | MigrationId::TableNote => RunsIn::Transaction, ``` SQLite has no `ADD COLUMN IF NOT EXISTS`, and a file with a plain `ADD COLUMN` fails when it runs a second time, so the SQLite body is code that checks for the column first. -`crates/storage-sqlite/src/migrations/m0006_table_note.rs`: +`crates/storage-sqlite/src/migrations/m0008_table_note.rs`: ```rust // Copyright 2026 ExtendDB contributors // SPDX-License-Identifier: Apache-2.0 -//! Migration 6, `table_note`, SQLite: a nullable `note` column on `tables`. +//! Migration 8, `table_note`, SQLite: a nullable `note` column on `tables`. //! //! SQLite has no `ADD COLUMN IF NOT EXISTS`, so the body reads the table's //! columns first and adds the column only when it is missing. A second run @@ -411,10 +432,10 @@ use extenddb_storage::migrations::{CodeBody, Done, MigrationError}; use super::SqliteStores; /// This file, whose SHA-256 the checksum test pins. -pub(crate) const SOURCE: &str = include_str!("m0006_table_note.rs"); +pub(crate) const SOURCE: &str = include_str!("m0008_table_note.rs"); /// Recorded as this migration's checksum. Changes only with a new migration. -const FINGERPRINT: &str = "0006_table_note/sqlite/1"; +const FINGERPRINT: &str = "0008_table_note/sqlite/1"; pub(crate) struct TableNote; @@ -453,7 +474,7 @@ beside the others: ```rust #[rustfmt::skip] #[allow(clippy::all, deprecated)] -mod m0006_table_note; +mod m0008_table_note; ``` and the arm in `migration()`: @@ -464,8 +485,8 @@ and the arm in `migration()`: convergent: true, bodies: vec![StoreBody::code( StoreKind::Catalog, - &m0006_table_note::TableNote, - m0006_table_note::SOURCE, + &m0008_table_note::TableNote, + m0008_table_note::SOURCE, )], }, ``` @@ -477,8 +498,8 @@ rows of `tables`: ```rust #[tokio::test] -async fn migration_6_adds_the_note_column_to_tables() { - let db = temp_db("m6"); +async fn migration_8_adds_the_note_column_to_tables() { + let db = temp_db("m8"); load_dump(&db, DEV_0_1_14).await; run(&db).await.expect("migrate"); let mut text = dump( @@ -488,12 +509,12 @@ async fn migration_6_adds_the_note_column_to_tables() { ) .await; text.push_str(&dump(&db, "SELECT table_name, note FROM tables ORDER BY table_name").await); - golden("0006_table_note", &text); + golden("0008_table_note", &text); } ``` -`EXTENDDB_BLESS_EXPECTED=1 cargo test -p extenddb-storage-sqlite --lib migration_6` -writes `crates/storage-sqlite/tests/fixtures/golden/0006_table_note.txt`, one +`EXTENDDB_BLESS_EXPECTED=1 cargo test -p extenddb-storage-sqlite --lib migration_8` +writes `crates/storage-sqlite/tests/fixtures/golden/0008_table_note.txt`, one tab-separated line per row under each query: ``` @@ -508,15 +529,15 @@ The PostgreSQL body is a file, so it has no golden test. The example declares `Kind::Breaking`. A nullable column can be additive by the rule in step 4, but declaring it additive needs the old-binary job, which this example does not add. As a breaking migration it makes a 1.x binary -without migration 6 refuse a catalog that has it, and a rollback restores the +without migration 8 refuse a catalog that has it, and a rollback restores the backup. The same PR changes the code that reads and writes `tables.note` in each backend's catalog store, and commits the `docs/schema/sqlite.txt` and `docs/schema/postgres.txt` that the integration jobs regenerate. It changes no per-table DDL, because `tables` is a catalog table, and adds no `RELEASED` -entry. `extenddb migrate --list` then shows migration 6, and on a catalog at -migration 5 `extenddb migrate` lists migration 6 as pending and exits 3. +entry. `extenddb migrate --list` then shows migration 8, and on a catalog at +migration 7 `extenddb migrate` lists migration 8 as pending and exits 3. ### Running the migration tests diff --git a/Cargo.lock b/Cargo.lock index 6cb6f19e4..35d034748 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1660,11 +1660,13 @@ dependencies = [ "clap", "daemonize", "extenddb-auth", + "extenddb-backup-store", "extenddb-config", "extenddb-core", "extenddb-engine", "extenddb-server", "extenddb-storage", + "futures", "libc", "rcgen", "rustls", @@ -1740,9 +1742,11 @@ version = "0.1.14" dependencies = [ "anyhow", "config", + "extenddb-backup-store", "extenddb-core", "extenddb-storage", "serde", + "tempfile", "toml", "tracing", ] @@ -1768,16 +1772,20 @@ name = "extenddb-engine" version = "0.1.14" dependencies = [ "base64 0.22.1", + "bytes", "crc32fast", "extenddb-auth", + "extenddb-backup-store", "extenddb-core", "extenddb-storage", + "futures", "hex", "hmac 0.12.1", "serde", "serde_json", "sha2 0.10.9", "tokio", + "tokio-util", "tracing", "uuid", ] @@ -1794,6 +1802,7 @@ dependencies = [ "bcrypt", "crc32fast", "extenddb-auth", + "extenddb-backup-store", "extenddb-cache", "extenddb-config", "extenddb-core", @@ -1808,6 +1817,7 @@ dependencies = [ "serde", "serde_json", "syslog-tracing", + "tempfile", "time", "tokio", "tower", @@ -1911,7 +1921,9 @@ dependencies = [ "bcrypt", "crc32fast", "extenddb-auth", + "extenddb-backup-store", "extenddb-core", + "extenddb-engine", "extenddb-storage", "extenddb-storage-postgres", "futures", @@ -1921,6 +1933,7 @@ dependencies = [ "serde", "serde_json", "sqlx", + "tempfile", "time", "tokio", "toml", @@ -1940,7 +1953,9 @@ dependencies = [ "bigdecimal", "crc32fast", "extenddb-auth", + "extenddb-backup-store", "extenddb-core", + "extenddb-engine", "extenddb-storage", "extenddb-storage-sqlite", "futures", diff --git a/SOFTWARE-LICENSE-NOTICES-CASSANDRA.html b/SOFTWARE-LICENSE-NOTICES-CASSANDRA.html index 80174e5eb..8a5f1b144 100644 --- a/SOFTWARE-LICENSE-NOTICES-CASSANDRA.html +++ b/SOFTWARE-LICENSE-NOTICES-CASSANDRA.html @@ -49,14 +49,14 @@

ExtendDB Software License Notices

Overview of licenses:

All license text:

@@ -1741,6 +1741,424 @@

Used by:

See the License for the specific language governing permissions and limitations under the License. + + +
  • +

    Apache License 2.0

    +

    Used by:

    + +
                                     Apache License
    +                           Version 2.0, January 2004
    +                        http://www.apache.org/licenses/
    +
    +   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
    +
    +   1. Definitions.
    +
    +      "License" shall mean the terms and conditions for use, reproduction,
    +      and distribution as defined by Sections 1 through 9 of this document.
    +
    +      "Licensor" shall mean the copyright owner or entity authorized by
    +      the copyright owner that is granting the License.
    +
    +      "Legal Entity" shall mean the union of the acting entity and all
    +      other entities that control, are controlled by, or are under common
    +      control with that entity. For the purposes of this definition,
    +      "control" means (i) the power, direct or indirect, to cause the
    +      direction or management of such entity, whether by contract or
    +      otherwise, or (ii) ownership of fifty percent (50%) or more of the
    +      outstanding shares, or (iii) beneficial ownership of such entity.
    +
    +      "You" (or "Your") shall mean an individual or Legal Entity
    +      exercising permissions granted by this License.
    +
    +      "Source" form shall mean the preferred form for making modifications,
    +      including but not limited to software source code, documentation
    +      source, and configuration files.
    +
    +      "Object" form shall mean any form resulting from mechanical
    +      transformation or translation of a Source form, including but
    +      not limited to compiled object code, generated documentation,
    +      and conversions to other media types.
    +
    +      "Work" shall mean the work of authorship, whether in Source or
    +      Object form, made available under the License, as indicated by a
    +      copyright notice that is included in or attached to the work
    +      (an example is provided in the Appendix below).
    +
    +      "Derivative Works" shall mean any work, whether in Source or Object
    +      form, that is based on (or derived from) the Work and for which the
    +      editorial revisions, annotations, elaborations, or other modifications
    +      represent, as a whole, an original work of authorship. For the purposes
    +      of this License, Derivative Works shall not include works that remain
    +      separable from, or merely link (or bind by name) to the interfaces of,
    +      the Work and Derivative Works thereof.
    +
    +      "Contribution" shall mean any work of authorship, including
    +      the original version of the Work and any modifications or additions
    +      to that Work or Derivative Works thereof, that is intentionally
    +      submitted to Licensor for inclusion in the Work by the copyright owner
    +      or by an individual or Legal Entity authorized to submit on behalf of
    +      the copyright owner. For the purposes of this definition, "submitted"
    +      means any form of electronic, verbal, or written communication sent
    +      to the Licensor or its representatives, including but not limited to
    +      communication on electronic mailing lists, source code control systems,
    +      and issue tracking systems that are managed by, or on behalf of, the
    +      Licensor for the purpose of discussing and improving the Work, but
    +      excluding communication that is conspicuously marked or otherwise
    +      designated in writing by the copyright owner as "Not a Contribution."
    +
    +      "Contributor" shall mean Licensor and any individual or Legal Entity
    +      on behalf of whom a Contribution has been received by Licensor and
    +      subsequently incorporated within the Work.
    +
    +   2. Grant of Copyright License. Subject to the terms and conditions of
    +      this License, each Contributor hereby grants to You a perpetual,
    +      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
    +      copyright license to reproduce, prepare Derivative Works of,
    +      publicly display, publicly perform, sublicense, and distribute the
    +      Work and such Derivative Works in Source or Object form.
    +
    +   3. Grant of Patent License. Subject to the terms and conditions of
    +      this License, each Contributor hereby grants to You a perpetual,
    +      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
    +      (except as stated in this section) patent license to make, have made,
    +      use, offer to sell, sell, import, and otherwise transfer the Work,
    +      where such license applies only to those patent claims licensable
    +      by such Contributor that are necessarily infringed by their
    +      Contribution(s) alone or by combination of their Contribution(s)
    +      with the Work to which such Contribution(s) was submitted. If You
    +      institute patent litigation against any entity (including a
    +      cross-claim or counterclaim in a lawsuit) alleging that the Work
    +      or a Contribution incorporated within the Work constitutes direct
    +      or contributory patent infringement, then any patent licenses
    +      granted to You under this License for that Work shall terminate
    +      as of the date such litigation is filed.
    +
    +   4. Redistribution. You may reproduce and distribute copies of the
    +      Work or Derivative Works thereof in any medium, with or without
    +      modifications, and in Source or Object form, provided that You
    +      meet the following conditions:
    +
    +      (a) You must give any other recipients of the Work or
    +          Derivative Works a copy of this License; and
    +
    +      (b) You must cause any modified files to carry prominent notices
    +          stating that You changed the files; and
    +
    +      (c) You must retain, in the Source form of any Derivative Works
    +          that You distribute, all copyright, patent, trademark, and
    +          attribution notices from the Source form of the Work,
    +          excluding those notices that do not pertain to any part of
    +          the Derivative Works; and
    +
    +      (d) If the Work includes a "NOTICE" text file as part of its
    +          distribution, then any Derivative Works that You distribute must
    +          include a readable copy of the attribution notices contained
    +          within such NOTICE file, excluding those notices that do not
    +          pertain to any part of the Derivative Works, in at least one
    +          of the following places: within a NOTICE text file distributed
    +          as part of the Derivative Works; within the Source form or
    +          documentation, if provided along with the Derivative Works; or,
    +          within a display generated by the Derivative Works, if and
    +          wherever such third-party notices normally appear. The contents
    +          of the NOTICE file are for informational purposes only and
    +          do not modify the License. You may add Your own attribution
    +          notices within Derivative Works that You distribute, alongside
    +          or as an addendum to the NOTICE text from the Work, provided
    +          that such additional attribution notices cannot be construed
    +          as modifying the License.
    +
    +      You may add Your own copyright statement to Your modifications and
    +      may provide additional or different license terms and conditions
    +      for use, reproduction, or distribution of Your modifications, or
    +      for any such Derivative Works as a whole, provided Your use,
    +      reproduction, and distribution of the Work otherwise complies with
    +      the conditions stated in this License.
    +
    +   5. Submission of Contributions. Unless You explicitly state otherwise,
    +      any Contribution intentionally submitted for inclusion in the Work
    +      by You to the Licensor shall be under the terms and conditions of
    +      this License, without any additional terms or conditions.
    +      Notwithstanding the above, nothing herein shall supersede or modify
    +      the terms of any separate license agreement you may have executed
    +      with Licensor regarding such Contributions.
    +
    +   6. Trademarks. This License does not grant permission to use the trade
    +      names, trademarks, service marks, or product names of the Licensor,
    +      except as required for reasonable and customary use in describing the
    +      origin of the Work and reproducing the content of the NOTICE file.
    +
    +   7. Disclaimer of Warranty. Unless required by applicable law or
    +      agreed to in writing, Licensor provides the Work (and each
    +      Contributor provides its Contributions) on an "AS IS" BASIS,
    +      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
    +      implied, including, without limitation, any warranties or conditions
    +      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
    +      PARTICULAR PURPOSE. You are solely responsible for determining the
    +      appropriateness of using or redistributing the Work and assume any
    +      risks associated with Your exercise of permissions under this License.
    +
    +   8. Limitation of Liability. In no event and under no legal theory,
    +      whether in tort (including negligence), contract, or otherwise,
    +      unless required by applicable law (such as deliberate and grossly
    +      negligent acts) or agreed to in writing, shall any Contributor be
    +      liable to You for damages, including any direct, indirect, special,
    +      incidental, or consequential damages of any character arising as a
    +      result of this License or out of the use or inability to use the
    +      Work (including but not limited to damages for loss of goodwill,
    +      work stoppage, computer failure or malfunction, or any and all
    +      other commercial damages or losses), even if such Contributor
    +      has been advised of the possibility of such damages.
    +
    +   9. Accepting Warranty or Additional Liability. While redistributing
    +      the Work or Derivative Works thereof, You may choose to offer,
    +      and charge a fee for, acceptance of support, warranty, indemnity,
    +      or other liability obligations and/or rights consistent with this
    +      License. However, in accepting such obligations, You may act only
    +      on Your own behalf and on Your sole responsibility, not on behalf
    +      of any other Contributor, and only if You agree to indemnify,
    +      defend, and hold each Contributor harmless for any liability
    +      incurred by, or claims asserted against, such Contributor by reason
    +      of your accepting any such warranty or additional liability.
    +
    +   END OF TERMS AND CONDITIONS
    +
    +   APPENDIX: How to apply the Apache License to your work.
    +
    +      To apply the Apache License to your work, attach the following
    +      boilerplate notice, with the fields enclosed by brackets "[]"
    +      replaced with your own identifying information. (Don't include
    +      the brackets!)  The text should be enclosed in the appropriate
    +      comment syntax for the file format. We also recommend that a
    +      file or class name and description of purpose be included on the
    +      same "printed page" as the copyright notice for easier
    +      identification within third-party archives.
    +
    +   Copyright [yyyy] [name of copyright owner]
    +
    +   Licensed under the Apache License, Version 2.0 (the "License");
    +   you may not use this file except in compliance with the License.
    +   You may obtain a copy of the License at
    +
    +       http://www.apache.org/licenses/LICENSE-2.0
    +
    +   Unless required by applicable law or agreed to in writing, software
    +   distributed under the License is distributed on an "AS IS" BASIS,
    +   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    +   See the License for the specific language governing permissions and
    +   limitations under the License.
    +
    +
  • +
  • +

    Apache License 2.0

    +

    Used by:

    + +
                                     Apache License
    +                           Version 2.0, January 2004
    +                        http://www.apache.org/licenses/
    +
    +   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
    +
    +   1. Definitions.
    +
    +      "License" shall mean the terms and conditions for use, reproduction,
    +      and distribution as defined by Sections 1 through 9 of this document.
    +
    +      "Licensor" shall mean the copyright owner or entity authorized by
    +      the copyright owner that is granting the License.
    +
    +      "Legal Entity" shall mean the union of the acting entity and all
    +      other entities that control, are controlled by, or are under common
    +      control with that entity. For the purposes of this definition,
    +      "control" means (i) the power, direct or indirect, to cause the
    +      direction or management of such entity, whether by contract or
    +      otherwise, or (ii) ownership of fifty percent (50%) or more of the
    +      outstanding shares, or (iii) beneficial ownership of such entity.
    +
    +      "You" (or "Your") shall mean an individual or Legal Entity
    +      exercising permissions granted by this License.
    +
    +      "Source" form shall mean the preferred form for making modifications,
    +      including but not limited to software source code, documentation
    +      source, and configuration files.
    +
    +      "Object" form shall mean any form resulting from mechanical
    +      transformation or translation of a Source form, including but
    +      not limited to compiled object code, generated documentation,
    +      and conversions to other media types.
    +
    +      "Work" shall mean the work of authorship, whether in Source or
    +      Object form, made available under the License, as indicated by a
    +      copyright notice that is included in or attached to the work
    +      (an example is provided in the Appendix below).
    +
    +      "Derivative Works" shall mean any work, whether in Source or Object
    +      form, that is based on (or derived from) the Work and for which the
    +      editorial revisions, annotations, elaborations, or other modifications
    +      represent, as a whole, an original work of authorship. For the purposes
    +      of this License, Derivative Works shall not include works that remain
    +      separable from, or merely link (or bind by name) to the interfaces of,
    +      the Work and Derivative Works thereof.
    +
    +      "Contribution" shall mean any work of authorship, including
    +      the original version of the Work and any modifications or additions
    +      to that Work or Derivative Works thereof, that is intentionally
    +      submitted to Licensor for inclusion in the Work by the copyright owner
    +      or by an individual or Legal Entity authorized to submit on behalf of
    +      the copyright owner. For the purposes of this definition, "submitted"
    +      means any form of electronic, verbal, or written communication sent
    +      to the Licensor or its representatives, including but not limited to
    +      communication on electronic mailing lists, source code control systems,
    +      and issue tracking systems that are managed by, or on behalf of, the
    +      Licensor for the purpose of discussing and improving the Work, but
    +      excluding communication that is conspicuously marked or otherwise
    +      designated in writing by the copyright owner as "Not a Contribution."
    +
    +      "Contributor" shall mean Licensor and any individual or Legal Entity
    +      on behalf of whom a Contribution has been received by Licensor and
    +      subsequently incorporated within the Work.
    +
    +   2. Grant of Copyright License. Subject to the terms and conditions of
    +      this License, each Contributor hereby grants to You a perpetual,
    +      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
    +      copyright license to reproduce, prepare Derivative Works of,
    +      publicly display, publicly perform, sublicense, and distribute the
    +      Work and such Derivative Works in Source or Object form.
    +
    +   3. Grant of Patent License. Subject to the terms and conditions of
    +      this License, each Contributor hereby grants to You a perpetual,
    +      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
    +      (except as stated in this section) patent license to make, have made,
    +      use, offer to sell, sell, import, and otherwise transfer the Work,
    +      where such license applies only to those patent claims licensable
    +      by such Contributor that are necessarily infringed by their
    +      Contribution(s) alone or by combination of their Contribution(s)
    +      with the Work to which such Contribution(s) was submitted. If You
    +      institute patent litigation against any entity (including a
    +      cross-claim or counterclaim in a lawsuit) alleging that the Work
    +      or a Contribution incorporated within the Work constitutes direct
    +      or contributory patent infringement, then any patent licenses
    +      granted to You under this License for that Work shall terminate
    +      as of the date such litigation is filed.
    +
    +   4. Redistribution. You may reproduce and distribute copies of the
    +      Work or Derivative Works thereof in any medium, with or without
    +      modifications, and in Source or Object form, provided that You
    +      meet the following conditions:
    +
    +      (a) You must give any other recipients of the Work or
    +          Derivative Works a copy of this License; and
    +
    +      (b) You must cause any modified files to carry prominent notices
    +          stating that You changed the files; and
    +
    +      (c) You must retain, in the Source form of any Derivative Works
    +          that You distribute, all copyright, patent, trademark, and
    +          attribution notices from the Source form of the Work,
    +          excluding those notices that do not pertain to any part of
    +          the Derivative Works; and
    +
    +      (d) If the Work includes a "NOTICE" text file as part of its
    +          distribution, then any Derivative Works that You distribute must
    +          include a readable copy of the attribution notices contained
    +          within such NOTICE file, excluding those notices that do not
    +          pertain to any part of the Derivative Works, in at least one
    +          of the following places: within a NOTICE text file distributed
    +          as part of the Derivative Works; within the Source form or
    +          documentation, if provided along with the Derivative Works; or,
    +          within a display generated by the Derivative Works, if and
    +          wherever such third-party notices normally appear. The contents
    +          of the NOTICE file are for informational purposes only and
    +          do not modify the License. You may add Your own attribution
    +          notices within Derivative Works that You distribute, alongside
    +          or as an addendum to the NOTICE text from the Work, provided
    +          that such additional attribution notices cannot be construed
    +          as modifying the License.
    +
    +      You may add Your own copyright statement to Your modifications and
    +      may provide additional or different license terms and conditions
    +      for use, reproduction, or distribution of Your modifications, or
    +      for any such Derivative Works as a whole, provided Your use,
    +      reproduction, and distribution of the Work otherwise complies with
    +      the conditions stated in this License.
    +
    +   5. Submission of Contributions. Unless You explicitly state otherwise,
    +      any Contribution intentionally submitted for inclusion in the Work
    +      by You to the Licensor shall be under the terms and conditions of
    +      this License, without any additional terms or conditions.
    +      Notwithstanding the above, nothing herein shall supersede or modify
    +      the terms of any separate license agreement you may have executed
    +      with Licensor regarding such Contributions.
    +
    +   6. Trademarks. This License does not grant permission to use the trade
    +      names, trademarks, service marks, or product names of the Licensor,
    +      except as required for reasonable and customary use in describing the
    +      origin of the Work and reproducing the content of the NOTICE file.
    +
    +   7. Disclaimer of Warranty. Unless required by applicable law or
    +      agreed to in writing, Licensor provides the Work (and each
    +      Contributor provides its Contributions) on an "AS IS" BASIS,
    +      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
    +      implied, including, without limitation, any warranties or conditions
    +      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
    +      PARTICULAR PURPOSE. You are solely responsible for determining the
    +      appropriateness of using or redistributing the Work and assume any
    +      risks associated with Your exercise of permissions under this License.
    +
    +   8. Limitation of Liability. In no event and under no legal theory,
    +      whether in tort (including negligence), contract, or otherwise,
    +      unless required by applicable law (such as deliberate and grossly
    +      negligent acts) or agreed to in writing, shall any Contributor be
    +      liable to You for damages, including any direct, indirect, special,
    +      incidental, or consequential damages of any character arising as a
    +      result of this License or out of the use or inability to use the
    +      Work (including but not limited to damages for loss of goodwill,
    +      work stoppage, computer failure or malfunction, or any and all
    +      other commercial damages or losses), even if such Contributor
    +      has been advised of the possibility of such damages.
    +
    +   9. Accepting Warranty or Additional Liability. While redistributing
    +      the Work or Derivative Works thereof, You may choose to offer,
    +      and charge a fee for, acceptance of support, warranty, indemnity,
    +      or other liability obligations and/or rights consistent with this
    +      License. However, in accepting such obligations, You may act only
    +      on Your own behalf and on Your sole responsibility, not on behalf
    +      of any other Contributor, and only if You agree to indemnify,
    +      defend, and hold each Contributor harmless for any liability
    +      incurred by, or claims asserted against, such Contributor by reason
    +      of your accepting any such warranty or additional liability.
    +
    +   END OF TERMS AND CONDITIONS
    +
    +   APPENDIX: How to apply the Apache License to your work.
    +
    +      To apply the Apache License to your work, attach the following
    +      boilerplate notice, with the fields enclosed by brackets "{}"
    +      replaced with your own identifying information. (Don't include
    +      the brackets!)  The text should be enclosed in the appropriate
    +      comment syntax for the file format. We also recommend that a
    +      file or class name and description of purpose be included on the
    +      same "printed page" as the copyright notice for easier
    +      identification within third-party archives.
    +
    +   Copyright 2017 Juniper Networks, Inc.
    +
    +   Licensed under the Apache License, Version 2.0 (the "License");
    +   you may not use this file except in compliance with the License.
    +   You may obtain a copy of the License at
    +
    +       http://www.apache.org/licenses/LICENSE-2.0
    +
    +   Unless required by applicable law or agreed to in writing, software
    +   distributed under the License is distributed on an "AS IS" BASIS,
    +   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    +   See the License for the specific language governing permissions and
    +   limitations under the License.
     
  • @@ -1767,7 +2185,220 @@

    Used by:

  • toml_edit 0.22.27
  • toml_write 0.1.2
  • -
                                     Apache License
    +                
                                     Apache License
    +                           Version 2.0, January 2004
    +                        http://www.apache.org/licenses/
    +
    +   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
    +
    +   1. Definitions.
    +
    +      "License" shall mean the terms and conditions for use, reproduction,
    +      and distribution as defined by Sections 1 through 9 of this document.
    +
    +      "Licensor" shall mean the copyright owner or entity authorized by
    +      the copyright owner that is granting the License.
    +
    +      "Legal Entity" shall mean the union of the acting entity and all
    +      other entities that control, are controlled by, or are under common
    +      control with that entity. For the purposes of this definition,
    +      "control" means (i) the power, direct or indirect, to cause the
    +      direction or management of such entity, whether by contract or
    +      otherwise, or (ii) ownership of fifty percent (50%) or more of the
    +      outstanding shares, or (iii) beneficial ownership of such entity.
    +
    +      "You" (or "Your") shall mean an individual or Legal Entity
    +      exercising permissions granted by this License.
    +
    +      "Source" form shall mean the preferred form for making modifications,
    +      including but not limited to software source code, documentation
    +      source, and configuration files.
    +
    +      "Object" form shall mean any form resulting from mechanical
    +      transformation or translation of a Source form, including but
    +      not limited to compiled object code, generated documentation,
    +      and conversions to other media types.
    +
    +      "Work" shall mean the work of authorship, whether in Source or
    +      Object form, made available under the License, as indicated by a
    +      copyright notice that is included in or attached to the work
    +      (an example is provided in the Appendix below).
    +
    +      "Derivative Works" shall mean any work, whether in Source or Object
    +      form, that is based on (or derived from) the Work and for which the
    +      editorial revisions, annotations, elaborations, or other modifications
    +      represent, as a whole, an original work of authorship. For the purposes
    +      of this License, Derivative Works shall not include works that remain
    +      separable from, or merely link (or bind by name) to the interfaces of,
    +      the Work and Derivative Works thereof.
    +
    +      "Contribution" shall mean any work of authorship, including
    +      the original version of the Work and any modifications or additions
    +      to that Work or Derivative Works thereof, that is intentionally
    +      submitted to Licensor for inclusion in the Work by the copyright owner
    +      or by an individual or Legal Entity authorized to submit on behalf of
    +      the copyright owner. For the purposes of this definition, "submitted"
    +      means any form of electronic, verbal, or written communication sent
    +      to the Licensor or its representatives, including but not limited to
    +      communication on electronic mailing lists, source code control systems,
    +      and issue tracking systems that are managed by, or on behalf of, the
    +      Licensor for the purpose of discussing and improving the Work, but
    +      excluding communication that is conspicuously marked or otherwise
    +      designated in writing by the copyright owner as "Not a Contribution."
    +
    +      "Contributor" shall mean Licensor and any individual or Legal Entity
    +      on behalf of whom a Contribution has been received by Licensor and
    +      subsequently incorporated within the Work.
    +
    +   2. Grant of Copyright License. Subject to the terms and conditions of
    +      this License, each Contributor hereby grants to You a perpetual,
    +      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
    +      copyright license to reproduce, prepare Derivative Works of,
    +      publicly display, publicly perform, sublicense, and distribute the
    +      Work and such Derivative Works in Source or Object form.
    +
    +   3. Grant of Patent License. Subject to the terms and conditions of
    +      this License, each Contributor hereby grants to You a perpetual,
    +      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
    +      (except as stated in this section) patent license to make, have made,
    +      use, offer to sell, sell, import, and otherwise transfer the Work,
    +      where such license applies only to those patent claims licensable
    +      by such Contributor that are necessarily infringed by their
    +      Contribution(s) alone or by combination of their Contribution(s)
    +      with the Work to which such Contribution(s) was submitted. If You
    +      institute patent litigation against any entity (including a
    +      cross-claim or counterclaim in a lawsuit) alleging that the Work
    +      or a Contribution incorporated within the Work constitutes direct
    +      or contributory patent infringement, then any patent licenses
    +      granted to You under this License for that Work shall terminate
    +      as of the date such litigation is filed.
    +
    +   4. Redistribution. You may reproduce and distribute copies of the
    +      Work or Derivative Works thereof in any medium, with or without
    +      modifications, and in Source or Object form, provided that You
    +      meet the following conditions:
    +
    +      (a) You must give any other recipients of the Work or
    +          Derivative Works a copy of this License; and
    +
    +      (b) You must cause any modified files to carry prominent notices
    +          stating that You changed the files; and
    +
    +      (c) You must retain, in the Source form of any Derivative Works
    +          that You distribute, all copyright, patent, trademark, and
    +          attribution notices from the Source form of the Work,
    +          excluding those notices that do not pertain to any part of
    +          the Derivative Works; and
    +
    +      (d) If the Work includes a "NOTICE" text file as part of its
    +          distribution, then any Derivative Works that You distribute must
    +          include a readable copy of the attribution notices contained
    +          within such NOTICE file, excluding those notices that do not
    +          pertain to any part of the Derivative Works, in at least one
    +          of the following places: within a NOTICE text file distributed
    +          as part of the Derivative Works; within the Source form or
    +          documentation, if provided along with the Derivative Works; or,
    +          within a display generated by the Derivative Works, if and
    +          wherever such third-party notices normally appear. The contents
    +          of the NOTICE file are for informational purposes only and
    +          do not modify the License. You may add Your own attribution
    +          notices within Derivative Works that You distribute, alongside
    +          or as an addendum to the NOTICE text from the Work, provided
    +          that such additional attribution notices cannot be construed
    +          as modifying the License.
    +
    +      You may add Your own copyright statement to Your modifications and
    +      may provide additional or different license terms and conditions
    +      for use, reproduction, or distribution of Your modifications, or
    +      for any such Derivative Works as a whole, provided Your use,
    +      reproduction, and distribution of the Work otherwise complies with
    +      the conditions stated in this License.
    +
    +   5. Submission of Contributions. Unless You explicitly state otherwise,
    +      any Contribution intentionally submitted for inclusion in the Work
    +      by You to the Licensor shall be under the terms and conditions of
    +      this License, without any additional terms or conditions.
    +      Notwithstanding the above, nothing herein shall supersede or modify
    +      the terms of any separate license agreement you may have executed
    +      with Licensor regarding such Contributions.
    +
    +   6. Trademarks. This License does not grant permission to use the trade
    +      names, trademarks, service marks, or product names of the Licensor,
    +      except as required for reasonable and customary use in describing the
    +      origin of the Work and reproducing the content of the NOTICE file.
    +
    +   7. Disclaimer of Warranty. Unless required by applicable law or
    +      agreed to in writing, Licensor provides the Work (and each
    +      Contributor provides its Contributions) on an "AS IS" BASIS,
    +      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
    +      implied, including, without limitation, any warranties or conditions
    +      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
    +      PARTICULAR PURPOSE. You are solely responsible for determining the
    +      appropriateness of using or redistributing the Work and assume any
    +      risks associated with Your exercise of permissions under this License.
    +
    +   8. Limitation of Liability. In no event and under no legal theory,
    +      whether in tort (including negligence), contract, or otherwise,
    +      unless required by applicable law (such as deliberate and grossly
    +      negligent acts) or agreed to in writing, shall any Contributor be
    +      liable to You for damages, including any direct, indirect, special,
    +      incidental, or consequential damages of any character arising as a
    +      result of this License or out of the use or inability to use the
    +      Work (including but not limited to damages for loss of goodwill,
    +      work stoppage, computer failure or malfunction, or any and all
    +      other commercial damages or losses), even if such Contributor
    +      has been advised of the possibility of such damages.
    +
    +   9. Accepting Warranty or Additional Liability. While redistributing
    +      the Work or Derivative Works thereof, You may choose to offer,
    +      and charge a fee for, acceptance of support, warranty, indemnity,
    +      or other liability obligations and/or rights consistent with this
    +      License. However, in accepting such obligations, You may act only
    +      on Your own behalf and on Your sole responsibility, not on behalf
    +      of any other Contributor, and only if You agree to indemnify,
    +      defend, and hold each Contributor harmless for any liability
    +      incurred by, or claims asserted against, such Contributor by reason
    +      of your accepting any such warranty or additional liability.
    +
    +   END OF TERMS AND CONDITIONS
    +
    +   APPENDIX: How to apply the Apache License to your work.
    +
    +      To apply the Apache License to your work, attach the following
    +      boilerplate notice, with the fields enclosed by brackets "{}"
    +      replaced with your own identifying information. (Don't include
    +      the brackets!)  The text should be enclosed in the appropriate
    +      comment syntax for the file format. We also recommend that a
    +      file or class name and description of purpose be included on the
    +      same "printed page" as the copyright notice for easier
    +      identification within third-party archives.
    +
    +   Copyright {yyyy} {name of copyright owner}
    +
    +   Licensed under the Apache License, Version 2.0 (the "License");
    +   you may not use this file except in compliance with the License.
    +   You may obtain a copy of the License at
    +
    +       http://www.apache.org/licenses/LICENSE-2.0
    +
    +   Unless required by applicable law or agreed to in writing, software
    +   distributed under the License is distributed on an "AS IS" BASIS,
    +   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    +   See the License for the specific language governing permissions and
    +   limitations under the License.
    +
    +
    + +
  • +

    Apache License 2.0

    +

    Used by:

    + +
                                    Apache License
                                Version 2.0, January 2004
                             http://www.apache.org/licenses/
     
    @@ -1955,7 +2586,7 @@ 

    Used by:

    same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright {yyyy} {name of copyright owner} + Copyright 2018-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -1975,10 +2606,227 @@

    Used by:

    Apache License 2.0

    Used by:

    +
                                  Apache License
    +                        Version 2.0 January 2004
    +                     http://www.apache.org/licenses/
    +
    +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
    +
    +1. Definitions.
    +
    +   "License" shall mean the terms and conditions for use, reproduction,
    +   and distribution as defined by Sections 1 through 9 of this document.
    +
    +   "Licensor" shall mean the copyright owner or entity authorized by
    +   the copyright owner that is granting the License.
    +
    +   "Legal Entity" shall mean the union of the acting entity and all
    +   other entities that control, are controlled by, or are under common
    +   control with that entity. For the purposes of this definition,
    +   "control" means (i) the power, direct or indirect, to cause the
    +   direction or management of such entity, whether by contract or
    +   otherwise, or (ii) ownership of fifty percent (50%) or more of the
    +   outstanding shares, or (iii) beneficial ownership of such entity.
    +
    +   "You" (or "Your") shall mean an individual or Legal Entity
    +   exercising permissions granted by this License.
    +
    +   "Source" form shall mean the preferred form for making modifications,
    +   including but not limited to software source code, documentation
    +   source, and configuration files.
    +
    +   "Object" form shall mean any form resulting from mechanical
    +   transformation or translation of a Source form, including but
    +   not limited to compiled object code, generated documentation,
    +   and conversions to other media types.
    +
    +   "Work" shall mean the work of authorship, whether in Source or
    +   Object form, made available under the License, as indicated by a
    +   copyright notice that is included in or attached to the work
    +   (an example is provided in the Appendix below).
    +
    +   "Derivative Works" shall mean any work, whether in Source or Object
    +   form, that is based on (or derived from) the Work and for which the
    +   editorial revisions, annotations, elaborations, or other modifications
    +   represent, as a whole, an original work of authorship. For the purposes
    +   of this License, Derivative Works shall not include works that remain
    +   separable from, or merely link (or bind by name) to the interfaces of,
    +   the Work and Derivative Works thereof.
    +
    +   "Contribution" shall mean any work of authorship, including
    +   the original version of the Work and any modifications or additions
    +   to that Work or Derivative Works thereof, that is intentionally
    +   submitted to Licensor for inclusion in the Work by the copyright owner
    +   or by an individual or Legal Entity authorized to submit on behalf of
    +   the copyright owner. For the purposes of this definition, "submitted"
    +   means any form of electronic, verbal, or written communication sent
    +   to the Licensor or its representatives, including but not limited to
    +   communication on electronic mailing lists, source code control systems,
    +   and issue tracking systems that are managed by, or on behalf of, the
    +   Licensor for the purpose of discussing and improving the Work, but
    +   excluding communication that is conspicuously marked or otherwise
    +   designated in writing by the copyright owner as "Not a Contribution."
    +
    +   "Contributor" shall mean Licensor and any individual or Legal Entity
    +   on behalf of whom a Contribution has been received by Licensor and
    +   subsequently incorporated within the Work.
    +
    +2. Grant of Copyright License. Subject to the terms and conditions of
    +   this License, each Contributor hereby grants to You a perpetual,
    +   worldwide, non-exclusive, no-charge, royalty-free, irrevocable
    +   copyright license to reproduce, prepare Derivative Works of,
    +   publicly display, publicly perform, sublicense, and distribute the
    +   Work and such Derivative Works in Source or Object form.
    +
    +3. Grant of Patent License. Subject to the terms and conditions of
    +   this License, each Contributor hereby grants to You a perpetual,
    +   worldwide, non-exclusive, no-charge, royalty-free, irrevocable
    +   (except as stated in this section) patent license to make, have made,
    +   use, offer to sell, sell, import, and otherwise transfer the Work,
    +   where such license applies only to those patent claims licensable
    +   by such Contributor that are necessarily infringed by their
    +   Contribution(s) alone or by combination of their Contribution(s)
    +   with the Work to which such Contribution(s) was submitted. If You
    +   institute patent litigation against any entity (including a
    +   cross-claim or counterclaim in a lawsuit) alleging that the Work
    +   or a Contribution incorporated within the Work constitutes direct
    +   or contributory patent infringement, then any patent licenses
    +   granted to You under this License for that Work shall terminate
    +   as of the date such litigation is filed.
    +
    +4. Redistribution. You may reproduce and distribute copies of the
    +   Work or Derivative Works thereof in any medium, with or without
    +   modifications, and in Source or Object form, provided that You
    +   meet the following conditions:
    +
    +   (a) You must give any other recipients of the Work or
    +       Derivative Works a copy of this License; and
    +
    +   (b) You must cause any modified files to carry prominent notices
    +       stating that You changed the files; and
    +
    +   (c) You must retain, in the Source form of any Derivative Works
    +       that You distribute, all copyright, patent, trademark, and
    +       attribution notices from the Source form of the Work,
    +       excluding those notices that do not pertain to any part of
    +       the Derivative Works; and
    +
    +   (d) If the Work includes a "NOTICE" text file as part of its
    +       distribution, then any Derivative Works that You distribute must
    +       include a readable copy of the attribution notices contained
    +       within such NOTICE file, excluding those notices that do not
    +       pertain to any part of the Derivative Works, in at least one
    +       of the following places: within a NOTICE text file distributed
    +       as part of the Derivative Works; within the Source form or
    +       documentation, if provided along with the Derivative Works; or,
    +       within a display generated by the Derivative Works, if and
    +       wherever such third-party notices normally appear. The contents
    +       of the NOTICE file are for informational purposes only and
    +       do not modify the License. You may add Your own attribution
    +       notices within Derivative Works that You distribute, alongside
    +       or as an addendum to the NOTICE text from the Work, provided
    +       that such additional attribution notices cannot be construed
    +       as modifying the License.
    +
    +   You may add Your own copyright statement to Your modifications and
    +   may provide additional or different license terms and conditions
    +   for use, reproduction, or distribution of Your modifications, or
    +   for any such Derivative Works as a whole, provided Your use,
    +   reproduction, and distribution of the Work otherwise complies with
    +   the conditions stated in this License.
    +
    +5. Submission of Contributions. Unless You explicitly state otherwise,
    +   any Contribution intentionally submitted for inclusion in the Work
    +   by You to the Licensor shall be under the terms and conditions of
    +   this License, without any additional terms or conditions.
    +   Notwithstanding the above, nothing herein shall supersede or modify
    +   the terms of any separate license agreement you may have executed
    +   with Licensor regarding such Contributions.
    +
    +6. Trademarks. This License does not grant permission to use the trade
    +   names, trademarks, service marks, or product names of the Licensor,
    +   except as required for reasonable and customary use in describing the
    +   origin of the Work and reproducing the content of the NOTICE file.
    +
    +7. Disclaimer of Warranty. Unless required by applicable law or
    +   agreed to in writing, Licensor provides the Work (and each
    +   Contributor provides its Contributions) on an "AS IS" BASIS,
    +   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
    +   implied, including, without limitation, any warranties or conditions
    +   of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
    +   PARTICULAR PURPOSE. You are solely responsible for determining the
    +   appropriateness of using or redistributing the Work and assume any
    +   risks associated with Your exercise of permissions under this License.
    +
    +8. Limitation of Liability. In no event and under no legal theory,
    +   whether in tort (including negligence), contract, or otherwise,
    +   unless required by applicable law (such as deliberate and grossly
    +   negligent acts) or agreed to in writing, shall any Contributor be
    +   liable to You for damages, including any direct, indirect, special,
    +   incidental, or consequential damages of any character arising as a
    +   result of this License or out of the use or inability to use the
    +   Work (including but not limited to damages for loss of goodwill,
    +   work stoppage, computer failure or malfunction, or any and all
    +   other commercial damages or losses), even if such Contributor
    +   has been advised of the possibility of such damages.
    +
    +9. Accepting Warranty or Additional Liability. While redistributing
    +   the Work or Derivative Works thereof, You may choose to offer,
    +   and charge a fee for, acceptance of support, warranty, indemnity,
    +   or other liability obligations and/or rights consistent with this
    +   License. However, in accepting such obligations, You may act only
    +   on Your own behalf and on Your sole responsibility, not on behalf
    +   of any other Contributor, and only if You agree to indemnify,
    +   defend, and hold each Contributor harmless for any liability
    +   incurred by, or claims asserted against, such Contributor by reason
    +   of your accepting any such warranty or additional liability.
    +
    +END OF TERMS AND CONDITIONS
    +
    +APPENDIX: How to apply the Apache License to your work.
    +
    +   To apply the Apache License to your work, attach the following
    +   boilerplate notice, with the fields enclosed by brackets "[]"
    +   replaced with your own identifying information. (Don't include
    +   the brackets!)  The text should be enclosed in the appropriate
    +   comment syntax for the file format. We also recommend that a
    +   file or class name and description of purpose be included on the
    +   same "printed page" as the copyright notice for easier
    +   identification within third-party archives.
    +
    +Copyright [yyyy] [name of copyright owner]
    +
    +Licensed under the Apache License, Version 2.0 (the "License");
    +you may not use this file except in compliance with the License.
    +You may obtain a copy of the License at
    +
    +	http://www.apache.org/licenses/LICENSE-2.0
    +
    +Unless required by applicable law or agreed to in writing, software
    +distributed under the License is distributed on an "AS IS" BASIS,
    +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    +See the License for the specific language governing permissions and
    +limitations under the License.
    +
    +
  • +
  • +

    Apache License 2.0

    +

    Used by:

    +
                                  Apache License
    -                        Version 2.0 January 2004
    +                        Version 2.0, January 2004
                          http://www.apache.org/licenses/
     
     TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
    @@ -2165,7 +3013,8 @@ 

    Used by:

    same "printed page" as the copyright notice for easier identification within third-party archives. -Copyright [yyyy] [name of copyright owner] +Copyright (c) 2016 Alex Crichton +Copyright (c) 2017 The Tokio Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -2184,15 +3033,7 @@

    Used by:

    Apache License 2.0

    Used by:

                                  Apache License
                             Version 2.0, January 2004
    @@ -2382,8 +3223,7 @@ 

    Used by:

    same "printed page" as the copyright notice for easier identification within third-party archives. -Copyright (c) 2016 Alex Crichton -Copyright (c) 2017 The Tokio Authors +Copyright (c) 2019 Matthias Einwag Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -2402,7 +3242,7 @@

    Used by:

    Apache License 2.0

    Used by:

                                  Apache License
                             Version 2.0, January 2004
    @@ -2592,7 +3432,7 @@ 

    Used by:

    same "printed page" as the copyright notice for easier identification within third-party archives. -Copyright (c) 2019 Matthias Einwag +Copyright 2014 Paho Lurie-Gregg Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -2604,14 +3444,13 @@

    Used by:

    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and -limitations under the License. -
    +limitations under the License.
  • Apache License 2.0

    Used by:

                                  Apache License
                             Version 2.0, January 2004
    @@ -2801,7 +3640,7 @@ 

    Used by:

    same "printed page" as the copyright notice for easier identification within third-party archives. -Copyright 2014 Paho Lurie-Gregg +Copyright 2017 The Rust Project Developers Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -2813,13 +3652,15 @@

    Used by:

    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and -limitations under the License.
    +limitations under the License. +
  • Apache License 2.0

    Used by:

                                  Apache License
                             Version 2.0, January 2004
    @@ -3009,7 +3850,7 @@ 

    Used by:

    same "printed page" as the copyright notice for easier identification within third-party archives. -Copyright 2017 The Rust Project Developers +Copyright 2017 http-rs authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -3028,7 +3869,7 @@

    Used by:

    Apache License 2.0

    Used by:

                                  Apache License
                             Version 2.0, January 2004
    @@ -3218,7 +4059,7 @@ 

    Used by:

    same "printed page" as the copyright notice for easier identification within third-party archives. -Copyright 2017 http-rs authors +Copyright 2017 quininer kel Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -3237,7 +4078,7 @@

    Used by:

    Apache License 2.0

    Used by:

                                  Apache License
                             Version 2.0, January 2004
    @@ -3427,7 +4268,7 @@ 

    Used by:

    same "printed page" as the copyright notice for easier identification within third-party archives. -Copyright 2017 quininer kel +Copyright 2018 The pin-utils authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -4503,6 +5344,7 @@

    Used by:

  • base64 0.21.7
  • base64 0.22.1
  • bitflags 2.11.1
  • +
  • bytes-utils 0.1.4
  • cc 1.2.62
  • cfg-if 1.0.4
  • cmake 0.1.58
  • @@ -4521,15 +5363,18 @@

    Used by:

  • errno 0.3.14
  • event-listener-strategy 0.5.4
  • event-listener 5.4.2
  • +
  • fastrand 2.5.0
  • find-msvc-tools 0.1.9
  • flate2 1.1.9
  • fnv 1.0.7
  • form_urlencoded 1.2.2
  • hashbrown 0.14.5
  • hashbrown 0.15.5
  • +
  • hashbrown 0.16.1
  • hashbrown 0.17.1
  • heck 0.5.0
  • httparse 1.10.1
  • +
  • hyper-rustls 0.27.10
  • idna 1.1.0
  • idna_adapter 1.2.2
  • indexmap 2.14.0
  • @@ -4544,6 +5389,7 @@

    Used by:

  • num-integer 0.1.46
  • num-traits 0.2.19
  • once_cell 1.21.4
  • +
  • openssl-probe 0.2.1
  • parking 2.2.1
  • parking_lot 0.12.5
  • parking_lot_core 0.9.12
  • @@ -4554,9 +5400,12 @@

    Used by:

  • pest_meta 2.8.6
  • pkg-config 0.3.33
  • regex-automata 0.4.14
  • +
  • regex-lite 0.1.9
  • regex-syntax 0.8.10
  • +
  • ring 0.17.14
  • ron 0.8.1
  • rustc_version 0.4.1
  • +
  • rustls-native-certs 0.8.4
  • rustls 0.23.46
  • scopeguard 1.2.0
  • signal-hook-registry 1.4.8
  • @@ -4574,6 +5423,7 @@

    Used by:

  • url 2.5.8
  • uuid 1.23.1
  • version_check 0.9.5
  • +
  • xmlparser 0.13.6
  • yasna 0.6.0
  •                               Apache License
    @@ -5420,6 +6270,7 @@ 

    Used by:

  • hmac 0.12.1
  • inout 0.1.4
  • md-5 0.10.6
  • +
  • sha1 0.10.6
  • sha2 0.10.9
  •                               Apache License
    @@ -6950,6 +7801,7 @@ 

    Used by:

    +
  • +
  • +

    ISC License

    +

    Used by:

    + +
    Copyright 2015-2025 Brian Smith.
    +
    +Permission to use, copy, modify, and/or distribute this software for any
    +purpose with or without fee is hereby granted, provided that the above
    +copyright notice and this permission notice appear in all copies.
    +
    +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
    +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
    +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
    +SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
    +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
    +OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
    +CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
     
  • @@ -7951,6 +8841,63 @@

    Used by:

    OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +
  • +
  • +

    MIT License

    +

    Used by:

    + +
    Copyright (c) 2018-2019 Sean McArthur
    +
    +Permission is hereby granted, free of charge, to any person obtaining a copy
    +of this software and associated documentation files (the "Software"), to deal
    +in the Software without restriction, including without limitation the rights
    +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
    +copies of the Software, and to permit persons to whom the Software is
    +furnished to do so, subject to the following conditions:
    +
    +The above copyright notice and this permission notice shall be included in
    +all copies or substantial portions of the Software.
    +
    +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
    +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
    +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
    +THE SOFTWARE.
    +
    +
    +
  • +
  • +

    MIT License

    +

    Used by:

    + +
    Copyright (c) 2018-2023 Sean McArthur
    +Copyright (c) 2016 Alex Crichton
    +
    +Permission is hereby granted, free of charge, to any person obtaining a copy
    +of this software and associated documentation files (the "Software"), to deal
    +in the Software without restriction, including without limitation the rights
    +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
    +copies of the Software, and to permit persons to whom the Software is
    +furnished to do so, subject to the following conditions:
    +
    +The above copyright notice and this permission notice shall be included in
    +all copies or substantial portions of the Software.
    +
    +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
    +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
    +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
    +THE SOFTWARE.
    +
     
  • @@ -8038,6 +8985,39 @@

    Used by:

    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +
  • +
  • +

    MIT License

    +

    Used by:

    + +
    Copyright (c) 2019 Hyper Contributors
    +
    +Permission is hereby granted, free of charge, to any
    +person obtaining a copy of this software and associated
    +documentation files (the "Software"), to deal in the
    +Software without restriction, including without
    +limitation the rights to use, copy, modify, merge,
    +publish, distribute, sublicense, and/or sell copies of
    +the Software, and to permit persons to whom the Software
    +is furnished to do so, subject to the following
    +conditions:
    +
    +The above copyright notice and this permission notice
    +shall be included in all copies or substantial portions
    +of the Software.
    +
    +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF
    +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED
    +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
    +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
    +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
    +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
    +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR
    +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
    +DEALINGS IN THE SOFTWARE.
     
  • @@ -8355,6 +9335,34 @@

    Used by:

    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +
  • +
  • +

    MIT License

    +

    Used by:

    + +
    MIT License
    +
    +Copyright (c) 2016 Jerome Froelich
    +
    +Permission is hereby granted, free of charge, to any person obtaining a copy
    +of this software and associated documentation files (the "Software"), to deal
    +in the Software without restriction, including without limitation the rights
    +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
    +copies of the Software, and to permit persons to whom the Software is
    +furnished to do so, subject to the following conditions:
    +
    +The above copyright notice and this permission notice shall be included in all
    +copies or substantial portions of the Software.
    +
    +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
    +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
    +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
    +SOFTWARE.
  • MIT License

    @@ -8564,6 +9572,34 @@

    Used by:

    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +
  • +
  • +

    MIT License

    +

    Used by:

    + +
    MIT License
    +
    +Copyright (c) 2022 Nugine
    +
    +Permission is hereby granted, free of charge, to any person obtaining a copy
    +of this software and associated documentation files (the "Software"), to deal
    +in the Software without restriction, including without limitation the rights
    +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
    +copies of the Software, and to permit persons to whom the Software is
    +furnished to do so, subject to the following conditions:
    +
    +The above copyright notice and this permission notice shall be included in all
    +copies or substantial portions of the Software.
    +
    +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
    +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
    +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
    +SOFTWARE.
  • MIT License

    @@ -8597,7 +9633,9 @@

    Used by:

    MIT License

    Used by:

    MIT License
     
    @@ -8736,6 +9774,34 @@ 

    Used by:

    OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
    +
  • +
  • +

    MIT License

    +

    Used by:

    + +
    The MIT License (MIT)
    +
    +Copyright (c) 2014 Mathijs van de Nes
    +
    +Permission is hereby granted, free of charge, to any person obtaining a copy
    +of this software and associated documentation files (the "Software"), to deal
    +in the Software without restriction, including without limitation the rights
    +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
    +copies of the Software, and to permit persons to whom the Software is
    +furnished to do so, subject to the following conditions:
    +
    +The above copyright notice and this permission notice shall be included in all
    +copies or substantial portions of the Software.
    +
    +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
    +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
    +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
    +SOFTWARE.
  • MIT License

    @@ -9107,6 +10173,34 @@

    Used by:

    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +
  • +
  • +

    MIT License

    +

    Used by:

    + +
    © 2016 Bertram Truong
    +© 2021 Kornel Lesiński
    +
    +Permission is hereby granted, free of charge, to any person obtaining a copy
    +of this software and associated documentation files (the "Software"), to deal
    +in the Software without restriction, including without limitation the rights
    +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
    +copies of the Software, and to permit persons to whom the Software is
    +furnished to do so, subject to the following conditions:
    +
    +The above copyright notice and this permission notice shall be included in
    +all copies or substantial portions of the Software.
    +
    +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
    +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
    +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
    +THE SOFTWARE.
    +
  • Unicode License v3

    @@ -9231,6 +10325,7 @@

    zlib License

    Used by:

    Copyright (c) 2024 Orson Peters
     
    diff --git a/SOFTWARE-LICENSE-NOTICES-DEV.html b/SOFTWARE-LICENSE-NOTICES-DEV.html
    index 9ff5a037d..1aa37ca78 100644
    --- a/SOFTWARE-LICENSE-NOTICES-DEV.html
    +++ b/SOFTWARE-LICENSE-NOTICES-DEV.html
    @@ -49,14 +49,14 @@ 

    ExtendDB Software License Notices

    Overview of licenses:

    All license text:

    @@ -1537,25 +1537,216 @@

    Used by:

    Apache License 2.0

    Used by:

    +
                                     Apache License
    +                           Version 2.0, January 2004
    +                        http://www.apache.org/licenses/
    +
    +   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
    +
    +   1. Definitions.
    +
    +      "License" shall mean the terms and conditions for use, reproduction,
    +      and distribution as defined by Sections 1 through 9 of this document.
    +
    +      "Licensor" shall mean the copyright owner or entity authorized by
    +      the copyright owner that is granting the License.
    +
    +      "Legal Entity" shall mean the union of the acting entity and all
    +      other entities that control, are controlled by, or are under common
    +      control with that entity. For the purposes of this definition,
    +      "control" means (i) the power, direct or indirect, to cause the
    +      direction or management of such entity, whether by contract or
    +      otherwise, or (ii) ownership of fifty percent (50%) or more of the
    +      outstanding shares, or (iii) beneficial ownership of such entity.
    +
    +      "You" (or "Your") shall mean an individual or Legal Entity
    +      exercising permissions granted by this License.
    +
    +      "Source" form shall mean the preferred form for making modifications,
    +      including but not limited to software source code, documentation
    +      source, and configuration files.
    +
    +      "Object" form shall mean any form resulting from mechanical
    +      transformation or translation of a Source form, including but
    +      not limited to compiled object code, generated documentation,
    +      and conversions to other media types.
    +
    +      "Work" shall mean the work of authorship, whether in Source or
    +      Object form, made available under the License, as indicated by a
    +      copyright notice that is included in or attached to the work
    +      (an example is provided in the Appendix below).
    +
    +      "Derivative Works" shall mean any work, whether in Source or Object
    +      form, that is based on (or derived from) the Work and for which the
    +      editorial revisions, annotations, elaborations, or other modifications
    +      represent, as a whole, an original work of authorship. For the purposes
    +      of this License, Derivative Works shall not include works that remain
    +      separable from, or merely link (or bind by name) to the interfaces of,
    +      the Work and Derivative Works thereof.
    +
    +      "Contribution" shall mean any work of authorship, including
    +      the original version of the Work and any modifications or additions
    +      to that Work or Derivative Works thereof, that is intentionally
    +      submitted to Licensor for inclusion in the Work by the copyright owner
    +      or by an individual or Legal Entity authorized to submit on behalf of
    +      the copyright owner. For the purposes of this definition, "submitted"
    +      means any form of electronic, verbal, or written communication sent
    +      to the Licensor or its representatives, including but not limited to
    +      communication on electronic mailing lists, source code control systems,
    +      and issue tracking systems that are managed by, or on behalf of, the
    +      Licensor for the purpose of discussing and improving the Work, but
    +      excluding communication that is conspicuously marked or otherwise
    +      designated in writing by the copyright owner as "Not a Contribution."
    +
    +      "Contributor" shall mean Licensor and any individual or Legal Entity
    +      on behalf of whom a Contribution has been received by Licensor and
    +      subsequently incorporated within the Work.
    +
    +   2. Grant of Copyright License. Subject to the terms and conditions of
    +      this License, each Contributor hereby grants to You a perpetual,
    +      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
    +      copyright license to reproduce, prepare Derivative Works of,
    +      publicly display, publicly perform, sublicense, and distribute the
    +      Work and such Derivative Works in Source or Object form.
    +
    +   3. Grant of Patent License. Subject to the terms and conditions of
    +      this License, each Contributor hereby grants to You a perpetual,
    +      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
    +      (except as stated in this section) patent license to make, have made,
    +      use, offer to sell, sell, import, and otherwise transfer the Work,
    +      where such license applies only to those patent claims licensable
    +      by such Contributor that are necessarily infringed by their
    +      Contribution(s) alone or by combination of their Contribution(s)
    +      with the Work to which such Contribution(s) was submitted. If You
    +      institute patent litigation against any entity (including a
    +      cross-claim or counterclaim in a lawsuit) alleging that the Work
    +      or a Contribution incorporated within the Work constitutes direct
    +      or contributory patent infringement, then any patent licenses
    +      granted to You under this License for that Work shall terminate
    +      as of the date such litigation is filed.
    +
    +   4. Redistribution. You may reproduce and distribute copies of the
    +      Work or Derivative Works thereof in any medium, with or without
    +      modifications, and in Source or Object form, provided that You
    +      meet the following conditions:
    +
    +      (a) You must give any other recipients of the Work or
    +          Derivative Works a copy of this License; and
    +
    +      (b) You must cause any modified files to carry prominent notices
    +          stating that You changed the files; and
    +
    +      (c) You must retain, in the Source form of any Derivative Works
    +          that You distribute, all copyright, patent, trademark, and
    +          attribution notices from the Source form of the Work,
    +          excluding those notices that do not pertain to any part of
    +          the Derivative Works; and
    +
    +      (d) If the Work includes a "NOTICE" text file as part of its
    +          distribution, then any Derivative Works that You distribute must
    +          include a readable copy of the attribution notices contained
    +          within such NOTICE file, excluding those notices that do not
    +          pertain to any part of the Derivative Works, in at least one
    +          of the following places: within a NOTICE text file distributed
    +          as part of the Derivative Works; within the Source form or
    +          documentation, if provided along with the Derivative Works; or,
    +          within a display generated by the Derivative Works, if and
    +          wherever such third-party notices normally appear. The contents
    +          of the NOTICE file are for informational purposes only and
    +          do not modify the License. You may add Your own attribution
    +          notices within Derivative Works that You distribute, alongside
    +          or as an addendum to the NOTICE text from the Work, provided
    +          that such additional attribution notices cannot be construed
    +          as modifying the License.
    +
    +      You may add Your own copyright statement to Your modifications and
    +      may provide additional or different license terms and conditions
    +      for use, reproduction, or distribution of Your modifications, or
    +      for any such Derivative Works as a whole, provided Your use,
    +      reproduction, and distribution of the Work otherwise complies with
    +      the conditions stated in this License.
    +
    +   5. Submission of Contributions. Unless You explicitly state otherwise,
    +      any Contribution intentionally submitted for inclusion in the Work
    +      by You to the Licensor shall be under the terms and conditions of
    +      this License, without any additional terms or conditions.
    +      Notwithstanding the above, nothing herein shall supersede or modify
    +      the terms of any separate license agreement you may have executed
    +      with Licensor regarding such Contributions.
    +
    +   6. Trademarks. This License does not grant permission to use the trade
    +      names, trademarks, service marks, or product names of the Licensor,
    +      except as required for reasonable and customary use in describing the
    +      origin of the Work and reproducing the content of the NOTICE file.
    +
    +   7. Disclaimer of Warranty. Unless required by applicable law or
    +      agreed to in writing, Licensor provides the Work (and each
    +      Contributor provides its Contributions) on an "AS IS" BASIS,
    +      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
    +      implied, including, without limitation, any warranties or conditions
    +      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
    +      PARTICULAR PURPOSE. You are solely responsible for determining the
    +      appropriateness of using or redistributing the Work and assume any
    +      risks associated with Your exercise of permissions under this License.
    +
    +   8. Limitation of Liability. In no event and under no legal theory,
    +      whether in tort (including negligence), contract, or otherwise,
    +      unless required by applicable law (such as deliberate and grossly
    +      negligent acts) or agreed to in writing, shall any Contributor be
    +      liable to You for damages, including any direct, indirect, special,
    +      incidental, or consequential damages of any character arising as a
    +      result of this License or out of the use or inability to use the
    +      Work (including but not limited to damages for loss of goodwill,
    +      work stoppage, computer failure or malfunction, or any and all
    +      other commercial damages or losses), even if such Contributor
    +      has been advised of the possibility of such damages.
    +
    +   9. Accepting Warranty or Additional Liability. While redistributing
    +      the Work or Derivative Works thereof, You may choose to offer,
    +      and charge a fee for, acceptance of support, warranty, indemnity,
    +      or other liability obligations and/or rights consistent with this
    +      License. However, in accepting such obligations, You may act only
    +      on Your own behalf and on Your sole responsibility, not on behalf
    +      of any other Contributor, and only if You agree to indemnify,
    +      defend, and hold each Contributor harmless for any liability
    +      incurred by, or claims asserted against, such Contributor by reason
    +      of your accepting any such warranty or additional liability.
    +
    +   END OF TERMS AND CONDITIONS
    +
    +   APPENDIX: How to apply the Apache License to your work.
    +
    +      To apply the Apache License to your work, attach the following
    +      boilerplate notice, with the fields enclosed by brackets "[]"
    +      replaced with your own identifying information. (Don't include
    +      the brackets!)  The text should be enclosed in the appropriate
    +      comment syntax for the file format. We also recommend that a
    +      file or class name and description of purpose be included on the
    +      same "printed page" as the copyright notice for easier
    +      identification within third-party archives.
    +
    +   Copyright [yyyy] [name of copyright owner]
    +
    +   Licensed under the Apache License, Version 2.0 (the "License");
    +   you may not use this file except in compliance with the License.
    +   You may obtain a copy of the License at
    +
    +       http://www.apache.org/licenses/LICENSE-2.0
    +
    +   Unless required by applicable law or agreed to in writing, software
    +   distributed under the License is distributed on an "AS IS" BASIS,
    +   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    +   See the License for the specific language governing permissions and
    +   limitations under the License.
    +
    +
  • +
  • +

    Apache License 2.0

    +

    Used by:

    +
                                     Apache License
                                Version 2.0, January 2004
    @@ -1745,30 +1936,687 @@ 

    Used by:

    same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright {yyyy} {name of copyright owner} + Copyright 2017 Juniper Networks, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +
    +
  • +
  • +

    Apache License 2.0

    +

    Used by:

    + +
                                     Apache License
    +                           Version 2.0, January 2004
    +                        http://www.apache.org/licenses/
    +
    +   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
    +
    +   1. Definitions.
    +
    +      "License" shall mean the terms and conditions for use, reproduction,
    +      and distribution as defined by Sections 1 through 9 of this document.
    +
    +      "Licensor" shall mean the copyright owner or entity authorized by
    +      the copyright owner that is granting the License.
    +
    +      "Legal Entity" shall mean the union of the acting entity and all
    +      other entities that control, are controlled by, or are under common
    +      control with that entity. For the purposes of this definition,
    +      "control" means (i) the power, direct or indirect, to cause the
    +      direction or management of such entity, whether by contract or
    +      otherwise, or (ii) ownership of fifty percent (50%) or more of the
    +      outstanding shares, or (iii) beneficial ownership of such entity.
    +
    +      "You" (or "Your") shall mean an individual or Legal Entity
    +      exercising permissions granted by this License.
    +
    +      "Source" form shall mean the preferred form for making modifications,
    +      including but not limited to software source code, documentation
    +      source, and configuration files.
    +
    +      "Object" form shall mean any form resulting from mechanical
    +      transformation or translation of a Source form, including but
    +      not limited to compiled object code, generated documentation,
    +      and conversions to other media types.
    +
    +      "Work" shall mean the work of authorship, whether in Source or
    +      Object form, made available under the License, as indicated by a
    +      copyright notice that is included in or attached to the work
    +      (an example is provided in the Appendix below).
    +
    +      "Derivative Works" shall mean any work, whether in Source or Object
    +      form, that is based on (or derived from) the Work and for which the
    +      editorial revisions, annotations, elaborations, or other modifications
    +      represent, as a whole, an original work of authorship. For the purposes
    +      of this License, Derivative Works shall not include works that remain
    +      separable from, or merely link (or bind by name) to the interfaces of,
    +      the Work and Derivative Works thereof.
    +
    +      "Contribution" shall mean any work of authorship, including
    +      the original version of the Work and any modifications or additions
    +      to that Work or Derivative Works thereof, that is intentionally
    +      submitted to Licensor for inclusion in the Work by the copyright owner
    +      or by an individual or Legal Entity authorized to submit on behalf of
    +      the copyright owner. For the purposes of this definition, "submitted"
    +      means any form of electronic, verbal, or written communication sent
    +      to the Licensor or its representatives, including but not limited to
    +      communication on electronic mailing lists, source code control systems,
    +      and issue tracking systems that are managed by, or on behalf of, the
    +      Licensor for the purpose of discussing and improving the Work, but
    +      excluding communication that is conspicuously marked or otherwise
    +      designated in writing by the copyright owner as "Not a Contribution."
    +
    +      "Contributor" shall mean Licensor and any individual or Legal Entity
    +      on behalf of whom a Contribution has been received by Licensor and
    +      subsequently incorporated within the Work.
    +
    +   2. Grant of Copyright License. Subject to the terms and conditions of
    +      this License, each Contributor hereby grants to You a perpetual,
    +      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
    +      copyright license to reproduce, prepare Derivative Works of,
    +      publicly display, publicly perform, sublicense, and distribute the
    +      Work and such Derivative Works in Source or Object form.
    +
    +   3. Grant of Patent License. Subject to the terms and conditions of
    +      this License, each Contributor hereby grants to You a perpetual,
    +      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
    +      (except as stated in this section) patent license to make, have made,
    +      use, offer to sell, sell, import, and otherwise transfer the Work,
    +      where such license applies only to those patent claims licensable
    +      by such Contributor that are necessarily infringed by their
    +      Contribution(s) alone or by combination of their Contribution(s)
    +      with the Work to which such Contribution(s) was submitted. If You
    +      institute patent litigation against any entity (including a
    +      cross-claim or counterclaim in a lawsuit) alleging that the Work
    +      or a Contribution incorporated within the Work constitutes direct
    +      or contributory patent infringement, then any patent licenses
    +      granted to You under this License for that Work shall terminate
    +      as of the date such litigation is filed.
    +
    +   4. Redistribution. You may reproduce and distribute copies of the
    +      Work or Derivative Works thereof in any medium, with or without
    +      modifications, and in Source or Object form, provided that You
    +      meet the following conditions:
    +
    +      (a) You must give any other recipients of the Work or
    +          Derivative Works a copy of this License; and
    +
    +      (b) You must cause any modified files to carry prominent notices
    +          stating that You changed the files; and
    +
    +      (c) You must retain, in the Source form of any Derivative Works
    +          that You distribute, all copyright, patent, trademark, and
    +          attribution notices from the Source form of the Work,
    +          excluding those notices that do not pertain to any part of
    +          the Derivative Works; and
    +
    +      (d) If the Work includes a "NOTICE" text file as part of its
    +          distribution, then any Derivative Works that You distribute must
    +          include a readable copy of the attribution notices contained
    +          within such NOTICE file, excluding those notices that do not
    +          pertain to any part of the Derivative Works, in at least one
    +          of the following places: within a NOTICE text file distributed
    +          as part of the Derivative Works; within the Source form or
    +          documentation, if provided along with the Derivative Works; or,
    +          within a display generated by the Derivative Works, if and
    +          wherever such third-party notices normally appear. The contents
    +          of the NOTICE file are for informational purposes only and
    +          do not modify the License. You may add Your own attribution
    +          notices within Derivative Works that You distribute, alongside
    +          or as an addendum to the NOTICE text from the Work, provided
    +          that such additional attribution notices cannot be construed
    +          as modifying the License.
    +
    +      You may add Your own copyright statement to Your modifications and
    +      may provide additional or different license terms and conditions
    +      for use, reproduction, or distribution of Your modifications, or
    +      for any such Derivative Works as a whole, provided Your use,
    +      reproduction, and distribution of the Work otherwise complies with
    +      the conditions stated in this License.
    +
    +   5. Submission of Contributions. Unless You explicitly state otherwise,
    +      any Contribution intentionally submitted for inclusion in the Work
    +      by You to the Licensor shall be under the terms and conditions of
    +      this License, without any additional terms or conditions.
    +      Notwithstanding the above, nothing herein shall supersede or modify
    +      the terms of any separate license agreement you may have executed
    +      with Licensor regarding such Contributions.
    +
    +   6. Trademarks. This License does not grant permission to use the trade
    +      names, trademarks, service marks, or product names of the Licensor,
    +      except as required for reasonable and customary use in describing the
    +      origin of the Work and reproducing the content of the NOTICE file.
    +
    +   7. Disclaimer of Warranty. Unless required by applicable law or
    +      agreed to in writing, Licensor provides the Work (and each
    +      Contributor provides its Contributions) on an "AS IS" BASIS,
    +      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
    +      implied, including, without limitation, any warranties or conditions
    +      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
    +      PARTICULAR PURPOSE. You are solely responsible for determining the
    +      appropriateness of using or redistributing the Work and assume any
    +      risks associated with Your exercise of permissions under this License.
    +
    +   8. Limitation of Liability. In no event and under no legal theory,
    +      whether in tort (including negligence), contract, or otherwise,
    +      unless required by applicable law (such as deliberate and grossly
    +      negligent acts) or agreed to in writing, shall any Contributor be
    +      liable to You for damages, including any direct, indirect, special,
    +      incidental, or consequential damages of any character arising as a
    +      result of this License or out of the use or inability to use the
    +      Work (including but not limited to damages for loss of goodwill,
    +      work stoppage, computer failure or malfunction, or any and all
    +      other commercial damages or losses), even if such Contributor
    +      has been advised of the possibility of such damages.
    +
    +   9. Accepting Warranty or Additional Liability. While redistributing
    +      the Work or Derivative Works thereof, You may choose to offer,
    +      and charge a fee for, acceptance of support, warranty, indemnity,
    +      or other liability obligations and/or rights consistent with this
    +      License. However, in accepting such obligations, You may act only
    +      on Your own behalf and on Your sole responsibility, not on behalf
    +      of any other Contributor, and only if You agree to indemnify,
    +      defend, and hold each Contributor harmless for any liability
    +      incurred by, or claims asserted against, such Contributor by reason
    +      of your accepting any such warranty or additional liability.
    +
    +   END OF TERMS AND CONDITIONS
    +
    +   APPENDIX: How to apply the Apache License to your work.
    +
    +      To apply the Apache License to your work, attach the following
    +      boilerplate notice, with the fields enclosed by brackets "{}"
    +      replaced with your own identifying information. (Don't include
    +      the brackets!)  The text should be enclosed in the appropriate
    +      comment syntax for the file format. We also recommend that a
    +      file or class name and description of purpose be included on the
    +      same "printed page" as the copyright notice for easier
    +      identification within third-party archives.
    +
    +   Copyright {yyyy} {name of copyright owner}
    +
    +   Licensed under the Apache License, Version 2.0 (the "License");
    +   you may not use this file except in compliance with the License.
    +   You may obtain a copy of the License at
    +
    +       http://www.apache.org/licenses/LICENSE-2.0
    +
    +   Unless required by applicable law or agreed to in writing, software
    +   distributed under the License is distributed on an "AS IS" BASIS,
    +   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    +   See the License for the specific language governing permissions and
    +   limitations under the License.
    +
    +
    +
  • +
  • +

    Apache License 2.0

    +

    Used by:

    + +
                                    Apache License
    +                           Version 2.0, January 2004
    +                        http://www.apache.org/licenses/
    +
    +   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
    +
    +   1. Definitions.
    +
    +      "License" shall mean the terms and conditions for use, reproduction,
    +      and distribution as defined by Sections 1 through 9 of this document.
    +
    +      "Licensor" shall mean the copyright owner or entity authorized by
    +      the copyright owner that is granting the License.
    +
    +      "Legal Entity" shall mean the union of the acting entity and all
    +      other entities that control, are controlled by, or are under common
    +      control with that entity. For the purposes of this definition,
    +      "control" means (i) the power, direct or indirect, to cause the
    +      direction or management of such entity, whether by contract or
    +      otherwise, or (ii) ownership of fifty percent (50%) or more of the
    +      outstanding shares, or (iii) beneficial ownership of such entity.
    +
    +      "You" (or "Your") shall mean an individual or Legal Entity
    +      exercising permissions granted by this License.
    +
    +      "Source" form shall mean the preferred form for making modifications,
    +      including but not limited to software source code, documentation
    +      source, and configuration files.
    +
    +      "Object" form shall mean any form resulting from mechanical
    +      transformation or translation of a Source form, including but
    +      not limited to compiled object code, generated documentation,
    +      and conversions to other media types.
    +
    +      "Work" shall mean the work of authorship, whether in Source or
    +      Object form, made available under the License, as indicated by a
    +      copyright notice that is included in or attached to the work
    +      (an example is provided in the Appendix below).
    +
    +      "Derivative Works" shall mean any work, whether in Source or Object
    +      form, that is based on (or derived from) the Work and for which the
    +      editorial revisions, annotations, elaborations, or other modifications
    +      represent, as a whole, an original work of authorship. For the purposes
    +      of this License, Derivative Works shall not include works that remain
    +      separable from, or merely link (or bind by name) to the interfaces of,
    +      the Work and Derivative Works thereof.
    +
    +      "Contribution" shall mean any work of authorship, including
    +      the original version of the Work and any modifications or additions
    +      to that Work or Derivative Works thereof, that is intentionally
    +      submitted to Licensor for inclusion in the Work by the copyright owner
    +      or by an individual or Legal Entity authorized to submit on behalf of
    +      the copyright owner. For the purposes of this definition, "submitted"
    +      means any form of electronic, verbal, or written communication sent
    +      to the Licensor or its representatives, including but not limited to
    +      communication on electronic mailing lists, source code control systems,
    +      and issue tracking systems that are managed by, or on behalf of, the
    +      Licensor for the purpose of discussing and improving the Work, but
    +      excluding communication that is conspicuously marked or otherwise
    +      designated in writing by the copyright owner as "Not a Contribution."
    +
    +      "Contributor" shall mean Licensor and any individual or Legal Entity
    +      on behalf of whom a Contribution has been received by Licensor and
    +      subsequently incorporated within the Work.
    +
    +   2. Grant of Copyright License. Subject to the terms and conditions of
    +      this License, each Contributor hereby grants to You a perpetual,
    +      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
    +      copyright license to reproduce, prepare Derivative Works of,
    +      publicly display, publicly perform, sublicense, and distribute the
    +      Work and such Derivative Works in Source or Object form.
    +
    +   3. Grant of Patent License. Subject to the terms and conditions of
    +      this License, each Contributor hereby grants to You a perpetual,
    +      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
    +      (except as stated in this section) patent license to make, have made,
    +      use, offer to sell, sell, import, and otherwise transfer the Work,
    +      where such license applies only to those patent claims licensable
    +      by such Contributor that are necessarily infringed by their
    +      Contribution(s) alone or by combination of their Contribution(s)
    +      with the Work to which such Contribution(s) was submitted. If You
    +      institute patent litigation against any entity (including a
    +      cross-claim or counterclaim in a lawsuit) alleging that the Work
    +      or a Contribution incorporated within the Work constitutes direct
    +      or contributory patent infringement, then any patent licenses
    +      granted to You under this License for that Work shall terminate
    +      as of the date such litigation is filed.
    +
    +   4. Redistribution. You may reproduce and distribute copies of the
    +      Work or Derivative Works thereof in any medium, with or without
    +      modifications, and in Source or Object form, provided that You
    +      meet the following conditions:
    +
    +      (a) You must give any other recipients of the Work or
    +          Derivative Works a copy of this License; and
    +
    +      (b) You must cause any modified files to carry prominent notices
    +          stating that You changed the files; and
    +
    +      (c) You must retain, in the Source form of any Derivative Works
    +          that You distribute, all copyright, patent, trademark, and
    +          attribution notices from the Source form of the Work,
    +          excluding those notices that do not pertain to any part of
    +          the Derivative Works; and
    +
    +      (d) If the Work includes a "NOTICE" text file as part of its
    +          distribution, then any Derivative Works that You distribute must
    +          include a readable copy of the attribution notices contained
    +          within such NOTICE file, excluding those notices that do not
    +          pertain to any part of the Derivative Works, in at least one
    +          of the following places: within a NOTICE text file distributed
    +          as part of the Derivative Works; within the Source form or
    +          documentation, if provided along with the Derivative Works; or,
    +          within a display generated by the Derivative Works, if and
    +          wherever such third-party notices normally appear. The contents
    +          of the NOTICE file are for informational purposes only and
    +          do not modify the License. You may add Your own attribution
    +          notices within Derivative Works that You distribute, alongside
    +          or as an addendum to the NOTICE text from the Work, provided
    +          that such additional attribution notices cannot be construed
    +          as modifying the License.
    +
    +      You may add Your own copyright statement to Your modifications and
    +      may provide additional or different license terms and conditions
    +      for use, reproduction, or distribution of Your modifications, or
    +      for any such Derivative Works as a whole, provided Your use,
    +      reproduction, and distribution of the Work otherwise complies with
    +      the conditions stated in this License.
    +
    +   5. Submission of Contributions. Unless You explicitly state otherwise,
    +      any Contribution intentionally submitted for inclusion in the Work
    +      by You to the Licensor shall be under the terms and conditions of
    +      this License, without any additional terms or conditions.
    +      Notwithstanding the above, nothing herein shall supersede or modify
    +      the terms of any separate license agreement you may have executed
    +      with Licensor regarding such Contributions.
    +
    +   6. Trademarks. This License does not grant permission to use the trade
    +      names, trademarks, service marks, or product names of the Licensor,
    +      except as required for reasonable and customary use in describing the
    +      origin of the Work and reproducing the content of the NOTICE file.
    +
    +   7. Disclaimer of Warranty. Unless required by applicable law or
    +      agreed to in writing, Licensor provides the Work (and each
    +      Contributor provides its Contributions) on an "AS IS" BASIS,
    +      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
    +      implied, including, without limitation, any warranties or conditions
    +      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
    +      PARTICULAR PURPOSE. You are solely responsible for determining the
    +      appropriateness of using or redistributing the Work and assume any
    +      risks associated with Your exercise of permissions under this License.
    +
    +   8. Limitation of Liability. In no event and under no legal theory,
    +      whether in tort (including negligence), contract, or otherwise,
    +      unless required by applicable law (such as deliberate and grossly
    +      negligent acts) or agreed to in writing, shall any Contributor be
    +      liable to You for damages, including any direct, indirect, special,
    +      incidental, or consequential damages of any character arising as a
    +      result of this License or out of the use or inability to use the
    +      Work (including but not limited to damages for loss of goodwill,
    +      work stoppage, computer failure or malfunction, or any and all
    +      other commercial damages or losses), even if such Contributor
    +      has been advised of the possibility of such damages.
    +
    +   9. Accepting Warranty or Additional Liability. While redistributing
    +      the Work or Derivative Works thereof, You may choose to offer,
    +      and charge a fee for, acceptance of support, warranty, indemnity,
    +      or other liability obligations and/or rights consistent with this
    +      License. However, in accepting such obligations, You may act only
    +      on Your own behalf and on Your sole responsibility, not on behalf
    +      of any other Contributor, and only if You agree to indemnify,
    +      defend, and hold each Contributor harmless for any liability
    +      incurred by, or claims asserted against, such Contributor by reason
    +      of your accepting any such warranty or additional liability.
    +
    +   END OF TERMS AND CONDITIONS
    +
    +   APPENDIX: How to apply the Apache License to your work.
    +
    +      To apply the Apache License to your work, attach the following
    +      boilerplate notice, with the fields enclosed by brackets "{}"
    +      replaced with your own identifying information. (Don't include
    +      the brackets!)  The text should be enclosed in the appropriate
    +      comment syntax for the file format. We also recommend that a
    +      file or class name and description of purpose be included on the
    +      same "printed page" as the copyright notice for easier
    +      identification within third-party archives.
    +
    +   Copyright 2018-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
    +
    +   Licensed under the Apache License, Version 2.0 (the "License");
    +   you may not use this file except in compliance with the License.
    +   You may obtain a copy of the License at
    +
    +       http://www.apache.org/licenses/LICENSE-2.0
    +
    +   Unless required by applicable law or agreed to in writing, software
    +   distributed under the License is distributed on an "AS IS" BASIS,
    +   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    +   See the License for the specific language governing permissions and
    +   limitations under the License.
    +
    +
    +
  • +
  • +

    Apache License 2.0

    +

    Used by:

    + +
                                  Apache License
    +                        Version 2.0 January 2004
    +                     http://www.apache.org/licenses/
    +
    +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
    +
    +1. Definitions.
    +
    +   "License" shall mean the terms and conditions for use, reproduction,
    +   and distribution as defined by Sections 1 through 9 of this document.
    +
    +   "Licensor" shall mean the copyright owner or entity authorized by
    +   the copyright owner that is granting the License.
    +
    +   "Legal Entity" shall mean the union of the acting entity and all
    +   other entities that control, are controlled by, or are under common
    +   control with that entity. For the purposes of this definition,
    +   "control" means (i) the power, direct or indirect, to cause the
    +   direction or management of such entity, whether by contract or
    +   otherwise, or (ii) ownership of fifty percent (50%) or more of the
    +   outstanding shares, or (iii) beneficial ownership of such entity.
    +
    +   "You" (or "Your") shall mean an individual or Legal Entity
    +   exercising permissions granted by this License.
    +
    +   "Source" form shall mean the preferred form for making modifications,
    +   including but not limited to software source code, documentation
    +   source, and configuration files.
    +
    +   "Object" form shall mean any form resulting from mechanical
    +   transformation or translation of a Source form, including but
    +   not limited to compiled object code, generated documentation,
    +   and conversions to other media types.
    +
    +   "Work" shall mean the work of authorship, whether in Source or
    +   Object form, made available under the License, as indicated by a
    +   copyright notice that is included in or attached to the work
    +   (an example is provided in the Appendix below).
    +
    +   "Derivative Works" shall mean any work, whether in Source or Object
    +   form, that is based on (or derived from) the Work and for which the
    +   editorial revisions, annotations, elaborations, or other modifications
    +   represent, as a whole, an original work of authorship. For the purposes
    +   of this License, Derivative Works shall not include works that remain
    +   separable from, or merely link (or bind by name) to the interfaces of,
    +   the Work and Derivative Works thereof.
    +
    +   "Contribution" shall mean any work of authorship, including
    +   the original version of the Work and any modifications or additions
    +   to that Work or Derivative Works thereof, that is intentionally
    +   submitted to Licensor for inclusion in the Work by the copyright owner
    +   or by an individual or Legal Entity authorized to submit on behalf of
    +   the copyright owner. For the purposes of this definition, "submitted"
    +   means any form of electronic, verbal, or written communication sent
    +   to the Licensor or its representatives, including but not limited to
    +   communication on electronic mailing lists, source code control systems,
    +   and issue tracking systems that are managed by, or on behalf of, the
    +   Licensor for the purpose of discussing and improving the Work, but
    +   excluding communication that is conspicuously marked or otherwise
    +   designated in writing by the copyright owner as "Not a Contribution."
    +
    +   "Contributor" shall mean Licensor and any individual or Legal Entity
    +   on behalf of whom a Contribution has been received by Licensor and
    +   subsequently incorporated within the Work.
    +
    +2. Grant of Copyright License. Subject to the terms and conditions of
    +   this License, each Contributor hereby grants to You a perpetual,
    +   worldwide, non-exclusive, no-charge, royalty-free, irrevocable
    +   copyright license to reproduce, prepare Derivative Works of,
    +   publicly display, publicly perform, sublicense, and distribute the
    +   Work and such Derivative Works in Source or Object form.
    +
    +3. Grant of Patent License. Subject to the terms and conditions of
    +   this License, each Contributor hereby grants to You a perpetual,
    +   worldwide, non-exclusive, no-charge, royalty-free, irrevocable
    +   (except as stated in this section) patent license to make, have made,
    +   use, offer to sell, sell, import, and otherwise transfer the Work,
    +   where such license applies only to those patent claims licensable
    +   by such Contributor that are necessarily infringed by their
    +   Contribution(s) alone or by combination of their Contribution(s)
    +   with the Work to which such Contribution(s) was submitted. If You
    +   institute patent litigation against any entity (including a
    +   cross-claim or counterclaim in a lawsuit) alleging that the Work
    +   or a Contribution incorporated within the Work constitutes direct
    +   or contributory patent infringement, then any patent licenses
    +   granted to You under this License for that Work shall terminate
    +   as of the date such litigation is filed.
    +
    +4. Redistribution. You may reproduce and distribute copies of the
    +   Work or Derivative Works thereof in any medium, with or without
    +   modifications, and in Source or Object form, provided that You
    +   meet the following conditions:
    +
    +   (a) You must give any other recipients of the Work or
    +       Derivative Works a copy of this License; and
    +
    +   (b) You must cause any modified files to carry prominent notices
    +       stating that You changed the files; and
    +
    +   (c) You must retain, in the Source form of any Derivative Works
    +       that You distribute, all copyright, patent, trademark, and
    +       attribution notices from the Source form of the Work,
    +       excluding those notices that do not pertain to any part of
    +       the Derivative Works; and
    +
    +   (d) If the Work includes a "NOTICE" text file as part of its
    +       distribution, then any Derivative Works that You distribute must
    +       include a readable copy of the attribution notices contained
    +       within such NOTICE file, excluding those notices that do not
    +       pertain to any part of the Derivative Works, in at least one
    +       of the following places: within a NOTICE text file distributed
    +       as part of the Derivative Works; within the Source form or
    +       documentation, if provided along with the Derivative Works; or,
    +       within a display generated by the Derivative Works, if and
    +       wherever such third-party notices normally appear. The contents
    +       of the NOTICE file are for informational purposes only and
    +       do not modify the License. You may add Your own attribution
    +       notices within Derivative Works that You distribute, alongside
    +       or as an addendum to the NOTICE text from the Work, provided
    +       that such additional attribution notices cannot be construed
    +       as modifying the License.
    +
    +   You may add Your own copyright statement to Your modifications and
    +   may provide additional or different license terms and conditions
    +   for use, reproduction, or distribution of Your modifications, or
    +   for any such Derivative Works as a whole, provided Your use,
    +   reproduction, and distribution of the Work otherwise complies with
    +   the conditions stated in this License.
    +
    +5. Submission of Contributions. Unless You explicitly state otherwise,
    +   any Contribution intentionally submitted for inclusion in the Work
    +   by You to the Licensor shall be under the terms and conditions of
    +   this License, without any additional terms or conditions.
    +   Notwithstanding the above, nothing herein shall supersede or modify
    +   the terms of any separate license agreement you may have executed
    +   with Licensor regarding such Contributions.
    +
    +6. Trademarks. This License does not grant permission to use the trade
    +   names, trademarks, service marks, or product names of the Licensor,
    +   except as required for reasonable and customary use in describing the
    +   origin of the Work and reproducing the content of the NOTICE file.
    +
    +7. Disclaimer of Warranty. Unless required by applicable law or
    +   agreed to in writing, Licensor provides the Work (and each
    +   Contributor provides its Contributions) on an "AS IS" BASIS,
    +   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
    +   implied, including, without limitation, any warranties or conditions
    +   of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
    +   PARTICULAR PURPOSE. You are solely responsible for determining the
    +   appropriateness of using or redistributing the Work and assume any
    +   risks associated with Your exercise of permissions under this License.
    +
    +8. Limitation of Liability. In no event and under no legal theory,
    +   whether in tort (including negligence), contract, or otherwise,
    +   unless required by applicable law (such as deliberate and grossly
    +   negligent acts) or agreed to in writing, shall any Contributor be
    +   liable to You for damages, including any direct, indirect, special,
    +   incidental, or consequential damages of any character arising as a
    +   result of this License or out of the use or inability to use the
    +   Work (including but not limited to damages for loss of goodwill,
    +   work stoppage, computer failure or malfunction, or any and all
    +   other commercial damages or losses), even if such Contributor
    +   has been advised of the possibility of such damages.
    +
    +9. Accepting Warranty or Additional Liability. While redistributing
    +   the Work or Derivative Works thereof, You may choose to offer,
    +   and charge a fee for, acceptance of support, warranty, indemnity,
    +   or other liability obligations and/or rights consistent with this
    +   License. However, in accepting such obligations, You may act only
    +   on Your own behalf and on Your sole responsibility, not on behalf
    +   of any other Contributor, and only if You agree to indemnify,
    +   defend, and hold each Contributor harmless for any liability
    +   incurred by, or claims asserted against, such Contributor by reason
    +   of your accepting any such warranty or additional liability.
    +
    +END OF TERMS AND CONDITIONS
    +
    +APPENDIX: How to apply the Apache License to your work.
     
    -   Licensed under the Apache License, Version 2.0 (the "License");
    -   you may not use this file except in compliance with the License.
    -   You may obtain a copy of the License at
    +   To apply the Apache License to your work, attach the following
    +   boilerplate notice, with the fields enclosed by brackets "[]"
    +   replaced with your own identifying information. (Don't include
    +   the brackets!)  The text should be enclosed in the appropriate
    +   comment syntax for the file format. We also recommend that a
    +   file or class name and description of purpose be included on the
    +   same "printed page" as the copyright notice for easier
    +   identification within third-party archives.
     
    -       http://www.apache.org/licenses/LICENSE-2.0
    +Copyright [yyyy] [name of copyright owner]
     
    -   Unless required by applicable law or agreed to in writing, software
    -   distributed under the License is distributed on an "AS IS" BASIS,
    -   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    -   See the License for the specific language governing permissions and
    -   limitations under the License.
    +Licensed under the Apache License, Version 2.0 (the "License");
    +you may not use this file except in compliance with the License.
    +You may obtain a copy of the License at
    +
    +	http://www.apache.org/licenses/LICENSE-2.0
     
    +Unless required by applicable law or agreed to in writing, software
    +distributed under the License is distributed on an "AS IS" BASIS,
    +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    +See the License for the specific language governing permissions and
    +limitations under the License.
     
  • Apache License 2.0

    Used by:

                                  Apache License
    -                        Version 2.0 January 2004
    +                        Version 2.0, January 2004
                          http://www.apache.org/licenses/
     
     TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
    @@ -1955,7 +2803,8 @@ 

    Used by:

    same "printed page" as the copyright notice for easier identification within third-party archives. -Copyright [yyyy] [name of copyright owner] +Copyright (c) 2016 Alex Crichton +Copyright (c) 2017 The Tokio Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -1974,15 +2823,7 @@

    Used by:

    Apache License 2.0

    Used by:

                                  Apache License
                             Version 2.0, January 2004
    @@ -2172,8 +3013,7 @@ 

    Used by:

    same "printed page" as the copyright notice for easier identification within third-party archives. -Copyright (c) 2016 Alex Crichton -Copyright (c) 2017 The Tokio Authors +Copyright (c) 2019 Matthias Einwag Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -2192,7 +3032,7 @@

    Used by:

    Apache License 2.0

    Used by:

                                  Apache License
                             Version 2.0, January 2004
    @@ -2382,7 +3222,7 @@ 

    Used by:

    same "printed page" as the copyright notice for easier identification within third-party archives. -Copyright (c) 2019 Matthias Einwag +Copyright 2014 Paho Lurie-Gregg Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -2394,14 +3234,13 @@

    Used by:

    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and -limitations under the License. -
    +limitations under the License.
  • Apache License 2.0

    Used by:

                                  Apache License
                             Version 2.0, January 2004
    @@ -2591,7 +3430,7 @@ 

    Used by:

    same "printed page" as the copyright notice for easier identification within third-party archives. -Copyright 2014 Paho Lurie-Gregg +Copyright 2017 The Rust Project Developers Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -2603,13 +3442,15 @@

    Used by:

    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and -limitations under the License.
    +limitations under the License. +
  • Apache License 2.0

    Used by:

                                  Apache License
                             Version 2.0, January 2004
    @@ -2799,7 +3640,7 @@ 

    Used by:

    same "printed page" as the copyright notice for easier identification within third-party archives. -Copyright 2017 The Rust Project Developers +Copyright 2017 http-rs authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -2818,7 +3659,7 @@

    Used by:

    Apache License 2.0

    Used by:

                                  Apache License
                             Version 2.0, January 2004
    @@ -3008,7 +3849,7 @@ 

    Used by:

    same "printed page" as the copyright notice for easier identification within third-party archives. -Copyright 2017 http-rs authors +Copyright 2017 quininer kel Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -3027,7 +3868,7 @@

    Used by:

    Apache License 2.0

    Used by:

                                  Apache License
                             Version 2.0, January 2004
    @@ -3217,7 +4058,7 @@ 

    Used by:

    same "printed page" as the copyright notice for easier identification within third-party archives. -Copyright 2017 quininer kel +Copyright 2018 The pin-utils authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -4083,6 +4924,7 @@

    Used by:

  • base64 0.21.7
  • base64 0.22.1
  • bitflags 2.11.1
  • +
  • bytes-utils 0.1.4
  • cc 1.2.62
  • cfg-if 1.0.4
  • cmake 0.1.58
  • @@ -4100,15 +4942,18 @@

    Used by:

  • errno 0.3.14
  • event-listener-strategy 0.5.4
  • event-listener 5.4.2
  • +
  • fastrand 2.5.0
  • find-msvc-tools 0.1.9
  • flate2 1.1.9
  • fnv 1.0.7
  • form_urlencoded 1.2.2
  • hashbrown 0.14.5
  • hashbrown 0.15.5
  • +
  • hashbrown 0.16.1
  • hashbrown 0.17.1
  • heck 0.5.0
  • httparse 1.10.1
  • +
  • hyper-rustls 0.27.10
  • idna 1.1.0
  • idna_adapter 1.2.2
  • indexmap 2.14.0
  • @@ -4121,6 +4966,7 @@

    Used by:

  • num-integer 0.1.46
  • num-traits 0.2.19
  • once_cell 1.21.4
  • +
  • openssl-probe 0.2.1
  • parking 2.2.1
  • parking_lot 0.12.5
  • parking_lot_core 0.9.12
  • @@ -4131,8 +4977,12 @@

    Used by:

  • pest_meta 2.8.6
  • pkg-config 0.3.33
  • regex-automata 0.4.14
  • +
  • regex-lite 0.1.9
  • regex-syntax 0.8.10
  • +
  • ring 0.17.14
  • ron 0.8.1
  • +
  • rustc_version 0.4.1
  • +
  • rustls-native-certs 0.8.4
  • rustls 0.23.46
  • scopeguard 1.2.0
  • signal-hook-registry 1.4.8
  • @@ -4148,6 +4998,7 @@

    Used by:

  • url 2.5.8
  • uuid 1.23.1
  • version_check 0.9.5
  • +
  • xmlparser 0.13.6
  • yasna 0.6.0
  •                               Apache License
    @@ -4992,6 +5843,7 @@ 

    Used by:

  • hmac 0.12.1
  • inout 0.1.4
  • md-5 0.10.6
  • +
  • sha1 0.10.6
  • sha2 0.10.9
  •                               Apache License
    @@ -6870,6 +7722,7 @@ 

    Used by:

    +
  • +
  • +

    ISC License

    +

    Used by:

    + +
    Copyright 2015-2025 Brian Smith.
    +
    +Permission to use, copy, modify, and/or distribute this software for any
    +purpose with or without fee is hereby granted, provided that the above
    +copyright notice and this permission notice appear in all copies.
    +
    +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
    +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
    +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
    +SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
    +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
    +OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
    +CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
     
  • @@ -7608,6 +8500,63 @@

    Used by:

    OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +
  • +
  • +

    MIT License

    +

    Used by:

    + +
    Copyright (c) 2018-2019 Sean McArthur
    +
    +Permission is hereby granted, free of charge, to any person obtaining a copy
    +of this software and associated documentation files (the "Software"), to deal
    +in the Software without restriction, including without limitation the rights
    +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
    +copies of the Software, and to permit persons to whom the Software is
    +furnished to do so, subject to the following conditions:
    +
    +The above copyright notice and this permission notice shall be included in
    +all copies or substantial portions of the Software.
    +
    +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
    +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
    +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
    +THE SOFTWARE.
    +
    +
    +
  • +
  • +

    MIT License

    +

    Used by:

    + +
    Copyright (c) 2018-2023 Sean McArthur
    +Copyright (c) 2016 Alex Crichton
    +
    +Permission is hereby granted, free of charge, to any person obtaining a copy
    +of this software and associated documentation files (the "Software"), to deal
    +in the Software without restriction, including without limitation the rights
    +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
    +copies of the Software, and to permit persons to whom the Software is
    +furnished to do so, subject to the following conditions:
    +
    +The above copyright notice and this permission notice shall be included in
    +all copies or substantial portions of the Software.
    +
    +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
    +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
    +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
    +THE SOFTWARE.
    +
     
  • @@ -7695,6 +8644,39 @@

    Used by:

    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +
  • +
  • +

    MIT License

    +

    Used by:

    + +
    Copyright (c) 2019 Hyper Contributors
    +
    +Permission is hereby granted, free of charge, to any
    +person obtaining a copy of this software and associated
    +documentation files (the "Software"), to deal in the
    +Software without restriction, including without
    +limitation the rights to use, copy, modify, merge,
    +publish, distribute, sublicense, and/or sell copies of
    +the Software, and to permit persons to whom the Software
    +is furnished to do so, subject to the following
    +conditions:
    +
    +The above copyright notice and this permission notice
    +shall be included in all copies or substantial portions
    +of the Software.
    +
    +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF
    +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED
    +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
    +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
    +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
    +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
    +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR
    +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
    +DEALINGS IN THE SOFTWARE.
     
  • @@ -8012,6 +8994,34 @@

    Used by:

    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +
  • +
  • +

    MIT License

    +

    Used by:

    + +
    MIT License
    +
    +Copyright (c) 2016 Jerome Froelich
    +
    +Permission is hereby granted, free of charge, to any person obtaining a copy
    +of this software and associated documentation files (the "Software"), to deal
    +in the Software without restriction, including without limitation the rights
    +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
    +copies of the Software, and to permit persons to whom the Software is
    +furnished to do so, subject to the following conditions:
    +
    +The above copyright notice and this permission notice shall be included in all
    +copies or substantial portions of the Software.
    +
    +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
    +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
    +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
    +SOFTWARE.
  • MIT License

    @@ -8226,7 +9236,37 @@

    Used by:

    MIT License

    Used by:

    +
    MIT License
    +
    +Copyright (c) 2022 Nugine
    +
    +Permission is hereby granted, free of charge, to any person obtaining a copy
    +of this software and associated documentation files (the "Software"), to deal
    +in the Software without restriction, including without limitation the rights
    +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
    +copies of the Software, and to permit persons to whom the Software is
    +furnished to do so, subject to the following conditions:
    +
    +The above copyright notice and this permission notice shall be included in all
    +copies or substantial portions of the Software.
    +
    +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
    +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
    +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
    +SOFTWARE.
    +
  • +
  • +

    MIT License

    +

    Used by:

    +
    MIT License
     
    @@ -8370,6 +9410,7 @@ 

    Used by:

    MIT License

    Used by:

    The MIT License (MIT)
    @@ -8646,6 +9687,34 @@ 

    Used by:

    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
    +
  • +
  • +

    MIT License

    +

    Used by:

    + +
    © 2016 Bertram Truong
    +© 2021 Kornel Lesiński
    +
    +Permission is hereby granted, free of charge, to any person obtaining a copy
    +of this software and associated documentation files (the "Software"), to deal
    +in the Software without restriction, including without limitation the rights
    +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
    +copies of the Software, and to permit persons to whom the Software is
    +furnished to do so, subject to the following conditions:
    +
    +The above copyright notice and this permission notice shall be included in
    +all copies or substantial portions of the Software.
    +
    +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
    +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
    +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
    +THE SOFTWARE.
    +
  • Unicode License v3

    @@ -8770,6 +9839,7 @@

    zlib License

    Used by:

    Copyright (c) 2024 Orson Peters
     
    diff --git a/SOFTWARE-LICENSE-NOTICES-MONGODB.html b/SOFTWARE-LICENSE-NOTICES-MONGODB.html
    index 6f1570169..7920a20c8 100644
    --- a/SOFTWARE-LICENSE-NOTICES-MONGODB.html
    +++ b/SOFTWARE-LICENSE-NOTICES-MONGODB.html
    @@ -49,14 +49,14 @@ 

    ExtendDB Software License Notices

    Overview of licenses:

    All license text:

    @@ -1534,6 +1534,215 @@

    Used by:

    See the License for the specific language governing permissions and limitations under the License. +
    +
  • +
  • +

    Apache License 2.0

    +

    Used by:

    + +
                                     Apache License
    +                           Version 2.0, January 2004
    +                        http://www.apache.org/licenses/
    +
    +   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
    +
    +   1. Definitions.
    +
    +      "License" shall mean the terms and conditions for use, reproduction,
    +      and distribution as defined by Sections 1 through 9 of this document.
    +
    +      "Licensor" shall mean the copyright owner or entity authorized by
    +      the copyright owner that is granting the License.
    +
    +      "Legal Entity" shall mean the union of the acting entity and all
    +      other entities that control, are controlled by, or are under common
    +      control with that entity. For the purposes of this definition,
    +      "control" means (i) the power, direct or indirect, to cause the
    +      direction or management of such entity, whether by contract or
    +      otherwise, or (ii) ownership of fifty percent (50%) or more of the
    +      outstanding shares, or (iii) beneficial ownership of such entity.
    +
    +      "You" (or "Your") shall mean an individual or Legal Entity
    +      exercising permissions granted by this License.
    +
    +      "Source" form shall mean the preferred form for making modifications,
    +      including but not limited to software source code, documentation
    +      source, and configuration files.
    +
    +      "Object" form shall mean any form resulting from mechanical
    +      transformation or translation of a Source form, including but
    +      not limited to compiled object code, generated documentation,
    +      and conversions to other media types.
    +
    +      "Work" shall mean the work of authorship, whether in Source or
    +      Object form, made available under the License, as indicated by a
    +      copyright notice that is included in or attached to the work
    +      (an example is provided in the Appendix below).
    +
    +      "Derivative Works" shall mean any work, whether in Source or Object
    +      form, that is based on (or derived from) the Work and for which the
    +      editorial revisions, annotations, elaborations, or other modifications
    +      represent, as a whole, an original work of authorship. For the purposes
    +      of this License, Derivative Works shall not include works that remain
    +      separable from, or merely link (or bind by name) to the interfaces of,
    +      the Work and Derivative Works thereof.
    +
    +      "Contribution" shall mean any work of authorship, including
    +      the original version of the Work and any modifications or additions
    +      to that Work or Derivative Works thereof, that is intentionally
    +      submitted to Licensor for inclusion in the Work by the copyright owner
    +      or by an individual or Legal Entity authorized to submit on behalf of
    +      the copyright owner. For the purposes of this definition, "submitted"
    +      means any form of electronic, verbal, or written communication sent
    +      to the Licensor or its representatives, including but not limited to
    +      communication on electronic mailing lists, source code control systems,
    +      and issue tracking systems that are managed by, or on behalf of, the
    +      Licensor for the purpose of discussing and improving the Work, but
    +      excluding communication that is conspicuously marked or otherwise
    +      designated in writing by the copyright owner as "Not a Contribution."
    +
    +      "Contributor" shall mean Licensor and any individual or Legal Entity
    +      on behalf of whom a Contribution has been received by Licensor and
    +      subsequently incorporated within the Work.
    +
    +   2. Grant of Copyright License. Subject to the terms and conditions of
    +      this License, each Contributor hereby grants to You a perpetual,
    +      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
    +      copyright license to reproduce, prepare Derivative Works of,
    +      publicly display, publicly perform, sublicense, and distribute the
    +      Work and such Derivative Works in Source or Object form.
    +
    +   3. Grant of Patent License. Subject to the terms and conditions of
    +      this License, each Contributor hereby grants to You a perpetual,
    +      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
    +      (except as stated in this section) patent license to make, have made,
    +      use, offer to sell, sell, import, and otherwise transfer the Work,
    +      where such license applies only to those patent claims licensable
    +      by such Contributor that are necessarily infringed by their
    +      Contribution(s) alone or by combination of their Contribution(s)
    +      with the Work to which such Contribution(s) was submitted. If You
    +      institute patent litigation against any entity (including a
    +      cross-claim or counterclaim in a lawsuit) alleging that the Work
    +      or a Contribution incorporated within the Work constitutes direct
    +      or contributory patent infringement, then any patent licenses
    +      granted to You under this License for that Work shall terminate
    +      as of the date such litigation is filed.
    +
    +   4. Redistribution. You may reproduce and distribute copies of the
    +      Work or Derivative Works thereof in any medium, with or without
    +      modifications, and in Source or Object form, provided that You
    +      meet the following conditions:
    +
    +      (a) You must give any other recipients of the Work or
    +          Derivative Works a copy of this License; and
    +
    +      (b) You must cause any modified files to carry prominent notices
    +          stating that You changed the files; and
    +
    +      (c) You must retain, in the Source form of any Derivative Works
    +          that You distribute, all copyright, patent, trademark, and
    +          attribution notices from the Source form of the Work,
    +          excluding those notices that do not pertain to any part of
    +          the Derivative Works; and
    +
    +      (d) If the Work includes a "NOTICE" text file as part of its
    +          distribution, then any Derivative Works that You distribute must
    +          include a readable copy of the attribution notices contained
    +          within such NOTICE file, excluding those notices that do not
    +          pertain to any part of the Derivative Works, in at least one
    +          of the following places: within a NOTICE text file distributed
    +          as part of the Derivative Works; within the Source form or
    +          documentation, if provided along with the Derivative Works; or,
    +          within a display generated by the Derivative Works, if and
    +          wherever such third-party notices normally appear. The contents
    +          of the NOTICE file are for informational purposes only and
    +          do not modify the License. You may add Your own attribution
    +          notices within Derivative Works that You distribute, alongside
    +          or as an addendum to the NOTICE text from the Work, provided
    +          that such additional attribution notices cannot be construed
    +          as modifying the License.
    +
    +      You may add Your own copyright statement to Your modifications and
    +      may provide additional or different license terms and conditions
    +      for use, reproduction, or distribution of Your modifications, or
    +      for any such Derivative Works as a whole, provided Your use,
    +      reproduction, and distribution of the Work otherwise complies with
    +      the conditions stated in this License.
    +
    +   5. Submission of Contributions. Unless You explicitly state otherwise,
    +      any Contribution intentionally submitted for inclusion in the Work
    +      by You to the Licensor shall be under the terms and conditions of
    +      this License, without any additional terms or conditions.
    +      Notwithstanding the above, nothing herein shall supersede or modify
    +      the terms of any separate license agreement you may have executed
    +      with Licensor regarding such Contributions.
    +
    +   6. Trademarks. This License does not grant permission to use the trade
    +      names, trademarks, service marks, or product names of the Licensor,
    +      except as required for reasonable and customary use in describing the
    +      origin of the Work and reproducing the content of the NOTICE file.
    +
    +   7. Disclaimer of Warranty. Unless required by applicable law or
    +      agreed to in writing, Licensor provides the Work (and each
    +      Contributor provides its Contributions) on an "AS IS" BASIS,
    +      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
    +      implied, including, without limitation, any warranties or conditions
    +      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
    +      PARTICULAR PURPOSE. You are solely responsible for determining the
    +      appropriateness of using or redistributing the Work and assume any
    +      risks associated with Your exercise of permissions under this License.
    +
    +   8. Limitation of Liability. In no event and under no legal theory,
    +      whether in tort (including negligence), contract, or otherwise,
    +      unless required by applicable law (such as deliberate and grossly
    +      negligent acts) or agreed to in writing, shall any Contributor be
    +      liable to You for damages, including any direct, indirect, special,
    +      incidental, or consequential damages of any character arising as a
    +      result of this License or out of the use or inability to use the
    +      Work (including but not limited to damages for loss of goodwill,
    +      work stoppage, computer failure or malfunction, or any and all
    +      other commercial damages or losses), even if such Contributor
    +      has been advised of the possibility of such damages.
    +
    +   9. Accepting Warranty or Additional Liability. While redistributing
    +      the Work or Derivative Works thereof, You may choose to offer,
    +      and charge a fee for, acceptance of support, warranty, indemnity,
    +      or other liability obligations and/or rights consistent with this
    +      License. However, in accepting such obligations, You may act only
    +      on Your own behalf and on Your sole responsibility, not on behalf
    +      of any other Contributor, and only if You agree to indemnify,
    +      defend, and hold each Contributor harmless for any liability
    +      incurred by, or claims asserted against, such Contributor by reason
    +      of your accepting any such warranty or additional liability.
    +
    +   END OF TERMS AND CONDITIONS
    +
    +   APPENDIX: How to apply the Apache License to your work.
    +
    +      To apply the Apache License to your work, attach the following
    +      boilerplate notice, with the fields enclosed by brackets "[]"
    +      replaced with your own identifying information. (Don't include
    +      the brackets!)  The text should be enclosed in the appropriate
    +      comment syntax for the file format. We also recommend that a
    +      file or class name and description of purpose be included on the
    +      same "printed page" as the copyright notice for easier
    +      identification within third-party archives.
    +
    +   Copyright [yyyy] [name of copyright owner]
    +
    +   Licensed under the Apache License, Version 2.0 (the "License");
    +   you may not use this file except in compliance with the License.
    +   You may obtain a copy of the License at
    +
    +       http://www.apache.org/licenses/LICENSE-2.0
    +
    +   Unless required by applicable law or agreed to in writing, software
    +   distributed under the License is distributed on an "AS IS" BASIS,
    +   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    +   See the License for the specific language governing permissions and
    +   limitations under the License.
     
  • @@ -2177,24 +2386,454 @@

    Used by:

    you may not use this file except in compliance with the License. You may obtain a copy of the License at - https://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +
  • +
  • +

    Apache License 2.0

    +

    Used by:

    + +
                                    Apache License
    +                           Version 2.0, January 2004
    +                        http://www.apache.org/licenses/
    +
    +   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
    +
    +   1. Definitions.
    +
    +      "License" shall mean the terms and conditions for use, reproduction,
    +      and distribution as defined by Sections 1 through 9 of this document.
    +
    +      "Licensor" shall mean the copyright owner or entity authorized by
    +      the copyright owner that is granting the License.
    +
    +      "Legal Entity" shall mean the union of the acting entity and all
    +      other entities that control, are controlled by, or are under common
    +      control with that entity. For the purposes of this definition,
    +      "control" means (i) the power, direct or indirect, to cause the
    +      direction or management of such entity, whether by contract or
    +      otherwise, or (ii) ownership of fifty percent (50%) or more of the
    +      outstanding shares, or (iii) beneficial ownership of such entity.
    +
    +      "You" (or "Your") shall mean an individual or Legal Entity
    +      exercising permissions granted by this License.
    +
    +      "Source" form shall mean the preferred form for making modifications,
    +      including but not limited to software source code, documentation
    +      source, and configuration files.
    +
    +      "Object" form shall mean any form resulting from mechanical
    +      transformation or translation of a Source form, including but
    +      not limited to compiled object code, generated documentation,
    +      and conversions to other media types.
    +
    +      "Work" shall mean the work of authorship, whether in Source or
    +      Object form, made available under the License, as indicated by a
    +      copyright notice that is included in or attached to the work
    +      (an example is provided in the Appendix below).
    +
    +      "Derivative Works" shall mean any work, whether in Source or Object
    +      form, that is based on (or derived from) the Work and for which the
    +      editorial revisions, annotations, elaborations, or other modifications
    +      represent, as a whole, an original work of authorship. For the purposes
    +      of this License, Derivative Works shall not include works that remain
    +      separable from, or merely link (or bind by name) to the interfaces of,
    +      the Work and Derivative Works thereof.
    +
    +      "Contribution" shall mean any work of authorship, including
    +      the original version of the Work and any modifications or additions
    +      to that Work or Derivative Works thereof, that is intentionally
    +      submitted to Licensor for inclusion in the Work by the copyright owner
    +      or by an individual or Legal Entity authorized to submit on behalf of
    +      the copyright owner. For the purposes of this definition, "submitted"
    +      means any form of electronic, verbal, or written communication sent
    +      to the Licensor or its representatives, including but not limited to
    +      communication on electronic mailing lists, source code control systems,
    +      and issue tracking systems that are managed by, or on behalf of, the
    +      Licensor for the purpose of discussing and improving the Work, but
    +      excluding communication that is conspicuously marked or otherwise
    +      designated in writing by the copyright owner as "Not a Contribution."
    +
    +      "Contributor" shall mean Licensor and any individual or Legal Entity
    +      on behalf of whom a Contribution has been received by Licensor and
    +      subsequently incorporated within the Work.
    +
    +   2. Grant of Copyright License. Subject to the terms and conditions of
    +      this License, each Contributor hereby grants to You a perpetual,
    +      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
    +      copyright license to reproduce, prepare Derivative Works of,
    +      publicly display, publicly perform, sublicense, and distribute the
    +      Work and such Derivative Works in Source or Object form.
    +
    +   3. Grant of Patent License. Subject to the terms and conditions of
    +      this License, each Contributor hereby grants to You a perpetual,
    +      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
    +      (except as stated in this section) patent license to make, have made,
    +      use, offer to sell, sell, import, and otherwise transfer the Work,
    +      where such license applies only to those patent claims licensable
    +      by such Contributor that are necessarily infringed by their
    +      Contribution(s) alone or by combination of their Contribution(s)
    +      with the Work to which such Contribution(s) was submitted. If You
    +      institute patent litigation against any entity (including a
    +      cross-claim or counterclaim in a lawsuit) alleging that the Work
    +      or a Contribution incorporated within the Work constitutes direct
    +      or contributory patent infringement, then any patent licenses
    +      granted to You under this License for that Work shall terminate
    +      as of the date such litigation is filed.
    +
    +   4. Redistribution. You may reproduce and distribute copies of the
    +      Work or Derivative Works thereof in any medium, with or without
    +      modifications, and in Source or Object form, provided that You
    +      meet the following conditions:
    +
    +      (a) You must give any other recipients of the Work or
    +          Derivative Works a copy of this License; and
    +
    +      (b) You must cause any modified files to carry prominent notices
    +          stating that You changed the files; and
    +
    +      (c) You must retain, in the Source form of any Derivative Works
    +          that You distribute, all copyright, patent, trademark, and
    +          attribution notices from the Source form of the Work,
    +          excluding those notices that do not pertain to any part of
    +          the Derivative Works; and
    +
    +      (d) If the Work includes a "NOTICE" text file as part of its
    +          distribution, then any Derivative Works that You distribute must
    +          include a readable copy of the attribution notices contained
    +          within such NOTICE file, excluding those notices that do not
    +          pertain to any part of the Derivative Works, in at least one
    +          of the following places: within a NOTICE text file distributed
    +          as part of the Derivative Works; within the Source form or
    +          documentation, if provided along with the Derivative Works; or,
    +          within a display generated by the Derivative Works, if and
    +          wherever such third-party notices normally appear. The contents
    +          of the NOTICE file are for informational purposes only and
    +          do not modify the License. You may add Your own attribution
    +          notices within Derivative Works that You distribute, alongside
    +          or as an addendum to the NOTICE text from the Work, provided
    +          that such additional attribution notices cannot be construed
    +          as modifying the License.
    +
    +      You may add Your own copyright statement to Your modifications and
    +      may provide additional or different license terms and conditions
    +      for use, reproduction, or distribution of Your modifications, or
    +      for any such Derivative Works as a whole, provided Your use,
    +      reproduction, and distribution of the Work otherwise complies with
    +      the conditions stated in this License.
    +
    +   5. Submission of Contributions. Unless You explicitly state otherwise,
    +      any Contribution intentionally submitted for inclusion in the Work
    +      by You to the Licensor shall be under the terms and conditions of
    +      this License, without any additional terms or conditions.
    +      Notwithstanding the above, nothing herein shall supersede or modify
    +      the terms of any separate license agreement you may have executed
    +      with Licensor regarding such Contributions.
    +
    +   6. Trademarks. This License does not grant permission to use the trade
    +      names, trademarks, service marks, or product names of the Licensor,
    +      except as required for reasonable and customary use in describing the
    +      origin of the Work and reproducing the content of the NOTICE file.
    +
    +   7. Disclaimer of Warranty. Unless required by applicable law or
    +      agreed to in writing, Licensor provides the Work (and each
    +      Contributor provides its Contributions) on an "AS IS" BASIS,
    +      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
    +      implied, including, without limitation, any warranties or conditions
    +      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
    +      PARTICULAR PURPOSE. You are solely responsible for determining the
    +      appropriateness of using or redistributing the Work and assume any
    +      risks associated with Your exercise of permissions under this License.
    +
    +   8. Limitation of Liability. In no event and under no legal theory,
    +      whether in tort (including negligence), contract, or otherwise,
    +      unless required by applicable law (such as deliberate and grossly
    +      negligent acts) or agreed to in writing, shall any Contributor be
    +      liable to You for damages, including any direct, indirect, special,
    +      incidental, or consequential damages of any character arising as a
    +      result of this License or out of the use or inability to use the
    +      Work (including but not limited to damages for loss of goodwill,
    +      work stoppage, computer failure or malfunction, or any and all
    +      other commercial damages or losses), even if such Contributor
    +      has been advised of the possibility of such damages.
    +
    +   9. Accepting Warranty or Additional Liability. While redistributing
    +      the Work or Derivative Works thereof, You may choose to offer,
    +      and charge a fee for, acceptance of support, warranty, indemnity,
    +      or other liability obligations and/or rights consistent with this
    +      License. However, in accepting such obligations, You may act only
    +      on Your own behalf and on Your sole responsibility, not on behalf
    +      of any other Contributor, and only if You agree to indemnify,
    +      defend, and hold each Contributor harmless for any liability
    +      incurred by, or claims asserted against, such Contributor by reason
    +      of your accepting any such warranty or additional liability.
    +
    +   END OF TERMS AND CONDITIONS
    +
    +   APPENDIX: How to apply the Apache License to your work.
    +
    +      To apply the Apache License to your work, attach the following
    +      boilerplate notice, with the fields enclosed by brackets "{}"
    +      replaced with your own identifying information. (Don't include
    +      the brackets!)  The text should be enclosed in the appropriate
    +      comment syntax for the file format. We also recommend that a
    +      file or class name and description of purpose be included on the
    +      same "printed page" as the copyright notice for easier
    +      identification within third-party archives.
    +
    +   Copyright 2018-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
    +
    +   Licensed under the Apache License, Version 2.0 (the "License");
    +   you may not use this file except in compliance with the License.
    +   You may obtain a copy of the License at
    +
    +       http://www.apache.org/licenses/LICENSE-2.0
    +
    +   Unless required by applicable law or agreed to in writing, software
    +   distributed under the License is distributed on an "AS IS" BASIS,
    +   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    +   See the License for the specific language governing permissions and
    +   limitations under the License.
    +
    +
    +
  • +
  • +

    Apache License 2.0

    +

    Used by:

    + +
                                  Apache License
    +                        Version 2.0 January 2004
    +                     http://www.apache.org/licenses/
    +
    +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
    +
    +1. Definitions.
    +
    +   "License" shall mean the terms and conditions for use, reproduction,
    +   and distribution as defined by Sections 1 through 9 of this document.
    +
    +   "Licensor" shall mean the copyright owner or entity authorized by
    +   the copyright owner that is granting the License.
    +
    +   "Legal Entity" shall mean the union of the acting entity and all
    +   other entities that control, are controlled by, or are under common
    +   control with that entity. For the purposes of this definition,
    +   "control" means (i) the power, direct or indirect, to cause the
    +   direction or management of such entity, whether by contract or
    +   otherwise, or (ii) ownership of fifty percent (50%) or more of the
    +   outstanding shares, or (iii) beneficial ownership of such entity.
    +
    +   "You" (or "Your") shall mean an individual or Legal Entity
    +   exercising permissions granted by this License.
    +
    +   "Source" form shall mean the preferred form for making modifications,
    +   including but not limited to software source code, documentation
    +   source, and configuration files.
    +
    +   "Object" form shall mean any form resulting from mechanical
    +   transformation or translation of a Source form, including but
    +   not limited to compiled object code, generated documentation,
    +   and conversions to other media types.
    +
    +   "Work" shall mean the work of authorship, whether in Source or
    +   Object form, made available under the License, as indicated by a
    +   copyright notice that is included in or attached to the work
    +   (an example is provided in the Appendix below).
    +
    +   "Derivative Works" shall mean any work, whether in Source or Object
    +   form, that is based on (or derived from) the Work and for which the
    +   editorial revisions, annotations, elaborations, or other modifications
    +   represent, as a whole, an original work of authorship. For the purposes
    +   of this License, Derivative Works shall not include works that remain
    +   separable from, or merely link (or bind by name) to the interfaces of,
    +   the Work and Derivative Works thereof.
    +
    +   "Contribution" shall mean any work of authorship, including
    +   the original version of the Work and any modifications or additions
    +   to that Work or Derivative Works thereof, that is intentionally
    +   submitted to Licensor for inclusion in the Work by the copyright owner
    +   or by an individual or Legal Entity authorized to submit on behalf of
    +   the copyright owner. For the purposes of this definition, "submitted"
    +   means any form of electronic, verbal, or written communication sent
    +   to the Licensor or its representatives, including but not limited to
    +   communication on electronic mailing lists, source code control systems,
    +   and issue tracking systems that are managed by, or on behalf of, the
    +   Licensor for the purpose of discussing and improving the Work, but
    +   excluding communication that is conspicuously marked or otherwise
    +   designated in writing by the copyright owner as "Not a Contribution."
    +
    +   "Contributor" shall mean Licensor and any individual or Legal Entity
    +   on behalf of whom a Contribution has been received by Licensor and
    +   subsequently incorporated within the Work.
    +
    +2. Grant of Copyright License. Subject to the terms and conditions of
    +   this License, each Contributor hereby grants to You a perpetual,
    +   worldwide, non-exclusive, no-charge, royalty-free, irrevocable
    +   copyright license to reproduce, prepare Derivative Works of,
    +   publicly display, publicly perform, sublicense, and distribute the
    +   Work and such Derivative Works in Source or Object form.
    +
    +3. Grant of Patent License. Subject to the terms and conditions of
    +   this License, each Contributor hereby grants to You a perpetual,
    +   worldwide, non-exclusive, no-charge, royalty-free, irrevocable
    +   (except as stated in this section) patent license to make, have made,
    +   use, offer to sell, sell, import, and otherwise transfer the Work,
    +   where such license applies only to those patent claims licensable
    +   by such Contributor that are necessarily infringed by their
    +   Contribution(s) alone or by combination of their Contribution(s)
    +   with the Work to which such Contribution(s) was submitted. If You
    +   institute patent litigation against any entity (including a
    +   cross-claim or counterclaim in a lawsuit) alleging that the Work
    +   or a Contribution incorporated within the Work constitutes direct
    +   or contributory patent infringement, then any patent licenses
    +   granted to You under this License for that Work shall terminate
    +   as of the date such litigation is filed.
    +
    +4. Redistribution. You may reproduce and distribute copies of the
    +   Work or Derivative Works thereof in any medium, with or without
    +   modifications, and in Source or Object form, provided that You
    +   meet the following conditions:
    +
    +   (a) You must give any other recipients of the Work or
    +       Derivative Works a copy of this License; and
    +
    +   (b) You must cause any modified files to carry prominent notices
    +       stating that You changed the files; and
    +
    +   (c) You must retain, in the Source form of any Derivative Works
    +       that You distribute, all copyright, patent, trademark, and
    +       attribution notices from the Source form of the Work,
    +       excluding those notices that do not pertain to any part of
    +       the Derivative Works; and
    +
    +   (d) If the Work includes a "NOTICE" text file as part of its
    +       distribution, then any Derivative Works that You distribute must
    +       include a readable copy of the attribution notices contained
    +       within such NOTICE file, excluding those notices that do not
    +       pertain to any part of the Derivative Works, in at least one
    +       of the following places: within a NOTICE text file distributed
    +       as part of the Derivative Works; within the Source form or
    +       documentation, if provided along with the Derivative Works; or,
    +       within a display generated by the Derivative Works, if and
    +       wherever such third-party notices normally appear. The contents
    +       of the NOTICE file are for informational purposes only and
    +       do not modify the License. You may add Your own attribution
    +       notices within Derivative Works that You distribute, alongside
    +       or as an addendum to the NOTICE text from the Work, provided
    +       that such additional attribution notices cannot be construed
    +       as modifying the License.
    +
    +   You may add Your own copyright statement to Your modifications and
    +   may provide additional or different license terms and conditions
    +   for use, reproduction, or distribution of Your modifications, or
    +   for any such Derivative Works as a whole, provided Your use,
    +   reproduction, and distribution of the Work otherwise complies with
    +   the conditions stated in this License.
    +
    +5. Submission of Contributions. Unless You explicitly state otherwise,
    +   any Contribution intentionally submitted for inclusion in the Work
    +   by You to the Licensor shall be under the terms and conditions of
    +   this License, without any additional terms or conditions.
    +   Notwithstanding the above, nothing herein shall supersede or modify
    +   the terms of any separate license agreement you may have executed
    +   with Licensor regarding such Contributions.
    +
    +6. Trademarks. This License does not grant permission to use the trade
    +   names, trademarks, service marks, or product names of the Licensor,
    +   except as required for reasonable and customary use in describing the
    +   origin of the Work and reproducing the content of the NOTICE file.
    +
    +7. Disclaimer of Warranty. Unless required by applicable law or
    +   agreed to in writing, Licensor provides the Work (and each
    +   Contributor provides its Contributions) on an "AS IS" BASIS,
    +   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
    +   implied, including, without limitation, any warranties or conditions
    +   of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
    +   PARTICULAR PURPOSE. You are solely responsible for determining the
    +   appropriateness of using or redistributing the Work and assume any
    +   risks associated with Your exercise of permissions under this License.
    +
    +8. Limitation of Liability. In no event and under no legal theory,
    +   whether in tort (including negligence), contract, or otherwise,
    +   unless required by applicable law (such as deliberate and grossly
    +   negligent acts) or agreed to in writing, shall any Contributor be
    +   liable to You for damages, including any direct, indirect, special,
    +   incidental, or consequential damages of any character arising as a
    +   result of this License or out of the use or inability to use the
    +   Work (including but not limited to damages for loss of goodwill,
    +   work stoppage, computer failure or malfunction, or any and all
    +   other commercial damages or losses), even if such Contributor
    +   has been advised of the possibility of such damages.
    +
    +9. Accepting Warranty or Additional Liability. While redistributing
    +   the Work or Derivative Works thereof, You may choose to offer,
    +   and charge a fee for, acceptance of support, warranty, indemnity,
    +   or other liability obligations and/or rights consistent with this
    +   License. However, in accepting such obligations, You may act only
    +   on Your own behalf and on Your sole responsibility, not on behalf
    +   of any other Contributor, and only if You agree to indemnify,
    +   defend, and hold each Contributor harmless for any liability
    +   incurred by, or claims asserted against, such Contributor by reason
    +   of your accepting any such warranty or additional liability.
    +
    +END OF TERMS AND CONDITIONS
    +
    +APPENDIX: How to apply the Apache License to your work.
    +
    +   To apply the Apache License to your work, attach the following
    +   boilerplate notice, with the fields enclosed by brackets "[]"
    +   replaced with your own identifying information. (Don't include
    +   the brackets!)  The text should be enclosed in the appropriate
    +   comment syntax for the file format. We also recommend that a
    +   file or class name and description of purpose be included on the
    +   same "printed page" as the copyright notice for easier
    +   identification within third-party archives.
    +
    +Copyright [yyyy] [name of copyright owner]
     
    -   Unless required by applicable law or agreed to in writing, software
    -   distributed under the License is distributed on an "AS IS" BASIS,
    -   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    -   See the License for the specific language governing permissions and
    -   limitations under the License.
    +Licensed under the Apache License, Version 2.0 (the "License");
    +you may not use this file except in compliance with the License.
    +You may obtain a copy of the License at
    +
    +	http://www.apache.org/licenses/LICENSE-2.0
     
    +Unless required by applicable law or agreed to in writing, software
    +distributed under the License is distributed on an "AS IS" BASIS,
    +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    +See the License for the specific language governing permissions and
    +limitations under the License.
     
  • Apache License 2.0

    Used by:

                                  Apache License
    -                        Version 2.0 January 2004
    +                        Version 2.0, January 2004
                          http://www.apache.org/licenses/
     
     TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
    @@ -2381,7 +3020,8 @@ 

    Used by:

    same "printed page" as the copyright notice for easier identification within third-party archives. -Copyright [yyyy] [name of copyright owner] +Copyright (c) 2016 Alex Crichton +Copyright (c) 2017 The Tokio Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -2400,15 +3040,7 @@

    Used by:

    Apache License 2.0

    Used by:

                                  Apache License
                             Version 2.0, January 2004
    @@ -2598,8 +3230,7 @@ 

    Used by:

    same "printed page" as the copyright notice for easier identification within third-party archives. -Copyright (c) 2016 Alex Crichton -Copyright (c) 2017 The Tokio Authors +Copyright (c) 2019 Matthias Einwag Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -2618,7 +3249,7 @@

    Used by:

    Apache License 2.0

    Used by:

                                  Apache License
                             Version 2.0, January 2004
    @@ -2808,7 +3439,7 @@ 

    Used by:

    same "printed page" as the copyright notice for easier identification within third-party archives. -Copyright (c) 2019 Matthias Einwag +Copyright 2014 Paho Lurie-Gregg Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -2820,14 +3451,13 @@

    Used by:

    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and -limitations under the License. -
    +limitations under the License.
  • Apache License 2.0

    Used by:

                                  Apache License
                             Version 2.0, January 2004
    @@ -3017,7 +3647,7 @@ 

    Used by:

    same "printed page" as the copyright notice for easier identification within third-party archives. -Copyright 2014 Paho Lurie-Gregg +Copyright 2017 The Rust Project Developers Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -3029,13 +3659,15 @@

    Used by:

    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and -limitations under the License.
    +limitations under the License. +
  • Apache License 2.0

    Used by:

                                  Apache License
                             Version 2.0, January 2004
    @@ -3225,7 +3857,7 @@ 

    Used by:

    same "printed page" as the copyright notice for easier identification within third-party archives. -Copyright 2017 The Rust Project Developers +Copyright 2017 http-rs authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -3244,7 +3876,7 @@

    Used by:

    Apache License 2.0

    Used by:

                                  Apache License
                             Version 2.0, January 2004
    @@ -3434,7 +4066,7 @@ 

    Used by:

    same "printed page" as the copyright notice for easier identification within third-party archives. -Copyright 2017 http-rs authors +Copyright 2017 quininer kel Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -3453,7 +4085,7 @@

    Used by:

    Apache License 2.0

    Used by:

                                  Apache License
                             Version 2.0, January 2004
    @@ -3643,7 +4275,7 @@ 

    Used by:

    same "printed page" as the copyright notice for easier identification within third-party archives. -Copyright 2017 quininer kel +Copyright 2018 The pin-utils authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -4509,6 +5141,7 @@

    Used by:

  • base64 0.21.7
  • base64 0.22.1
  • bitflags 2.11.1
  • +
  • bytes-utils 0.1.4
  • cc 1.2.62
  • cfg-if 1.0.4
  • cmake 0.1.58
  • @@ -4528,15 +5161,18 @@

    Used by:

  • errno 0.3.14
  • event-listener-strategy 0.5.4
  • event-listener 5.4.2
  • +
  • fastrand 2.5.0
  • find-msvc-tools 0.1.9
  • flate2 1.1.9
  • fnv 1.0.7
  • form_urlencoded 1.2.2
  • hashbrown 0.14.5
  • hashbrown 0.15.5
  • +
  • hashbrown 0.16.1
  • hashbrown 0.17.1
  • heck 0.5.0
  • httparse 1.10.1
  • +
  • hyper-rustls 0.27.10
  • idna 1.1.0
  • idna_adapter 1.2.2
  • indexmap 2.14.0
  • @@ -4549,6 +5185,7 @@

    Used by:

  • num-integer 0.1.46
  • num-traits 0.2.19
  • once_cell 1.21.4
  • +
  • openssl-probe 0.2.1
  • parking 2.2.1
  • parking_lot 0.12.5
  • parking_lot_core 0.9.12
  • @@ -4559,10 +5196,12 @@

    Used by:

  • pest_meta 2.8.6
  • pkg-config 0.3.33
  • regex-automata 0.4.14
  • +
  • regex-lite 0.1.9
  • regex-syntax 0.8.10
  • ring 0.17.14
  • ron 0.8.1
  • rustc_version 0.4.1
  • +
  • rustls-native-certs 0.8.4
  • rustls 0.23.46
  • scopeguard 1.2.0
  • serde_with 3.21.0
  • @@ -4583,6 +5222,7 @@

    Used by:

  • url 2.5.8
  • uuid 1.23.1
  • version_check 0.9.5
  • +
  • xmlparser 0.13.6
  • yasna 0.6.0
  •                               Apache License
    @@ -5437,6 +6077,7 @@ 

    Used by:

  • md-5 0.10.6
  • md-5 0.11.0
  • pbkdf2 0.13.0
  • +
  • sha1 0.10.6
  • sha1 0.11.0
  • sha2 0.10.9
  • sha2 0.11.0
  • @@ -6899,6 +7540,7 @@

    Used by:

    +
  • +
  • +

    MIT License

    +

    Used by:

    + +
    Copyright (c) 2018-2019 Sean McArthur
    +
    +Permission is hereby granted, free of charge, to any person obtaining a copy
    +of this software and associated documentation files (the "Software"), to deal
    +in the Software without restriction, including without limitation the rights
    +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
    +copies of the Software, and to permit persons to whom the Software is
    +furnished to do so, subject to the following conditions:
    +
    +The above copyright notice and this permission notice shall be included in
    +all copies or substantial portions of the Software.
    +
    +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
    +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
    +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
    +THE SOFTWARE.
    +
    +
    +
  • +
  • +

    MIT License

    +

    Used by:

    + +
    Copyright (c) 2018-2023 Sean McArthur
    +Copyright (c) 2016 Alex Crichton
    +
    +Permission is hereby granted, free of charge, to any person obtaining a copy
    +of this software and associated documentation files (the "Software"), to deal
    +in the Software without restriction, including without limitation the rights
    +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
    +copies of the Software, and to permit persons to whom the Software is
    +furnished to do so, subject to the following conditions:
    +
    +The above copyright notice and this permission notice shall be included in
    +all copies or substantial portions of the Software.
    +
    +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
    +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
    +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
    +THE SOFTWARE.
    +
     
  • @@ -7724,6 +8440,39 @@

    Used by:

    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +
  • +
  • +

    MIT License

    +

    Used by:

    + +
    Copyright (c) 2019 Hyper Contributors
    +
    +Permission is hereby granted, free of charge, to any
    +person obtaining a copy of this software and associated
    +documentation files (the "Software"), to deal in the
    +Software without restriction, including without
    +limitation the rights to use, copy, modify, merge,
    +publish, distribute, sublicense, and/or sell copies of
    +the Software, and to permit persons to whom the Software
    +is furnished to do so, subject to the following
    +conditions:
    +
    +The above copyright notice and this permission notice
    +shall be included in all copies or substantial portions
    +of the Software.
    +
    +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF
    +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED
    +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
    +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
    +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
    +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
    +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR
    +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
    +DEALINGS IN THE SOFTWARE.
     
  • @@ -8068,6 +8817,34 @@

    Used by:

    DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +
  • +
  • +

    MIT License

    +

    Used by:

    + +
    MIT License
    +
    +Copyright (c) 2016 Jerome Froelich
    +
    +Permission is hereby granted, free of charge, to any person obtaining a copy
    +of this software and associated documentation files (the "Software"), to deal
    +in the Software without restriction, including without limitation the rights
    +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
    +copies of the Software, and to permit persons to whom the Software is
    +furnished to do so, subject to the following conditions:
    +
    +The above copyright notice and this permission notice shall be included in all
    +copies or substantial portions of the Software.
    +
    +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
    +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
    +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
    +SOFTWARE.
  • MIT License

    @@ -8454,6 +9231,34 @@

    Used by:

    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +
  • +
  • +

    MIT License

    +

    Used by:

    + +
    MIT License
    +
    +Copyright (c) 2022 Nugine
    +
    +Permission is hereby granted, free of charge, to any person obtaining a copy
    +of this software and associated documentation files (the "Software"), to deal
    +in the Software without restriction, including without limitation the rights
    +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
    +copies of the Software, and to permit persons to whom the Software is
    +furnished to do so, subject to the following conditions:
    +
    +The above copyright notice and this permission notice shall be included in all
    +copies or substantial portions of the Software.
    +
    +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
    +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
    +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
    +SOFTWARE.
  • MIT License

    @@ -8487,7 +9292,9 @@

    Used by:

    MIT License

    Used by:

    MIT License
     
    @@ -8626,6 +9433,34 @@ 

    Used by:

    OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
    +
  • +
  • +

    MIT License

    +

    Used by:

    + +
    The MIT License (MIT)
    +
    +Copyright (c) 2014 Mathijs van de Nes
    +
    +Permission is hereby granted, free of charge, to any person obtaining a copy
    +of this software and associated documentation files (the "Software"), to deal
    +in the Software without restriction, including without limitation the rights
    +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
    +copies of the Software, and to permit persons to whom the Software is
    +furnished to do so, subject to the following conditions:
    +
    +The above copyright notice and this permission notice shall be included in all
    +copies or substantial portions of the Software.
    +
    +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
    +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
    +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
    +SOFTWARE.
  • MIT License

    @@ -9027,6 +9862,34 @@

    Used by:

    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +
  • +
  • +

    MIT License

    +

    Used by:

    + +
    © 2016 Bertram Truong
    +© 2021 Kornel Lesiński
    +
    +Permission is hereby granted, free of charge, to any person obtaining a copy
    +of this software and associated documentation files (the "Software"), to deal
    +in the Software without restriction, including without limitation the rights
    +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
    +copies of the Software, and to permit persons to whom the Software is
    +furnished to do so, subject to the following conditions:
    +
    +The above copyright notice and this permission notice shall be included in
    +all copies or substantial portions of the Software.
    +
    +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
    +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
    +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
    +THE SOFTWARE.
     
  • @@ -9152,6 +10015,7 @@

    zlib License

    Used by:

    Copyright (c) 2024 Orson Peters
     
    diff --git a/SOFTWARE-LICENSE-NOTICES.html b/SOFTWARE-LICENSE-NOTICES.html
    index 47ee4fe1c..9103a0534 100644
    --- a/SOFTWARE-LICENSE-NOTICES.html
    +++ b/SOFTWARE-LICENSE-NOTICES.html
    @@ -49,14 +49,14 @@ 

    ExtendDB Software License Notices

    Overview of licenses:

    All license text:

    @@ -1538,25 +1538,216 @@

    Used by:

    Apache License 2.0

    Used by:

    +
                                     Apache License
    +                           Version 2.0, January 2004
    +                        http://www.apache.org/licenses/
    +
    +   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
    +
    +   1. Definitions.
    +
    +      "License" shall mean the terms and conditions for use, reproduction,
    +      and distribution as defined by Sections 1 through 9 of this document.
    +
    +      "Licensor" shall mean the copyright owner or entity authorized by
    +      the copyright owner that is granting the License.
    +
    +      "Legal Entity" shall mean the union of the acting entity and all
    +      other entities that control, are controlled by, or are under common
    +      control with that entity. For the purposes of this definition,
    +      "control" means (i) the power, direct or indirect, to cause the
    +      direction or management of such entity, whether by contract or
    +      otherwise, or (ii) ownership of fifty percent (50%) or more of the
    +      outstanding shares, or (iii) beneficial ownership of such entity.
    +
    +      "You" (or "Your") shall mean an individual or Legal Entity
    +      exercising permissions granted by this License.
    +
    +      "Source" form shall mean the preferred form for making modifications,
    +      including but not limited to software source code, documentation
    +      source, and configuration files.
    +
    +      "Object" form shall mean any form resulting from mechanical
    +      transformation or translation of a Source form, including but
    +      not limited to compiled object code, generated documentation,
    +      and conversions to other media types.
    +
    +      "Work" shall mean the work of authorship, whether in Source or
    +      Object form, made available under the License, as indicated by a
    +      copyright notice that is included in or attached to the work
    +      (an example is provided in the Appendix below).
    +
    +      "Derivative Works" shall mean any work, whether in Source or Object
    +      form, that is based on (or derived from) the Work and for which the
    +      editorial revisions, annotations, elaborations, or other modifications
    +      represent, as a whole, an original work of authorship. For the purposes
    +      of this License, Derivative Works shall not include works that remain
    +      separable from, or merely link (or bind by name) to the interfaces of,
    +      the Work and Derivative Works thereof.
    +
    +      "Contribution" shall mean any work of authorship, including
    +      the original version of the Work and any modifications or additions
    +      to that Work or Derivative Works thereof, that is intentionally
    +      submitted to Licensor for inclusion in the Work by the copyright owner
    +      or by an individual or Legal Entity authorized to submit on behalf of
    +      the copyright owner. For the purposes of this definition, "submitted"
    +      means any form of electronic, verbal, or written communication sent
    +      to the Licensor or its representatives, including but not limited to
    +      communication on electronic mailing lists, source code control systems,
    +      and issue tracking systems that are managed by, or on behalf of, the
    +      Licensor for the purpose of discussing and improving the Work, but
    +      excluding communication that is conspicuously marked or otherwise
    +      designated in writing by the copyright owner as "Not a Contribution."
    +
    +      "Contributor" shall mean Licensor and any individual or Legal Entity
    +      on behalf of whom a Contribution has been received by Licensor and
    +      subsequently incorporated within the Work.
    +
    +   2. Grant of Copyright License. Subject to the terms and conditions of
    +      this License, each Contributor hereby grants to You a perpetual,
    +      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
    +      copyright license to reproduce, prepare Derivative Works of,
    +      publicly display, publicly perform, sublicense, and distribute the
    +      Work and such Derivative Works in Source or Object form.
    +
    +   3. Grant of Patent License. Subject to the terms and conditions of
    +      this License, each Contributor hereby grants to You a perpetual,
    +      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
    +      (except as stated in this section) patent license to make, have made,
    +      use, offer to sell, sell, import, and otherwise transfer the Work,
    +      where such license applies only to those patent claims licensable
    +      by such Contributor that are necessarily infringed by their
    +      Contribution(s) alone or by combination of their Contribution(s)
    +      with the Work to which such Contribution(s) was submitted. If You
    +      institute patent litigation against any entity (including a
    +      cross-claim or counterclaim in a lawsuit) alleging that the Work
    +      or a Contribution incorporated within the Work constitutes direct
    +      or contributory patent infringement, then any patent licenses
    +      granted to You under this License for that Work shall terminate
    +      as of the date such litigation is filed.
    +
    +   4. Redistribution. You may reproduce and distribute copies of the
    +      Work or Derivative Works thereof in any medium, with or without
    +      modifications, and in Source or Object form, provided that You
    +      meet the following conditions:
    +
    +      (a) You must give any other recipients of the Work or
    +          Derivative Works a copy of this License; and
    +
    +      (b) You must cause any modified files to carry prominent notices
    +          stating that You changed the files; and
    +
    +      (c) You must retain, in the Source form of any Derivative Works
    +          that You distribute, all copyright, patent, trademark, and
    +          attribution notices from the Source form of the Work,
    +          excluding those notices that do not pertain to any part of
    +          the Derivative Works; and
    +
    +      (d) If the Work includes a "NOTICE" text file as part of its
    +          distribution, then any Derivative Works that You distribute must
    +          include a readable copy of the attribution notices contained
    +          within such NOTICE file, excluding those notices that do not
    +          pertain to any part of the Derivative Works, in at least one
    +          of the following places: within a NOTICE text file distributed
    +          as part of the Derivative Works; within the Source form or
    +          documentation, if provided along with the Derivative Works; or,
    +          within a display generated by the Derivative Works, if and
    +          wherever such third-party notices normally appear. The contents
    +          of the NOTICE file are for informational purposes only and
    +          do not modify the License. You may add Your own attribution
    +          notices within Derivative Works that You distribute, alongside
    +          or as an addendum to the NOTICE text from the Work, provided
    +          that such additional attribution notices cannot be construed
    +          as modifying the License.
    +
    +      You may add Your own copyright statement to Your modifications and
    +      may provide additional or different license terms and conditions
    +      for use, reproduction, or distribution of Your modifications, or
    +      for any such Derivative Works as a whole, provided Your use,
    +      reproduction, and distribution of the Work otherwise complies with
    +      the conditions stated in this License.
    +
    +   5. Submission of Contributions. Unless You explicitly state otherwise,
    +      any Contribution intentionally submitted for inclusion in the Work
    +      by You to the Licensor shall be under the terms and conditions of
    +      this License, without any additional terms or conditions.
    +      Notwithstanding the above, nothing herein shall supersede or modify
    +      the terms of any separate license agreement you may have executed
    +      with Licensor regarding such Contributions.
    +
    +   6. Trademarks. This License does not grant permission to use the trade
    +      names, trademarks, service marks, or product names of the Licensor,
    +      except as required for reasonable and customary use in describing the
    +      origin of the Work and reproducing the content of the NOTICE file.
    +
    +   7. Disclaimer of Warranty. Unless required by applicable law or
    +      agreed to in writing, Licensor provides the Work (and each
    +      Contributor provides its Contributions) on an "AS IS" BASIS,
    +      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
    +      implied, including, without limitation, any warranties or conditions
    +      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
    +      PARTICULAR PURPOSE. You are solely responsible for determining the
    +      appropriateness of using or redistributing the Work and assume any
    +      risks associated with Your exercise of permissions under this License.
    +
    +   8. Limitation of Liability. In no event and under no legal theory,
    +      whether in tort (including negligence), contract, or otherwise,
    +      unless required by applicable law (such as deliberate and grossly
    +      negligent acts) or agreed to in writing, shall any Contributor be
    +      liable to You for damages, including any direct, indirect, special,
    +      incidental, or consequential damages of any character arising as a
    +      result of this License or out of the use or inability to use the
    +      Work (including but not limited to damages for loss of goodwill,
    +      work stoppage, computer failure or malfunction, or any and all
    +      other commercial damages or losses), even if such Contributor
    +      has been advised of the possibility of such damages.
    +
    +   9. Accepting Warranty or Additional Liability. While redistributing
    +      the Work or Derivative Works thereof, You may choose to offer,
    +      and charge a fee for, acceptance of support, warranty, indemnity,
    +      or other liability obligations and/or rights consistent with this
    +      License. However, in accepting such obligations, You may act only
    +      on Your own behalf and on Your sole responsibility, not on behalf
    +      of any other Contributor, and only if You agree to indemnify,
    +      defend, and hold each Contributor harmless for any liability
    +      incurred by, or claims asserted against, such Contributor by reason
    +      of your accepting any such warranty or additional liability.
    +
    +   END OF TERMS AND CONDITIONS
    +
    +   APPENDIX: How to apply the Apache License to your work.
    +
    +      To apply the Apache License to your work, attach the following
    +      boilerplate notice, with the fields enclosed by brackets "[]"
    +      replaced with your own identifying information. (Don't include
    +      the brackets!)  The text should be enclosed in the appropriate
    +      comment syntax for the file format. We also recommend that a
    +      file or class name and description of purpose be included on the
    +      same "printed page" as the copyright notice for easier
    +      identification within third-party archives.
    +
    +   Copyright [yyyy] [name of copyright owner]
    +
    +   Licensed under the Apache License, Version 2.0 (the "License");
    +   you may not use this file except in compliance with the License.
    +   You may obtain a copy of the License at
    +
    +       http://www.apache.org/licenses/LICENSE-2.0
    +
    +   Unless required by applicable law or agreed to in writing, software
    +   distributed under the License is distributed on an "AS IS" BASIS,
    +   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    +   See the License for the specific language governing permissions and
    +   limitations under the License.
    +
    +
  • +
  • +

    Apache License 2.0

    +

    Used by:

    +
                                     Apache License
                                Version 2.0, January 2004
    @@ -1746,30 +1937,687 @@ 

    Used by:

    same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright {yyyy} {name of copyright owner} + Copyright 2017 Juniper Networks, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +
    +
  • +
  • +

    Apache License 2.0

    +

    Used by:

    + +
                                     Apache License
    +                           Version 2.0, January 2004
    +                        http://www.apache.org/licenses/
    +
    +   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
    +
    +   1. Definitions.
    +
    +      "License" shall mean the terms and conditions for use, reproduction,
    +      and distribution as defined by Sections 1 through 9 of this document.
    +
    +      "Licensor" shall mean the copyright owner or entity authorized by
    +      the copyright owner that is granting the License.
    +
    +      "Legal Entity" shall mean the union of the acting entity and all
    +      other entities that control, are controlled by, or are under common
    +      control with that entity. For the purposes of this definition,
    +      "control" means (i) the power, direct or indirect, to cause the
    +      direction or management of such entity, whether by contract or
    +      otherwise, or (ii) ownership of fifty percent (50%) or more of the
    +      outstanding shares, or (iii) beneficial ownership of such entity.
    +
    +      "You" (or "Your") shall mean an individual or Legal Entity
    +      exercising permissions granted by this License.
    +
    +      "Source" form shall mean the preferred form for making modifications,
    +      including but not limited to software source code, documentation
    +      source, and configuration files.
    +
    +      "Object" form shall mean any form resulting from mechanical
    +      transformation or translation of a Source form, including but
    +      not limited to compiled object code, generated documentation,
    +      and conversions to other media types.
    +
    +      "Work" shall mean the work of authorship, whether in Source or
    +      Object form, made available under the License, as indicated by a
    +      copyright notice that is included in or attached to the work
    +      (an example is provided in the Appendix below).
    +
    +      "Derivative Works" shall mean any work, whether in Source or Object
    +      form, that is based on (or derived from) the Work and for which the
    +      editorial revisions, annotations, elaborations, or other modifications
    +      represent, as a whole, an original work of authorship. For the purposes
    +      of this License, Derivative Works shall not include works that remain
    +      separable from, or merely link (or bind by name) to the interfaces of,
    +      the Work and Derivative Works thereof.
    +
    +      "Contribution" shall mean any work of authorship, including
    +      the original version of the Work and any modifications or additions
    +      to that Work or Derivative Works thereof, that is intentionally
    +      submitted to Licensor for inclusion in the Work by the copyright owner
    +      or by an individual or Legal Entity authorized to submit on behalf of
    +      the copyright owner. For the purposes of this definition, "submitted"
    +      means any form of electronic, verbal, or written communication sent
    +      to the Licensor or its representatives, including but not limited to
    +      communication on electronic mailing lists, source code control systems,
    +      and issue tracking systems that are managed by, or on behalf of, the
    +      Licensor for the purpose of discussing and improving the Work, but
    +      excluding communication that is conspicuously marked or otherwise
    +      designated in writing by the copyright owner as "Not a Contribution."
    +
    +      "Contributor" shall mean Licensor and any individual or Legal Entity
    +      on behalf of whom a Contribution has been received by Licensor and
    +      subsequently incorporated within the Work.
    +
    +   2. Grant of Copyright License. Subject to the terms and conditions of
    +      this License, each Contributor hereby grants to You a perpetual,
    +      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
    +      copyright license to reproduce, prepare Derivative Works of,
    +      publicly display, publicly perform, sublicense, and distribute the
    +      Work and such Derivative Works in Source or Object form.
    +
    +   3. Grant of Patent License. Subject to the terms and conditions of
    +      this License, each Contributor hereby grants to You a perpetual,
    +      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
    +      (except as stated in this section) patent license to make, have made,
    +      use, offer to sell, sell, import, and otherwise transfer the Work,
    +      where such license applies only to those patent claims licensable
    +      by such Contributor that are necessarily infringed by their
    +      Contribution(s) alone or by combination of their Contribution(s)
    +      with the Work to which such Contribution(s) was submitted. If You
    +      institute patent litigation against any entity (including a
    +      cross-claim or counterclaim in a lawsuit) alleging that the Work
    +      or a Contribution incorporated within the Work constitutes direct
    +      or contributory patent infringement, then any patent licenses
    +      granted to You under this License for that Work shall terminate
    +      as of the date such litigation is filed.
    +
    +   4. Redistribution. You may reproduce and distribute copies of the
    +      Work or Derivative Works thereof in any medium, with or without
    +      modifications, and in Source or Object form, provided that You
    +      meet the following conditions:
    +
    +      (a) You must give any other recipients of the Work or
    +          Derivative Works a copy of this License; and
    +
    +      (b) You must cause any modified files to carry prominent notices
    +          stating that You changed the files; and
    +
    +      (c) You must retain, in the Source form of any Derivative Works
    +          that You distribute, all copyright, patent, trademark, and
    +          attribution notices from the Source form of the Work,
    +          excluding those notices that do not pertain to any part of
    +          the Derivative Works; and
    +
    +      (d) If the Work includes a "NOTICE" text file as part of its
    +          distribution, then any Derivative Works that You distribute must
    +          include a readable copy of the attribution notices contained
    +          within such NOTICE file, excluding those notices that do not
    +          pertain to any part of the Derivative Works, in at least one
    +          of the following places: within a NOTICE text file distributed
    +          as part of the Derivative Works; within the Source form or
    +          documentation, if provided along with the Derivative Works; or,
    +          within a display generated by the Derivative Works, if and
    +          wherever such third-party notices normally appear. The contents
    +          of the NOTICE file are for informational purposes only and
    +          do not modify the License. You may add Your own attribution
    +          notices within Derivative Works that You distribute, alongside
    +          or as an addendum to the NOTICE text from the Work, provided
    +          that such additional attribution notices cannot be construed
    +          as modifying the License.
    +
    +      You may add Your own copyright statement to Your modifications and
    +      may provide additional or different license terms and conditions
    +      for use, reproduction, or distribution of Your modifications, or
    +      for any such Derivative Works as a whole, provided Your use,
    +      reproduction, and distribution of the Work otherwise complies with
    +      the conditions stated in this License.
    +
    +   5. Submission of Contributions. Unless You explicitly state otherwise,
    +      any Contribution intentionally submitted for inclusion in the Work
    +      by You to the Licensor shall be under the terms and conditions of
    +      this License, without any additional terms or conditions.
    +      Notwithstanding the above, nothing herein shall supersede or modify
    +      the terms of any separate license agreement you may have executed
    +      with Licensor regarding such Contributions.
    +
    +   6. Trademarks. This License does not grant permission to use the trade
    +      names, trademarks, service marks, or product names of the Licensor,
    +      except as required for reasonable and customary use in describing the
    +      origin of the Work and reproducing the content of the NOTICE file.
    +
    +   7. Disclaimer of Warranty. Unless required by applicable law or
    +      agreed to in writing, Licensor provides the Work (and each
    +      Contributor provides its Contributions) on an "AS IS" BASIS,
    +      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
    +      implied, including, without limitation, any warranties or conditions
    +      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
    +      PARTICULAR PURPOSE. You are solely responsible for determining the
    +      appropriateness of using or redistributing the Work and assume any
    +      risks associated with Your exercise of permissions under this License.
    +
    +   8. Limitation of Liability. In no event and under no legal theory,
    +      whether in tort (including negligence), contract, or otherwise,
    +      unless required by applicable law (such as deliberate and grossly
    +      negligent acts) or agreed to in writing, shall any Contributor be
    +      liable to You for damages, including any direct, indirect, special,
    +      incidental, or consequential damages of any character arising as a
    +      result of this License or out of the use or inability to use the
    +      Work (including but not limited to damages for loss of goodwill,
    +      work stoppage, computer failure or malfunction, or any and all
    +      other commercial damages or losses), even if such Contributor
    +      has been advised of the possibility of such damages.
    +
    +   9. Accepting Warranty or Additional Liability. While redistributing
    +      the Work or Derivative Works thereof, You may choose to offer,
    +      and charge a fee for, acceptance of support, warranty, indemnity,
    +      or other liability obligations and/or rights consistent with this
    +      License. However, in accepting such obligations, You may act only
    +      on Your own behalf and on Your sole responsibility, not on behalf
    +      of any other Contributor, and only if You agree to indemnify,
    +      defend, and hold each Contributor harmless for any liability
    +      incurred by, or claims asserted against, such Contributor by reason
    +      of your accepting any such warranty or additional liability.
    +
    +   END OF TERMS AND CONDITIONS
    +
    +   APPENDIX: How to apply the Apache License to your work.
    +
    +      To apply the Apache License to your work, attach the following
    +      boilerplate notice, with the fields enclosed by brackets "{}"
    +      replaced with your own identifying information. (Don't include
    +      the brackets!)  The text should be enclosed in the appropriate
    +      comment syntax for the file format. We also recommend that a
    +      file or class name and description of purpose be included on the
    +      same "printed page" as the copyright notice for easier
    +      identification within third-party archives.
    +
    +   Copyright {yyyy} {name of copyright owner}
    +
    +   Licensed under the Apache License, Version 2.0 (the "License");
    +   you may not use this file except in compliance with the License.
    +   You may obtain a copy of the License at
    +
    +       http://www.apache.org/licenses/LICENSE-2.0
    +
    +   Unless required by applicable law or agreed to in writing, software
    +   distributed under the License is distributed on an "AS IS" BASIS,
    +   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    +   See the License for the specific language governing permissions and
    +   limitations under the License.
    +
    +
    +
  • +
  • +

    Apache License 2.0

    +

    Used by:

    + +
                                    Apache License
    +                           Version 2.0, January 2004
    +                        http://www.apache.org/licenses/
    +
    +   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
    +
    +   1. Definitions.
    +
    +      "License" shall mean the terms and conditions for use, reproduction,
    +      and distribution as defined by Sections 1 through 9 of this document.
    +
    +      "Licensor" shall mean the copyright owner or entity authorized by
    +      the copyright owner that is granting the License.
    +
    +      "Legal Entity" shall mean the union of the acting entity and all
    +      other entities that control, are controlled by, or are under common
    +      control with that entity. For the purposes of this definition,
    +      "control" means (i) the power, direct or indirect, to cause the
    +      direction or management of such entity, whether by contract or
    +      otherwise, or (ii) ownership of fifty percent (50%) or more of the
    +      outstanding shares, or (iii) beneficial ownership of such entity.
    +
    +      "You" (or "Your") shall mean an individual or Legal Entity
    +      exercising permissions granted by this License.
    +
    +      "Source" form shall mean the preferred form for making modifications,
    +      including but not limited to software source code, documentation
    +      source, and configuration files.
    +
    +      "Object" form shall mean any form resulting from mechanical
    +      transformation or translation of a Source form, including but
    +      not limited to compiled object code, generated documentation,
    +      and conversions to other media types.
    +
    +      "Work" shall mean the work of authorship, whether in Source or
    +      Object form, made available under the License, as indicated by a
    +      copyright notice that is included in or attached to the work
    +      (an example is provided in the Appendix below).
    +
    +      "Derivative Works" shall mean any work, whether in Source or Object
    +      form, that is based on (or derived from) the Work and for which the
    +      editorial revisions, annotations, elaborations, or other modifications
    +      represent, as a whole, an original work of authorship. For the purposes
    +      of this License, Derivative Works shall not include works that remain
    +      separable from, or merely link (or bind by name) to the interfaces of,
    +      the Work and Derivative Works thereof.
    +
    +      "Contribution" shall mean any work of authorship, including
    +      the original version of the Work and any modifications or additions
    +      to that Work or Derivative Works thereof, that is intentionally
    +      submitted to Licensor for inclusion in the Work by the copyright owner
    +      or by an individual or Legal Entity authorized to submit on behalf of
    +      the copyright owner. For the purposes of this definition, "submitted"
    +      means any form of electronic, verbal, or written communication sent
    +      to the Licensor or its representatives, including but not limited to
    +      communication on electronic mailing lists, source code control systems,
    +      and issue tracking systems that are managed by, or on behalf of, the
    +      Licensor for the purpose of discussing and improving the Work, but
    +      excluding communication that is conspicuously marked or otherwise
    +      designated in writing by the copyright owner as "Not a Contribution."
    +
    +      "Contributor" shall mean Licensor and any individual or Legal Entity
    +      on behalf of whom a Contribution has been received by Licensor and
    +      subsequently incorporated within the Work.
    +
    +   2. Grant of Copyright License. Subject to the terms and conditions of
    +      this License, each Contributor hereby grants to You a perpetual,
    +      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
    +      copyright license to reproduce, prepare Derivative Works of,
    +      publicly display, publicly perform, sublicense, and distribute the
    +      Work and such Derivative Works in Source or Object form.
    +
    +   3. Grant of Patent License. Subject to the terms and conditions of
    +      this License, each Contributor hereby grants to You a perpetual,
    +      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
    +      (except as stated in this section) patent license to make, have made,
    +      use, offer to sell, sell, import, and otherwise transfer the Work,
    +      where such license applies only to those patent claims licensable
    +      by such Contributor that are necessarily infringed by their
    +      Contribution(s) alone or by combination of their Contribution(s)
    +      with the Work to which such Contribution(s) was submitted. If You
    +      institute patent litigation against any entity (including a
    +      cross-claim or counterclaim in a lawsuit) alleging that the Work
    +      or a Contribution incorporated within the Work constitutes direct
    +      or contributory patent infringement, then any patent licenses
    +      granted to You under this License for that Work shall terminate
    +      as of the date such litigation is filed.
    +
    +   4. Redistribution. You may reproduce and distribute copies of the
    +      Work or Derivative Works thereof in any medium, with or without
    +      modifications, and in Source or Object form, provided that You
    +      meet the following conditions:
    +
    +      (a) You must give any other recipients of the Work or
    +          Derivative Works a copy of this License; and
    +
    +      (b) You must cause any modified files to carry prominent notices
    +          stating that You changed the files; and
    +
    +      (c) You must retain, in the Source form of any Derivative Works
    +          that You distribute, all copyright, patent, trademark, and
    +          attribution notices from the Source form of the Work,
    +          excluding those notices that do not pertain to any part of
    +          the Derivative Works; and
    +
    +      (d) If the Work includes a "NOTICE" text file as part of its
    +          distribution, then any Derivative Works that You distribute must
    +          include a readable copy of the attribution notices contained
    +          within such NOTICE file, excluding those notices that do not
    +          pertain to any part of the Derivative Works, in at least one
    +          of the following places: within a NOTICE text file distributed
    +          as part of the Derivative Works; within the Source form or
    +          documentation, if provided along with the Derivative Works; or,
    +          within a display generated by the Derivative Works, if and
    +          wherever such third-party notices normally appear. The contents
    +          of the NOTICE file are for informational purposes only and
    +          do not modify the License. You may add Your own attribution
    +          notices within Derivative Works that You distribute, alongside
    +          or as an addendum to the NOTICE text from the Work, provided
    +          that such additional attribution notices cannot be construed
    +          as modifying the License.
    +
    +      You may add Your own copyright statement to Your modifications and
    +      may provide additional or different license terms and conditions
    +      for use, reproduction, or distribution of Your modifications, or
    +      for any such Derivative Works as a whole, provided Your use,
    +      reproduction, and distribution of the Work otherwise complies with
    +      the conditions stated in this License.
    +
    +   5. Submission of Contributions. Unless You explicitly state otherwise,
    +      any Contribution intentionally submitted for inclusion in the Work
    +      by You to the Licensor shall be under the terms and conditions of
    +      this License, without any additional terms or conditions.
    +      Notwithstanding the above, nothing herein shall supersede or modify
    +      the terms of any separate license agreement you may have executed
    +      with Licensor regarding such Contributions.
    +
    +   6. Trademarks. This License does not grant permission to use the trade
    +      names, trademarks, service marks, or product names of the Licensor,
    +      except as required for reasonable and customary use in describing the
    +      origin of the Work and reproducing the content of the NOTICE file.
    +
    +   7. Disclaimer of Warranty. Unless required by applicable law or
    +      agreed to in writing, Licensor provides the Work (and each
    +      Contributor provides its Contributions) on an "AS IS" BASIS,
    +      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
    +      implied, including, without limitation, any warranties or conditions
    +      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
    +      PARTICULAR PURPOSE. You are solely responsible for determining the
    +      appropriateness of using or redistributing the Work and assume any
    +      risks associated with Your exercise of permissions under this License.
    +
    +   8. Limitation of Liability. In no event and under no legal theory,
    +      whether in tort (including negligence), contract, or otherwise,
    +      unless required by applicable law (such as deliberate and grossly
    +      negligent acts) or agreed to in writing, shall any Contributor be
    +      liable to You for damages, including any direct, indirect, special,
    +      incidental, or consequential damages of any character arising as a
    +      result of this License or out of the use or inability to use the
    +      Work (including but not limited to damages for loss of goodwill,
    +      work stoppage, computer failure or malfunction, or any and all
    +      other commercial damages or losses), even if such Contributor
    +      has been advised of the possibility of such damages.
    +
    +   9. Accepting Warranty or Additional Liability. While redistributing
    +      the Work or Derivative Works thereof, You may choose to offer,
    +      and charge a fee for, acceptance of support, warranty, indemnity,
    +      or other liability obligations and/or rights consistent with this
    +      License. However, in accepting such obligations, You may act only
    +      on Your own behalf and on Your sole responsibility, not on behalf
    +      of any other Contributor, and only if You agree to indemnify,
    +      defend, and hold each Contributor harmless for any liability
    +      incurred by, or claims asserted against, such Contributor by reason
    +      of your accepting any such warranty or additional liability.
    +
    +   END OF TERMS AND CONDITIONS
    +
    +   APPENDIX: How to apply the Apache License to your work.
    +
    +      To apply the Apache License to your work, attach the following
    +      boilerplate notice, with the fields enclosed by brackets "{}"
    +      replaced with your own identifying information. (Don't include
    +      the brackets!)  The text should be enclosed in the appropriate
    +      comment syntax for the file format. We also recommend that a
    +      file or class name and description of purpose be included on the
    +      same "printed page" as the copyright notice for easier
    +      identification within third-party archives.
    +
    +   Copyright 2018-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
    +
    +   Licensed under the Apache License, Version 2.0 (the "License");
    +   you may not use this file except in compliance with the License.
    +   You may obtain a copy of the License at
    +
    +       http://www.apache.org/licenses/LICENSE-2.0
    +
    +   Unless required by applicable law or agreed to in writing, software
    +   distributed under the License is distributed on an "AS IS" BASIS,
    +   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    +   See the License for the specific language governing permissions and
    +   limitations under the License.
    +
    +
    +
  • +
  • +

    Apache License 2.0

    +

    Used by:

    + +
                                  Apache License
    +                        Version 2.0 January 2004
    +                     http://www.apache.org/licenses/
    +
    +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
    +
    +1. Definitions.
    +
    +   "License" shall mean the terms and conditions for use, reproduction,
    +   and distribution as defined by Sections 1 through 9 of this document.
    +
    +   "Licensor" shall mean the copyright owner or entity authorized by
    +   the copyright owner that is granting the License.
    +
    +   "Legal Entity" shall mean the union of the acting entity and all
    +   other entities that control, are controlled by, or are under common
    +   control with that entity. For the purposes of this definition,
    +   "control" means (i) the power, direct or indirect, to cause the
    +   direction or management of such entity, whether by contract or
    +   otherwise, or (ii) ownership of fifty percent (50%) or more of the
    +   outstanding shares, or (iii) beneficial ownership of such entity.
    +
    +   "You" (or "Your") shall mean an individual or Legal Entity
    +   exercising permissions granted by this License.
    +
    +   "Source" form shall mean the preferred form for making modifications,
    +   including but not limited to software source code, documentation
    +   source, and configuration files.
    +
    +   "Object" form shall mean any form resulting from mechanical
    +   transformation or translation of a Source form, including but
    +   not limited to compiled object code, generated documentation,
    +   and conversions to other media types.
    +
    +   "Work" shall mean the work of authorship, whether in Source or
    +   Object form, made available under the License, as indicated by a
    +   copyright notice that is included in or attached to the work
    +   (an example is provided in the Appendix below).
    +
    +   "Derivative Works" shall mean any work, whether in Source or Object
    +   form, that is based on (or derived from) the Work and for which the
    +   editorial revisions, annotations, elaborations, or other modifications
    +   represent, as a whole, an original work of authorship. For the purposes
    +   of this License, Derivative Works shall not include works that remain
    +   separable from, or merely link (or bind by name) to the interfaces of,
    +   the Work and Derivative Works thereof.
    +
    +   "Contribution" shall mean any work of authorship, including
    +   the original version of the Work and any modifications or additions
    +   to that Work or Derivative Works thereof, that is intentionally
    +   submitted to Licensor for inclusion in the Work by the copyright owner
    +   or by an individual or Legal Entity authorized to submit on behalf of
    +   the copyright owner. For the purposes of this definition, "submitted"
    +   means any form of electronic, verbal, or written communication sent
    +   to the Licensor or its representatives, including but not limited to
    +   communication on electronic mailing lists, source code control systems,
    +   and issue tracking systems that are managed by, or on behalf of, the
    +   Licensor for the purpose of discussing and improving the Work, but
    +   excluding communication that is conspicuously marked or otherwise
    +   designated in writing by the copyright owner as "Not a Contribution."
    +
    +   "Contributor" shall mean Licensor and any individual or Legal Entity
    +   on behalf of whom a Contribution has been received by Licensor and
    +   subsequently incorporated within the Work.
    +
    +2. Grant of Copyright License. Subject to the terms and conditions of
    +   this License, each Contributor hereby grants to You a perpetual,
    +   worldwide, non-exclusive, no-charge, royalty-free, irrevocable
    +   copyright license to reproduce, prepare Derivative Works of,
    +   publicly display, publicly perform, sublicense, and distribute the
    +   Work and such Derivative Works in Source or Object form.
    +
    +3. Grant of Patent License. Subject to the terms and conditions of
    +   this License, each Contributor hereby grants to You a perpetual,
    +   worldwide, non-exclusive, no-charge, royalty-free, irrevocable
    +   (except as stated in this section) patent license to make, have made,
    +   use, offer to sell, sell, import, and otherwise transfer the Work,
    +   where such license applies only to those patent claims licensable
    +   by such Contributor that are necessarily infringed by their
    +   Contribution(s) alone or by combination of their Contribution(s)
    +   with the Work to which such Contribution(s) was submitted. If You
    +   institute patent litigation against any entity (including a
    +   cross-claim or counterclaim in a lawsuit) alleging that the Work
    +   or a Contribution incorporated within the Work constitutes direct
    +   or contributory patent infringement, then any patent licenses
    +   granted to You under this License for that Work shall terminate
    +   as of the date such litigation is filed.
    +
    +4. Redistribution. You may reproduce and distribute copies of the
    +   Work or Derivative Works thereof in any medium, with or without
    +   modifications, and in Source or Object form, provided that You
    +   meet the following conditions:
    +
    +   (a) You must give any other recipients of the Work or
    +       Derivative Works a copy of this License; and
    +
    +   (b) You must cause any modified files to carry prominent notices
    +       stating that You changed the files; and
    +
    +   (c) You must retain, in the Source form of any Derivative Works
    +       that You distribute, all copyright, patent, trademark, and
    +       attribution notices from the Source form of the Work,
    +       excluding those notices that do not pertain to any part of
    +       the Derivative Works; and
    +
    +   (d) If the Work includes a "NOTICE" text file as part of its
    +       distribution, then any Derivative Works that You distribute must
    +       include a readable copy of the attribution notices contained
    +       within such NOTICE file, excluding those notices that do not
    +       pertain to any part of the Derivative Works, in at least one
    +       of the following places: within a NOTICE text file distributed
    +       as part of the Derivative Works; within the Source form or
    +       documentation, if provided along with the Derivative Works; or,
    +       within a display generated by the Derivative Works, if and
    +       wherever such third-party notices normally appear. The contents
    +       of the NOTICE file are for informational purposes only and
    +       do not modify the License. You may add Your own attribution
    +       notices within Derivative Works that You distribute, alongside
    +       or as an addendum to the NOTICE text from the Work, provided
    +       that such additional attribution notices cannot be construed
    +       as modifying the License.
    +
    +   You may add Your own copyright statement to Your modifications and
    +   may provide additional or different license terms and conditions
    +   for use, reproduction, or distribution of Your modifications, or
    +   for any such Derivative Works as a whole, provided Your use,
    +   reproduction, and distribution of the Work otherwise complies with
    +   the conditions stated in this License.
    +
    +5. Submission of Contributions. Unless You explicitly state otherwise,
    +   any Contribution intentionally submitted for inclusion in the Work
    +   by You to the Licensor shall be under the terms and conditions of
    +   this License, without any additional terms or conditions.
    +   Notwithstanding the above, nothing herein shall supersede or modify
    +   the terms of any separate license agreement you may have executed
    +   with Licensor regarding such Contributions.
    +
    +6. Trademarks. This License does not grant permission to use the trade
    +   names, trademarks, service marks, or product names of the Licensor,
    +   except as required for reasonable and customary use in describing the
    +   origin of the Work and reproducing the content of the NOTICE file.
    +
    +7. Disclaimer of Warranty. Unless required by applicable law or
    +   agreed to in writing, Licensor provides the Work (and each
    +   Contributor provides its Contributions) on an "AS IS" BASIS,
    +   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
    +   implied, including, without limitation, any warranties or conditions
    +   of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
    +   PARTICULAR PURPOSE. You are solely responsible for determining the
    +   appropriateness of using or redistributing the Work and assume any
    +   risks associated with Your exercise of permissions under this License.
    +
    +8. Limitation of Liability. In no event and under no legal theory,
    +   whether in tort (including negligence), contract, or otherwise,
    +   unless required by applicable law (such as deliberate and grossly
    +   negligent acts) or agreed to in writing, shall any Contributor be
    +   liable to You for damages, including any direct, indirect, special,
    +   incidental, or consequential damages of any character arising as a
    +   result of this License or out of the use or inability to use the
    +   Work (including but not limited to damages for loss of goodwill,
    +   work stoppage, computer failure or malfunction, or any and all
    +   other commercial damages or losses), even if such Contributor
    +   has been advised of the possibility of such damages.
    +
    +9. Accepting Warranty or Additional Liability. While redistributing
    +   the Work or Derivative Works thereof, You may choose to offer,
    +   and charge a fee for, acceptance of support, warranty, indemnity,
    +   or other liability obligations and/or rights consistent with this
    +   License. However, in accepting such obligations, You may act only
    +   on Your own behalf and on Your sole responsibility, not on behalf
    +   of any other Contributor, and only if You agree to indemnify,
    +   defend, and hold each Contributor harmless for any liability
    +   incurred by, or claims asserted against, such Contributor by reason
    +   of your accepting any such warranty or additional liability.
    +
    +END OF TERMS AND CONDITIONS
    +
    +APPENDIX: How to apply the Apache License to your work.
    +
    +   To apply the Apache License to your work, attach the following
    +   boilerplate notice, with the fields enclosed by brackets "[]"
    +   replaced with your own identifying information. (Don't include
    +   the brackets!)  The text should be enclosed in the appropriate
    +   comment syntax for the file format. We also recommend that a
    +   file or class name and description of purpose be included on the
    +   same "printed page" as the copyright notice for easier
    +   identification within third-party archives.
     
    -   Licensed under the Apache License, Version 2.0 (the "License");
    -   you may not use this file except in compliance with the License.
    -   You may obtain a copy of the License at
    +Copyright [yyyy] [name of copyright owner]
     
    -       http://www.apache.org/licenses/LICENSE-2.0
    +Licensed under the Apache License, Version 2.0 (the "License");
    +you may not use this file except in compliance with the License.
    +You may obtain a copy of the License at
     
    -   Unless required by applicable law or agreed to in writing, software
    -   distributed under the License is distributed on an "AS IS" BASIS,
    -   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    -   See the License for the specific language governing permissions and
    -   limitations under the License.
    +	http://www.apache.org/licenses/LICENSE-2.0
     
    +Unless required by applicable law or agreed to in writing, software
    +distributed under the License is distributed on an "AS IS" BASIS,
    +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    +See the License for the specific language governing permissions and
    +limitations under the License.
     
  • Apache License 2.0

    Used by:

                                  Apache License
    -                        Version 2.0 January 2004
    +                        Version 2.0, January 2004
                          http://www.apache.org/licenses/
     
     TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
    @@ -1956,7 +2804,8 @@ 

    Used by:

    same "printed page" as the copyright notice for easier identification within third-party archives. -Copyright [yyyy] [name of copyright owner] +Copyright (c) 2016 Alex Crichton +Copyright (c) 2017 The Tokio Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -1975,15 +2824,7 @@

    Used by:

    Apache License 2.0

    Used by:

                                  Apache License
                             Version 2.0, January 2004
    @@ -2173,8 +3014,7 @@ 

    Used by:

    same "printed page" as the copyright notice for easier identification within third-party archives. -Copyright (c) 2016 Alex Crichton -Copyright (c) 2017 The Tokio Authors +Copyright (c) 2019 Matthias Einwag Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -2193,7 +3033,7 @@

    Used by:

    Apache License 2.0

    Used by:

                                  Apache License
                             Version 2.0, January 2004
    @@ -2383,7 +3223,7 @@ 

    Used by:

    same "printed page" as the copyright notice for easier identification within third-party archives. -Copyright (c) 2019 Matthias Einwag +Copyright 2014 Paho Lurie-Gregg Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -2395,14 +3235,13 @@

    Used by:

    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and -limitations under the License. -
    +limitations under the License.
  • Apache License 2.0

    Used by:

                                  Apache License
                             Version 2.0, January 2004
    @@ -2592,7 +3431,7 @@ 

    Used by:

    same "printed page" as the copyright notice for easier identification within third-party archives. -Copyright 2014 Paho Lurie-Gregg +Copyright 2017 The Rust Project Developers Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -2604,13 +3443,15 @@

    Used by:

    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and -limitations under the License.
    +limitations under the License. +
  • Apache License 2.0

    Used by:

                                  Apache License
                             Version 2.0, January 2004
    @@ -2800,7 +3641,7 @@ 

    Used by:

    same "printed page" as the copyright notice for easier identification within third-party archives. -Copyright 2017 The Rust Project Developers +Copyright 2017 http-rs authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -2819,7 +3660,7 @@

    Used by:

    Apache License 2.0

    Used by:

                                  Apache License
                             Version 2.0, January 2004
    @@ -3009,7 +3850,7 @@ 

    Used by:

    same "printed page" as the copyright notice for easier identification within third-party archives. -Copyright 2017 http-rs authors +Copyright 2017 quininer kel Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -3028,7 +3869,7 @@

    Used by:

    Apache License 2.0

    Used by:

                                  Apache License
                             Version 2.0, January 2004
    @@ -3218,7 +4059,7 @@ 

    Used by:

    same "printed page" as the copyright notice for easier identification within third-party archives. -Copyright 2017 quininer kel +Copyright 2018 The pin-utils authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -4084,6 +4925,7 @@

    Used by:

  • base64 0.21.7
  • base64 0.22.1
  • bitflags 2.11.1
  • +
  • bytes-utils 0.1.4
  • cc 1.2.62
  • cfg-if 1.0.4
  • cmake 0.1.58
  • @@ -4101,15 +4943,18 @@

    Used by:

  • errno 0.3.14
  • event-listener-strategy 0.5.4
  • event-listener 5.4.2
  • +
  • fastrand 2.5.0
  • find-msvc-tools 0.1.9
  • flate2 1.1.9
  • fnv 1.0.7
  • form_urlencoded 1.2.2
  • hashbrown 0.14.5
  • hashbrown 0.15.5
  • +
  • hashbrown 0.16.1
  • hashbrown 0.17.1
  • heck 0.5.0
  • httparse 1.10.1
  • +
  • hyper-rustls 0.27.10
  • idna 1.1.0
  • idna_adapter 1.2.2
  • indexmap 2.14.0
  • @@ -4122,6 +4967,7 @@

    Used by:

  • num-integer 0.1.46
  • num-traits 0.2.19
  • once_cell 1.21.4
  • +
  • openssl-probe 0.2.1
  • parking 2.2.1
  • parking_lot 0.12.5
  • parking_lot_core 0.9.12
  • @@ -4132,8 +4978,12 @@

    Used by:

  • pest_meta 2.8.6
  • pkg-config 0.3.33
  • regex-automata 0.4.14
  • +
  • regex-lite 0.1.9
  • regex-syntax 0.8.10
  • +
  • ring 0.17.14
  • ron 0.8.1
  • +
  • rustc_version 0.4.1
  • +
  • rustls-native-certs 0.8.4
  • rustls 0.23.46
  • scopeguard 1.2.0
  • signal-hook-registry 1.4.8
  • @@ -4149,6 +4999,7 @@

    Used by:

  • url 2.5.8
  • uuid 1.23.1
  • version_check 0.9.5
  • +
  • xmlparser 0.13.6
  • yasna 0.6.0
  •                               Apache License
    @@ -4993,6 +5844,7 @@ 

    Used by:

  • hmac 0.12.1
  • inout 0.1.4
  • md-5 0.10.6
  • +
  • sha1 0.10.6
  • sha2 0.10.9
  •                               Apache License
    @@ -6452,6 +7304,7 @@ 

    Used by:

    +
  • +
  • +

    ISC License

    +

    Used by:

    + +
    Copyright 2015-2025 Brian Smith.
    +
    +Permission to use, copy, modify, and/or distribute this software for any
    +purpose with or without fee is hereby granted, provided that the above
    +copyright notice and this permission notice appear in all copies.
    +
    +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
    +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
    +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
    +SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
    +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
    +OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
    +CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
     
  • @@ -7163,6 +8055,63 @@

    Used by:

    OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +
  • +
  • +

    MIT License

    +

    Used by:

    + +
    Copyright (c) 2018-2019 Sean McArthur
    +
    +Permission is hereby granted, free of charge, to any person obtaining a copy
    +of this software and associated documentation files (the "Software"), to deal
    +in the Software without restriction, including without limitation the rights
    +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
    +copies of the Software, and to permit persons to whom the Software is
    +furnished to do so, subject to the following conditions:
    +
    +The above copyright notice and this permission notice shall be included in
    +all copies or substantial portions of the Software.
    +
    +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
    +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
    +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
    +THE SOFTWARE.
    +
    +
    +
  • +
  • +

    MIT License

    +

    Used by:

    + +
    Copyright (c) 2018-2023 Sean McArthur
    +Copyright (c) 2016 Alex Crichton
    +
    +Permission is hereby granted, free of charge, to any person obtaining a copy
    +of this software and associated documentation files (the "Software"), to deal
    +in the Software without restriction, including without limitation the rights
    +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
    +copies of the Software, and to permit persons to whom the Software is
    +furnished to do so, subject to the following conditions:
    +
    +The above copyright notice and this permission notice shall be included in
    +all copies or substantial portions of the Software.
    +
    +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
    +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
    +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
    +THE SOFTWARE.
    +
     
  • @@ -7250,6 +8199,39 @@

    Used by:

    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +
  • +
  • +

    MIT License

    +

    Used by:

    + +
    Copyright (c) 2019 Hyper Contributors
    +
    +Permission is hereby granted, free of charge, to any
    +person obtaining a copy of this software and associated
    +documentation files (the "Software"), to deal in the
    +Software without restriction, including without
    +limitation the rights to use, copy, modify, merge,
    +publish, distribute, sublicense, and/or sell copies of
    +the Software, and to permit persons to whom the Software
    +is furnished to do so, subject to the following
    +conditions:
    +
    +The above copyright notice and this permission notice
    +shall be included in all copies or substantial portions
    +of the Software.
    +
    +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF
    +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED
    +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
    +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
    +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
    +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
    +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR
    +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
    +DEALINGS IN THE SOFTWARE.
     
  • @@ -7567,6 +8549,34 @@

    Used by:

    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +
  • +
  • +

    MIT License

    +

    Used by:

    + +
    MIT License
    +
    +Copyright (c) 2016 Jerome Froelich
    +
    +Permission is hereby granted, free of charge, to any person obtaining a copy
    +of this software and associated documentation files (the "Software"), to deal
    +in the Software without restriction, including without limitation the rights
    +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
    +copies of the Software, and to permit persons to whom the Software is
    +furnished to do so, subject to the following conditions:
    +
    +The above copyright notice and this permission notice shall be included in all
    +copies or substantial portions of the Software.
    +
    +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
    +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
    +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
    +SOFTWARE.
  • MIT License

    @@ -7781,7 +8791,37 @@

    Used by:

    MIT License

    Used by:

    +
    MIT License
    +
    +Copyright (c) 2022 Nugine
    +
    +Permission is hereby granted, free of charge, to any person obtaining a copy
    +of this software and associated documentation files (the "Software"), to deal
    +in the Software without restriction, including without limitation the rights
    +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
    +copies of the Software, and to permit persons to whom the Software is
    +furnished to do so, subject to the following conditions:
    +
    +The above copyright notice and this permission notice shall be included in all
    +copies or substantial portions of the Software.
    +
    +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
    +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
    +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
    +SOFTWARE.
    +
  • +
  • +

    MIT License

    +

    Used by:

    +
    MIT License
     
    @@ -7920,6 +8960,34 @@ 

    Used by:

    OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
    +
  • +
  • +

    MIT License

    +

    Used by:

    + +
    The MIT License (MIT)
    +
    +Copyright (c) 2014 Mathijs van de Nes
    +
    +Permission is hereby granted, free of charge, to any person obtaining a copy
    +of this software and associated documentation files (the "Software"), to deal
    +in the Software without restriction, including without limitation the rights
    +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
    +copies of the Software, and to permit persons to whom the Software is
    +furnished to do so, subject to the following conditions:
    +
    +The above copyright notice and this permission notice shall be included in all
    +copies or substantial portions of the Software.
    +
    +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
    +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
    +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
    +SOFTWARE.
  • MIT License

    @@ -8325,6 +9393,7 @@

    zlib License

    Used by:

    Copyright (c) 2024 Orson Peters
     
    diff --git a/crates/app/Cargo.toml b/crates/app/Cargo.toml
    index a1e66490f..fef29af1e 100644
    --- a/crates/app/Cargo.toml
    +++ b/crates/app/Cargo.toml
    @@ -13,6 +13,7 @@ license.workspace = true
     # the thin bin enforces that it is paired with a dev/CI backend.
     dev-mode = ["extenddb-config/dev-mode"]
     mongodb-test-hooks = ["extenddb-server/mongodb-test-hooks"]
    +backup-test-hooks = ["extenddb-server/backup-test-hooks"]
     
     [dependencies]
     extenddb-auth = { workspace = true }
    @@ -21,6 +22,8 @@ extenddb-engine = { workspace = true }
     extenddb-storage = { workspace = true }
     extenddb-config = { workspace = true }
     extenddb-server = { workspace = true }
    +extenddb-backup-store = { workspace = true }
    +futures = { workspace = true }
     tokio = { workspace = true }
     anyhow = { workspace = true }
     clap = { workspace = true }
    diff --git a/crates/app/src/cmd_backup.rs b/crates/app/src/cmd_backup.rs
    new file mode 100644
    index 000000000..4af501e68
    --- /dev/null
    +++ b/crates/app/src/cmd_backup.rs
    @@ -0,0 +1,1827 @@
    +// Copyright 2026 ExtendDB contributors
    +// SPDX-License-Identifier: Apache-2.0
    +
    +//! `extenddb backup`: the operator's commands over the configured backup
    +//! store.
    +//!
    +//! `backup list` prints the catalog's rows for the configured store's kind
    +//! and root, every status and every store id the rows carry, with `--failed`
    +//! narrowing to the FAILED tombstones the reconciler keeps for the operator;
    +//! each line carries the `store_root` and the `store_id` so an operator sees
    +//! which root and which id a row names. It is a read: it opens the catalog
    +//! beside a running server (on SQLite without the serve lock and without the
    +//! startup recovery a server runs), reads the root's identity file without
    +//! creating one, and runs on an unhealthy store, printing the store's
    +//! problem in its header, since an operator whose store is unhealthy is the
    +//! one who most wants to see which rows name the root.
    +//!
    +//! `backup purge --confirm` deletes every backup of the configured store: the
    +//! prefix of each row (`///`, derived from the
    +//! ARN, never a whole account prefix, so a prefix with no row survives) and
    +//! the row itself, outright, leaving no tombstone. It leaves
    +//! `extenddb-store.json`, which is under no account prefix, so the root
    +//! keeps its identity, and the `exports/` component. It needs the store
    +//! healthy, its identity known: a purge over an unhealthy store would act on
    +//! rows it cannot trust. Before it deletes it takes the catalog-level
    +//! maintenance claim, which it heartbeats on a timer for the claim's whole
    +//! life, at the period the jobs heartbeat their rows at
    +//! (`extenddb_storage::backup_catalog::JOB_HEARTBEAT_INTERVAL`, so the
    +//! `stale_job_after_secs` floor that keeps a live job from being declared
    +//! dead keeps this claim held too), so every reader of the claim sees it
    +//! held from the take to the release, through a deletion phase of any
    +//! length; a heartbeat that finds the claim lost, or fails, stops the
    +//! deletion loop before its next row. It then waits, up to
    +//! `2 * stale_job_after_secs`, until no CREATING or FAILED backup row of the
    +//! store has a live owner (a heartbeat within `stale_job_after_secs` on the
    +//! catalog's clock, as the backend's `list_backup_rows_older_than` reads it; a
    +//! FAILED row's heartbeat may still be refreshed by a job that has not yet
    +//! seen its claim lost, and its prefix must not be deleted under a manifest
    +//! it may still write) and no restore job is RUNNING; it refuses and
    +//! releases the claim if they have not gone by then, printing the wait. A
    +//! CREATING row the wait saw no live owner for is deleted too: the row
    +//! first (`remove_dead_creating_backup_row`, on the catalog's clock, which
    +//! leaves a row whose heartbeat is young after all, and its objects with
    +//! it), then its prefix. A DELETING row is not
    +//! waited on: nothing heartbeats one, the reconciler finishes any it finds,
    +//! and the deletion is the same idempotent sequence purge runs on every row
    +//! (two deleters of one prefix do not fail each other, and the row's moves
    +//! are conditional), so purge finishes it itself; for the same reason a row
    +//! a DeleteBackup moved to DELETING or DELETED, or removed, between the
    +//! command's read and its turn is finished and counted as deleted, not left
    +//! (`extenddb_engine::maintenance::purge_row` states each state's step).
    +//! Every backup and restore
    +//! job checks the claim before it starts and refuses while it is held
    +//! (`LimitExceededException`; the engine's backup module states where in
    +//! the job's order the check runs). The wait ends early, with the claim
    +//! released and nothing deleted, when a heartbeat finds the claim lost, when
    +//! the rows cannot be read, or on `SIGINT` (Ctrl-C) or `SIGTERM`; the same
    +//! signal during the deletion ends the loop before its next row, never
    +//! inside one, so a stopped command leaves no half-deleted row and no claim
    +//! for the next command or server to wait out. Without `--confirm` the
    +//! command is a
    +//! read like `list` (no serve lock, nothing created): it prints what it
    +//! would delete, changes nothing and exits 0, with no store configured as
    +//! with one. Each deletion is logged under
    +//! the `extenddb::audit` target with the account and the ARN.
    +//!
    +//! On SQLite purge opens the database the way `serve` does, which takes the
    +//! serve lock: it refuses while a server runs on the file, and a server
    +//! refuses to start while it runs. Under that lock every job owner is dead
    +//! (one process serves the file, and this is it), and no reconciler runs
    +//! while the command holds the file, so before the wait the command runs
    +//! the reconciler's duties itself with a zero stale window, as the holder
    +//! of the maintenance claim (the cycle takes its held branch and does not
    +//! read the claim back through that window): a restore the stopped server
    +//! left RUNNING is failed and its target removed, a store-backed one too,
    +//! which the held branch fails rather than resumes (a record with no
    +//! recorded progress over a target that holds items is failed
    +//! `target_not_empty` and its target left for `remove-target`), a
    +//! target a FAILED restore left CREATING is removed, an in-catalog CREATING
    +//! backup it left is removed, and a DELETING row is finished; the wait and
    +//! the CREATING-row removal then also use a zero window, so a CREATING or
    +//! FAILED row whose heartbeat is young (the stopped server's job wrote it)
    +//! is neither waited on nor left, and the wait has nothing to wait for. The
    +//! server's own startup under the same lock clears a maintenance claim a
    +//! stopped command left, for the same reason, and the command's own take
    +//! of the claim takes over a row a killed command left. On the other
    +//! backends purge runs beside live servers; the maintenance claim is the
    +//! exclusion.
    +//!
    +//! Every command opens the catalog through the backend's factory, under its own
    +//! name (`extenddb backup list`, `extenddb backup purge`, `extenddb backup
    +//! remove-target`, `extenddb backup sync`), and none runs the startup recovery
    +//! a server runs (`ServerComponentsOptions::startup_recovery` off): that work
    +//! belongs to the servers the command runs beside, and a command that ran it
    +//! could exit with it half done. On PostgreSQL, MongoDB and Cassandra the
    +//! command holds a serving registration for its run, as a server does, so a
    +//! migration waits for it, and the registration carries the command's name,
    +//! which the migration's wait prints (PostgreSQL's session `application_name`,
    +//! MongoDB's `extenddb_servers` document, Cassandra's `extenddb_servers` row).
    +//! The command releases the registration when it ends, whatever the outcome, so
    +//! a migration does not wait for a row or document to expire. A catalog the
    +//! factory refuses (a migration running, the catalog at another migration than
    +//! this binary's) ends the command with the message and the exit code `serve`
    +//! gives for it: 6 while a migration runs, 3 for a catalog behind this binary.
    +//! A registration lost during the command does not stop it: no supervisor runs
    +//! in a command, so a command that outlasts a lost registration runs beside a
    +//! migration that started after the loss. The signal listener is up before the
    +//! open in every subcommand, so a Ctrl-C during the open or the identity read
    +//! ends the command once the open returns, with the registration released.
    +//!
    +//! `extenddb destroy` removes the databases and leaves the store untouched:
    +//! the store is the copy that survives destroy, and this command is the one
    +//! that deletes it.
    +//!
    +//! `backup remove-target --account  
    ` removes a restore target +//! DeleteTable refuses (a CREATING table) with its restore record, under the +//! maintenance claim like `purge`: a FAILED record's target whatever the +//! reason (`target_not_empty` included, the one state the reconciler leaves +//! for this command: on MongoDB and Cassandra a crash before the first +//! progress record, the whole first data file on MongoDB and the first +//! 10,000 items or 32 MiB on Cassandra, ends in it), a RUNNING record's +//! target whose owner is dead (no heartbeat within `stale_job_after_secs`; +//! on SQLite under the serve lock every owner is dead), and a legacy +//! record's target the same way. It refuses a RUNNING record whose owner is +//! live, a DONE record and a table no record names ("not a restore target"; +//! an ACTIVE table is one or the other). It needs no store: it opens the +//! catalog as `serve` does (on SQLite under the serve lock) whether or not +//! `[backup.store]` is configured, prints the claim it took and the one line +//! of what it removed (`extenddb_engine::maintenance::remove_target` states +//! each step), and logs the removal under `extenddb::audit` with the +//! account, the table and the backup's ARN. +//! +//! `backup sync` rebuilds the catalog from the configured store: it gives an +//! AVAILABLE row to every backup prefix of the store that has a verified +//! manifest and no row, and reports the prefixes it skips with the reason +//! and the AVAILABLE rows whose manifest is gone (`extenddb_engine::sync` +//! states what it adopts and never adopts). It opens the catalog as `serve` +//! does (on SQLite under the serve lock, after the dead-owner resolutions +//! purge runs) and needs the store healthy: an unhealthy store refuses it, +//! naming the problem, with no row changed. On PostgreSQL, MongoDB and +//! Cassandra every sync takes the maintenance claim and heartbeats it, so +//! no purge removes a row and deletes its bytes between this sync's read +//! and its insert; plain sync waits for no job (its insert-if-absent orders +//! it against a CreateBackup), and while it holds the claim the jobs +//! refuse, as during purge. `--prune-missing` and `--adopt-store` also wait +//! for running jobs as purge does. A first `SIGINT` or `SIGTERM` stops any +//! sync before its next prefix (or the wait at once), releases the claim +//! and exits 1. `--prune-missing` removes each AVAILABLE row whose manifest +//! is gone outright and deletes no object, and reports a row a restore is +//! reading or another writer moved. `--adopt-store` is the way a durable +//! catalog takes a root whose identity file names another catalog (a server +//! on an in-memory database adopts, at startup, a root another in-memory +//! catalog wrote): it rewrites the file's owner +//! (`extenddb_engine::store_identity::adopt`), logged under +//! `extenddb::audit`, syncs, then reads the file once more and fails naming +//! the owner if another catalog adopted the root meanwhile; on a root this +//! catalog already owns it says so and syncs. Every server of the catalog +//! the file names is to be stopped before the adoption: the previous owner +//! acts on the root until a later check that can read its catalog finds +//! the store owned by another catalog, except where its deletion fence +//! stops it first (a re-read of the file before each deletion, before +//! DeleteBackup's and purge's row change and before a CreateBackup's first +//! put, which leaves a gap of one read). The refusal and the output say so. Sync +//! exits 0 once it ran, skipped +//! prefixes included, and 1 on a refusal, a stop, a listing that failed, a +//! row `--prune-missing` left for a later run (a store or catalog call +//! failed, or a restore is reading the backup), or a prefix skipped for a +//! store or catalog error (the run is incomplete; a second run adopts it). +//! A row `--prune-missing` leaves because its manifest is back or another +//! writer moved it is reported, and the exit stays 0. +//! +//! Every `backup` command refuses an in-memory SQLite database (a +//! `[storage.sqlite] path` naming no file) before it opens the catalog: it +//! would open a new, empty catalog that ends with the command. A server on +//! such a database syncs with its store itself at every start. + +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use clap::{Args, Subcommand}; +use extenddb_config as config; +use extenddb_engine::maintenance::{ + Interrupt, RemoveTarget, Stop, Wait, WaitOutcome, purge_rows, remove_target, wait_for_quiet, +}; +use extenddb_engine::{BackupRuntime, ConfiguredStore, HeldClaim, StoreHealth, StoreProblem}; +use extenddb_storage::StorageEngine; +use extenddb_storage::backup_catalog::{ + BackupRow, BackupStatus, JOB_HEARTBEAT_INTERVAL, JobStatus, StoreIdentity, +}; +use extenddb_storage::error::StorageError; +use extenddb_storage::hooks::ServingRegistrar; +use futures::TryStreamExt; + +#[derive(Args)] +#[allow(clippy::doc_markdown)] // Clap help text, not rustdoc +pub struct BackupArgs { + /// Path to configuration file (accepted before or after the subcommand) + #[arg(short, long, default_value = "extenddb.toml", global = true)] + config: String, + + #[command(subcommand)] + command: BackupCommand, +} + +#[derive(Subcommand)] +#[allow(clippy::doc_markdown)] // Clap help text, not rustdoc +enum BackupCommand { + /// List the catalog's backup rows for the configured backup store. A read that runs + /// beside a server, on an unhealthy store too. + List { + /// Only the FAILED rows (backups whose creation failed, kept for the operator) + #[arg(long)] + failed: bool, + }, + /// Delete every backup of the configured backup store: each backup's objects and its + /// catalog row. Leaves extenddb-store.json, exports/ and any prefix with no row. + /// Takes the maintenance claim first and waits for running jobs to finish. + Purge { + /// Confirm the deletion (required; without it the command prints what it would delete + /// and exits 0) + #[arg(long)] + confirm: bool, + }, + /// Remove a restore target DeleteTable refuses (a CREATING table) and its restore record: + /// the target of a FAILED restore (target_not_empty included) or of a RUNNING restore whose + /// owner is dead. Takes the maintenance claim first. Refuses a RUNNING restore with a live + /// owner, a DONE restore and a table no restore record names. + RemoveTarget { + /// The account the table belongs to + #[arg(long)] + account: String, + /// The table's name + table: String, + }, + /// Give a catalog row to every backup of the configured backup store that has none: each + /// prefix with a verified manifest (its sidecar matching, every data file present at its + /// size, its ARN naming the prefix's account, table and backup id) gets an AVAILABLE row. + /// Reports the prefixes it skips and the AVAILABLE rows whose manifest is gone. Refuses on + /// an unhealthy store. Beside servers (PostgreSQL, MongoDB, Cassandra) it takes the + /// maintenance claim; on SQLite it needs the server stopped. + Sync { + /// Make this catalog the owner of a store root whose extenddb-store.json names another + /// catalog, then sync. Stop every server of the catalog the file names first. Takes the + /// maintenance claim first. + #[arg(long)] + adopt_store: bool, + /// Also remove the AVAILABLE rows whose manifest is gone from the store. Each row is + /// removed outright and no object is deleted (the bytes may be on another host's root + /// of the same path). Takes the maintenance claim first and waits for running jobs. + #[arg(long)] + prune_missing: bool, + }, +} + +/// How often the waiting purge re-reads the rows. +const WAIT_POLL: Duration = Duration::from_secs(2); + +/// The commands' names, as the catalog's other sessions see them: on +/// PostgreSQL, MongoDB and Cassandra every command registers as the +/// catalog's server does, so a migration waits for it, and the registration +/// carries this name, which the migration prints while it waits. +const LIST: &str = "extenddb backup list"; +const PURGE: &str = "extenddb backup purge"; +const REMOVE_TARGET: &str = "extenddb backup remove-target"; +const SYNC: &str = "extenddb backup sync"; + +pub async fn run(args: BackupArgs, release: &'static str) -> anyhow::Result<()> { + if !std::path::Path::new(&args.config).exists() { + anyhow::bail!( + "Config file '{}' not found; use --config to specify its location.", + args.config, + ); + } + let app_config = config::load(&args.config)?; + // An in-memory database is a new catalog for each process that opens + // it: a command would read an empty catalog, and a writing one would + // record its work in a catalog that ends with it (an adopted root + // owned by a catalog nothing can open again). A location that does not + // parse is not in memory: its own error is printed, as `serve` does. + if app_config.storage.backend == "sqlite" { + match app_config.storage.database_location() { + Ok(Some(_)) => {} + Ok(None) => anyhow::bail!( + "the SQLite database is in memory ([storage.sqlite] path names no file), so \ + this command would open a new, empty catalog that ends with the command; a \ + server on an in-memory database syncs its catalog with the configured backup \ + store itself at every start" + ), + Err(e) => anyhow::bail!("{e}"), + } + } + // The signal listener is up before the catalog is opened, in every + // subcommand: a signal during the open or the identity read is acted on + // once the open returns, with the serving registration released, where + // a listener started after the open would leave the registration for + // `migrate --yes` to wait out. During the purge the same handle ends the + // wait at once and the deletion loop before its next row. + let interrupt = listen_for_signals(); + if let BackupCommand::RemoveTarget { account, table } = &args.command { + // The catalog as `serve` opens it; the store is not needed, so it is + // described and never checked, and the default root and an identity + // file are not created. + let (catalog, _) = open_catalog( + &app_config, + &args.config, + release, + REMOVE_TARGET, + Mode::WriteCatalogOnly, + ) + .await?; + if let Some(signal) = interrupt.signalled() { + catalog.close().await; + anyhow::bail!("{signal} received while opening the catalog; nothing was changed"); + } + let done = remove_restore_target(&catalog, &app_config, account, table).await; + catalog.close().await; + return done; + } + let (command, mode) = match args.command { + BackupCommand::List { .. } => (LIST, Mode::Read), + BackupCommand::Purge { confirm: false } => (PURGE, Mode::Read), + BackupCommand::Purge { confirm: true } => (PURGE, Mode::Write), + BackupCommand::Sync { .. } => (SYNC, Mode::Sync), + BackupCommand::RemoveTarget { .. } => unreachable!("handled above"), + }; + let opening = open(&app_config, &args.config, release, command, mode).await?; + if let Some(signal) = interrupt.signalled() { + if let Opening::Store(opened) = opening { + opened.close().await; + } + anyhow::bail!("{signal} received while opening the catalog; nothing was changed"); + } + match (args.command, opening) { + (BackupCommand::List { failed }, Opening::Store(opened)) => { + let done = list(&opened, failed).await; + opened.close().await; + done + } + (BackupCommand::List { .. }, Opening::NoStore(reason)) => { + println!("=== backups in the configured store: none ({reason}) ==="); + println!(" (none)"); + Ok(()) + } + // The dry run is a read, as `list` is: no serve lock, nothing + // created, the plan from the rows of the root; with no store, the + // plan is empty and says why, as `list` does. + (BackupCommand::Purge { confirm: false }, Opening::Store(opened)) => { + let done = dry_run(&opened).await; + opened.close().await; + done + } + (BackupCommand::Purge { confirm: false }, Opening::NoStore(reason)) => { + println!("=== extenddb backup purge ==="); + println!("Store: none ({reason})"); + println!(); + println!("--- Backups to delete (each backup's objects and its catalog row):"); + println!(" (none)"); + println!(); + println!("Dry run: nothing was deleted."); + Ok(()) + } + (BackupCommand::Purge { confirm: true }, Opening::Store(opened)) => { + let done = purge(&opened, &app_config, interrupt).await; + opened.close().await; + done + } + (BackupCommand::Purge { confirm: true }, Opening::NoStore(reason)) => { + anyhow::bail!("{reason}") + } + (BackupCommand::RemoveTarget { .. }, _) => unreachable!("handled above"), + ( + BackupCommand::Sync { + adopt_store, + prune_missing, + }, + Opening::Store(opened), + ) => { + let done = sync(&opened, &app_config, adopt_store, prune_missing, interrupt).await; + opened.close().await; + done + } + (BackupCommand::Sync { .. }, Opening::NoStore(reason)) => anyhow::bail!("{reason}"), + } +} + +/// What `open` found: the configured store, or why there is none. +enum Opening { + Store(Opened), + NoStore(String), +} + +/// How a command opens the catalog. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Mode { + /// A read beside a running server: no serve lock, no startup recovery, + /// the identity file read and never written. + Read, + /// A write: on SQLite under the serve lock, as `serve` opens it, but + /// without the startup recovery a server runs (on PostgreSQL the vector + /// index rebuild, the control-plane recovery and the GSI queue workers, + /// on MongoDB the key-repair pass, on Cassandra the control-plane + /// transitions), which belongs to the servers the command runs beside + /// and which the command must not exit with half done; the store's + /// check may write the identity file of an empty root. + Write, + /// As `Write`, for `backup sync`: a defaulted root another catalog owns + /// is configured, unhealthy, rather than left unconfigured with the + /// warning (`extenddb_server::backup_store::Caller::SyncCommand`). + Sync, + /// As `serve` opens it, for a command that needs no store + /// (`remove-target`): the store is described as `Read` leaves it, so + /// neither the default root nor an identity file is created. + WriteCatalogOnly, +} + +/// The open catalog: the storage engine, the backup runtime (with the +/// configured store when there is one), the backend's name, and the serving +/// registration the command holds for its run. +struct Catalog { + storage: Arc, + runtime: Arc, + backend: String, + registration: Option>, +} + +impl Catalog { + /// End the command's use of the catalog: release its serving + /// registration. + async fn close(self) { + release_registration(self.registration).await; + } +} + +/// Open the catalog as `command` in `mode` and describe the configured +/// store without checking it; the second member is why no store is +/// configured, when none is. A catalog the factory refuses (a migration +/// running, or the catalog at another migration than this binary's) is an +/// [`extenddb_server::CatalogRefusal`], with the message and exit code +/// `serve` gives for the same status. +async fn open_catalog( + app_config: &config::AppConfig, + config_path: &str, + release: &'static str, + command: &'static str, + mode: Mode, +) -> anyhow::Result<( + Catalog, + Option, +)> { + let mut options = extenddb_storage::server_components::ServerComponentsOptions::default(); + options.release = release; + options.max_concurrent_jobs = app_config.backup.max_concurrent_jobs; + options.beside_server = mode == Mode::Read; + options.startup_recovery = false; + options.command = Some(command); + let mut storage_config = app_config.storage.clone(); + storage_config.set_instance_id(&format!( + "{}:backup-command", + extenddb_server::backup_store::hostname() + )); + let components = extenddb_storage::create_server_components( + storage_config.as_trait(), + &app_config.server.region, + options, + ) + .await; + let components = match components { + Ok(components) => components, + Err(extenddb_storage::server_components::BackendError::Catalog(error)) => { + return Err(extenddb_server::catalog_refusal( + &error, + release, + app_config, + Some(config_path), + ) + .await + .into()); + } + Err(e) => anyhow::bail!("cannot open the catalog: {e}"), + }; + // The registration is the runtime hooks', which the command does not + // otherwise keep: held here, it lasts until `Catalog::close`. + let registration = components + .runtime_hooks + .as_ref() + .and_then(|hooks| hooks.serving_registrar()); + let storage = components.engine; + let backend = app_config.storage.backend.clone(); + let setup = extenddb_server::backup_store::configure_described( + &app_config.backup, + app_config.storage.database_file().as_deref(), + &backend, + &*storage, + match mode { + Mode::Read | Mode::WriteCatalogOnly => { + extenddb_server::backup_store::Caller::ReadOnlyCommand + } + Mode::Write => extenddb_server::backup_store::Caller::Command, + Mode::Sync => extenddb_server::backup_store::Caller::SyncCommand, + }, + ) + .await; + let setup = match setup { + Ok(setup) => setup, + Err(e) => { + release_registration(registration).await; + return Err(e); + } + }; + Ok(( + Catalog { + storage, + runtime: Arc::new(setup.runtime), + backend, + registration, + }, + setup.not_configured, + )) +} + +/// The catalog with the configured store and the store's identity as the +/// command read it. +struct Opened { + catalog: Catalog, + identity: Identity, +} + +/// The store's identity as the command read it. +enum Identity { + /// The check passed, or the file was read and agrees with the rows. + Known(StoreIdentity), + /// The root has no identity file and the catalog no rows for it: a + /// fresh root nothing has written to. + Empty, + /// The check failed (a listing still runs; a purge refuses). + Problem(StoreProblem), +} + +impl Opened { + /// End the command's use of the catalog: release its serving + /// registration. + async fn close(self) { + self.catalog.close().await; + } + + fn store(&self) -> &ConfiguredStore { + self.catalog + .runtime + .configured_store() + .expect("open returns only with a configured store") + } +} + +async fn open( + app_config: &config::AppConfig, + config_path: &str, + release: &'static str, + command: &'static str, + mode: Mode, +) -> anyhow::Result { + let (catalog, not_configured) = + open_catalog(app_config, config_path, release, command, mode).await?; + let Some(store) = catalog.runtime.configured_store() else { + // Why there is none, as `configure_described` found it: the section + // absent or disabled, the default root not yet created, or the + // default root failed its check with no rows for it. + let why = not_configured.expect("a runtime without a store carries the reason"); + catalog.close().await; + return Ok(Opening::NoStore(why.to_string())); + }; + let storage = &catalog.storage; + let backend = &catalog.backend; + // The server's startup leaves an S3 store's check to its reconciler; + // the command runs it here. A read leaves the root as it finds it: the + // identity file is read, never created, under the store's check bound as + // the server's check is, so an endpoint that accepts and never answers + // or a mount that has stopped answering costs the command the bound and + // not the store's retries. + let identity = match (mode, store.health()) { + (Mode::Read | Mode::WriteCatalogOnly, _) => { + match extenddb_engine::read_identity_bounded(store, &**storage).await { + Ok(Some(identity)) => Identity::Known(identity), + Ok(None) => Identity::Empty, + Err(problem) => Identity::Problem(problem), + } + } + (Mode::Write | Mode::Sync, StoreHealth::Checking) => { + let outcome = extenddb_engine::check_configured_store(store, &**storage, backend).await; + store.record_check(outcome.clone()); + match outcome { + Ok(identity) => Identity::Known(identity), + Err(problem) => Identity::Problem(problem), + } + } + (Mode::Write | Mode::Sync, StoreHealth::Healthy) => match store.identity() { + Some(identity) => Identity::Known(identity), + None => Identity::Problem(StoreProblem::Catalog( + "the store's id is unknown".to_owned(), + )), + }, + (Mode::Write | Mode::Sync, StoreHealth::Unhealthy(problem)) => Identity::Problem(problem), + }; + Ok(Opening::Store(Opened { catalog, identity })) +} + +/// How long a command waits for its registration's release: a row or +/// document whose delete does not answer in time expires on its own (45 s +/// after its last renewal on Cassandra and on MongoDB), and PostgreSQL's +/// session ends with the process. +const RELEASE_TIMEOUT: Duration = Duration::from_secs(10); + +/// Release a command's serving registration, if it holds one, so a +/// migration that waits for the registrations does not wait for it to +/// expire. Bounded by [`RELEASE_TIMEOUT`]. +async fn release_registration(registration: Option>) { + release_registration_within(registration, RELEASE_TIMEOUT).await; +} + +async fn release_registration_within( + registration: Option>, + bound: Duration, +) { + if let Some(registration) = registration + && tokio::time::timeout(bound, registration.release()) + .await + .is_err() + { + tracing::warn!( + "the command's serving registration was not released within {} s; it expires on \ + its own", + bound.as_secs() + ); + } +} + +async fn rows_of( + storage: &dyn StorageEngine, + identity: &StoreIdentity, + status: Option, +) -> anyhow::Result> { + storage + .list_backup_rows(status, identity) + .try_collect() + .await + .map_err(|e| anyhow::anyhow!("cannot list the catalog's backup rows: {e}")) +} + +fn print_row(row: &BackupRow) { + println!( + "{:<9} {:<12} {:<32} {:>12} {:>10} {} store_root={} store_id={}{}", + row.status, + row.account_id, + row.table_name, + row.size_bytes, + row.item_count, + row.backup_arn, + row.store.store_root, + row.store.store_id, + row.failure_reason + .as_deref() + .map(|r| format!(" failure_reason={r:?}")) + .unwrap_or_default() + ); +} + +/// Every row of the store's kind and root, whatever id it carries: the ids +/// the rows record, plus the file's when the check read one. +async fn rows_of_root( + opened: &Opened, + status: Option, +) -> anyhow::Result<(Vec, Vec)> { + let store = opened.store(); + let mut ids = opened + .catalog + .storage + .read_store_ids(store.kind, &store.root) + .await + .map_err(|e| anyhow::anyhow!("cannot read the catalog's store ids: {e}"))?; + if let Identity::Known(identity) = &opened.identity + && !ids.contains(&identity.store_id) + { + ids.push(identity.store_id.clone()); + } + let mut rows = Vec::new(); + for id in &ids { + let identity = StoreIdentity { + store_kind: store.kind, + store_root: store.root.clone(), + store_id: id.clone(), + }; + rows.extend(rows_of(&*opened.catalog.storage, &identity, status).await?); + } + Ok((ids, rows)) +} + +async fn list(opened: &Opened, failed: bool) -> anyhow::Result<()> { + let status = failed.then_some(BackupStatus::Failed); + let store = opened.store(); + let (ids, rows) = rows_of_root(opened, status).await?; + match &opened.identity { + Identity::Known(identity) => println!( + "=== backups in the {} store {} (store id {}) ===", + store.kind, store.root, identity.store_id + ), + Identity::Empty => println!( + "=== backups in the {} store {} (the root has no {} yet; a server's first check \ + creates it) ===", + store.kind, + store.root, + extenddb_engine::STORE_IDENTITY_KEY + ), + Identity::Problem(problem) => println!( + "=== backups in the {} store {} (the store is unhealthy: {problem}; rows of every \ + id the catalog records for the root are listed) ===", + store.kind, store.root + ), + } + if ids.len() > 1 { + println!( + " the catalog's rows for this root carry {} store ids: {}", + ids.len(), + ids.join(", ") + ); + } + if rows.is_empty() { + println!(" (none)"); + return Ok(()); + } + for row in &rows { + print_row(row); + } + println!("{} row(s)", rows.len()); + Ok(()) +} + +/// The plan a purge prints before it deletes, and the dry run's whole +/// output: the store, its rows, and what stays. +async fn print_plan(opened: &Opened) -> anyhow::Result<()> { + let configured = opened.store(); + println!("=== extenddb backup purge ==="); + match &opened.identity { + Identity::Known(identity) => println!( + "Store: {} {} (store id {})", + identity.store_kind, identity.store_root, identity.store_id + ), + Identity::Empty => println!( + "Store: {} {} (no {} yet)", + configured.kind, + configured.root, + extenddb_engine::STORE_IDENTITY_KEY + ), + Identity::Problem(problem) => println!( + "Store: {} {} (unhealthy: {problem})", + configured.kind, configured.root + ), + } + let (_, planned) = rows_of_root(opened, None).await?; + println!(); + println!("--- Backups to delete (each backup's objects and its catalog row):"); + if planned.is_empty() { + println!(" (none)"); + } + for row in &planned { + print_row(row); + } + println!(); + println!( + "Left in place: {}, exports/, and any prefix the catalog has no row for.", + extenddb_engine::STORE_IDENTITY_KEY + ); + println!(); + Ok(()) +} + +/// `backup purge` without `--confirm`: the plan, nothing changed, exit 0. +/// On an unhealthy store the plan still lists the rows, and the closing +/// line says that `--confirm` refuses until the store's check passes. +async fn dry_run(opened: &Opened) -> anyhow::Result<()> { + print_plan(opened).await?; + match &opened.identity { + Identity::Problem(problem) => println!( + "Dry run: nothing was deleted. The store is unhealthy ({}); --confirm refuses until \ + the store's check passes.", + problem.category() + ), + Identity::Known(_) | Identity::Empty => println!( + "Dry run: nothing was deleted. Run again with --confirm to permanently delete every \ + backup of the configured store listed above." + ), + } + Ok(()) +} + +async fn purge( + opened: &Opened, + app_config: &config::AppConfig, + mut interrupt: Interrupt, +) -> anyhow::Result<()> { + let backup = &app_config.backup; + let storage = &*opened.catalog.storage; + let configured = opened.store(); + let identity = match &opened.identity { + Identity::Known(identity) => identity, + Identity::Empty => anyhow::bail!( + "the {} backup store {} has no {} and the catalog has no rows for it: nothing to purge", + configured.kind, + configured.root, + extenddb_engine::STORE_IDENTITY_KEY + ), + Identity::Problem(problem) => anyhow::bail!( + "the {} backup store {} is unhealthy: {problem}; fix that before purging its backups", + configured.kind, + configured.root + ), + }; + let store = &*configured.store; + let fence = configured.fence(); + print_plan(opened).await?; + + let stale_after = Duration::from_secs(backup.stale_job_after_secs); + let holder = format!( + "extenddb backup purge on {} (pid {})", + extenddb_server::backup_store::hostname(), + std::process::id() + ); + // The signal listener has been up since before the open, so a signal + // from here on ends the command with the claim released: at once during + // the wait, before the next row during the deletion. + let claim = storage + .maintenance_claim(&holder, stale_after) + .await + .map_err(|e| match e { + StorageError::AlreadyExists(m) => anyhow::anyhow!( + "{m}; one maintenance command runs at a time, and a claim whose holder died \ + expires after stale_job_after_secs ({} s)", + backup.stale_job_after_secs + ), + other => anyhow::anyhow!("cannot take the maintenance claim: {other}"), + })?; + println!("Took the maintenance claim as {holder:?}."); + let claim = HeldClaim::start(claim, JOB_HEARTBEAT_INTERVAL); + + // On SQLite the command holds the file alone: every job owner is dead + // and no reconciler runs, so their resolution is this command's, and the + // owner of any row is dead whatever its heartbeat says, so the window a + // live owner is told apart by is zero. On the other backends purge runs + // beside live servers, whose jobs heartbeat within `stale_after`. + let owner_window = if opened.catalog.backend == "sqlite" { + resolve_dead_owners(opened).await; + Duration::ZERO + } else { + stale_after + }; + + // Wait for the jobs the claim excludes from starting to finish. + let wait_limit = stale_after * 2; + let started = Instant::now(); + let outcome = wait_for_quiet( + storage, + identity, + &claim, + Wait { + owner_window, + limit: wait_limit, + poll: WAIT_POLL, + }, + &mut interrupt, + |remaining| { + println!( + "Waiting for running jobs to finish: {}", + remaining.join("; ") + ) + }, + ) + .await; + match outcome { + WaitOutcome::Quiet => {} + WaitOutcome::Busy(remaining) => { + release(claim).await; + anyhow::bail!( + "jobs were still running after waiting {} s (2 * stale_job_after_secs): {}. The \ + maintenance claim is released; retry once they finish", + started.elapsed().as_secs(), + remaining.join("; ") + ); + } + WaitOutcome::Lost(reason) => { + // The release is conditioned on the generation: another holder's + // claim is left to it. + release(claim).await; + anyhow::bail!("stopped before any deletion: {reason}; retry once the claim is free"); + } + WaitOutcome::CatalogError(e) => { + release(claim).await; + anyhow::bail!( + "stopped before any deletion: {e}. The maintenance claim is released; retry once \ + the catalog answers" + ); + } + WaitOutcome::Interrupted(signal) => { + release(claim).await; + anyhow::bail!( + "stopped before any deletion: {signal} received while waiting. The maintenance \ + claim is released" + ); + } + } + + // Re-read under the claim: the plan printed before it may be stale. + let rows = match rows_of(storage, identity, None).await { + Ok(rows) => rows, + Err(e) => { + release(claim).await; + return Err(e); + } + }; + let outcome = purge_rows( + storage, + store, + &fence, + &rows, + owner_window, + &claim, + &interrupt, + |row| { + tracing::info!( + target: "extenddb::audit", + action = "backup purge", + account_id = %row.account_id, + backup_arn = %row.backup_arn, + store_root = %row.store.store_root, + "deleted backup objects and catalog row" + ); + println!("deleted {} ({})", row.backup_arn, row.status); + }, + ) + .await; + release(claim).await; + let deleted = outcome.deleted.len(); + println!(); + println!( + "=== extenddb backup purge complete: {deleted} backup(s) deleted, {} left ===", + rows.len() - deleted + ); + match outcome.stopped { + Some(Stop::Lost(reason)) => { + anyhow::bail!( + "stopped before the remaining rows: {reason}; retry once the claim is free" + ) + } + Some(Stop::Interrupted(signal)) => anyhow::bail!( + "stopped before the remaining rows: {signal} received. The maintenance claim is \ + released; run the command again for the rest" + ), + None => {} + } + if !outcome.skipped.is_empty() { + for line in &outcome.skipped { + println!(" left: {line}"); + } + anyhow::bail!( + "{} backup(s) could not be deleted; see above", + outcome.skipped.len() + ); + } + Ok(()) +} + +/// The handle every subcommand reads a signal from, fed by a task that waits +/// for `SIGINT` (Ctrl-C) or `SIGTERM`. The first signal stops the command at +/// its next safe point (after the open, the wait at once, the deletion loop +/// before its next row) with its serving registration and, in a purge, the +/// claim released; a second one exits the process at once, the registration +/// then expiring on its own and the claim by the stale rule (on SQLite the +/// next server start clears it). +fn listen_for_signals() -> Interrupt { + let (tx, interrupt) = Interrupt::new(); + tokio::spawn(async move { + let name = signal_name().await; + println!( + "{name} received; finishing the current step and releasing what the command holds (a \ + second {name} exits at once)." + ); + let _ = tx.send(Some(name)); + let again = signal_name().await; + println!( + "{again} received again; exiting now. The serving registration expires on its own and \ + a maintenance claim by the stale rule." + ); + std::process::exit(130); + }); + interrupt +} + +#[cfg(unix)] +async fn signal_name() -> &'static str { + use tokio::signal::unix::{SignalKind, signal}; + let ctrl_c = tokio::signal::ctrl_c(); + let Ok(mut sigterm) = signal(SignalKind::terminate()) else { + let _ = ctrl_c.await; + return "SIGINT"; + }; + tokio::select! { + _ = ctrl_c => "SIGINT", + _ = sigterm.recv() => "SIGTERM", + } +} + +#[cfg(not(unix))] +async fn signal_name() -> &'static str { + let _ = tokio::signal::ctrl_c().await; + "Ctrl-C" +} + +/// The reconciler's duties, run once with a zero stale window: on SQLite +/// under the serve lock every owner is dead. The command holds the +/// maintenance claim, and its state says so, so the cycle takes the held +/// branch without reading the claim (read through a zero window, the +/// command's own claim would read as held only within one instant of its +/// heartbeat). What was resolved is printed; a failure is printed and the +/// wait decides. +async fn resolve_dead_owners(opened: &Opened) { + match extenddb_engine::reconciler::cycle( + &opened.catalog.storage, + &opened.catalog.runtime, + Duration::ZERO, + &opened.catalog.backend, + &mut extenddb_engine::reconciler::ReconcilerState::for_maintenance_holder(), + ) + .await + { + Ok(report) => { + for table in &report.legacy_restores_removed { + println!("removed the restore target {table}: its restore did not finish"); + } + for table in &report.failed_targets_removed { + println!("removed the restore target {table}: its restore had failed"); + } + for table in &report.targets_left_to_owner { + println!( + "left the table {table}: it is no longer the restore target its record names" + ); + } + for arn in &report.legacy_backups_removed { + println!("removed the in-catalog backup {arn}: its create did not finish"); + } + for arn in &report.deletions_finished { + println!("finished the deletion of {arn}"); + } + for (arn, status) in &report.creating_resolved { + println!( + "resolved the CREATING backup {arn} to {status}: its owner is dead{}", + match status { + extenddb_storage::backup_catalog::BackupStatus::Available => + " and the store holds its complete manifest", + _ => + " and the store holds no complete manifest; its objects are \ + deleted once the row's heartbeat is stale", + } + ); + } + for arn in &report.creating_left_to_owner { + println!( + "left the CREATING backup {arn}: another owner took it over before this \ + command's write" + ); + } + } + Err(e) => println!("could not resolve the jobs the stopped server left: {e}"), + } +} + +/// Release the claim; a failure is printed, since the claim then expires by +/// the stale rule and the command's own outcome is what the operator needs. +async fn release(claim: HeldClaim) { + if let Err(e) = claim.release().await { + println!("could not release the maintenance claim ({e}); it expires on its own"); + } +} + +/// `backup remove-target --account
    ` (module doc). +async fn remove_restore_target( + catalog: &Catalog, + app_config: &config::AppConfig, + account_id: &str, + table_name: &str, +) -> anyhow::Result<()> { + let stale_after = Duration::from_secs(app_config.backup.stale_job_after_secs); + let holder = format!( + "extenddb backup remove-target on {} (pid {})", + extenddb_server::backup_store::hostname(), + std::process::id() + ); + println!("=== extenddb backup remove-target ==="); + println!("Taking the maintenance claim as {holder:?}."); + // On SQLite the command holds the file alone, so every job owner is + // dead whatever its heartbeat says; on the other backends a RUNNING + // record whose heartbeat is within `stale_after` has a live owner. + let owner_window = if catalog.backend == "sqlite" { + Duration::ZERO + } else { + stale_after + }; + let removed = remove_target( + &*catalog.storage, + &RemoveTarget { + account_id: account_id.to_owned(), + table_name: table_name.to_owned(), + holder, + stale_after, + owner_window, + heartbeat_interval: JOB_HEARTBEAT_INTERVAL, + }, + ) + .await + .map_err(|e| anyhow::anyhow!("{e}"))?; + let state = match (removed.status, removed.failure_reason.as_deref()) { + (JobStatus::Running, _) => "RUNNING with a dead owner".to_owned(), + (status, Some(reason)) => format!("{status} ({reason})"), + (status, None) => status.to_string(), + }; + let kind = match removed.kind { + extenddb_storage::backup_catalog::RestoreKind::Manifest => "store-backed", + extenddb_storage::backup_catalog::RestoreKind::Legacy => "in-catalog", + }; + println!( + "removed the restore target {table_name} (account {account_id}, table id {}) and its \ + record: its {kind} restore from {} was {state}", + removed.table_id, removed.backup_arn + ); + println!("The maintenance claim is released."); + Ok(()) +} + +/// `backup sync [--adopt-store] [--prune-missing]` (module doc). +#[allow(clippy::too_many_lines)] +async fn sync( + opened: &Opened, + app_config: &config::AppConfig, + adopt_store: bool, + prune_missing: bool, + mut interrupt: Interrupt, +) -> anyhow::Result<()> { + let storage = &*opened.catalog.storage; + let configured = opened.store(); + println!("=== extenddb backup sync ==="); + // The identity sync writes rows under, or the root to adopt. + let mut adopting = false; + let known = match &opened.identity { + Identity::Known(identity) => { + println!( + "Store: {} {} (store id {})", + identity.store_kind, identity.store_root, identity.store_id + ); + if adopt_store { + println!( + "The root's {} already names this catalog: there is nothing to adopt.", + extenddb_engine::STORE_IDENTITY_KEY + ); + } + Some(identity.clone()) + } + Identity::Problem(problem @ StoreProblem::ForeignCatalog { .. }) => { + println!( + "Store: {} {} (owned by another catalog: {problem})", + configured.kind, configured.root + ); + if !adopt_store { + anyhow::bail!( + "the {} backup store {} is owned by another catalog: {problem}. Sync changes \ + no row of a root another catalog owns. Stop every server of the catalog the \ + file names, then run with --adopt-store to make this catalog its owner: a \ + running server of that catalog acts on the root until a later check that \ + can read its catalog finds it owned by another catalog (its deletion fence refuses a deletion once the \ + file names another catalog, but a write already past the fence's read \ + lands). The default root beside a SQLite database file is shared by every \ + database file in its directory, and the adoption takes it from whichever \ + catalog wrote the file", + configured.kind, + configured.root + ); + } + adopting = true; + None + } + Identity::Problem(problem) => anyhow::bail!( + "the {} backup store {} is unhealthy: {problem}; sync changes no row while the store \ + is unhealthy", + configured.kind, + configured.root + ), + Identity::Empty => anyhow::bail!( + "the {} backup store {} has no {}: its check did not run", + configured.kind, + configured.root, + extenddb_engine::STORE_IDENTITY_KEY + ), + }; + + // On SQLite the command holds the serve lock, which already excludes + // every server and every other command. Elsewhere every sync takes the + // maintenance claim, so no purge removes a row between this sync's row + // read and its insert and then deletes the bytes it adopted; plain sync + // waits for no job, since its insert-if-absent orders it against a + // CreateBackup. The options wait for running jobs as purge does. + let sqlite = opened.catalog.backend == "sqlite"; + let waits = adopt_store || prune_missing; + let mut claim = None; + if !sqlite || waits { + let backup = &app_config.backup; + let stale_after = Duration::from_secs(backup.stale_job_after_secs); + let holder = format!( + "extenddb backup sync on {} (pid {})", + extenddb_server::backup_store::hostname(), + std::process::id() + ); + let taken = storage + .maintenance_claim(&holder, stale_after) + .await + .map_err(|e| match e { + StorageError::AlreadyExists(m) => anyhow::anyhow!( + "{m}; one maintenance command runs at a time, and a claim whose holder died \ + expires after stale_job_after_secs ({} s)", + backup.stale_job_after_secs + ), + other => anyhow::anyhow!("cannot take the maintenance claim: {other}"), + })?; + println!("Took the maintenance claim as {holder:?}."); + let held = HeldClaim::start(taken, JOB_HEARTBEAT_INTERVAL); + if waits + && let Err(refusal) = wait_for_jobs( + opened, + configured, + known.as_ref(), + &held, + &mut interrupt, + stale_after, + ) + .await + { + release(held).await; + return Err(refusal); + } + claim = Some(held); + } else { + resolve_dead_owners(opened).await; + } + + let done = sync_held( + opened, + app_config, + known, + adopting, + prune_missing, + claim.as_ref(), + &interrupt, + ) + .await; + if let Some(claim) = claim { + release(claim).await; + } + done +} + +/// The wait `--adopt-store` and `--prune-missing` run under the claim, as +/// purge's: on SQLite after the dead-owner resolutions with a zero window. +/// The refusal is the command's error, the claim still held. +async fn wait_for_jobs( + opened: &Opened, + configured: &ConfiguredStore, + known: Option<&StoreIdentity>, + claim: &HeldClaim, + interrupt: &mut Interrupt, + stale_after: Duration, +) -> anyhow::Result<()> { + let storage = &*opened.catalog.storage; + // The identity the wait reads rows of: the file's id, before the + // adoption as after it. + let wait_identity = match known { + Some(identity) => identity.clone(), + None => { + let file = tokio::time::timeout( + configured.check_timeout(), + extenddb_engine::store_identity::read_identity_file(&*configured.store), + ) + .await + .map_err(|_| { + anyhow::anyhow!( + "the store's {} could not be read within {} s", + extenddb_engine::STORE_IDENTITY_KEY, + configured.check_timeout().as_secs() + ) + })? + .map_err(|e| anyhow::anyhow!("{e}"))? + .ok_or_else(|| { + anyhow::anyhow!( + "the root's {} is gone; run the command again", + extenddb_engine::STORE_IDENTITY_KEY + ) + })?; + StoreIdentity { + store_kind: configured.kind, + store_root: configured.root.clone(), + store_id: file.store_id, + } + } + }; + let owner_window = if opened.catalog.backend == "sqlite" { + resolve_dead_owners(opened).await; + Duration::ZERO + } else { + stale_after + }; + let started = Instant::now(); + let outcome = wait_for_quiet( + storage, + &wait_identity, + claim, + Wait { + owner_window, + limit: stale_after * 2, + poll: WAIT_POLL, + }, + interrupt, + |remaining| { + println!( + "Waiting for running jobs to finish: {}", + remaining.join("; ") + ) + }, + ) + .await; + match outcome { + WaitOutcome::Quiet => Ok(()), + WaitOutcome::Busy(remaining) => Err(anyhow::anyhow!( + "jobs were still running after waiting {} s (2 * stale_job_after_secs): {}. The \ + maintenance claim is released; retry once they finish", + started.elapsed().as_secs(), + remaining.join("; ") + )), + WaitOutcome::Lost(reason) => Err(anyhow::anyhow!( + "stopped before any change: {reason}; retry once the claim is free" + )), + WaitOutcome::CatalogError(e) => Err(anyhow::anyhow!( + "stopped before any change: {e}. The maintenance claim is released; retry once the \ + catalog answers" + )), + WaitOutcome::Interrupted(signal) => Err(anyhow::anyhow!( + "stopped before any change: {signal} received while waiting. The maintenance claim \ + is released" + )), + } +} + +/// The adoption, the sync and the prune, with the claim held (on +/// PostgreSQL, MongoDB and Cassandra) or under the serve lock (SQLite); the +/// caller releases the claim. +#[allow(clippy::too_many_lines)] +async fn sync_held( + opened: &Opened, + app_config: &config::AppConfig, + known: Option, + adopting: bool, + prune_missing: bool, + claim: Option<&HeldClaim>, + interrupt: &Interrupt, +) -> anyhow::Result<()> { + let storage = &*opened.catalog.storage; + let configured = opened.store(); + let store = &*configured.store; + let catalog_id = storage + .catalog_id() + .await + .map_err(|e| anyhow::anyhow!("the catalog could not be read: {e}"))?; + let identity = match known { + Some(identity) => identity, + None => { + debug_assert!(adopting); + let adopted = tokio::time::timeout( + configured.check_timeout(), + extenddb_engine::store_identity::adopt( + store, + configured.kind, + &configured.root, + storage, + &opened.catalog.backend, + false, + extenddb_engine::store_identity::AdoptRule::Command, + ), + ) + .await; + let adopted = match adopted { + Ok(Ok(adopted)) => adopted, + Ok(Err( + refusal @ extenddb_engine::store_identity::AdoptRefusal::AdoptedUnhealthy { + .. + }, + )) => { + anyhow::bail!( + "--adopt-store rewrote the identity file, so the root is taken from its \ + previous owner, but the sync did not run: {refusal}" + ); + } + Ok(Err(refusal)) => { + anyhow::bail!("--adopt-store did not adopt the root: {refusal}"); + } + Err(_) => anyhow::bail!( + "--adopt-store did not finish within {} s (the store stopped answering); \ + the audit log's intent line says whether the rewrite was started. Run the \ + command again", + configured.check_timeout().as_secs() + ), + }; + let identity = match adopted { + extenddb_engine::store_identity::Adoption::Adopted { + identity, + previous_catalog_id, + } => { + println!( + "Adopted the root: {} named catalog {previous_catalog_id} and now names \ + this catalog (store id {}). A server of catalog {previous_catalog_id} \ + that still runs acts on the root until a later check that can read its \ + catalog finds it owned by another catalog; its deletion fence refuses a deletion and a \ + CreateBackup once it re-reads the file. Stop every such server before \ + an adoption, and start a new in-memory server only after the old one has \ + exited.", + extenddb_engine::STORE_IDENTITY_KEY, + identity.store_id + ); + identity + } + extenddb_engine::store_identity::Adoption::AlreadyOwned(identity) => { + println!( + "The root's {} names this catalog now: there is nothing to adopt.", + extenddb_engine::STORE_IDENTITY_KEY + ); + identity + } + }; + configured.record_check(Ok(identity.clone())); + identity + } + }; + if let Some(reason) = claim.and_then(HeldClaim::lost) { + anyhow::bail!("stopped before the sync: {reason}; retry once the claim is free"); + } + + let stop = || { + claim.and_then(HeldClaim::lost).or_else(|| { + interrupt + .signalled() + .map(|signal| format!("{signal} received")) + }) + }; + let report = extenddb_engine::sync::sync( + storage, + extenddb_engine::sync::SyncTarget { + store, + identity: &identity, + metrics: None, + region: Some(&app_config.server.region), + stop: Some(&stop), + }, + ) + .await; + let report = match report { + Ok(report) => report, + Err(extenddb_engine::sync::SyncError::Listing { error, partial }) => { + print_sync_report(&partial, prune_missing, false); + anyhow::bail!( + "the store's listing failed: {error}. {} backup(s) listed before the failure \ + were adopted, and no AVAILABLE row was examined for a missing manifest; run the \ + command again", + partial.adopted.len() + ); + } + Err(e) => anyhow::bail!("{e}"), + }; + print_sync_report(&report, prune_missing, report.stopped.is_none()); + if let Some(reason) = &report.stopped { + anyhow::bail!( + "stopped: {reason}. {} backup(s) were adopted before the stop, and no AVAILABLE row \ + was examined for a missing manifest; run the command again for the rest", + report.adopted.len() + ); + } + + let mut pruned = 0usize; + let mut prune_left = 0usize; + let mut prune_in_use = 0usize; + let mut stopped = None; + if prune_missing && !report.missing_manifests.is_empty() { + let (outcomes, stop_reason) = extenddb_engine::sync::prune_missing( + storage, + store, + &configured.root, + &report.missing_manifests, + &stop, + ) + .await; + println!(); + println!("--- --prune-missing (rows removed; no object is deleted):"); + for (arn, outcome) in &outcomes { + println!(" {arn}: {outcome}"); + if *outcome == extenddb_engine::sync::PruneOutcome::Pruned { + pruned += 1; + } else if outcome.needs_another_run() { + prune_left += 1; + if matches!(outcome, extenddb_engine::sync::PruneOutcome::InUse(_)) { + prune_in_use += 1; + } + } + } + stopped = stop_reason; + } + + // The file once more: an adopter whose put landed after this one's + // re-read and during the sync is detected here. + if adopting { + let file = tokio::time::timeout( + configured.check_timeout(), + extenddb_engine::store_identity::read_identity_file(store), + ) + .await + .ok(); + owner_after_sync(file, &catalog_id, configured.check_timeout())?; + } + + println!(); + println!( + "=== extenddb backup sync {}: {} backup(s) adopted, {} prefix(es) skipped, {} \ + AVAILABLE row(s) whose manifest is gone{} ===", + if stopped.is_some() { + "stopped" + } else { + "complete" + }, + report.adopted.len(), + report.skipped.len(), + report.missing_manifests.len(), + match (prune_missing, &stopped) { + (false, _) => String::new(), + (true, None) => format!(" ({pruned} removed)"), + (true, Some(_)) => format!(" ({pruned} removed before the stop)"), + }, + ); + if let Some(reason) = stopped { + anyhow::bail!("--prune-missing stopped before the remaining rows: {reason}"); + } + if prune_left > 0 { + anyhow::bail!( + "{prune_left} row(s) whose manifest is gone were left for a later run (see above){}", + if prune_in_use > 0 { + format!( + "; {prune_in_use} of them are being restored from: run the command again \ + once the restore ends" + ) + } else { + "; run the command again once the store and the catalog answer".to_owned() + } + ); + } + complete_run(&report) +} + +/// The adopting sync's last read of `extenddb-store.json` (`file`, `None` +/// when the read did not finish within `bound`): `Ok` when it still names +/// this catalog (`catalog_id`), and otherwise the error naming what it +/// found, so an adopter whose put landed during the sync is reported. +fn owner_after_sync( + file: Option< + Result< + Option, + extenddb_engine::StoreProblem, + >, + >, + catalog_id: &str, + bound: Duration, +) -> anyhow::Result<()> { + match file { + Some(Ok(Some(file))) if file.catalog_id == catalog_id => Ok(()), + Some(Ok(Some(file))) => anyhow::bail!( + "after the sync, {} names catalog {} (store id {}): another catalog adopted the root \ + during the sync, and the rows this sync inserted are this catalog's rows of a root \ + it does not own", + extenddb_engine::STORE_IDENTITY_KEY, + file.catalog_id, + file.store_id + ), + Some(Ok(None)) => anyhow::bail!( + "after the sync, the root has no {}", + extenddb_engine::STORE_IDENTITY_KEY + ), + Some(Err(problem)) => anyhow::bail!( + "after the sync, {} could not be read: {problem}", + extenddb_engine::STORE_IDENTITY_KEY + ), + None => anyhow::bail!( + "after the sync, {} could not be read within {} s", + extenddb_engine::STORE_IDENTITY_KEY, + bound.as_secs() + ), + } +} + +/// A sync that ran to its end is complete only when no prefix was skipped +/// for a store or catalog error; otherwise the command exits 1, since a +/// later run adopts those prefixes. +fn complete_run(report: &extenddb_engine::sync::SyncReport) -> anyhow::Result<()> { + let transient = report.transient_skips(); + if transient > 0 { + anyhow::bail!( + "the run is incomplete: {transient} prefix(es) were skipped for a store or catalog \ + error (see above); run the command again once the store and the catalog answer" + ); + } + Ok(()) +} + +/// The report of a sync: what it adopted, what it skipped and why, the +/// AVAILABLE rows whose manifest is gone (`examined` is false when the run +/// ended before the end of the listing, which leaves them unexamined). +fn print_sync_report( + report: &extenddb_engine::sync::SyncReport, + prune_missing: bool, + examined: bool, +) { + println!(); + println!("--- Adopted (an AVAILABLE row inserted for each):"); + if report.adopted.is_empty() { + println!(" (none)"); + } + for adopted in &report.adopted { + match &adopted.foreign { + Some(note) => println!(" {} ({note})", adopted.backup_arn), + None => println!(" {}", adopted.backup_arn), + } + } + println!(); + println!("--- Skipped (not adopted):"); + if report.skipped.is_empty() { + println!(" (none)"); + } + for skipped in &report.skipped { + println!(" {}: {}", skipped.prefix, skipped.reason); + } + println!(); + println!( + "--- AVAILABLE rows whose manifest is not in the store (the rows were AVAILABLE when the \ + catalog was read):" + ); + if !examined { + println!(" (not examined)"); + } else if report.missing_manifests.is_empty() { + println!(" (none)"); + } + for key in &report.missing_manifests { + println!(" {} (account {})", key.backup_arn, key.account_id); + } + if !report.missing_manifests.is_empty() && !prune_missing { + println!( + " (left AVAILABLE; run with --prune-missing to remove the rows, which deletes no \ + object)" + ); + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::time::Duration; + + use async_trait::async_trait; + use extenddb_storage::hooks::{Reregistration, ServingRegistrar}; + + use super::{ + complete_run, owner_after_sync, release_registration, release_registration_within, + }; + + /// The identity file as a store holds it, naming `catalog_id`. + fn file(catalog_id: &str) -> extenddb_engine::StoreIdentityFile { + extenddb_engine::StoreIdentityFile { + format_version: 1, + store_id: "store-1".to_owned(), + catalog_id: catalog_id.to_owned(), + backend: "postgres".to_owned(), + created_at_ms: 0, + ephemeral_catalog: false, + } + } + + /// The adopting sync's last read: the file still naming this catalog + /// passes; another catalog (an adopter whose put landed during the + /// sync), no file, an unreadable file and a read past the bound each + /// fail, the first naming the new owner. + #[test] + fn the_adopting_syncs_last_read_fails_unless_the_file_names_this_catalog() { + let bound = Duration::from_secs(10); + owner_after_sync(Some(Ok(Some(file("catalog-1")))), "catalog-1", bound).unwrap(); + let err = owner_after_sync(Some(Ok(Some(file("adopter")))), "catalog-1", bound) + .unwrap_err() + .to_string(); + assert!( + err.starts_with( + "after the sync, extenddb-store.json names catalog adopter (store id \ + store-1): another catalog adopted the root during the sync" + ), + "{err}" + ); + let err = owner_after_sync(Some(Ok(None)), "catalog-1", bound) + .unwrap_err() + .to_string(); + assert_eq!(err, "after the sync, the root has no extenddb-store.json"); + let err = owner_after_sync( + Some(Err(extenddb_engine::StoreProblem::Transport( + "down".to_owned(), + ))), + "catalog-1", + bound, + ) + .unwrap_err() + .to_string(); + assert!( + err.starts_with("after the sync, extenddb-store.json could not be read: "), + "{err}" + ); + let err = owner_after_sync(None, "catalog-1", bound) + .unwrap_err() + .to_string(); + assert_eq!( + err, + "after the sync, extenddb-store.json could not be read within 10 s" + ); + } + + /// A sync that skipped a prefix for a store or catalog error exits 1, + /// naming the count; skips of any other reason leave it complete. + #[test] + fn a_sync_with_a_transient_skip_is_incomplete() { + use extenddb_engine::sync::{SkipReason, SkippedPrefix, SyncReport}; + let skipped = |reason| SkippedPrefix { + prefix: "123456789012/T/01700000000000-0000000a/".to_owned(), + reason, + }; + let lasting = || { + vec![ + skipped(SkipReason::Incomplete("no manifest".to_owned())), + skipped(SkipReason::RowExists("a row".to_owned())), + ] + }; + complete_run(&SyncReport { + skipped: lasting(), + ..SyncReport::default() + }) + .unwrap(); + for reason in [ + SkipReason::Store("get failed".to_owned()), + SkipReason::Catalog("insert failed".to_owned()), + ] { + let mut skips = lasting(); + skips.push(skipped(reason)); + let report = SyncReport { + skipped: skips, + ..SyncReport::default() + }; + let err = complete_run(&report).unwrap_err().to_string(); + assert!( + err.starts_with( + "the run is incomplete: 1 prefix(es) were skipped for a store or \ + catalog error" + ), + "{err}" + ); + } + } + + /// A registrar that counts its releases, each taking `takes`. + struct Counting { + released: AtomicUsize, + takes: Duration, + } + + #[async_trait] + impl ServingRegistrar for Counting { + async fn lost(&self) -> String { + std::future::pending().await + } + async fn register_again(&self) -> Reregistration { + Reregistration::Registered + } + async fn release(&self) { + tokio::time::sleep(self.takes).await; + self.released.fetch_add(1, Ordering::SeqCst); + } + } + + /// A command's registration is released when the command ends, so a + /// migration does not wait for its row or document to expire; a release + /// that does not answer is abandoned at the bound. + #[tokio::test] + async fn a_commands_registration_is_released_within_the_bound() { + let quick = Arc::new(Counting { + released: AtomicUsize::new(0), + takes: Duration::ZERO, + }); + release_registration(Some(quick.clone() as Arc)).await; + assert_eq!(quick.released.load(Ordering::SeqCst), 1); + + let stalled = Arc::new(Counting { + released: AtomicUsize::new(0), + takes: Duration::from_secs(3600), + }); + let bound = Duration::from_millis(100); + let started = std::time::Instant::now(); + release_registration_within(Some(stalled.clone() as Arc), bound) + .await; + assert_eq!(stalled.released.load(Ordering::SeqCst), 0); + assert!( + started.elapsed() < Duration::from_secs(5), + "{:?}", + started.elapsed() + ); + + release_registration(None).await; + } +} diff --git a/crates/app/src/cmd_destroy.rs b/crates/app/src/cmd_destroy.rs index a9f95e023..38ebc6adf 100755 --- a/crates/app/src/cmd_destroy.rs +++ b/crates/app/src/cmd_destroy.rs @@ -3,7 +3,10 @@ //! `extenddb destroy` — tear down a extenddb deployment (REQ-CAT-012). //! -//! Reads config, enumerates tables, requires `--yes` to confirm, drops both databases. +//! Reads config, enumerates tables, requires `--yes` to confirm, drops both +//! databases. The backup store (`[backup.store]`) is not touched: it is the +//! copy that survives destroy, and `extenddb backup purge --confirm` is the +//! command that deletes it. use clap::Args; @@ -25,7 +28,8 @@ pub struct DestroyArgs { #[arg(long)] pg_pass: Option, - /// Confirm destruction (required, no interactive prompt) + /// Confirm destruction (required, no interactive prompt). The backup store is left + /// untouched; use `extenddb backup purge --confirm` to delete its backups. #[arg(long)] yes: bool, } @@ -90,6 +94,13 @@ pub async fn run(args: DestroyArgs) -> anyhow::Result<()> { println!(); println!("WARNING: This will permanently destroy ALL data in both databases."); + println!( + "The backup store ([backup.store]) is left untouched; `extenddb backup purge --confirm` \ + deletes its backups. Its extenddb-store.json names this catalog, so a catalog \ + re-created with `init` finds the store owned by another catalog (on the default \ + SQLite root, the server's warning, which leaves the store unconfigured) until \ + `extenddb backup sync --adopt-store` makes it the owner and gives its backups rows." + ); println!(); if !args.yes { diff --git a/crates/app/src/cmd_serve.rs b/crates/app/src/cmd_serve.rs index a754aa4c2..4fd7ad61d 100755 --- a/crates/app/src/cmd_serve.rs +++ b/crates/app/src/cmd_serve.rs @@ -174,6 +174,13 @@ pub fn run(args: &ServeArgs, build: BuildInfo) -> anyhow::Result<()> { backend, config::redact_password(app_config.storage.connection_config()), ); + let banner_line3 = format!( + " {}", + extenddb_server::backup_store::banner_line( + &app_config.backup, + app_config.storage.database_file().as_deref(), + ) + ); let mut warnings = app_config.storage.startup_warnings(); warnings.extend(app_config.server.startup_warnings()); let startup_warnings = render_startup_warnings(&warnings); @@ -181,11 +188,13 @@ pub fn run(args: &ServeArgs, build: BuildInfo) -> anyhow::Result<()> { eprint!("{wordmark}"); eprintln!("{banner_line1}"); eprintln!("{banner_line2}"); + eprintln!("{banner_line3}"); eprint!("{startup_warnings}"); } else { print!("{wordmark}"); println!("{banner_line1}"); println!("{banner_line2}"); + println!("{banner_line3}"); print!("{startup_warnings}"); } if cfg!(feature = "dev-mode") { @@ -275,10 +284,14 @@ pub fn run(args: &ServeArgs, build: BuildInfo) -> anyhow::Result<()> { if config_exists { params = params.with_config_path(&args.config); } - let result = tokio::runtime::Builder::new_multi_thread() + // The runtime is ended with a bound (`run_then_shut_down`): a backup + // store check abandoned on a mount that has stopped answering leaves a + // blocking thread parked there, and a plain drop of the runtime would + // join it, holding the process open after the drain until SIGKILL. + let runtime = tokio::runtime::Builder::new_multi_thread() .enable_all() - .build()? - .block_on(extenddb_server::serve(params)); + .build()?; + let result = crate::run_then_shut_down(runtime, extenddb_server::serve(params)); // `serve` has printed the refusal; end with its code, which `migrate` // and `verify` use for the same status. if let Err(e) = &result diff --git a/crates/app/src/cmd_stop.rs b/crates/app/src/cmd_stop.rs index 5fa856d72..a3963ac0c 100755 --- a/crates/app/src/cmd_stop.rs +++ b/crates/app/src/cmd_stop.rs @@ -18,9 +18,19 @@ use crate::util::is_process_alive; #[cfg(unix)] use extenddb_config as config; -/// Maximum time to wait for the process to exit after SIGTERM. +/// Maximum time to wait for the process to exit after SIGTERM: the sum of +/// the server's shutdown bounds, in the order it runs them (the listener's +/// drain of active connections, the backup jobs' failure paths, the +/// registration supervisor's return with the background workers' stop and +/// the serving registration's release, the runtime's blocking threads), so +/// `stop` does not give up on a server that is still inside one of them. #[cfg(unix)] -const SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(10); +const SHUTDOWN_TIMEOUT: Duration = Duration::from_secs( + extenddb_server::CONNECTION_DRAIN_TIMEOUT.as_secs() + + extenddb_server::JOB_DRAIN_TIMEOUT.as_secs() + + extenddb_server::WORKER_STOP_TIMEOUT.as_secs() + + crate::RUNTIME_SHUTDOWN_TIMEOUT.as_secs(), +); /// Poll interval when waiting for process exit. #[cfg(unix)] @@ -132,11 +142,46 @@ pub fn run(args: &StopArgs) { } if Instant::now() >= deadline { eprintln!( - "extenddb (pid {pid}) did not exit within {}s after SIGTERM", + "extenddb (pid {pid}) did not exit within {}s after SIGTERM (the server \ + drains connections for up to {}s, waits up to {}s for its backup jobs to end, \ + then up to {}s to stop its background workers and release its serving \ + registration, and {}s for its runtime)", SHUTDOWN_TIMEOUT.as_secs(), + extenddb_server::CONNECTION_DRAIN_TIMEOUT.as_secs(), + extenddb_server::JOB_DRAIN_TIMEOUT.as_secs(), + extenddb_server::WORKER_STOP_TIMEOUT.as_secs(), + crate::RUNTIME_SHUTDOWN_TIMEOUT.as_secs(), ); std::process::exit(1); } thread::sleep(POLL_INTERVAL); } } + +#[cfg(all(test, unix))] +mod tests { + use super::SHUTDOWN_TIMEOUT; + use std::time::Duration; + + /// `stop` waits at least as long as the server's shutdown can take with + /// a backup job in its failure path: 5 s of connection drain, 30 s of + /// job drain, 36 s after it (10 s for the supervisor's return, 16 s for + /// the workers' stop: 5 s of drain, 1 s of abort and 10 s behind a + /// supervisor still stopping them, and 10 s for the registration's + /// release) and 5 s for the runtime. + #[test] + fn stop_waits_for_every_shutdown_bound() { + assert_eq!( + extenddb_server::WORKER_STOP_TIMEOUT, + Duration::from_secs(36) + ); + assert_eq!(SHUTDOWN_TIMEOUT, Duration::from_secs(76)); + assert!( + SHUTDOWN_TIMEOUT + >= extenddb_server::JOB_DRAIN_TIMEOUT + + extenddb_server::WORKER_STOP_TIMEOUT + + crate::RUNTIME_SHUTDOWN_TIMEOUT + + extenddb_server::CONNECTION_DRAIN_TIMEOUT + ); + } +} diff --git a/crates/app/src/init_helpers.rs b/crates/app/src/init_helpers.rs index 406db8b0e..1e93d2777 100755 --- a/crates/app/src/init_helpers.rs +++ b/crates/app/src/init_helpers.rs @@ -300,6 +300,44 @@ key_path = "{tls_key}" # max_import_item_count = 10000000 # max_export_item_count = 10000000 +# [backup] +# Backup and restore jobs. stale_job_after_secs must be at least three +# heartbeat periods, the larger of heartbeat_interval_secs and the 15 s the +# in-catalog jobs heartbeat at (45 at the least); pool_size is raised to +# at least 2 * max_concurrent_jobs + 10 on PostgreSQL; max_concurrent_jobs +# is at most 1024. +# data_file_target_bytes = 67108864 # Uncompressed bytes per data file (minimum 32 MiB) +# heartbeat_interval_secs = 15 # Job heartbeat timer +# stale_job_after_secs = 300 # Heartbeat age after which a job's owner is dead +# failed_row_retention_secs = 604800 # FAILED tombstones kept for the operator (7 days) +# reconcile_interval_secs = 60 # Reconciler cycle +# max_concurrent_jobs = 4 # Backup and restore jobs running at once on this instance +# spool_path = "/var/lib/extenddb/spool" # Restore download area for an S3 store; required +# # with an S3 store except on file-backed SQLite, where +# # it defaults to backup-spool/ beside the database file + +# [backup.store] +# Where CreateBackup writes its backups on PostgreSQL, SQLite and MongoDB +# (CreateBackup answers CREATING and a job of the server uploads; Cassandra +# writes in-catalog backups with or without a store), and where restores and +# deletes of store-backed backups read. Absent on file-backed SQLite: a filesystem +# store at backups/ beside the database file. Absent on PostgreSQL, MongoDB, +# Cassandra and an in-memory SQLite database: no store, and CreateBackup +# writes in-catalog backups with a deprecation warning. kind = "disabled" +# opts out on any backend. Instances that share a catalog must share one +# store root (an S3 bucket and prefix, or a shared mount). +# kind = "filesystem" # "filesystem" | "s3" | "disabled" +# [backup.store.filesystem] +# path = "/var/lib/extenddb/backups" # Must exist, owned by the extenddb user, mode 0700 +# [backup.store.s3] +# bucket = "my-extenddb-backups" +# prefix = "extenddb/" # Optional key prefix +# region = "us-east-1" # Optional; the AWS SDK's chain otherwise +# endpoint = "http://127.0.0.1:9000" # Optional, for an S3-compatible server +# force_path_style = true # Required with an http:// endpoint +# upload_part_size_bytes = 8388608 # 5 MiB to 5 GiB +# max_concurrent_uploads = 4 + # [import] # paths = [] # Allowed directories for import operations diff --git a/crates/app/src/lib.rs b/crates/app/src/lib.rs index 157f29d9b..6dfcc54c0 100644 --- a/crates/app/src/lib.rs +++ b/crates/app/src/lib.rs @@ -3,9 +3,9 @@ //! ExtendDB application/CLI library. //! -//! Owns the command-line interface (`serve`, `init`, `destroy`, `verify`, -//! `migrate`, `status`, `stop`, `settings`, `manage`, `catalog-check`, -//! `healthcheck`) and the +//! Owns the command-line interface (`serve`, `init`, `destroy`, `backup`, +//! `verify`, `migrate`, `status`, `stop`, `settings`, `manage`, +//! `catalog-check`, `healthcheck`) and the //! subcommand dispatch. It is backend-agnostic: a backend's thin `main` //! installs its backend with //! [`set_backend`](extenddb_storage::set_backend) and then calls [`run`]: @@ -22,6 +22,7 @@ //! } //! ``` +mod cmd_backup; mod cmd_catalog_check; mod cmd_destroy; mod cmd_healthcheck; @@ -66,8 +67,11 @@ enum Command { Serve(cmd_serve::ServeArgs), /// Initialize a new extenddb deployment Init(cmd_init::InitArgs), - /// Tear down a extenddb deployment + /// Tear down a extenddb deployment (the databases; the backup store is left untouched) Destroy(cmd_destroy::DestroyArgs), + /// List, purge or sync the backups of the configured backup store, or remove a restore + /// target + Backup(cmd_backup::BackupArgs), /// Validate a extenddb deployment Verify(cmd_verify::VerifyArgs), /// Apply catalog migrations (exit 0 up to date or applied, 1 error, 3 pending without --yes, @@ -116,6 +120,19 @@ pub fn run(build: BuildInfo) -> anyhow::Result<()> { Ok(()) } Command::Destroy(args) => run_interactive(cmd_destroy::run(args)), + Command::Backup(args) => { + let result = run_interactive(cmd_backup::run(args, build.version)); + // A catalog the backend refused: the message `serve` prints for + // the status, and its exit code (3 behind, 6 while a migration + // runs). + if let Err(e) = &result + && let Some(refusal) = e.downcast_ref::() + { + eprintln!("Error: cannot open the catalog: {refusal}"); + std::process::exit(i32::from(refusal.exit_code)); + } + result + } Command::Verify(args) => { let code = run_interactive(cmd_verify::run(args, build.version))?; if code != 0 { @@ -181,6 +198,13 @@ fn print_version(build: BuildInfo) { "disabled" } ); + // The backup jobs' pause points (`EXTENDDB_BACKUP_PAUSE`) are this + // crate's feature, not the binary's, so they are read here. The line is + // printed only on such a build; `test-hooks` above stays the MongoDB + // hooks' line that `devtools/run-mongodb-tests` reads. + if cfg!(feature = "backup-test-hooks") { + println!("backup-test-hooks enabled"); + } } /// Run an async subcommand with a single-threaded tokio runtime and stderr logging. @@ -191,8 +215,75 @@ fn run_interactive( tracing_subscriber::fmt() .try_init() .unwrap_or_else(|e| eprintln!("Warning: logging init failed: {e}")); - tokio::runtime::Builder::new_current_thread() + let runtime = tokio::runtime::Builder::new_current_thread() .enable_all() - .build()? - .block_on(future) + .build()?; + run_then_shut_down(runtime, future) +} + +/// How long the process waits at exit for the runtime's blocking threads +/// after the command's future returns. A thread parked in a filesystem call +/// on a mount that has stopped answering (a backup store check the engine +/// abandoned at its bound) returns only when the mount does; past this bound +/// the thread is detached and the process exits without it. +pub(crate) const RUNTIME_SHUTDOWN_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5); + +/// Run `future` to completion on `runtime`, then end the runtime within +/// [`RUNTIME_SHUTDOWN_TIMEOUT`]. Dropping the runtime instead joins every +/// blocking thread, so one parked syscall held the process open past its +/// shutdown until SIGKILL. +pub(crate) fn run_then_shut_down( + runtime: tokio::runtime::Runtime, + future: impl std::future::Future, +) -> T { + let output = runtime.block_on(future); + runtime.shutdown_timeout(RUNTIME_SHUTDOWN_TIMEOUT); + output +} + +#[cfg(test)] +mod tests { + use super::{RUNTIME_SHUTDOWN_TIMEOUT, run_then_shut_down}; + + /// A blocking task that never returns (a filesystem call on a mount that + /// has stopped answering) whose handle the caller dropped does not hold + /// the runtime's end past the bound, on both runtimes the binary builds + /// (`serve`'s multi-thread one and the commands' current-thread one). + #[test] + fn a_parked_blocking_thread_does_not_hold_the_runtime_past_its_bound() { + let builders = [ + tokio::runtime::Builder::new_multi_thread(), + tokio::runtime::Builder::new_current_thread(), + ]; + for mut builder in builders { + let runtime = builder.enable_all().build().unwrap(); + let (done, finished) = std::sync::mpsc::channel(); + let started = std::time::Instant::now(); + std::thread::spawn(move || { + run_then_shut_down(runtime, async { + // Parked for good; the thread is leaked on purpose and + // dies with the test process. + drop(tokio::task::spawn_blocking(|| { + loop { + std::thread::park(); + } + })); + // Let the blocking pool pick the task up before the + // runtime ends, so a thread is parked when it does. + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + }); + let _ = done.send(()); + }); + // The test's own guard, well past the bound: a shutdown that + // joins the parked thread fails here instead of hanging. + finished + .recv_timeout(RUNTIME_SHUTDOWN_TIMEOUT * 6) + .expect("the runtime ends within its shutdown bound"); + let elapsed = started.elapsed(); + assert!( + elapsed < RUNTIME_SHUTDOWN_TIMEOUT * 3, + "the runtime's end took {elapsed:?}" + ); + } + } } diff --git a/crates/backup-store/README.md b/crates/backup-store/README.md index dd6251534..ae741ecfa 100644 --- a/crates/backup-store/README.md +++ b/crates/backup-store/README.md @@ -77,6 +77,26 @@ store's callers honour; this README is the record of what the store does. - `put` returns the stored object's metadata (bytes written and, on S3, the ETag with quotes stripped; a multipart ETag is not an MD5) so a manifest can be written from the put alone. +- `put_if_absent` creates the object only if no object is under the key and + returns `None`, leaving the existing object untouched, if one is; of two + concurrent calls on one key exactly one stores its body. The filesystem + store writes and syncs its temporary as `put` does (created + `O_CREAT | O_EXCL`) and then `link`s it to the key, which fails if + anything is there, so the object appears complete or not at all; the root + must therefore be on a filesystem that supports hard links (some CIFS and + FUSE mounts do not, and the call fails there with `EPERM`, which the + server reports as the store having failed validation, with the `link` + error in the log line). The S3 + store sends `PutObject` (or `CompleteMultipartUpload` for a body over one + part) with `If-None-Match: *`: a 412 is the key present, and a 409 + `ConditionalRequestConflict` from a concurrent conditional write is + re-sent after a short back-off, up to eight times. An S3-compatible + server that ignores `If-None-Match` on `PutObject` turns the call into a + plain put, last writer wins; moto 5.2 honours it (the conformance suite's + race cell checks that exactly one of eight concurrent callers stores its + body), other servers have not been run. The caller that depends on the + guarantee re-reads the object after its call and compares, which is what + catches a server that lost the race silently. - The S3 store takes credentials from the AWS SDK's default chain (the `s3` module doc names its members, among them `credential_process`, which runs a command named in the shared config file, and the SSO diff --git a/crates/backup-store/src/config.rs b/crates/backup-store/src/config.rs index 002a43e59..927329603 100644 --- a/crates/backup-store/src/config.rs +++ b/crates/backup-store/src/config.rs @@ -11,7 +11,9 @@ //! //! ```toml //! [backup.store] -//! kind = "filesystem" # "filesystem" | "s3"; filesystem when absent +//! kind = "filesystem" # "filesystem" | "s3" | "disabled"; filesystem when absent +//! # ("disabled" configures no store on purpose; the server +//! # then creates in-catalog backups and logs that) //! //! [backup.store.filesystem] //! path = "/backups" # required when kind = "filesystem"; the server defaults it @@ -167,6 +169,21 @@ pub enum BackupStoreKind { Filesystem, /// S3 bucket store. S3, + /// No store, on purpose: the opt-out of the server's per-backend default + /// root. [`crate::open`] refuses it; a caller checks the kind first. + Disabled, +} + +impl BackupStoreKind { + /// The value `kind` takes in the configuration. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Filesystem => "filesystem", + Self::S3 => "s3", + Self::Disabled => "disabled", + } + } } /// The `[backup.store]` table. @@ -175,7 +192,7 @@ pub enum BackupStoreKind { /// survive `serde(flatten)`, so the server nests this struct under its own /// key (`store`) in its `[backup]` model rather than flattening it; that /// keeps one definition of these fields and unknown-field rejection on both. -#[derive(Debug, Clone, Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] #[serde(deny_unknown_fields)] pub struct BackupStoreConfig { /// Selected store kind; `filesystem` when absent. @@ -190,7 +207,7 @@ pub struct BackupStoreConfig { } /// `[backup.store.filesystem]` settings. -#[derive(Debug, Clone, Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] #[serde(deny_unknown_fields)] pub struct FilesystemStoreConfig { /// Root directory for backup objects. Must exist at startup. @@ -201,7 +218,7 @@ pub struct FilesystemStoreConfig { /// from the standard AWS SDK chain. `Debug` prints the endpoint without its /// user information, which `open` refuses but a log of the configuration /// written before `open` would otherwise carry. -#[derive(Clone, Deserialize)] +#[derive(Clone, PartialEq, Eq, Deserialize)] #[serde(deny_unknown_fields)] pub struct S3StoreConfig { /// Destination bucket. @@ -375,6 +392,15 @@ mod tests { assert!(toml::from_str::("kind = \"gcs\"").is_err()); } + #[tokio::test] + async fn disabled_parses_and_does_not_open() { + let config: BackupStoreConfig = toml::from_str("kind = \"disabled\"").unwrap(); + assert_eq!(config.kind, BackupStoreKind::Disabled); + assert_eq!(BackupStoreKind::Disabled.as_str(), "disabled"); + let err = open_err(&config).await; + assert!(err.to_string().contains("disabled"), "{err}"); + } + #[tokio::test] async fn open_requires_the_matching_section() { let missing_fs = BackupStoreConfig { diff --git a/crates/backup-store/src/fs.rs b/crates/backup-store/src/fs.rs index 48bcb5895..fff5547dc 100644 --- a/crates/backup-store/src/fs.rs +++ b/crates/backup-store/src/fs.rs @@ -80,6 +80,16 @@ //! concurrent put into the same new directory, and could not cover a //! failure inside the creation walk itself.) //! +//! `put_if_absent` writes and syncs its temporary the same way, created +//! `O_CREAT | O_EXCL` as every temporary is, and then `link`s it to the key +//! instead of renaming: `link(2)` fails with `EEXIST` when anything is at +//! the new path, so of two concurrent calls on one key exactly one links and +//! the other reads `None`, the first body stays, and the object appears +//! under its key complete or not at all. The temporary is unlinked after the +//! link on every path (the two names are one inode, so the object stays). A +//! regular file already at the destination is reported `None` before the +//! body is read. +//! //! Where this store narrows the shared key space (temporary names, a file //! and a directory at one path, the filesystem's own name limit and case //! folding) is listed in the `key` module doc. @@ -142,10 +152,15 @@ impl FilesystemStore { /// /// # Errors /// - /// Returns an error when the root does not exist, cannot be resolved, or - /// is not a directory. + /// Returns an error when the root path is empty, the root does not exist, + /// cannot be resolved, or is not a directory. pub async fn open(root: impl AsRef) -> Result { let root = root.as_ref(); + if root.as_os_str().is_empty() { + return Err(StoreError::Other( + "the backup root path is empty; name the directory".to_owned(), + )); + } let canonical = tokio::fs::canonicalize(root).await.map_err(|e| { StoreError::io(format!("backup root {} is not usable", root.display()), e) })?; @@ -165,6 +180,14 @@ impl FilesystemStore { }) } + /// The root label a catalog row records for this store: the canonical + /// root directory. Two servers configured for one directory (by any + /// path that resolves to it) spell the same label. + #[must_use] + pub fn store_root(&self) -> String { + self.root.display().to_string() + } + /// Run `hook` on the blocking thread of every `delete_prefix` walk /// right before each removal (a file, a symlink or a directory), with /// the number of removals that walk has made so far. The crate's own @@ -313,7 +336,14 @@ impl FilesystemStore { Ok(current) } - async fn put_impl(&self, key: &str, mut body: ByteStream) -> Result { + /// Resolve `key` to the directory its object is written into and the + /// object's final path, creating the directories (`prepare_parent`) and + /// refusing a symlink or a directory at the destination: a put never + /// resolves through a link, and a directory there is a key-space + /// collision (`a/b` exists, `a` is put), refused before the body is read + /// rather than after the rename or link fails. Reports whether a regular + /// file is already at the destination, which `put_if_absent` reads. + async fn destination(&self, key: &str) -> Result { validate_key(key)?; let components: Vec<&str> = key.split('/').collect(); let (file_name, parent_components) = components @@ -321,12 +351,7 @@ impl FilesystemStore { .expect("a validated key has at least one component"); let parent = self.prepare_parent(parent_components).await?; let final_path = parent.join(file_name); - - // A symlink at the destination, in-root or escaping, is refused: a - // put never resolves through a link. A directory there is a key-space - // collision (`a/b` exists, `a` is put), refused before the body is - // read rather than after the rename fails. - match tokio::fs::symlink_metadata(&final_path).await { + let exists = match tokio::fs::symlink_metadata(&final_path).await { Ok(meta) if meta.is_symlink() => { return Err(symlink_refused(&final_path)); } @@ -339,13 +364,101 @@ impl FilesystemStore { ), )); } - Ok(_) | Err(_) => {} - } + Ok(_) => true, + Err(_) => false, + }; + Ok(Destination { + parent, + final_path, + exists, + }) + } + /// Write `body` to a new temporary sibling of `final_path`, flushed and + /// synced, and return its path, the guard that removes it, and the + /// number of bytes written. The temporary is created `O_CREAT | O_EXCL` + /// with mode `0600` (`create_private_file`). + /// + /// The temporary and its guard come back together. The guard removes + /// the temporary on every exit the caller does not disarm it for: an + /// error, the put's future dropped (a caller's timeout, a cancelled + /// job), including while the open itself is in flight on the blocking + /// pool, or a body that panics. + async fn write_temporary( + parent: &Path, + mut body: ByteStream, + ) -> Result<(PathBuf, RemoveOnDrop, u64), StoreError> { let temp_path = parent.join(format!( "{TEMP_PREFIX}{}{TEMP_SUFFIX}", uuid::Uuid::new_v4() )); + let (mut file, temp_guard) = create_private_file(&temp_path).await?; + let mut size: u64 = 0; + while let Some(chunk) = body.next().await { + let chunk = chunk?; + file.write_all(&chunk) + .await + .map_err(|e| StoreError::io(format!("write {}", temp_path.display()), e))?; + size += chunk.len() as u64; + } + // tokio's `File` hands each write to a background task and + // reports its error on the next operation. `sync_all` waits for + // the last background write but does not return its error (tokio + // keeps only its kind for a later write or flush), so without a + // flush before the drop a short file would be renamed into place + // on a full disk (or past RLIMIT_FSIZE). `flush` returns it. + file.flush() + .await + .map_err(|e| StoreError::io(format!("write {}", temp_path.display()), e))?; + // Defence in depth, no test: the fsync of the temporary matters + // only across a crash or power loss, which the suite cannot + // inject. + file.sync_all() + .await + .map_err(|e| StoreError::io(format!("sync {}", temp_path.display()), e))?; + drop(file); + Ok((temp_path, temp_guard, size)) + } + + /// Sync `parent` and every directory between it and the root, so the + /// entry a rename or link just made, and the directories a put created + /// on the way, are durable. Syncing every ancestor is a few extra + /// fsyncs per put (one per path component) against a bounded depth; + /// cheaper than tracking which ones this put created, and correct when a + /// concurrent put created them. Defence in depth, no test: it matters + /// only across a crash. + async fn sync_parent_chain(&self, parent: &Path) -> Result<(), StoreError> { + sync_dir(parent).await?; + let mut dir = parent.to_path_buf(); + while let Some(p) = dir.parent() { + if !p.starts_with(&self.root) || p == self.root { + sync_dir(&self.root).await?; + break; + } + sync_dir(p).await?; + dir = p.to_path_buf(); + } + Ok(()) + } + + /// The metadata of the object now at `final_path`. + async fn stored_meta(key: &str, final_path: &Path, size: u64) -> ObjectMeta { + let last_modified = tokio::fs::symlink_metadata(final_path) + .await + .ok() + .and_then(|m| m.modified().ok()); + ObjectMeta { + key: key.to_owned(), + size, + last_modified, + etag: None, + } + } + + async fn put_impl(&self, key: &str, body: ByteStream) -> Result { + let Destination { + parent, final_path, .. + } = self.destination(key).await?; // Durability contract: the temp file is synced before the rename and // the parent directory is synced after it, so an Ok return means both @@ -353,37 +466,9 @@ impl FilesystemStore { // directory sync the rename itself could be lost on power failure // even though the file data had been written. async { - // The temporary and its guard come back together. The guard - // removes it on every exit but the rename: an error (`?` below), - // the put's future dropped (a caller's timeout, a cancelled - // job), including while the open itself is in flight on the - // blocking pool, or a body that panics. It lives inside this - // block so dropping the block's future drops it. - let (mut file, mut temp_guard) = create_private_file(&temp_path).await?; - let mut size: u64 = 0; - while let Some(chunk) = body.next().await { - let chunk = chunk?; - file.write_all(&chunk) - .await - .map_err(|e| StoreError::io(format!("write {}", temp_path.display()), e))?; - size += chunk.len() as u64; - } - // tokio's `File` hands each write to a background task and - // reports its error on the next operation. `sync_all` waits for - // the last background write but does not return its error (tokio - // keeps only its kind for a later write or flush), so without a - // flush before the drop a short file would be renamed into place - // on a full disk (or past RLIMIT_FSIZE). `flush` returns it. - file.flush() - .await - .map_err(|e| StoreError::io(format!("write {}", temp_path.display()), e))?; - // Defence in depth, no test: the fsync of the temporary matters - // only across a crash or power loss, which the suite cannot - // inject. - file.sync_all() - .await - .map_err(|e| StoreError::io(format!("sync {}", temp_path.display()), e))?; - drop(file); + // The guard lives inside this block so dropping the block's + // future drops it (`write_temporary`). + let (temp_path, mut temp_guard, size) = Self::write_temporary(&parent, body).await?; tokio::fs::rename(&temp_path, &final_path) .await .map_err(|e| { @@ -400,32 +485,56 @@ impl FilesystemStore { // before the put into one that holds nothing, which is worse // than either complete object (the trait contract is "no partial // object", not "no object"). - sync_dir(&parent).await?; - // The directories between the root and the parent are durable - // only once their parents' entries are synced too. Syncing every - // ancestor is a few extra fsyncs per put (one per path component) - // against a bounded depth; cheaper than tracking which ones this - // put created, and correct when a concurrent put created them. - // Defence in depth, no test: it matters only across a crash. - let mut dir = parent.clone(); - while let Some(p) = dir.parent() { - if !p.starts_with(&self.root) || p == self.root { - sync_dir(&self.root).await?; - break; + self.sync_parent_chain(&parent).await?; + Ok(Self::stored_meta(key, &final_path, size).await) + } + .await + } + + /// `put_if_absent`: the temporary is written and synced as `put` writes + /// it, then `link`ed to the key. `link(2)` fails with `EEXIST` when + /// anything is at the new path (a file, a directory, a symlink, whatever + /// it points at), so the object appears under its key complete or not at + /// all, and an object that was there first is untouched. A regular file + /// already at the destination skips the body: the link would fail, and + /// writing the body first would only cost disk. The temporary is removed + /// after the link on every path; the link and the temporary are one + /// inode, so the object stays. + async fn put_if_absent_impl( + &self, + key: &str, + body: ByteStream, + ) -> Result, StoreError> { + let Destination { + parent, + final_path, + exists, + } = self.destination(key).await?; + if exists { + return Ok(None); + } + async { + let (temp_path, temp_guard, size) = Self::write_temporary(&parent, body).await?; + let linked = tokio::fs::hard_link(&temp_path, &final_path).await; + // The temporary's name is surplus from here on whether the + // link landed or not; the guard removes it when it drops at the + // end of this block (and on every early return below). + match linked { + Ok(()) => {} + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => { + drop(temp_guard); + return Ok(None); + } + Err(e) => { + return Err(StoreError::io( + format!("link {} to {}", temp_path.display(), final_path.display()), + e, + )); } - sync_dir(p).await?; - dir = p.to_path_buf(); } - let last_modified = tokio::fs::symlink_metadata(&final_path) - .await - .ok() - .and_then(|m| m.modified().ok()); - Ok(ObjectMeta { - key: key.to_owned(), - size, - last_modified, - etag: None, - }) + drop(temp_guard); + self.sync_parent_chain(&parent).await?; + Ok(Some(Self::stored_meta(key, &final_path, size).await)) } .await } @@ -512,15 +621,22 @@ impl FilesystemStore { } async fn validate_impl(&self) -> Result<(), StoreError> { - let meta = tokio::fs::symlink_metadata(&self.root).await.map_err(|e| { - StoreError::io( - format!("backup root {} is not usable", self.root.display()), - e, - ) - })?; - check_root_is_directory(&self.root, &meta)?; - check_root_location(&self.root)?; - check_root_ownership(&self.root, &meta)?; + // The root's own checks are synchronous filesystem calls (`lstat` of + // the root and each ancestor, `realpath`); they run on the blocking + // pool so a root on a mount that has stopped answering parks a + // blocking thread and not the runtime, and a caller's timeout on + // this future can fire. + let root = self.root.clone(); + tokio::task::spawn_blocking(move || { + let meta = std::fs::symlink_metadata(&root).map_err(|e| { + StoreError::io(format!("backup root {} is not usable", root.display()), e) + })?; + check_root_is_directory(&root, &meta)?; + check_root_location(&root)?; + check_root_ownership(&root, &meta) + }) + .await + .map_err(|e| StoreError::Other(format!("root check task failed: {e}")))??; let probe = self.root.join(format!( "{TEMP_PREFIX}validate-{}{TEMP_SUFFIX}", uuid::Uuid::new_v4() @@ -574,6 +690,15 @@ impl FilesystemStore { } } +/// Where a put writes: the directory the object goes into (created and +/// checked by `prepare_parent`), the object's final path, and whether a +/// regular file is already there. +struct Destination { + parent: PathBuf, + final_path: PathBuf, + exists: bool, +} + impl BackupStore for FilesystemStore { fn put<'a>( &'a self, @@ -583,6 +708,14 @@ impl BackupStore for FilesystemStore { self.put_impl(key, body).boxed() } + fn put_if_absent<'a>( + &'a self, + key: &'a str, + body: ByteStream, + ) -> BoxFuture<'a, Result, StoreError>> { + self.put_if_absent_impl(key, body).boxed() + } + fn get<'a>(&'a self, key: &'a str) -> BoxFuture<'a, Result> { self.get_impl(key).boxed() } @@ -624,6 +757,28 @@ impl BackupStore for FilesystemStore { /// cannot detect. Messages name the path and the fix. No check is made on /// other platforms. fn check_root_ownership(root: &Path, meta: &std::fs::Metadata) -> Result<(), StoreError> { + check_private_tree(root, meta, "backup root") +} + +/// The owner-only rule of a directory the server keeps private, by its +/// canonical path: `check_root_ownership`'s rule (the directory owned by +/// the effective user with no group or other permission bits; every ancestor +/// a directory owned by the effective user or by root, not writable by group +/// or others unless sticky) for a directory that is not a store root, named +/// `what` in the messages (the server's restore spool). No check is made on +/// other platforms. +/// +/// # Errors +/// +/// `PermissionDenied` naming the directory or ancestor and the mode it +/// needs; `Io` when a directory on the way cannot be read. +pub fn check_private_directory(dir: &Path, what: &str) -> Result<(), StoreError> { + let meta = std::fs::symlink_metadata(dir) + .map_err(|e| StoreError::io(format!("stat {}", dir.display()), e))?; + check_private_tree(dir, &meta, what) +} + +fn check_private_tree(root: &Path, meta: &std::fs::Metadata, what: &str) -> Result<(), StoreError> { #[cfg(unix)] { use std::os::unix::fs::MetadataExt; @@ -637,8 +792,8 @@ fn check_root_ownership(root: &Path, meta: &std::fs::Metadata) -> Result<(), Sto // only a swap between the two reaches this (module doc). if m.is_symlink() { return Err(StoreError::PermissionDenied(format!( - "{} (an ancestor of the backup root) is a symlink; configure the backup \ - root by its resolved path", + "{} (an ancestor of the {what}) is a symlink; configure the {what} by its \ + resolved path", dir.display() ))); } @@ -646,17 +801,17 @@ fn check_root_ownership(root: &Path, meta: &std::fs::Metadata) -> Result<(), Sto let sticky = mode & 0o1000 != 0; if m.uid() != euid && m.uid() != 0 { return Err(StoreError::PermissionDenied(format!( - "{} (an ancestor of the backup root) is owned by uid {}; the backup root \ - must sit under directories owned by root or by the extenddb user", + "{} (an ancestor of the {what}) is owned by uid {}; the {what} must sit \ + under directories owned by root or by the extenddb user", dir.display(), m.uid() ))); } if mode & 0o022 != 0 && !sticky { return Err(StoreError::PermissionDenied(format!( - "{} (an ancestor of the backup root) has mode {mode:04o} and is writable by \ - group or others without the sticky bit; another user could replace the \ - root. chmod it to remove group and world write, or move the backup root", + "{} (an ancestor of the {what}) has mode {mode:04o} and is writable by group \ + or others without the sticky bit; another user could replace the {what}. \ + chmod it to remove group and world write, or move the {what}", dir.display() ))); } @@ -665,7 +820,7 @@ fn check_root_ownership(root: &Path, meta: &std::fs::Metadata) -> Result<(), Sto } #[cfg(not(unix))] { - let _ = (root, meta); + let _ = (root, meta, what); } Ok(()) } diff --git a/crates/backup-store/src/lib.rs b/crates/backup-store/src/lib.rs index 5793dce10..23e7526ad 100644 --- a/crates/backup-store/src/lib.rs +++ b/crates/backup-store/src/lib.rs @@ -33,8 +33,11 @@ mod s3; pub use config::{BackupStoreConfig, BackupStoreKind, FilesystemStoreConfig, S3StoreConfig}; pub use error::StoreError; -pub use fs::FilesystemStore; -pub use s3::{MAX_PART_SIZE_BYTES, MAX_STORE_PREFIX_LEN, MIN_PART_SIZE_BYTES, S3Store}; +pub use fs::{FilesystemStore, check_private_directory}; +pub use s3::{ + ATTEMPT_TIMEOUT as S3_ATTEMPT_TIMEOUT, MAX_PART_SIZE_BYTES, MAX_STORE_PREFIX_LEN, + MIN_PART_SIZE_BYTES, S3Store, +}; /// Byte stream flowing into and out of a store. /// @@ -106,6 +109,31 @@ pub trait BackupStore: Send + Sync { body: ByteStream, ) -> BoxFuture<'a, Result>; + /// Store `body` under `key` only if no object is there, and return the + /// stored object's metadata; return `None`, leaving the existing object + /// untouched and reading none of `body` the store did not already read, + /// if one is. Of two concurrent calls on one key exactly one stores its + /// body. The object appears under its key complete or not at all, as + /// with [`BackupStore::put`], and a call that is dropped or fails before + /// the object is in place leaves nothing (the same guards as `put`); a + /// filesystem call whose `link` succeeded and whose parent-directory + /// sync then fails returns `Err` with the object in place, as `put` + /// does after its `rename`. Written for the store identity file; the + /// mechanics are the stores': S3 sends `PutObject` (or, for a body over + /// one part, `CompleteMultipartUpload`) with `If-None-Match: *`, where a + /// 412 means the key is present and a 409 `ConditionalRequestConflict` + /// from a concurrent conditional write is retried; the filesystem store + /// writes and syncs its temporary as `put` does, created + /// `O_CREAT | O_EXCL`, then `link`s it to the key, which fails if the + /// key exists. An S3-compatible server that ignores `If-None-Match` on + /// `PutObject` makes this a plain put, last writer wins; the README + /// names it, and the caller's re-read is what catches it. + fn put_if_absent<'a>( + &'a self, + key: &'a str, + body: ByteStream, + ) -> BoxFuture<'a, Result, StoreError>>; + /// Stream the object stored under `key`. /// /// Returns [`StoreError::NotFound`] when no object exists. @@ -183,13 +211,37 @@ pub trait BackupStore: Send + Sync { /// /// # Errors /// -/// Returns an error when the section matching the selected store kind is +/// Returns an error when `kind` is `disabled` (no store to open; the caller +/// checks the kind first), when the section matching the selected store kind +/// is /// absent, a value fails validation (an empty bucket, an S3 Express bucket /// name, a part size outside 5 MiB to 5 GiB, zero concurrency, a malformed /// prefix, an endpoint setting that breaks the rules in the `config` module /// doc, the SDK's own and its credential providers' and IMDS client's /// included), or the filesystem root is unusable. pub async fn open(config: &BackupStoreConfig) -> Result, StoreError> { + open_described(config).await.map(|opened| opened.store) +} + +/// A store [`open_described`] built, with the two facts a catalog row records +/// about it. +pub struct OpenedStore { + /// The store. + pub store: Arc, + /// Which kind it is (never [`BackupStoreKind::Disabled`]). + pub kind: BackupStoreKind, + /// The root label: the canonical directory of a filesystem store + /// ([`FilesystemStore::store_root`]), `s3:///` with an + /// optional `?endpoint=` for an S3 store ([`S3Store::store_root`]). + pub root: String, +} + +/// [`open`], also returning the store's kind and root label. +/// +/// # Errors +/// +/// As [`open`]. +pub async fn open_described(config: &BackupStoreConfig) -> Result { match config.kind { BackupStoreKind::Filesystem => { let fs_config = config.filesystem.as_ref().ok_or_else(|| { @@ -198,7 +250,13 @@ pub async fn open(config: &BackupStoreConfig) -> Result, St .to_owned(), ) })?; - Ok(Arc::new(FilesystemStore::open(&fs_config.path).await?)) + let store = FilesystemStore::open(&fs_config.path).await?; + let root = store.store_root(); + Ok(OpenedStore { + store: Arc::new(store), + kind: BackupStoreKind::Filesystem, + root, + }) } BackupStoreKind::S3 => { let s3_config = config.s3.as_ref().ok_or_else(|| { @@ -206,7 +264,16 @@ pub async fn open(config: &BackupStoreConfig) -> Result, St "[backup.store.s3] with a bucket is required when kind = \"s3\"".to_owned(), ) })?; - Ok(Arc::new(S3Store::open(s3_config).await?)) + let store = S3Store::open(s3_config).await?; + let root = store.store_root().to_owned(); + Ok(OpenedStore { + store: Arc::new(store), + kind: BackupStoreKind::S3, + root, + }) } + BackupStoreKind::Disabled => Err(StoreError::Other( + "[backup.store] kind = \"disabled\" configures no store; nothing to open".to_owned(), + )), } } diff --git a/crates/backup-store/src/s3.rs b/crates/backup-store/src/s3.rs index 64c7e2a9d..465a23f32 100644 --- a/crates/backup-store/src/s3.rs +++ b/crates/backup-store/src/s3.rs @@ -166,7 +166,10 @@ const _: () = { /// would otherwise hang a put for as long as the server keeps the socket. /// The attempt timeout includes sending the request body, so each part must /// upload within 60 s (see `upload_part_size_bytes` in the config module). -const ATTEMPT_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(60); +/// Exported as the crate's `S3_ATTEMPT_TIMEOUT`: a caller that bounds a +/// sequence of small requests (the engine's store check) stays under one +/// attempt, so its bound fires before the SDK starts a retry. +pub const ATTEMPT_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(60); const OPERATION_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(300); /// Longest wait for the next chunk of a `GetObject` body. The two timeouts @@ -232,6 +235,8 @@ pub struct S3Store { /// Longest wait for the next chunk of a `GetObject` body; /// [`BODY_IDLE_TIMEOUT`] except in the crate's own tests. body_idle_timeout: std::time::Duration, + /// The root label catalog rows record for this store ([`Self::store_root`]). + root: String, } impl S3Store { @@ -436,6 +441,14 @@ impl S3Store { // this keeps a client that reaches one (the test-only `open`) // from sending the request at all. .disable_s3_express_session_auth(true); + // The label names the endpoint the client uses, whichever source + // supplied it, so two servers reaching one bucket through one + // endpoint spell one label however each was told about it. + let root = store_root_label( + &config.bucket, + &prefix, + endpoint.as_ref().map(|(endpoint, _)| endpoint.as_str()), + ); builder.set_endpoint_url(endpoint.map(|(endpoint, _)| endpoint)); // The SDK's default client sends through the proxy named in // `HTTP_PROXY`, `http_proxy`, `ALL_PROXY` or `all_proxy`, which @@ -472,9 +485,24 @@ impl S3Store { max_parts: MAX_PARTS, page_size: None, body_idle_timeout: BODY_IDLE_TIMEOUT, + root, }) } + /// The root label a catalog row records for this store: + /// `s3:///`, with `?endpoint=` appended when the + /// client uses an endpoint other than the SDK's default (from `endpoint` + /// or from the SDK's environment and shared config), the URL in one + /// spelling: lowercase scheme and host, no default port, no `/` for an + /// empty path, no user information, query or fragment. Two servers + /// reaching the same bucket and prefix through the same endpoint spell + /// the same label however each was told about the endpoint, and so + /// address the same rows. + #[must_use] + pub fn store_root(&self) -> &str { + &self.root + } + /// Bucket-side key for a store key. S3 bounds the whole object key at /// [`crate::key::MAX_KEY_LEN`] bytes, prefix included, so a key that fits /// the shared rule can still be too long under a long prefix; refuse it @@ -493,6 +521,36 @@ impl S3Store { } async fn put_impl(&self, key: &str, body: ByteStream) -> Result { + // `put_inner` answers `None` only to a conditional put. + self.put_inner(key, body, false) + .await? + .ok_or_else(|| StoreError::Other(format!("s3 PutObject of {key} returned no object"))) + } + + async fn put_if_absent_impl( + &self, + key: &str, + body: ByteStream, + ) -> Result, StoreError> { + self.put_inner(key, body, true).await + } + + /// The one upload path behind `put` and `put_if_absent`. With + /// `if_none_match`, the request that creates the object (`PutObject` for + /// a body of one part, `CompleteMultipartUpload` otherwise) carries + /// `If-None-Match: *`: a 412 means an object is under the key and `None` + /// is returned (the multipart upload is aborted first, since the parts + /// would otherwise stay); a 409 `ConditionalRequestConflict` means a + /// concurrent conditional write on the key was in flight, and the request + /// is re-sent after a short back-off, up to [`CONDITIONAL_RETRIES`] times + /// (the parts already uploaded are reused), after which the last answer + /// is reported as the error it is. + async fn put_inner( + &self, + key: &str, + body: ByteStream, + if_none_match: bool, + ) -> Result, StoreError> { validate_key(key)?; let full_key = self.full_key(key)?; let mut chunks = std::pin::pin!(chunk_stream(body, self.part_size)); @@ -506,21 +564,38 @@ impl S3Store { let Some(second) = second else { // The whole body fits in one part: a single PutObject. let size = first.len() as u64; - let output = self - .client - .put_object() - .bucket(&self.bucket) - .key(&full_key) - .body(first.into()) - .send() - .await - .map_err(|e| map_sdk_error("PutObject", e))?; - return Ok(ObjectMeta { - key: key.to_owned(), - size, - last_modified: None, - etag: output.e_tag().map(strip_etag_quotes), - }); + let mut attempt = 0; + loop { + let mut request = self + .client + .put_object() + .bucket(&self.bucket) + .key(&full_key) + .body(first.clone().into()); + if if_none_match { + request = request.if_none_match("*"); + } + match request.send().await { + Ok(output) => { + return Ok(Some(ObjectMeta { + key: key.to_owned(), + size, + last_modified: None, + etag: output.e_tag().map(strip_etag_quotes), + })); + } + Err(e) => match conditional_outcome(&e) { + Some(Conditional::Present) if if_none_match => return Ok(None), + Some(Conditional::Conflict) + if if_none_match && attempt < CONDITIONAL_RETRIES => + { + attempt += 1; + tokio::time::sleep(conditional_backoff(attempt)).await; + } + _ => return Err(map_sdk_error("PutObject", e)), + }, + } + } }; // Every failure after CreateMultipartUpload must reach the abort, @@ -537,38 +612,62 @@ impl S3Store { // is in flight. let mut abort_guard = self.create_upload_guarded(&full_key).await?; let upload_id = abort_guard.upload_id.clone(); - let outcome: Result = async { + let outcome: Result, StoreError> = async { let (parts, size) = self .upload_parts(&full_key, &upload_id, first, second, chunks) .await?; - let output = self - .client - .complete_multipart_upload() - .bucket(&self.bucket) - .key(&full_key) - .upload_id(&upload_id) - .multipart_upload( - CompletedMultipartUpload::builder() - .set_parts(Some(parts)) - .build(), - ) - .send() - .await - .map_err(|e| map_sdk_error("CompleteMultipartUpload", e))?; - Ok(ObjectMeta { - key: key.to_owned(), - size, - last_modified: None, - etag: output.e_tag().map(strip_etag_quotes), - }) + let mut attempt = 0; + loop { + let mut request = self + .client + .complete_multipart_upload() + .bucket(&self.bucket) + .key(&full_key) + .upload_id(&upload_id) + .multipart_upload( + CompletedMultipartUpload::builder() + .set_parts(Some(parts.clone())) + .build(), + ); + if if_none_match { + request = request.if_none_match("*"); + } + match request.send().await { + Ok(output) => { + return Ok(Some(ObjectMeta { + key: key.to_owned(), + size, + last_modified: None, + etag: output.e_tag().map(strip_etag_quotes), + })); + } + Err(e) => match conditional_outcome(&e) { + Some(Conditional::Present) if if_none_match => return Ok(None), + Some(Conditional::Conflict) + if if_none_match && attempt < CONDITIONAL_RETRIES => + { + attempt += 1; + tokio::time::sleep(conditional_backoff(attempt)).await; + } + _ => return Err(map_sdk_error("CompleteMultipartUpload", e)), + }, + } + } } .await; // The guard stays armed until the inline abort has returned: a put // dropped while that abort is in flight (a caller's timeout) still - // has the guard abort the upload. + // has the guard abort the upload. A conditional complete that found + // the key present leaves its upload open too, so it is aborted the + // same way. let meta = match outcome { - Ok(meta) => meta, + Ok(Some(meta)) => meta, + Ok(None) => { + abort_upload(&self.client, &self.bucket, &full_key, &upload_id).await; + abort_guard.disarm(); + return Ok(None); + } Err(e) => { // Best effort: the original failure is what the caller needs. abort_upload(&self.client, &self.bucket, &full_key, &upload_id).await; @@ -577,7 +676,7 @@ impl S3Store { } }; abort_guard.disarm(); - Ok(meta) + Ok(Some(meta)) } /// Send `CreateMultipartUpload` for the bucket-side key `full_key` and @@ -1061,10 +1160,7 @@ impl S3Store { ({}; {reason})", self.bucket, PROBE_LIST_OBJECTS.grant )), - other => StoreError::Other(format!( - "backup bucket {} failed validation: {other}", - self.bucket - )), + other => validation_failure(&self.bucket, other), } })?; self.validate_write_grants().await @@ -1080,10 +1176,7 @@ impl S3Store { "the configured credentials lack {grant} on backup bucket {} ({reason}){extra}", self.bucket )), - other => StoreError::Other(format!( - "backup bucket {} failed validation: {other}", - self.bucket - )), + other => validation_failure(&self.bucket, other), }; self.client .delete_object() @@ -1183,6 +1276,14 @@ impl BackupStore for S3Store { self.put_impl(key, body).boxed() } + fn put_if_absent<'a>( + &'a self, + key: &'a str, + body: ByteStream, + ) -> BoxFuture<'a, Result, StoreError>> { + self.put_if_absent_impl(key, body).boxed() + } + fn get<'a>(&'a self, key: &'a str) -> BoxFuture<'a, Result> { self.get_impl(key).boxed() } @@ -1258,6 +1359,59 @@ impl BackupStore for S3Store { } } +/// [`S3Store::store_root`]'s rule, over the normalized prefix and the +/// endpoint the client uses (the configured one, or the one the SDK's +/// environment and shared config supplied). The endpoint is spelled one +/// way whatever the operator typed: scheme and host lowercased, a default +/// port (`80` for `http`, `443` for `https`) dropped, an empty path spelled +/// without its `/`, and (defence in depth for a label built before `open`'s +/// checks ran) user information, query and fragment removed. An endpoint +/// `http::Uri` accepts but `url` does not parse is spelled as typed, minus +/// those three parts. +fn store_root_label(bucket: &str, prefix: &str, endpoint: Option<&str>) -> String { + let mut label = format!("s3://{bucket}/{prefix}"); + if let Some(endpoint) = endpoint { + label.push_str("?endpoint="); + label.push_str(&normalized_endpoint(endpoint.trim())); + } + label +} + +/// The one spelling of an endpoint URL the root label carries +/// (`store_root_label`). +fn normalized_endpoint(endpoint: &str) -> String { + if let Ok(url) = url::Url::parse(endpoint) + && let Some(host) = url.host_str() + { + // `Url` lowercases the scheme and the host (a domain, or an IPv6 + // address's hex digits), and reports no port when it is the scheme's + // default. + let mut spelled = format!("{}://{host}", url.scheme()); + if let Some(port) = url.port() { + spelled.push(':'); + spelled.push_str(&port.to_string()); + } + if url.path() != "/" { + spelled.push_str(url.path()); + } + return spelled; + } + let authority_start = endpoint.find("//").map_or(0, |at| at + 2); + let rest = &endpoint[authority_start..]; + let authority_end = rest.find(['/', '?', '#']).unwrap_or(rest.len()); + let authority = &rest[..authority_end]; + let authority = authority + .rfind('@') + .map_or(authority, |at| &authority[at + 1..]); + let path_end = rest.find(['?', '#']).unwrap_or(rest.len()); + let path = &rest[authority_end..path_end]; + format!( + "{}{}{path}", + endpoint[..authority_start].to_ascii_lowercase(), + authority.to_ascii_lowercase() + ) +} + /// Normalize the configured bucket prefix: strip leading `/`, require a /// trailing `/` when non-empty, and hold it to the same component rules as /// keys. @@ -1955,6 +2109,46 @@ fn is_no_such_upload(err: &SdkError) -> bool { matches!(err, SdkError::ServiceError(ctx) if ctx.err().code() == Some("NoSuchUpload")) } +/// Why a request sent with `If-None-Match: *` did not store its object. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Conditional { + /// 412 `PreconditionFailed`: an object is under the key. + Present, + /// 409 `ConditionalRequestConflict`: another conditional write on the + /// key was in flight; S3 asks the caller to retry. + Conflict, +} + +/// Classify a conditional write's refusal, or `None` for any other error. +/// S3 answers a failed `If-None-Match` with 412 and the code +/// `PreconditionFailed`, and a collision of conditional writes with 409 and +/// `ConditionalRequestConflict`; either the status or the code identifies +/// the case, so a response carrying only one of them still classifies. +fn conditional_outcome(err: &SdkError) -> Option { + let SdkError::ServiceError(ctx) = err else { + return None; + }; + match (ctx.raw().status().as_u16(), ctx.err().code()) { + (412, _) | (_, Some("PreconditionFailed")) => Some(Conditional::Present), + (409, Some("ConditionalRequestConflict")) | (_, Some("ConditionalRequestConflict")) => { + Some(Conditional::Conflict) + } + _ => None, + } +} + +/// How many times a conditional write answered `ConditionalRequestConflict` +/// is re-sent before the conflict is reported. Each retry waits +/// [`conditional_backoff`]; the eight waits sum to 7.9 s. +const CONDITIONAL_RETRIES: u32 = 8; + +/// The wait before retry `attempt` (1-based) of a conflicting conditional +/// write: 100 ms, doubling each retry (100, 200, 400, 800 ms), capped at +/// 1.6 s from the fifth retry on. +fn conditional_backoff(attempt: u32) -> std::time::Duration { + std::time::Duration::from_millis(50u64.saturating_mul(1u64 << attempt.min(5))) +} + /// Best-effort abort of a multipart upload, after a put failure or from a /// drop guard; the original failure (if any) is what the caller needs, so /// an abort error is only logged. `NoSuchUpload` means the upload is no @@ -2152,6 +2346,42 @@ where } } +/// A `validate` failure that is not a denial, with the bucket named. A +/// transport failure keeps its class: the caller tells a store it could not +/// reach (which degrades the store) from a configuration it refuses (which +/// stops startup), so the bucket goes in as a link of the error chain rather +/// than flattening the error into text. +fn validation_failure(bucket: &str, error: StoreError) -> StoreError { + match error { + StoreError::Transport { source } => StoreError::Transport { + source: Box::new(ValidationContext { + context: format!("backup bucket {bucket} failed validation"), + source, + }), + }, + other => StoreError::Other(format!("backup bucket {bucket} failed validation: {other}")), + } +} + +/// The context link `validation_failure` adds to a transport error's chain. +#[derive(Debug)] +struct ValidationContext { + context: String, + source: Box, +} + +impl std::fmt::Display for ValidationContext { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.context) + } +} + +impl std::error::Error for ValidationContext { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + Some(self.source.as_ref()) + } +} + /// The SDK's non-service errors display as a bare category ("dispatch /// failure"); the cause (no region configured, no credentials, connection /// refused) is in the `source()` chain. Join the chain so an operator reads @@ -2174,6 +2404,81 @@ pub(crate) fn describe_error_chain(error: &(dyn std::error::Error + 'static)) -> mod tests { use super::*; + /// A transport failure through `validate` keeps its class and names the + /// bucket first in its chain; any other failure is `Other` with the same + /// prefix. + #[test] + fn a_validation_transport_failure_keeps_its_class() { + let transport: SdkError = + SdkError::construction_failure("no region"); + let wrapped = validation_failure("b", map_sdk_error("Op", transport)); + assert!( + matches!(wrapped, StoreError::Transport { .. }), + "{wrapped:?}" + ); + let text = wrapped.to_string(); + assert!( + text.starts_with("transport error: backup bucket b failed validation: "), + "{text}" + ); + assert!(text.contains("no region"), "{text}"); + let other = validation_failure("b", StoreError::Other("x".to_owned())); + assert!( + matches!(&other, StoreError::Other(m) if m == "backup bucket b failed validation: x"), + "{other:?}" + ); + } + + /// The label spells the endpoint one way: the two spellings an operator + /// may type for one endpoint (a trailing `/`, upper-case letters, the + /// scheme's default port) give one label, and a credential, query or + /// fragment never reaches it. + #[test] + fn the_store_root_label_names_bucket_prefix_and_endpoint() { + assert_eq!(store_root_label("b", "", None), "s3://b/"); + assert_eq!(store_root_label("b", "extenddb/", None), "s3://b/extenddb/"); + assert_eq!( + store_root_label("b", "p/", Some("http://127.0.0.1:9000")), + "s3://b/p/?endpoint=http://127.0.0.1:9000" + ); + assert_eq!( + store_root_label("b", "p/", Some("https://u:S3cretPW@h:1/base?x=1#f")), + "s3://b/p/?endpoint=https://h:1/base" + ); + for spelling in [ + "http://127.0.0.1:9000", + "http://127.0.0.1:9000/", + "HTTP://127.0.0.1:9000/", + " http://127.0.0.1:9000 ", + ] { + assert_eq!( + store_root_label("b", "p/", Some(spelling)), + "s3://b/p/?endpoint=http://127.0.0.1:9000", + "{spelling:?}" + ); + } + for spelling in [ + "https://S3.Example.com", + "https://s3.example.com/", + "https://s3.example.com:443", + "HTTPS://s3.example.com:443/", + ] { + assert_eq!( + store_root_label("b", "", Some(spelling)), + "s3://b/?endpoint=https://s3.example.com", + "{spelling:?}" + ); + } + assert_eq!( + store_root_label("b", "", Some("http://LocalHost:80/base/")), + "s3://b/?endpoint=http://localhost/base/" + ); + assert_eq!( + store_root_label("b", "", Some("http://[::1]:9000")), + "s3://b/?endpoint=http://[::1]:9000" + ); + } + #[test] fn store_prefix_normalization() { assert_eq!(normalize_store_prefix("").unwrap(), ""); @@ -2246,6 +2551,47 @@ mod tests { ) } + /// The two answers a conditional write classifies, from synthetic + /// service errors, by status alone, by code alone, and by both; every + /// other answer is `None` and takes `map_sdk_error`'s path. + #[test] + fn conditional_outcomes_from_a_reply_document() { + for (status, code, expected) in [ + (412, Some("PreconditionFailed"), Some(Conditional::Present)), + (412, None, Some(Conditional::Present)), + (400, Some("PreconditionFailed"), Some(Conditional::Present)), + ( + 409, + Some("ConditionalRequestConflict"), + Some(Conditional::Conflict), + ), + ( + 503, + Some("ConditionalRequestConflict"), + Some(Conditional::Conflict), + ), + // A bare 409 is some other conflict (a bucket operation's), not + // a conditional one. + (409, None, None), + (409, Some("BucketAlreadyOwnedByYou"), None), + (403, Some("AccessDenied"), None), + (404, Some("NoSuchKey"), None), + (500, Some("InternalError"), None), + ] { + assert_eq!( + conditional_outcome(&service_error(status, code)), + expected, + "{status} {code:?}" + ); + } + let transport: SdkError = + SdkError::construction_failure("no region"); + assert_eq!(conditional_outcome(&transport), None); + assert_eq!(conditional_backoff(1).as_millis(), 100); + assert_eq!(conditional_backoff(5).as_millis(), 1600); + assert_eq!(conditional_backoff(9).as_millis(), 1600, "capped"); + } + /// Every cell of `map_sdk_error`'s table, from synthetic service errors, /// so the mapping is pinned without an endpoint. #[test] diff --git a/crates/backup-store/tests/conformance.rs b/crates/backup-store/tests/conformance.rs index 1511b6164..30adcdd53 100644 --- a/crates/backup-store/tests/conformance.rs +++ b/crates/backup-store/tests/conformance.rs @@ -441,6 +441,146 @@ mod suite { ); assert!(list_keys(store, "aborted").await.is_empty()); } + + /// `put_if_absent` on an empty key stores the body and reports it; on a + /// key that holds an object (its own, or one `put` wrote) it reports + /// `None` and leaves the first body in place, whatever body the second + /// call carried, and the key's metadata is the first object's. The cell + /// over one part runs the multipart path on S3 (a conditional + /// `CompleteMultipartUpload`) and a plain link on the filesystem store; + /// its second call carries a body over one part too, so on S3 a + /// conditional complete meets the present key and has an upload to + /// abort, and its third call a body of one part. + pub async fn put_if_absent_creates_once(store: &dyn BackupStore) { + let small = b"first body".to_vec(); + let large = patterned_bytes(usize::try_from(PART_SIZE).unwrap() * 2 + 4096); + let mut other_large = patterned_bytes(usize::try_from(PART_SIZE).unwrap() * 2 + 2048); + other_large.reverse(); + for (key, first, seconds) in [ + ( + "ident/small.json", + small.clone(), + vec![b"second body, longer".to_vec()], + ), + ( + "ident/large.bin", + large.clone(), + vec![other_large, small.clone()], + ), + ] { + let stored = store + .put_if_absent(key, chunked_body(first.clone())) + .await + .expect("first put_if_absent succeeds") + .expect("an empty key is created"); + assert_eq!(stored.key, key); + assert_eq!(stored.size, first.len() as u64); + for second in seconds { + let again = store + .put_if_absent(key, chunked_body(second.clone())) + .await + .expect("a later put_if_absent succeeds"); + assert!(again.is_none(), "{key}: a present key reports None"); + let read = read_all(store.get(key).await.unwrap()).await; + assert_eq!(read, first, "{key}: the first body stays"); + } + let meta = store.head(key).await.unwrap().expect("object exists"); + assert_eq!(meta.size, first.len() as u64); + assert_eq!(list_keys(store, key).await, vec![key.to_owned()]); + } + // A key `put` wrote is present to `put_if_absent` too. + store + .put("ident/put-first", chunked_body(b"by put")) + .await + .unwrap(); + assert!( + store + .put_if_absent("ident/put-first", chunked_body(b"by put_if_absent")) + .await + .unwrap() + .is_none() + ); + assert_eq!( + read_all(store.get("ident/put-first").await.unwrap()).await, + b"by put" + ); + // And a key `put_if_absent` wrote is replaced by `put`, as any object + // is. + store + .put("ident/small.json", chunked_body(b"replaced")) + .await + .unwrap(); + assert_eq!( + read_all(store.get("ident/small.json").await.unwrap()).await, + b"replaced" + ); + // A body that fails leaves nothing under the key and the key stays + // free for the next call. + let err = store + .put_if_absent("ident/failing", failing_body(b"partial")) + .await + .expect_err("mid-body failure surfaces"); + assert!(matches!(&err, StoreError::Other(msg) if msg.contains("injected"))); + assert!(store.head("ident/failing").await.unwrap().is_none()); + assert!( + store + .put_if_absent("ident/failing", chunked_body(b"ok")) + .await + .unwrap() + .is_some() + ); + } + + /// Concurrent `put_if_absent` calls on one key: exactly one stores its + /// body, every other reads `None`, and the object under the key is the + /// winner's body. The bodies are distinct so the winner is identifiable; + /// the race is driven by starting every call before any is polled to + /// completion (`join_all`), and repeated so a store whose calls happen to + /// serialise once is still exercised across rounds. + pub async fn concurrent_put_if_absent_leaves_the_first_body(store: &dyn BackupStore) { + const CALLERS: usize = 8; + const ROUNDS: usize = 3; + for round in 0..ROUNDS { + let key = format!("race/round-{round}/extenddb-store.json"); + let bodies: Vec> = (0..CALLERS) + .map(|i| format!("{{\"round\":{round},\"caller\":{i}}}").into_bytes()) + .collect(); + let outcomes = futures::future::join_all( + bodies + .iter() + .map(|body| store.put_if_absent(&key, chunked_body(body.clone()))), + ) + .await; + let winners: Vec = outcomes + .iter() + .enumerate() + .filter_map(|(i, outcome)| { + outcome + .as_ref() + .expect("every call succeeds") + .as_ref() + .map(|_| i) + }) + .collect(); + assert_eq!( + winners.len(), + 1, + "round {round}: exactly one caller stores its body, got {winners:?}" + ); + let read = read_all(store.get(&key).await.unwrap()).await; + assert_eq!( + read, bodies[winners[0]], + "round {round}: the object is the winner's body" + ); + let meta = store.head(&key).await.unwrap().expect("object exists"); + assert_eq!(meta.size, bodies[winners[0]].len() as u64); + assert_eq!( + list_keys(store, &format!("race/round-{round}")).await, + vec![key.clone()], + "round {round}: one object under the prefix" + ); + } + } } #[cfg(unix)] @@ -557,6 +697,65 @@ mod filesystem { } } + #[tokio::test] + async fn put_if_absent_creates_once() { + let (dir, store) = fs_store().await; + suite::put_if_absent_creates_once(&store).await; + // The link discipline: no temporary stays, whether the call stored + // its object, found the key present, or failed mid-body, and the + // object the winning call linked is private. + assert!(temporaries_under(dir.path()).is_empty()); + assert_eq!( + mode_of(&dir.path().join("ident/large.bin")) & 0o777, + 0o600, + "a linked object keeps the temporary's mode" + ); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn concurrent_put_if_absent_leaves_the_first_body() { + let (dir, store) = fs_store().await; + suite::concurrent_put_if_absent_leaves_the_first_body(&store).await; + assert!(temporaries_under(dir.path()).is_empty()); + } + + /// `put_if_absent` applies the destination rules `put` does: a symlink at + /// the key is refused without being followed and its target untouched, a + /// directory at the key is a key-space collision, and an object at an + /// ancestor of the key makes the key unusable. None of the three + /// consumes the body or leaves a temporary. + #[tokio::test] + async fn put_if_absent_refuses_what_put_refuses() { + let (dir, store) = fs_store().await; + let outside = private_tempdir(); + let target = outside.path().join("target"); + std::fs::write(&target, b"outside").unwrap(); + private_dir(&dir.path().join("a")); + std::os::unix::fs::symlink(&target, dir.path().join("a/link")).unwrap(); + let err = store + .put_if_absent("a/link", chunked_body(b"x")) + .await + .expect_err("a symlink at the key is refused"); + assert!(matches!(err, StoreError::PermissionDenied(_)), "{err}"); + assert_eq!(std::fs::read(&target).unwrap(), b"outside"); + + store + .put("dir/under/obj", chunked_body(b"x")) + .await + .unwrap(); + let err = store + .put_if_absent("dir/under", chunked_body(b"x")) + .await + .expect_err("a directory at the key is refused"); + assert!(matches!(err, StoreError::Io { .. }), "{err}"); + let err = store + .put_if_absent("dir/under/obj/below", chunked_body(b"x")) + .await + .expect_err("an object at an ancestor is refused"); + assert!(matches!(err, StoreError::Io { .. }), "{err}"); + assert!(temporaries_under(dir.path()).is_empty()); + } + /// Every `.put-*` entry under `root`, at any depth. fn temporaries_under(root: &std::path::Path) -> Vec { let mut found = Vec::new(); @@ -1547,6 +1746,14 @@ mod filesystem { assert!(FilesystemStore::open(&missing).await.is_err()); } + /// An empty root path is refused by name, not as a stat of "" (whose + /// message would show an empty name between two spaces). + #[tokio::test] + async fn open_refuses_an_empty_root_path() { + let err = FilesystemStore::open("").await.err().expect("refused"); + assert!(err.to_string().contains("root path is empty"), "{err}"); + } + #[tokio::test] async fn symlinked_directory_escape_is_refused() { let outside = tempfile::tempdir().expect("outside dir"); @@ -2553,6 +2760,41 @@ mod s3 { suite::list_order_is_key_bytes(&store).await; } + /// Both cells of the conditional write: a single `PutObject` and a + /// `CompleteMultipartUpload`, each with `If-None-Match: *`. A present key + /// answers 412 to either; the multipart cell's upload is aborted on that + /// answer, which the listing of open uploads below checks. + #[tokio::test] + async fn put_if_absent_creates_once() { + let Some(endpoint) = endpoint_or_skip() else { + return; + }; + let prefix = unique_prefix(); + let config = test_config(endpoint, test_bucket(), prefix.clone()); + let store = S3Store::open(&config).await.expect("S3Store::open"); + ensure_bucket(&config).await; + suite::put_if_absent_creates_once(&store).await; + let open = raw_client(&config) + .await + .list_multipart_uploads() + .bucket(&config.bucket) + .prefix(&prefix) + .send() + .await + .expect("list_multipart_uploads"); + assert!( + open.uploads().is_empty(), + "a conditional complete that found the key present aborts its upload: {:?}", + open.uploads() + ); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn concurrent_put_if_absent_leaves_the_first_body() { + let Some(store) = store().await else { return }; + suite::concurrent_put_if_absent_leaves_the_first_body(&store).await; + } + #[tokio::test] async fn delete_prefix_removes_and_counts() { let Some(store) = store().await else { return }; @@ -2859,6 +3101,11 @@ mod s3 { .validate() .await .expect_err("nothing listens on port 9"); + assert!( + matches!(err, StoreError::Transport { .. }), + "a transport failure keeps its class through validate, so a server can tell it \ + from a configuration error: {err:?}" + ); let text = err.to_string(); assert!(text.contains("unused"), "names the bucket: {text}"); assert!( @@ -2867,6 +3114,284 @@ mod s3 { ); } + /// The 409 `ConditionalRequestConflict` loop of `put_if_absent`, driven + /// by a server of the test's own that answers from a script (moto never + /// answers 409): a `PutObject` answered 409 twice is re-sent and the + /// third answer stands; a `CompleteMultipartUpload` answered 409 and + /// then 412 is "present" and its upload is aborted; a `PutObject` + /// answered 409 nine times is the error the ninth answer is, after + /// eight retries. Runs in a child with the suite's own credentials, as + /// every request needs them. No endpoint is needed. + #[tokio::test] + async fn a_conditional_request_conflict_is_retried_from_a_reply_script() { + const NAME: &str = "s3::a_conditional_request_conflict_is_retried_from_a_reply_script"; + if !is_child(NAME) { + run_with_test_credentials(NAME); + return; + } + let server = ScriptedS3::start().await; + let config = test_config(server.endpoint(), "scripted".to_owned(), String::new()); + let store = S3Store::open(&config).await.expect("S3Store::open"); + + // Two conflicts, then created. + server.conflicts_before_success(2); + let meta = store + .put_if_absent("k/one-part", chunked_body(b"body")) + .await + .expect("the third answer is 200") + .expect("created"); + assert_eq!(meta.size, 4); + let puts = server.requests("PUT /scripted/k/one-part?x-id=PutObject"); + assert_eq!(puts, 3, "one send and two re-sends"); + + // The multipart cell: the complete answers 409 once, then 412. + server.conflicts_then_present(1); + let large = patterned_bytes(usize::try_from(PART_SIZE).unwrap() + 16); + let again = store + .put_if_absent("k/two-parts", chunked_body(large)) + .await + .expect("412 is an answer, not an error"); + assert!(again.is_none(), "a present key reports None"); + assert_eq!( + server.requests("POST /scripted/k/two-parts?uploadId="), + 2, + "the complete was re-sent once" + ); + assert_eq!( + server.requests("DELETE /scripted/k/two-parts?x-id=AbortMultipartUpload"), + 1, + "the upload the present key refused was aborted" + ); + + // Conflicts that never end: eight retries, then the error. + server.conflicts_before_success(u32::MAX); + let started = std::time::Instant::now(); + let err = store + .put_if_absent("k/never", chunked_body(b"body")) + .await + .expect_err("the ninth 409 is reported"); + let text = err.to_string(); + assert!( + text.contains("ConditionalRequestConflict"), + "names the conflict: {text}" + ); + assert_eq!(server.requests("PUT /scripted/k/never"), 9); + let waited = started.elapsed(); + assert!( + waited >= std::time::Duration::from_millis(7_000) + && waited < std::time::Duration::from_secs(20), + "the back-offs sum to about 7.9 s, measured {waited:?}" + ); + } + + /// An HTTP/1.1 server that plays S3 from a script: every request's + /// method, path and query are recorded; a `PutObject` or + /// `CompleteMultipartUpload` is answered 409 `ConditionalRequestConflict` + /// a scripted number of times, then 200, or 412 `PreconditionFailed`; + /// `CreateMultipartUpload`, `UploadPart` and `AbortMultipartUpload` are + /// answered as S3 answers them. + struct ScriptedS3 { + port: u16, + state: std::sync::Arc>, + } + + #[derive(Default)] + struct ScriptState { + /// Conflicts left to answer before the terminal answer. + conflicts: u32, + /// Whether the terminal answer is 412 rather than 200. + then_present: bool, + /// `" "` of every request, in order. + requests: Vec, + } + + impl ScriptedS3 { + async fn start() -> Self { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind"); + let port = listener.local_addr().unwrap().port(); + let state = std::sync::Arc::new(std::sync::Mutex::new(ScriptState::default())); + let shared = std::sync::Arc::clone(&state); + tokio::spawn(async move { + loop { + let Ok((socket, _)) = listener.accept().await else { + return; + }; + tokio::spawn(Self::serve(socket, std::sync::Arc::clone(&shared))); + } + }); + Self { port, state } + } + + fn endpoint(&self) -> String { + format!("http://127.0.0.1:{}", self.port) + } + + fn conflicts_before_success(&self, n: u32) { + let mut state = self.state.lock().unwrap(); + state.conflicts = n; + state.then_present = false; + } + + fn conflicts_then_present(&self, n: u32) { + let mut state = self.state.lock().unwrap(); + state.conflicts = n; + state.then_present = true; + } + + /// How many recorded requests start with `prefix`. + fn requests(&self, prefix: &str) -> usize { + self.state + .lock() + .unwrap() + .requests + .iter() + .filter(|r| r.starts_with(prefix)) + .count() + } + + async fn serve( + mut socket: tokio::net::TcpStream, + state: std::sync::Arc>, + ) { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let mut buffer = Vec::new(); + loop { + // Read one request: the head, then the body it announces. + let head_end = loop { + if let Some(at) = buffer.windows(4).position(|w| w == b"\r\n\r\n") { + break at + 4; + } + let mut chunk = [0u8; 8192]; + match socket.read(&mut chunk).await { + Ok(0) | Err(_) => return, + Ok(n) => buffer.extend_from_slice(&chunk[..n]), + } + }; + let head = String::from_utf8_lossy(&buffer[..head_end]).into_owned(); + let mut lines = head.split("\r\n"); + let request_line = lines.next().unwrap_or_default().to_owned(); + let mut content_length = 0usize; + let mut chunked = false; + for line in lines { + let Some((name, value)) = line.split_once(':') else { + continue; + }; + let value = value.trim(); + if name.eq_ignore_ascii_case("content-length") { + content_length = value.parse().unwrap_or(0); + } else if name.eq_ignore_ascii_case("transfer-encoding") + && value.eq_ignore_ascii_case("chunked") + { + chunked = true; + } + } + buffer.drain(..head_end); + if chunked { + // Read until the terminating chunk. + loop { + if buffer.windows(5).any(|w| w == b"0\r\n\r\n") { + let at = buffer.windows(5).position(|w| w == b"0\r\n\r\n").unwrap(); + buffer.drain(..at + 5); + break; + } + let mut chunk = [0u8; 65536]; + match socket.read(&mut chunk).await { + Ok(0) | Err(_) => return, + Ok(n) => buffer.extend_from_slice(&chunk[..n]), + } + } + } else { + while buffer.len() < content_length { + let mut chunk = [0u8; 65536]; + match socket.read(&mut chunk).await { + Ok(0) | Err(_) => return, + Ok(n) => buffer.extend_from_slice(&chunk[..n]), + } + } + buffer.drain(..content_length); + } + + let mut parts = request_line.split(' '); + let method = parts.next().unwrap_or_default().to_owned(); + let target = parts.next().unwrap_or_default().to_owned(); + let (status, body) = { + let mut state = state.lock().unwrap(); + state.requests.push(format!("{method} {target}")); + Self::answer(&mut state, &method, &target) + }; + let reason = match status { + 200 => "OK", + 204 => "No Content", + 409 => "Conflict", + 412 => "Precondition Failed", + _ => "Error", + }; + let response = format!( + "HTTP/1.1 {status} {reason}\r\nContent-Type: application/xml\r\n\ + Content-Length: {}\r\nETag: \"etag-1\"\r\nConnection: keep-alive\r\n\r\n{body}", + body.len() + ); + if socket.write_all(response.as_bytes()).await.is_err() { + return; + } + } + } + + /// The scripted answer to one request. + fn answer(state: &mut ScriptState, method: &str, target: &str) -> (u16, String) { + let conflict = || { + ( + 409, + "ConditionalRequestConflictConditional request \ + cannot succeed due to a conflicting operation against this \ + resource." + .to_owned(), + ) + }; + let present = || { + ( + 412, + "PreconditionFailedAt least one of the \ + pre-conditions you specified did not hold" + .to_owned(), + ) + }; + let conditional = |state: &mut ScriptState, success: String| { + if state.conflicts > 0 { + state.conflicts = state.conflicts.saturating_sub(1); + conflict() + } else if state.then_present { + present() + } else { + (200, success) + } + }; + match method { + "PUT" if target.contains("partNumber=") => (200, String::new()), + "PUT" => conditional(state, String::new()), + "POST" if target.ends_with("?uploads") => ( + 200, + "scriptedk\ + upload-1" + .to_owned(), + ), + "POST" if target.contains("uploadId=") => conditional( + state, + "scriptedk\ + \"etag-1\"" + .to_owned(), + ), + "DELETE" => (204, String::new()), + _ => ( + 404, + "NoSuchKeyno".to_owned(), + ), + } + } + } + /// A put whose future is dropped mid-upload (a cancelled job) aborts its /// multipart upload from the drop guard, and `delete_prefix` aborts any /// upload a crashed process left, so no parts accrue storage. @@ -5172,6 +5697,16 @@ mod s3 { }; println!("request URI: {uri}"); assert!(uri.starts_with(expected), "{uri} does not go to {expected}"); + // The root label names the endpoint the request went to, in its one + // spelling, whichever source supplied it. + let endpoint = expected.trim_end_matches('/'); + let label = store.store_root(); + println!("store root: {label}"); + assert_eq!( + label, + format!("s3://env-bucket/?endpoint={endpoint}"), + "the label names the effective endpoint" + ); } /// Rewrites every request's URI, after the endpoint is resolved and diff --git a/crates/bin/src/main.rs b/crates/bin/src/main.rs index 403986e24..15876a052 100755 --- a/crates/bin/src/main.rs +++ b/crates/bin/src/main.rs @@ -75,6 +75,8 @@ fn main() -> anyhow::Result<()> { version: env!("CARGO_PKG_VERSION"), git_hash: env!("EXTENDDB_GIT_HASH"), build_time: env!("EXTENDDB_BUILD_TIME"), + // The MongoDB hooks; the backup jobs' hooks are `extenddb-app`'s + // `backup-test-hooks` feature, which `version` reports itself. test_hooks_enabled: cfg!(feature = "mongodb-test-hooks"), }) } diff --git a/crates/config/Cargo.toml b/crates/config/Cargo.toml index 1c2c59ebf..f9f086124 100644 --- a/crates/config/Cargo.toml +++ b/crates/config/Cargo.toml @@ -15,8 +15,12 @@ dev-mode = [] [dependencies] extenddb-core = { workspace = true } extenddb-storage = { workspace = true } +extenddb-backup-store = { workspace = true } serde = { workspace = true } toml = { workspace = true } config = { workspace = true } anyhow = { workspace = true } tracing = { workspace = true } + +[dev-dependencies] +tempfile = "3" diff --git a/crates/config/src/backup.rs b/crates/config/src/backup.rs new file mode 100644 index 000000000..098acd631 --- /dev/null +++ b/crates/config/src/backup.rs @@ -0,0 +1,1032 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The `[backup]` section: the knobs backup and restore jobs read, and the +//! store they write to. +//! +//! ```toml +//! [backup] +//! data_file_target_bytes = 67108864 # uncompressed bytes per data file; minimum 32 MiB +//! heartbeat_interval_secs = 15 # job heartbeat timer +//! stale_job_after_secs = 300 # heartbeat age after which a job's owner is dead +//! failed_row_retention_secs = 604800 # FAILED tombstones kept for the operator, 7 days +//! reconcile_interval_secs = 60 # reconciler cycle +//! max_concurrent_jobs = 4 # backup and restore jobs running at once on this instance +//! # spool_path = "/var/lib/extenddb/spool" # restore download area for an S3 store +//! +//! [backup.store] +//! kind = "filesystem" # "filesystem" | "s3" | "disabled" +//! [backup.store.filesystem] +//! path = "/var/lib/extenddb/backups" +//! ``` +//! +//! `stale_job_after_secs` must be at least three heartbeat periods: a job +//! checks before each store write that its latest heartbeat confirmed its +//! claim recently enough (within two periods: one for the timer, one for the +//! beat to return), and the margin is what makes that detection precede a +//! reconciler's deletion of the job's prefix. The period the floor is +//! computed from is the one the jobs run at: the larger of +//! `heartbeat_interval_secs` and [`JOB_HEARTBEAT_INTERVAL`] (15 s). The +//! store-backed backup job heartbeats at that period; the in-catalog backup +//! and restore paths heartbeat at the constant whatever the knob says, and a +//! floor on a shorter configured interval would let the reconciler declare +//! one of those dead between two of its heartbeats. The floor is a floor: on +//! SQLite a heartbeat waits for the engine write lock like any writer, so a +//! long write transaction delays it, and a `stale_job_after_secs` near three +//! periods on a busy file can declare a live restore dead. +//! +//! `data_file_target_bytes` is read by the store-backed backup job (a data +//! file closes once its uncompressed bytes reach it, so a file exceeds it by +//! less than one line); `failed_row_retention_secs` by the reconciler, which +//! removes a FAILED or DELETED row whose prefix is empty once its +//! `completed_at` is older than that. The retention is at least +//! `stale_job_after_secs`: a job stopped and resumed (SIGSTOP) can land a put +//! after its row was failed, and the reconciler's FAILED-prefix duty deletes +//! that object only while the row exists, so the row must outlive the +//! window in which the stopped owner can still write. +//! +//! `max_concurrent_jobs` is at most [`MAX_CONCURRENT_JOBS`]: a value over +//! the runtime's permit ceiling would stop the server at startup rather than +//! at load, and the PostgreSQL pool rule makes anything over a few hundred +//! meaningless anyway. +//! +//! The store. `[backup.store]` is the backup store crate's +//! [`BackupStoreConfig`], nested under its own key so the one definition of +//! the shape stays in that crate and `deny_unknown_fields` holds on both +//! tables (`serde(flatten)` would drop it). Its keys, defaults and `open`'s +//! refusals are that crate's `config` module doc. What this module adds is +//! the per-backend rule for an absent section and the spool +//! ([`BackupConfig::resolve_store`]): +//! +//! - On a backend whose database is one file on this host (file-backed +//! SQLite), a missing `[backup.store]` is `kind = "filesystem"` with +//! `path = "backups/"` beside the database file. That defaulted root is +//! created `0700` if missing, and a failure of its validation is a logged +//! warning that leaves the store unconfigured, not a startup error, so an +//! upgrade never stops a server that never configured backups; an +//! explicitly configured root that fails validation is a startup error. +//! - On PostgreSQL, MongoDB, Cassandra and an in-memory SQLite database +//! (no file, so no default root), a missing section leaves the store +//! unconfigured and startup logs that. +//! - `kind = "disabled"` leaves the store unconfigured on purpose, on any +//! backend. +//! - With a configured store, CreateBackup on PostgreSQL, SQLite and +//! MongoDB writes the backup to the store and returns CREATING; without +//! one, and on Cassandra always, it writes the in-catalog format and logs +//! a deprecation warning naming `[backup.store]` on each call; backups of +//! either format stay readable, listable, restorable and deletable. +//! +//! The spool. Restores from an S3 store download to `spool_path`, created +//! `0700` with files `0600`: one entry `restore-
    /` per running +//! restore, holding the manifest, its sidecar and each data file with a +//! `.verified` marker once its digests matched, so the spool costs the +//! compressed size of the backups being restored at once; the job removes +//! its entry when it ends, a server stopped by SIGTERM keeps it for the +//! resume, and the reconciler removes entries of restores no longer RUNNING +//! with this host's `spool_host`. On file-backed SQLite it defaults to +//! `backup-spool/` beside the database file; on every other backend an S3 +//! store needs `spool_path` and startup refuses without it (a filesystem +//! store needs no spool: files are hashed in place and read again for the +//! load, so a filesystem store's `get` errors, which are local I/O errors, +//! are not retried as an S3 store's transport errors are). A `spool_path` that +//! contains or is contained in a filesystem store root is refused (the store +//! would list and delete it, and a sync would read its first component as an +//! account id), and so is one that is or contains the database file's +//! directory. A filesystem store root that is or contains the database +//! file's directory is refused too: the database and its sidecars would sit +//! among the backups. An empty root is compared with nothing (it names no +//! directory) and left for `open` to refuse by name. The spool's `.lock` +//! another live instance holds is refused at startup by the server, which +//! owns the lock file. +//! +//! A relative `[backup.store.filesystem] path` or `spool_path` is made +//! absolute against the working directory at config load +//! ([`BackupConfig::make_paths_absolute`]), where `storage.path` is: `serve` +//! daemonizes after loading, and the child's working directory is `/`. + +use std::path::{Path, PathBuf}; + +pub use extenddb_backup_store::{ + BackupStoreConfig, BackupStoreKind, FilesystemStoreConfig, S3StoreConfig, +}; +use extenddb_storage::backup_catalog::JOB_HEARTBEAT_INTERVAL; +use serde::Deserialize; + +/// The smallest `data_file_target_bytes`: the cap on one line of a data +/// file, so a file closes at most one line past the knob. +pub const MIN_DATA_FILE_TARGET_BYTES: u64 = 32 * 1024 * 1024; + +/// The smallest ratio of `stale_job_after_secs` to the heartbeat period +/// ([`BackupConfig::effective_heartbeat_interval_secs`]). +pub const MIN_STALE_HEARTBEATS: u64 = 3; + +/// The largest `max_concurrent_jobs`. +pub const MAX_CONCURRENT_JOBS: usize = 1024; + +/// The directory name of the defaulted filesystem store root beside a +/// database file. +pub const DEFAULT_ROOT_DIR_NAME: &str = "backups"; + +/// The directory name of the defaulted restore spool beside a database file. +pub const DEFAULT_SPOOL_DIR_NAME: &str = "backup-spool"; + +#[derive(Debug, Clone, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct BackupConfig { + /// Uncompressed bytes per data file; a file closes once it reaches this, + /// so it exceeds it by less than one line. + #[serde(default = "default_data_file_target_bytes")] + pub data_file_target_bytes: u64, + /// The job heartbeat timer. + #[serde(default = "default_heartbeat_interval_secs")] + pub heartbeat_interval_secs: u64, + /// Heartbeat age after which a job's owner is dead, and the delay before + /// the reconciler deletes a dead job's prefix. + #[serde(default = "default_stale_job_after_secs")] + pub stale_job_after_secs: u64, + /// How long FAILED tombstones are kept for the operator. + #[serde(default = "default_failed_row_retention_secs")] + pub failed_row_retention_secs: u64, + /// The reconciler's cycle. + #[serde(default = "default_reconcile_interval_secs")] + pub reconcile_interval_secs: u64, + /// Backup and restore jobs running at once on this instance, the + /// in-catalog creates and restores included; a further CreateBackup or + /// RestoreTableFromBackup is `LimitExceededException`. + #[serde(default = "default_max_concurrent_jobs")] + pub max_concurrent_jobs: usize, + /// Restore download area for an S3 store. Defaults beside the database + /// file on SQLite; required on the other backends with an S3 store. + #[serde(default)] + pub spool_path: Option, + /// `[backup.store]`: the store this instance writes backups to. `None` + /// when the section is absent, which [`BackupConfig::resolve_store`] + /// turns into the per-backend default. + #[serde(default)] + pub store: Option, +} + +impl Default for BackupConfig { + fn default() -> Self { + Self { + data_file_target_bytes: default_data_file_target_bytes(), + heartbeat_interval_secs: default_heartbeat_interval_secs(), + stale_job_after_secs: default_stale_job_after_secs(), + failed_row_retention_secs: default_failed_row_retention_secs(), + reconcile_interval_secs: default_reconcile_interval_secs(), + max_concurrent_jobs: default_max_concurrent_jobs(), + spool_path: None, + store: None, + } + } +} + +/// Why no store is configured. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Unconfigured { + /// `[backup.store]` is absent and this backend has no default root (a + /// server database, or an in-memory SQLite database). + Absent, + /// `kind = "disabled"`. + Disabled, +} + +/// The store `[backup.store]` and the backend together select. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum EffectiveStore { + /// No store: CreateBackup writes the in-catalog format. + Unconfigured(Unconfigured), + /// A store to open and validate at startup. + Configured { + config: BackupStoreConfig, + /// The section was absent and the root is the default beside the + /// database file: the server creates it if missing and treats a + /// validation failure as a warning that leaves the store + /// unconfigured. + defaulted: bool, + }, +} + +/// What [`BackupConfig::resolve_store`] decided. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ResolvedStore { + pub store: EffectiveStore, + /// The restore spool, when the configured store is S3. + pub spool_path: Option, +} + +impl BackupConfig { + /// The period the store-backed backup job heartbeats at, and the floor's + /// unit: the larger of `heartbeat_interval_secs` and + /// [`JOB_HEARTBEAT_INTERVAL`] (the in-catalog paths run at the constant). + #[must_use] + pub fn effective_heartbeat_interval_secs(&self) -> u64 { + self.heartbeat_interval_secs + .max(JOB_HEARTBEAT_INTERVAL.as_secs()) + } + + /// Make `[backup.store.filesystem] path` and `spool_path` absolute + /// against `cwd` (`~` expanded first), so a relative value written in + /// the config file names the directory it does from the shell that + /// started the server, in the daemonized child too. An absolute value + /// is left as it is, and so is an empty one, which `open` refuses as it + /// did (it would otherwise name the working directory as the root). + pub fn make_paths_absolute(&mut self, cwd: &Path) { + if let Some(fs) = self.store.as_mut().and_then(|s| s.filesystem.as_mut()) + && !fs.path.is_absolute() + && !fs.path.as_os_str().is_empty() + { + let expanded = PathBuf::from(crate::expand_tilde(&fs.path.to_string_lossy())); + fs.path = if expanded.is_absolute() { + expanded + } else { + cwd.join(expanded) + }; + } + if let Some(spool) = self.spool_path.as_mut() { + let expanded = PathBuf::from(crate::expand_tilde(spool)); + *spool = if expanded.is_absolute() { + expanded + } else { + cwd.join(expanded) + } + .to_string_lossy() + .into_owned(); + } + } + + /// The cross-field rules, checked at load. + /// + /// # Errors + /// + /// A message naming the key and the rule it breaks. + pub fn validate(&self) -> Result<(), String> { + if self.heartbeat_interval_secs == 0 { + return Err("backup.heartbeat_interval_secs must be at least 1".to_owned()); + } + let period = self.effective_heartbeat_interval_secs(); + let floor = period.saturating_mul(MIN_STALE_HEARTBEATS); + if self.stale_job_after_secs < floor { + return Err(format!( + "backup.stale_job_after_secs = {} is under {MIN_STALE_HEARTBEATS} * the job \ + heartbeat period of {period} s = {floor} (the larger of \ + backup.heartbeat_interval_secs = {} and the {} s the in-catalog jobs \ + heartbeat at); a reconciler could declare a running job dead between two \ + of its heartbeats", + self.stale_job_after_secs, + self.heartbeat_interval_secs, + JOB_HEARTBEAT_INTERVAL.as_secs() + )); + } + if self.failed_row_retention_secs < self.stale_job_after_secs { + return Err(format!( + "backup.failed_row_retention_secs = {} is under backup.stale_job_after_secs = \ + {}; a FAILED row removed before its owner's last heartbeat is stale leaves no \ + row for the reconciler to delete that owner's late objects under", + self.failed_row_retention_secs, self.stale_job_after_secs + )); + } + if self.data_file_target_bytes < MIN_DATA_FILE_TARGET_BYTES { + return Err(format!( + "backup.data_file_target_bytes = {} is under the minimum of \ + {MIN_DATA_FILE_TARGET_BYTES} (32 MiB, the cap on one line of a data file)", + self.data_file_target_bytes + )); + } + if self.reconcile_interval_secs == 0 { + return Err("backup.reconcile_interval_secs must be at least 1".to_owned()); + } + if self.max_concurrent_jobs == 0 { + return Err("backup.max_concurrent_jobs must be at least 1".to_owned()); + } + if self.max_concurrent_jobs > MAX_CONCURRENT_JOBS { + return Err(format!( + "backup.max_concurrent_jobs = {} is over the maximum of {MAX_CONCURRENT_JOBS}", + self.max_concurrent_jobs + )); + } + if let Some(spool) = &self.spool_path + && spool.trim().is_empty() + { + return Err("backup.spool_path must not be empty".to_owned()); + } + Ok(()) + } + + /// The store and spool this configuration selects on a backend whose + /// database is `database_file` (`Some` for file-backed SQLite, `None` + /// for a server database or an in-memory one), by the rules in the + /// module doc. + /// + /// # Errors + /// + /// A message naming the key: an S3 store without `spool_path` on a + /// backend with no default spool; a `spool_path` inside or containing a + /// filesystem store root, or inside or containing the database file's + /// directory; a filesystem root that is or contains the database file's + /// directory. + pub fn resolve_store(&self, database_file: Option<&Path>) -> Result { + let database_dir = database_file.and_then(Path::parent).map(Path::to_path_buf); + let store = match &self.store { + None => match &database_dir { + Some(dir) => EffectiveStore::Configured { + config: BackupStoreConfig { + kind: BackupStoreKind::Filesystem, + filesystem: Some(FilesystemStoreConfig { + path: dir.join(DEFAULT_ROOT_DIR_NAME), + }), + s3: None, + }, + defaulted: true, + }, + None => EffectiveStore::Unconfigured(Unconfigured::Absent), + }, + Some(config) if config.kind == BackupStoreKind::Disabled => { + EffectiveStore::Unconfigured(Unconfigured::Disabled) + } + Some(config) => EffectiveStore::Configured { + config: config.clone(), + defaulted: false, + }, + }; + + let explicit_spool = self + .spool_path + .as_deref() + .map(|p| PathBuf::from(crate::expand_tilde(p))); + let filesystem_root = match &store { + EffectiveStore::Configured { config, .. } + if config.kind == BackupStoreKind::Filesystem => + { + // An empty `path` names no directory: it is compared with + // nothing here (compared as written it would be the working + // directory, and refused by name as the database directory + // when the server starts there), and `open` refuses it. + config + .filesystem + .as_ref() + .map(|f| f.path.clone()) + .filter(|path| !path.as_os_str().is_empty()) + } + _ => None, + }; + let is_s3 = matches!( + &store, + EffectiveStore::Configured { config, .. } if config.kind == BackupStoreKind::S3 + ); + + // The spool rules apply to every explicit `spool_path`, so a + // configuration that breaks them is refused before the store it + // will one day be used with is selected. Each message spells both + // paths as they were compared (resolved through existing symlinks). + if let Some(spool) = &explicit_spool { + if let Some(root) = &filesystem_root + && paths_overlap(spool, root) + { + return Err(format!( + "backup.spool_path {} is inside or contains the filesystem backup store \ + root {}; the store would list and delete the spool. Put the spool \ + outside the root", + comparable_path(spool).display(), + comparable_path(root).display() + )); + } + if let Some(dir) = &database_dir + && path_is_or_contains(spool, dir) + { + return Err(format!( + "backup.spool_path {} is or contains the database directory {}; put the \ + spool in a directory of its own", + comparable_path(spool).display(), + comparable_path(dir).display() + )); + } + } + // A filesystem root that is the database directory or an ancestor of + // it would hold the database file and its sidecars among the + // backups: a listing would read the directory's entries as account + // ids and a deletion of the root's whole prefix would walk them. The + // default root is a child of the directory and passes. + if let (Some(root), Some(dir)) = (&filesystem_root, &database_dir) + && path_is_or_contains(root, dir) + { + return Err(format!( + "backup.store.filesystem.path {} is or contains the database directory {}; \ + put the backup root in a directory of its own (the default is \ + {DEFAULT_ROOT_DIR_NAME}/ beside the database file)", + comparable_path(root).display(), + comparable_path(dir).display() + )); + } + + let spool_path = if is_s3 { + match (explicit_spool, &database_dir) { + (Some(spool), _) => Some(spool), + (None, Some(dir)) => Some(dir.join(DEFAULT_SPOOL_DIR_NAME)), + (None, None) => { + return Err( + "backup.spool_path is required with an S3 backup store on this \ + backend: a restore downloads its files there before loading them, \ + and only a file-backed SQLite database has a default beside its file" + .to_owned(), + ); + } + } + } else { + None + }; + Ok(ResolvedStore { store, spool_path }) + } +} + +/// Whether one path is the other or an ancestor of it, after each is made +/// absolute and lexically normalized (`.` and `..` resolved without +/// consulting the filesystem, since neither directory need exist yet) and +/// canonicalized when it exists, so a symlinked spelling of the same +/// directory compares equal. +#[must_use] +pub fn paths_overlap(a: &Path, b: &Path) -> bool { + let a = comparable_path(a); + let b = comparable_path(b); + a.starts_with(&b) || b.starts_with(&a) +} + +/// Whether `a` is `b` or an ancestor of it, compared as [`paths_overlap`] +/// compares. +#[must_use] +pub fn path_is_or_contains(a: &Path, b: &Path) -> bool { + comparable_path(b).starts_with(comparable_path(a)) +} + +fn comparable_path(path: &Path) -> PathBuf { + let absolute = if path.is_absolute() { + path.to_path_buf() + } else { + std::env::current_dir() + .map(|cwd| cwd.join(path)) + .unwrap_or_else(|_| path.to_path_buf()) + }; + let mut normalized = PathBuf::new(); + for component in absolute.components() { + match component { + std::path::Component::CurDir => {} + std::path::Component::ParentDir => { + normalized.pop(); + } + other => normalized.push(other.as_os_str()), + } + } + // The deepest existing ancestor decides the canonical spelling; the + // rest is appended as written. + let mut existing = normalized.clone(); + let mut tail = Vec::new(); + loop { + if let Ok(canonical) = std::fs::canonicalize(&existing) { + let mut out = canonical; + for part in tail.iter().rev() { + out.push(part); + } + return out; + } + match (existing.file_name(), existing.parent()) { + (Some(name), Some(parent)) => { + tail.push(name.to_os_string()); + existing = parent.to_path_buf(); + } + _ => return normalized, + } + } +} + +fn default_data_file_target_bytes() -> u64 { + 64 * 1024 * 1024 +} +fn default_heartbeat_interval_secs() -> u64 { + 15 +} +fn default_stale_job_after_secs() -> u64 { + 300 +} +fn default_failed_row_retention_secs() -> u64 { + 7 * 24 * 3600 +} +fn default_reconcile_interval_secs() -> u64 { + 60 +} +fn default_max_concurrent_jobs() -> usize { + 4 +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_defaults_are_the_documented_ones() { + let c = BackupConfig::default(); + assert_eq!(c.data_file_target_bytes, 67_108_864); + assert_eq!(c.heartbeat_interval_secs, 15); + assert_eq!(c.stale_job_after_secs, 300); + assert_eq!(c.failed_row_retention_secs, 604_800); + assert_eq!(c.reconcile_interval_secs, 60); + assert_eq!(c.max_concurrent_jobs, 4); + assert_eq!(c.spool_path, None); + assert_eq!(c.store, None); + c.validate().unwrap(); + let parsed: BackupConfig = toml::from_str("").unwrap(); + assert_eq!(parsed, c); + } + + /// A relative root or spool is made absolute against the directory + /// given (the shell's at load); an absolute one, and `~`, resolve as + /// written. + #[test] + fn relative_store_and_spool_paths_are_made_absolute_at_load() { + let mut c: BackupConfig = toml::from_str( + "spool_path = \"rel-spool\"\n[store]\nkind = \"filesystem\"\n\ + [store.filesystem]\npath = \"rel-backups\"\n", + ) + .unwrap(); + c.make_paths_absolute(Path::new("/srv/extenddb")); + assert_eq!( + c.store.as_ref().unwrap().filesystem.as_ref().unwrap().path, + PathBuf::from("/srv/extenddb/rel-backups") + ); + assert_eq!(c.spool_path.as_deref(), Some("/srv/extenddb/rel-spool")); + + let mut absolute: BackupConfig = toml::from_str( + "spool_path = \"~/spool\"\n[store]\nkind = \"filesystem\"\n\ + [store.filesystem]\npath = \"/var/lib/extenddb/backups\"\n", + ) + .unwrap(); + absolute.make_paths_absolute(Path::new("/srv/extenddb")); + assert_eq!( + absolute + .store + .as_ref() + .unwrap() + .filesystem + .as_ref() + .unwrap() + .path, + PathBuf::from("/var/lib/extenddb/backups") + ); + let home = std::env::var("HOME").unwrap(); + assert_eq!( + absolute.spool_path.as_deref(), + Some(format!("{home}/spool").as_str()) + ); + // An S3 store has no path to resolve; `None` stays `None`. + let mut none = BackupConfig::default(); + none.make_paths_absolute(Path::new("/srv")); + assert_eq!(none, BackupConfig::default()); + // An empty root stays empty, for `open` to refuse; it must not + // become the working directory. + let mut empty: BackupConfig = + toml::from_str("[store]\nkind = \"filesystem\"\n[store.filesystem]\npath = \"\"\n") + .unwrap(); + empty.make_paths_absolute(Path::new("/srv/extenddb")); + assert_eq!( + empty + .store + .as_ref() + .unwrap() + .filesystem + .as_ref() + .unwrap() + .path, + PathBuf::from("") + ); + } + + #[test] + fn an_unknown_key_is_refused_at_every_level() { + let e = toml::from_str::("store_kind = \"s3\"\n").unwrap_err(); + assert!(e.to_string().contains("store_kind"), "{e}"); + let e = toml::from_str::("[store]\nkid = \"s3\"\n").unwrap_err(); + assert!(e.to_string().contains("kid"), "{e}"); + let e = toml::from_str::( + "[store]\nkind = \"s3\"\n[store.s3]\nbucket = \"b\"\nbukcet = \"b\"\n", + ) + .unwrap_err(); + assert!(e.to_string().contains("bukcet"), "{e}"); + } + + #[test] + fn the_store_section_is_the_crates_shape() { + let c: BackupConfig = + toml::from_str("[store]\nkind = \"s3\"\n[store.s3]\nbucket = \"b\"\nprefix = \"p/\"\n") + .unwrap(); + let store = c.store.unwrap(); + assert_eq!(store.kind, BackupStoreKind::S3); + assert_eq!(store.s3.unwrap().bucket, "b"); + let c: BackupConfig = toml::from_str("[store]\nkind = \"disabled\"\n").unwrap(); + assert_eq!(c.store.unwrap().kind, BackupStoreKind::Disabled); + } + + #[test] + fn a_stale_window_under_three_heartbeats_is_refused() { + let c = BackupConfig { + heartbeat_interval_secs: 20, + stale_job_after_secs: 59, + ..BackupConfig::default() + }; + let e = c.validate().unwrap_err(); + assert!(e.contains("stale_job_after_secs = 59"), "{e}"); + assert!(e.contains("= 60"), "{e}"); + let ok = BackupConfig { + stale_job_after_secs: 60, + ..c + }; + ok.validate().unwrap(); + } + + /// The floor is computed from the period the jobs run at, not from a + /// configured interval shorter than it: `heartbeat_interval_secs = 1` + /// with `stale_job_after_secs = 3` would let the reconciler remove a live + /// restore between two 15 s heartbeats, and is refused naming the 15 s + /// period and the 45 s floor. + #[test] + fn the_stale_floor_is_three_times_the_period_the_jobs_heartbeat_at() { + let c = BackupConfig { + heartbeat_interval_secs: 1, + stale_job_after_secs: 3, + ..BackupConfig::default() + }; + assert_eq!(c.effective_heartbeat_interval_secs(), 15); + let e = c.validate().unwrap_err(); + assert!(e.contains("stale_job_after_secs = 3 "), "{e}"); + assert!(e.contains("period of 15 s = 45"), "{e}"); + assert!(e.contains("heartbeat_interval_secs = 1 "), "{e}"); + let under = BackupConfig { + stale_job_after_secs: 44, + ..c.clone() + }; + assert!(under.validate().is_err()); + let ok = BackupConfig { + stale_job_after_secs: 45, + ..c + }; + ok.validate().unwrap(); + // A configured interval over the constant is the period. + let long = BackupConfig { + heartbeat_interval_secs: 40, + stale_job_after_secs: 120, + ..BackupConfig::default() + }; + assert_eq!(long.effective_heartbeat_interval_secs(), 40); + long.validate().unwrap(); + } + + /// The retention is at least the stale window: a FAILED row removed + /// sooner leaves a stopped owner's late object with no row under which + /// the reconciler deletes it. + #[test] + fn a_retention_under_the_stale_window_is_refused() { + let c = BackupConfig { + stale_job_after_secs: 300, + failed_row_retention_secs: 299, + ..BackupConfig::default() + }; + let e = c.validate().unwrap_err(); + assert!(e.contains("failed_row_retention_secs = 299"), "{e}"); + assert!(e.contains("stale_job_after_secs = 300"), "{e}"); + let ok = BackupConfig { + failed_row_retention_secs: 300, + ..c + }; + ok.validate().unwrap(); + let zero = BackupConfig { + failed_row_retention_secs: 0, + ..BackupConfig::default() + }; + assert!(zero.validate().is_err()); + } + + #[test] + fn max_concurrent_jobs_has_an_upper_bound() { + let at_most = BackupConfig { + max_concurrent_jobs: MAX_CONCURRENT_JOBS, + ..BackupConfig::default() + }; + at_most.validate().unwrap(); + let over = BackupConfig { + max_concurrent_jobs: MAX_CONCURRENT_JOBS + 1, + ..BackupConfig::default() + }; + let e = over.validate().unwrap_err(); + assert!(e.contains("max_concurrent_jobs = 1025"), "{e}"); + assert!(e.contains("1024"), "{e}"); + let huge = BackupConfig { + max_concurrent_jobs: usize::MAX, + ..BackupConfig::default() + }; + assert!(huge.validate().is_err()); + } + + #[test] + fn the_other_floors_are_checked() { + let small_file = BackupConfig { + data_file_target_bytes: MIN_DATA_FILE_TARGET_BYTES - 1, + ..BackupConfig::default() + }; + assert!( + small_file + .validate() + .unwrap_err() + .contains("data_file_target_bytes") + ); + let no_jobs = BackupConfig { + max_concurrent_jobs: 0, + ..BackupConfig::default() + }; + assert!( + no_jobs + .validate() + .unwrap_err() + .contains("max_concurrent_jobs") + ); + let no_heartbeat = BackupConfig { + heartbeat_interval_secs: 0, + ..BackupConfig::default() + }; + assert!( + no_heartbeat + .validate() + .unwrap_err() + .contains("heartbeat_interval_secs") + ); + let no_cycle = BackupConfig { + reconcile_interval_secs: 0, + ..BackupConfig::default() + }; + assert!( + no_cycle + .validate() + .unwrap_err() + .contains("reconcile_interval_secs") + ); + let empty_spool = BackupConfig { + spool_path: Some(" ".to_owned()), + ..BackupConfig::default() + }; + assert!(empty_spool.validate().unwrap_err().contains("spool_path")); + } + + fn s3_store() -> BackupStoreConfig { + toml::from_str("kind = \"s3\"\n[s3]\nbucket = \"b\"\n").unwrap() + } + + fn fs_store(path: &str) -> BackupStoreConfig { + toml::from_str(&format!( + "kind = \"filesystem\"\n[filesystem]\npath = {path:?}\n" + )) + .unwrap() + } + + /// An absent section on a file-backed database is the filesystem store + /// beside the file, marked defaulted; on a server or in-memory database + /// it is unconfigured; `disabled` is unconfigured on both. + #[test] + fn an_absent_section_defaults_per_backend() { + let db = Path::new("/var/lib/extenddb/data.sqlite"); + let resolved = BackupConfig::default().resolve_store(Some(db)).unwrap(); + assert_eq!( + resolved.store, + EffectiveStore::Configured { + config: fs_store("/var/lib/extenddb/backups"), + defaulted: true, + } + ); + assert_eq!( + resolved.spool_path, None, + "a filesystem store needs no spool" + ); + + let resolved = BackupConfig::default().resolve_store(None).unwrap(); + assert_eq!( + resolved.store, + EffectiveStore::Unconfigured(Unconfigured::Absent) + ); + assert_eq!(resolved.spool_path, None); + + let disabled = BackupConfig { + store: Some(toml::from_str("kind = \"disabled\"").unwrap()), + ..BackupConfig::default() + }; + for database in [Some(db), None] { + assert_eq!( + disabled.resolve_store(database).unwrap().store, + EffectiveStore::Unconfigured(Unconfigured::Disabled) + ); + } + + let explicit = BackupConfig { + store: Some(fs_store("/srv/backups")), + ..BackupConfig::default() + }; + assert_eq!( + explicit.resolve_store(Some(db)).unwrap().store, + EffectiveStore::Configured { + config: fs_store("/srv/backups"), + defaulted: false, + } + ); + } + + /// An S3 store's spool: explicit on any backend, defaulted beside a + /// database file, required otherwise with the key named. + #[test] + fn the_spool_is_required_with_s3_except_beside_a_database_file() { + let db = Path::new("/var/lib/extenddb/data.sqlite"); + let s3 = BackupConfig { + store: Some(s3_store()), + ..BackupConfig::default() + }; + assert_eq!( + s3.resolve_store(Some(db)).unwrap().spool_path, + Some(PathBuf::from("/var/lib/extenddb/backup-spool")) + ); + let e = s3.resolve_store(None).unwrap_err(); + assert!(e.contains("backup.spool_path is required"), "{e}"); + + let with_spool = BackupConfig { + spool_path: Some("/srv/spool".to_owned()), + ..s3.clone() + }; + for database in [Some(db), None] { + assert_eq!( + with_spool.resolve_store(database).unwrap().spool_path, + Some(PathBuf::from("/srv/spool")) + ); + } + } + + /// Every cell of the spool placement rule: inside the root, containing + /// the root, equal to the root; the database directory itself, an + /// ancestor of it; and the accepted shapes (a sibling of the root, a + /// directory under the database directory, a path elsewhere), each + /// against a filesystem store and against an S3 store. + #[test] + fn a_spool_inside_or_containing_the_root_or_the_database_directory_is_refused() { + let db = Path::new("/var/lib/extenddb/data.sqlite"); + let resolve = |store: BackupStoreConfig, spool: &str| { + BackupConfig { + store: Some(store), + spool_path: Some(spool.to_owned()), + ..BackupConfig::default() + } + .resolve_store(Some(db)) + }; + for spool in [ + "/srv/backups/spool", + "/srv", + "/srv/backups", + "/srv/backups/../backups", + ] { + let e = resolve(fs_store("/srv/backups"), spool).unwrap_err(); + assert!(e.contains("filesystem backup store root"), "{spool}: {e}"); + } + for spool in ["/var/lib/extenddb", "/var/lib", "/var/lib/extenddb/./"] { + let e = resolve(fs_store("/srv/backups"), spool).unwrap_err(); + assert!(e.contains("database directory"), "{spool}: {e}"); + let e = resolve(s3_store(), spool).unwrap_err(); + assert!(e.contains("database directory"), "{spool}: {e}"); + } + for spool in [ + "/srv/backup-spool", + "/var/lib/extenddb/backup-spool", + "/tmp/extenddb-spool", + ] { + resolve(fs_store("/srv/backups"), spool).unwrap_or_else(|e| panic!("{spool}: {e}")); + let resolved = resolve(s3_store(), spool).unwrap_or_else(|e| panic!("{spool}: {e}")); + assert_eq!(resolved.spool_path, Some(PathBuf::from(spool))); + } + // The defaulted root beside the database file is a root too. + let e = BackupConfig { + spool_path: Some("/var/lib/extenddb/backups/spool".to_owned()), + ..BackupConfig::default() + } + .resolve_store(Some(db)) + .unwrap_err(); + assert!(e.contains("filesystem backup store root"), "{e}"); + } + + #[test] + fn path_overlap_follows_an_existing_symlink_and_normalizes_dots() { + let dir = tempfile::tempdir().unwrap(); + let real = dir.path().join("real"); + std::fs::create_dir(&real).unwrap(); + let link = dir.path().join("link"); + std::os::unix::fs::symlink(&real, &link).unwrap(); + assert!(paths_overlap(&link.join("spool"), &real)); + assert!(paths_overlap(&real, &link)); + assert!(paths_overlap( + &dir.path().join("real/x/../y"), + &dir.path().join("real/y/z") + )); + assert!(!paths_overlap( + &dir.path().join("real-2"), + &dir.path().join("real") + )); + } + + /// A filesystem root that is the database directory (`"."` with the + /// database file in the working directory, or the directory spelled + /// out) or an ancestor of it is refused naming the key and both + /// directories; a child of the directory (the default root among them), + /// a sibling and a root elsewhere are accepted, and the rule does not + /// apply to an S3 store or to a server database (no directory). + #[test] + fn a_root_that_is_or_contains_the_database_directory_is_refused() { + let db = Path::new("/var/lib/extenddb/data.sqlite"); + let resolve = |path: &str| { + BackupConfig { + store: Some(fs_store(path)), + ..BackupConfig::default() + } + .resolve_store(Some(db)) + }; + for root in ["/var/lib/extenddb", "/var/lib/extenddb/./", "/var/lib", "/"] { + let e = resolve(root).unwrap_err(); + assert!(e.contains("backup.store.filesystem.path"), "{root}: {e}"); + assert!( + e.contains("database directory /var/lib/extenddb"), + "{root}: {e}" + ); + } + for root in [ + "/var/lib/extenddb/backups", + "/var/lib/extenddb-backups", + "/srv/backups", + ] { + resolve(root).unwrap_or_else(|e| panic!("{root}: {e}")); + } + BackupConfig::default() + .resolve_store(Some(db)) + .expect("the default root is a child of the directory"); + BackupConfig { + store: Some(fs_store("/var/lib/extenddb")), + ..BackupConfig::default() + } + .resolve_store(None) + .expect("a server database has no directory to protect"); + } + + /// An empty root is compared with nothing: a server started from the + /// database's directory (`cd /var/lib/extenddb && extenddb serve`, the + /// layout `init` leaves) with `path = ""` is not refused as a root that + /// is the database directory, a path the operator never wrote; the + /// configuration resolves with the empty path, which `open` refuses by + /// name. The spool rule likewise compares nothing with it. + #[test] + fn an_empty_root_is_left_for_open_to_refuse() { + let cwd = std::env::current_dir().unwrap(); + let db = cwd.join("data.sqlite"); + let resolved = BackupConfig { + store: Some(fs_store("")), + ..BackupConfig::default() + } + .resolve_store(Some(&db)) + .expect("an empty root is not the working directory"); + assert!(matches!( + resolved.store, + EffectiveStore::Configured { config, defaulted: false } + if config.filesystem.as_ref().unwrap().path.as_os_str().is_empty() + )); + BackupConfig { + spool_path: Some("/srv/spool".to_owned()), + store: Some(fs_store("")), + ..BackupConfig::default() + } + .resolve_store(Some(&db)) + .expect("a spool is not inside an empty root"); + } + + /// A refusal spells both paths as they were compared, so a spool given + /// through a symlink and a database directory given by its real path do + /// not read as two different places. + #[test] + fn a_placement_refusal_spells_both_paths_the_same_way() { + let dir = tempfile::tempdir().unwrap(); + let real = dir.path().join("real"); + std::fs::create_dir(&real).unwrap(); + let link = dir.path().join("link"); + std::os::unix::fs::symlink(&real, &link).unwrap(); + let e = BackupConfig { + spool_path: Some(link.display().to_string()), + store: Some(toml::from_str("kind = \"disabled\"").unwrap()), + ..BackupConfig::default() + } + .resolve_store(Some(&real.join("data.sqlite"))) + .unwrap_err(); + let real = real.canonicalize().unwrap().display().to_string(); + assert!( + e.contains(&format!( + "backup.spool_path {real} is or contains the database directory {real}" + )), + "{e}" + ); + assert!(!e.contains("link"), "{e}"); + } +} diff --git a/crates/config/src/lib.rs b/crates/config/src/lib.rs index 4cc5cf123..c04dd0ccd 100644 --- a/crates/config/src/lib.rs +++ b/crates/config/src/lib.rs @@ -29,6 +29,9 @@ pub struct AppConfig { pub limits: LimitsConfig, #[serde(default)] pub logging: LoggingConfig, + /// The knobs backup and restore jobs read ([`backup::BackupConfig`]). + #[serde(default)] + pub backup: backup::BackupConfig, /// Import configuration. Lists allowed source directories for import /// operations. If empty or absent, imports are denied (secure default). #[serde(default, rename = "import")] @@ -92,8 +95,8 @@ impl Default for ServerConfig { impl ServerConfig { /// Operator-facing warnings for server settings that work but cost - /// something: today a `region` outside `[a-z0-9-]`, which makes the - /// backup ARNs the server issues unresolvable (see + /// something: today a `region` outside `[a-z0-9-]`, under which + /// `CreateBackup` is refused (see /// [`extenddb_storage::config::server_region_warning`]). Shown in the /// `serve` banner; the same warning is logged when the storage components /// are created. @@ -176,6 +179,23 @@ impl StorageConfig { self.config.startup_warnings() } + /// The local file the database lives in, when the backend's database is + /// one file on this host ([`extenddb_storage::config::StorageConfig::database_file`]). + #[must_use] + pub fn database_file(&self) -> Option { + self.config.database_file() + } + + /// [`Self::database_file`] with the reason a location names no usable + /// file ([`extenddb_storage::config::StorageConfig::database_location`]). + /// + /// # Errors + /// + /// The backend's reason the location is not usable. + pub fn database_location(&self) -> Result, String> { + self.config.database_location() + } + /// Get a reference to the underlying trait object for factory calls. pub fn as_trait(&self) -> &(dyn extenddb_storage::config::StorageConfig + 'static) { &*self.config @@ -417,7 +437,17 @@ pub fn load(config_path: &str) -> anyhow::Result { .add_source(config::File::with_name(config_path).required(false)) .add_source(config::Environment::with_prefix("EXTENDDB").separator("__")) .build()?; - Ok(config.try_deserialize()?) + validated(config.try_deserialize()?) +} + +/// The cross-field checks a deserialized config still has to pass, and the +/// paths made absolute while the working directory is still the shell's. +fn validated(mut config: AppConfig) -> anyhow::Result { + config.backup.validate().map_err(|e| anyhow::anyhow!(e))?; + if let Ok(cwd) = std::env::current_dir() { + config.backup.make_paths_absolute(&cwd); + } + Ok(config) } /// Load `AppConfig` entirely from built-in defaults and environment variables, @@ -448,7 +478,7 @@ pub fn load_builtin_defaults() -> anyhow::Result { )) .add_source(config::Environment::with_prefix("EXTENDDB").separator("__")) .build()?; - Ok(config.try_deserialize()?) + validated(config.try_deserialize()?) } /// Redact password from a connection string for safe logging (REQ-LOG-002). @@ -495,6 +525,7 @@ pub fn pid_file_path_default(port: u16) -> PathBuf { pid_file_path(&ServerConfig::default().run_dir, port) } +pub mod backup; mod display; pub use display::{MASK, build_config_entries, should_redact}; diff --git a/crates/core/src/backup_format/data_file.rs b/crates/core/src/backup_format/data_file.rs index 10054b8e9..70a226541 100644 --- a/crates/core/src/backup_format/data_file.rs +++ b/crates/core/src/backup_format/data_file.rs @@ -137,7 +137,8 @@ pub const MAX_LINE_BYTES: usize = 32 * 1024 * 1024; /// the two paths share one encoding by construction. The line is not checked /// against [`MAX_LINE_BYTES`]; [`DataFileEncoder::write_item`] does that. /// [`encode_item_line_into`] writes the same bytes into a buffer the caller -/// keeps. +/// keeps; [`encode_item_line_len_bounded`] measures them against a cap +/// without a buffer. /// /// # Errors /// @@ -165,6 +166,65 @@ pub fn encode_item_line_into(item: &Item, line: &mut Vec) -> Result<(), serd Ok(()) } +/// The length of [`encode_item_line`]'s line for `item` when it is at most +/// `cap` bytes, measured without a buffer: the line is serialized into a +/// writer that counts and refuses the write that would take the count past +/// `cap`, at which `serde_json::to_writer` stops, so an item whose line is +/// many times the cap costs no memory and no more work than the cap's +/// worth. The backup job measures an item over the default item limit this +/// way before the encoder sees it; for a line within the cap the length is +/// the one `encode_item_line_into` would leave. +/// +/// # Errors +/// +/// Returns [`FormatError::LineTooLong`] once the count would pass `cap` and +/// [`FormatError::Json`] when the item cannot be serialized. +pub fn encode_item_line_len_bounded(item: &Item, cap: usize) -> Result { + let mut counter = LineCounter { + written: 0, + cap, + over: false, + }; + let counted = serde_json::to_writer(&mut counter, &ItemLineRef { item }) + .map_err(FormatError::from) + .and_then(|()| { + counter + .write_all(b"\n") + .map_err(|e| FormatError::Encode(e.to_string())) + }); + if counter.over { + return Err(FormatError::LineTooLong { cap }); + } + counted.map(|()| counter.written) +} + +/// The writer behind [`encode_item_line_len_bounded`]: counts the bytes +/// written and refuses the write that would take the count past `cap`, +/// setting `over`; no byte past the cap is counted and nothing is kept. +struct LineCounter { + written: usize, + cap: usize, + over: bool, +} + +impl Write for LineCounter { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + if buf.len() > self.cap - self.written { + self.over = true; + return Err(std::io::Error::new( + std::io::ErrorKind::WriteZero, + "the line cap", + )); + } + self.written += buf.len(); + Ok(buf.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + /// Lowercase hex encoding of a byte slice. pub(super) fn hex_encode(bytes: &[u8]) -> String { use std::fmt::Write as _; @@ -905,6 +965,14 @@ impl DataFileDecoder { self.reader.get_ref().get_ref().hasher.is_some() } + /// The uncompressed bytes decoded so far: every line consumed, its + /// newline included. The difference across one [`Self::next_item`] is + /// that item's line length, the unit a reader bounds its memory in. + #[must_use] + pub fn bytes_decoded(&self) -> u64 { + self.bytes_decoded + } + /// The error for a failed read: [`FormatError::CompressedSizeMismatch`] /// when the source ran past its recorded size, otherwise /// [`FormatError::Decode`]. @@ -1991,6 +2059,65 @@ mod tests { assert!(numbers_line < line, "{numbers_line}"); } + /// The bounded measurement is the line's length up to the cap and + /// `LineTooLong` past it, at the byte (the newline counted, as the + /// encoder counts it), and it holds no line: the counter refuses the + /// write that would pass the cap and counts nothing past it, so a + /// 600 KB item of the worst shape (a 40 MB line) is refused with at most + /// the cap's worth of bytes counted and no buffer. + #[test] + fn the_bounded_line_measurement_stops_at_the_cap_without_a_buffer() { + let small = item(vec![("pk", AttributeValue::S("p".into()))]); + let len = encode_item_line(&small).unwrap().len(); + assert_eq!(encode_item_line_len_bounded(&small, len).unwrap(), len); + assert_eq!( + encode_item_line_len_bounded(&small, MAX_LINE_BYTES).unwrap(), + len + ); + assert!(matches!( + encode_item_line_len_bounded(&small, len - 1), + Err(FormatError::LineTooLong { cap }) if cap == len - 1 + )); + // The counter itself: a write that would pass the cap is refused + // and leaves the count where it was. + let mut counter = LineCounter { + written: 0, + cap: 10, + over: false, + }; + counter.write_all(b"12345").unwrap(); + counter.write_all(b"67890").unwrap(); + assert_eq!((counter.written, counter.over), (10, false)); + assert!(counter.write_all(b"x").is_err()); + assert_eq!((counter.written, counter.over), (10, true)); + // The worst shape over the default limit: 3,700 sets of 81 two-digit + // values at `E-130`, a 40 MB line for a 600 KB item. + let set: BTreeSet = (11..100) + .filter(|m| m % 10 != 0) + .map(|m| format!("0.{}{m}", "0".repeat(129))) + .collect(); + let mut worst = item(vec![("pk", AttributeValue::S("p".into()))]); + worst.insert( + "l".to_owned(), + AttributeValue::L(vec![AttributeValue::NS(set); 3_700]), + ); + assert!(crate::types::item_size_bytes(&worst) > crate::limits::DEFAULT_MAX_ITEM_SIZE_BYTES); + assert!(matches!( + encode_item_line_len_bounded(&worst, MAX_LINE_BYTES), + Err(FormatError::LineTooLong { cap }) if cap == MAX_LINE_BYTES + )); + // A cap of 100 bytes stops the serializer after at most 100 bytes + // of a 40 MB line. + let mut counter = LineCounter { + written: 0, + cap: 100, + over: false, + }; + assert!(serde_json::to_writer(&mut counter, &ItemLineRef { item: &worst }).is_err()); + assert!(counter.over); + assert!(counter.written <= 100, "{}", counter.written); + } + /// Dropping an encoder that did not finish writes nothing more, whether /// it is fresh or has items in its buffer: an abandoned encoder must not /// complete a valid gzip file in the sink. diff --git a/crates/core/src/backup_format/key_order.rs b/crates/core/src/backup_format/key_order.rs index b0060e30c..d8990bdde 100644 --- a/crates/core/src/backup_format/key_order.rs +++ b/crates/core/src/backup_format/key_order.rs @@ -151,6 +151,15 @@ pub fn hash_key_text(value: &AttributeValue) -> Option> { } } +/// An untrusted string bounded for an error message, the format module's +/// rule (`error::truncated`) reached from outside the module: the string +/// when short, a prefix and the total length otherwise. Call sites format +/// the result with `{:?}`. +#[must_use] +pub fn shown(s: &str) -> String { + truncated(s) +} + /// A table's key shape, resolved once from its key schema and attribute /// definitions, able to compare and validate items' keys. #[derive(Debug, Clone, PartialEq, Eq)] @@ -433,9 +442,17 @@ fn parse_number(name: &str, n: &str) -> Result { } /// One key attribute's value checked as [`KeyOrder::validate_key_item`] -/// checks it: the declared scalar type, not empty, and for `N` a valid -/// number in normalized form. The error is the reason alone. -fn check_key_value( +/// checks it: not empty, the declared scalar type, and for `N` a valid +/// number in normalized form, in that order, so a text under a key declared +/// `S` is a value of the wrong type and not a malformed number. The error is +/// the reason alone. Public for the restorable-item validator +/// (`crate::validation::restorable_item`), which checks a multi-part key's +/// attributes one by one with the same rule. +/// +/// # Errors +/// +/// [`FormatError::InvalidKey`] naming the attribute and what is wrong. +pub fn check_key_value( name: &str, ty: ScalarAttributeType, value: &AttributeValue, @@ -454,32 +471,7 @@ fn check_key_value( } let actual = match value { AttributeValue::S(_) => ScalarAttributeType::S, - AttributeValue::N(n) => { - let normalized = validate_and_normalize_number(n).map_err(|e| { - // The reason for a malformed number ends in the input, which - // is shown once, bounded, before it; the other reasons - // (overflow, underflow, too many digits) are fixed texts. - let reason = e.to_string(); - let reason = match reason.strip_suffix(n.as_str()) { - Some(rest) => rest.strip_suffix(": ").unwrap_or(rest).to_owned(), - None => reason, - }; - FormatError::InvalidKey(format!( - "key attribute {:?} value {:?} is not a valid number: {reason}", - shown(), - truncated(n), - )) - })?; - if normalized != *n { - return Err(FormatError::InvalidKey(format!( - "key attribute {:?} value {:?} is not in normalized form ({:?})", - shown(), - truncated(n), - truncated(&normalized) - ))); - } - ScalarAttributeType::N - } + AttributeValue::N(_) => ScalarAttributeType::N, AttributeValue::B(_) => ScalarAttributeType::B, _ => { return Err(FormatError::InvalidKey(format!( @@ -494,6 +486,33 @@ fn check_key_value( shown() ))); } + if let AttributeValue::N(n) = value { + let normalized = validate_and_normalize_number(n).map_err(|e| { + // The reason for a malformed number ends in the input, which is + // shown once, bounded, before it, so the input and the colon + // before it are taken off the end (the wire appends the input + // as it is, trailing whitespace included); the other reasons + // (overflow, underflow, too many digits) are fixed texts. + let reason = e.to_string(); + let reason = match reason.strip_suffix(n.as_str()) { + Some(rest) => rest.strip_suffix(": ").unwrap_or(rest).to_owned(), + None => reason, + }; + FormatError::InvalidKey(format!( + "key attribute {:?} value {:?} is not a valid number: {reason}", + shown(), + truncated(n), + )) + })?; + if normalized != *n { + return Err(FormatError::InvalidKey(format!( + "key attribute {:?} value {:?} is not in normalized form ({:?})", + shown(), + truncated(n), + truncated(&normalized) + ))); + } + } Ok(()) } @@ -973,6 +992,48 @@ mod tests { assert!(err.to_string().len() < 400, "{}", err.to_string().len()); } + /// `check_key_value`'s texts: the declared type is compared before an + /// `N` is parsed, so a text under an `S` key is a value of the wrong + /// type; a malformed `N` under an `N` key gives the wire's reason without + /// the input the wire appends (shown once, bounded, before it) and + /// without the colon that preceded it, whatever whitespace the input + /// ends in; the fixed reasons are carried whole. + #[test] + fn check_key_value_compares_the_type_before_parsing_and_strips_the_echoed_input() { + let reason = |ty, value: AttributeValue| match check_key_value("k", ty, &value) { + Err(FormatError::InvalidKey(reason)) => reason, + other => panic!("{other:?}"), + }; + assert_eq!( + reason(ScalarAttributeType::S, AttributeValue::N("abc".into())), + r#"key attribute "k" is N, the schema declares S"# + ); + for bad in ["abc", "5 ", " 5", "5\t", "1e"] { + assert_eq!( + reason(ScalarAttributeType::N, AttributeValue::N(bad.into())), + format!( + "key attribute \"k\" value {bad:?} is not a valid number: The parameter \ + cannot be converted to a numeric value" + ), + "{bad:?}" + ); + } + assert_eq!( + reason(ScalarAttributeType::N, AttributeValue::N("1E126".into())), + "key attribute \"k\" value \"1E126\" is not a valid number: Number overflow. \ + Attempting to store a number with magnitude larger than supported range" + ); + assert_eq!( + reason(ScalarAttributeType::N, AttributeValue::N(String::new())), + "key attribute \"k\" value \"\" is not a valid number: The parameter cannot be \ + converted to a numeric value" + ); + assert_eq!( + reason(ScalarAttributeType::N, AttributeValue::N("5.0".into())), + r#"key attribute "k" value "5.0" is not in normalized form ("5")"# + ); + } + #[test] fn duplicate_attribute_definitions_are_unorderable() { let ks = vec![KeySchemaElement { diff --git a/crates/core/src/backup_format/layout.rs b/crates/core/src/backup_format/layout.rs index 7dd912bae..41113d02f 100644 --- a/crates/core/src/backup_format/layout.rs +++ b/crates/core/src/backup_format/layout.rs @@ -47,15 +47,25 @@ pub const FILES_MANIFEST_FILE: &str = "manifest-files.json"; /// sees a manifest. pub const DATA_PREFIX: &str = "data/"; +/// The first component of a store key that is reserved for exports +/// (`/exports/...`). No account id is spelled this way (account ids +/// are digits), and [`classify_store_key`] names it rather than relying on +/// that. +pub const EXPORTS_COMPONENT: &str = "exports"; + +/// The third component of a store key that is reserved for the change-log +/// archive (`//
    /changelog/...`). It is not a backup id. +pub const CHANGELOG_COMPONENT: &str = "changelog"; + /// Longest table name `DynamoDB` accepts, kept here as a path-safety bound: /// a longer component would exceed common filesystem name limits. The /// engine's configurable minimum length is not enforced here; a shorter name /// the engine accepted is a valid ARN component. The engine's /// `max_table_name_length` can be raised above 255; a table with a longer /// name has a backup ARN `check_backup_arn` accepts and this type refuses, -/// so the off-host backup path cannot write it (the design has `CreateBackup` -/// refuse it up front). -const MAX_TABLE_NAME_LEN: usize = 255; +/// so the off-host backup path cannot write it (`CreateBackup` refuses it up +/// front through [`check_store_prefix_table_name`]). +pub const MAX_TABLE_NAME_LEN: usize = 255; /// The key of the `index`-th data file within a backup prefix: /// `data/000001.json.gz` for index 1. @@ -195,11 +205,41 @@ fn ascii_word(s: &str, allowed: &[u8]) -> bool { } /// The region shape a backup ARN may carry: non-empty `[a-z0-9-]`. The -/// manifest checks a table ARN's region with the same predicate. -pub(super) fn is_region(s: &str) -> bool { +/// manifest checks a table ARN's region with the same predicate, the engine +/// checks `server.region` with it before forming a backup ARN at +/// `CreateBackup`, and the startup warning about `server.region` speaks for +/// it. +#[must_use] +pub fn is_region(s: &str) -> bool { ascii_word(s, b"-") && !s.bytes().any(|b| b.is_ascii_uppercase()) } +/// The store-prefix rules on a table name, the two inputs [`check_backup_arn`] +/// accepts that a [`BackupArn`] refuses: a name over [`MAX_TABLE_NAME_LEN`] +/// bytes, or made only of dots (`.` and `..` are path components; a longer +/// run of dots is a legal `DynamoDB` name but is refused to keep the rule +/// simple). The engine applies them at `CreateBackup`, before the ARN is +/// formed, so the refusal names the rule whatever the name's length. +/// +/// # Errors +/// +/// [`FormatError::InvalidArn`] naming the rule the name breaks. +pub fn check_store_prefix_table_name(table_name: &str) -> Result<(), FormatError> { + if table_name.len() > MAX_TABLE_NAME_LEN { + return Err(FormatError::InvalidArn(format!( + "table name of {} bytes is longer than {MAX_TABLE_NAME_LEN}", + table_name.len() + ))); + } + if table_name.bytes().all(|b| b == b'.') { + return Err(FormatError::InvalidArn(format!( + "table name {:?} is only dots", + truncated(table_name) + ))); + } + Ok(()) +} + /// Strip surrounding ASCII whitespace from a `BackupArn`. Measured on /// `DynamoDB`: an ARN with a trailing newline or CRLF resolves, and one with a /// leading space or tab is refused with `Invalid Backup ARN`. This parser @@ -377,22 +417,12 @@ impl BackupArn { }) } - /// The store-prefix rules over already-checked parts: the two rules - /// `check_backup_arn` does not apply, and the only two inputs it accepts - /// that this type refuses. + /// The store-prefix rules over already-checked parts + /// ([`check_store_prefix_table_name`]): the two rules `check_backup_arn` + /// does not apply, and the only two inputs it accepts that this type + /// refuses. fn from_parts(parts: &BackupArnParts<'_>) -> Result { - if parts.table_name.len() > MAX_TABLE_NAME_LEN { - return Err(FormatError::InvalidArn(format!( - "table name of {} bytes is longer than {MAX_TABLE_NAME_LEN}", - parts.table_name.len() - ))); - } - if parts.table_name.bytes().all(|b| b == b'.') { - return Err(FormatError::InvalidArn(format!( - "table name {:?} is only dots", - truncated(parts.table_name) - ))); - } + check_store_prefix_table_name(parts.table_name)?; Ok(Self { partition: parts.partition.to_owned(), region: parts.region.to_owned(), @@ -469,6 +499,133 @@ impl std::str::FromStr for BackupArn { } } +// --------------------------------------------------------------------------- +// Store keys +// --------------------------------------------------------------------------- + +/// What a key of a store listing is, relative to the backup prefixes +/// [`BackupArn::store_prefix`] forms. See [`classify_store_key`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum StoreKey<'a> { + /// Fewer than four components: a file at the root (the store's identity + /// file), or one directly in an account's or a table's directory. No + /// object of a backup has such a key. + Outside, + /// Under a reserved component, the one named: [`EXPORTS_COMPONENT`] first, + /// or [`CHANGELOG_COMPONENT`] third. + Reserved(&'static str), + /// Under `///`, whose three components are not the prefix of a + /// backup a manifest can describe. + NotABackup { + /// `///`, a slice of the key. + prefix: &'a str, + /// Why, with the untrusted components cut to a bounded length. + reason: String, + }, + /// Under the prefix of a backup. + Backup { + /// `/
    //`, a slice of the key, equal to + /// [`BackupArn::store_prefix`] of every ARN with these components. + prefix: &'a str, + /// The account id component. + account_id: &'a str, + /// The table name component. + table_name: &'a str, + /// The backup id component, in the 14-digit shape + /// [`format_backup_id`] produces. + backup_id: &'a str, + }, +} + +/// Classify a key of a store listing (relative to the store's root): under a +/// backup prefix `/
    //`, under a reserved component, +/// under three components that are not a backup prefix, or outside every +/// prefix. +/// +/// A backup prefix is one whose components [`BackupArn::new`] accepts, in any +/// partition and region (so a component carrying a separator, surrounding +/// whitespace, a control or non-ASCII character, or a traversal component +/// such as `..` is refused), and whose id has the 14-digit shape: only that +/// shape carries the instant a manifest's `backup_creation_date_time` must +/// equal, so a prefix under an earlier build's id (13 digits, with or without +/// a suffix) holds no manifest this build can read. +#[must_use] +pub fn classify_store_key(key: &str) -> StoreKey<'_> { + let mut parts = key.splitn(4, '/'); + let account_id = parts.next().unwrap_or_default(); + let table_name = parts.next(); + if account_id == EXPORTS_COMPONENT && table_name.is_some() { + return StoreKey::Reserved(EXPORTS_COMPONENT); + } + let (Some(table_name), Some(backup_id), Some(_)) = (table_name, parts.next(), parts.next()) + else { + return StoreKey::Outside; + }; + if backup_id == CHANGELOG_COMPONENT { + return StoreKey::Reserved(CHANGELOG_COMPONENT); + } + let prefix = &key[..account_id.len() + table_name.len() + backup_id.len() + 3]; + match store_prefix_problem(account_id, table_name, backup_id) { + Some(reason) => StoreKey::NotABackup { prefix, reason }, + None => StoreKey::Backup { + prefix, + account_id, + table_name, + backup_id, + }, + } +} + +/// The account id, table name and backup id of the backup prefix `key` is +/// under, or `None` when [`classify_store_key`] finds it outside every backup +/// prefix, under a reserved component, or under components that are not a +/// backup prefix. +#[must_use] +pub fn parse_store_prefix(key: &str) -> Option<(&str, &str, &str)> { + match classify_store_key(key) { + StoreKey::Backup { + account_id, + table_name, + backup_id, + .. + } => Some((account_id, table_name, backup_id)), + StoreKey::Outside | StoreKey::Reserved(_) | StoreKey::NotABackup { .. } => None, + } +} + +/// Why three path components are not a backup prefix, or `None` when they +/// are. The rules are [`BackupArn::new`]'s; the partition and region given to +/// it are placeholders that pass, since a prefix carries neither. +fn store_prefix_problem(account_id: &str, table_name: &str, backup_id: &str) -> Option { + const PARTITION: &str = "aws"; + const REGION: &str = "us-east-1"; + if BackupArn::new(PARTITION, REGION, account_id, table_name, backup_id).is_ok() { + return backup_id_epoch_millis(backup_id).is_none().then(|| { + format!( + "backup id {:?} has an earlier build's shape, which no manifest carries", + truncated(backup_id) + ) + }); + } + // Name the class of the refusal without the placeholder partition and + // region, which `BackupArn::new`'s own message would show. + let formatted = format_backup_arn(PARTITION, REGION, account_id, table_name, backup_id); + Some(match check_backup_arn(&formatted) { + Err(BackupArnError::Id) => format!("{:?} is not a backup id", truncated(backup_id)), + Err(BackupArnError::Invalid) => format!( + "account id {:?}, table name {:?} and backup id {:?} are not the components of a \ + backup ARN", + truncated(account_id), + truncated(table_name), + truncated(backup_id) + ), + Ok(_) => match check_store_prefix_table_name(table_name) { + Err(e) => e.to_string(), + Ok(()) => "a component has surrounding whitespace".to_owned(), + }, + }) +} + #[cfg(test)] mod tests { use super::*; @@ -752,8 +909,22 @@ mod tests { assert!(BackupArn::new(" aws", "us-east-1", "1234", "Music", ID).is_err()); assert!(BackupArn::new("aws", "us-east-1", "1234", "Music", &format!("{ID}\n")).is_err()); assert!(BackupArn::new("aws", "us-east-1", "1234", "Music", &format!("{ID} ")).is_err()); - // Dots inside names are fine; DynamoDB allows them. - BackupArn::new("aws", "us-east-1", "1234", "a.b.c", ID).unwrap(); + // Dots inside names are fine; DynamoDB allows them. So are a leading + // dot, a trailing dot and a run of dots beside other characters: the + // rule refuses a name made only of dots, not one that starts with + // them. + for dotted in ["a.b.c", ".cfg", "a.", "a..b", "..a", "a.."] { + BackupArn::new("aws", "us-east-1", "1234", dotted, ID) + .unwrap_or_else(|e| panic!("{dotted}: {e}")); + check_store_prefix_table_name(dotted).unwrap_or_else(|e| panic!("{dotted}: {e}")); + } + for only_dots in [".", "..", "...", "...."] { + let err = check_store_prefix_table_name(only_dots).unwrap_err(); + assert!( + err.to_string().contains("is only dots"), + "{only_dots}: {err}" + ); + } } #[test] @@ -763,5 +934,239 @@ mod tests { assert_eq!(SUMMARY_MANIFEST_FILE, "manifest-summary.json"); assert_eq!(FILES_MANIFEST_FILE, "manifest-files.json"); assert_eq!(DATA_PREFIX, "data/"); + assert_eq!(EXPORTS_COMPONENT, "exports"); + assert_eq!(CHANGELOG_COMPONENT, "changelog"); + } + + const ID: &str = "01489602797149-73d8d5bc"; + + fn not_a_backup(key: &str) -> String { + match classify_store_key(key) { + StoreKey::NotABackup { prefix, reason } => { + assert!(key.starts_with(prefix), "{key:?}: {prefix:?}"); + assert_eq!(prefix.matches('/').count(), 3, "{key:?}: {prefix:?}"); + assert!(prefix.ends_with('/'), "{key:?}: {prefix:?}"); + assert_eq!(parse_store_prefix(key), None, "{key:?}"); + reason + } + other => panic!("{key:?}: expected NotABackup, got {other:?}"), + } + } + + #[test] + fn store_keys_under_a_backup_prefix_parse_to_its_components() { + for name in [ + EXTENDDB_MANIFEST_FILE, + MANIFEST_CHECKSUM_FILE, + "data/000001.json.gz", + "data/a/b/c", + ] { + let key = format!("123456789012/Music/{ID}/{name}"); + assert_eq!( + classify_store_key(&key), + StoreKey::Backup { + prefix: &format!("123456789012/Music/{ID}/"), + account_id: "123456789012", + table_name: "Music", + backup_id: ID, + }, + "{key}" + ); + assert_eq!( + parse_store_prefix(&key), + Some(("123456789012", "Music", ID)), + "{key}" + ); + } + // The prefix is the one `BackupArn::store_prefix` forms, whatever the + // ARN's partition and region. + for (partition, region, table) in [ + ("aws", "us-east-1", "Music"), + ("aws-cn", "cn-north-1", "My.Table-2"), + ("aws-us-gov", "us-gov-west-1", "a_b"), + ] { + let arn = BackupArn::new(partition, region, "000000000001", table, ID).unwrap(); + let key = format!("{}{EXTENDDB_MANIFEST_FILE}", arn.store_prefix()); + let StoreKey::Backup { prefix, .. } = classify_store_key(&key) else { + panic!("{key}"); + }; + assert_eq!(prefix, arn.store_prefix()); + assert_eq!( + parse_store_prefix(&key), + Some((arn.account_id(), arn.table_name(), arn.backup_id())) + ); + } + // A table named like a reserved component is a table: `exports` is + // reserved only first and `changelog` only third. + for table in [EXPORTS_COMPONENT, CHANGELOG_COMPONENT] { + let key = format!("123/{table}/{ID}/{EXTENDDB_MANIFEST_FILE}"); + assert_eq!(parse_store_prefix(&key), Some(("123", table, ID)), "{key}"); + } + } + + #[test] + fn store_keys_outside_every_prefix_and_under_reserved_components_are_skipped() { + for key in [ + "", + "extenddb-store.json", + "123456789012", + "123456789012/Music", + "123456789012/Music/file", + &format!("123456789012/Music/{ID}"), + // A file named like a reserved component, not under it. + EXPORTS_COMPONENT, + "123456789012/Music/changelog", + ] { + assert_eq!(classify_store_key(key), StoreKey::Outside, "{key:?}"); + assert_eq!(parse_store_prefix(key), None, "{key:?}"); + } + for key in [ + "exports/", + "exports/x", + "exports/123456789012/Music", + &format!("exports/123456789012/{ID}/{EXTENDDB_MANIFEST_FILE}"), + &format!("exports/Music/{ID}/{EXTENDDB_MANIFEST_FILE}"), + ] { + assert_eq!( + classify_store_key(key), + StoreKey::Reserved(EXPORTS_COMPONENT), + "{key:?}" + ); + assert_eq!(parse_store_prefix(key), None, "{key:?}"); + } + for key in [ + "123456789012/Music/changelog/", + "123456789012/Music/changelog/000001.json.gz", + &format!("123456789012/Music/changelog/{ID}/{EXTENDDB_MANIFEST_FILE}"), + ] { + assert_eq!( + classify_store_key(key), + StoreKey::Reserved(CHANGELOG_COMPONENT), + "{key:?}" + ); + assert_eq!(parse_store_prefix(key), None, "{key:?}"); + } + // The reserved words are matched whole and by case: another + // component in their place is an account or an id like any other. + for key in [ + "Exports/Music/x/y", + "exportsx/Music/x/y", + "123/Music/Changelog/x", + "123/Music/changelogs/x", + ] { + not_a_backup(key); + } + } + + /// The id grid of `id_shape_grid_and_error_classes`, applied to the third + /// component: only the 14-digit shape is a backup prefix; the shapes an + /// earlier build issued are refused with their own reason, and every + /// other string is not a backup id. + #[test] + fn store_prefix_ids_are_the_fourteen_digit_shape_only() { + let mut ids: Vec = Vec::new(); + for millis_len in 0..=16usize { + for suffix_len in 0..=10usize { + ids.push(format!( + "{}-{}", + "1".repeat(millis_len), + "a".repeat(suffix_len) + )); + } + ids.push("1".repeat(millis_len)); + } + ids.extend( + [ + "01489602797149-73D8D5BC", + "01489602797149-73d8d5bg", + "0148960279714x-73d8d5bc", + "01489602797149_73d8d5bc", + "01489602797149-73d8d5bc-0", + "-01489602797149-73d8d5bc", + "a4b1d6c2-0000-4000-8000-000000000000", + ] + .map(str::to_owned), + ); + let (mut adopted, mut earlier, mut refused) = (0, 0, 0); + for id in &ids { + let key = format!("123456789012/Music/{id}/{EXTENDDB_MANIFEST_FILE}"); + if backup_id_epoch_millis(id).is_some() { + assert_eq!( + parse_store_prefix(&key), + Some(("123456789012", "Music", id.as_str())), + "{id}" + ); + adopted += 1; + } else if is_backup_id(id) { + let reason = not_a_backup(&key); + assert!(reason.contains("earlier build's shape"), "{id}: {reason}"); + earlier += 1; + } else { + let reason = not_a_backup(&key); + // An empty component is refused as a malformed path before + // the id rule is reached. + let expected = if id.is_empty() { + "are not the components of a backup ARN" + } else { + "not a backup id" + }; + assert!(reason.contains(expected), "{id}: {reason}"); + refused += 1; + } + } + // Every class was reached: the 14-digit form; 13 digits with a + // suffix and 13 bare digits; the rest. + assert_eq!((adopted, earlier), (1, 2), "{ids:?}"); + assert!(refused > 100, "{refused}"); + } + + #[test] + fn store_prefix_components_with_traversal_or_separators_are_refused() { + let long = "a".repeat(MAX_TABLE_NAME_LEN + 1); + for key in [ + // Traversal components. + format!("123/../{ID}/x"), + format!("123/./{ID}/x"), + format!("123/..../{ID}/x"), + format!("../Music/{ID}/x"), + format!("./Music/{ID}/x"), + "123/Music/../x".to_owned(), + "123/Music/./x".to_owned(), + // Empty components. + format!("/Music/{ID}/x"), + format!("123//{ID}/x"), + "123/Music//x".to_owned(), + // Separators and characters a component cannot carry. + format!("12:3/Music/{ID}/x"), + format!("123/Mu:sic/{ID}/x"), + format!("123/Music/{ID}:x/x"), + format!("12\\3/Music/{ID}/x"), + format!("123/Mu\\sic/{ID}/x"), + format!("123/Music/{ID}\\x/x"), + format!("123/Mu sic/{ID}/x"), + format!("123/Music\t/{ID}/x"), + format!("123/Music/{ID} /x"), + format!("123/Music/{ID}\n/x"), + format!(" 123/Music/{ID}/x"), + format!("123/Mu\u{0}sic/{ID}/x"), + format!("123/Mus\u{e9}c/{ID}/x"), + // An account is digits; a table name is at most 255 bytes. + format!("12345678901x/Music/{ID}/x"), + format!("arn:aws/Music/{ID}/x"), + format!("123/{long}/{ID}/x"), + ] { + let reason = not_a_backup(&key); + assert!(!reason.is_empty(), "{key:?}"); + // The placeholder partition and region never reach the reason. + assert!(!reason.contains("us-east-1"), "{key:?}: {reason}"); + } + // The reasons name the rule where one rule applies. + assert!(not_a_backup(&format!("123/../{ID}/x")).contains("is only dots")); + assert!(not_a_backup(&format!("123/Music/{ID}\n/x")).contains("surrounding whitespace")); + assert!(not_a_backup(&format!("123/{long}/{ID}/x")).contains("longer than 255")); + // The reason is bounded however long the components are. + let huge = "x".repeat(100_000); + let reason = not_a_backup(&format!("{huge}/{huge}/{huge}/x")); + assert!(reason.len() < 1_000, "{}", reason.len()); } } diff --git a/crates/core/src/backup_format/manifest.rs b/crates/core/src/backup_format/manifest.rs index 28658df85..5afa2afd9 100644 --- a/crates/core/src/backup_format/manifest.rs +++ b/crates/core/src/backup_format/manifest.rs @@ -588,6 +588,13 @@ impl DataFileReader { } checked } + + /// The uncompressed bytes decoded so far + /// ([`DataFileDecoder::bytes_decoded`]). + #[must_use] + pub fn bytes_decoded(&self) -> u64 { + self.decoder.bytes_decoded() + } } impl Iterator for DataFileReader { diff --git a/crates/core/src/backup_format/mod.rs b/crates/core/src/backup_format/mod.rs index 9287a9124..ed2e02e5b 100644 --- a/crates/core/src/backup_format/mod.rs +++ b/crates/core/src/backup_format/mod.rs @@ -73,7 +73,7 @@ mod golden_tests; pub use data_file::{ CompressedDigest, CompressedHasher, DataFileChecksums, DataFileDecoder, DataFileEncoder, DataFileExpectations, GZIP_LEVEL, MAX_LINE_BYTES, decode_data_file, encode_data_file, - encode_item_line, encode_item_line_into, + encode_item_line, encode_item_line_into, encode_item_line_len_bounded, }; pub use ddb_export::{ DYNAMODB_JSON_OUTPUT_FORMAT, EXPORT_SUMMARY_VERSION, ExportFileEntry, ExportSummaryManifest, @@ -81,12 +81,14 @@ pub use ddb_export::{ }; pub use documents::{BackupDocuments, ManifestHeader}; pub use error::{FormatError, JsonError}; -pub use key_order::{KeyOrder, hash_key_text}; +pub use key_order::{KeyOrder, check_key_value, hash_key_text, shown}; pub use layout::{ - BACKUP_ID_MILLIS_DIGITS, BackupArn, BackupArnError, BackupArnParts, DATA_PREFIX, - EXTENDDB_MANIFEST_FILE, FILES_MANIFEST_FILE, MANIFEST_CHECKSUM_FILE, MAX_BACKUP_ARN_LEN, - SUMMARY_MANIFEST_FILE, backup_creation_date_time, backup_id_epoch_millis, check_backup_arn, - data_file_key, format_backup_arn, format_backup_id, is_backup_id, parse_backup_arn, + BACKUP_ID_MILLIS_DIGITS, BackupArn, BackupArnError, BackupArnParts, CHANGELOG_COMPONENT, + DATA_PREFIX, EXPORTS_COMPONENT, EXTENDDB_MANIFEST_FILE, FILES_MANIFEST_FILE, + MANIFEST_CHECKSUM_FILE, MAX_BACKUP_ARN_LEN, MAX_TABLE_NAME_LEN, SUMMARY_MANIFEST_FILE, + StoreKey, backup_creation_date_time, backup_id_epoch_millis, check_backup_arn, + check_store_prefix_table_name, classify_store_key, data_file_key, format_backup_arn, + format_backup_id, is_backup_id, is_region, parse_backup_arn, parse_store_prefix, trim_backup_arn, }; pub use manifest::{ diff --git a/crates/core/src/limits/mod.rs b/crates/core/src/limits/mod.rs index a894f1b83..5805508bb 100755 --- a/crates/core/src/limits/mod.rs +++ b/crates/core/src/limits/mod.rs @@ -2,6 +2,12 @@ // SPDX-License-Identifier: Apache-2.0 use serde::Deserialize; +/// The default `max_item_size_bytes` (400 KB as `DynamoDB` sizes items), the +/// largest item the backup format's line cap is sized for: the backup job +/// measures the data file line of an item over this size and of no item +/// within it. +pub const DEFAULT_MAX_ITEM_SIZE_BYTES: usize = 409_600; + /// All configurable limits with DynamoDB-compatible defaults. #[derive(Debug, Clone, Deserialize)] #[serde(deny_unknown_fields)] @@ -98,7 +104,7 @@ impl Default for LimitsConfig { } fn default_max_item_size() -> usize { - 409_600 + DEFAULT_MAX_ITEM_SIZE_BYTES } fn default_max_pk_size() -> usize { 2048 diff --git a/crates/core/src/metrics/collector.rs b/crates/core/src/metrics/collector.rs index c5981cdcf..f6a3e6f68 100755 --- a/crates/core/src/metrics/collector.rs +++ b/crates/core/src/metrics/collector.rs @@ -348,6 +348,81 @@ impl MetricsCollector { ); } + /// Record the end of a store-backed backup job: one `BackupJobs` under + /// the outcome and its duration in microseconds under the same + /// dimension. The job measures the duration from the snapshot's open to + /// the end of its last catalog or store call: the AVAILABLE write for a + /// job that succeeded, the FAILED write and the prefix's deletion for + /// one that failed. + pub fn record_backup_job(&self, outcome: super::types::BackupJobOutcome, duration_us: f64) { + let outcome = outcome.to_string(); + self.record(MetricName::BackupJobs, 1.0, None, None, Some(&outcome)); + self.record( + MetricName::BackupJobDuration, + duration_us, + None, + None, + Some(&outcome), + ); + } + + /// Record bytes a backup job wrote to the store; the job records them + /// once it ended AVAILABLE, so a failed job's uploaded and then deleted + /// bytes are not counted. + #[allow(clippy::cast_precision_loss)] // byte counts of one backup + pub fn record_backup_bytes_uploaded(&self, bytes: u64) { + self.record( + MetricName::BackupBytesUploaded, + bytes as f64, + None, + None, + None, + ); + } + + /// Record a running backup job's snapshot age in seconds (gauge). + pub fn record_backup_snapshot_age(&self, age_secs: f64) { + self.record(MetricName::BackupSnapshotAge, age_secs, None, None, None); + } + + /// Record a failed store call of a backup job, a restore job or the + /// backup reconciler; `op` names the call (a backup job's `put` and + /// `delete_prefix`, a restore's `get`, the reconciler's `get`, `list` + /// and `delete_prefix`). + pub fn record_backup_store_error(&self, op: &str) { + self.record(MetricName::BackupStoreErrors, 1.0, None, None, Some(op)); + } + + /// Record the end of a store-backed restore job: one `RestoreJobs` under + /// the outcome and its duration in microseconds under the same + /// dimension, as [`Self::record_backup_job`] records a backup's. The job + /// measures the duration from its registration to the end of its last + /// catalog or store call. + pub fn record_restore_job(&self, outcome: super::types::BackupJobOutcome, duration_us: f64) { + let outcome = outcome.to_string(); + self.record(MetricName::RestoreJobs, 1.0, None, None, Some(&outcome)); + self.record( + MetricName::RestoreJobDuration, + duration_us, + None, + None, + Some(&outcome), + ); + } + + /// Record bytes a restore job read from the store, as each download + /// arrived. + #[allow(clippy::cast_precision_loss)] // byte counts of one backup + pub fn record_restore_bytes_downloaded(&self, bytes: u64) { + self.record( + MetricName::RestoreBytesDownloaded, + bytes as f64, + None, + None, + None, + ); + } + /// Record per-segment latency breakdown for a request. pub fn record_segments( &self, diff --git a/crates/core/src/metrics/collector_tests.rs b/crates/core/src/metrics/collector_tests.rs index 26a50b7f6..1683aa25f 100755 --- a/crates/core/src/metrics/collector_tests.rs +++ b/crates/core/src/metrics/collector_tests.rs @@ -175,3 +175,118 @@ fn segments_multiple_operations() { assert_eq!(s.count, 1); } } + +/// The backup job's recorders land under their names and dimensions: one +/// `BackupJobs` and one `BackupJobDuration` (microseconds) per job with the +/// outcome as the operation dimension, the bytes and the snapshot age +/// undimensioned, a store error under the call's name. +#[test] +fn backup_job_recorders_land_under_the_outcome_and_the_store_call() { + use crate::metrics::types::BackupJobOutcome; + + let c = MetricsCollector::new(); + c.record_backup_job(BackupJobOutcome::Succeeded, 1_500.0); + c.record_backup_job(BackupJobOutcome::Failed, 300.0); + c.record_backup_bytes_uploaded(4096); + c.record_backup_snapshot_age(12.5); + c.record_backup_store_error("put"); + + let by = |metric: MetricName| { + let mut snaps = c.query(&MetricsQuery { + metric: Some(metric), + window: Some(TimeWindow::AllTime), + ..Default::default() + }); + snaps.sort_by(|a, b| format!("{:?}", a.dimensions).cmp(&format!("{:?}", b.dimensions))); + snaps + }; + let jobs = by(MetricName::BackupJobs); + let outcomes: Vec = jobs.iter().map(|s| format!("{:?}", s.dimensions)).collect(); + assert_eq!( + outcomes, + [ + "[Operation(\"failed\")]".to_owned(), + "[Operation(\"succeeded\")]".to_owned() + ] + ); + assert!(jobs.iter().all(|s| s.count == 1 && s.sum == 1.0)); + let durations = by(MetricName::BackupJobDuration); + assert_eq!(durations.len(), 2); + assert_eq!(durations.iter().map(|s| s.sum).sum::(), 1_800.0); + let bytes = by(MetricName::BackupBytesUploaded); + assert_eq!(bytes.len(), 1); + assert!(bytes[0].dimensions.is_empty()); + assert_eq!(bytes[0].sum, 4096.0); + let age = by(MetricName::BackupSnapshotAge); + assert_eq!(age[0].max, 12.5); + let errors = by(MetricName::BackupStoreErrors); + assert_eq!( + format!("{:?}", errors[0].dimensions), + "[Operation(\"put\")]" + ); + for (outcome, text) in [ + (BackupJobOutcome::Succeeded, "succeeded"), + (BackupJobOutcome::Failed, "failed"), + (BackupJobOutcome::Cancelled, "cancelled"), + ] { + assert_eq!(outcome.to_string(), text); + } +} + +/// The restore job's recorders land as the backup job's do: one +/// `RestoreJobs` and one `RestoreJobDuration` per job under the outcome, +/// the bytes undimensioned, a `get` failure under `BackupStoreErrors`; and +/// the names render as the `/metrics` endpoint prints them. +#[test] +fn restore_job_recorders_land_under_the_outcome_and_render_their_names() { + use crate::metrics::types::{BackupJobOutcome, QuerySource}; + + let c = MetricsCollector::new(); + c.record_restore_job(BackupJobOutcome::Succeeded, 2_000.0); + c.record_restore_job(BackupJobOutcome::Cancelled, 500.0); + c.record_restore_bytes_downloaded(8192); + c.record_backup_store_error("get"); + + let by = |metric: MetricName| { + let mut snaps = c.query(&MetricsQuery { + metric: Some(metric), + window: Some(TimeWindow::AllTime), + ..Default::default() + }); + snaps.sort_by(|a, b| format!("{:?}", a.dimensions).cmp(&format!("{:?}", b.dimensions))); + snaps + }; + let jobs = by(MetricName::RestoreJobs); + let outcomes: Vec = jobs.iter().map(|s| format!("{:?}", s.dimensions)).collect(); + assert_eq!( + outcomes, + [ + "[Operation(\"cancelled\")]".to_owned(), + "[Operation(\"succeeded\")]".to_owned() + ] + ); + assert!(jobs.iter().all(|s| s.count == 1 && s.sum == 1.0)); + let durations = by(MetricName::RestoreJobDuration); + assert_eq!(durations.len(), 2); + assert_eq!(durations.iter().map(|s| s.sum).sum::(), 2_500.0); + let bytes = by(MetricName::RestoreBytesDownloaded); + assert_eq!(bytes.len(), 1); + assert!(bytes[0].dimensions.is_empty()); + assert_eq!(bytes[0].sum, 8192.0); + let errors = by(MetricName::BackupStoreErrors); + assert_eq!( + format!("{:?}", errors[0].dimensions), + "[Operation(\"get\")]" + ); + assert!(by(MetricName::BackupJobs).is_empty()); + assert_eq!(MetricName::RestoreJobs.to_string(), "RestoreJobs"); + assert_eq!( + MetricName::RestoreJobDuration.to_string(), + "RestoreJobDuration" + ); + assert_eq!( + MetricName::RestoreBytesDownloaded.to_string(), + "RestoreBytesDownloaded" + ); + assert_eq!(QuerySource::RestoreJob.to_string(), "RestoreJob"); +} diff --git a/crates/core/src/metrics/mod.rs b/crates/core/src/metrics/mod.rs index bf1ff0272..637a97c29 100755 --- a/crates/core/src/metrics/mod.rs +++ b/crates/core/src/metrics/mod.rs @@ -17,7 +17,7 @@ mod types; pub use collector::MetricsCollector; pub use types::{ - Dimension, FlushBucket, LatencySegments, MetricName, MetricSnapshot, MetricsBucket, - MetricsQuery, MetricsResponse, OperationSegments, Percentiles, QueryCategory, QuerySource, - TimeWindow, auto_granularity, parse_granularity, window_duration_secs, + BackupJobOutcome, Dimension, FlushBucket, LatencySegments, MetricName, MetricSnapshot, + MetricsBucket, MetricsQuery, MetricsResponse, OperationSegments, Percentiles, QueryCategory, + QuerySource, TimeWindow, auto_granularity, parse_granularity, window_duration_secs, }; diff --git a/crates/core/src/metrics/types.rs b/crates/core/src/metrics/types.rs index 39d9d07f4..0030ce1b2 100755 --- a/crates/core/src/metrics/types.rs +++ b/crates/core/src/metrics/types.rs @@ -55,6 +55,76 @@ pub enum MetricName { WorkerCycleLatency, /// P120e: Worker error count. WorkerErrorCount, + /// Store-backed backup jobs ended, one per job; the operation dimension + /// carries the outcome ([`BackupJobOutcome`]: `succeeded`, `failed` or + /// `cancelled`). `/metrics` emits it under this name as JSON, as it does + /// every metric (the endpoint is not Prometheus exposition format); a + /// converter would give it a name like `backup_jobs_total{outcome}`, and + /// nothing in the server renders that. + BackupJobs, + /// Duration of a store-backed backup job in microseconds, from the + /// snapshot's open to the end of its last catalog or store call (the + /// AVAILABLE write, or the FAILED write and the prefix's deletion); + /// recorded for every outcome, with the outcome as the operation + /// dimension. + BackupJobDuration, + /// Bytes a backup job that ended AVAILABLE wrote to the store: every + /// data file and document as the store reported it stored; a failed + /// job's uploaded and then deleted bytes are not counted. + BackupBytesUploaded, + /// Seconds between a running backup job's snapshot instant and now + /// (gauge), recorded at each confirmed heartbeat. + BackupSnapshotAge, + /// Store operations a backup or restore job or the backup reconciler + /// saw fail; the operation dimension names the store call (`put` by a + /// backup job's uploads, `delete_prefix` by its failure path and the + /// reconciler's deleting duties, `get` by a restore's reads of the + /// manifest and its sidecar, in the request and in a resume, by a + /// restore job's downloads and by the reconciler's manifest probe, and + /// `list` by the reconciler's manifest probe and deleting duties). + BackupStoreErrors, + /// Store-backed restore jobs ended, one per job; the operation dimension + /// carries the outcome ([`BackupJobOutcome`]: `succeeded` for a DONE + /// record, `failed` for a FAILED one or a job that stopped without a + /// status write of its own, `cancelled` for one the shutdown stopped, + /// whose record stays RUNNING for a resume). Named as `BackupJobs` is + /// (the `/metrics` name; a converter's would be + /// `restore_jobs_total{outcome}`). + RestoreJobs, + /// Duration of a store-backed restore job in microseconds, from its + /// registration to the end of its last catalog or store call, with the + /// outcome as the operation dimension. + RestoreJobDuration, + /// Bytes of each data-file download attempt of a restore job as they + /// arrived, verified or not, so an attempt the job retried counts its + /// bytes too; the manifest and sidecar reads, and a filesystem store's + /// second read of each file for the load, are not counted. + RestoreBytesDownloaded, +} + +/// How a store-backed backup or restore job ended, the operation dimension +/// of [`MetricName::BackupJobs`], [`MetricName::BackupJobDuration`], +/// [`MetricName::RestoreJobs`] and [`MetricName::RestoreJobDuration`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum BackupJobOutcome { + /// The manifest was written and the row is AVAILABLE. + Succeeded, + /// The row is FAILED, or the job stopped without a status write of its + /// own (its claim lost). + Failed, + /// The server's shutdown stopped the job; a backup row is FAILED with + /// reason `shutdown`, a restore record stays RUNNING for a resume. + Cancelled, +} + +impl std::fmt::Display for BackupJobOutcome { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + Self::Succeeded => "succeeded", + Self::Failed => "failed", + Self::Cancelled => "cancelled", + }) + } } impl std::fmt::Display for MetricName { @@ -86,6 +156,14 @@ impl std::fmt::Display for MetricName { Self::WorkerLastSuccess => f.write_str("WorkerLastSuccess"), Self::WorkerCycleLatency => f.write_str("WorkerCycleLatency"), Self::WorkerErrorCount => f.write_str("WorkerErrorCount"), + Self::BackupJobs => f.write_str("BackupJobs"), + Self::BackupJobDuration => f.write_str("BackupJobDuration"), + Self::BackupBytesUploaded => f.write_str("BackupBytesUploaded"), + Self::BackupSnapshotAge => f.write_str("BackupSnapshotAge"), + Self::BackupStoreErrors => f.write_str("BackupStoreErrors"), + Self::RestoreJobs => f.write_str("RestoreJobs"), + Self::RestoreJobDuration => f.write_str("RestoreJobDuration"), + Self::RestoreBytesDownloaded => f.write_str("RestoreBytesDownloaded"), } } } @@ -129,6 +207,14 @@ pub enum QuerySource { ControlPlane, /// Background: login attempt cleanup. LoginAttemptCleanup, + /// Background: the backup reconciler. + BackupReconciler, + /// A store-backed backup job (the worker metrics are recorded once per + /// job, at its end). + BackupJob, + /// A store-backed restore job (the worker metrics are recorded once per + /// job, at its end; a job the shutdown stopped records none). + RestoreJob, /// Management CLI commands. Management, } @@ -149,6 +235,9 @@ impl std::fmt::Display for QuerySource { Self::TableSizeRefresh => f.write_str("TableSizeRefresh"), Self::ControlPlane => f.write_str("ControlPlane"), Self::LoginAttemptCleanup => f.write_str("LoginAttemptCleanup"), + Self::BackupReconciler => f.write_str("BackupReconciler"), + Self::BackupJob => f.write_str("BackupJob"), + Self::RestoreJob => f.write_str("RestoreJob"), Self::Management => f.write_str("Management"), } } diff --git a/crates/core/src/validation/mod.rs b/crates/core/src/validation/mod.rs index a849f2c95..e4c694209 100755 --- a/crates/core/src/validation/mod.rs +++ b/crates/core/src/validation/mod.rs @@ -1,8 +1,13 @@ // Copyright 2026 ExtendDB contributors // SPDX-License-Identifier: Apache-2.0 pub mod number; +pub mod restorable_item; pub mod vector_item; +pub use restorable_item::{ + RestorableItemError, RestorableItemReason, RestorableKeySchema, RestorableLimits, + SHOWN_VALUE_BYTES, validate_restorable_item, +}; pub use vector_item::{ MAX_HASH_KEY_SIZE, MAX_INLINE_FILTER_SIZE, validate_vector_write, validate_vector_write_changed, vector_components, vector_norm, diff --git a/crates/core/src/validation/restorable_item.rs b/crates/core/src/validation/restorable_item.rs new file mode 100644 index 000000000..5d2b4f671 --- /dev/null +++ b/crates/core/src/validation/restorable_item.rs @@ -0,0 +1,1514 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Whether a stored item can be restored: the check a backup job runs on +//! every item it reads, before the item is written to a data file, and the +//! check PR 9a's restore binds for every item it reads back. +//! +//! The write paths let some items in that a restore's `PutItem`-shaped write +//! would refuse: an `N` value that is not a valid number (`1E126`, `abc`; +//! #618 names the four paths that store one unvalidated: `BatchWriteItem`, +//! `TransactWriteItems`, `UpdateItem` on an absent item, `ImportTable`), an +//! item over the configured size limit stored under a larger one, a key +//! attribute or attribute name over this instance's limits stored under +//! larger ones, and an attribute name over the limit that `UpdateItem` and +//! a `TransactWriteItems` `Update` stored unchecked before every write path +//! checked it (#433). The key-shape rules (a key attribute missing, of the wrong +//! type or empty, an `N` key not in normalized form) cover rows of earlier +//! builds and rows edited outside the wire: every write path of this tree +//! refuses such a key, and the snapshot streams hand every valid `N` over +//! normalized, so the normalized-form branch is unreachable from a +//! snapshot; it stays because it is the format's own rule +//! ([`crate::backup_format::check_key_value`]). A backup that carried such +//! an item could not be restored in full, so the job refuses the backup +//! instead and names the item (its key), the attribute path and the reason. +//! The base key alone is checked: an index key attribute of the wrong type +//! or empty is left to the restore's index build. +//! +//! One walk over the item: every `N` and every `NS` member is parsed once +//! on the accepting path (a key `N` that fails the key rule is parsed a +//! second time to classify the failure), the nesting depth is counted as it +//! goes, every attribute name is measured against `max_attribute_name_bytes` +//! and the key attributes are checked with the backup format's own key rule +//! and against the hash and range key size limits, the rules the wire's +//! `check_key_size` and `check_attribute_name_size` apply, so a key that +//! passes here is one a key-ordered file's range can carry and one the +//! target's write accepts. The item's size is one call on the whole item +//! (`item_size_bytes`). The request path's `validate_put_item` stays what it +//! is: it returns the wire error with no attribute path and is not called +//! here. +//! +//! Every untrusted text an error carries (a key value, an attribute name, a +//! path segment, a number the wire's reason echoes) is bounded on its own +//! ([`SHOWN_VALUE_BYTES`]; a key value at [`SHOWN_KEY_VALUE_BYTES`]) and +//! rendered as a JSON string's content, with the line separators U+0085, +//! U+2028 and U+2029 escaped beside the control characters, so neither a +//! control character nor a Unicode line separator in a name or a number +//! can end the log line it is written in, and a key of any +//! legal size appears whole while only an oversized value is cut. The +//! bound is per text, so the whole message is bounded by the item's own +//! size times a small factor (a byte renders as at most six), not by a +//! constant: a path of thirty nested maps with names at the bound is about +//! 127 KB. + +use std::borrow::Cow; +use std::fmt; + +use crate::backup_format::{FormatError, check_key_value, hash_key_text, shown}; +use crate::limits::LimitsConfig; +use crate::types::{ + AttributeDefinition, AttributeValue, Item, KeySchemaElement, KeyType, ScalarAttributeType, + item_size_bytes, +}; +use crate::validation::number::validate_and_normalize_number; + +/// Longest rendering of one untrusted text in a [`RestorableItemError`] +/// other than a key value (an attribute name, a path segment, a set member, +/// the number a reason echoes), in the bytes the message writes: the text +/// as a JSON string's content, quotes not counted. A round figure that +/// keeps a path segment to four times the longest key attribute name +/// (255 bytes), after which a name is cut. +pub const SHOWN_VALUE_BYTES: usize = 4096; + +/// Longest rendering of a key attribute's value, in the same measure: six +/// times the 2048-byte hash key limit, the most a byte escapes to +/// (`\u00XX`), so a key of any legal size renders whole whatever its +/// bytes, and the 2732-character base64 of a 2048-byte `B` key fits. +pub const SHOWN_KEY_VALUE_BYTES: usize = 6 * 2048; + +/// The prefix kept of a text over its bound, in the same measure, followed +/// by `... ` with the text's length. +const SHOWN_VALUE_PREFIX_BYTES: usize = 1024; + +/// The instance's limits a restore would hold an item to, for +/// [`validate_restorable_item`]: the item size, the hash and range key +/// sizes and the attribute name size of `LimitsConfig`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct RestorableLimits { + pub max_item_size_bytes: usize, + pub max_partition_key_size_bytes: usize, + pub max_sort_key_size_bytes: usize, + pub max_attribute_name_bytes: usize, +} + +impl From<&LimitsConfig> for RestorableLimits { + fn from(limits: &LimitsConfig) -> Self { + Self { + max_item_size_bytes: limits.max_item_size_bytes, + max_partition_key_size_bytes: limits.max_partition_key_size_bytes, + max_sort_key_size_bytes: limits.max_sort_key_size_bytes, + max_attribute_name_bytes: limits.max_attribute_name_bytes, + } + } +} + +/// One key attribute of a [`RestorableKeySchema`]. +#[derive(Debug, Clone, PartialEq, Eq)] +struct KeyAttribute { + name: String, + ty: ScalarAttributeType, + key_type: KeyType, +} + +/// The key attributes of a table, resolved once from its key schema and +/// attribute definitions, for [`validate_restorable_item`]. Unlike +/// `KeyOrder`, which only a hash-only or single-range-key table has, every +/// table has one of these: a multi-part key is a list of more than two +/// attributes. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RestorableKeySchema { + attributes: Vec, +} + +impl RestorableKeySchema { + /// Resolve the key attributes, in key-schema order. + /// + /// # Errors + /// + /// The name of a key attribute that has no attribute definition, or more + /// than one; `CreateTable` and the manifest reader both refuse such a + /// schema, so a table's own schema never fails here. + pub fn resolve( + key_schema: &[KeySchemaElement], + attribute_definitions: &[AttributeDefinition], + ) -> Result { + let mut attributes = Vec::with_capacity(key_schema.len()); + for element in key_schema { + let name = element.attribute_name.as_str(); + let mut matching = attribute_definitions + .iter() + .filter(|d| d.attribute_name == name) + .map(|d| d.attribute_type); + let ty = matching.next().ok_or_else(|| { + format!( + "key attribute {:?} has no attribute definition", + shown(name) + ) + })?; + if matching.next().is_some() { + return Err(format!( + "key attribute {:?} has more than one attribute definition", + shown(name) + )); + } + attributes.push(KeyAttribute { + name: name.to_owned(), + ty, + key_type: element.key_type, + }); + } + Ok(Self { attributes }) + } + + /// Names of the key attributes, in key-schema order. + pub fn attribute_names(&self) -> impl Iterator { + self.attributes.iter().map(|a| a.name.as_str()) + } + + /// Whether `name` is a key attribute. + fn is_key(&self, name: &str) -> bool { + self.attributes.iter().any(|a| a.name == name) + } + + /// The key attributes of `item` as compact JSON (`{"pk":{"S":"a"}}`) + /// for an error message or a log line, each name bounded on its own + /// ([`SHOWN_VALUE_BYTES`]) and each value on its own + /// ([`SHOWN_KEY_VALUE_BYTES`]), so every key attribute appears and only + /// an oversized value is cut; a key attribute the item lacks is left + /// out. A `B` value is its base64 text, as the format writes it; a value + /// that is not a scalar (a key attribute of the wrong type) is its own + /// JSON, cut to a JSON string when oversized. + #[must_use] + pub fn shown_key(&self, item: &Item) -> String { + let mut out = String::from("{"); + let mut first = true; + for attribute in &self.attributes { + let Some(value) = item.get(&attribute.name) else { + continue; + }; + if !first { + out.push(','); + } + first = false; + push_json_string(&mut out, &attribute.name, SHOWN_VALUE_BYTES); + out.push(':'); + let tag = match value { + AttributeValue::S(_) => Some("S"), + AttributeValue::N(_) => Some("N"), + AttributeValue::B(_) => Some("B"), + _ => None, + }; + match (tag, hash_key_text(value)) { + (Some(tag), Some(text)) => { + out.push_str("{\""); + out.push_str(tag); + out.push_str("\":"); + push_json_string(&mut out, &text, SHOWN_KEY_VALUE_BYTES); + out.push('}'); + } + _ => { + let text = serde_json::to_string(value).unwrap_or_default(); + if text.len() <= SHOWN_KEY_VALUE_BYTES { + out.push_str(&text); + } else { + push_json_string(&mut out, &text, SHOWN_KEY_VALUE_BYTES); + } + } + } + } + out.push('}'); + out + } +} + +/// Append `s` to `out` as a JSON string: quotes around [`bounded_text`] +/// with `bound`. +fn push_json_string(out: &mut String, s: &str, bound: usize) { + out.push('"'); + out.push_str(&bounded_text(s, bound)); + out.push('"'); +} + +/// Bytes a message writes for `c` inside a JSON string: `\"`, `\\` and +/// the five short escapes take two, the other control characters and the +/// three line separators `\u00XX`-style escapes of six, everything else +/// its UTF-8 length. +fn json_escaped_len(c: char) -> usize { + match c { + '"' | '\\' | '\n' | '\r' | '\t' | '\u{8}' | '\u{c}' => 2, + c if (c as u32) < 0x20 || is_line_separator(c) => 6, + c => c.len_utf8(), + } +} + +/// The Unicode line separators `serde_json` leaves raw and a reader that +/// breaks lines the Unicode way (Python's `str.splitlines`) breaks on: +/// NEXT LINE, LINE SEPARATOR and PARAGRAPH SEPARATOR. +fn is_line_separator(c: char) -> bool { + matches!(c, '\u{85}' | '\u{2028}' | '\u{2029}') +} + +/// `s` as a JSON string's content: the escapes `serde_json` writes, then +/// the three line separators as `\u0085`, `\u2028` and `\u2029` (valid JSON +/// escapes `serde_json` does not write), no quotes, so neither a control +/// character nor a line separator in a tenant's text can stand as a line +/// of its own in a log; borrowed when nothing in it escapes. +fn json_escaped(s: &str) -> Cow<'_, str> { + if s.chars().all(|c| json_escaped_len(c) == c.len_utf8()) { + return Cow::Borrowed(s); + } + let mut quoted = serde_json::to_string(s).unwrap_or_default(); + quoted.pop(); + if !quoted.is_empty() { + quoted.remove(0); + } + if quoted.chars().any(is_line_separator) { + quoted = quoted + .replace('\u{85}', r"\u0085") + .replace('\u{2028}', r"\u2028") + .replace('\u{2029}', r"\u2029"); + } + Cow::Owned(quoted) +} + +/// `s` as a message writes it, a JSON string's content ([`json_escaped`]), +/// bounded: whole when its escaped text is at most `bound` bytes, otherwise +/// a prefix of at most [`SHOWN_VALUE_PREFIX_BYTES`] of escaped text and +/// `... ` with the text's length. +fn bounded_text(s: &str, bound: usize) -> Cow<'_, str> { + let mut rendered = 0; + let mut prefix = 0; + for c in s.chars() { + rendered += json_escaped_len(c); + if rendered > bound { + return Cow::Owned(format!( + "{}... <{} bytes>", + json_escaped(&s[..prefix]), + s.len() + )); + } + if rendered <= SHOWN_VALUE_PREFIX_BYTES { + prefix += c.len_utf8(); + } + } + json_escaped(s) +} + +/// Why an item cannot be restored. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum RestorableItemReason { + /// An `N` value, or a member of an `NS`, that is not a valid `DynamoDB` + /// number: the wire's own reason text, with the member appended for an + /// `NS` when the text does not already end in it (the #618 case; a + /// FAILED backup names the issue when its first refused item has this + /// reason). + InvalidNumber(String), + /// A key attribute missing, of the wrong type, empty, an `N` key not in + /// normalized form (the backup format's key rule's reason text), or a + /// key value over the hash or range key size limit. + KeyShape(String), + /// An attribute name (the path's last segment) longer than the + /// attribute name limit. + AttributeNameTooLong { size: usize, max: usize }, + /// The item is larger than the configured limit. + TooLarge { size: usize, max: usize }, + /// A map or list nests deeper than the item nesting limit. + TooDeep, + /// The item's data file line would exceed the format's line cap + /// (`MAX_LINE_BYTES`): the backup job measures the line of an item over + /// the default item limit, the largest the cap is sized for, and stops + /// measuring once the cap is passed, so the line's own length is not + /// known. + LineTooLong { cap: usize }, +} + +impl fmt::Display for RestorableItemReason { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::InvalidNumber(reason) | Self::KeyShape(reason) => f.write_str(reason), + Self::AttributeNameTooLong { size, max } => { + write!( + f, + "attribute name of {size} bytes exceeds the limit of {max} bytes" + ) + } + Self::TooLarge { size, max } => { + write!(f, "item size {size} bytes exceeds the limit of {max} bytes") + } + Self::TooDeep => write!( + f, + "attributes nest beyond {} levels", + super::MAX_ITEM_NESTING_DEPTH + ), + Self::LineTooLong { cap } => { + write!(f, "data file line exceeds the line cap of {cap} bytes") + } + } + } +} + +/// An item [`validate_restorable_item`] refused: which item (its key +/// attributes as compact JSON, each name and value bounded), where (the +/// attribute path, `a.b[2].c` for a member of a list inside a map, each +/// segment bounded, empty for a rule about the whole item) and why. +/// `Display` writes `key {key}, attribute {path}: {reason}`, without the +/// attribute clause when the path is empty. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RestorableItemError { + pub key: String, + pub path: String, + pub reason: RestorableItemReason, +} + +impl fmt::Display for RestorableItemError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + if self.path.is_empty() { + write!(f, "key {}: {}", self.key, self.reason) + } else { + write!( + f, + "key {}, attribute {}: {}", + self.key, self.path, self.reason + ) + } + } +} + +impl std::error::Error for RestorableItemError {} + +/// Check that `item` is one a restore under `limits` can write: its key +/// attributes present with the declared types, not empty, an `N` key +/// normalized, each within its key size limit; every attribute name within +/// the name limit; every `N` and `NS` member a valid number; no map or list +/// nested beyond the item nesting limit; the item within the item size +/// limit. The first failure, in that order (key attributes in key-schema +/// order, then the other attributes in name order, then the size), is the +/// error. +/// +/// # Errors +/// +/// [`RestorableItemError`] naming the item's key, the attribute path and the +/// reason. +pub fn validate_restorable_item( + item: &Item, + key: &RestorableKeySchema, + limits: &RestorableLimits, +) -> Result<(), RestorableItemError> { + let refuse = |path: String, reason: RestorableItemReason| RestorableItemError { + key: key.shown_key(item), + path, + reason, + }; + for attribute in &key.attributes { + let name = attribute.name.as_str(); + let path = || bounded_text(name, SHOWN_VALUE_BYTES).into_owned(); + if name.len() > limits.max_attribute_name_bytes { + return Err(refuse( + path(), + RestorableItemReason::AttributeNameTooLong { + size: name.len(), + max: limits.max_attribute_name_bytes, + }, + )); + } + let Some(value) = item.get(name) else { + return Err(refuse( + path(), + RestorableItemReason::KeyShape(format!("key attribute {:?} missing", shown(name))), + )); + }; + if let Err(e) = check_key_value(name, attribute.ty, value) { + let reason = match e { + FormatError::InvalidKey(reason) => reason, + other => other.to_string(), + }; + // Classified by the declared type first: a text that is not a + // number under an `N` key is the #618 case; under any other + // declared type it is a key of the wrong type. + let reason = match (value, attribute.ty) { + (AttributeValue::N(n), ScalarAttributeType::N) => { + match validate_and_normalize_number(n) { + Err(wire) => RestorableItemReason::InvalidNumber(number_reason(n, &wire)), + Ok(_) => RestorableItemReason::KeyShape(reason), + } + } + _ => RestorableItemReason::KeyShape(reason), + }; + return Err(refuse(path(), reason)); + } + let (max, kind) = match attribute.key_type { + KeyType::Hash => (limits.max_partition_key_size_bytes, "hash"), + KeyType::Range => (limits.max_sort_key_size_bytes, "range"), + }; + let size = super::key_value_byte_size(value); + if size > max { + return Err(refuse( + path(), + RestorableItemReason::KeyShape(format!( + "key attribute {:?} has {size} bytes; the {kind} key limit is {max} bytes", + shown(name) + )), + )); + } + } + let mut path = String::new(); + for (name, value) in item { + if key.is_key(name) { + continue; + } + path.clear(); + push_segment(&mut path, name); + if let Err(reason) = + check_name(name, limits).and_then(|()| walk(value, &mut path, 0, limits)) + { + return Err(refuse(std::mem::take(&mut path), reason)); + } + } + let size = item_size_bytes(item); + if size > limits.max_item_size_bytes { + return Err(refuse( + String::new(), + RestorableItemReason::TooLarge { + size, + max: limits.max_item_size_bytes, + }, + )); + } + Ok(()) +} + +/// The attribute name rule the wire's `check_attribute_name_size` applies, +/// at the top level and for every map member. +fn check_name(name: &str, limits: &RestorableLimits) -> Result<(), RestorableItemReason> { + if name.len() > limits.max_attribute_name_bytes { + return Err(RestorableItemReason::AttributeNameTooLong { + size: name.len(), + max: limits.max_attribute_name_bytes, + }); + } + Ok(()) +} + +/// Append one attribute name to `path`, bounded and escaped +/// ([`bounded_text`]). +fn push_segment(path: &mut String, name: &str) { + path.push_str(&bounded_text(name, SHOWN_VALUE_BYTES)); +} + +/// Visit `value` and everything under it. On an error `path` names the +/// offending value; otherwise it is left as it was on entry. +fn walk( + value: &AttributeValue, + path: &mut String, + depth: usize, + limits: &RestorableLimits, +) -> Result<(), RestorableItemReason> { + match value { + AttributeValue::N(n) => number(n), + AttributeValue::NS(set) => { + for (i, n) in set.iter().enumerate() { + if let Err(RestorableItemReason::InvalidNumber(text)) = number(n) { + path.push_str(&format!("[{i}]")); + let member = bounded_text(n, SHOWN_VALUE_BYTES); + let text = if text.ends_with(&*member) { + text + } else { + format!("{text} (member {member})") + }; + return Err(RestorableItemReason::InvalidNumber(text)); + } + } + Ok(()) + } + AttributeValue::L(list) => { + let next = depth + 1; + if next >= super::MAX_ITEM_NESTING_DEPTH { + return Err(RestorableItemReason::TooDeep); + } + let len = path.len(); + for (i, v) in list.iter().enumerate() { + path.push_str(&format!("[{i}]")); + walk(v, path, next, limits)?; + path.truncate(len); + } + Ok(()) + } + AttributeValue::M(map) => { + let next = depth + 1; + if next >= super::MAX_ITEM_NESTING_DEPTH { + return Err(RestorableItemReason::TooDeep); + } + let len = path.len(); + for (name, v) in map { + path.push('.'); + push_segment(path, name); + check_name(name, limits)?; + walk(v, path, next, limits)?; + path.truncate(len); + } + Ok(()) + } + AttributeValue::S(_) + | AttributeValue::B(_) + | AttributeValue::SS(_) + | AttributeValue::BS(_) + | AttributeValue::Bool(_) + | AttributeValue::Null => Ok(()), + } +} + +fn number(n: &str) -> Result<(), RestorableItemReason> { + validate_and_normalize_number(n) + .map(drop) + .map_err(|e| RestorableItemReason::InvalidNumber(number_reason(n, &e))) +} + +/// The wire's reason for `n`, which ends in `n` for a malformed number: that +/// echo is bounded and escaped like every other untrusted text +/// ([`bounded_text`]), so the text is the wire's byte for byte up to +/// [`SHOWN_VALUE_BYTES`] for an input with nothing to escape. +fn number_reason(n: &str, e: &crate::error::DynamoDbError) -> String { + let text = e.message(); + match text.strip_suffix(n) { + Some(head) => format!("{head}{}", bounded_text(n, SHOWN_VALUE_BYTES)), + None => text.to_owned(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::types::KeyType; + use std::collections::{BTreeMap, BTreeSet}; + + const MAX: usize = 400 * 1024; + + /// The default limits with `max_item_size_bytes` at `max`. + fn limits(max: usize) -> RestorableLimits { + RestorableLimits { + max_item_size_bytes: max, + ..RestorableLimits::from(&LimitsConfig::default()) + } + } + + fn element(name: &str, key_type: KeyType) -> KeySchemaElement { + KeySchemaElement { + attribute_name: name.to_owned(), + key_type, + } + } + + fn definition(name: &str, ty: ScalarAttributeType) -> AttributeDefinition { + AttributeDefinition { + attribute_name: name.to_owned(), + attribute_type: ty, + } + } + + /// `pk` S alone. + fn hash_only() -> RestorableKeySchema { + RestorableKeySchema::resolve( + &[element("pk", KeyType::Hash)], + &[definition("pk", ScalarAttributeType::S)], + ) + .unwrap() + } + + /// `pk` S, `sk` N. + fn composite() -> RestorableKeySchema { + RestorableKeySchema::resolve( + &[element("pk", KeyType::Hash), element("sk", KeyType::Range)], + &[ + definition("pk", ScalarAttributeType::S), + definition("sk", ScalarAttributeType::N), + ], + ) + .unwrap() + } + + /// `pk` S, `sk` N, `sk2` B: a key `KeyOrder` cannot resolve. + fn multi_part() -> RestorableKeySchema { + RestorableKeySchema::resolve( + &[ + element("pk", KeyType::Hash), + element("sk", KeyType::Range), + element("sk2", KeyType::Range), + ], + &[ + definition("pk", ScalarAttributeType::S), + definition("sk", ScalarAttributeType::N), + definition("sk2", ScalarAttributeType::B), + ], + ) + .unwrap() + } + + fn s(v: &str) -> AttributeValue { + AttributeValue::S(v.to_owned()) + } + + fn n(v: &str) -> AttributeValue { + AttributeValue::N(v.to_owned()) + } + + fn item(pairs: &[(&str, AttributeValue)]) -> Item { + pairs + .iter() + .map(|(k, v)| ((*k).to_owned(), v.clone())) + .collect() + } + + fn refused(item: &Item, key: &RestorableKeySchema, max: usize) -> RestorableItemError { + validate_restorable_item(item, key, &limits(max)).expect_err("refused") + } + + #[test] + fn a_plain_item_passes_on_every_key_shape() { + let it = item(&[ + ("pk", s("a")), + ("sk", n("1")), + ("sk2", AttributeValue::B(vec![1])), + ("x", n("1E125")), + ( + "y", + AttributeValue::NS(BTreeSet::from(["1".to_owned(), "2.5".to_owned()])), + ), + ]); + validate_restorable_item(&it, &hash_only(), &limits(MAX)).unwrap(); + validate_restorable_item(&it, &composite(), &limits(MAX)).unwrap(); + validate_restorable_item(&it, &multi_part(), &limits(MAX)).unwrap(); + } + + #[test] + fn an_overflowing_number_in_a_non_key_attribute_is_refused_with_its_path() { + let it = item(&[("pk", s("a")), ("amount", n("1E126"))]); + let e = refused(&it, &hash_only(), MAX); + assert_eq!(e.key, r#"{"pk":{"S":"a"}}"#); + assert_eq!(e.path, "amount"); + let RestorableItemReason::InvalidNumber(reason) = &e.reason else { + panic!("{e:?}"); + }; + assert!(reason.starts_with("Number overflow"), "{reason}"); + assert_eq!( + e.to_string(), + format!(r#"key {{"pk":{{"S":"a"}}}}, attribute amount: {reason}"#) + ); + } + + #[test] + fn a_non_numeric_text_is_refused_with_the_wire_reason() { + let it = item(&[("pk", s("a")), ("amount", n("abc"))]); + let e = refused(&it, &hash_only(), MAX); + assert_eq!(e.path, "amount"); + assert_eq!( + e.reason, + RestorableItemReason::InvalidNumber( + "The parameter cannot be converted to a numeric value: abc".to_owned() + ) + ); + } + + #[test] + fn a_nested_number_names_its_path_through_maps_and_lists() { + let inner = AttributeValue::M(BTreeMap::from([("c".to_owned(), n("abc"))])); + let list = AttributeValue::L(vec![n("1"), n("2"), inner]); + let a = AttributeValue::M(BTreeMap::from([("b".to_owned(), list)])); + let it = item(&[("pk", s("a")), ("a", a)]); + let e = refused(&it, &hash_only(), MAX); + assert_eq!(e.path, "a.b[2].c"); + assert!(matches!(e.reason, RestorableItemReason::InvalidNumber(_))); + } + + /// A set member's reason carries the member, since the position is the + /// sorted set's and the overflow text alone does not name the value; a + /// text that already ends in the member (the malformed case) is left as + /// it is. + #[test] + fn a_number_set_member_names_its_position_and_the_member() { + let set = AttributeValue::NS(BTreeSet::from([ + "1".to_owned(), + "2".to_owned(), + "x".to_owned(), + ])); + let it = item(&[("pk", s("a")), ("ns", set)]); + let e = refused(&it, &hash_only(), MAX); + assert_eq!(e.path, "ns[2]"); + assert_eq!( + e.reason, + RestorableItemReason::InvalidNumber( + "The parameter cannot be converted to a numeric value: x".to_owned() + ) + ); + let set = AttributeValue::NS(BTreeSet::from(["01E126".to_owned(), "1E126".to_owned()])); + let it = item(&[("pk", s("a")), ("ns", set)]); + let e = refused(&it, &hash_only(), MAX); + assert_eq!(e.path, "ns[0]"); + let RestorableItemReason::InvalidNumber(reason) = &e.reason else { + panic!("{e:?}"); + }; + assert!(reason.starts_with("Number overflow"), "{reason}"); + assert!(reason.ends_with(" (member 01E126)"), "{reason}"); + } + + /// The path is rebuilt from the attribute for each sibling: a list whose + /// first member is a valid map and whose second member is bad names the + /// second, not a path that still carries the first's members. + #[test] + fn the_path_does_not_carry_a_sibling_that_passed() { + let ok = AttributeValue::M(BTreeMap::from([("deep".to_owned(), n("1"))])); + let list = AttributeValue::L(vec![ok, n("abc")]); + let it = item(&[("pk", s("a")), ("l", list)]); + let e = refused(&it, &hash_only(), MAX); + assert_eq!(e.path, "l[1]"); + } + + #[test] + fn validation_does_not_stop_at_the_first_valid_number() { + // Three valid numbers before the bad one, in name order. + let it = item(&[ + ("pk", s("a")), + ("n1", n("1")), + ("n2", n("2")), + ("n3", n("3")), + ("n4", n("1E126")), + ]); + assert_eq!(refused(&it, &hash_only(), MAX).path, "n4"); + } + + #[test] + fn an_item_over_the_size_limit_is_refused_without_a_path() { + let it = item(&[("pk", s("a")), ("blob", s(&"x".repeat(100)))]); + let size = item_size_bytes(&it); + validate_restorable_item(&it, &hash_only(), &limits(size)).unwrap(); + let e = refused(&it, &hash_only(), size - 1); + assert_eq!(e.path, ""); + assert_eq!( + e.reason, + RestorableItemReason::TooLarge { + size, + max: size - 1 + } + ); + assert_eq!( + e.to_string(), + format!( + r#"key {{"pk":{{"S":"a"}}}}: item size {size} bytes exceeds the limit of {} bytes"#, + size - 1 + ) + ); + } + + #[test] + fn nesting_at_the_limit_is_refused_and_one_short_of_it_passes() { + let nest = |levels: usize| { + let mut v = n("1"); + for _ in 0..levels { + v = AttributeValue::M(BTreeMap::from([("m".to_owned(), v)])); + } + item(&[("pk", s("a")), ("deep", v)]) + }; + let limit = super::super::MAX_ITEM_NESTING_DEPTH; + validate_restorable_item(&nest(limit - 1), &hash_only(), &limits(MAX)).unwrap(); + let e = refused(&nest(limit), &hash_only(), MAX); + assert_eq!(e.reason, RestorableItemReason::TooDeep); + assert_eq!(e.path, format!("deep{}", ".m".repeat(limit - 1))); + // The same rule the request path applies. + super::super::validate_item_nesting_depth(&nest(limit - 1)).unwrap(); + super::super::validate_item_nesting_depth(&nest(limit)).unwrap_err(); + } + + #[test] + fn a_missing_key_attribute_is_refused() { + let it = item(&[("pk", s("a"))]); + let e = refused(&it, &composite(), MAX); + assert_eq!(e.path, "sk"); + assert_eq!(e.key, r#"{"pk":{"S":"a"}}"#); + assert_eq!( + e.reason, + RestorableItemReason::KeyShape(r#"key attribute "sk" missing"#.to_owned()) + ); + } + + #[test] + fn a_key_attribute_of_the_wrong_type_is_refused() { + let it = item(&[("pk", s("a")), ("sk", s("1"))]); + let e = refused(&it, &composite(), MAX); + assert_eq!(e.path, "sk"); + assert_eq!( + e.reason, + RestorableItemReason::KeyShape( + r#"key attribute "sk" is S, the schema declares N"#.to_owned() + ) + ); + assert_eq!(e.key, r#"{"pk":{"S":"a"},"sk":{"S":"1"}}"#); + } + + /// A text that is not a number under a key declared `S` is a key of the + /// wrong type, not the #618 case: the declared type decides first. + #[test] + fn a_non_numeric_n_under_an_s_key_is_a_key_shape_error() { + let it = item(&[("pk", n("abc"))]); + let e = refused(&it, &hash_only(), MAX); + assert_eq!(e.path, "pk"); + assert_eq!( + e.reason, + RestorableItemReason::KeyShape( + r#"key attribute "pk" is N, the schema declares S"#.to_owned() + ) + ); + } + + #[test] + fn an_empty_key_value_is_refused() { + let it = item(&[("pk", s(""))]); + let e = refused(&it, &hash_only(), MAX); + assert_eq!(e.path, "pk"); + assert_eq!( + e.reason, + RestorableItemReason::KeyShape(r#"key attribute "pk" is empty"#.to_owned()) + ); + } + + /// A bad number in the range key is the #618 case, not a key-shape one: + /// the backup's FAILED reason names the issue by this variant. + #[test] + fn an_invalid_number_in_a_range_key_is_an_invalid_number() { + for bad in ["abc", "1E126"] { + let it = item(&[("pk", s("a")), ("sk", n(bad))]); + let e = refused(&it, &composite(), MAX); + assert_eq!(e.path, "sk"); + assert!( + matches!(e.reason, RestorableItemReason::InvalidNumber(_)), + "{bad}: {e:?}" + ); + } + } + + #[test] + fn a_valid_but_unnormalized_number_key_is_a_key_shape_error() { + let it = item(&[("pk", s("a")), ("sk", n("1.0"))]); + let e = refused(&it, &composite(), MAX); + assert_eq!(e.path, "sk"); + let RestorableItemReason::KeyShape(reason) = &e.reason else { + panic!("{e:?}"); + }; + assert!(reason.contains("not in normalized form"), "{reason}"); + } + + #[test] + fn a_multi_part_key_is_checked_attribute_by_attribute() { + let it = item(&[ + ("pk", s("a")), + ("sk", n("1")), + ("sk2", AttributeValue::B(vec![])), + ]); + let e = refused(&it, &multi_part(), MAX); + assert_eq!(e.path, "sk2"); + assert_eq!( + e.reason, + RestorableItemReason::KeyShape(r#"key attribute "sk2" is empty"#.to_owned()) + ); + assert_eq!(e.key, r#"{"pk":{"S":"a"},"sk":{"N":"1"},"sk2":{"B":""}}"#); + } + + #[test] + fn key_attributes_are_checked_before_the_other_attributes() { + let it = item(&[("amount", n("abc")), ("pk", s(""))]); + assert_eq!(refused(&it, &hash_only(), MAX).path, "pk"); + } + + #[test] + fn a_schema_whose_key_attribute_lacks_a_definition_does_not_resolve() { + let err = RestorableKeySchema::resolve(&[element("pk", KeyType::Hash)], &[]).unwrap_err(); + assert_eq!(err, r#"key attribute "pk" has no attribute definition"#); + let err = RestorableKeySchema::resolve( + &[element("pk", KeyType::Hash)], + &[ + definition("pk", ScalarAttributeType::S), + definition("pk", ScalarAttributeType::N), + ], + ) + .unwrap_err(); + assert_eq!( + err, + r#"key attribute "pk" has more than one attribute definition"# + ); + } + + /// The key sizes against the instance's limits, the rule the wire's + /// `check_key_size` applies: at the limit the key passes, one byte over + /// it is refused, hash and range each against their own limit; a `B` + /// key's size is its bytes. + #[test] + fn a_key_over_its_size_limit_is_refused_at_the_boundary() { + let limits = RestorableLimits { + max_partition_key_size_bytes: 10, + max_sort_key_size_bytes: 4, + ..limits(MAX) + }; + let ok = item(&[("pk", s(&"p".repeat(10))), ("sk", n("1234"))]); + validate_restorable_item(&ok, &composite(), &limits).unwrap(); + let hash = item(&[("pk", s(&"p".repeat(11))), ("sk", n("1"))]); + let e = validate_restorable_item(&hash, &composite(), &limits).unwrap_err(); + assert_eq!(e.path, "pk"); + assert_eq!( + e.reason, + RestorableItemReason::KeyShape( + r#"key attribute "pk" has 11 bytes; the hash key limit is 10 bytes"#.to_owned() + ) + ); + let range = item(&[("pk", s("p")), ("sk", n("12345"))]); + let e = validate_restorable_item(&range, &composite(), &limits).unwrap_err(); + assert_eq!(e.path, "sk"); + assert_eq!( + e.reason, + RestorableItemReason::KeyShape( + r#"key attribute "sk" has 5 bytes; the range key limit is 4 bytes"#.to_owned() + ) + ); + let binary = item(&[ + ("pk", s("p")), + ("sk", n("1")), + ("sk2", AttributeValue::B(vec![0; 5])), + ]); + let e = validate_restorable_item(&binary, &multi_part(), &limits).unwrap_err(); + assert_eq!(e.path, "sk2"); + assert!( + e.to_string().contains("has 5 bytes; the range key limit"), + "{e}" + ); + // The defaults: a 2048-byte hash key and a 1024-byte range key pass. + let largest = item(&[("pk", s(&"p".repeat(2048))), ("sk", n(&"1".repeat(38)))]); + validate_restorable_item(&largest, &composite(), &self::limits(MAX)).unwrap(); + let over = item(&[("pk", s(&"p".repeat(2049))), ("sk", n("1"))]); + assert!(validate_restorable_item(&over, &composite(), &self::limits(MAX)).is_err()); + } + + /// The attribute name rule the wire's `check_attribute_name_size` + /// applies: a name at the limit passes, one byte over is refused, at the + /// top level, inside a map and for a key attribute. + #[test] + fn an_attribute_name_over_its_limit_is_refused_at_the_boundary() { + let limits = RestorableLimits { + max_attribute_name_bytes: 6, + ..limits(MAX) + }; + let ok = item(&[("pk", s("a")), ("abcdef", n("1"))]); + validate_restorable_item(&ok, &hash_only(), &limits).unwrap(); + let top = item(&[("pk", s("a")), ("abcdefg", n("1"))]); + let e = validate_restorable_item(&top, &hash_only(), &limits).unwrap_err(); + assert_eq!(e.path, "abcdefg"); + assert_eq!( + e.reason, + RestorableItemReason::AttributeNameTooLong { size: 7, max: 6 } + ); + assert_eq!( + e.to_string(), + r#"key {"pk":{"S":"a"}}, attribute abcdefg: attribute name of 7 bytes exceeds the limit of 6 bytes"# + ); + let nested = AttributeValue::M(BTreeMap::from([("abcdefg".to_owned(), n("1"))])); + let it = item(&[("pk", s("a")), ("m", nested)]); + let e = validate_restorable_item(&it, &hash_only(), &limits).unwrap_err(); + assert_eq!(e.path, "m.abcdefg"); + assert_eq!( + e.reason, + RestorableItemReason::AttributeNameTooLong { size: 7, max: 6 } + ); + let key = RestorableKeySchema::resolve( + &[element("abcdefg", KeyType::Hash)], + &[definition("abcdefg", ScalarAttributeType::S)], + ) + .unwrap(); + let it = item(&[("abcdefg", s("a"))]); + let e = validate_restorable_item(&it, &key, &limits).unwrap_err(); + assert_eq!(e.path, "abcdefg"); + assert_eq!( + e.reason, + RestorableItemReason::AttributeNameTooLong { size: 7, max: 6 } + ); + } + + /// The limits are the instance's: `RestorableLimits` carries the four + /// sizes of `LimitsConfig`. + #[test] + fn restorable_limits_are_built_from_the_limits_config() { + let config = LimitsConfig { + max_item_size_bytes: 1, + max_partition_key_size_bytes: 2, + max_sort_key_size_bytes: 3, + max_attribute_name_bytes: 4, + ..LimitsConfig::default() + }; + assert_eq!( + RestorableLimits::from(&config), + RestorableLimits { + max_item_size_bytes: 1, + max_partition_key_size_bytes: 2, + max_sort_key_size_bytes: 3, + max_attribute_name_bytes: 4, + } + ); + } + + /// Keys of ordinary size are shown whole, every attribute present, as + /// `Display` writes them: a UUID hash key with a ten-digit range key, a + /// 58-byte hash key with range `1`, a 2048-byte hash key with a + /// 1024-byte range key (the largest legal key), and a hash key of + /// quotes, whose escaped text is twice its bytes. + #[test] + fn the_shown_key_carries_every_attribute_of_a_legal_key_whole() { + let uuid = "39518456-7a1b-4c2d-9e3f-0a1b2c3d832c"; + let it = item(&[ + ("pk", s(uuid)), + ("sk", n("1761955200")), + ("amount", n("abc")), + ]); + let e = refused(&it, &composite(), MAX); + assert_eq!( + e.key, + format!(r#"{{"pk":{{"S":"{uuid}"}},"sk":{{"N":"1761955200"}}}}"#) + ); + assert_eq!( + e.to_string(), + format!( + r#"key {{"pk":{{"S":"{uuid}"}},"sk":{{"N":"1761955200"}}}}, attribute amount: The parameter cannot be converted to a numeric value: abc"# + ) + ); + let tenant = "tenant-42/orders/2026-10-31/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"; + assert_eq!(tenant.len(), 58); + let it = item(&[("pk", s(tenant)), ("sk", n("1")), ("amount", n("abc"))]); + let e = refused(&it, &composite(), MAX); + assert_eq!( + e.key, + format!(r#"{{"pk":{{"S":"{tenant}"}},"sk":{{"N":"1"}}}}"#) + ); + let hash = "h".repeat(2048); + let range = "r".repeat(1024); + let key = RestorableKeySchema::resolve( + &[element("pk", KeyType::Hash), element("sk", KeyType::Range)], + &[ + definition("pk", ScalarAttributeType::S), + definition("sk", ScalarAttributeType::S), + ], + ) + .unwrap(); + let it = item(&[("pk", s(&hash)), ("sk", s(&range)), ("amount", n("abc"))]); + let e = refused(&it, &key, MAX); + assert_eq!( + e.key, + format!(r#"{{"pk":{{"S":"{hash}"}},"sk":{{"S":"{range}"}}}}"#) + ); + let quotes = "\"".repeat(2048); + let it = item(&[("pk", s("es)), ("amount", n("abc"))]); + let e = refused(&it, &hash_only(), MAX); + assert_eq!( + e.key, + format!( + r#"{{"pk":{{"S":{}}}}}"#, + serde_json::to_string("es).unwrap() + ) + ); + // A `B` key is its base64 text, the format's rendering. + let it = item(&[ + ("pk", s("a")), + ("sk", n("1")), + ("sk2", AttributeValue::B(vec![0xff; 2048])), + ("amount", n("abc")), + ]); + let e = refused(&it, &multi_part(), MAX); + assert!( + e.key + .ends_with(&format!(r#""sk2":{{"B":"{}"}}}}"#, "/".repeat(2730) + "8=")), + "{}", + e.key + ); + } + + /// Only an oversized value is cut, and on its own: a 13,000-byte hash + /// key (over `SHOWN_KEY_VALUE_BYTES`, under a raised key limit) is cut + /// to its prefix and length while the range key beside it is shown + /// whole; a 3000-byte key, over the service's limit but under the + /// bound, is whole; the bound is on the escaped text, so a key of 6145 + /// quotes (12,290 escaped bytes) is cut and a key of 6144 is not; a cut + /// value is still a JSON string. A key attribute's name is bounded at + /// `SHOWN_VALUE_BYTES` (one of 4097 bytes is cut), and a key attribute + /// holding a list is its JSON, cut to a JSON string when oversized. + #[test] + fn an_oversized_key_value_is_cut_on_its_own() { + let limits = RestorableLimits { + max_partition_key_size_bytes: 20_000, + ..limits(10) + }; + let long = "k".repeat(13_000); + let it = item(&[("pk", s(&long)), ("sk", n("7"))]); + let e = validate_restorable_item(&it, &composite(), &limits).unwrap_err(); + assert!(matches!( + e.reason, + RestorableItemReason::TooLarge { max: 10, .. } + )); + assert_eq!( + e.key, + format!( + r#"{{"pk":{{"S":"{}... <13000 bytes>"}},"sk":{{"N":"7"}}}}"#, + "k".repeat(SHOWN_VALUE_PREFIX_BYTES) + ) + ); + let over_the_service_limit = "k".repeat(3000); + let it = item(&[("pk", s(&over_the_service_limit)), ("sk", n("7"))]); + let e = validate_restorable_item(&it, &composite(), &limits).unwrap_err(); + assert_eq!( + e.key, + format!(r#"{{"pk":{{"S":"{over_the_service_limit}"}},"sk":{{"N":"7"}}}}"#) + ); + let parsed: serde_json::Value = serde_json::from_str(&e.key).unwrap(); + assert_eq!(parsed["sk"]["N"], "7"); + let plain = "p".repeat(SHOWN_KEY_VALUE_BYTES); + let it = item(&[("pk", s(&plain)), ("sk", n("7"))]); + let e = validate_restorable_item(&it, &composite(), &limits).unwrap_err(); + assert_eq!( + e.key, + format!(r#"{{"pk":{{"S":"{plain}"}},"sk":{{"N":"7"}}}}"#) + ); + let quotes = "\"".repeat(SHOWN_KEY_VALUE_BYTES / 2); + let it = item(&[("pk", s("es)), ("sk", n("7"))]); + let e = validate_restorable_item(&it, &composite(), &limits).unwrap_err(); + assert_eq!( + e.key, + format!( + r#"{{"pk":{{"S":{}}},"sk":{{"N":"7"}}}}"#, + serde_json::to_string("es).unwrap() + ) + ); + let quotes = "\"".repeat(SHOWN_KEY_VALUE_BYTES / 2 + 1); + let it = item(&[("pk", s("es)), ("sk", n("7"))]); + let e = validate_restorable_item(&it, &composite(), &limits).unwrap_err(); + assert!(e.key.contains("... <6145 bytes>"), "{}", e.key); + assert!(e.key.ends_with(r#","sk":{"N":"7"}}"#), "{}", e.key); + let parsed: serde_json::Value = serde_json::from_str(&e.key).unwrap(); + assert_eq!( + parsed["pk"]["S"].as_str().unwrap().len(), + 512 + "... <6145 bytes>".len() + ); + // The whole key stays bounded whatever the key: two 100,000-byte + // values and a 100,000-byte name. + let huge = "x".repeat(100_000); + let key = RestorableKeySchema::resolve( + &[element(&huge, KeyType::Hash), element("sk", KeyType::Range)], + &[ + definition(&huge, ScalarAttributeType::S), + definition("sk", ScalarAttributeType::S), + ], + ) + .unwrap(); + let it = item(&[(&huge, s(&huge)), ("sk", s(&huge))]); + let limits = RestorableLimits { + max_partition_key_size_bytes: 200_000, + max_sort_key_size_bytes: 200_000, + max_attribute_name_bytes: 200_000, + ..self::limits(10) + }; + let e = validate_restorable_item(&it, &key, &limits).unwrap_err(); + assert!( + e.key.len() < 3 * (SHOWN_VALUE_PREFIX_BYTES + 40), + "{}", + e.key.len() + ); + assert_eq!(e.key.matches("... <100000 bytes>").count(), 3); + // A key attribute's name is bounded at `SHOWN_VALUE_BYTES`, the + // bound a name gets, not the key value's: one of 4097 bytes is cut. + let long_name = "k".repeat(SHOWN_VALUE_BYTES + 1); + let key = RestorableKeySchema::resolve( + &[element(&long_name, KeyType::Hash)], + &[definition(&long_name, ScalarAttributeType::S)], + ) + .unwrap(); + let it = item(&[(long_name.as_str(), s("a")), ("amount", n("abc"))]); + let limits = RestorableLimits { + max_attribute_name_bytes: 10_000, + ..self::limits(MAX) + }; + let e = validate_restorable_item(&it, &key, &limits).unwrap_err(); + assert_eq!(e.path, "amount"); + assert_eq!( + e.key, + format!( + r#"{{"{}... <4097 bytes>":{{"S":"a"}}}}"#, + "k".repeat(SHOWN_VALUE_PREFIX_BYTES) + ) + ); + // A key attribute holding a value that is not a scalar (a list of + // 20,000 bytes under `pk`) renders as its JSON cut to a JSON string: + // the key still parses and stays under 1,200 bytes. + let list = AttributeValue::L(vec![s(&"x".repeat(20_000))]); + let list_json_len = serde_json::to_string(&list).unwrap().len(); + assert!(list_json_len > SHOWN_KEY_VALUE_BYTES); + let it = item(&[("pk", list), ("sk", n("7"))]); + let e = validate_restorable_item(&it, &composite(), &self::limits(MAX)).unwrap_err(); + assert!(e.key.len() < 1200, "{}", e.key.len()); + let parsed: serde_json::Value = serde_json::from_str(&e.key).unwrap(); + assert!( + parsed["pk"] + .as_str() + .unwrap() + .ends_with(&format!("... <{list_json_len} bytes>")), + "{}", + e.key + ); + assert_eq!(parsed["sk"]["N"], "7"); + } + + /// The attribute path is bounded segment by segment with the same rule: + /// a 60,000-byte attribute name holding a map with a 60,000-byte member + /// gives a path of two cut segments, each with its length; a name of + /// `SHOWN_VALUE_BYTES` bytes is whole and one byte more is cut. + #[test] + fn the_attribute_path_is_bounded_segment_by_segment() { + let limits = RestorableLimits { + max_attribute_name_bytes: 100_000, + ..limits(MAX) + }; + let long_name = "a".repeat(60_000); + let long_member = "b".repeat(60_000); + let inner = AttributeValue::M(BTreeMap::from([(long_member, n("abc"))])); + let it = item(&[("pk", s("a")), (&long_name, inner)]); + let e = validate_restorable_item(&it, &hash_only(), &limits).unwrap_err(); + assert_eq!( + e.path, + format!( + "{}... <60000 bytes>.{}... <60000 bytes>", + "a".repeat(SHOWN_VALUE_PREFIX_BYTES), + "b".repeat(SHOWN_VALUE_PREFIX_BYTES) + ) + ); + assert!(e.to_string().len() < 3000, "{}", e.to_string().len()); + let whole = "w".repeat(SHOWN_VALUE_BYTES); + let it = item(&[("pk", s("a")), (&whole, n("abc"))]); + assert_eq!( + validate_restorable_item(&it, &hash_only(), &limits) + .unwrap_err() + .path, + whole + ); + let over = "w".repeat(SHOWN_VALUE_BYTES + 1); + let it = item(&[("pk", s("a")), (&over, n("abc"))]); + assert_eq!( + validate_restorable_item(&it, &hash_only(), &limits) + .unwrap_err() + .path, + format!("{}... <4097 bytes>", "w".repeat(SHOWN_VALUE_PREFIX_BYTES)) + ); + // A set member over the bound is cut in the reason, as a plain `N` + // over it is: the wire's echo of the input is the one text that can + // grow with the item. + let member = format!("1E{}", "9".repeat(SHOWN_VALUE_BYTES)); + let set = AttributeValue::NS(BTreeSet::from([member.clone()])); + let it = item(&[("pk", s("a")), ("ns", set)]); + let e = validate_restorable_item(&it, &hash_only(), &limits).unwrap_err(); + assert!(e.to_string().len() < 2000, "{}", e.to_string().len()); + assert!( + e.to_string() + .ends_with(&format!("<{} bytes>", member.len())), + "{e}" + ); + let it = item(&[("pk", s("a")), ("n", n(&member))]); + let e = validate_restorable_item(&it, &hash_only(), &limits).unwrap_err(); + assert_eq!( + e.reason, + RestorableItemReason::InvalidNumber(format!( + "The parameter cannot be converted to a numeric value: {}... <{} bytes>", + &member[..SHOWN_VALUE_PREFIX_BYTES], + member.len() + )) + ); + // In a key the value is echoed twice: whole in the key (under the + // key bound) and cut in the reason. + let it = item(&[("pk", s("a")), ("sk", n(&member))]); + let e = validate_restorable_item(&it, &composite(), &limits).unwrap_err(); + assert!( + e.to_string().len() < member.len() + SHOWN_VALUE_PREFIX_BYTES + 400, + "{}", + e.to_string().len() + ); + assert!(e.key.contains(&member), "{}", e.key); + assert_eq!(e.to_string().matches("... <4098 bytes>").count(), 1, "{e}"); + } + + /// `LineTooLong`, the reason the backup job's feeder builds for an item + /// whose data file line would exceed the cap, names the cap: the + /// measurement stops at the cap, so the line's own length is unknown. + #[test] + fn a_line_too_long_reason_names_the_cap() { + let e = RestorableItemError { + key: r#"{"pk":{"S":"a"}}"#.to_owned(), + path: String::new(), + reason: RestorableItemReason::LineTooLong { cap: 33_554_432 }, + }; + assert_eq!( + e.to_string(), + r#"key {"pk":{"S":"a"}}: data file line exceeds the line cap of 33554432 bytes"# + ); + } + + /// A control character, or one of the line separators U+0085, U+2028 and + /// U+2029, in an attribute name or in the number the wire's reason + /// echoes is written escaped, as a JSON string's content (`\n`, `\t`, + /// `\"`, `\\`, `\u00XX`, `\u2028`), so a tenant's item cannot add a + /// line of its own to the log: the message holds no raw newline and no + /// separator, and the key beside it is the same escaped rendering. + #[test] + fn a_control_character_in_a_name_or_a_number_is_escaped_in_the_message() { + let it = item(&[ + ("pk", s("a\nb")), + ("amount", n("abc\nFORGED LINE: looks like another record")), + ]); + let e = refused(&it, &hash_only(), MAX); + let message = e.to_string(); + assert!(!message.contains('\n'), "{message:?}"); + assert_eq!( + message, + r#"key {"pk":{"S":"a\nb"}}, attribute amount: The parameter cannot be converted to a numeric value: abc\nFORGED LINE: looks like another record"# + ); + let name = "att\nFORGED LINE TWO\t\"quoted\" \\ \u{1}"; + let it = item(&[("pk", s("a")), (name, n("abc"))]); + let e = refused(&it, &hash_only(), MAX); + assert_eq!(e.path, r#"att\nFORGED LINE TWO\t\"quoted\" \\ \u0001"#); + assert!(!e.to_string().contains('\n'), "{e}"); + // In a map, in a set member and in a key attribute's name the same. + let set = AttributeValue::NS(BTreeSet::from(["1".to_owned(), "x\ny".to_owned()])); + let it = item(&[("pk", s("a")), ("ns", set)]); + let e = refused(&it, &hash_only(), MAX); + assert_eq!(e.path, "ns[1]"); + assert_eq!( + e.reason, + RestorableItemReason::InvalidNumber( + r"The parameter cannot be converted to a numeric value: x\ny".to_owned() + ) + ); + let inner = AttributeValue::M(BTreeMap::from([("c\rd".to_owned(), n("abc"))])); + let it = item(&[("pk", s("a")), ("m\u{c}", inner)]); + let e = refused(&it, &hash_only(), MAX); + assert_eq!(e.path, r"m\f.c\rd"); + let key = RestorableKeySchema::resolve( + &[element("p\nk", KeyType::Hash)], + &[definition("p\nk", ScalarAttributeType::S)], + ) + .unwrap(); + let it = item(&[("p\nk", s(""))]); + let e = validate_restorable_item(&it, &key, &limits(MAX)).unwrap_err(); + assert_eq!(e.path, r"p\nk"); + assert_eq!(e.key, r#"{"p\nk":{"S":""}}"#); + // The three Unicode line separators, which `serde_json` leaves raw + // and a reader that breaks lines the Unicode way breaks on, in a + // key value, a name and a number: each is written as its `\uXXXX` + // escape and none reaches the message. + let separators = "\u{85}\u{2028}\u{2029}"; + let name = format!("att{separators}"); + let it = item(&[ + ("pk", s(&format!("a{separators}b"))), + (name.as_str(), n(&format!("abc{separators}FORGED LINE"))), + ]); + let e = refused(&it, &hash_only(), MAX); + let message = e.to_string(); + assert!( + !message + .chars() + .any(|c| matches!(c, '\u{85}' | '\u{2028}' | '\u{2029}')), + "{message:?}" + ); + assert_eq!(e.key, r#"{"pk":{"S":"a\u0085\u2028\u2029b"}}"#); + assert_eq!(e.path, r"att\u0085\u2028\u2029"); + assert_eq!( + e.reason, + RestorableItemReason::InvalidNumber( + r"The parameter cannot be converted to a numeric value: abc\u0085\u2028\u2029FORGED LINE" + .to_owned() + ) + ); + } + + /// A key value is bounded at `SHOWN_KEY_VALUE_BYTES`, six times the hash + /// key limit: a 700-byte hash key of U+0001 (4,200 escaped bytes, over + /// the bound a name gets) is shown whole, and a 2049-byte one (12,294 + /// escaped bytes, over the legal key size) is cut. + #[test] + fn a_legal_key_of_control_characters_is_shown_whole() { + let limits = RestorableLimits { + max_partition_key_size_bytes: 4096, + ..limits(MAX) + }; + let legal = "\u{1}".repeat(700); + let it = item(&[("pk", s(&legal)), ("amount", n("abc"))]); + let e = validate_restorable_item(&it, &hash_only(), &limits).unwrap_err(); + assert_eq!(e.path, "amount"); + assert_eq!( + e.key, + format!(r#"{{"pk":{{"S":"{}"}}}}"#, r"\u0001".repeat(700)) + ); + let largest = "\u{1}".repeat(2048); + let it = item(&[("pk", s(&largest)), ("amount", n("abc"))]); + let e = validate_restorable_item(&it, &hash_only(), &limits).unwrap_err(); + assert_eq!( + e.key, + format!(r#"{{"pk":{{"S":"{}"}}}}"#, r"\u0001".repeat(2048)) + ); + let over = "\u{1}".repeat(2049); + let it = item(&[("pk", s(&over)), ("amount", n("abc"))]); + let e = validate_restorable_item(&it, &hash_only(), &limits).unwrap_err(); + assert_eq!( + e.key, + format!( + r#"{{"pk":{{"S":"{}... <2049 bytes>"}}}}"#, + r"\u0001".repeat(SHOWN_VALUE_PREFIX_BYTES / 6) + ) + ); + let parsed: serde_json::Value = serde_json::from_str(&e.key).unwrap(); + assert_eq!(parsed["pk"]["S"].as_str().unwrap().len(), 170 + 16); + } + + /// The depth rule through lists, and through lists and maps mixed: at + /// the limit refused, one short of it passed, the path naming each + /// level, and the request path's rule agreeing on every shape. + #[test] + fn nesting_through_lists_and_mixed_levels_is_measured_the_same() { + let limit = super::super::MAX_ITEM_NESTING_DEPTH; + let lists = |levels: usize| { + let mut v = n("1"); + for _ in 0..levels { + v = AttributeValue::L(vec![v]); + } + item(&[("pk", s("a")), ("deep", v)]) + }; + let mixed = |levels: usize| { + let mut v = n("1"); + for i in 0..levels { + v = if i % 2 == 0 { + AttributeValue::L(vec![v]) + } else { + AttributeValue::M(BTreeMap::from([("m".to_owned(), v)])) + }; + } + item(&[("pk", s("a")), ("deep", v)]) + }; + for (what, nest) in [ + ("lists", &lists as &dyn Fn(usize) -> Item), + ("mixed", &mixed as &dyn Fn(usize) -> Item), + ] { + validate_restorable_item(&nest(limit - 1), &hash_only(), &limits(MAX)) + .unwrap_or_else(|e| panic!("{what}, one short of the limit: {e}")); + let e = refused(&nest(limit), &hash_only(), MAX); + assert_eq!(e.reason, RestorableItemReason::TooDeep, "{what}"); + super::super::validate_item_nesting_depth(&nest(limit - 1)) + .unwrap_or_else(|e| panic!("{what}: the request path refuses one short: {e}")); + super::super::validate_item_nesting_depth(&nest(limit)).unwrap_err(); + } + let e = refused(&lists(limit), &hash_only(), MAX); + assert_eq!(e.path, format!("deep{}", "[0]".repeat(limit - 1))); + let e = refused(&mixed(limit), &hash_only(), MAX); + // Built inside out: the outermost level is the last one wrapped. + let mut path = "deep".to_owned(); + for i in (0..limit - 1).rev() { + path.push_str(if (i + 1) % 2 == 0 { "[0]" } else { ".m" }); + } + assert_eq!(e.path, path); + } +} diff --git a/crates/engine/Cargo.toml b/crates/engine/Cargo.toml index 3634c9152..1efd942e8 100755 --- a/crates/engine/Cargo.toml +++ b/crates/engine/Cargo.toml @@ -7,10 +7,16 @@ edition.workspace = true rust-version.workspace = true license.workspace = true +[features] +# Kill-point pauses in the backup job (`EXTENDDB_BACKUP_PAUSE`), for the +# binary-level crash tests; never enabled in a release build. +backup-test-hooks = [] + [dependencies] extenddb-auth = { workspace = true } extenddb-core = { workspace = true } extenddb-storage = { workspace = true } +extenddb-backup-store = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } tracing = { workspace = true } @@ -21,3 +27,13 @@ hmac = "0.12" sha2 = "0.10" hex = "0.4" tokio = { workspace = true } +# `TaskTracker` for the backup jobs the server drains at shutdown; +# `StreamReader` and `SyncIoBridge` for the restore's in-place decode of a +# store object on the blocking pool. +tokio-util = { workspace = true, features = ["rt", "io", "io-util"] } +bytes = { workspace = true } +futures = { workspace = true } + +[dev-dependencies] +# `start_paused` for the tests of timers that must not depend on the wall clock. +tokio = { workspace = true, features = ["test-util"] } diff --git a/crates/engine/src/backup.rs b/crates/engine/src/backup.rs index 2e3780e02..cc6c31760 100755 --- a/crates/engine/src/backup.rs +++ b/crates/engine/src/backup.rs @@ -16,6 +16,7 @@ use std::sync::Arc; +use extenddb_core::backup_format::{BackupArn, check_store_prefix_table_name, is_region}; use extenddb_core::error::{DynamoDbError, ErrorMessageKey, error_message}; use extenddb_core::limits::LimitsConfig; use extenddb_core::types::{ @@ -23,11 +24,27 @@ use extenddb_core::types::{ LocalSecondaryIndexInfo, ProvisionedThroughput, RestoreSummary, SourceTableFeatureDetails, SseDescription, TableDescription, TableStatus, }; -use extenddb_storage::BackupEngine; +use extenddb_storage::backup_catalog::{ + BackupKey, BackupRow, BackupStatus, DeleteStart, TimeBounds, is_backup_not_found, +}; use extenddb_storage::error::StorageError; -use extenddb_storage::{BackupRestoreSource, RestoreRequest, RestoreTableOverrides}; +use extenddb_storage::{ + BackupCatalog, BackupRestoreSource, DataEngine, LegacyBackupReader, MetadataEngine, + RestoreEngine, RestoreRequest, RestoreSource, RestoreTableOverrides, SnapshotEngine, + TableEngine, +}; +use futures::TryStreamExt; use serde_json::{Value, json}; +pub use job::KillPoint; +pub use maintenance::HeldClaim; +pub use restore::{OTHER_STORE, RESTORE_REGISTRATION_TIMEOUT, TARGET_NOT_EMPTY}; +pub use runtime::{ + BackupRuntime, ConfiguredStore, JobSettings, LOCAL_CHECK_TIMEOUT, S3_CHECK_TIMEOUT, Spool, + StoreFence, StoreHealth, StoreHealthReport, check_configured_store, read_identity_bounded, +}; +pub use store_identity::{STORE_IDENTITY_KEY, StoreIdentityFile, StoreProblem}; + use crate::{OperationContext, serialize_output}; /// `ListBackups` page size when the request gives no `Limit`, and the largest @@ -84,20 +101,37 @@ fn backup_arn_field( /// The spellings of one `BackupArn` the engine looks a backup up under: /// the ARN this deployment issues for the backup the caller named /// ([`canonical_backup_arn`]) and, when it differs, the ARN as the caller -/// wrote it, trimmed. +/// wrote it, trimmed. The strings are what the legacy reader is addressed +/// by, built as main built them; each is also parsed once, here, into the +/// [`BackupArn`] the catalog index is addressed by. `BackupArn` applies two +/// store-prefix rules `check_backup_arn` does not (a table name over 255 +/// bytes, or made only of dots), which a legacy backup can break: main's +/// CreateTable and CreateBackup accept `...`, and an operator can raise +/// `limits.max_table_name_length`. A spelling that fails them (or one built +/// under a `server.region` outside the ARN alphabet, which the legacy path +/// never checked) has no parsed form and no catalog-index read (no index row +/// can carry such an ARN: the store-backed writer, the one path that forms a +/// store prefix, refuses the name before it writes), and the legacy reader is +/// tried under its string as on main. #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) struct BackupArnSpellings { canonical: String, as_written: Option, + canonical_arn: Option, + as_written_arn: Option, } impl BackupArnSpellings { fn new(parts: &extenddb_storage::util::BackupArnParts<'_>, region: &str) -> Self { let canonical = canonical_backup_arn(parts, region); let as_written = (parts.arn != canonical).then(|| parts.arn.to_owned()); + let canonical_arn = BackupArn::parse(&canonical).ok(); + let as_written_arn = as_written.as_deref().and_then(|a| BackupArn::parse(a).ok()); Self { canonical, as_written, + canonical_arn, + as_written_arn, } } @@ -105,11 +139,62 @@ impl BackupArnSpellings { fn canonical(&self) -> &str { &self.canonical } + + /// The parsed spellings in lookup order: canonical, then as written; + /// none for a name the store-prefix rules refuse. + fn arns(&self) -> impl Iterator { + self.canonical_arn + .iter() + .chain(self.as_written_arn.as_ref()) + } +} + +/// The catalog row one of `arn`'s spellings names, read in [`resolve_backup`]'s +/// order, with the spelling that resolved. A row read by account and backup +/// id is returned by the backend only when its account, table and ARN equal +/// the spelling's, so an ARN of account B carrying the id of account A's +/// backup finds nothing (and never A's row). `None` when neither spelling +/// has a row: the caller falls back to the legacy reader. +async fn read_catalog_row( + storage: &S, + arn: &BackupArnSpellings, +) -> Result, StorageError> { + for spelling in arn.arns() { + if let Some(row) = storage.read_backup_row(spelling).await? { + return Ok(Some((spelling.clone(), row))); + } + } + Ok(None) +} + +/// The error for a backup the catalog index has no describable row for. +fn backup_not_found(arn: &str) -> DynamoDbError { + DynamoDbError::BackupNotFoundException(format!( + "{}: {arn}", + extenddb_storage::backup_catalog::BACKUP_NOT_FOUND + )) } -/// Whether a backend error says the backup is not there. -fn is_backup_not_found(e: &StorageError) -> bool { - matches!(e, StorageError::Validation(m) if m.contains("Backup not found")) +/// The error for a CREATING row, or one whose backup a CREATING restore +/// target still reads: `BackupInUseException`, as the service answers. The +/// in-use text names the backup, not the target (the service's text); the +/// target's name goes to the log. +fn backup_in_use(arn: &str, restoring_into: Option<&str>) -> DynamoDbError { + DynamoDbError::BackupInUseException(match restoring_into { + Some(table) => { + tracing::info!("backup {arn} is in use: a restore into {table} is still CREATING"); + extenddb_storage::backup_catalog::backup_in_use_message(arn) + } + None => format!("Backup is being created: {arn}"), + }) +} + +/// The error for a row of a store other than the configured one. Names no +/// root: an API error never does. +fn backup_in_another_store(arn: &str) -> DynamoDbError { + DynamoDbError::ValidationException(format!( + "Backup {arn} is in a backup store other than the one this server is configured with" + )) } /// Look a backup up by `lookup`, first under the canonical ARN, then, if that @@ -136,8 +221,8 @@ where } } -/// [`resolve_backup`]'s lookup for one `BackupEngine` method taking the ARN: -/// `by!(storage, account_id, method)`. +/// [`resolve_backup`]'s lookup for one [`LegacyBackupReader`] method taking +/// the ARN: `by!(storage, account_id, method)`. macro_rules! by { ($storage:expr, $account_id:expr, $method:ident) => {{ let storage = $storage; @@ -255,18 +340,23 @@ fn table_name_bounds(limits: &LimitsConfig) -> (usize, usize) { (limits.min_table_name_length, limits.max_table_name_length) } -/// The `TableName` of `CreateBackup`, which the service accepts as the -/// table's name or its ARN (measured on DynamoDB; the model +/// The `TableName` of `CreateBackup` and `ListBackups`, which the service +/// accepts as the table's name or its ARN (measured on DynamoDB; the model /// gives the member the `TableName` shape, max 255, so an SDK built from it /// refuses the ARN form client-side and only a hand-built request sends it). /// An ARN is `arn:aws:dynamodb:::table/`: another /// account's is `AccessDeniedException`, as a backup ARN of another account -/// is, and another region's resolves, as a backup ARN's does (what the -/// service answers for those two was not measured). The name part is then -/// checked as a name. +/// is, checked first (an ARN of another account in another region is +/// `AccessDeniedException` on the service, measured on DynamoDB for +/// CreateBackup and ListBackups); one whose region is not this server's +/// (`region`, compared as written, so `US-EAST-1` differs from `us-east-1`) +/// is `ValidationException` with the service's text (`1 validation error +/// detected: Invalid AWS region in ''`, measured the same way). The +/// name part is then checked as a name. fn create_backup_table_name<'a>( body: &'a Value, account_id: &str, + region: &str, limits: &LimitsConfig, ) -> Result<&'a str, DynamoDbError> { let value = required_str(body, "TableName", "tableName")?; @@ -280,11 +370,11 @@ fn create_backup_table_name<'a>( ) }; let fields: Vec<&str> = rest.split(':').collect(); - let [partition, "dynamodb", region, arn_account, resource] = fields.as_slice() else { + let [partition, "dynamodb", arn_region, arn_account, resource] = fields.as_slice() else { return Err(not_valid()); }; if *partition != extenddb_storage::util::ARN_PARTITION - || region.is_empty() + || arn_region.is_empty() || arn_account.is_empty() || !arn_account.bytes().all(|b| b.is_ascii_digit()) { @@ -296,12 +386,20 @@ fn create_backup_table_name<'a>( if name.contains('/') { return Err(not_valid()); } - validate_name_member(name, "tableName", table_name_bounds(limits))?; + // The account before the region: another account's ARN in another + // region is AccessDeniedException on the service (measured on DynamoDB + // for CreateBackup and ListBackups, twice each). if *arn_account != account_id { return Err(DynamoDbError::AccessDeniedException( "Access is denied".to_owned(), )); } + if *arn_region != region { + return Err(DynamoDbError::ValidationException(format!( + "1 validation error detected: Invalid AWS region in '{value}'" + ))); + } + validate_name_member(name, "tableName", table_name_bounds(limits))?; Ok(name) } @@ -310,16 +408,10 @@ pub(crate) async fn handle_create_backup( body: Value, ctx: &OperationContext, ) -> Result { - let table_name = create_backup_table_name(&body, &ctx.account_id, &ctx.limits)?; + let table_name = create_backup_table_name(&body, &ctx.account_id, &ctx.region, &ctx.limits)?; let backup_name = required_str(&body, "BackupName", "backupName")?; validate_name_member(backup_name, "backupName", BACKUP_NAME_BOUNDS)?; - - // A backup of a table with vector indexes could never be restored (the - // restore refuses it), so refuse it here with the restore's own reason - // rather than hand back an AVAILABLE backup the customer finds out about - // at restore time. A lookup failure is left to create_backup, which - // reports a missing table in its own terms. - if let Ok(desc) = ctx + let probe = ctx .storage .describe_table( &ctx.account_id, @@ -327,18 +419,223 @@ pub(crate) async fn handle_create_backup( table_name: table_name.to_owned(), }, ) - .await - { - refuse_backup_of_vector_table(&desc)?; + .await; + let details = create_backup( + &ctx.storage, + &ctx.backup, + &ctx.account_id, + &ctx.region, + &ctx.limits, + table_name, + backup_name, + probe, + ) + .await?; + serialize_output(&json!({ "BackupDetails": details })) +} + +/// One CreateBackup after its members are checked, in the service's order, +/// every check before the permit: the table exists and is ACTIVE for the +/// account (`TableNotFoundException`, "Table not found: ", otherwise, +/// whether or not a permit is free); the server's region fits a backup ARN +/// (`check_backup_arn_fits`: a `server.region` outside `[a-z0-9-]` is a +/// `ValidationException` naming the setting); on the store-backed path, the +/// table name fits a store prefix (`check_store_prefix_fits`: a name over 255 +/// bytes or made only of dots is a `ValidationException` naming the rule; the +/// in-catalog path writes no prefix and backs such a table up, as the service +/// does); on the in-catalog path, the table has no vector index (the +/// in-catalog format cannot restore one); then the job permit +/// (`LimitExceededException` when none is freed within the runtime's wait). +/// `probe` is the `DescribeTable` of the table, which the handler ran. +/// +/// With a store configured and the backend not bound to the in-catalog +/// create, the create is store-backed: the store must be healthy +/// ([`BackupRuntime::require_healthy_store`], refused before a slot is +/// taken as every operation that touches the store is), the maintenance +/// claim must not be held ([`refuse_while_maintenance_held`], before the +/// permit as on the in-catalog path, so a refusal inserts no row), then the +/// permit (`LimitExceededException` when none is free), then the detached +/// task of [`job::start`] (the snapshot's open, the row under the job's +/// claim, the response CREATING) whose answer the request awaits. A vector +/// table is backed up on this path (the manifest records its indexes). +/// +/// Otherwise the in-catalog create: the table has no vector index (the +/// in-catalog format cannot restore one), the maintenance claim is not held +/// ([`refuse_while_maintenance_held`], last before the permit), the permit, +/// then `LegacyBackupReader::create`; each call logs that the format is +/// deprecated. `probe` is the `DescribeTable` of the table, which the +/// handler ran. +#[allow(clippy::too_many_arguments)] +async fn create_backup( + storage: &Arc, + runtime: &BackupRuntime, + account_id: &str, + region: &str, + limits: &LimitsConfig, + table_name: &str, + backup_name: &str, + probe: Result, +) -> Result +where + S: BackupCatalog + + LegacyBackupReader + + SnapshotEngine + + MetadataEngine + + ?Sized + + Send + + Sync + + 'static, +{ + let desc = match probe { + Ok(desc) if desc.table_status == TableStatus::Active => desc, + Ok(_) => { + return Err(create_backup_storage_err_to_dynamo( + StorageError::TableNotActive(table_name.to_owned()), + table_name, + )); + } + Err(e) => return Err(create_backup_storage_err_to_dynamo(e, table_name)), + }; + check_backup_arn_fits(region)?; + let store_backed = runtime.configured_store().is_some() && !runtime.legacy_create_only(); + if store_backed { + check_store_prefix_fits(region, account_id, table_name)?; + let store = runtime.require_healthy_store()?; + let env = job::JobEnv::new(runtime, store)?; + refuse_while_maintenance_held(&**storage, runtime).await?; + let permit = runtime.take_permit().await?; + let (reply, answer) = tokio::sync::oneshot::channel(); + let request = job::CreateRequest { + account_id: account_id.to_owned(), + region: region.to_owned(), + table_name: table_name.to_owned(), + backup_name: backup_name.to_owned(), + limits: limits.into(), + }; + runtime.spawn_job(job::start(Arc::clone(storage), env, permit, request, reply)); + return answer.await.unwrap_or_else(|_| { + tracing::error!( + "CreateBackup of {table_name} (account {account_id}): the job task ended \ + without answering" + ); + Err(DynamoDbError::InternalServerError( + "Internal server error".to_owned(), + )) + }); } + refuse_backup_of_vector_table(&desc)?; + refuse_while_maintenance_held(&**storage, runtime).await?; + let permit = runtime.take_permit().await?; + match runtime.configured_store() { + None => tracing::warn!( + "CreateBackup of {table_name} (account {account_id}) is writing the in-catalog \ + backup format, which a later release stops creating; configure [backup.store] to \ + write backups to a backup store" + ), + Some(store) => tracing::info!( + "CreateBackup of {table_name} (account {account_id}) is writing the in-catalog \ + backup format: this backend is bound to it; the configured {} backup store serves \ + restores and deletes of the backups it holds", + store.kind + ), + } + storage + .create(account_id, table_name, backup_name, permit) + .await + .map_err(|e| create_backup_storage_err_to_dynamo(e, table_name)) +} - let details = ctx - .storage - .create_backup(&ctx.account_id, table_name, backup_name) +/// The jobs' side of the catalog-level maintenance claim: a backup or +/// restore job is refused with `LimitExceededException` +/// ([`maintenance_held`]) while a maintenance command (`extenddb backup +/// purge`) holds the claim, read with `stale_job_after_secs` so a dead +/// command's claim expires. The check runs after every other refusal and +/// before the permit, which is before the job's row exists: the in-catalog +/// create and restore commit their row inside the backend and detach, so the +/// engine has no point between the row and the copy to check at, and the +/// store-backed create checks here too rather than after its insert, since +/// a row written FAILED for the refusal carried the insert's heartbeat and +/// `backup purge`'s wait counted it as a live job for the stale window. A +/// claim taken between this check and the row's insert is therefore not +/// seen by the job; the in-catalog create that slips through writes no +/// object of the store the command deletes from, and a restore or a +/// store-backed create that slips through is a RUNNING record or a CREATING +/// row the command's wait lists, so the window costs nothing. +async fn refuse_while_maintenance_held( + storage: &S, + runtime: &BackupRuntime, +) -> Result<(), DynamoDbError> { + let held = storage + .check_maintenance_claim(runtime.stale_job_after()) .await - .map_err(|e| create_backup_storage_err_to_dynamo(e, table_name))?; + .map_err(storage_err_to_dynamo)?; + if held { + return Err(maintenance_held()); + } + Ok(()) +} - serialize_output(&json!({ "BackupDetails": details })) +/// The refusal every job answers while the maintenance claim is held. +fn maintenance_held() -> DynamoDbError { + DynamoDbError::LimitExceededException( + "A backup maintenance command holds this catalog's maintenance claim; backup and \ + restore jobs are refused until it finishes" + .to_owned(), + ) +} + +/// The one backup ARN rule CreateBackup applies on this path: the server's +/// own region (`server.region` outside `[a-z0-9-]` forms no backup ARN +/// whatever the name, so the message names the setting, not the table). The +/// store-prefix rules on the name (over 255 bytes, or made only of dots; +/// `extenddb_core::backup_format::check_store_prefix_table_name`) are not +/// applied here: the in-catalog create writes no store prefix, and the +/// service backs up any table it can create (measured on DynamoDB: a table +/// named `...` backs up), so the engine keeps main's behaviour for it, and +/// the store-backed writer, the one path that forms a prefix, applies them +/// before it writes. +fn check_backup_arn_fits(region: &str) -> Result<(), DynamoDbError> { + if !is_region(region) { + return Err(DynamoDbError::ValidationException(format!( + "CreateBackup is refused on this server: server.region {region:?} is outside \ + [a-z0-9-], so no backup ARN can be formed; set server.region to a value in \ + that class" + ))); + } + Ok(()) +} + +/// The store-backed create's name rules, applied before the job's row exists: +/// the table name has to fit a store prefix +/// (`extenddb_core::backup_format::check_store_prefix_table_name`: over 255 +/// bytes, or made only of dots, fits none) and the backup ARN has to form +/// (checked with a placeholder id, so the message names the limit whatever +/// the length). The in-catalog create applies neither: it writes no prefix, +/// and the service backs up any table it can create. +fn check_store_prefix_fits( + region: &str, + account_id: &str, + table_name: &str, +) -> Result<(), DynamoDbError> { + check_store_prefix_table_name(table_name).map_err(|e| { + DynamoDbError::ValidationException(format!( + "TableName cannot be backed up: a backup ARN cannot be formed for it: {e}" + )) + })?; + let placeholder = extenddb_core::backup_format::format_backup_id(0, 0); + BackupArn::new( + extenddb_storage::util::ARN_PARTITION, + region, + account_id, + table_name, + &placeholder, + ) + .map(drop) + .map_err(|e| { + DynamoDbError::ValidationException(format!( + "TableName cannot be backed up: a backup ARN cannot be formed for it: {e}" + )) + }) } /// [`storage_err_to_dynamo`] for CreateBackup, where a table that does not @@ -355,7 +652,8 @@ fn create_backup_storage_err_to_dynamo(e: StorageError, table_name: &str) -> Dyn } } -/// The CreateBackup refusal for a table with vector indexes. +/// The in-catalog create's refusal for a table with vector indexes (the +/// in-catalog format cannot restore one); the store-backed path has none. fn refuse_backup_of_vector_table(desc: &TableDescription) -> Result<(), DynamoDbError> { let count = desc.vector_indexes.as_ref().map_or(0, Vec::len); if count == 0 { @@ -379,21 +677,33 @@ pub(crate) async fn handle_describe_backup( serialize_output(&json!({ "BackupDescription": desc })) } -/// The DescribeBackup body: the backend's description, with +/// The DescribeBackup body. The catalog index is consulted first: a row of +/// one of the ARN's spellings answers on its own, derived from its recorded +/// source table ([`BackupRow::description`]), with DELETING reported as +/// DELETED and a DELETED or FAILED row `BackupNotFoundException`. With no +/// row, the legacy reader's description, with /// `SourceTableDetails.ProvisionedThroughput` and `SourceTableFeatureDetails` /// filled from the backup's recorded source when the backup is AVAILABLE. /// The backup is looked up under the ARN's spellings in [`resolve_backup`]'s /// order; the source is then read under the spelling that resolved. -async fn describe_backup( +async fn describe_backup( storage: &S, account_id: &str, region: &str, backup_arn: &BackupArnSpellings, ) -> Result { - let (backup_arn, mut desc) = - resolve_backup(backup_arn, by!(storage, account_id, describe_backup)) - .await - .map_err(storage_err_to_dynamo)?; + if let Some((arn, row)) = read_catalog_row(storage, backup_arn) + .await + .map_err(storage_err_to_dynamo)? + { + if !row.status.is_described() { + return Err(backup_not_found(&arn.to_arn_string())); + } + return row.description().map_err(storage_err_to_dynamo); + } + let (backup_arn, mut desc) = resolve_backup(backup_arn, by!(storage, account_id, describe)) + .await + .map_err(storage_err_to_dynamo)?; let backup_arn = backup_arn.as_str(); // A CREATING or DELETED backup has no restore source to read; it is @@ -408,9 +718,9 @@ async fn describe_backup( // answer for a database failure, while RestoreTableFromBackup on the same // read is a 500. let source = if desc.backup_details.backup_status == "AVAILABLE" { - match storage.backup_restore_source(account_id, backup_arn).await { + match storage.restore_source(account_id, backup_arn).await { Ok(source) => Some(source), - Err(StorageError::Validation(m)) if m.contains("Backup not found") => None, + Err(e) if is_backup_not_found(&e) => None, Err(StorageError::Unsupported(m)) => { tracing::warn!("DescribeBackup {backup_arn}: definition not described: {m}"); None @@ -524,10 +834,14 @@ pub(crate) async fn handle_list_backups( serialize_output(&page) } -/// `TableName` here is a table name only: unlike `CreateBackup`, the ARN form -/// is refused with the pattern clause (the service's behaviour for -/// `ListBackups` with an ARN was not measured). -async fn list_backups( +/// `TableName` is a table name or a table ARN, as `CreateBackup` accepts it +/// (the service accepts both, measured on DynamoDB). The listing merges the +/// catalog index's rows (CREATING and AVAILABLE ones, summarized from their +/// recorded source table) with the legacy reader's summaries into one list, +/// which is then filtered and paged in (`BackupCreationDateTime`, ARN) order: +/// `created_at` is the id's instant on both sides, so a cursor places itself +/// in the merged order the same way whichever side issued it. +async fn list_backups( storage: &S, account_id: &str, region: &str, @@ -560,10 +874,7 @@ async fn list_backups( }; let table_name = match body.get("TableName") { None | Some(Value::Null) => None, - Some(Value::String(name)) => { - validate_name_member(name, "tableName", table_name_bounds(limits))?; - Some(name.as_str()) - } + Some(Value::String(_)) => Some(create_backup_table_name(body, account_id, region, limits)?), Some(_) => { return Err(DynamoDbError::SerializationException( "Start of structure or map found where not expected: TableName".to_owned(), @@ -584,10 +895,41 @@ async fn list_backups( }; let filter = ListBackupsFilter::parse(body)?; - let summaries = storage - .list_backups(account_id, table_name) + let bounds = TimeBounds { + lower: filter.lower, + upper: filter.upper, + }; + // A row whose summary does not derive (a `source_table_details_json` + // this build cannot read) is logged and left out, so one such row does + // not turn every ListBackups of the account into an internal error; a + // failure to read the rows propagates. + let mut summaries: Vec = storage + .list_account_backup_rows(account_id, table_name, &bounds) + .try_filter_map(|row| async move { + if !row.status.is_listed() { + return Ok(None); + } + match row.summary() { + Ok(summary) => Ok(Some(summary)), + Err(e) => { + tracing::warn!( + "ListBackups (account {account_id}): backup {} left out of the \ + listing: {e}", + row.backup_arn + ); + Ok(None) + } + } + }) + .try_collect() .await .map_err(storage_err_to_dynamo)?; + summaries.extend( + storage + .list(account_id, table_name) + .await + .map_err(storage_err_to_dynamo)?, + ); let start = start.map(|cursor| cursor.resolve(&summaries)).transpose()?; let summaries = summaries.into_iter().filter(|s| filter.keeps(s)).collect(); let (page, last) = page_backups( @@ -812,7 +1154,14 @@ pub(crate) async fn handle_delete_backup( ctx: &OperationContext, ) -> Result { let backup_arn = backup_arn_field(&body, &ctx.account_id, &ctx.region)?; - let desc = delete_backup(&*ctx.storage, &ctx.account_id, &ctx.region, &backup_arn).await?; + let desc = delete_backup( + &*ctx.storage, + &ctx.backup, + &ctx.account_id, + &ctx.region, + &backup_arn, + ) + .await?; serialize_output(&json!({ "BackupDescription": desc })) } @@ -820,6 +1169,9 @@ pub(crate) async fn handle_delete_backup( /// as it was, with `BackupStatus` `DELETED` (measured on DynamoDB). `SourceTableDetails.ProvisionedThroughput` is a required member /// of the shape and a typed SDK can fail to unmarshal a response without it. /// +/// The catalog index is consulted first ([`delete_catalog_backup`]); with no +/// row of either spelling, the legacy reader's delete. +/// /// The restore source is read before the delete, since it is gone after, /// under the ARN spelling that resolves ([`resolve_backup`]); the delete then /// names that spelling. A backup the read does not find under either @@ -832,14 +1184,21 @@ pub(crate) async fn handle_delete_backup( /// means a damaged `PROVISIONED` backup is described with on-demand `{0, 0}` /// (ExtendDB's choice: deletability over the throughput member); a failure to /// reach the database propagates, as the delete itself would fail on it. -async fn delete_backup( +async fn delete_backup( storage: &S, + runtime: &BackupRuntime, account_id: &str, region: &str, backup_arn: &BackupArnSpellings, ) -> Result { + if let Some((arn, row)) = read_catalog_row(storage, backup_arn) + .await + .map_err(storage_err_to_dynamo)? + { + return delete_catalog_backup(storage, runtime, &arn, &row).await; + } let (resolved, source) = - match resolve_backup(backup_arn, by!(storage, account_id, backup_restore_source)).await { + match resolve_backup(backup_arn, by!(storage, account_id, restore_source)).await { Ok((resolved, source)) => (Some(resolved), Some(source)), Err(e) if is_backup_not_found(&e) => (None, None), Err(StorageError::Unsupported(m) | StorageError::Internal(m)) => { @@ -852,8 +1211,8 @@ async fn delete_backup( Err(e) => return Err(storage_err_to_dynamo(e)), }; let mut desc = match resolved { - Some(resolved) => storage.delete_backup(account_id, &resolved).await, - None => resolve_backup(backup_arn, by!(storage, account_id, delete_backup)) + Some(resolved) => storage.delete(account_id, &resolved).await, + None => resolve_backup(backup_arn, by!(storage, account_id, delete)) .await .map(|(_, desc)| desc), } @@ -863,6 +1222,220 @@ async fn delete_backup( Ok(desc) } +/// DeleteBackup of a backup the catalog index has a row for. In the order +/// the request's checks run, the same order as a restore's +/// ([`check_catalog_row_for_restore`]): a DELETED or FAILED row is +/// `BackupNotFoundException`; a CREATING row is `BackupInUseException`; a +/// backup a CREATING restore target still reads is `BackupInUseException` +/// naming that table ([`BackupCatalog::backup_in_use`]); a row of a store +/// other than the configured one is a `ValidationException` naming no root, +/// and a row of the configured store while that store is unhealthy or its +/// first check pending is a `ValidationException` naming the store's kind; +/// then the in-use check runs again with the AVAILABLE-to-DELETING write in +/// one transaction under the row's lock +/// ([`BackupCatalog::start_backup_delete`]), so a restore's registration and +/// this order wholly one before the other: a registration that won leaves +/// this `BackupInUseException` naming its target. A DELETING row (a retried +/// DeleteBackup) resumes the deletion. The deletion fence +/// ([`StoreFence`]) runs before the DELETING write and again right before +/// the first deletion. At the first, a root it finds owned by another +/// catalog is refused with the row as it was, answered as the unhealthy +/// store's `ValidationException`; a file it cannot read is a store failure +/// and the request goes on to the DELETING write, the second fence then +/// refusing the deletion. A refusal at the second (another catalog adopted +/// the root in between, or the file still cannot be read) leaves the row +/// DELETING with nothing deleted, which the reconciler finishes once its +/// own fence passes (for an adopted root, only if this catalog owns the +/// root again). Then the deletion itself +/// ([`finish_backup_delete`], through the store's bounded calls, +/// [`ConfiguredStore::calls`]): the manifest first, so the prefix is no +/// longer a complete backup to any reader, then the rest of the prefix, +/// then the row to DELETED once a listing of the prefix returns nothing. The +/// request answers the description with `BackupStatus` DELETED once the +/// DELETING write has committed, whatever the deletion does, the second +/// fence's refusal included: every reader already reports the backup +/// deleted from that write on, and the service has no counterpart for a +/// failed object deletion. If a step fails (the store not answering, a +/// listing that still shows an object) the row stays DELETING, the failure +/// is logged, and the reconciler finishes the deletion at its next cycle. +/// +/// The description is built before the DELETING write: a row whose +/// `source_table_details_json` this build cannot read is answered as an +/// internal error with the row still AVAILABLE, rather than moved to DELETING +/// and then refused on every retry. +async fn delete_catalog_backup( + storage: &S, + runtime: &BackupRuntime, + arn: &BackupArn, + row: &BackupRow, +) -> Result { + let arn_text = arn.to_arn_string(); + match row.status { + BackupStatus::Deleted | BackupStatus::Failed => return Err(backup_not_found(&arn_text)), + BackupStatus::Creating => return Err(backup_in_use(&arn_text, None)), + BackupStatus::Available | BackupStatus::Deleting => {} + } + let key = BackupKey::from_arn(arn); + if let Some(table) = storage + .backup_in_use(&key) + .await + .map_err(storage_err_to_dynamo)? + { + return Err(backup_in_use(&arn_text, Some(&table))); + } + if !runtime.is_configured_store(&row.store) { + return Err(backup_in_another_store(&arn_text)); + } + let store = runtime.require_healthy_store()?; + let mut desc = row.description().map_err(storage_err_to_dynamo)?; + // The fence before the DELETING write as well as before the deletion: a + // root it finds owned by another catalog (or by this one under another + // store id) is refused here with the row as it was, so the API never + // answers that refusal with a DELETED backup. A file it cannot read, or + // a root with no file, is a store failure: the request goes on as it + // does when the store does not answer the deletion, and the fence + // before the deletion refuses it, leaving the row DELETING for the + // reconciler, whose deletion runs the fence again. + let fence = store.fence(); + if let Err(problem) = fence.check_catalog(storage).await { + if let Some(refusal) = fence.refusal() { + tracing::error!( + "DeleteBackup {arn_text}: refused before the row was changed; nothing was \ + deleted: {problem}" + ); + return Err(refusal); + } + if let StoreProblem::Catalog(e) = &problem { + tracing::error!("DeleteBackup {arn_text}: the catalog id could not be read: {e}"); + return Err(DynamoDbError::InternalServerError( + "Internal server error".to_owned(), + )); + } + tracing::warn!( + "DeleteBackup {arn_text}: the deletion fence could not read the store's identity \ + file before the DELETING write ({problem}); the deletion is fenced again before \ + any object is deleted" + ); + } + match storage + .start_backup_delete(&key) + .await + .map_err(storage_err_to_dynamo)? + { + DeleteStart::Deleting | DeleteStart::Status(BackupStatus::Deleting) => {} + DeleteStart::InUse(table) => return Err(backup_in_use(&arn_text, Some(&table))), + DeleteStart::Status(BackupStatus::Creating) => { + return Err(backup_in_use(&arn_text, None)); + } + DeleteStart::Status(BackupStatus::Deleted | BackupStatus::Failed) => { + return Err(backup_not_found(&arn_text)); + } + // The contract says an AVAILABLE row becomes `Deleting`; a backend + // that answers its status instead has broken it, which is reported + // as the fault it is rather than as an absent backup. + DeleteStart::Status(BackupStatus::Available) => { + return Err(storage_err_to_dynamo(StorageError::Internal(format!( + "start_backup_delete of {arn_text} answered Status(AVAILABLE), which the \ + contract excludes" + )))); + } + } + if let Err(e) = finish_backup_delete(storage, &store.calls("deletion"), &fence, arn).await { + match e { + FinishError::Fenced(problem) => tracing::error!( + "DeleteBackup {arn_text}: the row is DELETING and no object was deleted (another \ + catalog adopted the root after the first fence); the deletion finishes if this \ + catalog owns the root again: {problem}" + ), + FinishError::Failed(e) => tracing::error!( + "DeleteBackup {arn_text}: the row is DELETING but its objects could not be \ + deleted; the reconciler retries: {e}" + ), + } + } + desc.backup_details.backup_status = BackupStatus::Deleted.as_str().to_owned(); + Ok(desc) +} + +/// Why [`finish_backup_delete`] left the row DELETING. +#[derive(Debug)] +pub(crate) enum FinishError { + /// The deletion fence refused before any object was deleted + /// ([`StoreFence`]): the store is not healthy, or its identity file + /// names another catalog or could not be read. + Fenced(StoreProblem), + /// A store or catalog call failed, or the prefix still lists an object. + Failed(String), +} + +impl std::fmt::Display for FinishError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Fenced(problem) => write!(f, "the deletion was refused: {problem}"), + Self::Failed(e) => f.write_str(e), + } + } +} + +/// The deletion of a DELETING row's objects and the row's move to DELETED: +/// the deletion fence (`fence`, which re-reads `extenddb-store.json` and +/// refuses when the root is no longer this catalog's), then +/// `extenddb-manifest.json` (a `delete_prefix` on its exact key, so the +/// prefix is no longer a complete backup to any reader), then the rest of the +/// prefix, then, once a listing of the prefix returns nothing, +/// `set_backup_deleted`. Shared by DeleteBackup and the reconciler's duty for +/// a DELETING row; either may run it on a row the other started, and a run +/// that finds the row no longer DELETING (`false` from the catalog) changes +/// nothing more. Returns whether this run moved the row. +/// +/// # Errors +/// +/// The fence's refusal, with nothing deleted; a store failure, a listing +/// that still shows objects (a put that overlapped the deletion, which the +/// next run removes), or a catalog write failure. The row stays DELETING +/// for the next run. +pub(crate) async fn finish_backup_delete( + storage: &S, + store: &dyn extenddb_backup_store::BackupStore, + fence: &StoreFence, + arn: &BackupArn, +) -> Result { + let prefix = arn.store_prefix(); + let manifest_key = format!( + "{prefix}{}", + extenddb_core::backup_format::EXTENDDB_MANIFEST_FILE + ); + fence + .check_catalog(storage) + .await + .map_err(FinishError::Fenced)?; + let failed = + |what: &str, e: &dyn std::fmt::Display| FinishError::Failed(format!("{what}: {e}")); + store + .delete_prefix(&manifest_key) + .await + .map_err(|e| failed("deleting the manifest", &e))?; + store + .delete_prefix(&prefix) + .await + .map_err(|e| failed("deleting the prefix", &e))?; + let remaining = store + .list(&prefix) + .try_next() + .await + .map_err(|e| failed("listing the prefix after the deletion", &e))?; + if let Some(object) = remaining { + return Err(FinishError::Failed(format!( + "the prefix still lists {} after the deletion", + object.key + ))); + } + storage + .set_backup_deleted(&BackupKey::from_arn(arn)) + .await + .map_err(|e| failed("marking the row DELETED", &e)) +} + /// Overrides refused outright. `VectorIndexOverride` is not in the botocore /// 1.33.13 model; newer models and SDKs send it. The index overrides and /// `OnDemandThroughputOverride` are applied as the service applies them; see @@ -969,6 +1542,7 @@ pub(crate) async fn handle_restore_table_from_backup( let auth_cache = ctx.auth_cache.clone(); restore_table_from_backup_detached( Arc::clone(&ctx.storage), + Arc::clone(&ctx.backup), Arc::clone(&ctx.account_id), Arc::clone(&ctx.region), Arc::clone(&ctx.limits), @@ -1001,6 +1575,7 @@ pub(crate) async fn handle_restore_table_from_backup( /// the request before the backend registered the restore, and no restore ran. async fn restore_table_from_backup_detached( storage: Arc, + runtime: Arc, account_id: Arc, region: Arc, limits: Arc, @@ -1008,13 +1583,22 @@ async fn restore_table_from_backup_detached( after: F, ) -> Result where - S: BackupEngine + ?Sized + Send + Sync + 'static, + S: BackupCatalog + + LegacyBackupReader + + TableEngine + + DataEngine + + RestoreEngine + + ?Sized + + Send + + Sync + + 'static, F: FnOnce(Arc, String) -> Fut + Send + 'static, Fut: std::future::Future + Send, { tokio::spawn(async move { let (target_table_name, response) = - restore_table_from_backup(&*storage, &account_id, ®ion, &limits, &body).await?; + restore_table_from_backup(&storage, &runtime, &account_id, ®ion, &limits, &body) + .await?; after(account_id, target_table_name).await; Ok(response) }) @@ -1029,22 +1613,65 @@ where /// order: (1) parse the overrides and check their shape, with no storage /// read; (2) check `BackupArn`: missing or malformed is a /// `ValidationException`, another account's is an `AccessDeniedException`; -/// (3) check `TargetTableName`; (4) read the backup's restore source; -/// (5) refuse what no backend restores faithfully (multi-part key, vector -/// indexes); (6) check the throughput override against the backup's billing -/// mode, build the target's `CreateTableInput`, and apply the index and -/// on-demand throughput overrides to it, refusing what the service refuses -/// ([`overrides`]); (7) validate that input as CreateTable does; (8) ask the -/// backend to create and register the target; (9) map its errors, a taken -/// name being `TableAlreadyExistsException`, a name being restored into -/// `TableInUseException`, a backup being restored from `BackupInUseException`. -async fn restore_table_from_backup( - storage: &S, +/// (3) check `TargetTableName`'s syntax; (4) read the backup: the catalog +/// index's row first ([`check_catalog_row_status`]: a CREATING row is +/// `BackupInUseException`, a DELETING, DELETED or FAILED row +/// `BackupNotFoundException`), else the legacy reader's restore source, a +/// missing backup being `BackupNotFoundException` whatever the target +/// name's state; (5) check the target name under the account through +/// `describe_table` ([`check_restore_target_name`]: a table of that name is +/// `TableAlreadyExistsException`, a CREATING restore target +/// `TableInUseException` naming its source backup), after the backup is +/// known to exist and before its in-use rule and the overrides are checked +/// against it, the order the service uses (measured on DynamoDB: a restore +/// of a missing backup into an existing table, into a table CreateTable is +/// still creating and into a CREATING restore target all answer +/// `BackupNotFoundException`; an in-use backup into an existing table +/// answers `TableAlreadyExistsException`, and a restore of a backup into +/// its own CREATING target `TableInUseException`); then, for a row, the +/// rest of its checks ([`check_catalog_row_for_restore`]: a backup in use is +/// `BackupInUseException`, a row of another store a `ValidationException`, +/// a row of the configured store while it is unhealthy one naming the +/// store's kind), and an AVAILABLE row of the healthy configured store is +/// restored from the store by the restore job ([`restore::request`]: the +/// permit (when none is freed within the runtime's wait, +/// `LimitExceededException`), the manifest and its checks, the vector +/// refusal of a target without vector support, the overrides, the detached +/// registration whose answer the request awaits; the maintenance claim is +/// checked by the job once its record exists, the design's 5.3 step 4); the +/// steps below are the legacy path's: (6) refuse what no backend restores +/// faithfully from an in-catalog backup (multi-part key, vector indexes); +/// (7) check the throughput override against the backup's billing mode, +/// build the target's `CreateTableInput`, and apply the index and on-demand +/// throughput overrides to it, refusing what the service refuses +/// ([`overrides`]); (8) validate that input as CreateTable does; (9) refuse +/// while a maintenance command holds the catalog's claim +/// ([`refuse_while_maintenance_held`]), then take the job permit, last among +/// the checks (`LimitExceededException` for either; the permit when none is +/// freed within the runtime's wait), and ask the backend to create and +/// register the target; (10) map its errors, a taken name being +/// `TableAlreadyExistsException`, a name being restored into +/// `TableInUseException` (the backend checks the name again, fenced, under +/// the registration), a backup being restored from `BackupInUseException`. +async fn restore_table_from_backup( + storage: &Arc, + runtime: &BackupRuntime, account_id: &str, region: &str, limits: &LimitsConfig, body: &Value, -) -> Result<(String, Value), DynamoDbError> { +) -> Result<(String, Value), DynamoDbError> +where + S: BackupCatalog + + LegacyBackupReader + + TableEngine + + DataEngine + + RestoreEngine + + ?Sized + + Send + + Sync + + 'static, +{ // 1. The overrides, parsed and shape-checked: no storage I/O, so a // malformed request is refused before a missing backup can take // precedence over the request's own error. @@ -1056,50 +1683,174 @@ async fn restore_table_from_backup( // first (measured on DynamoDB with two invalid inputs). let backup_arn = backup_arn_field(body, account_id, region)?; - // 3. + // 3. The name's syntax, before the backup is read. let target_table_name = required_str(body, "TargetTableName", "targetTableName")?; extenddb_core::validation::validate_table_name(target_table_name, limits)?; - // 4. Under the ARN spelling that resolves; the restore is registered - // against that spelling, the one the backend stored. + // 4. The catalog index first: a row of the healthy configured store is + // restored from the store. With no row, the legacy reader under the ARN + // spelling that resolves; the restore is registered against that + // spelling, the one the backend stored. + if let Some((arn, row)) = read_catalog_row(&**storage, &backup_arn) + .await + .map_err(storage_err_to_dynamo)? + { + check_catalog_row_status(&arn, &row)?; + // 5, for a row: the target name once the backup is known to exist, + // before the backup's in-use rule (measured on DynamoDB: an in-use + // backup into an existing table is TableAlreadyExistsException, a + // backup into its own CREATING target TableInUseException, not + // BackupInUseException) and before the manifest is read. + check_restore_target_name(&**storage, account_id, target_table_name).await?; + let store = check_catalog_row_for_restore(&**storage, runtime, &arn, &row).await?; + return restore::request( + storage, + runtime, + store, + account_id, + limits, + target_table_name, + &arn, + &overrides, + &index_overrides, + ) + .await; + } + let storage = &**storage; let (backup_arn, source) = - resolve_backup(&backup_arn, by!(storage, account_id, backup_restore_source)) + resolve_backup(&backup_arn, by!(storage, account_id, restore_source)) .await .map_err(storage_err_to_dynamo)?; - // 5. Refusals every backend gets, whatever it stores. + // 5. The target name, after the backup resolved and before the overrides + // are checked against it. + check_restore_target_name(storage, account_id, target_table_name).await?; + + // 6. Refusals every backend gets, whatever it stores. source.ensure_restorable().map_err(storage_err_to_dynamo)?; - // 6. + // 7. validate_override_for_source(&overrides, &source.billing_mode)?; let mut input = source.restore_input(target_table_name, &overrides); index_overrides.apply(&mut input, overrides.billing_mode)?; - // 7. A stored definition the service would refuse at CreateTable is + // 8. A stored definition the service would refuse at CreateTable is // refused here too (the service re-validates on restore): a backup row // is data, and data can be wrong. Only CreateTable's rules apply, so a // table CreateTable or UpdateTable accepted restores. extenddb_core::validation::validate_create_table(&input, limits)?; let on_demand = input.billing_mode == Some(BillingMode::PayPerRequest); - // 8. + // 9. The maintenance claim, then the permit last: every refusal above + // needs no slot, and the request holds nothing while it waits for one. let request = RestoreRequest { backup_arn: backup_arn.clone(), source_table_arn: source.source_table_arn.clone(), restore_date_time: source.backup_creation_date_time, input, }; + refuse_while_maintenance_held(storage, runtime).await?; + let permit = runtime.take_permit().await?; let desc = storage - .restore_table_from_backup(account_id, request) + .restore(account_id, request, permit) .await - // 9. + // 10. .map_err(restore_storage_err_to_dynamo)?; - let response = restore_response(desc, &source, on_demand)?; + let response = restore_response(desc, &RestoreSource::from(&source), on_demand)?; Ok((target_table_name.to_owned(), response)) } -/// The RestoreTableFromBackup response body for the backend's description. +/// Step 4 of a restore for a backup the catalog index has a row for, the +/// row's own state, before the target name is read: a CREATING row is +/// `BackupInUseException` ("Backup is being created"); a DELETING, DELETED +/// or FAILED row is `BackupNotFoundException`; an AVAILABLE row passes. +fn check_catalog_row_status(arn: &BackupArn, row: &BackupRow) -> Result<(), DynamoDbError> { + let arn_text = arn.to_arn_string(); + match row.status { + BackupStatus::Creating => Err(backup_in_use(&arn_text, None)), + BackupStatus::Deleting | BackupStatus::Deleted | BackupStatus::Failed => { + Err(backup_not_found(&arn_text)) + } + BackupStatus::Available => Ok(()), + } +} + +/// Step 4 of a restore for an AVAILABLE row ([`check_catalog_row_status`]), +/// after the target name and before the permit, reading no store: a backup +/// a CREATING restore target still reads is `BackupInUseException` naming +/// that table in the log ([`BackupCatalog::backup_in_use`]); a row of a +/// store other than the configured one is a `ValidationException` naming no +/// root, and a row of the configured store while that store is unhealthy or +/// its first check pending is a `ValidationException` naming the store's +/// kind ([`BackupRuntime::require_healthy_store`]). An AVAILABLE row of the +/// healthy configured store passes, and that store is returned: the restore +/// reads the backup from it ([`restore::request`]). +async fn check_catalog_row_for_restore<'r, S: BackupCatalog + ?Sized>( + storage: &S, + runtime: &'r BackupRuntime, + arn: &BackupArn, + row: &BackupRow, +) -> Result<&'r Arc, DynamoDbError> { + let arn_text = arn.to_arn_string(); + if let Some(table) = storage + .backup_in_use(&BackupKey::from_arn(arn)) + .await + .map_err(storage_err_to_dynamo)? + { + return Err(backup_in_use(&arn_text, Some(&table))); + } + if !runtime.is_configured_store(&row.store) { + return Err(backup_in_another_store(&arn_text)); + } + runtime.require_healthy_store() +} + +/// Step 5 of a restore, the target name under the account, read through +/// `describe_table` after the backup resolved and before the permit: a +/// CREATING table with a +/// `RestoreSummary` is a restore target, `TableInUseException` naming its +/// source backup (the text the backend's own registration answers with, +/// [`StorageError::TableBeingRestored`]); any other table of that name is +/// `TableAlreadyExistsException`; no table passes. Unfenced: the backend +/// checks the name again under its registration, so two restores into one +/// free name still have one winner. +async fn check_restore_target_name( + storage: &S, + account_id: &str, + target_table_name: &str, +) -> Result<(), DynamoDbError> { + let probe = storage + .describe_table( + account_id, + extenddb_core::types::DescribeTableInput { + table_name: target_table_name.to_owned(), + }, + ) + .await; + match probe { + Ok(desc) => { + let source = desc + .restore_summary + .as_ref() + .filter(|_| desc.table_status == TableStatus::Creating) + .and_then(|s| s.source_backup_arn.clone()); + Err(restore_storage_err_to_dynamo(match source { + Some(backup_arn) => StorageError::TableBeingRestored { + table_name: target_table_name.to_owned(), + backup_arn, + }, + None => StorageError::TableAlreadyExists(target_table_name.to_owned()), + })) + } + Err(StorageError::TableNotFound(_)) => Ok(()), + Err(e) => Err(storage_err_to_dynamo(e)), + } +} + +/// The RestoreTableFromBackup response body for the backend's description, +/// on both paths (`source` is the legacy reader's source or the manifest's +/// three members). /// /// The service returns the target CREATING with `RestoreSummary` /// { `SourceBackupArn`, `SourceTableArn`, `RestoreDateTime` = the backup's @@ -1110,7 +1861,7 @@ async fn restore_table_from_backup( /// ground-truth runs of 2026-08-24). fn restore_response( mut desc: TableDescription, - source: &BackupRestoreSource, + source: &RestoreSource, on_demand: bool, ) -> Result { desc.table_status = TableStatus::Creating; @@ -1133,9 +1884,9 @@ fn restore_response( } // The readiness invariant is applied on every path that hands a - // description to a client. A restored table has no vector indexes (the - // restore refuses them), so this holds trivially, but the invariant claims - // to cover every such path. + // description to a client: a legacy restore's target has no vector + // indexes (the reader refuses them); a store-backed restore's may, and + // the backend reports them as its CreateTable does. desc.validate_vector_index_readiness()?; // Same rules as every other path that emits a table description; the // target is CREATING, so the summary set above is kept. @@ -1206,7 +1957,7 @@ fn storage_err_to_dynamo(e: StorageError) -> DynamoDbError { // A missing (or deleted) backup surfaces from the backend as a // Validation error carrying "Backup not found"; DynamoDB reports // this as BackupNotFoundException, not ResourceNotFoundException. - if msg.contains("Backup not found") { + if msg.contains(extenddb_storage::backup_catalog::BACKUP_NOT_FOUND) { DynamoDbError::BackupNotFoundException(msg) } else { DynamoDbError::ValidationException(msg) @@ -1227,8 +1978,15 @@ fn storage_err_to_dynamo(e: StorageError) -> DynamoDbError { } } +mod job; +pub mod maintenance; mod overrides; mod pitr; +pub mod reconciler; +mod restore; +mod runtime; +pub mod store_identity; +pub mod sync; #[cfg(test)] mod tests; diff --git a/crates/engine/src/backup/job.rs b/crates/engine/src/backup/job.rs new file mode 100644 index 000000000..7c01e1081 --- /dev/null +++ b/crates/engine/src/backup/job.rs @@ -0,0 +1,763 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The store-backed CreateBackup job: the detached task the request spawns +//! once its checks have passed and the permit is held. +//! +//! The task ([`start`]) opens the table snapshot, mints the backup id from +//! the millisecond the open returned, reads the table's TTL, builds the +//! manifest's `source_table` from the snapshot's description +//! ([`source_table_schema`]), runs the deletion fence (a root another +//! catalog adopted since the store's last check is refused before any +//! write, `StoreFence`), inserts the CREATING row under the job's claim +//! (`claim_backup`), answers the request through a one-shot channel +//! (`BackupDetails` CREATING, or the refusal) and then goes on as the job +//! ([`run`]) with the same owned values: nothing crosses a thread or a +//! channel between the request and the job but the response (the +//! pipeline's item and chunk channels are the job's own), and a client that +//! disconnects mid-request cannot leave a row whose owner was never +//! spawned. The maintenance claim is checked by the request before the +//! permit, as the in-catalog paths check it, so a held claim refuses the +//! request and inserts no row (a FAILED row written for the refusal was +//! counted by `backup purge`'s wait as a live job for the stale window). +//! The open and the insert are selected against the shutdown token, so a +//! server that is stopping neither waits on an open the table's lock holds +//! nor inserts a row it would fail at once. The `ServiceUnavailable` that +//! refusal answers reaches a client only when the token is cancelled while +//! the listener still serves, which is a registration loss (the restartable +//! worker set stopping calls `BackupRuntime::cancel_jobs`); on SIGTERM the +//! listener's 5 s connection drain closes the connection before +//! `drain_then_release` cancels the token (the maintainer measured a +//! `CreateBackup` waiting in the open: its connection closed 8.2 s after it +//! was sent), so there the refusal's effect is the row not written. +//! +//! The job: the data files through the pipeline ([`writer`]), then the +//! three documents `BackupDocuments::build` returns in its order, a re-check +//! of the claim, the manifest (the commit record), and the AVAILABLE write +//! with the size, the count and the manifest's `source_table` +//! ([`failure::write_status`] gives every status write one meaning; an +//! AVAILABLE write whose outcome is unknown is re-read, one that matches no +//! row ends the job with no store access). Before every put the job checks +//! the store as the runtime holds it: healthy, and its identity the one the +//! row records ([`writer::Pipeline::store_check`]); a store the reconciler +//! has marked unhealthy, or whose identity file now names another store (a +//! root swapped under a running job), fails the row with the store's reason +//! and makes no store call, the put or the prefix's deletion included (the +//! FAILED-prefix duty deletes the prefix once the store is healthy again). +//! The heartbeat runs on a timer for the job's life ([`heartbeat`]); a +//! heartbeat that does not confirm the claim, or does not return within one +//! period, ends the job with no further store or catalog access. Any other +//! failure, and the shutdown token (reason `shutdown`), runs the failure +//! path ([`failure::fail_row`]): FAILED, then the prefix's deletion only +//! when that write matched and the fence passed. +//! +//! Each job records `BackupJobs` and `BackupJobDuration` (microseconds, +//! from the snapshot's open to the end of its last catalog or store call: +//! the AVAILABLE write, or the FAILED write and the prefix's deletion) +//! under its outcome, `BackupBytesUploaded` once it ended AVAILABLE, +//! `BackupSnapshotAge` at each heartbeat, and the worker metrics under +//! `QuerySource::BackupJob` (`WorkerLastSuccess` on success, +//! `WorkerErrorCount` on a failure; a job the shutdown cancelled is neither); +//! one info line at start (ARN, the table's item count as the stats pass +//! last counted it, the order) and at the end (ARN, the items written, +//! bytes, duration), one error line per failure. +//! +//! Kill points ([`kill_point`]) pause the job for the binary-level crash +//! tests under the `backup-test-hooks` feature and are no-ops without it. +//! The restore job ([`super::restore`]) shares this module's environment +//! ([`JobEnv`]), its kill points, its heartbeat and its status writes. + +pub(crate) mod failure; +pub(crate) mod heartbeat; +pub(crate) mod writer; + +use std::sync::Arc; + +use extenddb_backup_store::{BackupStore, byte_stream_from}; +use extenddb_core::backup_format::{ + BackupArn, BackupDocuments, ItemOrder, KeyOrder, KeyRangeTracker, ManifestHeader, + SourceTableSchema, backup_creation_date_time, +}; +use extenddb_core::error::DynamoDbError; +use extenddb_core::metrics::{BackupJobOutcome, MetricsCollector, QuerySource}; +use extenddb_core::types::BackupDetails; +use extenddb_core::validation::{RestorableKeySchema, RestorableLimits}; +use extenddb_storage::backup_catalog::{ + BACKUP_TYPE_USER, BackupRow, BackupStatus, JobStatus, StatusFields, StoreIdentity, + UNOWNED_GENERATION, +}; +use extenddb_storage::error::StorageError; +use extenddb_storage::snapshot::source_table_schema; +use extenddb_storage::{ + BackupCatalog, CancellationToken, JobClaim, JobPermit, MetadataEngine, SnapshotEngine, + TableSnapshot, +}; +use tokio::sync::oneshot; +use tokio::time::Instant; + +use super::runtime::{BackupRuntime, ConfiguredStore, JobSettings, Spool, StoreFence}; +use super::{create_backup_storage_err_to_dynamo, storage_err_to_dynamo}; +use failure::{FailurePath, Written}; +use heartbeat::{HeartbeatHandle, HeartbeatMetric}; +use writer::{DataFiles, Pipeline, Stop}; + +/// The reason a restore record carries when the maintenance claim was held +/// as the job started (the design's 5.3 step 4; the backup job checks the +/// claim before its row exists and writes no such row). +pub(crate) const MAINTENANCE: &str = "maintenance"; + +/// The reason a row carries when the server's shutdown stopped the job. +pub(crate) const SHUTDOWN: &str = "shutdown"; + +/// What the runtime hands a job: the store as the runtime holds it (its +/// health and identity, read before every put), its identity as the row +/// records it and its manifest names, the restore spool, the settings, the +/// collector, the shutdown token and the source names. +#[derive(Clone)] +pub(crate) struct JobEnv { + pub configured: Arc, + pub store: Arc, + /// The store's deletion fence, which CreateBackup runs before its first + /// put and the failure path before its deletion. + pub fence: StoreFence, + pub identity: StoreIdentity, + pub bucket: String, + pub key_prefix: String, + /// The restore spool of an S3 store; `None` for a filesystem store, + /// whose files a restore reads in place. + pub spool: Option, + pub settings: JobSettings, + pub metrics: Option>, + pub shutdown: CancellationToken, + pub source_backend: String, + pub source_extenddb_version: String, +} + +impl JobEnv { + /// From the runtime and its configured store, whose identity must be + /// known (it is once a check has passed, which `require_healthy_store` + /// has established). + pub(crate) fn new( + runtime: &BackupRuntime, + store: &Arc, + ) -> Result { + let identity = store.identity().ok_or_else(|| { + tracing::error!("the configured store is healthy but its id is unknown"); + DynamoDbError::InternalServerError("Internal server error".to_owned()) + })?; + let (source_backend, source_extenddb_version) = runtime.source(); + Ok(Self { + configured: Arc::clone(store), + store: Arc::clone(&store.store), + fence: store.fence(), + identity, + bucket: store.bucket.clone(), + key_prefix: store.key_prefix.clone(), + spool: store.spool.clone(), + settings: runtime.job_settings(), + metrics: runtime.metrics().cloned(), + shutdown: runtime.shutdown(), + source_backend: source_backend.to_owned(), + source_extenddb_version: source_extenddb_version.to_owned(), + }) + } +} + +/// The request's members the task needs. +pub(crate) struct CreateRequest { + pub account_id: String, + pub region: String, + pub table_name: String, + pub backup_name: String, + /// The instance's limits the validator holds every item to. + pub limits: RestorableLimits, +} + +/// The one-shot the task answers the request through. +pub(crate) type Reply = oneshot::Sender>; + +/// The detached task (module doc): everything from the open to the +/// response, then the job. +pub(crate) async fn start( + storage: Arc, + env: JobEnv, + permit: JobPermit, + request: CreateRequest, + reply: Reply, +) where + S: BackupCatalog + SnapshotEngine + MetadataEngine + ?Sized + Send + Sync + 'static, +{ + let job = match open_and_claim(&*storage, &env, &request).await { + Ok(job) => { + let details = job.details(); + // A request whose client went away still has its job. + let _ = reply.send(Ok(details)); + *job + } + Err(e) => { + drop(permit); + let _ = reply.send(Err(e)); + return; + } + }; + run(job).await; + drop(permit); +} + +/// The job once its row exists. +pub(crate) struct Job { + claim: Arc, + snapshot: TableSnapshot, + arn: BackupArn, + backup_name: String, + source_table: SourceTableSchema, + key_schema: RestorableKeySchema, + /// The table's key order when the stream is key-ordered. + key_order: Option, + limits: RestorableLimits, + env: JobEnv, + /// This instance's catalog id, which the failure path's fence compares + /// the identity file with. + catalog_id: String, + /// The id's instant, the row's `created_at`. + created_at: f64, + started: Instant, + /// The instant taken before the claim's insert, the heartbeat's first + /// confirmed beat. + claimed_at: Instant, +} + +impl Job { + /// The response the request answers: the row as inserted, CREATING. + fn details(&self) -> BackupDetails { + BackupDetails { + backup_arn: self.arn.to_arn_string(), + backup_name: self.backup_name.clone(), + backup_status: BackupStatus::Creating.as_str().to_owned(), + backup_type: BACKUP_TYPE_USER.to_owned(), + backup_size_bytes: 0, + backup_creation_date_time: self.created_at, + } + } +} + +/// The open, the id, the TTL read, the row and the claim (module doc). The +/// open and the insert are selected against the shutdown token. +async fn open_and_claim( + storage: &S, + env: &JobEnv, + request: &CreateRequest, +) -> Result, DynamoDbError> +where + S: BackupCatalog + SnapshotEngine + MetadataEngine + ?Sized, +{ + let CreateRequest { + account_id, + region, + table_name, + backup_name, + limits, + } = request; + let started = Instant::now(); + // The open and the claim's insert are selected against the shutdown + // token: an open that waits on a lock the table holds (an `ACCESS + // EXCLUSIVE` on PostgreSQL) does not hold the drain for its whole bound, + // and no row is inserted for a server that is stopping. The answer + // reaches a client only after a registration loss (module doc): on + // SIGTERM the listener's drain has closed the connection by the time the + // token is cancelled. + let snapshot = tokio::select! { + biased; + () = env.shutdown.cancelled() => return Err(shutting_down(table_name)), + opened = storage.open_table_snapshot(account_id, table_name) => { + opened.map_err(|e| create_backup_storage_err_to_dynamo(e, table_name))? + } + }; + // The id carries the millisecond the open returned. + let backup_id = extenddb_storage::util::new_backup_id(); + let created_at = backup_creation_date_time(&backup_id) + .ok_or_else(|| internal("the backup id carries no instant"))?; + let arn = BackupArn::new( + extenddb_storage::util::ARN_PARTITION, + region, + account_id, + table_name, + &backup_id, + ) + .map_err(|e| internal(&format!("no backup ARN for {table_name}: {e}")))?; + let ttl = storage + .describe_ttl(account_id, table_name) + .await + .map_err(storage_err_to_dynamo)?; + let source_table = + source_table_schema(&snapshot.description, Some(ttl)).map_err(storage_err_to_dynamo)?; + let key_schema = RestorableKeySchema::resolve( + &source_table.key_schema, + &source_table.attribute_definitions, + ) + .map_err(|e| { + internal(&format!( + "the key schema of {table_name} cannot be checked: {e}" + )) + })?; + let key_order = snapshot + .key_ordered + .then(|| { + KeyOrder::resolve( + &source_table.key_schema, + &source_table.attribute_definitions, + ) + .ok() + }) + .flatten(); + let row = BackupRow { + backup_id: backup_id.clone(), + account_id: account_id.clone(), + table_name: table_name.clone(), + backup_arn: arn.to_arn_string(), + backup_name: backup_name.clone(), + status: BackupStatus::Creating, + store: env.identity.clone(), + size_bytes: 0, + item_count: 0, + created_at, + completed_at: None, + failure_reason: None, + source_table_details_json: serde_json::to_value(&source_table) + .map_err(|e| internal(&format!("the source table does not serialize: {e}")))?, + owner_generation: UNOWNED_GENERATION, + heartbeat_at: 0.0, + }; + // The fence before the row and the first put: a root another catalog + // adopted since the store's last check gets no prefix from this + // catalog, and the store is marked unhealthy. + let catalog_id = env + .fence + .catalog_id(storage) + .await + .map_err(|e| internal(&format!("the catalog id could not be read: {e}")))?; + if let Err(problem) = env.fence.check(&catalog_id).await { + return Err(env.fence.refusal().unwrap_or_else(|| { + internal(&format!( + "{table_name}: the store's identity file could not be read before the first \ + write: {problem}" + )) + })); + } + let claimed_at = Instant::now(); + // The insert too is selected against the token: a server that is + // stopping inserts no row it would have to fail at once. + let claimed = tokio::select! { + biased; + () = env.shutdown.cancelled() => return Err(shutting_down(table_name)), + claimed = storage.claim_backup(&row) => claimed, + }; + let claim: Arc = match claimed { + Ok(claim) => Arc::from(claim), + Err(StorageError::AlreadyExists(what)) => { + return Err(internal(&format!( + "backup id {backup_id} is already claimed: {what}" + ))); + } + Err(e) => return Err(storage_err_to_dynamo(e)), + }; + Ok(Box::new(Job { + claim, + snapshot, + arn, + backup_name: backup_name.clone(), + source_table, + key_schema, + key_order, + limits: *limits, + env: env.clone(), + catalog_id, + created_at, + started, + claimed_at, + })) +} + +fn internal(why: &str) -> DynamoDbError { + tracing::error!("CreateBackup: {why}"); + DynamoDbError::InternalServerError("Internal server error".to_owned()) +} + +/// The refusal of a request whose server is stopping before its row exists +/// (module doc); the client sees it only when the token was cancelled while +/// the listener still served, after a registration loss. +fn shutting_down(table_name: &str) -> DynamoDbError { + tracing::info!("CreateBackup of {table_name}: refused, the server is shutting down"); + DynamoDbError::ServiceUnavailable("The server is shutting down. Retry the request.".to_owned()) +} + +/// The FAILED write of a row whose job makes no store call: nothing of the +/// job's is in the store, or the store is not to be called, so there is +/// nothing to delete here. +async fn fail_without_store_access(claim: &dyn JobClaim, arn: &BackupArn, reason: &str) { + let fields = StatusFields { + failure_reason: Some(reason.to_owned()), + completed: true, + ..StatusFields::default() + }; + match failure::write_status(claim, JobStatus::Creating, JobStatus::Failed, &fields).await { + Written::Matched => {} + Written::NotMatched(why) | Written::Error(why) => tracing::error!( + "backup {}: the row could not be marked FAILED ({reason}): {why}", + arn.to_arn_string() + ), + } +} + +/// The job (module doc), from the data files to the final status write. +pub(crate) async fn run(job: Job) -> BackupJobOutcome { + let Job { + claim, + snapshot, + arn, + backup_name, + mut source_table, + key_schema, + key_order, + limits, + env, + catalog_id, + created_at, + started, + claimed_at, + } = job; + let arn_text = arn.to_arn_string(); + let prefix = arn.store_prefix(); + let metrics = env.metrics.as_deref(); + let heartbeat = HeartbeatHandle::start( + Arc::clone(&claim), + env.settings.heartbeat_interval, + match env.metrics.clone() { + Some(metrics) => HeartbeatMetric::BackupSnapshotAge { + metrics, + created_at, + }, + None => HeartbeatMetric::None, + }, + claimed_at, + ); + tracing::info!( + "backup {arn_text}: started, {} items in the table as last counted by the stats pass, {}", + snapshot.description.item_count, + if key_order.is_some() { + "key-ordered" + } else { + "unordered" + } + ); + let TableSnapshot { info, items, .. } = snapshot; + let pipeline = Pipeline { + store: &*env.store, + configured: &env.configured, + identity: &env.identity, + prefix: &prefix, + arn: &arn_text, + heartbeat: &heartbeat, + shutdown: &env.shutdown, + metrics, + target_bytes: env.settings.data_file_target_bytes, + key_prefix: &env.key_prefix, + bucket: &env.bucket, + }; + let tracker = key_order.as_ref().map(KeyRangeTracker::new); + let mut item_count = 0; + let end = match writer::write_data_files(&pipeline, items, key_schema, limits, tracker).await { + Ok(files) => { + item_count = files.item_count; + // Both figures are the stream's: the items written and their + // DynamoDB sizes summed, not the statistics pass's (which can + // be stale, or 0 on a fresh table). + source_table.item_count = i64::try_from(files.item_count).unwrap_or(i64::MAX); + source_table.table_size_bytes = i64::try_from(files.item_bytes).unwrap_or(i64::MAX); + let header = ManifestHeader { + backup_arn: arn.clone(), + backup_name, + source_backend: env.source_backend.clone(), + source_extenddb_version: env.source_extenddb_version.clone(), + snapshot: info, + item_order: if key_order.is_some() { + ItemOrder::KeyOrdered + } else { + ItemOrder::Unordered + }, + source_table, + }; + write_documents(&pipeline, &claim, header, files).await + } + Err(stop) => Err(stop), + }; + let (outcome, bytes) = match end { + Ok(bytes) => { + tracing::info!( + "backup {arn_text}: AVAILABLE, {item_count} items, {bytes} bytes written in {:.1} s", + started.elapsed().as_secs_f64() + ); + (BackupJobOutcome::Succeeded, bytes) + } + Err(Stop::ClaimLost(why)) => { + tracing::error!( + "backup {arn_text}: stopped with no further store or catalog access: {why}; the \ + row is left to its owner or to the reconciler" + ); + (BackupJobOutcome::Failed, 0) + } + Err(Stop::StoreUnhealthy(reason)) => { + // The store is not to be called: FAILED with the store's reason, + // the prefix left to the FAILED-prefix duty once the store is + // healthy again. + fail_without_store_access(&*claim, &arn, &reason).await; + tracing::error!( + "backup {arn_text}: FAILED ({reason}); no store call was made and its objects \ + are deleted by the reconciler once the store's check passes and the row's \ + heartbeat is stale" + ); + (BackupJobOutcome::Failed, 0) + } + Err(stop) => { + let (reason, outcome) = match stop { + Stop::Cancelled => (SHUTDOWN.to_owned(), BackupJobOutcome::Cancelled), + Stop::Fail(reason) => (reason, BackupJobOutcome::Failed), + Stop::ClaimLost(_) | Stop::StoreUnhealthy(_) => unreachable!("matched above"), + }; + match failure::fail_row( + &*claim, + &*env.store, + &env.fence, + &catalog_id, + &prefix, + &reason, + metrics, + ) + .await + { + FailurePath::Deleted => tracing::error!( + "backup {arn_text}: FAILED ({reason}); its objects were deleted" + ), + FailurePath::Fenced(why) => tracing::error!( + "backup {arn_text}: FAILED ({reason}); its objects were not deleted: {why}" + ), + FailurePath::DeleteFailed(e) => tracing::error!( + "backup {arn_text}: FAILED ({reason}); its objects could not be deleted and \ + the reconciler deletes them once the row's heartbeat is stale: {e}" + ), + FailurePath::NoStoreAccess(why) => tracing::error!( + "backup {arn_text}: failed ({reason}) and the row is not this job's to \ + write ({why}); no store access was made" + ), + } + (outcome, 0) + } + }; + // After the failure path too, so a FAILED job's duration covers its + // FAILED write and the prefix's deletion. + let duration = started.elapsed(); + drop(heartbeat); + if let Some(metrics) = metrics { + #[allow(clippy::cast_precision_loss)] + let duration_us = duration.as_micros() as f64; + metrics.record_backup_job(outcome, duration_us); + if bytes > 0 { + metrics.record_backup_bytes_uploaded(bytes); + } + match outcome { + BackupJobOutcome::Succeeded => { + metrics.record_worker_success(QuerySource::BackupJob, duration_us); + } + BackupJobOutcome::Failed => { + metrics.record_worker_error(QuerySource::BackupJob); + } + // A clean shutdown is not a worker error. + BackupJobOutcome::Cancelled => {} + } + } + outcome +} + +/// After the data files: the three documents, the claim re-check, the +/// manifest, the AVAILABLE write. Returns the bytes written over the whole +/// backup. +async fn write_documents( + p: &Pipeline<'_>, + claim: &Arc, + header: ManifestHeader, + files: DataFiles, +) -> Result { + let DataFiles { + entries, + item_count, + item_bytes: _, + mut bytes_uploaded, + } = files; + let docs = BackupDocuments::build(header, entries, p.key_prefix, p.bucket) + .map_err(|e| Stop::Fail(format!("building the manifest: {e}")))?; + for (name, bytes) in docs.documents_before_manifest() { + bytes_uploaded += put_document(p, name, bytes).await?; + } + // The re-check narrows the window in which a fenced-out job can still + // write a manifest; the bound is the backend's (`heartbeat_within`). + p.heartbeat + .recheck() + .await + .map_err(|why| Stop::ClaimLost(format!("the claim re-check before the manifest: {why}")))?; + kill_point(KillPoint::BeforeManifest, p.shutdown).await; + let (name, bytes) = docs.manifest_document(); + bytes_uploaded += put_document(p, name, bytes).await?; + kill_point(KillPoint::AfterManifest, p.shutdown).await; + let manifest = docs.manifest(); + let fields = StatusFields { + size_bytes: Some(manifest.backup_size_bytes), + item_count: Some(i64::try_from(item_count).unwrap_or(i64::MAX)), + failure_reason: None, + source_table_details_json: serde_json::to_value(&manifest.source_table).ok(), + completed: true, + }; + match failure::write_status(&**claim, JobStatus::Creating, JobStatus::Available, &fields).await + { + Written::Matched => Ok(bytes_uploaded), + Written::NotMatched(why) | Written::Error(why) => Err(Stop::ClaimLost(format!( + "the AVAILABLE write did not settle: {why}" + ))), + } +} + +/// One document's put under the backup prefix, after the pre-put check; +/// returns the bytes the store reported. +pub(crate) async fn put_document(p: &Pipeline<'_>, name: &str, bytes: &[u8]) -> Result { + if p.shutdown.is_cancelled() { + return Err(Stop::Cancelled); + } + p.store_check(name)?; + if !p.heartbeat.confirms_the_claim() { + return Err(Stop::ClaimLost(format!( + "no heartbeat confirmed the claim recently enough for {name}" + ))); + } + let key = format!("{}{name}", p.prefix); + tokio::select! { + biased; + () = p.shutdown.cancelled() => Err(Stop::Cancelled), + () = p.heartbeat.lost().cancelled() => Err(Stop::ClaimLost(p.heartbeat.reason())), + result = p.store.put(&key, byte_stream_from(bytes.to_vec())) => match result { + Ok(meta) => Ok(meta.size), + Err(e) => { + if let Some(metrics) = p.metrics { + metrics.record_backup_store_error("put"); + } + Err(Stop::Fail(format!("writing {name}: {e}"))) + } + }, + } +} + +/// The points at which a backup or restore job pauses under +/// `backup-test-hooks` when `EXTENDDB_BACKUP_PAUSE` names one, until the +/// file `EXTENDDB_BACKUP_PAUSE_RELEASE` names exists (or for good, for a +/// test that kills the server there). Each is logged as `job reached kill +/// point ` (the backend's own points log `restore job reached kill +/// point `; the harness matches `reached kill point `). This +/// enum is the engine's list: the points inside a backend's own writes are +/// paused by the backend through +/// `extenddb_storage::backup_catalog::restore_pause_point` under +/// `extenddb-storage/test-hooks`, and each backend's own point is named +/// beside `restore_pause_point`, once. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum KillPoint { + /// After the put of the first data file returned, before the next file + /// or the documents. + AfterDataFile, + /// After the three documents, before `extenddb-manifest.json`. + BeforeManifest, + /// After the manifest's put returned, before the AVAILABLE write. + AfterManifest, + /// Restore: after every data file downloaded and verified, before the + /// first `bulk_load`. + RestoreAfterDownload, + /// Restore: after the first file's `bulk_load` returned, before the + /// next file. + RestoreAfterFileCommit, + /// Restore: after a `finish_restore` that left the ACTIVE flip to the + /// backend's index backfill returned, before the job ends (MongoDB). + RestoreMongodbBeforeWorkerFlip, + /// Restore: after the FAILED write matched, before + /// `remove_restore_target`. + RestoreBetweenFailedAndRemoval, + /// Restore: before the DONE write, with the job's heartbeat stopped + /// first, so a reconciler takes the record over while the job pauses + /// and the job's write then matches no record. + RestoreFencedOut, +} + +impl KillPoint { + /// The value of `EXTENDDB_BACKUP_PAUSE` that names this point. + #[must_use] + pub fn name(self) -> &'static str { + match self { + Self::AfterDataFile => "after_data_file", + Self::BeforeManifest => "before_manifest", + Self::AfterManifest => "after_manifest", + Self::RestoreAfterDownload => "restore_after_download", + Self::RestoreAfterFileCommit => "restore_after_file_commit", + Self::RestoreMongodbBeforeWorkerFlip => "restore_mongodb_before_worker_flip", + Self::RestoreBetweenFailedAndRemoval => "restore_between_failed_and_removal", + Self::RestoreFencedOut => "restore_fenced_out", + } + } +} + +/// Whether the environment names `point` as the pause point (so a caller +/// can do what the pause needs first, as the fenced-out point stops the +/// heartbeat); `false` without `backup-test-hooks`. +pub(crate) fn kill_point_named(point: KillPoint) -> bool { + #[cfg(feature = "backup-test-hooks")] + { + std::env::var("EXTENDDB_BACKUP_PAUSE").ok().as_deref() == Some(point.name()) + } + #[cfg(not(feature = "backup-test-hooks"))] + { + let _ = point; + false + } +} + +/// Pause at `point` if the environment names it (type doc); a ready future +/// without `backup-test-hooks`. +#[cfg_attr(not(feature = "backup-test-hooks"), allow(clippy::unused_async))] +pub(crate) async fn kill_point(point: KillPoint, shutdown: &CancellationToken) { + #[cfg(feature = "backup-test-hooks")] + { + if std::env::var("EXTENDDB_BACKUP_PAUSE").ok().as_deref() != Some(point.name()) { + return; + } + let release = std::env::var("EXTENDDB_BACKUP_PAUSE_RELEASE").ok(); + tracing::warn!( + "job reached kill point {}; paused until {} exists", + point.name(), + release + .as_deref() + .unwrap_or("no release file (the pause does not end)") + ); + loop { + if let Some(release) = &release + && std::path::Path::new(release).exists() + { + tracing::warn!("job released from kill point {}", point.name()); + return; + } + if shutdown.is_cancelled() { + tracing::warn!( + "job at kill point {} released by the shutdown", + point.name() + ); + return; + } + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + } + } + #[cfg(not(feature = "backup-test-hooks"))] + let _ = (point, shutdown); +} diff --git a/crates/engine/src/backup/job/failure.rs b/crates/engine/src/backup/job/failure.rs new file mode 100644 index 000000000..27920fa62 --- /dev/null +++ b/crates/engine/src/backup/job/failure.rs @@ -0,0 +1,162 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The conditional status writes of a backup row and the job's failure +//! path. +//! +//! Every status write goes through [`write_status`]: the backend writes +//! conditioned on the claim's generation and on the record being in the +//! status the job holds it in (`from`: CREATING for a backup row, RUNNING +//! for a restore record), and this module gives the three outcomes one +//! meaning. `Matched` is the write. `NotMatched` means the record is no +//! longer this claim's (gone, another generation, or not in `from`) and the +//! caller makes no further store or table access of any kind. `Unknown` +//! (the connection failed after the statement was sent) is followed by a +//! re-read of the record, not treated as a failure: a record that shows the +//! write landed is `Matched`; one still in `from` under this generation did +//! not take the write, which is sent once more, at most; anything else is +//! `NotMatched`. Each unknown +//! outcome has its re-read, the second write's included, so a second +//! unknown outcome whose write landed is `Matched` (the row is in the +//! target status under this generation) and one whose write was lost is +//! `NotMatched`. An error on the claim connection, on the write or the +//! re-read, ends the caller's work the way `NotMatched` does. +//! +//! The second write is for a backend whose claim connection survives the +//! unknown outcome (MongoDB and Cassandra, whose claim writes go through +//! the shared client). On PostgreSQL an unknown outcome is an error after +//! which the claim session's one connection is dead, so the second write +//! fails with the session and its re-read settles it as `NotMatched`: the +//! job stops, the session's end frees the advisory lock, and the row is the +//! reconciler's (AVAILABLE from a complete manifest, FAILED otherwise). +//! +//! The backup job's failure path ([`fail_row`]) is the FAILED write +//! followed, only when it matched, by the deletion fence and then +//! `delete_prefix` of the backup's prefix; the holder of the claim owns the +//! prefix, so a job whose FAILED write did not match never deletes, and a +//! root another catalog adopted while the job ran is left as it is (the +//! fence marks the store unhealthy). A deletion that fails leaves the +//! prefix to the reconciler, which deletes a FAILED row's prefix once its +//! heartbeat is stale. The restore job's failure path, FAILED then the +//! target's removal under the same gate, is `super::super::restore::fail_restore`. + +use extenddb_backup_store::BackupStore; + +use super::super::StoreFence; +use extenddb_core::metrics::MetricsCollector; +use extenddb_storage::JobClaim; +use extenddb_storage::backup_catalog::{JobStatus, StatusFields, WriteOutcome}; + +/// What a conditional status write settled to (module doc). +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum Written { + /// The row was moved. + Matched, + /// The row is not this claim's to write; the text says what the row + /// showed. + NotMatched(String), + /// The claim connection failed on the write or the re-read. + Error(String), +} + +/// Move the record from `from` to `to` through `claim` (module doc). +pub(crate) async fn write_status( + claim: &dyn JobClaim, + from: JobStatus, + to: JobStatus, + fields: &StatusFields, +) -> Written { + for attempt in 0..2 { + let outcome = match claim.set_status(from, to, fields).await { + Ok(outcome) => outcome, + Err(e) => return Written::Error(format!("the {to} write failed: {e}")), + }; + match outcome { + WriteOutcome::Matched => return Written::Matched, + WriteOutcome::NotMatched => { + return Written::NotMatched(format!( + "the record is no longer {from} under this claim's generation" + )); + } + WriteOutcome::Unknown => {} + } + let row = match claim.read().await { + Ok(row) => row, + Err(e) => { + return Written::Error(format!( + "the {to} write's outcome is unknown and the row could not be re-read: {e}" + )); + } + }; + if row.owner_generation != claim.generation() { + return Written::NotMatched(format!( + "the row carries generation {} (this claim's is {})", + row.owner_generation, + claim.generation() + )); + } + if row.status == to { + return Written::Matched; + } + if row.status != from || attempt == 1 { + return Written::NotMatched(format!( + "the record is {} after a {to} write whose outcome was unknown", + row.status + )); + } + tracing::warn!( + "the {to} write's outcome was unknown and the record is still {from} under this \ + claim; writing once more" + ); + } + unreachable!("the loop returns on its second attempt") +} + +/// The end of the failure path. +#[derive(Debug)] +pub(crate) enum FailurePath { + /// FAILED written and the prefix deleted. + Deleted, + /// FAILED written; the deletion failed and is the reconciler's. + DeleteFailed(String), + /// FAILED written; the deletion fence refused, and nothing was + /// deleted. + Fenced(String), + /// FAILED not written by this claim (the row is another's, or the claim + /// connection failed): no store access was made. + NoStoreAccess(String), +} + +/// The failure path (module doc): FAILED with `reason`, then, when the +/// write matched, the fence for `catalog_id` and the deletion of `prefix`. +pub(crate) async fn fail_row( + claim: &dyn JobClaim, + store: &dyn BackupStore, + fence: &StoreFence, + catalog_id: &str, + prefix: &str, + reason: &str, + metrics: Option<&MetricsCollector>, +) -> FailurePath { + let fields = StatusFields { + failure_reason: Some(reason.to_owned()), + completed: true, + ..StatusFields::default() + }; + match write_status(claim, JobStatus::Creating, JobStatus::Failed, &fields).await { + Written::Matched => {} + Written::NotMatched(why) | Written::Error(why) => return FailurePath::NoStoreAccess(why), + } + if let Err(problem) = fence.check(catalog_id).await { + return FailurePath::Fenced(problem.to_string()); + } + match store.delete_prefix(prefix).await { + Ok(_) => FailurePath::Deleted, + Err(e) => { + if let Some(metrics) = metrics { + metrics.record_backup_store_error("delete_prefix"); + } + FailurePath::DeleteFailed(e.to_string()) + } + } +} diff --git a/crates/engine/src/backup/job/heartbeat.rs b/crates/engine/src/backup/job/heartbeat.rs new file mode 100644 index 000000000..41bdb72d5 --- /dev/null +++ b/crates/engine/src/backup/job/heartbeat.rs @@ -0,0 +1,355 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The job's heartbeat: a timer task that refreshes the claim every period +//! for as long as the job runs, independent of upload progress, and the +//! two things the job reads from it: the instant of the last confirmed beat +//! (checked before every put) and the signal that the claim is lost. +//! +//! One rule decides both: a beat that does not confirm the claim +//! ([`Heartbeat::Lost`] or an error) or that has not returned within one +//! period ends the timer and raises the lost signal, after which the job +//! makes no further store or catalog access. The pre-put check is the same +//! rule read the other way: the last confirmed beat is at most two periods +//! old (one for the timer to fire, one for the beat to return, the bound +//! the timer itself enforces), or a beat is in flight (the last paragraph), +//! and no loss has been raised. The claim's +//! insert stands for the first confirmed beat, and the two jobs hand its +//! instant in by their own convention: the backup job takes it before +//! `claim_backup` (the row's `heartbeat_at` is stamped inside the insert, so +//! the instant is at or before the beat and the maintenance check that runs +//! between the insert and the handle's start, one catalog query with no +//! bound of its own, counts against the two periods); the restore job takes +//! it after its first beat returned confirmed, a beat it sends itself right +//! after `claim_restore` returned (the record's `heartbeat_at` as the +//! registration stamped it can be old: on PostgreSQL `now()` is the +//! transaction's start, so a registration stalled on a lock commits a stale +//! beat; the first beat refreshes it on the catalog's clock), and after +//! `claim_restore_dead` returned for a resume (the takeover stamped +//! `heartbeat_at` inside that write). Either way the instant is at or after +//! the beat it stands for, by a write's own duration, and a slow +//! registration does not end the job at its first check. A job +//! whose first write comes within two periods of its instant needs no beat +//! of its own; `stale_job_after_secs` is at least three periods, so a row +//! that passes the check is never one a reconciler may take over. +//! +//! A process stopped and resumed (SIGSTOP, a paused VM) meets the same +//! rule. On PostgreSQL its claim session keeps the advisory lock while it +//! is stopped, so no takeover happens while the session lives. On resume a +//! job with no beat in flight at the stop runs on under its own generation +//! once its next beat succeeds, however long the stop was (a job frozen +//! 160 s finished AVAILABLE: the timer's next tick beats, the beat confirms, +//! and the puts that follow find a fresh confirmation); it stops as a lost +//! claim with no catalog write only when a beat in flight at the stop +//! outlives its one-period bound, or when a put comes before the next beat +//! and finds the last confirmed beat more than two periods old. In that +//! case its claim closes, the lock is freed and the reconciler resolves the +//! row (AVAILABLE from a complete manifest, FAILED otherwise) or resumes a +//! store-backed restore from its last committed batch. On MongoDB and +//! Cassandra the row may have been taken over during the stop: a put in +//! flight at the stop may still land (the reconciler's FAILED-prefix retry +//! deletes it), and the job's next beat and status write match no row. +//! +//! The one-period bound is the backend's to place +//! ([`JobClaim::heartbeat_within`]): on SQLite the beat is an `UPDATE` +//! under the engine write lock, and the bound starts once the beat holds +//! that lock, so a writer holding it past a period (a restore's write +//! batches, item writes, a GSI backfill, the TTL sweep, a table's drop) +//! delays the beat and does not end the job; the SQLite restore engine's +//! module doc gives the batch times the bench record measured. The re-check +//! before the manifest ([`HeartbeatHandle::recheck`]) is the same call, and +//! a confirmation it gets counts as a beat for the pre-put check. +//! +//! The pre-put check has a third clause for the same case: a beat that is +//! in flight and has not failed confirms the claim, however old the last +//! confirmed beat is. Without it a backup large enough to reach a put while +//! a writer holds the lock past two periods stopped at that put (`no +//! heartbeat confirmed the claim recently enough`), its beat parked on the +//! lock and the claim intact (the maintainer measured it twice at the third +//! or fourth data file of a 180,000-item table beside a GSI build). The +//! clause is safe because a beat that is in flight is one no backend has +//! answered: on SQLite the beat parks only on this process's own write +//! lock, one process serves the file (the serve lock; `backup purge` +//! refuses while a server runs on it) and the reconciler's liveness test is +//! the in-process slot the job holds (design 4(e)), so a parked beat cannot +//! coincide with a takeover, whatever `heartbeat_at` reads meanwhile; the +//! backend says so through [`JobClaim::heartbeat_parks_before_its_bound`], +//! and there the clause holds for as long as the beat waits. On PostgreSQL, +//! MongoDB and Cassandra no lock of this process sits in front of the beat +//! and `heartbeat_within` bounds the whole call, and there the clause holds +//! only within one period of the beat's start, the bound itself: a process +//! stopped mid-beat (SIGSTOP, a paused VM) and resumed keeps the beat in +//! flight until its timer runs again and raises the loss, a window of tens +//! of microseconds in which another owner may already hold the row, and a +//! clause that read the flight alone passed the checks in that window (the +//! maintainer measured 64 and 85 µs). A beat that returns Lost, fails or +//! outlives its bound raises the loss first, and the clause reads the loss +//! before the flight. A timer task that stops mid-beat (a panic in the +//! backend's beat) clears the flight on the way out, so the two-period +//! clause is what remains for it. + +use std::sync::{Arc, Mutex, PoisonError}; +use std::time::Duration; + +use extenddb_core::metrics::MetricsCollector; +use extenddb_storage::CancellationToken; +use extenddb_storage::JobClaim; +use extenddb_storage::backup_catalog::Heartbeat; +use extenddb_storage::error::StorageError; +use tokio::sync::watch; +use tokio::time::Instant; + +/// The job's side of the heartbeat timer. Dropping it stops the timer. +pub(crate) struct HeartbeatHandle { + claim: Arc, + confirmed: Arc>, + last_confirmed: watch::Receiver, + /// The instant the beat in flight (the timer's or the re-check's) + /// started, while one is; the pre-put check's third clause (module doc). + in_flight: Arc>>, + /// Whether the claim's beat can wait outside its bound + /// ([`JobClaim::heartbeat_parks_before_its_bound`]): the third clause + /// holds for as long as the beat is in flight where it can, and for one + /// period from the beat's start where it cannot. + parks: bool, + lost: CancellationToken, + reason: Arc>>, + period: Duration, + timer: tokio::task::JoinHandle<()>, +} + +/// Marks a beat in flight, with its start, for as long as it lives, and +/// clears the mark when dropped, by the beat's return or by its task's end. +struct InFlight<'a>(&'a Mutex>); + +impl<'a> InFlight<'a> { + fn mark(flag: &'a Mutex>) -> Self { + *flag.lock().unwrap_or_else(PoisonError::into_inner) = Some(Instant::now()); + Self(flag) + } +} + +impl Drop for InFlight<'_> { + fn drop(&mut self) { + *self.0.lock().unwrap_or_else(PoisonError::into_inner) = None; + } +} + +/// The third clause of the pre-put check (module doc): a beat in flight +/// since `since` confirms the claim for as long as it waits where the beat +/// parks before its bound, and within one `period` of its start otherwise. +fn in_flight_confirms(in_flight: &Mutex>, parks: bool, period: Duration) -> bool { + match *in_flight.lock().unwrap_or_else(PoisonError::into_inner) { + Some(since) => parks || since.elapsed() <= period, + None => false, + } +} + +/// Why a beat did not confirm the claim. +fn beat_failure(outcome: Option>, bound: Duration) -> String { + match outcome { + Some(Ok(Heartbeat::Confirmed)) => unreachable!("a confirmed beat is not a failure"), + Some(Ok(Heartbeat::Lost)) => { + "the claim was lost: the row is gone or carries another generation".to_owned() + } + Some(Err(e)) => format!("the heartbeat failed: {e}"), + None => format!("the heartbeat did not return within {} s", bound.as_secs()), + } +} + +/// What a confirmed beat records, if anything: a backup job records the +/// snapshot's age, a restore job records nothing per beat. +pub(crate) enum HeartbeatMetric { + /// `BackupSnapshotAge`, seconds since `created_at`, into `metrics`. + BackupSnapshotAge { + metrics: Arc, + created_at: f64, + }, + /// No per-beat metric. + None, +} + +impl HeartbeatHandle { + /// Start the timer on `claim`, beating every `period` from one period + /// after now; `claimed_at`, the instant of the claim's insert as the + /// job's convention takes it (module doc), stands for the first + /// confirmed beat. Each confirmed beat records `metric`. + pub(crate) fn start( + claim: Arc, + period: Duration, + metric: HeartbeatMetric, + claimed_at: Instant, + ) -> Self { + let (tx, last_confirmed) = watch::channel(claimed_at); + let confirmed = Arc::new(tx); + let in_flight = Arc::new(Mutex::new(None)); + let parks = claim.heartbeat_parks_before_its_bound(); + let lost = CancellationToken::new(); + let reason = Arc::new(Mutex::new(None)); + let timer = tokio::spawn({ + let claim = Arc::clone(&claim); + let confirmed = Arc::clone(&confirmed); + let in_flight = Arc::clone(&in_flight); + let lost = lost.clone(); + let reason = Arc::clone(&reason); + async move { + let mut ticks = tokio::time::interval(period); + ticks.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + ticks.tick().await; + loop { + ticks.tick().await; + let outcome = { + let _beat = InFlight::mark(&in_flight); + claim.heartbeat_within(period).await + }; + if let Some(Ok(Heartbeat::Confirmed)) = outcome { + let _ = confirmed.send(Instant::now()); + if let HeartbeatMetric::BackupSnapshotAge { + metrics, + created_at, + } = &metric + { + metrics.record_backup_snapshot_age(seconds_since(*created_at)); + } + continue; + } + let why = beat_failure(outcome, period); + *reason.lock().unwrap_or_else(PoisonError::into_inner) = Some(why.clone()); + tracing::warn!( + "the job's heartbeat raised the loss of its claim; the job stops at its \ + next check: {why}" + ); + lost.cancel(); + return; + } + } + }); + Self { + claim, + confirmed, + last_confirmed, + in_flight, + parks, + lost, + reason, + period, + timer, + } + } + + /// The signal that the claim is lost; select on `lost().cancelled()`. + pub(crate) fn lost(&self) -> &CancellationToken { + &self.lost + } + + /// Why the claim is lost, once it is. + pub(crate) fn reason(&self) -> String { + self.reason + .lock() + .unwrap_or_else(PoisonError::into_inner) + .clone() + .unwrap_or_else(|| "the claim is lost".to_owned()) + } + + /// Why the pre-put check failed, for the job's stop line: the timer's + /// own reason when it raised the loss ([`Self::reason`]), else the + /// second clause, that no beat confirmed the claim within two periods, + /// naming `step`, what the job was about to do ("to load X", "for the + /// DONE write"). + pub(crate) fn why_not_confirmed(&self, step: &str) -> String { + if self.lost.is_cancelled() { + self.reason() + } else { + format!("no heartbeat confirmed the claim recently enough {step}") + } + } + + /// The pre-put check (module doc): no loss raised, and either the last + /// confirmed beat at most two periods old or a beat in flight (parked on + /// SQLite's write lock behind a long writer; within one period of its + /// start on the other backends). The second clause is what catches a + /// timer task that stopped running without raising the loss; the third + /// never outlives its beat. + pub(crate) fn confirms_the_claim(&self) -> bool { + self.claim_check().confirms_the_claim() + } + + /// The pre-put check as a value a blocking thread can hold (the + /// restore's decoder checks it at every chunk boundary): the same three + /// clauses read from the same watch, token and flag. + pub(crate) fn claim_check(&self) -> ClaimCheck { + ClaimCheck { + last_confirmed: self.last_confirmed.clone(), + lost: self.lost.clone(), + in_flight: Arc::clone(&self.in_flight), + parks: self.parks, + period: self.period, + } + } + + /// One beat outside the timer, under the period's bound (the claim + /// re-check before the manifest): a confirmation counts as a beat for + /// [`Self::confirms_the_claim`]; anything else is the reason the claim + /// is lost, with the timer's own text. + pub(crate) async fn recheck(&self) -> Result<(), String> { + let outcome = { + let _beat = InFlight::mark(&self.in_flight); + self.claim.heartbeat_within(self.period).await + }; + if let Some(Ok(Heartbeat::Confirmed)) = outcome { + let _ = self.confirmed.send(Instant::now()); + return Ok(()); + } + Err(beat_failure(outcome, self.period)) + } + + /// End the timer without raising the loss: a timer task that stopped + /// running, for the test of the pre-put check's second clause and for + /// the restore's fenced-out kill point, which lets a reconciler take the + /// record over while the job pauses. + pub(crate) fn stop_timer(&self) { + self.timer.abort(); + } + + /// A handle that ends the timer as [`Self::stop_timer`] does, for a test + /// that holds no reference to the job's handle. + #[cfg(test)] + pub(crate) fn timer_abort_handle(&self) -> tokio::task::AbortHandle { + self.timer.abort_handle() + } +} + +/// [`HeartbeatHandle::confirms_the_claim`] detached from the handle. +#[derive(Clone)] +pub(crate) struct ClaimCheck { + last_confirmed: watch::Receiver, + lost: CancellationToken, + in_flight: Arc>>, + parks: bool, + period: Duration, +} + +impl ClaimCheck { + /// No loss raised, and either the last confirmed beat at most two + /// periods old or a beat in flight within its bound (module doc). + pub(crate) fn confirms_the_claim(&self) -> bool { + !self.lost.is_cancelled() + && (self.last_confirmed.borrow().elapsed() <= 2 * self.period + || in_flight_confirms(&self.in_flight, self.parks, self.period)) + } +} + +impl Drop for HeartbeatHandle { + fn drop(&mut self) { + self.timer.abort(); + } +} + +/// Seconds between `epoch_secs` and now, on the wall clock. +pub(crate) fn seconds_since(epoch_secs: f64) -> f64 { + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs_f64()) + .unwrap_or_default(); + (now - epoch_secs).max(0.0) +} diff --git a/crates/engine/src/backup/job/writer.rs b/crates/engine/src/backup/job/writer.rs new file mode 100644 index 000000000..ba479a168 --- /dev/null +++ b/crates/engine/src/backup/job/writer.rs @@ -0,0 +1,742 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The data-file pipeline of a backup job: items from the snapshot stream, +//! through the validator, into gzip data files, into the store. +//! +//! Three parts run at once. The feeder (a task of its own) reads the +//! snapshot stream, checks every item with +//! [`validate_restorable_item`] and sends the ones that pass over a bounded +//! item channel ([`ITEM_CHANNEL`] items). One encoder per data file runs on +//! the blocking pool: it takes items off that channel, writes them through +//! `DataFileEncoder` into a [`ChunkSink`] that hands [`CHUNK_BYTES`]-sized +//! chunks over a bounded chunk channel ([`CHUNK_CHANNEL`] chunks), and +//! closes the file once its uncompressed bytes reach +//! `data_file_target_bytes`, so a file exceeds the knob by less than one +//! line. The uploader (the job's own task) starts the store `put` of each +//! file when the file's first chunk arrives, with the chunk channel as the +//! body, so a file is never buffered whole; a file that gets no item (the +//! stream ended or stopped at a file boundary) is never started. The +//! memory the pipeline holds is the item channel's items, the chunk +//! channel's chunks and the encoder's own state, none of which grows with +//! the table or the file size; the feeder's measurement of an oversized +//! item's line (below) holds no line. +//! +//! The validator's rule: after the first refused item the job encodes and +//! uploads nothing more but keeps reading and checking the rest of the +//! stream, logs each refused item (key, attribute path, reason, backup ARN) +//! up to [`REFUSAL_LOG_CAP`] and counts the rest, and then fails with a +//! reason that gives the count and the first item, naming #618 only when +//! the first refusal is an invalid `N` value. An item whose data file line +//! would exceed the format's cap (`MAX_LINE_BYTES`) is refused the same way +//! (`RestorableItemReason::LineTooLong`): the feeder measures the line of +//! every item over the default item limit (`DEFAULT_MAX_ITEM_SIZE_BYTES`), +//! the largest item the cap is sized for, on the blocking pool and through +//! a writer that counts and stops at the cap, so the measurement holds no +//! line and an instance with a raised `max_item_size_bytes` names the item +//! instead of failing in the encoder with no item named, and an instance at +//! the default measures none. Such an item's validation runs on the +//! blocking pool with the measurement (its cost grows with the item: the +//! number rule over every member); an item within the default is validated +//! on the feeder's own task. +//! +//! Before every `put` the uploader checks the heartbeat +//! ([`HeartbeatHandle::confirms_the_claim`]) and starts none otherwise; a +//! lost claim or the shutdown token ends the pipeline at its next await. +//! When the uploader fails on its own (a put refused, an encoder error) it +//! cancels a token the feeder selects on, so the feeder stops reading at +//! once rather than at the stream's next item and the FAILED write does not +//! wait on a slow or dead snapshot stream; the token is cancelled before +//! the uploader awaits the encoder, since an encoder waiting on the item +//! channel returns only once the feeder closes it; the feeder's own refusal +//! leaves that token alone, since the rest of the stream is still to be +//! checked. +//! When the pipeline returns, no `put` is in flight: a stop lets the +//! current put end (its body ends when the encoder drops the sink) or drops +//! it, and the store's own rule for a dropped put applies (the filesystem +//! store removes its temporary, the S3 store aborts the upload, and at +//! shutdown the failure path's `delete_prefix` sweeps what is left). + +use std::io::Write; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; + +use bytes::{Bytes, BytesMut}; +use extenddb_backup_store::{BackupStore, ByteStream}; +use extenddb_core::backup_format::{ + DataFileChecksums, DataFileEncoder, DataFileEntry, FormatError, KeyRange, KeyRangeTracker, + MAX_LINE_BYTES, data_file_key, encode_item_line_len_bounded, +}; +use extenddb_core::limits::DEFAULT_MAX_ITEM_SIZE_BYTES; +use extenddb_core::metrics::MetricsCollector; +use extenddb_core::types::{Item, item_size_bytes}; +use extenddb_core::validation::{ + RestorableItemError, RestorableItemReason, RestorableKeySchema, RestorableLimits, + validate_restorable_item, +}; +use extenddb_storage::CancellationToken; +use extenddb_storage::backup_catalog::StoreIdentity; +use extenddb_storage::error::StorageError; +use futures::StreamExt; +use futures::stream::BoxStream; +use tokio::sync::mpsc; + +use super::super::runtime::{ConfiguredStore, StoreHealth}; +use super::heartbeat::HeartbeatHandle; +use super::{KillPoint, kill_point}; + +/// Bytes per chunk handed from the encoder to the upload. +pub(crate) const CHUNK_BYTES: usize = 1024 * 1024; +/// Chunks the chunk channel holds before the encoder waits. +pub(crate) const CHUNK_CHANNEL: usize = 4; +/// Items the item channel holds before the feeder waits. Four, not one: a +/// one-item channel made a 1M-row backup on SQLite take 75% longer (12.4 s +/// to 21.6 s, measured in both orders under one host state) for about +/// 50 MiB less at the peak of the worst-shape run (design section 7). +pub(crate) const ITEM_CHANNEL: usize = 4; +/// Refused items logged one by one; the rest are counted. +pub(crate) const REFUSAL_LOG_CAP: u64 = 1000; + +/// Why the job stops before the manifest. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum Stop { + /// The row is this job's and the failure path runs with this reason. + Fail(String), + /// The shutdown token fired: the failure path runs with reason + /// `shutdown`. + Cancelled, + /// The claim is lost or its heartbeat did not return: no further store + /// or catalog access. + ClaimLost(String), + /// The store, as the runtime holds it, is unhealthy or is no longer the + /// store the row records ([`Pipeline::store_check`]): FAILED with this + /// reason and no store call, the prefix's deletion included. + StoreUnhealthy(String), +} + +/// What the pipeline wrote. +#[derive(Debug, Default)] +pub(crate) struct DataFiles { + /// Each file's manifest entry with the etag the store reported. + pub entries: Vec<(DataFileEntry, Option)>, + /// Items in every file. + pub item_count: u64, + /// The DynamoDB sizes of those items, summed (the manifest's + /// `source_table.table_size_bytes`). + pub item_bytes: u64, + /// Bytes the store reported stored, over every file. + pub bytes_uploaded: u64, +} + +/// What the uploader needs from the job. +pub(crate) struct Pipeline<'a> { + pub store: &'a dyn BackupStore, + /// The store as the runtime holds it: its health and identity, read + /// before every put. + pub configured: &'a ConfiguredStore, + /// The store's identity as the row records it. + pub identity: &'a StoreIdentity, + /// The backup's store prefix (`BackupArn::store_prefix`). + pub prefix: &'a str, + /// The ARN, for the log lines. + pub arn: &'a str, + pub heartbeat: &'a HeartbeatHandle, + pub shutdown: &'a CancellationToken, + pub metrics: Option<&'a MetricsCollector>, + /// `data_file_target_bytes`. + pub target_bytes: u64, + /// The store's configured key prefix and the summary's bucket name + /// (`BackupDocuments::build`). + pub key_prefix: &'a str, + pub bucket: &'a str, +} + +impl Pipeline<'_> { + /// The check before every put: the store is healthy as the runtime holds + /// it (the reconciler's check sets it) and its identity is the one the + /// row records. A store marked unhealthy mid-job, or whose identity file + /// now names another store (a root swapped under the job), stops the + /// job before the put of `name` with the store's reason + /// ([`Stop::StoreUnhealthy`]): the objects a job would write into such a + /// store are never the backup the row describes. + pub(crate) fn store_check(&self, name: &str) -> Result<(), Stop> { + match self.configured.health() { + StoreHealth::Healthy => {} + StoreHealth::Checking => { + return Err(Stop::StoreUnhealthy(format!( + "the {} backup store has no passed check; {name} was not written", + self.configured.kind + ))); + } + StoreHealth::Unhealthy(problem) => { + return Err(Stop::StoreUnhealthy(format!( + "the {} backup store is unhealthy ({problem}); {name} was not written", + self.configured.kind + ))); + } + } + match self.configured.identity() { + Some(identity) if identity == *self.identity => Ok(()), + Some(identity) => Err(Stop::StoreUnhealthy(format!( + "the {} backup store's id is {} and the row's is {}; {name} was not written", + self.configured.kind, identity.store_id, self.identity.store_id + ))), + None => Err(Stop::StoreUnhealthy(format!( + "the {} backup store's id is unknown; {name} was not written", + self.configured.kind + ))), + } + } +} + +/// The reason the FAILED row carries for refused items (module doc). +fn refusal_reason(count: u64, first: &RestorableItemError) -> String { + let issue = if matches!(first.reason, RestorableItemReason::InvalidNumber(_)) { + " (#618)" + } else { + "" + }; + format!("{count} item(s) cannot be restored; first: {first}{issue}") +} + +/// What the feeder found by the end of the stream. +#[derive(Default)] +struct FeedEnd { + refusals: Option<(u64, RestorableItemError)>, + stream_error: Option, + interrupted: Option, +} + +/// The feeder (module doc). `stop` is shared with the encoders: the feeder +/// sets it at the first refusal, and stops sending when another part set +/// it. `failed` is the uploader's: cancelled when the upload failed, so the +/// feeder returns at once. +struct Feeder { + tx: mpsc::Sender, + key: RestorableKeySchema, + limits: RestorableLimits, + stop: Arc, + failed: CancellationToken, + lost: CancellationToken, + shutdown: CancellationToken, + arn: String, +} + +impl Feeder { + async fn run(self, mut items: BoxStream<'static, Result>) -> FeedEnd { + let Self { + tx, + mut key, + limits, + stop, + failed, + lost, + shutdown, + arn, + } = self; + let mut tx = Some(tx); + let mut end = FeedEnd::default(); + loop { + let next = tokio::select! { + biased; + () = shutdown.cancelled() => { + end.interrupted = Some(Stop::Cancelled); + break; + } + () = lost.cancelled() => { + end.interrupted = Some(Stop::ClaimLost(String::new())); + break; + } + () = failed.cancelled() => break, + next = items.next() => next, + }; + let mut item = match next { + None => break, + Some(Ok(item)) => item, + Some(Err(e)) => { + end.stream_error = Some(e); + break; + } + }; + let refusal = if item_size_bytes(&item) > DEFAULT_MAX_ITEM_SIZE_BYTES { + // An item over the default item limit: the validator's walk + // (the number rule over every member, CPU work proportional + // to the item's size) and then the line's measurement (up + // to the cap's bytes) run on the blocking pool as the + // encoder does; the item and the key come back with the + // result. + let measured = tokio::task::spawn_blocking(move || { + let refusal = validate_restorable_item(&item, &key, &limits) + .err() + .or_else(|| line_too_long(&item, &key)); + (item, key, refusal) + }) + .await; + match measured { + Ok((back, key_back, refusal)) => { + item = back; + key = key_back; + refusal + } + Err(e) => { + end.stream_error = Some(StorageError::Internal(format!( + "checking an item over the default item limit: {e}" + ))); + break; + } + } + } else { + validate_restorable_item(&item, &key, &limits).err() + }; + if let Some(e) = refusal { + match &mut end.refusals { + None => { + tracing::error!("backup {arn}: item cannot be restored: {e}"); + end.refusals = Some((1, e)); + stop.store(true, Ordering::Release); + tx = None; + } + Some((count, _)) => { + *count += 1; + if *count <= REFUSAL_LOG_CAP { + tracing::error!("backup {arn}: item cannot be restored: {e}"); + } + } + } + continue; + } + if end.refusals.is_some() { + continue; + } + if stop.load(Ordering::Acquire) { + break; + } + let Some(sender) = &tx else { break }; + if sender.send(item).await.is_err() { + break; + } + } + if let Some((count, _)) = &end.refusals + && *count > REFUSAL_LOG_CAP + { + tracing::error!( + "backup {arn}: {} more item(s) cannot be restored (not listed)", + count - REFUSAL_LOG_CAP + ); + } + drop(items); + end + } +} + +/// The line-cap rule (module doc) for an item over the default item limit: +/// its line is measured against `MAX_LINE_BYTES` through a counting writer +/// that holds no line ([`encode_item_line_len_bounded`]), and one whose +/// line would exceed the cap is refused with `LineTooLong`. The feeder +/// measures no item within the default limit: the cap is sized to hold +/// every line such an item can expand to (`MAX_LINE_BYTES`'s doc, pinned by +/// the format's own test). An item the serializer refuses is left to the +/// encoder, which reports it. +fn line_too_long(item: &Item, key: &RestorableKeySchema) -> Option { + match encode_item_line_len_bounded(item, MAX_LINE_BYTES) { + Err(FormatError::LineTooLong { cap }) => Some(RestorableItemError { + key: key.shown_key(item), + path: String::new(), + reason: RestorableItemReason::LineTooLong { cap }, + }), + Ok(_) | Err(_) => None, + } +} + +const STOPPED: &str = "the pipeline stopped"; + +/// The encoder's sink: chunks of [`CHUNK_BYTES`] over the chunk channel, the +/// remainder on `flush`. A write after the pipeline stopped, or after the +/// upload ended, fails and poisons the encoder. Both failures are +/// `BrokenPipe`, never `Interrupted`: the encoder retries `Interrupted` on +/// every path (it is std's "try again" kind), and a stopped pipeline does +/// not come back, so an `Interrupted` here would spin the compressor's +/// trailer write for as long as the drain lasts. +struct ChunkSink { + tx: mpsc::Sender, + buf: BytesMut, + stop: Arc, +} + +impl ChunkSink { + fn send(&self, chunk: Bytes) -> std::io::Result<()> { + self.tx + .blocking_send(chunk) + .map_err(|_| std::io::Error::new(std::io::ErrorKind::BrokenPipe, "the upload ended")) + } +} + +impl Write for ChunkSink { + fn write(&mut self, data: &[u8]) -> std::io::Result { + if self.stop.load(Ordering::Acquire) { + return Err(std::io::Error::new(std::io::ErrorKind::BrokenPipe, STOPPED)); + } + self.buf.extend_from_slice(data); + while self.buf.len() >= CHUNK_BYTES { + let chunk = self.buf.split_to(CHUNK_BYTES).freeze(); + self.send(chunk)?; + } + Ok(data.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + if !self.buf.is_empty() { + let chunk = self.buf.split().freeze(); + self.send(chunk)?; + } + Ok(()) + } +} + +/// How one encoder ended. +enum FileEnd { + /// No item reached the encoder: nothing was written. + Empty, + /// The file reached `data_file_target_bytes` and closed; the stream goes + /// on. + Closed { + checksums: DataFileChecksums, + range: Option, + /// The DynamoDB sizes of the file's items, summed. + item_bytes: u64, + }, + /// The stream ended inside this file, which closed. + Ended { + checksums: DataFileChecksums, + range: Option, + item_bytes: u64, + }, +} + +/// What an encoder hands back: the item channel and the tracker for the +/// next file, and how it ended. +type Encoded = ( + mpsc::Receiver, + Option, + Result, +); + +/// One data file's encoder (module doc), on the blocking pool. +fn encode_file( + mut items: mpsc::Receiver, + sink: ChunkSink, + mut tracker: Option, + target_bytes: u64, +) -> Encoded { + let stop = Arc::clone(&sink.stop); + let mut encoder = DataFileEncoder::new(sink); + let mut item_bytes: u64 = 0; + let result = loop { + if stop.load(Ordering::Acquire) { + break Err(FormatError::Encode(STOPPED.to_owned())); + } + let Some(item) = items.blocking_recv() else { + if encoder.item_count() == 0 { + break Ok(FileEnd::Empty); + } + break close(&mut encoder, &mut tracker).map(|(checksums, range)| FileEnd::Ended { + checksums, + range, + item_bytes, + }); + }; + if let Some(tracker) = &mut tracker + && let Err(e) = tracker.push(&item) + { + break Err(e); + } + if let Err(e) = encoder.write_item(&item) { + break Err(e); + } + item_bytes += item_size_bytes(&item) as u64; + if encoder.uncompressed_bytes() >= target_bytes { + break close(&mut encoder, &mut tracker).map(|(checksums, range)| FileEnd::Closed { + checksums, + range, + item_bytes, + }); + } + }; + // A finished encoder's drop writes nothing; an open one's drop takes + // the sink out first, so no trailer reaches the upload. + drop(encoder); + (items, tracker, result) +} + +/// Close the file: the trailer and the remainder go to the sink, the +/// tracker starts the next file. +fn close( + encoder: &mut DataFileEncoder, + tracker: &mut Option, +) -> Result<(DataFileChecksums, Option), FormatError> { + let checksums = encoder.try_finish()?; + Ok(( + checksums, + tracker.as_mut().and_then(KeyRangeTracker::next_file), + )) +} + +/// The pipeline (module doc). `tracker` is `Some` for a key-ordered backup. +pub(crate) async fn write_data_files( + p: &Pipeline<'_>, + items: BoxStream<'static, Result>, + key: RestorableKeySchema, + limits: RestorableLimits, + mut tracker: Option, +) -> Result { + let stop = Arc::new(AtomicBool::new(false)); + let failed = CancellationToken::new(); + let (item_tx, item_rx) = mpsc::channel(ITEM_CHANNEL); + let feeder = tokio::spawn( + Feeder { + tx: item_tx, + key, + limits, + stop: Arc::clone(&stop), + failed: failed.clone(), + lost: p.heartbeat.lost().clone(), + shutdown: p.shutdown.clone(), + arn: p.arn.to_owned(), + } + .run(items), + ); + let mut item_rx = Some(item_rx); + let mut files = DataFiles::default(); + let mut index: u32 = 1; + let uploaded: Result<(), Stop> = loop { + let (chunk_tx, mut chunk_rx) = mpsc::channel::(CHUNK_CHANNEL); + let sink = ChunkSink { + tx: chunk_tx, + buf: BytesMut::with_capacity(CHUNK_BYTES), + stop: Arc::clone(&stop), + }; + let rx = item_rx.take().expect("the previous encoder returned it"); + let file_tracker = tracker.take(); + let target = p.target_bytes; + let encoder = + tokio::task::spawn_blocking(move || encode_file(rx, sink, file_tracker, target)); + + // The file's first chunk, or the channel closing with none. + let first = tokio::select! { + biased; + () = p.shutdown.cancelled() => Err(Stop::Cancelled), + () = p.heartbeat.lost().cancelled() => Err(Stop::ClaimLost(p.heartbeat.reason())), + chunk = chunk_rx.recv() => Ok(chunk), + }; + let first = match first { + Ok(Some(first)) => first, + Ok(None) => { + let (_, _, end) = match encoder.await { + Ok(encoded) => encoded, + Err(e) => break Err(Stop::Fail(format!("the encoder task failed: {e}"))), + }; + break match end { + Ok(FileEnd::Empty) => Ok(()), + Ok(_) => Err(Stop::Fail(format!( + "data file {index} closed without a chunk" + ))), + Err(_) if stop.load(Ordering::Acquire) => Ok(()), + Err(e) => Err(Stop::Fail(format!("encoding data file {index}: {e}"))), + }; + } + Err(why) => { + stop.store(true, Ordering::Release); + drop(chunk_rx); + // The feeder first (module doc): an encoder waiting on the + // item channel returns only when the feeder closes it, and + // the feeder waits on the stream's next item until told. + failed.cancel(); + let _ = encoder.await; + break Err(why); + } + }; + + let name = data_file_key(index); + if let Err(why) = p.store_check(&name) { + stop.store(true, Ordering::Release); + drop(chunk_rx); + failed.cancel(); + let _ = encoder.await; + break Err(why); + } + if !p.heartbeat.confirms_the_claim() { + stop.store(true, Ordering::Release); + drop(chunk_rx); + failed.cancel(); + let _ = encoder.await; + break Err(Stop::ClaimLost(format!( + "no heartbeat confirmed the claim recently enough for data file {index}" + ))); + } + let key = format!("{}{name}", p.prefix); + let body: ByteStream = Box::pin(futures::stream::once(async move { Ok(first) }).chain( + futures::stream::unfold(chunk_rx, |mut rx| async move { + rx.recv().await.map(|chunk| (Ok(chunk), rx)) + }), + )); + let put = tokio::select! { + biased; + () = p.shutdown.cancelled() => Err(Stop::Cancelled), + () = p.heartbeat.lost().cancelled() => Err(Stop::ClaimLost(p.heartbeat.reason())), + result = p.store.put(&key, body) => result.map_err(|e| { + if let Some(metrics) = p.metrics { + metrics.record_backup_store_error("put"); + } + Stop::Fail(format!("writing {name}: {e}")) + }), + }; + let meta = match put { + Ok(meta) => meta, + Err(why) => { + stop.store(true, Ordering::Release); + // Before the encoder (module doc): a put that fails mid-file + // leaves the encoder in `blocking_recv`, which returns only + // once the feeder, told here, closes the channel. + failed.cancel(); + let _ = encoder.await; + break Err(why); + } + }; + // The put ended because the encoder dropped the sink, so the encoder + // has returned or is about to. + let (rx, file_tracker, end) = match encoder.await { + Ok(encoded) => encoded, + Err(e) => break Err(Stop::Fail(format!("the encoder task failed: {e}"))), + }; + item_rx = Some(rx); + tracker = file_tracker; + let (checksums, range, item_bytes, last) = match end { + Ok(FileEnd::Closed { + checksums, + range, + item_bytes, + }) => (checksums, range, item_bytes, false), + Ok(FileEnd::Ended { + checksums, + range, + item_bytes, + }) => (checksums, range, item_bytes, true), + Ok(FileEnd::Empty) => { + break Err(Stop::Fail(format!( + "data file {index} sent a chunk and then reported no item" + ))); + } + Err(_) if stop.load(Ordering::Acquire) => break Ok(()), + Err(e) => break Err(Stop::Fail(format!("encoding data file {index}: {e}"))), + }; + // The store's size is the manifest's: the reconciler's dead-owner + // probe refuses a data file whose stored size differs from the + // recorded one, so a live job must not commit a manifest that probe + // would call incomplete. + if meta.size != checksums.compressed_bytes { + break Err(Stop::Fail(format!( + "the store reports {} bytes for {name} and the encoder wrote {}", + meta.size, checksums.compressed_bytes + ))); + } + files.item_count += checksums.item_count; + files.item_bytes += item_bytes; + files.bytes_uploaded += meta.size; + files.entries.push(( + DataFileEntry::from_checksums(name, &checksums, range), + meta.etag, + )); + if index == 1 { + kill_point(KillPoint::AfterDataFile, p.shutdown).await; + } + if last { + break Ok(()); + } + index += 1; + }; + stop.store(true, Ordering::Release); + if uploaded.is_err() { + // The uploader's own failure: the feeder has nothing left to check. + failed.cancel(); + } + drop(item_rx); + let end = match feeder.await { + Ok(end) => end, + Err(e) => return Err(Stop::Fail(format!("the snapshot reader task failed: {e}"))), + }; + if let Err(why @ (Stop::Cancelled | Stop::ClaimLost(_) | Stop::StoreUnhealthy(_))) = uploaded { + return Err(why); + } + match end.interrupted { + Some(Stop::Cancelled) => return Err(Stop::Cancelled), + Some(Stop::ClaimLost(_)) => return Err(Stop::ClaimLost(p.heartbeat.reason())), + Some(Stop::Fail(_) | Stop::StoreUnhealthy(_)) | None => {} + } + match (end.refusals, end.stream_error) { + // A stream that fails after a refusal (the pass that checks the rest + // reads the whole table) keeps the refusals in the reason, counted + // over the items read before the failure, so `backup list --failed` + // still names the first refused item. + (Some((count, first)), Some(e)) => { + return Err(Stop::Fail(format!( + "{}; the snapshot then failed: {e}", + refusal_reason(count, &first) + ))); + } + (None, Some(e)) => return Err(Stop::Fail(format!("reading the snapshot: {e}"))), + (Some((count, first)), None) => { + return Err(Stop::Fail(refusal_reason(count, &first))); + } + (None, None) => {} + } + uploaded.map(|()| files) +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use extenddb_core::types::AttributeValue; + + use super::*; + + fn stopped_sink() -> ChunkSink { + let (tx, _rx) = mpsc::channel(CHUNK_CHANNEL); + ChunkSink { + tx, + buf: BytesMut::with_capacity(CHUNK_BYTES), + stop: Arc::new(AtomicBool::new(true)), + } + } + + /// A write after the stop is `BrokenPipe`, the kind nothing retries. + #[test] + fn a_write_after_the_stop_is_a_broken_pipe() { + let e = stopped_sink().write(b"x").unwrap_err(); + assert_eq!(e.kind(), std::io::ErrorKind::BrokenPipe); + assert_eq!(e.to_string(), STOPPED); + } + + /// An encoder holding items when the pipeline stops fails its close at + /// once instead of retrying the trailer write for the whole drain: the + /// shutdown of a job in the middle of a large table ends within the + /// server's drain window. + #[test] + fn closing_an_encoder_after_the_stop_fails_at_once() { + let (done_tx, done_rx) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let mut encoder = DataFileEncoder::new(stopped_sink()); + let item = Item::from([("pk".to_owned(), AttributeValue::S("a".to_owned()))]); + // The gzip header is the first write to reach the sink. + let written = encoder.write_item(&item); + let closed = encoder.try_finish().map(|_| ()); + drop(encoder); + let _ = done_tx.send((written, closed)); + }); + let (written, closed) = done_rx + .recv_timeout(Duration::from_secs(5)) + .expect("the close returned within the window"); + let e = written.and(closed).unwrap_err(); + assert!(e.to_string().contains(STOPPED), "{e}"); + } +} diff --git a/crates/engine/src/backup/maintenance.rs b/crates/engine/src/backup/maintenance.rs new file mode 100644 index 000000000..e76af3c1e --- /dev/null +++ b/crates/engine/src/backup/maintenance.rs @@ -0,0 +1,875 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The holder's side of the catalog-level maintenance claim, and the steps a +//! maintenance command (`extenddb backup purge`, `extenddb backup +//! remove-target`) runs under it. +//! +//! [`HeldClaim`] is the handle the command keeps from the take to the +//! release, with the heartbeat running on a timer for the handle's whole +//! life, so every reader of the claim (`check_maintenance_claim` on every +//! backend measures the heartbeat's age, PostgreSQL the lock first) sees it +//! held through a wait and a deletion phase of any length. A heartbeat that +//! finds the claim lost (another holder took it over after this one looked +//! dead), or fails, is recorded for the command to stop at before its next +//! destructive step. Dropping the handle without `release` stops the timer +//! (the claim then expires by the stale rule, as a dead holder's does); +//! `release` stops it, waits for a heartbeat in flight, then releases. +//! +//! [`wait_for_quiet`] is the wait for the jobs the claim excludes from +//! starting to finish, [`purge_row`] the deletion of one row's objects and +//! the row, and [`purge_rows`] the loop over the rows that stops at a lost +//! claim or a signal, each written against the catalog's trait so the +//! engine's tests run them against a double; the command prints what they +//! return. An +//! [`Interrupt`] carries a signal the command received to both, so an +//! operator's Ctrl-C ends the wait at once and the deletion loop before its +//! next row, never inside one (the row's own steps are the idempotent +//! sequence a second deleter finishes). +//! +//! [`remove_target`] is the whole of +//! `extenddb backup remove-target --account
    `: the claim +//! taken and held for its duration, the table's +//! restore record read, the refusals, the removal through +//! `remove_restore_target`, the release. It is the one command that removes +//! a restore target DeleteTable refuses (a CREATING table): a FAILED +//! record's target, whatever the reason (the reconciler leaves a +//! `target_not_empty` one for this command; on MongoDB and Cassandra a +//! crash before the first progress record, the whole first data file on +//! MongoDB and the first 10,000 items or 32 MiB on Cassandra, ends in that +//! state), and a RUNNING record's target whose owner is dead (the backend's +//! `list_dead_restore_jobs` names it under the owner window: no heartbeat +//! within the window on the catalog's clock); a legacy record's target the +//! same way. It refuses a RUNNING record whose owner is live, a DONE record +//! (the table is ACTIVE, or on MongoDB awaiting its index backfill) and a +//! table no restore record names, the latter two as "not a restore target". + +use std::sync::{Arc, Mutex, PoisonError}; +use std::time::Duration; + +use extenddb_backup_store::BackupStore; +use extenddb_core::backup_format::{BackupArn, EXTENDDB_MANIFEST_FILE}; +use extenddb_storage::backup_catalog::{ + BackupRow, BackupStatus, DeleteStart, Heartbeat, JobStatus, RestoreJobRow, RestoreKind, + StaleClock, StoreIdentity, is_backup_not_found, +}; +use extenddb_storage::error::StorageError; +use extenddb_storage::{BackupCatalog, MaintenanceClaim}; +use futures::TryStreamExt; +use tokio::sync::watch; +use tokio::time::Instant; + +use super::StoreFence; + +/// A maintenance claim with its heartbeat on a timer. +pub struct HeldClaim { + claim: Arc, + lost: Arc>>, + timer: tokio::task::JoinHandle<()>, +} + +impl HeldClaim { + /// Hold `claim`, heartbeating it every `interval` from now until + /// [`HeldClaim::release`] or the drop (the caller passes the jobs' + /// heartbeat period, `extenddb_storage::backup_catalog::JOB_HEARTBEAT_INTERVAL`, + /// which every stale window the config accepts is at least three of). + #[must_use] + pub fn start(claim: Box, interval: Duration) -> Self { + let claim: Arc = Arc::from(claim); + let lost = Arc::new(Mutex::new(None)); + let timer = tokio::spawn({ + let claim = Arc::clone(&claim); + let lost = Arc::clone(&lost); + async move { + let mut ticks = tokio::time::interval(interval); + ticks.tick().await; + loop { + ticks.tick().await; + let reason = match claim.heartbeat().await { + Ok(Heartbeat::Confirmed) => continue, + Ok(Heartbeat::Lost) => { + "the maintenance claim was lost (another holder took it over)" + .to_owned() + } + Err(e) => format!("the maintenance claim's heartbeat failed: {e}"), + }; + *lost.lock().unwrap_or_else(PoisonError::into_inner) = Some(reason); + return; + } + } + }); + Self { claim, lost, timer } + } + + /// The claim's fencing token. + #[must_use] + pub fn generation(&self) -> u64 { + self.claim.generation() + } + + /// Why the claim is no longer held, once a heartbeat said so; `None` + /// while every heartbeat has confirmed it. + #[must_use] + pub fn lost(&self) -> Option { + self.lost + .lock() + .unwrap_or_else(PoisonError::into_inner) + .clone() + } + + /// Stop the heartbeat, wait for one in flight to end, and release the + /// claim, under [`RELEASE_TIMEOUT`]: a catalog that does not answer the + /// release leaves the claim to expire by the stale rule, and the command + /// ends. + /// + /// # Errors + /// + /// The backend's, when the release could not be written, or the bound; + /// the claim then expires by the stale rule. + pub async fn release(mut self) -> Result<(), StorageError> { + self.timer.abort(); + // The abort lands at the task's next await; a heartbeat already sent + // finishes first, so the release never overlaps it on the claim's + // connection. A cancelled task is the expected outcome. + let _ = (&mut self.timer).await; + tokio::time::timeout(RELEASE_TIMEOUT, self.claim.release()) + .await + .unwrap_or_else(|_| { + Err(extenddb_storage::error::StorageError::Internal(format!( + "the release did not finish within {} s", + RELEASE_TIMEOUT.as_secs() + ))) + }) + } +} + +/// How long [`HeldClaim::release`] waits for the catalog: past it the claim +/// is left to the stale rule, as a dead holder's is, so a stalled database +/// does not hold a command that received a signal. +pub const RELEASE_TIMEOUT: Duration = Duration::from_secs(10); + +impl Drop for HeldClaim { + fn drop(&mut self) { + self.timer.abort(); + } +} + +/// A signal the command received (`SIGINT` or `SIGTERM`), for the wait and +/// the deletion loop to stop at: the wait at once, the loop before its next +/// row. The sender side is the command's signal listener; a test's is the +/// test. +#[derive(Clone)] +pub struct Interrupt { + rx: watch::Receiver>, +} + +impl Interrupt { + /// A pair: the sender a signal listener sets with the signal's name, and + /// the handle the wait and the loop read. + #[must_use] + pub fn new() -> (watch::Sender>, Self) { + let (tx, rx) = watch::channel(None); + (tx, Self { rx }) + } + + /// The signal's name once one was received. + #[must_use] + pub fn signalled(&self) -> Option<&'static str> { + *self.rx.borrow() + } + + /// Resolves with the signal's name once one is received; pending until + /// then (forever, when the sender is gone and none was). + pub async fn wait(&mut self) -> &'static str { + loop { + if let Some(name) = *self.rx.borrow_and_update() { + return name; + } + if self.rx.changed().await.is_err() { + return std::future::pending().await; + } + } + } +} + +/// How [`wait_for_quiet`] ended. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum WaitOutcome { + /// No job the claim excludes is running: the deletion may start. + Quiet, + /// The limit passed with these jobs still running (the operator's text, + /// one per job). + Busy(Vec), + /// A heartbeat found the claim lost or failed; the reason. + Lost(String), + /// The rows could not be read; the backend's error. + CatalogError(String), + /// The command received this signal. + Interrupted(&'static str), +} + +/// The durations of a [`wait_for_quiet`]. +#[derive(Debug, Clone, Copy)] +pub struct Wait { + /// The window a live owner is told apart by: a backup row the backend + /// does not list as older than this on the heartbeat clock has a live + /// owner; zero where every owner is known dead (SQLite under the serve + /// lock). + pub owner_window: Duration, + /// How long to wait before giving up with the jobs still running. + pub limit: Duration, + /// How often the rows are re-read. + pub poll: Duration, +} + +/// Wait until no CREATING or FAILED backup row of `identity` has a live +/// owner and no restore job is RUNNING, re-reading every `wait.poll`, for +/// at most `wait.limit`; the claim lost, a catalog read failing, or a +/// signal ends it early. The jobs found running are reported through +/// `announce` the first time they are seen. +pub async fn wait_for_quiet( + storage: &S, + identity: &StoreIdentity, + claim: &HeldClaim, + wait: Wait, + interrupt: &mut Interrupt, + announce: impl Fn(&[String]), +) -> WaitOutcome { + let started = Instant::now(); + let mut announced = false; + loop { + if let Some(reason) = claim.lost() { + return WaitOutcome::Lost(reason); + } + if let Some(signal) = interrupt.signalled() { + return WaitOutcome::Interrupted(signal); + } + let remaining = match busy_jobs(storage, identity, wait.owner_window).await { + Ok(remaining) => remaining, + Err(e) => return WaitOutcome::CatalogError(e), + }; + if remaining.is_empty() { + return WaitOutcome::Quiet; + } + if started.elapsed() >= wait.limit { + return WaitOutcome::Busy(remaining); + } + if !announced { + announce(&remaining); + announced = true; + } + tokio::select! { + () = tokio::time::sleep(wait.poll) => {} + signal = interrupt.wait() => return WaitOutcome::Interrupted(signal), + } + } +} + +/// The jobs purge waits on, as text for the operator. Every age is the +/// backend's comparison on the catalog's clock: a CREATING or FAILED backup +/// row has a live owner while `list_backup_rows_older_than` on the +/// heartbeat clock does not list it under the window (a FAILED row whose +/// heartbeat is young may still be written to by a job that has not seen +/// its claim lost; its prefix waits for the window as the reconciler's +/// deletion of it does), and a RUNNING restore record is listed by status +/// (`list_running_restore_jobs`), with no age. +async fn busy_jobs( + storage: &S, + identity: &StoreIdentity, + owner_window: Duration, +) -> Result, String> { + let mut busy = Vec::new(); + for status in [BackupStatus::Creating, BackupStatus::Failed] { + let rows: Vec = storage + .list_backup_rows(Some(status), identity) + .try_collect() + .await + .map_err(|e| format!("cannot list the catalog's backup rows: {e}"))?; + let dead: Vec = storage + .list_backup_rows_older_than(status, identity, StaleClock::Heartbeat, owner_window) + .try_collect() + .await + .map_err(|e| format!("cannot list the catalog's backup rows by age: {e}"))?; + for row in rows { + if dead.iter().any(|d| d.backup_arn == row.backup_arn) { + continue; + } + busy.push(format!( + "{} {} has a live owner (heartbeat within stale_job_after_secs)", + if status == BackupStatus::Creating { + "CreateBackup" + } else { + "the FAILED backup" + }, + row.backup_arn + )); + } + } + // Every RUNNING restore job, by status alone: a record stamped in the + // listing's own instant, or ahead of the catalog's clock, is a running + // job too, which an age-based listing with a zero window would miss. The + // FAILED records whose target is still CREATING are the reconciler's to + // finish, not jobs: nothing runs under one, and purge deletes no table, + // so they are not waited on. + let running = storage + .list_running_restore_jobs() + .await + .map_err(|e| format!("cannot list restore jobs: {e}"))?; + for job in running { + if job.status != JobStatus::Running { + continue; + } + busy.push(format!( + "a restore into {} (account {}) is RUNNING", + job.target.table_name, job.target.account_id + )); + } + Ok(busy) +} + +/// Delete one row's objects and the row: the deletion fence first +/// (`fence`, which re-reads `extenddb-store.json` and refuses, marking the +/// store unhealthy, once another catalog has adopted the root), before the +/// row of a CREATING or AVAILABLE row is changed and again right before the +/// first deletion, then the manifest, then the prefix, then the row through +/// the status machine to a tombstone and out. +/// The row is `row` as the command read it; the status another deleter may +/// have moved it to since is read from `start_backup_delete`'s answer, and +/// each state the row can be in when the command reaches it ends with no +/// row: AVAILABLE is moved to DELETING here; DELETING (another deleter is +/// mid-way) and DELETED (another finished) are finished here, the prefix +/// deletion being the same idempotent sequence on both sides; a row already +/// gone counts as deleted. A CREATING row's owner is dead (the wait saw no +/// live one): the row goes first, through the dead-owner removal on the +/// catalog's clock, and the prefix only once the row is gone, so an owner +/// that heartbeated after the wait leaves the row and its objects together +/// rather than a row over a deleted prefix. Nothing adopts the prefix +/// between the two steps: every `backup sync` takes the maintenance claim +/// purge holds (on SQLite both run under the serve lock), so no sync reads +/// the rows after the removal and lists the prefix before its deletion. A +/// prefix a failed deletion leaves after the row's removal is reported as +/// left; if its job had written the manifest before it died (the state the +/// dead-owner duty resolves to AVAILABLE), a later sync adopts it as a +/// complete backup, which the next purge deletes. +/// +/// # Errors +/// +/// The reason the row was left, for the operator: the fence's refusal +/// (nothing deleted; before the row was changed it is as read, and from the +/// second fence a CREATING row is already removed with its prefix left and +/// an AVAILABLE one is DELETING, which a later purge or the reconciler +/// finishes only once this catalog owns the root again), a FAILED or +/// CREATING +/// status another writer moved an AVAILABLE row to (neither follows +/// AVAILABLE; the row is left for a look), a restore reading the backup, a +/// CREATING row whose owner heartbeated after the wait, a store or catalog +/// call failing, or a prefix that still lists an object. +pub async fn purge_row( + storage: &S, + store: &dyn BackupStore, + fence: &StoreFence, + row: &BackupRow, + stale_after: Duration, +) -> Result<(), String> { + let arn = BackupArn::parse(&row.backup_arn) + .map_err(|e| format!("the row's ARN does not parse: {e}"))?; + let key = row.key(); + // The fence before the catalog write as well as before the deletion: a + // refusal here leaves the row as it was read. + if matches!(row.status, BackupStatus::Creating | BackupStatus::Available) { + fence.check_catalog(storage).await.map_err(|problem| { + format!("the deletion was refused: {problem}; the row is unchanged") + })?; + } + if row.status == BackupStatus::Creating + && !storage + .remove_dead_creating_backup_row(&key, stale_after) + .await + .map_err(|e| format!("cannot remove the CREATING row: {e}"))? + { + return Err( + "the CREATING row could not be removed (its owner heartbeated after the wait, or its \ + status changed); its objects are left with it" + .to_owned(), + ); + } + // The status to finish from: the row's as read, or as another deleter + // left it. + let mut status = row.status; + if status == BackupStatus::Available { + match storage.start_backup_delete(&key).await { + Ok(DeleteStart::Deleting | DeleteStart::Status(BackupStatus::Deleting)) => { + status = BackupStatus::Deleting; + } + Ok(DeleteStart::Status(BackupStatus::Deleted)) => status = BackupStatus::Deleted, + Ok(DeleteStart::InUse(table)) => { + return Err(format!("a restore into {table} is reading it")); + } + Ok(DeleteStart::Status(other)) => { + return Err(format!("its status changed to {other}")); + } + Err(e) if is_backup_not_found(&e) => return Ok(()), + Err(e) => return Err(format!("cannot mark the row DELETING: {e}")), + } + } + let prefix = arn.store_prefix(); + fence + .check_catalog(storage) + .await + .map_err(|problem| format!("the deletion was refused: {problem}"))?; + store + .delete_prefix(&format!("{prefix}{EXTENDDB_MANIFEST_FILE}")) + .await + .map_err(|e| format!("cannot delete the manifest: {e}"))?; + store + .delete_prefix(&prefix) + .await + .map_err(|e| format!("cannot delete the prefix: {e}"))?; + if let Some(left) = store + .list(&prefix) + .try_next() + .await + .map_err(|e| format!("cannot list the prefix after the deletion: {e}"))? + { + return Err(format!( + "the prefix still lists {} after the deletion", + left.key + )); + } + if status == BackupStatus::Creating { + return Ok(()); + } + if status == BackupStatus::Deleting { + storage + .set_backup_deleted(&key) + .await + .map_err(|e| format!("cannot mark the row DELETED: {e}"))?; + } + if storage + .remove_backup_row(&key) + .await + .map_err(|e| format!("cannot remove the row: {e}"))? + { + return Ok(()); + } + // Not a tombstone: another deleter removed it, or a writer moved it. + match storage + .read_backup_row(&arn) + .await + .map_err(|e| format!("cannot re-read the row: {e}"))? + { + None => Ok(()), + Some(current) => Err(format!( + "the row could not be removed (it is {} now, not a tombstone)", + current.status + )), + } +} + +/// What [`purge_rows`] did. +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub struct PurgeOutcome { + /// The rows deleted, in order. + pub deleted: Vec, + /// The rows left, each with the reason (`: `). + pub skipped: Vec, + /// Why the loop stopped before its next row, if it did: the claim lost + /// or its heartbeat failed, or the signal received. + pub stopped: Option, +} + +/// Why [`purge_rows`] stopped before its next row. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Stop { + /// A heartbeat found the claim lost or failed; the reason. + Lost(String), + /// The command received this signal. + Interrupted(&'static str), +} + +/// The deletion phase: [`purge_row`] on each of `rows` in order, each behind +/// the deletion fence (`fence`), stopping before the next row (never inside +/// one) once a heartbeat has found the claim lost or failed, or a signal has +/// been received. A row that could not +/// be deleted is skipped with its reason and the loop goes on; `deleted` is +/// called with each row deleted, for the command's audit line and output. +#[allow(clippy::too_many_arguments)] +pub async fn purge_rows( + storage: &S, + store: &dyn BackupStore, + fence: &StoreFence, + rows: &[BackupRow], + stale_after: Duration, + claim: &HeldClaim, + interrupt: &Interrupt, + mut deleted: impl FnMut(&BackupRow), +) -> PurgeOutcome { + let mut outcome = PurgeOutcome::default(); + for row in rows { + if let Some(reason) = claim.lost() { + outcome.stopped = Some(Stop::Lost(reason)); + break; + } + if let Some(signal) = interrupt.signalled() { + outcome.stopped = Some(Stop::Interrupted(signal)); + break; + } + match purge_row(storage, store, fence, row, stale_after).await { + Ok(()) => { + deleted(row); + outcome.deleted.push(row.backup_arn.clone()); + } + Err(e) => outcome.skipped.push(format!("{}: {e}", row.backup_arn)), + } + } + outcome +} + +/// What `extenddb backup remove-target` is asked to do. +#[derive(Debug, Clone)] +pub struct RemoveTarget { + /// The account the table belongs to. + pub account_id: String, + /// The table's name. + pub table_name: String, + /// The maintenance claim's holder label (the command and its host). + pub holder: String, + /// `stale_job_after_secs`: the claim's stale window. + pub stale_after: Duration, + /// The window `list_dead_restore_jobs` is asked with: a RUNNING record + /// the backend does not list as dead under it (a heartbeat within the + /// window on the catalog's clock) has a live owner and is refused; zero + /// where every owner is known dead (SQLite under the serve lock, where + /// a zero window lists every RUNNING record). + pub owner_window: Duration, + /// The claim's heartbeat period (the jobs', `JOB_HEARTBEAT_INTERVAL`). + pub heartbeat_interval: Duration, +} + +/// What [`remove_target`] removed: the record as it was read, for the +/// command's line. +#[derive(Debug, Clone, PartialEq)] +pub struct RemovedTarget { + /// The target's table id. + pub table_id: String, + /// The backup the restore read. + pub backup_arn: String, + /// The record's kind. + pub kind: RestoreKind, + /// The record's status when it was read: FAILED, or RUNNING with a + /// dead owner. + pub status: JobStatus, + /// The FAILED record's reason. + pub failure_reason: Option, +} + +/// `extenddb backup remove-target` (module doc): under the maintenance +/// claim, taken here and released before the return, remove the CREATING +/// target of the restore record that names `request`'s table and the record +/// with it, through `remove_restore_target`. +/// +/// # Errors +/// +/// The text for the operator: the claim held by a live holder; a table no +/// record names, a DONE record, or a target no longer a CREATING restore +/// target with the record's id ("not a restore target"); a CREATING restore +/// target whose owner's lock is still held (the record unchanged after the +/// backend refused the removal: wait for the owner or stop it); a RUNNING +/// record the backend does not list as dead under the owner window; the +/// claim lost before the removal; a catalog call failing. +pub async fn remove_target( + storage: &S, + request: &RemoveTarget, +) -> Result { + let claim = storage + .maintenance_claim(&request.holder, request.stale_after) + .await + .map_err(|e| match e { + StorageError::AlreadyExists(m) => format!( + "{m}; one maintenance command runs at a time, and a claim whose holder died \ + expires after stale_job_after_secs ({} s)", + request.stale_after.as_secs() + ), + other => format!("cannot take the maintenance claim: {other}"), + })?; + let claim = HeldClaim::start(claim, request.heartbeat_interval); + let outcome = remove_target_under(storage, request, &claim).await; + if let Err(e) = claim.release().await { + tracing::warn!("the maintenance claim could not be released; it expires on its own: {e}"); + } + outcome +} + +/// The steps of [`remove_target`] between the take and the release. +async fn remove_target_under( + storage: &S, + request: &RemoveTarget, + claim: &HeldClaim, +) -> Result { + let what = format!("{} (account {})", request.table_name, request.account_id); + let records: Vec = storage + .list_restore_jobs(None) + .await + .map_err(|e| format!("cannot read the restore job records: {e}"))?; + let record = records + .into_iter() + .find(|r| { + r.target.account_id == request.account_id && r.target.table_name == request.table_name + }) + .ok_or_else(|| format!("{what} is not a restore target: no restore record names it"))?; + match record.status { + JobStatus::Done => { + return Err(format!( + "{what} is not a restore target: its restore from {} is DONE (the table is \ + ACTIVE, or awaits its index backfill)", + record.source_backup_arn + )); + } + JobStatus::Running => { + // The owner is dead only if the backend's own listing says so: + // `list_dead_restore_jobs` compares the heartbeat with the + // catalog's clock, the one the owner stamps, so a skew of this + // host's clock can neither refuse a dead owner nor remove under + // a live one. A zero window (SQLite under the serve lock) lists + // every RUNNING record, as it does for the purge's wait. + let dead = storage + .list_dead_restore_jobs(request.owner_window) + .await + .map_err(|e| format!("cannot list the dead restore jobs: {e}"))? + .iter() + .any(|r| r.target.table_id == record.target.table_id); + if !dead { + return Err(format!( + "the restore into {what} from {} is RUNNING with a live owner (a heartbeat \ + within stale_job_after_secs); stop it or wait for it", + record.source_backup_arn + )); + } + } + JobStatus::Failed | JobStatus::Creating | JobStatus::Available => {} + } + if let Some(reason) = claim.lost() { + return Err(format!("stopped before the removal: {reason}")); + } + let removed = storage + .remove_restore_target_exclusive(&record.target) + .await + .map_err(|e| format!("cannot remove the restore target {what}: {e}"))?; + if !removed { + // The backend refused without saying which of its two conditions + // failed. The record tells them apart: the target and its record go + // together, and the ACTIVE flip writes DONE, so a record unchanged + // since the listing names a table that is still its CREATING restore + // target, and the refusal was the owner's lock (PostgreSQL's + // per-session advisory lock, held by a frozen owner's session). + let unchanged = storage + .read_restore_job(&record.target) + .await + .map_err(|e| format!("cannot re-read the restore record of {what}: {e}"))? + .is_some_and(|now| { + now.status == record.status && now.owner_generation == record.owner_generation + }); + if unchanged { + return Err(format!( + "the restore into {what} from {} is a CREATING restore target (table id {}) \ + whose owner's lock is still held: its owner's session is alive though its \ + heartbeat is stale; wait for the owner to stop, or stop it, and run the \ + command again", + record.source_backup_arn, record.target.table_id + )); + } + return Err(format!( + "{what} is not a restore target: it is no longer a CREATING restore target with \ + table id {}", + record.target.table_id + )); + } + tracing::info!( + target: "extenddb::audit", + action = "backup remove-target", + account_id = %record.target.account_id, + table_name = %record.target.table_name, + table_id = %record.target.table_id, + backup_arn = %record.source_backup_arn, + "removed the restore target and its record" + ); + Ok(RemovedTarget { + table_id: record.target.table_id, + backup_arn: record.source_backup_arn, + kind: record.kind, + status: record.status, + failure_reason: record.failure_reason, + }) +} + +#[cfg(test)] +mod tests { + use extenddb_storage::error::StorageError; + use futures::future::BoxFuture; + use tokio::time::Instant; + + use super::*; + + /// A claim whose heartbeats are timestamps on tokio's clock, with the age + /// rule every backend's `check_maintenance_claim` applies. + #[derive(Default)] + struct FakeClaim { + beats: Mutex>, + lost: Mutex, + released: Mutex, + } + + impl FakeClaim { + /// `check_maintenance_claim(stale)`: held while the last heartbeat is + /// within `stale`. + fn held(&self, stale: Duration) -> bool { + self.beats + .lock() + .unwrap() + .last() + .is_some_and(|at| at.elapsed() < stale) + } + fn beats(&self) -> usize { + self.beats.lock().unwrap().len() + } + } + + impl MaintenanceClaim for FakeClaim { + fn generation(&self) -> u64 { + 7 + } + fn heartbeat(&self) -> BoxFuture<'_, Result> { + let lost = *self.lost.lock().unwrap(); + if !lost { + self.beats.lock().unwrap().push(Instant::now()); + } + Box::pin(async move { + Ok(if lost { + Heartbeat::Lost + } else { + Heartbeat::Confirmed + }) + }) + } + fn release(&self) -> BoxFuture<'_, Result<(), StorageError>> { + *self.released.lock().unwrap() = true; + Box::pin(async { Ok(()) }) + } + } + + /// The claim object handed to `HeldClaim`, forwarding to the test's own. + struct ForwardingClaim(Arc); + + impl MaintenanceClaim for ForwardingClaim { + fn generation(&self) -> u64 { + self.0.generation() + } + fn heartbeat(&self) -> BoxFuture<'_, Result> { + self.0.heartbeat() + } + fn release(&self) -> BoxFuture<'_, Result<(), StorageError>> { + self.0.release() + } + } + + fn held_over(fake: &Arc, interval: Duration) -> HeldClaim { + fake.beats.lock().unwrap().push(Instant::now()); + HeldClaim::start(Box::new(ForwardingClaim(Arc::clone(fake))), interval) + } + + /// On a paused clock (the timer and the samples share it, so a stall of + /// the test process cannot read as a missed heartbeat): a held claim + /// stays "held" to the jobs' check through a run of thirty stale + /// windows with no heartbeat from the holder's own loop, a lost + /// heartbeat is reported, and the release stops the timer. + #[tokio::test(start_paused = true)] + async fn the_heartbeat_timer_keeps_the_claim_held_through_a_long_run() { + let fake = Arc::new(FakeClaim::default()); + let stale = Duration::from_secs(3); + let held = held_over(&fake, Duration::from_secs(1)); + assert_eq!(held.generation(), 7); + let started = Instant::now(); + let mut samples = 0; + while started.elapsed() < Duration::from_secs(90) { + assert!( + fake.held(stale), + "the claim read as free {:?} after the take", + started.elapsed() + ); + assert_eq!(held.lost(), None); + samples += 1; + tokio::time::sleep(Duration::from_millis(500)).await; + } + assert!(samples >= 150, "{samples}"); + assert!( + fake.beats() >= 85, + "the timer heartbeated: {}", + fake.beats() + ); + // Taken over: the next heartbeat finds the claim lost. + *fake.lost.lock().unwrap() = true; + tokio::time::sleep(Duration::from_millis(2_500)).await; + assert!( + held.lost().is_some_and(|reason| reason.contains("lost")), + "{:?}", + held.lost() + ); + let beats = fake.beats(); + held.release().await.unwrap(); + assert!(*fake.released.lock().unwrap()); + tokio::time::sleep(Duration::from_secs(5)).await; + assert_eq!(fake.beats(), beats, "no heartbeat after the release"); + } + + /// A handle dropped without `release` stops its timer: the claim gets no + /// further heartbeat and expires by the stale rule, as a dead holder's + /// does, rather than being kept alive by a detached task. + #[tokio::test(start_paused = true)] + async fn a_dropped_handle_stops_the_heartbeat() { + let fake = Arc::new(FakeClaim::default()); + let held = held_over(&fake, Duration::from_secs(1)); + tokio::time::sleep(Duration::from_millis(3_500)).await; + let before = fake.beats(); + assert!(before >= 3, "{before}"); + drop(held); + tokio::time::sleep(Duration::from_secs(10)).await; + assert_eq!(fake.beats(), before, "a dropped handle heartbeats no more"); + assert!(!*fake.released.lock().unwrap(), "a drop is not a release"); + assert!( + !fake.held(Duration::from_secs(3)), + "the claim expires by the stale rule" + ); + } + + /// A claim whose release never answers (a stalled database). + struct StalledRelease; + + impl MaintenanceClaim for StalledRelease { + fn generation(&self) -> u64 { + 1 + } + fn heartbeat(&self) -> BoxFuture<'_, Result> { + Box::pin(async { Ok(Heartbeat::Confirmed) }) + } + fn release(&self) -> BoxFuture<'_, Result<(), StorageError>> { + Box::pin(std::future::pending()) + } + } + + /// `release` is bounded: a catalog that does not answer it ends the + /// release at [`RELEASE_TIMEOUT`] with an error naming the bound, so a + /// command that received a signal exits and leaves the claim to the + /// stale rule, rather than holding on a stalled database. + #[tokio::test(start_paused = true)] + async fn a_release_the_catalog_does_not_answer_ends_at_its_bound() { + let held = HeldClaim::start(Box::new(StalledRelease), Duration::from_secs(1)); + let started = Instant::now(); + let err = held.release().await.unwrap_err(); + assert_eq!(started.elapsed(), RELEASE_TIMEOUT); + assert!( + err.to_string() + .contains("the release did not finish within 10 s"), + "{err}" + ); + } +} diff --git a/crates/engine/src/backup/reconciler.rs b/crates/engine/src/backup/reconciler.rs new file mode 100644 index 000000000..963bc5a0a --- /dev/null +++ b/crates/engine/src/backup/reconciler.rs @@ -0,0 +1,2014 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The reconciler: the periodic worker that resolves catalog rows whose +//! owner is dead, deletes what FAILED and DELETED rows leave in the store, +//! removes tombstones past their retention, and keeps the configured +//! store's health current. +//! +//! One cycle runs every `interval`. It first checks the catalog-level +//! maintenance claim; while a maintenance command holds it the reconciler +//! runs only the duties the command waits on or finishes itself (the +//! dead-owner duties, the store's check and the DELETING rows) and skips +//! the deleting duties it must not race (a FAILED row's prefix, a DELETED +//! row's objects, tombstone retention, the registrations sweep). +//! +//! The claim is read through `check_maintenance_claim(stale_job_after)`, +//! except in a cycle a maintenance command runs itself while it holds the +//! claim (`backup purge` on SQLite, with a zero dead-owner window): that +//! caller's state says so ([`ReconcilerState::for_maintenance_holder`]) and +//! the cycle takes the held branch without reading the claim. Read through +//! the clock, a zero window would see the command's own claim held only +//! when its heartbeat and the check land in the same instant, so the +//! branch would be decided by scheduler timing. +//! +//! The duties, in the order a cycle runs them: +//! +//! - With a store configured: the store's check runs again +//! ([`super::check_configured_store`]: `validate` while it has not passed, +//! then the identity check and manifest probe) and sets the store's +//! health, so a store whose checks run after the listener is up, or that +//! failed at startup or later, is healthy once a check passes, and one +//! that fails is refused from the rest of this cycle on: every duty below +//! that reads the store's health reads the health this cycle's check left. +//! The check runs under the store's bound +//! ([`super::ConfiguredStore::check_timeout`]: 5 s for a filesystem root, +//! 30 s for a bucket), so a store that has stopped answering costs each +//! cycle that bound and nothing more: the duties run every `interval` plus +//! the bound, the check is recorded as a transport failure, and a +//! filesystem check leaves one blocking thread parked on the mount until +//! it answers, after which no check starts until that call returns (the +//! runtime module's in-flight rule), so a mount that stays stalled parks +//! one thread and not one per cycle. The check is also selected against +//! the shutdown token, so a cycle that is checking when the server stops +//! returns at once with no further duty and the drain finishes. +//! - With the startup sync of an in-memory catalog pending and the store +//! healthy after the check: the sync (`super::sync::startup_sync`), +//! bounded by `super::sync::STARTUP_SYNC_TIMEOUT` (60 s) and selected +//! against the shutdown token, so a store that stalls during it delays +//! the duties below by at most the bound; one that stops before it has +//! listed the whole root, reaches the bound, or skips a prefix for a store +//! or catalog error stays pending for the next cycle. +//! - Over the records `list_dead_restore_jobs` returns: a RUNNING restore +//! job record whose `heartbeat_at` is older than `stale_job_after` (on +//! the catalog's clock) has a dead owner. One of kind `legacy` is marked +//! FAILED (`fail_restore_job`, conditioned on the `owner_generation` and +//! `heartbeat_at` the listing read, so a heartbeat that lands after the +//! listing keeps the record its owner's) and its target removed through +//! `remove_restore_target`, never resumed: an in-catalog copy has no data +//! files or progress to resume from. One of kind `manifest` is resumed (`super::restore::resume`), +//! in this order: once the server's shutdown has begun (the jobs' token is +//! cancelled) the record is left as it is, with no takeover and no FAILED +//! write, for the next instance's reconciler (a resume spawned on the way +//! out would stop at its first check and leave the record with a fresh +//! heartbeat, a second stale window for the next instance to wait); once +//! this worker's own token has fired with the jobs' token intact (the +//! server stopping its workers while it holds no serving registration) it +//! is left the same way, for the next set's reconciler, since a resume is +//! a job and the workers' stop would not stop it; +//! while the maintenance claim is held it is failed and its target +//! removed like a legacy one ([`OWNER_DEAD`]; a resume is a job, which +//! the claim excludes), except a record with no recorded progress at all +//! (no `file_index`, no progress row) whose target already holds items, +//! which is failed [`super::restore::TARGET_NOT_EMPTY`] through the same +//! unclaimed write and its target left for the operator, as below (the +//! command's wait is satisfied either way: the record is no longer +//! RUNNING); a record of a store other than the +//! configured one (no store configured included) is failed with +//! [`OTHER_STORE`] (configure that store and restore again; no command +//! re-points a record yet), its target removed by the FAILED-target duty +//! next; while the configured store is unhealthy, or not yet checked, +//! the record is skipped (one line per cycle, naming which); with no free +//! job permit it is skipped to the next cycle; then the backend's +//! `claim_restore_dead` is the test and the takeover (a fresh generation, +//! this host's `spool_host` and a fresh `heartbeat_at`, so a second taker +//! finds the record live at once), and a record it does not hand over has +//! a live owner; the instant the takeover returned is the resumed job's +//! first confirmed beat (the record's `heartbeat_at` was stamped inside +//! the write, so the instant is at or after the beat it stands for by the +//! write's own duration, and the reads that follow count against the +//! heartbeat's window rather than widen it); the takeover after this +//! instance's fifth of one record ([`RESTORE_TAKEOVERS`], counted in +//! [`ReconcilerState`]) fails the record [`TAKEOVERS_EXHAUSTED`] and +//! removes its target, since a record whose every resume ends RUNNING +//! again (a read or the count that fails every time) would otherwise be +//! retaken every stale window; under the claim, a record with no recorded progress at all +//! (no `file_index`, no progress row) whose target already holds more +//! items than the backup's row counts is FAILED +//! [`super::restore::TARGET_NOT_EMPTY`] and its target left for the +//! operator (`extenddb backup remove-target --account
    `), +//! since a catalog restored +//! from a dump can name a restore that finished and was written to since; +//! a target holding no more than the total is a first data file applied +//! in part before its progress was recorded (Cassandra), and the resume +//! re-applies that file with upserts (`exceeds_backup_total`); +//! otherwise the job is spawned through the runtime's tracker and runs +//! from the recorded progress. A FAILED restore job record whose table is +//! still a CREATING restore target, the FAILED-target duty: the FAILED +//! write landed (the job's own, or an earlier cycle's) and the target's +//! removal did not; the backend lists it once the record is stale, and the +//! target is removed through `remove_restore_target` with no second +//! FAILED write, except a record whose reason is `target_not_empty`, +//! whose target is the operator's and is logged with the command. +//! - An in-catalog CREATING backup row whose create is dead (reported by the +//! backend's `abandoned_creating_backups`) is removed through the reader's +//! `delete`, which takes its items with it, unless the backend removed it +//! in the call that reported it (`AbandonedLegacyBackup::removed`), when +//! the reconciler only logs it. +//! - With a store configured: while the store is unhealthy the reconciler +//! acts on no row of the store. While it is healthy, on the rows of the +//! configured store: +//! - Each DELETING row (a DeleteBackup whose deletion did not finish) is +//! finished (`finish_backup_delete` in the engine's backup module): the +//! manifest, then the prefix, then the row to DELETED. +//! - Each CREATING row whose owner is dead is resolved. The backend's +//! `claim_dead_backup` is the test and the takeover (a fresh generation +//! fences the dead job's writes out; `heartbeat_at` is left as it was); +//! a row it does not hand over has a live owner or is another's. Under +//! the claim the store is probed for a complete manifest: +//! `extenddb-manifest.json` and its sidecar present and agreeing, the +//! manifest parsing with the row's ARN, and every data file it lists +//! present under the prefix with the recorded size. A complete manifest +//! makes the row AVAILABLE with the manifest's size, item count and +//! `source_table`; anything else makes it FAILED with reason +//! [`OWNER_DEAD_NO_MANIFEST`], and nothing is deleted here: the +//! FAILED-prefix duty, which runs later in this cycle, deletes it once +//! the row's heartbeat is older than `stale_job_after` on the database's +//! clock: in this cycle on PostgreSQL, MongoDB and Cassandra, where a +//! heartbeat that old is what made the owner dead (on PostgreSQL +//! together with the owner's claim session gone, its advisory lock +//! free); on SQLite, where the slot did, once the dead owner's last +//! beat is that old. A store failure during the probe (not an absent +//! object) leaves the row for the next cycle. The two writes are +//! conditioned on the row being CREATING under the new generation +//! (`job::failure::write_status`); one that matches no row (another +//! reconciler took the row over in the gap, item 31) leaves the row to +//! that owner. +//! - Not while the maintenance claim is held: a FAILED row whose +//! `heartbeat_at` is older than `stale_job_after` (the backend's +//! `list_backup_rows_older_than` on the heartbeat clock) and whose +//! prefix lists objects has its prefix deleted, retried each cycle while +//! the listing shows objects; a DELETED row whose prefix lists objects +//! has its prefix deleted; a FAILED or DELETED row whose prefix lists +//! nothing and whose `completed_at` is older than `failed_row_retention` +//! (the completed clock) is removed. Every age is the database's +//! comparison; the engine holds no clock. +//! - Each deletion of a prefix and each removal of a tombstone, the +//! DELETING rows' included, runs the deletion fence first +//! ([`super::StoreFence`]): `extenddb-store.json` is re-read, and a +//! file that names another catalog (a root adopted since this cycle's +//! check) marks the store unhealthy with nothing deleted. The cycle +//! re-tests the store's health after each store duty, so the duties +//! after the refusal (the dead-owner duty's reads and writes among +//! them) do not run on the root; the refusing duty's later rows are +//! refused with no read of the file (the FAILED-prefix and +//! DELETED-prefix duties still list each prefix first). +//! - Not while the maintenance claim is held, on every backend: the +//! backend's `sweep_restore_registrations` (a no-op but on Cassandra); +//! the progress-row sweep (`sweep_restore_progress` with the table ids of +//! the RUNNING restore targets: PostgreSQL's `restore_progress` rows of +//! any other table go, the other backends keep progress on the record and +//! have nothing to sweep); and, with a spool configured, the spool sweep +//! (`super::restore::spool::sweep_entries`: entries whose table id is +//! not a RUNNING restore with this host's `spool_host`). Both sweeps read +//! their candidates before the records: the progress-row sweep's +//! candidates are the backend's, the spool's are listed +//! (`super::restore::spool::list_entries`) before `list_restore_jobs`, +//! so a restore whose registration commits between the two reads is +//! never a candidate of either. A backend that cannot list the records +//! yet logs once per cycle and the two sweeps are skipped. +//! +//! The worker runs in the server's restartable worker set: while the server +//! holds no serving registration (PostgreSQL, MongoDB and Cassandra) it is +//! stopped with the other workers and a new one started once the server +//! registers again, as `extenddb_server`'s `registration` module doc says; +//! a cycle that is running at the stop finishes or is aborted, and spawns +//! no resume after its token fired (above). Each start builds a fresh +//! [`ReconcilerState`], and its first cycle runs at once; no duty depends +//! on what an earlier set's cycles did. The backup and restore jobs, +//! resumes included, run on the jobs' token, a child of the server's +//! shutdown token, so the workers' stop does not stop them: a job that is +//! running when the registration is lost goes on writing the catalog and +//! the store until it ends or the server stops, beside a migration that +//! may already have taken the catalog (GitHub issue #755 is the gate). +//! +//! Every duty acts only on rows of the configured store (kind, root and +//! id), the legacy duty excepted. A cycle that fails logs and waits for the +//! next; each cycle records the worker metrics the other workers record +//! (`WorkerLastSuccess` and `WorkerCycleLatency` on success, +//! `WorkerErrorCount` on failure, under `QuerySource::BackupReconciler`); +//! a duty's store call that fails (a `get` or `list` of the probe or the +//! deleting duties, a `delete_prefix`; not an absent object, and not the +//! store's check, which sets the store's health instead) records +//! `BackupStoreErrors` under the call's name, as the job's calls do, so a +//! store that refuses one row's calls is counted though the cycle +//! completes; the worker stops at the shutdown token. + +use std::collections::HashSet; +use std::sync::Arc; +use std::time::Duration; + +use extenddb_backup_store::{BackupStore, StoreError}; +use extenddb_core::backup_format::{ + BackupArn, EXTENDDB_MANIFEST_FILE, ExtenddbManifest, MANIFEST_CHECKSUM_FILE, + verify_manifest_checksum, +}; +use extenddb_core::metrics::{MetricsCollector, QuerySource}; +use extenddb_storage::backup_catalog::{ + BackupRow, BackupStatus, JobStatus, RestoreJobRow, RestoreKind, StaleClock, StatusFields, + StoreIdentity, +}; +use extenddb_storage::error::StorageError; +use extenddb_storage::{ + BackupCatalog, CancellationToken, JobPermit, LegacyBackupReader, RestoreEngine, + sleep_or_shutdown, +}; +use futures::TryStreamExt; + +use super::job::JobEnv; +use super::job::failure::{Written, write_status}; +use super::restore::{self, OTHER_STORE, TARGET_NOT_EMPTY}; +use super::{BackupRuntime, ConfiguredStore, StoreFence}; + +/// The two durations a reconciler reads from `[backup]`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ReconcilerSettings { + /// `reconcile_interval_secs`. + pub interval: Duration, + /// `stale_job_after_secs`. + pub stale_job_after: Duration, +} + +/// The reason written on a restore job record whose owner died. +pub const OWNER_DEAD: &str = "owner dead: no heartbeat within stale_job_after_secs"; + +/// The reason written on a CREATING backup row whose owner died without a +/// complete manifest in the store. +pub const OWNER_DEAD_NO_MANIFEST: &str = + "owner dead: no heartbeat within stale_job_after_secs; no complete manifest"; + +/// How many times one instance resumes a dead store-backed restore record +/// before it fails the record instead: a resume that keeps ending with the +/// record RUNNING (a progress read or the final count that fails every time, +/// a load that stops `LeftRunning`) would otherwise be retaken every stale +/// window for as long as the server runs. The count is this instance's, +/// kept in [`ReconcilerState`]; a restart starts it over. +pub const RESTORE_TAKEOVERS: u32 = 5; + +/// The reason written on a RUNNING store-backed restore record that this +/// instance took over [`RESTORE_TAKEOVERS`] times without the restore +/// ending; the target is removed as for [`OWNER_DEAD`]. +pub const TAKEOVERS_EXHAUSTED: &str = + "owner dead: taken over 5 times by one instance without the restore ending"; + +/// What one cycle did. +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub struct CycleReport { + /// Whether the maintenance claim was held during the cycle. + pub maintenance_held: bool, + /// Legacy restore targets failed and removed, by table name. + pub legacy_restores_removed: Vec, + /// Targets of FAILED records removed (the FAILED-target duty), by table + /// name. + pub failed_targets_removed: Vec, + /// Targets a removal found no longer a CREATING restore target (whether + /// this cycle wrote the FAILED or found it), left to their current owner, + /// by table name. + pub targets_left_to_owner: Vec, + /// Manifest restore jobs with a dead owner this cycle took over and + /// spawned a resume of, by table name. + pub manifest_restores_resumed: Vec, + /// Manifest restore jobs with a dead owner this cycle marked FAILED and + /// left their target to the FAILED-target duty or the operator, by table + /// name with the reason: another store, or `target_not_empty`. + pub manifest_restores_failed: Vec<(String, String)>, + /// Manifest restore jobs with a dead owner this cycle marked FAILED + /// `OWNER_DEAD` under the maintenance claim and removed the target of, + /// as a legacy record is, by table name. + pub manifest_restores_failed_and_removed: Vec, + /// Manifest restore jobs with a dead owner skipped to the next cycle: + /// no job permit was free, by table name. + pub restores_skipped_no_permit: Vec, + /// Manifest restore jobs with a dead owner skipped because the + /// configured store is unhealthy, by table name. + pub restores_skipped_store_unhealthy: Vec, + /// FAILED `target_not_empty` records whose target is left for the + /// operator, by table name. + pub targets_left_for_operator: Vec, + /// Progress rows of tables that are not RUNNING restore targets removed + /// (PostgreSQL's `restore_progress`). + pub progress_rows_swept: u64, + /// Spool entries of restores not RUNNING with this host's `spool_host` + /// removed, by table id. + pub spool_entries_swept: Vec, + /// In-catalog CREATING backup rows removed, by ARN. + pub legacy_backups_removed: Vec, + /// The configured store's health after this cycle's check; `None` with + /// no store configured. + pub store_healthy: Option, + /// DELETING rows of the configured store moved to DELETED, by ARN. + pub deletions_finished: Vec, + /// CREATING rows whose owner was dead, resolved to the status given, by + /// ARN. + pub creating_resolved: Vec<(String, BackupStatus)>, + /// CREATING rows this cycle took over whose status write matched no row + /// (another owner took them over first), left to that owner, by ARN. + pub creating_left_to_owner: Vec, + /// FAILED rows past the stale window whose prefix was deleted, by ARN. + pub failed_prefixes_deleted: Vec, + /// DELETED rows whose prefix still listed objects and was deleted, by + /// ARN. + pub deleted_prefixes_swept: Vec, + /// FAILED and DELETED rows past their retention with an empty prefix, + /// removed, by ARN. + pub tombstones_removed: Vec, + /// Backups whose restore registrations the backend swept, by ARN. + pub registrations_swept: Vec, +} + +impl CycleReport { + /// Whether the cycle changed a row or an object. + #[must_use] + pub fn acted(&self) -> bool { + !(self.legacy_restores_removed.is_empty() + && self.failed_targets_removed.is_empty() + && self.manifest_restores_resumed.is_empty() + && self.manifest_restores_failed.is_empty() + && self.manifest_restores_failed_and_removed.is_empty() + && self.progress_rows_swept == 0 + && self.spool_entries_swept.is_empty() + && self.legacy_backups_removed.is_empty() + && self.deletions_finished.is_empty() + && self.creating_resolved.is_empty() + && self.failed_prefixes_deleted.is_empty() + && self.deleted_prefixes_swept.is_empty() + && self.tombstones_removed.is_empty() + && self.registrations_swept.is_empty()) + } +} + +/// What a reconciler carries from one cycle to the next: the worker's +/// shutdown token, which the store's check is selected against (the caller +/// of [`run`] owns the state and hands the same one to every worker it +/// starts, `serve` restarting its workers each time it takes the serving +/// registration again, and the token is set each cycle), and, for a +/// maintenance command's own cycle, that its caller holds the maintenance +/// claim. +#[derive(Debug, Default)] +pub struct ReconcilerState { + shutdown: Option, + /// `Some(true)` when the caller holds the maintenance claim: the cycle + /// takes the held branch and does not read the claim. `None` reads it + /// through `check_maintenance_claim`. + maintenance_held: Option, + /// How many times this instance took over each store-backed restore + /// record, by target table id, for the bound [`RESTORE_TAKEOVERS`]; an + /// entry is removed when the record is failed by the bound. + restore_takeovers: std::collections::HashMap, +} + +impl ReconcilerState { + /// A state whose cycles stop waiting on the store's check when `shutdown` + /// fires (the health is then left as the last check left it). A default + /// state, which commands and tests use, waits the check's bound. + #[must_use] + pub fn with_shutdown(shutdown: CancellationToken) -> Self { + Self { + shutdown: Some(shutdown), + ..Self::default() + } + } + + /// The state of a cycle that a maintenance command runs while it holds + /// the maintenance claim: the cycle takes the held branch without + /// reading the claim, whatever window it is given. + #[must_use] + pub fn for_maintenance_holder() -> Self { + Self { + maintenance_held: Some(true), + ..Self::default() + } + } +} + +/// Run cycles every `settings.interval` until `shutdown` fires. One cycle +/// runs at once at startup, so a restart resolves what the stopped process +/// left without waiting an interval. `backend` names the storage backend in +/// an identity file this worker has to rewrite. `state` is locked for each +/// cycle. +pub async fn run( + storage: Arc, + runtime: Arc, + settings: ReconcilerSettings, + backend: String, + metrics: Arc, + state: Arc>, + shutdown: CancellationToken, +) where + S: BackupCatalog + LegacyBackupReader + RestoreEngine + ?Sized + Send + Sync + 'static, +{ + loop { + let started = std::time::Instant::now(); + let outcome = { + let mut state = state.lock().await; + // The state outlives this worker (it is `serve`'s); the token is + // this worker's, so it is set here, each cycle. + state.shutdown = Some(shutdown.clone()); + cycle( + &storage, + &runtime, + settings.stale_job_after, + &backend, + &mut state, + ) + .await + }; + match outcome { + Ok(report) => { + #[allow(clippy::cast_precision_loss)] + let cycle_us = started.elapsed().as_micros() as f64; + metrics.record_worker_success(QuerySource::BackupReconciler, cycle_us); + if report.acted() { + tracing::info!( + "backup reconciler: failed and removed {} dead legacy restore(s), resumed \ + {}, failed {} and failed and removed {} (under the maintenance claim) \ + dead store-backed restore(s), removed the target(s) of {} FAILED restore \ + record(s) and {} abandoned in-catalog backup(s), finished {} \ + deletion(s), resolved {} dead CREATING row(s), deleted {} FAILED \ + prefix(es), swept {} DELETED prefix(es), removed {} tombstone(s), swept \ + the registrations of {} backup(s), {} progress row(s) and {} spool \ + entry(ies)", + report.legacy_restores_removed.len(), + report.manifest_restores_resumed.len(), + report.manifest_restores_failed.len(), + report.manifest_restores_failed_and_removed.len(), + report.failed_targets_removed.len(), + report.legacy_backups_removed.len(), + report.deletions_finished.len(), + report.creating_resolved.len(), + report.failed_prefixes_deleted.len(), + report.deleted_prefixes_swept.len(), + report.tombstones_removed.len(), + report.registrations_swept.len(), + report.progress_rows_swept, + report.spool_entries_swept.len() + ); + } + } + Err(e) => { + metrics.record_worker_error(QuerySource::BackupReconciler); + tracing::error!("backup reconciler cycle failed: {e}"); + } + } + if !sleep_or_shutdown(&shutdown, settings.interval).await { + return; + } + } +} + +/// One cycle: the maintenance check (or the caller's word that it holds +/// the claim, from `state`), then the duties. `stale_job_after` is the +/// dead-owner window; it is also the claim's stale window when the claim is +/// read. +/// +/// # Errors +/// +/// A catalog read that fails ends the cycle; a failed write on one row is +/// logged and the cycle goes on to the next row. +pub async fn cycle( + storage: &Arc, + runtime: &BackupRuntime, + stale_job_after: Duration, + backend: &str, + state: &mut ReconcilerState, +) -> Result +where + S: BackupCatalog + LegacyBackupReader + RestoreEngine + ?Sized + Send + Sync + 'static, +{ + let maintenance_held = match state.maintenance_held { + Some(held) => held, + None => storage.check_maintenance_claim(stale_job_after).await?, + }; + let mut report = CycleReport { + maintenance_held, + ..CycleReport::default() + }; + if report.maintenance_held { + if state.maintenance_held.is_some() { + tracing::info!( + "backup reconciler: run by a maintenance command before its own work; running \ + only the duties that resolve rows whose owner is dead, the store's check and \ + the DELETING rows" + ); + } else { + tracing::info!( + "backup reconciler: a maintenance claim is held; running only the duties that \ + resolve rows whose owner is dead, the store's check and the DELETING rows" + ); + } + } + + // The store's check first, so every duty of this cycle that reads the + // store's health reads what this cycle's check left. + if let Some(store) = runtime.configured_store() { + let check = super::check_configured_store(store, &**storage, backend); + let outcome = match &state.shutdown { + Some(shutdown) => tokio::select! { + outcome = check => Some(outcome), + () = shutdown.cancelled() => None, + }, + None => Some(check.await), + }; + match outcome { + Some(outcome) => { + // The defaulted SQLite root's rule, applied to the first + // check when the startup checks did not finish and left the + // store checking: a failure with no rows for the root is the + // warning the startup check would have given, and the store + // is left unconfigured rather than unhealthy. + if store.is_defaulted() + && store.health() == super::StoreHealth::Checking + && let Err(problem) = &outcome + && !matches!(problem, super::StoreProblem::Catalog(_)) + && storage + .read_store_ids(store.kind, &store.root) + .await + .is_ok_and(|ids| ids.is_empty()) + { + tracing::warn!( + "backup store: the default root {} failed its first check ({problem}); \ + the catalog has no backup rows for it, so the store is unconfigured and \ + CreateBackup writes in-catalog backups. Fix the root (owned by this \ + user, mode 0700, under directories others cannot write to, its \ + extenddb-store.json naming this catalog), configure [backup.store] to \ + choose another, or set kind = \"disabled\" to silence this", + store.root + ); + store.set_aside(); + return Ok(report); + } + store.record_check(outcome); + } + None => { + tracing::info!( + "backup store ({}) check was stopped by shutdown; its health is left as the \ + last check left it", + store.kind + ); + report.store_healthy = Some(store.is_healthy()); + return Ok(report); + } + } + report.store_healthy = Some(store.is_healthy()); + // The startup sync of an in-memory catalog, once its store is + // healthy: an S3 store's first check is this cycle's, and a sync + // the server's start did not finish is left pending for here + // (`super::sync::startup_sync`). + if store.startup_sync_pending() { + let sync = super::sync::startup_sync(&**storage, runtime); + match &state.shutdown { + Some(shutdown) => tokio::select! { + _ = sync => {} + () = shutdown.cancelled() => return Ok(report), + }, + None => { + sync.await; + } + } + } + } + + let mut store_not_ready_logged = false; + for job in storage.list_dead_restore_jobs(stale_job_after).await? { + match (job.status, job.kind) { + // A `target_not_empty` target is the operator's. + (JobStatus::Failed, _) if job.failure_reason.as_deref() == Some(TARGET_NOT_EMPTY) => { + tracing::warn!( + "the restore into {} (account {}) from {} is FAILED {TARGET_NOT_EMPTY}: its \ + target ({}) holds items no record of the restore accounts for and is left \ + for the operator; inspect it at the storage level and remove it with \ + extenddb backup remove-target --account {} {}", + job.target.table_name, + job.target.account_id, + job.source_backup_arn, + job.target.table_id, + job.target.account_id, + job.target.table_name + ); + report + .targets_left_for_operator + .push(job.target.table_name.clone()); + } + // The FAILED-target duty, whatever the record's kind: the job is + // over and only its target is left. + (JobStatus::Failed, _) => { + match remove_restore_target( + &**storage, + &job, + "was FAILED when the reconciler reached it", + ) + .await + { + Removal::Removed => report + .failed_targets_removed + .push(job.target.table_name.clone()), + Removal::LeftToOwner => report + .targets_left_to_owner + .push(job.target.table_name.clone()), + Removal::Failed => {} + } + } + (_, RestoreKind::Legacy) => match remove_dead_legacy_restore(&**storage, &job).await { + Some(Removal::Removed) => report + .legacy_restores_removed + .push(job.target.table_name.clone()), + Some(Removal::LeftToOwner) => report + .targets_left_to_owner + .push(job.target.table_name.clone()), + Some(Removal::Failed) | None => {} + }, + (_, RestoreKind::Manifest) => { + let name = job.target.table_name.clone(); + let stopping = state + .shutdown + .as_ref() + .is_some_and(CancellationToken::is_cancelled); + match resume_dead_manifest_restore( + storage, + runtime, + stale_job_after, + &job, + report.maintenance_held, + stopping, + &mut state.restore_takeovers, + ) + .await + { + Resumed::Spawned => report.manifest_restores_resumed.push(name), + Resumed::Failed(reason) => report.manifest_restores_failed.push((name, reason)), + Resumed::FailedAndRemoved(removal) => match removal { + Removal::Removed => report.manifest_restores_failed_and_removed.push(name), + Removal::LeftToOwner => report.targets_left_to_owner.push(name), + Removal::Failed => {} + }, + Resumed::NoPermit => report.restores_skipped_no_permit.push(name), + Resumed::StoreUnhealthy => { + if !store_not_ready_logged { + store_not_ready_logged = true; + let state = + match runtime.configured_store().map(ConfiguredStore::health) { + Some(super::StoreHealth::Checking) => "not yet checked", + _ => "unhealthy", + }; + tracing::warn!( + "the configured backup store is {state}; dead store-backed \ + restores are not resumed this cycle" + ); + } + report.restores_skipped_store_unhealthy.push(name); + } + Resumed::LeftToOwner | Resumed::Error => {} + } + } + } + } + + for backup in storage.abandoned_creating_backups(stale_job_after).await? { + if backup.removed { + // The backend removed the row in the call that reported it + // (SQLite: one fenced write under its lock). + tracing::warn!( + "removed in-catalog backup {} (account {}): its CreateBackup did not finish \ + and no process owns it", + backup.backup_arn, + backup.account_id + ); + report.legacy_backups_removed.push(backup.backup_arn); + continue; + } + match storage.delete(&backup.account_id, &backup.backup_arn).await { + Ok(_) => { + tracing::warn!( + "removed in-catalog backup {} (account {}): its CreateBackup did not finish \ + and no process owns it", + backup.backup_arn, + backup.account_id + ); + report.legacy_backups_removed.push(backup.backup_arn); + } + Err(e) => tracing::error!( + "could not remove the abandoned in-catalog backup {} (account {}): {e}", + backup.backup_arn, + backup.account_id + ), + } + } + + if let Some(store) = runtime.configured_store() + && let Some(identity) = store.identity().filter(|_| store.is_healthy()) + { + let calls = StoreCalls { + store: &*store.store, + metrics: runtime.metrics().map(|m| &**m), + }; + let fence = store.fence(); + // A fence refusal makes the store unhealthy (a root another catalog + // adopted since this cycle's check): the store duties after it do + // not run on the root, the dead-owner duty's reads included. + let healthy = || store.is_healthy(); + report.deletions_finished = + finish_deleting_rows(&**storage, store, &fence, &identity).await?; + if healthy() { + (report.creating_resolved, report.creating_left_to_owner) = + resolve_dead_creating_rows(&**storage, calls, &identity, stale_job_after).await?; + } + if !report.maintenance_held && healthy() { + let retention = runtime.job_settings().failed_row_retention; + let duty = Duty { + storage: &**storage, + store: calls, + fence: &fence, + }; + report.failed_prefixes_deleted = + delete_stale_failed_prefixes(duty, &identity, stale_job_after).await?; + if healthy() { + report.deleted_prefixes_swept = sweep_deleted_prefixes(duty, &identity).await?; + } + if healthy() { + report.tombstones_removed = remove_tombstones(duty, &identity, retention).await?; + } + } + } + + if !report.maintenance_held { + report.registrations_swept = storage.sweep_restore_registrations().await?; + (report.progress_rows_swept, report.spool_entries_swept) = + sweep_restore_leftovers(&**storage, runtime.configured_store()).await; + } + Ok(report) +} + +/// What the manifest arm did with one dead RUNNING record. +enum Resumed { + /// Taken over and the job spawned. + Spawned, + /// FAILED with the reason, the target left to the FAILED-target duty + /// (another store) or to the operator (`target_not_empty`). + Failed(String), + /// FAILED `OWNER_DEAD` and the target removed (or found moved, or the + /// removal failed), as a legacy record is, under the maintenance claim. + FailedAndRemoved(Removal), + /// No job permit was free; the next cycle retries. + NoPermit, + /// The configured store is unhealthy; the next cycle retries. + StoreUnhealthy, + /// The backend did not hand the record over: its owner is alive, or + /// another reconciler took it; or the server is shutting down and the + /// record is left for the next reconciler. + LeftToOwner, + /// A catalog or store call failed; logged, the next cycle retries. + Error, +} + +/// The manifest arm (module doc). `takeovers` counts this instance's +/// takeovers of each record for the bound [`RESTORE_TAKEOVERS`]. +async fn resume_dead_manifest_restore( + storage: &Arc, + runtime: &BackupRuntime, + stale_job_after: Duration, + job: &RestoreJobRow, + maintenance_held: bool, + stopping: bool, + takeovers: &mut std::collections::HashMap, +) -> Resumed +where + S: BackupCatalog + RestoreEngine + ?Sized + Send + Sync + 'static, +{ + let target = &job.target; + let what = format!( + "the restore of {} into {} (account {})", + job.source_backup_arn, target.table_name, target.account_id + ); + // Once the server's shutdown has begun the jobs' token is cancelled: a + // takeover now would spawn a resume that stops at its first check and + // leaves the record with this instance's fresh heartbeat, so the next + // instance waits a second stale window for it; a FAILED write on the + // way out is no better. The record is left as it is for the next + // reconciler to meet. The same holds once this reconciler's own token + // fires while the jobs' token does not (`stopping`: the server stops its + // workers while it holds no serving registration and starts a new set + // once it holds one): a resume is a job, on the jobs' token, which the + // workers' stop does not stop, so the record is left for the next set's + // reconciler. + if runtime.shutdown().is_cancelled() { + tracing::info!( + "{what} has no live owner and the server is shutting down; it is left for the next \ + reconciler" + ); + return Resumed::LeftToOwner; + } + if stopping { + tracing::info!( + "{what} has no live owner and the reconciler is stopping; it is left for the next \ + reconciler" + ); + return Resumed::LeftToOwner; + } + if maintenance_held { + tracing::info!( + "{what} has no live owner and a maintenance claim is held; it is failed rather than \ + resumed" + ); + // The `target_not_empty` guard first, as on the takeover arm: a + // record with no progress over a target that holds items is the + // operator's, and `OWNER_DEAD` with the target's removal would drop + // what the record does not account for. + match unaccounted_items(&**storage, job).await { + Ok(Some(count)) => { + return match storage + .fail_restore_job( + target, + job.owner_generation, + job.heartbeat_at, + TARGET_NOT_EMPTY, + ) + .await + { + Ok(true) => { + tracing::error!( + "{what} recorded no progress and its target ({}) holds {count} \ + item(s); the record is FAILED {TARGET_NOT_EMPTY} and the target is \ + left for the operator: extenddb backup remove-target --account {} {}", + target.table_id, + target.account_id, + target.table_name + ); + Resumed::Failed(TARGET_NOT_EMPTY.to_owned()) + } + Ok(false) => Resumed::LeftToOwner, + Err(e) => { + tracing::error!( + "{what}: the FAILED write failed; the next cycle retries: {e}" + ); + Resumed::Error + } + }; + } + Ok(None) => {} + Err(e) => { + tracing::error!( + "{what}: its progress or its target could not be read; the next cycle \ + retries: {e}" + ); + return Resumed::Error; + } + } + return match remove_dead_legacy_restore(&**storage, job).await { + Some(removal) => Resumed::FailedAndRemoved(removal), + None => Resumed::LeftToOwner, + }; + } + let store = runtime.configured_store_arc(); + // An instance with no configured store does not own any store-backed + // record: it leaves every one alone for the instance that holds its + // store, rather than failing records it cannot read (an instance without + // `[backup.store]` had failed every dead store-backed record it listed + // `OTHER_STORE`). Only an instance that has a store, and whose store is + // not the record's, fails the record OTHER_STORE. + if store.is_none() { + tracing::info!( + "{what} has no live owner and this instance has no configured store; it is left for \ + the instance that holds the record's store" + ); + return Resumed::LeftToOwner; + } + let of_this_store = job + .store + .as_ref() + .is_some_and(|row_store| runtime.is_configured_store(row_store)); + if !of_this_store { + tracing::error!( + "{what} has no live owner and reads a store other than the configured one; the \ + record is FAILED and its target removed once the record is stale: {OTHER_STORE}" + ); + // Conditioned on the claim the listing read, as the legacy arm's + // write is: a heartbeat that landed since keeps the record its + // owner's (`Ok(false)`). + return match storage + .fail_restore_job(target, job.owner_generation, job.heartbeat_at, OTHER_STORE) + .await + { + Ok(true) => Resumed::Failed(OTHER_STORE.to_owned()), + Ok(false) => Resumed::LeftToOwner, + Err(e) => { + tracing::error!("{what} could not be marked FAILED; the next cycle retries: {e}"); + Resumed::Error + } + }; + } + let Some(store) = store.filter(|store| store.is_healthy()) else { + return Resumed::StoreUnhealthy; + }; + let env = match JobEnv::new(runtime, store) { + Ok(env) => env, + Err(_) => return Resumed::StoreUnhealthy, + }; + let Some(permit) = JobPermit::try_acquire(runtime.slots()) else { + tracing::info!("{what} has no live owner; no job permit is free, the next cycle retries"); + return Resumed::NoPermit; + }; + let spool_host = env.spool.as_ref().map(|s| s.spool_host.clone()); + let claim = match storage + .claim_restore_dead(target, stale_job_after, spool_host.as_deref()) + .await + { + Ok(Some(claim)) => claim, + Ok(None) => return Resumed::LeftToOwner, + Err(e) => { + tracing::error!( + "could not test whether the owner of {what} is dead; the next cycle retries: {e}" + ); + return Resumed::Error; + } + }; + // The resumed job's first confirmed beat (module doc): the takeover + // stamped `heartbeat_at` before it returned. + let claimed_at = tokio::time::Instant::now(); + // The cycle read the maintenance claim once at its start; a command may + // have taken it since. Re-read it now the record is this instance's, as + // the request's `start` re-checks after its registration: held, the + // resume is left for the maintenance arm of a later cycle rather than + // run beside a command, and the record keeps this takeover's fresh + // heartbeat. + match storage.check_maintenance_claim(stale_job_after).await { + Ok(false) => {} + Ok(true) => { + tracing::info!( + "{what} was taken over but a maintenance claim is now held; the resume is left \ + for the next cycle" + ); + return Resumed::LeftToOwner; + } + Err(e) => { + tracing::error!( + "{what} was taken over but the maintenance claim could not be re-read; the next \ + cycle retries: {e}" + ); + return Resumed::Error; + } + } + // The bound on this instance's takeovers of the record + // (`RESTORE_TAKEOVERS`): the takeover after the fifth fails the record + // and removes its target as the arm below does for a target with no + // tables, since a record whose every resume ends RUNNING again would + // otherwise be retaken every stale window for as long as the server runs. + let taken = takeovers.entry(target.table_id.clone()).or_insert(0); + *taken = taken.saturating_add(1); + if *taken > RESTORE_TAKEOVERS { + takeovers.remove(&target.table_id); + let fields = StatusFields { + failure_reason: Some(TAKEOVERS_EXHAUSTED.to_owned()), + completed: true, + ..StatusFields::default() + }; + return match write_status(&*claim, JobStatus::Running, JobStatus::Failed, &fields).await { + Written::Matched => { + tracing::error!( + "{what} has no live owner and was taken over {RESTORE_TAKEOVERS} times by \ + this instance without ending; the record is FAILED and the target ({}) \ + removed", + target.table_id + ); + Resumed::FailedAndRemoved( + remove_restore_target(&**storage, job, "was marked FAILED by the reconciler") + .await, + ) + } + Written::NotMatched(why) => { + tracing::info!("{what} was no longer this cycle's ({why}); left to its owner"); + Resumed::LeftToOwner + } + Written::Error(e) => { + tracing::error!("{what}: the FAILED write failed; the next cycle retries: {e}"); + Resumed::Error + } + }; + } + // The progress under the claim: the record as it is now and, where the + // backend keeps them apart, its progress rows. + let (record, progress_rows) = match ( + storage.read_restore_job(target).await, + storage.read_restore_progress(target).await, + ) { + (Ok(Some(record)), Ok(rows)) => (record, rows), + (Ok(None), _) => { + tracing::info!("{what} was gone when its progress was read; left to whoever moved it"); + return Resumed::LeftToOwner; + } + (Err(e), _) | (_, Err(e)) => { + tracing::error!( + "{what} was taken over but its progress could not be read; the next cycle \ + retries once the record is stale again: {e}" + ); + return Resumed::Error; + } + }; + // The target's data tables must exist: every backend commits the record + // before the data DDL (the trait doc of `restore_target_ready`), so an + // owner killed in between leaves a RUNNING record over a target with no + // table, or on MongoDB with its index metadata missing. Such a record is + // never resumable, and the count below would fail on the missing table + // and leave the takeover's fresh heartbeat, so the record would be + // retaken every stale window and never fail. It is failed and its target + // removed instead, under the claim this takeover holds. + match storage.restore_target_ready(target).await { + Ok(true) => {} + Ok(false) => { + let fields = StatusFields { + failure_reason: Some(OWNER_DEAD.to_owned()), + completed: true, + ..StatusFields::default() + }; + return match write_status(&*claim, JobStatus::Running, JobStatus::Failed, &fields).await + { + Written::Matched => { + tracing::error!( + "{what} has no live owner and its target ({}) has no data tables (its \ + owner died between the registration and the data DDL); the record is \ + FAILED and the target removed", + target.table_id + ); + Resumed::FailedAndRemoved( + remove_restore_target( + &**storage, + job, + "was marked FAILED by the reconciler", + ) + .await, + ) + } + Written::NotMatched(why) => { + tracing::info!("{what} was no longer this cycle's ({why}); left to its owner"); + Resumed::LeftToOwner + } + Written::Error(e) => { + tracing::error!("{what}: the FAILED write failed; the next cycle retries: {e}"); + Resumed::Error + } + }; + } + Err(e) => { + tracing::error!( + "{what} was taken over but whether its target has data tables could not be read; \ + the next cycle retries: {e}" + ); + return Resumed::Error; + } + } + if has_no_progress(&record, &progress_rows) { + let count = match storage.count_restore_target(target).await { + Ok(count) => count, + Err(e) => { + tracing::error!( + "{what} was taken over but its target could not be counted; the next cycle \ + retries once the record is stale again: {e}" + ); + return Resumed::Error; + } + }; + let unaccounted = match exceeds_backup_total(&**storage, job, count).await { + Ok(unaccounted) => unaccounted, + Err(e) => { + tracing::error!( + "{what} was taken over but its backup's row could not be read; the next \ + cycle retries once the record is stale again: {e}" + ); + return Resumed::Error; + } + }; + if count > 0 && !unaccounted { + tracing::info!( + "{what} recorded no progress and its target ({}) holds {count} item(s), no more \ + than the backup's total: a first data file applied in part before its progress \ + was recorded; the resume re-applies it with upserts", + target.table_id + ); + } + if unaccounted { + let fields = StatusFields { + failure_reason: Some(TARGET_NOT_EMPTY.to_owned()), + completed: true, + ..StatusFields::default() + }; + return match write_status(&*claim, JobStatus::Running, JobStatus::Failed, &fields).await + { + Written::Matched => { + tracing::error!( + "{what} recorded no progress and its target ({}) holds {count} item(s); \ + the record is FAILED {TARGET_NOT_EMPTY} and the target is left for the \ + operator: extenddb backup remove-target --account {} {}", + target.table_id, + target.account_id, + target.table_name + ); + Resumed::Failed(TARGET_NOT_EMPTY.to_owned()) + } + Written::NotMatched(why) => { + tracing::info!("{what} was no longer this cycle's ({why}); left to its owner"); + Resumed::LeftToOwner + } + Written::Error(e) => { + tracing::error!("{what}: the FAILED write failed; the next cycle retries: {e}"); + Resumed::Error + } + }; + } + } + // The record's progress as recorded; the job's own start line names the + // first file it opens. + tracing::info!( + "{what} has no live owner; taken over by this instance and resumed with {}", + match (record.file_index, record.items_applied) { + (Some(file), Some(items)) => + format!("{items} items of data file {file} recorded as applied"), + (Some(file), None) => format!("data file {file} recorded as applied"), + (None, _) if !progress_rows.is_empty() => + format!("{} data file(s) recorded as applied", progress_rows.len()), + (None, _) => "nothing recorded as applied".to_owned(), + } + ); + let limits = (&**runtime.limits()).into(); + runtime.spawn_job(restore::resume( + Arc::clone(storage), + env, + permit, + claim, + restore::DeadRecord { + row: record, + recorded_spool_host: job.spool_host.clone(), + progress_rows, + claimed_at, + }, + limits, + )); + Resumed::Spawned +} + +/// Whether a RUNNING record has no progress recorded at all: no +/// `file_index` on the record and no progress row beside it. +fn has_no_progress(record: &RestoreJobRow, progress_rows: &[(u32, u64)]) -> bool { + record.file_index.is_none() && progress_rows.is_empty() +} + +/// The `target_not_empty` test as the maintenance-held arm runs it, with +/// no claim on the record: the record and its progress rows read, and when +/// [`has_no_progress`] the target counted; `Some(count)` when the target +/// holds items no record accounts for ([`exceeds_backup_total`]), `None` +/// when progress is recorded, the target is empty or holds no more than the +/// backup's total, or the record is gone (the caller's write then finds it +/// so). +async fn unaccounted_items( + storage: &S, + job: &RestoreJobRow, +) -> Result, StorageError> { + let target = &job.target; + let Some(record) = storage.read_restore_job(target).await? else { + return Ok(None); + }; + let progress_rows = storage.read_restore_progress(target).await?; + if !has_no_progress(&record, &progress_rows) { + return Ok(None); + } + let count = storage.count_restore_target(target).await?; + Ok(exceeds_backup_total(storage, job, count) + .await? + .then_some(count)) +} + +/// The `target_not_empty` rule's test: whether `count` items in a target +/// with no recorded progress are more than the backup holds (its row's +/// `item_count`). At or below the total they are a first data file applied +/// in part before its progress was recorded (Cassandra records a batch's +/// progress after the batch's writes, so a crash inside the first batch +/// leaves this state), and the resume re-applies the first file with +/// `reapplying` set, every backend's load being idempotent for a file +/// resumed that way. Above the total the target holds items the backup does +/// not account for (a catalog restored from a dump naming a restore that +/// finished and was written to since), the operator's. A count of zero is +/// never unaccounted; a backup row this catalog does not hold, or an ARN +/// this build cannot parse, leaves every item unaccounted. +async fn exceeds_backup_total( + storage: &S, + job: &RestoreJobRow, + count: u64, +) -> Result { + if count == 0 { + return Ok(false); + } + let Ok(arn) = BackupArn::parse(&job.source_backup_arn) else { + return Ok(true); + }; + let total = storage + .read_backup_row(&arn) + .await? + .map(|row| u64::try_from(row.item_count).unwrap_or(0)); + Ok(total.is_none_or(|total| count > total)) +} + +/// The progress-row and spool sweeps (module doc). The spool directory is +/// listed first, then the RUNNING records, and only entries of that listing +/// that no record of this host names are removed, so a restore whose +/// registration commits and whose entry opens between the two reads is +/// never a candidate (its entry is not in the listing, or its record is in +/// the records). A listing the backend cannot make is logged and skips +/// both. +async fn sweep_restore_leftovers( + storage: &S, + store: Option<&ConfiguredStore>, +) -> (u64, Vec) { + let spool = store.and_then(|store| store.spool.as_ref()); + let entries = match spool.map(|spool| restore::spool::list_entries(&spool.path)) { + None => Vec::new(), + Some(Ok(entries)) => entries, + Some(Err(e)) => { + tracing::error!( + "the restore spool {} could not be listed; the spool sweep is skipped this \ + cycle: {e}", + spool + .map(|s| s.path.display().to_string()) + .unwrap_or_default() + ); + Vec::new() + } + }; + let running = match storage.list_restore_jobs(Some(JobStatus::Running)).await { + Ok(rows) => rows, + Err(e) => { + tracing::warn!( + "the restore job records could not be listed; the progress-row and spool sweeps \ + are skipped this cycle: {e}" + ); + return (0, Vec::new()); + } + }; + let running_ids: Vec = running.iter().map(|r| r.target.table_id.clone()).collect(); + let progress_rows = match storage.sweep_restore_progress(&running_ids).await { + Ok(n) => { + if n > 0 { + tracing::info!( + "removed {n} restore progress row(s) of tables that are not RUNNING restore \ + targets" + ); + } + n + } + Err(e) => { + tracing::error!( + "the restore progress rows could not be swept; the next cycle retries: {e}" + ); + 0 + } + }; + let Some(spool) = spool else { + return (progress_rows, Vec::new()); + }; + let this_host: HashSet = running + .iter() + .filter(|r| r.spool_host.as_deref() == Some(spool.spool_host.as_str())) + .map(|r| r.target.table_id.clone()) + .collect(); + let swept = restore::spool::sweep_entries(entries, &this_host); + (progress_rows, swept) +} + +/// The duty for CREATING rows of the configured store whose owner is dead +/// (module doc). Returns the rows resolved with their new status and the +/// rows whose write matched no row, left to their new owner; a row the +/// backend does not hand over is skipped, a row whose probe or write fails +/// is logged and left for the next cycle. +async fn resolve_dead_creating_rows( + storage: &S, + store: StoreCalls<'_>, + identity: &StoreIdentity, + stale_job_after: Duration, +) -> Result<(Vec<(String, BackupStatus)>, Vec), StorageError> { + let rows: Vec = storage + .list_backup_rows(Some(BackupStatus::Creating), identity) + .try_collect() + .await?; + let mut resolved = Vec::new(); + let mut left_to_owner = Vec::new(); + for row in rows { + let claim = match storage.claim_dead_backup(&row.key(), stale_job_after).await { + Ok(Some(claim)) => claim, + Ok(None) => continue, + Err(e) => { + tracing::error!( + "could not test whether the owner of CREATING backup {} (account {}) is \ + dead; the next cycle retries: {e}", + row.backup_arn, + row.account_id + ); + continue; + } + }; + let arn = match BackupArn::parse(&row.backup_arn) { + Ok(arn) => arn, + Err(e) => { + tracing::error!( + "CREATING row {} carries an ARN this build cannot parse; left as it is: {e}", + row.backup_arn + ); + continue; + } + }; + let (to, fields) = match probe_manifest(store, &arn).await { + Ok(Probe::Complete(fields)) => (JobStatus::Available, fields), + Ok(Probe::Incomplete(why)) => { + tracing::warn!( + "backup {} (account {}): its owner is dead and the store holds no complete \ + manifest ({why}); the row is FAILED and its prefix is deleted once the row's \ + heartbeat is stale", + row.backup_arn, + row.account_id + ); + ( + JobStatus::Failed, + StatusFields { + failure_reason: Some(OWNER_DEAD_NO_MANIFEST.to_owned()), + completed: true, + ..StatusFields::default() + }, + ) + } + Err(e) => { + tracing::error!( + "backup {} (account {}): its owner is dead and the store could not be \ + probed for its manifest; the next cycle retries: {e}", + row.backup_arn, + row.account_id + ); + continue; + } + }; + match write_status(&*claim, JobStatus::Creating, to, &fields).await { + Written::Matched => { + tracing::info!( + "backup {} (account {}): its owner was dead; the row is now {to}", + row.backup_arn, + row.account_id + ); + let status = match to { + JobStatus::Available => BackupStatus::Available, + _ => BackupStatus::Failed, + }; + resolved.push((row.backup_arn, status)); + } + Written::NotMatched(why) => { + tracing::info!( + "backup {} was no longer this cycle's to resolve ({why}); left to its owner", + row.backup_arn + ); + left_to_owner.push(row.backup_arn); + } + Written::Error(e) => tracing::error!( + "backup {} (account {}): the {to} write failed; the next cycle retries: {e}", + row.backup_arn, + row.account_id + ), + } + drop(claim); + } + Ok((resolved, left_to_owner)) +} + +/// The configured store as the reconciler's duties call it: each failed +/// call (not an absent object) records `BackupStoreErrors` under the call's +/// name (`get`, `list` or `delete_prefix`), as the job records its own, so +/// a store that refuses the reconciler's calls moves the metric even when +/// every cycle completes. +#[derive(Clone, Copy)] +pub(super) struct StoreCalls<'a> { + pub(super) store: &'a dyn BackupStore, + pub(super) metrics: Option<&'a MetricsCollector>, +} + +impl StoreCalls<'_> { + /// `e`, with `BackupStoreErrors` recorded under `op` when metrics are + /// collected. + pub(super) fn failed(self, op: &str, e: StoreError) -> StoreError { + if let Some(metrics) = self.metrics { + metrics.record_backup_store_error(op); + } + e + } + + /// One object's bytes within `bound` ([`read_object`]). + async fn read_object(self, key: &str, bound: u64) -> Result { + read_object(self.store, key, bound) + .await + .map_err(|e| self.failed("get", e)) + } + + /// The prefix's objects, key to size. + async fn list_sizes( + self, + prefix: &str, + ) -> Result, StoreError> { + self.store + .list(prefix) + .map_ok(|meta| (meta.key, meta.size)) + .try_collect() + .await + .map_err(|e| self.failed("list", e)) + } + + /// Whether the prefix lists at least one object. + async fn prefix_has_objects(self, prefix: &str) -> Result { + match self.store.list(prefix).try_next().await { + Ok(first) => Ok(first.is_some()), + Err(e) => Err(self.failed("list", e)), + } + } + + async fn delete_prefix(self, prefix: &str) -> Result { + self.store + .delete_prefix(prefix) + .await + .map_err(|e| self.failed("delete_prefix", e)) + } +} + +/// What the manifest probe found under a dead owner's prefix. +enum Probe { + /// A complete, verified manifest: the AVAILABLE write's fields. + Complete(StatusFields), + /// No manifest, or one the reader refuses, or a data file missing or of + /// another size; the text says which. + Incomplete(String), +} + +/// The manifest probe of the dead-owner duty (module doc): the checks of +/// [`verify_prefix`], with the manifest's ARN required to be the row's (its +/// region and partition included) before the prefix is listed. A store error +/// other than an absent object is the `Err`. +async fn probe_manifest(store: StoreCalls<'_>, arn: &BackupArn) -> Result { + let prefix = arn.store_prefix(); + let manifest = match read_verified_manifest(store, &prefix).await? { + Verified::Complete(manifest) => manifest, + Verified::Incomplete(why) => return Ok(Probe::Incomplete(why)), + }; + match manifest.backup_arn() { + Ok(found) if found == *arn => {} + Ok(found) => { + return Ok(Probe::Incomplete(format!( + "the manifest names {}, not the row's backup", + found.to_arn_string() + ))); + } + Err(e) => { + return Ok(Probe::Incomplete(format!( + "the manifest's ARN is refused: {e}" + ))); + } + } + let listed = store.list_sizes(&prefix).await?; + if let Some(why) = missing_data_file(&manifest, &prefix, &listed) { + return Ok(Probe::Incomplete(why)); + } + Ok(Probe::Complete(StatusFields { + size_bytes: Some(manifest.backup_size_bytes), + item_count: Some(manifest_item_count(&manifest)), + failure_reason: None, + source_table_details_json: serde_json::to_value(&manifest.source_table).ok(), + completed: true, + })) +} + +/// What [`verify_prefix`] found under a prefix. +pub(super) enum Verified { + /// A manifest whose sidecar agrees, that the reader accepts, and whose + /// every data file is listed under the prefix at its recorded size. + Complete(Box), + /// No manifest, or one without its sidecar or not matching it, or one + /// the reader refuses, or a data file missing or of another size; the + /// text says which. + Incomplete(String), +} + +/// The checks a prefix must pass before anything reads it as a backup, in +/// order: `extenddb-manifest.json` read within [`MANIFEST_READ_BOUND`], its +/// sidecar within [`SIDECAR_READ_BOUND`], the sidecar's digest equal to the +/// manifest's, the manifest accepted by +/// [`ExtenddbManifest::from_json_bytes`] (which also checks that its ARN, +/// `source_table` and creation instant agree), then every `data_files` key +/// present under the prefix with its `bytes`. `listed` is the prefix's +/// objects (key to size) when the caller already listed them (sync, which +/// lists the whole root); `None` lists the prefix here. The manifest's ARN +/// is the caller's to compare: the dead-owner duty requires the row's ARN, +/// sync the listed path's components. +/// +/// # Errors +/// +/// A store error other than an absent object. +pub(super) async fn verify_prefix( + store: StoreCalls<'_>, + prefix: &str, + listed: Option<&std::collections::HashMap>, +) -> Result { + let manifest = match read_verified_manifest(store, prefix).await? { + Verified::Complete(manifest) => manifest, + incomplete @ Verified::Incomplete(_) => return Ok(incomplete), + }; + let listed_here; + let listed = match listed { + Some(listed) => listed, + None => { + listed_here = store.list_sizes(prefix).await?; + &listed_here + } + }; + Ok(match missing_data_file(&manifest, prefix, listed) { + Some(why) => Verified::Incomplete(why), + None => Verified::Complete(manifest), + }) +} + +/// The first three checks of [`verify_prefix`]: the manifest and its +/// sidecar read within their bounds, the digest, the reader's acceptance. +async fn read_verified_manifest( + store: StoreCalls<'_>, + prefix: &str, +) -> Result { + let manifest = match store + .read_object( + &format!("{prefix}{EXTENDDB_MANIFEST_FILE}"), + MANIFEST_READ_BOUND, + ) + .await? + { + ReadObject::Bytes(bytes) => bytes, + ReadObject::Absent => return Ok(Verified::Incomplete("no manifest".to_owned())), + ReadObject::Oversize => { + return Ok(Verified::Incomplete(format!( + "a manifest over {MANIFEST_READ_BOUND} bytes" + ))); + } + }; + let sidecar = match store + .read_object( + &format!("{prefix}{MANIFEST_CHECKSUM_FILE}"), + SIDECAR_READ_BOUND, + ) + .await? + { + ReadObject::Bytes(bytes) => bytes, + ReadObject::Absent => { + return Ok(Verified::Incomplete( + "a manifest without its sidecar".to_owned(), + )); + } + ReadObject::Oversize => { + return Ok(Verified::Incomplete(format!( + "a sidecar over {SIDECAR_READ_BOUND} bytes" + ))); + } + }; + if let Err(e) = verify_manifest_checksum(&manifest, &sidecar) { + return Ok(Verified::Incomplete(format!( + "the sidecar does not match: {e}" + ))); + } + Ok(match ExtenddbManifest::from_json_bytes(&manifest) { + Ok(manifest) => Verified::Complete(Box::new(manifest)), + Err(e) => Verified::Incomplete(format!("the manifest is refused: {e}")), + }) +} + +/// The first `data_files` entry `listed` lacks or holds at another size, as +/// the reason a prefix is incomplete; `None` when every one is there. +fn missing_data_file( + manifest: &ExtenddbManifest, + prefix: &str, + listed: &std::collections::HashMap, +) -> Option { + manifest + .data_files + .iter() + .find_map(|file| match listed.get(&format!("{prefix}{}", file.key)) { + Some(size) if *size == file.bytes => None, + Some(size) => Some(format!( + "data file {} has {size} bytes in the store and {} in the manifest", + file.key, file.bytes + )), + None => Some(format!("data file {} is not in the store", file.key)), + }) +} + +/// The manifest's item total, the sum over its data files. +pub(super) fn manifest_item_count(manifest: &ExtenddbManifest) -> i64 { + let total = manifest + .data_files + .iter() + .map(|f| f.item_count) + .sum::(); + i64::try_from(total).unwrap_or(i64::MAX) +} + +/// The most bytes a manifest read collects: a manifest this server wrote is +/// a few KiB per data file, so one over this names a prefix nothing this +/// server wrote. +pub(crate) const MANIFEST_READ_BOUND: u64 = 16 * 1024 * 1024; + +/// The most bytes a sidecar read collects (a sidecar is one line: the +/// digest and the manifest's name). +pub(crate) const SIDECAR_READ_BOUND: u64 = 4 * 1024; + +/// What a bounded read of one object found. +pub(crate) enum ReadObject { + /// The object's bytes, within the bound. + Bytes(Vec), + /// No object under the key. + Absent, + /// The object has more bytes than the bound; the read stopped there and + /// collected nothing. + Oversize, +} + +/// One object's bytes, collected only while they stay within `bound` +/// (the body is read in chunks and dropped once the count passes it). The +/// restore's request and resume read the manifest and sidecar through it +/// too, each with its own bound, so a prefix an attacker rewrote cannot make +/// a request hold an object of any size in memory. +pub(super) async fn read_object( + store: &dyn BackupStore, + key: &str, + bound: u64, +) -> Result { + let mut body = match store.get(key).await { + Ok(body) => body, + Err(StoreError::NotFound) => return Ok(ReadObject::Absent), + Err(e) => return Err(e), + }; + let mut bytes = Vec::new(); + while let Some(chunk) = body.try_next().await? { + if (bytes.len() as u64).saturating_add(chunk.len() as u64) > bound { + return Ok(ReadObject::Oversize); + } + bytes.extend_from_slice(&chunk); + } + Ok(ReadObject::Bytes(bytes)) +} + +/// What the duties that delete a prefix or remove a tombstone act through: +/// the catalog, the store's calls, and the deletion fence each runs right +/// before its deletion or removal. +struct Duty<'a, S: ?Sized> { + storage: &'a S, + store: StoreCalls<'a>, + fence: &'a StoreFence, +} + +impl Clone for Duty<'_, S> { + fn clone(&self) -> Self { + *self + } +} + +impl Copy for Duty<'_, S> {} + +impl Duty<'_, S> { + /// The fence before `row`'s deletion or removal: `true` when it passed; + /// a refusal is logged and acts on nothing. + async fn fenced(self, row: &BackupRow, status: &str, what: &str) -> bool { + match self.fence.check_catalog(self.storage).await { + Ok(()) => true, + Err(problem) => { + tracing::error!( + "did not {what} of {status} backup {} (account {}): {problem}", + row.backup_arn, + row.account_id + ); + false + } + } + } +} + +/// The FAILED-prefix duty (module doc): FAILED rows whose heartbeat is older +/// than `stale_job_after`, their prefix deleted while it lists objects. +async fn delete_stale_failed_prefixes( + duty: Duty<'_, S>, + identity: &StoreIdentity, + stale_job_after: Duration, +) -> Result, StorageError> { + let rows: Vec = duty + .storage + .list_backup_rows_older_than( + BackupStatus::Failed, + identity, + StaleClock::Heartbeat, + stale_job_after, + ) + .try_collect() + .await?; + let mut deleted = Vec::new(); + for row in rows { + if delete_prefix_if_listed(duty, &row, "FAILED").await { + deleted.push(row.backup_arn); + } + } + Ok(deleted) +} + +/// The DELETED-prefix duty (module doc): DELETED rows whose prefix still +/// lists objects, the prefix deleted. +async fn sweep_deleted_prefixes( + duty: Duty<'_, S>, + identity: &StoreIdentity, +) -> Result, StorageError> { + let rows: Vec = duty + .storage + .list_backup_rows(Some(BackupStatus::Deleted), identity) + .try_collect() + .await?; + let mut swept = Vec::new(); + for row in rows { + if delete_prefix_if_listed(duty, &row, "DELETED").await { + swept.push(row.backup_arn); + } + } + Ok(swept) +} + +/// Delete `row`'s prefix when it lists objects, once the fence passed; +/// returns whether a deletion ran. A store failure or the fence's refusal +/// is logged and left for the next cycle. +async fn delete_prefix_if_listed( + duty: Duty<'_, S>, + row: &BackupRow, + status: &str, +) -> bool { + let store = duty.store; + let prefix = match BackupArn::parse(&row.backup_arn) { + Ok(arn) => arn.store_prefix(), + Err(e) => { + tracing::error!( + "{status} row {} carries an ARN this build cannot parse; left as it is: {e}", + row.backup_arn + ); + return false; + } + }; + match store.prefix_has_objects(&prefix).await { + Ok(true) => {} + Ok(false) => return false, + Err(e) => { + tracing::error!( + "could not list the prefix of {status} backup {}; the next cycle retries: {e}", + row.backup_arn + ); + return false; + } + } + if !duty.fenced(row, status, "delete the prefix").await { + return false; + } + match store.delete_prefix(&prefix).await { + Ok(n) => { + tracing::info!( + "deleted {n} object(s) left under the prefix of {status} backup {} (account {})", + row.backup_arn, + row.account_id + ); + true + } + Err(e) => { + tracing::error!( + "could not delete the prefix of {status} backup {} (account {}); the next cycle \ + retries: {e}", + row.backup_arn, + row.account_id + ); + false + } + } +} + +/// The tombstone duty (module doc): FAILED and DELETED rows whose +/// `completed_at` is older than `retention` and whose prefix lists nothing +/// are removed. +async fn remove_tombstones( + duty: Duty<'_, S>, + identity: &StoreIdentity, + retention: Duration, +) -> Result, StorageError> { + let Duty { storage, store, .. } = duty; + let mut removed = Vec::new(); + for status in [BackupStatus::Failed, BackupStatus::Deleted] { + let rows: Vec = storage + .list_backup_rows_older_than(status, identity, StaleClock::Completed, retention) + .try_collect() + .await?; + for row in rows { + let prefix = match BackupArn::parse(&row.backup_arn) { + Ok(arn) => arn.store_prefix(), + Err(e) => { + tracing::error!( + "{status} row {} carries an ARN this build cannot parse; left as it \ + is: {e}", + row.backup_arn + ); + continue; + } + }; + match store.prefix_has_objects(&prefix).await { + Ok(false) => {} + Ok(true) => continue, + Err(e) => { + tracing::error!( + "could not list the prefix of {status} backup {} before removing its \ + row; the next cycle retries: {e}", + row.backup_arn + ); + continue; + } + } + // The empty listing is a fact about this catalog's root only + // while the root is this catalog's. + if !duty + .fenced(&row, &status.to_string(), "remove the row") + .await + { + continue; + } + match storage.remove_backup_row(&row.key()).await { + Ok(true) => { + tracing::info!( + "removed the {status} row of backup {} (account {}): its prefix is empty \ + and its retention has passed", + row.backup_arn, + row.account_id + ); + removed.push(row.backup_arn); + } + Ok(false) => {} + Err(e) => tracing::error!( + "could not remove the {status} row of backup {} (account {}); the next cycle \ + retries: {e}", + row.backup_arn, + row.account_id + ), + } + } + } + Ok(removed) +} + +/// The duty for DELETING rows of the configured store: finish each deletion. +/// Returns the ARNs moved to DELETED; a row whose deletion fails is logged +/// and left for the next cycle. +async fn finish_deleting_rows( + storage: &S, + store: &super::ConfiguredStore, + fence: &StoreFence, + identity: &extenddb_storage::backup_catalog::StoreIdentity, +) -> Result, StorageError> { + let rows: Vec<_> = storage + .list_backup_rows(Some(BackupStatus::Deleting), identity) + .try_collect() + .await?; + let mut finished = Vec::new(); + for row in rows { + let arn = match BackupArn::parse(&row.backup_arn) { + Ok(arn) => arn, + Err(e) => { + tracing::error!( + "DELETING row {} carries an ARN this build cannot parse; left as it is: {e}", + row.backup_arn + ); + continue; + } + }; + match super::finish_backup_delete(storage, &store.calls("deletion"), fence, &arn).await { + Ok(true) => { + tracing::info!( + "finished the deletion of backup {} (account {})", + row.backup_arn, + row.account_id + ); + finished.push(row.backup_arn); + } + Ok(false) => tracing::info!( + "backup {} was no longer DELETING when its objects were gone; left to its owner", + row.backup_arn + ), + Err(e) => tracing::error!( + "could not finish the deletion of backup {} (account {}); the next cycle \ + retries: {e}", + row.backup_arn, + row.account_id + ), + } + } + Ok(finished) +} + +/// Mark a dead legacy restore FAILED and remove its target. The FAILED write +/// is conditioned on the `owner_generation` and `heartbeat_at` the listing +/// read (design 4(e)), so a heartbeat that landed after the listing, which +/// says the owner is alive after all, makes it match nothing. `None` when the +/// FAILED write did not match (the record is no longer RUNNING under the +/// listed claim and belongs to whoever holds it) or errored; otherwise the +/// removal's outcome. +async fn remove_dead_legacy_restore( + storage: &S, + job: &RestoreJobRow, +) -> Option { + let target = &job.target; + match storage + .fail_restore_job(target, job.owner_generation, job.heartbeat_at, OWNER_DEAD) + .await + { + Ok(true) => {} + Ok(false) => { + tracing::info!( + "restore into {} (account {}) was no longer RUNNING under the listed claim \ + when the reconciler reached it (its owner heartbeated or moved it); left to \ + its owner", + target.table_name, + target.account_id + ); + return None; + } + Err(e) => { + tracing::error!( + "could not mark the abandoned restore into {} (account {}) FAILED: {e}", + target.table_name, + target.account_id + ); + return None; + } + } + Some(remove_restore_target(storage, job, "was marked FAILED by the reconciler").await) +} + +/// What [`remove_restore_target`] found. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Removal { + /// The CREATING target was removed. + Removed, + /// The target was no longer a CREATING restore target with this id; it + /// belongs to whoever moved it and the record is left as it is. + LeftToOwner, + /// The removal errored; the FAILED record and the target wait for a + /// later cycle. + Failed, +} + +/// Remove the CREATING target of a record that is FAILED (`how` says, for +/// the log, whether this cycle wrote the FAILED or found it). A target that +/// is no longer a CREATING restore target belongs to whoever moved it; a +/// removal that errors leaves the FAILED record and the target for a later +/// cycle, which lists the pair again once the record is stale. +async fn remove_restore_target( + storage: &S, + job: &RestoreJobRow, + how: &str, +) -> Removal { + let target = &job.target; + match storage.remove_restore_target(target).await { + Ok(true) => { + tracing::warn!( + "removed abandoned restore target {} ({}) in account {}, restoring from {}: \ + its restore did not finish and no process owns it", + target.table_name, + target.table_id, + target.account_id, + job.source_backup_arn + ); + Removal::Removed + } + Ok(false) => { + tracing::warn!( + "the abandoned restore into {} (account {}) {how} but its target ({}) was no \ + longer a CREATING restore target; left to its current owner", + target.table_name, + target.account_id, + target.table_id + ); + Removal::LeftToOwner + } + Err(e) => { + tracing::error!( + "the abandoned restore into {} (account {}) {how} but its target ({}) could \ + not be removed yet; a later cycle retries once the FAILED record is older \ + than stale_job_after_secs: {e}", + target.table_name, + target.account_id, + target.table_id + ); + Removal::Failed + } + } +} diff --git a/crates/engine/src/backup/restore.rs b/crates/engine/src/backup/restore.rs new file mode 100644 index 000000000..2233858d0 --- /dev/null +++ b/crates/engine/src/backup/restore.rs @@ -0,0 +1,1483 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The store-backed RestoreTableFromBackup: the request's manifest branch +//! and the restore job (the design's 5.3). +//! +//! The request ([`request`]) runs once `check_catalog_row_for_restore` has +//! accepted the backup's row: it takes the job permit, reads the sidecar and +//! the manifest from the store (a missing one is `BackupNotFoundException` +//! and an error line naming the remedies; each read +//! collects at most its bound, [`super::reconciler::SIDECAR_READ_BOUND`] and +//! [`super::reconciler::MANIFEST_READ_BOUND`], and an object over it is a +//! `ValidationException` naming a corrupt prefix, as a bad sidecar is), +//! verifies the +//! sidecar, parses the manifest (`ExtenddbManifest::from_json_bytes`, whose +//! own gates are the `format_version` and the identity rule that the ARN, +//! `source_table.table_name` and `source_table.table_arn` name one table), +//! checks that it names the row's backup, refuses a non-empty +//! `vector_indexes` into a backend without vector support with the text the +//! legacy reader uses, builds the target's `CreateTableInput` from the +//! manifest with the overrides applied, validates it as CreateTable does, +//! resolves the key schema the validator holds every item to, and spawns +//! the detached task ([`start`]) whose answer it awaits for at most +//! [`RESTORE_REGISTRATION_TIMEOUT`]; past that it answers +//! `InternalServerError` while the task goes on (the target then appears +//! through DescribeTable). The task registers the target and its RUNNING +//! record in one write (`claim_restore`, under the backup row's lock), sends +//! the job's first beat through the claim (the registration's own +//! `heartbeat_at` stamp can be old: the design's 4(e) states each backend's +//! clock, PostgreSQL's `now()` being the transaction's start, so a +//! registration stalled on the `backup_index` lock or the DDL commits a +//! stale beat that the reconciler under a maintenance claim and +//! `remove-target`, which judge the owner by the heartbeat alone, would +//! read as dead; the beat refreshes it at the registration's end on every +//! backend, and a beat that finds the claim lost or fails ends the task as +//! a lost claim before the reply, the record failed as the maintenance +//! refusal fails it and the request answered `InternalServerError`), then +//! checks the maintenance claim: held, the record is FAILED `maintenance`, +//! the target removed, and the request refused with +//! `LimitExceededException` (a job end: `RestoreJobs{failed}` and the worker +//! error are recorded); otherwise the request is answered CREATING and the +//! job runs ([`run`]). The heartbeat's first confirmed beat is that first +//! beat, and the instant it returned is what the pre-write check +//! (`heartbeat.rs`) counts the two periods from, so the registration's own +//! duration, however long, does not end the job at its first check. A +//! resumed job's instant is the takeover's return, taken by the reconciler +//! beside `claim_restore_dead` and passed in (the takeover stamped +//! `heartbeat_at` inside its write), so the reads between the takeover and +//! the job's start count against the window rather than widen it. +//! +//! The job: step 3, every data file downloaded and verified +//! ([`download`]: into the spool entry on an S3 store, hashed in place on a +//! filesystem store, the manifest and sidecar spooled first for a resume, +//! each as a data file is: `.partial`, synced, renamed; a resume that read +//! them from the spool does not write them again); +//! step 5, each file decoded, validated and written through one +//! `bulk_load` ([`load`]), skipping the files and items a resume has +//! recorded; the count check (`count_restore_target` against the manifest's +//! total, a shortfall being a key the backup holds twice); step 6, +//! `finish_restore` (DONE and ACTIVE, or DONE with the index backfill +//! pending on MongoDB). A `finish_restore` that returns an error is a +//! conditional write whose outcome is unknown, handled as `write_status` +//! handles one: the record is re-read through the claim; DONE under this +//! generation means the write landed and the restore is reported complete +//! with a WARN naming the error; RUNNING under this generation means it did +//! not, and the write is sent once more, at most, its own error re-read the +//! same way; anything else is the DONE write that matched no record. The +//! heartbeat runs on a timer for the job's life +//! and is checked before every download attempt, every `bulk_load`, the +//! count and the flip; a lost claim ends the job with no catalog or table +//! access. Step 7 ([`fail_restore`]): FAILED with the reason, then +//! `remove_restore_target` only when that write matched (the claim holder +//! owns the table); the spool entry is removed on every end but the +//! shutdown's and the ends at which the job learnt the record is another's +//! (a lost claim, a DONE or FAILED write that matched no record): a +//! same-host successor may be reading that entry, so it is left to the +//! reconciler's sweep, which lists the spool before it reads the records and +//! removes the entries it listed that no RUNNING record of this host names. +//! The shutdown token stops the job between download attempts, +//! files and chunks with no status write: the record stays RUNNING with its +//! spool and progress for a resume ([`resume`], the reconciler's entry). +//! +//! Each job records `RestoreJobs` and `RestoreJobDuration` under its +//! outcome, `RestoreBytesDownloaded` as the files arrive, +//! `BackupStoreErrors{get}` on every failed `get` of a restore job (the +//! request's and a resume's two document reads and each download attempt +//! that the store failed, whatever the store's error), and the worker +//! metrics under `QuerySource::RestoreJob` (`WorkerLastSuccess` on DONE, +//! `WorkerErrorCount` on FAILED or a job that ended without its write; a +//! stopped job records neither); one info line at start and end, one error +//! line per failure. + +pub(crate) mod download; +pub(crate) mod load; +pub(crate) mod spool; + +use std::sync::Arc; +use std::time::Duration; + +use extenddb_core::backup_format::{ + BackupArn, EXTENDDB_MANIFEST_FILE, ExtenddbManifest, ItemOrder, MANIFEST_CHECKSUM_FILE, + verify_manifest_checksum, +}; +use extenddb_core::error::DynamoDbError; +use extenddb_core::limits::LimitsConfig; +use extenddb_core::metrics::{BackupJobOutcome, QuerySource}; +use extenddb_core::types::{BillingMode, CreateTableInput, TableDescription}; +use extenddb_core::validation::{RestorableKeySchema, RestorableLimits}; +use extenddb_storage::backup_catalog::{ + BackupKey, Heartbeat, JobStatus, RestoreJobRow, RestoreRegistrationRow, RestoreTargetKey, + StatusFields, UNOWNED_GENERATION, billing_mode_name, +}; +use extenddb_storage::backup_definition::{restore_input_from_manifest, vector_index_refusal}; +use extenddb_storage::{ + BackupCatalog, CancellationToken, DataEngine, Finished, JobClaim, JobPermit, RestoreEngine, + RestoreSource, RestoreTableOverrides, +}; +use serde_json::Value; +use tokio::sync::oneshot; +use tokio::time::Instant; + +use super::job::failure::{Written, write_status}; +use super::job::heartbeat::{HeartbeatHandle, HeartbeatMetric}; +use super::job::{JobEnv, KillPoint, MAINTENANCE, kill_point, kill_point_named}; +use super::overrides::IndexOverrides; +use super::reconciler::{MANIFEST_READ_BOUND, ReadObject, SIDECAR_READ_BOUND}; +use super::runtime::{BackupRuntime, ConfiguredStore}; +use super::{ + backup_not_found, maintenance_held, restore_response, restore_storage_err_to_dynamo, + storage_err_to_dynamo, validate_override_for_source, +}; +use download::{Dest, Downloader, Stop}; +use load::{FileLoad, Source}; + +/// How long the request waits for the detached task to register the target +/// before it answers `InternalServerError` and leaves the task to finish +/// (against a catalog that has stopped handing out connections). One value +/// on every backend. +pub const RESTORE_REGISTRATION_TIMEOUT: Duration = Duration::from_secs(60); + +/// The reason a resumed restore's record carries when no progress was +/// recorded and the target already holds items (a catalog restored from a +/// dump can name a restore that finished and was written to since): the +/// target is left for the operator (`extenddb backup remove-target`). +pub const TARGET_NOT_EMPTY: &str = "target_not_empty"; + +/// The reason the reconciler writes on a RUNNING record whose store is not +/// the configured one. +pub const OTHER_STORE: &str = "the restore reads a store other than the configured one; configure that store in [backup.store] and restore again"; + +/// What the request resolved before it spawned the task. +pub(crate) struct RestoreRequestParts { + pub account_id: String, + pub arn: BackupArn, + pub backup: BackupKey, + pub manifest: Arc, + pub manifest_bytes: Vec, + pub sidecar_bytes: Vec, + pub input: CreateTableInput, + /// The key schema the validator holds every item to. + pub key_schema: RestorableKeySchema, + pub limits: RestorableLimits, + /// This instance's in-flight entry for the registration, which the task + /// ends with the registration's outcome; `None` for a caller outside + /// the request path. + pub registration: Option, +} + +/// The one-shot the task answers the request through. +pub(crate) type Reply = oneshot::Sender>; + +/// The request's manifest branch (module doc), from the permit to the +/// response body, for a row `check_catalog_row_for_restore` accepted whose +/// store is the healthy configured one. +#[allow(clippy::too_many_arguments)] +pub(super) async fn request( + storage: &Arc, + runtime: &BackupRuntime, + store: &Arc, + account_id: &str, + limits: &LimitsConfig, + target_table_name: &str, + arn: &BackupArn, + overrides: &RestoreTableOverrides, + index_overrides: &IndexOverrides, +) -> Result<(String, Value), DynamoDbError> +where + S: BackupCatalog + DataEngine + RestoreEngine + ?Sized + Send + Sync + 'static, +{ + let arn_text = arn.to_arn_string(); + let env = JobEnv::new(runtime, store)?; + // The permit, last among the checks that need no store access. + let permit = runtime.take_permit().await?; + + // The sidecar and the manifest; a missing one behind an AVAILABLE row is + // BackupNotFound and the operator's line; one over its bound is a + // prefix nothing this server wrote produces. + let prefix = arn.store_prefix(); + let read = async |name: &str, bound: u64| -> Result, DynamoDbError> { + match super::reconciler::read_object(&*env.store, &format!("{prefix}{name}"), bound).await { + Ok(ReadObject::Bytes(bytes)) => Ok(bytes), + Ok(ReadObject::Absent) if name == EXTENDDB_MANIFEST_FILE => { + tracing::error!( + "backup {arn_text} (account {account_id}): its row is AVAILABLE but the store \ + holds no {name} under its prefix; put the prefix back, or, unless it is the \ + store's newest backup (whose missing manifest makes the store unhealthy), \ + remove the row with backup sync --prune-missing" + ); + Err(backup_not_found(&arn_text)) + } + Ok(ReadObject::Absent) => { + tracing::error!( + "backup {arn_text} (account {account_id}): its row is AVAILABLE but the store \ + holds no {name} under its prefix; put the prefix back (backup sync \ + --prune-missing removes only a row whose manifest is gone)" + ); + Err(backup_not_found(&arn_text)) + } + Ok(ReadObject::Oversize) => Err(DynamoDbError::ValidationException(format!( + "Backup {arn_text} has a {name} over {bound} bytes, which no backup this server \ + wrote produces; the backup's prefix is corrupt" + ))), + Err(e) => { + if let Some(metrics) = runtime.metrics() { + metrics.record_backup_store_error("get"); + } + tracing::error!("backup {arn_text}: reading {name} from the store failed: {e}"); + Err(DynamoDbError::InternalServerError( + "Internal server error".to_owned(), + )) + } + } + }; + let sidecar_bytes = read(MANIFEST_CHECKSUM_FILE, SIDECAR_READ_BOUND).await?; + let manifest_bytes = read(EXTENDDB_MANIFEST_FILE, MANIFEST_READ_BOUND).await?; + verify_manifest_checksum(&manifest_bytes, &sidecar_bytes).map_err(|e| { + DynamoDbError::ValidationException(format!( + "Backup {arn_text} has a manifest whose checksum sidecar does not match: {e}" + )) + })?; + // The parser ran the version gate and the identity rule (the ARN's + // table is `source_table.table_name`); what is left is that the + // manifest's backup is the row's. + let manifest = ExtenddbManifest::from_json_bytes(&manifest_bytes).map_err(|e| { + DynamoDbError::ValidationException(format!( + "Backup {arn_text} has a manifest this server cannot restore from: {e}" + )) + })?; + match manifest.backup_arn() { + Ok(named) if named == *arn => {} + Ok(named) => { + return Err(DynamoDbError::ValidationException(format!( + "Backup {arn_text} has a manifest that names another backup, {}", + named.to_arn_string() + ))); + } + Err(e) => { + return Err(DynamoDbError::ValidationException(format!( + "Backup {arn_text} has a manifest whose backup ARN is refused: {e}" + ))); + } + } + + // Step 2: the one feature a target can lack. + let source_table = &manifest.source_table; + if crate::vector_gate::ensure_create_table_supported( + source_table.vector_indexes.as_ref(), + storage.as_vector_search(), + ) + .is_err() + { + let count = source_table.vector_indexes.as_ref().map_or(0, Vec::len); + return Err(storage_err_to_dynamo(vector_index_refusal( + &format!("backup {arn_text}"), + count, + ))); + } + + // The target's input, with the overrides, validated as CreateTable does. + validate_override_for_source(overrides, billing_mode_name(source_table.billing_mode))?; + let mut input = restore_input_from_manifest(source_table, target_table_name, overrides); + index_overrides.apply(&mut input, overrides.billing_mode)?; + extenddb_core::validation::validate_create_table(&input, limits)?; + // The key schema the validator holds every item to: a pure function of + // the manifest, resolved here with step 2's refusals rather than after + // the client was told the target is CREATING. + let key_schema = RestorableKeySchema::resolve( + &source_table.key_schema, + &source_table.attribute_definitions, + ) + .map_err(|e| { + DynamoDbError::ValidationException(format!( + "Backup {arn_text} has a manifest whose key schema cannot be checked: {e}" + )) + })?; + let on_demand = input.billing_mode == Some(BillingMode::PayPerRequest); + let source = RestoreSource { + backup_arn: arn_text.clone(), + source_table_arn: source_table.table_arn.clone(), + backup_creation_date_time: manifest.backup_creation_date_time, + }; + + // Step 4 in the detached task; the request waits for its answer. A + // registration of this instance already in flight for the name is met + // here: the same backup's is this request's own, retried by a client + // whose read timeout (botocore's default is this bound, 60 s) ended its + // first attempt while the registration waited on the `backup_index` + // lock or the DDL, and the retry waits for that registration's answer + // rather than registering a second target or being refused the name; + // another backup's is `TableInUseException`, as a committed registration + // answers it. + let guard = match runtime.begin_restore_registration(account_id, target_table_name, &arn_text) { + Ok(guard) => guard, + Err(in_flight) => { + drop(permit); + if in_flight.backup_arn != arn_text { + return Err(restore_storage_err_to_dynamo( + extenddb_storage::error::StorageError::TableBeingRestored { + table_name: target_table_name.to_owned(), + backup_arn: in_flight.backup_arn, + }, + )); + } + tracing::info!( + "RestoreTableFromBackup of {arn_text} into {target_table_name} (account \ + {account_id}): a registration of this request is in flight; this request waits \ + for its answer" + ); + let desc = + await_registration(in_flight, &arn_text, target_table_name, account_id).await?; + let response = restore_response(desc, &source, on_demand)?; + return Ok((target_table_name.to_owned(), response)); + } + }; + let parts = RestoreRequestParts { + account_id: account_id.to_owned(), + arn: arn.clone(), + backup: BackupKey::from_arn(arn), + manifest: Arc::new(manifest), + manifest_bytes, + sidecar_bytes, + input, + key_schema, + limits: limits.into(), + registration: Some(guard), + }; + let (reply, answer) = oneshot::channel(); + runtime.spawn_job(start(Arc::clone(storage), env, permit, parts, reply)); + let desc = match tokio::time::timeout(RESTORE_REGISTRATION_TIMEOUT, answer).await { + Ok(Ok(answer)) => answer?, + Ok(Err(_)) => { + tracing::error!( + "RestoreTableFromBackup of {arn_text} into {target_table_name} (account \ + {account_id}): the job task ended without answering" + ); + return Err(DynamoDbError::InternalServerError( + "Internal server error".to_owned(), + )); + } + Err(_) => { + tracing::error!( + "RestoreTableFromBackup of {arn_text} into {target_table_name} (account \ + {account_id}): the registration did not answer within {} s; the task goes on \ + and the target, if registered, appears through DescribeTable", + RESTORE_REGISTRATION_TIMEOUT.as_secs() + ); + return Err(DynamoDbError::InternalServerError( + "Internal server error".to_owned(), + )); + } + }; + let response = restore_response(desc, &source, on_demand)?; + Ok((target_table_name.to_owned(), response)) +} + +/// A retry's wait on its own registration in flight (module doc): the +/// registration's outcome, for at most [`RESTORE_REGISTRATION_TIMEOUT`] +/// from this request's arrival; a registration that returned no outcome +/// (its task ended) or none within the bound is `InternalServerError`, as +/// the registering request's own bound is. +async fn await_registration( + mut in_flight: super::runtime::InFlightRegistration, + arn_text: &str, + target_table_name: &str, + account_id: &str, +) -> Result { + let internal = || DynamoDbError::InternalServerError("Internal server error".to_owned()); + let outcome = async { + loop { + if let Some(outcome) = in_flight.outcome.borrow_and_update().clone() { + return Some(outcome); + } + if in_flight.outcome.changed().await.is_err() { + return in_flight.outcome.borrow().clone(); + } + } + }; + match tokio::time::timeout(RESTORE_REGISTRATION_TIMEOUT, outcome).await { + Ok(Some(outcome)) => outcome, + Ok(None) => { + tracing::error!( + "RestoreTableFromBackup of {arn_text} into {target_table_name} (account \ + {account_id}): the registration this request waited on ended without answering" + ); + Err(internal()) + } + Err(_) => { + tracing::error!( + "RestoreTableFromBackup of {arn_text} into {target_table_name} (account \ + {account_id}): the registration this request waited on did not answer within {} \ + s; it goes on and the target, if registered, appears through DescribeTable", + RESTORE_REGISTRATION_TIMEOUT.as_secs() + ); + Err(internal()) + } + } +} + +/// The detached task: the registration, the maintenance check, the reply, +/// then the job. +pub(crate) async fn start( + storage: Arc, + env: JobEnv, + permit: JobPermit, + parts: RestoreRequestParts, + reply: Reply, +) where + S: BackupCatalog + RestoreEngine + ?Sized + Send + Sync + 'static, +{ + let started = Instant::now(); + let RestoreRequestParts { + account_id, + arn, + backup, + manifest, + manifest_bytes, + sidecar_bytes, + input, + key_schema, + limits, + registration, + } = parts; + let arn_text = arn.to_arn_string(); + let target_table_name = input.table_name.clone(); + let spool_host = env.spool.as_ref().map(|s| s.spool_host.clone()); + let registration_row = RestoreRegistrationRow { + backup, + source_table_arn: manifest.source_table.table_arn.clone(), + restore_date_time: manifest.backup_creation_date_time, + store: env.identity.clone(), + spool_host: spool_host.clone(), + owner_generation: UNOWNED_GENERATION, + }; + let registered = storage + .claim_restore(&account_id, input, ®istration_row) + .await; + // The in-flight entry ends with the reply, for the retries waiting on + // it: the description, or the error the request itself is answered. + let (desc, claim) = match registered { + Ok(registered) => registered, + Err(e) => { + drop(permit); + let error = restore_storage_err_to_dynamo(e); + if let Some(registration) = registration { + registration.finish(Err(error.clone())); + } + if let Err(Err(error)) = reply.send(Err(error)) { + // The request has answered `InternalServerError` at its bound + // and told the operator the target appears if registered; + // this line is the one trace of why it will not. + tracing::error!( + "restore of {arn_text} into {target_table_name} (account {account_id}): the \ + registration failed after the request had stopped waiting for it; no target \ + was created: {error}" + ); + } + return; + } + }; + // The job's first beat, before the reply: the record's `heartbeat_at` as + // the registration stamped it is the catalog's clock at some point of + // the registration (on PostgreSQL `now()` is the transaction's start, so + // a registration stalled on the `backup_index` lock or the DDL commits a + // stale beat), and the two readers that judge the owner by the heartbeat + // alone (the reconciler under a maintenance claim, `remove-target`) + // would find a live job dead until its timer's first beat. A confirmed + // beat refreshes it at the registration's end on every backend, and its + // return is the instant the heartbeat's two-period rule counts from. A + // beat that finds the claim lost, or fails, ends the task as a lost + // claim before the reply: the record is failed as the maintenance + // refusal fails it (the write matches nothing when the claim is + // another's) and the request is answered `InternalServerError`. + let first_beat = match claim.heartbeat().await { + Ok(Heartbeat::Confirmed) => Ok(Instant::now()), + Ok(Heartbeat::Lost) => Err("the claim was lost before the job started".to_owned()), + Err(e) => Err(format!( + "the claim was lost before the job started: its first heartbeat failed: {e}" + )), + }; + let claim: Arc = Arc::from(claim); + let target = RestoreTargetKey { + account_id: account_id.clone(), + table_name: desc.table_name.clone(), + table_id: desc.table_id.clone(), + }; + // The maintenance claim, once the record exists so the command's wait + // sees it: held, the record is FAILED `maintenance`, the target removed + // and the request refused. + let (claimed_at, refused) = match first_beat { + Err(reason) => ( + started, + Some(( + reason, + DynamoDbError::InternalServerError("Internal server error".to_owned()), + )), + ), + Ok(beat) => ( + beat, + match storage + .check_maintenance_claim(env.settings.stale_job_after) + .await + { + Ok(false) => None, + Ok(true) => Some((MAINTENANCE.to_owned(), maintenance_held())), + Err(e) => Some(( + format!("the maintenance claim could not be read: {e}"), + storage_err_to_dynamo(e), + )), + }, + ), + }; + if let Some((reason, error)) = refused { + let path = fail_restore(&*storage, &*claim, &target, &reason, None, &env.shutdown).await; + tracing::error!( + "restore of {arn_text} into {target_table_name} (account {account_id}): FAILED \ + ({reason}) before it started; {path}" + ); + // A FAILED record is a job end: the outcome and the worker error. + if let Some(metrics) = env.metrics.as_deref() { + #[allow(clippy::cast_precision_loss)] + let duration_us = started.elapsed().as_micros() as f64; + metrics.record_restore_job(BackupJobOutcome::Failed, duration_us); + metrics.record_worker_error(QuerySource::RestoreJob); + } + drop(permit); + if let Some(registration) = registration { + registration.finish(Err(error.clone())); + } + let _ = reply.send(Err(error)); + return; + } + // A request whose client went away still has its job. + if let Some(registration) = registration { + registration.finish(Ok(desc.clone())); + } + let _ = reply.send(Ok(desc)); + let job = RestoreJob { + storage, + claim, + target, + arn, + manifest, + manifest_bytes, + sidecar_bytes, + key_schema, + limits, + env, + started, + claimed_at, + progress: None, + recorded_spool_host: spool_host, + documents_from_spool: false, + }; + run(job).await; + drop(permit); +} + +/// Per file, what an earlier run recorded as applied; `None` for a file +/// with no record. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct Progress { + recorded: Vec>, +} + +impl Progress { + /// From a dead owner's record and, on PostgreSQL, its progress rows: + /// every file before the record's `file_index` is complete, `file_index` + /// itself has `items_applied` of its items (all of them when the backend + /// records per file), and each progress row names its file's count. + pub(crate) fn from_records( + row: &RestoreJobRow, + progress_rows: &[(u32, u64)], + manifest: &ExtenddbManifest, + ) -> Self { + let counts: Vec = manifest.data_files.iter().map(|f| f.item_count).collect(); + let mut recorded = vec![None; counts.len()]; + if let Some(last) = row.file_index { + let last = last as usize; + for (i, slot) in recorded.iter_mut().enumerate().take(last.min(counts.len())) { + *slot = Some(counts[i]); + } + if let Some(slot) = recorded.get_mut(last) { + *slot = Some(row.items_applied.unwrap_or(counts[last])); + } + } + for (index, applied) in progress_rows { + if let Some(slot) = recorded.get_mut(*index as usize) { + *slot = Some((*applied).max(slot.unwrap_or(0))); + } + } + Self { recorded } + } + + /// The first file not complete, for the start line. + fn first_open(&self, counts: &[u64]) -> Option<(usize, u64)> { + self.recorded + .iter() + .enumerate() + .find_map(|(i, recorded)| match recorded { + Some(n) if *n >= counts[i] => None, + Some(n) => Some((i, *n)), + None => Some((i, 0)), + }) + } + + /// Per file, the items recorded as applied; `None` for a file with no + /// record. + #[cfg(test)] + pub(crate) fn recorded(&self) -> &[Option] { + &self.recorded + } + + /// [`Self::first_open`] against `manifest`'s counts. + #[cfg(test)] + pub(crate) fn first_open_in(&self, manifest: &ExtenddbManifest) -> Option<(usize, u64)> { + let counts: Vec = manifest.data_files.iter().map(|f| f.item_count).collect(); + self.first_open(&counts) + } +} + +/// The job once its record exists. +pub(crate) struct RestoreJob { + storage: Arc, + claim: Arc, + target: RestoreTargetKey, + arn: BackupArn, + manifest: Arc, + manifest_bytes: Vec, + sidecar_bytes: Vec, + key_schema: RestorableKeySchema, + limits: RestorableLimits, + env: JobEnv, + started: Instant, + /// The instant the heartbeat's two-period rule counts from: the return + /// of the job's first beat, sent right after the registration, for a + /// fresh job; the takeover's return for a resume (the takeover stamped + /// the record's `heartbeat_at` inside its write). + claimed_at: Instant, + /// A resumed job's recorded progress; `None` for a fresh one. + progress: Option, + /// The `spool_host` the record carried before this job's claim: the + /// registration's own for a fresh job, the dead owner's for a resume. + /// A spool entry of this table id is this job's to reuse only when it + /// is this host's. + recorded_spool_host: Option, + /// Whether the manifest and sidecar in hand were read from this host's + /// spool entry (a resume), in which case the job does not write them + /// there again. + documents_from_spool: bool, +} + +/// The job (module doc), from the download to the final write. +pub(crate) async fn run(job: RestoreJob) -> BackupJobOutcome +where + S: BackupCatalog + RestoreEngine + ?Sized + Send + Sync + 'static, +{ + let RestoreJob { + storage, + claim, + target, + arn, + manifest, + manifest_bytes, + sidecar_bytes, + key_schema, + limits, + env, + started, + claimed_at, + progress, + recorded_spool_host, + documents_from_spool, + } = job; + let arn_text = arn.to_arn_string(); + let name = format!( + "restore of {arn_text} into {} (account {})", + target.table_name, target.account_id + ); + let metrics = env.metrics.as_deref(); + let heartbeat = HeartbeatHandle::start( + Arc::clone(&claim), + env.settings.heartbeat_interval, + HeartbeatMetric::None, + claimed_at, + ); + #[cfg(test)] + test_support::register_timer(&target.table_id, &heartbeat); + let counts: Vec = manifest.data_files.iter().map(|f| f.item_count).collect(); + let total_items: u64 = counts.iter().sum(); + let resumed_from = progress.as_ref().and_then(|p| p.first_open(&counts)); + // The files a resume's progress records as complete: neither + // downloaded nor loaded again. + let complete: Vec = counts + .iter() + .enumerate() + .map(|(i, count)| { + progress + .as_ref() + .and_then(|p| p.recorded[i]) + .is_some_and(|n| n >= *count) + }) + .collect(); + tracing::info!( + "{name}: started, {} data file(s), {total_items} items, {}{}", + manifest.data_files.len(), + match manifest.item_order { + ItemOrder::KeyOrdered => "key-ordered", + ItemOrder::Unordered => "unordered", + }, + match (&progress, resumed_from) { + (None, _) => String::new(), + (Some(_), Some((file, items))) => + format!("; resumed from data file {file} after {items} of its items"), + (Some(_), None) => "; resumed with every file recorded".to_owned(), + } + ); + let prefix = arn.store_prefix(); + let mut entry = None; + let mut bytes_downloaded = 0; + let mut applied_total = 0; + let end: Result = async { + // Step 3. The entry is the job's from the open, so a document write + // that fails (the disk full, a name taken by a directory) reaches + // the failure path with the entry in hand and it is removed as any + // other failed end's is. + if let Some(spool) = &env.spool { + let opened = spool::Entry::open( + &spool.path, + &target.table_id, + recorded_spool_host.as_deref(), + &spool.spool_host, + ) + .map_err(|e| Stop::Fail(format!("opening the spool entry: {e}")))?; + let opened = entry.insert(opened); + if !documents_from_spool { + for (doc, bytes) in [ + (EXTENDDB_MANIFEST_FILE, &manifest_bytes), + (MANIFEST_CHECKSUM_FILE, &sidecar_bytes), + ] { + opened + .write_document(doc, bytes) + .await + .map_err(|e| Stop::Fail(format!("writing {doc} to the spool: {e}")))?; + } + } + } + let dest = entry.as_ref().map_or(Dest::InPlace, Dest::Spool); + let downloader = Downloader { + store: &*env.store, + prefix: &prefix, + heartbeat: &heartbeat, + shutdown: &env.shutdown, + metrics, + }; + bytes_downloaded = download::download_all(&downloader, &manifest, &dest, &complete).await?; + kill_point(KillPoint::RestoreAfterDownload, &env.shutdown).await; + + // Step 5. + #[cfg(not(test))] + let chunk_bytes = load::LOAD_CHUNK_BYTES; + #[cfg(test)] + let chunk_bytes = test_support::chunk_bytes_for(&target.table_id); + let mut first_loaded = true; + let mut reapplying = progress.is_some(); + for (i, file) in manifest.data_files.iter().enumerate() { + let recorded = progress.as_ref().and_then(|p| p.recorded[i]); + let start_at = match recorded { + Some(n) if n >= file.item_count => continue, + Some(n) => n, + None => 0, + }; + if env.shutdown.is_cancelled() { + return Err(Stop::Cancelled); + } + if !heartbeat.confirms_the_claim() { + return Err(Stop::ClaimLost( + heartbeat.why_not_confirmed(&format!("to load {}", file.key)), + )); + } + let file_index = + u32::try_from(i).map_err(|_| Stop::Fail("too many data files".to_owned()))?; + let source = match &entry { + Some(entry) => Source::Spool(entry.file_path(file_index)), + None => Source::InPlace { + store: Arc::clone(&env.store), + key: format!("{prefix}{}", file.key), + }, + }; + let applied = load::load_file( + &*storage, + &target, + &*claim, + FileLoad { + manifest: &manifest, + entry: file, + file_index, + start_at, + reapplying, + key_schema: &key_schema, + limits, + shutdown: &env.shutdown, + heartbeat: &heartbeat, + chunk_bytes, + }, + source, + ) + .await?; + applied_total += applied; + // Only the first file a resume meets may have been applied in + // part without a record. + reapplying = false; + if first_loaded { + first_loaded = false; + kill_point(KillPoint::RestoreAfterFileCommit, &env.shutdown).await; + } + } + + // The count: every restore ends with it, so a backend that kept one + // of two rows with one key, and a key-range check that missed, are + // both caught. A count that fails is retried; one that keeps + // failing stops the job with the record RUNNING, since the load is + // complete and nothing of it is in doubt (on Cassandra the count is a + // full `COUNT(*)`, which a node can time out), and the reconciler + // resumes the job once the record is stale. + if env.shutdown.is_cancelled() { + return Err(Stop::Cancelled); + } + if !heartbeat.confirms_the_claim() { + return Err(Stop::ClaimLost( + heartbeat.why_not_confirmed("to count the target"), + )); + } + let count = count_target(&*storage, &target, &heartbeat, &env.shutdown).await?; + if count != total_items { + return Err(Stop::Fail(format!( + "restored {count} items, the manifest lists {total_items}: {}", + if count < total_items { + "the backup holds a key twice" + } else { + "the target holds items the backup does not account for" + } + ))); + } + + // Step 6. The fenced-out kill point stops the heartbeat first, so a + // reconciler can take the record over while the job pauses, and the + // pre-write check is skipped so the conditional write itself is what + // the test observes. + let fenced = kill_point_named(KillPoint::RestoreFencedOut); + if fenced { + heartbeat.stop_timer(); + } + kill_point(KillPoint::RestoreFencedOut, &env.shutdown).await; + if env.shutdown.is_cancelled() { + return Err(Stop::Cancelled); + } + if !fenced && !heartbeat.confirms_the_claim() { + return Err(Stop::ClaimLost( + heartbeat.why_not_confirmed("for the DONE write"), + )); + } + // The restored table's ItemCount and TableSizeBytes are 0 at ACTIVE, + // as the service reports them (measured: 0 and 0 on 15 restored + // tables, one of them 10 minutes after it went ACTIVE); the + // statistics pass rewrites them later where the backend has one. + finish(&*storage, &*claim, &target, 0, 0).await + } + .await; + let elapsed = started.elapsed().as_secs_f64(); + let (outcome, remove_entry) = match end { + Ok(Ended::Written(Finished::Active)) => { + tracing::info!( + "{name}: ACTIVE, {total_items} items ({applied_total} applied by this run), \ + {bytes_downloaded} bytes downloaded, in {elapsed:.1} s" + ); + (BackupJobOutcome::Succeeded, true) + } + Ok(Ended::Written(Finished::Pending)) => { + tracing::info!( + "{name}: DONE, {total_items} items ({applied_total} applied by this run), \ + {bytes_downloaded} bytes downloaded, in {elapsed:.1} s; index backfill pending, \ + the backend's worker flips the table ACTIVE" + ); + kill_point(KillPoint::RestoreMongodbBeforeWorkerFlip, &env.shutdown).await; + (BackupJobOutcome::Succeeded, true) + } + // The write landed and its answer did not come back: the table is + // ACTIVE, or awaits the backend's worker, which this job did not see. + Ok(Ended::DoneOnReRead) => { + tracing::info!( + "{name}: complete (DONE), {total_items} items ({applied_total} applied by this \ + run), {bytes_downloaded} bytes downloaded, in {elapsed:.1} s; the DONE write's \ + answer was lost and the record was re-read DONE under this job's generation (the \ + table is ACTIVE, or the backend's worker flips it)" + ); + (BackupJobOutcome::Succeeded, true) + } + // The record is another's: its owner may be reading this host's + // entry, so the entry is left to the sweep. + Ok(Ended::Written(Finished::NotMatched(why))) => { + tracing::error!( + "{name}: the DONE write matched no record ({why}); the record is not this job's \ + to write and no table access was made" + ); + (BackupJobOutcome::Failed, false) + } + Err(Stop::ClaimLost(why)) => { + tracing::error!( + "{name}: stopped with no further catalog or table access: {why}; the record is \ + left to its owner or to the reconciler" + ); + (BackupJobOutcome::Failed, false) + } + // The load is complete and a read after it kept failing: no FAILED + // write, the record stays RUNNING with its progress for a resume. + Err(Stop::LeftRunning(why)) => { + tracing::error!( + "{name}: stopped with the record RUNNING and no FAILED written: {why}; the \ + reconciler resumes it once the record is stale" + ); + (BackupJobOutcome::Failed, false) + } + Err(Stop::Cancelled) => { + tracing::info!( + "{name}: stopped by the server's shutdown; the record stays RUNNING with its \ + spool and its progress for a resume" + ); + (BackupJobOutcome::Cancelled, false) + } + Err(Stop::Fail(reason)) => { + let path = fail_restore( + &*storage, + &*claim, + &target, + &reason, + entry.as_ref(), + &env.shutdown, + ) + .await; + tracing::error!("{name}: FAILED ({reason}); {path}"); + // `fail_restore` removed the entry, or left it to the sweep. + (BackupJobOutcome::Failed, false) + } + }; + if remove_entry && let Some(entry) = &entry { + remove_spool_entry(entry).await; + } + // After the failure path too, so a FAILED job's duration covers its + // FAILED write and the target's removal. + let duration = started.elapsed(); + drop(heartbeat); + if let Some(metrics) = metrics { + #[allow(clippy::cast_precision_loss)] + let duration_us = duration.as_micros() as f64; + metrics.record_restore_job(outcome, duration_us); + match outcome { + BackupJobOutcome::Succeeded => { + metrics.record_worker_success(QuerySource::RestoreJob, duration_us); + } + BackupJobOutcome::Failed => metrics.record_worker_error(QuerySource::RestoreJob), + // A clean shutdown is not a worker error. + BackupJobOutcome::Cancelled => {} + } + } + outcome +} + +/// How many times the final count is attempted before the job stops with +/// the record RUNNING. +const COUNT_ATTEMPTS: u32 = 3; + +/// The final count (module doc): `count_restore_target`, retried after a +/// short wait on an error, [`COUNT_ATTEMPTS`] times in all, the shutdown +/// token and the heartbeat read before each attempt; an error on the last +/// attempt is [`Stop::LeftRunning`], never a FAILED write, since every item +/// is loaded and recorded and the reconciler's resume counts again. +async fn count_target( + storage: &S, + target: &RestoreTargetKey, + heartbeat: &HeartbeatHandle, + shutdown: &CancellationToken, +) -> Result { + for attempt in 1..=COUNT_ATTEMPTS { + if attempt > 1 { + if shutdown.is_cancelled() { + return Err(Stop::Cancelled); + } + if !heartbeat.confirms_the_claim() { + return Err(Stop::ClaimLost( + heartbeat.why_not_confirmed("to count the target again"), + )); + } + } + match storage.count_restore_target(target).await { + Ok(count) => return Ok(count), + Err(e) if attempt == COUNT_ATTEMPTS => { + return Err(Stop::LeftRunning(format!( + "counting the target's items failed {COUNT_ATTEMPTS} times, the last: {e}" + ))); + } + Err(e) => { + let wait = Duration::from_secs(u64::from(attempt)); + tracing::warn!( + "counting the target's items failed (attempt {attempt} of \ + {COUNT_ATTEMPTS}); retried in {} s: {e}", + wait.as_secs() + ); + tokio::select! { + biased; + () = shutdown.cancelled() => return Err(Stop::Cancelled), + () = heartbeat.lost().cancelled() => return Err(Stop::ClaimLost(heartbeat.reason())), + () = tokio::time::sleep(wait) => {} + } + } + } + } + unreachable!("the attempt loop returns on its last attempt") +} + +/// What step 6 settled to. +enum Ended { + /// The backend's own answer to the DONE write. + Written(Finished), + /// The DONE write's answer was lost and the record re-read DONE under + /// this generation: the restore is complete; whether the table is ACTIVE + /// or awaits the backend's worker (MongoDB) is a fact the backend holds + /// and this job did not see. + DoneOnReRead, +} + +/// Step 6's write (module doc): `finish_restore`, with an error from it +/// handled as a conditional write whose outcome is unknown. The record is +/// re-read through the claim: DONE under this generation is the write +/// landed ([`Ended::DoneOnReRead`], with a WARN naming the error), RUNNING +/// under this generation is the write not taken, sent once more at most; +/// another generation, another status, a failed re-read, or a second error +/// after a RUNNING re-read is the DONE write that matched no record, after +/// which the job makes no table access. +async fn finish( + storage: &S, + claim: &dyn JobClaim, + target: &RestoreTargetKey, + item_count: u64, + size_bytes: u64, +) -> Result { + let not_matched = |why: String| Ok(Ended::Written(Finished::NotMatched(why))); + for attempt in 0..2 { + let error = match storage + .finish_restore(target, claim, item_count, size_bytes) + .await + { + Ok(finished) => return Ok(Ended::Written(finished)), + Err(e) => e, + }; + let row = match claim.read().await { + Ok(row) => row, + Err(e) => { + return not_matched(format!( + "the DONE write failed ({error}) and the record could not be re-read: {e}" + )); + } + }; + if row.owner_generation != claim.generation() { + return not_matched(format!( + "the DONE write failed ({error}) and the record carries generation {} (this \ + claim's is {})", + row.owner_generation, + claim.generation() + )); + } + if row.status == JobStatus::Done { + tracing::warn!( + "the DONE write returned an error ({error}) and the record is DONE under this \ + job's generation: the write landed and the restore is complete" + ); + return Ok(Ended::DoneOnReRead); + } + if row.status != JobStatus::Running { + return not_matched(format!( + "the DONE write failed ({error}) and the record is {} under this claim", + row.status + )); + } + if attempt == 1 { + return not_matched(format!( + "the DONE write failed twice ({error}) and the record is still RUNNING under \ + this claim; the reconciler resumes it once the record is stale" + )); + } + tracing::warn!( + "the DONE write returned an error ({error}) and the record is still RUNNING under \ + this claim; writing once more" + ); + } + unreachable!("the loop returns on its second attempt") +} + +/// The end of the restore's failure path. +#[derive(Debug)] +pub(crate) enum RestoreFailurePath { + /// FAILED written and the target removed. + Removed, + /// FAILED written; the target was no longer a CREATING restore target + /// with this id and belongs to whoever moved it. + LeftToOwner, + /// FAILED written; the removal failed and the reconciler's FAILED-target + /// duty removes the target once the record is stale. + RemoveFailed(String), + /// FAILED not written by this claim (the record is another's, or the + /// claim connection failed): no table access was made. + NoTableAccess(String), +} + +impl std::fmt::Display for RestoreFailurePath { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Removed => f.write_str("the target was removed"), + Self::LeftToOwner => f.write_str( + "the target was no longer a CREATING restore target with this id and is left to \ + its owner", + ), + Self::RemoveFailed(e) => write!( + f, + "the target could not be removed and the reconciler removes it once the record \ + is stale: {e}" + ), + Self::NoTableAccess(why) => write!( + f, + "the record is not this job's to write ({why}); no table access was made" + ), + } + } +} + +/// Step 7 (module doc): FAILED with `reason` through `claim`, then +/// `remove_restore_target` only when that write matched; the spool entry, +/// when there is one, removed when the write matched and left to the sweep +/// when it did not (the record is another's, whose job may be reading the +/// entry). +pub(crate) async fn fail_restore( + storage: &S, + claim: &dyn JobClaim, + target: &RestoreTargetKey, + reason: &str, + entry: Option<&spool::Entry>, + shutdown: &CancellationToken, +) -> RestoreFailurePath { + let fields = StatusFields { + failure_reason: Some(reason.to_owned()), + completed: true, + ..StatusFields::default() + }; + let path = match write_status(claim, JobStatus::Running, JobStatus::Failed, &fields).await { + Written::Matched => { + kill_point(KillPoint::RestoreBetweenFailedAndRemoval, shutdown).await; + match storage.remove_restore_target(target).await { + Ok(true) => RestoreFailurePath::Removed, + Ok(false) => RestoreFailurePath::LeftToOwner, + Err(e) => RestoreFailurePath::RemoveFailed(e.to_string()), + } + } + Written::NotMatched(why) | Written::Error(why) => RestoreFailurePath::NoTableAccess(why), + }; + if let (Some(entry), false) = (entry, matches!(path, RestoreFailurePath::NoTableAccess(_))) { + remove_spool_entry(entry).await; + } + path +} + +async fn remove_spool_entry(entry: &spool::Entry) { + if let Err(e) = entry.remove().await { + tracing::error!( + "the spool entry {} could not be removed; the reconciler's sweep retries: {e}", + entry.dir().display() + ); + } +} + +/// A dead owner's record as the reconciler hands it to [`resume`]. +pub(crate) struct DeadRecord { + /// The record as read under the takeover's claim. + pub row: RestoreJobRow, + /// The `spool_host` the record carried before the takeover (which wrote + /// this host's): whose spool entry, if any, holds the files. + pub recorded_spool_host: Option, + /// The progress rows a backend keeps apart from the record. + pub progress_rows: Vec<(u32, u64)>, + /// The instant `claim_restore_dead` returned: the resumed job's first + /// confirmed beat (module doc). + pub claimed_at: Instant, +} + +/// The reconciler's entry: a dead owner's RUNNING record taken over +/// (`claim`), resumed from its progress (the record and, on PostgreSQL, its +/// progress rows). The manifest and sidecar are read from this host's spool +/// entry when the record named this host before the takeover and both are +/// there and verify, else from the store; either way they are verified and +/// must name the record's backup. A manifest that cannot be read or is +/// refused fails the restore (step 7), and this host's entry, when the +/// record named this host, goes with it. The first confirmed beat is +/// `record.claimed_at`, the takeover's return. +pub(crate) async fn resume( + storage: Arc, + env: JobEnv, + permit: JobPermit, + claim: Box, + record: DeadRecord, + limits: RestorableLimits, +) where + S: BackupCatalog + RestoreEngine + ?Sized + Send + Sync + 'static, +{ + let started = Instant::now(); + let claim: Arc = Arc::from(claim); + let DeadRecord { + row, + recorded_spool_host, + progress_rows, + claimed_at, + } = record; + let target = row.target.clone(); + let name = format!( + "restore of {} into {} (account {})", + row.source_backup_arn, target.table_name, target.account_id + ); + // This host's entry, when the record named this host before the + // takeover: the spooled documents are read from it, and a resume that + // fails before it runs removes it with the failure path, as the job + // removes its entry at every failed end. An entry that cannot be opened + // is left to the sweep and the store's documents are read. + let entry = env + .spool + .as_ref() + .filter(|spool| recorded_spool_host.as_deref() == Some(spool.spool_host.as_str())) + .and_then(|spool| { + spool::Entry::open( + &spool.path, + &target.table_id, + recorded_spool_host.as_deref(), + &spool.spool_host, + ) + .ok() + }); + let prepared = prepare_resume(&env, &row, entry.as_ref(), &progress_rows).await; + let (arn, manifest, manifest_bytes, sidecar_bytes, key_schema, progress, from_spool) = + match prepared { + Ok(prepared) => prepared, + Err(reason) => { + let path = fail_restore( + &*storage, + &*claim, + &target, + &reason, + entry.as_ref(), + &env.shutdown, + ) + .await; + tracing::error!("{name}: FAILED on resume ({reason}); {path}"); + if let Some(metrics) = env.metrics.as_deref() { + metrics.record_worker_error(QuerySource::RestoreJob); + #[allow(clippy::cast_precision_loss)] + let duration_us = started.elapsed().as_micros() as f64; + metrics.record_restore_job(BackupJobOutcome::Failed, duration_us); + } + drop(permit); + return; + } + }; + tracing::info!( + "{name}: resumed by this instance; the manifest was read from {}", + if from_spool { + "this host's spool" + } else { + "the store" + } + ); + let job = RestoreJob { + storage, + claim, + target, + arn, + manifest: Arc::new(manifest), + manifest_bytes, + sidecar_bytes, + key_schema, + limits, + env, + started, + claimed_at, + progress: Some(progress), + recorded_spool_host, + documents_from_spool: from_spool, + }; + run(job).await; + drop(permit); +} + +/// What a resume reads before it runs: the manifest and sidecar (from this +/// host's spool entry when there is one and the spooled pair verifies, else +/// the store), verified, with the key schema and the progress, and whether +/// the pair came from the spool; the `Err` is the FAILED reason. +type Prepared = ( + BackupArn, + ExtenddbManifest, + Vec, + Vec, + RestorableKeySchema, + Progress, + bool, +); + +/// The manifest of a (manifest, sidecar) pair that verifies and names +/// `arn`; the `Err` says what failed. +fn verified_manifest( + manifest_bytes: &[u8], + sidecar_bytes: &[u8], + arn: &BackupArn, +) -> Result { + verify_manifest_checksum(manifest_bytes, sidecar_bytes) + .map_err(|e| format!("the manifest's checksum sidecar does not match: {e}"))?; + let manifest = ExtenddbManifest::from_json_bytes(manifest_bytes) + .map_err(|e| format!("the manifest is refused: {e}"))?; + match manifest.backup_arn() { + Ok(named) if named == *arn => Ok(manifest), + Ok(named) => Err(format!( + "the manifest names another backup, {}", + named.to_arn_string() + )), + Err(e) => Err(format!("the manifest's backup ARN is refused: {e}")), + } +} + +async fn prepare_resume( + env: &JobEnv, + row: &RestoreJobRow, + entry: Option<&spool::Entry>, + progress_rows: &[(u32, u64)], +) -> Result { + let target = &row.target; + let arn = BackupArn::parse(&row.source_backup_arn) + .map_err(|e| format!("the record's backup ARN is refused: {e}"))?; + let prefix = arn.store_prefix(); + let spooled = entry.and_then(|entry| { + let manifest = entry.read_document(EXTENDDB_MANIFEST_FILE).ok()??; + let sidecar = entry.read_document(MANIFEST_CHECKSUM_FILE).ok()??; + Some((manifest, sidecar)) + }); + // The spooled pair, when it verifies; a pair a crash left short, or + // that names another backup, is replaced by the store's. + let spooled = spooled.and_then(|(manifest_bytes, sidecar_bytes)| { + match verified_manifest(&manifest_bytes, &sidecar_bytes, &arn) { + Ok(manifest) => Some((manifest, manifest_bytes, sidecar_bytes)), + Err(e) => { + tracing::warn!( + "the spooled manifest and sidecar of the restore into {} (account {}) do not \ + verify ({e}); the store's are read instead", + target.table_name, + target.account_id + ); + None + } + } + }); + let from_spool = spooled.is_some(); + let (manifest, manifest_bytes, sidecar_bytes) = match spooled { + Some(documents) => documents, + None => { + let read = async |name: &str, bound: u64| -> Result, String> { + match super::reconciler::read_object(&*env.store, &format!("{prefix}{name}"), bound) + .await + { + Ok(ReadObject::Bytes(bytes)) => Ok(bytes), + Ok(ReadObject::Absent) => { + Err(format!("the store holds no {name} for the backup")) + } + Ok(ReadObject::Oversize) => Err(format!( + "the store's {name} for the backup is over {bound} bytes, which no \ + backup this server wrote produces" + )), + Err(e) => { + if let Some(metrics) = env.metrics.as_deref() { + metrics.record_backup_store_error("get"); + } + Err(format!("reading {name} from the store: {e}")) + } + } + }; + let manifest_bytes = read(EXTENDDB_MANIFEST_FILE, MANIFEST_READ_BOUND).await?; + let sidecar_bytes = read(MANIFEST_CHECKSUM_FILE, SIDECAR_READ_BOUND).await?; + let manifest = verified_manifest(&manifest_bytes, &sidecar_bytes, &arn)?; + (manifest, manifest_bytes, sidecar_bytes) + } + }; + let key_schema = RestorableKeySchema::resolve( + &manifest.source_table.key_schema, + &manifest.source_table.attribute_definitions, + ) + .map_err(|e| format!("the key schema cannot be checked: {e}"))?; + let progress = Progress::from_records(row, progress_rows, &manifest); + Ok(( + arn, + manifest, + manifest_bytes, + sidecar_bytes, + key_schema, + progress, + from_spool, + )) +} + +/// What a test can do to a running job by its target table id: end its +/// heartbeat timer without raising the loss (`HeartbeatHandle::stop_timer`) +/// while the job is held at a gate of the storage double, and give its +/// loads a smaller chunk byte bound than [`load::LOAD_CHUNK_BYTES`]. Tests +/// that use it give their target a name of their own; the maps are +/// process-wide. +#[cfg(test)] +pub(crate) mod test_support { + use std::collections::HashMap; + use std::sync::{LazyLock, Mutex, PoisonError}; + + use super::HeartbeatHandle; + + static TIMERS: LazyLock>> = + LazyLock::new(|| Mutex::new(HashMap::new())); + static CHUNK_BYTES: LazyLock>> = + LazyLock::new(|| Mutex::new(HashMap::new())); + + /// Give the loads of the job into `table_id` a chunk byte bound of + /// `bytes`. + pub(crate) fn set_chunk_bytes(table_id: &str, bytes: u64) { + CHUNK_BYTES + .lock() + .unwrap_or_else(PoisonError::into_inner) + .insert(table_id.to_owned(), bytes); + } + + pub(super) fn chunk_bytes_for(table_id: &str) -> u64 { + CHUNK_BYTES + .lock() + .unwrap_or_else(PoisonError::into_inner) + .get(table_id) + .copied() + .unwrap_or(super::load::LOAD_CHUNK_BYTES) + } + + pub(super) fn register_timer(table_id: &str, heartbeat: &HeartbeatHandle) { + TIMERS + .lock() + .unwrap_or_else(PoisonError::into_inner) + .insert(table_id.to_owned(), heartbeat.timer_abort_handle()); + } + + /// End the heartbeat timer of the running job into `table_id`. + pub(crate) fn stop_timer(table_id: &str) { + let handle = TIMERS + .lock() + .unwrap_or_else(PoisonError::into_inner) + .get(table_id) + .cloned() + .expect("a job into the table is running"); + handle.abort(); + } +} diff --git a/crates/engine/src/backup/restore/download.rs b/crates/engine/src/backup/restore/download.rs new file mode 100644 index 000000000..7cb42273f --- /dev/null +++ b/crates/engine/src/backup/restore/download.rs @@ -0,0 +1,320 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Step 3 of a restore: every data file read from the store and compared +//! with the manifest's `sha256`, `md5` and `bytes` before any file is +//! loaded. +//! +//! On an S3 store each file is written to the job's spool entry as it +//! arrives ([`Dest::Spool`]), and committed with its `.verified` marker once +//! the digests match; a resume skips the files an earlier run marked (the +//! load hashes the spooled bytes again as it decodes them, so a marker is a +//! reason not to download, not a reason not to check). On a +//! filesystem store ([`Dest::InPlace`]) the file is hashed as the store +//! streams it and nothing is written: the load reads the same file through +//! a second `get`, hashing it again, so a file replaced between the two +//! passes fails there. +//! +//! A `get` that fails as a transport error, at the call or while the body +//! arrives, is retried with a backoff of `min(2^n s, 60 s)` plus jitter for +//! as long as the job has been downloading for less than +//! [`DOWNLOAD_RETRY_BOUND`], counted from the first download attempt of the +//! job (one clock for every file, so a store that flaps through a backup of +//! many files costs the restore the bound once, not once per file), the +//! record RUNNING and heartbeated meanwhile; the shutdown token and the +//! heartbeat's loss end the wait. Every other failure fails the restore at +//! once: a digest or size mismatch (the file is not the backup's), an absent +//! file (a prefix changed under an AVAILABLE row), a permission or key +//! refusal, a local I/O error (the filesystem store's `Io`, or the spool's +//! disk), or any other store error. The filesystem store reports a mount +//! that stopped answering as `Io`, so the retry covers S3 stores only. +//! Every `get` the store fails, retried or not, is one `BackupStoreErrors{get}`; +//! a mismatch and the spool's own errors are not the store's. + +use std::time::Duration; + +use extenddb_backup_store::{BackupStore, StoreError}; +use extenddb_core::backup_format::{CompressedHasher, DataFileEntry, ExtenddbManifest}; +use extenddb_core::metrics::MetricsCollector; +use extenddb_storage::CancellationToken; +use futures::TryStreamExt; +use tokio::io::AsyncWriteExt; +use tokio::time::Instant; + +use super::super::job::heartbeat::HeartbeatHandle; +use super::spool::Entry; + +/// How long the job's downloads are retried over transport errors, from +/// the job's first download attempt, before the restore fails. +pub(crate) const DOWNLOAD_RETRY_BOUND: Duration = Duration::from_secs(15 * 60); + +/// The longest wait between two attempts. +const MAX_BACKOFF: Duration = Duration::from_secs(60); + +/// Where a verified file goes. +pub(crate) enum Dest<'a> { + /// Hashed as it streams, written nowhere: the load reads the store + /// again (a filesystem store). + InPlace, + /// Written to the job's spool entry and marked `.verified`. + Spool(&'a Entry), +} + +/// Why the download stopped. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum Stop { + /// The restore fails with this reason. + Fail(String), + /// The shutdown token fired. + Cancelled, + /// The heartbeat no longer confirms the claim. + ClaimLost(String), + /// The job stops with the record RUNNING and no FAILED written (the + /// final count kept failing after every item was loaded and recorded); + /// the reconciler resumes it once the record is stale. + LeftRunning(String), +} + +/// What one download runs with. +pub(crate) struct Downloader<'a> { + pub store: &'a dyn BackupStore, + /// The backup's store prefix, which every key is under. + pub prefix: &'a str, + pub heartbeat: &'a HeartbeatHandle, + pub shutdown: &'a CancellationToken, + pub metrics: Option<&'a MetricsCollector>, +} + +/// Download and verify every data file of `manifest` into `dest` (module +/// doc), except the files `complete` marks (a resume's progress records +/// them applied whole, so they are neither downloaded nor loaded) and the +/// files the spool holds verified with the manifest's digest. Returns the +/// bytes read from the store over every attempt. +pub(crate) async fn download_all( + d: &Downloader<'_>, + manifest: &ExtenddbManifest, + dest: &Dest<'_>, + complete: &[bool], +) -> Result { + // The retry bound's clock: one for the job, started here. + let started = Instant::now(); + let mut bytes = 0; + for (index, entry) in manifest.data_files.iter().enumerate() { + if complete.get(index).copied().unwrap_or(false) { + continue; + } + let index = + u32::try_from(index).map_err(|_| Stop::Fail("too many data files".to_owned()))?; + if let Dest::Spool(spool) = dest + && spool.is_verified(index, &entry.sha256) + { + continue; + } + bytes += fetch_verified(d, entry, index, dest, started).await?; + } + Ok(bytes) +} + +/// One file: attempts until the file verifies, a non-transport failure, the +/// retry bound (measured from `started`, the job's first attempt), the +/// shutdown or a lost claim. +async fn fetch_verified( + d: &Downloader<'_>, + entry: &DataFileEntry, + index: u32, + dest: &Dest<'_>, + started: Instant, +) -> Result { + let key = format!("{}{}", d.prefix, entry.key); + let mut bytes = 0; + for attempt in 0u32.. { + if d.shutdown.is_cancelled() { + return Err(Stop::Cancelled); + } + if !d.heartbeat.confirms_the_claim() { + return Err(Stop::ClaimLost( + d.heartbeat + .why_not_confirmed(&format!("to download {}", entry.key)), + )); + } + let outcome = tokio::select! { + biased; + () = d.shutdown.cancelled() => return Err(Stop::Cancelled), + () = d.heartbeat.lost().cancelled() => return Err(Stop::ClaimLost(d.heartbeat.reason())), + outcome = attempt_once(d, entry, index, &key, dest) => outcome, + }; + bytes += outcome.bytes; + if let Some(metrics) = d.metrics + && outcome.bytes > 0 + { + metrics.record_restore_bytes_downloaded(outcome.bytes); + } + let transport = match outcome.result { + Ok(()) => return Ok(bytes), + Err(Attempt::Fail(reason)) => return Err(Stop::Fail(reason)), + Err(Attempt::StoreFailed(reason)) => { + // The store's refusal counts as its error, as a transport + // failure does; the restore then fails at once. + if let Some(metrics) = d.metrics { + metrics.record_backup_store_error("get"); + } + return Err(Stop::Fail(reason)); + } + Err(Attempt::Transport(e)) => e, + }; + if let Some(metrics) = d.metrics { + metrics.record_backup_store_error("get"); + } + let elapsed = started.elapsed(); + if elapsed >= DOWNLOAD_RETRY_BOUND { + return Err(Stop::Fail(format!( + "downloading {}: {transport} (the job has retried its downloads for {} s; the \ + bound is {} s)", + entry.key, + elapsed.as_secs(), + DOWNLOAD_RETRY_BOUND.as_secs() + ))); + } + let wait = backoff(attempt); + tracing::warn!( + "downloading {key}: {transport}; attempt {} retried in {:.1} s", + attempt + 2, + wait.as_secs_f64() + ); + tokio::select! { + biased; + () = d.shutdown.cancelled() => return Err(Stop::Cancelled), + () = d.heartbeat.lost().cancelled() => return Err(Stop::ClaimLost(d.heartbeat.reason())), + () = tokio::time::sleep(wait) => {} + } + } + unreachable!("the attempt loop returns") +} + +/// `min(2^attempt s, 60 s)` plus a jitter of up to one second. +fn backoff(attempt: u32) -> Duration { + let base = Duration::from_secs(1u64 << attempt.min(6)); + let jitter_ms = u64::from(uuid::Uuid::new_v4().as_u128() as u32) % 1000; + base.min(MAX_BACKOFF) + Duration::from_millis(jitter_ms) +} + +/// How one attempt ended. +enum Attempt { + /// The restore fails with this reason; no retry, and not the store's + /// error (a digest or size mismatch, the spool's disk). + Fail(String), + /// The store failed the `get` with an error that is not retried (an + /// absent file, a refusal, an I/O error, any other); the restore fails + /// with this reason and the error is counted as the store's. + StoreFailed(String), + /// Retried within the bound. + Transport(StoreError), +} + +struct AttemptOutcome { + result: Result<(), Attempt>, + /// Bytes this attempt read from the store. + bytes: u64, +} + +/// One `get` of the file, hashed and compared, written to the spool when +/// `dest` is one. +async fn attempt_once( + d: &Downloader<'_>, + entry: &DataFileEntry, + index: u32, + key: &str, + dest: &Dest<'_>, +) -> AttemptOutcome { + let mut bytes = 0; + let result = attempt_inner(d, entry, index, key, dest, &mut bytes).await; + AttemptOutcome { result, bytes } +} + +async fn attempt_inner( + d: &Downloader<'_>, + entry: &DataFileEntry, + index: u32, + key: &str, + dest: &Dest<'_>, + bytes: &mut u64, +) -> Result<(), Attempt> { + let mut body = match d.store.get(key).await { + Ok(body) => body, + Err(e @ StoreError::Transport { .. }) => return Err(Attempt::Transport(e)), + Err(StoreError::NotFound) => { + return Err(Attempt::StoreFailed(format!( + "data file {} is not in the store", + entry.key + ))); + } + Err(e) => { + return Err(Attempt::StoreFailed(format!( + "reading data file {} from the store: {e}", + entry.key + ))); + } + }; + let mut file = match dest { + Dest::InPlace => None, + Dest::Spool(spool) => Some(spool.begin_file(index).await.map_err(|e| { + Attempt::Fail(format!("writing data file {} to the spool: {e}", entry.key)) + })?), + }; + let mut hasher = CompressedHasher::new(); + loop { + let chunk = match body.try_next().await { + Ok(Some(chunk)) => chunk, + Ok(None) => break, + Err(e @ StoreError::Transport { .. }) => return Err(Attempt::Transport(e)), + Err(e) => { + return Err(Attempt::StoreFailed(format!( + "reading data file {} from the store: {e}", + entry.key + ))); + } + }; + *bytes += chunk.len() as u64; + hasher.update(&chunk); + if let Some(file) = &mut file { + file.write_all(&chunk).await.map_err(|e| { + Attempt::Fail(format!("writing data file {} to the spool: {e}", entry.key)) + })?; + } + } + entry + .expectations() + .verify_digest(&hasher.finish()) + .map_err(|e| { + Attempt::Fail(format!( + "data file {} does not match the manifest: {e}", + entry.key + )) + })?; + if let (Some(file), Dest::Spool(spool)) = (file, dest) { + spool + .commit_file(index, file, &entry.sha256) + .await + .map_err(|e| { + Attempt::Fail(format!("writing data file {} to the spool: {e}", entry.key)) + })?; + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_backoff_doubles_to_a_minute_with_jitter() { + for (attempt, base) in [(0, 1), (1, 2), (3, 8), (5, 32), (6, 64), (20, 64)] { + let wait = backoff(attempt); + let base = Duration::from_secs(base).min(MAX_BACKOFF); + assert!( + wait >= base && wait < base + Duration::from_secs(1), + "{attempt}: {wait:?}" + ); + } + } +} diff --git a/crates/engine/src/backup/restore/load.rs b/crates/engine/src/backup/restore/load.rs new file mode 100644 index 000000000..3e5de40d4 --- /dev/null +++ b/crates/engine/src/backup/restore/load.rs @@ -0,0 +1,285 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Step 5 of a restore: one verified data file decoded, validated and +//! handed to the backend's `bulk_load`. +//! +//! The decoder is synchronous (`std::io::Read`), so it runs on the blocking +//! pool: over the spooled file, or, on a filesystem store, over a +//! second `get` of the store bridged to `Read`, in both cases through +//! `ExtenddbManifest::reader`, which hashes the bytes again as it decodes +//! them: a spooled file's `.verified` marker says the download matched the +//! manifest, not that the file in the spool still does (a resume on the +//! same host reads what the spool holds now). On both paths the +//! items are handed to `bulk_load` as they decode and the digest is checked +//! at the end of the stream, so the items of a file replaced between the two +//! passes reach the backend (and, on a backend that commits per batch, the +//! CREATING target) before the end-of-stream check fails the file and the +//! failure path removes the target; the store root's and the spool's +//! owner-only modes are what keep the file from changing. Each item past the resume's `start_at` is +//! validated against the target's limits and key schema +//! (`validate_restorable_item`), then collected into chunks of at most +//! [`LOAD_CHUNK_ITEMS`] items or [`LOAD_CHUNK_BYTES`] of line text (the +//! bytes the decoder read for the items, the unit the backup side budgets +//! in), whichever comes first, and sent over a channel of two chunks to the +//! async side, which flattens them into the item stream of one `bulk_load` +//! call per file. The memory a file's load holds is therefore the decoded +//! items of at most `LOAD_CHUNK_BYTES` plus one line of line text per chunk +//! (the line that crosses the bound is kept in the chunk it was read for) at +//! four chunk positions (two in the channel, the one being filled, the one +//! being drained), the decoder's line (at most `MAX_LINE_BYTES`) and the +//! deserializer's intermediate value for it, and the backend's own batch. +//! +//! A decode or validation failure, the shutdown token and the heartbeat +//! (its loss, or a timer that no longer confirms the claim, both read at +//! every chunk boundary) end the stream with an `Err` item, which the +//! backend answers by failing the file (the design: nothing of a failed file +//! is recorded where the backend writes a file in one transaction); the +//! reason is kept here so the job reports it rather than the backend's +//! wrapping of it. A decoder that stops without ending the stream itself (a +//! panic on the blocking thread) fails the file too: a clean end of stream +//! the backend saw is not taken for a complete file. The join of the +//! decoder's thread after `bulk_load` returned has no bound of its own: on +//! the in-place path the thread reads the store through the bridge, and a +//! filesystem mount that stopped answering holds it as it holds the store's +//! own `get`, until the mount answers. + +use std::io::Read; +use std::path::PathBuf; +use std::sync::{Arc, Mutex, PoisonError}; + +use extenddb_backup_store::BackupStore; +use extenddb_core::backup_format::{DataFileEntry, ExtenddbManifest}; +use extenddb_core::types::Item; +use extenddb_core::validation::{RestorableKeySchema, RestorableLimits, validate_restorable_item}; +use extenddb_storage::backup_catalog::RestoreTargetKey; +use extenddb_storage::error::StorageError; +use extenddb_storage::{BulkLoadFile, CancellationToken, JobClaim, RestoreEngine}; +use futures::{StreamExt, TryStreamExt}; +use tokio::sync::mpsc; + +use super::super::job::heartbeat::{ClaimCheck, HeartbeatHandle}; +pub(crate) use super::download::Stop; + +/// Items per chunk handed to the backend, at most. +pub(crate) const LOAD_CHUNK_ITEMS: usize = 1000; +/// Line bytes per chunk (the data file's text the decoder read for the +/// chunk's items), at most; a chunk closes at the first item that takes it +/// to this. +pub(crate) const LOAD_CHUNK_BYTES: u64 = 32 * 1024 * 1024; +/// Chunks the channel between the decoder and the backend holds. +const CHUNK_CHANNEL: usize = 2; + +/// Where the file's bytes are read from. +pub(crate) enum Source { + /// The spooled, verified file. + Spool(PathBuf), + /// The store again (a filesystem store, read in place). + InPlace { + store: Arc, + key: String, + }, +} + +/// One file's load. +pub(crate) struct FileLoad<'a> { + pub manifest: &'a Arc, + pub entry: &'a DataFileEntry, + pub file_index: u32, + /// Items of the file an earlier run applied, decoded and skipped. + pub start_at: u64, + pub reapplying: bool, + pub key_schema: &'a RestorableKeySchema, + pub limits: RestorableLimits, + pub shutdown: &'a CancellationToken, + pub heartbeat: &'a HeartbeatHandle, + /// The chunk's byte bound: [`LOAD_CHUNK_BYTES`], or a smaller bound a + /// test set for its job. + pub chunk_bytes: u64, +} + +/// Decode `source` from `load.start_at` on, validate every item, and write +/// the file through `storage.bulk_load` (module doc). Returns the items the +/// backend applied. +pub(crate) async fn load_file( + storage: &S, + target: &RestoreTargetKey, + claim: &dyn JobClaim, + load: FileLoad<'_>, + source: Source, +) -> Result { + let key = load.entry.key.clone(); + let read: Box = match source { + Source::Spool(path) => { + let file = std::fs::File::open(&path).map_err(|e| { + Stop::Fail(format!( + "opening data file {key} in the spool ({}): {e}", + path.display() + )) + })?; + Box::new(file) + } + Source::InPlace { + store, + key: store_key, + } => { + let body = store + .get(&store_key) + .await + .map_err(|e| Stop::Fail(format!("reading data file {key} from the store: {e}")))?; + let reader = tokio_util::io::StreamReader::new(body.map_err(std::io::Error::other)); + let bridge = tokio_util::io::SyncIoBridge::new_with_handle( + reader, + tokio::runtime::Handle::current(), + ); + Box::new(bridge) + } + }; + let stop: Arc>> = Arc::new(Mutex::new(None)); + let (tx, rx) = mpsc::channel::, StorageError>>(CHUNK_CHANNEL); + let feeder = Feeder { + manifest: Arc::clone(load.manifest), + entry: load.entry.clone(), + start_at: load.start_at, + key_schema: load.key_schema.clone(), + limits: load.limits, + shutdown: load.shutdown.clone(), + claim: load.heartbeat.claim_check(), + chunk_bytes: load.chunk_bytes, + stop: Arc::clone(&stop), + tx, + }; + let decoder = tokio::task::spawn_blocking(move || feeder.run(read)); + let items = futures::stream::unfold(rx, |mut rx| async move { + rx.recv().await.map(|chunk| (chunk, rx)) + }) + .flat_map(|chunk| match chunk { + Ok(items) => futures::stream::iter(items.into_iter().map(Ok)).boxed(), + Err(e) => futures::stream::once(async move { Err(e) }).boxed(), + }) + .boxed(); + let file = BulkLoadFile { + file_index: load.file_index, + start_at: load.start_at, + reapplying: load.reapplying, + }; + let result = tokio::select! { + biased; + () = load.heartbeat.lost().cancelled() => Err(None), + result = storage.bulk_load(target, claim, file, items) => result.map_err(Some), + }; + // The stream is dropped with the call, so a decoder blocked on the + // channel returns; its thread is joined before the file is reported. A + // decoder that did not return (it panicked) dropped the channel without + // an `Err` item, so the backend saw a clean end of a short stream: the + // file fails here, where a backend that writes per file rolls it back. + if decoder.await.is_err() { + return Err(Stop::Fail(format!( + "data file {key}: the decoder stopped unexpectedly" + ))); + } + let stopped = stop.lock().unwrap_or_else(PoisonError::into_inner).take(); + match (result, stopped) { + (Ok(applied), None) => Ok(applied), + (Ok(_), Some(stop)) | (Err(_), Some(stop)) => Err(stop), + (Err(None), None) => Err(Stop::ClaimLost(load.heartbeat.reason())), + (Err(Some(e)), None) => Err(Stop::Fail(format!("data file {key}: {e}"))), + } +} + +/// The blocking side: the decoder, the validator and the chunking. +struct Feeder { + manifest: Arc, + entry: DataFileEntry, + start_at: u64, + key_schema: RestorableKeySchema, + limits: RestorableLimits, + shutdown: CancellationToken, + /// The heartbeat's pre-write check, read at every chunk boundary: the + /// loss raised, or a timer that no longer confirms the claim. + claim: ClaimCheck, + chunk_bytes: u64, + stop: Arc>>, + tx: mpsc::Sender, StorageError>>, +} + +impl Feeder { + fn run(self, read: Box) { + let key = self.entry.key.clone(); + let mut reader = match self.manifest.reader(&self.entry, read) { + Ok(reader) => reader, + Err(e) => { + self.end(Stop::Fail(format!("data file {key}: {e}"))); + return; + } + }; + let mut index: u64 = 0; + let mut chunk = Vec::new(); + // The line bytes of the items in `chunk`, from the decoder's count. + let mut chunk_bytes: u64 = 0; + let mut decoded_before = reader.bytes_decoded(); + loop { + match reader.next_item() { + Ok(Some(item)) => { + let line_bytes = reader.bytes_decoded() - decoded_before; + decoded_before = reader.bytes_decoded(); + index += 1; + if index <= self.start_at { + continue; + } + if let Err(e) = validate_restorable_item(&item, &self.key_schema, &self.limits) + { + self.end(Stop::Fail(format!( + "data file {key}: item cannot be restored: {e}" + ))); + return; + } + chunk_bytes += line_bytes; + chunk.push(item); + if chunk.len() >= LOAD_CHUNK_ITEMS || chunk_bytes >= self.chunk_bytes { + if self.shutdown.is_cancelled() { + self.end(Stop::Cancelled); + return; + } + if !self.claim.confirms_the_claim() { + self.end(Stop::ClaimLost( + "the claim was lost while the file was being decoded".to_owned(), + )); + return; + } + if self + .tx + .blocking_send(Ok(std::mem::take(&mut chunk))) + .is_err() + { + // The backend ended the call; nothing more to feed. + return; + } + chunk_bytes = 0; + } + } + Ok(None) => { + if !chunk.is_empty() { + let _ = self.tx.blocking_send(Ok(chunk)); + } + return; + } + Err(e) => { + self.end(Stop::Fail(format!("data file {key}: {e}"))); + return; + } + } + } + } + + /// Record why the stream ends and end it with an `Err` item. + fn end(&self, stop: Stop) { + let text = match &stop { + Stop::Fail(reason) => reason.clone(), + Stop::Cancelled => "the restore was stopped by the server's shutdown".to_owned(), + Stop::ClaimLost(why) | Stop::LeftRunning(why) => why.clone(), + }; + *self.stop.lock().unwrap_or_else(PoisonError::into_inner) = Some(stop); + let _ = self.tx.blocking_send(Err(StorageError::Internal(text))); + } +} diff --git a/crates/engine/src/backup/restore/spool.rs b/crates/engine/src/backup/restore/spool.rs new file mode 100644 index 000000000..d13730864 --- /dev/null +++ b/crates/engine/src/backup/restore/spool.rs @@ -0,0 +1,516 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The restore spool of an S3 store: where a restore job downloads the +//! backup's files before it loads any of them. +//! +//! One restore owns one entry, `restore-
    /` under the spool +//! directory the server took at startup (`backup_store::take_spool`: owner +//! only, locked). The entry is created `0700` and every file in it `0600` +//! (the database and its sidecars are `0600`; the spool holds the same +//! data). A data file is written as `.json.gz.partial`, synced, +//! renamed to `.json.gz`, and only then marked `.json.gz.verified`, +//! after the job has compared its digests with the manifest's; the marker +//! holds the file's sha256, and a file counts as verified only while the +//! marker names the digest the manifest in hand records for it, so a +//! replaced manifest does not have an old entry's files accepted. A +//! `.partial`, an unmarked file or a marker naming another digest is one a +//! crash or another manifest left, and the next run of the job removes it +//! and downloads the file again. The manifest and its sidecar are written to +//! the entry too, the same way (`.partial`, synced, renamed), so a +//! crash during the write leaves the previous document or none and never a +//! truncated one, and a resume on the same host reads them from the spool +//! and verifies them again (a pair that fails verification is replaced by +//! the store's). An entry is this host's only while the record's +//! `spool_host` is this host's: a successor on another host finds no entry +//! of its own, and one whose own old entry survives from an earlier run of +//! the job (the record moved to another host and back) removes it before +//! use ([`Entry::open`]). The job removes its entry when it ends by any +//! path but the server's shutdown, which keeps it for a resume, and a job +//! that learnt the record is another's (its claim lost, its final write +//! matching no record), which leaves it to the sweep; the reconciler's sweep +//! ([`list_entries`], then [`sweep_entries`]) removes the entries it listed +//! before it read the records whose job is not RUNNING with this host's +//! `spool_host`, so an entry opened after the listing is never a candidate. + +use std::collections::HashSet; +use std::io; +use std::path::{Path, PathBuf}; + +use tokio::io::AsyncWriteExt; + +/// The prefix of an entry's directory name, followed by the table id. +const ENTRY_PREFIX: &str = "restore-"; +/// The suffix of a file being downloaded. +const PARTIAL: &str = ".partial"; +/// The suffix of the marker of a file whose digests the job verified. +const VERIFIED: &str = ".verified"; + +/// One restore's directory in the spool. +pub(crate) struct Entry { + dir: PathBuf, + /// Make the rename into place fail, so a test can see what a crash + /// there leaves. + #[cfg(test)] + pub(crate) fail_rename: std::sync::atomic::AtomicBool, +} + +impl Entry { + /// The entry of the restore into table `table_id`, created `0700` when + /// it does not exist. An existing entry is a resume's and is kept when + /// `record_spool_host`, the `spool_host` the record carried before this + /// job's claim, is `this_host`; otherwise the files in it were verified + /// by a run whose record has since been another host's, and the entry + /// is removed and created again, empty. + pub(crate) fn open( + spool: &Path, + table_id: &str, + record_spool_host: Option<&str>, + this_host: &str, + ) -> io::Result { + let dir = spool.join(format!("{ENTRY_PREFIX}{table_id}")); + if dir.is_dir() && record_spool_host != Some(this_host) { + tracing::info!( + "the spool entry {} is left from a run whose record moved to another host; it is \ + removed and the files downloaded again", + dir.display() + ); + std::fs::remove_dir_all(&dir)?; + } + if !dir.is_dir() { + let mut builder = std::fs::DirBuilder::new(); + #[cfg(unix)] + { + use std::os::unix::fs::DirBuilderExt; + builder.mode(0o700); + } + match builder.create(&dir) { + Ok(()) => {} + Err(e) if e.kind() == io::ErrorKind::AlreadyExists => {} + Err(e) => return Err(e), + } + } + Ok(Self { + dir, + #[cfg(test)] + fail_rename: std::sync::atomic::AtomicBool::new(false), + }) + } + + /// The entry's directory. + pub(crate) fn dir(&self) -> &Path { + &self.dir + } + + /// The path of data file `index` once downloaded and renamed. + pub(crate) fn file_path(&self, index: u32) -> PathBuf { + self.dir.join(format!("{index:06}.json.gz")) + } + + fn partial_path(&self, index: u32) -> PathBuf { + let mut name = self.file_path(index).into_os_string(); + name.push(PARTIAL); + PathBuf::from(name) + } + + fn verified_path(&self, index: u32) -> PathBuf { + let mut name = self.file_path(index).into_os_string(); + name.push(VERIFIED); + PathBuf::from(name) + } + + /// Whether data file `index` is in the entry with a `.verified` marker + /// naming `sha256`, the manifest's digest of the file: a file an earlier + /// run of the job downloaded and verified against the same manifest. + pub(crate) fn is_verified(&self, index: u32, sha256: &str) -> bool { + self.file_path(index).is_file() + && std::fs::read(self.verified_path(index)).is_ok_and(|marker| { + marker.strip_suffix(b"\n").unwrap_or(&marker) == sha256.as_bytes() + }) + } + + /// Write a document (the manifest, the sidecar) as `name` in the entry + /// the way a data file is written: `.partial`, `0600`, synced, + /// then renamed over `name` and the directory synced. An existing + /// document stands until the rename, so a crash during the write leaves + /// it whole or leaves the new one, never a truncated file. + pub(crate) async fn write_document(&self, name: &str, bytes: &[u8]) -> io::Result<()> { + let path = self.dir.join(name); + let mut partial = path.clone().into_os_string(); + partial.push(PARTIAL); + let partial = PathBuf::from(partial); + let mut file = open_private(&partial).await?; + file.write_all(bytes).await?; + file.sync_all().await?; + drop(file); + #[cfg(test)] + if self.fail_rename.load(std::sync::atomic::Ordering::Acquire) { + return Err(io::Error::other("the process stopped at the rename")); + } + tokio::fs::rename(&partial, &path).await?; + sync_dir(&self.dir).await + } + + /// The bytes of document `name`, `None` when the entry has no such file. + pub(crate) fn read_document(&self, name: &str) -> io::Result>> { + match std::fs::read(self.dir.join(name)) { + Ok(bytes) => Ok(Some(bytes)), + Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(None), + Err(e) => Err(e), + } + } + + /// Start downloading data file `index`: whatever an earlier attempt + /// left (a `.partial`, an unmarked file, a stale marker) is removed, and + /// the `.partial` is created `0600` for the caller to write. + pub(crate) async fn begin_file(&self, index: u32) -> io::Result { + for path in [ + self.verified_path(index), + self.file_path(index), + self.partial_path(index), + ] { + remove_if_present(&path).await?; + } + open_private(&self.partial_path(index)).await + } + + /// Finish data file `index` after its digests matched: sync the + /// `.partial`, rename it into place, then create and sync the + /// `.verified` marker holding `sha256`, and sync the directory, in that + /// order, so the marker exists only beside a complete file. + pub(crate) async fn commit_file( + &self, + index: u32, + mut file: tokio::fs::File, + sha256: &str, + ) -> io::Result<()> { + file.flush().await?; + file.sync_all().await?; + drop(file); + self.rename_into_place(index).await?; + let mut marker = open_private(&self.verified_path(index)).await?; + marker.write_all(sha256.as_bytes()).await?; + marker.write_all(b"\n").await?; + marker.sync_all().await?; + drop(marker); + sync_dir(&self.dir).await + } + + /// The `.partial` renamed to the file's name. + async fn rename_into_place(&self, index: u32) -> io::Result<()> { + #[cfg(test)] + if self.fail_rename.load(std::sync::atomic::Ordering::Acquire) { + return Err(io::Error::other("the process stopped at the rename")); + } + tokio::fs::rename(self.partial_path(index), self.file_path(index)).await + } + + /// Remove the entry and everything in it; an entry already gone is not + /// an error. + pub(crate) async fn remove(&self) -> io::Result<()> { + match tokio::fs::remove_dir_all(&self.dir).await { + Ok(()) => Ok(()), + Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(()), + Err(e) => Err(e), + } + } +} + +/// Create or truncate `path` with mode `0600`. +async fn open_private(path: &Path) -> io::Result { + let mut options = tokio::fs::OpenOptions::new(); + options.write(true).create(true).truncate(true); + #[cfg(unix)] + { + options.mode(0o600); + } + options.open(path).await +} + +async fn remove_if_present(path: &Path) -> io::Result<()> { + match tokio::fs::remove_file(path).await { + Ok(()) => Ok(()), + Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(()), + Err(e) => Err(e), + } +} + +/// Sync a directory so the entries it gained or lost are durable. +async fn sync_dir(dir: &Path) -> io::Result<()> { + tokio::fs::File::open(dir).await?.sync_all().await +} + +/// The table id of an entry's directory name, `None` for any other name in +/// the spool (the `.lock` file, a name an operator put there). +fn entry_table_id(name: &str) -> Option<&str> { + let id = name.strip_prefix(ENTRY_PREFIX)?; + (!id.is_empty() + && id + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'-' || b == b'_')) + .then_some(id) +} + +/// The entries under `spool`, as `(table id, path)`: the reconciler's +/// sweep lists them before it reads the RUNNING records, so an entry a job +/// opens after this listing (its record committed after the listing too) +/// is never a candidate of the sweep that follows. Other names in the spool +/// (the `.lock` file, a name an operator put there, a file under an +/// entry-shaped name) are not listed. +pub(crate) fn list_entries(spool: &Path) -> io::Result> { + let mut entries = Vec::new(); + for dir_entry in std::fs::read_dir(spool)? { + let dir_entry = dir_entry?; + let name = dir_entry.file_name(); + let Some(table_id) = name.to_str().and_then(entry_table_id) else { + continue; + }; + if !dir_entry.file_type()?.is_dir() { + continue; + } + entries.push((table_id.to_owned(), dir_entry.path())); + } + Ok(entries) +} + +/// The reconciler's sweep: remove every entry of `entries` (a +/// [`list_entries`] listing made before the records were read) whose table +/// id is not in `running` (the RUNNING restore targets whose `spool_host` is +/// this host's), and return the ids removed. An entry gone by the time of +/// its removal (its own job removed it between the listing and now) is done, +/// not an error, and not counted; one that cannot be removed is logged and +/// left for the next cycle. +pub(crate) fn sweep_entries( + entries: Vec<(String, PathBuf)>, + running: &HashSet, +) -> Vec { + let mut removed = Vec::new(); + for (table_id, path) in entries { + if running.contains(&table_id) { + continue; + } + match std::fs::remove_dir_all(&path) { + Ok(()) => { + tracing::info!( + "removed the spool entry of the restore into table id {table_id}: no \ + RUNNING restore of this host's spool names it" + ); + removed.push(table_id); + } + Err(e) if e.kind() == io::ErrorKind::NotFound => {} + Err(e) => tracing::error!( + "could not remove the spool entry of table id {table_id}; the next cycle \ + retries: {e}" + ), + } + } + removed +} + +#[cfg(test)] +pub(crate) mod tests { + use super::*; + + /// A scratch directory of this test's own, removed when dropped. + pub(crate) struct Scratch(PathBuf); + + impl Scratch { + pub(crate) fn new() -> Self { + let dir = std::env::temp_dir().join(format!( + "extenddb-engine-test-{}", + uuid::Uuid::new_v4().simple() + )); + std::fs::create_dir(&dir).unwrap(); + Self(dir) + } + + pub(crate) fn path(&self) -> &Path { + &self.0 + } + } + + impl Drop for Scratch { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } + } + + #[cfg(unix)] + fn mode(path: &Path) -> u32 { + use std::os::unix::fs::PermissionsExt; + std::fs::metadata(path).unwrap().permissions().mode() & 0o777 + } + + /// The entry is `0700`, its files `0600`; a committed file has its + /// marker naming the digest, a `.partial` has none, a marker naming + /// another digest does not verify the file, and `begin_file` removes + /// what an earlier attempt left. + #[tokio::test] + async fn an_entry_is_private_and_a_marker_follows_the_rename() { + let spool = Scratch::new(); + let entry = Entry::open(spool.path(), "tid-1", Some("host-a"), "host-a").unwrap(); + assert_eq!(mode(entry.dir()), 0o700); + entry + .write_document("extenddb-manifest.json", b"{}") + .await + .unwrap(); + assert_eq!(mode(&entry.dir().join("extenddb-manifest.json")), 0o600); + assert_eq!( + entry.read_document("extenddb-manifest.json").unwrap(), + Some(b"{}".to_vec()) + ); + assert_eq!(entry.read_document("missing").unwrap(), None); + + let mut file = entry.begin_file(3).await.unwrap(); + file.write_all(b"bytes").await.unwrap(); + assert_eq!(mode(&entry.dir().join("000003.json.gz.partial")), 0o600); + assert!(!entry.is_verified(3, "abc")); + entry.commit_file(3, file, "abc").await.unwrap(); + assert!(entry.is_verified(3, "abc")); + assert!( + !entry.is_verified(3, "abd"), + "the marker binds the digest, not the index" + ); + assert_eq!( + std::fs::read(entry.dir().join("000003.json.gz.verified")).unwrap(), + b"abc\n" + ); + assert_eq!(mode(&entry.file_path(3)), 0o600); + assert_eq!(mode(&entry.dir().join("000003.json.gz.verified")), 0o600); + assert!(!entry.dir().join("000003.json.gz.partial").exists()); + + // A second attempt removes the file and its marker before it writes. + let file = entry.begin_file(3).await.unwrap(); + assert!(!entry.is_verified(3, "abc")); + assert!(!entry.file_path(3).exists()); + drop(file); + // Reopening an existing entry of this host's record keeps it; one + // whose record was another host's is emptied. + let again = Entry::open(spool.path(), "tid-1", Some("host-a"), "host-a").unwrap(); + assert!(again.dir().join("000003.json.gz.partial").exists()); + let moved = Entry::open(spool.path(), "tid-1", Some("host-b"), "host-a").unwrap(); + assert!(moved.dir().is_dir()); + assert_eq!(mode(moved.dir()), 0o700); + assert!(!moved.dir().join("000003.json.gz.partial").exists()); + assert!(!moved.dir().join("extenddb-manifest.json").exists()); + let unknown = Entry::open(spool.path(), "tid-1", None, "host-a"); + assert!( + unknown.is_ok(), + "a record with no spool_host has no entry to keep" + ); + again.remove().await.unwrap(); + assert!(!again.dir().exists()); + again.remove().await.unwrap(); + } + + /// A crash at the rename leaves a `.partial` and no marker: the file is + /// never taken for verified, so the marker is written only once the + /// rename has landed. + #[tokio::test] + async fn the_marker_is_absent_until_the_file_is_renamed() { + let spool = Scratch::new(); + let entry = Entry::open(spool.path(), "tid-2", Some("host-a"), "host-a").unwrap(); + let mut file = entry.begin_file(0).await.unwrap(); + file.write_all(b"bytes").await.unwrap(); + entry + .fail_rename + .store(true, std::sync::atomic::Ordering::Release); + let err = entry.commit_file(0, file, "abc").await.unwrap_err(); + assert!(err.to_string().contains("at the rename"), "{err}"); + assert!(entry.dir().join("000000.json.gz.partial").exists()); + assert!(!entry.file_path(0).exists()); + assert!(!entry.dir().join("000000.json.gz.verified").exists()); + assert!(!entry.is_verified(0, "abc")); + } + + /// The sweep removes the entries of restores not RUNNING on this host + /// and leaves a live entry and every other name; an entry listed and + /// then removed by its own job before the sweep reached it is not an + /// error and not counted. + #[test] + fn the_sweep_removes_only_dead_entries() { + let spool = Scratch::new(); + for name in [ + "restore-live", + "restore-dead-1", + "restore-gone", + "other-dir", + ] { + std::fs::create_dir(spool.path().join(name)).unwrap(); + } + std::fs::write(spool.path().join("restore-dead-1/000000.json.gz"), b"x").unwrap(); + std::fs::write(spool.path().join(".lock"), b"").unwrap(); + // A file, not a directory, under an entry-shaped name is left. + std::fs::write(spool.path().join("restore-file"), b"").unwrap(); + let running: HashSet = ["live".to_owned()].into_iter().collect(); + let mut listed = list_entries(spool.path()).unwrap(); + listed.sort(); + assert_eq!( + listed.iter().map(|(id, _)| id.as_str()).collect::>(), + ["dead-1", "gone", "live"], + "entries only, in whatever order the directory gives" + ); + // Between the listing and the sweep, the job of `gone` removed its + // entry. + std::fs::remove_dir(spool.path().join("restore-gone")).unwrap(); + let mut removed = sweep_entries(listed, &running); + removed.sort(); + assert_eq!(removed, ["dead-1"]); + let mut left: Vec = std::fs::read_dir(spool.path()) + .unwrap() + .map(|e| e.unwrap().file_name().into_string().unwrap()) + .collect(); + left.sort(); + assert_eq!(left, [".lock", "other-dir", "restore-file", "restore-live"]); + assert_eq!(entry_table_id("restore-"), None); + assert_eq!(entry_table_id("restore-a/b"), None); + assert_eq!(entry_table_id("restore-ab_1-2"), Some("ab_1-2")); + } + + /// A document is written beside the one it replaces and renamed over + /// it: a write that stops at the rename leaves the earlier document + /// whole and a `.partial` beside it; a write that completes leaves the + /// new document and no `.partial`. + #[tokio::test] + async fn a_document_replaces_the_earlier_one_at_the_rename() { + let spool = Scratch::new(); + let entry = Entry::open(spool.path(), "tid-3", Some("host-a"), "host-a").unwrap(); + entry + .write_document("extenddb-manifest.json", b"first") + .await + .unwrap(); + entry + .fail_rename + .store(true, std::sync::atomic::Ordering::Release); + let err = entry + .write_document("extenddb-manifest.json", b"second") + .await + .unwrap_err(); + assert!(err.to_string().contains("at the rename"), "{err}"); + assert_eq!( + entry.read_document("extenddb-manifest.json").unwrap(), + Some(b"first".to_vec()), + "the earlier document stands until the rename" + ); + assert_eq!( + std::fs::read(entry.dir().join("extenddb-manifest.json.partial")).unwrap(), + b"second" + ); + assert_eq!( + mode(&entry.dir().join("extenddb-manifest.json.partial")), + 0o600 + ); + entry + .fail_rename + .store(false, std::sync::atomic::Ordering::Release); + entry + .write_document("extenddb-manifest.json", b"third") + .await + .unwrap(); + assert_eq!( + entry.read_document("extenddb-manifest.json").unwrap(), + Some(b"third".to_vec()) + ); + assert!(!entry.dir().join("extenddb-manifest.json.partial").exists()); + } +} diff --git a/crates/engine/src/backup/runtime.rs b/crates/engine/src/backup/runtime.rs new file mode 100644 index 000000000..b765ba2a1 --- /dev/null +++ b/crates/engine/src/backup/runtime.rs @@ -0,0 +1,1607 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The per-process state of the backup operations: the job slots every +//! backup and restore job holds one of, and the store this instance is +//! configured to write, with its health. +//! +//! The store's health is a state the checks set ([`super::store_identity`] +//! runs them; [`check_configured_store`] is the one the reconciler calls each +//! cycle): `Checking` until the first check of a store whose checks run after +//! the listener is up (an S3 store; the server binds and serves first, so an +//! endpoint that stalls never holds the data plane), `Healthy`, or +//! `Unhealthy` with the problem. A transition between the three is logged +//! once; while checking or unhealthy, the operations that touch the store +//! refuse with a `ValidationException` that names the store's kind and not +//! its root ([`BackupRuntime::require_healthy_store`]), `/health` reports the +//! `backup_store` component `checking` or `unhealthy` +//! ([`BackupRuntime::store_health`]), and the reconciler acts on no row of +//! the store. A check that cannot read the catalog changes nothing: the +//! condition is the catalog's, not the store's, and the next cycle reads +//! again. +//! +//! A store call that does not finish within the store's bound is abandoned, +//! and the check with it; the catalog's reads run beside the calls and not +//! under the bound, so a slow catalog never marks the store unhealthy. An +//! S3 store's stalled request ends with the dropped future, before the SDK +//! retries it, and the next cycle sends a fresh one. A filesystem store's +//! stalled call is a blocking thread parked in a filesystem call until the +//! mount answers, which dropping the future cannot end, so a filesystem +//! root's calls run as tasks of their own that the abandoned call does not +//! take with it, and the store remembers that an abandoned call is in flight +//! ([`ConfiguredStore::check_call_in_flight`]): no call starts while one is +//! (the next [`check_configured_store`] records the same `unreachable` the +//! abandoned one did), so a mount that stays stalled parks one thread, not +//! one per cycle. A filesystem root's calls also run one at a time: a call +//! that finds another running (a DeleteBackup beside a check, two +//! DeleteBackups together) waits for it under its own bound, so calls that +//! start before the first reaches its bound park no thread of their own +//! and answer at the bound with the same transport failure. The deletions +//! DeleteBackup and the reconciler run against the store go through the +//! same adapter ([`BoundedCalls`]), under the same bound and rules. The +//! thread costs nothing at exit: the binary ends its runtime with a bounded +//! shutdown that detaches it. + +use std::collections::HashMap; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, RwLock}; +use std::time::Duration; + +use extenddb_backup_store::{BackupStore, ByteStream, ObjectMeta, StoreError}; +use extenddb_core::error::DynamoDbError; +use extenddb_core::limits::LimitsConfig; +use extenddb_core::metrics::MetricsCollector; +use extenddb_core::types::TableDescription; +use extenddb_storage::backup_catalog::{ + JOB_HEARTBEAT_INTERVAL, JobBound, StoreIdentity, StoreKind, limit_exceeded_message, +}; +use extenddb_storage::{BackupCatalog, CancellationToken, JobPermit}; +use futures::StreamExt; +use futures::future::BoxFuture; +use futures::stream::BoxStream; +use tokio::sync::Semaphore; +use tokio_util::task::TaskTracker; + +use super::store_identity::{self, StoreProblem}; + +/// The configuration key of the job bound the runtime enforces, as the +/// refusal names it. +const MAX_CONCURRENT_JOBS_KEY: &str = "[backup] max_concurrent_jobs"; + +/// How long a CreateBackup or RestoreTableFromBackup waits for a job slot +/// before it answers `LimitExceededException`: the bound the PostgreSQL and +/// MongoDB restore slots of earlier releases waited. The service accepts +/// requests over this count (measured on DynamoDB: five restores and six +/// CreateBackup calls sent together were all accepted), so a burst a little +/// over the bound is served in turn rather than refused. +pub const PERMIT_WAIT: Duration = Duration::from_secs(30); + +/// The `[backup]` durations and sizes the store-backed backup job and the +/// reconciler read, as the server resolved them from the configuration. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct JobSettings { + /// `data_file_target_bytes`: a data file closes once its uncompressed + /// bytes reach this. + pub data_file_target_bytes: u64, + /// The job's heartbeat period: `max(heartbeat_interval_secs, + /// JOB_HEARTBEAT_INTERVAL)`, which the server computes; a heartbeat that + /// has not confirmed the claim within one period aborts the job. + pub heartbeat_interval: Duration, + /// `stale_job_after_secs`: the window the maintenance-claim check reads + /// with, and the heartbeat age after which a job's owner is dead. + pub stale_job_after: Duration, + /// `failed_row_retention_secs`: how long a FAILED or DELETED row whose + /// prefix is empty is kept. + pub failed_row_retention: Duration, +} + +impl Default for JobSettings { + /// The configuration's defaults: 64 MiB files, the 15 s heartbeat, + /// 300 s staleness, 7 days of retention. + fn default() -> Self { + Self { + data_file_target_bytes: 64 * 1024 * 1024, + heartbeat_interval: JOB_HEARTBEAT_INTERVAL, + stale_job_after: Duration::from_secs(300), + failed_row_retention: Duration::from_secs(7 * 24 * 3600), + } + } +} + +/// How long a filesystem store's check may run, at startup and in a +/// reconciler cycle, before the store is treated as not answering: a local +/// disk answers in milliseconds, and a network mount that takes longer is +/// one that has stopped answering. +pub const LOCAL_CHECK_TIMEOUT: Duration = Duration::from_secs(5); + +/// How long an S3 store's check may run in a reconciler cycle. The check +/// sends a few small requests (`validate`'s listing, the identity file's +/// `GetObject`, the manifest's `HeadObject`), each answered in well under a +/// second by a store that answers at all; the bound sits under the SDK's +/// per-attempt timeout ([`extenddb_backup_store::S3_ATTEMPT_TIMEOUT`], 60 s) +/// so a timed-out check is dropped before the SDK starts a retry of its +/// stalled request, and no retry outlives the check. A stalled endpoint thus +/// costs a cycle this bound, not the SDK's whole retry budget. +pub const S3_CHECK_TIMEOUT: Duration = Duration::from_secs(30); +const _: () = assert!( + S3_CHECK_TIMEOUT.as_secs() < extenddb_backup_store::S3_ATTEMPT_TIMEOUT.as_secs(), + "the S3 check bound must fire before the SDK's attempt timeout" +); + +/// One per server process, shared by every request through the operation +/// context. Besides the slots and the store it carries what a store-backed +/// backup job needs and no request hands it: the job settings, the metrics +/// collector, the shutdown token the jobs are cancelled on, the tracker the +/// server drains them through ([`BackupRuntime::drain_jobs`]), and the +/// backend and version the manifest names as its source. +pub struct BackupRuntime { + slots: Arc, + max_concurrent_jobs: usize, + /// How long a request waits for a slot ([`PERMIT_WAIT`]; tests shorten + /// it). + permit_wait: Duration, + /// The store this instance writes new backups to and deletes from; + /// `None` while no store is configured, when every catalog row is a row + /// of another store. Shared with the jobs, which read its health and + /// identity before every put. + configured_store: Option>, + /// Whether CreateBackup takes the in-catalog path whatever the store + /// configuration (a backend that cannot pin a table snapshot); such a + /// create is not refused by an unhealthy store. + legacy_create_only: bool, + job_settings: JobSettings, + /// The collector the jobs record into; `None` in commands and tests + /// without one. + metrics: Option>, + /// The token the server's shutdown and a registration loss cancel the + /// jobs on ([`Self::with_shutdown`] makes it a child of the server's + /// token): a running job then runs its failure path with reason + /// `shutdown`. A loss cancels and replaces it ([`Self::cancel_jobs`]), + /// so the jobs started under a new registration run on a fresh token. + shutdown: RwLock, + /// The server's token the jobs' token is a child of. + shutdown_parent: CancellationToken, + /// Every job task spawned through [`BackupRuntime::spawn_job`]. + jobs: TaskTracker, + /// The storage backend's name, the manifest's `source_backend`. + source_backend: String, + /// The build's version, the manifest's `source_extenddb_version`. + source_extenddb_version: String, + /// The instance's limits, which a restore the reconciler resumes + /// validates its items against (a request's restore reads them from + /// the operation context). + limits: Arc, + /// The store-backed restore registrations this instance has spawned and + /// not yet seen return, by (account, target table name): what a retry + /// of the same request meets while the first registration is still in + /// flight ([`Self::begin_restore_registration`]). + restore_registrations: Arc>>, + /// The server's region, which `backup sync` compares an adopted ARN's + /// region with (a backup from another region is adopted and logged); + /// `None` until set, when only the partition is compared. + region: Option, +} + +/// The registration outcome a waiting retry reads: `None` until the +/// registration returns. +pub(crate) type RegistrationOutcome = Option>; + +/// A restore registration this instance has spawned and not yet seen +/// return, as a retry of the request meets it. +#[derive(Clone)] +pub(crate) struct InFlightRegistration { + /// The backup the registration restores from. + pub backup_arn: String, + /// The registration's outcome once it returns. + pub outcome: tokio::sync::watch::Receiver, +} + +/// The registering request's side of an [`InFlightRegistration`]: the +/// registration is in flight while this exists, and +/// [`RegistrationGuard::finish`] publishes its outcome and ends it. Dropped +/// without `finish`, it ends the entry with no outcome (the waiters read the +/// closed channel). +pub(crate) struct RegistrationGuard { + runtime_registrations: Arc>>, + key: (String, String), + outcome: tokio::sync::watch::Sender, +} + +impl RegistrationGuard { + /// Publish the registration's outcome to the retries waiting on it and + /// end the entry. + pub(crate) fn finish(self, outcome: Result) { + let _ = self.outcome.send(Some(outcome)); + self.runtime_registrations + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .remove(&self.key); + } +} + +impl Drop for RegistrationGuard { + fn drop(&mut self) { + self.runtime_registrations + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .remove(&self.key); + } +} + +impl std::fmt::Debug for BackupRuntime { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("BackupRuntime") + .field("max_concurrent_jobs", &self.max_concurrent_jobs) + .field( + "configured_store", + &self.configured_store.as_ref().map(|store| store.identity()), + ) + .field("legacy_create_only", &self.legacy_create_only) + .field("job_settings", &self.job_settings) + .finish_non_exhaustive() + } +} + +/// The store this instance is configured with: its kind and root (the +/// configuration's), its id once the identity check has read it, the handle, +/// the spool for restores from it, the names the DynamoDB-shaped manifests +/// carry for it, and its health. +/// +/// The id is `None` until a check passes: a store whose startup check failed +/// (a transport failure, say) has an unknown id, and a row of its kind and +/// root is still a row of this store for the refusals, so those rows answer +/// "the store is unhealthy" rather than "another store". +pub struct ConfiguredStore { + /// `filesystem` or `s3`. + pub kind: StoreKind, + /// The root label rows record (`store_root`). + pub root: String, + /// The opened store. + pub store: Arc, + /// The restore spool for an S3 store: the directory and the + /// `:` label a restore job record names. + pub spool: Option, + /// `manifest-summary.json`'s `s3Bucket`: the bucket name on S3, the root + /// path on a filesystem store ([`Self::with_manifest_names`]; the root + /// label until set). + pub bucket: String, + /// The key prefix the store was opened with (`[backup.store.s3] prefix`; + /// a filesystem store has none), which every key of the DynamoDB-shaped + /// manifests starts with ([`Self::with_manifest_names`]; empty until + /// set). + pub key_prefix: String, + /// The id the last passing check settled; shared with every + /// [`StoreFence`] of the store, which compares the file's id with it. + store_id: Arc>>, + /// Shared with every [`StoreFence`] of the store, which records the + /// problem it finds here. + health: Arc>, + /// This instance's catalog id, read once by the first fence check that + /// needs it and kept for the process. A catalog re-created under a + /// running process (`destroy` and `init` beside it, which MongoDB and + /// Cassandra allow) gets a new id the cache does not see; the fence then + /// finds a root that names the new catalog foreign and refuses every + /// deletion and CreateBackup until the process restarts, which is the + /// safe direction. + catalog_id: Arc>, + /// Whether the store's `validate` has passed; the checks run it first + /// until it has. + validated: AtomicBool, + /// Whether a call on a filesystem root abandoned at its bound is still + /// running (module doc): set when the call is abandoned, cleared by the + /// call's own task when the mount answers. Never set on an S3 store. + check_call_in_flight: Arc, + /// The one slot a filesystem root's calls run in (module doc): a call + /// waits for it under its bound, held by the call's task until the + /// store answers, so callers that start before the first call reaches + /// its bound park no thread of their own. Unused on an S3 store. + call_slot: Arc, + /// Whether the store is the defaulted SQLite root beside the database + /// file, whose first failed check with no rows for it is a warning that + /// leaves the store unconfigured ([`ConfiguredStore::set_aside`]) rather + /// than an unhealthy configured store. + defaulted: bool, + /// Set once a defaulted root's first check failed with no rows for it: + /// the runtime then answers as a server with no store configured. + set_aside: AtomicBool, + /// Whether this instance's catalog is an in-memory SQLite database: the + /// identity file its checks write says so, and its checks adopt a root + /// another such catalog owned (`store_identity`'s module doc). + ephemeral_catalog: bool, + /// Whether the startup sync of an in-memory catalog is still to run + /// ([`crate::backup::sync::startup_sync`]): set with + /// `ephemeral_catalog`, cleared when a sync over a healthy store has + /// listed the whole root. + startup_sync_pending: AtomicBool, +} + +/// The restore spool of an S3 store. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Spool { + pub path: std::path::PathBuf, + /// `:`, the host whose spool holds a restore's + /// downloaded files. + pub spool_host: String, +} + +/// The store's health as the last check left it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum StoreHealth { + /// No check has finished yet: the store's checks run after the listener + /// is up and the first has not returned. + Checking, + Healthy, + Unhealthy(StoreProblem), +} + +impl StoreHealth { + /// The one or two words the API refusal and `/health` name: the + /// problem's category ([`StoreProblem::category`]), or `check pending`. + #[must_use] + pub fn category(&self) -> Option<&'static str> { + match self { + Self::Healthy => None, + Self::Checking => Some("check pending"), + Self::Unhealthy(problem) => Some(problem.category()), + } + } +} + +/// What `/health` reports for the `backup_store` component. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct StoreHealthReport { + /// `filesystem` or `s3`. + pub kind: &'static str, + /// `healthy`, `unhealthy` or `checking`. + pub status: &'static str, + /// The problem's category and nothing else: `/health` is unauthenticated, + /// and the ids, ARNs and catalog id the operator's log line carries stay + /// out of it. `None` when healthy or checking. + pub reason: Option<&'static str>, +} + +impl ConfiguredStore { + /// A store of `kind` at `root` whose startup check (`validate`, then + /// `establish`) ended in `outcome`: the identity it settled (healthy), or + /// the problem that makes the store unhealthy from the start (a transport + /// failure, or a failed identity check or probe: section 7's rule that a + /// store problem never stops the data plane). A problem leaves the id + /// unknown until a later check passes. Whether `validate` passed is read + /// from the outcome: a store whose startup failed as a transport failure, + /// a denial or a failed validation (the three problems `validate` can end + /// in; [`StoreProblem::from`]) runs `validate` again at its next check, + /// so a probe write that failed is not reported healthy by a later check + /// that only re-read the identity file. + #[must_use] + pub fn new( + kind: StoreKind, + root: String, + store: Arc, + spool: Option, + outcome: Result, + ) -> Self { + let validated = !matches!( + outcome, + Err(StoreProblem::Transport(_) | StoreProblem::Denied(_) | StoreProblem::Validate(_)) + ); + let (store_id, health) = match outcome { + Ok(identity) => (Some(identity.store_id), StoreHealth::Healthy), + Err(problem) => (None, StoreHealth::Unhealthy(problem)), + }; + Self { + kind, + bucket: root.clone(), + key_prefix: String::new(), + root, + store, + spool, + store_id: Arc::new(RwLock::new(store_id)), + health: Arc::new(RwLock::new(health)), + catalog_id: Arc::new(std::sync::OnceLock::new()), + validated: AtomicBool::new(validated), + check_call_in_flight: Arc::new(AtomicBool::new(false)), + call_slot: Arc::new(Semaphore::new(1)), + defaulted: false, + set_aside: AtomicBool::new(false), + ephemeral_catalog: false, + startup_sync_pending: AtomicBool::new(false), + } + } + + /// A store of `kind` at `root` that `open` accepted and whose checks + /// (`validate`, then the identity check) have not run: the first + /// [`check_configured_store`] runs both and records the result. Until + /// then the store is [`StoreHealth::Checking`]. + #[must_use] + pub fn checking( + kind: StoreKind, + root: String, + store: Arc, + spool: Option, + ) -> Self { + Self { + kind, + bucket: root.clone(), + key_prefix: String::new(), + root, + store, + spool, + store_id: Arc::new(RwLock::new(None)), + health: Arc::new(RwLock::new(StoreHealth::Checking)), + catalog_id: Arc::new(std::sync::OnceLock::new()), + validated: AtomicBool::new(false), + check_call_in_flight: Arc::new(AtomicBool::new(false)), + call_slot: Arc::new(Semaphore::new(1)), + defaulted: false, + set_aside: AtomicBool::new(false), + ephemeral_catalog: false, + startup_sync_pending: AtomicBool::new(false), + } + } + + /// The same store marked as the defaulted SQLite root (or not): the + /// first check's failure with no rows for the root then sets it aside + /// ([`ConfiguredStore::set_aside`]), as the startup check's would have. + #[must_use] + pub fn defaulted(mut self, defaulted: bool) -> Self { + self.defaulted = defaulted; + self + } + + /// Whether this is the defaulted SQLite root. + #[must_use] + pub fn is_defaulted(&self) -> bool { + self.defaulted + } + + /// Leave the store unconfigured from now on: the runtime answers as a + /// server with no store (`configured_store` is `None`, `/health` has no + /// component, CreateBackup logs the no-store warning). The reconciler + /// applies this to a defaulted root whose first check failed while the + /// catalog has no rows for it, as the server's startup applies it to a + /// check that failed before the listener. + pub fn set_aside(&self) { + self.set_aside.store(true, Ordering::Release); + } + + /// Whether the store was set aside. + #[must_use] + pub fn is_set_aside(&self) -> bool { + self.set_aside.load(Ordering::Acquire) + } + + /// The same store with the names the DynamoDB-shaped manifests carry + /// for it: `bucket` is `manifest-summary.json`'s `s3Bucket` (the bucket + /// on S3, the root path on a filesystem store) and `key_prefix` the + /// store's configured key prefix (empty when it has none). + #[must_use] + pub fn with_manifest_names(mut self, bucket: String, key_prefix: String) -> Self { + self.bucket = bucket; + self.key_prefix = key_prefix; + self + } + + /// The same store for an instance whose catalog is an in-memory SQLite + /// database (`ephemeral_catalog`): its checks write the marker and + /// adopt a root another in-memory catalog owned, and the startup sync + /// is pending ([`crate::backup::sync::startup_sync`]). + #[must_use] + pub fn with_ephemeral_catalog(mut self, ephemeral_catalog: bool) -> Self { + self.ephemeral_catalog = ephemeral_catalog; + self.startup_sync_pending = AtomicBool::new(ephemeral_catalog); + self + } + + /// Whether this instance's catalog is an in-memory SQLite database. + #[must_use] + pub fn ephemeral_catalog(&self) -> bool { + self.ephemeral_catalog + } + + /// Whether the startup sync is still to run. + #[must_use] + pub fn startup_sync_pending(&self) -> bool { + self.startup_sync_pending.load(Ordering::Acquire) + } + + /// Take the pending startup sync: `true` once, for the caller that runs + /// it, so two callers (the server's start and a reconciler cycle) never + /// run it at once. A caller whose sync did not list the whole root + /// hands it back with [`Self::retry_startup_sync`]. + pub(crate) fn take_startup_sync(&self) -> bool { + self.startup_sync_pending.swap(false, Ordering::AcqRel) + } + + /// Hand a taken startup sync back, for the next reconciler cycle. + pub(crate) fn retry_startup_sync(&self) { + self.startup_sync_pending.store(true, Ordering::Release); + } + + /// The store's identity, once a check has read its id. + #[must_use] + pub fn identity(&self) -> Option { + let id = self + .store_id + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone()?; + Some(StoreIdentity { + store_kind: self.kind, + store_root: self.root.clone(), + store_id: id, + }) + } + + /// Whether a row's store is this one: the kind and the root, and the id + /// when this store's is known. + #[must_use] + pub fn is_store_of(&self, row_store: &StoreIdentity) -> bool { + if row_store.store_kind != self.kind || row_store.store_root != self.root { + return false; + } + match self + .store_id + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .as_deref() + { + Some(id) => id == row_store.store_id, + None => true, + } + } + + /// The health as the last check left it. + #[must_use] + pub fn health(&self) -> StoreHealth { + self.health + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone() + } + + /// Whether the last check passed. + #[must_use] + pub fn is_healthy(&self) -> bool { + matches!(self.health(), StoreHealth::Healthy) + } + + /// Record the outcome of a check: the identity it settled (the id is + /// kept), or the problem. A transition is logged once: to unhealthy at + /// `error` with the problem's detail, to healthy at `info`; a check + /// whose outcome is in the last one's category logs nothing, so a store + /// that stays down costs one line however its error text varies between + /// cycles. A check that could not read the catalog + /// ([`StoreProblem::Catalog`]) leaves the health as it was and logs the + /// read failure: the condition is not the store's. Returns whether this + /// check changed the health's category (and so logged a transition). + pub fn record_check(&self, outcome: Result) -> bool { + let next = match outcome { + Ok(identity) => { + *self + .store_id + .write() + .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(identity.store_id); + StoreHealth::Healthy + } + Err(StoreProblem::Catalog(e)) => { + tracing::warn!( + "backup store ({}) check could not read the catalog ({e}); the store's \ + health is left as it was and the next cycle reads again", + self.kind + ); + return false; + } + Err(problem) => StoreHealth::Unhealthy(problem), + }; + transition(self.kind, &self.root, &self.health, next, || { + self.identity().map(|i| i.store_id).unwrap_or_default() + }) + } + + /// The deletion fence of this store ([`StoreFence`]), sharing its + /// health. + #[must_use] + pub fn fence(&self) -> StoreFence { + StoreFence { + store: Arc::clone(&self.store), + kind: self.kind, + root: self.root.clone(), + health: Arc::clone(&self.health), + store_id: Arc::clone(&self.store_id), + catalog_id: Arc::clone(&self.catalog_id), + in_flight: Arc::clone(&self.check_call_in_flight), + call_slot: Arc::clone(&self.call_slot), + park: self.kind == StoreKind::Filesystem, + bound: self.check_timeout(), + } + } + + /// Whether the store's `validate` has passed. + #[must_use] + pub fn is_validated(&self) -> bool { + self.validated.load(Ordering::Acquire) + } + + /// How long one check of this store may run before it is abandoned + /// ([`LOCAL_CHECK_TIMEOUT`] for a filesystem root, [`S3_CHECK_TIMEOUT`] + /// for a bucket). + #[must_use] + pub fn check_timeout(&self) -> Duration { + match self.kind { + StoreKind::Filesystem => LOCAL_CHECK_TIMEOUT, + StoreKind::S3 => S3_CHECK_TIMEOUT, + } + } + + /// Whether a call abandoned at its bound has not returned (module doc). + #[must_use] + pub fn check_call_in_flight(&self) -> bool { + self.check_call_in_flight.load(Ordering::Acquire) + } + + /// The store as a check, a command's identity read or a deletion calls + /// it: every store call under [`ConfiguredStore::check_timeout`] and the + /// in-flight rule (module doc), and nothing else under the bound, so a + /// slow catalog read beside the calls does not count against it. `what` + /// names the caller in the transport failure a call that does not finish + /// is recorded as ("the check did not finish within 5 s"). + pub(crate) fn calls(&self, what: &'static str) -> BoundedCalls<'_> { + BoundedCalls { + store: &self.store, + in_flight: &self.check_call_in_flight, + slot: &self.call_slot, + bound: self.check_timeout(), + what, + // A filesystem root's calls are blocking threads, which dropping + // the future cannot end; an S3 store's are the SDK's requests, + // which it does end, before the SDK retries them. + park: self.kind == StoreKind::Filesystem, + timed_out: AtomicBool::new(false), + } + } +} + +/// Set `health` to `next`, logging a transition between categories once +/// ([`ConfiguredStore::record_check`]); `store_id` names the id in the +/// first healthy line. Returns whether the category changed. +fn transition( + kind: StoreKind, + root: &str, + health: &RwLock, + next: StoreHealth, + store_id: impl FnOnce() -> String, +) -> bool { + let mut health = health + .write() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if health.category() == next.category() { + *health = next; + return false; + } + match (&*health, &next) { + (StoreHealth::Checking, StoreHealth::Healthy) => tracing::info!( + "backup store ({kind}) {root} passed its check (store id {}); it serves restores \ + and deletes of the backups it holds", + store_id() + ), + (_, StoreHealth::Healthy) => { + tracing::info!("backup store ({kind}) is healthy again"); + } + (_, StoreHealth::Unhealthy(problem)) => tracing::error!( + "backup store ({kind}) {root} is unhealthy: {problem}; restore and delete of a \ + backup in the store are refused until the check passes; the check runs again each \ + reconciler cycle" + ), + (_, StoreHealth::Checking) => {} + } + *health = next; + true +} + +/// The `ValidationException` an operation that touches a store of `kind` +/// answers while its health is not [`StoreHealth::Healthy`]; `None` when +/// it is. +fn health_refusal(kind: StoreKind, health: &StoreHealth) -> Option { + match health { + StoreHealth::Healthy => None, + StoreHealth::Checking => Some(DynamoDbError::ValidationException(format!( + "The {kind} backup store configured on this server has not passed a check yet (its \ + first check is pending); the operation is refused until it does" + ))), + StoreHealth::Unhealthy(problem) => Some(DynamoDbError::ValidationException(format!( + "The {kind} backup store configured on this server is unhealthy ({}); the operation \ + is refused until the store's check passes", + problem.category() + ))), + } +} + +/// The deletion fence of the configured store: a re-read of +/// `extenddb-store.json` right before a call that deletes from the store or +/// acts on its rows because of what the store lists. Its call sites: +/// DeleteBackup's DELETING write and its deletion of the backup's objects; +/// purge's row change (the removal of a dead CREATING row, the DELETING +/// write) and its deletion of the prefix; the reconciler's deletion of a +/// DELETING row's objects, its FAILED-prefix and DELETED-prefix duties and +/// its removal of tombstones; the backup job's failure path; and +/// CreateBackup's first put. `backup sync --prune-missing`'s removal of a +/// row is not fenced: it deletes no object, and a root it does not own +/// leaves it no bytes to lose. +/// +/// A store's health is the last check's, and the check runs once per +/// reconciler cycle; a root another catalog adopted since +/// (`extenddb backup sync --adopt-store`, or a second in-memory server's +/// startup) would otherwise be written and deleted under the new owner's +/// rows until this instance's next check. The fence reads the file (one +/// `get` through the store's bounded calls, [`BoundedCalls`], so on a +/// filesystem root it waits for the root's one call slot and parks no +/// second thread beside an abandoned call) and, when it names another +/// catalog, records [`StoreProblem::ForeignCatalog`] on the store, so every +/// operation that needs a healthy store is refused from then on, and +/// refuses the call. A file that names this catalog under another store id +/// than the one the store's check settled (a root replaced by another root +/// of the same catalog) is recorded and refused the same way, as +/// [`StoreProblem::IdMismatch`]. A file that cannot be read refuses the call without +/// changing the health (the next check decides), and so does a root with no +/// file: every fenced call acts on a row of this catalog for the root, and +/// rows with no file are a problem the check reports. A store that is not +/// healthy refuses at once, with no read. DeleteBackup takes a refusal that +/// recorded nothing, at its read before the DELETING write, as a store +/// failure: it writes DELETING and answers DELETED, and its read before the +/// deletion refuses the deletion (`delete_catalog_backup`). +/// +/// It narrows the window to the gap between one read and the call that +/// follows it; it does not close it. A put of the file that lands in that +/// gap is not seen. On S3 that gap is more than one request: a +/// `delete_prefix` is a listing and then a `DeleteObjects`, each under the +/// SDK's retries, so a throttled listing keeps it open for seconds. A +/// check that read the file before the adoption and finishes after a fence +/// recorded [`StoreProblem::ForeignCatalog`] records its own outcome over +/// it, healthy, until the next cycle's check; each deletion is still +/// fenced in that time, but a restore is accepted. +#[derive(Clone)] +pub struct StoreFence { + store: Arc, + kind: StoreKind, + root: String, + health: Arc>, + store_id: Arc>>, + catalog_id: Arc>, + /// The store's in-flight mark and call slot, which the fence's read + /// shares with the checks and the deletions ([`BoundedCalls`]). + in_flight: Arc, + call_slot: Arc, + park: bool, + bound: Duration, +} + +impl StoreFence { + /// This instance's catalog id: read from `catalog` once, then the + /// store's copy. + /// + /// # Errors + /// + /// [`StoreProblem::Catalog`] when the catalog cannot be read. + pub async fn catalog_id( + &self, + catalog: &C, + ) -> Result { + if let Some(id) = self.catalog_id.get() { + return Ok(id.clone()); + } + let id = catalog + .catalog_id() + .await + .map_err(|e| StoreProblem::Catalog(e.to_string()))?; + Ok(self.catalog_id.get_or_init(|| id).clone()) + } + + /// The fence for `catalog`'s id ([`Self::check`]). + /// + /// # Errors + /// + /// As [`Self::check`], or [`StoreProblem::Catalog`] when the id cannot + /// be read. + pub async fn check_catalog( + &self, + catalog: &C, + ) -> Result<(), StoreProblem> { + let catalog_id = self.catalog_id(catalog).await?; + self.check(&catalog_id).await + } + + /// The fence (type doc) for the catalog whose id is `catalog_id`. + /// + /// # Errors + /// + /// The store's problem when it is not healthy; + /// [`StoreProblem::ForeignCatalog`], recorded on the store, when the + /// file names another catalog, and [`StoreProblem::IdMismatch`], + /// recorded, when it names this one under another store id; the read's + /// problem when the file cannot be read within the bound, and + /// [`StoreProblem::FileUnreadable`] when the root has none (neither + /// recorded). + pub async fn check(&self, catalog_id: &str) -> Result<(), StoreProblem> { + match &*self + .health + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner) + { + StoreHealth::Healthy => {} + StoreHealth::Checking => { + return Err(StoreProblem::Transport( + "the store's first check is pending".to_owned(), + )); + } + StoreHealth::Unhealthy(problem) => return Err(problem.clone()), + } + let via = BoundedCalls { + store: &self.store, + in_flight: &self.in_flight, + slot: &self.call_slot, + bound: self.bound, + what: "identity file read", + park: self.park, + timed_out: AtomicBool::new(false), + }; + let read = store_identity::read_identity_file(&via).await; + if via.timed_out() { + return Err(via.did_not_finish_problem()); + } + let file = read?; + match file { + Some(file) if file.catalog_id != catalog_id => { + let problem = StoreProblem::ForeignCatalog { + file_catalog_id: file.catalog_id, + catalog_id: catalog_id.to_owned(), + }; + transition( + self.kind, + &self.root, + &self.health, + StoreHealth::Unhealthy(problem.clone()), + String::new, + ); + Err(problem) + } + Some(file) => { + let known = self + .store_id + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone(); + match known { + Some(known) if file.store_id != known => { + let problem = StoreProblem::IdMismatch { + file_id: file.store_id, + row_ids: vec![known], + }; + transition( + self.kind, + &self.root, + &self.health, + StoreHealth::Unhealthy(problem.clone()), + String::new, + ); + Err(problem) + } + _ => Ok(()), + } + } + None => Err(StoreProblem::FileUnreadable(format!( + "the root has no {}; the next check reports it", + store_identity::STORE_IDENTITY_KEY + ))), + } + } + + /// The refusal an operation answers once the fence refused it: the + /// store's health as the fence left it, as + /// [`BackupRuntime::require_healthy_store`] answers it; `None` while the + /// store is still healthy (the file could not be read, which the next + /// check decides), when the caller answers as for a failed store call. + pub(crate) fn refusal(&self) -> Option { + let health = self + .health + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone(); + health_refusal(self.kind, &health) + } +} + +/// The check of the configured store a reconciler cycle, and a command that +/// needs the store's identity, run and record through +/// [`ConfiguredStore::record_check`]: `validate` first while it has not +/// passed (its failure is the problem [`StoreProblem::from`] makes of the +/// store's error, a transport failure [`StoreProblem::Transport`] with the +/// cause chain, a denial [`StoreProblem::Denied`], anything else +/// [`StoreProblem::Validate`]; each is retried next cycle), then the +/// identity check, which is [`store_identity::recheck`] for a store whose +/// id is known and [`store_identity::establish`] for one whose checks have +/// not passed yet (it settles the id when it passes). `backend` names the +/// storage backend in an identity file the check has to write. +/// +/// The store's calls run under [`ConfiguredStore::check_timeout`] +/// ([`BoundedCalls`]): a store that does not answer a call within it (a +/// stalled endpoint, a mount that has stopped answering) ends the check as +/// [`StoreProblem::Transport`] naming the bound, and the caller's cycle goes +/// on. The catalog's reads run beside the calls and not under the bound, so +/// a slow catalog costs the check time and never marks the store unhealthy. +/// An S3 store's abandoned request is dropped with the call's future. A +/// filesystem root's calls run as tasks of their own, which the abandoned +/// call does not take with it: the call's synchronous work runs on the +/// blocking pool, where the thread stays parked until the mount answers, +/// and the task is what clears the store's in-flight mark when it does. +/// While that mark is set no call starts: the store is recorded +/// `unreachable` again, with the reason, and nothing new is parked. +/// +/// # Errors +/// +/// The problem that makes the store unhealthy +/// ([`store_identity`]'s module doc). +pub async fn check_configured_store( + store: &ConfiguredStore, + catalog: &C, + backend: &str, +) -> Result { + if store.check_call_in_flight() { + tracing::info!( + "backup store ({}) check was not started: a call of the check abandoned at an \ + earlier cycle has not returned (the store is still not answering); the next cycle \ + checks again", + store.kind + ); + return Err(StoreProblem::Transport( + "a call of an earlier check has not returned".to_owned(), + )); + } + let via = store.calls("check"); + let outcome = check_through(store, &via, catalog, backend).await; + if via.timed_out() { + tracing::info!( + "backup store ({}) check did not finish within {} s and was abandoned; the \ + cycle's other duties ran, and the next cycle checks again", + store.kind, + via.bound.as_secs() + ); + return Err(via.did_not_finish_problem()); + } + outcome +} + +/// [`check_configured_store`]'s steps, calling the store through `via`. +async fn check_through( + store: &ConfiguredStore, + via: &dyn BackupStore, + catalog: &C, + backend: &str, +) -> Result { + if !store.is_validated() { + match via.validate().await { + Ok(()) => store.validated.store(true, Ordering::Release), + Err(e) => return Err(StoreProblem::from(e)), + } + } + match store.identity() { + Some(identity) => { + store_identity::recheck(via, &identity, catalog, backend, store.ephemeral_catalog) + .await + .map(|()| identity) + } + None => { + store_identity::establish( + via, + store.kind, + &store.root, + catalog, + backend, + store.ephemeral_catalog, + ) + .await + } + } +} + +/// A command's read of the store's identity ([`store_identity::read_identity`], +/// which writes nothing), with the store's calls under +/// [`ConfiguredStore::check_timeout`] as the server's check runs them: a call +/// that does not finish within the bound is [`StoreProblem::Transport`] +/// naming it, where the store's own retries (the SDK's three attempts of +/// 60 s on an endpoint that accepts and never answers) or a stalled mount +/// would hold the command for minutes or for good. +/// +/// # Errors +/// +/// The problem that would make the store unhealthy +/// ([`store_identity`]'s module doc), or the bound. +pub async fn read_identity_bounded( + store: &ConfiguredStore, + catalog: &C, +) -> Result, StoreProblem> { + let via = store.calls("identity read"); + let read = store_identity::read_identity(&via, store.kind, &store.root, catalog).await; + if via.timed_out() { + return Err(via.did_not_finish_problem()); + } + read +} + +/// The configured store as the checks, a command's identity read, the +/// deletion fence and the deletions call it (module doc): each call runs under `bound`, and a call +/// that does not finish is a transport failure naming `what` and the bound. +/// On a filesystem root (`park`) each call runs as a task of its own, owning +/// the store handle, so a call abandoned at its bound leaves the task +/// running until the mount answers, marked in flight until then; a call that +/// finds the mark set (a call abandoned earlier has not returned) is refused +/// at once as a transport failure, so no second thread is parked beside the +/// first. The root's calls run one at a time (`slot`): a call that finds +/// another running waits for it under the bound and, once it holds the slot, +/// has the bound for itself, so two callers whose calls start before the +/// first reaches its bound park one thread between them, the second +/// answering at its wait's bound with nothing parked when the first never +/// returns, and finishing when the first does. On an S3 +/// store the call is the SDK's request, which the dropped future ends, so +/// nothing is marked and nothing waits. The check calls `validate`, `get`, +/// `head` and `put_if_absent`, and `put` when an in-memory catalog adopts +/// the root (`store_identity::adopt`); the fence calls `get`; the deletions +/// call `delete_prefix` and `list`. +pub(crate) struct BoundedCalls<'a> { + store: &'a Arc, + in_flight: &'a Arc, + slot: &'a Arc, + bound: std::time::Duration, + what: &'static str, + park: bool, + timed_out: AtomicBool, +} + +/// The call task's end, by any exit: clears the in-flight mark under the +/// call's own lock, so a caller that abandons the call at its bound marks it +/// only while the task still runs (a task that ended between the timeout's +/// last poll and the caller's mark would otherwise leave the mark set with +/// nothing left to clear it, and every later check refused until a restart). +struct ClearOnDrop { + in_flight: Arc, + running: Arc>, +} + +impl Drop for ClearOnDrop { + fn drop(&mut self) { + let mut running = self + .running + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + *running = false; + self.in_flight.store(false, Ordering::Release); + } +} + +impl BoundedCalls<'_> { + /// Whether a call of this adapter was abandoned at the bound. + pub(crate) fn timed_out(&self) -> bool { + self.timed_out.load(Ordering::Acquire) + } + + /// The text of a call that did not finish within the bound. + fn did_not_finish_text(&self) -> String { + format!( + "the {} did not finish within {} s", + self.what, + self.bound.as_secs() + ) + } + + /// The transport failure a call that did not finish within the bound is. + fn did_not_finish(&self) -> StoreError { + StoreError::Transport { + source: self.did_not_finish_text().into(), + } + } + + /// The problem the check or read is recorded as once a call was + /// abandoned: the transport failure naming the bound and nothing else + /// (the store error's own prefix left out, so the operator's line and + /// the recorded reason read as the bound). + pub(crate) fn did_not_finish_problem(&self) -> StoreProblem { + StoreProblem::Transport(self.did_not_finish_text()) + } + + /// Run `call` on the store under the bound. + fn run(&self, call: F) -> BoxFuture<'_, Result> + where + T: Send + 'static, + F: FnOnce(Arc) -> BoxFuture<'static, Result> + + Send + + 'static, + { + if self.in_flight.load(Ordering::Acquire) { + return Box::pin(std::future::ready(Err(StoreError::Transport { + source: "a call of an earlier check has not returned".into(), + }))); + } + let store = Arc::clone(self.store); + let bound = self.bound; + if !self.park { + return Box::pin(async move { + match tokio::time::timeout(bound, call(store)).await { + Ok(outcome) => outcome, + Err(_) => { + self.timed_out.store(true, Ordering::Release); + Err(self.did_not_finish()) + } + } + }); + } + // The task owns the slot and the mark's clearing, so a call + // abandoned below is marked, and holds the slot, until the mount + // answers it. A caller that finds the slot taken waits for it under + // the bound, and the call's own bound starts once it holds the slot + // (a 3 s call behind a 3 s call finishes, where one bound over both + // would abandon it); a wait that ends at the bound marks nothing, + // since the call holding the slot marks itself when it is abandoned. + let slot = Arc::clone(self.slot); + let in_flight = Arc::clone(self.in_flight); + Box::pin(async move { + let Ok(permit) = tokio::time::timeout(bound, slot.acquire_owned()).await else { + self.timed_out.store(true, Ordering::Release); + return Err(self.did_not_finish()); + }; + let permit = permit.expect("the store's call slot is never closed"); + let running = Arc::new(std::sync::Mutex::new(true)); + let clear = ClearOnDrop { + in_flight, + running: Arc::clone(&running), + }; + let task = tokio::spawn(async move { + let _permit = permit; + let _clear = clear; + call(store).await + }); + match tokio::time::timeout(bound, task).await { + Ok(joined) => { + joined.map_err(|e| StoreError::Other(format!("store call task failed: {e}")))? + } + Err(_) => { + // Under the task's lock: a task that has already ended + // (its guard cleared the mark) is not marked again. + let running = running + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if *running { + self.in_flight.store(true, Ordering::Release); + } + drop(running); + self.timed_out.store(true, Ordering::Release); + Err(self.did_not_finish()) + } + } + }) + } +} + +impl BackupStore for BoundedCalls<'_> { + fn put<'a>( + &'a self, + key: &'a str, + body: ByteStream, + ) -> BoxFuture<'a, Result> { + let key = key.to_owned(); + self.run(move |store| Box::pin(async move { store.put(&key, body).await })) + } + + fn put_if_absent<'a>( + &'a self, + key: &'a str, + body: ByteStream, + ) -> BoxFuture<'a, Result, StoreError>> { + let key = key.to_owned(); + self.run(move |store| Box::pin(async move { store.put_if_absent(&key, body).await })) + } + + fn get<'a>(&'a self, key: &'a str) -> BoxFuture<'a, Result> { + let key = key.to_owned(); + self.run(move |store| Box::pin(async move { store.get(&key).await })) + } + + fn head<'a>(&'a self, key: &'a str) -> BoxFuture<'a, Result, StoreError>> { + let key = key.to_owned(); + self.run(move |store| Box::pin(async move { store.head(&key).await })) + } + + fn list<'a>(&'a self, prefix: &'a str) -> BoxStream<'a, Result> { + let prefix = prefix.to_owned(); + let listed = self.run(move |store| { + Box::pin(async move { + use futures::TryStreamExt as _; + store.list(&prefix).try_collect::>().await + }) + }); + Box::pin( + futures::stream::once(listed).flat_map(|listed| match listed { + Ok(metas) => futures::stream::iter(metas.into_iter().map(Ok)).boxed(), + Err(e) => futures::stream::once(async move { Err(e) }).boxed(), + }), + ) + } + + fn delete_prefix<'a>(&'a self, prefix: &'a str) -> BoxFuture<'a, Result> { + let prefix = prefix.to_owned(); + self.run(move |store| Box::pin(async move { store.delete_prefix(&prefix).await })) + } + + fn validate(&self) -> BoxFuture<'_, Result<(), StoreError>> { + self.run(|store| Box::pin(async move { store.validate().await })) + } +} + +impl BackupRuntime { + /// `max_concurrent_jobs` slots, no configured store, the default job + /// settings, no metrics, a shutdown token nothing cancels, and `test` as + /// the source backend with this crate's version. + #[must_use] + pub fn new(max_concurrent_jobs: usize) -> Self { + Self { + slots: Arc::new(Semaphore::new(max_concurrent_jobs)), + max_concurrent_jobs, + permit_wait: PERMIT_WAIT, + configured_store: None, + legacy_create_only: false, + job_settings: JobSettings::default(), + metrics: None, + shutdown: RwLock::new(CancellationToken::new()), + shutdown_parent: CancellationToken::new(), + jobs: TaskTracker::new(), + source_backend: "test".to_owned(), + source_extenddb_version: env!("CARGO_PKG_VERSION").to_owned(), + limits: Arc::new(LimitsConfig::default()), + restore_registrations: Arc::default(), + region: None, + } + } + + /// Mark a store-backed restore registration of `backup_arn` into + /// (`account_id`, `table_name`) as in flight, or return the one already + /// in flight for that name (a retry of the request, or another backup's + /// registration into the same name). The registering request's task + /// ends the entry through [`RegistrationGuard::finish`]. + pub(crate) fn begin_restore_registration( + &self, + account_id: &str, + table_name: &str, + backup_arn: &str, + ) -> Result { + let key = (account_id.to_owned(), table_name.to_owned()); + let mut registrations = self + .restore_registrations + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if let Some(in_flight) = registrations.get(&key) { + return Err(in_flight.clone()); + } + let (outcome, receiver) = tokio::sync::watch::channel(None); + registrations.insert( + key.clone(), + InFlightRegistration { + backup_arn: backup_arn.to_owned(), + outcome: receiver, + }, + ); + Ok(RegistrationGuard { + runtime_registrations: Arc::clone(&self.restore_registrations), + key, + outcome, + }) + } + + /// The same runtime with the instance's limits, which a resumed restore + /// validates its items against (the default is `LimitsConfig::default`). + #[must_use] + pub fn with_limits(mut self, limits: Arc) -> Self { + self.limits = limits; + self + } + + /// The instance's limits. + #[must_use] + pub fn limits(&self) -> &Arc { + &self.limits + } + + /// The same runtime with `[backup] stale_job_after_secs` set, the window + /// the jobs' maintenance-claim check reads with (the default is the + /// configuration's default, 300 s). + #[must_use] + pub fn with_stale_job_after(mut self, stale_job_after: Duration) -> Self { + self.job_settings.stale_job_after = stale_job_after; + self + } + + /// `[backup] stale_job_after_secs`. + #[must_use] + pub fn stale_job_after(&self) -> Duration { + self.job_settings.stale_job_after + } + + /// The same runtime with every job setting set. + #[must_use] + pub fn with_job_settings(mut self, settings: JobSettings) -> Self { + self.job_settings = settings; + self + } + + /// The settings the jobs and the reconciler read. + #[must_use] + pub fn job_settings(&self) -> JobSettings { + self.job_settings + } + + /// The same runtime with the collector the jobs record into. + #[must_use] + pub fn with_metrics(mut self, metrics: Arc) -> Self { + self.metrics = Some(metrics); + self + } + + /// The collector the jobs record into, if any. + #[must_use] + pub fn metrics(&self) -> Option<&Arc> { + self.metrics.as_ref() + } + + /// The same runtime with the server's shutdown token: the jobs run on a + /// child of it, which [`Self::drain_jobs`] and [`Self::cancel_jobs`] + /// cancel. + #[must_use] + pub fn with_shutdown(mut self, shutdown: CancellationToken) -> Self { + self.shutdown = RwLock::new(shutdown.child_token()); + self.shutdown_parent = shutdown; + self + } + + /// The token the jobs started now are cancelled on. + #[must_use] + pub fn shutdown(&self) -> CancellationToken { + self.shutdown + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone() + } + + /// Cancel the jobs running now (each runs its failure path with reason + /// `shutdown`: FAILED, then its prefix deleted) and give the jobs that + /// start afterwards a fresh token. The server calls this when its + /// serving registration is lost: a migration can take the catalog once + /// the registration is gone, and a job's heartbeats and status writes + /// would otherwise go on beside it. Returns how many jobs were running. + pub fn cancel_jobs(&self) -> usize { + let fresh = self.shutdown_parent.child_token(); + let old = std::mem::replace( + &mut *self + .shutdown + .write() + .unwrap_or_else(std::sync::PoisonError::into_inner), + fresh, + ); + old.cancel(); + self.jobs.len() + } + + /// The same runtime with the backend name and build version the + /// manifests name as their source. + #[must_use] + pub fn with_source(mut self, backend: &str, version: &str) -> Self { + self.source_backend = backend.to_owned(); + self.source_extenddb_version = version.to_owned(); + self + } + + /// The manifest's `source_backend` and `source_extenddb_version`. + #[must_use] + pub fn source(&self) -> (&str, &str) { + (&self.source_backend, &self.source_extenddb_version) + } + + /// The same runtime with the server's region. + #[must_use] + pub fn with_region(mut self, region: &str) -> Self { + self.region = Some(region.to_owned()); + self + } + + /// The server's region, once set. + #[must_use] + pub fn region(&self) -> Option<&str> { + self.region.as_deref() + } + + /// Spawn a job task under the tracker [`Self::drain_jobs`] waits on. + /// Public for the server's test of its shutdown sequence, which spawns a + /// task that outlives the drain; the engine's own jobs come through + /// CreateBackup. + pub fn spawn_job(&self, job: F) -> tokio::task::JoinHandle + where + F: std::future::Future + Send + 'static, + F::Output: Send + 'static, + { + self.jobs.spawn(job) + } + + /// Jobs spawned and not yet ended. + #[must_use] + pub fn running_jobs(&self) -> usize { + self.jobs.len() + } + + /// Cancel the jobs' shutdown token and wait up to `bound` for every job + /// to end: a cancelled job runs its failure path (the FAILED write and + /// the deletion of its prefix), which is what sweeps an upload still in + /// flight on S3, where the runtime's own shutdown sends no abort. Job + /// tasks still running at the bound are logged by their count (the + /// tracker counts tasks, from the request's spawn to the task's end, + /// whether or not a row exists for one yet) and left to the runtime's + /// shutdown; a row such a task leaves is CREATING (a task still in its + /// pipeline) or FAILED (one still in its `delete_prefix`), and the + /// reconciler resolves or sweeps it once its heartbeat is stale. Returns + /// the number still running, for the caller's information: the server + /// reads the warn line and nothing else. + pub async fn drain_jobs(&self, bound: Duration) -> usize { + self.shutdown().cancel(); + self.jobs.close(); + let running = self.jobs.len(); + if running == 0 { + return 0; + } + tracing::info!( + "waiting up to {} s for {running} backup or restore job(s) to end", + bound.as_secs() + ); + if tokio::time::timeout(bound, self.jobs.wait()).await.is_err() { + let left = self.jobs.len(); + tracing::warn!( + "{left} job task(s) did not end within {} s (tasks spawned by CreateBackup and \ + RestoreTableFromBackup, counted whether or not a row exists for them yet); a \ + backup job's row left CREATING or FAILED is resolved or swept by the \ + reconciler once its heartbeat is stale, a restore job's record is RUNNING \ + with its progress and the reconciler resumes it", + bound.as_secs() + ); + return left; + } + 0 + } + + /// The same runtime with `store` configured. + #[must_use] + pub fn with_store(mut self, store: ConfiguredStore) -> Self { + self.configured_store = Some(Arc::new(store)); + self + } + + /// The same runtime waiting `wait` for a slot in place of + /// [`PERMIT_WAIT`]. + #[must_use] + pub fn with_permit_wait(mut self, wait: Duration) -> Self { + self.permit_wait = wait; + self + } + + /// The same runtime with CreateBackup bound to the in-catalog path + /// whatever the store configuration. + #[must_use] + pub fn with_legacy_create_only(mut self, legacy_create_only: bool) -> Self { + self.legacy_create_only = legacy_create_only; + self + } + + /// The slots jobs take permits from. + #[must_use] + pub fn slots(&self) -> &Arc { + &self.slots + } + + /// Permits no job holds right now. + #[must_use] + pub fn free_slots(&self) -> usize { + self.slots.available_permits() + } + + /// The configured store, if any; `None` too for a defaulted root set + /// aside after its first check ([`ConfiguredStore::set_aside`]). + #[must_use] + pub fn configured_store(&self) -> Option<&ConfiguredStore> { + self.configured_store + .as_deref() + .filter(|store| !store.is_set_aside()) + } + + /// [`Self::configured_store`] as the `Arc` a job's environment holds + /// ([`super::job::JobEnv::new`]). + #[must_use] + pub(crate) fn configured_store_arc(&self) -> Option<&Arc> { + self.configured_store + .as_ref() + .filter(|store| !store.is_set_aside()) + } + + /// The configured store, taken out of a runtime a command built only to + /// reach it. + #[must_use] + pub fn into_configured_store(self) -> Option> { + self.configured_store + } + + /// Whether CreateBackup takes the in-catalog path whatever the store + /// configuration. + #[must_use] + pub fn legacy_create_only(&self) -> bool { + self.legacy_create_only + } + + /// What `/health` reports for the `backup_store` component; `None` when + /// no store is configured. + #[must_use] + pub fn store_health(&self) -> Option { + let store = self.configured_store()?; + let kind = store.kind.as_str(); + Some(match store.health() { + StoreHealth::Healthy => StoreHealthReport { + kind, + status: "healthy", + reason: None, + }, + StoreHealth::Checking => StoreHealthReport { + kind, + status: "checking", + reason: None, + }, + StoreHealth::Unhealthy(problem) => StoreHealthReport { + kind, + status: "unhealthy", + reason: Some(problem.category()), + }, + }) + } + + /// Take a job slot, waiting up to `permit_wait` for one, or + /// `LimitExceededException` when none was freed within the bound. The + /// request holds nothing while it waits: the permit is the last of its + /// checks, before the backend opens a snapshot or a transaction. The text + /// is the one every refusal over a job bound answers + /// ([`limit_exceeded_message`]), naming this one. + pub(crate) async fn take_permit(&self) -> Result { + JobPermit::acquire_within(&self.slots, self.permit_wait) + .await + .ok_or_else(|| { + DynamoDbError::LimitExceededException(limit_exceeded_message(JobBound::Counted { + name: MAX_CONCURRENT_JOBS_KEY, + limit: self.max_concurrent_jobs, + })) + }) + } + + /// Whether `store` is the one this instance is configured with. With no + /// store configured, no row's store is. + pub(crate) fn is_configured_store(&self, store: &StoreIdentity) -> bool { + self.configured_store() + .is_some_and(|configured| configured.is_store_of(store)) + } + + /// The configured store, when its last check passed; otherwise (no + /// check has finished, or the last one failed) the `ValidationException` + /// the operations that touch the store answer, naming the store's kind + /// and the problem's category and not its root (no API error names a + /// root). Every caller has checked that a store is configured + /// (`is_configured_store`, or the store-backed create's own condition). + pub(crate) fn require_healthy_store(&self) -> Result<&Arc, DynamoDbError> { + let store = self + .configured_store + .as_ref() + .filter(|store| !store.is_set_aside()) + .expect("every caller checks that a store is configured first"); + match health_refusal(store.kind, &store.health()) { + None => Ok(store), + Some(refusal) => Err(refusal), + } + } +} diff --git a/crates/engine/src/backup/store_identity.rs b/crates/engine/src/backup/store_identity.rs new file mode 100644 index 000000000..889b443ac --- /dev/null +++ b/crates/engine/src/backup/store_identity.rs @@ -0,0 +1,1036 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The store identity file and the checks that decide a configured store's +//! health. +//! +//! A store's root carries `extenddb-store.json` ([`STORE_IDENTITY_KEY`]), +//! holding a random store id and the `catalog_id` of the catalog that owns +//! the root; every backup row records the store id in `store_id`. Server +//! startup ([`establish`]) and every reconciler cycle ([`recheck`]) read the +//! file and check it against the catalog. The store is unhealthy when: +//! +//! - the file names another catalog (a root another deployment owns, or a +//! copy of one; an operator adopts it on purpose, never a server by +//! accident); +//! - its store id differs from the `store_id` of the catalog's rows for the +//! configured kind and root (the root was replaced under the rows); +//! - the catalog has rows for the root and the root has no file (the root +//! was emptied or re-pointed); +//! - those rows carry more than one `store_id`; +//! - the manifest probe fails: the newest AVAILABLE row's +//! `extenddb-manifest.json` is not under the root. Instances that share a +//! catalog must share one store root; host-local roots at one path give +//! each instance different bytes under one `store_root`, which the ids +//! alone cannot see, so the probe reads the root independently of the +//! host; +//! - the store cannot be reached or read, or the file cannot be parsed. +//! +//! On a root with no file and a catalog with no rows for it, the server +//! creates the file with `put_if_absent`, then re-reads it and uses the id +//! it holds. Of two instances of one catalog starting against one empty root, +//! one creates the file and the other's create finds it present, and both +//! use the winner's id. An S3-compatible server that ignores `If-None-Match` +//! makes the create a plain put, last writer wins: the re-read catches a race +//! lost before it (the file holds another id than this instance wrote) and +//! marks the store unhealthy naming both ids; a race lost after it is caught +//! by the next check, which finds rows carrying an id the file does not. +//! +//! An unhealthy store is not a startup refusal: the caller logs the problem, +//! reports it on `/health`, refuses the operations that touch the store, and +//! runs the check again each cycle until it passes. +//! +//! A root whose file names another catalog changes owner only by +//! [`adopt`]: `extenddb backup sync --adopt-store`, under the maintenance +//! claim, for a root any other catalog owns; and a server whose catalog is +//! an in-memory SQLite database, for a root whose file another in-memory +//! catalog wrote. Such a catalog, and with it its `catalog_id`, is new at +//! every start, so the file it writes carries `ephemeral_catalog: true` +//! ([`StoreIdentityFile::ephemeral_catalog`]), and [`establish`] in such a +//! server adopts a root whose file carries the marker (the previous start's +//! catalog, or another in-memory server's), and only such a root: one a +//! durable catalog owns stays unhealthy there as anywhere. [`recheck`], +//! which runs once the identity is known, never adopts, so two in-memory +//! servers pointed at one root do not take it back and forth: the later +//! start owns it and the earlier finds it owned by another catalog. Until +//! that check, the previous owner acts on the root as its own, except that +//! its deletion fence (`super::StoreFence`) re-reads the file before each +//! deletion from the store and before a CreateBackup's first put, refuses +//! when the file names another catalog, and marks the store unhealthy; a +//! write already past the fence's read still lands. So every server of the +//! catalog a file names is to be stopped before `--adopt-store` takes its +//! root, and a new in-memory server started only after the old one has +//! exited. The +//! rewrite is a plain `put` (the store trait has no conditional replace), +//! so two adopters race and the last writer wins. Each re-reads the file +//! after its put, which detects another adopter's put that landed between +//! the two, and `backup sync --adopt-store` reads the file once more after +//! its sync, which detects one that landed during the sync; a put that +//! lands after both reads is found by the loser's next check, which finds +//! the store owned by another catalog. The audit line is written after the +//! re-read with what it found, and an intent line before the put, so a put +//! whose caller stopped waiting for it is on record too. + +use extenddb_backup_store::{BackupStore, StoreError, byte_stream_from}; +use extenddb_core::backup_format::{BackupArn, EXTENDDB_MANIFEST_FILE}; +use extenddb_storage::BackupCatalog; +use extenddb_storage::backup_catalog::{StoreIdentity, StoreKind}; +use futures::TryStreamExt; +use serde::{Deserialize, Serialize}; + +/// The key of the identity file, at the root, under no `/` prefix. +pub const STORE_IDENTITY_KEY: &str = "extenddb-store.json"; + +/// The one version of the file's shape this build writes and reads. +pub const STORE_IDENTITY_FORMAT_VERSION: u32 = 1; + +/// The most bytes a check reads of `extenddb-store.json`. The file this +/// build writes is under 300 bytes; an object larger than this under the +/// key is not an identity file, and is reported as unreadable without being +/// read into memory on every cycle. +pub const STORE_IDENTITY_MAX_BYTES: usize = 64 * 1024; + +/// The contents of `extenddb-store.json`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct StoreIdentityFile { + pub format_version: u32, + /// The random id every row of this store carries. + pub store_id: String, + /// The catalog that owns the root. + pub catalog_id: String, + /// The storage backend of the owning catalog, for the operator reading + /// the file; nothing is checked against it. + pub backend: String, + /// When the file was created, epoch milliseconds on the creating host. + pub created_at_ms: u64, + /// Whether the owning catalog is an in-memory SQLite database, new at + /// every start of its server (module doc). Absent in a file an earlier + /// build wrote, which no in-memory catalog of that build could adopt + /// again, so it reads as `false`. + #[serde(default)] + pub ephemeral_catalog: bool, +} + +impl StoreIdentityFile { + /// A new identity for `catalog_id`. + fn new(catalog_id: &str, backend: &str, ephemeral_catalog: bool) -> Self { + Self { + format_version: STORE_IDENTITY_FORMAT_VERSION, + store_id: uuid::Uuid::new_v4().to_string(), + catalog_id: catalog_id.to_owned(), + backend: backend.to_owned(), + created_at_ms: std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |d| u64::try_from(d.as_millis()).unwrap_or(u64::MAX)), + ephemeral_catalog, + } + } + + /// Parse the file's bytes. + /// + /// # Errors + /// + /// [`StoreProblem::FileUnreadable`] for bytes that are not the documented + /// shape, or carry a version this build does not read. + pub fn parse(bytes: &[u8]) -> Result { + let file: Self = serde_json::from_slice(bytes) + .map_err(|e| StoreProblem::FileUnreadable(format!("not a store identity file: {e}")))?; + if file.format_version != STORE_IDENTITY_FORMAT_VERSION { + return Err(StoreProblem::FileUnreadable(format!( + "format_version {} is not {STORE_IDENTITY_FORMAT_VERSION}", + file.format_version + ))); + } + if file.store_id.is_empty() || file.catalog_id.is_empty() { + return Err(StoreProblem::FileUnreadable( + "store_id and catalog_id must be non-empty".to_owned(), + )); + } + Ok(file) + } + + /// The file's bytes. + #[must_use] + pub fn to_bytes(&self) -> Vec { + let mut bytes = serde_json::to_vec_pretty(self).expect("a plain struct serializes"); + bytes.push(b'\n'); + bytes + } +} + +/// Why a store is unhealthy. `Display` is the operator's line (ids, the +/// catalog id, the store's own error text, never a credential); +/// [`StoreProblem::public_reason`] is the part of it that names no root, and +/// [`StoreProblem::category`] alone is what `/health` reports. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum StoreProblem { + /// The store could not be reached, read or written; the store's error. + Transport(String), + /// The store refused an operation: a root or a directory under it that + /// another user can reach, a denied grant; the store's error. + Denied(String), + /// The store's `validate`, or a request of the identity check, failed for + /// a reason that is neither a transport failure nor a denial (a bucket + /// that does not exist, an I/O error on a filesystem root); the store's + /// error. The configuration errors `validate` reports before the + /// listener stop startup instead. One category whether `validate` or a + /// later check's request found the condition, so a bucket deleted under + /// a running server reads as it does at startup. + Validate(String), + /// `extenddb-store.json` is there but is not a readable identity file. + FileUnreadable(String), + /// The file names a catalog other than this one. + ForeignCatalog { + file_catalog_id: String, + catalog_id: String, + }, + /// The file's id is not the id the catalog's rows for the root carry. + IdMismatch { + file_id: String, + row_ids: Vec, + }, + /// The catalog's rows for the root carry more than one id. + MultipleRowIds(Vec), + /// The catalog has rows for the root and the root has no file. + FileMissingWithRows(Vec), + /// This instance's create reported success and the re-read holds another + /// id: the store did not honour the conditional put. + CreateRaceLost { ours: String, file: String }, + /// The newest AVAILABLE row's manifest is not under the root. + ManifestMissing { backup_arn: String }, + /// The catalog could not be read. + Catalog(String), +} + +impl StoreProblem { + /// The problem's category, one or two words, for an API refusal. + #[must_use] + pub fn category(&self) -> &'static str { + match self { + Self::Transport(_) => "unreachable", + Self::Denied(_) => "permission denied", + Self::Validate(_) => "failed validation", + Self::FileUnreadable(_) => "identity file unreadable", + Self::ForeignCatalog { .. } => "owned by another catalog", + Self::IdMismatch { .. } | Self::MultipleRowIds(_) | Self::CreateRaceLost { .. } => { + "identity mismatch" + } + Self::FileMissingWithRows(_) => "identity file missing", + Self::ManifestMissing { .. } => "manifest probe failed", + Self::Catalog(_) => "catalog unreadable", + } + } + + /// The problem with the ids involved and never a root (a filesystem path + /// or a bucket) or the store's own error text, which may carry one: the + /// operator's line for the identity problems. Not the `/health` text, + /// which is [`StoreProblem::category`]. + #[must_use] + pub fn public_reason(&self) -> String { + match self { + Self::Transport(_) => "the store could not be reached".to_owned(), + Self::Denied(_) => "the store refused an operation (permission denied)".to_owned(), + Self::Validate(_) => "the store failed its validation".to_owned(), + Self::FileUnreadable(_) => format!("{STORE_IDENTITY_KEY} could not be read"), + Self::ForeignCatalog { + file_catalog_id, + catalog_id, + } => format!( + "{STORE_IDENTITY_KEY} names catalog {file_catalog_id}; this catalog is \ + {catalog_id}" + ), + Self::IdMismatch { file_id, row_ids } => format!( + "{STORE_IDENTITY_KEY} holds store id {file_id}; the catalog's rows for the root \ + carry {}", + row_ids.join(", ") + ), + Self::MultipleRowIds(ids) => format!( + "the catalog's rows for the root carry more than one store id: {}", + ids.join(", ") + ), + Self::FileMissingWithRows(ids) => format!( + "the root has no {STORE_IDENTITY_KEY} but the catalog has rows for it (store id \ + {})", + ids.join(", ") + ), + Self::CreateRaceLost { ours, file } => format!( + "this server created {STORE_IDENTITY_KEY} with store id {ours} but the file \ + holds {file}; the store did not honour the conditional put" + ), + Self::ManifestMissing { backup_arn } => { + format!("the manifest of {backup_arn} is not under the root") + } + Self::Catalog(_) => "the catalog could not be read".to_owned(), + } + } +} + +impl std::fmt::Display for StoreProblem { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Transport(e) => write!(f, "the store could not be reached: {e}"), + Self::Denied(e) => write!(f, "the store refused an operation: {e}"), + Self::Validate(e) => write!(f, "the store failed its validation: {e}"), + Self::FileUnreadable(e) => write!(f, "{STORE_IDENTITY_KEY} could not be read: {e}"), + Self::Catalog(e) => write!(f, "the catalog could not be read: {e}"), + other => f.write_str(&other.public_reason()), + } + } +} + +/// The problem a store's error is: a transport failure keeps its class with +/// the store error's whole cause chain (the SDK's top link is a bare +/// category such as "dispatch failure"; the cause is a link below it), a +/// denial is [`StoreProblem::Denied`], and every other refusal +/// ([`StoreProblem::Validate`]'s doc) is the same category `validate` gives +/// it. +impl From for StoreProblem { + fn from(e: StoreError) -> Self { + match e { + StoreError::PermissionDenied(m) => Self::Denied(m), + transport @ StoreError::Transport { .. } => Self::Transport(transport.to_string()), + other => Self::Validate(other.to_string()), + } + } +} + +/// Read the identity file, or `None` when the root has none. At most +/// [`STORE_IDENTITY_MAX_BYTES`] are read; a larger object is +/// [`StoreProblem::FileUnreadable`]. +/// +/// # Errors +/// +/// The store's error as a [`StoreProblem`], or a file that does not parse. +pub async fn read_identity_file( + store: &dyn BackupStore, +) -> Result, StoreProblem> { + read_file(store).await +} + +/// [`read_identity_file`]. +async fn read_file(store: &dyn BackupStore) -> Result, StoreProblem> { + let mut body = match store.get(STORE_IDENTITY_KEY).await { + Ok(body) => body, + Err(StoreError::NotFound) => return Ok(None), + Err(e) => return Err(e.into()), + }; + let mut bytes = Vec::new(); + while let Some(chunk) = body.try_next().await? { + if bytes.len() + chunk.len() > STORE_IDENTITY_MAX_BYTES { + return Err(StoreProblem::FileUnreadable(format!( + "larger than {STORE_IDENTITY_MAX_BYTES} bytes" + ))); + } + bytes.extend_from_slice(&chunk); + } + StoreIdentityFile::parse(&bytes).map(Some) +} + +/// The distinct ids the catalog's rows for the root carry, at most one. +async fn row_ids( + catalog: &C, + kind: StoreKind, + root: &str, +) -> Result, StoreProblem> { + let ids = catalog + .read_store_ids(kind, root) + .await + .map_err(|e| StoreProblem::Catalog(e.to_string()))?; + if ids.len() > 1 { + return Err(StoreProblem::MultipleRowIds(ids)); + } + Ok(ids) +} + +/// The startup check: `validate` has passed; read the file, create it when +/// the root is empty of both file and rows, compare it with the rows and +/// with `catalog_id`, then probe the newest manifest. Returns the store's +/// identity, the rows' and the file's. `backend` and `ephemeral_catalog` +/// describe this catalog in a file the check writes; a catalog with +/// `ephemeral_catalog` set (an in-memory SQLite database) also adopts a +/// root whose file another such catalog wrote (module doc, +/// [`AdoptRule::EphemeralOwner`]). +/// +/// # Errors +/// +/// The problem that makes the store unhealthy (module doc). +pub async fn establish( + store: &dyn BackupStore, + kind: StoreKind, + root: &str, + catalog: &C, + backend: &str, + ephemeral_catalog: bool, +) -> Result { + let catalog_id = catalog + .catalog_id() + .await + .map_err(|e| StoreProblem::Catalog(e.to_string()))?; + let row_ids = row_ids(catalog, kind, root).await?; + let file = match read_file(store).await? { + Some(file) => file, + None if !row_ids.is_empty() => { + return Err(StoreProblem::FileMissingWithRows(row_ids)); + } + None => { + let ours = StoreIdentityFile::new(&catalog_id, backend, ephemeral_catalog); + let created = store + .put_if_absent(STORE_IDENTITY_KEY, byte_stream_from(ours.to_bytes())) + .await? + .is_some(); + // The key holds nothing the store reads as an object and refuses + // the create (a FIFO or a socket at the path, say): nothing was + // created, and nothing can be read. + let file = read_file(store).await?.ok_or_else(|| { + StoreProblem::Transport(format!( + "{STORE_IDENTITY_KEY} is neither readable nor creatable" + )) + })?; + if created && file.store_id != ours.store_id { + return Err(StoreProblem::CreateRaceLost { + ours: ours.store_id, + file: file.store_id, + }); + } + if created { + tracing::info!( + "backup store ({kind}) root was empty; created {STORE_IDENTITY_KEY} with \ + store id {}", + file.store_id + ); + } else if file.catalog_id == catalog_id { + tracing::info!( + "backup store ({kind}) root already carried {STORE_IDENTITY_KEY} (store id \ + {}), created by another instance of this catalog", + file.store_id + ); + } else { + tracing::info!( + "backup store ({kind}) root already carried {STORE_IDENTITY_KEY} (store id \ + {}), created by catalog {} while this check created it", + file.store_id, + file.catalog_id + ); + } + file + } + }; + if ephemeral_catalog && file.catalog_id != catalog_id && file.ephemeral_catalog { + return match adopt( + store, + kind, + root, + catalog, + backend, + ephemeral_catalog, + AdoptRule::EphemeralOwner, + ) + .await + { + Ok(Adoption::Adopted { identity, .. } | Adoption::AlreadyOwned(identity)) => { + Ok(identity) + } + Err(refusal) => Err(refusal.into_problem(&catalog_id)), + }; + } + let identity = compare(&file, &catalog_id, row_ids, kind, root)?; + manifest_probe(store, &identity, catalog).await?; + Ok(identity) +} + +/// A read of the store's identity that writes nothing: the file, compared +/// with the catalog as [`establish`] compares it, without the create of a +/// missing file and without the manifest probe. `Ok(None)` for a root with +/// no file and no rows (a fresh root). For a command that reads beside a +/// server (`backup list`), which must leave the root as it found it. +/// +/// # Errors +/// +/// The problem that would make the store unhealthy (module doc). +pub async fn read_identity( + store: &dyn BackupStore, + kind: StoreKind, + root: &str, + catalog: &C, +) -> Result, StoreProblem> { + let catalog_id = catalog + .catalog_id() + .await + .map_err(|e| StoreProblem::Catalog(e.to_string()))?; + let row_ids = row_ids(catalog, kind, root).await?; + match read_file(store).await? { + Some(file) => compare(&file, &catalog_id, row_ids, kind, root).map(Some), + None if !row_ids.is_empty() => Err(StoreProblem::FileMissingWithRows(row_ids)), + None => Ok(None), + } +} + +/// Which owners [`adopt`] takes a root from. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AdoptRule { + /// Any other catalog: `extenddb backup sync --adopt-store`, an + /// operator's decision. + Command, + /// Only a catalog whose file carries `ephemeral_catalog`: the startup of + /// a server whose own catalog is in memory (module doc). + EphemeralOwner, +} + +impl AdoptRule { + /// The `action` of the audit line an adoption writes. + fn action(self) -> &'static str { + match self { + Self::Command => "backup sync --adopt-store", + Self::EphemeralOwner => "startup adoption by an in-memory catalog", + } + } +} + +/// What [`adopt`] did. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Adoption { + /// The file named `previous_catalog_id` and now names this catalog; the + /// store passed the check that follows (`compare`, the manifest probe). + Adopted { + identity: StoreIdentity, + previous_catalog_id: String, + }, + /// The file already named this catalog: nothing was written, and the + /// store passed the check as [`establish`] runs it. + AlreadyOwned(StoreIdentity), +} + +/// Why [`adopt`] did not leave the root this catalog's and healthy. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum AdoptRefusal { + /// The store could not be read or written, the file could not be + /// parsed, the catalog could not be read, or (the file already this + /// catalog's) the check found another problem. + Problem(StoreProblem), + /// The root has no identity file: there is no owner to take it from. + /// `row_ids` are the store ids this catalog's rows for the root carry: + /// with none, the root is a fresh one that the next check (plain sync's + /// included) gives a file; with some, the root lost its file under the + /// rows. + NoFile { row_ids: Vec }, + /// The rule is [`AdoptRule::EphemeralOwner`] and the file's catalog is + /// not an in-memory one. + DurableOwner { file_catalog_id: String }, + /// This catalog's rows for the root carry an id other than the file's: + /// the adopted root would be unhealthy at once (an identity mismatch). + RowsCarryOtherIds { + file_id: String, + row_ids: Vec, + }, + /// After the put, the file names another catalog or holds another store + /// id: a concurrent adopter wrote after this one (last writer wins). + RaceLost { + file_catalog_id: String, + file_store_id: String, + }, + /// The file now names this catalog, and the check that follows failed + /// (the manifest probe, say). + AdoptedUnhealthy { + previous_catalog_id: String, + problem: StoreProblem, + }, +} + +impl AdoptRefusal { + /// The store problem the refusal leaves, for a check that records one: + /// a root another catalog still owns is [`StoreProblem::ForeignCatalog`]. + #[must_use] + pub fn into_problem(self, catalog_id: &str) -> StoreProblem { + match self { + Self::Problem(problem) | Self::AdoptedUnhealthy { problem, .. } => problem, + Self::NoFile { row_ids } if !row_ids.is_empty() => { + StoreProblem::FileMissingWithRows(row_ids) + } + Self::NoFile { .. } => StoreProblem::FileUnreadable(format!( + "{STORE_IDENTITY_KEY} was removed while the root was being adopted; the next \ + check creates it" + )), + Self::DurableOwner { file_catalog_id } + | Self::RaceLost { + file_catalog_id, .. + } => StoreProblem::ForeignCatalog { + file_catalog_id, + catalog_id: catalog_id.to_owned(), + }, + Self::RowsCarryOtherIds { file_id, row_ids } => { + StoreProblem::IdMismatch { file_id, row_ids } + } + } + } +} + +impl std::fmt::Display for AdoptRefusal { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Problem(problem) => write!(f, "{problem}"), + Self::NoFile { row_ids } if row_ids.is_empty() => write!( + f, + "the root has no {STORE_IDENTITY_KEY}, so no catalog owns it and there is \ + nothing to adopt; plain sync creates the file" + ), + Self::NoFile { row_ids } => write!( + f, + "the root has no {STORE_IDENTITY_KEY}, and this catalog's rows for it carry \ + store id {}: the root lost its file under the rows, and there is no owner to \ + adopt it from", + row_ids.join(", ") + ), + Self::DurableOwner { file_catalog_id } => write!( + f, + "{STORE_IDENTITY_KEY} names catalog {file_catalog_id}, which is not an \ + in-memory catalog" + ), + Self::RowsCarryOtherIds { file_id, row_ids } => write!( + f, + "{STORE_IDENTITY_KEY} holds store id {file_id}, and this catalog's rows for the \ + root carry {}: adopting the root would leave it unhealthy (an identity \ + mismatch); nothing was written", + row_ids.join(", ") + ), + Self::RaceLost { + file_catalog_id, + file_store_id, + } => write!( + f, + "another writer replaced {STORE_IDENTITY_KEY} after this catalog's write: it \ + now names catalog {file_catalog_id} with store id {file_store_id}" + ), + Self::AdoptedUnhealthy { + previous_catalog_id, + problem, + } => write!( + f, + "{STORE_IDENTITY_KEY} now names this catalog (it named {previous_catalog_id}), \ + and the store is unhealthy: {problem}" + ), + } + } +} + +/// Make this catalog the root's owner: rewrite `extenddb-store.json`'s +/// `catalog_id` (and `backend` and `ephemeral_catalog`, which describe the +/// owner) with a plain `put`, keeping `store_id` and `created_at_ms`, then +/// re-read the file and run the check [`establish`] runs (`compare`, the +/// manifest probe). `rule` says which owners it takes the root from. The +/// rewrite is logged under the `extenddb::audit` target with the old and +/// the new catalog id. +/// +/// A file that already names this catalog is not written +/// ([`Adoption::AlreadyOwned`]). Nothing is written when the root has no +/// file, when the rule refuses the owner, or when this catalog's rows for +/// the root carry an id other than the file's. +/// +/// # Errors +/// +/// [`AdoptRefusal`], each case with what was and was not written. +pub async fn adopt( + store: &dyn BackupStore, + kind: StoreKind, + root: &str, + catalog: &C, + backend: &str, + ephemeral_catalog: bool, + rule: AdoptRule, +) -> Result { + let catalog_id = catalog + .catalog_id() + .await + .map_err(|e| AdoptRefusal::Problem(StoreProblem::Catalog(e.to_string())))?; + let row_ids = catalog + .read_store_ids(kind, root) + .await + .map_err(|e| AdoptRefusal::Problem(StoreProblem::Catalog(e.to_string())))?; + let file = read_file(store) + .await + .map_err(AdoptRefusal::Problem)? + .ok_or_else(|| AdoptRefusal::NoFile { + row_ids: row_ids.clone(), + })?; + if file.catalog_id == catalog_id { + if row_ids.len() > 1 { + return Err(AdoptRefusal::Problem(StoreProblem::MultipleRowIds(row_ids))); + } + let identity = + compare(&file, &catalog_id, row_ids, kind, root).map_err(AdoptRefusal::Problem)?; + manifest_probe(store, &identity, catalog) + .await + .map_err(AdoptRefusal::Problem)?; + return Ok(Adoption::AlreadyOwned(identity)); + } + if rule == AdoptRule::EphemeralOwner && !file.ephemeral_catalog { + return Err(AdoptRefusal::DurableOwner { + file_catalog_id: file.catalog_id, + }); + } + if row_ids.iter().any(|id| *id != file.store_id) { + return Err(AdoptRefusal::RowsCarryOtherIds { + file_id: file.store_id, + row_ids, + }); + } + let ours = StoreIdentityFile { + catalog_id: catalog_id.clone(), + backend: backend.to_owned(), + ephemeral_catalog, + ..file.clone() + }; + tracing::info!( + target: "extenddb::audit", + action = rule.action(), + outcome = "rewriting", + store_kind = %kind, + store_root = %root, + store_id = %file.store_id, + previous_catalog_id = %file.catalog_id, + catalog_id = %catalog_id, + "rewriting {STORE_IDENTITY_KEY} to name this catalog" + ); + store + .put(STORE_IDENTITY_KEY, byte_stream_from(ours.to_bytes())) + .await + .map_err(|e| AdoptRefusal::Problem(e.into()))?; + let reread = read_file(store) + .await + .map_err(AdoptRefusal::Problem)? + .ok_or_else(|| AdoptRefusal::NoFile { + row_ids: row_ids.clone(), + })?; + let won = reread.catalog_id == catalog_id && reread.store_id == file.store_id; + tracing::info!( + target: "extenddb::audit", + action = rule.action(), + outcome = if won { "adopted" } else { "race lost" }, + store_kind = %kind, + store_root = %root, + store_id = %file.store_id, + previous_catalog_id = %file.catalog_id, + catalog_id = %catalog_id, + file_catalog_id = %reread.catalog_id, + "rewrote {STORE_IDENTITY_KEY}; its re-read names catalog {}", + reread.catalog_id + ); + if !won { + return Err(AdoptRefusal::RaceLost { + file_catalog_id: reread.catalog_id, + file_store_id: reread.store_id, + }); + } + let unhealthy = |problem: StoreProblem| AdoptRefusal::AdoptedUnhealthy { + previous_catalog_id: file.catalog_id.clone(), + problem, + }; + let identity = compare(&reread, &catalog_id, row_ids, kind, root).map_err(&unhealthy)?; + manifest_probe(store, &identity, catalog) + .await + .map_err(&unhealthy)?; + tracing::info!( + "backup store ({kind}) {root}: adopted from catalog {} (store id {})", + file.catalog_id, + identity.store_id + ); + Ok(Adoption::Adopted { + identity, + previous_catalog_id: file.catalog_id, + }) +} + +/// The file against the catalog: it must name this catalog, and the rows +/// for the root, when there are any, must carry its id. +fn compare( + file: &StoreIdentityFile, + catalog_id: &str, + row_ids: Vec, + kind: StoreKind, + root: &str, +) -> Result { + if file.catalog_id != catalog_id { + return Err(StoreProblem::ForeignCatalog { + file_catalog_id: file.catalog_id.clone(), + catalog_id: catalog_id.to_owned(), + }); + } + if !row_ids.is_empty() && row_ids[0] != file.store_id { + return Err(StoreProblem::IdMismatch { + file_id: file.store_id.clone(), + row_ids, + }); + } + Ok(StoreIdentity { + store_kind: kind, + store_root: root.to_owned(), + store_id: file.store_id.clone(), + }) +} + +/// The check every reconciler cycle runs on a store whose identity startup +/// settled: the file is still there, names this catalog and holds the +/// identity's id; the rows for the root carry no other id; the newest +/// manifest is under the root. A root that lost its file while the catalog +/// has no rows for it gets the file back, written with the identity's id +/// (`put_if_absent`, so a file another instance wrote first wins and is then +/// compared like any other). It never adopts (module doc). +/// +/// # Errors +/// +/// The problem that makes the store unhealthy (module doc). +pub async fn recheck( + store: &dyn BackupStore, + identity: &StoreIdentity, + catalog: &C, + backend: &str, + ephemeral_catalog: bool, +) -> Result<(), StoreProblem> { + let catalog_id = catalog + .catalog_id() + .await + .map_err(|e| StoreProblem::Catalog(e.to_string()))?; + let row_ids = row_ids(catalog, identity.store_kind, &identity.store_root).await?; + let file = match read_file(store).await? { + Some(file) => file, + None if !row_ids.is_empty() => { + return Err(StoreProblem::FileMissingWithRows(row_ids)); + } + None => { + let ours = StoreIdentityFile { + store_id: identity.store_id.clone(), + ..StoreIdentityFile::new(&catalog_id, backend, ephemeral_catalog) + }; + store + .put_if_absent(STORE_IDENTITY_KEY, byte_stream_from(ours.to_bytes())) + .await?; + tracing::warn!( + "backup store ({}) root had lost {STORE_IDENTITY_KEY}; rewrote it with store id \ + {}", + identity.store_kind, + identity.store_id + ); + read_file(store).await?.ok_or_else(|| { + StoreProblem::Transport(format!( + "{STORE_IDENTITY_KEY} is neither readable nor creatable" + )) + })? + } + }; + if file.catalog_id != catalog_id { + return Err(StoreProblem::ForeignCatalog { + file_catalog_id: file.catalog_id, + catalog_id, + }); + } + if file.store_id != identity.store_id { + // The file changed under a running server: the root was replaced. + return Err(StoreProblem::IdMismatch { + file_id: file.store_id, + row_ids: vec![identity.store_id.clone()], + }); + } + if let Some(other) = row_ids.iter().find(|id| **id != identity.store_id) { + return Err(StoreProblem::IdMismatch { + file_id: file.store_id, + row_ids: vec![other.clone()], + }); + } + manifest_probe(store, identity, catalog).await +} + +/// `head` the manifest of the newest AVAILABLE row of the store; a root with +/// no AVAILABLE row passes (there is nothing to probe). +/// +/// # Errors +/// +/// [`StoreProblem::ManifestMissing`] when the manifest is absent, +/// [`StoreProblem::Transport`] when the store cannot be read, +/// [`StoreProblem::Catalog`] when the rows cannot be listed. +pub async fn manifest_probe( + store: &dyn BackupStore, + identity: &StoreIdentity, + catalog: &C, +) -> Result<(), StoreProblem> { + // One row, not the listing: the probe runs each reconciler cycle on + // every instance and in each command's identity check. + let Some(newest) = catalog + .newest_available_backup_row(identity) + .await + .map_err(|e| StoreProblem::Catalog(e.to_string()))? + else { + return Ok(()); + }; + let arn = BackupArn::parse(&newest.backup_arn).map_err(|e| { + StoreProblem::Catalog(format!("row {} carries a bad ARN: {e}", newest.backup_arn)) + })?; + let key = format!("{}{EXTENDDB_MANIFEST_FILE}", arn.store_prefix()); + match store.head(&key).await? { + Some(_) => Ok(()), + None => Err(StoreProblem::ManifestMissing { + backup_arn: newest.backup_arn.clone(), + }), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_file_round_trips_and_refuses_other_shapes() { + let file = StoreIdentityFile::new("cat-1", "postgres", false); + assert_eq!(file.format_version, 1); + assert_eq!(file.catalog_id, "cat-1"); + assert_eq!(file.backend, "postgres"); + assert!(!file.store_id.is_empty()); + let parsed = StoreIdentityFile::parse(&file.to_bytes()).unwrap(); + assert_eq!(parsed, file); + // The marker round-trips, and a file without it (an earlier build's) + // reads as a durable catalog's. + let memory = StoreIdentityFile::new("cat-2", "sqlite", true); + assert!( + StoreIdentityFile::parse(&memory.to_bytes()) + .unwrap() + .ephemeral_catalog + ); + let earlier = serde_json::json!({ + "format_version": 1, "store_id": "s", "catalog_id": "c", "backend": "sqlite", + "created_at_ms": 0 + }); + assert!( + !StoreIdentityFile::parse(earlier.to_string().as_bytes()) + .unwrap() + .ephemeral_catalog + ); + + let v2 = serde_json::json!({ + "format_version": 2, "store_id": "s", "catalog_id": "c", "backend": "b", + "created_at_ms": 0 + }); + let err = StoreIdentityFile::parse(v2.to_string().as_bytes()).unwrap_err(); + assert!(matches!(&err, StoreProblem::FileUnreadable(m) if m.contains("format_version 2"))); + let empty = serde_json::json!({ + "format_version": 1, "store_id": "", "catalog_id": "c", "backend": "b", + "created_at_ms": 0 + }); + assert!(matches!( + StoreIdentityFile::parse(empty.to_string().as_bytes()), + Err(StoreProblem::FileUnreadable(_)) + )); + assert!(matches!( + StoreIdentityFile::parse(b"not json"), + Err(StoreProblem::FileUnreadable(_)) + )); + } + + /// `public_reason` names ids and the catalog, never a root or the store's + /// own error text (which may carry a path or a bucket); `Display` carries + /// that text, and `/health` carries the category alone. + #[test] + fn the_public_reason_names_no_root() { + let transport = StoreProblem::Transport("connect /srv/backups: refused".to_owned()); + assert!(!transport.public_reason().contains("/srv")); + assert!(transport.to_string().contains("/srv/backups")); + assert_eq!(transport.category(), "unreachable"); + let foreign = StoreProblem::ForeignCatalog { + file_catalog_id: "other".to_owned(), + catalog_id: "mine".to_owned(), + }; + assert!(foreign.public_reason().contains("other")); + assert!(foreign.public_reason().contains("mine")); + assert_eq!(foreign.category(), "owned by another catalog"); + let probe = StoreProblem::ManifestMissing { + backup_arn: "arn:x".to_owned(), + }; + assert!(probe.public_reason().contains("arn:x")); + assert_eq!(probe.category(), "manifest probe failed"); + let unreadable = StoreProblem::FileUnreadable("at /srv/x".to_owned()); + assert!(!unreadable.public_reason().contains("/srv")); + let denied: StoreProblem = + StoreError::PermissionDenied("/srv/backups/a has mode 0755".to_owned()).into(); + assert_eq!(denied.category(), "permission denied"); + assert!(!denied.public_reason().contains("/srv")); + assert!(denied.to_string().contains("/srv/backups/a")); + // A transport failure keeps its whole cause chain: the SDK's top + // link names only the class, and the cause is the link below it. + let chain: StoreProblem = StoreError::Transport { + source: Box::new(TwoLinks), + } + .into(); + assert_eq!(chain.category(), "unreachable"); + assert!(chain.to_string().contains("dispatch failure"), "{chain}"); + assert!(chain.to_string().contains("connection refused"), "{chain}"); + // Any other store refusal is the category `validate` gives it, so a + // bucket that disappears under a running server reads as a bucket + // that was missing at startup. + let missing: StoreProblem = StoreError::Other("NoSuchBucket".to_owned()).into(); + assert_eq!(missing.category(), "failed validation"); + assert!(missing.to_string().contains("NoSuchBucket")); + } + + /// A two-link error chain: "dispatch failure" over "connection refused". + #[derive(Debug)] + struct TwoLinks; + + impl std::fmt::Display for TwoLinks { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("dispatch failure") + } + } + + impl std::error::Error for TwoLinks { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + Some(&CONNECTION_REFUSED) + } + } + + static CONNECTION_REFUSED: ConnectionRefused = ConnectionRefused; + + #[derive(Debug)] + struct ConnectionRefused; + + impl std::fmt::Display for ConnectionRefused { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("connection refused") + } + } + + impl std::error::Error for ConnectionRefused {} + + #[test] + fn compare_applies_the_three_rules() { + let file = StoreIdentityFile { + format_version: 1, + store_id: "s-1".to_owned(), + catalog_id: "c-1".to_owned(), + backend: "sqlite".to_owned(), + created_at_ms: 0, + ephemeral_catalog: false, + }; + let ok = compare(&file, "c-1", vec![], StoreKind::Filesystem, "/r").unwrap(); + assert_eq!(ok.store_id, "s-1"); + assert_eq!(ok.store_root, "/r"); + compare( + &file, + "c-1", + vec!["s-1".to_owned()], + StoreKind::Filesystem, + "/r", + ) + .unwrap(); + assert!(matches!( + compare(&file, "c-2", vec![], StoreKind::Filesystem, "/r"), + Err(StoreProblem::ForeignCatalog { .. }) + )); + assert!(matches!( + compare( + &file, + "c-1", + vec!["s-9".to_owned()], + StoreKind::S3, + "s3://b/" + ), + Err(StoreProblem::IdMismatch { .. }) + )); + } +} diff --git a/crates/engine/src/backup/sync.rs b/crates/engine/src/backup/sync.rs new file mode 100644 index 000000000..e96b70dfd --- /dev/null +++ b/crates/engine/src/backup/sync.rs @@ -0,0 +1,987 @@ +// SPDX-License-Identifier: Apache-2.0 + +//! `backup sync`: the catalog rebuilt from the store, for the backups the +//! catalog has no row for, and the report of the rows whose backup the store +//! no longer holds. +//! +//! The store is the source of truth and the row a cache (design 4(g)). +//! [`sync`] reads the rows first: every row of the configured kind and root, +//! of every status and every store id the catalog records for the root. It +//! keeps of each row only what it uses (the status, store id and table by +//! `(account_id, backup_id)`, and the key of each AVAILABLE row of the +//! configured store id) and drops the rest as the row is read; the +//! backends' `list_backup_rows` collect their whole result first, so the +//! read peaks above what is kept. It then lists the configured +//! store's whole root (the store trait has no delimiter listing) and handles +//! each `/
    //` prefix ([`classify_store_key`]) as +//! the listing passes it: the store lists in key order, so the keys of one +//! prefix arrive together, and sync holds the objects of one prefix with +//! their sizes and the prefix of each manifest it listed, never the whole +//! listing. A filesystem store builds its own listing in memory +//! before it returns the first key (the store trait's doc gives its cost); +//! an S3 store lists a page at a time. Keys outside every backup prefix +//! (`extenddb-store.json` at the root) are passed over; the reserved +//! components (`exports/` first, `changelog` third) and three components +//! that are not a backup prefix (an id of an earlier build's shape, a +//! component no ARN can carry) are reported as skipped, never read. +//! +//! A prefix with a row of any status (CREATING, DELETING, DELETED, FAILED, +//! AVAILABLE) is skipped with the row's status as the reason, so sync never +//! modifies a row nor adopts a prefix a row names. A prefix with no row is +//! read and verified as the dead-owner duty verifies one +//! (`reconciler::verify_prefix`: the manifest and its sidecar within their +//! bounds, the digest, the reader's acceptance, every listed data file at +//! its recorded size, from the prefix's objects already listed), and the +//! manifest's ARN must name the listed account, table and backup id. A +//! manifest whose ARN is in another partition than [`ARN_PARTITION`] is +//! skipped: every request refuses such an ARN before it reads a row, so no +//! request could address the row. Another region is adopted and logged: +//! requests find the row by its ARN as written. Before the insert the row is +//! read under each spelling of the ARN (this server's region and the +//! manifest's), and a backup the other spelling has a row for is skipped: +//! Cassandra keys its rows by the ARN and has no unique backup id, so there +//! the read is what keeps a second row of the backup id out of the catalog +//! under the manifest's account, table and id. A row of the same account +//! and id under another table that the rows read did not return (a row of +//! another root, or one whose list entries are missing) is not seen by it; +//! within one run, the row check keeps a second prefix of an id this run +//! adopted out. A row under the manifest's own spelling is left to the +//! insert. A +//! verified prefix gets an AVAILABLE row through +//! [`BackupCatalog::insert_backup_row`], unowned, in the configured store's +//! identity, with the manifest's name, ARN, size, item total and +//! `source_table`, and the id's instant as `created_at`. An insert that +//! answers `false` found a row that landed since the rows were read (a +//! CreateBackup on another instance), a row of another root, account or +//! table holding the backup id, or, on Cassandra, a row whose list and +//! status entries are missing and that is not this insert's (a +//! CreateBackup's claim that stopped between its two writes); the row is +//! re-read to say which, and is left as it is. On Cassandra an earlier +//! insert of this same row that stopped between the row and its entries +//! (a sync killed there, or an entry batch that failed twice) is not in +//! the listings the rows are read from, so the next sync of the prefix +//! reaches the insert, which writes the entries and answers `true` (the +//! trait's doc). A failed +//! read of one prefix's manifest or sidecar ([`SkipReason::Store`]), or a +//! failed catalog call for one prefix ([`SkipReason::Catalog`]), is +//! reported for that prefix and sync goes on; a later run adopts it once +//! the call succeeds ([`SyncReport::transient_skips`]). +//! +//! A listing that fails stops sync: the rows inserted for the prefixes the +//! listing passed before the failure stay (each was verified), the prefix +//! it was in is not read (its keys may be cut short), and no +//! missing-manifest report is made, since the listing did not reach every +//! prefix. A stop asked through [`SyncTarget::stop`] (the command's +//! signal) is honoured before each prefix the same way. +//! +//! The report also names the AVAILABLE rows of the configured store whose +//! manifest the listing did not hold. [`prune_missing`] (`--prune-missing`, +//! under the maintenance claim) removes each such row outright through +//! [`BackupCatalog::forget_available_backup_row`], the row's lock, status +//! check and in-use rule with the removal, and deletes nothing in the +//! store: the bytes may be under another host's root of the same path, and +//! no DELETING or DELETED tombstone is left for a deletion duty to act on +//! them. A later sync where the bytes are adopts them again. Without +//! `--prune-missing` such rows stay AVAILABLE. +//! +//! The command takes the maintenance claim for every sync on PostgreSQL, +//! MongoDB and Cassandra, where it runs beside servers (design 4(f)), and +//! runs under the serve lock on SQLite; so no purge runs between its row +//! read and its insert. A row it inserts is AVAILABLE and unowned +//! (`owner_generation` 0); no reconciler duty reads an AVAILABLE row's +//! generation, so the row is handled as a created one. +//! +//! [`startup_sync`] is the sync a server whose catalog is an in-memory +//! SQLite database runs itself once its store is healthy: the catalog is +//! new at every start, so it is how such a server serves the backups of its +//! previous starts. The check that made the store healthy adopted a root an +//! earlier in-memory catalog owned (`store_identity`'s module doc). Each run +//! is bounded by [`STARTUP_SYNC_TIMEOUT`]; one that does not finish, or +//! that skipped a prefix for a store or catalog error, is left pending and +//! runs again in the next reconciler cycle. Every run lists from the first +//! key, so the startup sync completes only while listing the whole root +//! fits in the bound; past that it stays pending and is retried every +//! cycle, and the backups beyond what a run reaches stay unserved. Until a +//! run finishes, requests +//! for the earlier backups answer `BackupNotFoundException`; the log line +//! "startup sync of the in-memory catalog: N backup(s) adopted" ends that +//! window. + +use std::collections::{HashMap, HashSet}; +use std::time::Duration; + +use extenddb_backup_store::{BackupStore, StoreError}; +use extenddb_core::backup_format::{ + BackupArn, CHANGELOG_COMPONENT, EXPORTS_COMPONENT, EXTENDDB_MANIFEST_FILE, StoreKey, + backup_creation_date_time, classify_store_key, +}; +use extenddb_core::metrics::MetricsCollector; +use extenddb_storage::BackupCatalog; +use extenddb_storage::backup_catalog::{ + BackupKey, BackupRow, BackupStatus, ForgetRow, StoreIdentity, UNOWNED_GENERATION, + is_backup_not_found, +}; +use extenddb_storage::error::StorageError; +use extenddb_storage::util::ARN_PARTITION; +use futures::TryStreamExt; + +use super::BackupRuntime; +use super::reconciler::{StoreCalls, Verified, manifest_item_count, verify_prefix}; + +/// The bound of one run of [`startup_sync`], before the server serves and in +/// each reconciler cycle. A run abandoned at the bound keeps the rows it +/// inserted, and the next run skips those prefixes on the row check, but +/// every run lists the root from its first key: a root whose listing alone +/// takes longer than the bound is never synced, and the sync stays pending +/// and is retried every cycle (on a filesystem store the whole listing is +/// built before the first key, so no prefix is reached). On a filesystem +/// store an abandoned run's listing is not stopped: the walk runs to its end +/// on the blocking pool, holding the listing it builds, and the next +/// cycle's run starts another walk, so on a root whose walk outlasts the +/// cycle period the walks overlap. +pub const STARTUP_SYNC_TIMEOUT: Duration = Duration::from_secs(60); + +/// Asked before each prefix: why to stop, or `None` to go on. +pub type Stop<'a> = &'a (dyn Fn() -> Option + Sync); + +/// The store a sync reads and the identity its rows are written under. +#[derive(Clone, Copy)] +pub struct SyncTarget<'a> { + /// The configured store. + pub store: &'a dyn BackupStore, + /// The configured store's identity, its id the identity file's. + pub identity: &'a StoreIdentity, + /// The collector failed store calls are recorded in; `None` in a + /// command. + pub metrics: Option<&'a MetricsCollector>, + /// The server's region: an adopted ARN's region is compared with it + /// (logged when they differ), and the row is looked up under the ARN + /// rebuilt in it before the insert. `None` does neither. + pub region: Option<&'a str>, + /// Asked before each prefix; `None` never stops. + pub stop: Option>, +} + +/// What a sync did. +#[derive(Debug, Default)] +pub struct SyncReport { + /// The backups that got an AVAILABLE row, in listing order. + pub adopted: Vec, + /// The prefixes that did not, in listing order, with the reason. + pub skipped: Vec, + /// The keys of the AVAILABLE rows of the configured store whose + /// manifest the listing does not hold; empty when the run stopped + /// before the end of the listing. + pub missing_manifests: Vec, + /// Why the run stopped before the end of the listing, when it did. + pub stopped: Option, +} + +impl SyncReport { + /// The prefixes skipped for a store or catalog error, which a later run + /// may adopt. + #[must_use] + pub fn transient_skips(&self) -> usize { + self.skipped + .iter() + .filter(|s| s.reason.is_transient()) + .count() + } +} + +/// A backup [`sync`] gave an AVAILABLE row. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AdoptedBackup { + /// The ARN, as the manifest spells it. + pub backup_arn: String, + /// Set when the ARN's region is not the server's: the sentence the log + /// carries. + pub foreign: Option, +} + +/// A prefix [`sync`] did not adopt. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SkippedPrefix { + /// The prefix, as the store's keys spell it (`exports/` for that + /// component). + pub prefix: String, + pub reason: SkipReason, +} + +/// Why a prefix was not adopted. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum SkipReason { + /// Under a reserved component, the one named. + Reserved(&'static str), + /// Three components that are not a backup prefix; why. + NotABackup(String), + /// The catalog has a row for the account and backup id: its status, + /// store id, and its table when that is not the prefix's. + HasRow { + status: BackupStatus, + store_id: String, + other_table: Option, + }, + /// A row the catalog's read had not returned holds the backup: what + /// the read under the ARN's spellings, or the insert's re-read, found. + RowExists(String), + /// The prefix failed verification; which check. + Incomplete(String), + /// The manifest names another backup than the path. + ArnMismatch(String), + /// The manifest's ARN is in another partition: no request to this + /// server can address it. + OtherPartition(String), + /// A read of the prefix's manifest or sidecar failed; the store's + /// error. + Store(String), + /// A catalog call for the prefix (the read under the ARN's spellings, + /// the insert) failed; the catalog's error. + Catalog(String), +} + +impl SkipReason { + /// Whether a later run may adopt the prefix without anything changing + /// in the store: a store or catalog call failed. + #[must_use] + pub fn is_transient(&self) -> bool { + matches!(self, Self::Store(_) | Self::Catalog(_)) + } +} + +impl std::fmt::Display for SkipReason { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Reserved(component) if *component == EXPORTS_COMPONENT => { + f.write_str("reserved for exports") + } + Self::Reserved(_) => f.write_str("reserved for the change-log archive"), + Self::NotABackup(why) => write!(f, "not a backup prefix: {why}"), + Self::HasRow { + status, + store_id, + other_table: None, + } => write!( + f, + "the catalog has a row for it ({status}, store id {store_id})" + ), + Self::HasRow { + status, + store_id, + other_table: Some(table), + } => write!( + f, + "the catalog has a row for its backup id under table {table} ({status}, store \ + id {store_id}); a backup id names one backup" + ), + Self::RowExists(what) => f.write_str(what), + Self::Incomplete(why) => write!(f, "not a complete backup: {why}"), + Self::ArnMismatch(why) | Self::OtherPartition(why) => f.write_str(why), + Self::Store(e) => write!(f, "the store could not be read: {e}"), + Self::Catalog(e) => write!(f, "the catalog could not be read or written: {e}"), + } + } +} + +/// Why a sync stopped before it reached the end of the listing. +#[derive(Debug)] +pub enum SyncError { + /// The store's listing failed; `partial` is what the run did with the + /// prefixes the listing passed before (no missing-manifest report). + Listing { + error: StoreError, + partial: SyncReport, + }, + /// The catalog's rows could not be read; nothing was listed or written. + Catalog(StorageError), +} + +impl std::fmt::Display for SyncError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Listing { error, partial } if partial.adopted.is_empty() => { + write!(f, "the store's listing failed: {error}; no row was written") + } + Self::Listing { error, partial } => write!( + f, + "the store's listing failed: {error}; {} backup(s) listed before it were adopted", + partial.adopted.len() + ), + Self::Catalog(e) => write!( + f, + "the catalog's rows could not be read: {e}; no row was written" + ), + } + } +} + +impl std::error::Error for SyncError {} + +/// The prefix a listed key belongs to, and what it is. +enum Group { + /// Under a reserved component. + Reserved(&'static str), + /// Under three components that are not a backup prefix. + NotABackup(String), + /// Under a backup prefix: its components and the objects listed so far + /// with their sizes. + Backup { + account_id: String, + table_name: String, + backup_id: String, + objects: HashMap, + }, +} + +/// The prefix of `key` and an empty group for it; `None` for a key outside +/// every backup prefix. +fn group_of(key: &str) -> Option<(String, Group)> { + match classify_store_key(key) { + StoreKey::Outside => None, + StoreKey::Reserved(component) => { + Some((reserved_prefix(key, component), Group::Reserved(component))) + } + StoreKey::NotABackup { prefix, reason } => { + Some((prefix.to_owned(), Group::NotABackup(reason))) + } + StoreKey::Backup { + prefix, + account_id, + table_name, + backup_id, + } => Some(( + prefix.to_owned(), + Group::Backup { + account_id: account_id.to_owned(), + table_name: table_name.to_owned(), + backup_id: backup_id.to_owned(), + objects: HashMap::new(), + }, + )), + } +} + +/// The prefix a key under a reserved component is reported under: +/// `exports/`, or `/
    /changelog/`. +fn reserved_prefix(key: &str, component: &str) -> String { + if component == EXPORTS_COMPONENT { + return format!("{EXPORTS_COMPONENT}/"); + } + let mut parts = key.splitn(3, '/'); + let account = parts.next().unwrap_or_default(); + let table = parts.next().unwrap_or_default(); + format!("{account}/{table}/{CHANGELOG_COMPONENT}/") +} + +/// The rows of the configured kind and root, every status and every store +/// id the catalog records for the root, kept as only what sync uses: the +/// row check's status, store id and table by `(account_id, backup_id)`, +/// and the keys of the configured store id's AVAILABLE rows (the +/// missing-manifest candidates). Each row is dropped as it is read, its +/// parsed `source_table_details_json` with it. +async fn rows_of_root( + storage: &S, + identity: &StoreIdentity, +) -> Result<(Known, Vec), StorageError> { + let mut ids = storage + .read_store_ids(identity.store_kind, &identity.store_root) + .await?; + if !ids.contains(&identity.store_id) { + ids.push(identity.store_id.clone()); + } + let mut known = Known::new(); + let mut available = Vec::new(); + for id in ids { + let of_id = StoreIdentity { + store_id: id, + ..identity.clone() + }; + let mut listed = storage.list_backup_rows(None, &of_id); + while let Some(row) = listed.try_next().await? { + if row.status == BackupStatus::Available && row.store.store_id == identity.store_id { + available.push(row.key()); + } + let BackupRow { + account_id, + backup_id, + table_name, + status, + store, + .. + } = row; + known.insert( + (account_id, backup_id), + (status, store.store_id, table_name), + ); + } + } + Ok((known, available)) +} + +/// What the row check knows of a row: its status, store id and table. +type Known = HashMap<(String, String), (BackupStatus, String, String)>; + +/// Adopt the backups of the configured store that have no row (module doc). +/// +/// # Errors +/// +/// [`SyncError`]: the catalog's read failed (nothing written), or the +/// listing failed (the rows inserted before it stay). +pub async fn sync( + storage: &S, + target: SyncTarget<'_>, +) -> Result { + let calls = StoreCalls { + store: target.store, + metrics: target.metrics, + }; + let identity = target.identity; + let (mut known, available) = rows_of_root(storage, identity) + .await + .map_err(SyncError::Catalog)?; + + let mut report = SyncReport::default(); + // The prefixes whose manifest the listing held: one string per listed + // manifest. + let mut manifests: HashSet = HashSet::new(); + let mut open: Option<(String, Group)> = None; + let mut listing = calls.store.list(""); + loop { + let next = match listing.try_next().await { + Ok(next) => next, + Err(e) => { + return Err(SyncError::Listing { + error: calls.failed("list", e), + partial: report, + }); + } + }; + let Some(meta) = next else { break }; + let Some((prefix, group)) = group_of(&meta.key) else { + continue; + }; + if open.as_ref().is_none_or(|(open, _)| *open != prefix) { + // The listing has passed the open prefix: every key of it is + // listed. + if let Some((done, group)) = open.take() + && let Some(reason) = + finish_group(storage, calls, target, &mut known, &mut report, done, group).await + { + report.stopped = Some(reason); + return Ok(report); + } + open = Some((prefix.clone(), group)); + } + if let Some((_, Group::Backup { objects, .. })) = &mut open { + if meta.key.len() == prefix.len() + EXTENDDB_MANIFEST_FILE.len() + && meta.key.ends_with(EXTENDDB_MANIFEST_FILE) + { + manifests.insert(prefix); + } + objects.insert(meta.key, meta.size); + } + } + if let Some((done, group)) = open.take() + && let Some(reason) = + finish_group(storage, calls, target, &mut known, &mut report, done, group).await + { + report.stopped = Some(reason); + return Ok(report); + } + + report.missing_manifests = available + .into_iter() + .filter(|key| { + !BackupArn::parse(&key.backup_arn) + .is_ok_and(|arn| manifests.contains(&arn.store_prefix())) + }) + .collect(); + Ok(report) +} + +/// One prefix the listing has passed: the stop, then the row check, then +/// the verification and the insert. Returns why the run stops, when the +/// stop asked it to (the prefix is then left unread). +async fn finish_group( + storage: &S, + calls: StoreCalls<'_>, + target: SyncTarget<'_>, + known: &mut Known, + report: &mut SyncReport, + prefix: String, + group: Group, +) -> Option { + if let Some(reason) = target.stop.and_then(|stop| stop()) { + return Some(reason); + } + let (account_id, table_name, backup_id, objects) = match group { + Group::Reserved(component) => { + report.skipped.push(SkippedPrefix { + prefix, + reason: SkipReason::Reserved(component), + }); + return None; + } + Group::NotABackup(why) => { + report.skipped.push(SkippedPrefix { + prefix, + reason: SkipReason::NotABackup(why), + }); + return None; + } + Group::Backup { + account_id, + table_name, + backup_id, + objects, + } => (account_id, table_name, backup_id, objects), + }; + // The row check, before anything under the prefix is read. + let key = (account_id, backup_id); + if let Some((status, store_id, row_table)) = known.get(&key) { + report.skipped.push(SkippedPrefix { + prefix, + reason: SkipReason::HasRow { + status: *status, + store_id: store_id.clone(), + other_table: (*row_table != table_name).then(|| row_table.clone()), + }, + }); + return None; + } + let outcome = adopt_prefix( + storage, + calls, + target, + &prefix, + (&key.0, &table_name, &key.1), + &objects, + ) + .await; + match outcome { + Ok(adopted) => { + known.insert( + key, + ( + BackupStatus::Available, + target.identity.store_id.clone(), + table_name, + ), + ); + report.adopted.push(adopted); + } + Err(reason) => report.skipped.push(SkippedPrefix { prefix, reason }), + } + None +} + +/// Verify one prefix with no row and insert its row. +async fn adopt_prefix( + storage: &S, + calls: StoreCalls<'_>, + target: SyncTarget<'_>, + prefix: &str, + (account_id, table_name, backup_id): (&str, &str, &str), + objects: &HashMap, +) -> Result { + let manifest = match verify_prefix(calls, prefix, Some(objects)).await { + Ok(Verified::Complete(manifest)) => manifest, + Ok(Verified::Incomplete(why)) => return Err(SkipReason::Incomplete(why)), + Err(e) => return Err(SkipReason::Store(e.to_string())), + }; + let arn = manifest + .backup_arn() + .map_err(|e| SkipReason::ArnMismatch(format!("the manifest's ARN is refused: {e}")))?; + if arn.account_id() != account_id + || arn.table_name() != table_name + || arn.backup_id() != backup_id + { + tracing::warn!( + "backup sync: {prefix} holds a manifest naming {}, whose account, table and backup \ + id are not the prefix's; not adopted", + arn.to_arn_string() + ); + return Err(SkipReason::ArnMismatch(format!( + "the manifest names {}, whose account, table and backup id are not the prefix's", + arn.to_arn_string() + ))); + } + let backup_arn = arn.to_arn_string(); + if arn.partition() != ARN_PARTITION { + tracing::warn!( + "backup sync: {prefix} holds a manifest naming {backup_arn}, in partition {}; not \ + adopted, since every request refuses an ARN outside partition {ARN_PARTITION}", + arn.partition() + ); + return Err(SkipReason::OtherPartition(format!( + "the manifest names {backup_arn}, in partition {}, and no request to this server \ + can address an ARN outside partition {ARN_PARTITION}", + arn.partition() + ))); + } + let created_at = backup_creation_date_time(backup_id).ok_or_else(|| { + SkipReason::NotABackup(format!("backup id {backup_id:?} carries no instant")) + })?; + // A row under the other spelling of the ARN: on Cassandra the insert + // decides per spelling only (the trait's doc). A row under the + // manifest's own spelling is left to the insert, which answers `false` + // for it, or, on Cassandra, completes it when it is an earlier insert + // of this same row that stopped before its list and status entries. + match row_under_a_spelling(storage, &arn, target.region).await { + Ok(Some((spelling, row))) if spelling != arn => { + return Err(SkipReason::RowExists( + describe_row(storage, target, &row).await, + )); + } + Ok(_) => {} + Err(e) => return Err(SkipReason::Catalog(e.to_string())), + } + let source_table_details_json = serde_json::to_value(&manifest.source_table) + .map_err(|e| SkipReason::Incomplete(format!("the source table does not serialize: {e}")))?; + let row = BackupRow { + backup_id: backup_id.to_owned(), + account_id: account_id.to_owned(), + table_name: table_name.to_owned(), + backup_arn: backup_arn.clone(), + backup_name: manifest.backup_name.clone(), + status: BackupStatus::Available, + store: target.identity.clone(), + size_bytes: manifest.backup_size_bytes, + item_count: manifest_item_count(&manifest), + created_at, + completed_at: None, + failure_reason: None, + source_table_details_json, + owner_generation: UNOWNED_GENERATION, + heartbeat_at: 0.0, + }; + match storage.insert_backup_row(&row).await { + Ok(true) => {} + Ok(false) => { + return Err(SkipReason::RowExists( + existing_row(storage, target, &arn).await, + )); + } + Err(e) => return Err(SkipReason::Catalog(e.to_string())), + } + let foreign = foreign_note(&arn, target.region); + match &foreign { + Some(note) => tracing::warn!( + "backup sync: adopted {backup_arn} (account {account_id}), {note}; requests find it \ + by its ARN as written" + ), + None => tracing::info!("backup sync: adopted {backup_arn} (account {account_id})"), + } + Ok(AdoptedBackup { + backup_arn, + foreign, + }) +} + +/// The row under `arn` rebuilt in `region` (the spelling every request tries +/// first) or under `arn` as the manifest writes it, in that order, with the +/// spelling it was found under. +async fn row_under_a_spelling( + storage: &S, + arn: &BackupArn, + region: Option<&str>, +) -> Result, StorageError> { + let written = arn.to_arn_string(); + if let Some(region) = region + && let Ok(parts) = extenddb_storage::util::check_backup_arn(&written) + { + let spellings = super::BackupArnSpellings::new(&parts, region); + return super::read_catalog_row(storage, &spellings).await; + } + Ok(storage + .read_backup_row(arn) + .await? + .map(|row| (arn.clone(), row))) +} + +/// The sentence for an ARN in another region than the server's, or `None`. +fn foreign_note(arn: &BackupArn, region: Option<&str>) -> Option { + region + .filter(|region| arn.region() != *region) + .map(|_| format!("whose region {} is not this server's", arn.region())) +} + +/// What holds the backup id when the insert answered `false`: re-read so +/// the report says which row it met. +async fn existing_row( + storage: &S, + target: SyncTarget<'_>, + arn: &BackupArn, +) -> String { + match storage.read_backup_row(arn).await { + Ok(Some(row)) => describe_row(storage, target, &row).await, + Ok(None) => format!( + "a row with backup id {} exists under another account, another table or another \ + spelling of the ARN, or an earlier release's in-catalog backup holds this ARN; it \ + is left as it is", + arn.backup_id() + ), + Err(e) => format!("a row exists for it (its re-read failed: {e}); it is left as it is"), + } +} + +/// The report's sentence for a row the row check did not see. +async fn describe_row( + storage: &S, + target: SyncTarget<'_>, + row: &BackupRow, +) -> String { + if row.store != *target.identity { + return format!( + "a row of another store holds it ({}; {} {}, store id {}); it is left as it is", + row.status, row.store.store_kind, row.store.store_root, row.store.store_id + ); + } + let listed = storage + .list_backup_rows(Some(row.status), target.identity) + .try_any(|listed| std::future::ready(listed.backup_arn == row.backup_arn)) + .await; + match listed { + Ok(true) => format!( + "a row for it ({}) landed after the catalog was read (a CreateBackup, say); it is \ + left as it is", + row.status + ), + Ok(false) => format!( + "the catalog holds a row for it ({}) that the catalog's listings do not return (a \ + write that ended before the row's list entries, such as a CreateBackup's claim); \ + it is not a row this sync's insert writes, so it is left as it is", + row.status + ), + Err(e) => format!( + "a row for it ({}) exists that the catalog's read did not return (its listing \ + failed: {e}); it is left as it is", + row.status + ), + } +} + +/// What [`prune_missing`] did with one row. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum PruneOutcome { + /// The row is gone; nothing in the store was deleted. + Pruned, + /// The manifest is under the prefix again; the row was left. + ManifestBack, + /// A restore target registered from the backup is CREATING (its name); + /// the row was left. + InUse(String), + /// The row was not AVAILABLE any more (its status); left as it is. + Status(BackupStatus), + /// The row was gone. + Gone, + /// A store or catalog call failed (why); the row was left. + Failed(String), +} + +impl std::fmt::Display for PruneOutcome { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Pruned => f.write_str("removed: the row is gone, and no object was deleted"), + Self::ManifestBack => f.write_str("left: the manifest is under its prefix again"), + Self::InUse(table) => write!( + f, + "left: the restore target {table} is being restored from it" + ), + Self::Status(status) => write!(f, "left: the row is {status} now"), + Self::Gone => f.write_str("left: the row is gone"), + Self::Failed(why) => write!(f, "left: {why}"), + } + } +} + +impl PruneOutcome { + /// Whether the row is still AVAILABLE with its manifest gone and a later + /// run may remove it: a store or catalog call failed + /// ([`Self::Failed`]), or a restore is reading the backup + /// ([`Self::InUse`]). The command exits 1 when any row ends so. + /// [`Self::ManifestBack`], [`Self::Status`] and [`Self::Gone`] leave + /// nothing for a later run, and the command exits 0 for them. + #[must_use] + pub fn needs_another_run(&self) -> bool { + matches!(self, Self::InUse(_) | Self::Failed(_)) + } +} + +/// `--prune-missing`: for each AVAILABLE row whose manifest the listing did +/// not hold, `head` the manifest once more, then remove the row outright +/// ([`BackupCatalog::forget_available_backup_row`], which refuses a backup a +/// restore is reading and a row another writer moved). A `head` that fails +/// leaves the row ([`PruneOutcome::Failed`]): the store has not said whether +/// the manifest is back, so the row is kept and a later run decides. +/// Nothing in the store is deleted (module doc). The caller holds the +/// maintenance claim; `stop` +/// is asked before each row and returns why to stop (the claim lost, a +/// signal), which ends the loop with the reason. Each removal is logged +/// under the `extenddb::audit` target, with `store_root` the configured +/// store's root. +pub async fn prune_missing( + storage: &S, + store: &dyn BackupStore, + store_root: &str, + rows: &[BackupKey], + stop: &dyn Fn() -> Option, +) -> (Vec<(String, PruneOutcome)>, Option) { + let mut done = Vec::new(); + for row in rows { + if let Some(reason) = stop() { + return (done, Some(reason)); + } + let outcome = prune_one(storage, store, row).await; + if outcome == PruneOutcome::Pruned { + tracing::info!( + target: "extenddb::audit", + action = "backup sync --prune-missing", + account_id = %row.account_id, + backup_arn = %row.backup_arn, + store_root = %store_root, + "removed an AVAILABLE row whose manifest is not under the root; no object was \ + deleted" + ); + } + done.push((row.backup_arn.clone(), outcome)); + } + (done, None) +} + +async fn prune_one( + storage: &S, + store: &dyn BackupStore, + row: &BackupKey, +) -> PruneOutcome { + let arn = match BackupArn::parse(&row.backup_arn) { + Ok(arn) => arn, + Err(e) => return PruneOutcome::Failed(format!("the row's ARN does not parse: {e}")), + }; + match store + .head(&format!("{}{EXTENDDB_MANIFEST_FILE}", arn.store_prefix())) + .await + { + Ok(Some(_)) => return PruneOutcome::ManifestBack, + Ok(None) => {} + Err(e) => return PruneOutcome::Failed(format!("the manifest's head failed: {e}")), + } + match storage.forget_available_backup_row(row).await { + Ok(ForgetRow::Removed) => PruneOutcome::Pruned, + Ok(ForgetRow::InUse(table)) => PruneOutcome::InUse(table), + Ok(ForgetRow::Status(status)) => PruneOutcome::Status(status), + Err(e) if is_backup_not_found(&e) => PruneOutcome::Gone, + Err(e) => PruneOutcome::Failed(format!("the row could not be removed: {e}")), + } +} + +/// Hands a taken startup sync back when the run ends without listing the +/// whole root, or with a prefix a later run may adopt; dropped with its +/// future included. +struct Rearm<'a> { + store: &'a super::ConfiguredStore, + done: bool, +} + +impl Drop for Rearm<'_> { + fn drop(&mut self) { + if !self.done { + self.store.retry_startup_sync(); + } + } +} + +/// The startup sync of a server whose catalog is an in-memory SQLite +/// database (module doc): once, when the configured store is healthy and +/// the sync is pending ([`super::ConfiguredStore::with_ephemeral_catalog`]), +/// bounded by [`STARTUP_SYNC_TIMEOUT`]. `None` when nothing ran (no store, +/// not pending, not healthy, or another caller is running it) and when a +/// run reached the bound (logged; the sync stays pending). A run that +/// stops before it listed the whole root (a listing or catalog failure, the +/// bound, or its future dropped at a shutdown), or that skipped a prefix for +/// a store or catalog error, leaves the sync pending for the next +/// reconciler cycle. The server calls it before it serves (the filesystem +/// store, whose check ran then) and every reconciler cycle calls it after +/// the store's check (the S3 store, whose first check runs there, and a sync +/// left pending). +pub async fn startup_sync( + storage: &S, + runtime: &BackupRuntime, +) -> Option> { + let store = runtime.configured_store()?; + if !store.startup_sync_pending() { + return None; + } + let identity = store.identity().filter(|_| store.is_healthy())?; + if !store.take_startup_sync() { + return None; + } + let mut rearm = Rearm { store, done: false }; + let run = sync( + storage, + SyncTarget { + store: &*store.store, + identity: &identity, + metrics: runtime.metrics().map(|m| &**m), + region: runtime.region(), + stop: None, + }, + ); + let Ok(outcome) = tokio::time::timeout(STARTUP_SYNC_TIMEOUT, run).await else { + tracing::warn!( + "backup store ({}) startup sync of the in-memory catalog did not finish within {} \ + s; the backups it adopted keep their rows, and the next reconciler cycle runs it \ + again from the first key of the root", + store.kind, + STARTUP_SYNC_TIMEOUT.as_secs() + ); + return None; + }; + match &outcome { + Ok(report) => { + let transient = report.transient_skips(); + rearm.done = transient == 0; + let mut reserved = 0usize; + for skipped in &report.skipped { + if matches!(skipped.reason, SkipReason::Reserved(_)) { + reserved += 1; + } else if skipped.reason.is_transient() { + tracing::warn!( + "backup store startup sync: {} not adopted: {}; the next reconciler \ + cycle tries it again", + skipped.prefix, + skipped.reason + ); + } else { + tracing::info!( + "backup store startup sync: {} not adopted: {}", + skipped.prefix, + skipped.reason + ); + } + } + tracing::info!( + "backup store ({}) startup sync of the in-memory catalog: {} backup(s) adopted, \ + {} prefix(es) skipped ({reserved} of them reserved for exports or the \ + change-log archive){}", + store.kind, + report.adopted.len(), + report.skipped.len(), + if transient == 0 { + String::new() + } else { + format!( + "; {transient} prefix(es) met a store or catalog error, and the next \ + reconciler cycle runs the sync again" + ) + } + ); + } + Err(e) => tracing::warn!( + "backup store ({}) startup sync of the in-memory catalog stopped: {e}; the next \ + reconciler cycle runs it again", + store.kind + ), + } + Some(outcome) +} diff --git a/crates/engine/src/backup/tests.rs b/crates/engine/src/backup/tests.rs index dd13791e7..077dc5d96 100644 --- a/crates/engine/src/backup/tests.rs +++ b/crates/engine/src/backup/tests.rs @@ -1,24 +1,40 @@ // Copyright 2026 ExtendDB contributors // SPDX-License-Identifier: Apache-2.0 -//! Engine backup rules, against a `BackupEngine` double that records what the -//! engine asked of it. +//! Engine backup rules, against a double of the catalog index and the legacy +//! reader that records what the engine asked of it. use std::collections::HashMap; -use std::sync::Mutex; +use std::sync::{Arc, Mutex}; +use std::time::Duration; +use extenddb_core::backup_format::BackupArn; use extenddb_core::error::DynamoDbError; use extenddb_core::limits::LimitsConfig; use extenddb_core::types::{ AttributeDefinition, BackupDescription, BackupDetails, BackupSummary, BillingMode, - BillingModeSummary, GsiDescription, GsiInput, KeySchemaElement, KeyType, LsiInput, - OnDemandThroughput, Projection, ProjectionType, ProvisionedThroughput, - ProvisionedThroughputDescription, ScalarAttributeType, SourceTableDetails, StreamSpecification, - StreamViewType, TableDescription, TableStatus, TimeToLiveDescription, TimeToLiveStatus, + BillingModeSummary, CreateTableInput, DeleteTableInput, DescribeTableInput, GsiDescription, + GsiInput, IndexInfo, Item, KeySchemaElement, KeyType, ListTablesInput, ListTablesOutput, + LsiInput, OnDemandThroughput, Projection, ProjectionType, ProvisionedThroughput, + ProvisionedThroughputDescription, RestoreSummary, ScalarAttributeType, SourceTableDetails, + StreamSpecification, StreamViewType, TableDescription, TableKeyInfo, TableStatus, + TimeToLiveDescription, TimeToLiveStatus, UpdateTableInput, +}; +use extenddb_storage::backup_catalog::{ + AbandonedLegacyBackup, BackupKey, BackupRow, BackupStatus, DeleteStart, ForgetRow, Heartbeat, + JobClaim, JobRow, JobStatus, RestoreJobRow, RestoreKind, RestoreRegistrationRow, + RestoreTargetKey, StaleClock, StatusFields, StoreIdentity, StoreKind, TimeBounds, + UNOWNED_GENERATION, WriteOutcome, new_generation, }; use extenddb_storage::backup_definition::{BACKUP_DEFINITION_VERSION, BackupTableDefinition}; use extenddb_storage::error::StorageError; -use extenddb_storage::{BackupEngine, BackupRestoreSource, BoxedFuture, RestoreRequest}; +use extenddb_storage::{ + BackupCatalog, BackupRestoreSource, BoxedFuture, BulkLoadFile, DataEngine, Finished, JobPermit, + LegacyBackupReader, MetadataEngine, RestoreEngine, RestoreRequest, SnapshotEngine, TableEngine, + TableSnapshot, +}; +use futures::stream::BoxStream; +use futures::{StreamExt, TryStreamExt}; use serde_json::{Value, json}; use super::*; @@ -39,3247 +55,22248 @@ fn source_arn() -> String { #[derive(Default)] struct Fake { + /// Cancelled by `list_dead_restore_jobs`: a token that fires in the + /// middle of a cycle, after its store check. + cancel_on_list_dead: Mutex>, sources: Mutex>, descriptions: Mutex>, summaries: Mutex>, restore_error: Mutex>, - /// Returned (once) by `backup_restore_source` instead of the stored source. + /// Returned (once) by `restore_source` instead of the stored source. source_error: Mutex>, restores: Mutex>, + /// The permits the legacy restore and create were handed, held as the + /// backend would hold them for the job's life. + permits: Mutex>, + creates: Mutex>, source_reads: Mutex, listed_for: Mutex>>, - /// When set, `backup_restore_source` waits for a permit before answering. + /// When set, `restore_source` waits for a permit before answering. source_gate: Option>, + /// The catalog index: rows by ARN, shared with the claims handed out. + rows: Arc>>, + /// `backup_in_use` answers: the CREATING target's name, by backup ARN. + in_use: Mutex>, + /// What `start_backup_delete` answers after the row check, by ARN. + delete_starts: Mutex>, + delete_starts_seen: Mutex>, + row_reads: Mutex>, + dead_jobs: Mutex>, + failed_jobs: Mutex>, + /// A heartbeat that lands on the record between the listing and the + /// FAILED write: the `heartbeat_at` the record carries when the write + /// arrives. + heartbeat_after_listing: Mutex>, + removed_targets: Mutex>, + /// Whether `remove_restore_target` answers `true` (the target was a + /// CREATING restore target with the key's id). + target_still_creating: Mutex, + /// Whether `fail_restore_job` answers `true` (the record was RUNNING); + /// separate from `target_still_creating` so the FAILED write matching + /// while the removal finds the target moved is a reachable state. + record_still_running: Mutex, + abandoned_backups: Mutex>, + deleted: Mutex>, + maintenance_held: Mutex, + /// The windows `check_maintenance_claim` was called with: one entry per + /// reconciler cycle, so a test can see that a cycle ran. + maintenance_checks: Mutex>, + /// When set, `check_maintenance_claim` fails with it (every call). + maintenance_error: Mutex>, + /// When set, the age of a live maintenance claim's last heartbeat, and + /// `check_maintenance_claim(stale_after)` answers held while the age is + /// under `stale_after`, the rule the backends apply; `maintenance_held` + /// answers otherwise. + maintenance_beat_age: Mutex>, + /// How far the catalog's clock runs ahead of this process's (negative: + /// behind), in seconds; `list_backup_rows_older_than` measures ages on + /// the catalog's clock. + catalog_clock_offset_secs: Mutex, + /// When set, `check_maintenance_claim` takes this long to answer (a + /// catalog query held behind the pool). + maintenance_delay: Mutex>, + /// The `TimeBounds` `list_account_backup_rows` received. + bounds_seen: Mutex>, + /// The account's tables, by name, as `describe_table` reports them; a + /// name absent is `TableNotFound`. + tables: Mutex>, + /// The names `describe_table` was asked for. + table_reads: Mutex>, + /// What `catalog_id` answers; `catalog-1` unless a test changes it. + catalog_id: Mutex>, + /// When set, `catalog_id` fails as a catalog that cannot be read. + catalog_id_error: Mutex, + /// Rows `set_backup_deleted` moved to DELETED, by ARN. + set_deleted: Mutex>, + /// Rows `remove_backup_row` removed, by ARN. + removed_rows: Mutex>, + /// Every `remove_backup_row` call, by ARN, refused ones included. + remove_attempts: Mutex>, + /// When set, `list_backup_rows` fails with it. + list_error: Mutex>, + /// The stores `newest_available_backup_row` was asked for. + newest_reads: Mutex>, + /// RUNNING restore records with a live heartbeat (a fresh one, say), + /// which `list_running_restore_jobs` lists and `list_dead_restore_jobs` + /// does not. + running_jobs: Mutex>, + /// The table snapshots `open_table_snapshot` serves, by table name; a + /// name absent is `TableNotFound`. + snapshots: Mutex>, + /// When set, `open_table_snapshot` fails with it (once). + snapshot_error: Mutex>, + /// Snapshots opened, by table name. + snapshots_opened: Mutex>, + /// What `describe_ttl` answers; disabled when unset. + ttl: Mutex>, + /// The rows `claim_backup` received, as given. + claims: Mutex>, + /// The ARNs `claim_dead_backup` hands over (the backend's liveness test + /// modelled as a set): a CREATING row named here has a dead owner. + dead: Mutex>, + /// The ARNs `claim_dead_backup` took over. + takeovers: Mutex>, + /// Every `set_status` a claim received, in order. + status_writes: Arc>>, + /// What a claim's `heartbeat` does. + heartbeat_answer: Arc>, + /// Releases a beat parked under `HeartbeatAnswer::Park`. + park_release: Arc, + /// What the fake's claims declare about their beat + /// (`JobClaim::heartbeat_parks_before_its_bound`): `true` models SQLite, + /// `false` (the default) the other backends. + parks_before_bound: Arc, + /// Heartbeats the claims received. + heartbeats: Arc, + /// When set, the next `set_status` reports `Unknown` (applied or not as + /// the flag says) instead of its outcome; taken by that write unless + /// `unknown_write_sticky` is set (a connection that fails on every + /// write). + unknown_write: Arc>>, + unknown_write_sticky: Arc, + /// When set, moved into `unknown_write` once a (non-sticky) unknown + /// write has been taken: the write sent after an unknown outcome is + /// itself unknown, in this way. + unknown_write_then: Arc>>, + /// When set, every `set_status` fails as a claim connection that was + /// reset. + status_write_error: Arc, + /// When set, `claim_dead_backup` hands the claim over and then gives + /// the row a newer generation: another reconciler took the row over in + /// the gap before this one's write (item 31's two-reconciler race). + takeover_lost_to_another: std::sync::atomic::AtomicBool, + /// What `sweep_restore_registrations` answers. + swept_registrations: Mutex>, + /// The store-backed restore records, by target table id: what + /// `claim_restore` writes and the restore's claims, `read_restore_job`, + /// `list_restore_jobs` and the reconciler read. + restore_jobs: Arc>>, + /// The restore targets' data, by table id: the key schema the duplicate + /// rule reads keys with, the items `bulk_load` applied, and the + /// statuses `finish_restore` flips. + restore_tables: Mutex>, + /// Every `claim_restore` call: the account and the input. + restore_claims: Mutex>, + /// When set, the next `claim_restore` fails with it instead of + /// registering (a name race the backend lost, a backup deleted under + /// the lock). + claim_restore_error: Mutex>, + /// Every `set_status` a restore claim received, in order. + restore_writes: Arc>>, + /// The table ids `claim_restore_dead` hands over (the backend's liveness + /// test modelled as a set). + restore_dead: Mutex>, + /// Every `claim_restore_dead` that handed a claim over: the table id and + /// the `spool_host` it was given. + restore_takeovers: Mutex)>>, + /// Every `bulk_load`: the file, and the items the stream yielded before + /// it ended or failed. + bulk_loads: Mutex>, + /// When set, every `bulk_load` waits for one permit before it reads its + /// stream, so a test can hold the job between files. + bulk_load_gate: Mutex>>, + /// When set, `bulk_load` takes one permit before each item it pulls + /// from its stream, so a test can hold the job at a known item of a + /// file; `bulk_load_items_seen` counts the items pulled so far. + bulk_load_item_gate: Mutex>>, + bulk_load_items_seen: std::sync::atomic::AtomicUsize, + /// When set, `count_restore_target` answers this instead of the table's + /// item count (a backend that kept one of two rows with one key). + count_override: Mutex>, + /// When set, `count_restore_target` waits for one permit before it + /// counts, so a test can hold the job after its last `bulk_load`. + count_gate: Mutex>>, + /// `count_restore_target` calls, counted before the gate. + count_calls: std::sync::atomic::AtomicUsize, + /// When set, `claim_restore` takes this long before it registers (a + /// registration held behind the catalog's lock or pool). + claim_restore_delay: Mutex>, + /// What `finish_restore` answers when its conditions hold: ACTIVE, or + /// DONE with the backfill pending (MongoDB). + finish_pending: std::sync::atomic::AtomicBool, + /// The progress rows a backend keeps apart from the record + /// (PostgreSQL's `restore_progress`), by table id. + progress_rows: Mutex>>, + /// The `running` lists `sweep_restore_progress` received. + progress_sweeps: Mutex>>, + /// Whether `as_vector_search` answers an engine: a target with vector + /// support. + vector_search: std::sync::atomic::AtomicBool, + /// Every `list_restore_jobs` call's filter. + restore_lists: Mutex>>, + /// When set, `list_restore_jobs` fails as a backend that has no restore + /// records yet (its `Unsupported` stub). + restore_list_error: std::sync::atomic::AtomicBool, + /// `bulk_load` calls begun, counted before the gate, so a test can tell + /// a job waiting at the gate. + bulk_loads_started: std::sync::atomic::AtomicUsize, + /// When set, run inside `list_restore_jobs` after it builds its answer, + /// that is, after the reconciler read the RUNNING records and before its + /// spool listing uses them: the moment a registration can land between + /// the sweep's two reads. + sweep_hook: Mutex>, + /// What the next `finish_restore` calls do instead of answering, in + /// order: fail after the flip landed, or fail without flipping. + finish_errors: Mutex>, + /// Every `finish_restore` call's `size_bytes`. + finish_sizes: Mutex>, + /// When set, `read_restore_job` takes this long to answer (a catalog + /// read held behind the pool). + read_restore_job_delay: Mutex>, + /// When set, `claim_restore_dead` hands the claim over and then removes + /// the record (gone between the takeover and the reads that follow). + takeover_removes_record: std::sync::atomic::AtomicBool, + /// The maintenance claims `maintenance_claim` handed out, as the test's + /// handles on them; `None` in the error slot makes the next call fail + /// as a claim a live holder has. + maintenance_holders: Mutex>>, + maintenance_claim_held_by: Arc>>, + /// When set, `claim_restore` stamps the record's `heartbeat_at` this + /// many seconds in the past: a registration that stalled on a lock, as + /// PostgreSQL's transaction-start `now()` stamps it. + registration_heartbeat_age: Mutex>, + /// Table ids whose data tables never came up: `restore_target_ready` + /// answers `false` for them, the state a crash between the + /// registration's commit and the data DDL leaves. + targets_without_tables: Mutex>, + /// When set, the next `maintenance_claim` hands out a holder whose + /// heartbeats report the claim lost at once: the HeldClaim's timer sees + /// it and `lost()` reports it before the removal. + next_claim_lost: Mutex, + /// When set, `list_restore_jobs` sleeps this long before it answers, so a + /// test can let the maintenance claim's heartbeat timer observe a lost + /// claim during the read that precedes the removal. + list_jobs_delay: Mutex>, + /// How many `count_restore_target` calls still fail with an error (a + /// `COUNT(*)` that times out), taken one per call. + count_errors: std::sync::atomic::AtomicU32, + /// When set, run inside `claim_restore_dead` once it has handed a claim + /// over, before the reconciler's second read of the maintenance claim: + /// the moment a command can take that claim during the cycle. + takeover_hook: Mutex>, + /// When set, `remove_restore_target_exclusive` answers `false` without + /// removing anything: the owner's lock (PostgreSQL's per-session + /// advisory lock) is still held by a frozen owner's session. + owner_lock_held: Mutex, + /// Rows `insert_backup_row` lands just before it decides, by ARN: a + /// CreateBackup on another instance whose row lands between sync's read + /// of the rows and its insert. + lands_before_insert: Mutex>, + /// ARNs `list_backup_rows` does not return though the row exists: a + /// Cassandra row whose list and status entries were never written. + unlisted: Mutex>, + /// When set, `insert_backup_row` that meets an unowned row under the + /// same ARN equal to the given one in every column the insert takes + /// answers `true` and lists it again (`unlisted` loses it): Cassandra's + /// completion of an insert that stopped before its entries. + completes_own_insert: std::sync::atomic::AtomicBool, + /// Every row `insert_backup_row` received, inserted or not. + insert_attempts: Mutex>, + /// When set, `insert_backup_row` fails with this error and writes + /// nothing. + insert_error: Mutex>, + /// Every key `forget_available_backup_row` received. + forgets_seen: Mutex>, + /// Preset answers of `forget_available_backup_row`, by ARN; the row is + /// left as the test set it. + forget_answers: Mutex>, + /// When set, `read_backup_row` fails with this error (every call). + read_row_error: Mutex>, + /// When set, rows are unique by ARN alone, as on Cassandra: + /// `insert_backup_row` answers `false` only for a row under the same + /// ARN, and `read_backup_row` finds a row by its ARN; otherwise by + /// backup id, as on PostgreSQL, SQLite and MongoDB. + unique_by_arn: std::sync::atomic::AtomicBool, } -impl Fake { - fn with_source(source: BackupRestoreSource) -> Self { - let fake = Self::default(); - fake.sources - .lock() - .unwrap() - .insert(source.backup_arn.clone(), source); - fake - } - fn restores(&self) -> Vec { - self.restores.lock().unwrap().clone() - } - fn source_reads(&self) -> usize { - *self.source_reads.lock().unwrap() - } +/// A closure the `Fake` runs inside `sweep_restore_progress`. +type SweepHook = Box; + +/// What a `finish_restore` of the `Fake` does instead of answering. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum FinishError { + /// The flip landed (the record DONE, the table ACTIVE) and the call + /// then failed: PostgreSQL's progress delete after the commit. + AfterFlip, + /// The call failed before anything was written. + BeforeFlip, + /// The call failed before anything was written, and another claim took + /// the record over and wrote DONE under its own generation before the + /// re-read. + TakenAndDone, } -fn unused() -> BoxedFuture<'static, Result> { - Box::pin(async { Err(StorageError::Internal("not used by these tests".to_owned())) }) +/// A restore target's data in the `Fake`. +#[derive(Debug, Clone)] +struct RestoreTable { + name: String, + key_attributes: Vec, + items: Vec, + active: bool, } -impl BackupEngine for Fake { - fn create_backup( - &self, - _: &str, - _: &str, - _: &str, - ) -> BoxedFuture<'_, Result> { - unused() - } +/// One `bulk_load` the `Fake` received. +#[derive(Debug, Clone, PartialEq)] +struct BulkLoadCall { + table_id: String, + file: BulkLoadFile, + /// The items the stream yielded. + items: Vec, + /// The text of the `Err` item that ended the stream, if one did. + stream_error: Option, +} - fn describe_backup( - &self, - _account_id: &str, - backup_arn: &str, - ) -> BoxedFuture<'_, Result> { - let found = self.descriptions.lock().unwrap().get(backup_arn).cloned(); - let backup_arn = backup_arn.to_owned(); - Box::pin(async move { - found.ok_or_else(|| StorageError::Validation(format!("Backup not found: {backup_arn}"))) - }) - } +/// A `Fake` claim on one restore record, by table id: the backends' +/// `heartbeat` (`status = RUNNING AND owner_generation = ?`), `set_status` +/// conditioned on the generation and `from`, and `read`. +struct FakeRestoreClaim { + records: Arc>>, + table_id: String, + generation: u64, + writes: Arc>>, + heartbeat_answer: Arc>, + heartbeats: Arc, + unknown_write: Arc>>, + status_write_error: Arc, +} - fn list_backups( - &self, - _account_id: &str, - table_name: Option<&str>, - ) -> BoxedFuture<'_, Result, StorageError>> { - self.listed_for - .lock() - .unwrap() - .push(table_name.map(str::to_owned)); - let all = self.summaries.lock().unwrap().clone(); - Box::pin(async move { Ok(all) }) +impl JobClaim for FakeRestoreClaim { + fn generation(&self) -> u64 { + self.generation } - /// Removes the backup and answers its description as the backends do: - /// the row's status (not yet `DELETED`), `SourceTableDetails` without - /// `ProvisionedThroughput`, no feature details. - fn delete_backup( - &self, - _account_id: &str, - backup_arn: &str, - ) -> BoxedFuture<'_, Result> { - let found = self.descriptions.lock().unwrap().remove(backup_arn); - self.sources.lock().unwrap().remove(backup_arn); - let backup_arn = backup_arn.to_owned(); + fn heartbeat(&self) -> BoxedFuture<'_, Result> { + self.heartbeats + .fetch_add(1, std::sync::atomic::Ordering::AcqRel); + let answer = *self.heartbeat_answer.lock().unwrap(); + let confirmed = match self.records.lock().unwrap().get_mut(&self.table_id) { + Some(record) + if answer == HeartbeatAnswer::Confirm + && record.owner_generation == self.generation + && record.status == JobStatus::Running => + { + record.heartbeat_at = seconds_since_epoch(); + true + } + _ => false, + }; Box::pin(async move { - found.ok_or_else(|| StorageError::Validation(format!("Backup not found: {backup_arn}"))) + match answer { + HeartbeatAnswer::Confirm if confirmed => Ok(Heartbeat::Confirmed), + HeartbeatAnswer::AlwaysConfirm => Ok(Heartbeat::Confirmed), + HeartbeatAnswer::Confirm | HeartbeatAnswer::Lost => Ok(Heartbeat::Lost), + HeartbeatAnswer::Stall | HeartbeatAnswer::Park => futures::future::pending().await, + HeartbeatAnswer::Error => Err(StorageError::Internal( + "the claim connection failed".to_owned(), + )), + } }) } - fn backup_restore_source( - &self, - _account_id: &str, - backup_arn: &str, - ) -> BoxedFuture<'_, Result> { - *self.source_reads.lock().unwrap() += 1; - let err = self.source_error.lock().unwrap().take(); - let found = self.sources.lock().unwrap().get(backup_arn).cloned(); - let backup_arn = backup_arn.to_owned(); - let gate = self.source_gate.clone(); - Box::pin(async move { - if let Some(gate) = gate { - let _permit = gate.acquire().await.expect("gate closed"); - } - if let Some(e) = err { - return Err(e); + fn set_status<'a>( + &'a self, + from: JobStatus, + to: JobStatus, + fields: &'a StatusFields, + ) -> BoxedFuture<'a, Result> { + self.writes.lock().unwrap().push(StatusWrite { + arn: self.table_id.clone(), + generation: self.generation, + from, + to, + fields: fields.clone(), + }); + if self + .status_write_error + .load(std::sync::atomic::Ordering::Acquire) + { + return Box::pin(async move { + Err(StorageError::Internal( + "claim session: connection reset".to_owned(), + )) + }); + } + let unknown = self.unknown_write.lock().unwrap().take(); + let apply = unknown != Some(UnknownWrite::Lost); + let mut outcome = WriteOutcome::NotMatched; + if let Some(record) = self.records.lock().unwrap().get_mut(&self.table_id) + && record.owner_generation == self.generation + && record.status == from + { + if unknown == Some(UnknownWrite::TakenAndWritten) { + record.owner_generation = new_generation(); + record.status = to; + } else if apply { + record.status = to; + if let Some(reason) = &fields.failure_reason { + record.failure_reason = Some(reason.clone()); + } + if fields.completed { + record.completed_at = Some(seconds_since_epoch()); + } + outcome = WriteOutcome::Matched; } - found.ok_or_else(|| StorageError::Validation(format!("Backup not found: {backup_arn}"))) - }) + } + if unknown.is_some() { + outcome = WriteOutcome::Unknown; + } + Box::pin(async move { Ok(outcome) }) } - fn restore_table_from_backup( - &self, - _account_id: &str, - request: RestoreRequest, - ) -> BoxedFuture<'_, Result> { - let err = self.restore_error.lock().unwrap().take(); - let desc = described(&request); - self.restores.lock().unwrap().push(request); + fn read(&self) -> BoxedFuture<'_, Result> { + let found = self + .records + .lock() + .unwrap() + .get(&self.table_id) + .map(|record| JobRow { + status: record.status, + owner_generation: record.owner_generation, + heartbeat_at: record.heartbeat_at, + failure_reason: record.failure_reason.clone(), + }); Box::pin(async move { - match err { - Some(e) => Err(e), - None => Ok(desc), - } + found.ok_or_else(|| StorageError::Internal("the record is gone".to_owned())) }) } } -/// What a backend reports for a CREATING target: deliberately the shape the -/// engine must correct (ACTIVE GSI status, a pay-per-request timestamp, a -/// backend-side summary with the wrong time). -fn described(request: &RestoreRequest) -> TableDescription { - let input = &request.input; - TableDescription { - table_name: input.table_name.clone(), - key_schema: input.key_schema.clone(), - attribute_definitions: input.attribute_definitions.clone(), - table_status: TableStatus::Creating, - billing_mode_summary: Some(BillingModeSummary { - billing_mode: input.billing_mode.unwrap_or(BillingMode::Provisioned), - last_update_to_pay_per_request_date_time: Some(1.0), - }), - provisioned_throughput: ProvisionedThroughputDescription { - read_capacity_units: 9, - write_capacity_units: 9, - ..Default::default() - }, - global_secondary_indexes: input.global_secondary_indexes.as_ref().map(|gsis| { - gsis.iter() - .map(|g| GsiDescription { - index_name: g.index_name.clone(), - key_schema: g.key_schema.clone(), - projection: g.projection.clone(), - index_status: "ACTIVE".to_owned(), - provisioned_throughput: Some(ProvisionedThroughputDescription { - read_capacity_units: 9, - write_capacity_units: 9, - ..Default::default() - }), - index_size_bytes: 0, - item_count: 0, - index_arn: String::new(), - }) - .collect() - }), - restore_summary: Some(RestoreSummary { - source_backup_arn: Some("wrong".to_owned()), - source_table_arn: None, - restore_date_time: 42.0, - restore_in_progress: false, - }), - ..Default::default() +impl Fake { + fn restore_claim_on(&self, table_id: &str, generation: u64) -> Box { + Box::new(FakeRestoreClaim { + records: Arc::clone(&self.restore_jobs), + table_id: table_id.to_owned(), + generation, + writes: Arc::clone(&self.restore_writes), + heartbeat_answer: Arc::clone(&self.heartbeat_answer), + heartbeats: Arc::clone(&self.heartbeats), + unknown_write: Arc::clone(&self.unknown_write), + status_write_error: Arc::clone(&self.status_write_error), + }) } -} - -// --- fixtures --------------------------------------------------------------- -fn key(name: &str, key_type: KeyType) -> KeySchemaElement { - KeySchemaElement { - attribute_name: name.to_owned(), - key_type, + /// The key of `item` under `attributes`, for the duplicate rule. + fn item_key(attributes: &[String], item: &Item) -> String { + attributes + .iter() + .map(|a| serde_json::to_string(&item.get(a)).unwrap_or_default()) + .collect::>() + .join("|") } -} -fn attr(name: &str) -> AttributeDefinition { - AttributeDefinition { - attribute_name: name.to_owned(), - attribute_type: ScalarAttributeType::S, + /// The restore record of the target named `table`, if any. + fn restore_job(&self, table: &str) -> Option { + self.restore_jobs + .lock() + .unwrap() + .values() + .find(|r| r.target.table_name == table) + .cloned() } -} -fn throughput(read: i64, write: i64) -> ProvisionedThroughput { - ProvisionedThroughput { - read_capacity_units: read, - write_capacity_units: write, + /// The items a restore target holds, by table name. + fn restored_items(&self, table: &str) -> Vec { + self.restore_tables + .lock() + .unwrap() + .values() + .find(|t| t.name == table) + .map(|t| t.items.clone()) + .unwrap_or_default() } -} -fn gsi(name: &str, pk: &str, pt: Option) -> GsiInput { - GsiInput { - index_name: name.to_owned(), - key_schema: vec![key(pk, KeyType::Hash)], - projection: Projection { - projection_type: ProjectionType::All, - non_key_attributes: None, - }, - provisioned_throughput: pt, + /// The `bulk_load` calls received, as `(file_index, start_at, reapplying, + /// item count)`. + fn bulk_load_files(&self) -> Vec<(u32, u64, bool, usize)> { + self.bulk_loads + .lock() + .unwrap() + .iter() + .map(|c| { + ( + c.file.file_index, + c.file.start_at, + c.file.reapplying, + c.items.len(), + ) + }) + .collect() } } -fn definition(billing: &str) -> BackupTableDefinition { - let provisioned = billing == "PROVISIONED"; - BackupTableDefinition { - version: BACKUP_DEFINITION_VERSION, - billing_mode: billing.to_owned(), - provisioned_throughput: provisioned.then(|| throughput(7, 8)), - global_secondary_indexes: vec![gsi("gsi1", "gpk", provisioned.then(|| throughput(3, 4)))], - local_secondary_indexes: vec![LsiInput { - index_name: "lsi1".to_owned(), - key_schema: vec![key("pk", KeyType::Hash), key("lsk", KeyType::Range)], - projection: Projection { - projection_type: ProjectionType::KeysOnly, - non_key_attributes: None, - }, - }], - table_class: None, - sse_specification: None, - on_demand_throughput: None, - vector_index_names: Vec::new(), - time_to_live: None, - stream_specification: None, - } +/// A table snapshot the `Fake` serves. +struct SnapshotFixture { + description: TableDescription, + key_ordered: bool, + items: Vec>, + /// When set, the stream takes one permit per item before yielding it, + /// so a test can hold the job inside the stream. + gate: Option>, } -fn source(definition: Option) -> BackupRestoreSource { - let billing = definition - .as_ref() - .map_or("PROVISIONED".to_owned(), |d| d.billing_mode.clone()); - BackupRestoreSource { - backup_arn: arn(ACCOUNT), - backup_creation_date_time: CREATED, - source_table_name: "Music".to_owned(), - source_table_arn: source_arn(), - source_table_id: "tid".to_owned(), - key_schema: vec![key("pk", KeyType::Hash), key("sk", KeyType::Range)], - attribute_definitions: if definition.is_some() { - vec![attr("pk"), attr("sk"), attr("gpk"), attr("lsk")] - } else { - vec![attr("pk"), attr("sk")] - }, - provisioned_throughput: definition - .as_ref() - .and_then(|d| d.provisioned_throughput.clone()), - billing_mode: billing, - item_count: 3, - table_size_bytes: 30, - table_creation_date_time: 1.0, - definition, - legacy_vector_index_count: 0, - } +/// One `set_status` a claim received. +#[derive(Debug, Clone)] +struct StatusWrite { + arn: String, + generation: u64, + from: JobStatus, + to: JobStatus, + fields: StatusFields, } -fn restore_body(extra: Value) -> Value { - let mut body = json!({ "TargetTableName": "MusicRestored", "BackupArn": arn(ACCOUNT) }); - for (k, v) in extra.as_object().unwrap() { - body[k] = v.clone(); - } - body +/// What a `Fake` claim's heartbeat does. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +enum HeartbeatAnswer { + /// Confirm while the row carries the claim's generation, whatever its + /// status (the backends' `UPDATE ... WHERE owner_generation = ?` has no + /// status clause, so a job's beats during its own `delete_prefix` + /// refresh the FAILED row), refreshing `heartbeat_at`; `Lost` otherwise. + #[default] + Confirm, + /// Confirm whatever the row carries (a connection whose session still + /// holds its lock while the row was rewritten). + AlwaysConfirm, + /// `Lost`, whatever the row. + Lost, + /// Never return. + Stall, + /// An error. + Error, + /// Park outside the bound, as SQLite's beat does behind the engine + /// write lock, until `Fake::park_release` is notified; then answer as + /// `heartbeat_answer` reads at that moment. + Park, } -async fn restore(fake: &Fake, body: Value) -> Result { - restore_table_from_backup(fake, ACCOUNT, REGION, &LimitsConfig::default(), &body) - .await - .map(|(_, v)| v) +/// How the next `set_status` reports `Unknown`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum UnknownWrite { + /// The write landed and the connection failed after it. + Applied, + /// The write was lost with the connection. + Lost, + /// The write was lost with the connection, and another claim took the + /// record over and wrote the same status under its own generation + /// before the re-read. + TakenAndWritten, } -fn validation_message(err: DynamoDbError) -> String { - match err { - DynamoDbError::ValidationException(m) => m, - other => panic!("expected ValidationException, got {other:?}"), - } +/// A `Fake` claim on one row. +struct FakeClaim { + rows: Arc>>, + arn: String, + generation: u64, + status_writes: Arc>>, + heartbeat_answer: Arc>, + park_release: Arc, + parks_before_bound: Arc, + heartbeats: Arc, + unknown_write: Arc>>, + unknown_write_sticky: Arc, + unknown_write_then: Arc>>, + status_write_error: Arc, } -// --- BackupArn --------------------------------------------------------------- - -/// The ARN the engine looks a `BackupArn` up under first: the one this -/// deployment issues for the backup named. -fn arn_field(body: &Value) -> Result { - backup_arn_field(body, ACCOUNT, REGION).map(|a| a.canonical().to_owned()) +impl Fake { + fn claim_on(&self, arn: &str, generation: u64) -> Box { + Box::new(FakeClaim { + rows: Arc::clone(&self.rows), + arn: arn.to_owned(), + generation, + status_writes: Arc::clone(&self.status_writes), + heartbeat_answer: Arc::clone(&self.heartbeat_answer), + park_release: Arc::clone(&self.park_release), + parks_before_bound: Arc::clone(&self.parks_before_bound), + heartbeats: Arc::clone(&self.heartbeats), + unknown_write: Arc::clone(&self.unknown_write), + unknown_write_sticky: Arc::clone(&self.unknown_write_sticky), + unknown_write_then: Arc::clone(&self.unknown_write_then), + status_write_error: Arc::clone(&self.status_write_error), + }) + } } -/// The spellings of an ARN as `backup_arn_field` resolves them for `arn`. -fn own(arn: &str) -> BackupArnSpellings { - backup_arn_field(&json!({ "BackupArn": arn }), ACCOUNT, REGION).unwrap() +/// `JobStatus` as a backup row carries it. +fn row_status(status: JobStatus) -> Option { + match status { + JobStatus::Creating => Some(BackupStatus::Creating), + JobStatus::Available => Some(BackupStatus::Available), + JobStatus::Failed => Some(BackupStatus::Failed), + JobStatus::Running | JobStatus::Done => None, + } } -fn limits() -> LimitsConfig { - LimitsConfig::default() +/// `BackupStatus` as a job record reports it. +fn job_status(status: BackupStatus) -> JobStatus { + match status { + BackupStatus::Creating => JobStatus::Creating, + BackupStatus::Available => JobStatus::Available, + BackupStatus::Failed => JobStatus::Failed, + BackupStatus::Deleting | BackupStatus::Deleted => JobStatus::Done, + } } -#[test] -fn own_account_arn_is_accepted() { - let body = json!({ "BackupArn": arn(ACCOUNT) }); - assert_eq!(arn_field(&body).unwrap(), arn(ACCOUNT)); -} +impl JobClaim for FakeClaim { + fn generation(&self) -> u64 { + self.generation + } + fn heartbeat(&self) -> BoxedFuture<'_, Result> { + self.heartbeats + .fetch_add(1, std::sync::atomic::Ordering::AcqRel); + let answer = *self.heartbeat_answer.lock().unwrap(); + let confirmed = match self.rows.lock().unwrap().get_mut(&self.arn) { + Some(row) + if answer == HeartbeatAnswer::Confirm + && row.owner_generation == self.generation => + { + row.heartbeat_at = seconds_since_epoch(); + true + } + _ => false, + }; + Box::pin(async move { + match answer { + HeartbeatAnswer::Confirm if confirmed => Ok(Heartbeat::Confirmed), + HeartbeatAnswer::AlwaysConfirm => Ok(Heartbeat::Confirmed), + HeartbeatAnswer::Confirm | HeartbeatAnswer::Lost => Ok(Heartbeat::Lost), + HeartbeatAnswer::Stall => futures::future::pending().await, + HeartbeatAnswer::Error => Err(StorageError::Internal( + "the claim connection failed".to_owned(), + )), + HeartbeatAnswer::Park => futures::future::pending().await, + } + }) + } + + /// SQLite's shape: under `Park` the beat waits outside the bound (the + /// engine write lock's queue) and the bound covers the answer alone. + fn heartbeat_within( + &self, + bound: Duration, + ) -> BoxedFuture<'_, Option>> { + Box::pin(async move { + if *self.heartbeat_answer.lock().unwrap() == HeartbeatAnswer::Park { + self.park_release.notified().await; + } + tokio::time::timeout(bound, self.heartbeat()).await.ok() + }) + } + + /// What the fake declares about its beat: `true` models SQLite (the + /// park is the write lock's queue), `false` the other backends, where a + /// beat still in flight past its bound is one the process was stopped + /// in (`Park` then stands for the stop). + fn heartbeat_parks_before_its_bound(&self) -> bool { + self.parks_before_bound + .load(std::sync::atomic::Ordering::Acquire) + } + + /// The backends' write: conditioned on the claim's generation and on + /// the row being in `from`; `completed` stamps `completed_at`. + fn set_status<'a>( + &'a self, + from: JobStatus, + to: JobStatus, + fields: &'a StatusFields, + ) -> BoxedFuture<'a, Result> { + self.status_writes.lock().unwrap().push(StatusWrite { + arn: self.arn.clone(), + generation: self.generation, + from, + to, + fields: fields.clone(), + }); + if self + .status_write_error + .load(std::sync::atomic::Ordering::Acquire) + { + return Box::pin(async move { + Err(StorageError::Internal( + "claim session: connection reset".to_owned(), + )) + }); + } + let unknown = if self + .unknown_write_sticky + .load(std::sync::atomic::Ordering::Acquire) + { + *self.unknown_write.lock().unwrap() + } else { + let taken = self.unknown_write.lock().unwrap().take(); + if taken.is_some() { + *self.unknown_write.lock().unwrap() = + self.unknown_write_then.lock().unwrap().take(); + } + taken + }; + let apply = unknown != Some(UnknownWrite::Lost); + let mut outcome = WriteOutcome::NotMatched; + if apply + && let Some(row) = self.rows.lock().unwrap().get_mut(&self.arn) + && row.owner_generation == self.generation + && Some(row.status) == row_status(from) + && let Some(to_status) = row_status(to) + { + row.status = to_status; + if let Some(size) = fields.size_bytes { + row.size_bytes = size; + } + if let Some(count) = fields.item_count { + row.item_count = count; + } + if let Some(reason) = &fields.failure_reason { + row.failure_reason = Some(reason.clone()); + } + if let Some(details) = &fields.source_table_details_json { + row.source_table_details_json = details.clone(); + } + if fields.completed { + row.completed_at = Some(seconds_since_epoch()); + } + outcome = WriteOutcome::Matched; + } + if unknown.is_some() { + outcome = WriteOutcome::Unknown; + } + Box::pin(async move { Ok(outcome) }) + } + + fn read(&self) -> BoxedFuture<'_, Result> { + let found = self.rows.lock().unwrap().get(&self.arn).map(|row| JobRow { + status: job_status(row.status), + owner_generation: row.owner_generation, + heartbeat_at: row.heartbeat_at, + failure_reason: row.failure_reason.clone(), + }); + Box::pin(async move { + found.ok_or_else(|| StorageError::Internal("the row is gone".to_owned())) + }) + } +} + +impl Fake { + /// A `Fake` serving `source` through the legacy reader, in the `Arc` the + /// restore request takes (its manifest branch spawns a task that owns a + /// clone). + fn with_source(source: BackupRestoreSource) -> Arc { + let fake = Self::default(); + fake.sources + .lock() + .unwrap() + .insert(source.backup_arn.clone(), source); + Arc::new(fake) + } + fn restores(&self) -> Vec { + self.restores.lock().unwrap().clone() + } + fn source_reads(&self) -> usize { + *self.source_reads.lock().unwrap() + } +} + +fn unused() -> BoxedFuture<'static, Result> { + Box::pin(async { Err(StorageError::Internal("not used by these tests".to_owned())) }) +} + +/// The runtime the tests run with: four slots, no configured store. +fn runtime() -> BackupRuntime { + BackupRuntime::new(4) +} + +fn store() -> StoreIdentity { + StoreIdentity { + store_kind: StoreKind::Filesystem, + store_root: "/var/lib/extenddb/backups".to_owned(), + store_id: "store-1".to_owned(), + } +} + +/// An in-memory `BackupStore` for the engine's tests: objects by key, a +/// switch that makes every call fail as a transport error, a switch that +/// makes every call wait forever (a store that has stopped answering), and +/// the keys each operation was called with. +#[derive(Default)] +struct MemStore { + objects: Mutex>>, + /// When set, every operation fails with this text as the cause under a + /// "dispatch failure" link, the shape the SDK's transport errors have. + down: Mutex>, + /// When set, every operation's future never completes; `stalls` counts + /// the calls that entered the stall. A stalled call returns once the + /// store is un-stalled, as a parked filesystem call returns when its + /// mount answers. + stalled: std::sync::atomic::AtomicBool, + stalls: std::sync::atomic::AtomicUsize, + /// When set, every operation takes this long before it answers: a + /// healthy root whose calls are slow. + delay: Mutex>, + deletes: Mutex>, + puts: Mutex>, + /// When set, `put` fails as a transport error after reading its body; + /// every other call answers. + fail_puts: std::sync::atomic::AtomicBool, + /// When set, `put` fails as a transport error 200 ms after reading its + /// body's first chunk, while the encoder waits for the file's next item + /// (a connection reset mid-upload). + fail_puts_after_first_chunk: std::sync::atomic::AtomicBool, + /// When set, `put` reports one byte more than it stored (a store whose + /// size accounting differs from the bytes written). + misreport_put_size: std::sync::atomic::AtomicBool, + /// When set, `get` of a key under this prefix fails as a transport + /// error; every other call answers (the identity file's read among + /// them, so the store stays healthy). + fail_gets_under: Mutex>, + /// When set, the next `n` `get`s of a data file (a key under a `data/` + /// component) fail at the call with the error kind given; every other + /// call answers. + fail_gets: Mutex>, + /// When set, the next `get` of a data file yields this many bytes and + /// then fails as a transport error mid-body (a `GetObject` body that + /// stopped arriving). + fail_get_after_bytes: Mutex>, + /// When set, every `get` of a data file fails at the call as a transport + /// error while the clock reads before this instant (an outage of a + /// known length under a paused clock). + fail_data_gets_until: Mutex>, + /// When set, every `get` of a key ending in this text fails at the call + /// as a transport error (one object the store never serves). + fail_gets_of: Mutex>, + /// When set, the body of the `get` with this ordinal (1-based, over + /// every `get` the store saw) panics after its first chunk: a reader on + /// the blocking pool that dies mid-file. + panic_body_of_get: std::sync::atomic::AtomicUsize, + /// Every `get`'s key, in order, failed ones included. + gets: Mutex>, + /// Called with the key of every `put` that stored its object, so a test + /// can act at a point of the job (after the sidecar's put, before the + /// claim re-check). + on_put: Mutex>, + /// Bytes `put` stored, by key, in the order the puts ended. + put_sizes: Mutex>, + /// When set, the bytes `put_if_absent` leaves under the key right after + /// it stored its own: a server that ignores `If-None-Match` and a second + /// writer that won the plain-put race. + overwrite_after_conditional_put: Mutex>>, + /// When set, `put_if_absent` finds the key taken (`None`) while `get` + /// and `head` find no object: something that is not an object (a FIFO, + /// a socket) sits at the path. + key_taken_by_a_non_object: std::sync::atomic::AtomicBool, + /// When set, `validate` fails with this text as a store condition (the + /// probe write failing with an I/O error) while every other call + /// answers. + validate_error: Mutex>, + /// When set, a `delete_prefix` of a prefix (not of a manifest key) is + /// followed by a put of this key under the prefix: a writer whose put + /// overlapped the deletion. + late_put_after_delete: Mutex>, + /// When set, `delete_prefix` takes this long before it answers (a slow + /// store's deletion). + delete_delay: Mutex>, + /// When set, `list` of this prefix fails as a transport error. + fail_lists_of: Mutex>, + /// When set, a `list` of the whole root fails as a transport error + /// once it has yielded the keys that sort before this one: a listing + /// that stops part-way through. + fail_root_list_at: Mutex>, + /// When set, `delete_prefix` of this prefix fails as a transport error. + fail_deletes_of: Mutex>, + /// When set, the identity file becomes these bytes right after the next + /// `get` of it has read the current ones: a root another catalog adopts + /// between a store's check and what follows it. + identity_after_read: Mutex>>, + /// When set, a `list` of the whole root never yields (a listing that + /// stalls); a `list` of any prefix answers. + stall_root_list: std::sync::atomic::AtomicBool, + /// When set, the `get`s of the identity file after the first `n` (the + /// count) fail as a transport error, or never answer (`stall`); every + /// other call answers. The deletion fence's read failing alone. + identity_get_failure: Mutex>, + /// When set, `head` of this key fails as a transport error; every other + /// call answers. + fail_heads_of: Mutex>, + /// Every `list`'s prefix, in order. + lists: Mutex>, +} + +/// How `MemStore::identity_get_failure` fails a read of the identity file. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum IdentityFailure { + Transport, + Stall, +} + +/// What `MemStore::on_put` runs after a stored `put`, with the object's key. +type OnPut = Box; + +/// How `MemStore::fail_gets_with` fails a data file's `get`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum GetFailure { + /// The SDK's transport shape, which the restore retries. + Transport, + /// A denial, which it does not. + PermissionDenied, +} + +/// A transport error with the SDK's shape: a top link naming only the class +/// and the cause one link below it. +#[derive(Debug)] +struct DispatchFailure(ConnectError); + +#[derive(Debug)] +struct ConnectError(String); + +impl std::fmt::Display for DispatchFailure { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("dispatch failure") + } +} + +impl std::error::Error for DispatchFailure { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + Some(&self.0) + } +} + +impl std::fmt::Display for ConnectError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.0) + } +} + +impl std::error::Error for ConnectError {} + +impl MemStore { + /// A store that is down fails every call as the S3 store fails one it + /// cannot send: a transport error whose cause is a link below the top. + fn check(&self) -> Result<(), extenddb_backup_store::StoreError> { + match self.down.lock().unwrap().as_ref() { + Some(text) => Err(extenddb_backup_store::StoreError::Transport { + source: Box::new(DispatchFailure(ConnectError(text.clone()))), + }), + None => Ok(()), + } + } + /// Wait while the store is stalled, and the delay if one is set; + /// otherwise return at once. + async fn stall(&self) { + let delay = *self.delay.lock().unwrap(); + if let Some(delay) = delay { + tokio::time::sleep(delay).await; + } + if self.stalled.load(std::sync::atomic::Ordering::Acquire) { + self.stalls + .fetch_add(1, std::sync::atomic::Ordering::AcqRel); + while self.stalled.load(std::sync::atomic::Ordering::Acquire) { + tokio::time::sleep(Duration::from_millis(10)).await; + } + } + } + fn set_down(&self, down: bool) { + *self.down.lock().unwrap() = down.then(|| "connection refused".to_owned()); + } + fn set_stalled(&self, stalled: bool) { + self.stalled + .store(stalled, std::sync::atomic::Ordering::Release); + } + fn set_delay(&self, delay: Option) { + *self.delay.lock().unwrap() = delay; + } + fn insert(&self, key: &str, body: &[u8]) { + self.objects + .lock() + .unwrap() + .insert(key.to_owned(), body.to_vec()); + } + /// The next `times` `get`s of a data file fail with `failure`. + fn fail_gets_with(&self, failure: GetFailure, times: usize) { + *self.fail_gets.lock().unwrap() = Some((failure, times)); + } + /// The next `get` of a data file stops after `n` bytes with a transport + /// error. + fn fail_get_after_bytes(&self, n: usize) { + *self.fail_get_after_bytes.lock().unwrap() = Some(n); + } + fn gets(&self) -> Vec { + self.gets.lock().unwrap().clone() + } + fn is_data_file(key: &str) -> bool { + key.contains("/data/") + } + fn keys(&self) -> Vec { + self.objects.lock().unwrap().keys().cloned().collect() + } + fn matches(key: &str, prefix: &str) -> bool { + let prefix = prefix.trim_end_matches('/'); + prefix.is_empty() || key == prefix || key.starts_with(&format!("{prefix}/")) + } +} + +async fn collect_body( + body: extenddb_backup_store::ByteStream, +) -> Result, extenddb_backup_store::StoreError> { + body.try_fold(Vec::new(), |mut acc, chunk| async move { + acc.extend_from_slice(&chunk); + Ok(acc) + }) + .await +} + +impl extenddb_backup_store::BackupStore for MemStore { + fn put<'a>( + &'a self, + key: &'a str, + body: extenddb_backup_store::ByteStream, + ) -> BoxedFuture<'a, Result> + { + Box::pin(async move { + self.stall().await; + self.check()?; + let mut body = body; + if self + .fail_puts_after_first_chunk + .load(std::sync::atomic::Ordering::Acquire) + { + let _first = body.try_next().await?; + tokio::time::sleep(Duration::from_millis(200)).await; + return Err(extenddb_backup_store::StoreError::Transport { + source: Box::new(DispatchFailure(ConnectError( + "connection reset mid-body".to_owned(), + ))), + }); + } + let bytes = collect_body(body).await?; + if self.fail_puts.load(std::sync::atomic::Ordering::Acquire) { + return Err(extenddb_backup_store::StoreError::Transport { + source: Box::new(DispatchFailure(ConnectError("put refused".to_owned()))), + }); + } + self.puts.lock().unwrap().push(key.to_owned()); + let size = bytes.len() as u64; + self.put_sizes.lock().unwrap().push((key.to_owned(), size)); + self.insert(key, &bytes); + if let Some(on_put) = self.on_put.lock().unwrap().as_ref() { + on_put(key); + } + let reported = if self + .misreport_put_size + .load(std::sync::atomic::Ordering::Acquire) + { + size + 1 + } else { + size + }; + Ok(extenddb_backup_store::ObjectMeta { + key: key.to_owned(), + size: reported, + last_modified: None, + etag: None, + }) + }) + } + + fn put_if_absent<'a>( + &'a self, + key: &'a str, + body: extenddb_backup_store::ByteStream, + ) -> BoxedFuture< + 'a, + Result, extenddb_backup_store::StoreError>, + > { + Box::pin(async move { + self.stall().await; + self.check()?; + let bytes = collect_body(body).await?; + if self + .key_taken_by_a_non_object + .load(std::sync::atomic::Ordering::Acquire) + { + return Ok(None); + } + let mut objects = self.objects.lock().unwrap(); + if objects.contains_key(key) { + return Ok(None); + } + self.puts.lock().unwrap().push(key.to_owned()); + let size = bytes.len() as u64; + objects.insert(key.to_owned(), bytes); + if let Some(other) = self.overwrite_after_conditional_put.lock().unwrap().clone() { + objects.insert(key.to_owned(), other); + } + Ok(Some(extenddb_backup_store::ObjectMeta { + key: key.to_owned(), + size, + last_modified: None, + etag: None, + })) + }) + } + + fn get<'a>( + &'a self, + key: &'a str, + ) -> BoxedFuture<'a, Result> + { + Box::pin(async move { + self.gets.lock().unwrap().push(key.to_owned()); + self.stall().await; + self.check()?; + if key == STORE_IDENTITY_KEY { + let failure = { + let mut slot = self.identity_get_failure.lock().unwrap(); + match slot.as_mut() { + Some((0, failure)) => Some(*failure), + Some((pass, _)) => { + *pass -= 1; + None + } + None => None, + } + }; + match failure { + Some(IdentityFailure::Transport) => { + return Err(extenddb_backup_store::StoreError::Transport { + source: Box::new(DispatchFailure(ConnectError( + "identity get refused".to_owned(), + ))), + }); + } + Some(IdentityFailure::Stall) => futures::future::pending::<()>().await, + None => {} + } + } + if self + .fail_gets_under + .lock() + .unwrap() + .as_deref() + .is_some_and(|prefix| key.starts_with(prefix)) + { + return Err(extenddb_backup_store::StoreError::Transport { + source: Box::new(DispatchFailure(ConnectError("get refused".to_owned()))), + }); + } + let ordinal = self.gets.lock().unwrap().len(); + if Self::is_data_file(key) + && (self + .fail_data_gets_until + .lock() + .unwrap() + .is_some_and(|until| tokio::time::Instant::now() < until) + || self + .fail_gets_of + .lock() + .unwrap() + .as_deref() + .is_some_and(|suffix| key.ends_with(suffix))) + { + return Err(extenddb_backup_store::StoreError::Transport { + source: Box::new(DispatchFailure(ConnectError( + "connection reset by peer".to_owned(), + ))), + }); + } + if Self::is_data_file(key) { + let mut fail = self.fail_gets.lock().unwrap(); + if let Some((failure, times)) = *fail { + *fail = (times > 1).then_some((failure, times - 1)); + return Err(match failure { + GetFailure::Transport => extenddb_backup_store::StoreError::Transport { + source: Box::new(DispatchFailure(ConnectError( + "connection reset by peer".to_owned(), + ))), + }, + GetFailure::PermissionDenied => { + extenddb_backup_store::StoreError::PermissionDenied(format!( + "GetObject {key}" + )) + } + }); + } + } + let bytes = self + .objects + .lock() + .unwrap() + .get(key) + .cloned() + .ok_or(extenddb_backup_store::StoreError::NotFound)?; + if key == STORE_IDENTITY_KEY + && let Some(next) = self.identity_after_read.lock().unwrap().take() + { + self.insert(key, &next); + } + if Self::is_data_file(key) + && let Some(n) = self.fail_get_after_bytes.lock().unwrap().take() + { + let head = bytes[..n.min(bytes.len())].to_vec(); + let stream = futures::stream::iter(vec![ + Ok(bytes::Bytes::from(head)), + Err(extenddb_backup_store::StoreError::Transport { + source: Box::new(DispatchFailure(ConnectError( + "the body stopped arriving".to_owned(), + ))), + }), + ]); + return Ok(Box::pin(stream) as extenddb_backup_store::ByteStream); + } + if ordinal + == self + .panic_body_of_get + .load(std::sync::atomic::Ordering::Acquire) + { + let head = bytes[..bytes.len().min(16)].to_vec(); + let stream = futures::stream::iter(vec![Ok(bytes::Bytes::from(head))]).chain( + futures::stream::once(async { + panic!("the body's reader died (a test fixture)") + }), + ); + return Ok(Box::pin(stream) as extenddb_backup_store::ByteStream); + } + Ok(extenddb_backup_store::byte_stream_from(bytes)) + }) + } + + fn head<'a>( + &'a self, + key: &'a str, + ) -> BoxedFuture< + 'a, + Result, extenddb_backup_store::StoreError>, + > { + Box::pin(async move { + self.stall().await; + self.check()?; + if self.fail_heads_of.lock().unwrap().as_deref() == Some(key) { + return Err(extenddb_backup_store::StoreError::Transport { + source: Box::new(DispatchFailure(ConnectError("head refused".to_owned()))), + }); + } + Ok(self.objects.lock().unwrap().get(key).map(|bytes| { + extenddb_backup_store::ObjectMeta { + key: key.to_owned(), + size: bytes.len() as u64, + last_modified: None, + etag: None, + } + })) + }) + } + + fn list<'a>( + &'a self, + prefix: &'a str, + ) -> BoxStream<'a, Result> + { + self.lists.lock().unwrap().push(prefix.to_owned()); + if let Err(e) = self.check() { + return Box::pin(futures::stream::once(async move { Err(e) })); + } + if prefix.is_empty() + && self + .stall_root_list + .load(std::sync::atomic::Ordering::Acquire) + { + return Box::pin(futures::stream::pending()); + } + if self.fail_lists_of.lock().unwrap().as_deref() == Some(prefix) { + return Box::pin(futures::stream::once(async move { + Err(extenddb_backup_store::StoreError::Transport { + source: Box::new(DispatchFailure(ConnectError("list refused".to_owned()))), + }) + })); + } + let fail_at = self + .fail_root_list_at + .lock() + .unwrap() + .clone() + .filter(|_| prefix.is_empty()); + let mut metas: Vec<_> = self + .objects + .lock() + .unwrap() + .iter() + .filter(|(k, _)| Self::matches(k, prefix)) + .take_while(|(k, _)| fail_at.as_ref().is_none_or(|at| *k < at)) + .map(|(k, v)| { + Ok(extenddb_backup_store::ObjectMeta { + key: k.clone(), + size: v.len() as u64, + last_modified: None, + etag: None, + }) + }) + .collect(); + if fail_at.is_some() { + metas.push(Err(extenddb_backup_store::StoreError::Transport { + source: Box::new(DispatchFailure(ConnectError("list stopped".to_owned()))), + })); + } + Box::pin(futures::stream::iter(metas)) + } + + fn delete_prefix<'a>( + &'a self, + prefix: &'a str, + ) -> BoxedFuture<'a, Result> { + Box::pin(async move { + self.stall().await; + self.check()?; + let delay = *self.delete_delay.lock().unwrap(); + if let Some(delay) = delay { + tokio::time::sleep(delay).await; + } + if self.fail_deletes_of.lock().unwrap().as_deref() == Some(prefix) { + return Err(extenddb_backup_store::StoreError::Transport { + source: Box::new(DispatchFailure(ConnectError("delete refused".to_owned()))), + }); + } + self.deletes.lock().unwrap().push(prefix.to_owned()); + let mut objects = self.objects.lock().unwrap(); + let before = objects.len(); + objects.retain(|k, _| !Self::matches(k, prefix)); + let deleted = (before - objects.len()) as u64; + if let Some(late) = self.late_put_after_delete.lock().unwrap().as_deref() + && !prefix.ends_with(".json") + { + objects.insert(format!("{prefix}{late}"), b"late".to_vec()); + } + Ok(deleted) + }) + } + + fn validate(&self) -> BoxedFuture<'_, Result<(), extenddb_backup_store::StoreError>> { + Box::pin(async move { + self.stall().await; + self.check()?; + match self.validate_error.lock().unwrap().clone() { + Some(text) => Err(extenddb_backup_store::StoreError::Other(text)), + None => Ok(()), + } + }) + } +} + +/// A runtime with `store` configured as `identity`, healthy. A root with no +/// identity file gets one naming the `Fake`'s catalog, as the check that +/// made it healthy leaves it, so the deletion fence passes. +fn runtime_on(mem: &Arc, identity: StoreIdentity) -> BackupRuntime { + if !mem.objects.lock().unwrap().contains_key(STORE_IDENTITY_KEY) { + mem.insert( + STORE_IDENTITY_KEY, + &identity_file(&identity.store_id, "catalog-1"), + ); + } + runtime().with_store(ConfiguredStore::new( + identity.store_kind, + identity.store_root.clone(), + Arc::clone(mem) as Arc, + None, + Ok(identity), + )) +} + +/// A runtime whose configured store (`store()`'s kind and root) failed its +/// startup check with `problem`: the id is unknown. +fn runtime_unhealthy(mem: &Arc, problem: StoreProblem) -> BackupRuntime { + runtime().with_store(ConfiguredStore::new( + store().store_kind, + store().store_root, + Arc::clone(mem) as Arc, + None, + Err(problem), + )) +} + +/// A deletion fence over a healthy store whose root's identity file names +/// the `Fake`'s catalog: it passes. For the tests that call a deleting step +/// with a store of their own. +fn passing_fence() -> super::StoreFence { + let mem = MemStore::default(); + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + ConfiguredStore::new( + store().store_kind, + store().store_root, + Arc::new(mem) as Arc, + None, + Ok(store()), + ) + .fence() +} + +/// The identity file of `store()` for catalog `catalog_id`, as bytes. +fn identity_file(store_id: &str, catalog_id: &str) -> Vec { + identity_file_of(store_id, catalog_id, false) +} + +/// [`identity_file`] with the in-memory catalog marker as given. +fn identity_file_of(store_id: &str, catalog_id: &str, ephemeral_catalog: bool) -> Vec { + StoreIdentityFile { + format_version: 1, + store_id: store_id.to_owned(), + catalog_id: catalog_id.to_owned(), + backend: "test".to_owned(), + created_at_ms: 0, + ephemeral_catalog, + } + .to_bytes() +} + +/// A catalog row for `arn`, with the manifest fixture's source table (the +/// `Music` table of account 123456789012). +fn row(arn: &str, status: BackupStatus) -> BackupRow { + let parsed = BackupArn::parse(arn).unwrap(); + let manifest: Value = serde_json::from_str(include_str!( + "../../../core/testdata/extenddb-manifest/v1.json" + )) + .unwrap(); + BackupRow { + backup_id: parsed.backup_id().to_owned(), + account_id: parsed.account_id().to_owned(), + table_name: parsed.table_name().to_owned(), + backup_arn: arn.to_owned(), + backup_name: "music-nightly".to_owned(), + status, + store: store(), + size_bytes: 61828, + item_count: 100_000, + created_at: extenddb_storage::util::backup_creation_date_time(arn).unwrap(), + completed_at: None, + failure_reason: None, + source_table_details_json: manifest["source_table"].clone(), + owner_generation: 7, + heartbeat_at: 1.0, + } +} + +impl Fake { + /// `claim_restore`'s registration once any delay has passed. + fn register_restore( + &self, + account_id: &str, + input: CreateTableInput, + registration: &RestoreRegistrationRow, + ) -> Result<(TableDescription, Box), StorageError> { + let arn = ®istration.backup.backup_arn; + let available = self + .rows + .lock() + .unwrap() + .get(arn) + .is_some_and(|r| r.status == BackupStatus::Available); + if !available { + return Err(extenddb_storage::backup_catalog::backup_not_found(arn)); + } + let in_use = self.in_use.lock().unwrap().get(arn).cloned().or_else(|| { + let jobs = self.restore_jobs.lock().unwrap(); + self.restore_tables + .lock() + .unwrap() + .iter() + .filter(|(_, t)| !t.active) + .find(|(id, _)| jobs.get(*id).is_some_and(|j| j.source_backup_arn == *arn)) + .map(|(_, t)| t.name.clone()) + }); + if let Some(table) = in_use { + return Err(extenddb_storage::backup_catalog::backup_in_use(&table, arn)); + } + let mut tables = self.tables.lock().unwrap(); + if let Some(taken) = tables.get(&input.table_name) { + let err = match taken + .restore_summary + .as_ref() + .filter(|_| taken.table_status == TableStatus::Creating) + .and_then(|s| s.source_backup_arn.clone()) + { + Some(backup_arn) => StorageError::TableBeingRestored { + table_name: input.table_name.clone(), + backup_arn, + }, + None => StorageError::TableAlreadyExists(input.table_name.clone()), + }; + return Err(err); + } + let table_id = format!("tid-{}", input.table_name); + let generation = new_generation(); + let mut desc = described(&RestoreRequest { + backup_arn: arn.clone(), + source_table_arn: registration.source_table_arn.clone(), + restore_date_time: registration.restore_date_time, + input: input.clone(), + }); + desc.table_id = table_id.clone(); + desc.restore_summary = Some(RestoreSummary { + source_backup_arn: Some(arn.clone()), + source_table_arn: Some(registration.source_table_arn.clone()), + restore_date_time: registration.restore_date_time, + restore_in_progress: true, + }); + tables.insert(input.table_name.clone(), desc.clone()); + drop(tables); + let target = RestoreTargetKey { + account_id: account_id.to_owned(), + table_name: input.table_name.clone(), + table_id: table_id.clone(), + }; + self.restore_jobs.lock().unwrap().insert( + table_id.clone(), + RestoreJobRow { + target, + source_backup_arn: arn.clone(), + kind: RestoreKind::Manifest, + status: JobStatus::Running, + owner_generation: generation, + heartbeat_at: seconds_since_epoch() + - self + .registration_heartbeat_age + .lock() + .unwrap() + .unwrap_or(0.0), + store: Some(registration.store.clone()), + spool_host: registration.spool_host.clone(), + file_index: None, + items_applied: None, + failure_reason: None, + completed_at: None, + }, + ); + self.restore_tables.lock().unwrap().insert( + table_id.clone(), + RestoreTable { + name: input.table_name.clone(), + key_attributes: input + .key_schema + .iter() + .map(|k| k.attribute_name.clone()) + .collect(), + items: Vec::new(), + active: false, + }, + ); + let claim = self.restore_claim_on(&table_id, generation); + Ok((desc, claim)) + } +} + +impl BackupCatalog for Fake { + fn catalog_id(&self) -> BoxedFuture<'_, Result> { + if *self.catalog_id_error.lock().unwrap() { + return Box::pin(async { Err(StorageError::Internal("pool exhausted".to_owned())) }); + } + let id = self + .catalog_id + .lock() + .unwrap() + .clone() + .unwrap_or_else(|| "catalog-1".to_owned()); + Box::pin(async move { Ok(id) }) + } + + /// The backends' insert: the row as given, CREATING under a fresh + /// generation with `heartbeat_at` now; `AlreadyExists` when the ARN has a + /// row. + fn claim_backup<'a>( + &'a self, + row: &'a BackupRow, + ) -> BoxedFuture<'a, Result, StorageError>> { + self.claims.lock().unwrap().push(row.clone()); + let mut rows = self.rows.lock().unwrap(); + if rows.contains_key(&row.backup_arn) { + let arn = row.backup_arn.clone(); + return Box::pin(async move { Err(StorageError::AlreadyExists(arn)) }); + } + let generation = new_generation(); + let mut inserted = row.clone(); + inserted.status = BackupStatus::Creating; + inserted.owner_generation = generation; + inserted.heartbeat_at = seconds_since_epoch(); + rows.insert(row.backup_arn.clone(), inserted); + drop(rows); + let claim = self.claim_on(&row.backup_arn, generation); + Box::pin(async move { Ok(claim) }) + } + + /// The backends' unowned insert: the row as given, owner + /// `UNOWNED_GENERATION`, `completed_at` and `heartbeat_at` now; `false` + /// when a row with the backup id exists (PostgreSQL's, SQLite's and + /// MongoDB's unique `backup_id`, which also covers the key), or with + /// `unique_by_arn` a row under the ARN (Cassandra's), unless + /// `completes_own_insert` is set and the row is this insert's own. + fn insert_backup_row<'a>( + &'a self, + row: &'a BackupRow, + ) -> BoxedFuture<'a, Result> { + self.insert_attempts.lock().unwrap().push(row.clone()); + if let Some(e) = self.insert_error.lock().unwrap().clone() { + return Box::pin(async move { Err(StorageError::Internal(e)) }); + } + let mut rows = self.rows.lock().unwrap(); + for landed in self.lands_before_insert.lock().unwrap().drain(..) { + rows.insert(landed.backup_arn.clone(), landed); + } + if self + .completes_own_insert + .load(std::sync::atomic::Ordering::Acquire) + && rows.get(&row.backup_arn).is_some_and(|stored| { + stored.owner_generation == UNOWNED_GENERATION + && BackupRow { + owner_generation: row.owner_generation, + completed_at: row.completed_at, + heartbeat_at: row.heartbeat_at, + ..stored.clone() + } == *row + }) + { + self.unlisted.lock().unwrap().remove(&row.backup_arn); + return Box::pin(async move { Ok(true) }); + } + let exists = if self + .unique_by_arn + .load(std::sync::atomic::Ordering::Acquire) + { + rows.contains_key(&row.backup_arn) + } else { + rows.values().any(|r| r.backup_id == row.backup_id) + }; + if !exists { + let now = seconds_since_epoch(); + let mut inserted = row.clone(); + inserted.owner_generation = UNOWNED_GENERATION; + inserted.completed_at = Some(now); + inserted.heartbeat_at = now; + rows.insert(row.backup_arn.clone(), inserted); + } + Box::pin(async move { Ok(!exists) }) + } + + /// The backends' registration: under the backup row's lock the row must + /// be AVAILABLE (else `Backup not found`) and not in use (else + /// `BackupInUse`), the name free (else `TableAlreadyExists`, or + /// `TableBeingRestored` for a CREATING restore target); then the target + /// CREATING with its `RestoreSummary` and the record RUNNING under a + /// fresh generation, `heartbeat_at` now, in one step. + fn claim_restore<'a>( + &'a self, + account_id: &'a str, + input: CreateTableInput, + registration: &'a RestoreRegistrationRow, + ) -> BoxedFuture<'a, Result<(TableDescription, Box), StorageError>> { + self.restore_claims + .lock() + .unwrap() + .push((account_id.to_owned(), input.clone())); + if let Some(e) = self.claim_restore_error.lock().unwrap().take() { + return Box::pin(async move { Err(e) }); + } + let delay = *self.claim_restore_delay.lock().unwrap(); + Box::pin(async move { + if let Some(delay) = delay { + tokio::time::sleep(delay).await; + } + self.register_restore(account_id, input, registration) + }) + } + + /// The takeover: a RUNNING manifest record whose table id is in the + /// `restore_dead` set gets a fresh generation, the claimant's + /// `spool_host` and a fresh `heartbeat_at` (so a second taker finds it + /// live at once); anything else is `None`. + fn claim_restore_dead<'a>( + &'a self, + key: &'a RestoreTargetKey, + _stale_after: Duration, + spool_host: Option<&'a str>, + ) -> BoxedFuture<'a, Result>, StorageError>> { + let dead = self.restore_dead.lock().unwrap().contains(&key.table_id); + let mut records = self.restore_jobs.lock().unwrap(); + let claim = match records.get_mut(&key.table_id) { + Some(record) + if dead + && record.status == JobStatus::Running + && record.kind == RestoreKind::Manifest => + { + record.owner_generation = new_generation(); + record.spool_host = spool_host.map(str::to_owned); + record.heartbeat_at = seconds_since_epoch(); + let generation = record.owner_generation; + if self + .takeover_removes_record + .load(std::sync::atomic::Ordering::Acquire) + { + records.remove(&key.table_id); + } + drop(records); + self.restore_takeovers + .lock() + .unwrap() + .push((key.table_id.clone(), spool_host.map(str::to_owned))); + if let Some(hook) = self.takeover_hook.lock().unwrap().take() { + hook(self); + } + Some(self.restore_claim_on(&key.table_id, generation)) + } + _ => None, + }; + Box::pin(async move { Ok(claim) }) + } + + fn read_restore_job<'a>( + &'a self, + key: &'a RestoreTargetKey, + ) -> BoxedFuture<'a, Result, StorageError>> { + let delay = *self.read_restore_job_delay.lock().unwrap(); + Box::pin(async move { + if let Some(delay) = delay { + tokio::time::sleep(delay).await; + } + Ok(self + .restore_jobs + .lock() + .unwrap() + .get(&key.table_id) + .cloned()) + }) + } + + fn list_restore_jobs<'a>( + &'a self, + status: Option, + ) -> BoxedFuture<'a, Result, StorageError>> { + self.restore_lists.lock().unwrap().push(status); + if self + .restore_list_error + .load(std::sync::atomic::Ordering::Acquire) + { + return Box::pin(async { + Err(StorageError::Unsupported("not implemented".to_owned())) + }); + } + let mut rows: Vec = self + .restore_jobs + .lock() + .unwrap() + .values() + .filter(|r| status.is_none_or(|s| r.status == s)) + .cloned() + .collect(); + rows.sort_by(|a, b| a.target.table_id.cmp(&b.target.table_id)); + // The moment a registration can land between the sweep's two reads: + // after the reconciler read the RUNNING records and before its spool + // listing uses them. The reconciler lists the spool first, so the + // entry a registration opens after this read is not in that listing + // and is never a sweep candidate; the hook plants such a + // registration so the test fails if the two reads are swapped. + if let Some(hook) = self.sweep_hook.lock().unwrap().take() { + hook(self); + } + let delay = *self.list_jobs_delay.lock().unwrap(); + Box::pin(async move { + if let Some(delay) = delay { + tokio::time::sleep(delay).await; + } + Ok(rows) + }) + } + + /// The takeover: a CREATING row whose ARN is in the `dead` set gets a + /// fresh generation, `heartbeat_at` untouched; anything else is `None`. + fn claim_dead_backup<'a>( + &'a self, + key: &'a BackupKey, + _stale_after: Duration, + ) -> BoxedFuture<'a, Result>, StorageError>> { + let dead = self.dead.lock().unwrap().contains(&key.backup_arn); + let mut rows = self.rows.lock().unwrap(); + let claim = match rows.get_mut(&key.backup_arn) { + Some(row) if dead && row.status == BackupStatus::Creating => { + row.owner_generation = new_generation(); + let generation = row.owner_generation; + if self + .takeover_lost_to_another + .load(std::sync::atomic::Ordering::Acquire) + { + row.owner_generation = new_generation(); + } + drop(rows); + self.takeovers.lock().unwrap().push(key.backup_arn.clone()); + Some(self.claim_on(&key.backup_arn, generation)) + } + _ => None, + }; + Box::pin(async move { Ok(claim) }) + } + + fn sweep_restore_registrations(&self) -> BoxedFuture<'_, Result, StorageError>> { + let swept = self.swept_registrations.lock().unwrap().clone(); + Box::pin(async move { Ok(swept) }) + } + + /// The backend's listing: rows of the store (kind, root and id), of the + /// status when given, in (`created_at`, `backup_arn`) order. + fn list_backup_rows<'a>( + &'a self, + status: Option, + store: &'a StoreIdentity, + ) -> BoxStream<'a, Result> { + if let Some(e) = self.list_error.lock().unwrap().clone() { + return Box::pin(futures::stream::once(async move { + Err(StorageError::Internal(e)) + })); + } + let mut rows: Vec = self + .rows + .lock() + .unwrap() + .values() + .filter(|r| r.store == *store) + .filter(|r| status.is_none_or(|s| r.status == s)) + .filter(|r| !self.unlisted.lock().unwrap().contains(&r.backup_arn)) + .cloned() + .collect(); + rows.sort_by(|a, b| { + a.created_at + .total_cmp(&b.created_at) + .then_with(|| a.backup_arn.cmp(&b.backup_arn)) + }); + Box::pin(futures::stream::iter(rows.into_iter().map(Ok))) + } + + /// The last AVAILABLE row of the store in the listing's order, read as + /// one row; `list_error` fails it as it fails the listing. + fn newest_available_backup_row<'a>( + &'a self, + store: &'a StoreIdentity, + ) -> BoxedFuture<'a, Result, StorageError>> { + self.newest_reads.lock().unwrap().push(store.clone()); + if let Some(e) = self.list_error.lock().unwrap().clone() { + return Box::pin(async move { Err(StorageError::Internal(e)) }); + } + let newest = self + .rows + .lock() + .unwrap() + .values() + .filter(|r| r.store == *store && r.status == BackupStatus::Available) + .max_by(|a, b| { + a.created_at + .total_cmp(&b.created_at) + .then_with(|| a.backup_arn.cmp(&b.backup_arn)) + }) + .cloned(); + Box::pin(async move { Ok(newest) }) + } + + fn list_account_backup_rows<'a>( + &'a self, + account_id: &'a str, + table_name: Option<&'a str>, + bounds: &'a TimeBounds, + ) -> BoxStream<'a, Result> { + self.bounds_seen.lock().unwrap().push(*bounds); + let rows: Vec = self + .rows + .lock() + .unwrap() + .values() + .filter(|r| r.account_id == account_id) + .filter(|r| table_name.is_none_or(|t| r.table_name == t)) + .filter(|r| bounds.contains(r.created_at)) + .cloned() + .collect(); + Box::pin(futures::stream::iter(rows.into_iter().map(Ok))) + } + + /// The backend's rule: the row of (account, id), and only when its + /// account, table and ARN equal the request's; with `unique_by_arn`, + /// the row of (account, ARN), as Cassandra keys it. + fn read_backup_row<'a>( + &'a self, + arn: &'a BackupArn, + ) -> BoxedFuture<'a, Result, StorageError>> { + self.row_reads.lock().unwrap().push(arn.to_arn_string()); + if let Some(e) = self.read_row_error.lock().unwrap().clone() { + return Box::pin(async move { Err(StorageError::Internal(e)) }); + } + let by_arn = self + .unique_by_arn + .load(std::sync::atomic::Ordering::Acquire); + let found = self + .rows + .lock() + .unwrap() + .values() + .find(|r| { + r.account_id == arn.account_id() + && if by_arn { + r.backup_arn == arn.to_arn_string() + } else { + r.backup_id == arn.backup_id() + } + }) + .filter(|r| r.table_name == arn.table_name() && r.backup_arn == arn.to_arn_string()) + .cloned(); + Box::pin(async move { Ok(found) }) + } + + fn start_backup_delete<'a>( + &'a self, + key: &'a BackupKey, + ) -> BoxedFuture<'a, Result> { + self.delete_starts_seen.lock().unwrap().push(key.clone()); + let answer = self + .delete_starts + .lock() + .unwrap() + .get(&key.backup_arn) + .cloned(); + let mut rows = self.rows.lock().unwrap(); + let status = rows.get(&key.backup_arn).map(|r| r.status); + // The natural answer moves an AVAILABLE row to DELETING, as the + // backends do; a preset answer leaves the row as the test set it. + if answer.is_none() + && status == Some(BackupStatus::Available) + && let Some(row) = rows.get_mut(&key.backup_arn) + { + row.status = BackupStatus::Deleting; + } + drop(rows); + Box::pin(async move { + match (answer, status) { + (Some(answer), _) => Ok(answer), + (None, Some(BackupStatus::Available)) => Ok(DeleteStart::Deleting), + (None, Some(status)) => Ok(DeleteStart::Status(status)), + (None, None) => Err(StorageError::Validation("Backup not found: x".to_owned())), + } + }) + } + + /// DELETING to DELETED, conditional, as the backends do. + fn set_backup_deleted<'a>( + &'a self, + key: &'a BackupKey, + ) -> BoxedFuture<'a, Result> { + let moved = match self.rows.lock().unwrap().get_mut(&key.backup_arn) { + Some(row) if row.status == BackupStatus::Deleting => { + row.status = BackupStatus::Deleted; + true + } + _ => false, + }; + if moved { + self.set_deleted + .lock() + .unwrap() + .push(key.backup_arn.clone()); + } + Box::pin(async move { Ok(moved) }) + } + + /// Remove a DELETED or FAILED tombstone, as the backends do. + fn remove_backup_row<'a>( + &'a self, + key: &'a BackupKey, + ) -> BoxedFuture<'a, Result> { + self.remove_attempts + .lock() + .unwrap() + .push(key.backup_arn.clone()); + let mut rows = self.rows.lock().unwrap(); + let removed = match rows.get(&key.backup_arn) { + Some(row) if matches!(row.status, BackupStatus::Deleted | BackupStatus::Failed) => { + rows.remove(&key.backup_arn); + true + } + _ => false, + }; + if removed { + self.removed_rows + .lock() + .unwrap() + .push(key.backup_arn.clone()); + } + Box::pin(async move { Ok(removed) }) + } + + /// The backends' outright removal, without the in-use rule; preset + /// `InUse` answers stand for it. A preset answer in `forget_answers` + /// leaves the row as the test set it; otherwise an AVAILABLE row goes, + /// another status is answered, and a missing row is `Backup not found`. + fn forget_available_backup_row<'a>( + &'a self, + key: &'a BackupKey, + ) -> BoxedFuture<'a, Result> { + self.forgets_seen.lock().unwrap().push(key.clone()); + let answer = self + .forget_answers + .lock() + .unwrap() + .get(&key.backup_arn) + .cloned(); + let mut rows = self.rows.lock().unwrap(); + let status = rows.get(&key.backup_arn).map(|r| r.status); + if answer.is_none() && status == Some(BackupStatus::Available) { + rows.remove(&key.backup_arn); + } + drop(rows); + Box::pin(async move { + match (answer, status) { + (Some(answer), _) => Ok(answer), + (None, Some(BackupStatus::Available)) => Ok(ForgetRow::Removed), + (None, Some(status)) => Ok(ForgetRow::Status(status)), + (None, None) => Err(StorageError::Validation("Backup not found: x".to_owned())), + } + }) + } + + /// Remove a CREATING row whose heartbeat is older than `stale_after` + /// (on this process's clock, which stands for the catalog's here). + fn remove_dead_creating_backup_row<'a>( + &'a self, + key: &'a BackupKey, + stale_after: Duration, + ) -> BoxedFuture<'a, Result> { + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_secs_f64(); + let mut rows = self.rows.lock().unwrap(); + let removed = match rows.get(&key.backup_arn) { + Some(row) + if row.status == BackupStatus::Creating + && row.heartbeat_at < now - stale_after.as_secs_f64() => + { + rows.remove(&key.backup_arn); + true + } + _ => false, + }; + Box::pin(async move { Ok(removed) }) + } + + /// The backends' listing on the named clock, the catalog's being this + /// process's moved by `catalog_clock_offset_secs`: rows of the store + /// with `status` whose `heartbeat_at`, or `completed_at` when set, is + /// older than `age`. + fn list_backup_rows_older_than<'a>( + &'a self, + status: BackupStatus, + store: &'a StoreIdentity, + clock: StaleClock, + age: Duration, + ) -> BoxStream<'a, Result> { + let cutoff = seconds_since_epoch() + *self.catalog_clock_offset_secs.lock().unwrap() + - age.as_secs_f64(); + let mut rows: Vec = self + .rows + .lock() + .unwrap() + .values() + .filter(|r| r.store == *store && r.status == status) + .filter(|r| match clock { + StaleClock::Heartbeat => r.heartbeat_at < cutoff, + StaleClock::Completed => r.completed_at.is_some_and(|c| c < cutoff), + }) + .cloned() + .collect(); + rows.sort_by(|a, b| { + a.created_at + .total_cmp(&b.created_at) + .then_with(|| a.backup_arn.cmp(&b.backup_arn)) + }); + Box::pin(futures::stream::iter(rows.into_iter().map(Ok))) + } + + /// The preset answer, or the name of a CREATING restore target the + /// `Fake` registered from the backup. + fn backup_in_use<'a>( + &'a self, + key: &'a BackupKey, + ) -> BoxedFuture<'a, Result, StorageError>> { + let found = self + .in_use + .lock() + .unwrap() + .get(&key.backup_arn) + .cloned() + .or_else(|| { + let jobs = self.restore_jobs.lock().unwrap(); + self.restore_tables + .lock() + .unwrap() + .iter() + .filter(|(_, t)| !t.active) + .find(|(id, _)| { + jobs.get(*id) + .is_some_and(|j| j.source_backup_arn == key.backup_arn) + }) + .map(|(_, t)| t.name.clone()) + }); + Box::pin(async move { Ok(found) }) + } + + /// A target the `Fake` registered is removed with its record and its + /// progress rows when it is still CREATING; any other key answers the + /// preset `target_still_creating`. + fn remove_restore_target<'a>( + &'a self, + key: &'a RestoreTargetKey, + ) -> BoxedFuture<'a, Result> { + self.removed_targets.lock().unwrap().push(key.clone()); + let mut targets = self.restore_tables.lock().unwrap(); + let answer = match targets.get(&key.table_id) { + Some(target) if !target.active && target.name == key.table_name => { + let target = targets.remove(&key.table_id).expect("present"); + self.tables.lock().unwrap().remove(&target.name); + self.restore_jobs.lock().unwrap().remove(&key.table_id); + self.progress_rows.lock().unwrap().remove(&key.table_id); + true + } + Some(_) => false, + // The table is no longer a CREATING restore target with this id: + // its record went with it (the flip wrote DONE, the removal took + // both), so a re-read finds it changed or gone. + None => { + let still = *self.target_still_creating.lock().unwrap(); + if !still { + self.restore_jobs.lock().unwrap().remove(&key.table_id); + } + still + } + }; + Box::pin(async move { Ok(answer) }) + } + + /// The command's removal: refused, nothing removed, while a test holds + /// the owner's lock (`owner_lock_held`); otherwise `remove_restore_target`. + fn remove_restore_target_exclusive<'a>( + &'a self, + key: &'a RestoreTargetKey, + ) -> BoxedFuture<'a, Result> { + if *self.owner_lock_held.lock().unwrap() { + return Box::pin(async { Ok(false) }); + } + self.remove_restore_target(key) + } + + /// The preset `dead_jobs`, plus the `Fake`'s own records that the + /// backends' rule would list: RUNNING ones in the `restore_dead` set, and + /// FAILED ones whose target is still CREATING. + fn list_dead_restore_jobs( + &self, + _stale_after: Duration, + ) -> BoxedFuture<'_, Result, StorageError>> { + if let Some(token) = self.cancel_on_list_dead.lock().unwrap().take() { + token.cancel(); + } + let mut jobs = self.dead_jobs.lock().unwrap().clone(); + let dead = self.restore_dead.lock().unwrap(); + let targets = self.restore_tables.lock().unwrap(); + let mut own: Vec = self + .restore_jobs + .lock() + .unwrap() + .values() + .filter(|r| match r.status { + JobStatus::Running => dead.contains(&r.target.table_id), + JobStatus::Failed => targets.get(&r.target.table_id).is_some_and(|t| !t.active), + _ => false, + }) + .cloned() + .collect(); + own.sort_by(|a, b| a.target.table_id.cmp(&b.target.table_id)); + jobs.extend(own); + Box::pin(async move { Ok(jobs) }) + } + + /// A RUNNING record of the `Fake`'s own is FAILED with the reason; any + /// other key answers the preset `record_still_running`. + /// The RUNNING records of `dead_jobs` and every record of + /// `running_jobs`, by status, whatever their heartbeat. + fn list_running_restore_jobs( + &self, + ) -> BoxedFuture<'_, Result, StorageError>> { + let mut jobs: Vec = self + .dead_jobs + .lock() + .unwrap() + .iter() + .filter(|j| j.status == extenddb_storage::backup_catalog::JobStatus::Running) + .cloned() + .collect(); + jobs.extend(self.running_jobs.lock().unwrap().iter().cloned()); + Box::pin(async move { Ok(jobs) }) + } + + /// Matches as a backend does: only a record still RUNNING under the + /// listed `owner_generation` and `heartbeat_at` (the record's current + /// values are those of its `dead_jobs` entry, moved by + /// `heartbeat_after_listing`). + fn fail_restore_job<'a>( + &'a self, + key: &'a RestoreTargetKey, + owner_generation: u64, + heartbeat_at: f64, + reason: &'a str, + ) -> BoxedFuture<'a, Result> { + self.failed_jobs + .lock() + .unwrap() + .push((key.clone(), reason.to_owned())); + // The fence: the write matches only the claim the listing read + // (`owner_generation`, `heartbeat_at`), a beat that landed since + // (`heartbeat_after_listing`) making it miss. + let listed = self + .dead_jobs + .lock() + .unwrap() + .iter() + .find(|j| j.target == *key) + .map(|j| (j.owner_generation, j.heartbeat_at)); + let beat = self.heartbeat_after_listing.lock().unwrap().remove(key); + let fence_holds = |generation: u64, heartbeat: f64| { + generation == owner_generation && beat.unwrap_or(heartbeat) == heartbeat_at + }; + let answer = match self.restore_jobs.lock().unwrap().get_mut(&key.table_id) { + Some(record) if record.status == JobStatus::Running => { + if fence_holds(record.owner_generation, record.heartbeat_at) { + record.status = JobStatus::Failed; + record.failure_reason = Some(reason.to_owned()); + record.completed_at = Some(seconds_since_epoch()); + true + } else { + false + } + } + Some(_) => false, + None => { + *self.record_still_running.lock().unwrap() + && listed + .is_some_and(|(generation, heartbeat)| fence_holds(generation, heartbeat)) + } + }; + Box::pin(async move { Ok(answer) }) + } + + fn read_store_ids<'a>( + &'a self, + store_kind: StoreKind, + store_root: &'a str, + ) -> BoxedFuture<'a, Result, StorageError>> { + let mut ids: Vec = self + .rows + .lock() + .unwrap() + .values() + .filter(|row| row.store.store_kind == store_kind && row.store.store_root == store_root) + .map(|row| row.store.store_id.clone()) + .collect(); + ids.sort(); + ids.dedup(); + Box::pin(async move { Ok(ids) }) + } + + /// A claim on the `Fake`'s maintenance record: refused naming the holder + /// while `maintenance_claim_held_by` names one, else a `FakeHolder` the + /// test keeps a handle on. + fn maintenance_claim<'a>( + &'a self, + holder: &'a str, + _stale_after: Duration, + ) -> BoxedFuture<'a, Result, StorageError>> { + if let Some(live) = self.maintenance_claim_held_by.lock().unwrap().clone() { + return Box::pin(async move { + Err(StorageError::AlreadyExists(format!( + "the maintenance claim is held by {live:?}" + ))) + }); + } + let fake = Arc::new(FakeHolder::default()); + *fake.lost.lock().unwrap() = *self.next_claim_lost.lock().unwrap(); + self.maintenance_holders + .lock() + .unwrap() + .push(Arc::clone(&fake)); + *self.maintenance_claim_held_by.lock().unwrap() = Some(holder.to_owned()); + let claim: Box = Box::new(ReleasingForward( + Arc::clone(&self.maintenance_claim_held_by), + fake, + )); + Box::pin(async move { Ok(claim) }) + } + + fn check_maintenance_claim( + &self, + stale_after: Duration, + ) -> BoxedFuture<'_, Result> { + self.maintenance_checks.lock().unwrap().push(stale_after); + let held = match *self.maintenance_beat_age.lock().unwrap() { + Some(age) => age < stale_after, + None => *self.maintenance_held.lock().unwrap(), + }; + let error = self.maintenance_error.lock().unwrap().clone(); + let delay = *self.maintenance_delay.lock().unwrap(); + Box::pin(async move { + if let Some(delay) = delay { + tokio::time::sleep(delay).await; + } + match error { + Some(e) => Err(StorageError::Internal(e)), + None => Ok(held), + } + }) + } +} + +impl RestoreEngine for Fake { + /// The backends' load: under the claim's generation and RUNNING, the + /// stream read whole; an `Err` item fails the file with nothing applied; + /// a key the target holds fails the file unless the file is + /// `reapplying`, where the duplicate is skipped; the progress recorded + /// on the record as SQLite records it (`file_index`, the running total). + fn bulk_load<'a>( + &'a self, + target: &'a RestoreTargetKey, + claim: &'a dyn JobClaim, + file: BulkLoadFile, + items: BoxStream<'a, Result>, + ) -> BoxedFuture<'a, Result> { + Box::pin(async move { + self.bulk_loads_started + .fetch_add(1, std::sync::atomic::Ordering::AcqRel); + let gate = self.bulk_load_gate.lock().unwrap().clone(); + if let Some(gate) = gate { + gate.acquire() + .await + .expect("the gate is never closed") + .forget(); + } + let mut call = BulkLoadCall { + table_id: target.table_id.clone(), + file, + items: Vec::new(), + stream_error: None, + }; + let mut items = items; + let mut failed = None; + let item_gate = self.bulk_load_item_gate.lock().unwrap().clone(); + loop { + if let Some(gate) = &item_gate { + gate.acquire() + .await + .expect("the gate is never closed") + .forget(); + } + let Some(next) = items.next().await else { + break; + }; + match next { + Ok(item) => { + self.bulk_load_items_seen + .fetch_add(1, std::sync::atomic::Ordering::AcqRel); + call.items.push(item); + } + Err(e) => { + call.stream_error = Some(e.to_string()); + failed = Some(e); + break; + } + } + } + let streamed = std::mem::take(&mut call.items); + call.items.clone_from(&streamed); + self.bulk_loads.lock().unwrap().push(call); + if let Some(e) = failed { + return Err(e); + } + let owned = self + .restore_jobs + .lock() + .unwrap() + .get(&target.table_id) + .is_some_and(|r| { + r.owner_generation == claim.generation() && r.status == JobStatus::Running + }); + if !owned { + return Err(StorageError::Internal( + "the restore record is not RUNNING under this claim".to_owned(), + )); + } + let mut targets = self.restore_tables.lock().unwrap(); + let Some(table) = targets.get_mut(&target.table_id) else { + return Err(StorageError::TableNotFound(target.table_name.clone())); + }; + let mut keys: std::collections::HashSet = table + .items + .iter() + .map(|i| Fake::item_key(&table.key_attributes, i)) + .collect(); + let mut applied = 0; + for item in streamed { + let key = Fake::item_key(&table.key_attributes, &item); + if !keys.insert(key.clone()) { + if file.reapplying { + continue; + } + return Err(StorageError::Validation(format!( + "holds a primary key the target already has (the backup holds a key \ + twice): {key}" + ))); + } + table.items.push(item); + applied += 1; + } + drop(targets); + if let Some(record) = self.restore_jobs.lock().unwrap().get_mut(&target.table_id) { + record.file_index = Some(file.file_index); + record.items_applied = Some(file.start_at + applied); + } + Ok(applied) + }) + } + + /// DONE and ACTIVE in one step under the claim's generation, RUNNING and + /// a CREATING target; `Pending` when the test says the backend's worker + /// flips the table. A queued `FinishError` makes the call fail, after + /// the flip landed or before anything was written. + fn finish_restore<'a>( + &'a self, + target: &'a RestoreTargetKey, + claim: &'a dyn JobClaim, + item_count: u64, + size_bytes: u64, + ) -> BoxedFuture<'a, Result> { + self.finish_sizes.lock().unwrap().push(size_bytes); + let error = self.finish_errors.lock().unwrap().pop_front(); + if error == Some(FinishError::TakenAndDone) + && let Some(record) = self.restore_jobs.lock().unwrap().get_mut(&target.table_id) + && record.owner_generation == claim.generation() + { + record.owner_generation = new_generation(); + record.status = JobStatus::Done; + } + if matches!( + error, + Some(FinishError::BeforeFlip | FinishError::TakenAndDone) + ) { + return Box::pin(async { + Err(StorageError::Internal( + "the claim connection failed before the DONE write".to_owned(), + )) + }); + } + let mut records = self.restore_jobs.lock().unwrap(); + let mut targets = self.restore_tables.lock().unwrap(); + let finished = match ( + records.get_mut(&target.table_id), + targets.get_mut(&target.table_id), + ) { + (Some(record), Some(table)) + if record.owner_generation == claim.generation() + && record.status == JobStatus::Running + && !table.active => + { + record.status = JobStatus::Done; + record.completed_at = Some(seconds_since_epoch()); + let pending = self + .finish_pending + .load(std::sync::atomic::Ordering::Acquire); + if !pending { + table.active = true; + if let Some(desc) = self.tables.lock().unwrap().get_mut(&table.name) { + desc.table_status = TableStatus::Active; + desc.item_count = i64::try_from(item_count).unwrap_or(i64::MAX); + for gsi in desc.global_secondary_indexes.iter_mut().flatten() { + gsi.index_status = "ACTIVE".to_owned(); + } + } + } + self.progress_rows.lock().unwrap().remove(&target.table_id); + if pending { + Finished::Pending + } else { + Finished::Active + } + } + (Some(record), _) => Finished::NotMatched(format!( + "the record is {} under generation {} (the claim's is {})", + record.status, + record.owner_generation, + claim.generation() + )), + (None, _) => Finished::NotMatched("the record is gone".to_owned()), + }; + Box::pin(async move { + if error == Some(FinishError::AfterFlip) { + return Err(StorageError::Internal( + "the data pool failed after the flip's commit".to_owned(), + )); + } + Ok(finished) + }) + } + + fn count_restore_target<'a>( + &'a self, + key: &'a RestoreTargetKey, + ) -> BoxedFuture<'a, Result> { + self.count_calls + .fetch_add(1, std::sync::atomic::Ordering::AcqRel); + let gate = self.count_gate.lock().unwrap().clone(); + Box::pin(async move { + if let Some(gate) = gate { + gate.acquire() + .await + .expect("the gate is never closed") + .forget(); + } + // Take one error from the budget, if any is left: a + // compare-exchange loop, since `fetch_update` is deprecated on + // current stable and its replacement is not on 1.88. + let mut left = self.count_errors.load(std::sync::atomic::Ordering::Acquire); + while let Some(next) = left.checked_sub(1) { + match self.count_errors.compare_exchange( + left, + next, + std::sync::atomic::Ordering::AcqRel, + std::sync::atomic::Ordering::Acquire, + ) { + Ok(_) => { + return Err(StorageError::Internal( + "count restore target: the read timed out".to_owned(), + )); + } + Err(now) => left = now, + } + } + match *self.count_override.lock().unwrap() { + Some(count) => Ok(count), + None => self + .restore_tables + .lock() + .unwrap() + .get(&key.table_id) + .map(|t| t.items.len() as u64) + .ok_or_else(|| StorageError::TableNotFound(key.table_name.clone())), + } + }) + } + + fn read_restore_progress<'a>( + &'a self, + key: &'a RestoreTargetKey, + ) -> BoxedFuture<'a, Result, StorageError>> { + let rows = self + .progress_rows + .lock() + .unwrap() + .get(&key.table_id) + .cloned() + .unwrap_or_default(); + Box::pin(async move { Ok(rows) }) + } + + fn restore_target_ready<'a>( + &'a self, + key: &'a RestoreTargetKey, + ) -> BoxedFuture<'a, Result> { + // The record must name a restore target with this id, as the + // backends' catalog read requires; a target the test put in + // `targets_without_tables` is one whose data DDL never ran (the + // owner died between the registration's commit and the DDL). + let known = self + .restore_jobs + .lock() + .unwrap() + .contains_key(&key.table_id); + let ready = !self + .targets_without_tables + .lock() + .unwrap() + .contains(&key.table_id); + let table_name = key.table_name.clone(); + let table_id = key.table_id.clone(); + Box::pin(async move { + if known { + Ok(ready) + } else { + Err(StorageError::TableNotFound(format!( + "{table_name} is not a CREATING restore target with table id {table_id}" + ))) + } + }) + } + + fn sweep_restore_progress<'a>( + &'a self, + running: &'a [String], + ) -> BoxedFuture<'a, Result> { + self.progress_sweeps.lock().unwrap().push(running.to_vec()); + let mut rows = self.progress_rows.lock().unwrap(); + let before = rows.len(); + rows.retain(|id, _| running.contains(id)); + let removed = (before - rows.len()) as u64; + Box::pin(async move { Ok(removed) }) + } +} + +/// A vector-search engine the `Fake` hands out when it stands for a target +/// with vector support; never searched. +struct CapableVectorSearch; + +impl extenddb_storage::VectorSearchEngine for CapableVectorSearch { + fn search_vectors( + &self, + _req: extenddb_storage::VectorSearch<'_>, + ) -> BoxedFuture<'_, extenddb_storage::VectorSearchResult> { + unused() + } +} + +static CAPABLE_VECTOR_SEARCH: CapableVectorSearch = CapableVectorSearch; + +/// The restore's vector gate reads `as_vector_search`; nothing else of the +/// data plane is used by these tests. +impl DataEngine for Fake { + fn put_item( + &self, + _: &TableKeyInfo, + _: Item, + _: bool, + _: Option<&extenddb_core::expression::Expr>, + _: &extenddb_core::expression::ExpressionMaps, + _: Option<&extenddb_storage::StreamCapture>, + ) -> BoxedFuture<'_, Result, StorageError>> { + unused() + } + + fn get_item( + &self, + _: &TableKeyInfo, + _: &Item, + ) -> BoxedFuture<'_, Result, StorageError>> { + unused() + } + + fn delete_item( + &self, + _: &TableKeyInfo, + _: &Item, + _: bool, + _: Option<&extenddb_core::expression::Expr>, + _: &extenddb_core::expression::ExpressionMaps, + _: Option<&extenddb_storage::StreamCapture>, + ) -> BoxedFuture<'_, Result, StorageError>> { + unused() + } + + fn update_item( + &self, + _: &TableKeyInfo, + _: &Item, + _: &[extenddb_core::expression::UpdateAction], + _: bool, + _: bool, + _: Option<&extenddb_core::expression::Expr>, + _: &extenddb_core::expression::ExpressionMaps, + _: Option<&extenddb_storage::StreamCapture>, + ) -> BoxedFuture<'_, extenddb_storage::ItemPairResult> { + unused() + } + + fn query( + &self, + _: &TableKeyInfo, + _: &extenddb_core::expression::KeyCondition, + _: &extenddb_core::expression::ExpressionMaps, + _: bool, + _: Option, + _: Option<&Item>, + _: Option<&str>, + ) -> BoxedFuture<'_, extenddb_storage::QueryResult> { + unused() + } + + fn scan( + &self, + _: &TableKeyInfo, + _: Option, + _: Option<&Item>, + _: Option, + _: Option, + _: Option<&str>, + ) -> BoxedFuture<'_, extenddb_storage::QueryResult> { + unused() + } + + fn scan_key_in_segment( + &self, + _: &TableKeyInfo, + _: &Item, + _: i64, + _: i64, + _: Option<&str>, + ) -> BoxedFuture<'_, Result> { + unused() + } + + fn as_vector_search(&self) -> Option<&dyn extenddb_storage::VectorSearchEngine> { + self.vector_search + .load(std::sync::atomic::Ordering::Acquire) + .then_some(&CAPABLE_VECTOR_SEARCH) + } + + fn transact_get_items( + &self, + _: &[extenddb_storage::TransactGetOp<'_>], + ) -> BoxedFuture<'_, Result>, StorageError>> { + unused() + } + + fn transact_write_items( + &self, + _: &[extenddb_storage::TransactWriteOp<'_>], + _: Option>, + ) -> BoxedFuture<'_, Result<(), StorageError>> { + unused() + } + + fn cleanup_expired_idempotency_tokens( + &self, + _: i64, + ) -> BoxedFuture<'_, Result> { + unused() + } +} + +impl SnapshotEngine for Fake { + /// The fixture for the table, or `TableNotFound`; the description and + /// the items are the fixture's, in its order, through the gate when it + /// has one. + fn open_table_snapshot( + &self, + _account_id: &str, + table_name: &str, + ) -> BoxedFuture<'_, Result> { + self.snapshots_opened + .lock() + .unwrap() + .push(table_name.to_owned()); + if let Some(e) = self.snapshot_error.lock().unwrap().take() { + return Box::pin(async move { Err(e) }); + } + let mut snapshots = self.snapshots.lock().unwrap(); + let Some(fixture) = snapshots.get_mut(table_name) else { + let name = table_name.to_owned(); + return Box::pin(async move { + Err(StorageError::TableNotFound(format!( + "Table not found: {name}" + ))) + }); + }; + let items = std::mem::take(&mut fixture.items); + let gate = fixture.gate.clone(); + let snapshot = TableSnapshot { + description: fixture.description.clone(), + info: extenddb_core::backup_format::SnapshotDescriptor { + kind: "test".to_owned(), + marker: String::new(), + }, + key_ordered: fixture.key_ordered, + items: Box::pin(futures::stream::unfold( + (items.into_iter(), gate), + |(mut items, gate)| async move { + let next = items.next()?; + if let Some(gate) = &gate { + gate.acquire() + .await + .expect("the gate is never closed") + .forget(); + } + Some((next, (items, gate))) + }, + )), + }; + Box::pin(async move { Ok(snapshot) }) + } +} + +impl MetadataEngine for Fake { + fn describe_ttl( + &self, + _account_id: &str, + _table_name: &str, + ) -> BoxedFuture<'_, Result> { + let ttl = self + .ttl + .lock() + .unwrap() + .clone() + .unwrap_or(TimeToLiveDescription { + time_to_live_status: TimeToLiveStatus::Disabled, + attribute_name: None, + }); + Box::pin(async move { Ok(ttl) }) + } + + fn update_ttl( + &self, + _: &str, + _: &str, + _: &str, + _: bool, + ) -> BoxedFuture<'_, Result<(), StorageError>> { + unused() + } + + fn tag_resource( + &self, + _: &str, + _: &[extenddb_core::types::Tag], + ) -> BoxedFuture<'_, Result<(), StorageError>> { + unused() + } + + fn untag_resource(&self, _: &str, _: &[String]) -> BoxedFuture<'_, Result<(), StorageError>> { + unused() + } + + fn list_tags( + &self, + _: &str, + ) -> BoxedFuture<'_, Result, StorageError>> { + unused() + } + + fn tables_with_ttl( + &self, + _: &str, + ) -> BoxedFuture<'_, Result, StorageError>> { + unused() + } + + fn all_tables_with_ttl( + &self, + ) -> BoxedFuture<'_, Result, StorageError>> { + unused() + } + + fn all_tables_with_ttl_index_ready( + &self, + ) -> BoxedFuture<'_, Result, StorageError>> { + unused() + } + + fn create_ttl_index( + &self, + _: &str, + _: &str, + _: &str, + ) -> BoxedFuture<'_, Result<(), StorageError>> { + unused() + } + + fn drop_ttl_index( + &self, + _: &str, + _: &str, + _: &str, + ) -> BoxedFuture<'_, Result<(), StorageError>> { + unused() + } + + fn find_expired_items_indexed( + &self, + _: &str, + _: &str, + _: &str, + _: usize, + ) -> BoxedFuture<'_, Result, StorageError>> { + unused() + } + + fn refresh_table_size(&self, _: &str, _: &str) -> BoxedFuture<'_, Result<(), StorageError>> { + unused() + } + + fn list_active_table_names( + &self, + _: &str, + ) -> BoxedFuture<'_, Result, StorageError>> { + unused() + } + + fn all_active_tables(&self) -> BoxedFuture<'_, Result, StorageError>> { + unused() + } +} + +impl TableEngine for Fake { + fn create_table( + &self, + _account_id: &str, + _input: CreateTableInput, + ) -> BoxedFuture<'_, Result> { + unused() + } + + fn delete_table( + &self, + _account_id: &str, + _input: DeleteTableInput, + ) -> BoxedFuture<'_, Result> { + unused() + } + + /// The account's table of that name, or `TableNotFound`. + fn describe_table( + &self, + _account_id: &str, + input: DescribeTableInput, + ) -> BoxedFuture<'_, Result> { + self.table_reads + .lock() + .unwrap() + .push(input.table_name.clone()); + let found = self.tables.lock().unwrap().get(&input.table_name).cloned(); + Box::pin(async move { + found.ok_or_else(|| { + StorageError::TableNotFound(format!("Table not found: {}", input.table_name)) + }) + }) + } + + fn list_tables( + &self, + _account_id: &str, + _input: ListTablesInput, + ) -> BoxedFuture<'_, Result> { + unused() + } + + fn update_table( + &self, + _account_id: &str, + _input: UpdateTableInput, + ) -> BoxedFuture<'_, Result> { + unused() + } + + fn table_key_info( + &self, + _account_id: &str, + _table_name: &str, + ) -> BoxedFuture<'_, Result> { + unused() + } + + fn index_info( + &self, + _account_id: &str, + _table_name: &str, + _index_name: &str, + ) -> BoxedFuture<'_, Result> { + unused() + } + + fn index_info_by_table_id( + &self, + _table_id: &str, + _index_name: &str, + ) -> BoxedFuture<'_, Result> { + unused() + } +} + +impl LegacyBackupReader for Fake { + fn create( + &self, + _: &str, + table_name: &str, + backup_name: &str, + permit: JobPermit, + ) -> BoxedFuture<'_, Result> { + self.creates + .lock() + .unwrap() + .push((table_name.to_owned(), backup_name.to_owned())); + self.permits.lock().unwrap().push(permit); + Box::pin(async { + Ok(BackupDetails { + backup_arn: arn(ACCOUNT), + backup_name: "nightly".to_owned(), + backup_status: "AVAILABLE".to_owned(), + backup_type: "USER".to_owned(), + backup_size_bytes: 0, + backup_creation_date_time: CREATED, + }) + }) + } + + fn describe( + &self, + _account_id: &str, + backup_arn: &str, + ) -> BoxedFuture<'_, Result> { + let found = self.descriptions.lock().unwrap().get(backup_arn).cloned(); + let backup_arn = backup_arn.to_owned(); + Box::pin(async move { + found.ok_or_else(|| StorageError::Validation(format!("Backup not found: {backup_arn}"))) + }) + } + + fn list( + &self, + _account_id: &str, + table_name: Option<&str>, + ) -> BoxedFuture<'_, Result, StorageError>> { + self.listed_for + .lock() + .unwrap() + .push(table_name.map(str::to_owned)); + let all = self.summaries.lock().unwrap().clone(); + Box::pin(async move { Ok(all) }) + } + + /// Removes the backup and answers its description as the backends do: + /// the row's status (not yet `DELETED`), `SourceTableDetails` without + /// `ProvisionedThroughput`, no feature details. + fn delete( + &self, + _account_id: &str, + backup_arn: &str, + ) -> BoxedFuture<'_, Result> { + self.deleted.lock().unwrap().push(backup_arn.to_owned()); + let found = self.descriptions.lock().unwrap().remove(backup_arn); + self.sources.lock().unwrap().remove(backup_arn); + let backup_arn = backup_arn.to_owned(); + Box::pin(async move { + found.ok_or_else(|| StorageError::Validation(format!("Backup not found: {backup_arn}"))) + }) + } + + fn restore_source( + &self, + _account_id: &str, + backup_arn: &str, + ) -> BoxedFuture<'_, Result> { + *self.source_reads.lock().unwrap() += 1; + let err = self.source_error.lock().unwrap().take(); + let found = self.sources.lock().unwrap().get(backup_arn).cloned(); + let backup_arn = backup_arn.to_owned(); + let gate = self.source_gate.clone(); + Box::pin(async move { + if let Some(gate) = gate { + let _permit = gate.acquire().await.expect("gate closed"); + } + if let Some(e) = err { + return Err(e); + } + found.ok_or_else(|| StorageError::Validation(format!("Backup not found: {backup_arn}"))) + }) + } + + fn restore( + &self, + _account_id: &str, + request: RestoreRequest, + permit: JobPermit, + ) -> BoxedFuture<'_, Result> { + let err = self.restore_error.lock().unwrap().take(); + let desc = described(&request); + self.restores.lock().unwrap().push(request); + if err.is_none() { + self.permits.lock().unwrap().push(permit); + } + Box::pin(async move { + match err { + Some(e) => Err(e), + None => Ok(desc), + } + }) + } + + fn abandoned_creating_backups( + &self, + _stale_after: Duration, + ) -> BoxedFuture<'_, Result, StorageError>> { + let found = self.abandoned_backups.lock().unwrap().clone(); + Box::pin(async move { Ok(found) }) + } +} + +/// What a backend reports for a CREATING target: deliberately the shape the +/// engine must correct (ACTIVE GSI status, a pay-per-request timestamp, a +/// backend-side summary with the wrong time). +fn described(request: &RestoreRequest) -> TableDescription { + let input = &request.input; + TableDescription { + table_name: input.table_name.clone(), + key_schema: input.key_schema.clone(), + attribute_definitions: input.attribute_definitions.clone(), + table_status: TableStatus::Creating, + billing_mode_summary: Some(BillingModeSummary { + billing_mode: input.billing_mode.unwrap_or(BillingMode::Provisioned), + last_update_to_pay_per_request_date_time: Some(1.0), + }), + provisioned_throughput: ProvisionedThroughputDescription { + read_capacity_units: 9, + write_capacity_units: 9, + ..Default::default() + }, + global_secondary_indexes: input.global_secondary_indexes.as_ref().map(|gsis| { + gsis.iter() + .map(|g| GsiDescription { + index_name: g.index_name.clone(), + key_schema: g.key_schema.clone(), + projection: g.projection.clone(), + index_status: "ACTIVE".to_owned(), + provisioned_throughput: Some(ProvisionedThroughputDescription { + read_capacity_units: 9, + write_capacity_units: 9, + ..Default::default() + }), + index_size_bytes: 0, + item_count: 0, + index_arn: String::new(), + }) + .collect() + }), + restore_summary: Some(RestoreSummary { + source_backup_arn: Some("wrong".to_owned()), + source_table_arn: None, + restore_date_time: 42.0, + restore_in_progress: false, + }), + ..Default::default() + } +} + +// --- fixtures --------------------------------------------------------------- + +fn key(name: &str, key_type: KeyType) -> KeySchemaElement { + KeySchemaElement { + attribute_name: name.to_owned(), + key_type, + } +} + +fn attr(name: &str) -> AttributeDefinition { + AttributeDefinition { + attribute_name: name.to_owned(), + attribute_type: ScalarAttributeType::S, + } +} + +fn throughput(read: i64, write: i64) -> ProvisionedThroughput { + ProvisionedThroughput { + read_capacity_units: read, + write_capacity_units: write, + } +} + +fn gsi(name: &str, pk: &str, pt: Option) -> GsiInput { + GsiInput { + index_name: name.to_owned(), + key_schema: vec![key(pk, KeyType::Hash)], + projection: Projection { + projection_type: ProjectionType::All, + non_key_attributes: None, + }, + provisioned_throughput: pt, + } +} + +fn definition(billing: &str) -> BackupTableDefinition { + let provisioned = billing == "PROVISIONED"; + BackupTableDefinition { + version: BACKUP_DEFINITION_VERSION, + billing_mode: billing.to_owned(), + provisioned_throughput: provisioned.then(|| throughput(7, 8)), + global_secondary_indexes: vec![gsi("gsi1", "gpk", provisioned.then(|| throughput(3, 4)))], + local_secondary_indexes: vec![LsiInput { + index_name: "lsi1".to_owned(), + key_schema: vec![key("pk", KeyType::Hash), key("lsk", KeyType::Range)], + projection: Projection { + projection_type: ProjectionType::KeysOnly, + non_key_attributes: None, + }, + }], + table_class: None, + sse_specification: None, + on_demand_throughput: None, + vector_index_names: Vec::new(), + time_to_live: None, + stream_specification: None, + } +} + +fn source(definition: Option) -> BackupRestoreSource { + let billing = definition + .as_ref() + .map_or("PROVISIONED".to_owned(), |d| d.billing_mode.clone()); + BackupRestoreSource { + backup_arn: arn(ACCOUNT), + backup_creation_date_time: CREATED, + source_table_name: "Music".to_owned(), + source_table_arn: source_arn(), + source_table_id: "tid".to_owned(), + key_schema: vec![key("pk", KeyType::Hash), key("sk", KeyType::Range)], + attribute_definitions: if definition.is_some() { + vec![attr("pk"), attr("sk"), attr("gpk"), attr("lsk")] + } else { + vec![attr("pk"), attr("sk")] + }, + provisioned_throughput: definition + .as_ref() + .and_then(|d| d.provisioned_throughput.clone()), + billing_mode: billing, + item_count: 3, + table_size_bytes: 30, + table_creation_date_time: 1.0, + definition, + legacy_vector_index_count: 0, + } +} + +fn restore_body(extra: Value) -> Value { + let mut body = json!({ "TargetTableName": "MusicRestored", "BackupArn": arn(ACCOUNT) }); + for (k, v) in extra.as_object().unwrap() { + body[k] = v.clone(); + } + body +} + +async fn restore(fake: &Arc, body: Value) -> Result { + restore_on(fake, &runtime(), &body).await +} + +/// `restore` on `runtime` rather than the tests' default one. +async fn restore_on( + fake: &Arc, + runtime: &BackupRuntime, + body: &Value, +) -> Result { + restore_table_from_backup( + fake, + runtime, + ACCOUNT, + REGION, + &LimitsConfig::default(), + body, + ) + .await + .map(|(_, v)| v) +} + +fn validation_message(err: DynamoDbError) -> String { + match err { + DynamoDbError::ValidationException(m) => m, + other => panic!("expected ValidationException, got {other:?}"), + } +} + +// --- BackupArn --------------------------------------------------------------- + +/// The ARN the engine looks a `BackupArn` up under first: the one this +/// deployment issues for the backup named. +fn arn_field(body: &Value) -> Result { + backup_arn_field(body, ACCOUNT, REGION).map(|a| a.canonical().to_owned()) +} + +/// The spellings of an ARN as `backup_arn_field` resolves them for `arn`. +fn own(arn: &str) -> BackupArnSpellings { + backup_arn_field(&json!({ "BackupArn": arn }), ACCOUNT, REGION).unwrap() +} + +fn limits() -> LimitsConfig { + LimitsConfig::default() +} + +#[test] +fn own_account_arn_is_accepted() { + let body = json!({ "BackupArn": arn(ACCOUNT) }); + assert_eq!(arn_field(&body).unwrap(), arn(ACCOUNT)); +} + +#[test] +fn other_account_arn_is_denied() { + let body = json!({ "BackupArn": arn("999999999999") }); + let err = arn_field(&body).unwrap_err(); + assert!( + matches!(err, DynamoDbError::AccessDeniedException(_)), + "{err:?}" + ); +} + +#[test] +fn account_prefix_does_not_match() { + let body = json!({ "BackupArn": arn("12345678901") }); + assert!(matches!( + arn_field(&body).unwrap_err(), + DynamoDbError::AccessDeniedException(_) + )); +} + +/// Measured on DynamoDB: the shapes the service refuses as `ValidationException` +/// with `Invalid Request: BackupArn is not valid` (not an ARN, oversize, +/// control characters, a slash or `..` in the region) or `Invalid Backup ARN` +/// (an id suffix of the wrong length or not hex, a wildcard id), on every +/// member that carries a backup ARN. Which text goes with which input was +/// not captured, so the split is the parser's (see `BackupArnError`). +#[tokio::test] +async fn malformed_arn_is_a_validation_exception_with_the_services_text() { + const NOT_VALID: &str = "Invalid Request: BackupArn is not valid"; + const INVALID_ID: &str = "Invalid Backup ARN"; + let id = "01791976431438-73d8d5bc"; + let long_table = "x".repeat(1000); + let cases: Vec<(String, &str)> = vec![ + ("not-an-arn".to_owned(), NOT_VALID), + ("arn:aws:dynamodb".to_owned(), NOT_VALID), + (format!("arn:aws:dynamodb:{REGION}"), NOT_VALID), + (String::new(), NOT_VALID), + ( + format!("arn:aws:dynamodb:{REGION}::table/Music/backup/{id}"), + NOT_VALID, + ), + ( + format!("arn:aws:dynamodb:{REGION}:table/Music/backup/{ACCOUNT}"), + NOT_VALID, + ), + ( + format!("arn:aws:dynamodb:{REGION}:{ACCOUNT}:table/Music"), + NOT_VALID, + ), + ( + format!("arn:aws:dynamodb:{REGION}:{ACCOUNT}:table/Music/stream/{id}"), + NOT_VALID, + ), + ( + format!("arn:aws:dynamodb:{REGION}:999999999999:table/Music/index/x"), + NOT_VALID, + ), + // Oversize (the model's max is 1024). + ( + format!("arn:aws:dynamodb:{REGION}:{ACCOUNT}:table/{long_table}/backup/{id}"), + NOT_VALID, + ), + // Control characters, inside and at the end (a NUL byte was a 500 on + // PostgreSQL before the check). + ( + format!("arn:aws:dynamodb:{REGION}:{ACCOUNT}:table/Music/backup/{id}\u{0}"), + NOT_VALID, + ), + ( + format!("arn:aws:dynamodb:{REGION}:{ACCOUNT}:table/Mu\u{1}sic/backup/{id}"), + NOT_VALID, + ), + ( + format!("arn:aws:dynamodb:{REGION}:{ACCOUNT}:table/Music/backup/{id}\n\u{7}"), + NOT_VALID, + ), + // A slash or `..` in the region. + ( + format!("arn:aws:dynamodb:us/east-1:{ACCOUNT}:table/Music/backup/{id}"), + NOT_VALID, + ), + ( + format!("arn:aws:dynamodb:..:{ACCOUNT}:table/Music/backup/{id}"), + NOT_VALID, + ), + ( + format!("arn:aws:dynamodb:us-east-1/..:{ACCOUNT}:table/Music/backup/{id}"), + NOT_VALID, + ), + // The id: wrong length, not hex, a wildcard. + ( + format!("arn:aws:dynamodb:{REGION}:{ACCOUNT}:table/Music/backup/1"), + INVALID_ID, + ), + ( + format!("arn:aws:dynamodb:{REGION}:{ACCOUNT}:table/Music/backup/*"), + INVALID_ID, + ), + ( + format!("arn:aws:dynamodb:{REGION}:{ACCOUNT}:table/Music/backup/01791976431438-*"), + INVALID_ID, + ), + ( + format!( + "arn:aws:dynamodb:{REGION}:{ACCOUNT}:table/Music/backup/01791976431438-73d8d5b" + ), + INVALID_ID, + ), + ( + format!( + "arn:aws:dynamodb:{REGION}:{ACCOUNT}:table/Music/backup/01791976431438-73d8d5bcd" + ), + INVALID_ID, + ), + ( + format!( + "arn:aws:dynamodb:{REGION}:{ACCOUNT}:table/Music/backup/01791976431438-73D8D5BC" + ), + INVALID_ID, + ), + ( + format!( + "arn:aws:dynamodb:{REGION}:{ACCOUNT}:table/Music/backup/01791976431438-73d8d5bg" + ), + INVALID_ID, + ), + ( + format!( + "arn:aws:dynamodb:{REGION}:{ACCOUNT}:table/Music/backup/0179197643143x-73d8d5bc" + ), + INVALID_ID, + ), + ]; + let fake = Fake::with_source(source(Some(definition("PROVISIONED")))); + for (candidate, expected) in &cases { + for (member, result) in [ + ("BackupArn", arn_field(&json!({ "BackupArn": candidate }))), + ( + "RestoreTableFromBackup", + restore(&fake, restore_body(json!({ "BackupArn": candidate }))) + .await + .map(|_| String::new()), + ), + ( + "ExclusiveStartBackupArn", + list(&fake, json!({ "ExclusiveStartBackupArn": candidate })) + .await + .map(|_| String::new()), + ), + ] { + let msg = validation_message(result.unwrap_err()); + assert_eq!(msg, *expected, "{member} {candidate:?}"); + } + } + assert_eq!(fake.source_reads(), 0); + assert!(fake.restores().is_empty()); +} + +/// Measured on DynamoDB: the service trims a trailing newline and resolves an ARN +/// naming another region of the caller's account. The engine hands the +/// backend the ARN it issued: trimmed, in its own region. +#[tokio::test] +async fn whitespace_is_trimmed_and_another_region_resolves() { + let ours = arn(ACCOUNT); + for candidate in [ + format!("{ours}\n"), + format!(" {ours}\t\r\n"), + format!("arn:aws:dynamodb:eu-west-1:{ACCOUNT}:table/Music/backup/01791976431438-73d8d5bc"), + format!( + "arn:aws:dynamodb:eu-west-1:{ACCOUNT}:table/Music/backup/01791976431438-73d8d5bc\n" + ), + ] { + assert_eq!( + arn_field(&json!({ "BackupArn": candidate })).unwrap(), + ours, + "{candidate:?}" + ); + let fake = Fake::with_source(source(Some(definition("PROVISIONED")))); + restore(&fake, restore_body(json!({ "BackupArn": candidate }))) + .await + .unwrap_or_else(|e| panic!("{candidate:?}: {e:?}")); + assert_eq!(fake.restores()[0].backup_arn, ours, "{candidate:?}"); + } + // Another region of another account is still that account's. + let foreign = + "arn:aws:dynamodb:eu-west-1:999999999999:table/Music/backup/01791976431438-73d8d5bc"; + assert!(matches!( + arn_field(&json!({ "BackupArn": foreign })).unwrap_err(), + DynamoDbError::AccessDeniedException(_) + )); + // Whitespace inside is not trimmed. + let inside = + format!("arn:aws:dynamodb:{REGION}:{ACCOUNT}:table/Mu sic/backup/01791976431438-73d8d5bc"); + assert_eq!( + validation_message(arn_field(&json!({ "BackupArn": inside })).unwrap_err()), + "Invalid Request: BackupArn is not valid" + ); +} + +/// A backup created under an earlier `server.region` is stored with that +/// region in its ARN, which `ListBackups` reports as is. `DescribeBackup`, +/// `DeleteBackup`, and `RestoreTableFromBackup` look the ARN up rebuilt in +/// the server's region first and, when that is not found, as written: the +/// stored backup stays reachable by the ARN the server reports for it, and +/// the engine acts on whichever spelling resolved. +#[tokio::test] +async fn a_backup_stored_under_another_region_resolves_as_written() { + let stored = + format!("arn:aws:dynamodb:eu-west-1:{ACCOUNT}:table/Music/backup/01791976431438-73d8d5bc"); + let ours = arn(ACCOUNT); + let spellings = own(&stored); + assert_eq!(spellings.canonical(), ours); + assert_eq!(spellings.as_written.as_deref(), Some(stored.as_str())); + // The server's own spelling has no second spelling to try. + assert_eq!(own(&ours).as_written, None); + assert_eq!(own(&format!(" {ours}\n")).as_written, None, "trimmed first"); + + let stored_source = || { + let mut src = source(Some(definition("PROVISIONED"))); + src.backup_arn.clone_from(&stored); + src + }; + let with_stored = || { + let fake = Fake::with_source(stored_source()); + fake.descriptions.lock().unwrap().insert( + stored.clone(), + backup_description("AVAILABLE", "PROVISIONED"), + ); + fake + }; + + // DescribeBackup: canonical not found, as written found. + let fake = with_stored(); + let desc = describe_backup(&*fake, ACCOUNT, REGION, &spellings) + .await + .unwrap(); + assert_eq!(desc.backup_details.backup_status, "AVAILABLE"); + assert!( + desc.source_table_feature_details.is_some(), + "source read under the stored ARN" + ); + assert_eq!(fake.source_reads(), 1); + + // RestoreTableFromBackup: registered against the stored spelling. + let fake = with_stored(); + restore(&fake, restore_body(json!({ "BackupArn": stored }))) + .await + .unwrap(); + assert_eq!(fake.restores()[0].backup_arn, stored); + assert_eq!( + fake.source_reads(), + 2, + "canonical (not found), then as written" + ); + + // DeleteBackup: the full description, and the stored row is gone. + let fake = with_stored(); + let deleted = delete_backup(&*fake, &runtime(), ACCOUNT, REGION, &spellings) + .await + .unwrap(); + assert_eq!(deleted.backup_details.backup_status, "DELETED"); + assert!(deleted.source_table_feature_details.is_some()); + assert!(fake.descriptions.lock().unwrap().is_empty()); + assert!(fake.sources.lock().unwrap().is_empty()); + + // The canonical spelling wins when both exist: the backend is asked for + // it alone (one read), and the stored other-region row is untouched. + let fake = with_stored(); + fake.sources + .lock() + .unwrap() + .insert(ours.clone(), source(Some(definition("PAY_PER_REQUEST")))); + fake.descriptions.lock().unwrap().insert( + ours.clone(), + backup_description("AVAILABLE", "PAY_PER_REQUEST"), + ); + restore(&fake, restore_body(json!({ "BackupArn": stored }))) + .await + .unwrap(); + assert_eq!(fake.restores()[0].backup_arn, ours); + assert_eq!(fake.source_reads(), 1); + let deleted = delete_backup(&*fake, &runtime(), ACCOUNT, REGION, &spellings) + .await + .unwrap(); + assert_eq!(deleted.backup_details.backup_status, "DELETED"); + assert!(fake.descriptions.lock().unwrap().contains_key(&stored)); + assert!(!fake.descriptions.lock().unwrap().contains_key(&ours)); + + // Neither spelling: BackupNotFoundException from the backend, after both + // were tried; the server's own spelling is tried once. + let fake = Arc::new(Fake::default()); + for (body, reads) in [(stored.clone(), 2), (ours.clone(), 1)] { + let before = fake.source_reads(); + let err = restore(&fake, restore_body(json!({ "BackupArn": body }))) + .await + .unwrap_err(); + assert!( + matches!(err, DynamoDbError::BackupNotFoundException(_)), + "{err:?}" + ); + assert_eq!(fake.source_reads() - before, reads); + assert!(matches!( + describe_backup(&*fake, ACCOUNT, REGION, &own(&body)) + .await + .unwrap_err(), + DynamoDbError::BackupNotFoundException(_) + )); + assert!(matches!( + delete_backup(&*fake, &runtime(), ACCOUNT, REGION, &own(&body)) + .await + .unwrap_err(), + DynamoDbError::BackupNotFoundException(_) + )); + } + + // An error other than not-found on the canonical spelling is not a + // reason to try the other one. + let fake = with_stored(); + *fake.source_error.lock().unwrap() = + Some(StorageError::Connection("connection refused".to_owned())); + let err = restore(&fake, restore_body(json!({ "BackupArn": stored }))) + .await + .unwrap_err(); + assert!( + matches!(err, DynamoDbError::InternalServerError(_)), + "{err:?}" + ); + assert_eq!(fake.source_reads(), 1); + assert!(fake.restores().is_empty()); +} + +/// An ARN in another partition names nothing here: `ValidationException` +/// (the service's class for one naming the caller's account), before the +/// account is compared, on every member that carries a backup ARN. +#[tokio::test] +async fn an_arn_in_another_partition_is_a_validation_exception() { + let ours = arn(ACCOUNT); + let fake = Fake::with_source(source(Some(definition("PROVISIONED")))); + fake.descriptions + .lock() + .unwrap() + .insert(ours.clone(), backup_description("AVAILABLE", "PROVISIONED")); + *fake.summaries.lock().unwrap() = vec![summary(1, 1)]; + for partition in ["aws-cn", "aws-us-gov", "aws-iso", "AWS", "x"] { + for account in [ACCOUNT, "999999999999"] { + let candidate = format!( + "arn:{partition}:dynamodb:{REGION}:{account}:table/Music/backup/01791976431438-73d8d5bc" + ); + let err = arn_field(&json!({ "BackupArn": candidate })).unwrap_err(); + assert_eq!( + validation_message(err), + "Invalid Request: BackupArn is not valid" + ); + let err = restore(&fake, restore_body(json!({ "BackupArn": candidate }))) + .await + .unwrap_err(); + assert!( + matches!(err, DynamoDbError::ValidationException(_)), + "{err:?}" + ); + let err = list(&fake, json!({ "ExclusiveStartBackupArn": candidate })) + .await + .unwrap_err(); + assert!( + matches!(err, DynamoDbError::ValidationException(_)), + "{err:?}" + ); + } + } + assert_eq!(fake.source_reads(), 0); + assert!(fake.restores().is_empty()); + // Liveness: the same ARN in this server's partition resolves. + assert_eq!( + ours, + extenddb_storage::util::backup_arn(REGION, ACCOUNT, "Music", "01791976431438-73d8d5bc") + ); + assert_eq!(arn_field(&json!({ "BackupArn": ours })).unwrap(), ours); + restore(&fake, restore_body(json!({}))).await.unwrap(); + assert_eq!(fake.restores().len(), 1); +} + +#[test] +fn missing_arn_field_is_a_validation_error() { + let err = arn_field(&json!({})).unwrap_err(); + assert!(matches!(err, DynamoDbError::ValidationException(m) if m.contains("backupArn"))); +} + +/// A request dropped while the engine is still reading the backup (the client +/// disconnected) does not cancel the restore: it still reaches the backend. +#[tokio::test] +async fn a_restore_request_dropped_while_the_backup_is_read_still_restores() { + use std::future::Future; + use std::sync::Arc; + use std::task::{Context, Waker}; + + let gate = Arc::new(tokio::sync::Semaphore::new(0)); + let mut fake = Fake::default(); + fake.sources + .lock() + .unwrap() + .insert(arn(ACCOUNT), source(Some(definition("PROVISIONED")))); + fake.source_gate = Some(Arc::clone(&gate)); + let fake = Arc::new(fake); + + let after_ran = Arc::new(tokio::sync::Notify::new()); + let notify = Arc::clone(&after_ran); + let mut request = Box::pin(restore_table_from_backup_detached( + Arc::clone(&fake), + Arc::new(runtime()), + Arc::from(ACCOUNT), + Arc::from(REGION), + Arc::new(LimitsConfig::default()), + restore_body(json!({})), + move |_, target| async move { + assert_eq!(target, "MusicRestored"); + notify.notify_one(); + }, + )); + // Poll once so the restore starts, wait until it is blocked reading the + // backup, then drop the request as a disconnecting client's would be. + let mut cx = Context::from_waker(Waker::noop()); + assert!(request.as_mut().poll(&mut cx).is_pending()); + for _ in 0..200 { + if fake.source_reads() == 1 { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(5)).await; + } + assert_eq!(fake.source_reads(), 1, "the restore never read the backup"); + drop(request); + assert!( + fake.restores().is_empty(), + "precondition: not registered yet" + ); + + gate.add_permits(1); + tokio::time::timeout(std::time::Duration::from_secs(5), after_ran.notified()) + .await + .expect("the dropped request's restore never reached the backend"); + assert_eq!(fake.restores().len(), 1); +} + +#[tokio::test] +async fn restore_checks_the_arn_before_the_target_name() { + let fake = Fake::with_source(source(Some(definition("PROVISIONED")))); + // "x" is shorter than the 3-character minimum table name. + let malformed = json!({ "TargetTableName": "x", "BackupArn": "garbage" }); + let msg = validation_message(restore(&fake, malformed).await.unwrap_err()); + assert!(msg.contains("BackupArn"), "{msg}"); + + let foreign = json!({ "TargetTableName": "x", "BackupArn": arn("999999999999") }); + assert!(matches!( + restore(&fake, foreign).await.unwrap_err(), + DynamoDbError::AccessDeniedException(_) + )); + + let missing_name = json!({ "BackupArn": arn("999999999999") }); + assert!(matches!( + restore(&fake, missing_name).await.unwrap_err(), + DynamoDbError::AccessDeniedException(_) + )); + + // With a good ARN, the bad name is what is reported. + let bad_name = json!({ "TargetTableName": "x", "BackupArn": arn(ACCOUNT) }); + let msg = validation_message(restore(&fake, bad_name).await.unwrap_err()); + assert!(!msg.contains("BackupArn"), "{msg}"); + assert_eq!(fake.source_reads(), 0, "no storage read for a bad request"); + assert!(fake.restores().is_empty()); +} + +// --- overrides --------------------------------------------------------------- + +#[test] +fn restore_request_without_overrides_is_accepted() { + assert!( + parse_restore_table_overrides(&restore_body(json!({}))) + .unwrap() + .is_empty() + ); +} + +#[test] +fn provisioned_override_with_throughput_is_accepted() { + let overrides = parse_restore_table_overrides(&json!({ + "BillingModeOverride": "PROVISIONED", + "ProvisionedThroughputOverride": { "ReadCapacityUnits": 5, "WriteCapacityUnits": 5 }, + })) + .unwrap(); + validate_override_shape(&overrides).unwrap(); + validate_override_for_source(&overrides, "PAY_PER_REQUEST").unwrap(); + assert_eq!(overrides.billing_mode, Some(BillingMode::Provisioned)); + assert_eq!(overrides.provisioned_throughput, Some(throughput(5, 5))); +} + +#[test] +fn provisioned_override_without_throughput_is_refused() { + let overrides = + parse_restore_table_overrides(&json!({ "BillingModeOverride": "PROVISIONED" })).unwrap(); + assert!(matches!( + validate_override_shape(&overrides).unwrap_err(), + DynamoDbError::ValidationException(_) + )); +} + +#[test] +fn throughput_override_under_pay_per_request_is_refused() { + let overrides = parse_restore_table_overrides(&json!({ + "BillingModeOverride": "PAY_PER_REQUEST", + "ProvisionedThroughputOverride": { "ReadCapacityUnits": 5, "WriteCapacityUnits": 5 }, + })) + .unwrap(); + let msg = + validation_message(validate_override_for_source(&overrides, "PROVISIONED").unwrap_err()); + assert!(msg.contains( + "ProvisionedThroughputOverride can only be specified with BillingModeOverride PROVISIONED" + )); +} + +#[test] +fn invalid_provisioned_throughput_override_matches_create_table() { + let expected = "One or more parameter values were invalid: ReadCapacityUnits and \ + WriteCapacityUnits must both be greater than or equal to 1 for table"; + for (read, write) in [(0, 5), (5, 0), (-1, 5), (5, -1)] { + let overrides = parse_restore_table_overrides(&json!({ + "BillingModeOverride": "PROVISIONED", + "ProvisionedThroughputOverride": { "ReadCapacityUnits": read, "WriteCapacityUnits": write }, + })) + .unwrap(); + assert_eq!( + validation_message(validate_override_shape(&overrides).unwrap_err()), + expected + ); + } +} + +#[test] +fn missing_provisioned_throughput_override_member_matches_create_table() { + for (member, body) in [ + ( + "ReadCapacityUnits", + json!({ "BillingModeOverride": "PROVISIONED", + "ProvisionedThroughputOverride": { "WriteCapacityUnits": 5 } }), + ), + ( + "WriteCapacityUnits", + json!({ "BillingModeOverride": "PROVISIONED", + "ProvisionedThroughputOverride": { "ReadCapacityUnits": 5 } }), + ), + ] { + match parse_restore_table_overrides(&body).unwrap_err() { + DynamoDbError::SerializationException(m) => { + assert!(m.contains(&format!("missing field `{member}`")), "{m}"); + } + other => panic!("expected SerializationException, got {other:?}"), + } + } +} + +#[test] +fn vector_index_override_is_refused() { + let body = restore_body(json!({ "VectorIndexOverride": {} })); + assert_eq!( + validation_message(parse_restore_table_overrides(&body).unwrap_err()), + "RestoreTableFromBackup does not support VectorIndexOverride on ExtendDB" + ); +} + +#[test] +fn null_override_members_are_absent() { + let body = restore_body(json!({ + "GlobalSecondaryIndexOverride": null, + "LocalSecondaryIndexOverride": null, + "SSESpecificationOverride": null, + "OnDemandThroughputOverride": null, + "VectorIndexOverride": null, + "BillingModeOverride": null, + "ProvisionedThroughputOverride": null, + })); + assert!(parse_restore_table_overrides(&body).unwrap().is_empty()); +} + +#[test] +fn index_and_on_demand_overrides_of_the_wrong_type_are_serialization_errors() { + for (field, value) in [ + ("GlobalSecondaryIndexOverride", json!("nope")), + ("LocalSecondaryIndexOverride", json!({})), + ("OnDemandThroughputOverride", json!([1])), + // An entry without its required members. + ( + "GlobalSecondaryIndexOverride", + json!([{ "IndexName": "gsi1" }]), + ), + ( + "LocalSecondaryIndexOverride", + json!([{ "IndexName": "lsi1" }]), + ), + ( + "OnDemandThroughputOverride", + json!({ "MaxReadRequestUnits": "x" }), + ), + ] { + let body = restore_body(json!({ field: value })); + assert!( + matches!( + overrides::IndexOverrides::parse(&body).unwrap_err(), + DynamoDbError::SerializationException(_) + ), + "{field} {value}" + ); + } +} + +#[tokio::test] +async fn a_malformed_index_override_is_refused_before_any_storage_read() { + let fake = Fake::with_source(source(Some(definition("PROVISIONED")))); + let err = restore( + &fake, + restore_body(json!({ "GlobalSecondaryIndexOverride": [{ "IndexName": "gsi1" }] })), + ) + .await + .unwrap_err(); + assert!( + matches!(err, DynamoDbError::SerializationException(_)), + "{err:?}" + ); + assert_eq!(fake.source_reads(), 0); +} + +/// The backup's GSI and LSI as a client would send them back: the shape +/// `definition` records, in the wire's own form. +fn gsi_entry(pt: Option<(i64, i64)>) -> Value { + let mut entry = json!({ + "IndexName": "gsi1", + "KeySchema": [{ "AttributeName": "gpk", "KeyType": "HASH" }], + "Projection": { "ProjectionType": "ALL" }, + }); + if let Some((r, w)) = pt { + entry["ProvisionedThroughput"] = json!({ "ReadCapacityUnits": r, "WriteCapacityUnits": w }); + } + entry +} + +fn lsi_entry() -> Value { + json!({ + "IndexName": "lsi1", + "KeySchema": [ + { "AttributeName": "pk", "KeyType": "HASH" }, + { "AttributeName": "lsk", "KeyType": "RANGE" }, + ], + "Projection": { "ProjectionType": "KEYS_ONLY" }, + }) +} + +/// The target input of a restore of `src` with `body`'s overrides, or the +/// engine's refusal, through both paths: the legacy reader's, and the +/// manifest path over a store-backed backup whose `source_table` is `src`'s +/// definition (`manifest_from_source`). The two must agree (the same input, +/// or the same refusal), so one list of override cases binds both; the +/// manifest records the SSE *description*, not the specification a legacy +/// definition keeps, so that member is compared only when an override set +/// it. A source without a definition (pre-0.0.4) has no manifest form and +/// runs the legacy path alone. +async fn restored_input( + src: BackupRestoreSource, + body: Value, +) -> Result { + let fake = Fake::with_source(src.clone()); + let legacy = restore(&fake, restore_body(body.clone())) + .await + .map(|_| fake.restores()[0].input.clone()); + if legacy.is_err() { + assert!( + fake.restores().is_empty(), + "the legacy path reached storage for a refused {body}" + ); + } + if src.definition.is_none() { + return legacy; + } + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let item: Item = src + .key_schema + .iter() + .map(|k| { + ( + k.attribute_name.clone(), + extenddb_core::types::AttributeValue::S("a".to_owned()), + ) + }) + .collect(); + seed_manifest_backup( + &fake, + &mem, + &arn(ACCOUNT), + manifest_from_source(&src), + &[vec![item]], + false, + ); + let runtime = job_runtime(&mem); + let manifest = restore_on(&fake, &runtime, &restore_body(body.clone())) + .await + .map(|_| fake.restore_claims.lock().unwrap()[0].1.clone()); + wait_for_jobs(&runtime).await; + // The manifest records the SSE description, from which the restore + // derives the enabled KMS specification, so the member is compared when + // the source had SSE enabled or an override set it; a legacy + // definition's disabled specification has no manifest form. + let source_sse_enabled = src + .definition + .as_ref() + .and_then(|d| d.sse_specification.as_ref()) + .and_then(|sse| sse.get("Enabled")) + .and_then(Value::as_bool) + == Some(true); + match (&legacy, &manifest) { + (Ok(a), Ok(b)) => { + let mut a = a.clone(); + let mut b = b.clone(); + if body.get("SSESpecificationOverride").is_none() && !source_sse_enabled { + a.sse_specification = None; + b.sse_specification = None; + } + assert_eq!(a, b, "the two paths built different inputs for {body}"); + } + (Err(a), Err(b)) => { + assert_eq!( + format!("{a:?}"), + format!("{b:?}"), + "the two paths refused differently for {body}" + ); + assert!( + fake.restore_claims.lock().unwrap().is_empty(), + "the manifest path registered a target it then refused for {body}" + ); + } + (a, b) => panic!("the two paths disagree for {body}: legacy {a:?}, manifest {b:?}"), + } + legacy +} + +/// A manifest's `source_table` for a legacy source with a definition: the +/// same keys, indexes, billing and throughput, so the manifest path builds +/// the input the legacy path builds. +fn manifest_from_source(src: &BackupRestoreSource) -> SourceTableSchema { + let def = src.definition.as_ref().expect("a definition"); + let sse_description = def + .sse_specification + .as_ref() + .filter(|s| s.get("Enabled").and_then(Value::as_bool) == Some(true)) + .map(|_| extenddb_core::types::SseDescription { + status: "ENABLED".to_owned(), + sse_type: Some(extenddb_core::types::SseType::KMS), + kms_master_key_arn: None, + }); + SourceTableSchema { + table_name: src.source_table_name.clone(), + table_id: src.source_table_id.clone(), + table_arn: src.source_table_arn.clone(), + table_creation_date_time: src.table_creation_date_time, + key_schema: src.key_schema.clone(), + attribute_definitions: src.attribute_definitions.clone(), + billing_mode: if def.billing_mode == "PAY_PER_REQUEST" { + BillingMode::PayPerRequest + } else { + BillingMode::Provisioned + }, + provisioned_throughput: def.provisioned_throughput.clone(), + on_demand_throughput: def.on_demand_throughput.clone(), + table_class: def.table_class.clone(), + sse_description, + global_secondary_indexes: def.global_secondary_indexes.clone(), + local_secondary_indexes: def.local_secondary_indexes.clone(), + vector_indexes: None, + stream_specification: None, + time_to_live: None, + item_count: src.item_count, + table_size_bytes: src.table_size_bytes, + } +} + +/// The attribute names `input` defines. +fn attr_names(input: &CreateTableInput) -> Vec<&str> { + input + .attribute_definitions + .iter() + .map(|a| a.attribute_name.as_str()) + .collect() +} + +fn gsi_names(input: &CreateTableInput) -> Vec<&str> { + input + .global_secondary_indexes + .iter() + .flatten() + .map(|g| g.index_name.as_str()) + .collect() +} + +fn gsi_throughputs(input: &CreateTableInput) -> Vec> { + input + .global_secondary_indexes + .iter() + .flatten() + .map(|g| { + g.provisioned_throughput + .as_ref() + .map(|t| (t.read_capacity_units, t.write_capacity_units)) + }) + .collect() +} + +/// A backup with two GSIs, the second with an INCLUDE projection. +fn two_gsis(billing: &str) -> BackupTableDefinition { + let mut def = definition(billing); + let mut second = gsi( + "gsi2", + "gpk", + (billing == "PROVISIONED").then(|| throughput(1, 2)), + ); + second.projection = Projection { + projection_type: ProjectionType::Include, + non_key_attributes: Some(vec!["a".to_owned(), "b".to_owned()]), + }; + def.global_secondary_indexes.push(second); + def +} + +fn gsi2_entry(nka: Value, pt: Option<(i64, i64)>) -> Value { + let mut entry = json!({ + "IndexName": "gsi2", + "KeySchema": [{ "AttributeName": "gpk", "KeyType": "HASH" }], + "Projection": { "ProjectionType": "INCLUDE", "NonKeyAttributes": nka }, + }); + if let Some((r, w)) = pt { + entry["ProvisionedThroughput"] = json!({ "ReadCapacityUnits": r, "WriteCapacityUnits": w }); + } + entry +} + +/// Measured on DynamoDB: an empty index override list +/// restores no index of that kind. The attribute definitions only those +/// indexes used go with them, or CreateTable's validation would refuse the +/// input. +#[tokio::test] +async fn an_empty_index_override_list_restores_no_indexes_of_that_kind() { + for def in [definition("PROVISIONED"), definition("PAY_PER_REQUEST")] { + let input = restored_input( + source(Some(def.clone())), + json!({ "GlobalSecondaryIndexOverride": [] }), + ) + .await + .unwrap(); + assert!(input.global_secondary_indexes.is_none(), "{input:?}"); + assert_eq!( + input.local_secondary_indexes.as_ref().map(Vec::len), + Some(1) + ); + assert_eq!(attr_names(&input), ["pk", "sk", "lsk"]); + + let input = restored_input( + source(Some(def.clone())), + json!({ "LocalSecondaryIndexOverride": [] }), + ) + .await + .unwrap(); + assert!(input.local_secondary_indexes.is_none()); + assert_eq!(gsi_names(&input), ["gsi1"]); + assert_eq!(attr_names(&input), ["pk", "sk", "gpk"]); + + let input = restored_input( + source(Some(def)), + json!({ "GlobalSecondaryIndexOverride": [], "LocalSecondaryIndexOverride": [] }), + ) + .await + .unwrap(); + assert!(input.global_secondary_indexes.is_none()); + assert!(input.local_secondary_indexes.is_none()); + assert_eq!(attr_names(&input), ["pk", "sk"]); + extenddb_core::validation::validate_create_table(&input, &LimitsConfig::default()) + .expect("CreateTable accepts the pruned input"); + } + // Also with a billing override to PROVISIONED on an on-demand backup: + // with no GSIs left there is nothing that needs throughput. + let input = restored_input( + source(Some(definition("PAY_PER_REQUEST"))), + json!({ + "BillingModeOverride": "PROVISIONED", + "ProvisionedThroughputOverride": { "ReadCapacityUnits": 5, "WriteCapacityUnits": 6 }, + "GlobalSecondaryIndexOverride": [], + }), + ) + .await + .unwrap(); + assert!(input.global_secondary_indexes.is_none()); + assert_eq!(input.provisioned_throughput, Some(throughput(5, 6))); +} + +/// Measured on DynamoDB: the service applies a changed +/// index list. A subset restores those indexes, a changed GSI throughput is +/// applied, the list's order is the restored order, and a list that +/// describes the backup's indexes unchanged restores the same table as no +/// list at all. +#[tokio::test] +async fn index_override_lists_are_applied() { + let plain = restored_input(source(Some(definition("PROVISIONED"))), json!({})) + .await + .unwrap(); + assert_eq!(gsi_throughputs(&plain), [Some((3, 4))]); + assert_eq!(attr_names(&plain), ["pk", "sk", "gpk", "lsk"]); + + // Unchanged, as the console sends it (the backup's throughput). + let same = restored_input( + source(Some(definition("PROVISIONED"))), + json!({ + "GlobalSecondaryIndexOverride": [gsi_entry(Some((3, 4)))], + "LocalSecondaryIndexOverride": [lsi_entry()], + }), + ) + .await + .unwrap(); + assert_eq!(same, plain); + + // A changed GSI throughput. + let changed = restored_input( + source(Some(definition("PROVISIONED"))), + json!({ "GlobalSecondaryIndexOverride": [gsi_entry(Some((30, 40)))] }), + ) + .await + .unwrap(); + assert_eq!(gsi_throughputs(&changed), [Some((30, 40))]); + assert_eq!(gsi_names(&changed), ["gsi1"]); + assert_eq!(attr_names(&changed), attr_names(&plain)); + + // A subset: the backup has two GSIs, the list names one. The other + // index's attribute definition goes with it. + let subset = restored_input( + source(Some(two_gsis("PROVISIONED"))), + json!({ "GlobalSecondaryIndexOverride": [gsi2_entry(json!(["b", "a"]), Some((1, 2)))] }), + ) + .await + .unwrap(); + assert_eq!(gsi_names(&subset), ["gsi2"]); + assert_eq!(gsi_throughputs(&subset), [Some((1, 2))]); + assert_eq!( + subset.global_secondary_indexes.as_ref().unwrap()[0] + .projection + .non_key_attributes, + Some(vec!["a".to_owned(), "b".to_owned()]), + "the backup's projection is restored, not the entry's spelling of it" + ); + assert_eq!(attr_names(&subset), ["pk", "sk", "gpk", "lsk"]); + extenddb_core::validation::validate_create_table(&subset, &LimitsConfig::default()).unwrap(); + + // The list's order is the restored order. + let reordered = restored_input( + source(Some(two_gsis("PROVISIONED"))), + json!({ "GlobalSecondaryIndexOverride": [ + gsi2_entry(json!(["a", "b"]), Some((1, 2))), + gsi_entry(Some((3, 4))), + ] }), + ) + .await + .unwrap(); + assert_eq!(gsi_names(&reordered), ["gsi2", "gsi1"]); + + // An LSI subset of one (the only one) and an on-demand backup's GSIs, + // listed without throughput as DescribeBackup's {0, 0} cannot be sent. + let on_demand = restored_input(source(Some(two_gsis("PAY_PER_REQUEST"))), json!({})) + .await + .unwrap(); + let listed = restored_input( + source(Some(two_gsis("PAY_PER_REQUEST"))), + json!({ + "GlobalSecondaryIndexOverride": [gsi_entry(None), gsi2_entry(json!(["a", "b"]), None)], + "LocalSecondaryIndexOverride": [lsi_entry()], + }), + ) + .await + .unwrap(); + assert_eq!(listed, on_demand); +} + +/// The DescribeBackup echo (a console or SDK restore flow) restores exactly +/// the table the restore creates without it, for both billing modes; sent +/// with the `{0, 0}` DescribeBackup reports for an on-demand backup's GSI it +/// is refused as CreateTable refuses `{0, 0}`. +#[tokio::test] +async fn index_overrides_echoing_describe_backup_restore_the_same_table() { + for def in [definition("PAY_PER_REQUEST"), definition("PROVISIONED")] { + let on_demand = def.billing_mode == "PAY_PER_REQUEST"; + let fake = fake_with_backup("AVAILABLE", source(Some(def.clone()))); + let features = describe(&fake).await["SourceTableFeatureDetails"].clone(); + let mut gsis = features["GlobalSecondaryIndexes"].clone(); + let lsis = features["LocalSecondaryIndexes"].clone(); + assert!( + gsis.as_array().is_some_and(|g| g.len() == 1), + "precondition, DescribeBackup reports the GSI: {features}" + ); + if on_demand { + assert_eq!( + gsis[0]["ProvisionedThroughput"], + json!({ "ReadCapacityUnits": 0, "WriteCapacityUnits": 0 }), + "precondition: an on-demand backup's GSI is reported as {{0, 0}}" + ); + let zero_echo = restored_input( + source(Some(def.clone())), + json!({ "GlobalSecondaryIndexOverride": gsis.clone() }), + ) + .await + .unwrap_err(); + let msg = validation_message(zero_echo); + assert!( + msg.contains("gsi1") && msg.contains("should not be specified"), + "{msg}" + ); + for g in gsis.as_array_mut().unwrap() { + g.as_object_mut().unwrap().remove("ProvisionedThroughput"); + } + } + let plain = restored_input(source(Some(def.clone())), json!({})) + .await + .unwrap(); + let echoed = restored_input( + source(Some(def)), + json!({ + "GlobalSecondaryIndexOverride": gsis, + "LocalSecondaryIndexOverride": lsis, + }), + ) + .await + .unwrap_or_else(|e| panic!("the DescribeBackup echo was refused: {e:?}")); + assert_eq!(echoed, plain); + } +} + +/// Measured on DynamoDB: refused by the service and by ExtendDB: an unknown or +/// changed index definition, a throughput member contradicting the billing +/// mode, `{0, 0}`. ExtendDB's own: a member it cannot apply. +#[tokio::test] +async fn index_overrides_the_service_refuses_are_refused() { + let entry_with = |key: &str, value: Value| { + let mut e = gsi_entry(Some((3, 4))); + e[key] = value; + e + }; + let cases: [(&str, Value, &str); 13] = [ + ( + "an index the backup does not have", + json!({ "LocalSecondaryIndexOverride": [], "GlobalSecondaryIndexOverride": [ + entry_with("IndexName", json!("other")), + ] }), + "global secondary index other is not in the backup", + ), + ( + "listed twice", + json!({ "GlobalSecondaryIndexOverride": [ + gsi_entry(Some((3, 4))), gsi_entry(Some((3, 4))), + ] }), + "global secondary index gsi1 is listed more than once", + ), + ( + "key schema", + json!({ "GlobalSecondaryIndexOverride": [entry_with( + "KeySchema", json!([{ "AttributeName": "pk", "KeyType": "HASH" }]), + )] }), + "index gsi1: KeySchema differs from the backup's", + ), + ( + "key schema order", + json!({ "LocalSecondaryIndexOverride": [{ + "IndexName": "lsi1", + "KeySchema": [ + { "AttributeName": "lsk", "KeyType": "RANGE" }, + { "AttributeName": "pk", "KeyType": "HASH" }, + ], + "Projection": { "ProjectionType": "KEYS_ONLY" }, + }] }), + "index lsi1: KeySchema differs from the backup's", + ), + ( + "projection type", + json!({ "GlobalSecondaryIndexOverride": [entry_with( + "Projection", json!({ "ProjectionType": "KEYS_ONLY" }), + )] }), + "index gsi1: Projection differs from the backup's", + ), + ( + "projected attributes", + json!({ "GlobalSecondaryIndexOverride": [entry_with( + "Projection", json!({ "ProjectionType": "ALL", "NonKeyAttributes": ["x"] }), + )] }), + "index gsi1: Projection differs from the backup's", + ), + ( + "throughput on an on-demand restore", + json!({ + "BillingModeOverride": "PAY_PER_REQUEST", + "GlobalSecondaryIndexOverride": [gsi_entry(Some((3, 4)))], + }), + "ProvisionedThroughput should not be specified for index: gsi1", + ), + ( + "{0, 0}", + json!({ "GlobalSecondaryIndexOverride": [gsi_entry(Some((0, 0)))] }), + "ReadCapacityUnits and WriteCapacityUnits must both be greater than or equal to 1", + ), + ( + "an index on-demand throughput on a PROVISIONED restore", + json!({ "GlobalSecondaryIndexOverride": [entry_with( + "OnDemandThroughput", json!({ "MaxReadRequestUnits": 10 }), + )] }), + "index gsi1: OnDemandThroughput can only be specified when the restored table's \ + BillingMode is PAY_PER_REQUEST", + ), + ( + "a member ExtendDB cannot apply", + json!({ "GlobalSecondaryIndexOverride": [entry_with( + "WarmThroughput", json!({ "ReadUnitsPerSecond": 12000 }), + )] }), + "index gsi1: member WarmThroughput is not supported", + ), + ( + "an LSI member ExtendDB cannot apply", + json!({ "LocalSecondaryIndexOverride": [{ + "IndexName": "lsi1", + "KeySchema": [ + { "AttributeName": "pk", "KeyType": "HASH" }, + { "AttributeName": "lsk", "KeyType": "RANGE" }, + ], + "Projection": { "ProjectionType": "KEYS_ONLY" }, + "ProvisionedThroughput": { "ReadCapacityUnits": 1, "WriteCapacityUnits": 1 }, + }] }), + "index lsi1: member ProvisionedThroughput is not supported", + ), + ( + "an LSI the backup does not have", + json!({ "LocalSecondaryIndexOverride": [{ + "IndexName": "lsi2", + "KeySchema": [ + { "AttributeName": "pk", "KeyType": "HASH" }, + { "AttributeName": "lsk", "KeyType": "RANGE" }, + ], + "Projection": { "ProjectionType": "KEYS_ONLY" }, + }] }), + "local secondary index lsi2 is not in the backup", + ), + ( + "an LSI listed twice", + json!({ "LocalSecondaryIndexOverride": [lsi_entry(), lsi_entry()] }), + "local secondary index lsi1 is listed more than once", + ), + ]; + for (what, body, expect) in cases { + let msg = validation_message( + restored_input(source(Some(definition("PROVISIONED"))), body) + .await + .unwrap_err(), + ); + assert!(msg.contains(expect), "{what}: {msg}"); + } + + // A legacy backup restores no index, so any listed one is not in it; an + // empty list changes nothing. + let fake = Fake::with_source(source(None)); + let msg = validation_message( + restore( + &fake, + restore_body(json!({ "GlobalSecondaryIndexOverride": [gsi_entry(Some((1, 1)))] })), + ) + .await + .unwrap_err(), + ); + assert!( + msg.ends_with("global secondary index gsi1 is not in the backup"), + "{msg}" + ); + let plain = restored_input(source(None), json!({})).await.unwrap(); + let empty = restored_input( + source(None), + json!({ "GlobalSecondaryIndexOverride": [], "LocalSecondaryIndexOverride": [] }), + ) + .await + .unwrap(); + assert_eq!(empty, plain); +} + +/// Measured on DynamoDB: the two requests the service refuses and ExtendDB +/// accepted: a GSI entry without throughput on a PROVISIONED restore, and +/// `BillingModeOverride` `PROVISIONED` with no GSI list on a backup with +/// GSIs, both with the service's text. The service's text conditions on the +/// override being present, so the rule applies whatever the backup's billing +/// mode and whatever throughput it recorded for the index (the PROVISIONED +/// backup and stale-throughput cases were not measured). The +/// on-demand-to-PROVISIONED route works when the list supplies each index's +/// throughput. +#[tokio::test] +async fn a_provisioned_restore_needs_each_gsis_throughput_overridden() { + let expected = "Invalid Request: Must override ProvisionedThroughput if BillingMode is \ + overridden to PROVISIONED for index gsi1"; + let provisioned = json!({ + "BillingModeOverride": "PROVISIONED", + "ProvisionedThroughputOverride": { "ReadCapacityUnits": 5, "WriteCapacityUnits": 5 }, + }); + let with = |extra: Value| { + let mut body = provisioned.clone(); + for (k, v) in extra.as_object().unwrap() { + body[k] = v.clone(); + } + body + }; + // An on-demand backup whose GSI still carries a throughput from before a + // billing switch (what UpdateTable to PAY_PER_REQUEST leaves behind). + let mut stale = definition("PAY_PER_REQUEST"); + stale.global_secondary_indexes[0].provisioned_throughput = Some(throughput(3, 4)); + let refused: [(&str, BackupTableDefinition, Value); 6] = [ + ( + "no GSI list, on-demand backup", + definition("PAY_PER_REQUEST"), + provisioned.clone(), + ), + ( + "no GSI list, PROVISIONED backup (the override, not the recorded throughput, decides)", + definition("PROVISIONED"), + provisioned.clone(), + ), + ( + "no GSI list, on-demand backup with a stale recorded throughput", + stale, + provisioned.clone(), + ), + ( + "a GSI entry without throughput, on-demand backup", + definition("PAY_PER_REQUEST"), + with(json!({ "GlobalSecondaryIndexOverride": [gsi_entry(None)] })), + ), + ( + "a GSI entry without throughput, PROVISIONED backup with the override", + definition("PROVISIONED"), + with(json!({ "GlobalSecondaryIndexOverride": [gsi_entry(None)] })), + ), + ( + "a GSI entry without throughput, PROVISIONED backup without the override", + definition("PROVISIONED"), + json!({ "GlobalSecondaryIndexOverride": [gsi_entry(None)] }), + ), + ]; + for (what, def, body) in refused { + let msg = validation_message(restored_input(source(Some(def)), body).await.unwrap_err()); + assert_eq!(msg, expected, "{what}"); + } + // The first GSI of two is the one named. + let msg = validation_message( + restored_input( + source(Some(two_gsis("PAY_PER_REQUEST"))), + provisioned.clone(), + ) + .await + .unwrap_err(), + ); + assert_eq!(msg, expected); + + // The route that works: the list supplies the throughput. + let input = restored_input( + source(Some(two_gsis("PAY_PER_REQUEST"))), + with(json!({ "GlobalSecondaryIndexOverride": [ + gsi_entry(Some((7, 8))), + gsi2_entry(json!(["a", "b"]), Some((9, 10))), + ] })), + ) + .await + .unwrap(); + assert_eq!(input.billing_mode, Some(BillingMode::Provisioned)); + assert_eq!(input.provisioned_throughput, Some(throughput(5, 5))); + assert_eq!(gsi_throughputs(&input), [Some((7, 8)), Some((9, 10))]); + extenddb_core::validation::validate_create_table(&input, &LimitsConfig::default()).unwrap(); + + // A PROVISIONED backup restored with no override keeps its recorded GSI + // throughput; only the override asks for the list. + let input = restored_input(source(Some(definition("PROVISIONED"))), json!({})) + .await + .unwrap(); + assert_eq!(gsi_throughputs(&input), [Some((3, 4))]); + // And with no GSIs, the billing override to PROVISIONED needs nothing. + let mut none = definition("PAY_PER_REQUEST"); + none.global_secondary_indexes.clear(); + let mut src = source(Some(none)); + src.attribute_definitions + .retain(|a| a.attribute_name != "gpk"); + let input = restored_input(src, provisioned).await.unwrap(); + assert!(input.global_secondary_indexes.is_none()); + assert_eq!(input.provisioned_throughput, Some(throughput(5, 5))); +} + +fn on_demand(read: Option, write: Option) -> OnDemandThroughput { + OnDemandThroughput { + max_read_request_units: read, + max_write_request_units: write, + } +} + +/// Measured on DynamoDB: the service applies `OnDemandThroughputOverride` on an +/// on-demand restore, changed or with one member, including on a PROVISIONED +/// backup restored as on-demand; a GSI entry's `OnDemandThroughput` is +/// accepted (and, ExtendDB's choice, not recorded). On a PROVISIONED restore +/// the member contradicts the billing mode and is refused. +#[tokio::test] +async fn an_on_demand_throughput_override_is_applied_on_an_on_demand_restore() { + let with = |recorded: Option| { + let mut def = definition("PAY_PER_REQUEST"); + def.on_demand_throughput = recorded; + source(Some(def)) + }; + let applied: [( + Option, + Value, + Option, + ); 6] = [ + ( + Some(on_demand(Some(50), Some(60))), + json!({ "MaxReadRequestUnits": 50, "MaxWriteRequestUnits": 60 }), + Some(on_demand(Some(50), Some(60))), + ), + ( + Some(on_demand(Some(50), Some(60))), + json!({ "MaxReadRequestUnits": 50, "MaxWriteRequestUnits": 61 }), + Some(on_demand(Some(50), Some(61))), + ), + ( + None, + json!({ "MaxReadRequestUnits": 10 }), + Some(on_demand(Some(10), None)), + ), + ( + Some(on_demand(Some(50), Some(60))), + json!({ "MaxWriteRequestUnits": -1 }), + Some(on_demand(None, Some(-1))), + ), + ( + None, + json!({ "MaxReadRequestUnits": -1, "MaxWriteRequestUnits": -1 }), + Some(on_demand(Some(-1), Some(-1))), + ), + // An empty override records no maxima, as CreateTable without the + // member does. + (Some(on_demand(Some(50), Some(60))), json!({}), None), + ]; + for (recorded, given, expected) in applied { + let input = restored_input( + with(recorded.clone()), + json!({ "OnDemandThroughputOverride": given }), + ) + .await + .unwrap_or_else(|e| panic!("{recorded:?} {given}: {e:?}")); + assert_eq!(input.on_demand_throughput, expected, "{recorded:?} {given}"); + assert_eq!(input.billing_mode, Some(BillingMode::PayPerRequest)); + } + // Without the member the backup's is restored. + let input = restored_input(with(Some(on_demand(Some(50), Some(60)))), json!({})) + .await + .unwrap(); + assert_eq!( + input.on_demand_throughput, + Some(on_demand(Some(50), Some(60))) + ); + + // A PROVISIONED backup restored as on-demand takes it. + let input = restored_input( + source(Some(definition("PROVISIONED"))), + json!({ + "BillingModeOverride": "PAY_PER_REQUEST", + "OnDemandThroughputOverride": { "MaxReadRequestUnits": 7, "MaxWriteRequestUnits": 8 }, + }), + ) + .await + .unwrap(); + assert_eq!(input.billing_mode, Some(BillingMode::PayPerRequest)); + assert!(input.provisioned_throughput.is_none()); + assert_eq!( + input.on_demand_throughput, + Some(on_demand(Some(7), Some(8))) + ); + + // A GSI entry's OnDemandThroughput on an on-demand restore: accepted, the + // index restored without a maximum (ExtendDB records none per index). + let plain = restored_input(source(Some(definition("PAY_PER_REQUEST"))), json!({})) + .await + .unwrap(); + let input = restored_input( + source(Some(definition("PAY_PER_REQUEST"))), + json!({ "GlobalSecondaryIndexOverride": [{ + "IndexName": "gsi1", + "KeySchema": [{ "AttributeName": "gpk", "KeyType": "HASH" }], + "Projection": { "ProjectionType": "ALL" }, + "OnDemandThroughput": { "MaxReadRequestUnits": 10, "MaxWriteRequestUnits": -1 }, + }] }), + ) + .await + .unwrap(); + assert_eq!(input, plain); + + // Refused: an unknown member, and the member on a PROVISIONED restore + // (from a PROVISIONED backup or by override). + let fake = Fake::with_source(with(None)); + let msg = validation_message( + restore( + &fake, + restore_body(json!({ "OnDemandThroughputOverride": { + "MaxReadRequestUnits": 50, "MaxWriteRequestUnits": 60, "Other": 1, + } })), + ) + .await + .unwrap_err(), + ); + assert!( + msg.ends_with("OnDemandThroughputOverride: member Other is not supported"), + "{msg}" + ); + assert!(fake.restores().is_empty()); + for (def, body) in [ + ( + definition("PROVISIONED"), + json!({ "OnDemandThroughputOverride": { "MaxReadRequestUnits": 1 } }), + ), + ( + definition("PAY_PER_REQUEST"), + json!({ + "OnDemandThroughputOverride": {}, + "BillingModeOverride": "PROVISIONED", + "ProvisionedThroughputOverride": { "ReadCapacityUnits": 1, "WriteCapacityUnits": 1 }, + }), + ), + ] { + let mut def = def; + def.global_secondary_indexes.clear(); + let fake = Fake::with_source(source(Some(def))); + let msg = validation_message(restore(&fake, restore_body(body)).await.unwrap_err()); + assert!( + msg.ends_with( + "OnDemandThroughputOverride can only be specified when the restored table's \ + BillingMode is PAY_PER_REQUEST" + ), + "{msg}" + ); + assert!(fake.restores().is_empty()); + } +} + +/// Measured on DynamoDB: the service restores `STANDARD` whatever the source's +/// table class; the recorded class is for DescribeBackup only. +#[tokio::test] +async fn table_class_is_not_restored() { + let mut def = definition("PROVISIONED"); + def.table_class = Some("STANDARD_INFREQUENT_ACCESS".to_owned()); + let input = restored_input(source(Some(def)), json!({})).await.unwrap(); + assert_eq!(input.table_class, None); +} + +/// A source with SSE enabled restores with an enabled KMS specification on +/// both paths: the legacy definition carries the specification, the +/// manifest the description it was derived from. +#[tokio::test] +async fn an_enabled_sse_source_restores_sse_on_both_paths() { + let mut def = definition("PROVISIONED"); + def.sse_specification = Some(json!({ "Enabled": true, "SSEType": "KMS" })); + let input = restored_input(source(Some(def)), json!({})).await.unwrap(); + assert_eq!( + input.sse_specification, + Some(json!({ "Enabled": true, "SSEType": "KMS" })) + ); +} + +#[tokio::test] +async fn sse_override_replaces_the_recorded_specification() { + let mut def = definition("PROVISIONED"); + def.sse_specification = Some(json!({ "Enabled": false })); + let sse = json!({ "Enabled": true, "SSEType": "KMS", "KMSMasterKeyId": "alias/k" }); + let input = restored_input( + source(Some(def)), + json!({ "SSESpecificationOverride": sse }), + ) + .await + .unwrap(); + assert_eq!(input.sse_specification, Some(sse)); + + // A legacy backup takes the override too. + let fake = Fake::with_source(source(None)); + restore( + &fake, + restore_body(json!({ "SSESpecificationOverride": { "Enabled": true } })), + ) + .await + .unwrap(); + assert_eq!( + fake.restores()[0].input.sse_specification, + Some(json!({ "Enabled": true })) + ); + + let body = restore_body(json!({ "SSESpecificationOverride": "on" })); + assert!(matches!( + parse_restore_table_overrides(&body).unwrap_err(), + DynamoDbError::SerializationException(_) + )); +} + +#[tokio::test] +async fn billing_overrides_shape_the_restore_input() { + // PROVISIONED -> PAY_PER_REQUEST drops table and GSI throughput. + let body = json!({ "BillingModeOverride": "PAY_PER_REQUEST" }); + let input = restored_input(source(Some(definition("PROVISIONED"))), body.clone()) + .await + .unwrap(); + assert_eq!(input.billing_mode, Some(BillingMode::PayPerRequest)); + assert!(input.provisioned_throughput.is_none()); + assert!( + input.global_secondary_indexes.as_ref().unwrap()[0] + .provisioned_throughput + .is_none() + ); + // The response zeros the throughput of an on-demand table and index. + let fake = Fake::with_source(source(Some(definition("PROVISIONED")))); + let out = restore(&fake, restore_body(body)).await.unwrap(); + let td = &out["TableDescription"]; + assert_eq!(td["ProvisionedThroughput"]["ReadCapacityUnits"], 0); + assert_eq!( + td["GlobalSecondaryIndexes"][0]["ProvisionedThroughput"]["ReadCapacityUnits"], + 0 + ); + + // A throughput override on a PROVISIONED backup replaces the table's. + let input = restored_input( + source(Some(definition("PROVISIONED"))), + json!({ + "ProvisionedThroughputOverride": { "ReadCapacityUnits": 11, "WriteCapacityUnits": 12 }, + }), + ) + .await + .unwrap(); + assert_eq!(input.provisioned_throughput, Some(throughput(11, 12))); + assert_eq!( + input.global_secondary_indexes.as_ref().unwrap()[0].provisioned_throughput, + Some(throughput(3, 4)) + ); + + // A throughput override on an on-demand backup without a billing override. + let err = restored_input( + source(Some(definition("PAY_PER_REQUEST"))), + json!({ + "ProvisionedThroughputOverride": { "ReadCapacityUnits": 1, "WriteCapacityUnits": 1 }, + }), + ) + .await + .unwrap_err(); + assert!(validation_message(err).contains("can only be specified")); +} + +#[tokio::test] +async fn provisioned_override_on_an_on_demand_gsi_has_the_services_text() { + let err = restored_input( + source(Some(definition("PAY_PER_REQUEST"))), + json!({ + "BillingModeOverride": "PROVISIONED", + "ProvisionedThroughputOverride": { "ReadCapacityUnits": 5, "WriteCapacityUnits": 5 }, + }), + ) + .await + .unwrap_err(); + assert_eq!( + validation_message(err), + "Invalid Request: Must override ProvisionedThroughput if BillingMode is overridden to \ + PROVISIONED for index gsi1" + ); +} + +#[tokio::test] +async fn override_shape_is_refused_before_any_storage_read() { + let fake = Arc::new(Fake::default()); + let err = restore( + &fake, + restore_body(json!({ "BillingModeOverride": "PROVISIONED" })), + ) + .await + .unwrap_err(); + assert!(validation_message(err).contains("ProvisionedThroughputOverride must")); + assert_eq!(fake.source_reads(), 0); + // The manifest path refuses the shape the same way, before any store or + // row read. + let err = restored_input( + source(Some(definition("PROVISIONED"))), + json!({ "BillingModeOverride": "PROVISIONED" }), + ) + .await + .unwrap_err(); + assert!(validation_message(err).contains("ProvisionedThroughputOverride must")); +} + +// --- refusals and validation -------------------------------------------------- + +#[tokio::test] +async fn multi_part_key_backups_are_refused_in_the_engine() { + let mut src = source(Some(definition("PROVISIONED"))); + src.key_schema.push(key("pk2", KeyType::Hash)); + let fake = Fake::with_source(src); + let msg = validation_message(restore(&fake, restore_body(json!({}))).await.unwrap_err()); + assert!(msg.contains("multi-part key"), "{msg}"); + assert!(fake.restores().is_empty()); +} + +#[tokio::test] +async fn a_stored_definition_create_table_would_refuse_is_refused() { + type Corrupt = fn(&mut BackupRestoreSource); + let cases: [(&str, Corrupt); 7] = [ + ( + "101 projected attributes split across a GSI and an LSI", + |s| { + let d = s.definition.as_mut().unwrap(); + d.global_secondary_indexes[0].projection = include(0..60); + d.local_secondary_indexes[0].projection = include(60..101); + }, + ), + ("GSI on an undefined attribute", |s| { + let d = s.definition.as_mut().unwrap(); + d.global_secondary_indexes[0].key_schema[0].attribute_name = "nowhere".to_owned(); + }), + ("negative GSI throughput", |s| { + let d = s.definition.as_mut().unwrap(); + d.global_secondary_indexes[0].provisioned_throughput = Some(throughput(-1, 1)); + }), + ("GSI throughput with one unit at 0", |s| { + let d = s.definition.as_mut().unwrap(); + d.global_secondary_indexes[0].provisioned_throughput = Some(throughput(0, 1)); + }), + ("negative table throughput", |s| { + s.definition.as_mut().unwrap().provisioned_throughput = Some(throughput(1, -1)); + }), + ("too many GSIs", |s| { + let d = s.definition.as_mut().unwrap(); + d.global_secondary_indexes = (0..30) + .map(|i| gsi(&format!("g{i}"), "gpk", Some(throughput(1, 1)))) + .collect(); + }), + ("too many projected attributes", |s| { + let d = s.definition.as_mut().unwrap(); + d.global_secondary_indexes[0].projection = Projection { + projection_type: ProjectionType::Include, + non_key_attributes: Some((0..101).map(|i| format!("a{i}")).collect()), + }; + }), + ]; + for (what, corrupt) in cases { + let mut src = source(Some(definition("PROVISIONED"))); + corrupt(&mut src); + let fake = Fake::with_source(src); + let err = restore(&fake, restore_body(json!({}))).await.unwrap_err(); + assert!( + matches!(err, DynamoDbError::ValidationException(_)), + "{what}: {err:?}" + ); + assert!(fake.restores().is_empty(), "{what}: reached storage"); + } +} + +fn include(range: std::ops::Range) -> Projection { + Projection { + projection_type: ProjectionType::Include, + non_key_attributes: Some(range.map(|i| format!("a{i}")).collect()), + } +} + +#[tokio::test] +async fn one_hundred_projected_attributes_are_accepted() { + let mut src = source(Some(definition("PROVISIONED"))); + let d = src.definition.as_mut().unwrap(); + d.global_secondary_indexes[0].projection = include(0..60); + d.local_secondary_indexes[0].projection = include(60..100); + let fake = Fake::with_source(src); + restore(&fake, restore_body(json!({}))).await.unwrap(); + assert_eq!(fake.restores().len(), 1); +} + +/// A PROVISIONED table whose GSI has no throughput (CreateTable accepts one) +/// or 0/0 (what UpdateTable to PROVISIONED leaves on an on-demand table's +/// GSIs) restores with no override, the GSI taking the target table's +/// throughput. With `BillingModeOverride` `PROVISIONED` and no GSI list it is +/// refused like any backup with a GSI (the override, not the recorded +/// throughput, decides); the list route restores it. +#[tokio::test] +async fn a_table_create_table_accepted_restores() { + // The CreateTable half: the shape is accepted there. + let mut created = source(Some(definition("PROVISIONED"))) + .restore_input("Music", &RestoreTableOverrides::default()); + created.global_secondary_indexes.as_mut().unwrap()[0].provisioned_throughput = None; + extenddb_core::validation::validate_create_table(&created, &LimitsConfig::default()) + .expect("CreateTable accepts a PROVISIONED GSI without throughput"); + + let provisioned = json!({ + "BillingModeOverride": "PROVISIONED", + "ProvisionedThroughputOverride": { "ReadCapacityUnits": 5, "WriteCapacityUnits": 6 }, + }); + for recorded in [None, Some(throughput(0, 0))] { + let src = || { + let mut src = source(Some(definition("PROVISIONED"))); + src.definition.as_mut().unwrap().global_secondary_indexes[0].provisioned_throughput = + recorded.clone(); + src + }; + let fake = Fake::with_source(src()); + restore(&fake, restore_body(json!({}))) + .await + .unwrap_or_else(|e| panic!("{recorded:?}: {e:?}")); + let input = &fake.restores()[0].input; + assert_eq!( + input.global_secondary_indexes.as_ref().unwrap()[0].provisioned_throughput, + Some(throughput(7, 8)), + "{recorded:?}" + ); + + let fake = Fake::with_source(src()); + let msg = validation_message( + restore(&fake, restore_body(provisioned.clone())) + .await + .unwrap_err(), + ); + assert_eq!( + msg, + "Invalid Request: Must override ProvisionedThroughput if BillingMode is overridden \ + to PROVISIONED for index gsi1", + "{recorded:?}" + ); + assert!(fake.restores().is_empty()); + + let mut with_list = provisioned.clone(); + with_list["GlobalSecondaryIndexOverride"] = json!([gsi_entry(Some((5, 6)))]); + let fake = Fake::with_source(src()); + restore(&fake, restore_body(with_list)) + .await + .unwrap_or_else(|e| panic!("{recorded:?}: {e:?}")); + let input = &fake.restores()[0].input; + assert_eq!( + input.global_secondary_indexes.as_ref().unwrap()[0].provisioned_throughput, + Some(throughput(5, 6)), + "{recorded:?}" + ); + } +} + +#[tokio::test] +async fn a_missing_backup_is_backup_not_found() { + let fake = Arc::new(Fake::default()); + assert!(matches!( + restore(&fake, restore_body(json!({}))).await.unwrap_err(), + DynamoDbError::BackupNotFoundException(_) + )); +} + +// --- the storage request and the response ------------------------------------ + +#[tokio::test] +async fn the_backend_receives_the_resolved_request() { + let fake = Fake::with_source(source(Some(definition("PROVISIONED")))); + restore(&fake, restore_body(json!({}))).await.unwrap(); + let [request] = fake.restores().try_into().unwrap(); + assert_eq!(request.backup_arn, arn(ACCOUNT)); + assert_eq!(request.source_table_arn, source_arn()); + assert!((request.restore_date_time - CREATED).abs() < f64::EPSILON); + let input = request.input; + assert_eq!(input.table_name, "MusicRestored"); + assert_eq!(input.key_schema.len(), 2); + assert_eq!(input.billing_mode, Some(BillingMode::Provisioned)); + assert_eq!(input.provisioned_throughput, Some(throughput(7, 8))); + assert_eq!( + input.global_secondary_indexes.unwrap()[0].provisioned_throughput, + Some(throughput(3, 4)) + ); + assert_eq!(input.local_secondary_indexes.unwrap().len(), 1); + assert!(input.stream_specification.is_none()); +} + +#[tokio::test] +async fn a_legacy_backup_restores_with_its_old_defaults() { + let fake = Fake::with_source(source(None)); + restore(&fake, restore_body(json!({}))).await.unwrap(); + let input = &fake.restores()[0].input; + assert_eq!(input.provisioned_throughput, Some(throughput(5, 5))); + assert!(input.global_secondary_indexes.is_none()); + assert!(input.local_secondary_indexes.is_none()); +} + +#[tokio::test] +async fn the_response_matches_the_service_shape() { + let fake = Fake::with_source(source(Some(definition("PROVISIONED")))); + let out = restore(&fake, restore_body(json!({}))).await.unwrap(); + let td = &out["TableDescription"]; + assert_eq!(td["TableStatus"], "CREATING"); + let summary = &td["RestoreSummary"]; + assert_eq!(summary["SourceBackupArn"], arn(ACCOUNT)); + assert_eq!(summary["SourceTableArn"], source_arn()); + assert_eq!(summary["RestoreDateTime"], CREATED); + assert_eq!(summary["RestoreInProgress"], true); + let gsi = td["GlobalSecondaryIndexes"][0].as_object().unwrap(); + assert!(!gsi.contains_key("IndexStatus"), "{gsi:?}"); + assert!(gsi.contains_key("IndexName")); + assert!( + td["BillingModeSummary"] + .get("LastUpdateToPayPerRequestDateTime") + .is_none() + ); + assert!( + td["TableThroughputModeSummary"] + .get("LastUpdateToPayPerRequestDateTime") + .is_none() + ); + assert_eq!( + td["TableThroughputModeSummary"]["TableThroughputMode"], + "PROVISIONED" + ); + // Omitted by the service here; the backend's description carries both. + assert!(td.get("DeletionProtectionEnabled").is_none(), "{td}"); + let pt = td["ProvisionedThroughput"].as_object().unwrap(); + assert!(!pt.contains_key("NumberOfDecreasesToday"), "{pt:?}"); + assert_eq!(pt["ReadCapacityUnits"], 9); + let backend = serde_json::to_value(described(&fake.restores()[0])).unwrap(); + assert_eq!(backend["DeletionProtectionEnabled"], false); + assert_eq!( + backend["ProvisionedThroughput"]["NumberOfDecreasesToday"], + 0 + ); +} + +#[tokio::test] +async fn a_taken_name_is_table_already_exists_on_restore_only() { + let fake = Fake::with_source(source(Some(definition("PROVISIONED")))); + *fake.restore_error.lock().unwrap() = + Some(StorageError::TableAlreadyExists("MusicRestored".to_owned())); + let err = restore(&fake, restore_body(json!({}))).await.unwrap_err(); + match &err { + DynamoDbError::TableAlreadyExistsException(m) => { + assert_eq!(m, "Table already exists: MusicRestored"); + } + other => panic!("expected TableAlreadyExistsException, got {other:?}"), + } + assert_eq!( + err.full_error_type(), + "com.amazonaws.dynamodb.v20120810#TableAlreadyExistsException" + ); + + // CreateTable keeps ResourceInUseException. + assert!(matches!( + crate::create_table::storage_err_to_dynamo(StorageError::TableAlreadyExists( + "t".to_owned() + )), + DynamoDbError::ResourceInUseException(_) + )); +} + +/// Measured on DynamoDB: a target name taken by a +/// `CREATING` restore is `TableInUseException` `Table: is already +/// being restored from backup: `; a backup with a restore from it still +/// `CREATING` is `BackupInUseException`. The backends raise the storage +/// errors; this is their mapping. +#[tokio::test] +async fn a_name_being_restored_is_table_in_use_and_a_backup_being_restored_is_backup_in_use() { + let fake = Fake::with_source(source(Some(definition("PROVISIONED")))); + *fake.restore_error.lock().unwrap() = Some(StorageError::TableBeingRestored { + table_name: "MusicRestored".to_owned(), + backup_arn: arn(ACCOUNT), + }); + let err = restore(&fake, restore_body(json!({}))).await.unwrap_err(); + match &err { + DynamoDbError::TableInUseException(m) => assert_eq!( + m, + &format!( + "Table: MusicRestored is already being restored from backup: {}", + arn(ACCOUNT) + ) + ), + other => panic!("expected TableInUseException, got {other:?}"), + } + assert_eq!( + err.full_error_type(), + "com.amazonaws.dynamodb.v20120810#TableInUseException" + ); + + let fake = Fake::with_source(source(Some(definition("PROVISIONED")))); + *fake.restore_error.lock().unwrap() = Some(StorageError::BackupInUse(format!( + "Backup {} is being restored", + arn(ACCOUNT) + ))); + let err = restore(&fake, restore_body(json!({}))).await.unwrap_err(); + assert!( + matches!(err, DynamoDbError::BackupInUseException(_)), + "{err:?}" + ); + + // Elsewhere the name is a table in use. A plain CreateTable into the + // name answers the service's text for a taken name (measured on + // DynamoDB), without the backup ARN. + assert!(matches!( + storage_err_to_dynamo(StorageError::TableBeingRestored { + table_name: "t".to_owned(), + backup_arn: "a".to_owned(), + }), + DynamoDbError::ResourceInUseException(_) + )); + match crate::create_table::storage_err_to_dynamo(StorageError::TableBeingRestored { + table_name: "MusicRestored".to_owned(), + backup_arn: arn(ACCOUNT), + }) { + DynamoDbError::ResourceInUseException(m) => { + assert_eq!(m, "Table already exists: MusicRestored"); + } + other => panic!("expected ResourceInUseException, got {other:?}"), + } +} + +#[tokio::test] +async fn synchronous_backend_refusals_keep_their_classes() { + for (storage, check) in [ + ( + StorageError::LimitExceeded("busy".to_owned()), + (|e: &DynamoDbError| matches!(e, DynamoDbError::LimitExceededException(_))) + as fn(&DynamoDbError) -> bool, + ), + ( + StorageError::Validation(format!("Backup not found: {}", arn(ACCOUNT))), + |e| matches!(e, DynamoDbError::BackupNotFoundException(_)), + ), + ] { + let fake = Fake::with_source(source(Some(definition("PROVISIONED")))); + *fake.restore_error.lock().unwrap() = Some(storage); + let err = restore(&fake, restore_body(json!({}))).await.unwrap_err(); + assert!(check(&err), "{err:?}"); + } +} + +#[test] +fn an_unsupported_feature_is_a_validation_exception() { + let msg = "restoring a table with vector indexes is not supported by this storage backend"; + assert_eq!( + validation_message(storage_err_to_dynamo(StorageError::Unsupported( + msg.to_owned() + ))), + msg + ); +} + +// --- CreateBackup ------------------------------------------------------------- + +/// Measured on DynamoDB: `BackupName` and `TableName` are +/// constrained as the service model says, with the service's texts; both +/// constraints failing join under `2 validation errors detected:`. +#[test] +fn backup_and_table_names_are_validated_as_the_model_says() { + let ok = ["abc", "a.b-c_D", &"x".repeat(255)]; + for name in ok { + validate_name_member(name, "backupName", BACKUP_NAME_BOUNDS) + .unwrap_or_else(|e| panic!("{name}: {e:?}")); + } + let constraint = |v: &str, path: &str, c: &str| { + format!("Value '{v}' at '{path}' failed to satisfy constraint: {c}") + }; + let short = "Member must have length greater than or equal to 3"; + let long = "Member must have length less than or equal to 255"; + let pattern = "Member must satisfy regular expression pattern: [a-zA-Z0-9_.-]+"; + let too_long = "x".repeat(256); + let huge = "y".repeat(200 * 1024); + let cases: Vec<(&str, &str, String)> = vec![ + ( + "", + "backupName", + format!( + "1 validation error detected: {}", + constraint("", "backupName", short) + ), + ), + ( + "ab", + "backupName", + format!( + "1 validation error detected: {}", + constraint("ab", "backupName", short) + ), + ), + ( + &too_long, + "tableName", + format!( + "1 validation error detected: {}", + constraint(&too_long, "tableName", long) + ), + ), + ( + "a b c", + "backupName", + format!( + "1 validation error detected: {}", + constraint("a b c", "backupName", pattern) + ), + ), + ( + "abc\u{0}", + "backupName", + format!( + "1 validation error detected: {}", + constraint("abc\u{0}", "backupName", pattern) + ), + ), + ( + "b\u{e9}k", + "backupName", + format!( + "1 validation error detected: {}", + constraint("b\u{e9}k", "backupName", pattern) + ), + ), + ( + "a!", + "backupName", + format!( + "2 validation errors detected: {}; {}", + constraint("a!", "backupName", short), + constraint("a!", "backupName", pattern) + ), + ), + ]; + for (value, path, expected) in cases { + assert_eq!( + validation_message(validate_name_member(value, path, BACKUP_NAME_BOUNDS).unwrap_err()), + expected, + "{value:?}" + ); + } + // A 200 KB name with a NUL byte (a 500 on Cassandra and PostgreSQL + // before the check) is refused with both constraints named. + let bad = format!("{huge}\u{0}"); + let msg = validation_message( + validate_name_member(&bad, "backupName", BACKUP_NAME_BOUNDS).unwrap_err(), + ); + assert!( + msg.starts_with("2 validation errors detected: "), + "{}", + &msg[..60] + ); + assert!(msg.contains(long) && msg.contains(pattern)); + + // Through the handlers' members. + let body = json!({ "TableName": "ok_table", "BackupName": "x" }); + let name = create_backup_table_name(&body, ACCOUNT, REGION, &limits()).unwrap(); + assert_eq!(name, "ok_table"); + assert!(matches!( + validate_name_member(body["BackupName"].as_str().unwrap(), "backupName", BACKUP_NAME_BOUNDS) + .unwrap_err(), + DynamoDbError::ValidationException(m) if m.contains("'backupName'") + )); + let msg = validation_message( + create_backup_table_name(&json!({ "TableName": "a*" }), ACCOUNT, REGION, &limits()) + .unwrap_err(), + ); + assert!( + msg.contains("'tableName'") && msg.contains(pattern), + "{msg}" + ); +} + +/// Length is counted in characters, as the model's bounds are: a two-byte +/// character is one. `TableName` members take the deployment's table-name +/// limits (`CreateBackup` and `ListBackups` agree with `CreateTable`); +/// `BackupName` keeps the model's 3 to 255. +#[tokio::test] +async fn name_lengths_are_counted_in_characters_and_table_names_take_the_limits() { + let pattern = "Member must satisfy regular expression pattern: [a-zA-Z0-9_.-]+"; + // Two characters, four bytes: too short, and not in the pattern (two + // clauses, not one). + let msg = validation_message( + validate_name_member("\u{e9}\u{e9}", "backupName", BACKUP_NAME_BOUNDS).unwrap_err(), + ); + assert!(msg.starts_with("2 validation errors detected: "), "{msg}"); + assert!(msg.contains("length greater than or equal to 3"), "{msg}"); + assert!(msg.contains(pattern), "{msg}"); + // 255 two-byte characters (510 bytes): within the length bound, so only + // the pattern clause. + let wide = "\u{e9}".repeat(255); + let msg = validation_message( + validate_name_member(&wide, "backupName", BACKUP_NAME_BOUNDS).unwrap_err(), + ); + assert_eq!( + msg, + format!( + "1 validation error detected: Value '{wide}' at 'backupName' failed to satisfy \ + constraint: {pattern}" + ) + ); + + // A deployment that raised the maximum: a 300-character table can be + // backed up and listed; the backup name stays at 255. + let raised = LimitsConfig { + max_table_name_length: 300, + min_table_name_length: 2, + ..Default::default() + }; + let long = "t".repeat(300); + assert_eq!( + create_backup_table_name(&json!({ "TableName": long }), ACCOUNT, REGION, &raised).unwrap(), + long + ); + assert_eq!( + create_backup_table_name(&json!({ "TableName": "ab" }), ACCOUNT, REGION, &raised).unwrap(), + "ab" + ); + let arn_form = format!("arn:aws:dynamodb:{REGION}:{ACCOUNT}:table/{long}"); + assert_eq!( + create_backup_table_name(&json!({ "TableName": arn_form }), ACCOUNT, REGION, &raised) + .unwrap(), + long + ); + let fake = fake_with_summaries(vec![]); + list_backups( + &*fake, + ACCOUNT, + REGION, + &raised, + &json!({ "TableName": long }), + ) + .await + .unwrap(); + assert_eq!( + fake.listed_for.lock().unwrap()[0].as_deref(), + Some(long.as_str()) + ); + // The defaults still refuse it, with the bound in the text. + let msg = validation_message( + create_backup_table_name(&json!({ "TableName": long }), ACCOUNT, REGION, &limits()) + .unwrap_err(), + ); + assert!(msg.contains("length less than or equal to 255"), "{msg}"); + let msg = validation_message( + list_backups( + &*fake, + ACCOUNT, + REGION, + &limits(), + &json!({ "TableName": long }), + ) + .await + .unwrap_err(), + ); + assert!(msg.contains("length less than or equal to 255"), "{msg}"); + // A lowered maximum is reported with its own number. + let lowered = LimitsConfig { + max_table_name_length: 10, + ..Default::default() + }; + let msg = validation_message( + create_backup_table_name( + &json!({ "TableName": "elevenchars" }), + ACCOUNT, + REGION, + &lowered, + ) + .unwrap_err(), + ); + assert!(msg.contains("length less than or equal to 10"), "{msg}"); + let msg = validation_message( + validate_name_member(&"b".repeat(300), "backupName", BACKUP_NAME_BOUNDS).unwrap_err(), + ); + assert!(msg.contains("length less than or equal to 255"), "{msg}"); +} + +/// Measured on DynamoDB: `CreateBackup` accepts the +/// table's ARN as `TableName`. Another account's is refused as a backup ARN +/// of another account is (ExtendDB's choice, not measured); another region's, +/// or the server's region in another case, is the service's +/// `ValidationException` naming the ARN (measured on DynamoDB, twice each). +#[test] +fn create_backup_table_name_may_be_the_tables_arn() { + let own = format!("arn:aws:dynamodb:{REGION}:{ACCOUNT}:table/Music"); + assert_eq!( + create_backup_table_name(&json!({ "TableName": own }), ACCOUNT, REGION, &limits()).unwrap(), + "Music" + ); + for other_region in [ + format!("arn:aws:dynamodb:us-west-2:{ACCOUNT}:table/Music"), + format!( + "arn:aws:dynamodb:{}:{ACCOUNT}:table/Music", + REGION.to_ascii_uppercase() + ), + ] { + let err = create_backup_table_name( + &json!({ "TableName": other_region }), + ACCOUNT, + REGION, + &limits(), + ) + .unwrap_err(); + assert!( + matches!(&err, DynamoDbError::ValidationException(m) + if *m == format!("1 validation error detected: Invalid AWS region in '{other_region}'")), + "{other_region}: {err:?}" + ); + } + let foreign = format!("arn:aws:dynamodb:{REGION}:999999999999:table/Music"); + assert!(matches!( + create_backup_table_name(&json!({ "TableName": foreign }), ACCOUNT, REGION, &limits()) + .unwrap_err(), + DynamoDbError::AccessDeniedException(_) + )); + for bad in [ + format!("arn:aws:dynamodb:{REGION}:{ACCOUNT}:table/Music/backup/01791976431438-73d8d5bc"), + format!("arn:aws:dynamodb:{REGION}:{ACCOUNT}:table/Music/index/i"), + format!("arn:aws:dynamodb:{REGION}:{ACCOUNT}:index/Music"), + format!("arn:aws:s3:{REGION}:{ACCOUNT}:table/Music"), + format!("arn:aws-cn:dynamodb:{REGION}:{ACCOUNT}:table/Music"), + format!("arn:aws:dynamodb::{ACCOUNT}:table/Music"), + format!("arn:aws:dynamodb:{REGION}:{ACCOUNT}"), + "arn:".to_owned(), + ] { + let msg = validation_message( + create_backup_table_name(&json!({ "TableName": bad }), ACCOUNT, REGION, &limits()) + .unwrap_err(), + ); + assert_eq!( + msg, "Invalid Request: TableName is not a valid table name or table ARN", + "{bad}" + ); + } + // The name part is checked as a name. + let short = format!("arn:aws:dynamodb:{REGION}:{ACCOUNT}:table/ab"); + let msg = validation_message( + create_backup_table_name(&json!({ "TableName": short }), ACCOUNT, REGION, &limits()) + .unwrap_err(), + ); + assert!( + msg.contains("'tableName'") && msg.contains("length greater than or equal to 3"), + "{msg}" + ); + // Missing: the null-member text. + let msg = validation_message( + create_backup_table_name(&json!({}), ACCOUNT, REGION, &limits()).unwrap_err(), + ); + assert!(msg.contains("Value null at 'tableName'"), "{msg}"); +} + +/// Measured on DynamoDB: `CreateBackup` on a table that +/// does not exist or is still `CREATING` is `TableNotFoundException` +/// `Table not found: `, whatever the backend's own payload says. +#[test] +fn create_backup_on_a_missing_or_creating_table_is_table_not_found() { + for storage in [ + // PostgreSQL's payload and the bare name the other backends give. + StorageError::TableNotFound("Table not found: Music".to_owned()), + StorageError::TableNotFound("Music".to_owned()), + StorageError::TableNotActive("Music".to_owned()), + ] { + let what = format!("{storage:?}"); + let err = create_backup_storage_err_to_dynamo(storage, "Music"); + match &err { + DynamoDbError::TableNotFoundException(m) => { + assert_eq!(m, "Table not found: Music", "{what}") + } + other => panic!("{what}: expected TableNotFoundException, got {other:?}"), + } + assert_eq!( + err.full_error_type(), + "com.amazonaws.dynamodb.v20120810#TableNotFoundException" + ); + } + // Other errors keep the generic mapping. + assert!(matches!( + create_backup_storage_err_to_dynamo(StorageError::LimitExceeded("x".to_owned()), "Music"), + DynamoDbError::LimitExceededException(_) + )); + // The data plane keeps ResourceNotFoundException for a missing table. + assert!(matches!( + storage_err_to_dynamo(StorageError::TableNotFound("Music".to_owned())), + DynamoDbError::ResourceNotFoundException(_) + )); +} + +// --- DeleteBackup ------------------------------------------------------------- + +/// Measured on DynamoDB: `DeleteBackup` answers the full +/// DescribeBackup description with `BackupStatus` `DELETED`, including the +/// required `SourceTableDetails.ProvisionedThroughput` and the +/// `SourceTableFeatureDetails` the backend's row alone does not carry. +#[tokio::test] +async fn delete_backup_answers_the_full_description_as_deleted() { + for def in [definition("PROVISIONED"), definition("PAY_PER_REQUEST")] { + let on_demand = def.billing_mode == "PAY_PER_REQUEST"; + let fake = fake_with_backup("AVAILABLE", source(Some(def))); + let described = describe(&fake).await; + let deleted = delete_backup(&*fake, &runtime(), ACCOUNT, REGION, &own(&arn(ACCOUNT))) + .await + .unwrap(); + let deleted = serde_json::to_value(deleted).unwrap(); + assert_eq!(deleted["BackupDetails"]["BackupStatus"], "DELETED"); + assert_eq!( + deleted["SourceTableDetails"]["ProvisionedThroughput"], + if on_demand { + json!({ "ReadCapacityUnits": 0, "WriteCapacityUnits": 0 }) + } else { + json!({ "ReadCapacityUnits": 7, "WriteCapacityUnits": 8 }) + } + ); + assert_eq!( + deleted["SourceTableFeatureDetails"]["GlobalSecondaryIndexes"][0]["IndexName"], + "gsi1" + ); + assert_eq!( + deleted["SourceTableFeatureDetails"]["LocalSecondaryIndexes"][0]["IndexName"], + "lsi1" + ); + // Everything but the status is what DescribeBackup said. + let mut expected = described.clone(); + expected["BackupDetails"]["BackupStatus"] = json!("DELETED"); + assert_eq!(deleted, expected); + // The backup is gone. + assert!(matches!( + delete_backup(&*fake, &runtime(), ACCOUNT, REGION, &own(&arn(ACCOUNT))) + .await + .unwrap_err(), + DynamoDbError::BackupNotFoundException(_) + )); + assert_eq!( + fake.source_reads(), + 3, + "one read by DescribeBackup, one per delete attempt" + ); + } + + // A backup whose definition this build cannot read, or that does not + // parse, is still deleted, described without feature details. + for err in [ + StorageError::Unsupported("version 9".to_owned()), + StorageError::Internal("does not parse".to_owned()), + ] { + let what = format!("{err:?}"); + let fake = fake_with_backup("AVAILABLE", source(Some(definition("PROVISIONED")))); + *fake.source_error.lock().unwrap() = Some(err); + let deleted = delete_backup(&*fake, &runtime(), ACCOUNT, REGION, &own(&arn(ACCOUNT))) + .await + .unwrap_or_else(|e| panic!("{what}: {e:?}")); + assert_eq!(deleted.backup_details.backup_status, "DELETED", "{what}"); + assert!(deleted.source_table_feature_details.is_none(), "{what}"); + assert!( + deleted + .source_table_details + .provisioned_throughput + .is_some(), + "{what}" + ); + assert!( + fake.descriptions.lock().unwrap().is_empty(), + "{what}: not deleted" + ); + } + // A failure to reach the database propagates, and nothing is deleted. + let fake = fake_with_backup("AVAILABLE", source(Some(definition("PROVISIONED")))); + *fake.source_error.lock().unwrap() = Some(StorageError::Connection("refused".to_owned())); + assert!(matches!( + delete_backup(&*fake, &runtime(), ACCOUNT, REGION, &own(&arn(ACCOUNT))) + .await + .unwrap_err(), + DynamoDbError::InternalServerError(_) + )); + assert_eq!(fake.descriptions.lock().unwrap().len(), 1); +} + +#[test] +fn create_backup_refuses_a_table_with_vector_indexes() { + let plain = TableDescription { + table_name: "T".to_owned(), + ..Default::default() + }; + assert!(refuse_backup_of_vector_table(&plain).is_ok()); + + let with_vectors: TableDescription = serde_json::from_value(json!({ + "TableName": "T", + "KeySchema": [], + "AttributeDefinitions": [], + "TableStatus": "ACTIVE", + "CreationDateTime": 0.0, + "TableSizeBytes": 0, + "ItemCount": 0, + "TableArn": "", + "TableId": "", + "ProvisionedThroughput": { "ReadCapacityUnits": 0, "WriteCapacityUnits": 0, + "NumberOfDecreasesToday": 0 }, + "DeletionProtectionEnabled": false, + "VectorIndexes": [{ + "IndexName": "v", + "VectorAttribute": { "AttributeName": "emb" }, + "Dimensions": 3, + "DistanceFunction": "COSINE", + "IndexStatus": "ACTIVE", + "IndexSizeBytes": 0, + "ItemCount": 0, + "IndexArn": "" + }] + })) + .unwrap(); + let msg = validation_message(refuse_backup_of_vector_table(&with_vectors).unwrap_err()); + assert!(msg.starts_with("table T has 1 vector index(es)"), "{msg}"); + + // The restore's refusal reads the same after the subject. + let mut def = definition("PAY_PER_REQUEST"); + def.vector_index_names.push("v".to_owned()); + let restore_msg = validation_message(storage_err_to_dynamo( + def.ensure_restorable(&arn(ACCOUNT)).unwrap_err(), + )); + assert_eq!( + msg.trim_start_matches("table T"), + restore_msg.trim_start_matches(&format!("backup {}", arn(ACCOUNT))) + ); +} + +// --- DescribeBackup ----------------------------------------------------------- + +fn backup_description(status: &str, billing: &str) -> BackupDescription { + BackupDescription { + backup_details: BackupDetails { + backup_arn: arn(ACCOUNT), + backup_name: "b".to_owned(), + backup_status: status.to_owned(), + backup_type: "USER".to_owned(), + backup_size_bytes: 30, + backup_creation_date_time: CREATED, + }, + source_table_details: SourceTableDetails { + table_name: "Music".to_owned(), + table_id: "tid".to_owned(), + table_arn: source_arn(), + billing_mode: Some(billing.to_owned()), + ..Default::default() + }, + source_table_feature_details: None, + } +} + +fn fake_with_backup(status: &str, src: BackupRestoreSource) -> Arc { + let billing = src.billing_mode.clone(); + let fake = Fake::with_source(src); + fake.descriptions + .lock() + .unwrap() + .insert(arn(ACCOUNT), backup_description(status, &billing)); + fake +} + +async fn describe(fake: &Fake) -> Value { + serde_json::to_value( + describe_backup(fake, ACCOUNT, REGION, &own(&arn(ACCOUNT))) + .await + .unwrap(), + ) + .unwrap() +} + +#[tokio::test] +async fn describe_backup_reports_features_of_an_on_demand_table() { + let mut def = definition("PAY_PER_REQUEST"); + def.time_to_live = Some(TimeToLiveDescription { + time_to_live_status: TimeToLiveStatus::Enabled, + attribute_name: Some("exp".to_owned()), + }); + def.stream_specification = Some(StreamSpecification { + stream_enabled: true, + stream_view_type: Some(StreamViewType::NewImage), + }); + def.sse_specification = Some(json!({ "Enabled": true })); + let fake = fake_with_backup("AVAILABLE", source(Some(def))); + let v = describe(&fake).await; + + let pt = &v["SourceTableDetails"]["ProvisionedThroughput"]; + assert_eq!( + pt, + &json!({ "ReadCapacityUnits": 0, "WriteCapacityUnits": 0 }) + ); + let f = &v["SourceTableFeatureDetails"]; + assert_eq!(f["GlobalSecondaryIndexes"][0]["IndexName"], "gsi1"); + assert_eq!( + f["GlobalSecondaryIndexes"][0]["KeySchema"][0]["AttributeName"], + "gpk" + ); + assert_eq!( + f["GlobalSecondaryIndexes"][0]["Projection"]["ProjectionType"], + "ALL" + ); + assert_eq!( + f["GlobalSecondaryIndexes"][0]["ProvisionedThroughput"], + json!({ "ReadCapacityUnits": 0, "WriteCapacityUnits": 0 }) + ); + assert_eq!(f["LocalSecondaryIndexes"][0]["IndexName"], "lsi1"); + assert!( + f["LocalSecondaryIndexes"][0] + .get("ProvisionedThroughput") + .is_none() + ); + assert_eq!( + f["TimeToLiveDescription"], + json!({ "TimeToLiveStatus": "ENABLED", "AttributeName": "exp" }) + ); + assert_eq!( + f["StreamDescription"], + json!({ "StreamEnabled": true, "StreamViewType": "NEW_IMAGE" }) + ); + assert_eq!(f["SSEDescription"]["Status"], "ENABLED"); + assert_eq!( + f["SSEDescription"]["KMSMasterKeyArn"], + format!("arn:aws:kms:{REGION}:{ACCOUNT}:key/default") + ); +} + +#[tokio::test] +async fn describe_backup_reports_provisioned_throughput_and_omits_absent_members() { + let fake = fake_with_backup("AVAILABLE", source(Some(definition("PROVISIONED")))); + let v = describe(&fake).await; + assert_eq!( + v["SourceTableDetails"]["ProvisionedThroughput"], + json!({ "ReadCapacityUnits": 7, "WriteCapacityUnits": 8 }) + ); + let f = v["SourceTableFeatureDetails"].as_object().unwrap(); + assert_eq!( + f["GlobalSecondaryIndexes"][0]["ProvisionedThroughput"], + json!({ "ReadCapacityUnits": 3, "WriteCapacityUnits": 4 }) + ); + for absent in [ + "TimeToLiveDescription", + "StreamDescription", + "SSEDescription", + ] { + assert!(!f.contains_key(absent), "{absent}: {f:?}"); + } +} + +#[tokio::test] +async fn describe_backup_of_a_legacy_backup_has_no_feature_details() { + let mut src = source(None); + src.provisioned_throughput = Some(throughput(2, 3)); + let fake = fake_with_backup("AVAILABLE", src); + let v = describe(&fake).await; + assert!(v.get("SourceTableFeatureDetails").is_none(), "{v}"); + assert_eq!( + v["SourceTableDetails"]["ProvisionedThroughput"], + json!({ "ReadCapacityUnits": 2, "WriteCapacityUnits": 3 }) + ); +} + +/// A definition this build cannot read (a later version) does not stop +/// DescribeBackup: the backend's BackupDetails are returned without feature +/// details. A definition that does not parse is `Internal`, and propagates as +/// a 500 like a failure to reach the database (`Transient`, `Connection`): +/// a wrong definition row is a database fault, and describing the backup as +/// on-demand `{0, 0}` without feature details would mask it as a wrong answer +/// while RestoreTableFromBackup on the same read is a 500. +/// The errors are the ones every backend returns for those cases, built by +/// `BackupTableDefinition::from_json` itself. +#[tokio::test] +async fn describe_backup_survives_a_definition_that_does_not_read() { + let arn_text = arn(ACCOUNT); + let undecodable = BackupTableDefinition::from_json( + json!({ "Version": BACKUP_DEFINITION_VERSION, "BillingMode": 7 }), + &arn_text, + ) + .expect_err("a member of the wrong type does not parse"); + assert!( + matches!(undecodable, StorageError::Internal(_)), + "{undecodable:?}" + ); + let unknown_version = BackupTableDefinition::from_json( + json!({ "Version": BACKUP_DEFINITION_VERSION + 1 }), + &arn_text, + ) + .expect_err("an unknown version is refused"); + assert!( + matches!(unknown_version, StorageError::Unsupported(_)), + "{unknown_version:?}" + ); + { + let err = unknown_version; + let what = format!("{err:?}"); + let fake = fake_with_backup("AVAILABLE", source(Some(definition("PROVISIONED")))); + *fake.source_error.lock().unwrap() = Some(err); + let desc = describe_backup(&*fake, ACCOUNT, REGION, &own(&arn_text)) + .await + .unwrap_or_else(|e| panic!("{what}: {e:?}")); + assert_eq!(fake.source_reads(), 1, "{what}"); + assert_eq!(desc.backup_details.backup_status, "AVAILABLE", "{what}"); + assert_eq!(desc.backup_details.backup_arn, arn_text, "{what}"); + assert!(desc.source_table_feature_details.is_none(), "{what}"); + assert!( + desc.source_table_details.provisioned_throughput.is_some(), + "{what}: required member" + ); + } + + for err in [ + undecodable, + StorageError::Internal("column \"definition\" does not exist".to_owned()), + StorageError::Transient( + "Database error: pool timed out while waiting for an open connection".to_owned(), + ), + StorageError::Connection("connection refused".to_owned()), + ] { + let what = format!("{err:?}"); + let fake = fake_with_backup("AVAILABLE", source(Some(definition("PROVISIONED")))); + *fake.source_error.lock().unwrap() = Some(err); + // The backup operations' mapping reports both as a 500 + // (`storage_err_to_dynamo` in backup.rs), which SDKs retry. + match describe_backup(&*fake, ACCOUNT, REGION, &own(&arn_text)).await { + Err(DynamoDbError::InternalServerError(_)) => {} + other => panic!("{what} must reach the client as an error: {other:?}"), + } + assert_eq!(fake.source_reads(), 1, "{what}"); + } +} + +#[tokio::test] +async fn describe_backup_of_a_backup_that_is_not_available_uses_the_backend_row() { + let fake = fake_with_backup("CREATING", source(Some(definition("PROVISIONED")))); + let v = describe(&fake).await; + assert_eq!(fake.source_reads(), 0); + assert!(v.get("SourceTableFeatureDetails").is_none()); + assert_eq!(v["BackupDetails"]["BackupStatus"], "CREATING"); + // Required member, still present. + assert!( + v["SourceTableDetails"] + .get("ProvisionedThroughput") + .is_some() + ); + + // AVAILABLE in describe, gone by the source read: described without + // feature details rather than failing. + let fake = fake_with_backup("AVAILABLE", source(Some(definition("PROVISIONED")))); + fake.sources.lock().unwrap().clear(); + let v = describe(&fake).await; + assert_eq!(fake.source_reads(), 1); + assert!(v.get("SourceTableFeatureDetails").is_none()); +} + +// --- ListBackups ---------------------------------------------------------------- + +fn summary(n: u64, millis: u64) -> BackupSummary { + let id = extenddb_storage::util::format_backup_id(millis, u32::try_from(n).unwrap()); + BackupSummary { + backup_arn: extenddb_storage::util::backup_arn(REGION, ACCOUNT, "Music", &id), + backup_name: format!("b{n}"), + table_name: "Music".to_owned(), + table_id: "tid".to_owned(), + table_arn: source_arn(), + backup_status: "AVAILABLE".to_owned(), + backup_type: "USER".to_owned(), + backup_size_bytes: 0, + backup_creation_date_time: extenddb_storage::util::backup_creation_date_time(&id).unwrap(), + } +} + +fn fake_with_summaries(summaries: Vec) -> Arc { + let fake = Arc::new(Fake::default()); + *fake.summaries.lock().unwrap() = summaries; + fake +} + +async fn list(fake: &Fake, body: Value) -> Result { + list_backups(fake, ACCOUNT, REGION, &limits(), &body).await +} + +fn arns(page: &Value) -> Vec { + page["BackupSummaries"] + .as_array() + .unwrap() + .iter() + .map(|s| s["BackupArn"].as_str().unwrap().to_owned()) + .collect() +} + +#[tokio::test] +async fn list_backups_defaults_to_one_hundred() { + let fake = fake_with_summaries((0..150).map(|n| summary(n, 1_000 + n)).collect()); + let page = list(&fake, json!({})).await.unwrap(); + assert_eq!(arns(&page).len(), 100); + assert_eq!( + page["LastEvaluatedBackupArn"].as_str(), + Some(arns(&page)[99].as_str()) + ); + assert_eq!(page["BackupSummaries"][0]["TableId"], "tid"); +} + +#[tokio::test] +async fn list_backups_limit_is_bounded() { + let fake = fake_with_summaries(vec![summary(1, 1)]); + for bad in [0, -1, 101] { + let msg = validation_message(list(&fake, json!({ "Limit": bad })).await.unwrap_err()); + assert!(msg.contains("'limit'"), "{msg}"); + } + assert!(matches!( + list(&fake, json!({ "Limit": "ten" })).await.unwrap_err(), + DynamoDbError::SerializationException(_) + )); + for ok in [1, 100] { + list(&fake, json!({ "Limit": ok })).await.unwrap(); + } +} + +#[tokio::test] +async fn list_backups_pages_in_a_stable_order_exactly_once() { + // Out of order, with two backups in the same millisecond. + let all = vec![ + summary(5, 500), + summary(1, 100), + summary(3, 300), + summary(2, 300), + summary(4, 400), + ]; + let mut expected: Vec = all.clone(); + expected.sort_by(|a, b| { + a.backup_creation_date_time + .total_cmp(&b.backup_creation_date_time) + .then_with(|| a.backup_arn.cmp(&b.backup_arn)) + }); + let expected: Vec = expected.into_iter().map(|s| s.backup_arn).collect(); + + let fake = fake_with_summaries(all); + let mut seen = Vec::new(); + let mut cursor: Option = None; + let mut pages = 0; + loop { + let mut body = json!({ "Limit": 2 }); + if let Some(c) = &cursor { + body["ExclusiveStartBackupArn"] = json!(c); + } + let page = list(&fake, body).await.unwrap(); + pages += 1; + seen.extend(arns(&page)); + match page.get("LastEvaluatedBackupArn") { + Some(c) => cursor = Some(c.as_str().unwrap().to_owned()), + None => break, + } + assert!(pages < 10, "paging did not terminate"); + } + assert_eq!(seen, expected); + assert_eq!(pages, 3); +} + +#[tokio::test] +async fn list_backups_last_page_has_no_cursor() { + let fake = fake_with_summaries((0..4).map(|n| summary(n, n)).collect()); + let first = list(&fake, json!({ "Limit": 2 })).await.unwrap(); + let cursor = first["LastEvaluatedBackupArn"].as_str().unwrap().to_owned(); + let second = list( + &fake, + json!({ "Limit": 2, "ExclusiveStartBackupArn": cursor }), + ) + .await + .unwrap(); + assert_eq!(arns(&second).len(), 2); + assert!(second.get("LastEvaluatedBackupArn").is_none()); + + // Exactly the limit, from the start. + let all = list(&fake, json!({ "Limit": 4 })).await.unwrap(); + assert!(all.get("LastEvaluatedBackupArn").is_none()); + // Empty listing. + let none = list(&fake_with_summaries(vec![]), json!({})).await.unwrap(); + assert_eq!(none, json!({ "BackupSummaries": [] })); +} + +#[tokio::test] +async fn list_backups_resumes_after_a_cursor_whose_backup_was_deleted() { + let all: Vec = (0..5).map(|n| summary(n, 100 * (n + 1))).collect(); + let deleted = all[2].backup_arn.clone(); + let remaining: Vec = all + .iter() + .filter(|s| s.backup_arn != deleted) + .cloned() + .collect(); + let fake = fake_with_summaries(remaining); + let page = list(&fake, json!({ "ExclusiveStartBackupArn": deleted })) + .await + .unwrap(); + assert_eq!( + arns(&page), + vec![all[3].backup_arn.clone(), all[4].backup_arn.clone()] + ); + + // A deleted cursor after every remaining backup: an empty last page. + let fake = fake_with_summaries(all[..2].to_vec()); + let page = list(&fake, json!({ "ExclusiveStartBackupArn": deleted })) + .await + .unwrap(); + assert!(arns(&page).is_empty()); + assert!(page.get("LastEvaluatedBackupArn").is_none()); +} + +#[tokio::test] +async fn list_backups_refuses_a_cursor_it_cannot_place() { + let fake = fake_with_summaries(vec![summary(1, 1)]); + // Well formed, ours, not listed, and its id (one of the two shapes + // earlier builds issued: 13 digits with the hex suffix, 13 bare digits) + // carries no time: `backup_id_epoch_millis` needs the 14-digit form. + for id in ["1791976431438-73d8d5bc", "1791976431438"] { + let legacy = format!("arn:aws:dynamodb:{REGION}:{ACCOUNT}:table/Music/backup/{id}"); + let msg = validation_message( + list(&fake, json!({ "ExclusiveStartBackupArn": legacy })) + .await + .unwrap_err(), + ); + assert!(msg.contains("ExclusiveStartBackupArn"), "{id}: {msg}"); + assert!(msg.contains("carries no creation time"), "{id}: {msg}"); + } + + assert_eq!( + validation_message( + list(&fake, json!({ "ExclusiveStartBackupArn": "garbage" })) + .await + .unwrap_err() + ), + "Invalid Request: BackupArn is not valid" + ); +} + +/// Measured on DynamoDB: a cursor of another account, +/// another region, or a table other than `TableName`'s is a +/// `ValidationException`, in that order; the account text is the service's, +/// the other two complete the fragments captured ("does not match the +/// region", "does not match the table name predicate"). +#[tokio::test] +async fn list_backups_cursor_must_match_the_account_region_and_table() { + let fake = fake_with_summaries(vec![summary(1, 1), summary(2, 2)]); + let id = "01791976431438-73d8d5bc"; + let cases = [ + ( + format!("arn:aws:dynamodb:{REGION}:999999999999:table/Music/backup/{id}"), + None, + "account", + ), + ( + format!("arn:aws:dynamodb:eu-west-1:{ACCOUNT}:table/Music/backup/{id}"), + None, + "region", + ), + ( + format!("arn:aws:dynamodb:{REGION}:{ACCOUNT}:table/Other/backup/{id}"), + Some("Music"), + "table name predicate", + ), + // Account before region before table. + ( + format!("arn:aws:dynamodb:eu-west-1:999999999999:table/Other/backup/{id}"), + Some("Music"), + "account", + ), + ( + format!("arn:aws:dynamodb:eu-west-1:{ACCOUNT}:table/Other/backup/{id}"), + Some("Music"), + "region", + ), + ]; + for (cursor, table, what) in cases { + let mut body = json!({ "ExclusiveStartBackupArn": cursor }); + if let Some(table) = table { + body["TableName"] = json!(table); + } + let msg = validation_message(list(&fake, body).await.unwrap_err()); + assert_eq!( + msg, + format!( + "Invalid Request: The supplied ExclusiveStartBackupArn does not match the {what}" + ), + "{cursor}" + ); + } + // Another table's cursor without TableName is fine (the listing is the + // account's), and a trailing newline is trimmed. + let cursor = format!("arn:aws:dynamodb:{REGION}:{ACCOUNT}:table/Other/backup/{id}\n"); + list(&fake, json!({ "ExclusiveStartBackupArn": cursor })) + .await + .unwrap(); + // The cursor of the listing itself resumes after it. + let first = summary(1, 1).backup_arn; + let page = list( + &fake, + json!({ "ExclusiveStartBackupArn": format!("{first}\n"), "TableName": "Music" }), + ) + .await + .unwrap(); + assert_eq!(arns(&page), vec![summary(2, 2).backup_arn]); +} + +/// A backup created under an earlier `server.region` keeps that region in its +/// stored ARN, and `ListBackups` hands that ARN out as the cursor; the next +/// page accepts it (trimmed). Another region's ARN that is not a listed +/// backup's is still refused. +#[tokio::test] +async fn list_backups_pages_past_a_backup_stored_under_another_region() { + let mut old = summary(2, 2); + old.backup_arn = old.backup_arn.replace(REGION, "eu-west-1"); + let all = vec![summary(1, 1), old.clone(), summary(3, 3)]; + let fake = fake_with_summaries(all); + let mut seen = Vec::new(); + let mut body = json!({ "Limit": 2 }); + let first = list(&fake, body.clone()).await.unwrap(); + seen.extend(arns(&first)); + assert_eq!( + first["LastEvaluatedBackupArn"].as_str(), + Some(old.backup_arn.as_str()) + ); + body["ExclusiveStartBackupArn"] = json!(format!("{}\n", old.backup_arn)); + for table in [None, Some("Music")] { + if let Some(t) = table { + body["TableName"] = json!(t); + } + let second = list(&fake, body.clone()).await.unwrap(); + assert_eq!(arns(&second), vec![summary(3, 3).backup_arn], "{body}"); + } + seen.extend(arns(&list(&fake, body.clone()).await.unwrap())); + assert_eq!(seen.len(), 3); + let stranger = summary(9, 9).backup_arn.replace(REGION, "eu-west-1"); + let msg = validation_message( + list(&fake, json!({ "ExclusiveStartBackupArn": stranger })) + .await + .unwrap_err(), + ); + assert_eq!( + msg, + "Invalid Request: The supplied ExclusiveStartBackupArn does not match the region" + ); +} + +/// Measured on DynamoDB: `TableName` on `ListBackups` is checked as the service +/// model constrains it. +#[tokio::test] +async fn list_backups_table_name_is_validated() { + let fake = fake_with_summaries(vec![summary(1, 1)]); + let msg = validation_message(list(&fake, json!({ "TableName": "a" })).await.unwrap_err()); + assert_eq!( + msg, + "1 validation error detected: Value 'a' at 'tableName' failed to satisfy constraint: \ + Member must have length greater than or equal to 3" + ); + assert!(fake.listed_for.lock().unwrap().is_empty()); + list(&fake, json!({ "TableName": "Music" })).await.unwrap(); + assert_eq!( + fake.listed_for.lock().unwrap().as_slice(), + [Some("Music".to_owned())] + ); +} + +#[tokio::test] +async fn list_backups_passes_the_table_filter_through() { + let fake = fake_with_summaries(vec![]); + list(&fake, json!({ "TableName": "Music" })).await.unwrap(); + list(&fake, json!({})).await.unwrap(); + assert_eq!( + *fake.listed_for.lock().unwrap(), + vec![Some("Music".to_owned()), None] + ); +} + +#[tokio::test] +async fn list_backups_filters_on_backup_type() { + let mut system = summary(2, 200); + system.backup_type = "SYSTEM".to_owned(); + let mut aws_backup = summary(3, 300); + aws_backup.backup_type = "AWS_BACKUP".to_owned(); + let user = summary(1, 100); + let fake = fake_with_summaries(vec![user.clone(), system.clone(), aws_backup.clone()]); + for (filter, expected) in [ + (json!(null), vec![user.backup_arn.clone()]), + (json!("USER"), vec![user.backup_arn.clone()]), + (json!("SYSTEM"), vec![system.backup_arn.clone()]), + (json!("AWS_BACKUP"), vec![aws_backup.backup_arn.clone()]), + ( + json!("ALL"), + vec![ + user.backup_arn.clone(), + system.backup_arn.clone(), + aws_backup.backup_arn.clone(), + ], + ), + ] { + let page = list(&fake, json!({ "BackupType": filter })).await.unwrap(); + assert_eq!(arns(&page), expected, "{filter}"); + } + let page = list(&fake, json!({})).await.unwrap(); + assert_eq!(arns(&page), vec![user.backup_arn.clone()]); + + let msg = validation_message( + list(&fake, json!({ "BackupType": "user" })) + .await + .unwrap_err(), + ); + assert_eq!( + msg, + "1 validation error detected: Value 'user' at 'backupType' failed to satisfy \ + constraint: Member must satisfy enum value set: [USER, SYSTEM, AWS_BACKUP, ALL]" + ); + assert!(matches!( + list(&fake, json!({ "BackupType": 1 })).await.unwrap_err(), + DynamoDbError::SerializationException(_) + )); +} + +#[tokio::test] +async fn list_backups_filters_on_the_time_range_before_paging() { + // Creation times 1.000, 2.000, 3.000, 4.000, 5.000 s. + let all: Vec = (1..=5).map(|n| summary(n, 1_000 * n)).collect(); + let arn_of = |n: usize| all[n - 1].backup_arn.clone(); + let fake = fake_with_summaries(all.clone()); + for (body, expected) in [ + // Lower bound inclusive, upper bound exclusive, at the millisecond. + (json!({ "TimeRangeLowerBound": 2.0 }), vec![2, 3, 4, 5]), + (json!({ "TimeRangeLowerBound": 2.001 }), vec![3, 4, 5]), + (json!({ "TimeRangeUpperBound": 4.0 }), vec![1, 2, 3]), + (json!({ "TimeRangeUpperBound": 4.001 }), vec![1, 2, 3, 4]), + ( + json!({ "TimeRangeLowerBound": 2, "TimeRangeUpperBound": 4 }), + vec![2, 3], + ), + ( + json!({ "TimeRangeLowerBound": 3, "TimeRangeUpperBound": 3 }), + vec![], + ), + ( + json!({ "TimeRangeLowerBound": null, "TimeRangeUpperBound": null }), + vec![1, 2, 3, 4, 5], + ), + ( + json!({ "TimeRangeLowerBound": 0, "TimeRangeUpperBound": 0 }), + vec![], + ), + ] { + let page = list(&fake, body.clone()).await.unwrap(); + let want: Vec = expected.into_iter().map(arn_of).collect(); + assert_eq!(arns(&page), want, "{body}"); + } + + // Measured on DynamoDB: a lower bound above the upper + // bound and a bound before the epoch are refused. + assert_eq!( + validation_message( + list( + &fake, + json!({ "TimeRangeLowerBound": 4, "TimeRangeUpperBound": 2 }) + ) + .await + .unwrap_err() + ), + "Time range lower bound must be less than or equal to upper bound" + ); + assert_eq!( + validation_message( + list( + &fake, + json!({ "TimeRangeLowerBound": 2.5, "TimeRangeUpperBound": 2.4999 }) + ) + .await + .unwrap_err() + ), + "Time range lower bound must be less than or equal to upper bound" + ); + for member in ["TimeRangeLowerBound", "TimeRangeUpperBound"] { + for at in [-1, -1_000_000_000] { + let msg = validation_message(list(&fake, json!({ member: at })).await.unwrap_err()); + assert!(msg.contains(member), "{member} {at}: {msg}"); + assert!(msg.contains("1970-01-01T00:00:00Z"), "{member} {at}: {msg}"); + } + } + + // Paging walks the filtered list: Limit counts only backups in range, and + // the last in-range page has no cursor although later backups exist. + let mut seen = Vec::new(); + let mut body = json!({ "TimeRangeLowerBound": 2, "TimeRangeUpperBound": 4.5, "Limit": 2 }); + let first = list(&fake, body.clone()).await.unwrap(); + seen.extend(arns(&first)); + assert_eq!( + first["LastEvaluatedBackupArn"].as_str(), + Some(arn_of(3).as_str()) + ); + body["ExclusiveStartBackupArn"] = first["LastEvaluatedBackupArn"].clone(); + let second = list(&fake, body).await.unwrap(); + seen.extend(arns(&second)); + assert!( + second.get("LastEvaluatedBackupArn").is_none(), + "the last page carries a cursor" + ); + assert_eq!(seen, vec![arn_of(2), arn_of(3), arn_of(4)]); + + for member in ["TimeRangeLowerBound", "TimeRangeUpperBound"] { + assert!(matches!( + list(&fake, json!({ member: "2026-01-01" })).await.unwrap_err(), + DynamoDbError::SerializationException(m) if m.contains(member) + )); + } +} + +// --- the catalog index --------------------------------------------------------- + +/// An ARN of the `Music` table in account `account` with backup id `id`. +fn music_arn(account: &str, id: &str) -> String { + format!("arn:aws:dynamodb:{REGION}:{account}:table/Music/backup/{id}") +} + +fn fake_with_row(status: BackupStatus) -> Arc { + let fake = Arc::new(Fake::default()); + fake.rows + .lock() + .unwrap() + .insert(arn(ACCOUNT), row(&arn(ACCOUNT), status)); + fake +} + +fn runtime_with_store() -> BackupRuntime { + runtime_on(&Arc::new(MemStore::default()), store()) +} + +/// DescribeBackup answers from the catalog row when one exists, derived from +/// the recorded source table, and never reads the legacy reader for it. +#[tokio::test] +async fn describe_backup_reads_the_catalog_row_first() { + let fake = fake_with_row(BackupStatus::Available); + // A legacy description under the same ARN would answer differently. + fake.descriptions.lock().unwrap().insert( + arn(ACCOUNT), + backup_description("AVAILABLE", "PAY_PER_REQUEST"), + ); + let desc = describe_backup(&*fake, ACCOUNT, REGION, &own(&arn(ACCOUNT))) + .await + .unwrap(); + assert_eq!(desc.backup_details.backup_name, "music-nightly"); + assert_eq!(desc.backup_details.backup_status, "AVAILABLE"); + assert_eq!(desc.backup_details.backup_size_bytes, 61828); + assert_eq!( + desc.source_table_details.table_id, + "12345a12-abcd-123a-ab12-1234abc12345" + ); + assert_eq!( + desc.source_table_details.billing_mode.as_deref(), + Some("PROVISIONED") + ); + let features = desc.source_table_feature_details.expect("features"); + assert_eq!( + features.global_secondary_indexes.unwrap()[0].index_name, + "GenreIndex" + ); + assert_eq!( + fake.source_reads(), + 0, + "the legacy reader was not consulted" + ); +} + +/// The status mapping of the catalog index: CREATING as itself, DELETING as +/// DELETED, DELETED and FAILED as `BackupNotFoundException`, with no fall +/// back to the legacy reader for a tombstone. +#[tokio::test] +async fn describe_backup_maps_the_row_status() { + let fake = fake_with_row(BackupStatus::Creating); + let desc = describe_backup(&*fake, ACCOUNT, REGION, &own(&arn(ACCOUNT))) + .await + .unwrap(); + assert_eq!(desc.backup_details.backup_status, "CREATING"); + + let fake = fake_with_row(BackupStatus::Deleting); + let desc = describe_backup(&*fake, ACCOUNT, REGION, &own(&arn(ACCOUNT))) + .await + .unwrap(); + assert_eq!(desc.backup_details.backup_status, "DELETED"); + + for status in [BackupStatus::Deleted, BackupStatus::Failed] { + let fake = fake_with_row(status); + fake.descriptions + .lock() + .unwrap() + .insert(arn(ACCOUNT), backup_description("AVAILABLE", "PROVISIONED")); + let err = describe_backup(&*fake, ACCOUNT, REGION, &own(&arn(ACCOUNT))) + .await + .unwrap_err(); + assert!( + matches!(err, DynamoDbError::BackupNotFoundException(ref m) if m.contains(&arn(ACCOUNT))), + "{status}: {err:?}" + ); + assert_eq!(fake.source_reads(), 0, "{status}"); + } +} + +/// An ARN of account B that carries the id of account A's backup is read by +/// B's account and the id, matches no row, and is BackupNotFound: never A's +/// row. +#[tokio::test] +async fn describe_backup_of_another_accounts_id_is_not_found() { + const OTHER: &str = "999999999999"; + let fake = fake_with_row(BackupStatus::Available); + let other = own_for(&music_arn(OTHER, "01791976431438-73d8d5bc"), OTHER); + let err = describe_backup(&*fake, OTHER, REGION, &other) + .await + .unwrap_err(); + assert!( + matches!(err, DynamoDbError::BackupNotFoundException(_)), + "{err:?}" + ); + // The lookup carried B's account, so B's own rows (none) were read. + assert_eq!( + fake.row_reads.lock().unwrap().as_slice(), + [music_arn(OTHER, "01791976431438-73d8d5bc")] + ); +} + +/// `own` for another account. +fn own_for(arn: &str, account: &str) -> BackupArnSpellings { + backup_arn_field(&json!({ "BackupArn": arn }), account, REGION).unwrap() +} + +/// A row stored under another region resolves under the ARN as written, +/// after the canonical spelling found nothing, as a legacy backup does. +#[tokio::test] +async fn describe_backup_tries_the_row_under_the_arn_as_written() { + let stored = + format!("arn:aws:dynamodb:eu-west-1:{ACCOUNT}:table/Music/backup/01791976431438-73d8d5bc"); + let fake = Fake::default(); + fake.rows + .lock() + .unwrap() + .insert(stored.clone(), row(&stored, BackupStatus::Available)); + let desc = describe_backup(&fake, ACCOUNT, REGION, &own(&stored)) + .await + .unwrap(); + assert_eq!(desc.backup_details.backup_arn, stored); + assert_eq!( + fake.row_reads.lock().unwrap().as_slice(), + [arn(ACCOUNT), stored.clone()], + "canonical first, then as written" + ); +} + +/// ListBackups merges the catalog's rows with the legacy reader's summaries +/// into one (creation time, ARN) order, lists CREATING and AVAILABLE rows +/// only, and pages the merged list with one cursor. +#[tokio::test] +async fn list_backups_merges_the_catalog_rows_with_the_legacy_summaries() { + let fake = fake_with_summaries(vec![summary(1, 1_000), summary(3, 3_000)]); + let ids = |n: u32, millis: u64| extenddb_storage::util::format_backup_id(millis, n); + let catalog = |n: u32, millis: u64, status: BackupStatus| { + let a = music_arn(ACCOUNT, &ids(n, millis)); + row(&a, status) + }; + { + let mut rows = fake.rows.lock().unwrap(); + for r in [ + catalog(2, 2_000, BackupStatus::Available), + catalog(4, 4_000, BackupStatus::Creating), + catalog(5, 5_000, BackupStatus::Deleting), + catalog(6, 6_000, BackupStatus::Deleted), + catalog(7, 7_000, BackupStatus::Failed), + ] { + rows.insert(r.backup_arn.clone(), r); + } + } + let page = list(&fake, json!({ "Limit": 3 })).await.unwrap(); + assert_eq!( + arns(&page), + [ + summary(1, 1_000).backup_arn, + music_arn(ACCOUNT, &ids(2, 2_000)), + summary(3, 3_000).backup_arn, + ] + ); + let cursor = page["LastEvaluatedBackupArn"].as_str().unwrap().to_owned(); + let page = list(&fake, json!({ "ExclusiveStartBackupArn": cursor })) + .await + .unwrap(); + assert_eq!(arns(&page), [music_arn(ACCOUNT, &ids(4, 4_000))]); + assert!(page.get("LastEvaluatedBackupArn").is_none()); + let creating = &page["BackupSummaries"][0]; + assert_eq!(creating["BackupStatus"], "CREATING"); + assert_eq!(creating["TableId"], "12345a12-abcd-123a-ab12-1234abc12345"); +} + +/// `TableName` may be the table's ARN: it lists what the name lists, and the +/// cursor's table check runs against the resolved name. +#[tokio::test] +async fn list_backups_accepts_a_table_arn_as_table_name() { + let fake = fake_with_summaries(vec![summary(1, 1_000)]); + let by_name = list(&fake, json!({ "TableName": "Music" })).await.unwrap(); + let by_arn = list(&fake, json!({ "TableName": source_arn() })) + .await + .unwrap(); + assert_eq!(arns(&by_name), arns(&by_arn)); + assert_eq!( + fake.listed_for.lock().unwrap().as_slice(), + [Some("Music".to_owned()), Some("Music".to_owned())] + ); + let cursor = summary(1, 1_000).backup_arn; + list( + &fake, + json!({ "TableName": source_arn(), "ExclusiveStartBackupArn": cursor }), + ) + .await + .unwrap(); + let other = format!("arn:aws:dynamodb:{REGION}:{ACCOUNT}:table/Other"); + let msg = validation_message( + list( + &fake, + json!({ "TableName": other, "ExclusiveStartBackupArn": summary(1, 1_000).backup_arn }), + ) + .await + .unwrap_err(), + ); + assert!(msg.contains("table name predicate"), "{msg}"); + let foreign = format!("arn:aws:dynamodb:{REGION}:999999999999:table/Music"); + assert!(matches!( + list(&fake, json!({ "TableName": foreign })) + .await + .unwrap_err(), + DynamoDbError::AccessDeniedException(_) + )); + // Another region's ARN, or the server's region in another case: the + // service's ValidationException (measured on DynamoDB), not the + // server-region table's backups. + for other_region in [ + format!("arn:aws:dynamodb:us-west-2:{ACCOUNT}:table/Music"), + format!( + "arn:aws:dynamodb:{}:{ACCOUNT}:table/Music", + REGION.to_ascii_uppercase() + ), + ] { + let msg = validation_message( + list(&fake, json!({ "TableName": other_region })) + .await + .unwrap_err(), + ); + assert_eq!( + msg, + format!("1 validation error detected: Invalid AWS region in '{other_region}'") + ); + } + assert_eq!( + fake.listed_for.lock().unwrap().len(), + 3, + "the refused requests reached no listing" + ); +} + +/// The time range is handed to the catalog as bounds and applied to the +/// legacy summaries as before, so both sides are filtered before paging. +#[tokio::test] +async fn list_backups_applies_the_time_range_to_both_sources() { + let fake = fake_with_summaries(vec![summary(1, 1_000), summary(3, 3_000)]); + let a2 = music_arn(ACCOUNT, &extenddb_storage::util::format_backup_id(2_000, 2)); + let a4 = music_arn(ACCOUNT, &extenddb_storage::util::format_backup_id(4_000, 4)); + { + let mut rows = fake.rows.lock().unwrap(); + rows.insert(a2.clone(), row(&a2, BackupStatus::Available)); + rows.insert(a4.clone(), row(&a4, BackupStatus::Available)); + } + let page = list( + &fake, + json!({ "TimeRangeLowerBound": 2.0, "TimeRangeUpperBound": 4.0 }), + ) + .await + .unwrap(); + assert_eq!(arns(&page), [a2, summary(3, 3_000).backup_arn]); + assert_eq!( + fake.bounds_seen.lock().unwrap().as_slice(), + [TimeBounds { + lower: Some(2.0), + upper: Some(4.0), + }], + "the bounds reached the catalog as bounds" + ); +} + +/// DeleteBackup over a catalog row, in the request's order: a tombstone is +/// BackupNotFound, a CREATING row BackupInUse, a row of another store a +/// ValidationException that names no root, and nothing reaches the row's +/// lock for any of them. +#[tokio::test] +async fn delete_backup_over_a_row_refuses_before_the_lock() { + for status in [BackupStatus::Deleted, BackupStatus::Failed] { + let fake = fake_with_row(status); + let err = delete_backup( + &*fake, + &runtime_with_store(), + ACCOUNT, + REGION, + &own(&arn(ACCOUNT)), + ) + .await + .unwrap_err(); + assert!( + matches!(err, DynamoDbError::BackupNotFoundException(_)), + "{status}: {err:?}" + ); + assert!(fake.delete_starts_seen.lock().unwrap().is_empty()); + } + let fake = fake_with_row(BackupStatus::Creating); + let err = delete_backup( + &*fake, + &runtime_with_store(), + ACCOUNT, + REGION, + &own(&arn(ACCOUNT)), + ) + .await + .unwrap_err(); + assert!( + matches!(err, DynamoDbError::BackupInUseException(_)), + "{err:?}" + ); + assert!(fake.delete_starts_seen.lock().unwrap().is_empty()); + + // No store configured: every row is a row of another store. + let fake = fake_with_row(BackupStatus::Available); + let msg = validation_message( + delete_backup(&*fake, &runtime(), ACCOUNT, REGION, &own(&arn(ACCOUNT))) + .await + .unwrap_err(), + ); + assert!(msg.contains("store other than"), "{msg}"); + assert!( + !msg.contains(&store().store_root), + "no root is named: {msg}" + ); + assert!(fake.delete_starts_seen.lock().unwrap().is_empty()); + // The legacy reader was never asked. + assert!(fake.deleted.lock().unwrap().is_empty()); + assert_eq!(fake.source_reads(), 0); + + // The in-use check precedes the other-store check, as it does for a + // restore: a row of another store with a CREATING target registered + // from it (by an instance configured with that store) is BackupInUse + // naming the target, not the other-store refusal. + let fake = fake_with_row(BackupStatus::Available); + fake.in_use + .lock() + .unwrap() + .insert(arn(ACCOUNT), "Earlier".to_owned()); + let err = delete_backup(&*fake, &runtime(), ACCOUNT, REGION, &own(&arn(ACCOUNT))) + .await + .unwrap_err(); + assert!( + matches!(err, DynamoDbError::BackupInUseException(ref m) + if *m == format!("Backup is being used to restore another table: {}", arn(ACCOUNT))), + "the service's text, naming the backup and not the target: {err:?}" + ); + assert!(fake.delete_starts_seen.lock().unwrap().is_empty()); +} + +/// DeleteBackup over an AVAILABLE row of the configured store: the in-use +/// check and the DELETING write run under the row's lock +/// (`start_backup_delete`), whose answer decides: a registration that won is +/// BackupInUse naming its target; a row that was CREATING under the lock is +/// BackupInUse; one that moved to a tombstone is BackupNotFound; DELETING +/// (new or resumed) answers the full description as DELETED. +#[tokio::test] +async fn delete_backup_over_a_row_runs_the_in_use_check_under_the_lock() { + let fake = fake_with_row(BackupStatus::Available); + let desc = delete_backup( + &*fake, + &runtime_with_store(), + ACCOUNT, + REGION, + &own(&arn(ACCOUNT)), + ) + .await + .unwrap(); + assert_eq!(desc.backup_details.backup_status, "DELETED"); + assert_eq!(desc.backup_details.backup_name, "music-nightly"); + assert!(desc.source_table_details.provisioned_throughput.is_some()); + assert!(desc.source_table_feature_details.is_some()); + assert_eq!( + fake.delete_starts_seen.lock().unwrap().as_slice(), + [BackupKey::from_arn( + &BackupArn::parse(&arn(ACCOUNT)).unwrap() + )] + ); + + let fake = fake_with_row(BackupStatus::Deleting); + let desc = delete_backup( + &*fake, + &runtime_with_store(), + ACCOUNT, + REGION, + &own(&arn(ACCOUNT)), + ) + .await + .unwrap(); + assert_eq!( + desc.backup_details.backup_status, "DELETED", + "a retry resumes" + ); + + for (answer, want) in [ + ( + DeleteStart::InUse("MusicRestored".to_owned()), + "BackupInUseException", + ), + ( + DeleteStart::Status(BackupStatus::Creating), + "BackupInUseException", + ), + ( + DeleteStart::Status(BackupStatus::Deleted), + "BackupNotFoundException", + ), + ( + DeleteStart::Status(BackupStatus::Failed), + "BackupNotFoundException", + ), + ] { + let fake = fake_with_row(BackupStatus::Available); + fake.delete_starts + .lock() + .unwrap() + .insert(arn(ACCOUNT), answer.clone()); + let err = delete_backup( + &*fake, + &runtime_with_store(), + ACCOUNT, + REGION, + &own(&arn(ACCOUNT)), + ) + .await + .unwrap_err(); + let got = err.full_error_type(); + assert!(got.ends_with(want), "{answer:?}: {got}"); + if let DeleteStart::InUse(_) = &answer { + let DynamoDbError::BackupInUseException(m) = err else { + unreachable!() + }; + assert_eq!( + m, + format!( + "Backup is being used to restore another table: {}", + arn(ACCOUNT) + ), + "the service's text names the backup, not the target" + ); + } + } + + // `Status(AVAILABLE)` is an answer the contract excludes (an AVAILABLE + // row becomes `Deleting`): a backend that gives it has broken the + // contract, which is an internal error, not an absent backup. + let fake = fake_with_row(BackupStatus::Available); + fake.delete_starts + .lock() + .unwrap() + .insert(arn(ACCOUNT), DeleteStart::Status(BackupStatus::Available)); + let err = delete_backup( + &*fake, + &runtime_with_store(), + ACCOUNT, + REGION, + &own(&arn(ACCOUNT)), + ) + .await + .unwrap_err(); + assert!( + matches!(err, DynamoDbError::InternalServerError(_)), + "{err:?}" + ); +} + +/// A row whose `source_table_details_json` this build cannot describe: the +/// listing leaves it out and lists the rest (not an internal error for the +/// whole account), and DeleteBackup answers the internal error before the +/// DELETING write, so the row stays AVAILABLE and a retry sees the same. +#[tokio::test] +async fn an_unreadable_source_table_is_left_out_of_the_listing_and_stops_a_delete_before_the_write() +{ + let readable = music_arn(ACCOUNT, &extenddb_storage::util::format_backup_id(1_000, 1)); + let unreadable = music_arn(ACCOUNT, &extenddb_storage::util::format_backup_id(2_000, 2)); + let fake = Fake::default(); + { + let mut rows = fake.rows.lock().unwrap(); + rows.insert(readable.clone(), row(&readable, BackupStatus::Available)); + let mut damaged = row(&unreadable, BackupStatus::Available); + damaged.source_table_details_json = json!({ "table_name": "Music" }); + rows.insert(unreadable.clone(), damaged); + } + let page = list(&fake, json!({})).await.unwrap(); + assert_eq!(arns(&page), std::slice::from_ref(&readable)); + + let err = delete_backup( + &fake, + &runtime_with_store(), + ACCOUNT, + REGION, + &own(&unreadable), + ) + .await + .unwrap_err(); + assert!( + matches!(err, DynamoDbError::InternalServerError(_)), + "{err:?}" + ); + assert!( + fake.delete_starts_seen.lock().unwrap().is_empty(), + "the description is built before the DELETING write" + ); + assert_eq!( + fake.rows.lock().unwrap()[&unreadable].status, + BackupStatus::Available + ); + // A row the summary can still read (its `table_id` and `table_arn` are + // there, the rest is not) lists; its description is the internal error. + fake.rows + .lock() + .unwrap() + .get_mut(&unreadable) + .unwrap() + .source_table_details_json = json!({ + "table_id": "tid-2", + "table_arn": source_arn(), + }); + let page = list(&fake, json!({})).await.unwrap(); + assert_eq!(arns(&page), [readable, unreadable.clone()]); + assert_eq!(page["BackupSummaries"][1]["TableId"], "tid-2"); + let err = describe_backup(&fake, ACCOUNT, REGION, &own(&unreadable)) + .await + .unwrap_err(); + assert!( + matches!(err, DynamoDbError::InternalServerError(_)), + "{err:?}" + ); +} + +/// A legacy backup of a table whose name the store-prefix rules refuse (a +/// name made only of dots, which main's CreateTable and CreateBackup accept; +/// a name over 255 bytes under a raised `limits.max_table_name_length`) has +/// no parsed ARN and no catalog-index read, and describes, restores and +/// deletes through the legacy reader as on main. +#[tokio::test] +async fn a_backup_of_a_name_the_store_prefix_refuses_resolves_through_the_legacy_reader() { + for table in ["...", &"x".repeat(256)] { + let id = "01791976431438-73d8d5bc"; + let legacy_arn = format!("arn:aws:dynamodb:{REGION}:{ACCOUNT}:table/{table}/backup/{id}"); + assert!( + BackupArn::parse(&legacy_arn).is_err(), + "the store-prefix rules refuse {table:?}" + ); + let spellings = own(&legacy_arn); + assert_eq!(spellings.canonical(), legacy_arn); + assert_eq!(spellings.arns().count(), 0, "no parsed spelling"); + + let mut src = source(Some(definition("PROVISIONED"))); + src.backup_arn = legacy_arn.clone(); + src.source_table_name = table.to_owned(); + let fake = Fake::with_source(src); + let mut description = backup_description("AVAILABLE", "PROVISIONED"); + description.backup_details.backup_arn = legacy_arn.clone(); + description.source_table_details.table_name = table.to_owned(); + fake.descriptions + .lock() + .unwrap() + .insert(legacy_arn.clone(), description); + + let desc = describe_backup(&*fake, ACCOUNT, REGION, &spellings) + .await + .unwrap(); + assert_eq!(desc.backup_details.backup_arn, legacy_arn); + assert_eq!(desc.source_table_details.table_name, table); + assert!(desc.source_table_feature_details.is_some()); + + let limits = LimitsConfig { + max_table_name_length: 1024, + ..LimitsConfig::default() + }; + let (_, response) = restore_table_from_backup( + &fake, + &runtime(), + ACCOUNT, + REGION, + &limits, + &json!({ "BackupArn": legacy_arn, "TargetTableName": "Restored" }), + ) + .await + .unwrap(); + assert_eq!(response["TableDescription"]["TableName"], "Restored"); + assert_eq!(fake.restores().len(), 1); + assert_eq!(fake.restores()[0].backup_arn, legacy_arn); + + let desc = delete_backup(&*fake, &runtime(), ACCOUNT, REGION, &spellings) + .await + .unwrap(); + assert_eq!(desc.backup_details.backup_status, "DELETED"); + assert_eq!( + fake.deleted.lock().unwrap().as_slice(), + std::slice::from_ref(&legacy_arn) + ); + assert!( + fake.row_reads.lock().unwrap().is_empty(), + "the catalog index was not read for {table:?}" + ); + } +} + +/// RestoreTableFromBackup over a catalog row, before the permit and reading +/// no store: CREATING is BackupInUse; DELETING, DELETED and FAILED are +/// BackupNotFound; a backup in use is BackupInUse naming the target, whether +/// or not the row is of the configured store (the in-use check runs before +/// the other-store check, as the delete's does); a row +/// of another store is a ValidationException, with no store configured and +/// with a different one configured; and an AVAILABLE row of the healthy +/// configured store passes to the permit, the next check, with no store +/// read. The runtime has no free slot, so a permit taken anywhere before a +/// row's refusal, even one dropped again, would turn the refusal into +/// LimitExceeded; the legacy reader is never reached. +#[tokio::test] +async fn restore_over_a_row_refuses_before_the_permit() { + async fn check(fake: Arc, runtime: &BackupRuntime) -> DynamoDbError { + assert_eq!(runtime.free_slots(), 0, "every slot is held"); + let err = restore_table_from_backup( + &fake, + runtime, + ACCOUNT, + REGION, + &LimitsConfig::default(), + &restore_body(json!({})), + ) + .await + .unwrap_err(); + assert!( + !matches!(err, DynamoDbError::LimitExceededException(_)), + "the row's refusal, not the permit's: {err:?}" + ); + assert_eq!(fake.source_reads(), 0, "the legacy reader was not reached"); + assert!(fake.restores().is_empty()); + err + } + let configured_mem = Arc::new(MemStore::default()); + let configured = BackupRuntime::new(1) + .with_store(ConfiguredStore::new( + store().store_kind, + store().store_root, + Arc::clone(&configured_mem) as Arc, + None, + Ok(store()), + )) + .with_permit_wait(Duration::ZERO); + let _held_configured = JobPermit::try_acquire(configured.slots()).unwrap(); + let none = BackupRuntime::new(1).with_permit_wait(Duration::ZERO); + let _held_none = JobPermit::try_acquire(none.slots()).unwrap(); + let err = check(fake_with_row(BackupStatus::Creating), &configured).await; + assert!( + matches!(err, DynamoDbError::BackupInUseException(_)), + "{err:?}" + ); + for status in [ + BackupStatus::Deleting, + BackupStatus::Deleted, + BackupStatus::Failed, + ] { + let err = check(fake_with_row(status), &configured).await; + assert!( + matches!(err, DynamoDbError::BackupNotFoundException(_)), + "{status}: {err:?}" + ); + } + let fake = fake_with_row(BackupStatus::Available); + fake.in_use + .lock() + .unwrap() + .insert(arn(ACCOUNT), "Earlier".to_owned()); + let err = check(fake, &configured).await; + assert!( + matches!(err, DynamoDbError::BackupInUseException(ref m) + if *m == format!("Backup is being used to restore another table: {}", arn(ACCOUNT))), + "the service's text, naming the backup and not the target: {err:?}" + ); + // The in-use check runs before the other-store check, as the delete's + // does: a row of another store that is in use is BackupInUse, not the + // other-store ValidationException. + let fake = fake_with_row(BackupStatus::Available); + fake.in_use + .lock() + .unwrap() + .insert(arn(ACCOUNT), "Earlier".to_owned()); + let err = check(fake, &none).await; + assert!( + matches!(err, DynamoDbError::BackupInUseException(ref m) if m.contains("another table")), + "the in-use check runs before the other-store check: {err:?}" + ); + // An AVAILABLE row of the healthy configured store passes the row checks: + // the permit is the next check, before any store read, so with every + // slot held the request is LimitExceeded and the store saw no get. + let fake = fake_with_row(BackupStatus::Available); + let err = restore_table_from_backup( + &fake, + &configured, + ACCOUNT, + REGION, + &LimitsConfig::default(), + &restore_body(json!({})), + ) + .await + .unwrap_err(); + assert!( + matches!(err, DynamoDbError::LimitExceededException(_)), + "the permit is the check after the row's: {err:?}" + ); + assert!( + configured_mem.gets().is_empty(), + "no store read before the permit" + ); + assert!(fake.restore_claims.lock().unwrap().is_empty()); + let msg = validation_message(check(fake_with_row(BackupStatus::Available), &none).await); + assert!(msg.contains("store other than"), "{msg}"); + // A configured store that is not the row's: the row is of another store, + // not of "the" store because one is configured. + let other = BackupRuntime::new(1) + .with_store(ConfiguredStore::new( + store().store_kind, + store().store_root, + Arc::new(MemStore::default()) as Arc, + None, + Ok(StoreIdentity { + store_id: "store-2".to_owned(), + ..store() + }), + )) + .with_permit_wait(Duration::ZERO); + let _held_other = JobPermit::try_acquire(other.slots()).unwrap(); + let msg = validation_message(check(fake_with_row(BackupStatus::Available), &other).await); + assert!( + msg.contains("store other than"), + "a configured store other than the row's: {msg}" + ); +} + +/// The job permit is the last check of a restore: a request refused earlier +/// takes no slot, a request that reaches the backend hands its slot to the +/// backend for the job's life, and with no slot free the request is +/// LimitExceeded after every other check passed, with the backend untouched. +#[tokio::test] +async fn restore_takes_the_permit_last() { + async fn restore_with( + fake: &Arc, + runtime: &BackupRuntime, + ) -> Result<(String, Value), DynamoDbError> { + let body = restore_body(json!({})); + restore_table_from_backup( + fake, + runtime, + ACCOUNT, + REGION, + &LimitsConfig::default(), + &body, + ) + .await + } + let runtime = BackupRuntime::new(1).with_permit_wait(Duration::from_millis(50)); + let fake = Fake::with_source(source(Some(definition("PROVISIONED")))); + restore_with(&fake, &runtime).await.unwrap(); + assert_eq!(runtime.free_slots(), 0, "the backend holds the slot"); + assert_eq!(fake.restores().len(), 1); + + let err = restore_with(&fake, &runtime).await.unwrap_err(); + assert!( + matches!( + err, + DynamoDbError::LimitExceededException(ref m) + if m.contains("(at most 1 at once, the [backup] max_concurrent_jobs bound)") + ), + "the refusal names the bound: {err:?}" + ); + assert_eq!(fake.restores().len(), 1, "the backend was not asked"); + assert_eq!( + fake.source_reads(), + 2, + "the source was read before the permit" + ); + + // A refusal earlier in the order takes no slot: a missing backup. + let runtime = BackupRuntime::new(1); + let empty = Arc::new(Fake::default()); + let err = restore_with(&empty, &runtime).await.unwrap_err(); + assert!( + matches!(err, DynamoDbError::BackupNotFoundException(_)), + "{err:?}" + ); + assert_eq!(runtime.free_slots(), 1); + + // The backend's refusal returns the slot with the dropped permit. + let fake = Fake::with_source(source(Some(definition("PROVISIONED")))); + *fake.restore_error.lock().unwrap() = + Some(StorageError::TableAlreadyExists("MusicRestored".to_owned())); + let err = restore_with(&fake, &runtime).await.unwrap_err(); + assert!( + matches!(err, DynamoDbError::TableAlreadyExistsException(_)), + "{err:?}" + ); + assert_eq!(runtime.free_slots(), 1); + + // Dropping the backend's permit frees the slot. + fake.permits.lock().unwrap().clear(); + assert_eq!(runtime.free_slots(), 1); +} + +/// The target name is checked after the backup resolved and before the +/// overrides and the permit, the order the service uses: with every permit +/// held and the backup restorable, a restore into a name a table has is +/// TableAlreadyExists, a retry into a CREATING restore target is TableInUse +/// naming the target's source backup in the backend's own text, a CREATING +/// table with no RestoreSummary (an ordinary create) and an ACTIVE former +/// target (its summary no longer in progress) are TableAlreadyExists; the +/// backup was read once and the backend not asked. With a missing backup, +/// the same three targets answer BackupNotFound (measured on DynamoDB, twice +/// each, and on main), the name never read. A free name passes. +#[tokio::test] +async fn restore_checks_the_target_name_after_the_backup_and_before_the_permit() { + fn table(status: TableStatus, source_backup_arn: Option<&str>) -> TableDescription { + TableDescription { + table_name: "MusicRestored".to_owned(), + table_status: status, + restore_summary: source_backup_arn.map(|arn| RestoreSummary { + source_backup_arn: Some(arn.to_owned()), + source_table_arn: Some(source_arn()), + restore_date_time: CREATED, + restore_in_progress: status == TableStatus::Creating, + }), + ..Default::default() + } + } + async fn check(fake: Arc, target: TableDescription) -> (Arc, DynamoDbError) { + let full = BackupRuntime::new(1).with_permit_wait(Duration::ZERO); + let _held = full.take_permit().await.unwrap(); + fake.tables + .lock() + .unwrap() + .insert("MusicRestored".to_owned(), target); + let err = restore_table_from_backup( + &fake, + &full, + ACCOUNT, + REGION, + &LimitsConfig::default(), + &restore_body(json!({})), + ) + .await + .unwrap_err(); + assert!( + !matches!(err, DynamoDbError::LimitExceededException(_)), + "the request's refusal, not the permit's: {err:?}" + ); + assert!(fake.restores().is_empty(), "the backend was not asked"); + (fake, err) + } + let restorable = || Fake::with_source(source(Some(definition("PROVISIONED")))); + let targets = [ + table(TableStatus::Active, None), + table(TableStatus::Creating, None), + table(TableStatus::Creating, Some(&arn(ACCOUNT))), + table(TableStatus::Active, Some(&arn(ACCOUNT))), + ]; + + // The backup restorable: the name's refusal, after one read of the backup. + let in_use_text = format!( + "Table: MusicRestored is already being restored from backup: {}", + arn(ACCOUNT) + ); + for (i, target) in targets.iter().enumerate() { + let (fake, err) = check(restorable(), target.clone()).await; + assert_eq!(fake.source_reads(), 1, "{i}: the backup was read first"); + assert_eq!( + fake.table_reads.lock().unwrap().as_slice(), + ["MusicRestored"], + "{i}: the name was read once, under the request's spelling" + ); + if i == 2 { + assert!( + matches!(err, DynamoDbError::TableInUseException(ref m) if *m == in_use_text), + "a CREATING restore target: {err:?}" + ); + } else { + assert!( + matches!(err, DynamoDbError::TableAlreadyExistsException(ref m) if m.contains("MusicRestored")), + "{i}: {err:?}" + ); + } + } + + // The backup missing: BackupNotFound whatever the target's state, and + // the name is never read. + for (i, target) in targets.iter().enumerate() { + let (fake, err) = check(Arc::new(Fake::default()), target.clone()).await; + assert!( + matches!(err, DynamoDbError::BackupNotFoundException(ref m) + if *m == format!("Backup not found: {}", arn(ACCOUNT))), + "{i}: a missing backup is BackupNotFound before the name: {err:?}" + ); + assert!( + fake.table_reads.lock().unwrap().is_empty(), + "{i}: the name was not read" + ); + } + + // A free name: the check passes and the restore reaches the backend. + let runtime = BackupRuntime::new(1); + let fake = restorable(); + restore_table_from_backup( + &fake, + &runtime, + ACCOUNT, + REGION, + &LimitsConfig::default(), + &restore_body(json!({})), + ) + .await + .unwrap(); + assert_eq!( + fake.table_reads.lock().unwrap().as_slice(), + ["MusicRestored"] + ); + assert_eq!(fake.restores().len(), 1); +} + +/// A request over `max_concurrent_jobs` waits for a slot up to the runtime's +/// bound (30 s in the server; shortened here) rather than refusing at once: a +/// slot freed within the bound is taken and the request proceeds, and a bound +/// that passes with every slot held is LimitExceeded naming the bound. The +/// waiting request holds nothing: the backend is asked only once it has the +/// permit. +#[tokio::test] +async fn a_request_over_the_job_bound_waits_a_bounded_time_for_a_slot() { + let runtime = Arc::new(BackupRuntime::new(1).with_permit_wait(Duration::from_secs(5))); + let held = runtime.take_permit().await.unwrap(); + let fake = Arc::new(Fake::with_source(source(Some(definition("PROVISIONED"))))); + let waiting = { + let runtime = Arc::clone(&runtime); + let fake = Arc::clone(&fake); + tokio::spawn(async move { + restore_table_from_backup( + &*fake, + &runtime, + ACCOUNT, + REGION, + &LimitsConfig::default(), + &restore_body(json!({})), + ) + .await + }) + }; + tokio::time::sleep(Duration::from_millis(50)).await; + assert!(!waiting.is_finished(), "the request waits for the slot"); + assert!( + fake.restores().is_empty(), + "the backend is not asked while it waits" + ); + drop(held); + waiting + .await + .unwrap() + .expect("the request takes the freed slot and proceeds"); + assert_eq!(fake.restores().len(), 1); + assert_eq!(runtime.free_slots(), 0, "the backend holds the slot"); + + let bounded = BackupRuntime::new(1).with_permit_wait(Duration::from_millis(50)); + let _held = bounded.take_permit().await.unwrap(); + let started = tokio::time::Instant::now(); + let err = restore_table_from_backup( + &*fake, + &bounded, + ACCOUNT, + REGION, + &LimitsConfig::default(), + &restore_body(json!({})), + ) + .await + .unwrap_err(); + assert!( + started.elapsed() >= Duration::from_millis(50), + "the request waited the bound" + ); + assert!( + matches!(err, DynamoDbError::LimitExceededException(ref m) + if m.contains("(at most 1 at once, the [backup] max_concurrent_jobs bound)")), + "{err:?}" + ); + assert_eq!(fake.restores().len(), 1, "the backend was not asked again"); +} + +/// CreateBackup's checks run in the service's order, every one before the +/// permit: a missing or not-ACTIVE table is TableNotFound whether or not a +/// slot is free; a table with vector indexes is refused; then the permit, +/// LimitExceeded when none is free; then the in-catalog create holds the slot. +#[tokio::test] +async fn create_backup_checks_the_table_then_takes_the_permit_last() { + let active = TableDescription { + table_name: "Music".to_owned(), + table_status: TableStatus::Active, + ..Default::default() + }; + let full = BackupRuntime::new(1).with_permit_wait(Duration::ZERO); + let _held = full.take_permit().await.unwrap(); + let fake = Arc::new(Fake::default()); + + let err = create_backup( + &fake, + &full, + ACCOUNT, + REGION, + &LimitsConfig::default(), + "Music", + "nightly", + Err(StorageError::TableNotFound("Music".to_owned())), + ) + .await + .unwrap_err(); + assert!( + matches!(err, DynamoDbError::TableNotFoundException(ref m) if m == "Table not found: Music"), + "{err:?}" + ); + let creating = TableDescription { + table_status: TableStatus::Creating, + ..active.clone() + }; + let err = create_backup( + &fake, + &full, + ACCOUNT, + REGION, + &LimitsConfig::default(), + "Music", + "nightly", + Ok(creating), + ) + .await + .unwrap_err(); + assert!( + matches!(err, DynamoDbError::TableNotFoundException(_)), + "{err:?}" + ); + + let with_vectors: TableDescription = serde_json::from_value(json!({ + "TableName": "Music", + "KeySchema": [], + "AttributeDefinitions": [], + "TableStatus": "ACTIVE", + "CreationDateTime": 0.0, + "TableSizeBytes": 0, + "ItemCount": 0, + "TableArn": "", + "TableId": "", + "ProvisionedThroughput": { "ReadCapacityUnits": 0, "WriteCapacityUnits": 0, + "NumberOfDecreasesToday": 0 }, + "DeletionProtectionEnabled": false, + "VectorIndexes": [{ + "IndexName": "v", + "VectorAttribute": { "AttributeName": "emb" }, + "Dimensions": 3, + "DistanceFunction": "COSINE", + "IndexStatus": "ACTIVE", + "IndexSizeBytes": 0, + "ItemCount": 0, + "IndexArn": "" + }] + })) + .unwrap(); + let msg = validation_message( + create_backup( + &fake, + &full, + ACCOUNT, + REGION, + &LimitsConfig::default(), + "Music", + "nightly", + Ok(with_vectors), + ) + .await + .unwrap_err(), + ); + assert!(msg.contains("vector"), "{msg}"); + + let err = create_backup( + &fake, + &full, + ACCOUNT, + REGION, + &LimitsConfig::default(), + "Music", + "nightly", + Ok(active.clone()), + ) + .await + .unwrap_err(); + assert!( + matches!(err, DynamoDbError::LimitExceededException(_)), + "{err:?}" + ); + assert!( + fake.creates.lock().unwrap().is_empty(), + "the backend was not asked" + ); + + let free = BackupRuntime::new(1); + create_backup( + &fake, + &free, + ACCOUNT, + REGION, + &LimitsConfig::default(), + "Music", + "nightly", + Ok(active), + ) + .await + .unwrap(); + assert_eq!( + fake.creates.lock().unwrap().as_slice(), + [("Music".to_owned(), "nightly".to_owned())] + ); + assert_eq!(free.free_slots(), 0, "the backend holds the slot"); + fake.permits.lock().unwrap().clear(); + assert_eq!(free.free_slots(), 1); +} + +/// A table name that fits CreateTable but not a backup ARN's store prefix (over +/// 255 bytes under a raised `limits.max_table_name_length`, or made only of +/// dots) is backed up by the in-catalog create as on main: the service backs +/// up any table it can create (measured on DynamoDB: a table named `...` +/// answers 200 with BackupStatus CREATING), and the in-catalog format writes +/// no store prefix. The store-prefix rules belong to the store-backed writer. +#[tokio::test] +async fn create_backup_accepts_a_name_the_store_prefix_refuses() { + let fake = Arc::new(Fake::default()); + for name in ["x".repeat(256), "x".repeat(1024), "...".to_owned()] { + let runtime = BackupRuntime::new(1); + let desc = TableDescription { + table_name: name.clone(), + table_status: TableStatus::Active, + ..Default::default() + }; + create_backup( + &fake, + &runtime, + ACCOUNT, + REGION, + &LimitsConfig::default(), + &name, + "nightly", + Ok(desc), + ) + .await + .unwrap_or_else(|e| panic!("{name:?}: {e:?}")); + assert_eq!(runtime.free_slots(), 0, "the backend holds the slot"); + } + let created: Vec = fake + .creates + .lock() + .unwrap() + .iter() + .map(|(table, _)| table.clone()) + .collect(); + assert_eq!( + created, + ["x".repeat(256), "x".repeat(1024), "...".to_owned()] + ); +} + +/// The store-backed create applies the store-prefix rules the in-catalog one +/// does not (`check_store_prefix_fits`): a name over 255 bytes or made only +/// of dots fits no prefix, so it is refused before a snapshot is opened or a +/// row is written, naming the rule (the length before the ARN is formed, so +/// the limit is named whatever the length). +#[tokio::test] +async fn a_store_backed_create_refuses_a_name_no_store_prefix_can_carry() { + let fake = Arc::new(Fake::default()); + let mem = Arc::new(MemStore::default()); + let runtime = job_runtime(&mem); + for (name, needle) in [ + ("x".repeat(256), "256 bytes"), + ("x".repeat(1024), "1024 bytes"), + ("...".to_owned(), "only dots"), + ] { + let desc = TableDescription { + table_name: name.clone(), + table_status: TableStatus::Active, + ..Default::default() + }; + let msg = validation_message( + create_backup( + &fake, + &runtime, + ACCOUNT, + REGION, + &LimitsConfig::default(), + &name, + "nightly", + Ok(desc), + ) + .await + .unwrap_err(), + ); + assert!( + msg.contains("cannot be backed up") && msg.contains(needle), + "{name:?}: {msg}" + ); + assert!(!msg.contains("not a backup ARN"), "{msg}"); + } + assert!(fake.rows.lock().unwrap().is_empty(), "no row was claimed"); + assert_eq!( + mem.keys(), + [STORE_IDENTITY_KEY], + "nothing was written to the store but the identity file" + ); + assert_eq!(runtime.free_slots(), 4, "no slot was taken"); +} + +/// Under a `server.region` outside `[a-z0-9-]` no backup ARN can be formed +/// for any table, so CreateBackup is refused naming `server.region`, not the +/// table name, before the permit (the runtime's one permit is held). The +/// other operations are not affected for a backup created under a valid +/// region: its stored ARN is reached as written, the canonical spelling in +/// the server's region having no parsed form. A backup an earlier release +/// created under such a region keeps an ARN `check_backup_arn` refuses as +/// not valid, before any lookup. +#[tokio::test] +async fn create_backup_under_a_region_outside_the_arn_alphabet_names_the_setting() { + let runtime = BackupRuntime::new(1).with_permit_wait(Duration::ZERO); + let _held = runtime.take_permit().await.unwrap(); + let fake = Arc::new(Fake::default()); + let desc = TableDescription { + table_name: "Music".to_owned(), + table_status: TableStatus::Active, + ..Default::default() + }; + let msg = validation_message( + create_backup( + &fake, + &runtime, + ACCOUNT, + "LOCAL", + &LimitsConfig::default(), + "Music", + "nightly", + Ok(desc), + ) + .await + .unwrap_err(), + ); + assert!( + msg.contains("server.region") && msg.contains("\"LOCAL\"") && msg.contains("[a-z0-9-]"), + "{msg}" + ); + assert!( + !msg.contains("TableName"), + "the table is not the cause: {msg}" + ); + assert_eq!(runtime.free_slots(), 0, "the test still holds the permit"); + assert!(fake.creates.lock().unwrap().is_empty()); + + // A backup created under a valid region, described under "LOCAL": the + // row is read under the ARN as written, and only that spelling. + let stored = arn(ACCOUNT); + fake.rows + .lock() + .unwrap() + .insert(stored.clone(), row(&stored, BackupStatus::Available)); + let spellings = backup_arn_field(&json!({ "BackupArn": stored }), ACCOUNT, "LOCAL").unwrap(); + let desc = describe_backup(&*fake, ACCOUNT, "LOCAL", &spellings) + .await + .unwrap(); + assert_eq!(desc.backup_details.backup_arn, stored); + assert_eq!( + fake.row_reads.lock().unwrap().as_slice(), + std::slice::from_ref(&stored), + "as written only: the canonical spelling under \"LOCAL\" has no parsed form" + ); + + // A backup created under "LOCAL" by an earlier release: its ARN is + // refused before any lookup. + let local = + format!("arn:aws:dynamodb:LOCAL:{ACCOUNT}:table/Music/backup/01791976431438-73d8d5bc"); + let msg = validation_message( + backup_arn_field(&json!({ "BackupArn": local }), ACCOUNT, "LOCAL").unwrap_err(), + ); + assert_eq!(msg, BACKUP_ARN_NOT_VALID); + assert_eq!(fake.row_reads.lock().unwrap().len(), 1, "no further read"); +} + +/// While a maintenance command holds the catalog's claim, CreateBackup and +/// RestoreTableFromBackup are refused with LimitExceeded naming a maintenance command +/// (generically: every maintenance command holds the same claim), +/// after every other check (a missing table is still TableNotFound) and +/// before the permit (no slot is taken, and the backend is not asked); once +/// the claim is gone both run. +#[tokio::test] +async fn jobs_are_refused_while_a_maintenance_claim_is_held() { + let fake = Fake::with_source(source(Some(definition("PROVISIONED")))); + *fake.maintenance_held.lock().unwrap() = true; + let runtime = BackupRuntime::new(1).with_stale_job_after(Duration::from_secs(7)); + let err = create_backup( + &fake, + &runtime, + ACCOUNT, + REGION, + &LimitsConfig::default(), + "Music", + "nightly", + Ok(active_table("Music")), + ) + .await + .unwrap_err(); + assert!( + matches!(&err, DynamoDbError::LimitExceededException(m) + if m.contains("A backup maintenance command holds") && !m.contains("purge")), + "{err:?}" + ); + assert!(fake.creates.lock().unwrap().is_empty()); + assert_eq!(runtime.free_slots(), 1, "no slot was taken"); + let err = create_backup( + &fake, + &runtime, + ACCOUNT, + REGION, + &LimitsConfig::default(), + "Music", + "nightly", + Err(StorageError::TableNotFound("Music".to_owned())), + ) + .await + .unwrap_err(); + assert!( + matches!(err, DynamoDbError::TableNotFoundException(_)), + "the table check comes first: {err:?}" + ); + + let err = restore_on(&fake, &runtime, &restore_body(json!({}))) + .await + .unwrap_err(); + assert!( + matches!(&err, DynamoDbError::LimitExceededException(m) + if m.contains("A backup maintenance command holds") && !m.contains("purge")), + "{err:?}" + ); + assert!(fake.restores().is_empty(), "the backend was not asked"); + assert_eq!(runtime.free_slots(), 1); + assert_eq!( + *fake.maintenance_checks.lock().unwrap(), + vec![Duration::from_secs(7); 2], + "the claim is read with stale_job_after_secs" + ); + + *fake.maintenance_held.lock().unwrap() = false; + create_backup( + &fake, + &runtime, + ACCOUNT, + REGION, + &LimitsConfig::default(), + "Music", + "nightly", + Ok(active_table("Music")), + ) + .await + .unwrap(); + fake.permits.lock().unwrap().clear(); + restore_on(&fake, &runtime, &restore_body(json!({}))) + .await + .unwrap(); + assert_eq!(fake.creates.lock().unwrap().len(), 1); + assert_eq!(fake.restores().len(), 1); + + // Before the permit: with every slot held and the claim held, the + // refusal names the maintenance claim, not the slot limit, so the check cannot + // have run after `take_permit`. + *fake.maintenance_held.lock().unwrap() = true; + let full = BackupRuntime::new(1).with_stale_job_after(Duration::from_secs(7)); + let held_slot = full.take_permit().await.unwrap(); + assert_eq!(full.free_slots(), 0, "every slot is held"); + let err = create_backup( + &fake, + &full, + ACCOUNT, + REGION, + &LimitsConfig::default(), + "Music", + "nightly", + Ok(active_table("Music")), + ) + .await + .unwrap_err(); + assert!( + matches!(&err, DynamoDbError::LimitExceededException(m) + if m.contains("A backup maintenance command holds") && !m.contains("purge")), + "the claim is checked before the permit: {err:?}" + ); + let err = restore_on(&fake, &full, &restore_body(json!({}))) + .await + .unwrap_err(); + assert!( + matches!(&err, DynamoDbError::LimitExceededException(m) + if m.contains("A backup maintenance command holds") && !m.contains("purge")), + "the claim is checked before the permit: {err:?}" + ); + drop(held_slot); +} + +// --- purge: the wait and the row deletion (`maintenance`) ---------------------- + +/// A maintenance claim handle for `HeldClaim`: heartbeats confirm until +/// `lost` is set. +#[derive(Default)] +struct FakeHolder { + lost: Mutex, + released: Mutex, +} + +impl extenddb_storage::MaintenanceClaim for FakeHolder { + fn generation(&self) -> u64 { + 11 + } + fn heartbeat( + &self, + ) -> BoxedFuture<'_, Result> { + let lost = *self.lost.lock().unwrap(); + Box::pin(async move { + Ok(if lost { + extenddb_storage::backup_catalog::Heartbeat::Lost + } else { + extenddb_storage::backup_catalog::Heartbeat::Confirmed + }) + }) + } + fn release(&self) -> BoxedFuture<'_, Result<(), StorageError>> { + *self.released.lock().unwrap() = true; + Box::pin(async { Ok(()) }) + } +} + +/// A forwarding handle, so the test keeps its own reference to the holder. +struct Forward(Arc); + +impl extenddb_storage::MaintenanceClaim for Forward { + fn generation(&self) -> u64 { + self.0.generation() + } + fn heartbeat( + &self, + ) -> BoxedFuture<'_, Result> { + self.0.heartbeat() + } + fn release(&self) -> BoxedFuture<'_, Result<(), StorageError>> { + self.0.release() + } +} + +fn held(holder: &Arc) -> maintenance::HeldClaim { + maintenance::HeldClaim::start( + Box::new(Forward(Arc::clone(holder))), + Duration::from_secs(1), + ) +} + +/// The claim object the `Fake`'s `maintenance_claim` hands out: `Forward` +/// whose release also frees the `Fake`'s record, so a second command may +/// take the claim once the first released it. +struct ReleasingForward(Arc>>, Arc); + +impl extenddb_storage::MaintenanceClaim for ReleasingForward { + fn generation(&self) -> u64 { + self.1.generation() + } + fn heartbeat( + &self, + ) -> BoxedFuture<'_, Result> { + self.1.heartbeat() + } + fn release(&self) -> BoxedFuture<'_, Result<(), StorageError>> { + *self.0.lock().unwrap() = None; + self.1.release() + } +} + +fn seconds_since_epoch() -> f64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_secs_f64() +} + +/// A row of `store()` under ARN `arn` in `status`, with a manifest and a +/// data file under its prefix in `mem`. +fn seed(fake: &Fake, mem: &MemStore, arn: &str, status: BackupStatus) { + let prefix = BackupArn::parse(arn).unwrap().store_prefix(); + mem.insert(&format!("{prefix}extenddb-manifest.json"), b"{}"); + mem.insert(&format!("{prefix}data/00001.json.gz"), b"x"); + fake.rows + .lock() + .unwrap() + .insert(arn.to_owned(), row(arn, status)); +} + +fn arn_n(n: u32) -> String { + format!("arn:aws:dynamodb:{REGION}:{ACCOUNT}:table/Music/backup/0179197643143{n}-0000000{n}") +} + +/// The states a row can be in when purge reaches it after reading it +/// AVAILABLE, a DeleteBackup having run meanwhile: still AVAILABLE; moved to +/// DELETING (the other deleter mid-way); moved to DELETED (the other +/// finished); gone (finished and its tombstone removed). Each ends with no +/// row and no object, counted as deleted; a status that cannot follow +/// AVAILABLE (FAILED) is the one case left for a look. A tombstone read +/// and removed meanwhile (a second purge) counts as deleted too. +#[tokio::test] +async fn purge_finishes_a_row_another_deleter_moved_or_removed_meanwhile() { + let fake = Fake::default(); + let mem = MemStore::default(); + let arns: Vec = (1..=6).map(arn_n).collect(); + for arn in &arns[..5] { + seed(&fake, &mem, arn, BackupStatus::Available); + } + // A DELETED tombstone the command read, whose prefix is already empty. + fake.rows + .lock() + .unwrap() + .insert(arns[5].clone(), row(&arns[5], BackupStatus::Deleted)); + // The command's read under the claim. + let read: Vec = fake + .list_backup_rows(None, &store()) + .try_collect() + .await + .unwrap(); + assert_eq!(read.len(), 6); + // Between the read and the rows' turns, DeleteBackup moves rows 2 and 3 + // and finishes 4 (its tombstone removed too); a writer moves 5 to + // FAILED, which no DeleteBackup does; the tombstone 6 is removed by a + // second purge. + { + let mut rows = fake.rows.lock().unwrap(); + rows.get_mut(&arns[1]).unwrap().status = BackupStatus::Deleting; + rows.get_mut(&arns[2]).unwrap().status = BackupStatus::Deleted; + rows.remove(&arns[3]); + rows.get_mut(&arns[4]).unwrap().status = BackupStatus::Failed; + rows.remove(&arns[5]); + } + // The deleter that finished rows 3 and 4 deleted their objects. + for arn in [&arns[2], &arns[3]] { + let prefix = BackupArn::parse(arn).unwrap().store_prefix(); + extenddb_backup_store::BackupStore::delete_prefix(&mem, &prefix) + .await + .unwrap(); + } + + let mut outcomes = Vec::new(); + for row in &read { + outcomes.push( + maintenance::purge_row(&fake, &mem, &passing_fence(), row, Duration::from_secs(300)) + .await, + ); + } + assert_eq!( + outcomes[0], + Ok(()), + "still AVAILABLE: moved and deleted here" + ); + assert_eq!( + outcomes[1], + Ok(()), + "moved to DELETING by another deleter: finished here" + ); + assert_eq!( + outcomes[2], + Ok(()), + "moved to DELETED by another deleter: the tombstone removed here, counted as deleted" + ); + assert_eq!(outcomes[3], Ok(()), "gone: counted as deleted"); + assert_eq!( + outcomes[4], + Err("its status changed to FAILED".to_owned()), + "a status that cannot follow AVAILABLE is left" + ); + assert_eq!( + outcomes[5], + Ok(()), + "a tombstone removed meanwhile: counted as deleted" + ); + let left: Vec = fake.rows.lock().unwrap().keys().cloned().collect(); + assert_eq!(left, vec![arns[4].clone()], "no row of the five remains"); + let failed_prefix = BackupArn::parse(&arns[4]).unwrap().store_prefix(); + assert_eq!( + mem.keys(), + vec![ + format!("{failed_prefix}data/00001.json.gz"), + format!("{failed_prefix}extenddb-manifest.json"), + ], + "only the left row's objects remain" + ); + // The row moved to DELETED was never moved by purge (DELETING to + // DELETED is the other deleter's write); the one left DELETING and the + // one still AVAILABLE were finished by purge; the three tombstones were + // removed here. + assert_eq!( + *fake.set_deleted.lock().unwrap(), + vec![arns[0].clone(), arns[1].clone()] + ); + assert_eq!( + *fake.removed_rows.lock().unwrap(), + vec![arns[0].clone(), arns[1].clone(), arns[2].clone()] + ); +} + +/// A CREATING row with a dead owner goes through the dead-owner removal and +/// then its prefix; one whose heartbeat is young is left with its objects +/// under the live window, and removed under the zero window SQLite uses. +#[tokio::test] +async fn purge_removes_a_dead_creating_row_and_leaves_a_live_one() { + let fake = Fake::default(); + let mem = MemStore::default(); + let dead = arn_n(1); + let live = arn_n(2); + seed(&fake, &mem, &dead, BackupStatus::Creating); + seed(&fake, &mem, &live, BackupStatus::Creating); + fake.rows + .lock() + .unwrap() + .get_mut(&live) + .unwrap() + .heartbeat_at = seconds_since_epoch(); + let window = Duration::from_secs(300); + let dead_row = fake.rows.lock().unwrap()[&dead].clone(); + let live_row = fake.rows.lock().unwrap()[&live].clone(); + assert_eq!( + maintenance::purge_row(&fake, &mem, &passing_fence(), &dead_row, window).await, + Ok(()) + ); + let err = maintenance::purge_row(&fake, &mem, &passing_fence(), &live_row, window) + .await + .unwrap_err(); + assert!( + err.contains("the CREATING row could not be removed"), + "{err}" + ); + let left: Vec = fake.rows.lock().unwrap().keys().cloned().collect(); + assert_eq!(left, vec![live.clone()]); + assert_eq!(mem.keys().len(), 2, "the live row's objects stay"); + // A heartbeat written a moment ago is older than a zero window once a + // moment has passed. + tokio::time::sleep(Duration::from_millis(20)).await; + assert_eq!( + maintenance::purge_row(&fake, &mem, &passing_fence(), &live_row, Duration::ZERO).await, + Ok(()) + ); + assert!(fake.rows.lock().unwrap().is_empty()); + assert!(mem.keys().is_empty()); +} + +/// The wait's five outcomes are told apart: quiet, also with a FAILED +/// restore record whose target is still CREATING (the reconciler's duty, +/// not a running job) and a FAILED backup row whose owner is dead; busy at +/// the limit, with the jobs named (a CREATING row and a FAILED row with a +/// live heartbeat, a RUNNING restore) and announced once; the ages read on +/// the catalog's clock, not this host's (a skewed catalog clock decides +/// busy and quiet against what this host's clock would say); a signal, +/// during a sleep and before the first read; the catalog unreadable; the +/// claim lost. +#[tokio::test(start_paused = true)] +async fn the_wait_reports_each_outcome_as_itself() { + use extenddb_storage::backup_catalog::{JobStatus, RestoreKind}; + use maintenance::{Interrupt, Wait, WaitOutcome, wait_for_quiet}; + let fake = Fake::default(); + let holder = Arc::new(FakeHolder::default()); + let claim = held(&holder); + let limit = Duration::from_secs(20); + let wait = Wait { + owner_window: Duration::from_secs(10), + limit, + poll: Duration::from_secs(2), + }; + let announced: Mutex>> = Mutex::new(Vec::new()); + let announce = |jobs: &[String]| announced.lock().unwrap().push(jobs.to_vec()); + + let (_tx, mut quiet) = Interrupt::new(); + assert_eq!( + wait_for_quiet(&fake, &store(), &claim, wait, &mut quiet, announce).await, + WaitOutcome::Quiet + ); + // A FAILED record whose target is still CREATING is listed by the + // zero-window dead-job listing with the RUNNING ones; it is the + // reconciler's to finish and no job runs under it. A FAILED backup row + // whose heartbeat is older than the window has a dead owner. + let mut failed_record = dead_job(RestoreKind::Legacy, "FailedTarget"); + failed_record.status = JobStatus::Failed; + fake.dead_jobs.lock().unwrap().push(failed_record); + fake.rows + .lock() + .unwrap() + .insert(arn_n(3), row(&arn_n(3), BackupStatus::Failed)); + let started = tokio::time::Instant::now(); + assert_eq!( + wait_for_quiet(&fake, &store(), &claim, wait, &mut quiet, announce).await, + WaitOutcome::Quiet, + "a FAILED restore record and a FAILED row with a dead owner are not waited on" + ); + assert_eq!(started.elapsed(), Duration::ZERO, "quiet at the first read"); + assert!(announced.lock().unwrap().is_empty()); + + // Busy: a CREATING row and a FAILED row whose heartbeats are young, and + // a RUNNING restore. + for (n, status) in [(1, BackupStatus::Creating), (2, BackupStatus::Failed)] { + fake.rows.lock().unwrap().insert(arn_n(n), { + let mut r = row(&arn_n(n), status); + r.heartbeat_at = seconds_since_epoch(); + r + }); + } + fake.dead_jobs + .lock() + .unwrap() + .push(dead_job(RestoreKind::Legacy, "Target")); + let started = tokio::time::Instant::now(); + let busy = wait_for_quiet(&fake, &store(), &claim, wait, &mut quiet, announce).await; + let expected = vec![ + format!( + "CreateBackup {} has a live owner (heartbeat within stale_job_after_secs)", + arn_n(1) + ), + format!( + "the FAILED backup {} has a live owner (heartbeat within stale_job_after_secs)", + arn_n(2) + ), + format!("a restore into Target (account {ACCOUNT}) is RUNNING"), + ]; + assert_eq!(busy, WaitOutcome::Busy(expected.clone())); + assert!(started.elapsed() >= limit, "{:?}", started.elapsed()); + assert_eq!(*announced.lock().unwrap(), vec![expected]); + + // A signal during a sleep ends the wait before the limit. + let (tx, mut interrupted) = Interrupt::new(); + tokio::spawn(async move { + tokio::time::sleep(Duration::from_secs(3)).await; + let _ = tx.send(Some("SIGTERM")); + }); + let started = tokio::time::Instant::now(); + assert_eq!( + wait_for_quiet(&fake, &store(), &claim, wait, &mut interrupted, announce).await, + WaitOutcome::Interrupted("SIGTERM") + ); + assert!( + started.elapsed() < Duration::from_millis(3_500), + "the signal ends the sleep, not the next poll: {:?}", + started.elapsed() + ); + // One already received ends it before the first read. + assert_eq!(interrupted.signalled(), Some("SIGTERM")); + assert_eq!( + wait_for_quiet(&fake, &store(), &claim, wait, &mut interrupted, announce).await, + WaitOutcome::Interrupted("SIGTERM") + ); + + // The catalog unreadable. + *fake.list_error.lock().unwrap() = Some("pool exhausted".to_owned()); + assert_eq!( + wait_for_quiet(&fake, &store(), &claim, wait, &mut quiet, announce).await, + WaitOutcome::CatalogError( + "cannot list the catalog's backup rows: Internal error: pool exhausted".to_owned() + ) + ); + *fake.list_error.lock().unwrap() = None; + + // The claim lost: the next heartbeat records it, and the wait stops. + *holder.lost.lock().unwrap() = true; + tokio::time::sleep(Duration::from_millis(1_500)).await; + assert_eq!( + wait_for_quiet(&fake, &store(), &claim, wait, &mut quiet, announce).await, + WaitOutcome::Lost( + "the maintenance claim was lost (another holder took it over)".to_owned() + ) + ); + claim.release().await.unwrap(); + assert!(*holder.released.lock().unwrap()); +} + +// --- the reconciler ------------------------------------------------------------- + +fn dead_job(kind: extenddb_storage::backup_catalog::RestoreKind, name: &str) -> RestoreJobRow { + RestoreJobRow::reconciler_row( + RestoreTargetKey { + account_id: ACCOUNT.to_owned(), + table_name: name.to_owned(), + table_id: format!("tid-{name}"), + }, + arn(ACCOUNT), + kind, + extenddb_storage::backup_catalog::JobStatus::Running, + 0, + 0.0, + ) +} + +/// A legacy restore whose owner is dead is marked FAILED and its target +/// removed; a manifest restore whose store is not the configured one is +/// left alone when this instance has no store configured (it is the job of +/// the instance that holds the record's store, not this one's to fail); +/// an abandoned in-catalog CREATING backup is removed through the reader's +/// delete. +#[tokio::test] +async fn the_reconciler_fails_dead_legacy_restores_and_removes_abandoned_backups() { + use extenddb_storage::backup_catalog::RestoreKind; + let fake = Arc::new(Fake::default()); + *fake.target_still_creating.lock().unwrap() = true; + *fake.record_still_running.lock().unwrap() = true; + fake.dead_jobs.lock().unwrap().extend([ + dead_job(RestoreKind::Legacy, "LegacyTarget"), + dead_job(RestoreKind::Manifest, "ManifestTarget"), + ]); + fake.abandoned_backups + .lock() + .unwrap() + .push(AbandonedLegacyBackup { + account_id: ACCOUNT.to_owned(), + backup_arn: arn(ACCOUNT), + removed: false, + }); + fake.descriptions + .lock() + .unwrap() + .insert(arn(ACCOUNT), backup_description("CREATING", "PROVISIONED")); + + let report = reconciler::cycle( + &fake, + &runtime(), + Duration::from_secs(300), + "test", + &mut reconciler::ReconcilerState::default(), + ) + .await + .unwrap(); + assert!(!report.maintenance_held); + assert_eq!(report.legacy_restores_removed, ["LegacyTarget"]); + assert!(report.failed_targets_removed.is_empty()); + assert!(report.manifest_restores_resumed.is_empty()); + assert!( + report.manifest_restores_failed.is_empty(), + "an instance with no configured store leaves a store-backed record alone: {:?}", + report.manifest_restores_failed + ); + assert!( + fake.failed_jobs.lock().unwrap().is_empty() + || fake + .failed_jobs + .lock() + .unwrap() + .iter() + .all(|(_, reason)| reason != OTHER_STORE), + "no OTHER_STORE write when this instance has no store: {:?}", + fake.failed_jobs.lock().unwrap() + ); + assert_eq!(report.legacy_backups_removed, [arn(ACCOUNT)]); + + let failed = fake.failed_jobs.lock().unwrap().clone(); + assert_eq!( + failed.len(), + 1, + "only the legacy restore is failed: {failed:?}" + ); + assert_eq!(failed[0].0.table_name, "LegacyTarget"); + assert_eq!(failed[0].1, reconciler::OWNER_DEAD); + assert_eq!( + fake.removed_targets.lock().unwrap().as_slice(), + [dead_job(RestoreKind::Legacy, "LegacyTarget").target] + ); + assert_eq!(fake.deleted.lock().unwrap().as_slice(), [arn(ACCOUNT)]); +} + +/// A record that is no longer RUNNING when the reconciler reaches it belongs +/// to whoever moved it: the target is not removed. +#[tokio::test] +async fn the_reconciler_leaves_a_target_whose_record_moved() { + use extenddb_storage::backup_catalog::RestoreKind; + let fake = Arc::new(Fake::default()); + *fake.record_still_running.lock().unwrap() = false; + *fake.target_still_creating.lock().unwrap() = true; + fake.dead_jobs + .lock() + .unwrap() + .push(dead_job(RestoreKind::Legacy, "LegacyTarget")); + let report = reconciler::cycle( + &fake, + &runtime(), + Duration::from_secs(300), + "test", + &mut reconciler::ReconcilerState::default(), + ) + .await + .unwrap(); + assert!(report.legacy_restores_removed.is_empty()); + assert!(report.targets_left_to_owner.is_empty()); + assert_eq!(fake.failed_jobs.lock().unwrap().len(), 1); + assert!(fake.removed_targets.lock().unwrap().is_empty()); +} + +/// A removal that finds the target no longer a CREATING restore target +/// (`remove_restore_target` answered `false`: the target flipped ACTIVE, was +/// deleted, or was re-created under another id) leaves it to its current +/// owner: the target is not counted as removed, the cycle reports it as left, +/// and the record is not written again, whether this cycle wrote the FAILED +/// (a dead legacy restore) or found it (the FAILED-target duty). +#[tokio::test] +async fn the_reconciler_leaves_a_target_that_is_no_longer_creating_after_the_failed_write() { + use extenddb_storage::backup_catalog::{JobStatus, RestoreKind}; + let fake = Arc::new(Fake::default()); + *fake.record_still_running.lock().unwrap() = true; + *fake.target_still_creating.lock().unwrap() = false; + let dead = dead_job(RestoreKind::Legacy, "LegacyTarget"); + let mut failed = dead_job(RestoreKind::Legacy, "FailedTarget"); + failed.status = JobStatus::Failed; + fake.dead_jobs + .lock() + .unwrap() + .extend([dead.clone(), failed.clone()]); + let report = reconciler::cycle( + &fake, + &runtime(), + Duration::from_secs(300), + "test", + &mut reconciler::ReconcilerState::default(), + ) + .await + .unwrap(); + assert!( + report.legacy_restores_removed.is_empty() && report.failed_targets_removed.is_empty(), + "a target the removal found moved is not counted as removed: {report:?}" + ); + assert_eq!( + report.targets_left_to_owner, + ["LegacyTarget", "FailedTarget"] + ); + assert_eq!( + fake.removed_targets.lock().unwrap().as_slice(), + [dead.target.clone(), failed.target] + ); + assert_eq!( + fake.failed_jobs.lock().unwrap().as_slice(), + [(dead.target, reconciler::OWNER_DEAD.to_owned())], + "the dead record is written FAILED once; the FAILED record is not written again" + ); +} + +/// The maintenance claim is checked first and reported; the duties that +/// resolve dead owners and the DELETING rows of the configured store still +/// run while it is held (the command waits on the first and finishes the +/// second itself, so neither races it). +#[tokio::test] +async fn the_reconciler_reports_a_held_maintenance_claim_and_still_resolves_dead_owners() { + use extenddb_storage::backup_catalog::RestoreKind; + let mem = Arc::new(MemStore::default()); + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + mem.insert(&manifest_key(&arn(ACCOUNT)), b"{}"); + let fake = fake_with_rows(&[(arn(ACCOUNT), BackupStatus::Deleting)]); + *fake.maintenance_held.lock().unwrap() = true; + *fake.target_still_creating.lock().unwrap() = true; + *fake.record_still_running.lock().unwrap() = true; + fake.dead_jobs + .lock() + .unwrap() + .push(dead_job(RestoreKind::Legacy, "LegacyTarget")); + let report = reconciler::cycle( + &fake, + &runtime_on(&mem, store()), + Duration::from_secs(300), + "test", + &mut reconciler::ReconcilerState::default(), + ) + .await + .unwrap(); + assert!(report.maintenance_held); + assert_eq!(report.legacy_restores_removed, ["LegacyTarget"]); + assert_eq!( + report.deletions_finished, + [arn(ACCOUNT)], + "a DELETING row is finished while the claim is held" + ); + assert_eq!( + fake.rows.lock().unwrap()[&arn(ACCOUNT)].status, + BackupStatus::Deleted + ); + assert_eq!(mem.keys(), [STORE_IDENTITY_KEY]); +} + +/// The wait tells a live owner from a dead one on the catalog's clock, the +/// clock the owner's heartbeat was written on, and never on this host's: +/// with the catalog's clock 400 s behind this host's, a CREATING row whose +/// heartbeat is the catalog's now has a live owner although this host's +/// clock reads it 400 s old, past the 300 s window; with the catalog's clock +/// 400 s ahead, a row stamped with this host's now is 400 s old to the +/// catalog and its owner dead. +#[tokio::test(start_paused = true)] +async fn the_wait_reads_a_live_owner_on_the_catalogs_clock() { + use maintenance::{Interrupt, Wait, WaitOutcome, wait_for_quiet}; + let fake = Fake::default(); + let holder = Arc::new(FakeHolder::default()); + let claim = held(&holder); + let wait = Wait { + owner_window: Duration::from_secs(300), + limit: Duration::from_secs(4), + poll: Duration::from_secs(2), + }; + let (_tx, mut quiet) = Interrupt::new(); + + *fake.catalog_clock_offset_secs.lock().unwrap() = -400.0; + fake.rows.lock().unwrap().insert(arn_n(1), { + let mut r = row(&arn_n(1), BackupStatus::Creating); + r.heartbeat_at = seconds_since_epoch() - 400.0; + r + }); + assert_eq!( + wait_for_quiet(&fake, &store(), &claim, wait, &mut quiet, |_| {}).await, + WaitOutcome::Busy(vec![format!( + "CreateBackup {} has a live owner (heartbeat within stale_job_after_secs)", + arn_n(1) + )]), + "an owner whose heartbeat is young on the catalog's clock is live" + ); + + *fake.catalog_clock_offset_secs.lock().unwrap() = 400.0; + fake.rows + .lock() + .unwrap() + .get_mut(&arn_n(1)) + .unwrap() + .heartbeat_at = seconds_since_epoch(); + assert_eq!( + wait_for_quiet(&fake, &store(), &claim, wait, &mut quiet, |_| {}).await, + WaitOutcome::Quiet, + "an owner whose heartbeat is old on the catalog's clock is dead" + ); +} + +/// A cycle run by the holder of the maintenance claim with a zero dead-owner +/// window (`backup purge` on SQLite) takes the held branch without reading +/// the claim: the command's claim was heartbeated a millisecond ago, which +/// a zero window reads as stale, so a cycle that read it would take the +/// branch for a free claim. The held branch skips the deleting duties the +/// command must not race: a stale FAILED row keeps its prefix and the +/// registrations sweep does not run. The same cycle without the holder's +/// state reads the claim through the window, finds it free and runs them; +/// that is the read the holder's state replaces. +#[tokio::test] +async fn a_cycle_run_by_the_claim_holder_takes_the_held_branch_whatever_its_window() { + use extenddb_storage::backup_catalog::RestoreKind; + let mem = Arc::new(MemStore::default()); + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + mem.insert(&manifest_key(&arn(ACCOUNT)), b"{}"); + let fake = fake_with_rows(&[(arn(ACCOUNT), BackupStatus::Deleting)]); + let stale_failed = arn_n(1); + let mut r = row(&stale_failed, BackupStatus::Failed); + r.heartbeat_at = seconds_since_epoch() - 1000.0; + r.completed_at = Some(seconds_since_epoch()); + fake.rows.lock().unwrap().insert(stale_failed.clone(), r); + let prefix = BackupArn::parse(&stale_failed).unwrap().store_prefix(); + mem.insert(&format!("{prefix}data/000001.json.gz"), b"x"); + fake.swept_registrations.lock().unwrap().push(arn_n(9)); + *fake.maintenance_beat_age.lock().unwrap() = Some(Duration::from_millis(1)); + *fake.target_still_creating.lock().unwrap() = true; + *fake.record_still_running.lock().unwrap() = true; + fake.dead_jobs + .lock() + .unwrap() + .push(dead_job(RestoreKind::Legacy, "LegacyTarget")); + let report = reconciler::cycle( + &fake, + &runtime_on(&mem, store()), + Duration::ZERO, + "test", + &mut reconciler::ReconcilerState::for_maintenance_holder(), + ) + .await + .unwrap(); + assert!( + report.failed_prefixes_deleted.is_empty() && objects_under(&mem, &stale_failed).len() == 1, + "the holder's cycle deleted a FAILED row's prefix: {:?}", + report.failed_prefixes_deleted + ); + assert!( + report.registrations_swept.is_empty(), + "the holder's cycle swept the registrations" + ); + assert!( + report.maintenance_held, + "the holder's cycle took the branch for a free claim" + ); + assert!( + fake.maintenance_checks.lock().unwrap().is_empty(), + "the holder's cycle read the claim: {:?}", + fake.maintenance_checks.lock().unwrap() + ); + // The duties the command waits on still ran under the zero window. + assert_eq!(report.legacy_restores_removed, ["LegacyTarget"]); + assert_eq!(report.deletions_finished, [arn(ACCOUNT)]); + + // Without the holder's state the zero window reads the same claim as + // free. + let report = reconciler::cycle( + &fake, + &runtime_on(&mem, store()), + Duration::ZERO, + "test", + &mut reconciler::ReconcilerState::default(), + ) + .await + .unwrap(); + assert!(!report.maintenance_held); + assert_eq!( + *fake.maintenance_checks.lock().unwrap(), + [Duration::ZERO], + "a default state reads the claim through the window it is given" + ); + assert_eq!( + report.failed_prefixes_deleted, + std::slice::from_ref(&stale_failed) + ); + assert!(objects_under(&mem, &stale_failed).is_empty()); + assert_eq!(report.registrations_swept, [arn_n(9)]); +} + +/// The restore arm of a cycle run by the holder of the maintenance claim with +/// a zero dead-owner window (`backup purge` on SQLite): a RUNNING `manifest` +/// record whose owner is dead is FAILED `OWNER_DEAD` and its target removed, +/// as 4(f) asks of a held claim, and not taken over and resumed inside the +/// command. The command's claim was heartbeated a millisecond ago, which a +/// zero window read through the clock finds stale; a cycle that read it +/// would take the arm for a free claim and spawn a resume. +#[tokio::test] +async fn a_cycle_run_by_the_claim_holder_fails_a_dead_manifest_restore_rather_than_resuming_it() { + let mem = Arc::new(MemStore::default()); + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + let fake = Arc::new(Fake::default()); + seed_manifest_backup( + &fake, + &mem, + &arn(ACCOUNT), + music_source_table(), + &[songs(2)], + true, + ); + let runtime = job_runtime(&mem); + let gate = closed_gate(); + *fake.bulk_load_gate.lock().unwrap() = Some(Arc::clone(&gate)); + restore_on(&fake, &runtime, &restore_body(json!({}))) + .await + .unwrap(); + wait_for_bulk_loads_started(&fake, 1).await; + runtime.drain_jobs(Duration::from_millis(50)).await; + gate.add_permits(10); + wait_for_jobs(&runtime).await; + *fake.bulk_load_gate.lock().unwrap() = None; + assert_eq!( + fake.restore_job("MusicRestored").unwrap().status, + JobStatus::Running, + "precondition: the stopped restore left its record RUNNING" + ); + fake.restore_dead + .lock() + .unwrap() + .insert("tid-MusicRestored".to_owned()); + *fake.maintenance_beat_age.lock().unwrap() = Some(Duration::from_millis(1)); + // The restore's own request read the claim; the cycle must not. + fake.maintenance_checks.lock().unwrap().clear(); + + let command = job_runtime(&mem); + let report = reconciler::cycle( + &fake, + &command, + Duration::ZERO, + "test", + &mut reconciler::ReconcilerState::for_maintenance_holder(), + ) + .await + .unwrap(); + wait_for_jobs(&command).await; + assert!( + report.manifest_restores_resumed.is_empty(), + "the holder's cycle resumed a restore: {:?}", + report.manifest_restores_resumed + ); + assert!(fake.restore_takeovers.lock().unwrap().is_empty()); + assert_eq!( + report.manifest_restores_failed_and_removed, + ["MusicRestored"] + ); + assert!(fake.restore_job("MusicRestored").is_none()); + assert!(!fake.tables.lock().unwrap().contains_key("MusicRestored")); + assert!( + fake.failed_jobs + .lock() + .unwrap() + .iter() + .any(|(key, reason)| key.table_name == "MusicRestored" + && reason == reconciler::OWNER_DEAD), + "{:?}", + fake.failed_jobs.lock().unwrap() + ); + assert!(fake.maintenance_checks.lock().unwrap().is_empty()); +} + +/// The worker runs a cycle at once (observed through the maintenance check, +/// the cycle's first read), records the worker metrics the other workers +/// record, then waits an interval, and stops at the shutdown token. +#[tokio::test] +async fn the_reconciler_worker_runs_at_startup_and_stops_on_shutdown() { + use extenddb_core::metrics::{MetricName, MetricsCollector}; + let fake = Arc::new(Fake::default()); + *fake.maintenance_held.lock().unwrap() = true; + let metrics = Arc::new(MetricsCollector::new()); + let shutdown = extenddb_storage::CancellationToken::new(); + let worker = tokio::spawn(reconciler::run( + Arc::clone(&fake), + Arc::new(runtime()), + reconciler::ReconcilerSettings { + interval: Duration::from_secs(3600), + stale_job_after: Duration::from_secs(300), + }, + "test".to_owned(), + Arc::clone(&metrics), + Arc::new(tokio::sync::Mutex::new( + reconciler::ReconcilerState::default(), + )), + shutdown.clone(), + )); + let deadline = tokio::time::Instant::now() + Duration::from_secs(5); + while fake.maintenance_checks.lock().unwrap().is_empty() { + assert!( + tokio::time::Instant::now() < deadline, + "the first cycle did not run" + ); + tokio::time::sleep(Duration::from_millis(5)).await; + } + assert_eq!( + fake.maintenance_checks.lock().unwrap().len(), + 1, + "one cycle before the first interval" + ); + shutdown.cancel(); + tokio::time::timeout(Duration::from_secs(5), worker) + .await + .expect("the worker stops at shutdown") + .unwrap(); + assert_eq!(fake.maintenance_checks.lock().unwrap().len(), 1); + let recorded: Vec = metrics + .drain(Duration::ZERO) + .into_iter() + .filter(|b| b.operation == "BackupReconciler") + .map(|b| b.metric) + .collect(); + assert!( + recorded.contains(&MetricName::WorkerLastSuccess) + && recorded.contains(&MetricName::WorkerCycleLatency), + "{recorded:?}" + ); + assert!(!recorded.contains(&MetricName::WorkerErrorCount)); +} + +/// A cycle that fails is recorded as a worker error, as the other workers +/// record theirs, and the worker goes on to the next cycle. +#[tokio::test] +async fn a_failed_reconciler_cycle_records_a_worker_error() { + use extenddb_core::metrics::{MetricName, MetricsCollector}; + let fake = Arc::new(Fake::default()); + *fake.maintenance_error.lock().unwrap() = Some("catalog unreachable".to_owned()); + let metrics = Arc::new(MetricsCollector::new()); + let shutdown = extenddb_storage::CancellationToken::new(); + let worker = tokio::spawn(reconciler::run( + Arc::clone(&fake), + Arc::new(runtime()), + reconciler::ReconcilerSettings { + interval: Duration::from_millis(5), + stale_job_after: Duration::from_secs(300), + }, + "test".to_owned(), + Arc::clone(&metrics), + Arc::new(tokio::sync::Mutex::new( + reconciler::ReconcilerState::default(), + )), + shutdown.clone(), + )); + let deadline = tokio::time::Instant::now() + Duration::from_secs(5); + while fake.maintenance_checks.lock().unwrap().len() < 2 { + assert!( + tokio::time::Instant::now() < deadline, + "the worker did not go on after a failed cycle" + ); + tokio::time::sleep(Duration::from_millis(5)).await; + } + shutdown.cancel(); + tokio::time::timeout(Duration::from_secs(5), worker) + .await + .expect("the worker stops at shutdown") + .unwrap(); + let errors: f64 = metrics + .drain(Duration::ZERO) + .into_iter() + .filter(|b| b.operation == "BackupReconciler") + .filter(|b| b.metric == MetricName::WorkerErrorCount) + .map(|b| b.sum) + .sum(); + assert!(errors >= 2.0, "{errors} worker errors recorded"); +} + +/// The FAILED-target duty: a FAILED record whose target is still CREATING +/// (the FAILED write landed, the removal did not) is listed by the backend +/// once stale, and the reconciler removes the target with no second FAILED +/// write. +#[tokio::test] +async fn the_reconciler_removes_the_target_of_a_failed_record_without_writing_failed_again() { + use extenddb_storage::backup_catalog::{JobStatus, RestoreKind}; + let fake = Arc::new(Fake::default()); + *fake.target_still_creating.lock().unwrap() = true; + let mut failed = dead_job(RestoreKind::Legacy, "FailedTarget"); + failed.status = JobStatus::Failed; + let mut failed_manifest = dead_job(RestoreKind::Manifest, "FailedManifestTarget"); + failed_manifest.status = JobStatus::Failed; + fake.dead_jobs + .lock() + .unwrap() + .extend([failed.clone(), failed_manifest.clone()]); + let report = reconciler::cycle( + &fake, + &runtime(), + Duration::from_secs(300), + "test", + &mut reconciler::ReconcilerState::default(), + ) + .await + .unwrap(); + assert!( + fake.failed_jobs.lock().unwrap().is_empty(), + "a FAILED record is not written FAILED again" + ); + assert_eq!( + report.failed_targets_removed, + ["FailedTarget", "FailedManifestTarget"] + ); + assert!(report.legacy_restores_removed.is_empty()); + assert!(report.manifest_restores_resumed.is_empty()); + assert!(report.manifest_restores_failed.is_empty()); + assert_eq!( + fake.removed_targets.lock().unwrap().as_slice(), + [failed.target, failed_manifest.target] + ); +} + +/// The FAILED write for a dead legacy restore is conditioned on the +/// `owner_generation` and `heartbeat_at` the listing read (design 4(e)): a +/// heartbeat that lands between the listing and the write says the owner is +/// alive, the write matches nothing, and the target is left to it. The same +/// record without the late heartbeat is failed and removed. +#[tokio::test] +async fn the_reconciler_leaves_a_record_whose_owner_heartbeated_after_the_listing() { + use extenddb_storage::backup_catalog::RestoreKind; + let fake = Arc::new(Fake::default()); + *fake.target_still_creating.lock().unwrap() = true; + *fake.record_still_running.lock().unwrap() = true; + let mut listed = dead_job(RestoreKind::Legacy, "LateBeat"); + listed.owner_generation = 77; + listed.heartbeat_at = 1_000.0; + fake.dead_jobs.lock().unwrap().push(listed.clone()); + fake.heartbeat_after_listing + .lock() + .unwrap() + .insert(listed.target.clone(), 1_011.6); + + let report = reconciler::cycle( + &fake, + &runtime(), + Duration::from_secs(300), + "test", + &mut reconciler::ReconcilerState::default(), + ) + .await + .unwrap(); + assert!( + report.legacy_restores_removed.is_empty() && report.targets_left_to_owner.is_empty(), + "{report:?}" + ); + assert_eq!( + fake.failed_jobs.lock().unwrap().len(), + 1, + "the FAILED write was attempted under the listed claim" + ); + assert!( + fake.removed_targets.lock().unwrap().is_empty(), + "a record whose owner heartbeated keeps its target" + ); + + // Without the late heartbeat the listed values still match: failed and + // removed. + let report = reconciler::cycle( + &fake, + &runtime(), + Duration::from_secs(300), + "test", + &mut reconciler::ReconcilerState::default(), + ) + .await + .unwrap(); + assert_eq!(report.legacy_restores_removed, ["LateBeat"]); + assert_eq!( + fake.removed_targets.lock().unwrap().as_slice(), + [listed.target] + ); +} + +// --- the store identity file and the store's health --------------------------- + +const MANIFEST_KEY_SUFFIX: &str = "extenddb-manifest.json"; + +/// The manifest key of `arn`'s prefix. +fn manifest_key(arn: &str) -> String { + format!( + "{}{MANIFEST_KEY_SUFFIX}", + BackupArn::parse(arn).unwrap().store_prefix() + ) +} + +/// An ACTIVE table description, the probe CreateBackup's handler ran. +fn active_table(name: &str) -> TableDescription { + TableDescription { + table_name: name.to_owned(), + table_status: TableStatus::Active, + ..Default::default() + } +} + +/// A second, newer backup of the `Music` table. +fn newer_arn(account: &str) -> String { + format!("arn:aws:dynamodb:{REGION}:{account}:table/Music/backup/01791976999999-0badf00d") +} + +fn fake_with_rows(rows: &[(String, BackupStatus)]) -> Arc { + let fake = Arc::new(Fake::default()); + for (arn, status) in rows { + fake.rows + .lock() + .unwrap() + .insert(arn.clone(), row(arn, *status)); + } + fake +} + +/// A restore of a store-backed row checks the target name once the row is +/// known to exist and AVAILABLE, before the backup's in-use rule and before +/// the store is read (the service's order: a restore of a backup into its +/// own CREATING target is `TableInUseException`, measured): an existing +/// table of the name is `TableAlreadyExistsException`, a CREATING restore +/// target `TableInUseException` naming its source backup, and a free name +/// goes on to the manifest read (here `BackupNotFoundException`: the row is +/// AVAILABLE and the store holds nothing under its prefix). A row that is +/// not AVAILABLE answers for itself before the name is read. +#[tokio::test] +async fn a_store_backed_restore_checks_the_target_name_after_the_row_and_before_the_store() { + let mem = Arc::new(MemStore::default()); + let runtime = runtime_on(&mem, store()); + let fake = fake_with_rows(&[(arn(ACCOUNT), BackupStatus::Available)]); + + fake.tables.lock().unwrap().insert( + "MusicRestored".to_owned(), + TableDescription { + table_name: "MusicRestored".to_owned(), + table_status: TableStatus::Active, + ..Default::default() + }, + ); + let err = restore_on(&fake, &runtime, &restore_body(json!({}))) + .await + .unwrap_err(); + assert!( + matches!(err, DynamoDbError::TableAlreadyExistsException(_)), + "{err:?}" + ); + + fake.tables.lock().unwrap().insert( + "MusicRestored".to_owned(), + TableDescription { + table_name: "MusicRestored".to_owned(), + table_status: TableStatus::Creating, + restore_summary: Some(RestoreSummary { + source_backup_arn: Some(arn(ACCOUNT)), + source_table_arn: None, + restore_date_time: 0.0, + restore_in_progress: true, + }), + ..Default::default() + }, + ); + let err = restore_on(&fake, &runtime, &restore_body(json!({}))) + .await + .unwrap_err(); + assert!( + matches!(err, DynamoDbError::TableInUseException(ref m) if m.contains(&arn(ACCOUNT))), + "{err:?}" + ); + + fake.tables.lock().unwrap().remove("MusicRestored"); + assert!(mem.gets().is_empty(), "no store read before the name"); + let err = restore_on(&fake, &runtime, &restore_body(json!({}))) + .await + .unwrap_err(); + assert!( + matches!(err, DynamoDbError::BackupNotFoundException(_)), + "{err:?}" + ); + assert!(!mem.gets().is_empty(), "the free name reached the store"); + + // A row that is not AVAILABLE: its own answer, the name unread. + let fake = fake_with_rows(&[(arn(ACCOUNT), BackupStatus::Deleted)]); + fake.tables.lock().unwrap().insert( + "MusicRestored".to_owned(), + TableDescription { + table_name: "MusicRestored".to_owned(), + table_status: TableStatus::Active, + ..Default::default() + }, + ); + let err = restore_on(&fake, &runtime, &restore_body(json!({}))) + .await + .unwrap_err(); + assert!( + matches!(err, DynamoDbError::BackupNotFoundException(_)), + "{err:?}" + ); + assert!(fake.table_reads.lock().unwrap().is_empty()); +} + +/// An empty root and a catalog with no rows for it: startup creates the file +/// with this catalog's id and a fresh store id, re-reads it and uses that id. +#[tokio::test] +async fn establish_creates_the_identity_file_on_an_empty_root() { + let mem = MemStore::default(); + let fake = Fake::default(); + let identity = + store_identity::establish(&mem, StoreKind::Filesystem, "/r", &fake, "test", false) + .await + .unwrap(); + assert_eq!(identity.store_kind, StoreKind::Filesystem); + assert_eq!(identity.store_root, "/r"); + assert_eq!(mem.keys(), [STORE_IDENTITY_KEY]); + let file = StoreIdentityFile::parse(&mem.objects.lock().unwrap()[STORE_IDENTITY_KEY]).unwrap(); + assert_eq!(file.store_id, identity.store_id); + assert_eq!(file.catalog_id, "catalog-1"); + assert_eq!(file.backend, "test"); + assert_eq!(mem.puts.lock().unwrap().as_slice(), [STORE_IDENTITY_KEY]); + // A second startup against the same root reads the file and uses its id. + let again = store_identity::establish(&mem, StoreKind::Filesystem, "/r", &fake, "test", false) + .await + .unwrap(); + assert_eq!(again, identity); + assert_eq!( + mem.puts.lock().unwrap().len(), + 1, + "the file is not rewritten" + ); +} + +/// Two instances of one catalog starting against one empty root: one creates +/// the file, the other finds it present, and both use the winner's id. +#[tokio::test] +async fn two_instances_on_one_empty_root_end_with_one_id() { + let mem = Arc::new(MemStore::default()); + let fakes = [Fake::default(), Fake::default()]; + let (a, b) = tokio::join!( + store_identity::establish(&*mem, StoreKind::S3, "s3://b/", &fakes[0], "test", false), + store_identity::establish(&*mem, StoreKind::S3, "s3://b/", &fakes[1], "test", false), + ); + let (a, b) = (a.unwrap(), b.unwrap()); + assert_eq!(a, b); + assert_eq!(mem.keys(), [STORE_IDENTITY_KEY]); + assert_eq!( + mem.puts.lock().unwrap().len(), + 1, + "one create stored its file" + ); +} + +/// A root whose file names another catalog is unhealthy, and stays so until +/// the file is restored; the problem names both catalogs and no root. +#[tokio::test] +async fn a_file_naming_another_catalog_is_unhealthy_until_restored() { + let mem = MemStore::default(); + mem.insert( + STORE_IDENTITY_KEY, + &identity_file("store-1", "another-catalog"), + ); + let fake = fake_with_row(BackupStatus::Available); + let err = store_identity::establish(&mem, StoreKind::Filesystem, "/r", &*fake, "test", false) + .await + .unwrap_err(); + assert_eq!( + err, + StoreProblem::ForeignCatalog { + file_catalog_id: "another-catalog".to_owned(), + catalog_id: "catalog-1".to_owned(), + } + ); + assert_eq!(err.category(), "owned by another catalog"); + assert!(!err.public_reason().contains("/r")); + // Restored: the check passes. + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + mem.insert(&manifest_key(&arn(ACCOUNT)), b"{}"); + let identity = store_identity::recheck(&mem, &store(), &*fake, "test", false).await; + assert_eq!(identity, Ok(())); +} + +/// Something that is not an object at the identity file's key (a FIFO, a +/// socket): the create finds the key taken and the read finds no object, so +/// nothing was created and nothing can be read, and the problem says so +/// rather than claiming a file was created. +#[tokio::test] +async fn a_non_object_at_the_identity_key_is_neither_readable_nor_creatable() { + let mem = MemStore::default(); + mem.key_taken_by_a_non_object + .store(true, std::sync::atomic::Ordering::Release); + let fake = Fake::default(); + let err = store_identity::establish(&mem, StoreKind::Filesystem, "/r", &fake, "test", false) + .await + .unwrap_err(); + assert_eq!( + err, + StoreProblem::Transport("extenddb-store.json is neither readable nor creatable".to_owned()) + ); + assert!(mem.keys().is_empty(), "nothing was created"); +} + +/// The rows for the root carry an id the file does not, or more than one id, +/// or the root has no file while rows exist: each is unhealthy. +#[tokio::test] +async fn ids_that_disagree_or_a_missing_file_with_rows_are_unhealthy() { + // The file holds another id than the rows. + let mem = MemStore::default(); + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-2", "catalog-1")); + let fake = fake_with_row(BackupStatus::Available); + let err = store_identity::establish( + &mem, + StoreKind::Filesystem, + store().store_root.as_str(), + &*fake, + "test", + false, + ) + .await + .unwrap_err(); + assert_eq!( + err, + StoreProblem::IdMismatch { + file_id: "store-2".to_owned(), + row_ids: vec!["store-1".to_owned()], + } + ); + assert_eq!(err.category(), "identity mismatch"); + + // No file, rows exist. + let mem = MemStore::default(); + let err = store_identity::establish( + &mem, + StoreKind::Filesystem, + store().store_root.as_str(), + &*fake, + "test", + false, + ) + .await + .unwrap_err(); + assert_eq!( + err, + StoreProblem::FileMissingWithRows(vec!["store-1".to_owned()]) + ); + assert!(mem.keys().is_empty(), "nothing is created over rows"); + // The file put back: healthy. + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + mem.insert(&manifest_key(&arn(ACCOUNT)), b"{}"); + store_identity::establish( + &mem, + StoreKind::Filesystem, + store().store_root.as_str(), + &*fake, + "test", + false, + ) + .await + .unwrap(); + + // Rows carrying two ids. + let fake = fake_with_row(BackupStatus::Available); + let mut other = row(&newer_arn(ACCOUNT), BackupStatus::Available); + other.store.store_id = "store-9".to_owned(); + fake.rows.lock().unwrap().insert(newer_arn(ACCOUNT), other); + let err = store_identity::establish( + &mem, + StoreKind::Filesystem, + store().store_root.as_str(), + &*fake, + "test", + false, + ) + .await + .unwrap_err(); + assert_eq!( + err, + StoreProblem::MultipleRowIds(vec!["store-1".to_owned(), "store-9".to_owned()]) + ); +} + +/// The manifest probe: the newest AVAILABLE row's manifest must be under the +/// root. A second instance with a host-local root at the same path and the +/// first's file copied to it has the file and none of the objects, which is +/// what the probe catches; a root with no AVAILABLE row is not probed. +#[tokio::test] +async fn the_manifest_probe_reads_the_newest_available_rows_manifest() { + let mem = MemStore::default(); + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + let fake = fake_with_rows(&[ + (arn(ACCOUNT), BackupStatus::Available), + (newer_arn(ACCOUNT), BackupStatus::Available), + ( + music_arn(ACCOUNT, "01791976999999-0bad0bad"), + BackupStatus::Creating, + ), + ]); + // Only the older manifest is there: the newest is probed and missing. + mem.insert(&manifest_key(&arn(ACCOUNT)), b"{}"); + let err = store_identity::establish( + &mem, + StoreKind::Filesystem, + &store().store_root, + &*fake, + "test", + false, + ) + .await + .unwrap_err(); + assert_eq!( + err, + StoreProblem::ManifestMissing { + backup_arn: newer_arn(ACCOUNT), + } + ); + assert_eq!(err.category(), "manifest probe failed"); + mem.insert(&manifest_key(&newer_arn(ACCOUNT)), b"{}"); + store_identity::establish( + &mem, + StoreKind::Filesystem, + &store().store_root, + &*fake, + "test", + false, + ) + .await + .unwrap(); + // The CREATING row's manifest is not probed: it has none yet. + // A root with rows but none AVAILABLE is not probed either. + let fake = fake_with_rows(&[(arn(ACCOUNT), BackupStatus::Creating)]); + let empty = MemStore::default(); + empty.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + store_identity::establish( + &empty, + StoreKind::Filesystem, + &store().store_root, + &*fake, + "test", + false, + ) + .await + .unwrap(); +} + +/// A store that ignores the conditional put makes the create a plain put: a +/// race lost before the re-read leaves another id in the file, which the +/// re-read catches, naming both ids. +#[tokio::test] +async fn a_create_race_lost_on_a_store_ignoring_the_condition_is_caught_by_the_re_read() { + let mem = MemStore::default(); + *mem.overwrite_after_conditional_put.lock().unwrap() = + Some(identity_file("the-other-instance", "catalog-1")); + let fake = Fake::default(); + let err = store_identity::establish(&mem, StoreKind::S3, "s3://b/", &fake, "test", false) + .await + .unwrap_err(); + match err { + StoreProblem::CreateRaceLost { ours, file } => { + assert_eq!(file, "the-other-instance"); + assert_ne!(ours, file); + let reason = StoreProblem::CreateRaceLost { + ours: ours.clone(), + file: file.clone(), + } + .public_reason(); + assert!(reason.contains(&ours) && reason.contains(&file), "{reason}"); + } + other => panic!("expected CreateRaceLost, got {other:?}"), + } +} + +/// An unreachable store is unhealthy with the store's error in the log line +/// and not in the `/health` text; the check that runs again (the +/// reconciler's) recovers it once the store answers. +#[tokio::test] +async fn an_unreachable_store_is_unhealthy_and_recovers_on_recheck() { + let mem = Arc::new(MemStore::default()); + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + mem.set_down(true); + let fake = Arc::new(Fake::default()); + let err = store_identity::establish( + &*mem, + StoreKind::Filesystem, + &store().store_root, + &*fake, + "test", + false, + ) + .await + .unwrap_err(); + // The operator's line carries the whole cause chain: the top link names + // the class ("dispatch failure"), the cause is the link below it. + assert!( + matches!(&err, StoreProblem::Transport(m) + if m.contains("dispatch failure") && m.contains("connection refused")), + "{err:?}" + ); + assert_eq!(err.public_reason(), "the store could not be reached"); + assert!(err.to_string().contains("connection refused")); + + let runtime = runtime_unhealthy(&mem, err); + assert_eq!( + runtime.configured_store().unwrap().identity(), + None, + "the id is unknown until a check passes" + ); + let report = runtime.store_health().unwrap(); + assert_eq!(report.kind, "filesystem"); + assert_eq!(report.status, "unhealthy"); + assert_eq!(report.reason, Some("unreachable")); + + // Still down: the cycle leaves it unhealthy. + let report = reconciler::cycle( + &fake, + &runtime, + Duration::from_secs(300), + "test", + &mut reconciler::ReconcilerState::default(), + ) + .await + .unwrap(); + assert_eq!(report.store_healthy, Some(false)); + // Up again: the cycle's recheck passes and the component recovers. + mem.set_down(false); + let report = reconciler::cycle( + &fake, + &runtime, + Duration::from_secs(300), + "test", + &mut reconciler::ReconcilerState::default(), + ) + .await + .unwrap(); + assert_eq!(report.store_healthy, Some(true)); + assert_eq!(runtime.store_health().unwrap().status, "healthy"); + assert_eq!(runtime.store_health().unwrap().reason, None); + assert_eq!( + runtime.configured_store().unwrap().identity(), + Some(store()), + "the check that passed settled the id from the file" + ); + // And back to unhealthy when it goes away again. + mem.set_down(true); + let report = reconciler::cycle( + &fake, + &runtime, + Duration::from_secs(300), + "test", + &mut reconciler::ReconcilerState::default(), + ) + .await + .unwrap(); + assert_eq!(report.store_healthy, Some(false)); +} + +/// A store that has stopped answering costs a cycle its check bound and +/// nothing more: the dead-owner duties ran before the check, the check is +/// abandoned at `check_timeout` (5 s for a filesystem root) and recorded as +/// a transport failure naming the bound, the cycle returns, and the next +/// cycle on a store that answers again recovers it. +#[tokio::test(start_paused = true)] +async fn a_stalled_store_check_is_abandoned_at_its_bound() { + use extenddb_storage::backup_catalog::RestoreKind; + let mem = Arc::new(MemStore::default()); + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + mem.set_stalled(true); + let runtime = runtime_on(&mem, store()); + let configured = runtime.configured_store().unwrap(); + assert_eq!(configured.check_timeout(), LOCAL_CHECK_TIMEOUT); + let fake = Arc::new(Fake::default()); + *fake.record_still_running.lock().unwrap() = true; + *fake.target_still_creating.lock().unwrap() = true; + fake.dead_jobs + .lock() + .unwrap() + .push(dead_job(RestoreKind::Legacy, "LegacyTarget")); + + let started = tokio::time::Instant::now(); + // The test's own bound, far past the check's: a cycle that waits on the + // store for longer than it fails here instead of hanging. + let report = tokio::time::timeout( + Duration::from_secs(120), + reconciler::cycle( + &fake, + &runtime, + Duration::from_secs(300), + "test", + &mut reconciler::ReconcilerState::default(), + ), + ) + .await + .expect("the cycle returns within the check's bound") + .unwrap(); + assert_eq!( + started.elapsed(), + LOCAL_CHECK_TIMEOUT, + "the check's bound is the whole cost" + ); + assert_eq!( + report.legacy_restores_removed, + ["LegacyTarget"], + "the duties ran" + ); + assert_eq!(report.store_healthy, Some(false)); + assert_eq!( + configured.health(), + StoreHealth::Unhealthy(StoreProblem::Transport( + "the check did not finish within 5 s".to_owned() + )) + ); + assert_eq!(runtime.store_health().unwrap().reason, Some("unreachable")); + assert_eq!(mem.stalls.load(std::sync::atomic::Ordering::Acquire), 1); + // The abandoned check's call is still in the store (on a filesystem + // root, a thread parked on the mount). A cycle during the stall starts + // no second check: it returns at once, the store stays `unreachable` + // with the reason, and the store sees no second call. + assert!(configured.check_call_in_flight()); + let started = tokio::time::Instant::now(); + let report = reconciler::cycle( + &fake, + &runtime, + Duration::from_secs(300), + "test", + &mut reconciler::ReconcilerState::default(), + ) + .await + .unwrap(); + assert_eq!( + started.elapsed(), + Duration::ZERO, + "a cycle that starts no check waits no bound" + ); + assert_eq!(report.store_healthy, Some(false)); + assert_eq!( + configured.health(), + StoreHealth::Unhealthy(StoreProblem::Transport( + "a call of an earlier check has not returned".to_owned() + )) + ); + assert_eq!( + mem.stalls.load(std::sync::atomic::Ordering::Acquire), + 1, + "no second call reached the store while the first was in flight" + ); + // Checking stores wait the same bound: an S3 store's is 30 s, under the + // SDK's attempt timeout so no retry outlives the check. + let s3 = ConfiguredStore::checking( + StoreKind::S3, + "s3://b/".to_owned(), + Arc::clone(&mem) as Arc, + None, + ); + assert_eq!(s3.check_timeout(), S3_CHECK_TIMEOUT); + assert_eq!(S3_CHECK_TIMEOUT, Duration::from_secs(30)); + assert!(S3_CHECK_TIMEOUT < extenddb_backup_store::S3_ATTEMPT_TIMEOUT); + // An S3 store's abandoned request is dropped with the check (the SDK + // would otherwise retry it past the bound), so nothing is in flight + // afterwards and the next cycle sends a fresh request. + let s3_runtime = BackupRuntime::new(4).with_store(s3); + let s3 = s3_runtime.configured_store().unwrap(); + let calls_before = mem.stalls.load(std::sync::atomic::Ordering::Acquire); + for _ in 0..2 { + let started = tokio::time::Instant::now(); + reconciler::cycle( + &fake, + &s3_runtime, + Duration::from_secs(300), + "test", + &mut reconciler::ReconcilerState::default(), + ) + .await + .unwrap(); + assert_eq!(started.elapsed(), S3_CHECK_TIMEOUT); + assert!( + !s3.check_call_in_flight(), + "an S3 request ends with the check" + ); + } + assert_eq!( + mem.stalls.load(std::sync::atomic::Ordering::Acquire), + calls_before + 2, + "each cycle's check reached the endpoint" + ); + + // The store answers again: the parked call returns (its task clears the + // in-flight mark), and the next cycle's check passes. + mem.set_stalled(false); + tokio::time::sleep(Duration::from_millis(20)).await; + assert!( + !configured.check_call_in_flight(), + "the parked call returned once the store answered" + ); + let report = reconciler::cycle( + &fake, + &runtime, + Duration::from_secs(300), + "test", + &mut reconciler::ReconcilerState::default(), + ) + .await + .unwrap(); + assert_eq!(report.store_healthy, Some(true)); + assert!(configured.is_healthy()); +} + +/// A shutdown that fires while a cycle waits on the store's check ends the +/// cycle at once (the health is left as it was), so the worker drains +/// without waiting the check's bound. +#[tokio::test(start_paused = true)] +async fn a_shutdown_during_the_store_check_ends_the_cycle() { + use extenddb_core::metrics::MetricsCollector; + let mem = Arc::new(MemStore::default()); + mem.set_stalled(true); + // An S3 store, whose bound (30 s) is the longer one. + let runtime = Arc::new(runtime().with_store(ConfiguredStore::checking( + StoreKind::S3, + "s3://b/".to_owned(), + Arc::clone(&mem) as Arc, + None, + ))); + let fake = Arc::new(Fake::default()); + let shutdown = extenddb_storage::CancellationToken::new(); + let worker = tokio::spawn(reconciler::run( + Arc::clone(&fake), + Arc::clone(&runtime), + reconciler::ReconcilerSettings { + interval: Duration::from_secs(3600), + stale_job_after: Duration::from_secs(300), + }, + "test".to_owned(), + Arc::new(MetricsCollector::new()), + Arc::new(tokio::sync::Mutex::new( + reconciler::ReconcilerState::default(), + )), + shutdown.clone(), + )); + let deadline = tokio::time::Instant::now() + Duration::from_secs(5); + while mem.stalls.load(std::sync::atomic::Ordering::Acquire) == 0 { + assert!( + tokio::time::Instant::now() < deadline, + "the first cycle's check reached the store" + ); + tokio::time::sleep(Duration::from_millis(5)).await; + } + let cancelled = tokio::time::Instant::now(); + shutdown.cancel(); + tokio::time::timeout(Duration::from_secs(5), worker) + .await + .expect("the worker stops at shutdown without waiting the check's bound") + .unwrap(); + assert!( + cancelled.elapsed() < Duration::from_secs(1), + "the drain took {:?}", + cancelled.elapsed() + ); + assert_eq!( + runtime.configured_store().unwrap().health(), + StoreHealth::Checking, + "a check shutdown ended records nothing" + ); +} + +/// A command's identity read (`backup list`, the purge's dry run) runs under +/// the store's check bound as the server's check does: a store that does not +/// answer costs the command the bound (5 s for a filesystem root) and +/// reports it, where the unbounded read would wait out the store's retries +/// or, on a stalled mount, never return; a store that answers is read. +#[tokio::test(start_paused = true)] +async fn a_commands_identity_read_is_bounded_as_the_check_is() { + let mem = Arc::new(MemStore::default()); + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + let fake = Fake::default(); + let configured = ConfiguredStore::checking( + StoreKind::Filesystem, + store().store_root, + Arc::clone(&mem) as Arc, + None, + ); + let read = runtime::read_identity_bounded(&configured, &fake) + .await + .unwrap(); + assert_eq!(read.map(|i| i.store_id), Some("store-1".to_owned())); + + mem.set_stalled(true); + let started = tokio::time::Instant::now(); + // The test's own bound, far past the read's: an unbounded read fails + // here instead of hanging. + let problem = tokio::time::timeout( + Duration::from_secs(120), + runtime::read_identity_bounded(&configured, &fake), + ) + .await + .expect("the read returns at its bound") + .unwrap_err(); + assert_eq!(started.elapsed(), LOCAL_CHECK_TIMEOUT); + assert_eq!( + problem, + StoreProblem::Transport("the identity read did not finish within 5 s".to_owned()) + ); + assert_eq!(problem.category(), "unreachable"); + assert_eq!(mem.stalls.load(std::sync::atomic::Ordering::Acquire), 1); + mem.set_stalled(false); +} + +/// `record_check` logs a transition once per category: it reports a change +/// of category, and not a new text within the same one, so a store that +/// stays down with a varying error costs one line. +#[test] +fn record_check_reports_a_transition_only_across_categories() { + let mem = Arc::new(MemStore::default()); + let runtime = runtime_on(&mem, store()); + let store = runtime.configured_store().unwrap(); + assert!(store.record_check(Err(StoreProblem::Transport("refused".to_owned())))); + assert!( + !store.record_check(Err(StoreProblem::Transport("timed out".to_owned()))), + "a new text in the same category is not a transition" + ); + assert_eq!( + store.health(), + StoreHealth::Unhealthy(StoreProblem::Transport("timed out".to_owned())), + "the latest text is kept" + ); + assert!(store.record_check(Err(StoreProblem::Validate("no bucket".to_owned())))); + assert!(store.record_check(Ok(self::store()))); + assert!(!store.record_check(Ok(self::store()))); + assert!( + !store.record_check(Err(StoreProblem::Catalog("pool exhausted".to_owned()))), + "a catalog read failure changes nothing" + ); +} + +/// A recheck of a root that lost its file while the catalog has no rows for +/// it writes the file back with the same id. +#[tokio::test] +async fn a_recheck_rewrites_a_lost_file_when_no_rows_exist() { + let mem = MemStore::default(); + let fake = Fake::default(); + store_identity::recheck(&mem, &store(), &fake, "test", false) + .await + .unwrap(); + let file = StoreIdentityFile::parse(&mem.objects.lock().unwrap()[STORE_IDENTITY_KEY]).unwrap(); + assert_eq!(file.store_id, "store-1"); + assert_eq!(file.catalog_id, "catalog-1"); + // A file that changed under the running server (the root replaced) is + // an id mismatch. + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-2", "catalog-1")); + let err = store_identity::recheck(&mem, &store(), &fake, "test", false) + .await + .unwrap_err(); + assert_eq!( + err, + StoreProblem::IdMismatch { + file_id: "store-2".to_owned(), + row_ids: vec!["store-1".to_owned()], + } + ); +} + +/// No store configured: `/health` has no component and no operation is +/// refused for the store. +#[test] +fn no_configured_store_reports_no_component() { + assert_eq!(runtime().store_health(), None); + assert_eq!( + runtime_with_store().store_health().unwrap().status, + "healthy" + ); +} + +/// The `/health` reason is the problem's category and nothing else: the +/// endpoint is unauthenticated, and the ARN (account, table and backup id) +/// a manifest probe names, and the catalog and store ids the identity +/// problems name, go to the operator's log line only. +#[test] +fn the_health_reason_is_the_category_only() { + let mem = Arc::new(MemStore::default()); + for (problem, category) in [ + ( + StoreProblem::ManifestMissing { + backup_arn: arn("999988887777"), + }, + "manifest probe failed", + ), + ( + StoreProblem::ForeignCatalog { + file_catalog_id: "cat-other".to_owned(), + catalog_id: "cat-mine".to_owned(), + }, + "owned by another catalog", + ), + ( + StoreProblem::IdMismatch { + file_id: "store-9".to_owned(), + row_ids: vec!["store-1".to_owned()], + }, + "identity mismatch", + ), + ( + StoreProblem::FileMissingWithRows(vec!["store-1".to_owned()]), + "identity file missing", + ), + ( + StoreProblem::Transport("connect /var/lib/extenddb: refused".to_owned()), + "unreachable", + ), + ] { + let report = runtime_unhealthy(&mem, problem.clone()) + .store_health() + .unwrap(); + assert_eq!(report.status, "unhealthy"); + assert_eq!(report.reason, Some(category), "{problem:?}"); + for secret in ["999988887777", "cat-", "store-", "/var/lib", "arn:"] { + assert!( + !report.reason.unwrap().contains(secret), + "{problem:?}: the reason carries {secret:?}" + ); + } + } +} + +/// An identity file over the read bound is reported unreadable without +/// being read in full; one under it parses. +#[tokio::test] +async fn an_oversized_identity_file_is_unreadable() { + let mem = MemStore::default(); + let fake = Fake::default(); + let mut big = identity_file("store-1", "catalog-1"); + big.resize(store_identity::STORE_IDENTITY_MAX_BYTES + 1, b' '); + mem.insert(STORE_IDENTITY_KEY, &big); + let err = store_identity::establish(&mem, StoreKind::Filesystem, "/r", &fake, "test", false) + .await + .unwrap_err(); + assert!( + matches!(&err, StoreProblem::FileUnreadable(m) if m.contains("larger than 65536 bytes")), + "{err:?}" + ); + assert_eq!(err.category(), "identity file unreadable"); + // Trailing blanks up to the bound are JSON whitespace: the file parses. + let mut padded = identity_file("store-1", "catalog-1"); + padded.resize(store_identity::STORE_IDENTITY_MAX_BYTES, b' '); + mem.insert(STORE_IDENTITY_KEY, &padded); + let identity = + store_identity::establish(&mem, StoreKind::Filesystem, "/r", &fake, "test", false) + .await + .unwrap(); + assert_eq!(identity.store_id, "store-1"); +} + +/// A check that could not read the catalog leaves the store's health as the +/// last check left it, healthy or unhealthy, and keeps its id. +#[tokio::test] +async fn a_catalog_read_failure_leaves_the_health_unchanged() { + let mem = Arc::new(MemStore::default()); + let healthy = runtime_on(&mem, store()); + let store = healthy.configured_store().unwrap(); + store.record_check(Err(StoreProblem::Catalog("pool exhausted".to_owned()))); + assert_eq!(store.health(), StoreHealth::Healthy); + assert_eq!(store.identity(), Some(self::store())); + let unhealthy = runtime_unhealthy(&mem, StoreProblem::Transport("refused".to_owned())); + let store = unhealthy.configured_store().unwrap(); + store.record_check(Err(StoreProblem::Catalog("pool exhausted".to_owned()))); + assert_eq!( + store.health(), + StoreHealth::Unhealthy(StoreProblem::Transport("refused".to_owned())) + ); + // And through a cycle: a catalog that fails `catalog_id` leaves a + // healthy store healthy. + let fake = Arc::new(Fake::default()); + *fake.catalog_id_error.lock().unwrap() = true; + let report = reconciler::cycle( + &fake, + &healthy, + Duration::from_secs(300), + "test", + &mut reconciler::ReconcilerState::default(), + ) + .await + .unwrap(); + assert_eq!(report.store_healthy, Some(true)); + // A problem of another category replaces the last one; one of the same + // category is kept as the latest text. + let store = healthy.configured_store().unwrap(); + store.record_check(Err(StoreProblem::Transport("refused".to_owned()))); + store.record_check(Err(StoreProblem::Transport("timed out".to_owned()))); + assert_eq!( + store.health(), + StoreHealth::Unhealthy(StoreProblem::Transport("timed out".to_owned())) + ); +} + +/// A store whose checks run after the listener is up is `checking` until +/// the first check records: `/health` says so with no reason, the +/// operations that touch the store refuse naming the pending check, and +/// the check runs `validate` first (a store that cannot be reached fails +/// there, with its id still unknown), then the identity check, which +/// settles the id. +#[tokio::test] +async fn a_store_checked_after_startup_is_checking_until_the_first_check() { + let mem = Arc::new(MemStore::default()); + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + mem.set_down(true); + let runtime = runtime().with_store(ConfiguredStore::checking( + store().store_kind, + store().store_root, + Arc::clone(&mem) as Arc, + None, + )); + let configured = runtime.configured_store().unwrap(); + assert_eq!(configured.health(), StoreHealth::Checking); + assert!(!configured.is_validated()); + let report = runtime.store_health().unwrap(); + assert_eq!((report.status, report.reason), ("checking", None)); + let err = runtime.require_healthy_store().err().unwrap(); + assert!( + matches!(&err, DynamoDbError::ValidationException(m) if m.contains("first check is pending")), + "{err:?}" + ); + // The restore check of a row of the store refuses the same way, before + // the permit: with no slot free, the refusal is still the store's. + let fake = fake_with_row(BackupStatus::Available); + let full = BackupRuntime::new(0).with_store(ConfiguredStore::checking( + store().store_kind, + store().store_root, + Arc::clone(&mem) as Arc, + None, + )); + let err = restore_on(&fake, &full, &restore_body(json!({}))) + .await + .unwrap_err(); + assert!( + matches!(&err, DynamoDbError::ValidationException(m) if m.contains("first check is pending")), + "{err:?}" + ); + + // The first check: `validate` fails on the unreachable store. + let fake = Arc::new(Fake::default()); + let report = reconciler::cycle( + &fake, + &runtime, + Duration::from_secs(300), + "test", + &mut reconciler::ReconcilerState::default(), + ) + .await + .unwrap(); + assert_eq!(report.store_healthy, Some(false)); + assert!(!configured.is_validated()); + assert_eq!(runtime.store_health().unwrap().reason, Some("unreachable")); + assert_eq!(configured.identity(), None); + // Reachable: `validate` passes once and is not run again, the identity + // check settles the id, and the store is healthy. + mem.set_down(false); + let report = reconciler::cycle( + &fake, + &runtime, + Duration::from_secs(300), + "test", + &mut reconciler::ReconcilerState::default(), + ) + .await + .unwrap(); + assert_eq!(report.store_healthy, Some(true)); + assert!(configured.is_validated()); + assert_eq!(configured.identity(), Some(store())); + assert_eq!(runtime.store_health().unwrap().status, "healthy"); +} + +/// An unhealthy store refuses the operations that touch it, before a slot is +/// taken and naming the store's kind and the problem's category and never +/// its root or the cause: CreateBackup (store-backed on this runtime), +/// DeleteBackup of a row of the store, and the restore of one (with no slot +/// free it is still the store's refusal, not LimitExceeded). A backend +/// bound to the in-catalog create is not refused: its create touches no +/// store. A row of another store is refused as another store's, whatever +/// the configured one's health. +#[tokio::test] +async fn an_unhealthy_store_refuses_create_delete_and_restore_unless_bound_to_the_in_catalog_create() + { + let mem = Arc::new(MemStore::default()); + let unhealthy = || { + runtime_unhealthy( + &mem, + StoreProblem::Transport("connect /var/lib/extenddb/backups: refused".to_owned()), + ) + }; + let check = |e: DynamoDbError| { + let DynamoDbError::ValidationException(m) = e else { + panic!("expected ValidationException, got {e:?}"); + }; + assert!(m.contains("filesystem backup store"), "{m}"); + assert!(m.contains("unreachable"), "{m}"); + assert!(!m.contains("/var/lib"), "{m}"); + assert!(!m.contains("connect"), "{m}"); + }; + + // The store-backed create is refused before the permit and the open. + let fake = Arc::new(Fake::default()); + let runtime = unhealthy(); + let err = create_backup( + &fake, + &runtime, + ACCOUNT, + REGION, + &LimitsConfig::default(), + "Music", + "nightly", + Ok(active_table("Music")), + ) + .await + .unwrap_err(); + check(err); + assert_eq!(runtime.free_slots(), 4, "no slot was taken"); + assert!(fake.snapshots_opened.lock().unwrap().is_empty()); + assert!(fake.creates.lock().unwrap().is_empty()); + + // A backend bound to the in-catalog create runs it, taking its slot. + let runtime = unhealthy().with_legacy_create_only(true); + create_backup( + &fake, + &runtime, + ACCOUNT, + REGION, + &LimitsConfig::default(), + "Music", + "nightly", + Ok(active_table("Music")), + ) + .await + .unwrap(); + assert_eq!(fake.creates.lock().unwrap().len(), 1); + assert_eq!(runtime.free_slots(), 3, "the create holds its slot"); + fake.permits.lock().unwrap().clear(); + + // DeleteBackup of a row of the store. + let fake = fake_with_row(BackupStatus::Available); + let err = delete_backup(&*fake, &unhealthy(), ACCOUNT, REGION, &own(&arn(ACCOUNT))) + .await + .unwrap_err(); + check(err); + assert!( + fake.delete_starts_seen.lock().unwrap().is_empty(), + "no DELETING write" + ); + + // The restore of a row of the store, with no slot free: the refusal is + // the store's, so it came before the permit. + let full = BackupRuntime::new(0).with_store(ConfiguredStore::new( + store().store_kind, + store().store_root, + Arc::clone(&mem) as Arc, + None, + Err(StoreProblem::Transport( + "connect /var/lib/extenddb/backups: refused".to_owned(), + )), + )); + let err = restore_on(&fake, &full, &restore_body(json!({}))) + .await + .unwrap_err(); + check(err); + assert!(fake.restores().is_empty(), "the backend was not asked"); + + // A row of another store is the other-store refusal, whatever the health + // of the configured one. + let mut other = row(&arn(ACCOUNT), BackupStatus::Available); + other.store.store_root = "/elsewhere".to_owned(); + let fake = Arc::new(Fake::default()); + fake.rows.lock().unwrap().insert(arn(ACCOUNT), other); + let err = delete_backup(&*fake, &unhealthy(), ACCOUNT, REGION, &own(&arn(ACCOUNT))) + .await + .unwrap_err(); + assert!( + matches!(&err, DynamoDbError::ValidationException(m) if m.contains("other than")), + "{err:?}" + ); +} + +/// With a healthy store configured, CreateBackup is store-backed: the +/// legacy reader is not asked, the table's snapshot is opened and the answer +/// is CREATING; a table with vector indexes is backed up on this path (the +/// refusal belongs to the in-catalog create alone). +#[tokio::test] +async fn a_healthy_store_makes_create_backup_store_backed() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let mut table = music_table(); + table.vector_indexes = Some(vec![vector_index()]); + snapshot_fixture(&fake, &table, Vec::new(), true, None); + let runtime = job_runtime(&mem); + let details = create_backup( + &fake, + &runtime, + ACCOUNT, + REGION, + &LimitsConfig::default(), + "Music", + "nightly", + Ok(table), + ) + .await + .unwrap(); + assert_eq!(details.backup_status, "CREATING"); + assert!( + fake.creates.lock().unwrap().is_empty(), + "the legacy reader was not asked" + ); + assert_eq!(fake.snapshots_opened.lock().unwrap().as_slice(), ["Music"]); + wait_for_jobs(&runtime).await; + let row = fake.rows.lock().unwrap()[&details.backup_arn].clone(); + assert_eq!(row.status, BackupStatus::Available); + assert_eq!( + row.source_table_details_json["vector_indexes"][0]["IndexName"], + "v" + ); +} + +/// DeleteBackup of a row of the configured store deletes the manifest first +/// (a `delete_prefix` on its exact key), then the prefix, and moves the row +/// to DELETED once the prefix lists nothing; a retry on a DELETING row runs +/// the same deletion. The identity file and other backups' prefixes stay. +#[tokio::test] +async fn delete_backup_of_a_configured_store_row_deletes_manifest_prefix_then_marks_deleted() { + let mem = Arc::new(MemStore::default()); + let prefix = BackupArn::parse(&arn(ACCOUNT)).unwrap().store_prefix(); + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + mem.insert(&manifest_key(&arn(ACCOUNT)), b"{}"); + mem.insert(&format!("{prefix}data/00001.json.gz"), b"gz"); + mem.insert(&format!("{prefix}manifest-files.json"), b"{}"); + mem.insert(&manifest_key(&newer_arn(ACCOUNT)), b"{}"); + let fake = fake_with_rows(&[ + (arn(ACCOUNT), BackupStatus::Deleting), + (newer_arn(ACCOUNT), BackupStatus::Available), + ]); + let runtime = runtime_on(&mem, store()); + let desc = delete_backup(&*fake, &runtime, ACCOUNT, REGION, &own(&arn(ACCOUNT))) + .await + .unwrap(); + assert_eq!(desc.backup_details.backup_status, "DELETED"); + assert_eq!( + mem.deletes.lock().unwrap().as_slice(), + [manifest_key(&arn(ACCOUNT)), prefix.clone()], + "the manifest first, then the prefix" + ); + assert_eq!( + mem.keys(), + [ + manifest_key(&newer_arn(ACCOUNT)), + STORE_IDENTITY_KEY.to_owned() + ], + "the other backup and the identity file stay" + ); + assert_eq!(fake.set_deleted.lock().unwrap().as_slice(), [arn(ACCOUNT)]); + assert_eq!( + fake.rows.lock().unwrap()[&arn(ACCOUNT)].status, + BackupStatus::Deleted + ); +} + +/// A store failure during the deletion leaves the row DELETING and still +/// answers the description with `BackupStatus` DELETED (the service has no +/// counterpart for a failed object deletion, and every reader reports the +/// backup deleted from the DELETING write on); the reconciler's next cycle +/// finishes it once the store answers, and leaves it alone while the store +/// is unhealthy. +#[tokio::test] +async fn a_failed_deletion_stays_deleting_and_the_reconciler_finishes_it() { + let mem = Arc::new(MemStore::default()); + let prefix = BackupArn::parse(&arn(ACCOUNT)).unwrap().store_prefix(); + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + mem.insert(&manifest_key(&arn(ACCOUNT)), b"{}"); + mem.insert(&format!("{prefix}data/00001.json.gz"), b"gz"); + let fake = fake_with_rows(&[(arn(ACCOUNT), BackupStatus::Deleting)]); + let runtime = runtime_on(&mem, store()); + + mem.set_down(true); + let out = delete_backup(&*fake, &runtime, ACCOUNT, REGION, &own(&arn(ACCOUNT))) + .await + .unwrap(); + assert_eq!( + out.backup_details.backup_status, "DELETED", + "the description answers DELETED once DELETING committed" + ); + assert_eq!( + fake.rows.lock().unwrap()[&arn(ACCOUNT)].status, + BackupStatus::Deleting + ); + assert_eq!(mem.keys().len(), 3, "nothing was deleted"); + + // The reconciler finds the store down: unhealthy, and no row touched. + let report = reconciler::cycle( + &fake, + &runtime, + Duration::from_secs(300), + "test", + &mut reconciler::ReconcilerState::default(), + ) + .await + .unwrap(); + assert_eq!(report.store_healthy, Some(false)); + assert!(report.deletions_finished.is_empty()); + assert!(runtime.require_healthy_store().is_err()); + + // The store answers again: the cycle recovers the health and finishes + // the deletion. + mem.set_down(false); + let report = reconciler::cycle( + &fake, + &runtime, + Duration::from_secs(300), + "test", + &mut reconciler::ReconcilerState::default(), + ) + .await + .unwrap(); + assert_eq!(report.store_healthy, Some(true)); + assert_eq!(report.deletions_finished, [arn(ACCOUNT)]); + assert_eq!(mem.keys(), [STORE_IDENTITY_KEY]); + assert_eq!( + fake.rows.lock().unwrap()[&arn(ACCOUNT)].status, + BackupStatus::Deleted + ); + // Nothing left to finish. + let report = reconciler::cycle( + &fake, + &runtime, + Duration::from_secs(300), + "test", + &mut reconciler::ReconcilerState::default(), + ) + .await + .unwrap(); + assert!(report.deletions_finished.is_empty()); +} + +/// The reconciler finishes DELETING rows of the configured store only: a +/// DELETING row of another store is left alone, and so is one whose prefix +/// still lists an object after the deletion (a put that overlapped it). +#[tokio::test] +async fn the_reconciler_finishes_only_the_configured_stores_deleting_rows() { + let mem = Arc::new(MemStore::default()); + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + let fake = fake_with_rows(&[(arn(ACCOUNT), BackupStatus::Deleting)]); + let mut other = row(&newer_arn(ACCOUNT), BackupStatus::Deleting); + other.store.store_id = "store-9".to_owned(); + other.store.store_root = "/elsewhere".to_owned(); + fake.rows.lock().unwrap().insert(newer_arn(ACCOUNT), other); + let runtime = runtime_on(&mem, store()); + let report = reconciler::cycle( + &fake, + &runtime, + Duration::from_secs(300), + "test", + &mut reconciler::ReconcilerState::default(), + ) + .await + .unwrap(); + assert_eq!(report.deletions_finished, [arn(ACCOUNT)]); + assert_eq!( + fake.rows.lock().unwrap()[&newer_arn(ACCOUNT)].status, + BackupStatus::Deleting, + "another store's row is not touched" + ); + assert!( + mem.deletes + .lock() + .unwrap() + .iter() + .all(|p| !p.contains("0badf00d")), + "nothing of the other store's prefix was deleted" + ); +} + +/// `finish_backup_delete` reports a prefix that still lists an object after +/// the deletion and leaves the row DELETING for the next run. +#[tokio::test] +async fn a_prefix_that_still_lists_an_object_is_left_deleting() { + struct Refilling(MemStore); + impl extenddb_backup_store::BackupStore for Refilling { + fn put<'a>( + &'a self, + key: &'a str, + body: extenddb_backup_store::ByteStream, + ) -> BoxedFuture< + 'a, + Result, + > { + self.0.put(key, body) + } + fn put_if_absent<'a>( + &'a self, + key: &'a str, + body: extenddb_backup_store::ByteStream, + ) -> BoxedFuture< + 'a, + Result, extenddb_backup_store::StoreError>, + > { + self.0.put_if_absent(key, body) + } + fn get<'a>( + &'a self, + key: &'a str, + ) -> BoxedFuture< + 'a, + Result, + > { + self.0.get(key) + } + fn head<'a>( + &'a self, + key: &'a str, + ) -> BoxedFuture< + 'a, + Result, extenddb_backup_store::StoreError>, + > { + self.0.head(key) + } + fn list<'a>( + &'a self, + prefix: &'a str, + ) -> BoxStream< + 'a, + Result, + > { + self.0.list(prefix) + } + /// The deletion runs, and a late put lands under the prefix after + /// it. + fn delete_prefix<'a>( + &'a self, + prefix: &'a str, + ) -> BoxedFuture<'a, Result> { + Box::pin(async move { + let n = self.0.delete_prefix(prefix).await?; + if !prefix.ends_with(MANIFEST_KEY_SUFFIX) { + self.0 + .insert(&format!("{prefix}data/late.json.gz"), b"late"); + } + Ok(n) + }) + } + fn validate(&self) -> BoxedFuture<'_, Result<(), extenddb_backup_store::StoreError>> { + self.0.validate() + } + } + let store_handle = Refilling(MemStore::default()); + let fake = fake_with_rows(&[(arn(ACCOUNT), BackupStatus::Deleting)]); + let parsed = BackupArn::parse(&arn(ACCOUNT)).unwrap(); + let err = finish_backup_delete(&*fake, &store_handle, &passing_fence(), &parsed) + .await + .unwrap_err(); + assert!(err.to_string().contains("still lists"), "{err}"); + assert_eq!( + fake.rows.lock().unwrap()[&arn(ACCOUNT)].status, + BackupStatus::Deleting + ); +} + +// --- the second review round: the store's health, the purge's steps, the orders -- + +mod review_round_2 { + use super::*; + use extenddb_storage::backup_catalog::{JobStatus, RestoreKind}; + use maintenance::{ + Interrupt, PurgeOutcome, Stop, Wait, WaitOutcome, purge_rows, wait_for_quiet, + }; + + async fn cycle_on(fake: &Arc, runtime: &BackupRuntime) -> reconciler::CycleReport { + reconciler::cycle( + fake, + runtime, + Duration::from_secs(300), + "test", + &mut reconciler::ReconcilerState::default(), + ) + .await + .unwrap() + } + + /// A store whose startup `validate` failed (an I/O error on the probe + /// write) is built unvalidated, so the next check runs `validate` again + /// and the store stays unhealthy while the probe fails, where a store + /// built validated would re-read the identity file alone and report + /// itself healthy after one cycle. Once the probe passes the store is + /// healthy. + #[tokio::test] + async fn a_store_whose_startup_validate_failed_is_rechecked_by_validate() { + let mem = Arc::new(MemStore::default()); + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + *mem.validate_error.lock().unwrap() = + Some("write probe: Input/output error (os error 5)".to_owned()); + let fake = Arc::new(Fake::default()); + let runtime = runtime_unhealthy( + &mem, + StoreProblem::Validate("write probe: Input/output error (os error 5)".to_owned()), + ); + let configured = runtime.configured_store().unwrap(); + assert!( + !configured.is_validated(), + "a failed validate leaves the store unvalidated" + ); + let report = cycle_on(&fake, &runtime).await; + assert_eq!(report.store_healthy, Some(false)); + assert_eq!( + configured.health(), + StoreHealth::Unhealthy(StoreProblem::Validate( + "write probe: Input/output error (os error 5)".to_owned() + )), + "validate ran again and failed again" + ); + assert!(!configured.is_validated()); + assert_eq!(runtime.store_health().unwrap().status, "unhealthy"); + + *mem.validate_error.lock().unwrap() = None; + let report = cycle_on(&fake, &runtime).await; + assert_eq!(report.store_healthy, Some(true)); + assert!(configured.is_validated()); + assert_eq!( + configured.identity().map(|i| i.store_id), + Some("store-1".to_owned()) + ); + + // A startup check that failed after `validate` passed (an identity + // problem) is built validated: the probe is not repeated. + let identity_problem = runtime_unhealthy( + &mem, + StoreProblem::ForeignCatalog { + file_catalog_id: "other".to_owned(), + catalog_id: "catalog-1".to_owned(), + }, + ); + assert!(identity_problem.configured_store().unwrap().is_validated()); + for problem in [ + StoreProblem::Transport("refused".to_owned()), + StoreProblem::Denied("mode 0755".to_owned()), + ] { + assert!( + !runtime_unhealthy(&mem, problem) + .configured_store() + .unwrap() + .is_validated(), + "the problems validate can end in leave the store unvalidated" + ); + } + } + + /// `purge_row` deletes the manifest before the rest of the prefix (so + /// the prefix is no longer a complete backup to any reader while the + /// rest goes), lists the prefix after the deletion and leaves a row whose + /// prefix still holds an object (a put that overlapped the deletion), + /// and does not continue past `DeleteStart::InUse`: a restore reading + /// the backup keeps its source and the row stays AVAILABLE. + #[tokio::test] + async fn purge_row_deletes_manifest_first_lists_after_and_stops_at_in_use() { + // The order: the manifest key, then the prefix. + let fake = Fake::default(); + let mem = MemStore::default(); + let arn = arn_n(1); + seed(&fake, &mem, &arn, BackupStatus::Available); + let prefix = BackupArn::parse(&arn).unwrap().store_prefix(); + let row = fake.rows.lock().unwrap()[&arn].clone(); + maintenance::purge_row( + &fake, + &mem, + &passing_fence(), + &row, + Duration::from_secs(300), + ) + .await + .unwrap(); + assert_eq!( + *mem.deletes.lock().unwrap(), + vec![format!("{prefix}{MANIFEST_KEY_SUFFIX}"), prefix.clone()], + "the manifest first, then the prefix" + ); + assert!(mem.keys().is_empty()); + assert!(fake.rows.lock().unwrap().is_empty()); + + // An object left under the prefix: the row is left, after the + // DELETING write, for the operator and the reconciler. + let fake = Fake::default(); + let mem = MemStore::default(); + *mem.late_put_after_delete.lock().unwrap() = Some("data/late.json.gz".to_owned()); + seed(&fake, &mem, &arn, BackupStatus::Available); + let row = fake.rows.lock().unwrap()[&arn].clone(); + let err = maintenance::purge_row( + &fake, + &mem, + &passing_fence(), + &row, + Duration::from_secs(300), + ) + .await + .unwrap_err(); + assert_eq!( + err, + format!("the prefix still lists {prefix}data/late.json.gz after the deletion") + ); + assert_eq!( + fake.rows.lock().unwrap()[&arn].status, + BackupStatus::Deleting, + "the row is left DELETING, not removed, over objects" + ); + assert!(fake.set_deleted.lock().unwrap().is_empty()); + assert!(fake.removed_rows.lock().unwrap().is_empty()); + + // In use: a restore reads the backup, so nothing is deleted. + let fake = Fake::default(); + let mem = MemStore::default(); + seed(&fake, &mem, &arn, BackupStatus::Available); + fake.delete_starts + .lock() + .unwrap() + .insert(arn.clone(), DeleteStart::InUse("Restored".to_owned())); + let row = fake.rows.lock().unwrap()[&arn].clone(); + let err = maintenance::purge_row( + &fake, + &mem, + &passing_fence(), + &row, + Duration::from_secs(300), + ) + .await + .unwrap_err(); + assert_eq!(err, "a restore into Restored is reading it"); + assert!(mem.deletes.lock().unwrap().is_empty(), "no delete ran"); + assert_eq!(mem.keys().len(), 2, "the objects stay"); + assert_eq!( + fake.rows.lock().unwrap()[&arn].status, + BackupStatus::Available + ); + } + + /// The reconciler's DELETING duty runs only while the store is healthy: + /// with a foreign `extenddb-store.json` under the root (another + /// catalog's), and a store that answers deletes, no delete runs and the + /// row stays DELETING, so the reconciler never deletes under a root + /// another catalog owns. + #[tokio::test] + async fn an_unhealthy_store_whose_deletes_would_answer_has_no_row_touched() { + let mem = Arc::new(MemStore::default()); + mem.insert( + STORE_IDENTITY_KEY, + &identity_file("store-1", "other-catalog"), + ); + mem.insert(&manifest_key(&arn(ACCOUNT)), b"{}"); + let fake = fake_with_rows(&[(arn(ACCOUNT), BackupStatus::Deleting)]); + let runtime = runtime_on(&mem, store()); + let report = cycle_on(&fake, &runtime).await; + assert_eq!(report.store_healthy, Some(false)); + assert_eq!( + runtime.configured_store().unwrap().health(), + StoreHealth::Unhealthy(StoreProblem::ForeignCatalog { + file_catalog_id: "other-catalog".to_owned(), + catalog_id: "catalog-1".to_owned(), + }) + ); + assert!( + report.deletions_finished.is_empty() && mem.deletes.lock().unwrap().is_empty(), + "no delete ran under a root another catalog owns" + ); + assert_eq!( + fake.rows.lock().unwrap()[&arn(ACCOUNT)].status, + BackupStatus::Deleting + ); + assert_eq!(mem.keys().len(), 2, "the manifest and the file stay"); + } + + /// The maintenance refusal comes after every other check: with the claim + /// held, a create under a `server.region` `check_backup_arn_fits` + /// refuses and a restore whose throughput override does not fit the + /// backup's billing mode (`validate_override_for_source`) each answer + /// their own ValidationException, not LimitExceededException. + #[tokio::test] + async fn the_maintenance_refusal_comes_after_the_requests_own_checks() { + let fake = Fake::with_source(source(Some(definition("PAY_PER_REQUEST")))); + *fake.maintenance_held.lock().unwrap() = true; + let runtime = BackupRuntime::new(1).with_stale_job_after(Duration::from_secs(7)); + + let err = create_backup( + &fake, + &runtime, + ACCOUNT, + "US_EAST_1", + &LimitsConfig::default(), + "Music", + "nightly", + Ok(active_table("Music")), + ) + .await + .unwrap_err(); + assert!( + matches!(&err, DynamoDbError::ValidationException(m) if m.contains("server.region")), + "the region rule answers before the claim: {err:?}" + ); + + let body = restore_body(json!({ + "BillingModeOverride": "PAY_PER_REQUEST", + "ProvisionedThroughputOverride": { + "ReadCapacityUnits": 5, "WriteCapacityUnits": 5 + }, + })); + let err = restore_on(&fake, &runtime, &body).await.unwrap_err(); + assert!( + matches!(&err, DynamoDbError::ValidationException(m) + if m.contains("ProvisionedThroughputOverride can only be specified")), + "the override rule answers before the claim: {err:?}" + ); + assert!( + fake.maintenance_checks.lock().unwrap().is_empty(), + "the claim was not read for a request refused on its own terms" + ); + + // The same requests with their own faults removed are refused on the + // claim, so the claim is the next check. + let err = create_backup( + &fake, + &runtime, + ACCOUNT, + REGION, + &LimitsConfig::default(), + "Music", + "nightly", + Ok(active_table("Music")), + ) + .await + .unwrap_err(); + assert!(matches!(err, DynamoDbError::LimitExceededException(_))); + let err = restore_on(&fake, &runtime, &restore_body(json!({}))) + .await + .unwrap_err(); + assert!(matches!(err, DynamoDbError::LimitExceededException(_))); + assert_eq!(fake.maintenance_checks.lock().unwrap().len(), 2); + } + + /// `recheck` on a store startup settled: rows that arrive carrying + /// another id are an identity mismatch; a file deleted while rows exist + /// is `identity file missing` and is not rewritten; a file replaced with + /// another catalog's is `owned by another catalog`. Each runs through + /// the reconciler, as a running server runs it. + #[tokio::test] + async fn a_recheck_catches_rows_with_another_id_a_lost_file_and_a_foreign_file() { + let mem = Arc::new(MemStore::default()); + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + let fake = Arc::new(Fake::default()); + let runtime = runtime_on(&mem, store()); + let configured = runtime.configured_store().unwrap(); + assert_eq!(cycle_on(&fake, &runtime).await.store_healthy, Some(true)); + + // Rows of the root with another id arrive after startup. + let mut foreign_row = row(&arn(ACCOUNT), BackupStatus::Available); + foreign_row.store.store_id = "store-2".to_owned(); + mem.insert(&manifest_key(&arn(ACCOUNT)), b"{}"); + fake.rows.lock().unwrap().insert(arn(ACCOUNT), foreign_row); + assert_eq!(cycle_on(&fake, &runtime).await.store_healthy, Some(false)); + assert_eq!( + configured.health(), + StoreHealth::Unhealthy(StoreProblem::IdMismatch { + file_id: "store-1".to_owned(), + row_ids: vec!["store-2".to_owned()], + }) + ); + fake.rows.lock().unwrap().clear(); + assert_eq!(cycle_on(&fake, &runtime).await.store_healthy, Some(true)); + + // The file is deleted while rows exist: not rewritten. + fake.rows + .lock() + .unwrap() + .insert(arn(ACCOUNT), row(&arn(ACCOUNT), BackupStatus::Available)); + let puts_before = mem.puts.lock().unwrap().len(); + mem.objects.lock().unwrap().remove(STORE_IDENTITY_KEY); + assert_eq!(cycle_on(&fake, &runtime).await.store_healthy, Some(false)); + assert_eq!( + configured.health(), + StoreHealth::Unhealthy(StoreProblem::FileMissingWithRows(vec![ + "store-1".to_owned() + ])) + ); + assert_eq!( + mem.puts.lock().unwrap().len(), + puts_before, + "a root that lost its file while rows exist gets no new one" + ); + assert!(!mem.keys().contains(&STORE_IDENTITY_KEY.to_owned())); + assert_eq!( + runtime.store_health().unwrap().reason, + Some("identity file missing") + ); + + // The file is replaced with another catalog's. + mem.insert( + STORE_IDENTITY_KEY, + &identity_file("store-1", "other-catalog"), + ); + assert_eq!(cycle_on(&fake, &runtime).await.store_healthy, Some(false)); + assert_eq!( + configured.health(), + StoreHealth::Unhealthy(StoreProblem::ForeignCatalog { + file_catalog_id: "other-catalog".to_owned(), + catalog_id: "catalog-1".to_owned(), + }) + ); + // Restored, the store is healthy again. + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + assert_eq!(cycle_on(&fake, &runtime).await.store_healthy, Some(true)); + } + + /// The wait reads RUNNING restores by status, with no age: a record + /// stamped this instant (one the dead-job listing never returns, its + /// heartbeat being within any window) holds the wait, and the wait + /// reports it. + #[tokio::test(start_paused = true)] + async fn the_wait_sees_a_running_restore_with_a_fresh_heartbeat() { + let fake = Fake::default(); + let holder = Arc::new(FakeHolder::default()); + let claim = held(&holder); + let mut fresh = dead_job(RestoreKind::Legacy, "Fresh"); + fresh.heartbeat_at = seconds_since_epoch(); + fresh.status = JobStatus::Running; + fake.running_jobs.lock().unwrap().push(fresh); + let wait = Wait { + owner_window: Duration::from_secs(10), + limit: Duration::from_secs(4), + poll: Duration::from_secs(2), + }; + let (_tx, mut quiet) = Interrupt::new(); + let outcome = wait_for_quiet(&fake, &store(), &claim, wait, &mut quiet, |_| {}).await; + assert_eq!( + outcome, + WaitOutcome::Busy(vec![format!( + "a restore into Fresh (account {ACCOUNT}) is RUNNING" + )]) + ); + fake.running_jobs.lock().unwrap().clear(); + assert_eq!( + wait_for_quiet(&fake, &store(), &claim, wait, &mut quiet, |_| {}).await, + WaitOutcome::Quiet + ); + claim.release().await.unwrap(); + } + + /// The deletion loop stops before its next row, never inside one: once a + /// heartbeat has found the claim lost, and once a signal has been + /// received (here sent as the first row's deletion is reported, so the + /// loop stops before the second). A row left for a reason is skipped + /// and the loop goes on. + #[tokio::test(start_paused = true)] + async fn the_deletion_loop_stops_at_a_lost_claim_and_at_a_signal() { + let fake = Fake::default(); + let mem = MemStore::default(); + let arns: Vec = (1..=3).map(arn_n).collect(); + for arn in &arns { + seed(&fake, &mem, arn, BackupStatus::Available); + } + let rows: Vec = fake + .list_backup_rows(None, &store()) + .try_collect() + .await + .unwrap(); + assert_eq!(rows.len(), 3); + + // The claim lost before the loop: no row is deleted. + let holder = Arc::new(FakeHolder::default()); + let claim = held(&holder); + *holder.lost.lock().unwrap() = true; + tokio::time::sleep(Duration::from_millis(1_500)).await; + assert!(claim.lost().is_some()); + let (_tx, interrupt) = Interrupt::new(); + let mut reported = Vec::new(); + let outcome = purge_rows( + &fake, + &mem, + &passing_fence(), + &rows, + Duration::from_secs(300), + &claim, + &interrupt, + |row| reported.push(row.backup_arn.clone()), + ) + .await; + assert_eq!( + outcome, + PurgeOutcome { + deleted: vec![], + skipped: vec![], + stopped: Some(Stop::Lost( + "the maintenance claim was lost (another holder took it over)".to_owned() + )), + } + ); + assert!(reported.is_empty()); + assert_eq!(fake.rows.lock().unwrap().len(), 3, "no row was touched"); + assert_eq!(mem.keys().len(), 6, "no object was deleted"); + claim.release().await.unwrap(); + + // A signal received as the first row is reported stops the loop + // before the second; the third is untouched. + let holder = Arc::new(FakeHolder::default()); + let claim = held(&holder); + let (tx, interrupt) = Interrupt::new(); + let mut reported = Vec::new(); + let outcome = purge_rows( + &fake, + &mem, + &passing_fence(), + &rows, + Duration::from_secs(300), + &claim, + &interrupt, + |row| { + reported.push(row.backup_arn.clone()); + let _ = tx.send(Some("SIGTERM")); + }, + ) + .await; + assert_eq!( + outcome, + PurgeOutcome { + deleted: vec![arns[0].clone()], + skipped: vec![], + stopped: Some(Stop::Interrupted("SIGTERM")), + } + ); + assert_eq!(reported, vec![arns[0].clone()]); + let mut left: Vec = fake.rows.lock().unwrap().keys().cloned().collect(); + left.sort(); + assert_eq!(left, vec![arns[1].clone(), arns[2].clone()]); + assert_eq!(mem.keys().len(), 4); + + // Nothing stops it: the rest goes, a row left for a reason skipped. + fake.delete_starts + .lock() + .unwrap() + .insert(arns[1].clone(), DeleteStart::InUse("Restored".to_owned())); + let (_tx, interrupt) = Interrupt::new(); + let rest: Vec = rows[1..].to_vec(); + let outcome = purge_rows( + &fake, + &mem, + &passing_fence(), + &rest, + Duration::from_secs(300), + &claim, + &interrupt, + |_| {}, + ) + .await; + assert_eq!( + outcome, + PurgeOutcome { + deleted: vec![arns[2].clone()], + skipped: vec![format!( + "{}: a restore into Restored is reading it", + arns[1] + )], + stopped: None, + } + ); + claim.release().await.unwrap(); + } + + /// A defaulted root whose startup checks did not finish starts checking + /// with the rule kept: a first check that fails while the catalog has no + /// rows for the root sets the store aside (no component, no store), as + /// the startup failure would have; with rows for it the same failure is + /// an unhealthy configured store. + #[tokio::test] + async fn a_defaulted_root_set_aside_after_its_first_failed_check() { + let mem = Arc::new(MemStore::default()); + mem.insert( + STORE_IDENTITY_KEY, + &identity_file("store-1", "other-catalog"), + ); + let fake = Arc::new(Fake::default()); + let checking = || { + ConfiguredStore::checking( + store().store_kind, + store().store_root, + Arc::clone(&mem) as Arc, + None, + ) + .defaulted(true) + }; + let runtime = BackupRuntime::new(4).with_store(checking()); + assert!(runtime.configured_store().is_some()); + assert_eq!(runtime.store_health().unwrap().status, "checking"); + let report = cycle_on(&fake, &runtime).await; + assert_eq!( + report.store_healthy, None, + "the store is set aside, as none configured" + ); + assert!(runtime.configured_store().is_none()); + assert!(runtime.store_health().is_none()); + assert!(!runtime.is_configured_store(&store())); + // The next cycle runs no check: the store is unconfigured for good. + let puts = mem.puts.lock().unwrap().len(); + assert_eq!(cycle_on(&fake, &runtime).await.store_healthy, None); + assert_eq!(mem.puts.lock().unwrap().len(), puts); + + // With rows for the root, the same failure is an unhealthy store. + fake.rows + .lock() + .unwrap() + .insert(arn(ACCOUNT), row(&arn(ACCOUNT), BackupStatus::Available)); + let runtime = BackupRuntime::new(4).with_store(checking()); + let report = cycle_on(&fake, &runtime).await; + assert_eq!(report.store_healthy, Some(false)); + assert_eq!( + runtime.store_health().unwrap().reason, + Some("owned by another catalog") + ); + // A root that is not the defaulted one is never set aside. + let explicit = BackupRuntime::new(4).with_store(checking().defaulted(false)); + fake.rows.lock().unwrap().clear(); + assert_eq!(cycle_on(&fake, &explicit).await.store_healthy, Some(false)); + assert!(explicit.configured_store().is_some()); + } + + /// The deletions run under the store's bound and the in-flight rule: + /// DeleteBackup on a stalled store answers DELETED at the bound with the + /// row left DELETING, two DeleteBackups together park one thread between + /// them (the second waits for the first's slot and answers at its bound + /// with nothing parked), and the reconciler's deletion of those rows + /// returns at the bound too, with no second call parked beside the first. + #[tokio::test(start_paused = true)] + async fn the_deletions_run_under_the_stores_bound() { + let mem = Arc::new(MemStore::default()); + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + mem.insert(&manifest_key(&arn(ACCOUNT)), b"{}"); + mem.insert(&manifest_key(&newer_arn(ACCOUNT)), b"{}"); + let fake = fake_with_rows(&[ + (arn(ACCOUNT), BackupStatus::Available), + (newer_arn(ACCOUNT), BackupStatus::Available), + ]); + let runtime = runtime_on(&mem, store()); + let configured = runtime.configured_store().unwrap(); + mem.set_stalled(true); + let started = tokio::time::Instant::now(); + let (first, second) = tokio::time::timeout( + Duration::from_secs(120), + futures::future::join( + delete_backup(&*fake, &runtime, ACCOUNT, REGION, &own(&arn(ACCOUNT))), + delete_backup(&*fake, &runtime, ACCOUNT, REGION, &own(&newer_arn(ACCOUNT))), + ), + ) + .await + .expect("both deletes return at the store's bound"); + assert_eq!(started.elapsed(), LOCAL_CHECK_TIMEOUT); + assert_eq!(first.unwrap().backup_details.backup_status, "DELETED"); + assert_eq!(second.unwrap().backup_details.backup_status, "DELETED"); + for row in [arn(ACCOUNT), newer_arn(ACCOUNT)] { + assert_eq!( + fake.rows.lock().unwrap()[&row].status, + BackupStatus::Deleting, + "the row is left for the reconciler" + ); + } + assert!( + configured.check_call_in_flight(), + "the abandoned call is parked" + ); + assert_eq!( + mem.stalls.load(std::sync::atomic::Ordering::Acquire), + 1, + "the second delete's call waited for the first's slot and parked nothing" + ); + + // The reconciler's cycle: the check is refused at once (a call is + // parked), so no deletion runs and nothing new is parked. + let started = tokio::time::Instant::now(); + let report = cycle_on(&fake, &runtime).await; + assert_eq!(started.elapsed(), Duration::ZERO); + assert_eq!(report.store_healthy, Some(false)); + assert!(report.deletions_finished.is_empty()); + assert_eq!(mem.stalls.load(std::sync::atomic::Ordering::Acquire), 1); + + // The mount answers: the parked call returns, the next cycle checks + // and finishes the deletions. + mem.set_stalled(false); + tokio::time::sleep(Duration::from_millis(50)).await; + assert!(!configured.check_call_in_flight()); + let report = cycle_on(&fake, &runtime).await; + assert_eq!(report.store_healthy, Some(true)); + let mut finished = report.deletions_finished.clone(); + finished.sort(); + let mut expected = [arn(ACCOUNT), newer_arn(ACCOUNT)]; + expected.sort(); + assert_eq!(finished, expected); + assert_eq!(mem.keys(), [STORE_IDENTITY_KEY]); + } + + /// A call's bound starts once it holds the root's slot: on a healthy + /// root whose calls take 3 s under a 5 s bound, two DeleteBackups + /// together both finish (the second waits 3 s for the slot and then has + /// the bound for its own 3 s call), both rows reach DELETED, nothing is + /// marked in flight and the next check finds the store healthy. With one + /// bound over the wait and the call the second was abandoned at 5 s, its + /// row left DELETING, and the next check marked the store unreachable. + #[tokio::test(start_paused = true)] + async fn a_calls_bound_starts_when_it_holds_the_slot() { + let mem = Arc::new(MemStore::default()); + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + mem.insert(&manifest_key(&arn(ACCOUNT)), b"{}"); + mem.insert(&manifest_key(&newer_arn(ACCOUNT)), b"{}"); + let fake = fake_with_rows(&[ + (arn(ACCOUNT), BackupStatus::Available), + (newer_arn(ACCOUNT), BackupStatus::Available), + ]); + let runtime = runtime_on(&mem, store()); + let configured = runtime.configured_store().unwrap(); + mem.set_delay(Some(Duration::from_secs(3))); + let started = tokio::time::Instant::now(); + let (first, second) = futures::future::join( + delete_backup(&*fake, &runtime, ACCOUNT, REGION, &own(&arn(ACCOUNT))), + delete_backup(&*fake, &runtime, ACCOUNT, REGION, &own(&newer_arn(ACCOUNT))), + ) + .await; + assert_eq!(first.unwrap().backup_details.backup_status, "DELETED"); + assert_eq!(second.unwrap().backup_details.backup_status, "DELETED"); + assert!( + started.elapsed() > LOCAL_CHECK_TIMEOUT, + "the second call ran after the first's 3 s: {:?}", + started.elapsed() + ); + for row in [arn(ACCOUNT), newer_arn(ACCOUNT)] { + assert_eq!( + fake.rows.lock().unwrap()[&row].status, + BackupStatus::Deleted, + "both deletions finished" + ); + } + assert!(!configured.check_call_in_flight(), "nothing abandoned"); + mem.set_delay(None); + let report = cycle_on(&fake, &runtime).await; + assert_eq!(report.store_healthy, Some(true)); + } + + /// The manifest probe reads one row (`newest_available_backup_row`) and + /// not the listing: the catalog double records the read, and the probe + /// heads the newest AVAILABLE row's manifest. + #[tokio::test] + async fn the_manifest_probe_reads_one_row() { + let mem = Arc::new(MemStore::default()); + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + mem.insert(&manifest_key(&arn(ACCOUNT)), b"{}"); + let fake = fake_with_rows(&[ + (arn(ACCOUNT), BackupStatus::Available), + (newer_arn(ACCOUNT), BackupStatus::Available), + ]); + let runtime = runtime_on(&mem, store()); + let report = cycle_on(&fake, &runtime).await; + assert_eq!(report.store_healthy, Some(false)); + assert_eq!( + runtime.configured_store().unwrap().health(), + StoreHealth::Unhealthy(StoreProblem::ManifestMissing { + backup_arn: newer_arn(ACCOUNT), + }), + "the newest row's manifest is the one probed" + ); + assert_eq!(fake.newest_reads.lock().unwrap().as_slice(), [store()]); + mem.insert(&manifest_key(&newer_arn(ACCOUNT)), b"{}"); + assert_eq!(cycle_on(&fake, &runtime).await.store_healthy, Some(true)); + } +} + +// --- the store-backed create and its job ------------------------------------ + +/// The `Music` table as a snapshot describes it: composite key, one GSI, +/// provisioned. +fn music_table() -> TableDescription { + TableDescription { + table_name: "Music".to_owned(), + key_schema: vec![ + key("Artist", KeyType::Hash), + key("SongTitle", KeyType::Range), + ], + attribute_definitions: vec![attr("Artist"), attr("SongTitle"), attr("Genre")], + table_status: TableStatus::Active, + creation_date_time: 1_757_400_000.0, + table_size_bytes: 123_456, + item_count: 100_000, + table_arn: source_arn(), + table_id: "12345a12-abcd-123a-ab12-1234abc12345".to_owned(), + provisioned_throughput: ProvisionedThroughputDescription { + read_capacity_units: 5, + write_capacity_units: 5, + ..Default::default() + }, + billing_mode_summary: Some(BillingModeSummary { + billing_mode: BillingMode::Provisioned, + last_update_to_pay_per_request_date_time: None, + }), + global_secondary_indexes: Some(vec![GsiDescription { + index_name: "GenreIndex".to_owned(), + key_schema: vec![key("Genre", KeyType::Hash)], + projection: Projection { + projection_type: ProjectionType::All, + non_key_attributes: None, + }, + index_status: "ACTIVE".to_owned(), + provisioned_throughput: Some(ProvisionedThroughputDescription { + read_capacity_units: 1, + write_capacity_units: 1, + ..Default::default() + }), + index_size_bytes: 0, + item_count: 0, + index_arn: String::new(), + }]), + ..Default::default() + } +} + +/// A vector index description for a table with vector support. +fn vector_index() -> extenddb_core::types::VectorIndexDescription { + serde_json::from_value(json!({ + "IndexName": "v", + "VectorAttribute": { "AttributeName": "emb" }, + "Dimensions": 3, + "DistanceFunction": "COSINE", + "IndexStatus": "ACTIVE", + "IndexSizeBytes": 0, + "ItemCount": 0, + "IndexArn": "" + })) + .unwrap() +} + +/// An item of the `Music` table with a `Year` number. +fn song(artist: &str, title: &str) -> Item { + use extenddb_core::types::AttributeValue; + Item::from([ + ("Artist".to_owned(), AttributeValue::S(artist.to_owned())), + ("SongTitle".to_owned(), AttributeValue::S(title.to_owned())), + ("Year".to_owned(), AttributeValue::N("1999".to_owned())), + ]) +} + +/// `song` with a `Year` the wire refuses (#618's shape). +fn song_with_bad_number(artist: &str, title: &str, year: &str) -> Item { + use extenddb_core::types::AttributeValue; + let mut item = song(artist, title); + item.insert("Year".to_owned(), AttributeValue::N(year.to_owned())); + item +} + +/// `n` songs in key order. +fn songs(n: usize) -> Vec { + (0..n) + .map(|i| song(&format!("artist-{i:05}"), &format!("title-{i:05}"))) + .collect() +} + +/// Make `fake` serve a snapshot of `table` yielding `items` (through `gate` +/// when given). +fn snapshot_fixture( + fake: &Fake, + table: &TableDescription, + items: Vec, + key_ordered: bool, + gate: Option>, +) { + fake.snapshots.lock().unwrap().insert( + table.table_name.clone(), + SnapshotFixture { + description: table.clone(), + key_ordered, + items: items.into_iter().map(Ok).collect(), + gate, + }, + ); +} + +/// A gate with no permit: the stream yields nothing until the test adds +/// some. +fn closed_gate() -> Arc { + Arc::new(tokio::sync::Semaphore::new(0)) +} + +/// The settings the job tests run with: small files, a short heartbeat. +fn job_settings() -> JobSettings { + JobSettings { + data_file_target_bytes: 64 * 1024 * 1024, + heartbeat_interval: Duration::from_millis(50), + stale_job_after: Duration::from_secs(300), + failed_row_retention: Duration::from_secs(7 * 24 * 3600), + } +} + +/// A healthy runtime on `mem` with the job settings above and a collector. +fn job_runtime(mem: &Arc) -> BackupRuntime { + runtime_on(mem, store()) + .with_job_settings(job_settings()) + .with_metrics(Arc::new(extenddb_core::metrics::MetricsCollector::new())) +} + +/// CreateBackup of `Music` through `runtime`. +async fn create_music_backup( + fake: &Arc, + runtime: &BackupRuntime, +) -> Result { + create_backup( + fake, + runtime, + ACCOUNT, + REGION, + &LimitsConfig::default(), + "Music", + "nightly", + Ok(music_table()), + ) + .await +} + +/// Wait until no job task of `runtime` runs, failing after 30 s: a hang +/// guard for jobs that take well under a second alone. A test whose job +/// takes seconds calls [`wait_for_jobs_within`] with a bound at least ten +/// times its time alone on an idle core, so a loaded host cannot reach it. +async fn wait_for_jobs(runtime: &BackupRuntime) { + wait_for_jobs_within(runtime, Duration::from_secs(30)).await; +} + +/// Wait until no job task of `runtime` runs, failing after `bound`. +async fn wait_for_jobs_within(runtime: &BackupRuntime, bound: Duration) { + let deadline = tokio::time::Instant::now() + bound; + while runtime.running_jobs() > 0 { + assert!( + tokio::time::Instant::now() < deadline, + "a job did not end within {bound:?}" + ); + tokio::time::sleep(Duration::from_millis(5)).await; + } +} + +/// The objects under the backup's prefix, by key. +fn objects_under(mem: &MemStore, arn: &str) -> Vec { + let prefix = BackupArn::parse(arn).unwrap().store_prefix(); + mem.keys() + .into_iter() + .filter(|k| k.starts_with(&prefix)) + .collect() +} + +/// The stored manifest of `arn`, parsed. +fn stored_manifest(mem: &MemStore, arn: &str) -> extenddb_core::backup_format::ExtenddbManifest { + let bytes = mem.objects.lock().unwrap()[&manifest_key(arn)].clone(); + extenddb_core::backup_format::ExtenddbManifest::from_json_bytes(&bytes).unwrap() +} + +/// The status writes the fake's claims received, as `(to, failure_reason)`. +fn writes(fake: &Fake) -> Vec<(JobStatus, Option)> { + fake.status_writes + .lock() + .unwrap() + .iter() + .map(|w| (w.to, w.fields.failure_reason.clone())) + .collect() +} + +/// The `BackupJobs` outcomes the collector drained, with their count. +fn job_outcomes(runtime: &BackupRuntime) -> Vec<(String, f64)> { + use extenddb_core::metrics::MetricName; + let mut outcomes: Vec<(String, f64)> = runtime + .metrics() + .unwrap() + .drain(Duration::ZERO) + .into_iter() + .filter(|b| b.metric == MetricName::BackupJobs) + .map(|b| (b.operation, b.sum)) + .collect(); + outcomes.sort_by(|a, b| a.0.cmp(&b.0)); + outcomes +} + +/// The request's order and the job's happy path. The checks before the +/// permit: a missing table is TableNotFound with every slot held and the +/// store untouched. Then the detached task: the snapshot is opened, the row +/// is inserted CREATING under the claim with the manifest's `source_table` +/// (the TTL read beside the open), the maintenance claim is read, and the +/// request is answered CREATING with the id's instant. The job then writes +/// the data file, `manifest-files.json`, `manifest-summary.json`, the +/// sidecar and the manifest, in that order and nothing else, and moves the +/// row to AVAILABLE with the manifest's size, the stream's count and the +/// manifest's `source_table`; the ARN's id, the row's `created_at`, the +/// response's time and the manifest's `backup_creation_date_time` are one +/// instant; the data file decodes back to the items; the slot is freed; the +/// metrics name the outcome, and `BackupSnapshotAge` was recorded by the +/// beats the job's three periods inside the stream gave it. +#[tokio::test] +async fn a_store_backed_create_answers_creating_and_its_job_writes_the_backup() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let items = songs(5); + let gate = closed_gate(); + snapshot_fixture( + &fake, + &music_table(), + items.clone(), + true, + Some(Arc::clone(&gate)), + ); + *fake.ttl.lock().unwrap() = Some(TimeToLiveDescription { + time_to_live_status: TimeToLiveStatus::Enabled, + attribute_name: Some("expires_at".to_owned()), + }); + let full = job_runtime(&mem); + let mut _held = Vec::new(); + for _ in 0..4 { + _held.push(full.take_permit().await.unwrap()); + } + let err = create_backup( + &fake, + &full, + ACCOUNT, + REGION, + &LimitsConfig::default(), + "Music", + "nightly", + Err(StorageError::TableNotFound("Music".to_owned())), + ) + .await + .unwrap_err(); + assert!( + matches!(err, DynamoDbError::TableNotFoundException(_)), + "{err:?}" + ); + let err = create_music_backup(&fake, &full).await.unwrap_err(); + assert!( + matches!(err, DynamoDbError::LimitExceededException(_)), + "{err:?}" + ); + assert!( + fake.snapshots_opened.lock().unwrap().is_empty(), + "nothing was opened" + ); + drop(_held); + + let runtime = job_runtime(&mem); + let details = create_music_backup(&fake, &runtime).await.unwrap(); + assert_eq!(details.backup_status, "CREATING"); + assert_eq!(details.backup_type, "USER"); + assert_eq!(details.backup_name, "nightly"); + assert_eq!(details.backup_size_bytes, 0); + let arn = BackupArn::parse(&details.backup_arn).unwrap(); + assert_eq!( + extenddb_storage::util::backup_creation_date_time(&details.backup_arn), + Some(details.backup_creation_date_time), + "the response's time is the id's instant" + ); + let claimed = fake.claims.lock().unwrap().clone(); + assert_eq!(claimed.len(), 1); + assert_eq!(claimed[0].status, BackupStatus::Creating); + assert_eq!(claimed[0].store, store()); + assert_eq!(claimed[0].created_at, details.backup_creation_date_time); + assert_eq!(claimed[0].item_count, 0); + assert_eq!(claimed[0].source_table_details_json["item_count"], 100_000); + assert_eq!( + claimed[0].source_table_details_json["time_to_live"]["AttributeName"], + "expires_at" + ); + assert_eq!( + fake.maintenance_checks.lock().unwrap().as_slice(), + [Duration::from_secs(300), Duration::from_secs(300)], + "the maintenance claim is read before the permit, by the refused request too" + ); + + // Three heartbeat periods inside the stream, so the timer beats before + // the job ends. + tokio::time::sleep(3 * job_settings().heartbeat_interval).await; + gate.add_permits(items.len()); + wait_for_jobs(&runtime).await; + assert_eq!(runtime.free_slots(), 4, "the job freed its slot"); + let prefix = arn.store_prefix(); + assert_eq!( + mem.puts.lock().unwrap().as_slice(), + [ + format!("{prefix}data/000001.json.gz"), + format!("{prefix}manifest-files.json"), + format!("{prefix}manifest-summary.json"), + format!("{prefix}extenddb-manifest.sha256"), + format!("{prefix}extenddb-manifest.json"), + ], + "the write order" + ); + assert!(mem.deletes.lock().unwrap().is_empty()); + let manifest = stored_manifest(&mem, &details.backup_arn); + assert_eq!(manifest.backup_arn, details.backup_arn); + assert_eq!( + manifest.backup_creation_date_time, + details.backup_creation_date_time + ); + assert_eq!(manifest.source_table.item_count, 5, "the stream's count"); + assert_eq!( + manifest.source_table.table_size_bytes, + items + .iter() + .map(|item| extenddb_core::types::item_size_bytes(item) as i64) + .sum::(), + "the stream's size, not the stats pass's 123,456" + ); + assert_eq!(manifest.source_backend, "test"); + assert_eq!( + manifest.item_order, + extenddb_core::backup_format::ItemOrder::KeyOrdered + ); + assert_eq!(manifest.data_files.len(), 1); + assert!(manifest.data_files[0].key_range.is_some()); + let file = mem.objects.lock().unwrap()[&format!("{prefix}data/000001.json.gz")].clone(); + assert_eq!(manifest.data_files[0].bytes, file.len() as u64); + let mut reader = manifest.reader(&manifest.data_files[0], &file[..]).unwrap(); + let mut decoded = Vec::new(); + while let Some(item) = reader.next_item().unwrap() { + decoded.push(item); + } + assert_eq!(decoded, items); + let manifest_bytes = + mem.objects.lock().unwrap()[&format!("{prefix}extenddb-manifest.json")].clone(); + let sidecar = mem.objects.lock().unwrap()[&format!("{prefix}extenddb-manifest.sha256")].clone(); + extenddb_core::backup_format::verify_manifest_checksum(&manifest_bytes, &sidecar).unwrap(); + + let row = fake.rows.lock().unwrap()[&details.backup_arn].clone(); + assert_eq!(row.status, BackupStatus::Available); + assert_eq!(row.size_bytes, manifest.backup_size_bytes); + assert_eq!(row.item_count, 5); + assert_eq!( + row.source_table_details_json, + serde_json::to_value(&manifest.source_table).unwrap() + ); + assert!(row.completed_at.is_some()); + assert_eq!(writes(&fake), [(JobStatus::Available, None)]); + let write = fake.status_writes.lock().unwrap()[0].clone(); + assert_eq!( + (write.arn.as_str(), write.from, write.generation), + ( + details.backup_arn.as_str(), + JobStatus::Creating, + row.owner_generation + ), + "every status write is conditioned on CREATING under the claim's generation" + ); + assert!(fake.heartbeats.load(std::sync::atomic::Ordering::Acquire) >= 1); + + use extenddb_core::metrics::MetricName; + let drained = runtime.metrics().unwrap().drain(Duration::ZERO); + let find = |metric: MetricName| { + drained + .iter() + .filter(|b| b.metric == metric) + .map(|b| (b.operation.clone(), b.sum)) + .collect::>() + }; + assert_eq!( + find(MetricName::BackupJobs), + [("succeeded".to_owned(), 1.0)] + ); + assert_eq!(find(MetricName::BackupJobDuration).len(), 1); + let total: u64 = mem.put_sizes.lock().unwrap().iter().map(|(_, n)| n).sum(); + assert_eq!( + find(MetricName::BackupBytesUploaded), + [(String::new(), total as f64)] + ); + assert_eq!( + find(MetricName::WorkerLastSuccess) + .iter() + .map(|(op, _)| op.as_str()) + .collect::>(), + ["BackupJob"] + ); + assert!(find(MetricName::WorkerErrorCount).is_empty()); + let beats = fake.heartbeats.load(std::sync::atomic::Ordering::Acquire); + let ages: usize = drained + .iter() + .filter(|b| b.metric == MetricName::BackupSnapshotAge) + .map(|b| b.count as usize) + .sum(); + assert!( + ages >= 1 && ages < beats, + "the snapshot's age is recorded at each confirmed timer beat and not at the re-check \ + before the manifest: {ages} recorded, {beats} beats" + ); +} + +/// The maintenance claim is checked before the permit, as on the +/// in-catalog path: held, the request is LimitExceeded naming a backup +/// maintenance command and the claim, no snapshot is opened, no row is +/// inserted (a FAILED `maintenance` row carried the insert's heartbeat and +/// `backup purge`'s wait counted it as a live job), nothing reaches the +/// store and every slot is free. A claim that cannot be read answers the +/// read's error, with no row either. +#[tokio::test] +async fn a_store_backed_create_under_a_maintenance_claim_is_refused_before_its_row() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + snapshot_fixture(&fake, &music_table(), songs(3), true, None); + *fake.maintenance_held.lock().unwrap() = true; + let runtime = job_runtime(&mem); + let err = create_music_backup(&fake, &runtime).await.unwrap_err(); + assert!( + matches!(&err, DynamoDbError::LimitExceededException(m) + if m.contains("A backup maintenance command holds") && !m.contains("purge")), + "{err:?}" + ); + wait_for_jobs(&runtime).await; + assert!(fake.snapshots_opened.lock().unwrap().is_empty()); + assert!(fake.rows.lock().unwrap().is_empty(), "no row is inserted"); + assert!(fake.claims.lock().unwrap().is_empty()); + assert!(mem.puts.lock().unwrap().is_empty()); + assert!(mem.deletes.lock().unwrap().is_empty()); + assert_eq!(runtime.free_slots(), 4); + assert_eq!(job_outcomes(&runtime), []); + + *fake.maintenance_held.lock().unwrap() = false; + *fake.maintenance_error.lock().unwrap() = Some("the claim table is locked".to_owned()); + let err = create_music_backup(&fake, &runtime).await.unwrap_err(); + assert!( + matches!(&err, DynamoDbError::InternalServerError(_)), + "{err:?}" + ); + assert!(fake.rows.lock().unwrap().is_empty(), "no row is inserted"); + assert_eq!(runtime.free_slots(), 4); +} + +/// A snapshot the backend refuses to open answers the request with the +/// open's error (here the backend's bound, LimitExceeded) and leaves no row +/// and no object; the slot is freed. +#[tokio::test] +async fn an_open_that_fails_answers_the_request_and_leaves_nothing() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + *fake.snapshot_error.lock().unwrap() = + Some(StorageError::LimitExceeded("too many snapshots".to_owned())); + let runtime = job_runtime(&mem); + let err = create_music_backup(&fake, &runtime).await.unwrap_err(); + assert!( + matches!(&err, DynamoDbError::LimitExceededException(m) if m == "too many snapshots"), + "{err:?}" + ); + wait_for_jobs(&runtime).await; + assert!(fake.rows.lock().unwrap().is_empty()); + assert!(fake.claims.lock().unwrap().is_empty()); + assert_eq!( + mem.keys(), + [STORE_IDENTITY_KEY], + "no object but the identity file" + ); + assert_eq!(runtime.free_slots(), 4); +} + +/// The file close rule: a file closes at the first item that takes its +/// uncompressed bytes to or past `data_file_target_bytes` (a knob of exactly +/// three lines closes at the third), so every file but the last is at least +/// the knob and under the knob plus one line, and the items are split across +/// files in order. At the knob's minimum (32 MiB) with 5 MiB items, each +/// file takes the items until the sum passes 32 MiB. +#[tokio::test] +async fn a_data_file_closes_at_the_first_item_that_reaches_the_knob() { + use extenddb_core::types::AttributeValue; + let line_bytes = |item: &Item| { + extenddb_core::backup_format::encode_item_line(item) + .unwrap() + .len() as u64 + }; + + // A low knob and small items: the exact rule. + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let items = songs(10); + let line = line_bytes(&items[0]); + let target = 3 * line; + snapshot_fixture(&fake, &music_table(), items.clone(), true, None); + let runtime = job_runtime(&mem).with_job_settings(JobSettings { + data_file_target_bytes: target, + ..job_settings() + }); + let details = create_music_backup(&fake, &runtime).await.unwrap(); + wait_for_jobs(&runtime).await; + { + let row = fake.rows.lock().unwrap()[&details.backup_arn].clone(); + assert_eq!( + (row.status, row.failure_reason.clone()), + (BackupStatus::Available, None), + "{:?}", + mem.puts.lock().unwrap() + ); + } + let manifest = stored_manifest(&mem, &details.backup_arn); + let counts: Vec = manifest.data_files.iter().map(|f| f.item_count).collect(); + assert_eq!( + counts, + [3, 3, 3, 1], + "the third line takes the file to the knob, so it closes there" + ); + for file in &manifest.data_files[..3] { + assert!(file.uncompressed_bytes >= target); + assert!(file.uncompressed_bytes - line < target); + } + assert_eq!( + manifest + .data_files + .iter() + .map(|f| f.key.as_str()) + .collect::>(), + [ + "data/000001.json.gz", + "data/000002.json.gz", + "data/000003.json.gz", + "data/000004.json.gz" + ] + ); + let prefix = BackupArn::parse(&details.backup_arn) + .unwrap() + .store_prefix(); + let mut decoded = Vec::new(); + for file in &manifest.data_files { + let bytes = mem.objects.lock().unwrap()[&format!("{prefix}{}", file.key)].clone(); + let mut reader = manifest.reader(file, &bytes[..]).unwrap(); + while let Some(item) = reader.next_item().unwrap() { + decoded.push(item); + } + } + assert_eq!(decoded, items); + assert_eq!( + fake.rows.lock().unwrap()[&details.backup_arn].item_count, + 10 + ); + + // The knob's minimum, 32 MiB, with 5 MiB binary items (a 6.7 MiB + // line each): five lines pass 32 MiB, then three. + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let items: Vec = (0..8) + .map(|i| { + let mut item = song("a", &format!("t{i}")); + item.insert( + "blob".to_owned(), + AttributeValue::B(vec![0u8; 5 * 1024 * 1024]), + ); + item + }) + .collect(); + let line = line_bytes(&items[0]); + snapshot_fixture(&fake, &music_table(), items, true, None); + let runtime = job_runtime(&mem).with_job_settings(JobSettings { + data_file_target_bytes: 32 * 1024 * 1024, + ..job_settings() + }); + let limits = LimitsConfig { + max_item_size_bytes: 8 * 1024 * 1024, + ..LimitsConfig::default() + }; + let details = create_backup( + &fake, + &runtime, + ACCOUNT, + REGION, + &limits, + "Music", + "nightly", + Ok(music_table()), + ) + .await + .unwrap(); + // The whole test takes about 5 s alone on an idle core (eight 6.7 MiB + // lines encoded and compressed into two files), so the guard is 120 s. + wait_for_jobs_within(&runtime, Duration::from_secs(120)).await; + assert_eq!( + fake.rows.lock().unwrap()[&details.backup_arn].failure_reason, + None + ); + let manifest = stored_manifest(&mem, &details.backup_arn); + let counts: Vec = manifest.data_files.iter().map(|f| f.item_count).collect(); + assert_eq!(counts, [5, 3]); + assert!(manifest.data_files[0].uncompressed_bytes >= 32 * 1024 * 1024); + assert!(manifest.data_files[0].uncompressed_bytes - line < 32 * 1024 * 1024); +} + +/// An unordered snapshot writes `unordered` with no key ranges; an ordered +/// one `key_ordered` with a range per file. +#[tokio::test] +async fn the_manifest_records_the_snapshots_order() { + for key_ordered in [false, true] { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + snapshot_fixture(&fake, &music_table(), songs(3), key_ordered, None); + let runtime = job_runtime(&mem); + let details = create_music_backup(&fake, &runtime).await.unwrap(); + wait_for_jobs(&runtime).await; + let manifest = stored_manifest(&mem, &details.backup_arn); + let expected = if key_ordered { + extenddb_core::backup_format::ItemOrder::KeyOrdered + } else { + extenddb_core::backup_format::ItemOrder::Unordered + }; + assert_eq!(manifest.item_order, expected); + assert_eq!(manifest.data_files[0].key_range.is_some(), key_ordered); + } +} + +/// A snapshot stream that fails after a refused item: the FAILED reason +/// carries both, the refusals' count and first item and then the stream's +/// error, so the item the operator is told to find is not lost behind the +/// read failure. Nothing is uploaded and the prefix is deleted. +#[tokio::test] +async fn a_stream_error_after_a_refusal_keeps_the_refusal_in_the_reason() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let items = vec![ + song("a", "1"), + song_with_bad_number("a", "2", "1E126"), + song_with_bad_number("a", "3", "abc"), + ]; + snapshot_fixture(&fake, &music_table(), items, true, None); + fake.snapshots + .lock() + .unwrap() + .get_mut("Music") + .unwrap() + .items + .push(Err(StorageError::Internal( + "the page read failed".to_owned(), + ))); + let runtime = job_runtime(&mem); + let details = create_music_backup(&fake, &runtime).await.unwrap(); + wait_for_jobs(&runtime).await; + let row = fake.rows.lock().unwrap()[&details.backup_arn].clone(); + assert_eq!(row.status, BackupStatus::Failed); + let reason = row.failure_reason.unwrap(); + assert!( + reason.starts_with( + "2 item(s) cannot be restored; first: key {\"Artist\":{\"S\":\"a\"},\"SongTitle\":{\"S\":\"2\"}}, \ + attribute Year: " + ), + "{reason}" + ); + assert!( + reason.ends_with(" (#618); the snapshot then failed: Internal error: the page read failed"), + "{reason}" + ); + assert!(mem.puts.lock().unwrap().is_empty()); + assert!(objects_under(&mem, &details.backup_arn).is_empty()); + assert_eq!(job_outcomes(&runtime), [("failed".to_owned(), 1.0)]); + + // The stream's error alone keeps its own reason. + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + snapshot_fixture(&fake, &music_table(), songs(1), true, None); + fake.snapshots + .lock() + .unwrap() + .get_mut("Music") + .unwrap() + .items + .push(Err(StorageError::Internal( + "the page read failed".to_owned(), + ))); + let runtime = job_runtime(&mem); + let details = create_music_backup(&fake, &runtime).await.unwrap(); + wait_for_jobs(&runtime).await; + assert_eq!( + fake.rows.lock().unwrap()[&details.backup_arn] + .failure_reason + .as_deref(), + Some("reading the snapshot: Internal error: the page read failed") + ); +} + +/// The validator's rule: with three items the wire let in, the job reads +/// and checks the whole stream past the first refusal (every gate permit is +/// taken), encodes nothing more, and fails the row with the count and the +/// first item, naming #618 for an invalid number; the FAILED write comes +/// first and the prefix is deleted after it. The stream is fed one item +/// every 20 ms, slower than the pipeline, so the encoder's stop lands +/// before the stream's end and an uploader that read that stop as its own +/// failure would end the feeder with the rest unchecked. An item refused +/// for its size gives a reason that does not name #618. +#[tokio::test] +async fn the_validator_fails_the_backup_after_the_rest_is_validated() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let items = vec![ + song("a", "1"), + song_with_bad_number("a", "2", "1E126"), + song("a", "3"), + song_with_bad_number("a", "4", "abc"), + song_with_bad_number("a", "5", "1E126"), + song("a", "6"), + ]; + let count = items.len(); + let gate = closed_gate(); + snapshot_fixture(&fake, &music_table(), items, true, Some(Arc::clone(&gate))); + let runtime = job_runtime(&mem); + let details = create_music_backup(&fake, &runtime).await.unwrap(); + let feed = tokio::spawn({ + let gate = Arc::clone(&gate); + async move { + for _ in 0..count { + tokio::time::sleep(Duration::from_millis(20)).await; + gate.add_permits(1); + } + } + }); + wait_for_jobs(&runtime).await; + feed.await.unwrap(); + assert_eq!(gate.available_permits(), 0, "every item was read"); + let row = fake.rows.lock().unwrap()[&details.backup_arn].clone(); + assert_eq!(row.status, BackupStatus::Failed); + let reason = row.failure_reason.unwrap(); + assert!( + reason.starts_with( + "3 item(s) cannot be restored; first: key {\"Artist\":{\"S\":\"a\"},\"SongTitle\":{\"S\":\"2\"}}, \ + attribute Year: " + ), + "{reason}" + ); + assert!(reason.ends_with(" (#618)"), "{reason}"); + let prefix = BackupArn::parse(&details.backup_arn) + .unwrap() + .store_prefix(); + assert_eq!(writes(&fake).len(), 1); + assert_eq!(writes(&fake)[0].0, JobStatus::Failed); + assert_eq!( + mem.deletes.lock().unwrap().as_slice(), + std::slice::from_ref(&prefix) + ); + assert!(objects_under(&mem, &details.backup_arn).is_empty()); + assert!( + mem.puts.lock().unwrap().is_empty(), + "after the first refused item nothing is uploaded, the first file with its one valid \ + item included: {:?}", + mem.puts.lock().unwrap() + ); + assert_eq!(job_outcomes(&runtime), [("failed".to_owned(), 1.0)]); + + // Over `max_item_size_bytes`: the reason names the size rule, not #618. + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + snapshot_fixture(&fake, &music_table(), songs(2), true, None); + let runtime = job_runtime(&mem); + let limits = LimitsConfig { + max_item_size_bytes: 20, + ..LimitsConfig::default() + }; + let details = create_backup( + &fake, + &runtime, + ACCOUNT, + REGION, + &limits, + "Music", + "nightly", + Ok(music_table()), + ) + .await + .unwrap(); + wait_for_jobs(&runtime).await; + let row = fake.rows.lock().unwrap()[&details.backup_arn].clone(); + assert_eq!(row.status, BackupStatus::Failed); + let reason = row.failure_reason.unwrap(); + assert!( + reason.starts_with("2 item(s) cannot be restored; first: key "), + "{reason}" + ); + assert!(reason.contains("20 bytes"), "{reason}"); + assert!(!reason.contains("#618"), "{reason}"); +} + +/// A put that fails fails the row with the file named, and only then +/// deletes the prefix; the store error is counted. +#[tokio::test] +async fn a_put_failure_fails_the_row_then_deletes_the_prefix() { + let mem = Arc::new(MemStore::default()); + mem.fail_puts + .store(true, std::sync::atomic::Ordering::Release); + let fake = Arc::new(Fake::default()); + snapshot_fixture(&fake, &music_table(), songs(3), true, None); + let runtime = job_runtime(&mem); + let details = create_music_backup(&fake, &runtime).await.unwrap(); + wait_for_jobs(&runtime).await; + let row = fake.rows.lock().unwrap()[&details.backup_arn].clone(); + assert_eq!(row.status, BackupStatus::Failed); + assert!( + row.failure_reason + .as_deref() + .unwrap() + .starts_with("writing data/000001.json.gz: "), + "{:?}", + row.failure_reason + ); + let prefix = BackupArn::parse(&details.backup_arn) + .unwrap() + .store_prefix(); + assert_eq!( + writes(&fake), + [(JobStatus::Failed, row.failure_reason.clone())] + ); + assert_eq!(mem.deletes.lock().unwrap().as_slice(), [prefix]); + use extenddb_core::metrics::MetricName; + let drained = runtime.metrics().unwrap().drain(Duration::ZERO); + assert!(drained.iter().any(|b| { + b.metric == MetricName::BackupStoreErrors && b.operation == "put" && b.sum == 1.0 + })); + assert!( + drained + .iter() + .any(|b| { b.metric == MetricName::BackupJobs && b.operation == "failed" }) + ); + assert!( + drained + .iter() + .any(|b| { b.metric == MetricName::WorkerErrorCount && b.operation == "BackupJob" }) + ); +} + +/// A FAILED write that matches no row (the row's generation changed under +/// a heartbeat that kept confirming) ends the job with no store access: no +/// `delete_prefix`, the row untouched. +#[tokio::test] +async fn a_failed_write_that_does_not_match_makes_no_store_call() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let gate = closed_gate(); + snapshot_fixture( + &fake, + &music_table(), + songs(3), + true, + Some(Arc::clone(&gate)), + ); + *fake.heartbeat_answer.lock().unwrap() = HeartbeatAnswer::AlwaysConfirm; + let runtime = job_runtime(&mem); + let details = create_music_backup(&fake, &runtime).await.unwrap(); + // Another generation takes the row while the job waits on its stream; + // the put then fails. + fake.rows + .lock() + .unwrap() + .get_mut(&details.backup_arn) + .unwrap() + .owner_generation = 999; + mem.fail_puts + .store(true, std::sync::atomic::Ordering::Release); + gate.add_permits(3); + wait_for_jobs(&runtime).await; + assert_eq!(writes(&fake).len(), 1); + assert_eq!(writes(&fake)[0].0, JobStatus::Failed); + assert!( + mem.deletes.lock().unwrap().is_empty(), + "no store access after NotMatched" + ); + let row = fake.rows.lock().unwrap()[&details.backup_arn].clone(); + assert_eq!(row.status, BackupStatus::Creating); + assert_eq!(row.owner_generation, 999); + assert_eq!(job_outcomes(&runtime), [("failed".to_owned(), 1.0)]); +} + +/// A status write whose outcome is unknown is re-read, not treated as a +/// failure: an AVAILABLE write that landed is AVAILABLE with no store +/// access after it; one that was lost is written once more. The same for +/// the FAILED write: landed, the deletion follows; lost, it is written again +/// and the deletion follows. +#[tokio::test] +async fn an_unknown_write_is_re_read() { + for (unknown, writes_expected) in [(UnknownWrite::Applied, 1), (UnknownWrite::Lost, 2)] { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + snapshot_fixture(&fake, &music_table(), songs(2), true, None); + *fake.unknown_write.lock().unwrap() = Some(unknown); + let runtime = job_runtime(&mem); + let details = create_music_backup(&fake, &runtime).await.unwrap(); + wait_for_jobs(&runtime).await; + let row = fake.rows.lock().unwrap()[&details.backup_arn].clone(); + assert_eq!(row.status, BackupStatus::Available, "{unknown:?}"); + assert_eq!(writes(&fake).len(), writes_expected, "{unknown:?}"); + assert!(mem.deletes.lock().unwrap().is_empty()); + assert_eq!(objects_under(&mem, &details.backup_arn).len(), 5); + assert_eq!(job_outcomes(&runtime), [("succeeded".to_owned(), 1.0)]); + } + for (unknown, writes_expected) in [(UnknownWrite::Applied, 1), (UnknownWrite::Lost, 2)] { + let mem = Arc::new(MemStore::default()); + mem.fail_puts + .store(true, std::sync::atomic::Ordering::Release); + let fake = Arc::new(Fake::default()); + snapshot_fixture(&fake, &music_table(), songs(2), true, None); + *fake.unknown_write.lock().unwrap() = Some(unknown); + let runtime = job_runtime(&mem); + let details = create_music_backup(&fake, &runtime).await.unwrap(); + wait_for_jobs(&runtime).await; + let row = fake.rows.lock().unwrap()[&details.backup_arn].clone(); + assert_eq!(row.status, BackupStatus::Failed, "{unknown:?}"); + assert_eq!(writes(&fake).len(), writes_expected, "{unknown:?}"); + let prefix = BackupArn::parse(&details.backup_arn) + .unwrap() + .store_prefix(); + assert_eq!(mem.deletes.lock().unwrap().as_slice(), [prefix]); + } +} + +/// A heartbeat that reports the claim lost, or that does not return within +/// a period, ends the job with no put and no catalog write: the row is left +/// as it is for its owner or the reconciler. +#[tokio::test] +async fn a_lost_or_stalled_heartbeat_stops_the_job_with_no_put() { + for answer in [ + HeartbeatAnswer::Lost, + HeartbeatAnswer::Stall, + HeartbeatAnswer::Error, + ] { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let gate = closed_gate(); + snapshot_fixture(&fake, &music_table(), songs(3), true, Some(gate)); + let runtime = job_runtime(&mem); + let details = create_music_backup(&fake, &runtime).await.unwrap(); + // The job waits on its stream (the encoder is a blocking task, so + // the clock runs: a paused clock would not advance behind it); the + // first beat, a period later, finds the claim lost (or never + // returns: a period more). + *fake.heartbeat_answer.lock().unwrap() = answer; + wait_for_jobs(&runtime).await; + assert!(mem.puts.lock().unwrap().is_empty(), "{answer:?}: no put"); + assert!( + mem.deletes.lock().unwrap().is_empty(), + "{answer:?}: no delete" + ); + assert!(writes(&fake).is_empty(), "{answer:?}: no catalog write"); + let row = fake.rows.lock().unwrap()[&details.backup_arn].clone(); + assert_eq!(row.status, BackupStatus::Creating, "{answer:?}"); + assert_eq!(runtime.free_slots(), 4); + assert_eq!(job_outcomes(&runtime), [("failed".to_owned(), 1.0)]); + } +} + +/// The pre-put check's numbers: with the claim confirming, the last +/// confirmed beat stays recent however long the job runs; a timer that +/// stopped running leaves the check true for two periods after the last +/// confirmed beat and false after; with a beat that never returns, the +/// check still passes until the timer has waited one period for the beat +/// (two periods from the last confirmation), and the claim is lost from +/// then on. +#[tokio::test(start_paused = true)] +async fn the_heartbeat_handle_confirms_the_claim_within_two_periods() { + let fake = Arc::new(Fake::default()); + fake.rows + .lock() + .unwrap() + .insert(arn(ACCOUNT), row(&arn(ACCOUNT), BackupStatus::Creating)); + let claim: Arc = Arc::from(fake.claim_on(&arn(ACCOUNT), 7)); + let period = Duration::from_secs(10); + let handle = job::heartbeat::HeartbeatHandle::start( + Arc::clone(&claim), + period, + job::heartbeat::HeartbeatMetric::None, + tokio::time::Instant::now(), + ); + // Half a period past the tenth tick, so that tick's beat has confirmed. + tokio::time::sleep(10 * period + period / 2).await; + assert!(handle.confirms_the_claim()); + assert!(fake.heartbeats.load(std::sync::atomic::Ordering::Acquire) >= 10); + // A timer that stopped running (no loss raised): the check holds for two + // periods after the last confirmed beat, the second period's end + // included, and then refuses. + handle.stop_timer(); + tokio::time::sleep(period).await; + assert!(handle.confirms_the_claim(), "a period and a half"); + tokio::time::sleep(period / 2).await; + assert!( + handle.confirms_the_claim(), + "exactly two periods: the bound is inclusive" + ); + tokio::time::sleep(period / 2).await; + assert!(!handle.lost().is_cancelled()); + assert!( + !handle.confirms_the_claim(), + "two and a half periods without a confirmed beat" + ); + drop(handle); + + *fake.heartbeat_answer.lock().unwrap() = HeartbeatAnswer::Stall; + let handle = job::heartbeat::HeartbeatHandle::start( + claim, + period, + job::heartbeat::HeartbeatMetric::None, + tokio::time::Instant::now(), + ); + tokio::time::sleep(period + period / 2).await; + assert!(handle.confirms_the_claim(), "the beat may still return"); + assert!(!handle.lost().is_cancelled()); + tokio::time::sleep(period).await; + assert!(!handle.confirms_the_claim()); + assert!(handle.lost().is_cancelled()); + assert!(handle.reason().contains("did not return")); +} + +/// The shutdown token runs the failure path: FAILED with reason `shutdown`, +/// then the prefix deleted; the outcome is `cancelled`. `drain_jobs` is what +/// the server calls: it cancels and waits for the job. +#[tokio::test] +async fn cancellation_runs_the_failure_path() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let gate = Arc::new(tokio::sync::Semaphore::new(1)); + snapshot_fixture( + &fake, + &music_table(), + songs(3), + true, + Some(Arc::clone(&gate)), + ); + let runtime = job_runtime(&mem); + let details = create_music_backup(&fake, &runtime).await.unwrap(); + // The job holds its first item and waits for the second. + let deadline = tokio::time::Instant::now() + Duration::from_secs(5); + while gate.available_permits() > 0 { + assert!(tokio::time::Instant::now() < deadline); + tokio::time::sleep(Duration::from_millis(5)).await; + } + assert_eq!(runtime.running_jobs(), 1); + let left = runtime.drain_jobs(Duration::from_secs(10)).await; + assert_eq!(left, 0); + assert_eq!(runtime.running_jobs(), 0); + let row = fake.rows.lock().unwrap()[&details.backup_arn].clone(); + assert_eq!(row.status, BackupStatus::Failed); + assert_eq!(row.failure_reason.as_deref(), Some("shutdown")); + let prefix = BackupArn::parse(&details.backup_arn) + .unwrap() + .store_prefix(); + assert_eq!( + writes(&fake), + [(JobStatus::Failed, Some("shutdown".to_owned()))] + ); + assert_eq!(mem.deletes.lock().unwrap().as_slice(), [prefix]); + assert!(objects_under(&mem, &details.backup_arn).is_empty()); + use extenddb_core::metrics::MetricName; + let drained = runtime.metrics().unwrap().drain(Duration::ZERO); + assert_eq!( + drained + .iter() + .filter(|b| b.metric == MetricName::BackupJobs) + .map(|b| (b.operation.clone(), b.sum)) + .collect::>(), + [("cancelled".to_owned(), 1.0)] + ); + assert!( + !drained + .iter() + .any(|b| b.metric == MetricName::WorkerErrorCount), + "a clean shutdown is not a worker error" + ); + assert!( + !drained + .iter() + .any(|b| b.metric == MetricName::WorkerLastSuccess), + "nor a success" + ); +} + +/// With `max_concurrent_jobs = 1`, a second CreateBackup while a job runs +/// is LimitExceeded naming the bound; once the job ends the slot is free. +#[tokio::test] +async fn a_second_create_over_the_job_bound_is_limit_exceeded() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let gate = closed_gate(); + snapshot_fixture( + &fake, + &music_table(), + songs(2), + true, + Some(Arc::clone(&gate)), + ); + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + let runtime = BackupRuntime::new(1) + .with_store(ConfiguredStore::new( + store().store_kind, + store().store_root, + Arc::clone(&mem) as Arc, + None, + Ok(store()), + )) + .with_job_settings(job_settings()); + create_music_backup(&fake, &runtime).await.unwrap(); + let err = create_music_backup(&fake, &runtime).await.unwrap_err(); + assert!( + matches!(&err, DynamoDbError::LimitExceededException(m) if m.contains("max_concurrent_jobs")), + "{err:?}" + ); + assert_eq!( + fake.snapshots_opened.lock().unwrap().len(), + 1, + "the second opened nothing" + ); + gate.add_permits(2); + wait_for_jobs(&runtime).await; + assert_eq!(runtime.free_slots(), 1); + snapshot_fixture(&fake, &music_table(), songs(1), true, None); + create_music_backup(&fake, &runtime).await.unwrap(); + wait_for_jobs(&runtime).await; + assert_eq!(fake.rows.lock().unwrap().len(), 2); +} + +/// A `tracing` subscriber that keeps every event's level and message, for +/// the tests of the job's log lines. Set as the thread's default for the +/// test's life through [`LogLines::capture`]; a `#[tokio::test]` runtime +/// runs the job's tasks on the test's thread, so their events reach it (the +/// encoders, on the blocking pool, log nothing). +#[derive(Clone, Default)] +struct LogLines(Arc>>); + +/// The process's global `tracing` subscriber while these tests run: +/// interested in every callsite and enabled for none. `tracing` caches each +/// callsite's interest the first time any thread reaches it; while exactly +/// one scoped subscriber is registered in the process, that cache is +/// computed from the default of the thread that got there first. A test +/// without a subscriber running beside a `LogLines` test on another thread +/// would so cache the job's log callsites as "never", and the `LogLines` +/// test would then see no line from them (tokio-rs/tracing#2743). With this +/// global default the cache reads "sometimes" instead, and each event asks +/// the thread's own default, the `LogLines` where one is set. +struct InterestedInEveryCallsite; + +impl tracing::Subscriber for InterestedInEveryCallsite { + fn register_callsite( + &self, + _: &'static tracing::Metadata<'static>, + ) -> tracing::subscriber::Interest { + tracing::subscriber::Interest::sometimes() + } + fn enabled(&self, _: &tracing::Metadata<'_>) -> bool { + false + } + fn new_span(&self, _: &tracing::span::Attributes<'_>) -> tracing::span::Id { + tracing::span::Id::from_u64(1) + } + fn record(&self, _: &tracing::span::Id, _: &tracing::span::Record<'_>) {} + fn record_follows_from(&self, _: &tracing::span::Id, _: &tracing::span::Id) {} + fn event(&self, _: &tracing::Event<'_>) {} + fn enter(&self, _: &tracing::span::Id) {} + fn exit(&self, _: &tracing::span::Id) {} +} + +impl LogLines { + /// A fresh `LogLines` set as this thread's `tracing` default until the + /// guard drops, with the process's global default installed once. + fn capture() -> (Self, tracing::subscriber::DefaultGuard) { + static GLOBAL: std::sync::Once = std::sync::Once::new(); + GLOBAL.call_once(|| { + // Another global may already be set by a test outside this + // module; then its interest rule applies and this one is moot. + let _ = tracing::subscriber::set_global_default(InterestedInEveryCallsite); + }); + let lines = Self::default(); + let guard = tracing::subscriber::set_default(lines.clone()); + (lines, guard) + } + + /// The messages containing `needle`, in order. + fn containing(&self, needle: &str) -> Vec { + self.0 + .lock() + .unwrap() + .iter() + .filter(|(_, m)| m.contains(needle)) + .map(|(_, m)| m.clone()) + .collect() + } + + /// The messages logged at `level`, in order. + fn at_level(&self, level: tracing::Level) -> Vec { + self.0 + .lock() + .unwrap() + .iter() + .filter(|(l, _)| *l == level) + .map(|(_, m)| m.clone()) + .collect() + } +} + +struct MessageVisitor<'a>(&'a mut String); + +/// A callsite met first by a thread without a scoped subscriber is still +/// seen by a `LogLines` on another thread (the global default's interest +/// rule; the doc of `InterestedInEveryCallsite`). Without that default the +/// callsite's interest is cached as "never" whenever this is the only +/// `LogLines` alive in the process, and nothing is captured. #[test] -fn other_account_arn_is_denied() { - let body = json!({ "BackupArn": arn("999999999999") }); - let err = arn_field(&body).unwrap_err(); +fn a_callsite_first_met_on_another_thread_still_reaches_the_lines() { + fn shared_callsite(n: u32) { + tracing::error!("probe {n}: a callsite shared by two threads"); + } + let (lines, _guard) = LogLines::capture(); + std::thread::spawn(|| shared_callsite(1)).join().unwrap(); + shared_callsite(2); + assert_eq!( + lines.containing("probe "), + ["probe 2: a callsite shared by two threads"] + ); +} + +impl tracing::field::Visit for MessageVisitor<'_> { + fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn std::fmt::Debug) { + if field.name() == "message" { + use std::fmt::Write as _; + let _ = write!(self.0, "{value:?}"); + } + } +} + +impl tracing::Subscriber for LogLines { + fn enabled(&self, _: &tracing::Metadata<'_>) -> bool { + true + } + fn new_span(&self, _: &tracing::span::Attributes<'_>) -> tracing::span::Id { + tracing::span::Id::from_u64(1) + } + fn record(&self, _: &tracing::span::Id, _: &tracing::span::Record<'_>) {} + fn record_follows_from(&self, _: &tracing::span::Id, _: &tracing::span::Id) {} + fn event(&self, event: &tracing::Event<'_>) { + let mut message = String::new(); + event.record(&mut MessageVisitor(&mut message)); + self.0 + .lock() + .unwrap() + .push((*event.metadata().level(), message)); + } + fn enter(&self, _: &tracing::span::Id) {} + fn exit(&self, _: &tracing::span::Id) {} +} + +/// The bytes of one item's data file line. +fn line_bytes(item: &Item) -> u64 { + extenddb_core::backup_format::encode_item_line(item) + .unwrap() + .len() as u64 +} + +/// A put that fails while the snapshot stream is between pages ends the job +/// at once: the stream holds one permit, so it yields the first item and +/// then waits; the first file closes at that item (a one-line knob), its put +/// fails, and the FAILED write lands with the stream still held, no permit +/// released. Without the uploader's stop reaching the feeder, the feeder +/// would wait for the stream's next item and the row would stay CREATING +/// with a live heartbeat for as long as the stream did. +#[tokio::test] +async fn a_put_failure_ends_the_job_without_waiting_for_the_streams_next_item() { + let mem = Arc::new(MemStore::default()); + mem.fail_puts + .store(true, std::sync::atomic::Ordering::Release); + let fake = Arc::new(Fake::default()); + let items = songs(3); + let target = line_bytes(&items[0]); + let gate = Arc::new(tokio::sync::Semaphore::new(1)); + snapshot_fixture(&fake, &music_table(), items, true, Some(Arc::clone(&gate))); + let runtime = job_runtime(&mem).with_job_settings(JobSettings { + data_file_target_bytes: target, + ..job_settings() + }); + let details = create_music_backup(&fake, &runtime).await.unwrap(); + wait_for_jobs(&runtime).await; + assert_eq!(gate.available_permits(), 0, "the stream was never released"); + let row = fake.rows.lock().unwrap()[&details.backup_arn].clone(); + assert_eq!(row.status, BackupStatus::Failed); + assert!( + row.failure_reason + .as_deref() + .unwrap() + .starts_with("writing data/000001.json.gz: "), + "{:?}", + row.failure_reason + ); + let prefix = BackupArn::parse(&details.backup_arn) + .unwrap() + .store_prefix(); + assert_eq!(mem.deletes.lock().unwrap().as_slice(), [prefix]); + assert_eq!(job_outcomes(&runtime), [("failed".to_owned(), 1.0)]); +} + +/// `BackupJobDuration` of a FAILED job covers its failure path: with the +/// store's deletion taking 300 ms, the duration recorded under `failed` is +/// at least that, so the metric measures the job to the end of its last +/// store call for either outcome. +#[tokio::test] +async fn a_failed_jobs_duration_covers_its_failed_write_and_deletion() { + use extenddb_core::metrics::MetricName; + let mem = Arc::new(MemStore::default()); + mem.fail_puts + .store(true, std::sync::atomic::Ordering::Release); + *mem.delete_delay.lock().unwrap() = Some(Duration::from_millis(300)); + let fake = Arc::new(Fake::default()); + snapshot_fixture(&fake, &music_table(), songs(2), true, None); + let runtime = job_runtime(&mem); + let details = create_music_backup(&fake, &runtime).await.unwrap(); + wait_for_jobs(&runtime).await; + let row = fake.rows.lock().unwrap()[&details.backup_arn].clone(); + assert_eq!(row.status, BackupStatus::Failed); + assert_eq!(mem.deletes.lock().unwrap().len(), 1); + let drained = runtime.metrics().unwrap().drain(Duration::ZERO); + let durations: Vec<(String, f64)> = drained + .iter() + .filter(|b| b.metric == MetricName::BackupJobDuration) + .map(|b| (b.operation.clone(), b.sum)) + .collect(); + assert_eq!(durations.len(), 1, "{durations:?}"); + assert_eq!(durations[0].0, "failed"); + assert!( + durations[0].1 >= 300_000.0, + "{} µs: the duration stops before the deletion", + durations[0].1 + ); +} + +/// The two info lines: the start line gives the table's count as the stats +/// pass last counted it (the description's figure, here 100,000), the +/// completion line the stream's count (the items written) and the bytes. +#[tokio::test] +async fn the_start_and_completion_lines_name_the_two_item_counts() { + let (lines, _guard) = LogLines::capture(); + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + snapshot_fixture(&fake, &music_table(), songs(5), true, None); + let runtime = job_runtime(&mem); + let details = create_music_backup(&fake, &runtime).await.unwrap(); + wait_for_jobs(&runtime).await; + let arn = &details.backup_arn; + let started = lines.containing(": started, "); + assert_eq!( + started, + [format!( + "backup {arn}: started, 100000 items in the table as last counted by the stats pass, \ + key-ordered" + )] + ); + let total: u64 = mem.put_sizes.lock().unwrap().iter().map(|(_, n)| n).sum(); + let available = lines.containing(": AVAILABLE, "); + assert_eq!(available.len(), 1, "{available:?}"); + assert!( + available[0].starts_with(&format!( + "backup {arn}: AVAILABLE, 5 items, {total} bytes written in " + )), + "{}", + available[0] + ); +} + +/// The refusal log cap: with 1,001 refused items the first 1,000 are +/// logged one by one, one line counts the remainder, the reason carries the +/// full count and the first item, the stream is read to its end and nothing +/// is uploaded. +#[tokio::test] +async fn refusals_past_the_log_cap_are_counted_in_one_line() { + let (lines, _guard) = LogLines::capture(); + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let mut items = vec![song("a", "0")]; + items.extend((1..=1001).map(|i| song_with_bad_number("a", &format!("{i:05}"), "1E126"))); + items.push(song("a", "zzz")); + let gate = Arc::new(tokio::sync::Semaphore::new(items.len())); + snapshot_fixture(&fake, &music_table(), items, true, Some(Arc::clone(&gate))); + let runtime = job_runtime(&mem); + let details = create_music_backup(&fake, &runtime).await.unwrap(); + wait_for_jobs(&runtime).await; + assert_eq!(gate.available_permits(), 0, "every item was read"); + let logged = lines.containing(": item cannot be restored: key "); + assert_eq!(logged.len(), 1000); + assert!( + logged[0].contains(r#""SongTitle":{"S":"00001"}"#) + && logged[0].contains(", attribute Year: "), + "{}", + logged[0] + ); + assert!( + logged[999].contains(r#""SongTitle":{"S":"01000"}"#), + "{}", + logged[999] + ); + assert_eq!( + lines.containing(" more item(s) cannot be restored (not listed)"), + [format!( + "backup {}: 1 more item(s) cannot be restored (not listed)", + details.backup_arn + )] + ); + let row = fake.rows.lock().unwrap()[&details.backup_arn].clone(); + assert_eq!(row.status, BackupStatus::Failed); + let reason = row.failure_reason.unwrap(); + assert!( + reason.starts_with("1001 item(s) cannot be restored; first: key "), + "{reason}" + ); + assert!(reason.contains(r#""SongTitle":{"S":"00001"}"#), "{reason}"); + assert!(mem.puts.lock().unwrap().is_empty()); +} + +/// An item whose data file line would exceed the format's cap is refused +/// through the validator's path, not the encoder's: under a raised item +/// limit, a 600 KB item of the worst-expansion shape (a list of number +/// sets, a 40 MB line) is counted, logged with its key and the cap, the +/// rest of the stream is checked, nothing is uploaded and the FAILED +/// reason names the item without #618. The item passes the validator +/// itself (its numbers are valid, its size within the limit). The line is +/// measured through a counting writer that stops at the cap (the format's +/// own test pins that it holds no line), so the reason carries the cap and +/// not the line's length. +#[tokio::test] +async fn an_item_whose_line_exceeds_the_cap_is_refused_and_named() { + use extenddb_core::backup_format::MAX_LINE_BYTES; + use extenddb_core::types::AttributeValue; + use std::collections::BTreeSet; + let (lines, _guard) = LogLines::capture(); + // 81 two-digit values at `E-130`, 133 characters each in normalized + // form, 2 bytes of item size each (the `MAX_LINE_BYTES` doc's shape). + let set: BTreeSet = (11..100) + .filter(|m| m % 10 != 0) + .map(|m| format!("0.{}{m}", "0".repeat(129))) + .collect(); + assert_eq!(set.len(), 81); + let mut worst = song("a", "2"); + worst.insert( + "l".to_owned(), + AttributeValue::L(vec![AttributeValue::NS(set); 3_700]), + ); + let size = extenddb_core::types::item_size_bytes(&worst); + assert!(size > 409_600 && size < 2 * 1024 * 1024, "{size}"); + let line = line_bytes(&worst); + assert!(line > MAX_LINE_BYTES as u64, "{line}"); + let items = vec![song("a", "1"), worst, song("a", "3")]; + let gate = Arc::new(tokio::sync::Semaphore::new(items.len())); + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + snapshot_fixture(&fake, &music_table(), items, true, Some(Arc::clone(&gate))); + let runtime = job_runtime(&mem); + let limits = LimitsConfig { + max_item_size_bytes: 2 * 1024 * 1024, + ..LimitsConfig::default() + }; + let details = create_backup( + &fake, + &runtime, + ACCOUNT, + REGION, + &limits, + "Music", + "nightly", + Ok(music_table()), + ) + .await + .unwrap(); + // The test takes about 20 s alone on an idle core, so the guard is + // 300 s. + wait_for_jobs_within(&runtime, Duration::from_secs(300)).await; + assert_eq!( + gate.available_permits(), + 0, + "the rest of the stream was checked" + ); + let row = fake.rows.lock().unwrap()[&details.backup_arn].clone(); + assert_eq!(row.status, BackupStatus::Failed); + let reason = row.failure_reason.unwrap(); + assert_eq!( + reason, + format!( + "1 item(s) cannot be restored; first: key {{\"Artist\":{{\"S\":\"a\"}},\"SongTitle\":\ + {{\"S\":\"2\"}}}}: data file line exceeds the line cap of {MAX_LINE_BYTES} bytes" + ) + ); + assert_eq!(lines.containing(": item cannot be restored: key ").len(), 1); + assert!( + mem.puts.lock().unwrap().is_empty(), + "{:?}", + mem.puts.lock().unwrap() + ); + assert_eq!(job_outcomes(&runtime), [("failed".to_owned(), 1.0)]); +} + +/// A status write that errors on the claim connection makes no store call, +/// whichever write it is. The FAILED write: no `delete_prefix`, the row +/// CREATING for the reconciler. The AVAILABLE write: the manifest was +/// already put, so the five objects stay for the reconciler's probe, which +/// is the order the design rests on (the manifest is the commit record, the +/// AVAILABLE write follows it). +#[tokio::test] +async fn a_status_write_error_makes_no_store_call_and_leaves_the_manifest() { + let mem = Arc::new(MemStore::default()); + mem.fail_puts + .store(true, std::sync::atomic::Ordering::Release); + let fake = Arc::new(Fake::default()); + snapshot_fixture(&fake, &music_table(), songs(2), true, None); + fake.status_write_error + .store(true, std::sync::atomic::Ordering::Release); + let runtime = job_runtime(&mem); + let details = create_music_backup(&fake, &runtime).await.unwrap(); + wait_for_jobs(&runtime).await; + let row = fake.rows.lock().unwrap()[&details.backup_arn].clone(); + assert_eq!(row.status, BackupStatus::Creating); + assert_eq!(writes(&fake).len(), 1); + assert!( + mem.deletes.lock().unwrap().is_empty(), + "no delete after a claim error on the FAILED write" + ); + assert_eq!(job_outcomes(&runtime), [("failed".to_owned(), 1.0)]); + + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + snapshot_fixture(&fake, &music_table(), songs(2), true, None); + fake.status_write_error + .store(true, std::sync::atomic::Ordering::Release); + let runtime = job_runtime(&mem); + let details = create_music_backup(&fake, &runtime).await.unwrap(); + wait_for_jobs(&runtime).await; + let row = fake.rows.lock().unwrap()[&details.backup_arn].clone(); + assert_eq!(row.status, BackupStatus::Creating); + assert_eq!(writes(&fake), [(JobStatus::Available, None)]); + assert!(mem.deletes.lock().unwrap().is_empty()); + let prefix = BackupArn::parse(&details.backup_arn) + .unwrap() + .store_prefix(); + assert_eq!( + mem.puts.lock().unwrap().as_slice(), + [ + format!("{prefix}data/000001.json.gz"), + format!("{prefix}manifest-files.json"), + format!("{prefix}manifest-summary.json"), + format!("{prefix}extenddb-manifest.sha256"), + format!("{prefix}extenddb-manifest.json"), + ], + "the manifest was put before the AVAILABLE write" + ); + assert_eq!(objects_under(&mem, &details.backup_arn).len(), 5); + assert_eq!(job_outcomes(&runtime), [("failed".to_owned(), 1.0)]); +} + +/// A second unknown outcome, on the write sent once more, counts as +/// matching no row: two writes and no third, the row left as it is, no +/// delete, the outcome `failed` (the job ended by its own path, not a +/// panic). +#[tokio::test] +async fn a_second_unknown_outcome_ends_the_job_without_a_third_write() { + let mem = Arc::new(MemStore::default()); + mem.fail_puts + .store(true, std::sync::atomic::Ordering::Release); + let fake = Arc::new(Fake::default()); + snapshot_fixture(&fake, &music_table(), songs(2), true, None); + *fake.unknown_write.lock().unwrap() = Some(UnknownWrite::Lost); + fake.unknown_write_sticky + .store(true, std::sync::atomic::Ordering::Release); + let runtime = job_runtime(&mem); + let details = create_music_backup(&fake, &runtime).await.unwrap(); + wait_for_jobs(&runtime).await; + assert_eq!(writes(&fake).len(), 2); + let row = fake.rows.lock().unwrap()[&details.backup_arn].clone(); + assert_eq!(row.status, BackupStatus::Creating); + assert!(mem.deletes.lock().unwrap().is_empty()); + assert_eq!(job_outcomes(&runtime), [("failed".to_owned(), 1.0)]); +} + +/// A second unknown outcome whose write landed counts as matched: the first +/// FAILED write is lost with its connection (the re-read shows CREATING, so +/// the write is sent once more), the second lands and its outcome is +/// unknown too, and the second re-read shows the row FAILED under this +/// claim, so the failure path goes on to the one `delete_prefix`. The row +/// is FAILED under this generation and the prefix is the job's, so the +/// deletion is right; a job that treated the second unknown outcome as +/// matching no row would leave the prefix to the reconciler for nothing. +#[tokio::test] +async fn a_second_unknown_outcome_whose_write_landed_is_matched() { + let mem = Arc::new(MemStore::default()); + mem.fail_puts + .store(true, std::sync::atomic::Ordering::Release); + let fake = Arc::new(Fake::default()); + snapshot_fixture(&fake, &music_table(), songs(2), true, None); + *fake.unknown_write.lock().unwrap() = Some(UnknownWrite::Lost); + *fake.unknown_write_then.lock().unwrap() = Some(UnknownWrite::Applied); + let runtime = job_runtime(&mem); + let details = create_music_backup(&fake, &runtime).await.unwrap(); + wait_for_jobs(&runtime).await; + assert_eq!( + writes(&fake).iter().map(|w| w.0).collect::>(), + [JobStatus::Failed, JobStatus::Failed] + ); + let row = fake.rows.lock().unwrap()[&details.backup_arn].clone(); + assert_eq!(row.status, BackupStatus::Failed); + let prefix = BackupArn::parse(&details.backup_arn) + .unwrap() + .store_prefix(); + assert_eq!( + mem.deletes.lock().unwrap().as_slice(), + [prefix], + "the second re-read shows FAILED under this claim: the deletion follows" + ); + assert!(objects_under(&mem, &details.backup_arn).is_empty()); + assert_eq!(job_outcomes(&runtime), [("failed".to_owned(), 1.0)]); +} + +/// The pre-put check in its two places: with the heartbeat timer stopped +/// without raising the loss (a timer task that died) and two periods gone, +/// neither a data file's put nor a document's put starts, each stopping the +/// job as a lost claim naming the object. +#[tokio::test] +async fn a_stopped_heartbeat_timer_starts_no_put() { + let mem = Arc::new(MemStore::default()); + let fake = Fake::default(); + let arn = arn(ACCOUNT); + fake.rows + .lock() + .unwrap() + .insert(arn.clone(), row(&arn, BackupStatus::Creating)); + let claim: Arc = Arc::from(fake.claim_on(&arn, 7)); + let period = Duration::from_millis(50); + let heartbeat = job::heartbeat::HeartbeatHandle::start( + claim, + period, + job::heartbeat::HeartbeatMetric::None, + tokio::time::Instant::now(), + ); + heartbeat.stop_timer(); + tokio::time::sleep(3 * period).await; + assert!(!heartbeat.lost().is_cancelled()); + let shutdown = extenddb_storage::CancellationToken::new(); + let parsed = BackupArn::parse(&arn).unwrap(); + let prefix = parsed.store_prefix(); + let runtime = runtime_on(&mem, store()); + let identity = store(); + let pipeline = job::writer::Pipeline { + store: &*mem, + configured: runtime.configured_store().unwrap(), + identity: &identity, + prefix: &prefix, + arn: &arn, + heartbeat: &heartbeat, + shutdown: &shutdown, + metrics: None, + target_bytes: 64 * 1024 * 1024, + key_prefix: "", + bucket: &store().store_root, + }; + let table = music_table(); + let key = extenddb_core::validation::RestorableKeySchema::resolve( + &table.key_schema, + &table.attribute_definitions, + ) + .unwrap(); + let stream: BoxStream<'static, Result> = + Box::pin(futures::stream::iter(songs(1).into_iter().map(Ok))); + let limits = extenddb_core::validation::RestorableLimits::from(&LimitsConfig::default()); + let stop = job::writer::write_data_files(&pipeline, stream, key, limits, None) + .await + .unwrap_err(); + assert_eq!( + stop, + job::writer::Stop::ClaimLost( + "no heartbeat confirmed the claim recently enough for data file 1".to_owned() + ) + ); + assert!(mem.puts.lock().unwrap().is_empty()); + let stop = job::put_document(&pipeline, "manifest-files.json", b"{}") + .await + .unwrap_err(); + assert_eq!( + stop, + job::writer::Stop::ClaimLost( + "no heartbeat confirmed the claim recently enough for manifest-files.json".to_owned() + ) + ); + assert!(mem.puts.lock().unwrap().is_empty()); + assert_eq!( + mem.keys(), + [STORE_IDENTITY_KEY], + "no object but the identity file" + ); +} + +/// The maintenance check runs before the permit and the row, so a slow +/// check delays the request and not the job's first put: with a period of +/// 100 ms, `stale_job_after` of 300 ms (the floor's ratio) and a check of +/// 400 ms, the request takes at least the check, the row is inserted after +/// it (its heartbeat the insert's, within two periods of the first put), and +/// the backup completes AVAILABLE. +#[tokio::test] +async fn a_slow_maintenance_check_delays_the_request_and_not_the_first_put() { + let period = Duration::from_millis(100); + let settings = JobSettings { + heartbeat_interval: period, + stale_job_after: 3 * period, + ..job_settings() + }; + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + snapshot_fixture(&fake, &music_table(), songs(2), true, None); + *fake.maintenance_delay.lock().unwrap() = Some(4 * period); + let runtime = job_runtime(&mem).with_job_settings(settings); + let asked = tokio::time::Instant::now(); + let details = create_music_backup(&fake, &runtime).await.unwrap(); + assert!( + asked.elapsed() >= 4 * period, + "the request waited for the check" + ); + assert_eq!(fake.maintenance_checks.lock().unwrap().len(), 1); + wait_for_jobs(&runtime).await; + let row = fake.rows.lock().unwrap()[&details.backup_arn].clone(); + assert_eq!( + (row.status, row.failure_reason), + (BackupStatus::Available, None) + ); + assert_eq!(objects_under(&mem, &details.backup_arn).len(), 5); + assert_eq!(job_outcomes(&runtime), [("succeeded".to_owned(), 1.0)]); + assert_eq!(runtime.free_slots(), 4); +} + +/// The claim re-check before the manifest: the heartbeat answers `Lost` +/// from the sidecar's put on, with a period long enough that no timer beat +/// runs during the job, so the re-check is the one beat the job sends; it +/// finds the claim lost, the manifest is not written, the row is left +/// CREATING with its four objects and no catalog write. +#[tokio::test] +async fn the_claim_re_check_before_the_manifest_stops_a_lost_claim() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + snapshot_fixture(&fake, &music_table(), songs(2), true, None); + let answer = Arc::clone(&fake.heartbeat_answer); + *mem.on_put.lock().unwrap() = Some(Box::new(move |key: &str| { + if key.ends_with("extenddb-manifest.sha256") { + *answer.lock().unwrap() = HeartbeatAnswer::Lost; + } + })); + let runtime = job_runtime(&mem).with_job_settings(JobSettings { + heartbeat_interval: Duration::from_secs(10), + ..job_settings() + }); + let details = create_music_backup(&fake, &runtime).await.unwrap(); + wait_for_jobs(&runtime).await; + assert_eq!( + fake.heartbeats.load(std::sync::atomic::Ordering::Acquire), + 1, + "the re-check is the one beat" + ); + let row = fake.rows.lock().unwrap()[&details.backup_arn].clone(); + assert_eq!(row.status, BackupStatus::Creating); + assert!( + writes(&fake).is_empty(), + "no catalog write after a lost claim" + ); + assert!(mem.deletes.lock().unwrap().is_empty()); + let objects = objects_under(&mem, &details.backup_arn); + assert_eq!(objects.len(), 4, "{objects:?}"); + assert!( + !objects + .iter() + .any(|k| k.ends_with("extenddb-manifest.json")), + "the manifest was not written" + ); + assert_eq!(job_outcomes(&runtime), [("failed".to_owned(), 1.0)]); +} + +// --- the reconciler's duties over the store's rows ---------------------------- + +/// A CREATING row whose owner is dead, with a complete, verified backup +/// under its prefix (the golden manifest's documents and data files of the +/// recorded sizes), in `mem` and `fake`; returns its ARN. +fn seed_dead_creating_with_manifest(fake: &Fake, mem: &MemStore, complete: bool) -> String { + use extenddb_core::backup_format::{BackupDocuments, ExtenddbManifest, ManifestHeader}; + let golden = ExtenddbManifest::from_json_bytes(include_bytes!( + "../../../core/testdata/extenddb-manifest/v1.json" + )) + .unwrap(); + let docs = BackupDocuments::build( + ManifestHeader { + backup_arn: golden.backup_arn().unwrap(), + backup_name: golden.backup_name.clone(), + source_backend: golden.source_backend.clone(), + source_extenddb_version: golden.source_extenddb_version.clone(), + snapshot: golden.snapshot.clone(), + item_order: golden.item_order, + source_table: golden.source_table.clone(), + }, + golden + .data_files + .iter() + .cloned() + .map(|entry| (entry, None)) + .collect(), + "", + &store().store_root, + ) + .unwrap(); + let arn = golden.backup_arn.clone(); + let prefix = golden.backup_arn().unwrap().store_prefix(); + for (i, file) in golden.data_files.iter().enumerate() { + let size = if complete || i == 0 { + file.bytes + } else { + file.bytes - 1 + }; + mem.insert( + &format!("{prefix}{}", file.key), + &vec![0u8; usize::try_from(size).unwrap()], + ); + } + for (name, bytes) in docs.documents() { + mem.insert(&format!("{prefix}{name}"), bytes); + } + let mut dead = row(&arn, BackupStatus::Creating); + dead.heartbeat_at = 1234.5; + dead.size_bytes = 0; + dead.item_count = 0; + fake.rows.lock().unwrap().insert(arn.clone(), dead); + fake.dead.lock().unwrap().insert(arn.clone()); + arn +} + +/// One cycle on `fake` and `mem`, the store healthy. +async fn cycle_on(fake: &Arc, runtime: &BackupRuntime) -> reconciler::CycleReport { + reconciler::cycle( + fake, + runtime, + Duration::from_secs(300), + "test", + &mut reconciler::ReconcilerState::default(), + ) + .await + .unwrap() +} + +/// The reconciler's store calls that fail (not an absent object) record +/// `BackupStoreErrors` under the call's name, as the job's do: a `get` of a +/// dead owner's manifest, a `list` of a DELETED row's prefix, a +/// `delete_prefix` of a stale FAILED row's prefix. Each row is left for the +/// next cycle and the cycle itself completes, so `WorkerErrorCount` would +/// not move; the store's health is untouched. The next cycle, with the +/// store answering, finishes each duty and records nothing more. +#[tokio::test] +async fn the_reconcilers_failed_store_calls_are_counted_by_call() { + use extenddb_core::metrics::{MetricName, MetricsCollector}; + let mem = Arc::new(MemStore::default()); + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + let fake = Arc::new(Fake::default()); + let now = seconds_since_epoch(); + let dead = seed_dead_creating_with_manifest(&fake, &mem, true); + let dead_prefix = BackupArn::parse(&dead).unwrap().store_prefix(); + let stale_failed = arn_n(1); + let deleted = arn_n(2); + for (a, status) in [ + (&stale_failed, BackupStatus::Failed), + (&deleted, BackupStatus::Deleted), + ] { + let mut r = row(a, status); + r.heartbeat_at = now - 1000.0; + r.completed_at = Some(now); + fake.rows.lock().unwrap().insert(a.clone(), r); + let prefix = BackupArn::parse(a).unwrap().store_prefix(); + mem.insert(&format!("{prefix}data/000001.json.gz"), b"x"); + } + let failed_prefix = BackupArn::parse(&stale_failed).unwrap().store_prefix(); + let deleted_prefix = BackupArn::parse(&deleted).unwrap().store_prefix(); + *mem.fail_gets_under.lock().unwrap() = Some(dead_prefix.clone()); + *mem.fail_deletes_of.lock().unwrap() = Some(failed_prefix.clone()); + *mem.fail_lists_of.lock().unwrap() = Some(deleted_prefix.clone()); + let metrics = Arc::new(MetricsCollector::new()); + let runtime = runtime_on(&mem, store()).with_metrics(Arc::clone(&metrics)); + + let report = cycle_on(&fake, &runtime).await; + assert_eq!(report.store_healthy, Some(true)); + assert!(report.creating_resolved.is_empty()); + assert!(report.failed_prefixes_deleted.is_empty()); + assert!(report.deleted_prefixes_swept.is_empty()); + let errors = |drained: &[extenddb_core::metrics::FlushBucket]| { + let mut ops: Vec<(String, f64)> = drained + .iter() + .filter(|b| b.metric == MetricName::BackupStoreErrors) + .map(|b| (b.operation.clone(), b.sum)) + .collect(); + ops.sort_by(|a, b| a.0.cmp(&b.0)); + ops + }; + assert_eq!( + errors(&metrics.drain(Duration::ZERO)), + [ + ("delete_prefix".to_owned(), 1.0), + ("get".to_owned(), 1.0), + ("list".to_owned(), 1.0), + ] + ); + + *mem.fail_gets_under.lock().unwrap() = None; + *mem.fail_deletes_of.lock().unwrap() = None; + *mem.fail_lists_of.lock().unwrap() = None; + let report = cycle_on(&fake, &runtime).await; + assert_eq!(report.creating_resolved, [(dead, BackupStatus::Available)]); + assert_eq!(report.failed_prefixes_deleted, [stale_failed]); + assert_eq!(report.deleted_prefixes_swept, [deleted]); + assert!(errors(&metrics.drain(Duration::ZERO)).is_empty()); +} + +/// A dead CREATING row whose prefix holds a complete, verified manifest is +/// AVAILABLE with the manifest's size, count and `source_table` (the row's +/// own `source_table_details_json`, the stats pass's, is replaced: it is +/// seeded here with another object so the write is seen to carry the +/// manifest's); the takeover leaves `heartbeat_at` as the dead owner wrote +/// it and heartbeats nothing; the objects stay. A row whose owner is alive +/// is left alone. +#[tokio::test] +async fn the_reconciler_resolves_a_dead_creating_row_with_a_manifest_to_available() { + let mem = Arc::new(MemStore::default()); + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + let fake = Arc::new(Fake::default()); + let dead = seed_dead_creating_with_manifest(&fake, &mem, true); + let stats_pass = serde_json::json!({"TableName": "Music", "ItemCount": 0}); + fake.rows + .lock() + .unwrap() + .get_mut(&dead) + .unwrap() + .source_table_details_json = stats_pass.clone(); + let live = arn(ACCOUNT); + fake.rows + .lock() + .unwrap() + .insert(live.clone(), row(&live, BackupStatus::Creating)); + let objects_before = mem.keys().len(); + let runtime = runtime_on(&mem, store()); + let report = cycle_on(&fake, &runtime).await; + assert_eq!( + report.creating_resolved, + [(dead.clone(), BackupStatus::Available)] + ); + assert_eq!( + fake.takeovers.lock().unwrap().as_slice(), + std::slice::from_ref(&dead) + ); + let manifest = stored_manifest(&mem, &dead); + let row = fake.rows.lock().unwrap()[&dead].clone(); + assert_eq!(row.status, BackupStatus::Available); + assert_eq!(row.size_bytes, manifest.backup_size_bytes); + assert_eq!(row.item_count, 100_000); + assert_ne!( + stats_pass, + serde_json::to_value(&manifest.source_table).unwrap() + ); + assert_eq!( + row.source_table_details_json, + serde_json::to_value(&manifest.source_table).unwrap(), + "the AVAILABLE write carries the manifest's source_table" + ); + assert_eq!(row.heartbeat_at, 1234.5, "the takeover left the heartbeat"); + assert_ne!(row.owner_generation, 7, "a fresh generation"); + assert!(row.completed_at.is_some()); + assert_eq!( + fake.heartbeats.load(std::sync::atomic::Ordering::Acquire), + 0, + "the reconciler heartbeats no takeover claim" + ); + assert_eq!(mem.keys().len(), objects_before, "nothing deleted"); + assert_eq!( + fake.rows.lock().unwrap()[&live].status, + BackupStatus::Creating + ); + assert!(mem.deletes.lock().unwrap().is_empty()); +} + +/// A dead CREATING row without a complete manifest is FAILED with the +/// owner-dead reason and `completed_at`, and the resolution itself deletes +/// nothing; the FAILED-prefix duty deletes the prefix once it runs, the +/// heartbeat the owner left being stale. Every guard of the probe has its +/// shape: no manifest; a manifest without its sidecar; a sidecar that does +/// not match; a manifest the reader refuses under a sidecar computed over +/// its bytes (a sidecar matches what was stored, not what parses); a data +/// file missing; a data file shorter than recorded; one larger than +/// recorded; a manifest naming another backup under the row's prefix. A +/// store error during the probe (not an absent object) is none of these: +/// the row stays CREATING for the next cycle, nothing written. +#[tokio::test] +async fn the_reconciler_fails_a_dead_creating_row_without_a_complete_manifest() { + for shape in [ + "none", + "no sidecar", + "sidecar mismatch", + "unparseable manifest", + "missing data file", + "short file", + "oversized file", + "foreign ARN", + ] { + let mem = Arc::new(MemStore::default()); + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + let fake = Arc::new(Fake::default()); + let mut dead = seed_dead_creating_with_manifest(&fake, &mem, shape != "short file"); + let prefix = BackupArn::parse(&dead).unwrap().store_prefix(); + let golden = stored_manifest(&mem, &dead); + match shape { + "none" => { + mem.objects + .lock() + .unwrap() + .remove(&format!("{prefix}extenddb-manifest.json")); + } + "no sidecar" => { + mem.objects + .lock() + .unwrap() + .remove(&format!("{prefix}extenddb-manifest.sha256")); + } + "sidecar mismatch" => { + mem.insert(&format!("{prefix}extenddb-manifest.sha256"), b"0000"); + } + "unparseable manifest" => { + use sha2::Digest as _; + let garbage = b"{\"format_version\":1,\"backup_arn\":"; + mem.insert(&format!("{prefix}extenddb-manifest.json"), garbage); + let sidecar = format!( + "{} extenddb-manifest.json\n", + hex::encode(sha2::Sha256::digest(garbage)) + ); + mem.insert( + &format!("{prefix}extenddb-manifest.sha256"), + sidecar.as_bytes(), + ); + } + "missing data file" => { + mem.objects + .lock() + .unwrap() + .remove(&format!("{prefix}{}", golden.data_files[1].key)); + } + "oversized file" => { + let key = format!("{prefix}{}", golden.data_files[1].key); + let bytes = usize::try_from(golden.data_files[1].bytes).unwrap() + 1; + mem.insert(&key, &vec![0u8; bytes]); + } + "foreign ARN" => { + // The golden backup's objects, whole, under another row's + // prefix: that row's manifest names the golden backup. + let other = arn_n(3); + let other_prefix = BackupArn::parse(&other).unwrap().store_prefix(); + let objects: Vec<(String, Vec)> = mem + .objects + .lock() + .unwrap() + .iter() + .filter(|(k, _)| k.starts_with(&prefix)) + .map(|(k, v)| (k.clone(), v.clone())) + .collect(); + for (k, v) in objects { + mem.objects.lock().unwrap().remove(&k); + mem.insert(&format!("{other_prefix}{}", &k[prefix.len()..]), &v); + } + fake.rows.lock().unwrap().remove(&dead); + fake.dead.lock().unwrap().remove(&dead); + let mut r = row(&other, BackupStatus::Creating); + r.heartbeat_at = 1234.5; + fake.rows.lock().unwrap().insert(other.clone(), r); + fake.dead.lock().unwrap().insert(other.clone()); + dead = other; + } + _ => {} + } + let objects_before = mem.keys().len(); + // Under the maintenance claim, so the FAILED-prefix duty of the same + // cycle does not run and the resolution's own effect is visible. + *fake.maintenance_held.lock().unwrap() = true; + let runtime = runtime_on(&mem, store()); + let report = cycle_on(&fake, &runtime).await; + assert_eq!( + report.creating_resolved, + [(dead.clone(), BackupStatus::Failed)], + "{shape}" + ); + assert!(report.failed_prefixes_deleted.is_empty()); + let row = fake.rows.lock().unwrap()[&dead].clone(); + assert_eq!(row.status, BackupStatus::Failed, "{shape}"); + assert_eq!( + row.failure_reason.as_deref(), + Some(reconciler::OWNER_DEAD_NO_MANIFEST) + ); + assert!(row.completed_at.is_some()); + assert_eq!(row.heartbeat_at, 1234.5); + assert_eq!(mem.keys().len(), objects_before, "{shape}: nothing deleted"); + assert!(mem.deletes.lock().unwrap().is_empty(), "{shape}"); + + // The next cycle without the claim: the FAILED row's heartbeat is + // stale (the dead owner's), the prefix lists objects, so it is + // deleted. + *fake.maintenance_held.lock().unwrap() = false; + let report = cycle_on(&fake, &runtime).await; + assert_eq!( + report.failed_prefixes_deleted, + std::slice::from_ref(&dead), + "{shape}" + ); + assert!(objects_under(&mem, &dead).is_empty(), "{shape}"); + assert_eq!(mem.keys(), [STORE_IDENTITY_KEY]); + } + + // A store error during the probe: the row is taken over and left + // CREATING, no status write, nothing deleted; the next cycle, with the + // store answering, resolves it. + let mem = Arc::new(MemStore::default()); + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + let fake = Arc::new(Fake::default()); + let dead = seed_dead_creating_with_manifest(&fake, &mem, true); + *mem.fail_gets_under.lock().unwrap() = Some(BackupArn::parse(&dead).unwrap().store_prefix()); + let runtime = runtime_on(&mem, store()); + let report = cycle_on(&fake, &runtime).await; + assert!(report.creating_resolved.is_empty()); + assert!( + report.creating_left_to_owner.is_empty(), + "a probe that failed is neither resolved nor left to an owner" + ); + assert_eq!( + fake.takeovers.lock().unwrap().as_slice(), + std::slice::from_ref(&dead) + ); + let row = fake.rows.lock().unwrap()[&dead].clone(); + assert_eq!( + row.status, + BackupStatus::Creating, + "left for the next cycle" + ); + assert!(fake.status_writes.lock().unwrap().is_empty()); + assert!(mem.deletes.lock().unwrap().is_empty()); + *mem.fail_gets_under.lock().unwrap() = None; + let report = cycle_on(&fake, &runtime).await; + assert_eq!(report.creating_resolved, [(dead, BackupStatus::Available)]); +} + +/// A dead CREATING row whose status write fails on the claim connection is +/// counted neither as resolved nor as left to its owner: the row stays +/// CREATING for the next cycle (its takeover done, one write attempted), +/// nothing deleted; with the connection back the next cycle resolves it. +#[tokio::test] +async fn a_dead_creating_row_whose_status_write_errors_is_left_for_the_next_cycle() { + let mem = Arc::new(MemStore::default()); + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + let fake = Arc::new(Fake::default()); + let dead = seed_dead_creating_with_manifest(&fake, &mem, true); + fake.status_write_error + .store(true, std::sync::atomic::Ordering::Release); + let runtime = runtime_on(&mem, store()); + let report = cycle_on(&fake, &runtime).await; + assert!(report.creating_resolved.is_empty(), "{report:?}"); + assert!(report.creating_left_to_owner.is_empty(), "{report:?}"); + assert!(!report.acted(), "nothing changed"); + assert_eq!( + fake.takeovers.lock().unwrap().as_slice(), + std::slice::from_ref(&dead) + ); + assert_eq!(writes(&fake), [(JobStatus::Available, None)]); + let row = fake.rows.lock().unwrap()[&dead].clone(); + assert_eq!(row.status, BackupStatus::Creating); + assert!(mem.deletes.lock().unwrap().is_empty()); + fake.status_write_error + .store(false, std::sync::atomic::Ordering::Release); + let report = cycle_on(&fake, &runtime).await; + assert_eq!(report.creating_resolved, [(dead, BackupStatus::Available)]); +} + +/// Within one cycle the dead-owner duty runs before the FAILED-prefix duty, +/// so a row the cycle just failed, whose heartbeat the dead owner left +/// stale, loses its prefix in that same cycle (the design states this as a +/// property of the cycle, not a convenience). +#[tokio::test] +async fn a_row_failed_by_the_cycle_loses_its_prefix_in_that_cycle() { + let mem = Arc::new(MemStore::default()); + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + let fake = Arc::new(Fake::default()); + let dead = seed_dead_creating_with_manifest(&fake, &mem, false); + assert!(!objects_under(&mem, &dead).is_empty()); + let runtime = runtime_on(&mem, store()); + let report = cycle_on(&fake, &runtime).await; + assert_eq!( + report.creating_resolved, + [(dead.clone(), BackupStatus::Failed)] + ); + assert_eq!( + report.failed_prefixes_deleted, + std::slice::from_ref(&dead), + "the same cycle" + ); + assert!(objects_under(&mem, &dead).is_empty()); + assert_eq!( + fake.rows.lock().unwrap()[&dead].status, + BackupStatus::Failed + ); +} + +/// A dead CREATING row whose status write matches no row (another +/// reconciler took the row over in the gap before this one's write, item +/// 29's race on MongoDB and Cassandra) is left to that owner: counted as +/// such and not as resolved, the row still CREATING under the other +/// generation, nothing deleted. +#[tokio::test] +async fn a_dead_creating_row_taken_over_by_another_is_left_to_that_owner() { + let mem = Arc::new(MemStore::default()); + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + let fake = Arc::new(Fake::default()); + let dead = seed_dead_creating_with_manifest(&fake, &mem, true); + fake.takeover_lost_to_another + .store(true, std::sync::atomic::Ordering::Release); + let runtime = runtime_on(&mem, store()); + let report = cycle_on(&fake, &runtime).await; + assert!(report.creating_resolved.is_empty()); + assert_eq!(report.creating_left_to_owner, std::slice::from_ref(&dead)); + assert!(!report.acted(), "nothing changed"); + let writes = fake.status_writes.lock().unwrap().clone(); + assert_eq!(writes.len(), 1); + assert_eq!(writes[0].to, JobStatus::Available); + let row = fake.rows.lock().unwrap()[&dead].clone(); + assert_eq!(row.status, BackupStatus::Creating); + assert_ne!(row.owner_generation, writes[0].generation); + assert!(mem.deletes.lock().unwrap().is_empty()); +} + +/// The deleting duties and their gates. A FAILED row's prefix is deleted +/// only once its heartbeat is older than `stale_job_after` and only while +/// no maintenance claim is held; a DELETED row's objects are swept; a FAILED +/// or DELETED row whose prefix is empty is removed once its `completed_at` +/// is older than `failed_row_retention` (a prefix emptied earlier in the +/// cycle counts), and kept while the retention has not passed or while its +/// prefix still lists objects (a FAILED row past its retention whose +/// heartbeat is fresh: no duty empties its prefix, so the tombstone duty +/// keeps the row); an AVAILABLE row is no tombstone, whatever its clock and +/// prefix. The dead-CREATING duty runs under the maintenance claim; +/// the deleting duties and the registrations sweep do not. With the store +/// unhealthy none of them runs. +#[tokio::test] +async fn the_reconciler_deletes_failed_and_deleted_prefixes_and_removes_tombstones_on_their_clocks() +{ + let mem = Arc::new(MemStore::default()); + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + let fake = Arc::new(Fake::default()); + let now = seconds_since_epoch(); + let stale_failed = arn_n(1); + let fresh_failed = arn_n(2); + let deleted_with_objects = arn_n(3); + let old_failed_tombstone = arn_n(4); + let old_deleted_tombstone = arn_n(5); + let young_tombstone = arn_n(6); + let old_tombstone_with_objects = arn_n(7); + let old_failed_with_objects_and_a_live_heartbeat = arn_n(8); + for (a, status, heartbeat_at, completed_at, objects) in [ + ( + &stale_failed, + BackupStatus::Failed, + now - 1000.0, + Some(now - 1000.0), + true, + ), + (&fresh_failed, BackupStatus::Failed, now, Some(now), true), + ( + &deleted_with_objects, + BackupStatus::Deleted, + now - 1000.0, + Some(now - 1000.0), + true, + ), + ( + &old_failed_tombstone, + BackupStatus::Failed, + now - 1000.0, + Some(now - 1000.0), + false, + ), + ( + &old_deleted_tombstone, + BackupStatus::Deleted, + now - 1000.0, + Some(now - 1000.0), + false, + ), + ( + &young_tombstone, + BackupStatus::Failed, + now - 1000.0, + Some(now - 10.0), + false, + ), + ( + &old_tombstone_with_objects, + BackupStatus::Deleted, + now - 1000.0, + Some(now - 1000.0), + true, + ), + ( + &old_failed_with_objects_and_a_live_heartbeat, + BackupStatus::Failed, + now, + Some(now - 1000.0), + true, + ), + ] { + let mut r = row(a, status); + r.heartbeat_at = heartbeat_at; + r.completed_at = completed_at; + fake.rows.lock().unwrap().insert(a.clone(), r); + if objects { + let prefix = BackupArn::parse(a).unwrap().store_prefix(); + mem.insert(&format!("{prefix}data/000001.json.gz"), b"x"); + } + } + let retention = Duration::from_secs(600); + let settings = JobSettings { + failed_row_retention: retention, + ..job_settings() + }; + + // Unhealthy (the root's identity file names another store while the + // store answers): no duty over the store's rows runs, the dead-CREATING + // one included. + let dead = seed_dead_creating_with_manifest(&fake, &mem, true); + mem.insert( + STORE_IDENTITY_KEY, + &identity_file("another-store", "catalog-1"), + ); + let runtime = runtime_on(&mem, store()).with_job_settings(settings); + let report = cycle_on(&fake, &runtime).await; + assert_eq!(report.store_healthy, Some(false)); + assert!(report.creating_resolved.is_empty()); + assert!(report.failed_prefixes_deleted.is_empty()); + assert!(report.deleted_prefixes_swept.is_empty()); + assert!(report.tombstones_removed.is_empty()); + assert!(mem.deletes.lock().unwrap().is_empty()); + assert_eq!(fake.rows.lock().unwrap().len(), 9); + assert_eq!( + fake.rows.lock().unwrap()[&dead].status, + BackupStatus::Creating + ); + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + + // Under the maintenance claim, the store healthy again: the + // dead-CREATING duty runs, the deleting duties and the registrations + // sweep (which the backend has something for) do not. + fake.swept_registrations.lock().unwrap().push(arn_n(9)); + *fake.maintenance_held.lock().unwrap() = true; + let report = cycle_on(&fake, &runtime).await; + assert_eq!(report.store_healthy, Some(true)); + assert!(report.maintenance_held); + assert_eq!(report.creating_resolved, [(dead, BackupStatus::Available)]); + assert!(report.failed_prefixes_deleted.is_empty()); + assert!(report.deleted_prefixes_swept.is_empty()); + assert!(report.tombstones_removed.is_empty()); + assert!( + report.registrations_swept.is_empty(), + "the sweep does not run under the claim" + ); + assert!(mem.deletes.lock().unwrap().is_empty()); + assert_eq!(fake.rows.lock().unwrap().len(), 9); + + // An AVAILABLE row past the retention with an empty prefix: a tombstone + // is a FAILED or DELETED row and nothing else. It enters the catalog + // here, older than the row the cycle above resolved, because the + // store's check heads the newest AVAILABLE row's manifest and this row + // has none. + let old_available_with_an_empty_prefix = + format!("arn:aws:dynamodb:{REGION}:{ACCOUNT}:table/Music/backup/01700000000000-00000000"); + let mut r = row(&old_available_with_an_empty_prefix, BackupStatus::Available); + r.heartbeat_at = now - 1000.0; + r.completed_at = Some(now - 1000.0); + fake.rows + .lock() + .unwrap() + .insert(old_available_with_an_empty_prefix.clone(), r); + + // The claim released: each duty on its clock. + *fake.maintenance_held.lock().unwrap() = false; + let report = cycle_on(&fake, &runtime).await; + assert_eq!( + report.failed_prefixes_deleted, + std::slice::from_ref(&stale_failed) + ); + assert_eq!( + report.deleted_prefixes_swept, + [ + deleted_with_objects.clone(), + old_tombstone_with_objects.clone() + ] + ); + // The tombstone duty runs after the two deleting duties, so a prefix + // they emptied this cycle counts as empty here. + let mut removed = report.tombstones_removed.clone(); + removed.sort(); + assert_eq!( + removed, + [ + stale_failed.clone(), + deleted_with_objects.clone(), + old_failed_tombstone.clone(), + old_deleted_tombstone.clone(), + old_tombstone_with_objects.clone(), + ] + ); + assert_eq!(report.registrations_swept, [arn_n(9)]); + assert!(report.acted()); + assert_eq!( + objects_under(&mem, &fresh_failed).len(), + 1, + "a FAILED row with a live heartbeat keeps its prefix" + ); + assert!(objects_under(&mem, &stale_failed).is_empty()); + assert!(objects_under(&mem, &deleted_with_objects).is_empty()); + assert_eq!( + objects_under(&mem, &old_failed_with_objects_and_a_live_heartbeat).len(), + 1, + "no duty empties the prefix of a FAILED row with a live heartbeat" + ); + { + let rows = fake.rows.lock().unwrap(); + assert!(rows.contains_key(&fresh_failed), "its heartbeat is live"); + assert!( + rows.contains_key(&young_tombstone), + "its retention has not passed" + ); + assert!( + rows.contains_key(&old_failed_with_objects_and_a_live_heartbeat), + "its prefix still lists an object" + ); + assert!( + rows.contains_key(&old_available_with_an_empty_prefix), + "AVAILABLE is not a tombstone status" + ); + assert_eq!( + rows[&old_available_with_an_empty_prefix].status, + BackupStatus::Available + ); + assert_eq!(rows.len(), 5, "the four kept and the resolved CREATING row"); + } + assert!( + !fake + .remove_attempts + .lock() + .unwrap() + .contains(&old_available_with_an_empty_prefix), + "the duty asks the catalog to remove FAILED and DELETED rows only; the backends would \ + refuse an AVAILABLE one, and the duty does not lean on that" + ); + + // The next cycle has nothing left to do. + let report = cycle_on(&fake, &runtime).await; + assert!(report.tombstones_removed.is_empty()); + assert!(report.deleted_prefixes_swept.is_empty()); + assert!(report.failed_prefixes_deleted.is_empty()); + assert!(report.creating_resolved.is_empty()); +} + +// --- the store-backed restore --------------------------------------------------- + +use extenddb_core::backup_format::{ + BackupDocuments, DataFileEntry, EXTENDDB_MANIFEST_FILE, ExtenddbManifest, ItemOrder, KeyOrder, + KeyRangeTracker, MANIFEST_CHECKSUM_FILE, ManifestHeader, SnapshotDescriptor, SourceTableSchema, + data_file_key, encode_data_file, +}; +use extenddb_core::metrics::MetricName; +use extenddb_storage::snapshot::source_table_schema; + +/// The `Music` table's source schema as a manifest carries it. +fn music_source_table() -> SourceTableSchema { + source_table_schema(&music_table(), None).unwrap() +} + +/// `music_source_table` as an on-demand table (vector indexes need one). +fn on_demand_source_table() -> SourceTableSchema { + let mut source_table = music_source_table(); + source_table.billing_mode = BillingMode::PayPerRequest; + source_table.provisioned_throughput = None; + for gsi in &mut source_table.global_secondary_indexes { + gsi.provisioned_throughput = None; + } + source_table +} + +/// A backup of `source_table` under `mem` for `arn`, as the writer leaves +/// one: one data file per entry of `files`, encoded with the format's +/// encoder; key-ordered with the tracker's ranges when `key_ordered`; the +/// four documents built by `BackupDocuments::build`; and the row AVAILABLE +/// in `fake` with the manifest's `source_table`. Returns the manifest. +fn seed_manifest_backup( + fake: &Fake, + mem: &MemStore, + arn: &str, + mut source_table: SourceTableSchema, + files: &[Vec], + key_ordered: bool, +) -> ExtenddbManifest { + let parsed = BackupArn::parse(arn).unwrap(); + let prefix = parsed.store_prefix(); + let mut tracker = key_ordered.then(|| { + KeyRangeTracker::new( + &KeyOrder::resolve( + &source_table.key_schema, + &source_table.attribute_definitions, + ) + .unwrap(), + ) + }); + let mut entries = Vec::new(); + let mut total = 0; + for (i, items) in files.iter().enumerate() { + let (bytes, checksums) = encode_data_file(items.iter().cloned()).unwrap(); + let range = tracker.as_mut().map(|t| { + for item in items { + t.push(item).unwrap(); + } + t.next_file().expect("a file with items has a range") + }); + let key = data_file_key(u32::try_from(i + 1).unwrap()); + mem.insert(&format!("{prefix}{key}"), &bytes); + total += items.len(); + entries.push((DataFileEntry::from_checksums(key, &checksums, range), None)); + } + source_table.item_count = i64::try_from(total).unwrap(); + seed_manifest_documents(fake, mem, arn, source_table, entries, key_ordered) +} + +/// The documents of a backup whose data files are already under the prefix, +/// from `entries`, and the AVAILABLE row; the manifest as built. +fn seed_manifest_documents( + fake: &Fake, + mem: &MemStore, + arn: &str, + source_table: SourceTableSchema, + entries: Vec<(DataFileEntry, Option)>, + key_ordered: bool, +) -> ExtenddbManifest { + let parsed = BackupArn::parse(arn).unwrap(); + let prefix = parsed.store_prefix(); + let total: u64 = entries.iter().map(|(e, _)| e.item_count).sum(); + let docs = BackupDocuments::build( + ManifestHeader { + backup_arn: parsed, + backup_name: "nightly".to_owned(), + source_backend: "test".to_owned(), + source_extenddb_version: "0.0.0".to_owned(), + snapshot: SnapshotDescriptor { + kind: "test".to_owned(), + marker: String::new(), + }, + item_order: if key_ordered { + ItemOrder::KeyOrdered + } else { + ItemOrder::Unordered + }, + source_table, + }, + entries, + "", + &store().store_root, + ) + .unwrap(); + for (name, bytes) in docs.documents() { + mem.insert(&format!("{prefix}{name}"), bytes); + } + let manifest = docs.manifest().clone(); + let mut available = row(arn, BackupStatus::Available); + available.source_table_details_json = serde_json::to_value(&manifest.source_table).unwrap(); + available.size_bytes = manifest.backup_size_bytes; + available.item_count = i64::try_from(total).unwrap(); + fake.rows.lock().unwrap().insert(arn.to_owned(), available); + manifest +} + +/// `n` songs from `from`, in key order. +fn songs_from(from: usize, n: usize) -> Vec { + (from..from + n) + .map(|i| song(&format!("artist-{i:05}"), &format!("title-{i:05}"))) + .collect() +} + +/// A healthy runtime on `mem` with a spool at `spool` (an S3 store's shape), +/// the job settings and a collector. +fn spooled_runtime(mem: &Arc, spool: &std::path::Path) -> BackupRuntime { + runtime() + .with_store( + ConfiguredStore::new( + StoreKind::S3, + "bucket/prefix".to_owned(), + Arc::clone(mem) as Arc, + Some(Spool { + path: spool.to_path_buf(), + spool_host: "host-a:/var/spool".to_owned(), + }), + Ok(StoreIdentity { + store_kind: StoreKind::S3, + store_root: "bucket/prefix".to_owned(), + store_id: "store-1".to_owned(), + }), + ) + .with_manifest_names("bucket".to_owned(), "prefix/".to_owned()), + ) + .with_job_settings(job_settings()) + .with_metrics(Arc::new(extenddb_core::metrics::MetricsCollector::new())) +} + +/// The `s3_row` of `arn`: the row with the spooled runtime's identity. +fn s3_identity() -> StoreIdentity { + StoreIdentity { + store_kind: StoreKind::S3, + store_root: "bucket/prefix".to_owned(), + store_id: "store-1".to_owned(), + } +} + +/// The `RestoreJobs` outcomes the collector drained, with their count. +fn restore_outcomes(runtime: &BackupRuntime) -> Vec<(String, f64)> { + let mut outcomes: Vec<(String, f64)> = runtime + .metrics() + .unwrap() + .drain(Duration::ZERO) + .into_iter() + .filter(|b| b.metric == MetricName::RestoreJobs) + .map(|b| (b.operation, b.sum)) + .collect(); + outcomes.sort_by(|a, b| a.0.cmp(&b.0)); + outcomes +} + +/// Every metric bucket the collector drained, as `(name, operation, sum)`. +fn drained(runtime: &BackupRuntime) -> Vec<(MetricName, String, f64)> { + runtime + .metrics() + .unwrap() + .drain(Duration::ZERO) + .into_iter() + .map(|b| (b.metric, b.operation, b.sum)) + .collect() +} + +/// Wait until `n` `bulk_load` calls have begun (a job waiting at the gate +/// counts). +async fn wait_for_bulk_loads_started(fake: &Fake, n: usize) { + let deadline = tokio::time::Instant::now() + Duration::from_secs(30); + while fake + .bulk_loads_started + .load(std::sync::atomic::Ordering::Acquire) + < n + { + assert!(tokio::time::Instant::now() < deadline, "no bulk_load began"); + tokio::time::sleep(Duration::from_millis(5)).await; + } +} + +/// The status writes the restore claims received, as `(from, to, reason)`. +fn restore_writes(fake: &Fake) -> Vec<(JobStatus, JobStatus, Option)> { + fake.restore_writes + .lock() + .unwrap() + .iter() + .map(|w| (w.from, w.to, w.fields.failure_reason.clone())) + .collect() +} + +/// The store keys `mem` saw `get` for, without the prefix. +fn gets_under(mem: &MemStore, arn: &str) -> Vec { + let prefix = BackupArn::parse(arn).unwrap().store_prefix(); + mem.gets() + .into_iter() + .filter_map(|k| k.strip_prefix(&prefix).map(str::to_owned)) + .collect() +} + +/// The happy path. The request: the row passes, the permit is taken, the +/// sidecar and manifest are read, the registration carries the manifest's +/// table (key schema, GSI, billing) and the row's store, the maintenance +/// claim is read once after it, and the response is CREATING with the +/// manifest's provenance in its `RestoreSummary`. The job: every file is +/// read once to hash it (in order), then once more to decode it, each file +/// goes to one `bulk_load` from item 0 with `reapplying` false, the items +/// come back whole and in order, the target is counted, the record is DONE +/// and the table ACTIVE with its GSI, no status write was needed, the slot +/// is free; `RestoreJobs{succeeded}`, `RestoreBytesDownloaded` of every +/// file's bytes, `WorkerLastSuccess{RestoreJob}`. +#[tokio::test] +async fn a_store_backed_restore_answers_creating_and_its_job_loads_the_backup() { + let (lines, _guard) = LogLines::capture(); + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let files = [songs_from(0, 3), songs_from(3, 2), songs_from(5, 4)]; + let manifest = seed_manifest_backup( + &fake, + &mem, + &arn(ACCOUNT), + music_source_table(), + &files, + true, + ); + let runtime = job_runtime(&mem); + + let response = restore_on(&fake, &runtime, &restore_body(json!({}))) + .await + .unwrap(); + let table = &response["TableDescription"]; + assert_eq!(table["TableStatus"], "CREATING"); + assert_eq!(table["TableName"], "MusicRestored"); + assert_eq!(table["RestoreSummary"]["SourceBackupArn"], arn(ACCOUNT)); + assert_eq!(table["RestoreSummary"]["SourceTableArn"], source_arn()); + assert_eq!( + table["RestoreSummary"]["RestoreDateTime"], + json!(manifest.backup_creation_date_time) + ); + assert_eq!(table["RestoreSummary"]["RestoreInProgress"], true); + let claims = fake.restore_claims.lock().unwrap().clone(); + assert_eq!(claims.len(), 1); + assert_eq!(claims[0].0, ACCOUNT); + let input = &claims[0].1; + assert_eq!(input.table_name, "MusicRestored"); + assert_eq!(input.key_schema, music_table().key_schema); + assert_eq!(input.billing_mode, Some(BillingMode::Provisioned)); + assert_eq!(gsi_names(input), ["GenreIndex"]); + assert_eq!(input.vector_indexes, None); + let record = fake.restore_job("MusicRestored").unwrap(); + assert_eq!(record.kind, RestoreKind::Manifest); + assert_eq!(record.store, Some(store())); + assert_eq!(record.spool_host, None, "a filesystem store has no spool"); + assert_eq!( + fake.maintenance_checks.lock().unwrap().as_slice(), + [job_settings().stale_job_after], + "the maintenance claim was read once, after the registration" + ); + + wait_for_jobs(&runtime).await; + assert_eq!( + fake.bulk_load_files(), + [(0, 0, false, 3), (1, 0, false, 2), (2, 0, false, 4)] + ); + let all: Vec = files.iter().flatten().cloned().collect(); + assert_eq!(fake.restored_items("MusicRestored"), all); + let record = fake.restore_job("MusicRestored").unwrap(); + assert_eq!(record.status, JobStatus::Done); + assert!(record.completed_at.is_some()); + assert_eq!(record.file_index, Some(2)); + assert_eq!(record.items_applied, Some(4)); + let desc = fake.tables.lock().unwrap()["MusicRestored"].clone(); + assert_eq!(desc.table_status, TableStatus::Active); + assert_eq!( + desc.item_count, 0, + "the restored table's ItemCount is 0 at ACTIVE, as the service reports it" + ); + assert!( + restore_writes(&fake).is_empty(), + "the flip is the backend's one write" + ); + assert_eq!( + fake.finish_sizes.lock().unwrap().as_slice(), + [0], + "the restored table's TableSizeBytes is 0 at ACTIVE, as the service reports it" + ); + assert_eq!(runtime.free_slots(), 4); + let reads = gets_under(&mem, &arn(ACCOUNT)); + assert_eq!( + reads, + [ + MANIFEST_CHECKSUM_FILE, + EXTENDDB_MANIFEST_FILE, + "data/000001.json.gz", + "data/000002.json.gz", + "data/000003.json.gz", + "data/000001.json.gz", + "data/000002.json.gz", + "data/000003.json.gz", + ], + "every file hashed before any is decoded, each read twice in place" + ); + let bytes: u64 = manifest.data_files.iter().map(|f| f.bytes).sum(); + let metrics = drained(&runtime); + assert_eq!( + metrics + .iter() + .filter(|(m, _, _)| *m == MetricName::RestoreJobs) + .map(|(_, op, sum)| (op.clone(), *sum)) + .collect::>(), + [("succeeded".to_owned(), 1.0)] + ); + // Summed across the drain's buckets (one per wall-clock minute). + let downloaded: f64 = metrics + .iter() + .filter(|(m, _, _)| *m == MetricName::RestoreBytesDownloaded) + .map(|(_, _, sum)| *sum) + .sum(); + assert!( + (downloaded - bytes as f64).abs() < f64::EPSILON, + "{metrics:?}" + ); + assert!( + metrics + .iter() + .any(|(m, op, _)| *m == MetricName::WorkerLastSuccess && op == "RestoreJob"), + "{metrics:?}" + ); + let started = lines.containing(": started, 3 data file(s), 9 items, key-ordered"); + assert_eq!(started.len(), 1, "{started:?}"); + let active = lines.containing(": ACTIVE, 9 items (9 applied by this run), "); + assert_eq!(active.len(), 1, "{active:?}"); +} + +/// The request's order on the manifest path, each refusal before the next +/// step and the registration: a table of the target's name is +/// `TableAlreadyExists` with no row read; a missing sidecar or manifest is +/// `BackupNotFound` with the operator's line naming `--prune-missing`; a +/// sidecar that does not match, a manifest naming another backup or another +/// table are `ValidationException`s; none of them registers anything or +/// takes a slot for longer than the request. +#[tokio::test] +async fn the_manifest_request_refuses_in_order_before_the_registration() { + let (lines, _guard) = LogLines::capture(); + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + seed_manifest_backup( + &fake, + &mem, + &arn(ACCOUNT), + music_source_table(), + &[songs(2)], + true, + ); + let runtime = job_runtime(&mem); + let prefix = BackupArn::parse(&arn(ACCOUNT)).unwrap().store_prefix(); + + // A taken name: before the row. + fake.tables.lock().unwrap().insert( + "MusicRestored".to_owned(), + TableDescription { + table_name: "MusicRestored".to_owned(), + table_status: TableStatus::Active, + ..Default::default() + }, + ); + let err = restore_on(&fake, &runtime, &restore_body(json!({}))) + .await + .unwrap_err(); + assert!( + matches!(err, DynamoDbError::TableAlreadyExistsException(_)), + "{err:?}" + ); + // The name is checked after the row is read (the service's order: a + // missing backup answers first) and before any store read. + assert_eq!(fake.row_reads.lock().unwrap().len(), 1); + assert!(mem.gets().is_empty()); + fake.row_reads.lock().unwrap().clear(); + fake.tables.lock().unwrap().remove("MusicRestored"); + + // A missing sidecar, then a missing manifest. + let sidecar_key = format!("{prefix}{MANIFEST_CHECKSUM_FILE}"); + let manifest_key = format!("{prefix}{EXTENDDB_MANIFEST_FILE}"); + let sidecar = mem.objects.lock().unwrap().remove(&sidecar_key).unwrap(); + let err = restore_on(&fake, &runtime, &restore_body(json!({}))) + .await + .unwrap_err(); + assert!( + matches!(err, DynamoDbError::BackupNotFoundException(ref m) if m.contains(&arn(ACCOUNT))), + "{err:?}" + ); + let prune = lines.containing("backup sync --prune-missing"); + assert_eq!(prune.len(), 1, "{prune:?}"); + assert!(prune[0].contains(MANIFEST_CHECKSUM_FILE), "{}", prune[0]); + mem.insert(&sidecar_key, &sidecar); + let manifest = mem.objects.lock().unwrap().remove(&manifest_key).unwrap(); + let err = restore_on(&fake, &runtime, &restore_body(json!({}))) + .await + .unwrap_err(); + assert!( + matches!(err, DynamoDbError::BackupNotFoundException(_)), + "{err:?}" + ); + assert_eq!(lines.containing("backup sync --prune-missing").len(), 2); + + // A sidecar that does not match the manifest. + mem.insert(&manifest_key, &manifest); + mem.insert( + &sidecar_key, + b"0000000000000000000000000000000000000000000000000000000000000000\n", + ); + let msg = validation_message( + restore_on(&fake, &runtime, &restore_body(json!({}))) + .await + .unwrap_err(), + ); + assert!(msg.contains("checksum sidecar does not match"), "{msg}"); + mem.insert(&sidecar_key, &sidecar); + + // A manifest naming another backup (another account's, the same id), + // with a sidecar that matches it. + let other = arn("999999999999"); + let mut parsed: Value = serde_json::from_slice(&manifest).unwrap(); + parsed["backup_arn"] = json!(other); + parsed["source_table"]["table_arn"] = json!(format!( + "arn:aws:dynamodb:{REGION}:999999999999:table/Music" + )); + let other_bytes = serde_json::to_vec(&parsed).unwrap(); + let other_sidecar = { + use sha2::Digest; + format!( + "{:x} {EXTENDDB_MANIFEST_FILE}\n", + sha2::Sha256::digest(&other_bytes) + ) + }; + mem.insert(&manifest_key, &other_bytes); + mem.insert(&sidecar_key, other_sidecar.as_bytes()); + let msg = validation_message( + restore_on(&fake, &runtime, &restore_body(json!({}))) + .await + .unwrap_err(), + ); + assert!( + msg.contains("names another backup") && msg.contains(&other), + "{msg}" + ); + + // A manifest whose source table is not the ARN's: the reader's own + // identity rule refuses it at the parse. + let mut parsed: Value = serde_json::from_slice(&manifest).unwrap(); + parsed["source_table"]["table_name"] = json!("Other"); + let renamed = serde_json::to_vec(&parsed).unwrap(); + let renamed_sidecar = { + use sha2::Digest; + format!( + "{:x} {EXTENDDB_MANIFEST_FILE}\n", + sha2::Sha256::digest(&renamed) + ) + }; + mem.insert(&manifest_key, &renamed); + mem.insert(&sidecar_key, renamed_sidecar.as_bytes()); + let msg = validation_message( + restore_on(&fake, &runtime, &restore_body(json!({}))) + .await + .unwrap_err(), + ); + assert!( + msg.contains("cannot restore from") + && msg.contains("names table \"Music\" but source_table.table_name is \"Other\""), + "{msg}" + ); + + assert!( + fake.restore_claims.lock().unwrap().is_empty(), + "nothing was registered" + ); + assert_eq!(runtime.free_slots(), 4, "every refusal returned its slot"); + assert_eq!(runtime.running_jobs(), 0); +} + +/// The maintenance claim is checked after the registration, not before: a +/// held claim leaves a record that was FAILED `maintenance` from RUNNING +/// under the job's claim, its target removed, and the request answered +/// `LimitExceededException`; the registration ran (the row is what the +/// command's wait sees). A claim that cannot be read fails the record with +/// the read's error and answers it. +#[tokio::test] +async fn a_manifest_restore_under_a_maintenance_claim_fails_its_record_after_registering() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + seed_manifest_backup( + &fake, + &mem, + &arn(ACCOUNT), + music_source_table(), + &[songs(2)], + true, + ); + let runtime = job_runtime(&mem); + *fake.maintenance_held.lock().unwrap() = true; + let err = restore_on(&fake, &runtime, &restore_body(json!({}))) + .await + .unwrap_err(); + assert!( + matches!(err, DynamoDbError::LimitExceededException(ref m) if m.contains("maintenance claim")), + "{err:?}" + ); + wait_for_jobs(&runtime).await; + assert_eq!( + fake.restore_claims.lock().unwrap().len(), + 1, + "the registration ran first" + ); + assert_eq!( + restore_writes(&fake), + [( + JobStatus::Running, + JobStatus::Failed, + Some(job::MAINTENANCE.to_owned()) + )] + ); + assert_eq!(fake.removed_targets.lock().unwrap().len(), 1); + assert!( + fake.restore_job("MusicRestored").is_none(), + "the target and its record are gone" + ); + assert!(!fake.tables.lock().unwrap().contains_key("MusicRestored")); + assert!(fake.bulk_loads.lock().unwrap().is_empty()); + assert_eq!(runtime.free_slots(), 4); + // A FAILED record is a job end: the outcome beside the worker error. + let metrics = drained(&runtime); + assert_eq!( + metrics + .iter() + .filter(|(m, _, _)| *m == MetricName::RestoreJobs) + .map(|(_, op, sum)| (op.clone(), *sum)) + .collect::>(), + [("failed".to_owned(), 1.0)], + "{metrics:?}" + ); + assert!( + metrics + .iter() + .any(|(m, op, _)| *m == MetricName::WorkerErrorCount && op == "RestoreJob"), + "{metrics:?}" + ); + assert!( + metrics + .iter() + .any(|(m, op, _)| *m == MetricName::RestoreJobDuration && op == "failed"), + "{metrics:?}" + ); + + *fake.maintenance_held.lock().unwrap() = false; + *fake.maintenance_error.lock().unwrap() = Some("pool exhausted".to_owned()); + let err = restore_on(&fake, &runtime, &restore_body(json!({}))) + .await + .unwrap_err(); + assert!( + matches!(err, DynamoDbError::InternalServerError(_)), + "{err:?}" + ); + wait_for_jobs(&runtime).await; + let writes = restore_writes(&fake); + assert_eq!(writes.len(), 2); + assert!( + writes[1].2.as_deref().unwrap().contains("pool exhausted"), + "{writes:?}" + ); +} + +/// The registration's own refusals reach the client as the legacy +/// backend's do: a name taken under the registration is +/// `TableAlreadyExistsException`, a CREATING restore target +/// `TableInUseException`, a backup deleted under the lock +/// `BackupNotFoundException`, a backup in use `BackupInUseException`; the +/// slot is returned each time and no job runs. With no slot free the +/// request is `LimitExceededException` before any store read. +#[tokio::test] +async fn the_registrations_refusals_keep_their_classes_and_return_the_slot() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + seed_manifest_backup( + &fake, + &mem, + &arn(ACCOUNT), + music_source_table(), + &[songs(2)], + true, + ); + let runtime = job_runtime(&mem); + type Expected = fn(&DynamoDbError) -> bool; + let cases: [(StorageError, Expected); 4] = [ + ( + StorageError::TableAlreadyExists("MusicRestored".to_owned()), + |e| matches!(e, DynamoDbError::TableAlreadyExistsException(_)), + ), + ( + StorageError::TableBeingRestored { + table_name: "MusicRestored".to_owned(), + backup_arn: arn(ACCOUNT), + }, + |e| matches!(e, DynamoDbError::TableInUseException(_)), + ), + ( + extenddb_storage::backup_catalog::backup_not_found(&arn(ACCOUNT)), + |e| matches!(e, DynamoDbError::BackupNotFoundException(_)), + ), + ( + extenddb_storage::backup_catalog::backup_in_use("Earlier", &arn(ACCOUNT)), + |e| matches!(e, DynamoDbError::BackupInUseException(_)), + ), + ]; + for (error, expected) in cases { + let shown = format!("{error:?}"); + *fake.claim_restore_error.lock().unwrap() = Some(error); + let err = restore_on(&fake, &runtime, &restore_body(json!({}))) + .await + .unwrap_err(); + assert!(expected(&err), "{shown}: {err:?}"); + wait_for_jobs(&runtime).await; + assert_eq!(runtime.free_slots(), 4, "{shown}"); + } + assert!(fake.bulk_loads.lock().unwrap().is_empty()); + assert!(restore_writes(&fake).is_empty()); + + // No wait for a slot: the refusal is what the cell reads. + let full = job_runtime(&mem).with_permit_wait(Duration::ZERO); + let mut held = Vec::new(); + for _ in 0..4 { + held.push(full.take_permit().await.unwrap()); + } + let before = mem.gets().len(); + let err = restore_on(&fake, &full, &restore_body(json!({}))) + .await + .unwrap_err(); + assert!( + matches!(err, DynamoDbError::LimitExceededException(_)), + "{err:?}" + ); + assert_eq!(mem.gets().len(), before, "no store read before the permit"); +} + +/// A second restore while the first's target is CREATING is +/// `BackupInUseException` naming the backup, through the catalog's in-use +/// rule the registration wrote; a second restore into the same name, of the +/// same backup or of another, is `TableInUseException` (the target name is +/// checked once the backup is known to exist and before its in-use rule, the +/// service's measured order); once the first is ACTIVE the backup is free +/// again. +#[tokio::test] +async fn a_backup_being_restored_is_in_use_until_its_target_is_active() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + seed_manifest_backup( + &fake, + &mem, + &arn(ACCOUNT), + music_source_table(), + &[songs(2)], + true, + ); + seed_manifest_backup( + &fake, + &mem, + &arn_n(2), + music_source_table(), + &[songs(2)], + true, + ); + let runtime = job_runtime(&mem); + let gate = closed_gate(); + *fake.bulk_load_gate.lock().unwrap() = Some(Arc::clone(&gate)); + restore_on(&fake, &runtime, &restore_body(json!({}))) + .await + .unwrap(); + wait_for_bulk_loads_started(&fake, 1).await; + assert_eq!( + runtime.free_slots(), + 3, + "the job holds its slot past the reply, until it ends" + ); + let err = restore_on( + &fake, + &runtime, + &restore_body(json!({ "TargetTableName": "Second" })), + ) + .await + .unwrap_err(); + // The service's text names the backup, not the target (the log line + // names the target). + let in_use = extenddb_storage::backup_catalog::backup_in_use_message(&arn(ACCOUNT)); + assert!( + matches!(err, DynamoDbError::BackupInUseException(ref m) if *m == in_use), + "{err:?}" + ); + assert_eq!(runtime.free_slots(), 3, "a refusal returns its own slot"); + let err = restore_on(&fake, &runtime, &restore_body(json!({}))) + .await + .unwrap_err(); + assert!( + matches!(err, DynamoDbError::TableInUseException(_)), + "the same backup into its own CREATING target (measured on DynamoDB): {err:?}" + ); + let err = restore_on( + &fake, + &runtime, + &restore_body(json!({ "BackupArn": arn_n(2) })), + ) + .await + .unwrap_err(); + assert!( + matches!(err, DynamoDbError::TableInUseException(_)), + "another backup into the name being restored into: {err:?}" + ); + gate.add_permits(1); + wait_for_jobs(&runtime).await; + assert_eq!( + fake.restore_job("MusicRestored").unwrap().status, + JobStatus::Done + ); + assert_eq!( + runtime.free_slots(), + 4, + "the slot is free once the job ended" + ); + *fake.bulk_load_gate.lock().unwrap() = None; + restore_on( + &fake, + &runtime, + &restore_body(json!({ "TargetTableName": "Second" })), + ) + .await + .unwrap(); + wait_for_jobs(&runtime).await; + assert_eq!(fake.restored_items("Second").len(), 2); +} + +/// The tamper cell: file 2 of 3 does not hash to the manifest's digest. +/// Every file is verified before any is loaded, so no `bulk_load` is +/// called at all, the record is FAILED naming the file, the target is +/// removed (the FAILED write matched), and the failed outcome is recorded. +#[tokio::test] +async fn a_corrupt_second_file_fails_before_any_load() { + let (lines, _guard) = LogLines::capture(); + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let files = [songs_from(0, 3), songs_from(3, 3), songs_from(6, 3)]; + seed_manifest_backup( + &fake, + &mem, + &arn(ACCOUNT), + music_source_table(), + &files, + true, + ); + let prefix = BackupArn::parse(&arn(ACCOUNT)).unwrap().store_prefix(); + let key = format!("{prefix}data/000002.json.gz"); + let mut bytes = mem.objects.lock().unwrap()[&key].clone(); + let last = bytes.len() - 1; + bytes[last] ^= 0xff; + mem.insert(&key, &bytes); + let runtime = job_runtime(&mem); + restore_on(&fake, &runtime, &restore_body(json!({}))) + .await + .unwrap(); + wait_for_jobs(&runtime).await; + assert!( + fake.bulk_loads.lock().unwrap().is_empty(), + "no file was loaded" + ); + let writes = restore_writes(&fake); + assert_eq!(writes.len(), 1); + assert_eq!( + (writes[0].0, writes[0].1), + (JobStatus::Running, JobStatus::Failed) + ); + let reason = writes[0].2.clone().unwrap(); + assert!( + reason.starts_with("data file data/000002.json.gz does not match the manifest: "), + "{reason}" + ); + assert_eq!(fake.removed_targets.lock().unwrap().len(), 1); + assert!(fake.restore_job("MusicRestored").is_none()); + assert!(!fake.tables.lock().unwrap().contains_key("MusicRestored")); + assert_eq!( + gets_under(&mem, &arn(ACCOUNT)).len(), + 4, + "sidecar, manifest, files 1 and 2" + ); + let metrics = drained(&runtime); + assert_eq!( + metrics + .iter() + .filter(|(m, _, _)| *m == MetricName::RestoreJobs) + .map(|(_, op, sum)| (op.clone(), *sum)) + .collect::>(), + [("failed".to_owned(), 1.0)] + ); + assert!( + metrics + .iter() + .any(|(m, op, sum)| *m == MetricName::WorkerErrorCount + && op == "RestoreJob" + && (*sum - 1.0).abs() < f64::EPSILON), + "a FAILED job is a worker error: {metrics:?}" + ); + let failed = lines.containing(": FAILED (data file data/000002.json.gz does not match"); + assert_eq!(failed.len(), 1, "{failed:?}"); + assert!( + failed[0].ends_with("the target was removed"), + "{}", + failed[0] + ); +} + +/// A transport failure at the `get` and one mid-body are each retried +/// after a backoff and the restore completes; the store's error is counted +/// once per failed attempt; a denial is not retried and fails the restore +/// naming the file; a file missing from the store fails it at once. +#[tokio::test] +async fn transport_failures_are_retried_and_other_failures_are_not() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let files = [songs_from(0, 2), songs_from(2, 2)]; + seed_manifest_backup( + &fake, + &mem, + &arn(ACCOUNT), + music_source_table(), + &files, + true, + ); + let runtime = job_runtime(&mem); + mem.fail_gets_with(GetFailure::Transport, 1); + restore_on(&fake, &runtime, &restore_body(json!({}))) + .await + .unwrap(); + // The first data-file get failed; its retry, the next data-file get, + // stops mid-body. + let deadline = tokio::time::Instant::now() + Duration::from_secs(10); + while gets_under(&mem, &arn(ACCOUNT)).len() < 3 { + assert!(tokio::time::Instant::now() < deadline); + tokio::time::sleep(Duration::from_millis(5)).await; + } + mem.fail_get_after_bytes(5); + wait_for_jobs(&runtime).await; + assert_eq!( + fake.restore_job("MusicRestored").unwrap().status, + JobStatus::Done + ); + assert_eq!(fake.restored_items("MusicRestored").len(), 4); + let reads = gets_under(&mem, &arn(ACCOUNT)); + assert_eq!( + reads, + [ + MANIFEST_CHECKSUM_FILE, + EXTENDDB_MANIFEST_FILE, + "data/000001.json.gz", + "data/000001.json.gz", + "data/000001.json.gz", + "data/000002.json.gz", + "data/000001.json.gz", + "data/000002.json.gz", + ], + "file 1 took three attempts" + ); + // Summed across the drain's buckets: the two errors are 1 to 2 s apart + // and a run that crosses a wall-clock minute puts them in two. + let errors: Vec<_> = drained(&runtime) + .into_iter() + .filter(|(m, _, _)| *m == MetricName::BackupStoreErrors) + .collect(); + assert!(errors.iter().all(|(_, op, _)| op == "get"), "{errors:?}"); + let total: f64 = errors.iter().map(|(_, _, sum)| *sum).sum(); + assert!((total - 2.0).abs() < f64::EPSILON, "{errors:?}"); + + // A denial: no retry. + let fake = Arc::new(Fake::default()); + let mem = Arc::new(MemStore::default()); + seed_manifest_backup( + &fake, + &mem, + &arn(ACCOUNT), + music_source_table(), + &files, + true, + ); + let runtime = job_runtime(&mem); + mem.fail_gets_with(GetFailure::PermissionDenied, 1); + restore_on(&fake, &runtime, &restore_body(json!({}))) + .await + .unwrap(); + wait_for_jobs(&runtime).await; + let writes = restore_writes(&fake); + assert_eq!(writes.len(), 1); + let reason = writes[0].2.clone().unwrap(); + assert!( + reason + .starts_with("reading data file data/000001.json.gz from the store: permission denied"), + "{reason}" + ); + assert_eq!(gets_under(&mem, &arn(ACCOUNT)).len(), 3); + let errors: Vec<_> = drained(&runtime) + .into_iter() + .filter(|(m, _, _)| *m == MetricName::BackupStoreErrors) + .collect(); + assert_eq!( + errors, + [(MetricName::BackupStoreErrors, "get".to_owned(), 1.0)], + "a denial is the store's error too" + ); + + // A file that is not there. + let fake = Arc::new(Fake::default()); + let mem = Arc::new(MemStore::default()); + seed_manifest_backup( + &fake, + &mem, + &arn(ACCOUNT), + music_source_table(), + &files, + true, + ); + let prefix = BackupArn::parse(&arn(ACCOUNT)).unwrap().store_prefix(); + mem.objects + .lock() + .unwrap() + .remove(&format!("{prefix}data/000002.json.gz")); + let runtime = job_runtime(&mem); + restore_on(&fake, &runtime, &restore_body(json!({}))) + .await + .unwrap(); + wait_for_jobs(&runtime).await; + let writes = restore_writes(&fake); + assert_eq!( + writes[0].2.as_deref(), + Some("data file data/000002.json.gz is not in the store") + ); + assert!(fake.bulk_loads.lock().unwrap().is_empty()); + let errors: Vec<_> = drained(&runtime) + .into_iter() + .filter(|(m, _, _)| *m == MetricName::BackupStoreErrors) + .collect(); + assert_eq!( + errors, + [(MetricName::BackupStoreErrors, "get".to_owned(), 1.0)], + "an absent file is the store's error too" + ); +} + +/// A store that keeps failing is retried for the bound and then fails the +/// restore naming the file and the bound, with the record heartbeated +/// meanwhile and no `bulk_load`; the backoff never exceeds a minute. +#[tokio::test(start_paused = true)] +async fn a_download_exhausting_the_retry_bound_fails_the_restore() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + seed_manifest_backup( + &fake, + &mem, + &arn(ACCOUNT), + music_source_table(), + &[songs(2)], + true, + ); + let runtime = job_runtime(&mem); + mem.fail_gets_with(GetFailure::Transport, usize::MAX); + restore_on(&fake, &runtime, &restore_body(json!({}))) + .await + .unwrap(); + let started = tokio::time::Instant::now(); + // The paused clock runs ahead through the backoffs, so the job's end is + // waited for without `wait_for_jobs`'s wall-clock bound. + while runtime.running_jobs() > 0 { + tokio::time::sleep(Duration::from_millis(50)).await; + } + let elapsed = started.elapsed(); + assert!( + elapsed >= Duration::from_secs(15 * 60) && elapsed < Duration::from_secs(16 * 60 + 2), + "{elapsed:?}" + ); + let writes = restore_writes(&fake); + assert_eq!(writes.len(), 1); + let reason = writes[0].2.clone().unwrap(); + assert!( + reason.starts_with("downloading data/000001.json.gz: transport error"), + "{reason}" + ); + assert!(reason.contains("the bound is 900 s"), "{reason}"); + let attempts = gets_under(&mem, &arn(ACCOUNT)).len() - 2; + // 1+2+4+8+16+32 s, then 60 s each: about 20 attempts in 15 minutes. + assert!((18..=24).contains(&attempts), "{attempts} attempts"); + assert!(fake.bulk_loads.lock().unwrap().is_empty()); + assert!( + fake.heartbeats.load(std::sync::atomic::Ordering::Acquire) > 100, + "the record was heartbeated through the wait" + ); + assert_eq!(restore_outcomes(&runtime), [("failed".to_owned(), 1.0)]); +} + +/// The count after the last file: a backend that reports one item fewer +/// than the manifest (one of two rows with one key kept) fails the restore +/// with the count reason, after every file was loaded; and the duplicate +/// rule of the backend itself (an unordered backup holding one key in two +/// files) fails the second file's load with the backend's text. +#[tokio::test] +async fn a_duplicate_key_fails_the_restore_by_the_count_or_by_the_load() { + // The count runs on both orders. + for key_ordered in [false, true] { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + seed_manifest_backup( + &fake, + &mem, + &arn(ACCOUNT), + music_source_table(), + &[songs_from(0, 2), songs_from(2, 2)], + key_ordered, + ); + let runtime = job_runtime(&mem); + *fake.count_override.lock().unwrap() = Some(3); + restore_on(&fake, &runtime, &restore_body(json!({}))) + .await + .unwrap(); + wait_for_jobs(&runtime).await; + assert_eq!( + fake.bulk_loads.lock().unwrap().len(), + 2, + "every file was loaded first ({key_ordered})" + ); + let writes = restore_writes(&fake); + assert_eq!( + writes[0].2.as_deref(), + Some("restored 3 items, the manifest lists 4: the backup holds a key twice"), + "{key_ordered}" + ); + assert!(fake.restore_job("MusicRestored").is_none()); + } + + let fake = Arc::new(Fake::default()); + let mem = Arc::new(MemStore::default()); + let mut second = songs_from(5, 2); + second.push(song("artist-00001", "title-00001")); + seed_manifest_backup( + &fake, + &mem, + &arn(ACCOUNT), + music_source_table(), + &[songs_from(0, 3), second], + false, + ); + let runtime = job_runtime(&mem); + restore_on(&fake, &runtime, &restore_body(json!({}))) + .await + .unwrap(); + wait_for_jobs(&runtime).await; + assert_eq!(fake.bulk_loads.lock().unwrap().len(), 2); + let writes = restore_writes(&fake); + let reason = writes[0].2.clone().unwrap(); assert!( - matches!(err, DynamoDbError::AccessDeniedException(_)), - "{err:?}" + reason.starts_with("data file data/000002.json.gz: ") + && reason.contains("holds a primary key the target already has"), + "{reason}" ); + assert!(fake.restore_job("MusicRestored").is_none()); + assert_eq!(fake.restored_items("MusicRestored").len(), 0); } -#[test] -fn account_prefix_does_not_match() { - let body = json!({ "BackupArn": arn("12345678901") }); - assert!(matches!( - arn_field(&body).unwrap_err(), - DynamoDbError::AccessDeniedException(_) - )); -} - -/// Measured on DynamoDB: the shapes the service refuses as `ValidationException` -/// with `Invalid Request: BackupArn is not valid` (not an ARN, oversize, -/// control characters, a slash or `..` in the region) or `Invalid Backup ARN` -/// (an id suffix of the wrong length or not hex, a wildcard id), on every -/// member that carries a backup ARN. Which text goes with which input was -/// not captured, so the split is the parser's (see `BackupArnError`). +/// A key-ordered file whose first item is below its recorded range fails +/// the file through the reader's range check, naming the file; nothing of +/// the file is applied and the target is removed. #[tokio::test] -async fn malformed_arn_is_a_validation_exception_with_the_services_text() { - const NOT_VALID: &str = "Invalid Request: BackupArn is not valid"; - const INVALID_ID: &str = "Invalid Backup ARN"; - let id = "01791976431438-73d8d5bc"; - let long_table = "x".repeat(1000); - let cases: Vec<(String, &str)> = vec![ - ("not-an-arn".to_owned(), NOT_VALID), - ("arn:aws:dynamodb".to_owned(), NOT_VALID), - (format!("arn:aws:dynamodb:{REGION}"), NOT_VALID), - (String::new(), NOT_VALID), - ( - format!("arn:aws:dynamodb:{REGION}::table/Music/backup/{id}"), - NOT_VALID, - ), - ( - format!("arn:aws:dynamodb:{REGION}:table/Music/backup/{ACCOUNT}"), - NOT_VALID, - ), - ( - format!("arn:aws:dynamodb:{REGION}:{ACCOUNT}:table/Music"), - NOT_VALID, - ), - ( - format!("arn:aws:dynamodb:{REGION}:{ACCOUNT}:table/Music/stream/{id}"), - NOT_VALID, - ), - ( - format!("arn:aws:dynamodb:{REGION}:999999999999:table/Music/index/x"), - NOT_VALID, - ), - // Oversize (the model's max is 1024). - ( - format!("arn:aws:dynamodb:{REGION}:{ACCOUNT}:table/{long_table}/backup/{id}"), - NOT_VALID, - ), - // Control characters, inside and at the end (a NUL byte was a 500 on - // PostgreSQL before the check). - ( - format!("arn:aws:dynamodb:{REGION}:{ACCOUNT}:table/Music/backup/{id}\u{0}"), - NOT_VALID, - ), - ( - format!("arn:aws:dynamodb:{REGION}:{ACCOUNT}:table/Mu\u{1}sic/backup/{id}"), - NOT_VALID, - ), - ( - format!("arn:aws:dynamodb:{REGION}:{ACCOUNT}:table/Music/backup/{id}\n\u{7}"), - NOT_VALID, - ), - // A slash or `..` in the region. - ( - format!("arn:aws:dynamodb:us/east-1:{ACCOUNT}:table/Music/backup/{id}"), - NOT_VALID, - ), - ( - format!("arn:aws:dynamodb:..:{ACCOUNT}:table/Music/backup/{id}"), - NOT_VALID, - ), - ( - format!("arn:aws:dynamodb:us-east-1/..:{ACCOUNT}:table/Music/backup/{id}"), - NOT_VALID, - ), - // The id: wrong length, not hex, a wildcard. - ( - format!("arn:aws:dynamodb:{REGION}:{ACCOUNT}:table/Music/backup/1"), - INVALID_ID, - ), - ( - format!("arn:aws:dynamodb:{REGION}:{ACCOUNT}:table/Music/backup/*"), - INVALID_ID, - ), - ( - format!("arn:aws:dynamodb:{REGION}:{ACCOUNT}:table/Music/backup/01791976431438-*"), - INVALID_ID, - ), - ( - format!( - "arn:aws:dynamodb:{REGION}:{ACCOUNT}:table/Music/backup/01791976431438-73d8d5b" - ), - INVALID_ID, - ), - ( - format!( - "arn:aws:dynamodb:{REGION}:{ACCOUNT}:table/Music/backup/01791976431438-73d8d5bcd" - ), - INVALID_ID, - ), - ( - format!( - "arn:aws:dynamodb:{REGION}:{ACCOUNT}:table/Music/backup/01791976431438-73D8D5BC" - ), - INVALID_ID, - ), - ( - format!( - "arn:aws:dynamodb:{REGION}:{ACCOUNT}:table/Music/backup/01791976431438-73d8d5bg" - ), - INVALID_ID, - ), - ( - format!( - "arn:aws:dynamodb:{REGION}:{ACCOUNT}:table/Music/backup/0179197643143x-73d8d5bc" - ), - INVALID_ID, - ), - ]; - let fake = Fake::with_source(source(Some(definition("PROVISIONED")))); - for (candidate, expected) in &cases { - for (member, result) in [ - ("BackupArn", arn_field(&json!({ "BackupArn": candidate }))), - ( - "RestoreTableFromBackup", - restore(&fake, restore_body(json!({ "BackupArn": candidate }))) - .await - .map(|_| String::new()), - ), - ( - "ExclusiveStartBackupArn", - list(&fake, json!({ "ExclusiveStartBackupArn": candidate })) - .await - .map(|_| String::new()), - ), - ] { - let msg = validation_message(result.unwrap_err()); - assert_eq!(msg, *expected, "{member} {candidate:?}"); +async fn a_file_leaving_its_key_range_fails_the_restore() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let files = [songs_from(0, 2), songs_from(2, 3)]; + let source_table = music_source_table(); + let parsed = BackupArn::parse(&arn(ACCOUNT)).unwrap(); + let prefix = parsed.store_prefix(); + let order = KeyOrder::resolve( + &source_table.key_schema, + &source_table.attribute_definitions, + ) + .unwrap(); + let mut tracker = KeyRangeTracker::new(&order); + let mut entries = Vec::new(); + for (i, items) in files.iter().enumerate() { + let (bytes, checksums) = encode_data_file(items.iter().cloned()).unwrap(); + for item in items { + tracker.push(item).unwrap(); } + let mut range = tracker.next_file().unwrap(); + if i == 1 { + // The recorded range starts at the file's second item. + range.first = order.endpoint_key(&items[1]).unwrap(); + } + let key = data_file_key(u32::try_from(i + 1).unwrap()); + mem.insert(&format!("{prefix}{key}"), &bytes); + entries.push(( + DataFileEntry::from_checksums(key, &checksums, Some(range)), + None, + )); } - assert_eq!(fake.source_reads(), 0); - assert!(fake.restores().is_empty()); + seed_manifest_documents(&fake, &mem, &arn(ACCOUNT), source_table, entries, true); + let runtime = job_runtime(&mem); + restore_on(&fake, &runtime, &restore_body(json!({}))) + .await + .unwrap(); + wait_for_jobs(&runtime).await; + let loads = fake.bulk_loads.lock().unwrap().clone(); + assert_eq!(loads.len(), 2); + assert!(loads[1].stream_error.is_some(), "{loads:?}"); + let writes = restore_writes(&fake); + let reason = writes[0].2.clone().unwrap(); + assert!( + reason.starts_with("data file data/000002.json.gz: "), + "{reason}" + ); + assert!( + reason.contains("range") || reason.contains("key"), + "{reason}" + ); + assert!(fake.restore_job("MusicRestored").is_none()); } -/// Measured on DynamoDB: the service trims a trailing newline and resolves an ARN -/// naming another region of the caller's account. The engine hands the -/// backend the ARN it issued: trimmed, in its own region. +/// An item the validator refuses (a `Year` of `1E126`, #618's shape) fails +/// the file before the backend sees it, naming the key and the attribute; +/// the backend's stream ended with the error and nothing was applied. #[tokio::test] -async fn whitespace_is_trimmed_and_another_region_resolves() { - let ours = arn(ACCOUNT); - for candidate in [ - format!("{ours}\n"), - format!(" {ours}\t\r\n"), - format!("arn:aws:dynamodb:eu-west-1:{ACCOUNT}:table/Music/backup/01791976431438-73d8d5bc"), - format!( - "arn:aws:dynamodb:eu-west-1:{ACCOUNT}:table/Music/backup/01791976431438-73d8d5bc\n" +async fn a_validator_refusal_fails_the_restore_naming_the_item() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let mut second = songs_from(2, 1); + second.push(song_with_bad_number("artist-00003", "title-00003", "1E126")); + seed_manifest_backup( + &fake, + &mem, + &arn(ACCOUNT), + music_source_table(), + &[songs_from(0, 2), second], + true, + ); + let runtime = job_runtime(&mem); + restore_on(&fake, &runtime, &restore_body(json!({}))) + .await + .unwrap(); + wait_for_jobs(&runtime).await; + let loads = fake.bulk_loads.lock().unwrap().clone(); + assert_eq!(loads.len(), 2); + assert_eq!(loads[1].items.len(), 0, "the chunk was never sent"); + let writes = restore_writes(&fake); + let reason = writes[0].2.clone().unwrap(); + assert!( + reason.starts_with( + "data file data/000002.json.gz: item cannot be restored: key \ + {\"Artist\":{\"S\":\"artist-00003\"},\"SongTitle\":{\"S\":\"title-00003\"}}, attribute Year: " ), - ] { - assert_eq!( - arn_field(&json!({ "BackupArn": candidate })).unwrap(), - ours, - "{candidate:?}" - ); - let fake = Fake::with_source(source(Some(definition("PROVISIONED")))); - restore(&fake, restore_body(json!({ "BackupArn": candidate }))) - .await - .unwrap_or_else(|e| panic!("{candidate:?}: {e:?}")); - assert_eq!(fake.restores()[0].backup_arn, ours, "{candidate:?}"); - } - // Another region of another account is still that account's. - let foreign = - "arn:aws:dynamodb:eu-west-1:999999999999:table/Music/backup/01791976431438-73d8d5bc"; - assert!(matches!( - arn_field(&json!({ "BackupArn": foreign })).unwrap_err(), - DynamoDbError::AccessDeniedException(_) - )); - // Whitespace inside is not trimmed. - let inside = - format!("arn:aws:dynamodb:{REGION}:{ACCOUNT}:table/Mu sic/backup/01791976431438-73d8d5bc"); + "{reason}" + ); assert_eq!( - validation_message(arn_field(&json!({ "BackupArn": inside })).unwrap_err()), - "Invalid Request: BackupArn is not valid" + loads[0].items.len(), + 2, + "file 1 was applied before file 2 failed" + ); + assert!( + fake.restore_job("MusicRestored").is_none(), + "the target was removed" ); } -/// A backup created under an earlier `server.region` is stored with that -/// region in its ARN, which `ListBackups` reports as is. `DescribeBackup`, -/// `DeleteBackup`, and `RestoreTableFromBackup` look the ARN up rebuilt in -/// the server's region first and, when that is not found, as written: the -/// stored backup stays reachable by the ARN the server reports for it, and -/// the engine acts on whichever spelling resolved. +/// The FAILED-write gate: a record another owner took (its generation +/// changed) is not this job's to write, so a failing job makes no table +/// access: no `remove_restore_target`, the record untouched. An unknown +/// outcome is re-read: a write that landed is matched and the target is +/// removed with one write; one that was lost is sent once more; one that +/// was lost and whose record a successor then wrote FAILED under its own +/// generation is not this job's, so no removal. #[tokio::test] -async fn a_backup_stored_under_another_region_resolves_as_written() { - let stored = - format!("arn:aws:dynamodb:eu-west-1:{ACCOUNT}:table/Music/backup/01791976431438-73d8d5bc"); - let ours = arn(ACCOUNT); - let spellings = own(&stored); - assert_eq!(spellings.canonical(), ours); - assert_eq!(spellings.as_written.as_deref(), Some(stored.as_str())); - // The server's own spelling has no second spelling to try. - assert_eq!(own(&ours).as_written, None); - assert_eq!(own(&format!(" {ours}\n")).as_written, None, "trimmed first"); - - let stored_source = || { - let mut src = source(Some(definition("PROVISIONED"))); - src.backup_arn.clone_from(&stored); - src +async fn a_failed_write_that_does_not_match_removes_no_target_and_an_unknown_one_is_re_read() { + let (lines, _guard) = LogLines::capture(); + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + seed_manifest_backup( + &fake, + &mem, + &arn(ACCOUNT), + music_source_table(), + &[songs(2)], + true, + ); + *fake.heartbeat_answer.lock().unwrap() = HeartbeatAnswer::AlwaysConfirm; + // The job is held at its first `bulk_load`; the record changes hands + // meanwhile, so the load fails under the old generation and the FAILED + // write matches no record. + let gate = closed_gate(); + *fake.bulk_load_gate.lock().unwrap() = Some(Arc::clone(&gate)); + let runtime = job_runtime(&mem); + restore_on(&fake, &runtime, &restore_body(json!({}))) + .await + .unwrap(); + wait_for_bulk_loads_started(&fake, 1).await; + let other = { + let mut records = fake.restore_jobs.lock().unwrap(); + let record = records.get_mut("tid-MusicRestored").unwrap(); + record.owner_generation = new_generation(); + record.owner_generation }; - let with_stored = || { - let fake = Fake::with_source(stored_source()); - fake.descriptions.lock().unwrap().insert( - stored.clone(), - backup_description("AVAILABLE", "PROVISIONED"), + gate.add_permits(10); + wait_for_jobs(&runtime).await; + let writes = restore_writes(&fake); + assert_eq!(writes.len(), 1); + assert_eq!(writes[0].1, JobStatus::Failed); + assert!( + fake.removed_targets.lock().unwrap().is_empty(), + "no table access" + ); + let record = fake.restore_job("MusicRestored").unwrap(); + assert_eq!(record.status, JobStatus::Running); + assert_eq!(record.owner_generation, other); + let line = lines.containing("no table access was made"); + assert_eq!(line.len(), 1, "{line:?}"); + + // A corrupt file, so the failure path runs with the claim intact. A + // write lost with its connection and followed by a takeover whose + // successor wrote FAILED itself re-reads FAILED under another + // generation: the record is not this job's, so no removal. + let prefix = BackupArn::parse(&arn(ACCOUNT)).unwrap().store_prefix(); + let key = format!("{prefix}data/000001.json.gz"); + { + let (lines, _guard) = LogLines::capture(); + let fake = Arc::new(Fake::default()); + let mem = Arc::new(MemStore::default()); + seed_manifest_backup( + &fake, + &mem, + &arn(ACCOUNT), + music_source_table(), + &[songs(2)], + true, ); - fake - }; + let mut bytes = mem.objects.lock().unwrap()[&key].clone(); + bytes[0] ^= 0xff; + mem.insert(&key, &bytes); + *fake.unknown_write.lock().unwrap() = Some(UnknownWrite::TakenAndWritten); + let runtime = job_runtime(&mem); + restore_on(&fake, &runtime, &restore_body(json!({}))) + .await + .unwrap(); + wait_for_jobs(&runtime).await; + assert_eq!(restore_writes(&fake).len(), 1, "no second write"); + assert!( + fake.removed_targets.lock().unwrap().is_empty(), + "no table access: the FAILED record is the successor's" + ); + assert!(fake.tables.lock().unwrap().contains_key("MusicRestored")); + let line = + lines.containing("the record is not this job's to write (the row carries generation"); + assert_eq!(line.len(), 1, "{line:?}"); + } + for (unknown, expected_writes) in [(UnknownWrite::Applied, 1), (UnknownWrite::Lost, 2)] { + let fake = Arc::new(Fake::default()); + let mem = Arc::new(MemStore::default()); + seed_manifest_backup( + &fake, + &mem, + &arn(ACCOUNT), + music_source_table(), + &[songs(2)], + true, + ); + let mut bytes = mem.objects.lock().unwrap()[&key].clone(); + bytes[0] ^= 0xff; + mem.insert(&key, &bytes); + *fake.unknown_write.lock().unwrap() = Some(unknown); + let runtime = job_runtime(&mem); + restore_on(&fake, &runtime, &restore_body(json!({}))) + .await + .unwrap(); + wait_for_jobs(&runtime).await; + assert_eq!(restore_writes(&fake).len(), expected_writes, "{unknown:?}"); + assert_eq!(fake.removed_targets.lock().unwrap().len(), 1, "{unknown:?}"); + assert!(fake.restore_job("MusicRestored").is_none(), "{unknown:?}"); + } +} - // DescribeBackup: canonical not found, as written found. - let fake = with_stored(); - let desc = describe_backup(&fake, ACCOUNT, REGION, &spellings) +/// A lost heartbeat ends the job with no catalog write and no table access: +/// the job is held inside the first `bulk_load`, the claim is reported lost +/// at the next beat, and the job stops; the record stays RUNNING under its +/// generation, nothing is removed, the outcome is `failed`. +#[tokio::test] +async fn a_lost_heartbeat_stops_the_restore_with_no_catalog_write_and_no_table_access() { + let (lines, _guard) = LogLines::capture(); + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + seed_manifest_backup( + &fake, + &mem, + &arn(ACCOUNT), + music_source_table(), + &[songs(2)], + true, + ); + let runtime = job_runtime(&mem); + let gate = closed_gate(); + *fake.bulk_load_gate.lock().unwrap() = Some(Arc::clone(&gate)); + restore_on(&fake, &runtime, &restore_body(json!({}))) .await .unwrap(); - assert_eq!(desc.backup_details.backup_status, "AVAILABLE"); + // The job reaches the gate; then the claim is lost at the next beat. + wait_for_bulk_loads_started(&fake, 1).await; + *fake.heartbeat_answer.lock().unwrap() = HeartbeatAnswer::Lost; + wait_for_jobs(&runtime).await; + assert!(restore_writes(&fake).is_empty(), "no catalog write"); assert!( - desc.source_table_feature_details.is_some(), - "source read under the stored ARN" + fake.removed_targets.lock().unwrap().is_empty(), + "no table access" ); - assert_eq!(fake.source_reads(), 1); + let record = fake.restore_job("MusicRestored").unwrap(); + assert_eq!(record.status, JobStatus::Running); + assert!(fake.restored_items("MusicRestored").is_empty()); + assert_eq!(restore_outcomes(&runtime), [("failed".to_owned(), 1.0)]); + // The stop line names the timer's reason, not the two-period clause. + let stopped = lines.containing( + "stopped with no further catalog or table access: the claim was lost: the row is gone \ + or carries another generation; the record is left to its owner or to the reconciler", + ); + assert_eq!(stopped.len(), 1, "{:?}", lines.containing("stopped with")); + let raised = lines.containing( + "the job's heartbeat raised the loss of its claim; the job stops at its next check: the \ + claim was lost: the row is gone or carries another generation", + ); + assert_eq!(raised.len(), 1, "{raised:?}"); + assert_eq!(lines.at_level(tracing::Level::WARN), raised); + assert_eq!(runtime.free_slots(), 4); +} - // RestoreTableFromBackup: registered against the stored spelling. - let fake = with_stored(); - restore(&fake, restore_body(json!({ "BackupArn": stored }))) +/// A loss the timer raised while the job was inside a step (here the count) +/// is named by the next check between steps: the pre-flip check's stop line +/// carries the timer's reason, not the two-period clause. +#[tokio::test] +async fn a_loss_raised_inside_a_step_is_named_by_the_next_check() { + let (lines, _guard) = LogLines::capture(); + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + seed_manifest_backup( + &fake, + &mem, + &arn(ACCOUNT), + music_source_table(), + &[songs(2)], + true, + ); + let runtime = job_runtime(&mem); + let count_gate = closed_gate(); + *fake.count_gate.lock().unwrap() = Some(Arc::clone(&count_gate)); + restore_on(&fake, &runtime, &restore_body(json!({}))) .await .unwrap(); - assert_eq!(fake.restores()[0].backup_arn, stored); + let deadline = tokio::time::Instant::now() + Duration::from_secs(30); + while fake.count_calls.load(std::sync::atomic::Ordering::Acquire) == 0 { + assert!(tokio::time::Instant::now() < deadline, "no count began"); + tokio::time::sleep(Duration::from_millis(5)).await; + } + // The claim is lost at the next beat, while the job waits in the count. + *fake.heartbeat_answer.lock().unwrap() = HeartbeatAnswer::Lost; + while lines.containing("raised the loss of its claim").is_empty() { + assert!( + tokio::time::Instant::now() < deadline, + "the loss was not raised" + ); + tokio::time::sleep(Duration::from_millis(5)).await; + } + count_gate.add_permits(1); + wait_for_jobs(&runtime).await; + assert_eq!(fake.finish_sizes.lock().unwrap().len(), 0, "no DONE write"); + assert!(restore_writes(&fake).is_empty(), "no catalog write"); assert_eq!( - fake.source_reads(), - 2, - "canonical (not found), then as written" + fake.restore_job("MusicRestored").unwrap().status, + JobStatus::Running + ); + let stopped = lines.containing( + "stopped with no further catalog or table access: the claim was lost: the row is gone \ + or carries another generation; the record is left to its owner or to the reconciler", ); + assert_eq!(stopped.len(), 1, "{:?}", lines.containing("stopped with")); + assert_eq!(restore_outcomes(&runtime), [("failed".to_owned(), 1.0)]); +} - // DeleteBackup: the full description, and the stored row is gone. - let fake = with_stored(); - let deleted = delete_backup(&fake, ACCOUNT, REGION, &spellings) +/// Shutdown between files: the job stops at its next file with no status +/// write, the record stays RUNNING with its progress, the spool entry stays +/// with every file verified, the outcome is `cancelled` and no worker +/// error is recorded; `drain_jobs` returns with every job ended. +#[tokio::test] +async fn a_shutdown_leaves_the_record_running_with_its_spool() { + let scratch = restore::spool::tests::Scratch::new(); + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let s3_arn = arn(ACCOUNT); + let files = [songs_from(0, 2), songs_from(2, 2), songs_from(4, 2)]; + seed_manifest_backup(&fake, &mem, &s3_arn, music_source_table(), &files, true); + fake.rows.lock().unwrap().get_mut(&s3_arn).unwrap().store = s3_identity(); + let runtime = spooled_runtime(&mem, scratch.path()); + let gate = Arc::new(tokio::sync::Semaphore::new(1)); + *fake.bulk_load_gate.lock().unwrap() = Some(Arc::clone(&gate)); + restore_on(&fake, &runtime, &restore_body(json!({}))) .await .unwrap(); - assert_eq!(deleted.backup_details.backup_status, "DELETED"); - assert!(deleted.source_table_feature_details.is_some()); - assert!(fake.descriptions.lock().unwrap().is_empty()); - assert!(fake.sources.lock().unwrap().is_empty()); + assert_eq!( + fake.restore_job("MusicRestored") + .unwrap() + .spool_host + .as_deref(), + Some("host-a:/var/spool") + ); + // File 1 loads; file 2 waits at the gate. + wait_for_bulk_loads_started(&fake, 2).await; + let left = runtime.drain_jobs(Duration::from_millis(100)).await; + assert_eq!(left, 1, "the job waits at the gate past the bound"); + gate.add_permits(10); + wait_for_jobs(&runtime).await; + assert!( + restore_writes(&fake).is_empty(), + "no status write at shutdown" + ); + let record = fake.restore_job("MusicRestored").unwrap(); + assert_eq!(record.status, JobStatus::Running); + assert!( + record.file_index.is_some(), + "the progress of the loaded file(s) stays" + ); + assert!(fake.removed_targets.lock().unwrap().is_empty()); + let entry = scratch.path().join("restore-tid-MusicRestored"); + assert!(entry.is_dir(), "the spool entry stays"); + for i in 0..3 { + assert!(entry.join(format!("{i:06}.json.gz")).is_file()); + assert!(entry.join(format!("{i:06}.json.gz.verified")).is_file()); + } + assert!(entry.join(EXTENDDB_MANIFEST_FILE).is_file()); + assert!(entry.join(MANIFEST_CHECKSUM_FILE).is_file()); + // One drain, both assertions read from it (a second drain reads an + // empty collector and asserts nothing). + let metrics = drained(&runtime); + assert_eq!( + metrics + .iter() + .filter(|(m, _, _)| *m == MetricName::RestoreJobs) + .map(|(_, op, sum)| (op.clone(), *sum)) + .collect::>(), + [("cancelled".to_owned(), 1.0)] + ); + assert!( + !metrics + .iter() + .any(|(m, op, _)| *m == MetricName::WorkerErrorCount && op == "RestoreJob"), + "a clean shutdown is not a worker error: {metrics:?}" + ); +} - // The canonical spelling wins when both exist: the backend is asked for - // it alone (one read), and the stored other-region row is untouched. - let fake = with_stored(); - fake.sources +/// On an S3 store the files are spooled: each is downloaded once into the +/// entry, marked `.verified`, and the load reads the spool, not the store; +/// the entry is removed when the job ends; and a completed job's second +/// read of the store is never made. +#[tokio::test] +async fn an_s3_restore_spools_every_file_and_removes_its_entry_at_the_end() { + let scratch = restore::spool::tests::Scratch::new(); + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let files = [songs_from(0, 2), songs_from(2, 2)]; + seed_manifest_backup( + &fake, + &mem, + &arn(ACCOUNT), + music_source_table(), + &files, + true, + ); + fake.rows .lock() .unwrap() - .insert(ours.clone(), source(Some(definition("PAY_PER_REQUEST")))); - fake.descriptions.lock().unwrap().insert( - ours.clone(), - backup_description("AVAILABLE", "PAY_PER_REQUEST"), - ); - restore(&fake, restore_body(json!({ "BackupArn": stored }))) - .await - .unwrap(); - assert_eq!(fake.restores()[0].backup_arn, ours); - assert_eq!(fake.source_reads(), 1); - let deleted = delete_backup(&fake, ACCOUNT, REGION, &spellings) + .get_mut(&arn(ACCOUNT)) + .unwrap() + .store = s3_identity(); + let runtime = spooled_runtime(&mem, scratch.path()); + let gate = closed_gate(); + *fake.bulk_load_gate.lock().unwrap() = Some(Arc::clone(&gate)); + restore_on(&fake, &runtime, &restore_body(json!({}))) .await .unwrap(); - assert_eq!(deleted.backup_details.backup_status, "DELETED"); - assert!(fake.descriptions.lock().unwrap().contains_key(&stored)); - assert!(!fake.descriptions.lock().unwrap().contains_key(&ours)); - - // Neither spelling: BackupNotFoundException from the backend, after both - // were tried; the server's own spelling is tried once. - let fake = Fake::default(); - for (body, reads) in [(stored.clone(), 2), (ours.clone(), 1)] { - let before = fake.source_reads(); - let err = restore(&fake, restore_body(json!({ "BackupArn": body }))) - .await - .unwrap_err(); - assert!( - matches!(err, DynamoDbError::BackupNotFoundException(_)), - "{err:?}" + let entry = scratch.path().join("restore-tid-MusicRestored"); + let deadline = tokio::time::Instant::now() + Duration::from_secs(10); + while !entry.join("000001.json.gz.verified").is_file() { + assert!(tokio::time::Instant::now() < deadline); + tokio::time::sleep(Duration::from_millis(5)).await; + } + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + assert_eq!( + std::fs::metadata(&entry).unwrap().permissions().mode() & 0o777, + 0o700 + ); + assert_eq!( + std::fs::metadata(entry.join("000000.json.gz")) + .unwrap() + .permissions() + .mode() + & 0o777, + 0o600 ); - assert_eq!(fake.source_reads() - before, reads); - assert!(matches!( - describe_backup(&fake, ACCOUNT, REGION, &own(&body)) - .await - .unwrap_err(), - DynamoDbError::BackupNotFoundException(_) - )); - assert!(matches!( - delete_backup(&fake, ACCOUNT, REGION, &own(&body)) - .await - .unwrap_err(), - DynamoDbError::BackupNotFoundException(_) - )); } - - // An error other than not-found on the canonical spelling is not a - // reason to try the other one. - let fake = with_stored(); - *fake.source_error.lock().unwrap() = - Some(StorageError::Connection("connection refused".to_owned())); - let err = restore(&fake, restore_body(json!({ "BackupArn": stored }))) - .await - .unwrap_err(); - assert!( - matches!(err, DynamoDbError::InternalServerError(_)), - "{err:?}" + gate.add_permits(10); + wait_for_jobs(&runtime).await; + assert_eq!( + fake.restore_job("MusicRestored").unwrap().status, + JobStatus::Done ); - assert_eq!(fake.source_reads(), 1); - assert!(fake.restores().is_empty()); + assert_eq!(fake.restored_items("MusicRestored").len(), 4); + assert_eq!( + gets_under(&mem, &arn(ACCOUNT)), + [ + MANIFEST_CHECKSUM_FILE, + EXTENDDB_MANIFEST_FILE, + "data/000001.json.gz", + "data/000002.json.gz", + ], + "each file read from the store once" + ); + assert!(!entry.exists(), "the entry is removed at the end"); } -/// An ARN in another partition names nothing here: `ValidationException` -/// (the service's class for one naming the caller's account), before the -/// account is compared, on every member that carries a backup ARN. +/// A resumed job skips the files and items the record and the progress rows +/// account for: the first open file is loaded from its `start_at` with +/// `reapplying`, the rest from 0 without it, and the count covers the whole +/// manifest. #[tokio::test] -async fn an_arn_in_another_partition_is_a_validation_exception() { - let ours = arn(ACCOUNT); - let fake = Fake::with_source(source(Some(definition("PROVISIONED")))); - fake.descriptions +async fn a_resume_loads_from_the_recorded_progress() { + let mem = Arc::new(MemStore::default()); + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + let fake = Arc::new(Fake::default()); + let files = [songs_from(0, 3), songs_from(3, 3), songs_from(6, 3)]; + seed_manifest_backup( + &fake, + &mem, + &arn(ACCOUNT), + music_source_table(), + &files, + true, + ); + let runtime = job_runtime(&mem); + // A dead owner's record: file 0 complete, one item of file 1 applied, + // the target holding those four items. + restore_on(&fake, &runtime, &restore_body(json!({}))) + .await + .unwrap(); + wait_for_jobs(&runtime).await; + { + let mut records = fake.restore_jobs.lock().unwrap(); + let record = records.get_mut("tid-MusicRestored").unwrap(); + record.status = JobStatus::Running; + record.completed_at = None; + record.file_index = Some(1); + record.items_applied = Some(1); + let mut targets = fake.restore_tables.lock().unwrap(); + let target = targets.get_mut("tid-MusicRestored").unwrap(); + target.items.truncate(4); + target.active = false; + } + fake.bulk_loads.lock().unwrap().clear(); + fake.restore_dead .lock() .unwrap() - .insert(ours.clone(), backup_description("AVAILABLE", "PROVISIONED")); - *fake.summaries.lock().unwrap() = vec![summary(1, 1)]; - for partition in ["aws-cn", "aws-us-gov", "aws-iso", "AWS", "x"] { - for account in [ACCOUNT, "999999999999"] { - let candidate = format!( - "arn:{partition}:dynamodb:{REGION}:{account}:table/Music/backup/01791976431438-73d8d5bc" - ); - let err = arn_field(&json!({ "BackupArn": candidate })).unwrap_err(); - assert_eq!( - validation_message(err), - "Invalid Request: BackupArn is not valid" - ); - let err = restore(&fake, restore_body(json!({ "BackupArn": candidate }))) - .await - .unwrap_err(); - assert!( - matches!(err, DynamoDbError::ValidationException(_)), - "{err:?}" - ); - let err = list(&fake, json!({ "ExclusiveStartBackupArn": candidate })) - .await - .unwrap_err(); - assert!( - matches!(err, DynamoDbError::ValidationException(_)), - "{err:?}" - ); - } - } - assert_eq!(fake.source_reads(), 0); - assert!(fake.restores().is_empty()); - // Liveness: the same ARN in this server's partition resolves. + .insert("tid-MusicRestored".to_owned()); + let report = cycle_on(&fake, &runtime).await; + assert_eq!(report.manifest_restores_resumed, ["MusicRestored"]); + wait_for_jobs(&runtime).await; + assert_eq!(fake.bulk_load_files(), [(1, 1, true, 2), (2, 0, false, 3)]); + let all: Vec = files.iter().flatten().cloned().collect(); + assert_eq!(fake.restored_items("MusicRestored"), all); assert_eq!( - ours, - extenddb_storage::util::backup_arn(REGION, ACCOUNT, "Music", "01791976431438-73d8d5bc") + fake.restore_job("MusicRestored").unwrap().status, + JobStatus::Done ); - assert_eq!(arn_field(&json!({ "BackupArn": ours })).unwrap(), ours); - restore(&fake, restore_body(json!({}))).await.unwrap(); - assert_eq!(fake.restores().len(), 1); } -#[test] -fn missing_arn_field_is_a_validation_error() { - let err = arn_field(&json!({})).unwrap_err(); - assert!(matches!(err, DynamoDbError::ValidationException(m) if m.contains("backupArn"))); +/// The vector refusal on the manifest path, narrowed to a target without +/// vector support: a manifest with a non-empty `vector_indexes` into a +/// target whose engine has no vector search is the legacy text's +/// `ValidationException` with nothing registered; the same manifest into a +/// target with vector support registers the table with its vector indexes; +/// `null` and `[]` register on both. +#[tokio::test] +async fn vector_index_backups_are_refused_in_the_engine() { + let spec: extenddb_core::types::VectorIndexSpecification = serde_json::from_value(json!({ + "IndexName": "vidx", + "VectorAttribute": { "AttributeName": "emb" }, + "Dimensions": 3, + "DistanceFunction": "COSINE", + "Projection": { "ProjectionType": "ALL" } + })) + .unwrap(); + let cases: [( + Option>, + bool, + bool, + ); 6] = [ + (Some(vec![spec.clone()]), false, false), + (Some(vec![spec.clone()]), true, true), + (None, false, true), + (None, true, true), + (Some(Vec::new()), false, true), + (Some(Vec::new()), true, true), + ]; + for (vector_indexes, capable, registered) in cases { + let what = format!("{vector_indexes:?} into a target with vector support {capable}"); + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + fake.vector_search + .store(capable, std::sync::atomic::Ordering::Release); + let mut source_table = on_demand_source_table(); + source_table.vector_indexes = vector_indexes.clone(); + seed_manifest_backup(&fake, &mem, &arn(ACCOUNT), source_table, &[songs(1)], true); + let runtime = job_runtime(&mem); + let result = restore_on(&fake, &runtime, &restore_body(json!({}))).await; + wait_for_jobs(&runtime).await; + if registered { + result.unwrap_or_else(|e| panic!("{what}: {e:?}")); + let claims = fake.restore_claims.lock().unwrap(); + assert_eq!(claims.len(), 1, "{what}"); + let expected = vector_indexes.filter(|v| !v.is_empty()); + assert_eq!(claims[0].1.vector_indexes, expected, "{what}"); + } else { + let msg = validation_message(result.unwrap_err()); + assert_eq!( + msg, + format!( + "backup {} has 1 vector index(es); restoring a table with vector indexes is \ + not supported by this storage backend", + arn(ACCOUNT) + ), + "{what}" + ); + assert!(fake.restore_claims.lock().unwrap().is_empty(), "{what}"); + } + assert_eq!(runtime.free_slots(), 4, "{what}"); + } } -/// A request dropped while the engine is still reading the backup (the client -/// disconnected) does not cancel the restore: it still reaches the backend. +/// The legacy reader's vector refusals stay as they were: a definition +/// naming a vector index and a pre-0.0.4 source counting one. #[tokio::test] -async fn a_restore_request_dropped_while_the_backup_is_read_still_restores() { - use std::future::Future; - use std::sync::Arc; - use std::task::{Context, Waker}; +async fn legacy_vector_backups_keep_the_reader_refusal() { + let mut def = definition("PAY_PER_REQUEST"); + def.vector_index_names.push("v".to_owned()); + let fake = Fake::with_source(source(Some(def))); + let msg = validation_message(restore(&fake, restore_body(json!({}))).await.unwrap_err()); + assert!(msg.contains("vector index"), "{msg}"); + assert!(fake.restores().is_empty()); - let gate = Arc::new(tokio::sync::Semaphore::new(0)); - let mut fake = Fake::with_source(source(Some(definition("PROVISIONED")))); - fake.source_gate = Some(Arc::clone(&gate)); - let fake = Arc::new(fake); + let mut legacy = source(None); + legacy.legacy_vector_index_count = 1; + let fake = Fake::with_source(legacy); + let msg = validation_message(restore(&fake, restore_body(json!({}))).await.unwrap_err()); + assert!(msg.contains("vector index"), "{msg}"); + assert!(fake.restores().is_empty()); +} - let after_ran = Arc::new(tokio::sync::Notify::new()); - let notify = Arc::clone(&after_ran); - let mut request = Box::pin(restore_table_from_backup_detached( - Arc::clone(&fake), - Arc::from(ACCOUNT), - Arc::from(REGION), - Arc::new(LimitsConfig::default()), - restore_body(json!({})), - move |_, target| async move { - assert_eq!(target, "MusicRestored"); - notify.notify_one(); - }, - )); - // Poll once so the restore starts, wait until it is blocked reading the - // backup, then drop the request as a disconnecting client's would be. - let mut cx = Context::from_waker(Waker::noop()); - assert!(request.as_mut().poll(&mut cx).is_pending()); - for _ in 0..200 { - if fake.source_reads() == 1 { - break; - } - tokio::time::sleep(std::time::Duration::from_millis(5)).await; +/// A multi-part-key table (two HASH attributes, backed up unordered) +/// registers on the manifest path when the target's limits allow the shape: +/// `ensure_single_part_base_key` is the legacy reader's rule alone. +#[tokio::test] +async fn a_multi_part_key_manifest_is_registered() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let mut source_table = music_source_table(); + source_table.key_schema = vec![ + key("Artist", KeyType::Hash), + key("Label", KeyType::Hash), + key("SongTitle", KeyType::Range), + ]; + source_table.attribute_definitions.push(attr("Label")); + source_table + .attribute_definitions + .retain(|a| a.attribute_name != "Genre"); + source_table.global_secondary_indexes.clear(); + let mut items = songs(2); + for item in &mut items { + item.insert( + "Label".to_owned(), + extenddb_core::types::AttributeValue::S("indie".to_owned()), + ); } - assert_eq!(fake.source_reads(), 1, "the restore never read the backup"); - drop(request); + seed_manifest_backup( + &fake, + &mem, + &arn(ACCOUNT), + source_table, + &[items.clone()], + false, + ); + let runtime = job_runtime(&mem); + let limits = LimitsConfig { + allow_multipart_table_keys: true, + ..LimitsConfig::default() + }; + let (_, response) = restore_table_from_backup( + &fake, + &runtime, + ACCOUNT, + REGION, + &limits, + &restore_body(json!({})), + ) + .await + .unwrap(); + assert_eq!(response["TableDescription"]["TableStatus"], "CREATING"); + wait_for_jobs(&runtime).await; + let claims = fake.restore_claims.lock().unwrap(); + assert_eq!(claims[0].1.key_schema.len(), 3); + assert_eq!(fake.restored_items("MusicRestored"), items); +} + +/// The override contract on the manifest path is the legacy path's: +/// `restored_input` runs every case through both and asserts they agree, +/// so one list of cases binds both (the override tests above). This cell +/// pins the comparison itself with a case of each kind: an applied +/// override and a refused one. +#[tokio::test] +async fn the_override_cases_run_through_both_paths() { + let input = restored_input( + source(Some(definition("PAY_PER_REQUEST"))), + json!({ + "BillingModeOverride": "PROVISIONED", + "ProvisionedThroughputOverride": { "ReadCapacityUnits": 11, "WriteCapacityUnits": 12 }, + "GlobalSecondaryIndexOverride": [gsi_entry(Some((3, 4)))], + }), + ) + .await + .unwrap(); + assert_eq!(input.billing_mode, Some(BillingMode::Provisioned)); + assert_eq!(input.provisioned_throughput, Some(throughput(11, 12))); + let err = restored_input( + source(Some(definition("PAY_PER_REQUEST"))), + json!({ "ProvisionedThroughputOverride": { "ReadCapacityUnits": 1, "WriteCapacityUnits": 1 } }), + ) + .await + .unwrap_err(); assert!( - fake.restores().is_empty(), - "precondition: not registered yet" + matches!(err, DynamoDbError::ValidationException(_)), + "{err:?}" ); - - gate.add_permits(1); - tokio::time::timeout(std::time::Duration::from_secs(5), after_ran.notified()) - .await - .expect("the dropped request's restore never reached the backend"); - assert_eq!(fake.restores().len(), 1); } -#[tokio::test] -async fn restore_checks_the_arn_before_the_target_name() { - let fake = Fake::with_source(source(Some(definition("PROVISIONED")))); - // "x" is shorter than the 3-character minimum table name. - let malformed = json!({ "TargetTableName": "x", "BackupArn": "garbage" }); - let msg = validation_message(restore(&fake, malformed).await.unwrap_err()); - assert!(msg.contains("BackupArn"), "{msg}"); - - let foreign = json!({ "TargetTableName": "x", "BackupArn": arn("999999999999") }); - assert!(matches!( - restore(&fake, foreign).await.unwrap_err(), - DynamoDbError::AccessDeniedException(_) - )); - - let missing_name = json!({ "BackupArn": arn("999999999999") }); - assert!(matches!( - restore(&fake, missing_name).await.unwrap_err(), - DynamoDbError::AccessDeniedException(_) - )); +// --- the reconciler's restore duties ------------------------------------------- - // With a good ARN, the bad name is what is reported. - let bad_name = json!({ "TargetTableName": "x", "BackupArn": arn(ACCOUNT) }); - let msg = validation_message(restore(&fake, bad_name).await.unwrap_err()); - assert!(!msg.contains("BackupArn"), "{msg}"); - assert_eq!(fake.source_reads(), 0, "no storage read for a bad request"); - assert!(fake.restores().is_empty()); -} +/// A dead RUNNING manifest record of the configured store whose owner was +/// on another host: the reconciler takes it over (a fresh generation, this +/// host's `spool_host`, `heartbeat_at` refreshed so a second taker finds the +/// record live) and spawns the resume, which reads the manifest from the +/// store, removes the entry this host's earlier run of the job left (its +/// files were verified under a record that has since been another host's), +/// downloads every file again and finishes the restore from the recorded +/// progress; a live record is left alone. +#[tokio::test] +async fn the_reconciler_resumes_a_dead_manifest_restore_with_its_spool_host() { + let (lines, _guard) = LogLines::capture(); + let scratch = restore::spool::tests::Scratch::new(); + let mem = Arc::new(MemStore::default()); + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + let fake = Arc::new(Fake::default()); + let files = [songs_from(0, 2), songs_from(2, 2)]; + seed_manifest_backup( + &fake, + &mem, + &arn(ACCOUNT), + music_source_table(), + &files, + true, + ); + fake.rows + .lock() + .unwrap() + .get_mut(&arn(ACCOUNT)) + .unwrap() + .store = s3_identity(); + let runtime = spooled_runtime(&mem, scratch.path()); + // A record left by a dead owner on another host, file 0 applied. + let gate = Arc::new(tokio::sync::Semaphore::new(1)); + *fake.bulk_load_gate.lock().unwrap() = Some(Arc::clone(&gate)); + restore_on(&fake, &runtime, &restore_body(json!({}))) + .await + .unwrap(); + wait_for_bulk_loads_started(&fake, 2).await; + runtime.drain_jobs(Duration::from_millis(100)).await; + gate.add_permits(10); + wait_for_jobs(&runtime).await; + let before = fake.restore_job("MusicRestored").unwrap(); + assert_eq!(before.status, JobStatus::Running); + assert_eq!( + before.file_index, + Some(1), + "file 1 was in its load when the shutdown fired" + ); + *fake.bulk_load_gate.lock().unwrap() = None; + fake.bulk_loads.lock().unwrap().clear(); + let entry = scratch.path().join("restore-tid-MusicRestored"); -// --- overrides --------------------------------------------------------------- + // A live owner on this host: nothing happens, and the sweep keeps the + // entry of a RUNNING record that names this host. + let runtime2 = spooled_runtime(&mem, scratch.path()); + let report = cycle_on(&fake, &runtime2).await; + assert!(report.manifest_restores_resumed.is_empty()); + assert!(fake.restore_takeovers.lock().unwrap().is_empty()); + assert!(report.spool_entries_swept.is_empty()); + assert!(entry.join("000000.json.gz.verified").is_file()); + assert!(entry.join("000001.json.gz.verified").is_file()); -#[test] -fn restore_request_without_overrides_is_accepted() { - assert!( - parse_restore_table_overrides(&restore_body(json!({}))) + { + // As a backend whose file 1 write did not land would have left it: + // the record at file 0, the target holding file 0's items, the + // owner on another host (the record moved there and its job died), + // its heartbeat old. This host's entry from the stopped job still + // holds both files verified; it is not the record's and the resume + // removes it before use (the sweep would, a cycle later). + let mut records = fake.restore_jobs.lock().unwrap(); + let record = records.get_mut("tid-MusicRestored").unwrap(); + record.spool_host = Some("host-b:/var/spool".to_owned()); + record.heartbeat_at = 1234.5; + record.file_index = Some(0); + record.items_applied = Some(2); + fake.restore_tables + .lock() .unwrap() - .is_empty() + .get_mut("tid-MusicRestored") + .unwrap() + .items + .truncate(2); + } + mem.gets.lock().unwrap().clear(); + fake.restore_dead + .lock() + .unwrap() + .insert("tid-MusicRestored".to_owned()); + let report = cycle_on(&fake, &runtime2).await; + assert_eq!(report.manifest_restores_resumed, ["MusicRestored"]); + assert!(report.acted()); + let after = fake.restore_job("MusicRestored").unwrap(); + assert_ne!(after.owner_generation, before.owner_generation); + assert_eq!(after.spool_host.as_deref(), Some("host-a:/var/spool")); + assert!( + after.heartbeat_at > 1234.5, + "the takeover refreshed the heartbeat: {}", + after.heartbeat_at + ); + assert_eq!( + fake.restore_takeovers.lock().unwrap().as_slice(), + [( + "tid-MusicRestored".to_owned(), + Some("host-a:/var/spool".to_owned()) + )] + ); + wait_for_jobs(&runtime2).await; + assert_eq!(fake.bulk_load_files(), [(1, 0, true, 2)]); + let all: Vec = files.iter().flatten().cloned().collect(); + assert_eq!(fake.restored_items("MusicRestored"), all); + assert_eq!( + fake.restore_job("MusicRestored").unwrap().status, + JobStatus::Done + ); + assert_eq!( + gets_under(&mem, &arn(ACCOUNT)), + [ + EXTENDDB_MANIFEST_FILE, + MANIFEST_CHECKSUM_FILE, + "data/000002.json.gz", + ], + "the manifest from the store; file 0 was recorded applied, so only file 1 is \ + downloaded (the resume downloads only the files its progress leaves to load)" + ); + assert_eq!( + lines + .containing( + "has no live owner; taken over by this instance and resumed with 2 items of data \ + file 0 recorded as applied" + ) + .len(), + 1, + "the reconciler's line states the record's progress" + ); + assert_eq!( + lines + .containing("started, 2 data file(s), 4 items, key-ordered; resumed from data file 1 after 0 of its items") + .len(), + 1, + "the job's line names the first open file" + ); + assert_eq!( + lines + .containing("resumed by this instance; the manifest was read from the store") + .len(), + 1 + ); + assert_eq!( + lines + .containing("is left from a run whose record moved to another host") + .len(), + 1 ); + assert!(!entry.exists()); } -#[test] -fn provisioned_override_with_throughput_is_accepted() { - let overrides = parse_restore_table_overrides(&json!({ - "BillingModeOverride": "PROVISIONED", - "ProvisionedThroughputOverride": { "ReadCapacityUnits": 5, "WriteCapacityUnits": 5 }, - })) - .unwrap(); - validate_override_shape(&overrides).unwrap(); - validate_override_for_source(&overrides, "PAY_PER_REQUEST").unwrap(); - assert_eq!(overrides.billing_mode, Some(BillingMode::Provisioned)); - assert_eq!(overrides.provisioned_throughput, Some(throughput(5, 5))); -} +/// The manifest arm's refusals: no free permit skips the record to the next +/// cycle with no takeover; a held maintenance claim fails it `OWNER_DEAD` +/// and removes its target like a legacy record; a record of another store +/// is FAILED with `OTHER_STORE` and its target left to the FAILED-target +/// duty, which removes it next cycle; an unhealthy store skips it. +#[tokio::test] +async fn the_reconciler_skips_or_fails_a_dead_manifest_restore_it_cannot_resume() { + let (lines, _guard) = LogLines::capture(); + let mem = Arc::new(MemStore::default()); + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + let fake = Arc::new(Fake::default()); + seed_manifest_backup( + &fake, + &mem, + &arn(ACCOUNT), + music_source_table(), + &[songs(2)], + true, + ); + let runtime = job_runtime(&mem); + let gate = closed_gate(); + *fake.bulk_load_gate.lock().unwrap() = Some(Arc::clone(&gate)); + restore_on(&fake, &runtime, &restore_body(json!({}))) + .await + .unwrap(); + wait_for_bulk_loads_started(&fake, 1).await; + runtime.drain_jobs(Duration::from_millis(50)).await; + gate.add_permits(10); + wait_for_jobs(&runtime).await; + *fake.bulk_load_gate.lock().unwrap() = None; + fake.restore_dead + .lock() + .unwrap() + .insert("tid-MusicRestored".to_owned()); -#[test] -fn provisioned_override_without_throughput_is_refused() { - let overrides = - parse_restore_table_overrides(&json!({ "BillingModeOverride": "PROVISIONED" })).unwrap(); - assert!(matches!( - validate_override_shape(&overrides).unwrap_err(), - DynamoDbError::ValidationException(_) - )); -} + // No permit. + let full = job_runtime(&mem); + let mut held = Vec::new(); + for _ in 0..4 { + held.push(full.take_permit().await.unwrap()); + } + let report = cycle_on(&fake, &full).await; + assert_eq!(report.restores_skipped_no_permit, ["MusicRestored"]); + assert!(fake.restore_takeovers.lock().unwrap().is_empty()); + assert!(!report.acted()); + drop(held); -#[test] -fn throughput_override_under_pay_per_request_is_refused() { - let overrides = parse_restore_table_overrides(&json!({ - "BillingModeOverride": "PAY_PER_REQUEST", - "ProvisionedThroughputOverride": { "ReadCapacityUnits": 5, "WriteCapacityUnits": 5 }, - })) - .unwrap(); - let msg = - validation_message(validate_override_for_source(&overrides, "PROVISIONED").unwrap_err()); - assert!(msg.contains( - "ProvisionedThroughputOverride can only be specified with BillingModeOverride PROVISIONED" - )); -} + // The store unhealthy: the cycle's own check, which runs first, finds + // the store down. + mem.set_down(true); + let unhealthy = runtime_unhealthy(&mem, StoreProblem::Transport("down".to_owned())) + .with_job_settings(job_settings()); + let report = cycle_on(&fake, &unhealthy).await; + assert_eq!(report.restores_skipped_store_unhealthy, ["MusicRestored"]); + assert_eq!(report.store_healthy, Some(false)); + assert_eq!( + lines.containing("dead store-backed restores are not resumed this cycle"), + [ + "the configured backup store is unhealthy; dead store-backed restores are not \ + resumed this cycle" + ] + ); + assert!(fake.restore_takeovers.lock().unwrap().is_empty()); + mem.set_down(false); -#[test] -fn invalid_provisioned_throughput_override_matches_create_table() { - let expected = "One or more parameter values were invalid: ReadCapacityUnits and \ - WriteCapacityUnits must both be greater than or equal to 1 for table"; - for (read, write) in [(0, 5), (5, 0), (-1, 5), (5, -1)] { - let overrides = parse_restore_table_overrides(&json!({ - "BillingModeOverride": "PROVISIONED", - "ProvisionedThroughputOverride": { "ReadCapacityUnits": read, "WriteCapacityUnits": write }, - })) + // The maintenance claim held: counted once, as failed and removed. + *fake.maintenance_held.lock().unwrap() = true; + let report = cycle_on(&fake, &full).await; + assert_eq!( + report.manifest_restores_failed_and_removed, + ["MusicRestored"] + ); + assert!(report.manifest_restores_failed.is_empty()); + assert!(report.legacy_restores_removed.is_empty()); + assert!(report.acted()); + assert!(fake.restore_job("MusicRestored").is_none()); + assert!(!fake.tables.lock().unwrap().contains_key("MusicRestored")); + *fake.maintenance_held.lock().unwrap() = false; + + // Another store. + let mem2 = Arc::new(MemStore::default()); + mem2.insert(STORE_IDENTITY_KEY, &identity_file("store-2", "catalog-1")); + let fake = Arc::new(Fake::default()); + seed_manifest_backup( + &fake, + &mem2, + &arn(ACCOUNT), + music_source_table(), + &[songs(2)], + true, + ); + let runtime = job_runtime(&mem2); + let gate = closed_gate(); + *fake.bulk_load_gate.lock().unwrap() = Some(Arc::clone(&gate)); + restore_on(&fake, &runtime, &restore_body(json!({}))) + .await .unwrap(); - assert_eq!( - validation_message(validate_override_shape(&overrides).unwrap_err()), - expected - ); - } + wait_for_bulk_loads_started(&fake, 1).await; + runtime.drain_jobs(Duration::from_millis(50)).await; + gate.add_permits(10); + wait_for_jobs(&runtime).await; + fake.restore_dead + .lock() + .unwrap() + .insert("tid-MusicRestored".to_owned()); + let other = runtime_on( + &mem2, + StoreIdentity { + store_id: "store-2".to_owned(), + ..store() + }, + ) + .with_job_settings(job_settings()); + let report = cycle_on(&fake, &other).await; + assert_eq!( + report.manifest_restores_failed, + [("MusicRestored".to_owned(), OTHER_STORE.to_owned())] + ); + let record = fake.restore_job("MusicRestored").unwrap(); + assert_eq!(record.status, JobStatus::Failed); + assert_eq!(record.failure_reason.as_deref(), Some(OTHER_STORE)); + assert!( + fake.tables.lock().unwrap().contains_key("MusicRestored"), + "the target waits for the FAILED-target duty" + ); + assert!(fake.restore_takeovers.lock().unwrap().is_empty()); + let report = cycle_on(&fake, &other).await; + assert_eq!(report.failed_targets_removed, ["MusicRestored"]); + assert!(!fake.tables.lock().unwrap().contains_key("MusicRestored")); } -#[test] -fn missing_provisioned_throughput_override_member_matches_create_table() { - for (member, body) in [ - ( - "ReadCapacityUnits", - json!({ "BillingModeOverride": "PROVISIONED", - "ProvisionedThroughputOverride": { "WriteCapacityUnits": 5 } }), - ), - ( - "WriteCapacityUnits", - json!({ "BillingModeOverride": "PROVISIONED", - "ProvisionedThroughputOverride": { "ReadCapacityUnits": 5 } }), - ), - ] { - match parse_restore_table_overrides(&body).unwrap_err() { - DynamoDbError::SerializationException(m) => { - assert!(m.contains(&format!("missing field `{member}`")), "{m}"); - } - other => panic!("expected SerializationException, got {other:?}"), - } - } -} +/// A dead record with no progress at all whose target already holds items +/// is FAILED `target_not_empty` under the takeover and its target left in +/// place; the FAILED-target duty skips it on the next cycle and logs the +/// operator's command; a record with no progress into an empty target is +/// resumed from the first file. +#[tokio::test] +async fn a_resume_without_progress_needs_an_empty_target() { + let (lines, _guard) = LogLines::capture(); + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + seed_manifest_backup( + &fake, + &mem, + &arn(ACCOUNT), + music_source_table(), + &[songs(2)], + true, + ); + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + let runtime = job_runtime(&mem); + restore_on(&fake, &runtime, &restore_body(json!({}))) + .await + .unwrap(); + wait_for_jobs(&runtime).await; + // The record as a dead owner that registered and recorded nothing left + // it, over a target that holds an item the record does not account for. + dead_record_without_progress(&fake, true); + fake.bulk_loads.lock().unwrap().clear(); + let runtime = job_runtime(&mem); + let report = cycle_on(&fake, &runtime).await; + assert_eq!( + report.manifest_restores_failed, + [("MusicRestored".to_owned(), TARGET_NOT_EMPTY.to_owned())] + ); + assert_eq!(fake.restore_takeovers.lock().unwrap().len(), 1); + let record = fake.restore_job("MusicRestored").unwrap(); + assert_eq!(record.status, JobStatus::Failed); + assert_eq!(record.failure_reason.as_deref(), Some(TARGET_NOT_EMPTY)); + assert!( + fake.tables.lock().unwrap().contains_key("MusicRestored"), + "the target is left" + ); + wait_for_jobs(&runtime).await; + assert!(fake.bulk_loads.lock().unwrap().is_empty()); + // The FAILED-target duty leaves it to the operator. + let report = cycle_on(&fake, &runtime).await; + assert_eq!(report.targets_left_for_operator, ["MusicRestored"]); + assert!(report.failed_targets_removed.is_empty()); + assert!(fake.tables.lock().unwrap().contains_key("MusicRestored")); + let command = lines.containing(&format!( + "extenddb backup remove-target --account {ACCOUNT} MusicRestored" + )); + assert_eq!( + command.len(), + 2, + "the takeover's line and the duty's: {command:?}" + ); -#[test] -fn vector_index_override_is_refused() { - let body = restore_body(json!({ "VectorIndexOverride": {} })); + // An empty target with no progress is resumed from the first file. + let fake = Arc::new(Fake::default()); + let mem = Arc::new(MemStore::default()); + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + seed_manifest_backup( + &fake, + &mem, + &arn(ACCOUNT), + music_source_table(), + &[songs(2)], + true, + ); + let runtime = job_runtime(&mem); + restore_on(&fake, &runtime, &restore_body(json!({}))) + .await + .unwrap(); + wait_for_jobs(&runtime).await; + dead_record_without_progress(&fake, false); + fake.bulk_loads.lock().unwrap().clear(); + let runtime = job_runtime(&mem); + let report = cycle_on(&fake, &runtime).await; + assert_eq!(report.manifest_restores_resumed, ["MusicRestored"]); + wait_for_jobs(&runtime).await; + assert_eq!(fake.bulk_load_files(), [(0, 0, true, 2)]); assert_eq!( - validation_message(parse_restore_table_overrides(&body).unwrap_err()), - "RestoreTableFromBackup does not support VectorIndexOverride on ExtendDB" + fake.restore_job("MusicRestored").unwrap().status, + JobStatus::Done ); } -#[test] -fn null_override_members_are_absent() { - let body = restore_body(json!({ - "GlobalSecondaryIndexOverride": null, - "LocalSecondaryIndexOverride": null, - "SSESpecificationOverride": null, - "OnDemandThroughputOverride": null, - "VectorIndexOverride": null, - "BillingModeOverride": null, - "ProvisionedThroughputOverride": null, - })); - assert!(parse_restore_table_overrides(&body).unwrap().is_empty()); -} +/// Under the maintenance claim a dead record with no progress at all whose +/// target holds items is FAILED `target_not_empty` through the unclaimed +/// write and its target left for the operator, not `OWNER_DEAD` with the +/// target removed: the command's wait is satisfied (the record is no longer +/// RUNNING) and `remove-target`, under the same claim, removes it; one +/// whose target is empty is `OWNER_DEAD` and removed, as before. +#[tokio::test] +async fn a_dead_record_without_progress_under_the_maintenance_claim_keeps_its_target() { + let (lines, _guard) = LogLines::capture(); + let mem = Arc::new(MemStore::default()); + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + let fake = Arc::new(Fake::default()); + seed_manifest_backup( + &fake, + &mem, + &arn(ACCOUNT), + music_source_table(), + &[songs(2)], + true, + ); + let runtime = job_runtime(&mem); + restore_on(&fake, &runtime, &restore_body(json!({}))) + .await + .unwrap(); + wait_for_jobs(&runtime).await; + dead_record_without_progress(&fake, true); + *fake.maintenance_held.lock().unwrap() = true; + let report = cycle_on(&fake, &runtime).await; + assert_eq!( + report.manifest_restores_failed, + [("MusicRestored".to_owned(), TARGET_NOT_EMPTY.to_owned())] + ); + assert!(report.manifest_restores_failed_and_removed.is_empty()); + let record = fake.restore_job("MusicRestored").unwrap(); + assert_eq!(record.status, JobStatus::Failed); + assert_eq!(record.failure_reason.as_deref(), Some(TARGET_NOT_EMPTY)); + assert!( + fake.tables.lock().unwrap().contains_key("MusicRestored"), + "the target is left" + ); + assert!(fake.removed_targets.lock().unwrap().is_empty()); + assert!( + fake.restore_takeovers.lock().unwrap().is_empty(), + "no takeover under the claim" + ); + assert_eq!( + fake.failed_jobs.lock().unwrap().as_slice(), + [(record.target.clone(), TARGET_NOT_EMPTY.to_owned())], + "the unclaimed write" + ); + assert_eq!( + lines + .containing(&format!( + "extenddb backup remove-target --account {ACCOUNT} MusicRestored" + )) + .len(), + 1 + ); + // The command removes it under the same claim. + let removed = maintenance::remove_target( + &*fake, + &remove_target_request("MusicRestored", Duration::from_secs(300)), + ) + .await + .unwrap(); + assert_eq!(removed.failure_reason.as_deref(), Some(TARGET_NOT_EMPTY)); + assert!(!fake.tables.lock().unwrap().contains_key("MusicRestored")); -#[test] -fn index_and_on_demand_overrides_of_the_wrong_type_are_serialization_errors() { - for (field, value) in [ - ("GlobalSecondaryIndexOverride", json!("nope")), - ("LocalSecondaryIndexOverride", json!({})), - ("OnDemandThroughputOverride", json!([1])), - // An entry without its required members. - ( - "GlobalSecondaryIndexOverride", - json!([{ "IndexName": "gsi1" }]), - ), - ( - "LocalSecondaryIndexOverride", - json!([{ "IndexName": "lsi1" }]), - ), - ( - "OnDemandThroughputOverride", - json!({ "MaxReadRequestUnits": "x" }), - ), - ] { - let body = restore_body(json!({ field: value })); - assert!( - matches!( - overrides::IndexOverrides::parse(&body).unwrap_err(), - DynamoDbError::SerializationException(_) - ), - "{field} {value}" - ); - } + // An empty target: `OWNER_DEAD` and removed. + let fake = Arc::new(Fake::default()); + seed_manifest_backup( + &fake, + &mem, + &arn(ACCOUNT), + music_source_table(), + &[songs(2)], + true, + ); + let runtime = job_runtime(&mem); + restore_on(&fake, &runtime, &restore_body(json!({}))) + .await + .unwrap(); + wait_for_jobs(&runtime).await; + dead_record_without_progress(&fake, false); + *fake.maintenance_held.lock().unwrap() = true; + let report = cycle_on(&fake, &runtime).await; + assert_eq!( + report.manifest_restores_failed_and_removed, + ["MusicRestored"] + ); + assert!(report.manifest_restores_failed.is_empty()); + assert!(fake.restore_job("MusicRestored").is_none()); + assert!(!fake.tables.lock().unwrap().contains_key("MusicRestored")); } +/// Once the server's shutdown has begun (the jobs' token is cancelled) a +/// cycle takes no dead record over, writes no FAILED on the maintenance or +/// other-store arms, and spawns nothing: the record stays RUNNING under its +/// owner's generation with its heartbeat as it was, for the next instance's +/// reconciler; the cycle's other duties still run. #[tokio::test] -async fn a_malformed_index_override_is_refused_before_any_storage_read() { - let fake = Fake::with_source(source(Some(definition("PROVISIONED")))); - let err = restore( +async fn the_reconciler_takes_nothing_over_once_the_shutdown_began() { + let (lines, _guard) = LogLines::capture(); + let mem = Arc::new(MemStore::default()); + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + let fake = Arc::new(Fake::default()); + seed_manifest_backup( &fake, - restore_body(json!({ "GlobalSecondaryIndexOverride": [{ "IndexName": "gsi1" }] })), + &mem, + &arn(ACCOUNT), + music_source_table(), + &[songs(2)], + true, + ); + let runtime = job_runtime(&mem); + restore_on(&fake, &runtime, &restore_body(json!({}))) + .await + .unwrap(); + wait_for_jobs(&runtime).await; + dead_record_without_progress(&fake, false); + let before = fake.restore_job("MusicRestored").unwrap(); + let checks_before = fake.maintenance_checks.lock().unwrap().len(); + let shutdown = extenddb_storage::CancellationToken::new(); + let draining = job_runtime(&mem).with_shutdown(shutdown.clone()); + shutdown.cancel(); + + // The takeover arm, the maintenance arm and the other-store arm, each + // with the token cancelled. + let report = cycle_on(&fake, &draining).await; + assert!(report.manifest_restores_resumed.is_empty(), "{report:?}"); + assert!(!report.acted(), "{report:?}"); + *fake.maintenance_held.lock().unwrap() = true; + let report = cycle_on(&fake, &draining).await; + assert!( + report.manifest_restores_failed_and_removed.is_empty(), + "{report:?}" + ); + assert!(report.manifest_restores_failed.is_empty(), "{report:?}"); + *fake.maintenance_held.lock().unwrap() = false; + let mem2 = Arc::new(MemStore::default()); + mem2.insert(STORE_IDENTITY_KEY, &identity_file("store-2", "catalog-1")); + let other = runtime_on( + &mem2, + StoreIdentity { + store_id: "store-2".to_owned(), + ..store() + }, + ) + .with_job_settings(job_settings()) + .with_shutdown(shutdown.clone()); + let report = cycle_on(&fake, &other).await; + assert!(report.manifest_restores_failed.is_empty(), "{report:?}"); + + assert!( + fake.restore_takeovers.lock().unwrap().is_empty(), + "no takeover" + ); + assert!( + fake.failed_jobs.lock().unwrap().is_empty(), + "no FAILED write" + ); + assert!(restore_writes(&fake).is_empty()); + assert_eq!(draining.running_jobs(), 0, "nothing spawned"); + assert_eq!(fake.restore_job("MusicRestored").unwrap(), before); + assert_eq!( + lines + .containing("has no live owner and the server is shutting down; it is left for the next reconciler") + .len(), + 3 + ); + assert_eq!( + fake.maintenance_checks.lock().unwrap().len() - checks_before, + 3, + "each cycle ran its other duties" + ); + + // The same runtime with the token intact takes it over. + let runtime = job_runtime(&mem); + let report = cycle_on(&fake, &runtime).await; + assert_eq!(report.manifest_restores_resumed, ["MusicRestored"]); + wait_for_jobs(&runtime).await; + assert_eq!( + fake.restore_job("MusicRestored").unwrap().status, + JobStatus::Done + ); +} + +/// A reconciler whose own token has fired while the jobs' token has not +/// (the server stopping its workers while it holds no serving +/// registration, to start a new set once it holds one) takes no dead record +/// over and spawns no resume: a resume is a job, on the jobs' token, which +/// the workers' stop would not stop. The record stays as it was, the cycle's +/// other duties still run, and the next set's reconciler (a fresh state) +/// takes it over. +#[tokio::test] +async fn a_stopping_reconciler_takes_nothing_over() { + let (lines, _guard) = LogLines::capture(); + let mem = Arc::new(MemStore::default()); + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + let fake = Arc::new(Fake::default()); + seed_manifest_backup( + &fake, + &mem, + &arn(ACCOUNT), + music_source_table(), + &[songs(2)], + true, + ); + let runtime = job_runtime(&mem); + restore_on(&fake, &runtime, &restore_body(json!({}))) + .await + .unwrap(); + wait_for_jobs(&runtime).await; + dead_record_without_progress(&fake, false); + let before = fake.restore_job("MusicRestored").unwrap(); + let checks_before = fake.maintenance_checks.lock().unwrap().len(); + // The token fires once the cycle lists the dead records, after its + // store check, as a stop that lands mid-cycle does. + let stopped = extenddb_storage::CancellationToken::new(); + *fake.cancel_on_list_dead.lock().unwrap() = Some(stopped.clone()); + + let report = reconciler::cycle( + &fake, + &runtime, + Duration::from_secs(300), + "test", + &mut reconciler::ReconcilerState::with_shutdown(stopped.clone()), ) .await - .unwrap_err(); + .unwrap(); + assert!(stopped.is_cancelled(), "the cycle reached the dead records"); + assert!(report.manifest_restores_resumed.is_empty(), "{report:?}"); assert!( - matches!(err, DynamoDbError::SerializationException(_)), - "{err:?}" + !runtime.shutdown().is_cancelled(), + "the jobs' token is intact" + ); + assert!( + fake.restore_takeovers.lock().unwrap().is_empty(), + "no takeover" + ); + assert_eq!(runtime.running_jobs(), 0, "nothing spawned"); + assert_eq!(fake.restore_job("MusicRestored").unwrap(), before); + assert_eq!( + lines + .containing("has no live owner and the reconciler is stopping; it is left for the next reconciler") + .len(), + 1 + ); + assert_eq!( + fake.maintenance_checks.lock().unwrap().len() - checks_before, + 1, + "the cycle ran its other duties" + ); + + // The next set's reconciler takes it over. + let report = cycle_on(&fake, &runtime).await; + assert_eq!(report.manifest_restores_resumed, ["MusicRestored"]); + wait_for_jobs(&runtime).await; + assert_eq!( + fake.restore_job("MusicRestored").unwrap().status, + JobStatus::Done ); - assert_eq!(fake.source_reads(), 0); } -/// The backup's GSI and LSI as a client would send them back: the shape -/// `definition` records, in the wire's own form. -fn gsi_entry(pt: Option<(i64, i64)>) -> Value { - let mut entry = json!({ - "IndexName": "gsi1", - "KeySchema": [{ "AttributeName": "gpk", "KeyType": "HASH" }], - "Projection": { "ProjectionType": "ALL" }, - }); - if let Some((r, w)) = pt { - entry["ProvisionedThroughput"] = json!({ "ReadCapacityUnits": r, "WriteCapacityUnits": w }); +/// Rewrite the finished restore into `MusicRestored` as a dead owner's +/// RUNNING record with no progress recorded, its target CREATING and +/// holding three items, one more than the backup's two, when `with_item`, +/// else empty. +fn dead_record_without_progress(fake: &Fake, with_item: bool) { + let mut records = fake.restore_jobs.lock().unwrap(); + let record = records.get_mut("tid-MusicRestored").unwrap(); + record.status = JobStatus::Running; + record.completed_at = None; + record.file_index = None; + record.items_applied = None; + let mut targets = fake.restore_tables.lock().unwrap(); + let target = targets.get_mut("tid-MusicRestored").unwrap(); + target.active = false; + target.items.clear(); + if with_item { + // Three items against the backup's two: more than the backup's total, + // which the `target_not_empty` rule requires (a count at or below it + // is a first file applied in part and is resumed). + target.items.push(song("x", "y")); + target.items.push(song("x", "z")); + target.items.push(song("y", "z")); } - entry + if let Some(desc) = fake.tables.lock().unwrap().get_mut("MusicRestored") { + desc.table_status = TableStatus::Creating; + } + fake.restore_dead + .lock() + .unwrap() + .insert("tid-MusicRestored".to_owned()); } -fn lsi_entry() -> Value { - json!({ - "IndexName": "lsi1", - "KeySchema": [ - { "AttributeName": "pk", "KeyType": "HASH" }, - { "AttributeName": "lsk", "KeyType": "RANGE" }, - ], - "Projection": { "ProjectionType": "KEYS_ONLY" }, - }) -} +/// The two sweeps: progress rows of tables that are not RUNNING restore +/// targets go, a RUNNING target's stay; spool entries of restores not +/// RUNNING with this host's `spool_host` go (a dead one, one taken over by +/// another host), a live one of this host and every other name stay; both +/// run only without the maintenance claim; a backend that cannot list the +/// records skips both and the cycle goes on. +#[tokio::test] +async fn the_reconciler_sweeps_progress_rows_and_spool_entries() { + let scratch = restore::spool::tests::Scratch::new(); + let mem = Arc::new(MemStore::default()); + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + let fake = Arc::new(Fake::default()); + let runtime = spooled_runtime(&mem, scratch.path()); + for (id, host) in [ + ("live", "host-a:/var/spool"), + ("moved", "host-b:/var/spool"), + ] { + fake.restore_jobs.lock().unwrap().insert( + id.to_owned(), + RestoreJobRow { + target: RestoreTargetKey { + account_id: ACCOUNT.to_owned(), + table_name: id.to_owned(), + table_id: id.to_owned(), + }, + source_backup_arn: arn(ACCOUNT), + kind: RestoreKind::Manifest, + status: JobStatus::Running, + owner_generation: 1, + heartbeat_at: seconds_since_epoch(), + store: Some(s3_identity()), + spool_host: Some(host.to_owned()), + file_index: None, + items_applied: None, + failure_reason: None, + completed_at: None, + }, + ); + } + { + let mut rows = fake.progress_rows.lock().unwrap(); + rows.insert("live".to_owned(), vec![(0, 10)]); + rows.insert("gone".to_owned(), vec![(0, 10), (1, 5)]); + } + for name in ["restore-live", "restore-moved", "restore-dead", "other"] { + std::fs::create_dir(scratch.path().join(name)).unwrap(); + } + std::fs::write(scratch.path().join(".lock"), b"").unwrap(); -/// The target input of a restore of `src` with `body`'s overrides, or the -/// engine's refusal. -async fn restored_input( - src: BackupRestoreSource, - body: Value, -) -> Result { - let fake = Fake::with_source(src); - restore(&fake, restore_body(body)).await?; - Ok(fake.restores()[0].input.clone()) + *fake.maintenance_held.lock().unwrap() = true; + let report = cycle_on(&fake, &runtime).await; + assert_eq!(report.progress_rows_swept, 0); + assert!(report.spool_entries_swept.is_empty()); + assert!(fake.progress_sweeps.lock().unwrap().is_empty()); + assert!(scratch.path().join("restore-dead").is_dir()); + *fake.maintenance_held.lock().unwrap() = false; + + let report = cycle_on(&fake, &runtime).await; + assert_eq!(report.progress_rows_swept, 1); + let mut swept = report.spool_entries_swept.clone(); + swept.sort(); + assert_eq!(swept, ["dead", "moved"]); + assert!(report.acted()); + assert_eq!( + fake.progress_sweeps.lock().unwrap().as_slice(), + [vec!["live".to_owned(), "moved".to_owned()]], + "the RUNNING targets' ids, in id order" + ); + assert_eq!( + fake.progress_rows + .lock() + .unwrap() + .keys() + .cloned() + .collect::>(), + ["live"] + ); + let mut left: Vec = std::fs::read_dir(scratch.path()) + .unwrap() + .map(|e| e.unwrap().file_name().into_string().unwrap()) + .collect(); + left.sort(); + assert_eq!(left, [".lock", "other", "restore-live"]); + assert_eq!( + fake.restore_lists.lock().unwrap().as_slice(), + [Some(JobStatus::Running)] + ); } -/// The attribute names `input` defines. -fn attr_names(input: &CreateTableInput) -> Vec<&str> { - input - .attribute_definitions - .iter() - .map(|a| a.attribute_name.as_str()) - .collect() +/// The sweeps on a backend whose records cannot be listed yet: the cycle +/// goes on, logs once, and sweeps nothing. +#[tokio::test] +async fn a_backend_that_cannot_list_restore_records_skips_the_sweeps() { + let (lines, _guard) = LogLines::capture(); + let fake = Arc::new(Fake::default()); + fake.progress_rows + .lock() + .unwrap() + .insert("gone".to_owned(), vec![(0, 1)]); + fake.restore_list_error + .store(true, std::sync::atomic::Ordering::Release); + let report = cycle_on(&fake, &runtime()).await; + assert_eq!(report.progress_rows_swept, 0); + assert_eq!( + lines + .containing("the restore job records could not be listed") + .len(), + 1 + ); + assert_eq!(fake.progress_rows.lock().unwrap().len(), 1, "nothing swept"); } -fn gsi_names(input: &CreateTableInput) -> Vec<&str> { - input - .global_secondary_indexes - .iter() - .flatten() - .map(|g| g.index_name.as_str()) - .collect() +// --- the restore's liveness checks, spool rule and load bounds ----------------- + +/// The heartbeat's first confirmed beat is the instant `claim_restore` +/// returned, not the instant before it: a registration that takes four +/// periods answers CREATING to a job that runs to DONE, where a clock taken +/// before the registration would end it at its first check as a lost claim. +#[tokio::test] +async fn a_slow_registration_does_not_end_the_job_at_its_first_check() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + seed_manifest_backup( + &fake, + &mem, + &arn(ACCOUNT), + music_source_table(), + &[songs(2)], + true, + ); + let period = job_settings().heartbeat_interval; + *fake.claim_restore_delay.lock().unwrap() = Some(4 * period); + let runtime = job_runtime(&mem); + let response = restore_on(&fake, &runtime, &restore_body(json!({}))) + .await + .unwrap(); + assert_eq!(response["TableDescription"]["TableStatus"], "CREATING"); + wait_for_jobs(&runtime).await; + assert_eq!( + fake.restore_job("MusicRestored").unwrap().status, + JobStatus::Done + ); + assert_eq!(fake.restored_items("MusicRestored").len(), 2); + assert_eq!(restore_outcomes(&runtime), [("succeeded".to_owned(), 1.0)]); } -fn gsi_throughputs(input: &CreateTableInput) -> Vec> { - input - .global_secondary_indexes - .iter() - .flatten() - .map(|g| { - g.provisioned_throughput - .as_ref() - .map(|t| (t.read_capacity_units, t.write_capacity_units)) - }) - .collect() +/// The job's first beat, sent right after the registration and before the +/// reply, refreshes the record's `heartbeat_at`: a registration that +/// stamped it three periods in the past (a transaction-start `now()` behind +/// a lock, as on PostgreSQL) answers CREATING with the record's heartbeat of +/// this instant, one beat received before the reply, and the job runs to +/// DONE; a first beat that finds the claim lost fails the record before the +/// job starts, with the reason, and answers `InternalServerError`. On a +/// paused clock, so no timer beat runs between the reply and the read of +/// the record. +#[tokio::test(start_paused = true)] +async fn the_jobs_first_beat_refreshes_the_registrations_heartbeat_before_the_reply() { + let (lines, _guard) = LogLines::capture(); + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + seed_manifest_backup( + &fake, + &mem, + &arn(ACCOUNT), + music_source_table(), + &[songs(2)], + true, + ); + let period = job_settings().heartbeat_interval; + *fake.registration_heartbeat_age.lock().unwrap() = Some(3.0 * period.as_secs_f64() + 10.0); + let runtime = job_runtime(&mem); + let gate = closed_gate(); + *fake.bulk_load_gate.lock().unwrap() = Some(Arc::clone(&gate)); + let response = restore_on(&fake, &runtime, &restore_body(json!({}))) + .await + .unwrap(); + assert_eq!(response["TableDescription"]["TableStatus"], "CREATING"); + let age = seconds_since_epoch() - fake.restore_job("MusicRestored").unwrap().heartbeat_at; + assert!( + age < 1.0, + "the record's heartbeat is this instant's by the reply, not the registration's: {age}" + ); + assert!( + fake.heartbeats.load(std::sync::atomic::Ordering::Acquire) >= 1, + "one beat before the reply" + ); + gate.add_permits(10); + wait_for_jobs(&runtime).await; + assert_eq!( + fake.restore_job("MusicRestored").unwrap().status, + JobStatus::Done + ); + assert_eq!(restore_outcomes(&runtime), [("succeeded".to_owned(), 1.0)]); + + // The first beat finds the claim lost. + let fake = Arc::new(Fake::default()); + seed_manifest_backup( + &fake, + &mem, + &arn(ACCOUNT), + music_source_table(), + &[songs(2)], + true, + ); + *fake.heartbeat_answer.lock().unwrap() = HeartbeatAnswer::Lost; + let runtime = job_runtime(&mem); + let err = restore_on(&fake, &runtime, &restore_body(json!({}))) + .await + .unwrap_err(); + assert!( + matches!(err, DynamoDbError::InternalServerError(_)), + "{err:?}" + ); + wait_for_jobs(&runtime).await; + assert_eq!( + restore_writes(&fake), + [( + JobStatus::Running, + JobStatus::Failed, + Some("the claim was lost before the job started".to_owned()) + )] + ); + assert!(fake.bulk_loads.lock().unwrap().is_empty(), "no load"); + assert_eq!( + lines + .containing("FAILED (the claim was lost before the job started) before it started;") + .len(), + 1 + ); + assert_eq!(restore_outcomes(&runtime), [("failed".to_owned(), 1.0)]); + assert_eq!(runtime.free_slots(), 4); } -/// A backup with two GSIs, the second with an INCLUDE projection. -fn two_gsis(billing: &str) -> BackupTableDefinition { - let mut def = definition(billing); - let mut second = gsi( - "gsi2", - "gpk", - (billing == "PROVISIONED").then(|| throughput(1, 2)), +/// The registration's bound: a `claim_restore` that takes longer than +/// `RESTORE_REGISTRATION_TIMEOUT` has the request answer +/// `InternalServerError` at the bound with the line naming it, while the +/// task goes on and the job runs to DONE once the registration returns. +#[tokio::test(start_paused = true)] +async fn the_request_stops_waiting_at_the_registration_bound_and_the_task_goes_on() { + let (lines, _guard) = LogLines::capture(); + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + seed_manifest_backup( + &fake, + &mem, + &arn(ACCOUNT), + music_source_table(), + &[songs(2)], + true, ); - second.projection = Projection { - projection_type: ProjectionType::Include, - non_key_attributes: Some(vec!["a".to_owned(), "b".to_owned()]), - }; - def.global_secondary_indexes.push(second); - def + // The design's figure, 60 s on every backend; the registration takes + // one second longer. + assert_eq!(RESTORE_REGISTRATION_TIMEOUT, Duration::from_secs(60)); + *fake.claim_restore_delay.lock().unwrap() = Some(Duration::from_secs(61)); + let runtime = job_runtime(&mem); + let started = tokio::time::Instant::now(); + let err = restore_on(&fake, &runtime, &restore_body(json!({}))) + .await + .unwrap_err(); + assert!( + matches!(err, DynamoDbError::InternalServerError(_)), + "{err:?}" + ); + assert!( + started.elapsed() >= Duration::from_secs(60) && started.elapsed() < Duration::from_secs(61), + "answered at the bound: {:?}", + started.elapsed() + ); + assert_eq!(runtime.running_jobs(), 1, "the task goes on"); + assert_eq!( + lines.containing("the registration did not answer within 60 s; the task goes on"), + [format!( + "RestoreTableFromBackup of {} into MusicRestored (account {ACCOUNT}): the \ + registration did not answer within 60 s; the task goes on and the target, if \ + registered, appears through DescribeTable", + arn(ACCOUNT) + )] + ); + wait_for_jobs(&runtime).await; + assert_eq!( + fake.restore_job("MusicRestored").unwrap().status, + JobStatus::Done + ); + assert_eq!(fake.restored_items("MusicRestored").len(), 2); + assert_eq!(runtime.free_slots(), 4); } -fn gsi2_entry(nka: Value, pt: Option<(i64, i64)>) -> Value { - let mut entry = json!({ - "IndexName": "gsi2", - "KeySchema": [{ "AttributeName": "gpk", "KeyType": "HASH" }], - "Projection": { "ProjectionType": "INCLUDE", "NonKeyAttributes": nka }, - }); - if let Some((r, w)) = pt { - entry["ProvisionedThroughput"] = json!({ "ReadCapacityUnits": r, "WriteCapacityUnits": w }); +/// The pre-write check's second clause on the restore: with the job held +/// inside a `bulk_load` and its heartbeat timer ended without raising the +/// loss, the next check (before the next file, before the count, before the +/// DONE write) finds the last confirmed beat more than two periods old and +/// ends the job as a lost claim naming what it was about to do, with no +/// further `bulk_load`, no count, no `finish_restore` and no status write. +#[tokio::test] +async fn a_stopped_heartbeat_timer_stops_the_restore_before_its_next_write() { + let (lines, _guard) = LogLines::capture(); + let period = job_settings().heartbeat_interval; + // (the target's name, the bulk_load the job is held in, the reason the + // next check names, the bulk_loads and counts made) + let cells: [(&str, usize, &str, usize, usize); 3] = [ + ( + "TimerStoppedBeforeFile", + 1, + "no heartbeat confirmed the claim recently enough to load data/000002.json.gz", + 1, + 0, + ), + ( + "TimerStoppedBeforeCount", + 2, + "no heartbeat confirmed the claim recently enough to count the target", + 2, + 0, + ), + ( + "TimerStoppedBeforeFlip", + 0, + "no heartbeat confirmed the claim recently enough for the DONE write", + 2, + 1, + ), + ]; + for (table, held_in, reason, loads, counts) in cells { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + seed_manifest_backup( + &fake, + &mem, + &arn(ACCOUNT), + music_source_table(), + &[songs_from(0, 2), songs_from(2, 2)], + true, + ); + let runtime = job_runtime(&mem); + let gate = Arc::new(tokio::sync::Semaphore::new(held_in.saturating_sub(1))); + let count_gate = closed_gate(); + if held_in > 0 { + *fake.bulk_load_gate.lock().unwrap() = Some(Arc::clone(&gate)); + } else { + *fake.count_gate.lock().unwrap() = Some(Arc::clone(&count_gate)); + } + restore_on( + &fake, + &runtime, + &restore_body(json!({ "TargetTableName": table })), + ) + .await + .unwrap(); + let table_id = format!("tid-{table}"); + if held_in > 0 { + wait_for_bulk_loads_started(&fake, held_in).await; + } else { + // Both files loaded; the job waits in the count. + let deadline = tokio::time::Instant::now() + Duration::from_secs(30); + while fake.bulk_loads.lock().unwrap().len() < 2 { + assert!(tokio::time::Instant::now() < deadline, "{table}"); + tokio::time::sleep(Duration::from_millis(5)).await; + } + tokio::time::sleep(period / 2).await; + } + restore::test_support::stop_timer(&table_id); + tokio::time::sleep(3 * period).await; + gate.add_permits(10); + count_gate.add_permits(10); + wait_for_jobs(&runtime).await; + assert_eq!(fake.bulk_loads.lock().unwrap().len(), loads, "{table}"); + assert_eq!( + fake.count_calls.load(std::sync::atomic::Ordering::Acquire), + counts, + "{table}" + ); + assert!(restore_writes(&fake).is_empty(), "{table}: no status write"); + assert!(fake.removed_targets.lock().unwrap().is_empty(), "{table}"); + let record = fake.restore_job(table).unwrap(); + assert_eq!(record.status, JobStatus::Running, "{table}"); + let stopped = lines.containing(&format!( + "into {table} (account {ACCOUNT}): stopped with no further catalog or table access: \ + {reason}" + )); + assert_eq!(stopped.len(), 1, "{table}: {stopped:?}"); + assert_eq!( + restore_outcomes(&runtime), + [("failed".to_owned(), 1.0)], + "{table}" + ); } - entry } -/// Measured on DynamoDB: an empty index override list -/// restores no index of that kind. The attribute definitions only those -/// indexes used go with them, or CreateTable's validation would refuse the -/// input. +/// The DONE write's `NotMatched` outcome: the generation moved after the +/// last `bulk_load` returned (the job held in the count), so +/// `finish_restore` matches no record; the job removes no target, writes no +/// status, records `RestoreJobs{failed}` and the worker error, logs the +/// "matched no record" line, and leaves its spool entry to the sweep (the +/// record's owner may be reading it). #[tokio::test] -async fn an_empty_index_override_list_restores_no_indexes_of_that_kind() { - for def in [definition("PROVISIONED"), definition("PAY_PER_REQUEST")] { - let input = restored_input( - source(Some(def.clone())), - json!({ "GlobalSecondaryIndexOverride": [] }), - ) +async fn a_done_write_that_matches_no_record_removes_no_target_and_keeps_the_entry() { + let (lines, _guard) = LogLines::capture(); + let scratch = restore::spool::tests::Scratch::new(); + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + seed_manifest_backup( + &fake, + &mem, + &arn(ACCOUNT), + music_source_table(), + &[songs(2)], + true, + ); + fake.rows + .lock() + .unwrap() + .get_mut(&arn(ACCOUNT)) + .unwrap() + .store = s3_identity(); + *fake.heartbeat_answer.lock().unwrap() = HeartbeatAnswer::AlwaysConfirm; + let count_gate = closed_gate(); + *fake.count_gate.lock().unwrap() = Some(Arc::clone(&count_gate)); + let runtime = spooled_runtime(&mem, scratch.path()); + restore_on(&fake, &runtime, &restore_body(json!({}))) .await .unwrap(); - assert!(input.global_secondary_indexes.is_none(), "{input:?}"); + let deadline = tokio::time::Instant::now() + Duration::from_secs(30); + while fake.bulk_loads.lock().unwrap().is_empty() { + assert!(tokio::time::Instant::now() < deadline); + tokio::time::sleep(Duration::from_millis(5)).await; + } + let other = { + let mut records = fake.restore_jobs.lock().unwrap(); + let record = records.get_mut("tid-MusicRestored").unwrap(); + record.owner_generation = new_generation(); + record.owner_generation + }; + count_gate.add_permits(10); + wait_for_jobs(&runtime).await; + assert_eq!( + fake.count_calls.load(std::sync::atomic::Ordering::Acquire), + 1 + ); + assert!( + fake.removed_targets.lock().unwrap().is_empty(), + "no table access" + ); + assert!(restore_writes(&fake).is_empty(), "no status write"); + let record = fake.restore_job("MusicRestored").unwrap(); + assert_eq!(record.status, JobStatus::Running); + assert_eq!(record.owner_generation, other); + assert!(!fake.restore_tables.lock().unwrap()["tid-MusicRestored"].active); + let line = lines.containing("the DONE write matched no record"); + assert_eq!(line.len(), 1, "{line:?}"); + assert!(line[0].contains("no table access was made"), "{}", line[0]); + let metrics = drained(&runtime); + assert_eq!( + metrics + .iter() + .filter(|(m, _, _)| *m == MetricName::RestoreJobs) + .map(|(_, op, sum)| (op.clone(), *sum)) + .collect::>(), + [("failed".to_owned(), 1.0)] + ); + assert!( + metrics + .iter() + .any(|(m, op, _)| *m == MetricName::WorkerErrorCount && op == "RestoreJob"), + "{metrics:?}" + ); + assert!( + scratch.path().join("restore-tid-MusicRestored").is_dir(), + "the entry is left to the sweep" + ); + assert_eq!(runtime.free_slots(), 4); +} + +/// A job that learnt the record is another's leaves its spool entry in +/// place for the sweep: one whose claim is lost while it is held inside a +/// `bulk_load`, and one whose FAILED write matches no record (the +/// generation moved under it). Both end with no table access. +#[tokio::test] +async fn a_job_that_learnt_the_record_is_anothers_leaves_its_spool_entry() { + for lost_claim in [true, false] { + let scratch = restore::spool::tests::Scratch::new(); + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + seed_manifest_backup( + &fake, + &mem, + &arn(ACCOUNT), + music_source_table(), + &[songs(2)], + true, + ); + fake.rows + .lock() + .unwrap() + .get_mut(&arn(ACCOUNT)) + .unwrap() + .store = s3_identity(); + let gate = closed_gate(); + *fake.bulk_load_gate.lock().unwrap() = Some(Arc::clone(&gate)); + let runtime = spooled_runtime(&mem, scratch.path()); + restore_on(&fake, &runtime, &restore_body(json!({}))) + .await + .unwrap(); + wait_for_bulk_loads_started(&fake, 1).await; + let entry = scratch.path().join("restore-tid-MusicRestored"); + assert!(entry.join("000000.json.gz.verified").is_file()); + if lost_claim { + *fake.heartbeat_answer.lock().unwrap() = HeartbeatAnswer::Lost; + } else { + *fake.heartbeat_answer.lock().unwrap() = HeartbeatAnswer::AlwaysConfirm; + fake.restore_jobs + .lock() + .unwrap() + .get_mut("tid-MusicRestored") + .unwrap() + .owner_generation = new_generation(); + gate.add_permits(10); + } + wait_for_jobs(&runtime).await; + assert!( + fake.removed_targets.lock().unwrap().is_empty(), + "{lost_claim}: no table access" + ); assert_eq!( - input.local_secondary_indexes.as_ref().map(Vec::len), - Some(1) + restore_writes(&fake).len(), + usize::from(!lost_claim), + "{lost_claim}: a lost claim writes nothing, a failing job one FAILED write" ); - assert_eq!(attr_names(&input), ["pk", "sk", "lsk"]); + assert_eq!( + fake.restore_job("MusicRestored").unwrap().status, + JobStatus::Running, + "{lost_claim}" + ); + assert!( + entry.join("000000.json.gz.verified").is_file(), + "{lost_claim}: the entry is left to the sweep" + ); + assert_eq!( + restore_outcomes(&runtime), + [("failed".to_owned(), 1.0)], + "{lost_claim}" + ); + } +} - let input = restored_input( - source(Some(def.clone())), - json!({ "LocalSecondaryIndexOverride": [] }), - ) +/// `finish_restore` answering `Pending` (MongoDB: DONE with the index +/// backfill left to the backend's worker) is a success: the backfill line, +/// `RestoreJobs{succeeded}`, `WorkerLastSuccess{RestoreJob}`, the spool entry +/// removed and the record DONE, the table not flipped by the job. +#[tokio::test] +async fn a_pending_finish_is_a_success_whose_flip_is_the_backends() { + let (lines, _guard) = LogLines::capture(); + let scratch = restore::spool::tests::Scratch::new(); + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + seed_manifest_backup( + &fake, + &mem, + &arn(ACCOUNT), + music_source_table(), + &[songs(2)], + true, + ); + fake.rows + .lock() + .unwrap() + .get_mut(&arn(ACCOUNT)) + .unwrap() + .store = s3_identity(); + fake.finish_pending + .store(true, std::sync::atomic::Ordering::Release); + let runtime = spooled_runtime(&mem, scratch.path()); + restore_on(&fake, &runtime, &restore_body(json!({}))) .await .unwrap(); - assert!(input.local_secondary_indexes.is_none()); - assert_eq!(gsi_names(&input), ["gsi1"]); - assert_eq!(attr_names(&input), ["pk", "sk", "gpk"]); + wait_for_jobs(&runtime).await; + let record = fake.restore_job("MusicRestored").unwrap(); + assert_eq!(record.status, JobStatus::Done); + assert!(record.completed_at.is_some()); + assert!( + !fake.restore_tables.lock().unwrap()["tid-MusicRestored"].active, + "the flip is the backend's worker's" + ); + assert!(fake.removed_targets.lock().unwrap().is_empty()); + assert!(!scratch.path().join("restore-tid-MusicRestored").exists()); + let pending = + lines.containing("index backfill pending, the backend's worker flips the table ACTIVE"); + assert_eq!(pending.len(), 1, "{pending:?}"); + assert!( + pending[0].contains(": DONE, 2 items (2 applied by this run)"), + "{}", + pending[0] + ); + let metrics = drained(&runtime); + assert_eq!( + metrics + .iter() + .filter(|(m, _, _)| *m == MetricName::RestoreJobs) + .map(|(_, op, sum)| (op.clone(), *sum)) + .collect::>(), + [("succeeded".to_owned(), 1.0)] + ); + assert!( + metrics + .iter() + .any(|(m, op, _)| *m == MetricName::WorkerLastSuccess && op == "RestoreJob"), + "{metrics:?}" + ); + assert!( + !metrics + .iter() + .any(|(m, op, _)| *m == MetricName::WorkerErrorCount && op == "RestoreJob"), + "{metrics:?}" + ); +} - let input = restored_input( - source(Some(def)), - json!({ "GlobalSecondaryIndexOverride": [], "LocalSecondaryIndexOverride": [] }), - ) +/// The retry bound is the job's, from its first download attempt: the first +/// file succeeds after ten minutes of transport failures, the second fails +/// from then on, and the restore is FAILED naming the second file and the +/// bound once fifteen minutes have passed since the job's first attempt +/// (a clock per file would run it fifteen minutes longer). +#[tokio::test(start_paused = true)] +async fn the_download_retry_bound_is_the_jobs_not_each_files() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + seed_manifest_backup( + &fake, + &mem, + &arn(ACCOUNT), + music_source_table(), + &[songs_from(0, 2), songs_from(2, 2)], + true, + ); + let runtime = job_runtime(&mem); + *mem.fail_data_gets_until.lock().unwrap() = + Some(tokio::time::Instant::now() + Duration::from_secs(10 * 60)); + *mem.fail_gets_of.lock().unwrap() = Some("data/000002.json.gz".to_owned()); + restore_on(&fake, &runtime, &restore_body(json!({}))) .await .unwrap(); - assert!(input.global_secondary_indexes.is_none()); - assert!(input.local_secondary_indexes.is_none()); - assert_eq!(attr_names(&input), ["pk", "sk"]); - extenddb_core::validation::validate_create_table(&input, &LimitsConfig::default()) - .expect("CreateTable accepts the pruned input"); + let started = tokio::time::Instant::now(); + while runtime.running_jobs() > 0 { + tokio::time::sleep(Duration::from_millis(50)).await; } - // Also with a billing override to PROVISIONED on an on-demand backup: - // with no GSIs left there is nothing that needs throughput. - let input = restored_input( - source(Some(definition("PAY_PER_REQUEST"))), - json!({ - "BillingModeOverride": "PROVISIONED", - "ProvisionedThroughputOverride": { "ReadCapacityUnits": 5, "WriteCapacityUnits": 6 }, - "GlobalSecondaryIndexOverride": [], - }), + let elapsed = started.elapsed(); + assert!( + elapsed >= Duration::from_secs(15 * 60) && elapsed < Duration::from_secs(16 * 60 + 2), + "{elapsed:?}" + ); + let writes = restore_writes(&fake); + assert_eq!(writes.len(), 1); + let reason = writes[0].2.clone().unwrap(); + assert!( + reason.starts_with("downloading data/000002.json.gz: transport error"), + "{reason}" + ); + assert!(reason.contains("the bound is 900 s"), "{reason}"); + let reads = gets_under(&mem, &arn(ACCOUNT)); + let first: Vec<&String> = reads + .iter() + .filter(|k| k.ends_with("000001.json.gz")) + .collect(); + let second: Vec<&String> = reads + .iter() + .filter(|k| k.ends_with("000002.json.gz")) + .collect(); + // The backoff restarts per file (1+2+4+8+16+32 s, then 60 s each): about + // ten minutes of attempts on the first file, five on the second. + assert!( + (13..=18).contains(&first.len()), + "{} attempts on file 1", + first.len() + ); + assert!( + (9..=13).contains(&second.len()), + "{} attempts on file 2", + second.len() + ); + assert!(fake.bulk_loads.lock().unwrap().is_empty()); + assert_eq!(restore_outcomes(&runtime), [("failed".to_owned(), 1.0)]); +} + +/// The download's own pre-attempt check: with the first attempt failed and +/// the job in its backoff, a heartbeat timer ended without raising the loss +/// stops the job before the next attempt, naming the file; no second +/// attempt is made and nothing is written. +#[tokio::test(start_paused = true)] +async fn a_stopped_heartbeat_timer_stops_the_download_before_its_next_attempt() { + let (lines, _guard) = LogLines::capture(); + let period = job_settings().heartbeat_interval; + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + seed_manifest_backup( + &fake, + &mem, + &arn(ACCOUNT), + music_source_table(), + &[songs(2)], + true, + ); + let runtime = job_runtime(&mem); + mem.fail_gets_with(GetFailure::Transport, 1); + restore_on( + &fake, + &runtime, + &restore_body(json!({ "TargetTableName": "TimerStoppedInBackoff" })), ) .await .unwrap(); - assert!(input.global_secondary_indexes.is_none()); - assert_eq!(input.provisioned_throughput, Some(throughput(5, 6))); + // The first attempt failed; the job sleeps its first backoff (at least + // one second), during which the timer is ended. + while gets_under(&mem, &arn(ACCOUNT)).len() < 3 { + tokio::time::sleep(Duration::from_millis(5)).await; + } + restore::test_support::stop_timer("tid-TimerStoppedInBackoff"); + tokio::time::sleep(3 * period).await; + while runtime.running_jobs() > 0 { + tokio::time::sleep(Duration::from_millis(5)).await; + } + assert_eq!( + gets_under(&mem, &arn(ACCOUNT)).len(), + 3, + "no second attempt" + ); + assert!(restore_writes(&fake).is_empty()); + assert!(fake.bulk_loads.lock().unwrap().is_empty()); + let stopped = lines.containing( + "stopped with no further catalog or table access: no heartbeat confirmed the claim \ + recently enough to download data/000001.json.gz", + ); + assert_eq!(stopped.len(), 1, "{stopped:?}"); + assert_eq!( + fake.restore_job("TimerStoppedInBackoff").unwrap().status, + JobStatus::Running + ); } -/// Measured on DynamoDB: the service applies a changed -/// index list. A subset restores those indexes, a changed GSI throughput is -/// applied, the list's order is the restored order, and a list that -/// describes the backup's indexes unchanged restores the same table as no -/// list at all. +/// A decoder that dies on the blocking pool (its reader panics mid-file) +/// drops the channel without an `Err` item, so the backend sees a clean end +/// of a short stream; the job fails the file rather than reporting it +/// applied, and the target is removed. #[tokio::test] -async fn index_override_lists_are_applied() { - let plain = restored_input(source(Some(definition("PROVISIONED"))), json!({})) +async fn a_decoder_that_dies_fails_the_file_rather_than_completing_it() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + seed_manifest_backup( + &fake, + &mem, + &arn(ACCOUNT), + music_source_table(), + &[songs(2)], + true, + ); + // The gets: the sidecar, the manifest, the file's hashing pass, then + // the file's load pass, whose body dies. + mem.panic_body_of_get + .store(4, std::sync::atomic::Ordering::Release); + let runtime = job_runtime(&mem); + restore_on(&fake, &runtime, &restore_body(json!({}))) .await .unwrap(); - assert_eq!(gsi_throughputs(&plain), [Some((3, 4))]); - assert_eq!(attr_names(&plain), ["pk", "sk", "gpk", "lsk"]); + wait_for_jobs(&runtime).await; + let loads = fake.bulk_loads.lock().unwrap().clone(); + assert_eq!(loads.len(), 1); + assert_eq!(loads[0].stream_error, None, "the backend saw a clean end"); + assert!(loads[0].items.len() < 2, "{} items", loads[0].items.len()); + let writes = restore_writes(&fake); + assert_eq!(writes.len(), 1); + assert_eq!( + writes[0].2.as_deref(), + Some("data file data/000001.json.gz: the decoder stopped unexpectedly") + ); + assert_eq!(fake.removed_targets.lock().unwrap().len(), 1); + assert!(fake.restore_job("MusicRestored").is_none()); + assert_eq!(restore_outcomes(&runtime), [("failed".to_owned(), 1.0)]); +} - // Unchanged, as the console sends it (the backup's throughput). - let same = restored_input( - source(Some(definition("PROVISIONED"))), - json!({ - "GlobalSecondaryIndexOverride": [gsi_entry(Some((3, 4)))], - "LocalSecondaryIndexOverride": [lsi_entry()], - }), - ) - .await - .unwrap(); - assert_eq!(same, plain); +/// Stop a restore into `MusicRestored` on `runtime` at its first `bulk_load` +/// through the shutdown, so its spool entry holds every file verified; then +/// rewrite its record as a dead owner's with no progress recorded over an +/// empty target (the shutdown lets the held file finish, so the record +/// would otherwise name file 0). +async fn stop_a_spooled_restore_before_its_first_load( + fake: &Arc, + runtime: &BackupRuntime, + gate: &Arc, +) { + *fake.bulk_load_gate.lock().unwrap() = Some(Arc::clone(gate)); + restore_on(fake, runtime, &restore_body(json!({}))) + .await + .unwrap(); + wait_for_bulk_loads_started(fake, 1).await; + runtime.drain_jobs(Duration::from_millis(100)).await; + gate.add_permits(10); + wait_for_jobs(runtime).await; + *fake.bulk_load_gate.lock().unwrap() = None; + fake.bulk_loads.lock().unwrap().clear(); + { + let mut records = fake.restore_jobs.lock().unwrap(); + let record = records.get_mut("tid-MusicRestored").unwrap(); + assert_eq!(record.status, JobStatus::Running); + record.file_index = None; + record.items_applied = None; + record.heartbeat_at = 1234.5; + let mut targets = fake.restore_tables.lock().unwrap(); + let target = targets.get_mut("tid-MusicRestored").unwrap(); + target.items.clear(); + target.active = false; + } + fake.restore_dead + .lock() + .unwrap() + .insert("tid-MusicRestored".to_owned()); +} - // A changed GSI throughput. - let changed = restored_input( - source(Some(definition("PROVISIONED"))), - json!({ "GlobalSecondaryIndexOverride": [gsi_entry(Some((30, 40)))] }), - ) - .await - .unwrap(); - assert_eq!(gsi_throughputs(&changed), [Some((30, 40))]); - assert_eq!(gsi_names(&changed), ["gsi1"]); - assert_eq!(attr_names(&changed), attr_names(&plain)); +/// A same-host resume with every file verified in the spool reads the +/// manifest and sidecar from the spool and downloads nothing: no store +/// read at all, no `RestoreBytesDownloaded`. +#[tokio::test] +async fn a_same_host_resume_reads_the_spool_and_downloads_nothing() { + let (lines, _guard) = LogLines::capture(); + let scratch = restore::spool::tests::Scratch::new(); + let mem = Arc::new(MemStore::default()); + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + let fake = Arc::new(Fake::default()); + let files = [songs_from(0, 2), songs_from(2, 2)]; + seed_manifest_backup( + &fake, + &mem, + &arn(ACCOUNT), + music_source_table(), + &files, + true, + ); + fake.rows + .lock() + .unwrap() + .get_mut(&arn(ACCOUNT)) + .unwrap() + .store = s3_identity(); + let runtime = spooled_runtime(&mem, scratch.path()); + stop_a_spooled_restore_before_its_first_load(&fake, &runtime, &closed_gate()).await; + mem.gets.lock().unwrap().clear(); + let _ = drained(&runtime); - // A subset: the backup has two GSIs, the list names one. The other - // index's attribute definition goes with it. - let subset = restored_input( - source(Some(two_gsis("PROVISIONED"))), - json!({ "GlobalSecondaryIndexOverride": [gsi2_entry(json!(["b", "a"]), Some((1, 2)))] }), - ) - .await - .unwrap(); - assert_eq!(gsi_names(&subset), ["gsi2"]); - assert_eq!(gsi_throughputs(&subset), [Some((1, 2))]); + let runtime2 = spooled_runtime(&mem, scratch.path()); + let report = cycle_on(&fake, &runtime2).await; + assert_eq!(report.manifest_restores_resumed, ["MusicRestored"]); + wait_for_jobs(&runtime2).await; + assert!( + gets_under(&mem, &arn(ACCOUNT)).is_empty(), + "{:?}", + gets_under(&mem, &arn(ACCOUNT)) + ); + assert_eq!(fake.bulk_load_files(), [(0, 0, true, 2), (1, 0, false, 2)]); + let all: Vec = files.iter().flatten().cloned().collect(); + assert_eq!(fake.restored_items("MusicRestored"), all); assert_eq!( - subset.global_secondary_indexes.as_ref().unwrap()[0] - .projection - .non_key_attributes, - Some(vec!["a".to_owned(), "b".to_owned()]), - "the backup's projection is restored, not the entry's spelling of it" + fake.restore_job("MusicRestored").unwrap().status, + JobStatus::Done ); - assert_eq!(attr_names(&subset), ["pk", "sk", "gpk", "lsk"]); - extenddb_core::validation::validate_create_table(&subset, &LimitsConfig::default()).unwrap(); + assert_eq!( + lines + .containing("resumed by this instance; the manifest was read from this host's spool") + .len(), + 1 + ); + assert!( + !drained(&runtime2) + .iter() + .any(|(m, _, _)| *m == MetricName::RestoreBytesDownloaded), + "nothing downloaded" + ); + let active = lines.containing(": ACTIVE, 4 items (4 applied by this run), 0 bytes downloaded"); + assert_eq!(active.len(), 1, "{active:?}"); +} - // The list's order is the restored order. - let reordered = restored_input( - source(Some(two_gsis("PROVISIONED"))), - json!({ "GlobalSecondaryIndexOverride": [ - gsi2_entry(json!(["a", "b"]), Some((1, 2))), - gsi_entry(Some((3, 4))), - ] }), - ) - .await - .unwrap(); - assert_eq!(gsi_names(&reordered), ["gsi2", "gsi1"]); +/// A `.verified` marker binds the digest: a marker naming another digest +/// does not make its file verified, and a same-host resume downloads that +/// file again and only it. +#[tokio::test] +async fn a_marker_naming_another_digest_has_its_file_downloaded_again() { + let scratch = restore::spool::tests::Scratch::new(); + let mem = Arc::new(MemStore::default()); + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + let fake = Arc::new(Fake::default()); + let files = [songs_from(0, 2), songs_from(2, 2)]; + seed_manifest_backup( + &fake, + &mem, + &arn(ACCOUNT), + music_source_table(), + &files, + true, + ); + fake.rows + .lock() + .unwrap() + .get_mut(&arn(ACCOUNT)) + .unwrap() + .store = s3_identity(); + let runtime = spooled_runtime(&mem, scratch.path()); + stop_a_spooled_restore_before_its_first_load(&fake, &runtime, &closed_gate()).await; + let entry = scratch.path().join("restore-tid-MusicRestored"); + let marker = entry.join("000000.json.gz.verified"); + let recorded = std::fs::read_to_string(&marker).unwrap(); + assert_eq!(recorded.trim_end(), files_sha256(&mem, &arn(ACCOUNT), 1)); + std::fs::write(&marker, format!("{}\n", "0".repeat(64))).unwrap(); + mem.gets.lock().unwrap().clear(); - // An LSI subset of one (the only one) and an on-demand backup's GSIs, - // listed without throughput as DescribeBackup's {0, 0} cannot be sent. - let on_demand = restored_input(source(Some(two_gsis("PAY_PER_REQUEST"))), json!({})) - .await - .unwrap(); - let listed = restored_input( - source(Some(two_gsis("PAY_PER_REQUEST"))), - json!({ - "GlobalSecondaryIndexOverride": [gsi_entry(None), gsi2_entry(json!(["a", "b"]), None)], - "LocalSecondaryIndexOverride": [lsi_entry()], - }), + let runtime2 = spooled_runtime(&mem, scratch.path()); + let report = cycle_on(&fake, &runtime2).await; + assert_eq!(report.manifest_restores_resumed, ["MusicRestored"]); + wait_for_jobs(&runtime2).await; + assert_eq!( + gets_under(&mem, &arn(ACCOUNT)), + ["data/000001.json.gz"], + "the file whose marker names another digest, and nothing else" + ); + let all: Vec = files.iter().flatten().cloned().collect(); + assert_eq!(fake.restored_items("MusicRestored"), all); + assert_eq!( + fake.restore_job("MusicRestored").unwrap().status, + JobStatus::Done + ); + assert!(!entry.exists(), "the entry is removed with the job's end"); +} + +/// The lowercase hex sha256 of data file `n` (1-based key) of `arn` in `mem`. +fn files_sha256(mem: &MemStore, arn: &str, n: u32) -> String { + use sha2::Digest; + let prefix = BackupArn::parse(arn).unwrap().store_prefix(); + let key = format!("{prefix}{}", data_file_key(n)); + let bytes = mem.objects.lock().unwrap()[&key].clone(); + format!("{:x}", sha2::Sha256::digest(&bytes)) +} + +/// The spool states a crash can leave, and what a same-host resume +/// downloads: file 0 verified (file and marker), file 1 a `.partial` only, +/// file 2 a file without its marker, file 3 a marker without its file. The +/// verified file is kept; the other three are downloaded again, every file +/// is loaded and the restore completes. +#[tokio::test] +async fn a_resume_downloads_the_spool_files_a_crash_left_unverified() { + let scratch = restore::spool::tests::Scratch::new(); + let mem = Arc::new(MemStore::default()); + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + let fake = Arc::new(Fake::default()); + let files = [ + songs_from(0, 2), + songs_from(2, 2), + songs_from(4, 2), + songs_from(6, 2), + ]; + seed_manifest_backup( + &fake, + &mem, + &arn(ACCOUNT), + music_source_table(), + &files, + true, + ); + fake.rows + .lock() + .unwrap() + .get_mut(&arn(ACCOUNT)) + .unwrap() + .store = s3_identity(); + let runtime = spooled_runtime(&mem, scratch.path()); + stop_a_spooled_restore_before_its_first_load(&fake, &runtime, &closed_gate()).await; + let entry = scratch.path().join("restore-tid-MusicRestored"); + for i in 0..4 { + assert!(entry.join(format!("{i:06}.json.gz.verified")).is_file()); + } + std::fs::rename( + entry.join("000001.json.gz"), + entry.join("000001.json.gz.partial"), ) - .await .unwrap(); - assert_eq!(listed, on_demand); + std::fs::remove_file(entry.join("000001.json.gz.verified")).unwrap(); + std::fs::remove_file(entry.join("000002.json.gz.verified")).unwrap(); + std::fs::remove_file(entry.join("000003.json.gz")).unwrap(); + mem.gets.lock().unwrap().clear(); + + let runtime2 = spooled_runtime(&mem, scratch.path()); + let report = cycle_on(&fake, &runtime2).await; + assert_eq!(report.manifest_restores_resumed, ["MusicRestored"]); + wait_for_jobs(&runtime2).await; + assert_eq!( + gets_under(&mem, &arn(ACCOUNT)), + [ + "data/000002.json.gz", + "data/000003.json.gz", + "data/000004.json.gz" + ], + "the verified file is kept; the .partial, the unmarked file and the marker without a \ + file are downloaded again" + ); + assert_eq!(fake.bulk_load_files().len(), 4); + let all: Vec = files.iter().flatten().cloned().collect(); + assert_eq!(fake.restored_items("MusicRestored"), all); + assert_eq!( + fake.restore_job("MusicRestored").unwrap().status, + JobStatus::Done + ); + assert!(!entry.exists()); } -/// The DescribeBackup echo (a console or SDK restore flow) restores exactly -/// the table the restore creates without it, for both billing modes; sent -/// with the `{0, 0}` DescribeBackup reports for an on-demand backup's GSI it -/// is refused as CreateTable refuses `{0, 0}`. -#[tokio::test] -async fn index_overrides_echoing_describe_backup_restore_the_same_table() { - for def in [definition("PAY_PER_REQUEST"), definition("PROVISIONED")] { - let on_demand = def.billing_mode == "PAY_PER_REQUEST"; - let fake = fake_with_backup("AVAILABLE", source(Some(def.clone()))); - let features = describe(&fake).await["SourceTableFeatureDetails"].clone(); - let mut gsis = features["GlobalSecondaryIndexes"].clone(); - let lsis = features["LocalSecondaryIndexes"].clone(); - assert!( - gsis.as_array().is_some_and(|g| g.len() == 1), - "precondition, DescribeBackup reports the GSI: {features}" - ); - if on_demand { - assert_eq!( - gsis[0]["ProvisionedThroughput"], - json!({ "ReadCapacityUnits": 0, "WriteCapacityUnits": 0 }), - "precondition: an on-demand backup's GSI is reported as {{0, 0}}" - ); - let zero_echo = restored_input( - source(Some(def.clone())), - json!({ "GlobalSecondaryIndexOverride": gsis.clone() }), - ) - .await - .unwrap_err(); - let msg = validation_message(zero_echo); - assert!( - msg.contains("gsi1") && msg.contains("should not be specified"), - "{msg}" +/// Songs `from..from + n` each carrying a `Blob` of `bytes` bytes. +fn padded_songs(from: usize, n: usize, bytes: usize, pattern: impl Fn(usize) -> u8) -> Vec { + songs_from(from, n) + .into_iter() + .enumerate() + .map(|(i, mut item)| { + let blob: Vec = (0..bytes) + .map(|j| pattern(i * 7919 + j * 104_729)) + .collect(); + item.insert( + "Blob".to_owned(), + extenddb_core::types::AttributeValue::B(blob), ); - for g in gsis.as_array_mut().unwrap() { - g.as_object_mut().unwrap().remove("ProvisionedThroughput"); + item + }) + .collect() +} + +/// `padded_songs` whose blob is `zeros` zero bytes followed by bytes from +/// `pattern`: the zeros compress to nothing, so one item's `zeros` tunes the +/// compressed length of a file by under a byte per step while the +/// uncompressed length and the item count stay the same. +fn padded_songs_with_zeros( + from: usize, + n: usize, + bytes: usize, + zeros_in_first: usize, + pattern: impl Fn(usize) -> u8, +) -> Vec { + let mut items = padded_songs(from, n, bytes, pattern); + if let Some(first) = items.first_mut() { + let blob: Vec = (0..bytes) + .map(|j| { + if j < zeros_in_first { + 0 + } else { + (j.wrapping_mul(2_654_435_761) >> 7) as u8 + } + }) + .collect(); + first.insert( + "Blob".to_owned(), + extenddb_core::types::AttributeValue::B(blob), + ); + } + items +} + +/// The blob byte pattern of `seed`: bytes with no structure a compressor +/// finds, so two seeds give files of nearly the same compressed length. +fn blob_pattern(seed: usize) -> impl Fn(usize) -> u8 + Copy { + move |x: usize| { + let mut z = (x as u64).wrapping_add((seed as u64).wrapping_mul(0x9E37_79B9_7F4A_7C15)); + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + (z ^ (z >> 31)) as u8 + } +} + +/// A data file of the same item count, uncompressed length and compressed +/// length as `original` (the file of `blob_pattern(0)`), holding other items +/// of the same keys and blob length (so only the digests differ): the blobs +/// are seeded differently per try, and within a try the zeros at the head of +/// the first item's blob are swept until the compressed length matches. +fn same_length_replacement(original: &[u8], from: usize, n: usize, bytes: usize) -> Vec { + for seed in 1usize..200 { + let pattern = blob_pattern(seed); + let (base, _) = + encode_data_file(padded_songs_with_zeros(from, n, bytes, 0, pattern)).unwrap(); + if base.len() < original.len() || base.len() > original.len() + bytes / 2 { + continue; + } + for zeros in 0..=bytes { + let (candidate, _) = + encode_data_file(padded_songs_with_zeros(from, n, bytes, zeros, pattern)).unwrap(); + if candidate.len() == original.len() { + assert_ne!(candidate, original); + return candidate; + } + if candidate.len() + 8 < original.len() { + break; } } - let plain = restored_input(source(Some(def.clone())), json!({})) - .await - .unwrap(); - let echoed = restored_input( - source(Some(def)), - json!({ - "GlobalSecondaryIndexOverride": gsis, - "LocalSecondaryIndexOverride": lsis, - }), - ) - .await - .unwrap_or_else(|e| panic!("the DescribeBackup echo was refused: {e:?}")); - assert_eq!(echoed, plain); } + panic!( + "no replacement of {} compressed bytes found", + original.len() + ) } -/// Measured on DynamoDB: refused by the service and by ExtendDB: an unknown or -/// changed index definition, a throughput member contradicting the billing -/// mode, `{0, 0}`. ExtendDB's own: a member it cannot apply. +/// On an in-place (filesystem) store a data file replaced between the +/// hashing pass and the load pass by another valid file of the same item +/// count, uncompressed length and compressed length (so only its digests +/// differ from the manifest's): the items of the replacement reach the +/// backend as they decode (two chunks of a thousand), the end-of-stream +/// check then fails the file on its sha256, since the load's second read is +/// hashed again, the restore is FAILED naming the file and the target is +/// removed. +/// +/// The job runs at a 1 s heartbeat, not the 50 ms of `job_settings`: the +/// hashing pass over both files runs before the first load's claim check, +/// on this test's one runtime thread with the heartbeat's timer, and under +/// a loaded host it can outlast two 50 ms periods, which ends the job as a +/// lost claim before any load. #[tokio::test] -async fn index_overrides_the_service_refuses_are_refused() { - let entry_with = |key: &str, value: Value| { - let mut e = gsi_entry(Some((3, 4))); - e[key] = value; - e - }; - let cases: [(&str, Value, &str); 13] = [ - ( - "an index the backup does not have", - json!({ "LocalSecondaryIndexOverride": [], "GlobalSecondaryIndexOverride": [ - entry_with("IndexName", json!("other")), - ] }), - "global secondary index other is not in the backup", - ), - ( - "listed twice", - json!({ "GlobalSecondaryIndexOverride": [ - gsi_entry(Some((3, 4))), gsi_entry(Some((3, 4))), - ] }), - "global secondary index gsi1 is listed more than once", - ), - ( - "key schema", - json!({ "GlobalSecondaryIndexOverride": [entry_with( - "KeySchema", json!([{ "AttributeName": "pk", "KeyType": "HASH" }]), - )] }), - "index gsi1: KeySchema differs from the backup's", - ), - ( - "key schema order", - json!({ "LocalSecondaryIndexOverride": [{ - "IndexName": "lsi1", - "KeySchema": [ - { "AttributeName": "lsk", "KeyType": "RANGE" }, - { "AttributeName": "pk", "KeyType": "HASH" }, - ], - "Projection": { "ProjectionType": "KEYS_ONLY" }, - }] }), - "index lsi1: KeySchema differs from the backup's", - ), - ( - "projection type", - json!({ "GlobalSecondaryIndexOverride": [entry_with( - "Projection", json!({ "ProjectionType": "KEYS_ONLY" }), - )] }), - "index gsi1: Projection differs from the backup's", - ), - ( - "projected attributes", - json!({ "GlobalSecondaryIndexOverride": [entry_with( - "Projection", json!({ "ProjectionType": "ALL", "NonKeyAttributes": ["x"] }), - )] }), - "index gsi1: Projection differs from the backup's", - ), - ( - "throughput on an on-demand restore", - json!({ - "BillingModeOverride": "PAY_PER_REQUEST", - "GlobalSecondaryIndexOverride": [gsi_entry(Some((3, 4)))], - }), - "ProvisionedThroughput should not be specified for index: gsi1", - ), - ( - "{0, 0}", - json!({ "GlobalSecondaryIndexOverride": [gsi_entry(Some((0, 0)))] }), - "ReadCapacityUnits and WriteCapacityUnits must both be greater than or equal to 1", - ), - ( - "an index on-demand throughput on a PROVISIONED restore", - json!({ "GlobalSecondaryIndexOverride": [entry_with( - "OnDemandThroughput", json!({ "MaxReadRequestUnits": 10 }), - )] }), - "index gsi1: OnDemandThroughput can only be specified when the restored table's \ - BillingMode is PAY_PER_REQUEST", - ), - ( - "a member ExtendDB cannot apply", - json!({ "GlobalSecondaryIndexOverride": [entry_with( - "WarmThroughput", json!({ "ReadUnitsPerSecond": 12000 }), - )] }), - "index gsi1: member WarmThroughput is not supported", - ), - ( - "an LSI member ExtendDB cannot apply", - json!({ "LocalSecondaryIndexOverride": [{ - "IndexName": "lsi1", - "KeySchema": [ - { "AttributeName": "pk", "KeyType": "HASH" }, - { "AttributeName": "lsk", "KeyType": "RANGE" }, - ], - "Projection": { "ProjectionType": "KEYS_ONLY" }, - "ProvisionedThroughput": { "ReadCapacityUnits": 1, "WriteCapacityUnits": 1 }, - }] }), - "index lsi1: member ProvisionedThroughput is not supported", - ), - ( - "an LSI the backup does not have", - json!({ "LocalSecondaryIndexOverride": [{ - "IndexName": "lsi2", - "KeySchema": [ - { "AttributeName": "pk", "KeyType": "HASH" }, - { "AttributeName": "lsk", "KeyType": "RANGE" }, - ], - "Projection": { "ProjectionType": "KEYS_ONLY" }, - }] }), - "local secondary index lsi2 is not in the backup", - ), - ( - "an LSI listed twice", - json!({ "LocalSecondaryIndexOverride": [lsi_entry(), lsi_entry()] }), - "local secondary index lsi1 is listed more than once", - ), +async fn a_file_replaced_between_the_two_passes_fails_in_the_load() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + // Payloads that compress little, so the compressed body is long and the + // decoder's first chunks are sent before the stream ends. + let blob = 256; + let files = [ + songs_from(0, 2), + padded_songs(2, 2500, blob, blob_pattern(0)), ]; - for (what, body, expect) in cases { - let fake = Fake::with_source(source(Some(definition("PROVISIONED")))); - let msg = validation_message(restore(&fake, restore_body(body)).await.unwrap_err()); - assert!(msg.contains(expect), "{what}: {msg}"); - assert!(fake.restores().is_empty(), "{what}: reached storage"); - } - - // A legacy backup restores no index, so any listed one is not in it; an - // empty list changes nothing. - let fake = Fake::with_source(source(None)); - let msg = validation_message( - restore( - &fake, - restore_body(json!({ "GlobalSecondaryIndexOverride": [gsi_entry(Some((1, 1)))] })), - ) + seed_manifest_backup( + &fake, + &mem, + &arn(ACCOUNT), + music_source_table(), + &files, + false, + ); + let prefix = BackupArn::parse(&arn(ACCOUNT)).unwrap().store_prefix(); + let key = format!("{prefix}data/000002.json.gz"); + let original = mem.objects.lock().unwrap()[&key].clone(); + let replacement = same_length_replacement(&original, 2, 2500, blob); + assert_eq!(replacement.len(), original.len()); + let runtime = job_runtime(&mem).with_job_settings(JobSettings { + heartbeat_interval: Duration::from_secs(1), + ..job_settings() + }); + let gate = closed_gate(); + *fake.bulk_load_gate.lock().unwrap() = Some(Arc::clone(&gate)); + restore_on(&fake, &runtime, &restore_body(json!({}))) .await - .unwrap_err(), + .unwrap(); + // Both files hashed (pass 1) and file 1's second get made; file 2 not + // yet re-read. + wait_for_bulk_loads_started(&fake, 1).await; + let reads = gets_under(&mem, &arn(ACCOUNT)); + assert_eq!( + reads + .iter() + .filter(|k| k.ends_with("000002.json.gz")) + .count(), + 1, + "{reads:?}" + ); + mem.insert(&key, &replacement); + gate.add_permits(10); + wait_for_jobs(&runtime).await; + let writes = restore_writes(&fake); + assert_eq!( + writes.len(), + 1, + "the second pass hashes the file again and fails the restore: {writes:?}" ); + let reason = writes[0].2.clone().unwrap(); assert!( - msg.ends_with("global secondary index gsi1 is not in the backup"), - "{msg}" + reason.starts_with("data file data/000002.json.gz: data file sha256 checksum mismatch"), + "the second pass hashes the file again: {reason}" ); - let plain = restored_input(source(None), json!({})).await.unwrap(); - let empty = restored_input( - source(None), - json!({ "GlobalSecondaryIndexOverride": [], "LocalSecondaryIndexOverride": [] }), + let loads = fake.bulk_loads.lock().unwrap().clone(); + assert_eq!(loads.len(), 2); + assert_eq!( + loads[1].items.len(), + 2000, + "two chunks of the replacement reached the backend before the end-of-stream check" + ); + assert!(loads[1].stream_error.is_some(), "{loads:?}"); + assert!( + fake.restore_job("MusicRestored").is_none(), + "target removed" + ); + assert_eq!(fake.removed_targets.lock().unwrap().len(), 1); +} + +/// A file over one chunk by line bytes (the job's bound set small for the +/// test; the items are well under the count bound) is fed in chunks that +/// close at the byte bound, and the shutdown is read at each chunk boundary. +/// The backend is held after it pulled the first item of the second chunk, +/// so the decoder has sent two chunks and holds at most two more in the +/// channel and one past its check; the shutdown fired then is seen at the +/// close of a later chunk, so the backend receives a whole number of chunks +/// (two to five of six) and then the shutdown's `Err` item, never the whole +/// file. +#[tokio::test] +async fn a_file_over_one_chunk_by_bytes_is_fed_in_chunks_and_stops_at_a_boundary() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + // The job's chunk bound is 32 KiB (the production bound, 32 MiB, would + // make this a 192 MiB file); items of 1,500-byte blobs whose lines are + // the same length, so a chunk is a fixed number of items. + let chunk_bytes: u64 = 32 * 1024; + restore::test_support::set_chunk_bytes("tid-ChunkedByBytes", chunk_bytes); + let blob = 1500; + let line = line_bytes(&padded_songs(0, 1, blob, |_| b'a')[0]); + let per_chunk = usize::try_from(chunk_bytes.div_ceil(line)).unwrap(); + assert!( + (10..=20).contains(&per_chunk), + "{per_chunk} items of {line} bytes per chunk" + ); + let total = 6 * per_chunk; + let items = padded_songs(0, total, blob, |_| b'a'); + seed_manifest_backup( + &fake, + &mem, + &arn(ACCOUNT), + music_source_table(), + &[items], + true, + ); + let runtime = job_runtime(&mem); + let item_gate = Arc::new(tokio::sync::Semaphore::new(per_chunk + 1)); + *fake.bulk_load_item_gate.lock().unwrap() = Some(Arc::clone(&item_gate)); + restore_on( + &fake, + &runtime, + &restore_body(json!({ "TargetTableName": "ChunkedByBytes" })), ) .await .unwrap(); - assert_eq!(empty, plain); + wait_for_items_seen(&fake, per_chunk + 1).await; + let left = runtime.drain_jobs(Duration::from_millis(100)).await; + assert_eq!(left, 1, "the job waits at the gate past the bound"); + item_gate.add_permits(total + 10); + wait_for_jobs(&runtime).await; + let loads = fake.bulk_loads.lock().unwrap().clone(); + assert_eq!(loads.len(), 1); + let received = loads[0].items.len(); + assert!( + received % per_chunk == 0 && received >= 2 * per_chunk && received < total, + "{received} items received of {total}, {per_chunk} per chunk" + ); + assert_eq!( + loads[0].stream_error.as_deref(), + Some("Internal error: the restore was stopped by the server's shutdown"), + "{:?}", + loads[0].stream_error + ); + assert!(restore_writes(&fake).is_empty()); + assert_eq!( + fake.restore_job("ChunkedByBytes").unwrap().status, + JobStatus::Running + ); + assert_eq!(restore_outcomes(&runtime), [("cancelled".to_owned(), 1.0)]); } -/// Measured on DynamoDB: the two requests the service refuses and ExtendDB -/// accepted: a GSI entry without throughput on a PROVISIONED restore, and -/// `BillingModeOverride` `PROVISIONED` with no GSI list on a backup with -/// GSIs, both with the service's text. The service's text conditions on the -/// override being present, so the rule applies whatever the backup's billing -/// mode and whatever throughput it recorded for the index (the PROVISIONED -/// backup and stale-throughput cases were not measured). The -/// on-demand-to-PROVISIONED route works when the list supplies each index's -/// throughput. -#[tokio::test] -async fn a_provisioned_restore_needs_each_gsis_throughput_overridden() { - let expected = "Invalid Request: Must override ProvisionedThroughput if BillingMode is \ - overridden to PROVISIONED for index gsi1"; - let provisioned = json!({ - "BillingModeOverride": "PROVISIONED", - "ProvisionedThroughputOverride": { "ReadCapacityUnits": 5, "WriteCapacityUnits": 5 }, - }); - let with = |extra: Value| { - let mut body = provisioned.clone(); - for (k, v) in extra.as_object().unwrap() { - body[k] = v.clone(); - } - body - }; - // An on-demand backup whose GSI still carries a throughput from before a - // billing switch (what UpdateTable to PAY_PER_REQUEST leaves behind). - let mut stale = definition("PAY_PER_REQUEST"); - stale.global_secondary_indexes[0].provisioned_throughput = Some(throughput(3, 4)); - let refused: [(&str, BackupTableDefinition, Value); 6] = [ - ( - "no GSI list, on-demand backup", - definition("PAY_PER_REQUEST"), - provisioned.clone(), - ), - ( - "no GSI list, PROVISIONED backup (the override, not the recorded throughput, decides)", - definition("PROVISIONED"), - provisioned.clone(), - ), - ( - "no GSI list, on-demand backup with a stale recorded throughput", - stale, - provisioned.clone(), - ), - ( - "a GSI entry without throughput, on-demand backup", - definition("PAY_PER_REQUEST"), - with(json!({ "GlobalSecondaryIndexOverride": [gsi_entry(None)] })), - ), - ( - "a GSI entry without throughput, PROVISIONED backup with the override", - definition("PROVISIONED"), - with(json!({ "GlobalSecondaryIndexOverride": [gsi_entry(None)] })), - ), - ( - "a GSI entry without throughput, PROVISIONED backup without the override", - definition("PROVISIONED"), - json!({ "GlobalSecondaryIndexOverride": [gsi_entry(None)] }), - ), - ]; - for (what, def, body) in refused { - let fake = Fake::with_source(source(Some(def))); - let msg = validation_message(restore(&fake, restore_body(body)).await.unwrap_err()); - assert_eq!(msg, expected, "{what}"); - assert!(fake.restores().is_empty(), "{what}: reached storage"); +/// Wait until `bulk_load` has pulled `n` items from its stream. +async fn wait_for_items_seen(fake: &Fake, n: usize) { + // 120 s: the deadline covers the decoder's hashing pass over the file and + // the first chunk's decoding on the test's one runtime thread, which under + // a loaded machine (the suite runs beside the other crates' tests in CI) + // outlasted the 30 s this had, failing the two cells below while the code + // was right. + let deadline = tokio::time::Instant::now() + Duration::from_secs(120); + while fake + .bulk_load_items_seen + .load(std::sync::atomic::Ordering::Acquire) + < n + { + assert!( + tokio::time::Instant::now() < deadline, + "the backend pulled no item" + ); + tokio::time::sleep(Duration::from_millis(5)).await; } - // The first GSI of two is the one named. - let fake = Fake::with_source(source(Some(two_gsis("PAY_PER_REQUEST")))); - let msg = validation_message( - restore(&fake, restore_body(provisioned.clone())) - .await - .unwrap_err(), - ); - assert_eq!(msg, expected); +} - // The route that works: the list supplies the throughput. - let input = restored_input( - source(Some(two_gsis("PAY_PER_REQUEST"))), - with(json!({ "GlobalSecondaryIndexOverride": [ - gsi_entry(Some((7, 8))), - gsi2_entry(json!(["a", "b"]), Some((9, 10))), - ] })), +/// The decoder's claim check at each chunk boundary: the backend is held +/// after it pulled the first item of the second chunk (the decoder has sent +/// two chunks of a thousand and holds at most two more and one past its +/// check), the heartbeat timer is ended without raising the loss, and the +/// decoder ends the stream at the close of a later chunk with the +/// claim-lost reason: a whole number of chunks (two to five of six) reach +/// the backend, then the `Err` item, and the job stops with no catalog +/// write. +#[tokio::test] +async fn a_feeder_whose_timer_stopped_ends_the_stream_with_the_claim_lost_reason() { + let (lines, _guard) = LogLines::capture(); + let period = job_settings().heartbeat_interval; + let per_chunk = restore::load::LOAD_CHUNK_ITEMS; + let total = 6 * per_chunk; + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + seed_manifest_backup( + &fake, + &mem, + &arn(ACCOUNT), + music_source_table(), + &[songs_from(0, total)], + true, + ); + let runtime = job_runtime(&mem); + let item_gate = Arc::new(tokio::sync::Semaphore::new(per_chunk + 1)); + *fake.bulk_load_item_gate.lock().unwrap() = Some(Arc::clone(&item_gate)); + restore_on( + &fake, + &runtime, + &restore_body(json!({ "TargetTableName": "FeederTimerStopped" })), ) .await .unwrap(); - assert_eq!(input.billing_mode, Some(BillingMode::Provisioned)); - assert_eq!(input.provisioned_throughput, Some(throughput(5, 5))); - assert_eq!(gsi_throughputs(&input), [Some((7, 8)), Some((9, 10))]); - extenddb_core::validation::validate_create_table(&input, &LimitsConfig::default()).unwrap(); + wait_for_items_seen(&fake, per_chunk + 1).await; + restore::test_support::stop_timer("tid-FeederTimerStopped"); + tokio::time::sleep(3 * period).await; + item_gate.add_permits(total + 10); + wait_for_jobs(&runtime).await; + let loads = fake.bulk_loads.lock().unwrap().clone(); + assert_eq!(loads.len(), 1); + let received = loads[0].items.len(); + assert!( + received % per_chunk == 0 && received >= 2 * per_chunk && received < total, + "{received} items received of {total}" + ); + assert_eq!( + loads[0].stream_error.as_deref(), + Some("Internal error: the claim was lost while the file was being decoded"), + "{:?}", + loads[0].stream_error + ); + assert!(restore_writes(&fake).is_empty()); + assert!(fake.removed_targets.lock().unwrap().is_empty()); + let stopped = lines.containing( + "stopped with no further catalog or table access: the claim was lost while the file \ + was being decoded", + ); + assert_eq!(stopped.len(), 1, "{stopped:?}"); + assert_eq!( + fake.restore_job("FeederTimerStopped").unwrap().status, + JobStatus::Running + ); +} - // A PROVISIONED backup restored with no override keeps its recorded GSI - // throughput; only the override asks for the list. - let input = restored_input(source(Some(definition("PROVISIONED"))), json!({})) +/// The manifest arm reads the health this cycle's check left: a store that +/// passed a check once (its identity known) and is down when the cycle +/// runs is unhealthy to the arm, which skips the record with the line +/// naming the state; a store whose check cannot read the catalog and so has +/// not yet been checked is skipped with "not yet checked"; once the store +/// answers again the next cycle resumes the record. +#[tokio::test] +async fn an_unhealthy_or_unchecked_store_with_a_known_identity_skips_the_resume() { + let (lines, _guard) = LogLines::capture(); + let mem = Arc::new(MemStore::default()); + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + let fake = Arc::new(Fake::default()); + seed_manifest_backup( + &fake, + &mem, + &arn(ACCOUNT), + music_source_table(), + &[songs(2)], + true, + ); + let runtime = job_runtime(&mem); + restore_on(&fake, &runtime, &restore_body(json!({}))) .await .unwrap(); - assert_eq!(gsi_throughputs(&input), [Some((3, 4))]); - // And with no GSIs, the billing override to PROVISIONED needs nothing. - let mut none = definition("PAY_PER_REQUEST"); - none.global_secondary_indexes.clear(); - let mut src = source(Some(none)); - src.attribute_definitions - .retain(|a| a.attribute_name != "gpk"); - let input = restored_input(src, provisioned).await.unwrap(); - assert!(input.global_secondary_indexes.is_none()); - assert_eq!(input.provisioned_throughput, Some(throughput(5, 5))); -} + wait_for_jobs(&runtime).await; + dead_record_without_progress(&fake, false); + fake.bulk_loads.lock().unwrap().clear(); -fn on_demand(read: Option, write: Option) -> OnDemandThroughput { - OnDemandThroughput { - max_read_request_units: read, - max_write_request_units: write, - } + // Healthy with its identity known, then down when the cycle runs. + let runtime = job_runtime(&mem); + assert_eq!( + runtime.configured_store().unwrap().identity(), + Some(store()) + ); + mem.set_down(true); + let report = cycle_on(&fake, &runtime).await; + assert_eq!(report.store_healthy, Some(false)); + assert_eq!(report.restores_skipped_store_unhealthy, ["MusicRestored"]); + assert!(report.manifest_restores_resumed.is_empty()); + assert!(fake.restore_takeovers.lock().unwrap().is_empty()); + assert_eq!( + lines.containing("dead store-backed restores are not resumed this cycle"), + [ + "the configured backup store is unhealthy; dead store-backed restores are not \ + resumed this cycle" + ] + ); + mem.set_down(false); + + // Not yet checked: the store's first check cannot read the catalog. + let checking = self::runtime() + .with_store(ConfiguredStore::checking( + StoreKind::Filesystem, + store().store_root, + Arc::clone(&mem) as Arc, + None, + )) + .with_job_settings(job_settings()) + .with_metrics(Arc::new(extenddb_core::metrics::MetricsCollector::new())); + *fake.catalog_id_error.lock().unwrap() = true; + let report = cycle_on(&fake, &checking).await; + assert_eq!(report.store_healthy, Some(false)); + assert_eq!(report.restores_skipped_store_unhealthy, ["MusicRestored"]); + assert!(fake.restore_takeovers.lock().unwrap().is_empty()); + assert_eq!( + lines + .containing("the configured backup store is not yet checked; dead store-backed restores are not resumed this cycle") + .len(), + 1 + ); + *fake.catalog_id_error.lock().unwrap() = false; + + // The store answers again: resumed. + let report = cycle_on(&fake, &runtime).await; + assert_eq!(report.store_healthy, Some(true)); + assert_eq!(report.manifest_restores_resumed, ["MusicRestored"]); + wait_for_jobs(&runtime).await; + assert_eq!( + fake.restore_job("MusicRestored").unwrap().status, + JobStatus::Done + ); } -/// Measured on DynamoDB: the service applies `OnDemandThroughputOverride` on an -/// on-demand restore, changed or with one member, including on a PROVISIONED -/// backup restored as on-demand; a GSI entry's `OnDemandThroughput` is -/// accepted (and, ExtendDB's choice, not recorded). On a PROVISIONED restore -/// the member contradicts the billing mode and is refused. +// --- the sweep's read order, the DONE write's unknown outcome, the resume's +// instant and documents, the request's bounds ----------------------------------- + +/// A restore that registers (RUNNING, this host's `spool_host`) and opens +/// its spool entry after the reconciler read the RUNNING records: the spool +/// was listed before the records, so the entry is not a candidate and the +/// live job keeps it (the design: the sweep never removes the spool of the +/// job that owns the record). #[tokio::test] -async fn an_on_demand_throughput_override_is_applied_on_an_on_demand_restore() { - let with = |recorded: Option| { - let mut def = definition("PAY_PER_REQUEST"); - def.on_demand_throughput = recorded; - source(Some(def)) - }; - let applied: [( - Option, - Value, - Option, - ); 6] = [ - ( - Some(on_demand(Some(50), Some(60))), - json!({ "MaxReadRequestUnits": 50, "MaxWriteRequestUnits": 60 }), - Some(on_demand(Some(50), Some(60))), - ), - ( - Some(on_demand(Some(50), Some(60))), - json!({ "MaxReadRequestUnits": 50, "MaxWriteRequestUnits": 61 }), - Some(on_demand(Some(50), Some(61))), - ), - ( - None, - json!({ "MaxReadRequestUnits": 10 }), - Some(on_demand(Some(10), None)), - ), - ( - Some(on_demand(Some(50), Some(60))), - json!({ "MaxWriteRequestUnits": -1 }), - Some(on_demand(None, Some(-1))), - ), - ( - None, - json!({ "MaxReadRequestUnits": -1, "MaxWriteRequestUnits": -1 }), - Some(on_demand(Some(-1), Some(-1))), - ), - // An empty override records no maxima, as CreateTable without the - // member does. - (Some(on_demand(Some(50), Some(60))), json!({}), None), - ]; - for (recorded, given, expected) in applied { - let input = restored_input( - with(recorded.clone()), - json!({ "OnDemandThroughputOverride": given }), +async fn a_restore_registered_between_the_two_reads_keeps_its_spool_entry() { + let scratch = restore::spool::tests::Scratch::new(); + let mem = Arc::new(MemStore::default()); + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + let fake = Arc::new(Fake::default()); + let runtime = spooled_runtime(&mem, scratch.path()); + // An entry of a dead restore, so the sweep has work to do in the same + // cycle and the live entry's survival is not the sweep doing nothing. + std::fs::create_dir(scratch.path().join("restore-dead")).unwrap(); + let spool_dir = scratch.path().to_path_buf(); + *fake.sweep_hook.lock().unwrap() = Some(Box::new(move |fake: &Fake| { + fake.restore_jobs.lock().unwrap().insert( + "late".to_owned(), + RestoreJobRow { + target: RestoreTargetKey { + account_id: ACCOUNT.to_owned(), + table_name: "late".to_owned(), + table_id: "late".to_owned(), + }, + source_backup_arn: arn(ACCOUNT), + kind: RestoreKind::Manifest, + status: JobStatus::Running, + owner_generation: 1, + heartbeat_at: seconds_since_epoch(), + store: Some(s3_identity()), + spool_host: Some("host-a:/var/spool".to_owned()), + file_index: None, + items_applied: None, + failure_reason: None, + completed_at: None, + }, + ); + // The job's `Entry::open` and its first download. + let entry = restore::spool::Entry::open( + &spool_dir, + "late", + Some("host-a:/var/spool"), + "host-a:/var/spool", ) - .await - .unwrap_or_else(|e| panic!("{recorded:?} {given}: {e:?}")); - assert_eq!(input.on_demand_throughput, expected, "{recorded:?} {given}"); - assert_eq!(input.billing_mode, Some(BillingMode::PayPerRequest)); - } - // Without the member the backup's is restored. - let input = restored_input(with(Some(on_demand(Some(50), Some(60)))), json!({})) - .await .unwrap(); + std::fs::write(entry.dir().join("000000.json.gz.partial"), b"bytes").unwrap(); + })); + let report = cycle_on(&fake, &runtime).await; + assert!( + fake.sweep_hook.lock().unwrap().is_none(), + "the registration landed between the two reads" + ); assert_eq!( - input.on_demand_throughput, - Some(on_demand(Some(50), Some(60))) + fake.restore_job("late").map(|r| r.status), + Some(JobStatus::Running), + "the record is RUNNING with this host's spool_host" + ); + assert_eq!( + report.spool_entries_swept, + ["dead"], + "the dead entry went; the live restore's entry did not" + ); + assert!( + scratch.path().join("restore-late").is_dir(), + "the live job's spool entry was removed from under it" + ); + assert!( + scratch + .path() + .join("restore-late/000000.json.gz.partial") + .is_file() ); + // The next cycle, with the record RUNNING and this host's, keeps it too. + let report = cycle_on(&fake, &runtime).await; + assert!(report.spool_entries_swept.is_empty()); + assert!(scratch.path().join("restore-late").is_dir()); +} - // A PROVISIONED backup restored as on-demand takes it. - let input = restored_input( - source(Some(definition("PROVISIONED"))), - json!({ - "BillingModeOverride": "PAY_PER_REQUEST", - "OnDemandThroughputOverride": { "MaxReadRequestUnits": 7, "MaxWriteRequestUnits": 8 }, - }), - ) - .await - .unwrap(); - assert_eq!(input.billing_mode, Some(BillingMode::PayPerRequest)); - assert!(input.provisioned_throughput.is_none()); +/// A `finish_restore` that returns an error is a write whose outcome is +/// unknown, and the record decides: (a) the flip landed and the call then +/// failed (PostgreSQL's progress delete after the commit): the re-read finds +/// DONE under this generation, the restore is complete and `succeeded`, with +/// a WARN naming the error, no FAILED write, no table access and the entry +/// removed; (b) the call failed before anything was written: the re-read +/// finds RUNNING, the write is sent once more and lands; (c) it fails twice: +/// the record stays RUNNING for the reconciler, no FAILED write, no +/// removal, the job `failed` with the worker error and the entry left; (d) +/// it fails and a successor wrote DONE under its own generation before the +/// re-read: the write matched no record, the job `failed`, no table access. +#[tokio::test] +async fn a_failed_done_write_is_re_read_and_the_record_decides() { + async fn run_with( + errors: &[FinishError], + ) -> (Arc, Vec<(MetricName, String, f64)>, LogLines) { + let (lines, _guard) = LogLines::capture(); + let scratch = restore::spool::tests::Scratch::new(); + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + seed_manifest_backup( + &fake, + &mem, + &arn(ACCOUNT), + music_source_table(), + &[songs(2)], + true, + ); + fake.rows + .lock() + .unwrap() + .get_mut(&arn(ACCOUNT)) + .unwrap() + .store = s3_identity(); + fake.finish_errors + .lock() + .unwrap() + .extend(errors.iter().copied()); + let runtime = spooled_runtime(&mem, scratch.path()); + restore_on(&fake, &runtime, &restore_body(json!({}))) + .await + .unwrap(); + wait_for_jobs(&runtime).await; + let entry_left = scratch.path().join("restore-tid-MusicRestored").exists(); + let mut metrics = drained(&runtime); + metrics.push(( + MetricName::RestoreJobs, + format!("entry_left={entry_left}"), + 0.0, + )); + (fake, metrics, lines) + } + fn restore_jobs(metrics: &[(MetricName, String, f64)]) -> Vec<(String, f64)> { + metrics + .iter() + .filter(|(m, op, _)| *m == MetricName::RestoreJobs && !op.starts_with("entry_left")) + .map(|(_, op, sum)| (op.clone(), *sum)) + .collect() + } + fn entry_left(metrics: &[(MetricName, String, f64)]) -> bool { + metrics.iter().any(|(_, op, _)| op == "entry_left=true") + } + + // (a) The flip landed, then the error. + let (fake, metrics, lines) = run_with(&[FinishError::AfterFlip]).await; + assert_eq!(fake.finish_sizes.lock().unwrap().len(), 1, "one DONE write"); + let record = fake.restore_job("MusicRestored").unwrap(); + assert_eq!(record.status, JobStatus::Done); + assert!(fake.restore_tables.lock().unwrap()["tid-MusicRestored"].active); + assert!( + restore_writes(&fake).is_empty(), + "no FAILED write over a DONE record: {:?}", + restore_writes(&fake) + ); + assert!(fake.removed_targets.lock().unwrap().is_empty()); + assert_eq!(restore_jobs(&metrics), [("succeeded".to_owned(), 1.0)]); + assert!( + metrics + .iter() + .any(|(m, op, _)| *m == MetricName::WorkerLastSuccess && op == "RestoreJob") + ); + assert!(!entry_left(&metrics), "the entry is removed on a success"); + let warned = lines.containing( + "the DONE write returned an error (Internal error: the data pool failed after the flip's \ + commit) and the record is DONE under this job's generation: the write landed and the \ + restore is complete", + ); + assert_eq!(warned.len(), 1, "{warned:?}"); assert_eq!( - input.on_demand_throughput, - Some(on_demand(Some(7), Some(8))) + lines + .containing(": complete (DONE), 2 items (2 applied by this run)") + .len(), + 1, + "the line says DONE, not ACTIVE: the table's flip is the backend's" ); + assert!(lines.containing(": ACTIVE, ").is_empty()); - // A GSI entry's OnDemandThroughput on an on-demand restore: accepted, the - // index restored without a maximum (ExtendDB records none per index). - let plain = restored_input(source(Some(definition("PAY_PER_REQUEST"))), json!({})) - .await - .unwrap(); - let input = restored_input( - source(Some(definition("PAY_PER_REQUEST"))), - json!({ "GlobalSecondaryIndexOverride": [{ - "IndexName": "gsi1", - "KeySchema": [{ "AttributeName": "gpk", "KeyType": "HASH" }], - "Projection": { "ProjectionType": "ALL" }, - "OnDemandThroughput": { "MaxReadRequestUnits": 10, "MaxWriteRequestUnits": -1 }, - }] }), - ) - .await - .unwrap(); - assert_eq!(input, plain); + // (b) The error before anything was written: once more, and it lands. + let (fake, metrics, lines) = run_with(&[FinishError::BeforeFlip]).await; + assert_eq!( + fake.finish_sizes.lock().unwrap().len(), + 2, + "the write sent once more" + ); + assert_eq!( + fake.restore_job("MusicRestored").unwrap().status, + JobStatus::Done + ); + assert!(restore_writes(&fake).is_empty()); + assert_eq!(restore_jobs(&metrics), [("succeeded".to_owned(), 1.0)]); + assert_eq!( + lines + .containing("the record is still RUNNING under this claim; writing once more") + .len(), + 1 + ); - // Refused: an unknown member, and the member on a PROVISIONED restore - // (from a PROVISIONED backup or by override). - let fake = Fake::with_source(with(None)); - let msg = validation_message( - restore( - &fake, - restore_body(json!({ "OnDemandThroughputOverride": { - "MaxReadRequestUnits": 50, "MaxWriteRequestUnits": 60, "Other": 1, - } })), - ) - .await - .unwrap_err(), + // (d) The error, and a takeover whose successor wrote DONE before the + // re-read: the record is DONE under another generation, so the write + // matched no record; no second write, no table access, the entry left. + let (fake, metrics, lines) = run_with(&[FinishError::TakenAndDone]).await; + assert_eq!(fake.finish_sizes.lock().unwrap().len(), 1, "one DONE write"); + assert_eq!( + fake.restore_job("MusicRestored").unwrap().status, + JobStatus::Done ); + assert!(restore_writes(&fake).is_empty()); + assert!(fake.removed_targets.lock().unwrap().is_empty()); + assert_eq!(restore_jobs(&metrics), [("failed".to_owned(), 1.0)]); + assert!(entry_left(&metrics), "the entry is left to the sweep"); + let ended = lines.containing( + "the DONE write matched no record (the DONE write failed (Internal error: the claim \ + connection failed before the DONE write) and the record carries generation", + ); + assert_eq!(ended.len(), 1, "{ended:?}"); assert!( - msg.ends_with("OnDemandThroughputOverride: member Other is not supported"), - "{msg}" + lines.containing("complete (DONE)").is_empty(), + "a successor's DONE is not this job's" ); - assert!(fake.restores().is_empty()); - for (def, body) in [ - ( - definition("PROVISIONED"), - json!({ "OnDemandThroughputOverride": { "MaxReadRequestUnits": 1 } }), - ), - ( - definition("PAY_PER_REQUEST"), - json!({ - "OnDemandThroughputOverride": {}, - "BillingModeOverride": "PROVISIONED", - "ProvisionedThroughputOverride": { "ReadCapacityUnits": 1, "WriteCapacityUnits": 1 }, - }), - ), - ] { - let mut def = def; - def.global_secondary_indexes.clear(); - let fake = Fake::with_source(source(Some(def))); - let msg = validation_message(restore(&fake, restore_body(body)).await.unwrap_err()); - assert!( - msg.ends_with( - "OnDemandThroughputOverride can only be specified when the restored table's \ - BillingMode is PAY_PER_REQUEST" - ), - "{msg}" + + // (c) Twice: no third write, no FAILED write, no removal; the record is + // the reconciler's. + let (fake, metrics, lines) = + run_with(&[FinishError::BeforeFlip, FinishError::BeforeFlip]).await; + assert_eq!( + fake.finish_sizes.lock().unwrap().len(), + 2, + "two writes at most" + ); + let record = fake.restore_job("MusicRestored").unwrap(); + assert_eq!(record.status, JobStatus::Running); + assert!( + restore_writes(&fake).is_empty(), + "no FAILED write: {:?}", + restore_writes(&fake) + ); + assert!(fake.removed_targets.lock().unwrap().is_empty()); + assert!( + fake.tables.lock().unwrap().contains_key("MusicRestored"), + "the target stands" + ); + assert_eq!(restore_jobs(&metrics), [("failed".to_owned(), 1.0)]); + assert!( + metrics + .iter() + .any(|(m, op, _)| *m == MetricName::WorkerErrorCount && op == "RestoreJob") + ); + assert!(entry_left(&metrics), "the entry is left to the sweep"); + let ended = lines.containing( + "the DONE write matched no record (the DONE write failed twice (Internal error: the claim \ + connection failed before the DONE write) and the record is still RUNNING under this \ + claim; the reconciler resumes it once the record is stale); the record is not this \ + job's to write and no table access was made", + ); + assert_eq!(ended.len(), 1, "{ended:?}"); +} + +/// A resumed job's first confirmed beat is the instant the takeover +/// returned, not the instant the job started: with the record read held +/// three periods after the takeover, the job's first check finds the beat +/// older than two periods and stops as a lost claim with no load and no +/// status write (the record is one a second taker may hold by then); with +/// the read held one period the job runs to DONE. +#[tokio::test(start_paused = true)] +async fn a_resumes_first_beat_is_the_takeovers_return() { + async fn resume_held(periods: u32) -> (Arc, Arc, LogLines) { + let (lines, _guard) = LogLines::capture(); + let mem = Arc::new(MemStore::default()); + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + let fake = Arc::new(Fake::default()); + seed_manifest_backup( + &fake, + &mem, + &arn(ACCOUNT), + music_source_table(), + &[songs_from(0, 2), songs_from(2, 2)], + true, ); - assert!(fake.restores().is_empty()); + let runtime = job_runtime(&mem); + restore_on(&fake, &runtime, &restore_body(json!({}))) + .await + .unwrap(); + wait_for_jobs(&runtime).await; + // The record as a dead owner left it after file 0. + { + let mut records = fake.restore_jobs.lock().unwrap(); + let record = records.get_mut("tid-MusicRestored").unwrap(); + record.status = JobStatus::Running; + record.completed_at = None; + record.file_index = Some(0); + record.items_applied = Some(2); + record.heartbeat_at = 1234.5; + let mut targets = fake.restore_tables.lock().unwrap(); + let target = targets.get_mut("tid-MusicRestored").unwrap(); + target.active = false; + target.items.truncate(2); + if let Some(desc) = fake.tables.lock().unwrap().get_mut("MusicRestored") { + desc.table_status = TableStatus::Creating; + } + } + fake.restore_dead + .lock() + .unwrap() + .insert("tid-MusicRestored".to_owned()); + fake.bulk_loads.lock().unwrap().clear(); + mem.gets.lock().unwrap().clear(); + *fake.read_restore_job_delay.lock().unwrap() = + Some(job_settings().heartbeat_interval * periods); + let runtime = job_runtime(&mem); + let report = cycle_on(&fake, &runtime).await; + assert_eq!(report.manifest_restores_resumed, ["MusicRestored"]); + wait_for_jobs(&runtime).await; + (fake, mem, lines) } -} -/// Measured on DynamoDB: the service restores `STANDARD` whatever the source's -/// table class; the recorded class is for DescribeBackup only. -#[tokio::test] -async fn table_class_is_not_restored() { - let mut def = definition("PROVISIONED"); - def.table_class = Some("STANDARD_INFREQUENT_ACCESS".to_owned()); - let input = restored_input(source(Some(def)), json!({})).await.unwrap(); - assert_eq!(input.table_class, None); + let (fake, mem, lines) = resume_held(3).await; + assert!( + fake.bulk_loads.lock().unwrap().is_empty(), + "no load under a beat three periods old: {:?}", + fake.bulk_load_files() + ); + assert_eq!( + fake.restore_job("MusicRestored").unwrap().status, + JobStatus::Running, + "no status write" + ); + assert!(restore_writes(&fake).is_empty()); + let stopped = lines.containing( + "stopped with no further catalog or table access: no heartbeat confirmed the claim \ + recently enough to download data/000002.json.gz", + ); + assert_eq!(stopped.len(), 1, "{stopped:?}"); + assert_eq!( + gets_under(&mem, &arn(ACCOUNT)), + [EXTENDDB_MANIFEST_FILE, MANIFEST_CHECKSUM_FILE], + "the two documents of the resume's preparation and no data file" + ); + + let (fake, _mem, _lines) = resume_held(1).await; + assert_eq!(fake.bulk_load_files(), [(1, 0, true, 2)]); + assert_eq!( + fake.restore_job("MusicRestored").unwrap().status, + JobStatus::Done + ); } +/// A same-host resume whose spooled manifest a crash left short reads the +/// store's manifest and sidecar instead (one WARN), writes them to the entry +/// again, and resumes with the spooled files; the restore completes. #[tokio::test] -async fn sse_override_replaces_the_recorded_specification() { - let mut def = definition("PROVISIONED"); - def.sse_specification = Some(json!({ "Enabled": false })); - let fake = Fake::with_source(source(Some(def))); - let sse = json!({ "Enabled": true, "SSEType": "KMS", "KMSMasterKeyId": "alias/k" }); - restore( +async fn a_truncated_spooled_manifest_has_the_resume_read_the_store() { + let (lines, _guard) = LogLines::capture(); + let scratch = restore::spool::tests::Scratch::new(); + let mem = Arc::new(MemStore::default()); + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + let fake = Arc::new(Fake::default()); + let files = [songs_from(0, 2), songs_from(2, 2)]; + seed_manifest_backup( &fake, - restore_body(json!({ "SSESpecificationOverride": sse })), + &mem, + &arn(ACCOUNT), + music_source_table(), + &files, + true, + ); + fake.rows + .lock() + .unwrap() + .get_mut(&arn(ACCOUNT)) + .unwrap() + .store = s3_identity(); + let runtime = spooled_runtime(&mem, scratch.path()); + stop_a_spooled_restore_before_its_first_load(&fake, &runtime, &closed_gate()).await; + mem.gets.lock().unwrap().clear(); + let entry = scratch.path().join("restore-tid-MusicRestored"); + let spooled_manifest = std::fs::read(entry.join(EXTENDDB_MANIFEST_FILE)).unwrap(); + // A crash during the manifest's write, as an in-place write leaves it. + std::fs::write( + entry.join(EXTENDDB_MANIFEST_FILE), + &spooled_manifest[..spooled_manifest.len() / 2], ) - .await .unwrap(); - assert_eq!(fake.restores()[0].input.sse_specification, Some(sse)); - // A legacy backup takes the override too. - let fake = Fake::with_source(source(None)); - restore( - &fake, - restore_body(json!({ "SSESpecificationOverride": { "Enabled": true } })), - ) - .await - .unwrap(); + let runtime2 = spooled_runtime(&mem, scratch.path()); + let report = cycle_on(&fake, &runtime2).await; + assert_eq!(report.manifest_restores_resumed, ["MusicRestored"]); + wait_for_jobs(&runtime2).await; assert_eq!( - fake.restores()[0].input.sse_specification, - Some(json!({ "Enabled": true })) + fake.restore_job("MusicRestored").map(|r| r.status), + Some(JobStatus::Done), + "the resume read the store's documents and completed: {:?}", + restore_writes(&fake) + ); + assert_eq!( + gets_under(&mem, &arn(ACCOUNT)), + [EXTENDDB_MANIFEST_FILE, MANIFEST_CHECKSUM_FILE], + "the two documents from the store, no data file (the spooled files are verified)" + ); + assert_eq!(fake.bulk_load_files(), [(0, 0, true, 2), (1, 0, false, 2)]); + assert_eq!( + lines + .containing( + "the spooled manifest and sidecar of the restore into MusicRestored (account \ + 123456789012) do not verify (the manifest's checksum sidecar does not match" + ) + .len(), + 1 + ); + assert_eq!( + lines + .containing("resumed by this instance; the manifest was read from the store") + .len(), + 1 ); - - let body = restore_body(json!({ "SSESpecificationOverride": "on" })); - assert!(matches!( - parse_restore_table_overrides(&body).unwrap_err(), - DynamoDbError::SerializationException(_) - )); } +/// A resume whose store documents fail verification is FAILED with the +/// reason, its target removed, and its duration recorded from the resume's +/// start: a sidecar that does not match the manifest; a manifest naming +/// another backup under a sidecar that matches it. #[tokio::test] -async fn billing_overrides_shape_the_restore_input() { - // PROVISIONED -> PAY_PER_REQUEST drops table and GSI throughput. - let fake = Fake::with_source(source(Some(definition("PROVISIONED")))); - let out = restore( +async fn a_resume_whose_store_documents_fail_verification_is_failed() { + async fn resume_over(tamper: impl Fn(&MemStore, &str)) -> (Arc, BackupRuntime, LogLines) { + let (lines, _guard) = LogLines::capture(); + let mem = Arc::new(MemStore::default()); + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + let fake = Arc::new(Fake::default()); + seed_manifest_backup( + &fake, + &mem, + &arn(ACCOUNT), + music_source_table(), + &[songs_from(0, 2), songs_from(2, 2)], + true, + ); + let runtime = job_runtime(&mem); + restore_on(&fake, &runtime, &restore_body(json!({}))) + .await + .unwrap(); + wait_for_jobs(&runtime).await; + { + let mut records = fake.restore_jobs.lock().unwrap(); + let record = records.get_mut("tid-MusicRestored").unwrap(); + record.status = JobStatus::Running; + record.completed_at = None; + record.file_index = Some(0); + record.items_applied = Some(2); + record.heartbeat_at = 1234.5; + let mut targets = fake.restore_tables.lock().unwrap(); + let target = targets.get_mut("tid-MusicRestored").unwrap(); + target.active = false; + target.items.truncate(2); + } + fake.restore_dead + .lock() + .unwrap() + .insert("tid-MusicRestored".to_owned()); + fake.bulk_loads.lock().unwrap().clear(); + let prefix = BackupArn::parse(&arn(ACCOUNT)).unwrap().store_prefix(); + tamper(&mem, &prefix); + let runtime = job_runtime(&mem); + let report = cycle_on(&fake, &runtime).await; + assert_eq!(report.manifest_restores_resumed, ["MusicRestored"]); + wait_for_jobs(&runtime).await; + drop(_guard); + (fake, runtime, lines) + } + fn failed_with(fake: &Fake, runtime: &BackupRuntime, reason_prefix: &str) { + let writes = restore_writes(fake); + assert_eq!(writes.len(), 1, "{writes:?}"); + assert_eq!( + (writes[0].0, writes[0].1), + (JobStatus::Running, JobStatus::Failed) + ); + let reason = writes[0].2.clone().unwrap(); + assert!(reason.starts_with(reason_prefix), "{reason}"); + assert!(fake.bulk_loads.lock().unwrap().is_empty(), "no load"); + assert!( + fake.restore_job("MusicRestored").is_none(), + "the target and its record are removed" + ); + assert_eq!(fake.removed_targets.lock().unwrap().len(), 1); + let metrics = drained(runtime); + let durations: Vec = metrics + .iter() + .filter(|(m, op, _)| *m == MetricName::RestoreJobDuration && op == "failed") + .map(|(_, _, sum)| *sum) + .collect(); + assert_eq!(durations.len(), 1, "{metrics:?}"); + assert!( + durations[0] > 0.0, + "the duration runs from the resume's start: {metrics:?}" + ); + assert!( + metrics + .iter() + .any(|(m, op, _)| *m == MetricName::WorkerErrorCount && op == "RestoreJob") + ); + } + + // The sidecar names another digest. + let (fake, runtime, lines) = resume_over(|mem, prefix| { + mem.insert( + &format!("{prefix}{MANIFEST_CHECKSUM_FILE}"), + format!("{} {EXTENDDB_MANIFEST_FILE}\n", "0".repeat(64)).as_bytes(), + ); + }) + .await; + failed_with( &fake, - restore_body(json!({ "BillingModeOverride": "PAY_PER_REQUEST" })), - ) - .await - .unwrap(); - let input = &fake.restores()[0].input; - assert_eq!(input.billing_mode, Some(BillingMode::PayPerRequest)); - assert!(input.provisioned_throughput.is_none()); - assert!( - input.global_secondary_indexes.as_ref().unwrap()[0] - .provisioned_throughput - .is_none() + &runtime, + "the manifest's checksum sidecar does not match", ); - let td = &out["TableDescription"]; - assert_eq!(td["ProvisionedThroughput"]["ReadCapacityUnits"], 0); assert_eq!( - td["GlobalSecondaryIndexes"][0]["ProvisionedThroughput"]["ReadCapacityUnits"], - 0 + lines + .containing("FAILED on resume (the manifest's checksum sidecar does not match") + .len(), + 1 ); - // A throughput override on a PROVISIONED backup replaces the table's. - let fake = Fake::with_source(source(Some(definition("PROVISIONED")))); - restore( - &fake, - restore_body(json!({ - "ProvisionedThroughputOverride": { "ReadCapacityUnits": 11, "WriteCapacityUnits": 12 }, - })), - ) - .await - .unwrap(); - let input = &fake.restores()[0].input; - assert_eq!(input.provisioned_throughput, Some(throughput(11, 12))); + // The manifest names another backup of the same table and instant + // (another id; the parser's identity rule admits it), under a sidecar + // that matches it. + let (fake, runtime, lines) = resume_over(|mem, prefix| { + let other = BackupArn::parse(&arn(ACCOUNT).replace("73d8d5bc", "0badf00d")).unwrap(); + let bytes = + mem.objects.lock().unwrap()[&format!("{prefix}{EXTENDDB_MANIFEST_FILE}")].clone(); + let text = String::from_utf8(bytes).unwrap(); + let own_id = BackupArn::parse(&arn(ACCOUNT)).unwrap(); + let renamed = text.replace( + own_id.to_arn_string().as_str(), + other.to_arn_string().as_str(), + ); + assert_ne!(renamed, text); + let manifest = renamed.into_bytes(); + let sidecar = { + use sha2::Digest; + format!( + "{:x} {EXTENDDB_MANIFEST_FILE}\n", + sha2::Sha256::digest(&manifest) + ) + }; + mem.insert(&format!("{prefix}{EXTENDDB_MANIFEST_FILE}"), &manifest); + mem.insert( + &format!("{prefix}{MANIFEST_CHECKSUM_FILE}"), + sidecar.as_bytes(), + ); + }) + .await; + failed_with(&fake, &runtime, "the manifest names another backup, "); assert_eq!( - input.global_secondary_indexes.as_ref().unwrap()[0].provisioned_throughput, - Some(throughput(3, 4)) + lines + .containing("FAILED on resume (the manifest names another backup, ") + .len(), + 1 ); - - // A throughput override on an on-demand backup without a billing override. - let fake = Fake::with_source(source(Some(definition("PAY_PER_REQUEST")))); - let err = restore( - &fake, - restore_body(json!({ - "ProvisionedThroughputOverride": { "ReadCapacityUnits": 1, "WriteCapacityUnits": 1 }, - })), - ) - .await - .unwrap_err(); - assert!(validation_message(err).contains("can only be specified")); - assert!(fake.restores().is_empty()); } +/// A same-host resume that fails before it runs (the spooled pair a crash +/// left short, and the store's sidecar naming another digest) removes this +/// host's spool entry with the failure path, as the job removes its entry +/// at every failed end: the record is FAILED naming the mismatch, the +/// target removed, and the entry gone with its verified files. #[tokio::test] -async fn provisioned_override_on_an_on_demand_gsi_has_the_services_text() { - let fake = Fake::with_source(source(Some(definition("PAY_PER_REQUEST")))); - let err = restore( +async fn a_resume_that_fails_before_it_runs_removes_this_hosts_spool_entry() { + let (lines, _guard) = LogLines::capture(); + let scratch = restore::spool::tests::Scratch::new(); + let mem = Arc::new(MemStore::default()); + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + let fake = Arc::new(Fake::default()); + seed_manifest_backup( &fake, - restore_body(json!({ - "BillingModeOverride": "PROVISIONED", - "ProvisionedThroughputOverride": { "ReadCapacityUnits": 5, "WriteCapacityUnits": 5 }, - })), + &mem, + &arn(ACCOUNT), + music_source_table(), + &[songs_from(0, 2), songs_from(2, 2)], + true, + ); + fake.rows + .lock() + .unwrap() + .get_mut(&arn(ACCOUNT)) + .unwrap() + .store = s3_identity(); + let runtime = spooled_runtime(&mem, scratch.path()); + stop_a_spooled_restore_before_its_first_load(&fake, &runtime, &closed_gate()).await; + let entry = scratch.path().join("restore-tid-MusicRestored"); + assert!(entry.join("000000.json.gz.verified").is_file()); + let spooled_manifest = std::fs::read(entry.join(EXTENDDB_MANIFEST_FILE)).unwrap(); + std::fs::write( + entry.join(EXTENDDB_MANIFEST_FILE), + &spooled_manifest[..spooled_manifest.len() / 2], ) - .await - .unwrap_err(); + .unwrap(); + let prefix = BackupArn::parse(&arn(ACCOUNT)).unwrap().store_prefix(); + mem.insert( + &format!("{prefix}{MANIFEST_CHECKSUM_FILE}"), + format!("{} {EXTENDDB_MANIFEST_FILE}\n", "0".repeat(64)).as_bytes(), + ); + + let runtime2 = spooled_runtime(&mem, scratch.path()); + let report = cycle_on(&fake, &runtime2).await; + assert_eq!(report.manifest_restores_resumed, ["MusicRestored"]); + wait_for_jobs(&runtime2).await; + let writes = restore_writes(&fake); + assert_eq!(writes.len(), 1, "{writes:?}"); assert_eq!( - validation_message(err), - "Invalid Request: Must override ProvisionedThroughput if BillingMode is overridden to \ - PROVISIONED for index gsi1" + (writes[0].0, writes[0].1), + (JobStatus::Running, JobStatus::Failed) ); - assert!(fake.restores().is_empty()); + let reason = writes[0].2.clone().unwrap(); + assert!( + reason.starts_with("the manifest's checksum sidecar does not match"), + "{reason}" + ); + assert_eq!(fake.removed_targets.lock().unwrap().len(), 1); + assert!(fake.bulk_loads.lock().unwrap().is_empty()); + assert!( + !entry.exists(), + "this host's entry goes with the failed resume" + ); + let ended = + lines.containing("FAILED on resume (the manifest's checksum sidecar does not match"); + assert_eq!(ended.len(), 1, "{ended:?}"); + assert!(ended[0].ends_with("; the target was removed"), "{ended:?}"); } +/// A spool document write that fails (the `.partial` path is taken by a +/// directory, as a disk full fails the same arm) fails the restore naming +/// the document, removes the target, and removes the entry the job had +/// opened, as every failed end does. #[tokio::test] -async fn override_shape_is_refused_before_any_storage_read() { - let fake = Fake::default(); - let err = restore( +async fn a_spool_document_write_failure_fails_the_restore_and_removes_the_entry() { + let scratch = restore::spool::tests::Scratch::new(); + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + seed_manifest_backup( &fake, - restore_body(json!({ "BillingModeOverride": "PROVISIONED" })), - ) - .await - .unwrap_err(); - assert!(validation_message(err).contains("ProvisionedThroughputOverride must")); - assert_eq!(fake.source_reads(), 0); + &mem, + &arn(ACCOUNT), + music_source_table(), + &[songs(2)], + true, + ); + fake.rows + .lock() + .unwrap() + .get_mut(&arn(ACCOUNT)) + .unwrap() + .store = s3_identity(); + let entry = scratch.path().join("restore-tid-MusicRestored"); + std::fs::create_dir_all(entry.join(format!("{EXTENDDB_MANIFEST_FILE}.partial"))).unwrap(); + let runtime = spooled_runtime(&mem, scratch.path()); + restore_on(&fake, &runtime, &restore_body(json!({}))) + .await + .unwrap(); + wait_for_jobs(&runtime).await; + let writes = restore_writes(&fake); + assert_eq!(writes.len(), 1, "{writes:?}"); + let reason = writes[0].2.clone().unwrap(); + assert!( + reason.starts_with(&format!("writing {EXTENDDB_MANIFEST_FILE} to the spool: ")), + "{reason}" + ); + assert_eq!(fake.removed_targets.lock().unwrap().len(), 1); + assert!(fake.bulk_loads.lock().unwrap().is_empty()); + assert!( + !entry.exists(), + "the entry is removed after a FAILED write that matched" + ); + assert_eq!(restore_outcomes(&runtime), [("failed".to_owned(), 1.0)]); } -// --- refusals and validation -------------------------------------------------- +/// `Progress::from_records` on each backend's record shape: PostgreSQL's +/// (the record carries no `file_index`; the progress rows name each +/// recorded file's count), MongoDB's (`file_index` with no `items_applied`: +/// the recorded file is complete), and SQLite's and Cassandra's (both, +/// mid-file); a `file_index` past the manifest records nothing more. +#[test] +fn progress_from_records_reads_each_backends_shape() { + let mem = MemStore::default(); + let fake = Fake::default(); + let manifest = seed_manifest_backup( + &fake, + &mem, + &arn(ACCOUNT), + music_source_table(), + &[songs_from(0, 3), songs_from(3, 2), songs_from(5, 4)], + true, + ); + let mut row = RestoreJobRow::reconciler_row( + RestoreTargetKey { + account_id: ACCOUNT.to_owned(), + table_name: "T".to_owned(), + table_id: "tid".to_owned(), + }, + arn(ACCOUNT), + RestoreKind::Manifest, + JobStatus::Running, + 1, + 0.0, + ); -#[tokio::test] -async fn multi_part_key_backups_are_refused_in_the_engine() { - let mut src = source(Some(definition("PROVISIONED"))); - src.key_schema.push(key("pk2", KeyType::Hash)); - let fake = Fake::with_source(src); - let msg = validation_message(restore(&fake, restore_body(json!({}))).await.unwrap_err()); - assert!(msg.contains("multi-part key"), "{msg}"); - assert!(fake.restores().is_empty()); + // PostgreSQL: files 0 and 2 recorded whole, file 1 not; the resume + // opens file 1 from 0. + let progress = restore::Progress::from_records(&row, &[(0, 3), (2, 4)], &manifest); + assert_eq!(progress.recorded(), [Some(3), None, Some(4)]); + assert_eq!(progress.first_open_in(&manifest), Some((1, 0))); + // A PostgreSQL row of a file a decoder panic left short (n1): the file + // opens from its row. + let progress = restore::Progress::from_records(&row, &[(0, 3), (1, 1)], &manifest); + assert_eq!(progress.recorded(), [Some(3), Some(1), None]); + assert_eq!(progress.first_open_in(&manifest), Some((1, 1))); + + // MongoDB: `file_index` 1 with no `items_applied`: files 0 and 1 are + // complete, file 2 opens from 0. + row.file_index = Some(1); + row.items_applied = None; + let progress = restore::Progress::from_records(&row, &[], &manifest); + assert_eq!(progress.recorded(), [Some(3), Some(2), None]); + assert_eq!(progress.first_open_in(&manifest), Some((2, 0))); + + // SQLite and Cassandra: file 1 after 1 of its items. + row.items_applied = Some(1); + let progress = restore::Progress::from_records(&row, &[], &manifest); + assert_eq!(progress.recorded(), [Some(3), Some(1), None]); + assert_eq!(progress.first_open_in(&manifest), Some((1, 1))); + + // Every file recorded: nothing opens. + row.file_index = Some(2); + row.items_applied = Some(4); + let progress = restore::Progress::from_records(&row, &[], &manifest); + assert_eq!(progress.first_open_in(&manifest), None); + + // A `file_index` past the manifest: every file complete, the count + // decides. + row.file_index = Some(7); + row.items_applied = Some(1); + let progress = restore::Progress::from_records(&row, &[], &manifest); + assert_eq!(progress.recorded(), [Some(3), Some(2), Some(4)]); + assert_eq!(progress.first_open_in(&manifest), None); + + // Nothing recorded. + row.file_index = None; + row.items_applied = None; + let progress = restore::Progress::from_records(&row, &[], &manifest); + assert_eq!(progress.recorded(), [None, None, None]); + assert_eq!(progress.first_open_in(&manifest), Some((0, 0))); + + // The record and a progress row for one file: the larger count is the + // file's, whichever source carries it (no backend writes both today). + row.file_index = Some(1); + row.items_applied = Some(2); + let progress = restore::Progress::from_records(&row, &[(1, 1)], &manifest); + assert_eq!( + progress.recorded(), + [Some(3), Some(2), None], + "a smaller progress row lowered the record's count" + ); + row.items_applied = Some(1); + let progress = restore::Progress::from_records(&row, &[(1, 2)], &manifest); + assert_eq!(progress.recorded(), [Some(3), Some(2), None]); } +/// A record gone between the takeover and the reads that follow (whoever +/// moved it owns it) is left to its owner with the info line, not reported +/// as a cycle error. #[tokio::test] -async fn vector_index_backups_are_refused_in_the_engine() { - let mut def = definition("PAY_PER_REQUEST"); - def.vector_index_names.push("v".to_owned()); - let fake = Fake::with_source(source(Some(def))); - let msg = validation_message(restore(&fake, restore_body(json!({}))).await.unwrap_err()); - assert!(msg.contains("vector index"), "{msg}"); - assert!(fake.restores().is_empty()); - - let mut legacy = source(None); - legacy.legacy_vector_index_count = 1; - let fake = Fake::with_source(legacy); - let msg = validation_message(restore(&fake, restore_body(json!({}))).await.unwrap_err()); - assert!(msg.contains("vector index"), "{msg}"); - assert!(fake.restores().is_empty()); +async fn a_record_gone_after_the_takeover_is_left_to_its_owner() { + let (lines, _guard) = LogLines::capture(); + let mem = Arc::new(MemStore::default()); + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + let fake = Arc::new(Fake::default()); + seed_manifest_backup( + &fake, + &mem, + &arn(ACCOUNT), + music_source_table(), + &[songs(2)], + true, + ); + let runtime = job_runtime(&mem); + restore_on(&fake, &runtime, &restore_body(json!({}))) + .await + .unwrap(); + wait_for_jobs(&runtime).await; + dead_record_without_progress(&fake, false); + fake.takeover_removes_record + .store(true, std::sync::atomic::Ordering::Release); + let report = cycle_on(&fake, &runtime).await; + assert_eq!(fake.restore_takeovers.lock().unwrap().len(), 1); + assert!(report.manifest_restores_resumed.is_empty()); + assert!(report.manifest_restores_failed.is_empty()); + assert_eq!( + lines + .containing("was gone when its progress was read; left to whoever moved it") + .len(), + 1 + ); + assert!( + lines + .containing("was taken over but its progress could not be read") + .is_empty() + ); + assert_eq!(lines.at_level(tracing::Level::ERROR), Vec::::new()); } +/// A registration that fails after the request stopped waiting for it +/// (the 60 s bound answered `InternalServerError`) is logged at ERROR with +/// the ARN, the target, the account and the error, since the request's own +/// line told the operator the target appears if registered. #[tokio::test] -async fn a_stored_definition_create_table_would_refuse_is_refused() { - type Corrupt = fn(&mut BackupRestoreSource); - let cases: [(&str, Corrupt); 7] = [ - ( - "101 projected attributes split across a GSI and an LSI", - |s| { - let d = s.definition.as_mut().unwrap(); - d.global_secondary_indexes[0].projection = include(0..60); - d.local_secondary_indexes[0].projection = include(60..101); - }, - ), - ("GSI on an undefined attribute", |s| { - let d = s.definition.as_mut().unwrap(); - d.global_secondary_indexes[0].key_schema[0].attribute_name = "nowhere".to_owned(); - }), - ("negative GSI throughput", |s| { - let d = s.definition.as_mut().unwrap(); - d.global_secondary_indexes[0].provisioned_throughput = Some(throughput(-1, 1)); - }), - ("GSI throughput with one unit at 0", |s| { - let d = s.definition.as_mut().unwrap(); - d.global_secondary_indexes[0].provisioned_throughput = Some(throughput(0, 1)); - }), - ("negative table throughput", |s| { - s.definition.as_mut().unwrap().provisioned_throughput = Some(throughput(1, -1)); - }), - ("too many GSIs", |s| { - let d = s.definition.as_mut().unwrap(); - d.global_secondary_indexes = (0..30) - .map(|i| gsi(&format!("g{i}"), "gpk", Some(throughput(1, 1)))) - .collect(); - }), - ("too many projected attributes", |s| { - let d = s.definition.as_mut().unwrap(); - d.global_secondary_indexes[0].projection = Projection { - projection_type: ProjectionType::Include, - non_key_attributes: Some((0..101).map(|i| format!("a{i}")).collect()), - }; - }), - ]; - for (what, corrupt) in cases { - let mut src = source(Some(definition("PROVISIONED"))); - corrupt(&mut src); - let fake = Fake::with_source(src); - let err = restore(&fake, restore_body(json!({}))).await.unwrap_err(); - assert!( - matches!(err, DynamoDbError::ValidationException(_)), - "{what}: {err:?}" - ); - assert!(fake.restores().is_empty(), "{what}: reached storage"); - } +async fn a_registration_that_fails_after_the_request_stopped_waiting_is_logged() { + let (lines, _guard) = LogLines::capture(); + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let manifest = seed_manifest_backup( + &fake, + &mem, + &arn(ACCOUNT), + music_source_table(), + &[songs(2)], + true, + ); + let runtime = job_runtime(&mem); + let store = runtime.configured_store_arc().unwrap(); + let env = job::JobEnv::new(&runtime, store).unwrap(); + let permit = runtime.take_permit().await.unwrap(); + let prefix = BackupArn::parse(&arn(ACCOUNT)).unwrap().store_prefix(); + let manifest_bytes = + mem.objects.lock().unwrap()[&format!("{prefix}{EXTENDDB_MANIFEST_FILE}")].clone(); + let sidecar_bytes = + mem.objects.lock().unwrap()[&format!("{prefix}{MANIFEST_CHECKSUM_FILE}")].clone(); + let input = extenddb_storage::backup_definition::restore_input_from_manifest( + &manifest.source_table, + "MusicRestored", + &RestoreTableOverrides::default(), + ); + let key_schema = extenddb_core::validation::RestorableKeySchema::resolve( + &manifest.source_table.key_schema, + &manifest.source_table.attribute_definitions, + ) + .unwrap(); + let parts = restore::RestoreRequestParts { + account_id: ACCOUNT.to_owned(), + arn: BackupArn::parse(&arn(ACCOUNT)).unwrap(), + backup: BackupKey::from_arn(&BackupArn::parse(&arn(ACCOUNT)).unwrap()), + manifest: Arc::new(manifest), + manifest_bytes, + sidecar_bytes, + input, + key_schema, + limits: (&LimitsConfig::default()).into(), + registration: None, + }; + *fake.claim_restore_error.lock().unwrap() = + Some(StorageError::TableAlreadyExists("MusicRestored".to_owned())); + let (reply, answer) = tokio::sync::oneshot::channel(); + // The request gave up at its bound. + drop(answer); + restore::start(Arc::clone(&fake), env, permit, parts, reply).await; + let logged = lines.containing( + "the registration failed after the request had stopped waiting for it; no target was \ + created", + ); + assert_eq!(logged.len(), 1, "{logged:?}"); + assert!( + logged[0].starts_with(&format!( + "restore of {} into MusicRestored (account {ACCOUNT}): ", + arn(ACCOUNT) + )), + "{}", + logged[0] + ); + assert!(logged[0].contains("MusicRestored"), "{}", logged[0]); + assert_eq!(runtime.free_slots(), 4, "the permit is returned"); } -fn include(range: std::ops::Range) -> Projection { - Projection { - projection_type: ProjectionType::Include, - non_key_attributes: Some(range.map(|i| format!("a{i}")).collect()), - } +/// The request's two document reads collect at most their bound: a sidecar +/// over 4 KiB or a manifest over 16 MiB behind an AVAILABLE row is refused +/// as a corrupt prefix (`ValidationException`), not read whole; a resume +/// meeting one is FAILED with the same reason. +#[tokio::test] +async fn the_document_reads_are_bounded() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + seed_manifest_backup( + &fake, + &mem, + &arn(ACCOUNT), + music_source_table(), + &[songs(2)], + true, + ); + let runtime = job_runtime(&mem); + let prefix = BackupArn::parse(&arn(ACCOUNT)).unwrap().store_prefix(); + let sidecar_key = format!("{prefix}{MANIFEST_CHECKSUM_FILE}"); + let good_sidecar = mem.objects.lock().unwrap()[&sidecar_key].clone(); + let mut oversize = good_sidecar.clone(); + oversize.extend(std::iter::repeat_n(b' ', 4 * 1024)); + mem.insert(&sidecar_key, &oversize); + let err = restore_on(&fake, &runtime, &restore_body(json!({}))) + .await + .unwrap_err(); + assert_eq!( + validation_message(err), + format!( + "Backup {} has a {MANIFEST_CHECKSUM_FILE} over 4096 bytes, which no backup this \ + server wrote produces; the backup's prefix is corrupt", + arn(ACCOUNT) + ) + ); + assert!(fake.restore_claims.lock().unwrap().is_empty()); + mem.insert(&sidecar_key, &good_sidecar); + + // The manifest's bound, one byte over. + let manifest_key = format!("{prefix}{EXTENDDB_MANIFEST_FILE}"); + let good_manifest = mem.objects.lock().unwrap()[&manifest_key].clone(); + let mut oversize = good_manifest.clone(); + oversize.resize(16 * 1024 * 1024 + 1, b' '); + mem.insert(&manifest_key, &oversize); + let err = restore_on(&fake, &runtime, &restore_body(json!({}))) + .await + .unwrap_err(); + assert_eq!( + validation_message(err), + format!( + "Backup {} has a {EXTENDDB_MANIFEST_FILE} over 16777216 bytes, which no backup this \ + server wrote produces; the backup's prefix is corrupt", + arn(ACCOUNT) + ) + ); + // At the bound, the read goes through and the checksum decides. + oversize.truncate(16 * 1024 * 1024); + mem.insert(&manifest_key, &oversize); + let err = restore_on(&fake, &runtime, &restore_body(json!({}))) + .await + .unwrap_err(); + assert!( + validation_message(err).contains("whose checksum sidecar does not match"), + "a manifest at the bound is read and verified" + ); + mem.insert(&manifest_key, &good_manifest); + + // A resume over an oversize sidecar is FAILED with the reason. + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + restore_on(&fake, &runtime, &restore_body(json!({}))) + .await + .unwrap(); + wait_for_jobs(&runtime).await; + dead_record_without_progress(&fake, false); + let mut oversize = good_sidecar.clone(); + oversize.extend(std::iter::repeat_n(b' ', 4 * 1024)); + mem.insert(&sidecar_key, &oversize); + let runtime = job_runtime(&mem); + let report = cycle_on(&fake, &runtime).await; + assert_eq!(report.manifest_restores_resumed, ["MusicRestored"]); + wait_for_jobs(&runtime).await; + let writes = restore_writes(&fake); + assert_eq!( + writes.last().and_then(|w| w.2.clone()).as_deref(), + Some( + "the store's extenddb-manifest.sha256 for the backup is over 4096 bytes, which no \ + backup this server wrote produces" + ), + "{writes:?}" + ); } -#[tokio::test] -async fn one_hundred_projected_attributes_are_accepted() { - let mut src = source(Some(definition("PROVISIONED"))); - let d = src.definition.as_mut().unwrap(); - d.global_secondary_indexes[0].projection = include(0..60); - d.local_secondary_indexes[0].projection = include(60..100); - let fake = Fake::with_source(src); - restore(&fake, restore_body(json!({}))).await.unwrap(); - assert_eq!(fake.restores().len(), 1); +// --- `backup remove-target` (`maintenance::remove_target`) ------------------------ + +/// The request `remove_target` runs with, against the `Fake`. +fn remove_target_request(table: &str, owner_window: Duration) -> maintenance::RemoveTarget { + maintenance::RemoveTarget { + account_id: ACCOUNT.to_owned(), + table_name: table.to_owned(), + holder: "extenddb backup remove-target on host-a (pid 1)".to_owned(), + stale_after: Duration::from_secs(300), + owner_window, + heartbeat_interval: Duration::from_millis(20), + } } -/// A PROVISIONED table whose GSI has no throughput (CreateTable accepts one) -/// or 0/0 (what UpdateTable to PROVISIONED leaves on an on-demand table's -/// GSIs) restores with no override, the GSI taking the target table's -/// throughput. With `BillingModeOverride` `PROVISIONED` and no GSI list it is -/// refused like any backup with a GSI (the override, not the recorded -/// throughput, decides); the list route restores it. +/// `backup remove-target` under the maintenance claim: a FAILED record's +/// CREATING target (`target_not_empty`, the reason the reconciler leaves +/// for it) is removed with its record, and the claim was taken for the +/// removal and released after it; a RUNNING record the backend's dead-job +/// listing does not name is refused and nothing is removed, whatever its +/// `heartbeat_at` reads on this host's clock (one second old; 1234.5, an +/// age a host whose clock runs ahead of the catalog's would compute for a +/// live owner); a RUNNING record the listing names is removed, with a +/// heartbeat this host's clock reads as fresh; a DONE record (the table +/// ACTIVE) and a table no record names are refused as "not a restore +/// target"; a claim a live holder has refuses the command before any read. #[tokio::test] -async fn a_table_create_table_accepted_restores() { - // The CreateTable half: the shape is accepted there. - let mut created = source(Some(definition("PROVISIONED"))) - .restore_input("Music", &RestoreTableOverrides::default()); - created.global_secondary_indexes.as_mut().unwrap()[0].provisioned_throughput = None; - extenddb_core::validation::validate_create_table(&created, &LimitsConfig::default()) - .expect("CreateTable accepts a PROVISIONED GSI without throughput"); +async fn remove_target_removes_a_failed_or_dead_target_under_the_claim_and_refuses_the_rest() { + let mem = Arc::new(MemStore::default()); + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + let fake = Arc::new(Fake::default()); + seed_manifest_backup( + &fake, + &mem, + &arn(ACCOUNT), + music_source_table(), + &[songs(2)], + true, + ); + let runtime = job_runtime(&mem); + restore_on(&fake, &runtime, &restore_body(json!({}))) + .await + .unwrap(); + wait_for_jobs(&runtime).await; - let provisioned = json!({ - "BillingModeOverride": "PROVISIONED", - "ProvisionedThroughputOverride": { "ReadCapacityUnits": 5, "WriteCapacityUnits": 6 }, - }); - for recorded in [None, Some(throughput(0, 0))] { - let src = || { - let mut src = source(Some(definition("PROVISIONED"))); - src.definition.as_mut().unwrap().global_secondary_indexes[0].provisioned_throughput = - recorded.clone(); - src - }; - let fake = Fake::with_source(src()); - restore(&fake, restore_body(json!({}))) - .await - .unwrap_or_else(|e| panic!("{recorded:?}: {e:?}")); - let input = &fake.restores()[0].input; + // DONE: the table is ACTIVE. + assert_eq!( + fake.restore_job("MusicRestored").unwrap().status, + JobStatus::Done + ); + let err = maintenance::remove_target( + &*fake, + &remove_target_request("MusicRestored", Duration::from_secs(300)), + ) + .await + .unwrap_err(); + assert_eq!( + err, + format!( + "MusicRestored (account {ACCOUNT}) is not a restore target: its restore from {} is \ + DONE (the table is ACTIVE, or awaits its index backfill)", + arn(ACCOUNT) + ) + ); + assert!(fake.removed_targets.lock().unwrap().is_empty()); + assert_eq!(fake.maintenance_holders.lock().unwrap().len(), 1); + assert!( + *fake.maintenance_holders.lock().unwrap()[0] + .released + .lock() + .unwrap(), + "the claim is released after a refusal too" + ); + + // No record names the table. + let err = maintenance::remove_target( + &*fake, + &remove_target_request("Nothing", Duration::from_secs(300)), + ) + .await + .unwrap_err(); + assert_eq!( + err, + format!("Nothing (account {ACCOUNT}) is not a restore target: no restore record names it") + ); + + // RUNNING with a live owner: the dead-job listing does not name the + // record (the backend's age test on the catalog's clock), whatever its + // heartbeat reads against this host's clock. + dead_record_without_progress(&fake, true); + fake.restore_dead.lock().unwrap().clear(); + for heartbeat_at in [seconds_since_epoch() - 1.0, 1234.5] { + fake.restore_jobs + .lock() + .unwrap() + .get_mut("tid-MusicRestored") + .unwrap() + .heartbeat_at = heartbeat_at; + let err = maintenance::remove_target( + &*fake, + &remove_target_request("MusicRestored", Duration::from_secs(300)), + ) + .await + .unwrap_err(); assert_eq!( - input.global_secondary_indexes.as_ref().unwrap()[0].provisioned_throughput, - Some(throughput(7, 8)), - "{recorded:?}" + err, + format!( + "the restore into MusicRestored (account {ACCOUNT}) from {} is RUNNING with a \ + live owner (a heartbeat within stale_job_after_secs); stop it or wait for it", + arn(ACCOUNT) + ), + "heartbeat_at {heartbeat_at}" ); + assert!(fake.removed_targets.lock().unwrap().is_empty()); + assert!(fake.tables.lock().unwrap().contains_key("MusicRestored")); + } + fake.restore_dead + .lock() + .unwrap() + .insert("tid-MusicRestored".to_owned()); - let fake = Fake::with_source(src()); - let msg = validation_message( - restore(&fake, restore_body(provisioned.clone())) - .await - .unwrap_err(), - ); - assert_eq!( - msg, - "Invalid Request: Must override ProvisionedThroughput if BillingMode is overridden \ - to PROVISIONED for index gsi1", - "{recorded:?}" + // The reconciler finds the dead owner, no progress and a non-empty + // target: FAILED `target_not_empty`, the target left; the command + // removes it. + let report = cycle_on(&fake, &runtime).await; + assert_eq!( + report.manifest_restores_failed, + [("MusicRestored".to_owned(), TARGET_NOT_EMPTY.to_owned())] + ); + assert!(fake.tables.lock().unwrap().contains_key("MusicRestored")); + let removed = maintenance::remove_target( + &*fake, + &remove_target_request("MusicRestored", Duration::from_secs(300)), + ) + .await + .unwrap(); + assert_eq!( + removed, + maintenance::RemovedTarget { + table_id: "tid-MusicRestored".to_owned(), + backup_arn: arn(ACCOUNT), + kind: RestoreKind::Manifest, + status: JobStatus::Failed, + failure_reason: Some(TARGET_NOT_EMPTY.to_owned()), + } + ); + assert!( + !fake.tables.lock().unwrap().contains_key("MusicRestored"), + "the target is gone" + ); + assert!( + fake.restore_job("MusicRestored").is_none(), + "the record is gone" + ); + assert_eq!( + fake.removed_targets.lock().unwrap().as_slice(), + [RestoreTargetKey { + account_id: ACCOUNT.to_owned(), + table_name: "MusicRestored".to_owned(), + table_id: "tid-MusicRestored".to_owned(), + }] + ); + { + let holders = fake.maintenance_holders.lock().unwrap(); + assert_eq!(holders.len(), 5, "a claim per command"); + assert!( + holders.iter().all(|h| *h.released.lock().unwrap()), + "every claim released" ); - assert!(fake.restores().is_empty()); + } + assert!( + fake.maintenance_claim_held_by.lock().unwrap().is_none(), + "no claim left held" + ); - let mut with_list = provisioned.clone(); - with_list["GlobalSecondaryIndexOverride"] = json!([gsi_entry(Some((5, 6)))]); - let fake = Fake::with_source(src()); - restore(&fake, restore_body(with_list)) + // RUNNING with a dead owner: the listing names the record (under the + // 300 s window with a heartbeat this host's clock reads as fresh; under + // the zero window SQLite's serve lock gives, with a heartbeat an hour + // ahead): removed with its record, no FAILED write. + for window in [Duration::from_secs(300), Duration::ZERO] { + let fake = Arc::new(Fake::default()); + let mem = Arc::new(MemStore::default()); + seed_manifest_backup( + &fake, + &mem, + &arn(ACCOUNT), + music_source_table(), + &[songs(2)], + true, + ); + let runtime = job_runtime(&mem); + restore_on(&fake, &runtime, &restore_body(json!({}))) .await - .unwrap_or_else(|e| panic!("{recorded:?}: {e:?}")); - let input = &fake.restores()[0].input; - assert_eq!( - input.global_secondary_indexes.as_ref().unwrap()[0].provisioned_throughput, - Some(throughput(5, 6)), - "{recorded:?}" + .unwrap(); + wait_for_jobs(&runtime).await; + dead_record_without_progress(&fake, true); + if window.is_zero() { + fake.restore_jobs + .lock() + .unwrap() + .get_mut("tid-MusicRestored") + .unwrap() + .heartbeat_at = seconds_since_epoch() + 3600.0; + } else { + fake.restore_jobs + .lock() + .unwrap() + .get_mut("tid-MusicRestored") + .unwrap() + .heartbeat_at = seconds_since_epoch(); + } + let removed = + maintenance::remove_target(&*fake, &remove_target_request("MusicRestored", window)) + .await + .unwrap(); + assert_eq!(removed.status, JobStatus::Running); + assert_eq!(removed.failure_reason, None); + assert!(fake.restore_job("MusicRestored").is_none()); + assert!(!fake.tables.lock().unwrap().contains_key("MusicRestored")); + assert!( + fake.failed_jobs.lock().unwrap().is_empty(), + "no FAILED write" ); } -} - -#[tokio::test] -async fn a_missing_backup_is_backup_not_found() { - let fake = Fake::default(); - assert!(matches!( - restore(&fake, restore_body(json!({}))).await.unwrap_err(), - DynamoDbError::BackupNotFoundException(_) - )); -} - -// --- the storage request and the response ------------------------------------ -#[tokio::test] -async fn the_backend_receives_the_resolved_request() { - let fake = Fake::with_source(source(Some(definition("PROVISIONED")))); - restore(&fake, restore_body(json!({}))).await.unwrap(); - let [request] = fake.restores().try_into().unwrap(); - assert_eq!(request.backup_arn, arn(ACCOUNT)); - assert_eq!(request.source_table_arn, source_arn()); - assert!((request.restore_date_time - CREATED).abs() < f64::EPSILON); - let input = request.input; - assert_eq!(input.table_name, "MusicRestored"); - assert_eq!(input.key_schema.len(), 2); - assert_eq!(input.billing_mode, Some(BillingMode::Provisioned)); - assert_eq!(input.provisioned_throughput, Some(throughput(7, 8))); + // A claim a live holder has: refused before any read. + let fake = Arc::new(Fake::default()); + *fake.maintenance_claim_held_by.lock().unwrap() = Some("extenddb backup purge".to_owned()); + let err = maintenance::remove_target( + &*fake, + &remove_target_request("MusicRestored", Duration::from_secs(300)), + ) + .await + .unwrap_err(); assert_eq!( - input.global_secondary_indexes.unwrap()[0].provisioned_throughput, - Some(throughput(3, 4)) + err, + "the maintenance claim is held by \"extenddb backup purge\"; one maintenance command \ + runs at a time, and a claim whose holder died expires after stale_job_after_secs (300 s)" ); - assert_eq!(input.local_secondary_indexes.unwrap().len(), 1); - assert!(input.stream_specification.is_none()); -} - -#[tokio::test] -async fn a_legacy_backup_restores_with_its_old_defaults() { - let fake = Fake::with_source(source(None)); - restore(&fake, restore_body(json!({}))).await.unwrap(); - let input = &fake.restores()[0].input; - assert_eq!(input.provisioned_throughput, Some(throughput(5, 5))); - assert!(input.global_secondary_indexes.is_none()); - assert!(input.local_secondary_indexes.is_none()); + assert!(fake.restore_lists.lock().unwrap().is_empty(), "no read"); } +/// A legacy record's target is removed the same way: the reconciler would +/// fail and remove it once stale; the command lets the operator do it now. #[tokio::test] -async fn the_response_matches_the_service_shape() { - let fake = Fake::with_source(source(Some(definition("PROVISIONED")))); - let out = restore(&fake, restore_body(json!({}))).await.unwrap(); - let td = &out["TableDescription"]; - assert_eq!(td["TableStatus"], "CREATING"); - let summary = &td["RestoreSummary"]; - assert_eq!(summary["SourceBackupArn"], arn(ACCOUNT)); - assert_eq!(summary["SourceTableArn"], source_arn()); - assert_eq!(summary["RestoreDateTime"], CREATED); - assert_eq!(summary["RestoreInProgress"], true); - let gsi = td["GlobalSecondaryIndexes"][0].as_object().unwrap(); - assert!(!gsi.contains_key("IndexStatus"), "{gsi:?}"); - assert!(gsi.contains_key("IndexName")); - assert!( - td["BillingModeSummary"] - .get("LastUpdateToPayPerRequestDateTime") - .is_none() - ); - assert!( - td["TableThroughputModeSummary"] - .get("LastUpdateToPayPerRequestDateTime") - .is_none() +async fn remove_target_removes_a_legacy_records_target() { + let fake = Arc::new(Fake::default()); + let target = RestoreTargetKey { + account_id: ACCOUNT.to_owned(), + table_name: "Legacy".to_owned(), + table_id: "tid-legacy".to_owned(), + }; + fake.restore_jobs.lock().unwrap().insert( + "tid-legacy".to_owned(), + RestoreJobRow::reconciler_row( + target.clone(), + arn(ACCOUNT), + RestoreKind::Legacy, + JobStatus::Running, + 1, + 1234.5, + ), ); + fake.restore_dead + .lock() + .unwrap() + .insert("tid-legacy".to_owned()); + *fake.target_still_creating.lock().unwrap() = true; + let removed = maintenance::remove_target( + &*fake, + &remove_target_request("Legacy", Duration::from_secs(300)), + ) + .await + .unwrap(); + assert_eq!(removed.kind, RestoreKind::Legacy); + assert_eq!(removed.table_id, "tid-legacy"); + assert_eq!(fake.removed_targets.lock().unwrap().as_slice(), [target]); + + // A target the backend finds no longer a CREATING restore target: the + // record went with it, so the re-read finds none and the refusal is not + // the lock's. + *fake.target_still_creating.lock().unwrap() = false; + let err = maintenance::remove_target( + &*fake, + &remove_target_request("Legacy", Duration::from_secs(300)), + ) + .await + .unwrap_err(); assert_eq!( - td["TableThroughputModeSummary"]["TableThroughputMode"], - "PROVISIONED" + err, + format!( + "Legacy (account {ACCOUNT}) is not a restore target: it is no longer a CREATING \ + restore target with table id tid-legacy" + ) ); - // Omitted by the service here; the backend's description carries both. - assert!(td.get("DeletionProtectionEnabled").is_none(), "{td}"); - let pt = td["ProvisionedThroughput"].as_object().unwrap(); - assert!(!pt.contains_key("NumberOfDecreasesToday"), "{pt:?}"); - assert_eq!(pt["ReadCapacityUnits"], 9); - let backend = serde_json::to_value(described(&fake.restores()[0])).unwrap(); - assert_eq!(backend["DeletionProtectionEnabled"], false); +} + +/// `remove-target` of a CREATING restore target whose owner's lock is still +/// held (a frozen owner whose session outlives its heartbeat, on +/// PostgreSQL) reports the held lock as its own case, not "not a restore +/// target": the backend refused the removal and the record is unchanged, so +/// the table is still the target and the operator is told to wait for the +/// owner or stop it. Nothing is removed. With the lock released the same +/// command removes the target. +#[tokio::test] +async fn remove_target_reports_a_held_owner_lock_as_its_own_case() { + let fake = Arc::new(Fake::default()); + seed_failed_target(&fake, ACCOUNT, "T", "tid-a"); + *fake.owner_lock_held.lock().unwrap() = true; + let err = maintenance::remove_target( + &*fake, + &remove_target_request("T", Duration::from_secs(300)), + ) + .await + .unwrap_err(); assert_eq!( - backend["ProvisionedThroughput"]["NumberOfDecreasesToday"], - 0 + err, + format!( + "the restore into T (account {ACCOUNT}) from {} is a CREATING restore target (table \ + id tid-a) whose owner's lock is still held: its owner's session is alive though \ + its heartbeat is stale; wait for the owner to stop, or stop it, and run the \ + command again", + arn(ACCOUNT) + ) + ); + assert!( + fake.restore_jobs.lock().unwrap().contains_key("tid-a"), + "the record stays" + ); + assert!( + fake.restore_tables.lock().unwrap().contains_key("tid-a"), + "the target stays" ); + + *fake.owner_lock_held.lock().unwrap() = false; + let removed = maintenance::remove_target( + &*fake, + &remove_target_request("T", Duration::from_secs(300)), + ) + .await + .unwrap(); + assert_eq!(removed.table_id, "tid-a"); } +// --- the manifest total, remove-target's scope and lost claim, a target without +// --- tables, the registration retry --------------------------------------------- + +/// The final count above the manifest's total fails the restore and removes +/// its target, as a count below it does: the count check is `!=`, not `<` +/// (five restored against four listed). A fresh restore starts on an empty +/// target, so a count over the total means the target holds items the +/// backup does not account for. #[tokio::test] -async fn a_taken_name_is_table_already_exists_on_restore_only() { - let fake = Fake::with_source(source(Some(definition("PROVISIONED")))); - *fake.restore_error.lock().unwrap() = - Some(StorageError::TableAlreadyExists("MusicRestored".to_owned())); - let err = restore(&fake, restore_body(json!({}))).await.unwrap_err(); - match &err { - DynamoDbError::TableAlreadyExistsException(m) => { - assert_eq!(m, "Table already exists: MusicRestored"); - } - other => panic!("expected TableAlreadyExistsException, got {other:?}"), - } +async fn a_count_above_the_manifest_total_fails_the_restore() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + seed_manifest_backup( + &fake, + &mem, + &arn(ACCOUNT), + music_source_table(), + &[songs_from(0, 2), songs_from(2, 2)], + true, + ); + let runtime = job_runtime(&mem); + // The manifest lists four items; the backend reports five. + *fake.count_override.lock().unwrap() = Some(5); + restore_on(&fake, &runtime, &restore_body(json!({}))) + .await + .unwrap(); + wait_for_jobs(&runtime).await; assert_eq!( - err.full_error_type(), - "com.amazonaws.dynamodb.v20120810#TableAlreadyExistsException" + fake.bulk_loads.lock().unwrap().len(), + 2, + "every file was loaded before the count" + ); + let writes = restore_writes(&fake); + assert_eq!( + writes[0].2.as_deref(), + Some( + "restored 5 items, the manifest lists 4: the target holds items the backup does not \ + account for" + ) + ); + assert!( + fake.restore_job("MusicRestored").is_none(), + "the record and its target are removed" ); + assert_eq!(restore_outcomes(&runtime), [("failed".to_owned(), 1.0)]); +} - // CreateTable keeps ResourceInUseException. - assert!(matches!( - crate::create_table::storage_err_to_dynamo(StorageError::TableAlreadyExists( - "t".to_owned() - )), - DynamoDbError::ResourceInUseException(_) - )); +/// Seed a FAILED `target_not_empty` restore record over a non-empty CREATING +/// target of `(account, name, id)`, the state `remove-target` acts on. +fn seed_failed_target(fake: &Fake, account: &str, name: &str, id: &str) { + fake.restore_jobs.lock().unwrap().insert( + id.to_owned(), + RestoreJobRow { + target: RestoreTargetKey { + account_id: account.to_owned(), + table_name: name.to_owned(), + table_id: id.to_owned(), + }, + source_backup_arn: arn(account), + kind: RestoreKind::Manifest, + status: JobStatus::Failed, + owner_generation: 1, + heartbeat_at: seconds_since_epoch(), + store: Some(s3_identity()), + spool_host: None, + file_index: None, + items_applied: None, + failure_reason: Some(TARGET_NOT_EMPTY.to_owned()), + completed_at: Some(seconds_since_epoch()), + }, + ); + fake.restore_tables.lock().unwrap().insert( + id.to_owned(), + RestoreTable { + name: name.to_owned(), + key_attributes: vec!["pk".to_owned()], + items: vec![song("x", "y")], + active: false, + }, + ); } -/// Measured on DynamoDB: a target name taken by a -/// `CREATING` restore is `TableInUseException` `Table: is already -/// being restored from backup: `; a backup with a restore from it still -/// `CREATING` is `BackupInUseException`. The backends raise the storage -/// errors; this is their mapping. +/// `remove-target --account A T` removes only account A's target: the record +/// `find` compares the account, so account B's target of the same name is +/// untouched, and a command for A when only B's target exists is refused: +/// the record's table name alone would select either account's target. #[tokio::test] -async fn a_name_being_restored_is_table_in_use_and_a_backup_being_restored_is_backup_in_use() { - let fake = Fake::with_source(source(Some(definition("PROVISIONED")))); - *fake.restore_error.lock().unwrap() = Some(StorageError::TableBeingRestored { - table_name: "MusicRestored".to_owned(), - backup_arn: arn(ACCOUNT), - }); - let err = restore(&fake, restore_body(json!({}))).await.unwrap_err(); - match &err { - DynamoDbError::TableInUseException(m) => assert_eq!( - m, - &format!( - "Table: MusicRestored is already being restored from backup: {}", - arn(ACCOUNT) - ) - ), - other => panic!("expected TableInUseException, got {other:?}"), - } +async fn remove_target_is_scoped_to_the_named_account() { + const OTHER: &str = "210987654321"; + let fake = Arc::new(Fake::default()); + seed_failed_target(&fake, ACCOUNT, "T", "tid-a"); + seed_failed_target(&fake, OTHER, "T", "tid-b"); + + // The command for account A removes A's target alone. + let removed = maintenance::remove_target( + &*fake, + &remove_target_request("T", Duration::from_secs(300)), + ) + .await + .unwrap(); + assert_eq!(removed.table_id, "tid-a"); assert_eq!( - err.full_error_type(), - "com.amazonaws.dynamodb.v20120810#TableInUseException" + fake.removed_targets.lock().unwrap().as_slice(), + [RestoreTargetKey { + account_id: ACCOUNT.to_owned(), + table_name: "T".to_owned(), + table_id: "tid-a".to_owned(), + }], + "only the named account's target was removed" ); - - let fake = Fake::with_source(source(Some(definition("PROVISIONED")))); - *fake.restore_error.lock().unwrap() = Some(StorageError::BackupInUse(format!( - "Backup {} is being restored", - arn(ACCOUNT) - ))); - let err = restore(&fake, restore_body(json!({}))).await.unwrap_err(); assert!( - matches!(err, DynamoDbError::BackupInUseException(_)), - "{err:?}" + fake.restore_jobs.lock().unwrap().contains_key("tid-b"), + "the other account's record is untouched" ); - // Elsewhere the name is a table in use. A plain CreateTable into the - // name answers the service's text for a taken name (measured on - // DynamoDB), without the backup ARN. - assert!(matches!( - storage_err_to_dynamo(StorageError::TableBeingRestored { - table_name: "t".to_owned(), - backup_arn: "a".to_owned(), - }), - DynamoDbError::ResourceInUseException(_) - )); - match crate::create_table::storage_err_to_dynamo(StorageError::TableBeingRestored { - table_name: "MusicRestored".to_owned(), - backup_arn: arn(ACCOUNT), - }) { - DynamoDbError::ResourceInUseException(m) => { - assert_eq!(m, "Table already exists: MusicRestored"); - } - other => panic!("expected ResourceInUseException, got {other:?}"), - } + // With only account B's target left, the command for account A finds no + // record of its own. + let err = maintenance::remove_target( + &*fake, + &remove_target_request("T", Duration::from_secs(300)), + ) + .await + .unwrap_err(); + assert_eq!( + err, + format!("T (account {ACCOUNT}) is not a restore target: no restore record names it") + ); + assert!(fake.restore_jobs.lock().unwrap().contains_key("tid-b")); } +/// `remove-target` whose maintenance claim is lost between the record checks +/// and the removal stops before the removal, leaving the target and its +/// record: the pre-removal `claim.lost()` check is what stops it. #[tokio::test] -async fn synchronous_backend_refusals_keep_their_classes() { - for (storage, check) in [ - ( - StorageError::LimitExceeded("busy".to_owned()), - (|e: &DynamoDbError| matches!(e, DynamoDbError::LimitExceededException(_))) - as fn(&DynamoDbError) -> bool, - ), - ( - StorageError::Validation(format!("Backup not found: {}", arn(ACCOUNT))), - |e| matches!(e, DynamoDbError::BackupNotFoundException(_)), - ), - ] { - let fake = Fake::with_source(source(Some(definition("PROVISIONED")))); - *fake.restore_error.lock().unwrap() = Some(storage); - let err = restore(&fake, restore_body(json!({}))).await.unwrap_err(); - assert!(check(&err), "{err:?}"); - } +async fn remove_target_stops_before_the_removal_when_the_claim_is_lost() { + let fake = Arc::new(Fake::default()); + seed_failed_target(&fake, ACCOUNT, "T", "tid-a"); + // The claim this command takes reports lost at once; the read before the + // removal runs long enough for the HeldClaim's timer to observe it. + *fake.next_claim_lost.lock().unwrap() = true; + *fake.list_jobs_delay.lock().unwrap() = Some(Duration::from_millis(80)); + let request = maintenance::RemoveTarget { + account_id: ACCOUNT.to_owned(), + table_name: "T".to_owned(), + holder: "extenddb backup remove-target on host-a (pid 1)".to_owned(), + stale_after: Duration::from_secs(300), + owner_window: Duration::from_secs(300), + heartbeat_interval: Duration::from_millis(5), + }; + let err = maintenance::remove_target(&*fake, &request) + .await + .unwrap_err(); + assert!(err.starts_with("stopped before the removal: "), "{err}"); + assert!( + fake.removed_targets.lock().unwrap().is_empty(), + "nothing was removed" + ); + assert!( + fake.restore_jobs.lock().unwrap().contains_key("tid-a"), + "the record stays" + ); + assert!( + fake.restore_tables.lock().unwrap().contains_key("tid-a"), + "the target stays" + ); } -#[test] -fn an_unsupported_feature_is_a_validation_exception() { - let msg = "restoring a table with vector indexes is not supported by this storage backend"; +/// A dead manifest restore whose target has no data tables (its owner died +/// between the registration's commit and the data DDL) is failed and its +/// target removed, not retaken every stale window. Without the readiness +/// check the count on the missing table errors and the refreshed heartbeat +/// leaves the record to be retaken, never failing. +#[tokio::test] +async fn the_reconciler_fails_a_dead_restore_whose_target_has_no_tables() { + let mem = Arc::new(MemStore::default()); + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + let fake = Arc::new(Fake::default()); + seed_manifest_backup( + &fake, + &mem, + &arn(ACCOUNT), + music_source_table(), + &[songs(2)], + true, + ); + let runtime = job_runtime(&mem); + restore_on(&fake, &runtime, &restore_body(json!({}))) + .await + .unwrap(); + wait_for_jobs(&runtime).await; + // The record is RUNNING with a dead owner, its target's data tables gone. + dead_record_without_progress(&fake, false); + fake.targets_without_tables + .lock() + .unwrap() + .insert("tid-MusicRestored".to_owned()); + fake.bulk_loads.lock().unwrap().clear(); + + let report = cycle_on(&fake, &runtime).await; assert_eq!( - validation_message(storage_err_to_dynamo(StorageError::Unsupported( - msg.to_owned() - ))), - msg + report.manifest_restores_failed_and_removed, + ["MusicRestored"] + ); + assert!(report.manifest_restores_resumed.is_empty()); + assert!( + fake.restore_job("MusicRestored").is_none(), + "the record is gone" + ); + assert!( + !fake.tables.lock().unwrap().contains_key("MusicRestored"), + "the target is removed" + ); + assert!( + fake.bulk_loads.lock().unwrap().is_empty(), + "no resume was spawned" ); } -// --- CreateBackup ------------------------------------------------------------- +/// A retry that meets its own registration in flight waits for it and is +/// answered the CREATING description, rather than registering a second +/// target or being refused; a restore of another backup into the same name +/// while that registration is in flight is `TableInUseException` (the 60 s +/// registration bound equals botocore's default read timeout, so a client +/// retries while the first registration still holds the lock). +#[tokio::test] +async fn a_retry_waits_on_its_registration_and_another_backup_is_refused() { + let mem = Arc::new(MemStore::default()); + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + let fake = Arc::new(Fake::default()); + seed_manifest_backup( + &fake, + &mem, + &arn(ACCOUNT), + music_source_table(), + &[songs(2)], + true, + ); + let arn_b = + format!("arn:aws:dynamodb:{REGION}:{ACCOUNT}:table/Music/backup/01791976431439-aaaaaaaa"); + seed_manifest_backup(&fake, &mem, &arn_b, music_source_table(), &[songs(2)], true); + let runtime = Arc::new(job_runtime(&mem)); + // Hold the first registration mid-flight so the target is not yet + // created when the retry's name check runs. + *fake.claim_restore_delay.lock().unwrap() = Some(Duration::from_millis(400)); -/// Measured on DynamoDB: `BackupName` and `TableName` are -/// constrained as the service model says, with the service's texts; both -/// constraints failing join under `2 validation errors detected:`. -#[test] -fn backup_and_table_names_are_validated_as_the_model_says() { - let ok = ["abc", "a.b-c_D", &"x".repeat(255)]; - for name in ok { - validate_name_member(name, "backupName", BACKUP_NAME_BOUNDS) - .unwrap_or_else(|e| panic!("{name}: {e:?}")); - } - let constraint = |v: &str, path: &str, c: &str| { - format!("Value '{v}' at '{path}' failed to satisfy constraint: {c}") + let first = { + let fake = Arc::clone(&fake); + let runtime = Arc::clone(&runtime); + tokio::spawn(async move { restore_on(&fake, &runtime, &restore_body(json!({}))).await }) }; - let short = "Member must have length greater than or equal to 3"; - let long = "Member must have length less than or equal to 255"; - let pattern = "Member must satisfy regular expression pattern: [a-zA-Z0-9_.-]+"; - let too_long = "x".repeat(256); - let huge = "y".repeat(200 * 1024); - let cases: Vec<(&str, &str, String)> = vec![ - ( - "", - "backupName", - format!( - "1 validation error detected: {}", - constraint("", "backupName", short) - ), - ), - ( - "ab", - "backupName", - format!( - "1 validation error detected: {}", - constraint("ab", "backupName", short) - ), - ), - ( - &too_long, - "tableName", - format!( - "1 validation error detected: {}", - constraint(&too_long, "tableName", long) - ), - ), - ( - "a b c", - "backupName", - format!( - "1 validation error detected: {}", - constraint("a b c", "backupName", pattern) - ), - ), - ( - "abc\u{0}", - "backupName", - format!( - "1 validation error detected: {}", - constraint("abc\u{0}", "backupName", pattern) - ), - ), - ( - "b\u{e9}k", - "backupName", - format!( - "1 validation error detected: {}", - constraint("b\u{e9}k", "backupName", pattern) - ), - ), - ( - "a!", - "backupName", - format!( - "2 validation errors detected: {}; {}", - constraint("a!", "backupName", short), - constraint("a!", "backupName", pattern) - ), - ), - ]; - for (value, path, expected) in cases { + // Wait for the first request's registration to be in flight. + let deadline = tokio::time::Instant::now() + Duration::from_secs(5); + while fake.restore_claims.lock().unwrap().is_empty() { + assert!( + tokio::time::Instant::now() < deadline, + "registration never started" + ); + tokio::time::sleep(Duration::from_millis(5)).await; + } + + // The retry of the same request waits and is answered CREATING. + let retry = restore_on(&fake, &runtime, &restore_body(json!({}))) + .await + .unwrap(); + assert_eq!(retry["TableDescription"]["TableStatus"], "CREATING"); + assert_eq!( + retry["TableDescription"]["RestoreSummary"]["SourceBackupArn"], + arn(ACCOUNT) + ); + + // A different backup into the same name is refused while the first is in + // flight. + let other = restore_on( + &fake, + &runtime, + &restore_body(json!({ "BackupArn": arn_b })), + ) + .await + .unwrap_err(); + assert!( + matches!(other, DynamoDbError::TableInUseException(_)), + "{other:?}" + ); + + let first = first.await.unwrap().unwrap(); + assert_eq!(first["TableDescription"]["TableStatus"], "CREATING"); + wait_for_jobs(&runtime).await; + // One target, one record: the retry registered nothing of its own. + assert_eq!(fake.restore_claims.lock().unwrap().len(), 1); +} + +// --- the count retry, the maintenance claim taken mid-cycle, the takeover +// --- bound, the first file resumed with upserts ---------------------------------- + +/// The final count that fails three times stops the job with the record +/// RUNNING and no FAILED write: every item is loaded and recorded, so the +/// reconciler's resume counts again rather than the restore failing and its +/// loaded target being removed. The count is attempted `COUNT_ATTEMPTS` +/// times (three), with a wait between them. +#[tokio::test] +async fn a_count_that_fails_three_times_leaves_the_record_running() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + seed_manifest_backup( + &fake, + &mem, + &arn(ACCOUNT), + music_source_table(), + &[songs_from(0, 2), songs_from(2, 2)], + true, + ); + let runtime = job_runtime(&mem); + fake.count_errors + .store(3, std::sync::atomic::Ordering::Release); + restore_on(&fake, &runtime, &restore_body(json!({}))) + .await + .unwrap(); + wait_for_jobs_within(&runtime, Duration::from_secs(20)).await; + assert_eq!( + fake.count_calls.load(std::sync::atomic::Ordering::Acquire), + 3, + "the count was attempted three times" + ); + assert_eq!( + fake.bulk_loads.lock().unwrap().len(), + 2, + "every file was loaded before the count" + ); + let record = fake.restore_job("MusicRestored").expect("the record stays"); + assert_eq!(record.status, JobStatus::Running, "no FAILED was written"); + assert_eq!( + (record.file_index, record.items_applied), + (Some(1), Some(2)), + "the progress of the last file stands for the resume" + ); + assert!( + restore_writes(&fake) + .iter() + .all(|(_, to, _)| *to != JobStatus::Failed && *to != JobStatus::Done), + "no status write ended the job: {:?}", + restore_writes(&fake) + ); + assert!( + fake.restore_tables + .lock() + .unwrap() + .contains_key("tid-MusicRestored"), + "the loaded target is kept" + ); + assert_eq!(restore_outcomes(&runtime), [("failed".to_owned(), 1.0)]); +} + +/// A command that takes the maintenance claim during the cycle, after the +/// cycle's first read of it and before the takeover, is seen by the second +/// read after `claim_restore_dead`: the record is taken over (its generation +/// moved, its heartbeat fresh) but not resumed, and is left for the +/// maintenance arm of a later cycle. +#[tokio::test] +async fn a_claim_taken_during_the_cycle_leaves_the_taken_over_record_unresumed() { + let mem = Arc::new(MemStore::default()); + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + let fake = Arc::new(Fake::default()); + seed_manifest_backup( + &fake, + &mem, + &arn(ACCOUNT), + music_source_table(), + &[songs(2)], + true, + ); + let runtime = job_runtime(&mem); + restore_on(&fake, &runtime, &restore_body(json!({}))) + .await + .unwrap(); + wait_for_jobs(&runtime).await; + dead_record_without_progress(&fake, false); + fake.bulk_loads.lock().unwrap().clear(); + let before = fake.restore_job("MusicRestored").unwrap().owner_generation; + // The command takes the claim inside the takeover, so the cycle's first + // read (not held) and its second read (held) disagree. + *fake.takeover_hook.lock().unwrap() = Some(Box::new(|fake: &Fake| { + *fake.maintenance_held.lock().unwrap() = true; + })); + + let report = cycle_on(&fake, &runtime).await; + assert!(!report.maintenance_held, "the first read found no claim"); + assert!( + fake.takeover_hook.lock().unwrap().is_none(), + "the takeover ran" + ); + assert!( + report.manifest_restores_resumed.is_empty(), + "the record was not resumed: {report:?}" + ); + assert!(report.manifest_restores_failed.is_empty()); + assert!(report.manifest_restores_failed_and_removed.is_empty()); + wait_for_jobs(&runtime).await; + assert!( + fake.bulk_loads.lock().unwrap().is_empty(), + "no resume was spawned" + ); + let record = fake.restore_job("MusicRestored").unwrap(); + assert_eq!(record.status, JobStatus::Running); + assert_ne!(record.owner_generation, before, "the takeover itself stood"); +} + +/// A record whose every resume ends with it RUNNING again (here the final +/// count fails on every attempt, so each resume stops `LeftRunning` and the +/// record is retaken every stale window) is failed by the takeover after +/// this instance's fifth: FAILED `TAKEOVERS_EXHAUSTED`, the target removed. +/// Five cycles take it over and leave it RUNNING; the sixth fails it. +#[tokio::test] +async fn the_takeover_after_the_fifth_fails_a_record_that_never_ends() { + let mem = Arc::new(MemStore::default()); + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + let fake = Arc::new(Fake::default()); + seed_manifest_backup( + &fake, + &mem, + &arn(ACCOUNT), + music_source_table(), + &[songs(2)], + true, + ); + let runtime = job_runtime(&mem); + restore_on(&fake, &runtime, &restore_body(json!({}))) + .await + .unwrap(); + wait_for_jobs(&runtime).await; + // The record is RUNNING with its progress and a dead owner; the target's + // count fails on every attempt, so every resume ends `LeftRunning`. + { + let mut records = fake.restore_jobs.lock().unwrap(); + let record = records.get_mut("tid-MusicRestored").unwrap(); + record.status = JobStatus::Running; + record.completed_at = None; + fake.restore_tables + .lock() + .unwrap() + .get_mut("tid-MusicRestored") + .unwrap() + .active = false; + if let Some(desc) = fake.tables.lock().unwrap().get_mut("MusicRestored") { + desc.table_status = TableStatus::Creating; + } + fake.restore_dead + .lock() + .unwrap() + .insert("tid-MusicRestored".to_owned()); + } + fake.count_errors + .store(u32::MAX, std::sync::atomic::Ordering::Release); + let mut state = reconciler::ReconcilerState::default(); + for cycle in 1..=reconciler::RESTORE_TAKEOVERS { + let report = reconciler::cycle( + &fake, + &runtime, + Duration::from_secs(300), + "test", + &mut state, + ) + .await + .unwrap(); assert_eq!( - validation_message(validate_name_member(value, path, BACKUP_NAME_BOUNDS).unwrap_err()), - expected, - "{value:?}" + report.manifest_restores_resumed, + ["MusicRestored"], + "cycle {cycle} took the record over and resumed it" + ); + wait_for_jobs_within(&runtime, Duration::from_secs(20)).await; + let record = fake.restore_job("MusicRestored").expect("the record stays"); + assert_eq!( + record.status, + JobStatus::Running, + "cycle {cycle}: the resume ended with the record RUNNING" ); } - // A 200 KB name with a NUL byte (a 500 on Cassandra and PostgreSQL - // before the check) is refused with both constraints named. - let bad = format!("{huge}\u{0}"); - let msg = validation_message( - validate_name_member(&bad, "backupName", BACKUP_NAME_BOUNDS).unwrap_err(), + assert_eq!(fake.restore_takeovers.lock().unwrap().len(), 5); + + let report = reconciler::cycle( + &fake, + &runtime, + Duration::from_secs(300), + "test", + &mut state, + ) + .await + .unwrap(); + assert_eq!( + report.manifest_restores_failed_and_removed, + ["MusicRestored"], + "the sixth takeover fails the record: {report:?}" ); + assert!(report.manifest_restores_resumed.is_empty()); assert!( - msg.starts_with("2 validation errors detected: "), - "{}", - &msg[..60] + fake.restore_job("MusicRestored").is_none(), + "the record and its target are removed" ); - assert!(msg.contains(long) && msg.contains(pattern)); + assert!( + !fake.tables.lock().unwrap().contains_key("MusicRestored"), + "the target is removed" + ); + let writes = restore_writes(&fake); + assert!( + writes.iter().any(|(from, to, reason)| { + *from == JobStatus::Running + && *to == JobStatus::Failed + && reason.as_deref() == Some(reconciler::TAKEOVERS_EXHAUSTED) + }), + "the FAILED write names the exhausted takeovers: {writes:?}" + ); +} - // Through the handlers' members. - let body = json!({ "TableName": "ok_table", "BackupName": "x" }); - let name = create_backup_table_name(&body, ACCOUNT, &limits()).unwrap(); - assert_eq!(name, "ok_table"); - assert!(matches!( - validate_name_member(body["BackupName"].as_str().unwrap(), "backupName", BACKUP_NAME_BOUNDS) - .unwrap_err(), - DynamoDbError::ValidationException(m) if m.contains("'backupName'") - )); - let msg = validation_message( - create_backup_table_name(&json!({ "TableName": "a*" }), ACCOUNT, &limits()).unwrap_err(), +/// A dead record with no progress recorded over a target that holds items, +/// no more of them than the backup's total, is the state a crash inside the +/// first batch leaves on Cassandra (progress is recorded after the batch's +/// writes): it is resumed, not failed `target_not_empty`, and the resume +/// re-applies the first data file with `reapplying` set (idempotent +/// upserts), so the restore ends ACTIVE with the exact count. Only a count +/// above the backup's total is `target_not_empty` +/// (`a_resume_without_progress_needs_an_empty_target`). +#[tokio::test] +async fn a_half_applied_first_file_with_no_progress_is_resumed_with_upserts() { + let mem = Arc::new(MemStore::default()); + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + let fake = Arc::new(Fake::default()); + seed_manifest_backup( + &fake, + &mem, + &arn(ACCOUNT), + music_source_table(), + &[songs(2)], + true, ); + let runtime = job_runtime(&mem); + restore_on(&fake, &runtime, &restore_body(json!({}))) + .await + .unwrap(); + wait_for_jobs(&runtime).await; + // The record with no progress, the target holding one of the backup's + // two items (the first batch's writes landed, its progress did not). + dead_record_without_progress(&fake, false); + let first = songs(2).remove(0); + fake.restore_tables + .lock() + .unwrap() + .get_mut("tid-MusicRestored") + .unwrap() + .items + .push(first); + fake.bulk_loads.lock().unwrap().clear(); + + let report = cycle_on(&fake, &runtime).await; + assert_eq!( + report.manifest_restores_resumed, + ["MusicRestored"], + "one item of two is resumed, not failed: {report:?}" + ); + assert!(report.manifest_restores_failed.is_empty()); + wait_for_jobs(&runtime).await; + let loads = fake.bulk_loads.lock().unwrap(); + assert_eq!(loads.len(), 1, "the one file was loaded again"); assert!( - msg.contains("'tableName'") && msg.contains(pattern), - "{msg}" + loads[0].file.reapplying, + "the first file is re-applied with upserts: {:?}", + loads[0].file + ); + assert_eq!((loads[0].file.file_index, loads[0].file.start_at), (0, 0)); + drop(loads); + let record = fake.restore_job("MusicRestored").unwrap(); + assert_eq!(record.status, JobStatus::Done, "{record:?}"); + assert_eq!( + fake.restore_tables.lock().unwrap()["tid-MusicRestored"] + .items + .len(), + 2, + "the duplicate of the half-applied item was suppressed and the count is exact" ); } -/// Length is counted in characters, as the model's bounds are: a two-byte -/// character is one. `TableName` members take the deployment's table-name -/// limits (`CreateBackup` and `ListBackups` agree with `CreateTable`); -/// `BackupName` keeps the model's 3 to 255. +// --- the review's cells (PR 8, Lee's items) ------------------------------- + +/// An item of the `Music` table carrying `n` pseudo-random bytes (a `B` +/// attribute gzip cannot shrink), under the default item limit at 300,000. +fn song_with_noise(i: usize, n: usize) -> Item { + use extenddb_core::types::AttributeValue; + let mut state: u64 = 0x9E37_79B9_7F4A_7C15 ^ (i as u64 + 1); + let noise: Vec = (0..n) + .map(|_| { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + (state >> 24) as u8 + }) + .collect(); + let mut item = song(&format!("artist-{i:05}"), &format!("title-{i:05}")); + item.insert("Noise".to_owned(), AttributeValue::B(noise)); + item +} + +/// The mid-file case of +/// `a_put_failure_ends_the_job_without_waiting_for_the_streams_next_item`: +/// the data file's first chunk is out during the third item (three +/// incompressible items of 380,000 bytes, 1.15 MB once base64-encoded and +/// gzipped), the stream is held before the fourth, and the put fails 200 ms +/// after reading that chunk, by when the encoder has finished the third +/// item and waits on the empty item channel while the feeder waits on the +/// stream, which the test never releases. The uploader cancels the feeder's +/// token before it awaits the encoder, so the feeder stops, closes the item +/// channel, the encoder returns, and the row is FAILED naming the file +/// within the test's bound; with the encoder awaited first the job runs for +/// as long as the stream is held. #[tokio::test] -async fn name_lengths_are_counted_in_characters_and_table_names_take_the_limits() { - let pattern = "Member must satisfy regular expression pattern: [a-zA-Z0-9_.-]+"; - // Two characters, four bytes: too short, and not in the pattern (two - // clauses, not one). - let msg = validation_message( - validate_name_member("\u{e9}\u{e9}", "backupName", BACKUP_NAME_BOUNDS).unwrap_err(), +async fn a_mid_file_put_failure_ends_the_job_while_the_stream_is_held() { + let mem = Arc::new(MemStore::default()); + mem.fail_puts_after_first_chunk + .store(true, std::sync::atomic::Ordering::Release); + let fake = Arc::new(Fake::default()); + let items: Vec = (0..5).map(|i| song_with_noise(i, 380_000)).collect(); + let gate = Arc::new(tokio::sync::Semaphore::new(3)); + snapshot_fixture(&fake, &music_table(), items, true, Some(Arc::clone(&gate))); + let runtime = job_runtime(&mem); + let details = create_music_backup(&fake, &runtime).await.unwrap(); + wait_for_jobs_within(&runtime, Duration::from_secs(10)).await; + assert_eq!(gate.available_permits(), 0, "the stream was never released"); + let row = fake.rows.lock().unwrap()[&details.backup_arn].clone(); + assert_eq!(row.status, BackupStatus::Failed); + assert!( + row.failure_reason + .as_deref() + .unwrap() + .starts_with("writing data/000001.json.gz: "), + "{:?}", + row.failure_reason ); - assert!(msg.starts_with("2 validation errors detected: "), "{msg}"); - assert!(msg.contains("length greater than or equal to 3"), "{msg}"); - assert!(msg.contains(pattern), "{msg}"); - // 255 two-byte characters (510 bytes): within the length bound, so only - // the pattern clause. - let wide = "\u{e9}".repeat(255); - let msg = validation_message( - validate_name_member(&wide, "backupName", BACKUP_NAME_BOUNDS).unwrap_err(), + let prefix = BackupArn::parse(&details.backup_arn) + .unwrap() + .store_prefix(); + assert_eq!(mem.deletes.lock().unwrap().as_slice(), [prefix]); + assert!(mem.puts.lock().unwrap().is_empty(), "the put never stored"); + assert_eq!(job_outcomes(&runtime), [("failed".to_owned(), 1.0)]); + assert_eq!(runtime.free_slots(), 4); +} + +/// The check before every put reads the store as the runtime holds it. A +/// job held in its stream while the reconciler marks the store unhealthy +/// (here `record_check` with the problem a check would report) fails its +/// row with the store's reason and makes no store call: no put, no +/// `delete_prefix` (the FAILED-prefix duty deletes the prefix once the store +/// is healthy again). A store whose check passed with another id (the +/// identity file names another store: a root swapped under the job) fails +/// the row the same way, naming both ids. +#[tokio::test] +async fn a_store_marked_unhealthy_or_renamed_mid_job_fails_the_row_with_no_store_call() { + // Unhealthy. + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let gate = closed_gate(); + snapshot_fixture( + &fake, + &music_table(), + songs(3), + true, + Some(Arc::clone(&gate)), + ); + let runtime = job_runtime(&mem); + let details = create_music_backup(&fake, &runtime).await.unwrap(); + runtime + .configured_store() + .unwrap() + .record_check(Err(StoreProblem::FileUnreadable( + "not an identity file".to_owned(), + ))); + assert!(!runtime.configured_store().unwrap().is_healthy()); + gate.add_permits(3); + wait_for_jobs(&runtime).await; + let row = fake.rows.lock().unwrap()[&details.backup_arn].clone(); + assert_eq!(row.status, BackupStatus::Failed); + let reason = row.failure_reason.clone().unwrap(); + assert!( + reason.starts_with("the filesystem backup store is unhealthy (") + && reason.contains("extenddb-store.json could not be read: not an identity file") + && reason.ends_with("); data/000001.json.gz was not written"), + "{reason}" ); + assert!(row.completed_at.is_some()); + assert!(mem.puts.lock().unwrap().is_empty(), "no put"); + assert!(mem.deletes.lock().unwrap().is_empty(), "no delete_prefix"); assert_eq!( - msg, - format!( - "1 validation error detected: Value '{wide}' at 'backupName' failed to satisfy \ - constraint: {pattern}" + mem.keys(), + [STORE_IDENTITY_KEY], + "no object but the identity file" + ); + assert_eq!( + writes(&fake), + [(JobStatus::Failed, Some(reason))], + "the FAILED write and nothing else" + ); + assert_eq!(job_outcomes(&runtime), [("failed".to_owned(), 1.0)]); + assert_eq!(runtime.free_slots(), 4); + + // Another id. + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let gate = closed_gate(); + snapshot_fixture( + &fake, + &music_table(), + songs(3), + true, + Some(Arc::clone(&gate)), + ); + let runtime = job_runtime(&mem); + let details = create_music_backup(&fake, &runtime).await.unwrap(); + let swapped = StoreIdentity { + store_id: "store-2".to_owned(), + ..store() + }; + runtime + .configured_store() + .unwrap() + .record_check(Ok(swapped)); + assert!(runtime.configured_store().unwrap().is_healthy()); + gate.add_permits(3); + wait_for_jobs(&runtime).await; + let row = fake.rows.lock().unwrap()[&details.backup_arn].clone(); + assert_eq!(row.status, BackupStatus::Failed); + assert_eq!( + row.failure_reason.as_deref(), + Some( + "the filesystem backup store's id is store-2 and the row's is store-1; \ + data/000001.json.gz was not written" ) ); + assert!(mem.puts.lock().unwrap().is_empty(), "no put"); + assert!(mem.deletes.lock().unwrap().is_empty(), "no delete_prefix"); + assert_eq!(job_outcomes(&runtime), [("failed".to_owned(), 1.0)]); +} - // A deployment that raised the maximum: a 300-character table can be - // backed up and listed; the backup name stays at 255. - let raised = LimitsConfig { - max_table_name_length: 300, - min_table_name_length: 2, - ..Default::default() +/// A key-ordered stream that is not in key order: the manifest would say +/// `KeyOrdered` over items a restore's key-range seek cannot trust, so the +/// tracker refuses the item and the job fails `encoding data file 1` with +/// an empty prefix (no chunk was out, so no put started). One cell with a +/// lower key after a higher one, one with a key repeated. +#[tokio::test] +async fn a_key_ordered_stream_out_of_order_fails_the_first_data_file() { + let lower = vec![ + song("artist-00002", "title-00002"), + song("artist-00001", "title-00001"), + ]; + let repeated = vec![ + song("artist-00001", "title-00001"), + song("artist-00001", "title-00001"), + ]; + for (what, items) in [("a lower key", lower), ("a repeated key", repeated)] { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + snapshot_fixture(&fake, &music_table(), items, true, None); + let runtime = job_runtime(&mem); + let details = create_music_backup(&fake, &runtime).await.unwrap(); + wait_for_jobs(&runtime).await; + let row = fake.rows.lock().unwrap()[&details.backup_arn].clone(); + assert_eq!(row.status, BackupStatus::Failed, "{what}"); + assert!( + row.failure_reason + .as_deref() + .unwrap() + .starts_with("encoding data file 1: "), + "{what}: {:?}", + row.failure_reason + ); + assert!(mem.puts.lock().unwrap().is_empty(), "{what}: no put"); + assert!( + objects_under(&mem, &details.backup_arn).is_empty(), + "{what}: an empty prefix" + ); + assert_eq!( + job_outcomes(&runtime), + [("failed".to_owned(), 1.0)], + "{what}" + ); + } +} + +/// `BackupStoreErrors` is recorded for the job's `delete_prefix` when its +/// failure path's deletion fails (the row FAILED, the prefix left to the +/// reconciler), and for the probe's `list` when the listing of a dead +/// owner's prefix fails after its manifest read (the row left for the next +/// cycle); each once, under the call's name. +#[tokio::test] +async fn the_jobs_failed_deletion_and_the_probes_failed_listing_are_counted() { + use extenddb_core::metrics::MetricName; + let errors = |drained: &[extenddb_core::metrics::FlushBucket]| { + let mut ops: Vec<(String, f64)> = drained + .iter() + .filter(|b| b.metric == MetricName::BackupStoreErrors) + .map(|b| (b.operation.clone(), b.sum)) + .collect(); + ops.sort_by(|a, b| a.0.cmp(&b.0)); + ops }; - let long = "t".repeat(300); + + // The job's deletion. + let mem = Arc::new(MemStore::default()); + mem.fail_puts + .store(true, std::sync::atomic::Ordering::Release); + let fake = Arc::new(Fake::default()); + snapshot_fixture(&fake, &music_table(), songs(2), true, None); + let runtime = job_runtime(&mem); + let details = create_music_backup(&fake, &runtime).await.unwrap(); + let prefix = BackupArn::parse(&details.backup_arn) + .unwrap() + .store_prefix(); + // Set before the job reaches its deletion: the put fails first and the + // FAILED write comes between; the prefix is known from the response. + *mem.fail_deletes_of.lock().unwrap() = Some(prefix.clone()); + wait_for_jobs(&runtime).await; + let row = fake.rows.lock().unwrap()[&details.backup_arn].clone(); + assert_eq!(row.status, BackupStatus::Failed); + assert!( + mem.deletes.lock().unwrap().is_empty(), + "the deletion failed" + ); assert_eq!( - create_backup_table_name(&json!({ "TableName": long }), ACCOUNT, &raised).unwrap(), - long + errors(&runtime.metrics().unwrap().drain(Duration::ZERO)), + [("delete_prefix".to_owned(), 1.0), ("put".to_owned(), 1.0)] ); + + // The probe's listing. + let mem = Arc::new(MemStore::default()); + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + let fake = Arc::new(Fake::default()); + let dead = seed_dead_creating_with_manifest(&fake, &mem, true); + let dead_prefix = BackupArn::parse(&dead).unwrap().store_prefix(); + *mem.fail_lists_of.lock().unwrap() = Some(dead_prefix); + let metrics = Arc::new(extenddb_core::metrics::MetricsCollector::new()); + let runtime = runtime_on(&mem, store()).with_metrics(Arc::clone(&metrics)); + let report = cycle_on(&fake, &runtime).await; + assert_eq!(report.store_healthy, Some(true)); + assert!(report.creating_resolved.is_empty(), "{report:?}"); assert_eq!( - create_backup_table_name(&json!({ "TableName": "ab" }), ACCOUNT, &raised).unwrap(), - "ab" + fake.rows.lock().unwrap()[&dead].status, + BackupStatus::Creating, + "left for the next cycle" ); - let arn_form = format!("arn:aws:dynamodb:{REGION}:{ACCOUNT}:table/{long}"); assert_eq!( - create_backup_table_name(&json!({ "TableName": arn_form }), ACCOUNT, &raised).unwrap(), - long + errors(&metrics.drain(Duration::ZERO)), + [("list".to_owned(), 1.0)] ); - let fake = fake_with_summaries(vec![]); - list_backups( - &fake, - ACCOUNT, - REGION, - &raised, - &json!({ "TableName": long }), + *mem.fail_lists_of.lock().unwrap() = None; + let report = cycle_on(&fake, &runtime).await; + assert_eq!(report.creating_resolved, [(dead, BackupStatus::Available)]); + assert!(errors(&metrics.drain(Duration::ZERO)).is_empty()); +} + +/// `write_status`'s generation check: a FAILED write whose outcome is +/// unknown is re-read, and a row that shows FAILED under another generation +/// is `NotMatched` (the reconciler resolved the same backup; the row is not +/// this claim's), so the failure path makes no `delete_prefix`. Without the +/// check the status alone would read as `Matched`. +#[tokio::test] +async fn a_re_read_showing_the_target_status_under_another_generation_is_not_matched() { + let mem = Arc::new(MemStore::default()); + let fake = Fake::default(); + let arn = arn(ACCOUNT); + let prefix = BackupArn::parse(&arn).unwrap().store_prefix(); + mem.insert(&format!("{prefix}data/000001.json.gz"), b"x"); + let mut r = row(&arn, BackupStatus::Failed); + r.owner_generation = 9; + fake.rows.lock().unwrap().insert(arn.clone(), r); + let claim = fake.claim_on(&arn, 7); + *fake.unknown_write.lock().unwrap() = Some(UnknownWrite::Lost); + let path = job::failure::fail_row( + &*claim, + &*mem, + &passing_fence(), + "catalog-1", + &prefix, + "a reason", + None, ) - .await - .unwrap(); + .await; + match path { + job::failure::FailurePath::NoStoreAccess(why) => { + assert_eq!(why, "the row carries generation 9 (this claim's is 7)"); + } + other => panic!("{other:?}"), + } + assert!(mem.deletes.lock().unwrap().is_empty(), "no delete_prefix"); + assert_eq!(objects_under(&mem, &arn).len(), 1); + assert_eq!(fake.status_writes.lock().unwrap().len(), 1, "one write"); +} + +/// A data file whose stored size, as the store reports it, differs from the +/// encoder's compressed count fails the job: the manifest records the +/// encoder's count, the reconciler's dead-owner probe refuses a data file +/// whose stored size differs from the recorded one, and a live job must not +/// commit a manifest that probe would call incomplete. FAILED names both +/// figures; the prefix is deleted. +#[tokio::test] +async fn a_data_file_whose_stored_size_differs_from_the_encoders_fails_the_job() { + let mem = Arc::new(MemStore::default()); + mem.misreport_put_size + .store(true, std::sync::atomic::Ordering::Release); + let fake = Arc::new(Fake::default()); + snapshot_fixture(&fake, &music_table(), songs(2), true, None); + let runtime = job_runtime(&mem); + let details = create_music_backup(&fake, &runtime).await.unwrap(); + wait_for_jobs(&runtime).await; + let row = fake.rows.lock().unwrap()[&details.backup_arn].clone(); + assert_eq!(row.status, BackupStatus::Failed); + let stored = mem.put_sizes.lock().unwrap()[0].1; assert_eq!( - fake.listed_for.lock().unwrap()[0].as_deref(), - Some(long.as_str()) - ); - // The defaults still refuse it, with the bound in the text. - let msg = validation_message( - create_backup_table_name(&json!({ "TableName": long }), ACCOUNT, &limits()).unwrap_err(), - ); - assert!(msg.contains("length less than or equal to 255"), "{msg}"); - let msg = validation_message( - list_backups( - &fake, - ACCOUNT, - REGION, - &limits(), - &json!({ "TableName": long }), + row.failure_reason.as_deref(), + Some( + format!( + "the store reports {} bytes for data/000001.json.gz and the encoder wrote {stored}", + stored + 1 + ) + .as_str() ) - .await - .unwrap_err(), ); - assert!(msg.contains("length less than or equal to 255"), "{msg}"); - // A lowered maximum is reported with its own number. - let lowered = LimitsConfig { - max_table_name_length: 10, - ..Default::default() + assert_eq!(mem.puts.lock().unwrap().len(), 1, "no document followed"); + assert!(objects_under(&mem, &details.backup_arn).is_empty()); + assert_eq!(job_outcomes(&runtime), [("failed".to_owned(), 1.0)]); +} + +// --- the maintainer's second round on PR 8: a beat parked on the write lock -- + +mod review_round_3 { + use super::*; + + /// The pre-put check's third clause: a beat parked outside its bound + /// (SQLite's engine write lock, held by a GSI build) confirms the claim + /// however old the last confirmed beat is, and stops confirming the + /// moment the beat returns `Lost`. A parked beat that confirms on its + /// release refreshes the last confirmed instant and clears the flight, + /// so a timer that then stops leaves the check to the two-period rule. + #[tokio::test(start_paused = true)] + async fn a_beat_parked_on_the_write_lock_confirms_the_claim_until_it_returns_lost() { + let fake = Arc::new(Fake::default()); + fake.rows + .lock() + .unwrap() + .insert(arn(ACCOUNT), row(&arn(ACCOUNT), BackupStatus::Creating)); + // SQLite's shape: the park is the write lock's queue. + fake.parks_before_bound + .store(true, std::sync::atomic::Ordering::Release); + let claim: Arc = Arc::from(fake.claim_on(&arn(ACCOUNT), 7)); + let period = Duration::from_secs(10); + *fake.heartbeat_answer.lock().unwrap() = HeartbeatAnswer::Park; + let handle = job::heartbeat::HeartbeatHandle::start( + Arc::clone(&claim), + period, + job::heartbeat::HeartbeatMetric::None, + tokio::time::Instant::now(), + ); + // The first tick's beat parks at one period; at two and a half the + // claim's insert is too old for the two-period clause alone. + tokio::time::sleep(2 * period + period / 2).await; + assert_eq!( + fake.heartbeats.load(std::sync::atomic::Ordering::Acquire), + 0 + ); + assert!(!handle.lost().is_cancelled()); + assert!( + handle.confirms_the_claim(), + "a beat parked on the lock confirms the claim" + ); + tokio::time::sleep(10 * period).await; + assert!(handle.confirms_the_claim(), "however long it is parked"); + assert!(!handle.lost().is_cancelled()); + // The lock is released and the beat finds the claim lost. + *fake.heartbeat_answer.lock().unwrap() = HeartbeatAnswer::Lost; + fake.park_release.notify_one(); + tokio::time::sleep(Duration::from_millis(1)).await; + assert_eq!( + fake.heartbeats.load(std::sync::atomic::Ordering::Acquire), + 1 + ); + assert!(handle.lost().is_cancelled()); + assert!( + !handle.confirms_the_claim(), + "the loss is read before the flight" + ); + assert!(handle.reason().contains("the claim was lost")); + drop(handle); + + // The same park, released with a confirmation: the check passes + // from the fresh beat, and once the timer stops the flight is gone + // and the two-period rule refuses two and a half periods later. + *fake.heartbeat_answer.lock().unwrap() = HeartbeatAnswer::Park; + let handle = job::heartbeat::HeartbeatHandle::start( + claim, + period, + job::heartbeat::HeartbeatMetric::None, + tokio::time::Instant::now(), + ); + tokio::time::sleep(5 * period).await; + assert!(handle.confirms_the_claim()); + *fake.heartbeat_answer.lock().unwrap() = HeartbeatAnswer::Confirm; + fake.park_release.notify_one(); + tokio::time::sleep(Duration::from_millis(1)).await; + assert!(!handle.lost().is_cancelled()); + assert!(handle.confirms_the_claim()); + handle.stop_timer(); + tokio::time::sleep(2 * period + period / 2).await; + assert!( + !handle.confirms_the_claim(), + "no beat in flight and the last confirmed beat too old" + ); + assert!(!handle.lost().is_cancelled()); + } + + /// On a backend whose beat does not park before its bound (PostgreSQL, + /// MongoDB, Cassandra), a beat in flight confirms the claim only within + /// one period of its start: a process stopped mid-beat (SIGSTOP, a + /// paused VM) and resumed keeps the beat in flight until its timer runs + /// again and raises the loss, and in that window another owner may hold + /// the row, so the check refuses it; the same beat on a backend whose + /// beat parks (SQLite) confirms the claim for as long as it waits. + #[tokio::test(start_paused = true)] + async fn a_beat_in_flight_past_its_bound_confirms_the_claim_only_where_the_beat_parks() { + let fake = Arc::new(Fake::default()); + fake.rows + .lock() + .unwrap() + .insert(arn(ACCOUNT), row(&arn(ACCOUNT), BackupStatus::Creating)); + let period = Duration::from_secs(10); + for parks in [false, true] { + fake.parks_before_bound + .store(parks, std::sync::atomic::Ordering::Release); + let claim: Arc = Arc::from(fake.claim_on(&arn(ACCOUNT), 7)); + // `Park` holds the beat outside the bound: the stop, on a backend + // that does not park, or the lock's queue on one that does. + *fake.heartbeat_answer.lock().unwrap() = HeartbeatAnswer::Park; + // The claim's insert is old enough that the two-period clause + // alone refuses from the first tick on. + let handle = job::heartbeat::HeartbeatHandle::start( + claim, + period, + job::heartbeat::HeartbeatMetric::None, + tokio::time::Instant::now() - 2 * period, + ); + // The first tick's beat starts at one period; half a period in it + // is within its bound on every backend. + tokio::time::sleep(period + period / 2).await; + assert!(!handle.lost().is_cancelled()); + assert!( + handle.confirms_the_claim(), + "parks={parks}: a beat within its bound confirms the claim" + ); + // One and a half periods into the beat: past its bound. + tokio::time::sleep(period).await; + assert!(!handle.lost().is_cancelled(), "the timer has not run again"); + assert_eq!( + handle.confirms_the_claim(), + parks, + "parks={parks}: a beat past its bound confirms the claim only where it parks" + ); + // The beat returns Lost: the loss is read before the flight. + *fake.heartbeat_answer.lock().unwrap() = HeartbeatAnswer::Lost; + fake.park_release.notify_one(); + tokio::time::sleep(Duration::from_millis(1)).await; + assert!(handle.lost().is_cancelled()); + assert!(!handle.confirms_the_claim()); + drop(handle); + *fake.heartbeat_answer.lock().unwrap() = HeartbeatAnswer::Confirm; + } + } +} + +/// A restore of a backup the catalog index has a row for checks the target +/// name once the row is known to exist and AVAILABLE, before the backup's +/// in-use rule and before the store checks, the service's order (measured on +/// DynamoDB: an in-use backup into an existing table is +/// `TableAlreadyExistsException`, a backup into its own CREATING target +/// `TableInUseException`): an existing table of the name answers +/// `TableAlreadyExistsException` though the backup is in use, a CREATING +/// restore target `TableInUseException` naming its source backup, and a free +/// name reaches the in-use rule and then the manifest read (here +/// `BackupNotFoundException`: the store holds nothing under the row's +/// prefix). A row that is not AVAILABLE answers for itself before the name +/// is read. +#[tokio::test] +async fn a_catalog_row_restore_checks_the_target_name_after_the_row_and_before_its_in_use_rule() { + // The row's store is the configured, healthy one, so a free name reaches + // the store after the in-use rule. + let mem = Arc::new(MemStore::default()); + let runtime = runtime_on(&mem, store()); + let fake = Arc::new(fake_with_row(BackupStatus::Available)); + fake.in_use + .lock() + .unwrap() + .insert(arn(ACCOUNT), "Elsewhere".to_owned()); + let limits = LimitsConfig::default(); + let body = restore_body(json!({})); + let restore_on = async |fake: &Arc| { + restore_table_from_backup(fake, &runtime, ACCOUNT, REGION, &limits, &body).await }; - let msg = validation_message( - create_backup_table_name(&json!({ "TableName": "elevenchars" }), ACCOUNT, &lowered) - .unwrap_err(), + + fake.tables.lock().unwrap().insert( + "MusicRestored".to_owned(), + TableDescription { + table_name: "MusicRestored".to_owned(), + table_status: TableStatus::Active, + ..Default::default() + }, ); - assert!(msg.contains("length less than or equal to 10"), "{msg}"); - let msg = validation_message( - validate_name_member(&"b".repeat(300), "backupName", BACKUP_NAME_BOUNDS).unwrap_err(), + let err = restore_on(&fake).await.unwrap_err(); + assert!( + matches!(err, DynamoDbError::TableAlreadyExistsException(_)), + "an in-use backup into an existing table: {err:?}" ); - assert!(msg.contains("length less than or equal to 255"), "{msg}"); -} -/// Measured on DynamoDB: `CreateBackup` accepts the -/// table's ARN as `TableName`. Another account's is refused as a backup ARN -/// of another account is; another region's resolves (both ExtendDB's choice, -/// not measured). -#[test] -fn create_backup_table_name_may_be_the_tables_arn() { - let own = format!("arn:aws:dynamodb:{REGION}:{ACCOUNT}:table/Music"); - assert_eq!( - create_backup_table_name(&json!({ "TableName": own }), ACCOUNT, &limits()).unwrap(), - "Music" + fake.tables.lock().unwrap().insert( + "MusicRestored".to_owned(), + TableDescription { + table_name: "MusicRestored".to_owned(), + table_status: TableStatus::Creating, + restore_summary: Some(RestoreSummary { + source_backup_arn: Some(arn(ACCOUNT)), + source_table_arn: None, + restore_date_time: 0.0, + restore_in_progress: true, + }), + ..Default::default() + }, ); - let other_region = format!("arn:aws:dynamodb:eu-west-1:{ACCOUNT}:table/Music"); - assert_eq!( - create_backup_table_name(&json!({ "TableName": other_region }), ACCOUNT, &limits()) - .unwrap(), - "Music" + let err = restore_on(&fake).await.unwrap_err(); + assert!( + matches!(err, DynamoDbError::TableInUseException(ref m) if m.contains(&arn(ACCOUNT))), + "{err:?}" ); - let foreign = format!("arn:aws:dynamodb:{REGION}:999999999999:table/Music"); - assert!(matches!( - create_backup_table_name(&json!({ "TableName": foreign }), ACCOUNT, &limits()).unwrap_err(), - DynamoDbError::AccessDeniedException(_) - )); - for bad in [ - format!("arn:aws:dynamodb:{REGION}:{ACCOUNT}:table/Music/backup/01791976431438-73d8d5bc"), - format!("arn:aws:dynamodb:{REGION}:{ACCOUNT}:table/Music/index/i"), - format!("arn:aws:dynamodb:{REGION}:{ACCOUNT}:index/Music"), - format!("arn:aws:s3:{REGION}:{ACCOUNT}:table/Music"), - format!("arn:aws-cn:dynamodb:{REGION}:{ACCOUNT}:table/Music"), - format!("arn:aws:dynamodb::{ACCOUNT}:table/Music"), - format!("arn:aws:dynamodb:{REGION}:{ACCOUNT}"), - "arn:".to_owned(), + + fake.tables.lock().unwrap().remove("MusicRestored"); + let err = restore_on(&fake).await.unwrap_err(); + assert!( + matches!(err, DynamoDbError::BackupInUseException(ref m) + if *m == extenddb_storage::backup_catalog::backup_in_use_message(&arn(ACCOUNT))), + "a free name reaches the in-use rule: {err:?}" + ); + fake.in_use.lock().unwrap().clear(); + assert!( + mem.gets().is_empty(), + "no store read before the in-use rule" + ); + let err = restore_on(&fake).await.unwrap_err(); + assert!( + matches!(err, DynamoDbError::BackupNotFoundException(_)), + "{err:?}" + ); + assert!(!mem.gets().is_empty(), "the free name reached the store"); + + // A row that is not AVAILABLE: its own answer, the name unread. A + // CREATING row is the backup being created (not measured on the service: + // its backups were AVAILABLE 4 ms after CreateBackup answered). + for (status, expect) in [ + (BackupStatus::Deleted, "not found"), + (BackupStatus::Creating, "being created"), ] { - let msg = validation_message( - create_backup_table_name(&json!({ "TableName": bad }), ACCOUNT, &limits()).unwrap_err(), + let fake = Arc::new(fake_with_row(status)); + fake.tables.lock().unwrap().insert( + "MusicRestored".to_owned(), + TableDescription { + table_name: "MusicRestored".to_owned(), + table_status: TableStatus::Active, + ..Default::default() + }, ); - assert_eq!( - msg, "Invalid Request: TableName is not a valid table name or table ARN", - "{bad}" + let err = restore_on(&fake).await.unwrap_err(); + match expect { + "not found" => assert!( + matches!(err, DynamoDbError::BackupNotFoundException(_)), + "{status:?}: {err:?}" + ), + _ => assert!( + matches!(err, DynamoDbError::BackupInUseException(ref m) + if *m == format!("Backup is being created: {}", arn(ACCOUNT))), + "{status:?}: {err:?}" + ), + } + assert!( + fake.table_reads.lock().unwrap().is_empty(), + "{status:?}: the name is not read" ); } - // The name part is checked as a name. - let short = format!("arn:aws:dynamodb:{REGION}:{ACCOUNT}:table/ab"); - let msg = validation_message( - create_backup_table_name(&json!({ "TableName": short }), ACCOUNT, &limits()).unwrap_err(), +} + +/// A `TableName` ARN of another account in another region is +/// `AccessDeniedException` on CreateBackup and ListBackups: the account is +/// checked before the region (measured on DynamoDB, twice each). +#[tokio::test] +async fn another_accounts_arn_in_another_region_is_access_denied_before_the_region_check() { + let fake = Fake::default(); + let arn = "arn:aws:dynamodb:eu-west-1:210987654321:table/Music"; + let err = create_backup_table_name( + &json!({ "TableName": arn, "BackupName": "nightly" }), + ACCOUNT, + REGION, + &LimitsConfig::default(), + ) + .unwrap_err(); + assert!( + matches!(err, DynamoDbError::AccessDeniedException(ref m) if m == "Access is denied"), + "{err:?}" ); + let err = list_backups( + &fake, + ACCOUNT, + REGION, + &LimitsConfig::default(), + &json!({ "TableName": arn }), + ) + .await + .unwrap_err(); assert!( - msg.contains("'tableName'") && msg.contains("length greater than or equal to 3"), - "{msg}" + matches!(err, DynamoDbError::AccessDeniedException(ref m) if m == "Access is denied"), + "{err:?}" ); - // Missing: the null-member text. - let msg = - validation_message(create_backup_table_name(&json!({}), ACCOUNT, &limits()).unwrap_err()); - assert!(msg.contains("Value null at 'tableName'"), "{msg}"); } -/// Measured on DynamoDB: `CreateBackup` on a table that -/// does not exist or is still `CREATING` is `TableNotFoundException` -/// `Table not found: `, whatever the backend's own payload says. -#[test] -fn create_backup_on_a_missing_or_creating_table_is_table_not_found() { - for storage in [ - // PostgreSQL's payload and the bare name the other backends give. - StorageError::TableNotFound("Table not found: Music".to_owned()), - StorageError::TableNotFound("Music".to_owned()), - StorageError::TableNotActive("Music".to_owned()), - ] { - let what = format!("{storage:?}"); - let err = create_backup_storage_err_to_dynamo(storage, "Music"); - match &err { - DynamoDbError::TableNotFoundException(m) => { - assert_eq!(m, "Table not found: Music", "{what}") - } - other => panic!("{what}: expected TableNotFoundException, got {other:?}"), +// --- backup sync (`sync`) ------------------------------------------------------ + +/// A backup ARN of `store()`'s account and table with a 14-digit id that +/// carries an instant, one per `n`. +fn sync_arn(n: u32) -> String { + format!("arn:aws:dynamodb:{REGION}:{ACCOUNT}:table/Music/backup/0179197643{n:04}-0000{n:04}") +} + +/// The objects of a complete backup of `arn`, keys relative to the root: the +/// golden manifest's data files at their recorded sizes and the four +/// documents `BackupDocuments::build` writes, the source table renamed to +/// `arn`'s table, account, region and partition, the name `sync-`. +fn backup_objects(arn: &str) -> Vec<(String, Vec)> { + use extenddb_core::backup_format::{BackupDocuments, ExtenddbManifest, ManifestHeader}; + let golden = ExtenddbManifest::from_json_bytes(include_bytes!( + "../../../core/testdata/extenddb-manifest/v1.json" + )) + .unwrap(); + let parsed = BackupArn::parse(arn).unwrap(); + let mut source_table = golden.source_table.clone(); + source_table.table_name = parsed.table_name().to_owned(); + source_table.table_arn = format!( + "arn:{}:dynamodb:{}:{}:table/{}", + parsed.partition(), + parsed.region(), + parsed.account_id(), + parsed.table_name() + ); + let docs = BackupDocuments::build( + ManifestHeader { + backup_arn: parsed.clone(), + backup_name: format!("sync-{}", parsed.backup_id()), + source_backend: golden.source_backend.clone(), + source_extenddb_version: golden.source_extenddb_version.clone(), + snapshot: golden.snapshot.clone(), + item_order: golden.item_order, + source_table, + }, + golden + .data_files + .iter() + .cloned() + .map(|entry| (entry, None)) + .collect(), + "", + &store().store_root, + ) + .unwrap(); + let prefix = parsed.store_prefix(); + let mut objects: Vec<(String, Vec)> = golden + .data_files + .iter() + .map(|file| { + ( + format!("{prefix}{}", file.key), + vec![0u8; usize::try_from(file.bytes).unwrap()], + ) + }) + .collect(); + for (name, bytes) in docs.documents() { + objects.push((format!("{prefix}{name}"), bytes.to_vec())); + } + objects +} + +/// Write the objects of a backup of `arn` into `mem` with its manifest +/// changed by `edit` and a sidecar that matches the changed manifest: a +/// manifest whose digest checks and whose content the reader judges. +fn put_backup_with_manifest(mem: &MemStore, arn: &str, edit: impl Fn(&mut Value)) { + use sha2::Digest as _; + for (key, bytes) in backup_objects(arn) { + if key.ends_with("extenddb-manifest.sha256") { + continue; + } + if key.ends_with("extenddb-manifest.json") { + let mut manifest: Value = serde_json::from_slice(&bytes).unwrap(); + edit(&mut manifest); + let edited = serde_json::to_vec(&manifest).unwrap(); + let sidecar = format!( + "{} extenddb-manifest.json\n", + hex::encode(sha2::Sha256::digest(&edited)) + ); + mem.insert(&key.replace(".json", ".sha256"), sidecar.as_bytes()); + mem.insert(&key, &edited); + } else { + mem.insert(&key, &bytes); } - assert_eq!( - err.full_error_type(), - "com.amazonaws.dynamodb.v20120810#TableNotFoundException" - ); } - // Other errors keep the generic mapping. - assert!(matches!( - create_backup_storage_err_to_dynamo(StorageError::LimitExceeded("x".to_owned()), "Music"), - DynamoDbError::LimitExceededException(_) - )); - // The data plane keeps ResourceNotFoundException for a missing table. - assert!(matches!( - storage_err_to_dynamo(StorageError::TableNotFound("Music".to_owned())), - DynamoDbError::ResourceNotFoundException(_) - )); } -// --- DeleteBackup ------------------------------------------------------------- +/// Write a complete backup of `arn` into `mem`. +fn put_backup(mem: &MemStore, arn: &str) { + for (key, bytes) in backup_objects(arn) { + mem.insert(&key, &bytes); + } +} -/// Measured on DynamoDB: `DeleteBackup` answers the full -/// DescribeBackup description with `BackupStatus` `DELETED`, including the -/// required `SourceTableDetails.ProvisionedThroughput` and the -/// `SourceTableFeatureDetails` the backend's row alone does not carry. +fn prefix_of(arn: &str) -> String { + BackupArn::parse(arn).unwrap().store_prefix() +} + +/// A sync of `mem` as `identity`, in the tests' region. +fn sync_target<'a>(mem: &'a MemStore, identity: &'a StoreIdentity) -> super::sync::SyncTarget<'a> { + super::sync::SyncTarget { + store: mem, + identity, + metrics: None, + region: Some(REGION), + stop: None, + } +} + +/// The reasons of a report's skipped prefixes, by prefix. +fn skipped_by_prefix(report: &super::sync::SyncReport) -> HashMap { + report + .skipped + .iter() + .map(|s| (s.prefix.clone(), s.reason.clone())) + .collect() +} + +/// A prefix with a verified manifest and no row gets an AVAILABLE row with +/// the values CreateBackup's AVAILABLE write leaves: the manifest's name, +/// ARN, size, item total and `source_table`, the configured store's +/// identity, `created_at` the id's instant, unowned, `completed_at` and +/// `heartbeat_at` the catalog's clock. A second sync finds the row and +/// skips the prefix; a reconciler cycle then leaves the row as it is, as it +/// leaves a created one. #[tokio::test] -async fn delete_backup_answers_the_full_description_as_deleted() { - for def in [definition("PROVISIONED"), definition("PAY_PER_REQUEST")] { - let on_demand = def.billing_mode == "PAY_PER_REQUEST"; - let fake = fake_with_backup("AVAILABLE", source(Some(def))); - let described = describe(&fake).await; - let deleted = delete_backup(&fake, ACCOUNT, REGION, &own(&arn(ACCOUNT))) - .await - .unwrap(); - let deleted = serde_json::to_value(deleted).unwrap(); - assert_eq!(deleted["BackupDetails"]["BackupStatus"], "DELETED"); - assert_eq!( - deleted["SourceTableDetails"]["ProvisionedThroughput"], - if on_demand { - json!({ "ReadCapacityUnits": 0, "WriteCapacityUnits": 0 }) - } else { - json!({ "ReadCapacityUnits": 7, "WriteCapacityUnits": 8 }) +async fn sync_adopts_a_verified_prefix_with_the_row_a_create_writes() { + let mem = Arc::new(MemStore::default()); + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + let fake = Arc::new(Fake::default()); + let adopted = sync_arn(1); + put_backup(&mem, &adopted); + let identity = store(); + + let report = super::sync::sync(&*fake, sync_target(&mem, &identity)) + .await + .unwrap(); + assert_eq!( + report.adopted, + [super::sync::AdoptedBackup { + backup_arn: adopted.clone(), + foreign: None, + }] + ); + assert!(report.skipped.is_empty(), "{:?}", report.skipped); + assert!(report.missing_manifests.is_empty()); + assert_eq!(report.stopped, None); + let manifest = stored_manifest(&mem, &adopted); + let parsed = BackupArn::parse(&adopted).unwrap(); + let row = fake.rows.lock().unwrap()[&adopted].clone(); + assert_eq!(row.status, BackupStatus::Available); + assert_eq!(row.backup_arn, adopted); + assert_eq!(row.account_id, parsed.account_id()); + assert_eq!(row.table_name, parsed.table_name()); + assert_eq!(row.backup_id, parsed.backup_id()); + assert_eq!(row.backup_name, manifest.backup_name); + assert_eq!(row.backup_name, format!("sync-{}", parsed.backup_id())); + assert_eq!(row.store, identity); + assert_eq!(row.size_bytes, manifest.backup_size_bytes); + assert_eq!(row.item_count, 100_000); + assert_eq!( + Some(row.created_at), + extenddb_core::backup_format::backup_creation_date_time(parsed.backup_id()) + ); + assert_eq!( + row.source_table_details_json, + serde_json::to_value(&manifest.source_table).unwrap() + ); + assert_eq!(row.owner_generation, UNOWNED_GENERATION); + assert!(row.completed_at.is_some()); + assert_eq!(row.failure_reason, None); + + let again = super::sync::sync(&*fake, sync_target(&mem, &identity)) + .await + .unwrap(); + assert!(again.adopted.is_empty()); + assert_eq!( + skipped_by_prefix(&again), + HashMap::from([( + prefix_of(&adopted), + super::sync::SkipReason::HasRow { + status: BackupStatus::Available, + store_id: "store-1".to_owned(), + other_table: None, } - ); - assert_eq!( - deleted["SourceTableFeatureDetails"]["GlobalSecondaryIndexes"][0]["IndexName"], - "gsi1" - ); - assert_eq!( - deleted["SourceTableFeatureDetails"]["LocalSecondaryIndexes"][0]["IndexName"], - "lsi1" - ); - // Everything but the status is what DescribeBackup said. - let mut expected = described.clone(); - expected["BackupDetails"]["BackupStatus"] = json!("DELETED"); - assert_eq!(deleted, expected); - // The backup is gone. - assert!(matches!( - delete_backup(&fake, ACCOUNT, REGION, &own(&arn(ACCOUNT))) - .await - .unwrap_err(), - DynamoDbError::BackupNotFoundException(_) - )); - assert_eq!( - fake.source_reads(), - 3, - "one read by DescribeBackup, one per delete attempt" - ); + )]) + ); + assert_eq!(fake.insert_attempts.lock().unwrap().len(), 1); + + let runtime = runtime_on(&mem, store()); + let cycle = cycle_on(&fake, &runtime).await; + assert_eq!(cycle.store_healthy, Some(true)); + assert_eq!( + fake.rows.lock().unwrap()[&adopted], + row, + "the cycle left it" + ); +} + +/// An ARN whose region is not the server's is adopted, and the report (and +/// the log) says so; with no region known, no region is compared. +#[tokio::test] +async fn sync_adopts_a_backup_of_another_region_and_says_so() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let other = + format!("arn:aws:dynamodb:eu-west-1:{ACCOUNT}:table/Music/backup/01791976430007-00000007"); + put_backup(&mem, &other); + let identity = store(); + let report = super::sync::sync(&*fake, sync_target(&mem, &identity)) + .await + .unwrap(); + assert_eq!(report.adopted.len(), 1); + assert_eq!(report.adopted[0].backup_arn, other); + let note = report.adopted[0].foreign.clone().unwrap(); + assert!(note.contains("eu-west-1"), "{note}"); + assert_eq!(fake.rows.lock().unwrap()[&other].backup_arn, other); + + let fake = Arc::new(Fake::default()); + let unknown_region = super::sync::SyncTarget { + region: None, + ..sync_target(&mem, &identity) + }; + let report = super::sync::sync(&*fake, unknown_region).await.unwrap(); + assert_eq!(report.adopted[0].foreign, None); +} + +/// Every prefix 4(g) says sync never adopts is skipped with its reason, and +/// the run goes on to adopt the one complete prefix among them: no manifest, +/// a manifest without its sidecar, a sidecar that does not match, a data +/// file missing, a data file of another size, a manifest naming another +/// backup than the path, a manifest the store fails to serve, a manifest +/// whose sidecar matches and that the reader refuses (a `source_table` of +/// another table, a creation time that is not the id's instant, a data-file +/// key that leaves the prefix, a size total that is not the files' sum), +/// the reserved `changelog` and `exports/` components, an id of an earlier +/// build's shape, and a complete prefix with a row of each status. The +/// manifest of a prefix with a row is never read: the row check comes +/// first. +#[tokio::test] +async fn sync_skips_every_prefix_it_must_not_adopt() { + use super::sync::SkipReason; + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let identity = store(); + let manifest_name = "extenddb-manifest.json"; + let sidecar_name = "extenddb-manifest.sha256"; + + let good = sync_arn(1); + put_backup(&mem, &good); + let none = sync_arn(2); + mem.insert(&format!("{}data/000001.json.gz", prefix_of(&none)), b"x"); + let no_sidecar = sync_arn(3); + for (key, bytes) in backup_objects(&no_sidecar) { + if !key.ends_with(sidecar_name) { + mem.insert(&key, &bytes); + } } + let bad_sidecar = sync_arn(4); + for (key, bytes) in backup_objects(&bad_sidecar) { + if key.ends_with(sidecar_name) { + mem.insert( + &key, + format!("{} {manifest_name}\n", "0".repeat(64)).as_bytes(), + ); + } else { + mem.insert(&key, &bytes); + } + } + let missing_file = sync_arn(5); + let mut objects = backup_objects(&missing_file); + let dropped = objects + .iter() + .position(|(k, _)| k.contains("/data/")) + .unwrap(); + objects.remove(dropped); + for (key, bytes) in objects { + mem.insert(&key, &bytes); + } + let short_file = sync_arn(6); + for (key, mut bytes) in backup_objects(&short_file) { + if key.contains("/data/") && bytes.len() > 1 { + bytes.pop(); + } + mem.insert(&key, &bytes); + } + // The documents and files of backup 8, under backup 7's prefix. + let mismatched = sync_arn(7); + for (key, bytes) in backup_objects(&sync_arn(8)) { + let moved = key.replacen(&prefix_of(&sync_arn(8)), &prefix_of(&mismatched), 1); + mem.insert(&moved, &bytes); + } + let unreadable = sync_arn(9); + put_backup(&mem, &unreadable); + *mem.fail_gets_under.lock().unwrap() = Some(prefix_of(&unreadable)); + let other_source = sync_arn(10); + put_backup_with_manifest(&mem, &other_source, |m| { + m["source_table"]["table_name"] = "Other".into(); + }); + let other_instant = sync_arn(11); + put_backup_with_manifest(&mem, &other_instant, |m| { + m["backup_creation_date_time"] = 1.5.into(); + }); + let traversal = sync_arn(12); + put_backup_with_manifest(&mem, &traversal, |m| { + m["data_files"][0]["key"] = "../../../../x/data/000001.json.gz".into(); + }); + let other_size = sync_arn(13); + put_backup_with_manifest(&mem, &other_size, |m| { + m["backup_size_bytes"] = 7.into(); + }); + mem.insert(&format!("{ACCOUNT}/Music/changelog/000001.json.gz"), b"x"); + mem.insert("exports/01791976430001/manifest-summary.json", b"{}"); + mem.insert( + &format!("{ACCOUNT}/Music/1791976431438/{manifest_name}"), + b"{}", + ); + let mut with_rows = Vec::new(); + for (n, status) in [ + (20, BackupStatus::Creating), + (21, BackupStatus::Deleting), + (22, BackupStatus::Deleted), + (23, BackupStatus::Failed), + (24, BackupStatus::Available), + ] { + let a = sync_arn(n); + put_backup(&mem, &a); + fake.rows.lock().unwrap().insert(a.clone(), row(&a, status)); + with_rows.push((a, status)); + } + let rows_before: Vec = fake.rows.lock().unwrap().values().cloned().collect(); - // A backup whose definition this build cannot read, or that does not - // parse, is still deleted, described without feature details. - for err in [ - StorageError::Unsupported("version 9".to_owned()), - StorageError::Internal("does not parse".to_owned()), + let report = super::sync::sync(&*fake, sync_target(&mem, &identity)) + .await + .unwrap(); + assert_eq!( + report + .adopted + .iter() + .map(|a| a.backup_arn.clone()) + .collect::>(), + std::slice::from_ref(&good) + ); + let skipped = skipped_by_prefix(&report); + let incomplete = |arn: &str| match &skipped[&prefix_of(arn)] { + SkipReason::Incomplete(why) => why.clone(), + other => panic!("{arn}: {other:?}"), + }; + assert_eq!(incomplete(&none), "no manifest"); + assert_eq!(incomplete(&no_sidecar), "a manifest without its sidecar"); + assert!( + incomplete(&bad_sidecar).starts_with("the sidecar does not match"), + "{}", + incomplete(&bad_sidecar) + ); + assert!( + incomplete(&missing_file).contains("is not in the store"), + "{}", + incomplete(&missing_file) + ); + assert!( + incomplete(&short_file).contains("bytes in the store"), + "{}", + incomplete(&short_file) + ); + assert!( + matches!(&skipped[&prefix_of(&mismatched)], SkipReason::ArnMismatch(why) if why.contains(&sync_arn(8))), + "{:?}", + skipped[&prefix_of(&mismatched)] + ); + assert!( + matches!(&skipped[&prefix_of(&unreadable)], SkipReason::Store(e) if e.contains("get refused")), + "{:?}", + skipped[&prefix_of(&unreadable)] + ); + for (refused, says) in [ + (&other_source, "source_table.table_name is \"Other\""), + (&other_instant, "does not match the instant in backup_arn"), + (&traversal, "must start with the data/ prefix"), + ( + &other_size, + "backup_size_bytes is 7 but the data files total", + ), ] { - let what = format!("{err:?}"); - let fake = fake_with_backup("AVAILABLE", source(Some(definition("PROVISIONED")))); - *fake.source_error.lock().unwrap() = Some(err); - let deleted = delete_backup(&fake, ACCOUNT, REGION, &own(&arn(ACCOUNT))) - .await - .unwrap_or_else(|e| panic!("{what}: {e:?}")); - assert_eq!(deleted.backup_details.backup_status, "DELETED", "{what}"); - assert!(deleted.source_table_feature_details.is_none(), "{what}"); + let why = incomplete(refused); assert!( - deleted - .source_table_details - .provisioned_throughput - .is_some(), - "{what}" + why.starts_with("the manifest is refused: ") && why.contains(says), + "{refused}: {why}" + ); + } + assert_eq!( + skipped[&format!("{ACCOUNT}/Music/changelog/")], + SkipReason::Reserved("changelog") + ); + assert_eq!(skipped["exports/"], SkipReason::Reserved("exports")); + assert!( + matches!(&skipped[&format!("{ACCOUNT}/Music/1791976431438/")], SkipReason::NotABackup(why) if why.contains("earlier build")), + "{:?}", + skipped[&format!("{ACCOUNT}/Music/1791976431438/")] + ); + for (a, status) in &with_rows { + assert_eq!( + skipped[&prefix_of(a)], + SkipReason::HasRow { + status: *status, + store_id: "store-1".to_owned(), + other_table: None, + }, + "{a}" ); assert!( - fake.descriptions.lock().unwrap().is_empty(), - "{what}: not deleted" + !mem.gets().iter().any(|k| k.starts_with(&prefix_of(a))), + "{a}: read before its row was checked" ); } - // A failure to reach the database propagates, and nothing is deleted. - let fake = fake_with_backup("AVAILABLE", source(Some(definition("PROVISIONED")))); - *fake.source_error.lock().unwrap() = Some(StorageError::Connection("refused".to_owned())); - assert!(matches!( - delete_backup(&fake, ACCOUNT, REGION, &own(&arn(ACCOUNT))) - .await - .unwrap_err(), - DynamoDbError::InternalServerError(_) - )); - assert_eq!(fake.descriptions.lock().unwrap().len(), 1); + assert_eq!(report.skipped.len(), 19, "{:?}", report.skipped); + // One insert, the good prefix's; every row as it was. + assert_eq!( + fake.insert_attempts + .lock() + .unwrap() + .iter() + .map(|r| r.backup_arn.clone()) + .collect::>(), + std::slice::from_ref(&good) + ); + let rows = fake.rows.lock().unwrap(); + for before in &rows_before { + assert_eq!(rows[&before.backup_arn], *before); + } + assert_eq!(rows.len(), rows_before.len() + 1); } -#[test] -fn create_backup_refuses_a_table_with_vector_indexes() { - let plain = TableDescription { - table_name: "T".to_owned(), - ..Default::default() - }; - assert!(refuse_backup_of_vector_table(&plain).is_ok()); - - let with_vectors: TableDescription = serde_json::from_value(json!({ - "TableName": "T", - "KeySchema": [], - "AttributeDefinitions": [], - "TableStatus": "ACTIVE", - "CreationDateTime": 0.0, - "TableSizeBytes": 0, - "ItemCount": 0, - "TableArn": "", - "TableId": "", - "ProvisionedThroughput": { "ReadCapacityUnits": 0, "WriteCapacityUnits": 0, - "NumberOfDecreasesToday": 0 }, - "DeletionProtectionEnabled": false, - "VectorIndexes": [{ - "IndexName": "v", - "VectorAttribute": { "AttributeName": "emb" }, - "Dimensions": 3, - "DistanceFunction": "COSINE", - "IndexStatus": "ACTIVE", - "IndexSizeBytes": 0, - "ItemCount": 0, - "IndexArn": "" - }] - })) - .unwrap(); - let msg = validation_message(refuse_backup_of_vector_table(&with_vectors).unwrap_err()); - assert!(msg.starts_with("table T has 1 vector index(es)"), "{msg}"); +/// The report names the AVAILABLE rows of the configured store whose +/// manifest the listing does not hold (no prefix at all, or a prefix +/// without its manifest), not one whose prefix is complete, nor a row of +/// the root under another store id; that row's prefix is skipped (the row +/// check covers every id of the root). A prefix whose backup id a row of +/// another root holds is skipped naming that store. Nothing is modified +/// without `--prune-missing`. +#[tokio::test] +async fn sync_reports_missing_manifests_and_rows_of_other_stores_and_changes_none() { + use super::sync::SkipReason; + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let identity = store(); + let no_prefix = sync_arn(1); + let no_manifest = sync_arn(2); + let complete = sync_arn(3); + let other_id = sync_arn(4); + let other_root = sync_arn(5); + for a in [&no_prefix, &no_manifest, &complete, &other_id] { + fake.rows + .lock() + .unwrap() + .insert(a.clone(), row(a, BackupStatus::Available)); + } + fake.rows + .lock() + .unwrap() + .get_mut(&other_id) + .unwrap() + .store + .store_id = "store-0".to_owned(); + let mut elsewhere = row(&other_root, BackupStatus::Available); + elsewhere.store.store_root = "/srv/elsewhere".to_owned(); + fake.rows + .lock() + .unwrap() + .insert(other_root.clone(), elsewhere.clone()); + mem.insert( + &format!("{}data/000001.json.gz", prefix_of(&no_manifest)), + b"x", + ); + // A key that ends in the manifest's name below the prefix is not the + // prefix's manifest: the row is still reported. + mem.insert( + &format!("{}data/x-extenddb-manifest.json", prefix_of(&no_manifest)), + b"{}", + ); + for a in [&complete, &other_id, &other_root] { + put_backup(&mem, a); + } + let rows_before: Vec = fake.rows.lock().unwrap().values().cloned().collect(); - // The restore's refusal reads the same after the subject. - let mut def = definition("PAY_PER_REQUEST"); - def.vector_index_names.push("v".to_owned()); - let restore_msg = validation_message(storage_err_to_dynamo( - def.ensure_restorable(&arn(ACCOUNT)).unwrap_err(), - )); + let report = super::sync::sync(&*fake, sync_target(&mem, &identity)) + .await + .unwrap(); + assert!(report.adopted.is_empty(), "{:?}", report.adopted); + let mut missing: Vec = report + .missing_manifests + .iter() + .map(|r| r.backup_arn.clone()) + .collect(); + missing.sort(); + assert_eq!(missing, [no_prefix.clone(), no_manifest.clone()]); + let skipped = skipped_by_prefix(&report); assert_eq!( - msg.trim_start_matches("table T"), - restore_msg.trim_start_matches(&format!("backup {}", arn(ACCOUNT))) + skipped[&prefix_of(&other_id)], + SkipReason::HasRow { + status: BackupStatus::Available, + store_id: "store-0".to_owned(), + other_table: None, + } + ); + assert!( + matches!(&skipped[&prefix_of(&other_root)], SkipReason::RowExists(what) if what.contains("of another store") && what.contains("/srv/elsewhere")), + "{:?}", + skipped[&prefix_of(&other_root)] ); + let rows = fake.rows.lock().unwrap(); + assert_eq!(rows.len(), rows_before.len()); + for before in &rows_before { + assert_eq!(rows[&before.backup_arn], *before); + } } -// --- DescribeBackup ----------------------------------------------------------- +/// The insert-if-absent decides against what the read of the rows did not +/// see, and each case leaves the row it met: a CreateBackup on another +/// instance whose CREATING row landed between the read and the insert; a +/// row the catalog holds that its listings do not return (Cassandra's +/// insert that ended between its two writes); a row of the backup id under +/// another account. +#[tokio::test] +async fn an_insert_that_meets_a_row_leaves_it_and_says_which() { + use super::sync::SkipReason; + let identity = store(); + let reason = |report: &super::sync::SyncReport, arn: &str| match &skipped_by_prefix(report) + [&prefix_of(arn)] + { + SkipReason::RowExists(what) => what.clone(), + other => panic!("{arn}: {other:?}"), + }; -fn backup_description(status: &str, billing: &str) -> BackupDescription { - BackupDescription { - backup_details: BackupDetails { - backup_arn: arn(ACCOUNT), - backup_name: "b".to_owned(), - backup_status: status.to_owned(), - backup_type: "USER".to_owned(), - backup_size_bytes: 30, - backup_creation_date_time: CREATED, - }, - source_table_details: SourceTableDetails { - table_name: "Music".to_owned(), - table_id: "tid".to_owned(), - table_arn: source_arn(), - billing_mode: Some(billing.to_owned()), - ..Default::default() - }, - source_table_feature_details: None, - } + // A CreateBackup racing the sync. + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let racing = sync_arn(1); + put_backup(&mem, &racing); + let creating = row(&racing, BackupStatus::Creating); + fake.lands_before_insert + .lock() + .unwrap() + .push(creating.clone()); + let report = super::sync::sync(&*fake, sync_target(&mem, &identity)) + .await + .unwrap(); + assert!(report.adopted.is_empty()); + let what = reason(&report, &racing); + assert!(what.contains("(CREATING) landed after"), "{what}"); + assert_eq!(fake.rows.lock().unwrap()[&racing], creating); + + // A row the listings do not return. + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let unlisted = sync_arn(2); + put_backup(&mem, &unlisted); + let available = row(&unlisted, BackupStatus::Available); + fake.rows + .lock() + .unwrap() + .insert(unlisted.clone(), available.clone()); + fake.unlisted.lock().unwrap().insert(unlisted.clone()); + let report = super::sync::sync(&*fake, sync_target(&mem, &identity)) + .await + .unwrap(); + assert!(report.adopted.is_empty()); + let what = reason(&report, &unlisted); + assert!(what.contains("listings do not return"), "{what}"); + assert_eq!(fake.rows.lock().unwrap()[&unlisted], available); + + // The backup id under another account. + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let ours = sync_arn(3); + put_backup(&mem, &ours); + let theirs = ours.replace(ACCOUNT, "210987654321"); + let other_account = row(&theirs, BackupStatus::Available); + fake.rows + .lock() + .unwrap() + .insert(theirs.clone(), other_account.clone()); + let report = super::sync::sync(&*fake, sync_target(&mem, &identity)) + .await + .unwrap(); + assert!(report.adopted.is_empty()); + let what = reason(&report, &ours); + assert!(what.contains("under another account"), "{what}"); + assert_eq!(fake.rows.lock().unwrap().len(), 1); + assert_eq!(fake.rows.lock().unwrap()[&theirs], other_account); } -fn fake_with_backup(status: &str, src: BackupRestoreSource) -> Fake { - let billing = src.billing_mode.clone(); - let fake = Fake::with_source(src); - fake.descriptions +/// A listing that fails stops the sync with no row written, and so does a +/// read of the rows that fails. +#[tokio::test] +async fn a_failed_listing_or_row_read_stops_sync_with_no_row_written() { + let identity = store(); + let mem = Arc::new(MemStore::default()); + put_backup(&mem, &sync_arn(1)); + let fake = Arc::new(Fake::default()); + *mem.fail_lists_of.lock().unwrap() = Some(String::new()); + let err = super::sync::sync(&*fake, sync_target(&mem, &identity)) + .await + .unwrap_err(); + assert!( + matches!(&err, super::sync::SyncError::Listing { partial, .. } if partial.adopted.is_empty()), + "{err}" + ); + assert!(err.to_string().contains("list refused"), "{err}"); + *mem.fail_lists_of.lock().unwrap() = None; + + *fake.list_error.lock().unwrap() = Some("pool exhausted".to_owned()); + let err = super::sync::sync(&*fake, sync_target(&mem, &identity)) + .await + .unwrap_err(); + assert!(matches!(err, super::sync::SyncError::Catalog(_)), "{err}"); + assert!(fake.insert_attempts.lock().unwrap().is_empty()); + assert!(fake.rows.lock().unwrap().is_empty()); +} + +/// `--prune-missing` over the rows a sync reported: a row whose manifest is +/// gone is removed outright, with no DELETING or DELETED tombstone and no +/// object deleted (the objects left under its prefix stay, as a host-local +/// root's bytes on another host would); a row a restore is reading, a row +/// another writer moved, a row whose manifest is back, a row gone meanwhile +/// and a row whose manifest's `head` fails are each reported and left; the +/// rows a restore is reading and the failed `head`'s row are the ones a +/// later run may remove (the command exits 1 for them). A stop asked before +/// the first row changes nothing. DeleteBackup's start is never called. +#[tokio::test] +async fn prune_missing_removes_rows_whose_manifest_is_gone_and_deletes_nothing() { + use super::sync::PruneOutcome; + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let gone = sync_arn(1); + let leftover = sync_arn(2); + let in_use = sync_arn(3); + let moved = sync_arn(4); + let back = sync_arn(5); + let removed = sync_arn(6); + let unanswered = sync_arn(7); + let rows: Vec = [ + &gone, + &leftover, + &in_use, + &moved, + &back, + &removed, + &unanswered, + ] + .into_iter() + .map(|a| row(a, BackupStatus::Available)) + .collect(); + for r in &rows { + fake.rows + .lock() + .unwrap() + .insert(r.backup_arn.clone(), r.clone()); + } + let leftover_key = format!("{}data/000001.json.gz", prefix_of(&leftover)); + mem.insert(&leftover_key, b"x"); + fake.forget_answers .lock() .unwrap() - .insert(arn(ACCOUNT), backup_description(status, &billing)); - fake + .insert(in_use.clone(), ForgetRow::InUse("Target".to_owned())); + fake.rows.lock().unwrap().get_mut(&moved).unwrap().status = BackupStatus::Deleting; + mem.insert(&manifest_key(&back), b"{}"); + fake.rows.lock().unwrap().remove(&removed); + *mem.fail_heads_of.lock().unwrap() = Some(manifest_key(&unanswered)); + + let keys: Vec = rows.iter().map(BackupRow::key).collect(); + let root = store().store_root; + let (none, stopped) = + super::sync::prune_missing(&*fake, &*mem, &root, &keys, &|| Some("stopped".to_owned())) + .await; + assert!(none.is_empty()); + assert_eq!(stopped.as_deref(), Some("stopped")); + assert!(fake.forgets_seen.lock().unwrap().is_empty()); + + let keys_before = mem.keys(); + let (outcomes, stopped) = + super::sync::prune_missing(&*fake, &*mem, &root, &keys, &|| None).await; + assert_eq!(stopped, None); + assert_eq!( + outcomes, + [ + (gone.clone(), PruneOutcome::Pruned), + (leftover.clone(), PruneOutcome::Pruned), + (in_use.clone(), PruneOutcome::InUse("Target".to_owned())), + (moved.clone(), PruneOutcome::Status(BackupStatus::Deleting)), + (back.clone(), PruneOutcome::ManifestBack), + (removed.clone(), PruneOutcome::Gone), + ( + unanswered.clone(), + PruneOutcome::Failed( + "the manifest's head failed: transport error: dispatch failure: head refused" + .to_owned() + ), + ), + ] + ); + assert_eq!( + outcomes + .iter() + .map(|(_, outcome)| outcome.needs_another_run()) + .collect::>(), + [false, false, true, false, false, false, true] + ); + assert!( + !fake + .forgets_seen + .lock() + .unwrap() + .iter() + .any(|key| key.backup_arn == unanswered), + "a failed head removes nothing" + ); + let after = fake.rows.lock().unwrap(); + assert!(!after.contains_key(&gone), "no tombstone is left"); + assert!(!after.contains_key(&leftover), "no tombstone is left"); + assert_eq!(after[&in_use].status, BackupStatus::Available); + assert_eq!(after[&moved].status, BackupStatus::Deleting); + assert_eq!(after[&back].status, BackupStatus::Available); + assert_eq!(after[&unanswered].status, BackupStatus::Available); + assert_eq!(mem.keys(), keys_before, "no object was deleted"); + assert!(mem.keys().contains(&leftover_key)); + assert!(fake.delete_starts_seen.lock().unwrap().is_empty()); } -async fn describe(fake: &Fake) -> Value { - serde_json::to_value( - describe_backup(fake, ACCOUNT, REGION, &own(&arn(ACCOUNT))) +/// `--adopt-store`'s rewrite: the file names this catalog, keeps its store +/// id and creation time, and is written once; the identity is the file's +/// id; the previous owner's next check finds the root owned by another +/// catalog. On a root this catalog owns nothing is written. A root whose +/// rows in this catalog carry another id, and a root with no file, are +/// refused with nothing written. +#[tokio::test] +async fn adopt_store_makes_this_catalog_the_owner_and_keeps_the_store_id() { + use super::store_identity::{AdoptRefusal, AdoptRule, Adoption, adopt}; + let file = StoreIdentityFile { + format_version: 1, + store_id: "store-7".to_owned(), + catalog_id: "previous".to_owned(), + backend: "postgres".to_owned(), + created_at_ms: 1234, + ephemeral_catalog: false, + }; + let mem = MemStore::default(); + mem.insert(STORE_IDENTITY_KEY, &file.to_bytes()); + let fake = Fake::default(); + let adopted = adopt( + &mem, + StoreKind::Filesystem, + "/r", + &fake, + "sqlite", + false, + AdoptRule::Command, + ) + .await + .unwrap(); + let identity = StoreIdentity { + store_kind: StoreKind::Filesystem, + store_root: "/r".to_owned(), + store_id: "store-7".to_owned(), + }; + assert_eq!( + adopted, + Adoption::Adopted { + identity: identity.clone(), + previous_catalog_id: "previous".to_owned(), + } + ); + let now = StoreIdentityFile::parse(&mem.objects.lock().unwrap()[STORE_IDENTITY_KEY]).unwrap(); + assert_eq!( + now, + StoreIdentityFile { + catalog_id: "catalog-1".to_owned(), + backend: "sqlite".to_owned(), + ..file.clone() + } + ); + assert_eq!(mem.puts.lock().unwrap().as_slice(), [STORE_IDENTITY_KEY]); + // The previous owner's next check. + let previous = Fake::default(); + *previous.catalog_id.lock().unwrap() = Some("previous".to_owned()); + assert_eq!( + store_identity::recheck(&mem, &identity, &previous, "postgres", false).await, + Err(StoreProblem::ForeignCatalog { + file_catalog_id: "catalog-1".to_owned(), + catalog_id: "previous".to_owned(), + }) + ); + // Already this catalog's: nothing written. + let again = adopt( + &mem, + StoreKind::Filesystem, + "/r", + &fake, + "sqlite", + false, + AdoptRule::Command, + ) + .await + .unwrap(); + assert_eq!(again, Adoption::AlreadyOwned(identity.clone())); + assert_eq!(mem.puts.lock().unwrap().len(), 1); + + // This catalog's rows for the root carry another id. + let mem = MemStore::default(); + mem.insert(STORE_IDENTITY_KEY, &file.to_bytes()); + let fake = fake_with_row(BackupStatus::Available); + let err = adopt( + &mem, + StoreKind::Filesystem, + &store().store_root, + &*fake, + "sqlite", + false, + AdoptRule::Command, + ) + .await + .unwrap_err(); + assert_eq!( + err, + AdoptRefusal::RowsCarryOtherIds { + file_id: "store-7".to_owned(), + row_ids: vec!["store-1".to_owned()], + } + ); + assert!(mem.puts.lock().unwrap().is_empty()); + assert_eq!( + StoreIdentityFile::parse(&mem.objects.lock().unwrap()[STORE_IDENTITY_KEY]).unwrap(), + file + ); + + // No file. + let mem = MemStore::default(); + let err = adopt( + &mem, + StoreKind::Filesystem, + "/r", + &Fake::default(), + "sqlite", + false, + AdoptRule::Command, + ) + .await + .unwrap_err(); + assert_eq!(err, AdoptRefusal::NoFile { row_ids: vec![] }); + assert!(matches!( + err.into_problem("catalog-1"), + StoreProblem::FileUnreadable(_) + )); + assert!(mem.keys().is_empty()); +} + +/// Two adopters of one root race through a plain put: the one whose write +/// is replaced before its re-read reports the race lost, naming the +/// catalog the file names now. +#[tokio::test] +async fn an_adoption_overwritten_before_its_re_read_reports_the_race_lost() { + use super::store_identity::{AdoptRefusal, AdoptRule, adopt}; + let mem = Arc::new(MemStore::default()); + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-7", "previous")); + let third = identity_file("store-7", "third"); + let racer = Arc::clone(&mem); + *mem.on_put.lock().unwrap() = Some(Box::new(move |key: &str| { + if key == STORE_IDENTITY_KEY { + racer.insert(STORE_IDENTITY_KEY, &third); + } + })); + let err = adopt( + &*mem, + StoreKind::Filesystem, + "/r", + &Fake::default(), + "sqlite", + false, + AdoptRule::Command, + ) + .await + .unwrap_err(); + *mem.on_put.lock().unwrap() = None; + assert_eq!( + err, + AdoptRefusal::RaceLost { + file_catalog_id: "third".to_owned(), + file_store_id: "store-7".to_owned(), + } + ); + assert_eq!( + err.into_problem("catalog-1"), + StoreProblem::ForeignCatalog { + file_catalog_id: "third".to_owned(), + catalog_id: "catalog-1".to_owned(), + } + ); +} + +/// A server whose own catalog is in memory adopts, in its startup check, a +/// root whose file another in-memory catalog wrote, keeping the store id +/// and writing the marker; a root a durable catalog owns stays owned by +/// another catalog, its file untouched; a durable server never adopts a +/// marked root; and once the identity is known the re-check never adopts, +/// so a second in-memory server's later adoption leaves the first one's +/// store owned by another catalog. +#[tokio::test] +async fn an_in_memory_catalog_adopts_only_a_root_another_in_memory_catalog_owned() { + // Marked: adopted. + let mem = MemStore::default(); + mem.insert( + STORE_IDENTITY_KEY, + &identity_file_of("store-7", "previous-start", true), + ); + let fake = Fake::default(); + let identity = + store_identity::establish(&mem, StoreKind::Filesystem, "/r", &fake, "sqlite", true) .await - .unwrap(), + .unwrap(); + assert_eq!(identity.store_id, "store-7"); + let file = StoreIdentityFile::parse(&mem.objects.lock().unwrap()[STORE_IDENTITY_KEY]).unwrap(); + assert_eq!(file.catalog_id, "catalog-1"); + assert_eq!(file.store_id, "store-7"); + assert!(file.ephemeral_catalog); + + // A durable owner: unhealthy, nothing written. + let mem = MemStore::default(); + mem.insert( + STORE_IDENTITY_KEY, + &identity_file_of("store-7", "durable", false), + ); + let err = store_identity::establish(&mem, StoreKind::Filesystem, "/r", &fake, "sqlite", true) + .await + .unwrap_err(); + assert_eq!( + err, + StoreProblem::ForeignCatalog { + file_catalog_id: "durable".to_owned(), + catalog_id: "catalog-1".to_owned(), + } + ); + assert!(mem.puts.lock().unwrap().is_empty()); + + // A durable server and a marked root: unhealthy, nothing written. + let mem = MemStore::default(); + mem.insert( + STORE_IDENTITY_KEY, + &identity_file_of("store-7", "previous-start", true), + ); + let err = + store_identity::establish(&mem, StoreKind::Filesystem, "/r", &fake, "postgres", false) + .await + .unwrap_err(); + assert!(matches!(err, StoreProblem::ForeignCatalog { .. }), "{err}"); + assert!(mem.puts.lock().unwrap().is_empty()); + + // An empty root: the file an in-memory catalog creates carries the + // marker. + let mem = MemStore::default(); + store_identity::establish(&mem, StoreKind::Filesystem, "/r", &fake, "sqlite", true) + .await + .unwrap(); + let created = + StoreIdentityFile::parse(&mem.objects.lock().unwrap()[STORE_IDENTITY_KEY]).unwrap(); + assert!(created.ephemeral_catalog); + + // The identity known: another in-memory server took the root since. + let mem = MemStore::default(); + mem.insert( + STORE_IDENTITY_KEY, + &identity_file_of("store-7", "second-server", true), + ); + let known = StoreIdentity { + store_kind: StoreKind::Filesystem, + store_root: "/r".to_owned(), + store_id: "store-7".to_owned(), + }; + let err = store_identity::recheck(&mem, &known, &fake, "sqlite", true) + .await + .unwrap_err(); + assert!(matches!(err, StoreProblem::ForeignCatalog { .. }), "{err}"); + assert!(mem.puts.lock().unwrap().is_empty()); +} + +/// A runtime whose store `mem` is unchecked, for an in-memory catalog when +/// `ephemeral` is set (`store()`'s kind and root). +fn runtime_checking(mem: &Arc, ephemeral: bool) -> BackupRuntime { + runtime().with_store( + ConfiguredStore::checking( + store().store_kind, + store().store_root, + Arc::clone(mem) as Arc, + None, + ) + .with_ephemeral_catalog(ephemeral), ) - .unwrap() } +/// The startup of a server whose catalog is in memory, against the root +/// the previous start owned: the first reconciler cycle's check adopts the +/// root and the startup sync gives the previous start's backup its row +/// again, under the file's store id; the next cycle syncs no more. A root a +/// durable catalog owns stays unhealthy and nothing is synced, the sync +/// left pending. A listing that fails leaves the sync pending, and the next +/// cycle runs it. #[tokio::test] -async fn describe_backup_reports_features_of_an_on_demand_table() { - let mut def = definition("PAY_PER_REQUEST"); - def.time_to_live = Some(TimeToLiveDescription { - time_to_live_status: TimeToLiveStatus::Enabled, - attribute_name: Some("exp".to_owned()), - }); - def.stream_specification = Some(StreamSpecification { - stream_enabled: true, - stream_view_type: Some(StreamViewType::NewImage), - }); - def.sse_specification = Some(json!({ "Enabled": true })); - let fake = fake_with_backup("AVAILABLE", source(Some(def))); - let v = describe(&fake).await; +async fn the_startup_sync_of_an_in_memory_catalog_serves_the_previous_starts_backups() { + let previous = sync_arn(1); - let pt = &v["SourceTableDetails"]["ProvisionedThroughput"]; - assert_eq!( - pt, - &json!({ "ReadCapacityUnits": 0, "WriteCapacityUnits": 0 }) + let mem = Arc::new(MemStore::default()); + mem.insert( + STORE_IDENTITY_KEY, + &identity_file_of("store-7", "previous-start", true), ); - let f = &v["SourceTableFeatureDetails"]; - assert_eq!(f["GlobalSecondaryIndexes"][0]["IndexName"], "gsi1"); - assert_eq!( - f["GlobalSecondaryIndexes"][0]["KeySchema"][0]["AttributeName"], - "gpk" + put_backup(&mem, &previous); + // A prefix with no manifest: skipped, and no later run adopts it + // unless the store changes, so it leaves the sync done. + let incomplete = sync_arn(2); + mem.insert( + &format!("{}data/000001.json.gz", prefix_of(&incomplete)), + b"x", ); - assert_eq!( - f["GlobalSecondaryIndexes"][0]["Projection"]["ProjectionType"], - "ALL" + let fake = Arc::new(Fake::default()); + let runtime = runtime_checking(&mem, true); + let store_of = + |runtime: &BackupRuntime| runtime.configured_store().unwrap().startup_sync_pending(); + assert!(store_of(&runtime)); + let report = cycle_on(&fake, &runtime).await; + assert_eq!(report.store_healthy, Some(true)); + let row = fake.rows.lock().unwrap()[&previous].clone(); + assert_eq!(row.status, BackupStatus::Available); + assert_eq!(row.store.store_id, "store-7"); + assert!(!fake.rows.lock().unwrap().contains_key(&incomplete)); + assert!( + !store_of(&runtime), + "an incomplete prefix leaves the sync done" ); - assert_eq!( - f["GlobalSecondaryIndexes"][0]["ProvisionedThroughput"], - json!({ "ReadCapacityUnits": 0, "WriteCapacityUnits": 0 }) + cycle_on(&fake, &runtime).await; + assert_eq!(fake.insert_attempts.lock().unwrap().len(), 1, "synced once"); + + // A durable owner. + let mem = Arc::new(MemStore::default()); + mem.insert( + STORE_IDENTITY_KEY, + &identity_file_of("store-7", "durable", false), ); - assert_eq!(f["LocalSecondaryIndexes"][0]["IndexName"], "lsi1"); - assert!( - f["LocalSecondaryIndexes"][0] - .get("ProvisionedThroughput") - .is_none() + put_backup(&mem, &previous); + let fake = Arc::new(Fake::default()); + let runtime = runtime_checking(&mem, true); + let report = cycle_on(&fake, &runtime).await; + assert_eq!(report.store_healthy, Some(false)); + assert!(fake.insert_attempts.lock().unwrap().is_empty()); + assert!(store_of(&runtime)); + + // The listing fails once. + let mem = Arc::new(MemStore::default()); + mem.insert( + STORE_IDENTITY_KEY, + &identity_file_of("store-7", "previous-start", true), ); + put_backup(&mem, &previous); + *mem.fail_lists_of.lock().unwrap() = Some(String::new()); + let fake = Arc::new(Fake::default()); + let runtime = runtime_checking(&mem, true); + let report = cycle_on(&fake, &runtime).await; + assert_eq!(report.store_healthy, Some(true)); + assert!(fake.rows.lock().unwrap().is_empty()); + assert!(store_of(&runtime)); + *mem.fail_lists_of.lock().unwrap() = None; + cycle_on(&fake, &runtime).await; assert_eq!( - f["TimeToLiveDescription"], - json!({ "TimeToLiveStatus": "ENABLED", "AttributeName": "exp" }) + fake.rows.lock().unwrap()[&previous].status, + BackupStatus::Available + ); + assert!(!store_of(&runtime)); + + // A durable catalog has no startup sync. + let mem = Arc::new(MemStore::default()); + put_backup(&mem, &previous); + let fake = Arc::new(Fake::default()); + let runtime = runtime_checking(&mem, false); + assert!(!store_of(&runtime)); + cycle_on(&fake, &runtime).await; + assert!(fake.insert_attempts.lock().unwrap().is_empty()); +} + +/// The server's own call of the startup sync (before it serves, a +/// filesystem store whose check passed) runs it once; a run abandoned at +/// the server's bound (its future dropped while the store does not answer) +/// leaves it pending for the reconciler. +#[tokio::test] +async fn a_startup_sync_abandoned_at_its_bound_stays_pending() { + let mem = Arc::new(MemStore::default()); + put_backup(&mem, &sync_arn(1)); + let fake = Arc::new(Fake::default()); + let runtime = runtime().with_store( + ConfiguredStore::new( + store().store_kind, + store().store_root, + Arc::clone(&mem) as Arc, + None, + Ok(store()), + ) + .with_ephemeral_catalog(true), + ); + let pending = || runtime.configured_store().unwrap().startup_sync_pending(); + mem.set_stalled(true); + let abandoned = tokio::time::timeout( + Duration::from_millis(50), + super::sync::startup_sync(&*fake, &runtime), + ) + .await; + assert!(abandoned.is_err()); + assert!(mem.stalls.load(std::sync::atomic::Ordering::Acquire) > 0); + assert!(pending(), "left pending"); + mem.set_stalled(false); + let ran = super::sync::startup_sync(&*fake, &runtime).await; + assert!(matches!(ran, Some(Ok(ref report)) if report.adopted.len() == 1)); + assert!(!pending()); + assert!(super::sync::startup_sync(&*fake, &runtime).await.is_none()); +} + +/// A reconciler cycle whose startup sync meets a root listing that never +/// answers returns once the sync's own bound passes, and the cycle's store +/// duties still run: the DELETING row is finished in the same cycle, and +/// the sync stays pending for the next. +#[tokio::test(start_paused = true)] +async fn a_cycles_startup_sync_is_cut_at_its_bound_and_the_duties_run() { + let mem = Arc::new(MemStore::default()); + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + let fake = Arc::new(Fake::default()); + let deleting = arn(ACCOUNT); + seed(&fake, &mem, &deleting, BackupStatus::Deleting); + put_backup(&mem, &sync_arn(1)); + let runtime = runtime().with_store( + ConfiguredStore::new( + store().store_kind, + store().store_root, + Arc::clone(&mem) as Arc, + None, + Ok(store()), + ) + .with_ephemeral_catalog(true), ); + mem.stall_root_list + .store(true, std::sync::atomic::Ordering::Release); + let started = tokio::time::Instant::now(); + // A guard far past the bound, so a startup sync with no bound fails + // here instead of hanging. + let report = tokio::time::timeout( + super::sync::STARTUP_SYNC_TIMEOUT * 5, + cycle_on(&fake, &runtime), + ) + .await + .expect("the startup sync is bounded"); + let took = started.elapsed(); assert_eq!( - f["StreamDescription"], - json!({ "StreamEnabled": true, "StreamViewType": "NEW_IMAGE" }) + report.store_healthy, + Some(true), + "{:?}", + runtime.configured_store().unwrap().health() ); - assert_eq!(f["SSEDescription"]["Status"], "ENABLED"); + assert!(took >= super::sync::STARTUP_SYNC_TIMEOUT, "{took:?}"); + assert!( + took < super::sync::STARTUP_SYNC_TIMEOUT + Duration::from_secs(30), + "{took:?}" + ); + assert_eq!(report.deletions_finished, [deleting.as_str()]); assert_eq!( - f["SSEDescription"]["KMSMasterKeyArn"], - format!("arn:aws:kms:{REGION}:{ACCOUNT}:key/default") + fake.rows.lock().unwrap()[&deleting].status, + BackupStatus::Deleted + ); + assert!( + runtime.configured_store().unwrap().startup_sync_pending(), + "left pending" ); + assert!(fake.insert_attempts.lock().unwrap().is_empty()); } +/// A manifest the path does not name is skipped whatever component differs +/// (another account, another table), and so is one in another partition +/// whose components match the path: every request refuses an ARN outside +/// `aws` before it reads a row, so no request could address the row. +/// Nothing is inserted. #[tokio::test] -async fn describe_backup_reports_provisioned_throughput_and_omits_absent_members() { - let fake = fake_with_backup("AVAILABLE", source(Some(definition("PROVISIONED")))); - let v = describe(&fake).await; - assert_eq!( - v["SourceTableDetails"]["ProvisionedThroughput"], - json!({ "ReadCapacityUnits": 7, "WriteCapacityUnits": 8 }) +async fn sync_skips_a_manifest_of_another_account_table_or_partition() { + use super::sync::SkipReason; + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let identity = store(); + let moved_under = |from: &str, to: &str| { + for (key, bytes) in backup_objects(from) { + mem.insert(&key.replacen(&prefix_of(from), &prefix_of(to), 1), &bytes); + } + }; + let account_path = sync_arn(1); + let other_account = account_path.replace(ACCOUNT, "210987654321"); + moved_under(&other_account, &account_path); + let table_path = sync_arn(2); + let other_table = table_path.replace("table/Music", "table/Other"); + moved_under(&other_table, &table_path); + let partition = sync_arn(3) + .replace("arn:aws:", "arn:aws-cn:") + .replace(REGION, "cn-north-1"); + put_backup(&mem, &partition); + assert_eq!(prefix_of(&partition), prefix_of(&sync_arn(3))); + + let report = super::sync::sync(&*fake, sync_target(&mem, &identity)) + .await + .unwrap(); + assert!(report.adopted.is_empty(), "{:?}", report.adopted); + let skipped = skipped_by_prefix(&report); + assert!( + matches!(&skipped[&prefix_of(&account_path)], SkipReason::ArnMismatch(why) if why.contains(&other_account)), + "{:?}", + skipped[&prefix_of(&account_path)] ); - let f = v["SourceTableFeatureDetails"].as_object().unwrap(); - assert_eq!( - f["GlobalSecondaryIndexes"][0]["ProvisionedThroughput"], - json!({ "ReadCapacityUnits": 3, "WriteCapacityUnits": 4 }) + assert!( + matches!(&skipped[&prefix_of(&table_path)], SkipReason::ArnMismatch(why) if why.contains(&other_table)), + "{:?}", + skipped[&prefix_of(&table_path)] ); - for absent in [ - "TimeToLiveDescription", - "StreamDescription", - "SSEDescription", + assert!( + matches!(&skipped[&prefix_of(&partition)], SkipReason::OtherPartition(why) + if why.contains("aws-cn") && why.contains("no request")), + "{:?}", + skipped[&prefix_of(&partition)] + ); + assert!(fake.insert_attempts.lock().unwrap().is_empty()); + assert!(fake.rows.lock().unwrap().is_empty()); +} + +/// Only an AVAILABLE row of the configured store is reported with a missing +/// manifest: a CREATING, DELETING, DELETED or FAILED row with no prefix is +/// not, so `--prune-missing` never examines it. +#[tokio::test] +async fn only_available_rows_are_reported_with_a_missing_manifest() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let identity = store(); + for (n, status) in [ + (1, BackupStatus::Creating), + (2, BackupStatus::Failed), + (3, BackupStatus::Deleting), + (4, BackupStatus::Deleted), + (5, BackupStatus::Available), ] { - assert!(!f.contains_key(absent), "{absent}: {f:?}"); + let a = sync_arn(n); + fake.rows.lock().unwrap().insert(a.clone(), row(&a, status)); } + let report = super::sync::sync(&*fake, sync_target(&mem, &identity)) + .await + .unwrap(); + assert_eq!( + report + .missing_manifests + .iter() + .map(|r| r.backup_arn.clone()) + .collect::>(), + [sync_arn(5)] + ); } +/// A prefix whose backup id a row of this root holds under another table +/// (the prefix copied under a second table directory) is skipped naming +/// the row's table, never read. #[tokio::test] -async fn describe_backup_of_a_legacy_backup_has_no_feature_details() { - let mut src = source(None); - src.provisioned_throughput = Some(throughput(2, 3)); - let fake = fake_with_backup("AVAILABLE", src); - let v = describe(&fake).await; - assert!(v.get("SourceTableFeatureDetails").is_none(), "{v}"); +async fn a_prefix_under_another_table_with_a_rows_backup_id_is_skipped_naming_the_table() { + use super::sync::SkipReason; + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let identity = store(); + let original = sync_arn(1); + fake.rows + .lock() + .unwrap() + .insert(original.clone(), row(&original, BackupStatus::Available)); + put_backup(&mem, &original); + let copy = original.replace("table/Music", "table/Copy"); + for (key, bytes) in backup_objects(&original) { + mem.insert( + &key.replacen(&prefix_of(&original), &prefix_of(©), 1), + &bytes, + ); + } + let report = super::sync::sync(&*fake, sync_target(&mem, &identity)) + .await + .unwrap(); + let skipped = skipped_by_prefix(&report); assert_eq!( - v["SourceTableDetails"]["ProvisionedThroughput"], - json!({ "ReadCapacityUnits": 2, "WriteCapacityUnits": 3 }) + skipped[&prefix_of(©)], + SkipReason::HasRow { + status: BackupStatus::Available, + store_id: "store-1".to_owned(), + other_table: Some("Music".to_owned()), + } + ); + assert!( + skipped[&prefix_of(©)] + .to_string() + .contains("under table Music"), + "{}", + skipped[&prefix_of(©)] ); + assert!(!mem.gets().iter().any(|k| k.starts_with(&prefix_of(©)))); } -/// A definition this build cannot read (a later version) does not stop -/// DescribeBackup: the backend's BackupDetails are returned without feature -/// details. A definition that does not parse is `Internal`, and propagates as -/// a 500 like a failure to reach the database (`Transient`, `Connection`): -/// a wrong definition row is a database fault, and describing the backup as -/// on-demand `{0, 0}` without feature details would mask it as a wrong answer -/// while RestoreTableFromBackup on the same read is a 500. -/// The errors are the ones every backend returns for those cases, built by -/// `BackupTableDefinition::from_json` itself. +/// Before the insert the row is read under each spelling of the manifest's +/// ARN: a row of another root under the ARN rebuilt in this server's region +/// holds the backup though the manifest spells another region, and the +/// prefix is skipped naming that store, with no insert (on Cassandra the +/// insert alone would not see it). #[tokio::test] -async fn describe_backup_survives_a_definition_that_does_not_read() { - let arn_text = arn(ACCOUNT); - let undecodable = BackupTableDefinition::from_json( - json!({ "Version": BACKUP_DEFINITION_VERSION, "BillingMode": 7 }), - &arn_text, - ) - .expect_err("a member of the wrong type does not parse"); +async fn sync_skips_a_backup_a_row_holds_under_another_spelling_of_its_arn() { + use super::sync::SkipReason; + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let identity = store(); + let canonical = sync_arn(1); + let spelled = canonical.replace(REGION, "eu-west-1"); + let mut elsewhere = row(&canonical, BackupStatus::Available); + elsewhere.store.store_root = "/srv/elsewhere".to_owned(); + fake.rows + .lock() + .unwrap() + .insert(canonical.clone(), elsewhere.clone()); + put_backup(&mem, &spelled); + let report = super::sync::sync(&*fake, sync_target(&mem, &identity)) + .await + .unwrap(); + assert!(report.adopted.is_empty()); assert!( - matches!(undecodable, StorageError::Internal(_)), - "{undecodable:?}" + matches!(&skipped_by_prefix(&report)[&prefix_of(&spelled)], SkipReason::RowExists(what) + if what.contains("of another store") && what.contains("/srv/elsewhere")), + "{:?}", + report.skipped ); - let unknown_version = BackupTableDefinition::from_json( - json!({ "Version": BACKUP_DEFINITION_VERSION + 1 }), - &arn_text, - ) - .expect_err("an unknown version is refused"); + assert!(fake.insert_attempts.lock().unwrap().is_empty()); assert!( - matches!(unknown_version, StorageError::Unsupported(_)), - "{unknown_version:?}" + fake.row_reads.lock().unwrap().contains(&canonical), + "the canonical spelling was read" ); - { - let err = unknown_version; - let what = format!("{err:?}"); - let fake = fake_with_backup("AVAILABLE", source(Some(definition("PROVISIONED")))); - *fake.source_error.lock().unwrap() = Some(err); - let desc = describe_backup(&fake, ACCOUNT, REGION, &own(&arn_text)) - .await - .unwrap_or_else(|e| panic!("{what}: {e:?}")); - assert_eq!(fake.source_reads(), 1, "{what}"); - assert_eq!(desc.backup_details.backup_status, "AVAILABLE", "{what}"); - assert_eq!(desc.backup_details.backup_arn, arn_text, "{what}"); - assert!(desc.source_table_feature_details.is_none(), "{what}"); + assert_eq!(fake.rows.lock().unwrap()[&canonical], elsewhere); +} + +/// An adopted row whose list and status entries are gone (a Cassandra +/// insert that stopped between the row and its entries) is not in the rows +/// sync reads first, and the read under the manifest's own spelling finds +/// it: the next sync passes it to the insert, which completes it, and the +/// backup is adopted and listed again. A row under the other spelling of +/// the ARN still skips the prefix with no insert. +#[tokio::test] +async fn the_next_sync_completes_its_own_insert_that_stopped_before_its_entries() { + use super::sync::SkipReason; + let identity = store(); + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + fake.completes_own_insert + .store(true, std::sync::atomic::Ordering::Release); + let torn = sync_arn(1); + put_backup(&mem, &torn); + let first = super::sync::sync(&*fake, sync_target(&mem, &identity)) + .await + .unwrap(); + assert_eq!(first.adopted.len(), 1); + fake.unlisted.lock().unwrap().insert(torn.clone()); + let stored = fake.rows.lock().unwrap()[&torn].clone(); + + let second = super::sync::sync(&*fake, sync_target(&mem, &identity)) + .await + .unwrap(); + assert_eq!( + second.adopted, + [super::sync::AdoptedBackup { + backup_arn: torn.clone(), + foreign: None, + }], + "{:?}", + second.skipped + ); + assert!(second.skipped.is_empty(), "{:?}", second.skipped); + assert_eq!(fake.insert_attempts.lock().unwrap().len(), 2); + assert!( + !fake.unlisted.lock().unwrap().contains(&torn), + "listed again" + ); + assert_eq!(fake.rows.lock().unwrap()[&torn], stored); + + // The other spelling: the canonical ARN's row is unlisted, the manifest + // spells another region. + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + fake.completes_own_insert + .store(true, std::sync::atomic::Ordering::Release); + let canonical = sync_arn(2); + let spelled = canonical.replace(REGION, "eu-west-1"); + let mut held = row(&canonical, BackupStatus::Available); + held.owner_generation = UNOWNED_GENERATION; + fake.rows + .lock() + .unwrap() + .insert(canonical.clone(), held.clone()); + fake.unlisted.lock().unwrap().insert(canonical.clone()); + put_backup(&mem, &spelled); + let report = super::sync::sync(&*fake, sync_target(&mem, &identity)) + .await + .unwrap(); + assert!(report.adopted.is_empty()); + assert!( + matches!(&skipped_by_prefix(&report)[&prefix_of(&spelled)], SkipReason::RowExists(what) + if what.contains("listings do not return")), + "{:?}", + report.skipped + ); + assert!(fake.insert_attempts.lock().unwrap().is_empty()); + assert_eq!(fake.rows.lock().unwrap()[&canonical], held); +} + +/// An insert that fails is a catalog skip for its prefix, and sync goes on +/// to the next; the report counts it as transient, so the command exits 1 +/// and a startup sync stays pending. +#[tokio::test] +async fn an_insert_that_fails_is_reported_and_sync_goes_on() { + use super::sync::SkipReason; + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let identity = store(); + put_backup(&mem, &sync_arn(1)); + put_backup(&mem, &sync_arn(2)); + *fake.insert_error.lock().unwrap() = Some("pool exhausted".to_owned()); + let report = super::sync::sync(&*fake, sync_target(&mem, &identity)) + .await + .unwrap(); + assert!(report.adopted.is_empty()); + let skipped = skipped_by_prefix(&report); + for n in [1, 2] { assert!( - desc.source_table_details.provisioned_throughput.is_some(), - "{what}: required member" + matches!(&skipped[&prefix_of(&sync_arn(n))], SkipReason::Catalog(e) if e.contains("pool exhausted")), + "{:?}", + skipped[&prefix_of(&sync_arn(n))] ); } - - for err in [ - undecodable, - StorageError::Internal("column \"definition\" does not exist".to_owned()), - StorageError::Transient( - "Database error: pool timed out while waiting for an open connection".to_owned(), - ), - StorageError::Connection("connection refused".to_owned()), - ] { - let what = format!("{err:?}"); - let fake = fake_with_backup("AVAILABLE", source(Some(definition("PROVISIONED")))); - *fake.source_error.lock().unwrap() = Some(err); - // The backup operations' mapping reports both as a 500 - // (`storage_err_to_dynamo` in backup.rs), which SDKs retry. - match describe_backup(&fake, ACCOUNT, REGION, &own(&arn_text)).await { - Err(DynamoDbError::InternalServerError(_)) => {} - other => panic!("{what} must reach the client as an error: {other:?}"), - } - assert_eq!(fake.source_reads(), 1, "{what}"); - } + assert_eq!(report.transient_skips(), 2); + assert_eq!(fake.insert_attempts.lock().unwrap().len(), 2); } +/// A stop asked before a prefix ends the run there: the prefixes before it +/// keep their rows, nothing after it is read, and no missing-manifest report +/// is made (the listing did not reach every prefix). #[tokio::test] -async fn describe_backup_of_a_backup_that_is_not_available_uses_the_backend_row() { - let fake = fake_with_backup("CREATING", source(Some(definition("PROVISIONED")))); - let v = describe(&fake).await; - assert_eq!(fake.source_reads(), 0); - assert!(v.get("SourceTableFeatureDetails").is_none()); - assert_eq!(v["BackupDetails"]["BackupStatus"], "CREATING"); - // Required member, still present. +async fn a_stop_ends_sync_before_the_next_prefix_with_no_missing_manifest_report() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let identity = store(); + put_backup(&mem, &sync_arn(1)); + put_backup(&mem, &sync_arn(2)); + let gone = sync_arn(9); + fake.rows + .lock() + .unwrap() + .insert(gone.clone(), row(&gone, BackupStatus::Available)); + let asked = std::sync::atomic::AtomicUsize::new(0); + let stop = || { + (asked.fetch_add(1, std::sync::atomic::Ordering::AcqRel) >= 1) + .then(|| "SIGTERM received".to_owned()) + }; + let target = super::sync::SyncTarget { + stop: Some(&stop), + ..sync_target(&mem, &identity) + }; + let report = super::sync::sync(&*fake, target).await.unwrap(); + assert_eq!(report.stopped.as_deref(), Some("SIGTERM received")); + assert_eq!( + report + .adopted + .iter() + .map(|a| a.backup_arn.clone()) + .collect::>(), + [sync_arn(1)] + ); + assert!(report.missing_manifests.is_empty()); assert!( - v["SourceTableDetails"] - .get("ProvisionedThroughput") - .is_some() + !mem.gets() + .iter() + .any(|k| k.starts_with(&prefix_of(&sync_arn(2)))) ); - - // AVAILABLE in describe, gone by the source read: described without - // feature details rather than failing. - let fake = fake_with_backup("AVAILABLE", source(Some(definition("PROVISIONED")))); - fake.sources.lock().unwrap().clear(); - let v = describe(&fake).await; - assert_eq!(fake.source_reads(), 1); - assert!(v.get("SourceTableFeatureDetails").is_none()); } -// --- ListBackups ---------------------------------------------------------------- - -fn summary(n: u64, millis: u64) -> BackupSummary { - let id = extenddb_storage::util::format_backup_id(millis, u32::try_from(n).unwrap()); - BackupSummary { - backup_arn: extenddb_storage::util::backup_arn(REGION, ACCOUNT, "Music", &id), - backup_name: format!("b{n}"), - table_name: "Music".to_owned(), - table_id: "tid".to_owned(), - table_arn: source_arn(), - backup_status: "AVAILABLE".to_owned(), - backup_type: "USER".to_owned(), - backup_size_bytes: 0, - backup_creation_date_time: extenddb_storage::util::backup_creation_date_time(&id).unwrap(), +/// A listing that fails part-way keeps the rows of the prefixes it passed +/// (each verified), reports them, and makes no missing-manifest report. The +/// prefix it was listing when it failed (here the second) is not read: its +/// keys may be cut short. +#[tokio::test] +async fn a_listing_that_fails_part_way_keeps_the_rows_before_it() { + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let identity = store(); + for n in 1..=3 { + put_backup(&mem, &sync_arn(n)); } + let gone = sync_arn(9); + fake.rows + .lock() + .unwrap() + .insert(gone.clone(), row(&gone, BackupStatus::Available)); + *mem.fail_root_list_at.lock().unwrap() = Some(prefix_of(&sync_arn(3))); + let err = super::sync::sync(&*fake, sync_target(&mem, &identity)) + .await + .unwrap_err(); + let super::sync::SyncError::Listing { partial, .. } = &err else { + panic!("{err}"); + }; + assert_eq!(partial.adopted.len(), 1, "{:?}", partial.adopted); + assert_eq!(partial.adopted[0].backup_arn, sync_arn(1)); + assert!(partial.missing_manifests.is_empty()); + assert!( + err.to_string().contains("1 backup(s) listed before it"), + "{err}" + ); + assert!(fake.rows.lock().unwrap().contains_key(&sync_arn(1))); + assert!(!fake.rows.lock().unwrap().contains_key(&sync_arn(2))); + assert!( + !mem.gets() + .iter() + .any(|k| k.starts_with(&prefix_of(&sync_arn(2)))) + ); +} + +/// The startup sync of an in-memory catalog stays pending when a prefix +/// was skipped for a store error, and the next cycle adopts it once its +/// reads answer; the prefixes adopted in the first run are skipped on the +/// row check then. +#[tokio::test] +async fn a_startup_sync_with_a_store_error_on_a_prefix_stays_pending() { + let mem = Arc::new(MemStore::default()); + mem.insert( + STORE_IDENTITY_KEY, + &identity_file_of("store-7", "previous-start", true), + ); + let ok = sync_arn(1); + let flaky = sync_arn(2); + put_backup(&mem, &ok); + put_backup(&mem, &flaky); + *mem.fail_gets_under.lock().unwrap() = Some(prefix_of(&flaky)); + let fake = Arc::new(Fake::default()); + let runtime = runtime_checking(&mem, true); + let pending = || runtime.configured_store().unwrap().startup_sync_pending(); + cycle_on(&fake, &runtime).await; + assert!(fake.rows.lock().unwrap().contains_key(&ok)); + assert!(!fake.rows.lock().unwrap().contains_key(&flaky)); + assert!(pending(), "a store error leaves the sync pending"); + *mem.fail_gets_under.lock().unwrap() = None; + cycle_on(&fake, &runtime).await; + assert_eq!( + fake.rows.lock().unwrap()[&flaky].status, + BackupStatus::Available + ); + assert!(!pending()); + assert_eq!(fake.insert_attempts.lock().unwrap().len(), 2); } -fn fake_with_summaries(summaries: Vec) -> Fake { - let fake = Fake::default(); - *fake.summaries.lock().unwrap() = summaries; - fake +/// A startup sync never runs while the store is unhealthy, though its +/// identity is known: a recheck that found the root owned by another +/// catalog (a second in-memory server took it) leaves the sync pending and +/// nothing inserted. +#[tokio::test] +async fn a_startup_sync_does_not_run_while_a_recheck_finds_the_root_foreign() { + let mem = Arc::new(MemStore::default()); + put_backup(&mem, &sync_arn(1)); + let fake = Arc::new(Fake::default()); + let runtime = runtime().with_store( + ConfiguredStore::new( + store().store_kind, + store().store_root, + Arc::clone(&mem) as Arc, + None, + Ok(store()), + ) + .with_ephemeral_catalog(true), + ); + let configured = runtime.configured_store().unwrap(); + configured.record_check(Err(StoreProblem::ForeignCatalog { + file_catalog_id: "second-server".to_owned(), + catalog_id: "catalog-1".to_owned(), + })); + assert!(configured.identity().is_some(), "the identity is known"); + assert!(super::sync::startup_sync(&*fake, &runtime).await.is_none()); + assert!(configured.startup_sync_pending()); + assert!(fake.insert_attempts.lock().unwrap().is_empty()); } -async fn list(fake: &Fake, body: Value) -> Result { - list_backups(fake, ACCOUNT, REGION, &limits(), &body).await +/// The dead-owner probe requires the manifest's whole ARN to be the row's: +/// a complete prefix whose manifest names the row's backup id in another +/// region does not make the row AVAILABLE (sync, which compares only the +/// path's components, would adopt that manifest as its own row). +#[tokio::test] +async fn the_dead_owner_probe_refuses_a_manifest_of_another_region() { + let mem = Arc::new(MemStore::default()); + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + let fake = Arc::new(Fake::default()); + let dead = sync_arn(1); + put_backup(&mem, &dead.replace(REGION, "eu-west-1")); + let mut creating = row(&dead, BackupStatus::Creating); + creating.heartbeat_at = 1234.5; + fake.rows.lock().unwrap().insert(dead.clone(), creating); + fake.dead.lock().unwrap().insert(dead.clone()); + let runtime = runtime_on(&mem, store()); + let report = cycle_on(&fake, &runtime).await; + assert_eq!( + report.creating_resolved, + [(dead.clone(), BackupStatus::Failed)] + ); + assert_eq!( + fake.rows.lock().unwrap()[&dead].status, + BackupStatus::Failed + ); } -fn arns(page: &Value) -> Vec { - page["BackupSummaries"] - .as_array() - .unwrap() - .iter() - .map(|s| s["BackupArn"].as_str().unwrap().to_owned()) - .collect() +// --- the deletion fence ------------------------------------------------------ + +/// The identity file of `store()` after catalog `adopter` adopted the root. +fn adopted_file() -> Vec { + identity_file("store-1", "adopter") } -#[tokio::test] -async fn list_backups_defaults_to_one_hundred() { - let fake = fake_with_summaries((0..150).map(|n| summary(n, 1_000 + n)).collect()); - let page = list(&fake, json!({})).await.unwrap(); - assert_eq!(arns(&page).len(), 100); +/// The store `runtime` is configured with is unhealthy because its root's +/// identity file names `adopter`, as the fence recorded it. +fn assert_fenced(runtime: &BackupRuntime) { assert_eq!( - page["LastEvaluatedBackupArn"].as_str(), - Some(arns(&page)[99].as_str()) + runtime.configured_store().unwrap().health(), + StoreHealth::Unhealthy(StoreProblem::ForeignCatalog { + file_catalog_id: "adopter".to_owned(), + catalog_id: "catalog-1".to_owned(), + }) ); - assert_eq!(page["BackupSummaries"][0]["TableId"], "tid"); } +/// DeleteBackup on a store whose last check passed, after another catalog +/// adopted the root: the fence re-reads the identity file before the +/// DELETING write and refuses. No object is deleted, the request is refused +/// as an unhealthy store's, the row stays AVAILABLE (so DescribeBackup does +/// not answer a refused request with DELETED), and the store is unhealthy, +/// so the next request is refused at the gate with no read. #[tokio::test] -async fn list_backups_limit_is_bounded() { - let fake = fake_with_summaries(vec![summary(1, 1)]); - for bad in [0, -1, 101] { - let msg = validation_message(list(&fake, json!({ "Limit": bad })).await.unwrap_err()); - assert!(msg.contains("'limit'"), "{msg}"); - } +async fn delete_backup_on_a_root_another_catalog_adopted_deletes_nothing() { + let mem = Arc::new(MemStore::default()); + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + let fake = Arc::new(Fake::default()); + seed(&fake, &mem, &arn(ACCOUNT), BackupStatus::Available); + let runtime = runtime_on(&mem, store()); + mem.insert(STORE_IDENTITY_KEY, &adopted_file()); + let err = delete_backup(&*fake, &runtime, ACCOUNT, REGION, &own(&arn(ACCOUNT))) + .await + .unwrap_err(); + assert!( + validation_message(err).contains("unhealthy (owned by another catalog)"), + "the refusal names the store's problem" + ); + assert!(mem.deletes.lock().unwrap().is_empty()); + assert_eq!(objects_under(&mem, &arn(ACCOUNT)).len(), 2); + assert_eq!( + fake.rows.lock().unwrap()[&arn(ACCOUNT)].status, + BackupStatus::Available + ); + assert!( + fake.delete_starts_seen.lock().unwrap().is_empty(), + "no DELETING write" + ); + assert_fenced(&runtime); + let gets = mem.gets().len(); + assert!( + delete_backup(&*fake, &runtime, ACCOUNT, REGION, &own(&arn(ACCOUNT))) + .await + .is_err() + ); + assert_eq!(mem.gets().len(), gets, "refused at the gate"); + // The fence of a store already unhealthy refuses with no read. + let fence = runtime.configured_store().unwrap().fence(); assert!(matches!( - list(&fake, json!({ "Limit": "ten" })).await.unwrap_err(), - DynamoDbError::SerializationException(_) + fence.check("catalog-1").await, + Err(StoreProblem::ForeignCatalog { .. }) )); - for ok in [1, 100] { - list(&fake, json!({ "Limit": ok })).await.unwrap(); - } + assert_eq!(mem.gets().len(), gets, "the fence read nothing"); } +/// The adoption lands between DeleteBackup's two fences: the first passes, +/// the row moves to DELETING, and the fence before the first `delete_prefix` +/// refuses. The request answers DELETED, as for any deletion that fails +/// after the DELETING write; nothing is deleted, the row stays DELETING, the +/// store is unhealthy; the reconciler's DELETING duty acts on no row of it. #[tokio::test] -async fn list_backups_pages_in_a_stable_order_exactly_once() { - // Out of order, with two backups in the same millisecond. - let all = vec![ - summary(5, 500), - summary(1, 100), - summary(3, 300), - summary(2, 300), - summary(4, 400), - ]; - let mut expected: Vec = all.clone(); - expected.sort_by(|a, b| { - a.backup_creation_date_time - .total_cmp(&b.backup_creation_date_time) - .then_with(|| a.backup_arn.cmp(&b.backup_arn)) - }); - let expected: Vec = expected.into_iter().map(|s| s.backup_arn).collect(); +async fn an_adoption_between_delete_backups_two_fences_leaves_the_row_deleting() { + let mem = Arc::new(MemStore::default()); + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + let fake = Arc::new(Fake::default()); + seed(&fake, &mem, &arn(ACCOUNT), BackupStatus::Available); + let runtime = runtime_on(&mem, store()); + *mem.identity_after_read.lock().unwrap() = Some(adopted_file()); + let desc = delete_backup(&*fake, &runtime, ACCOUNT, REGION, &own(&arn(ACCOUNT))) + .await + .unwrap(); + assert_eq!(desc.backup_details.backup_status, "DELETED"); + assert!(mem.deletes.lock().unwrap().is_empty()); + assert_eq!(objects_under(&mem, &arn(ACCOUNT)).len(), 2); + assert_eq!( + fake.rows.lock().unwrap()[&arn(ACCOUNT)].status, + BackupStatus::Deleting + ); + assert_fenced(&runtime); + let report = cycle_on(&fake, &runtime).await; + assert!(report.deletions_finished.is_empty()); + assert!(mem.deletes.lock().unwrap().is_empty()); +} - let fake = fake_with_summaries(all); - let mut seen = Vec::new(); - let mut cursor: Option = None; - let mut pages = 0; - loop { - let mut body = json!({ "Limit": 2 }); - if let Some(c) = &cursor { - body["ExclusiveStartBackupArn"] = json!(c); - } - let page = list(&fake, body).await.unwrap(); - pages += 1; - seen.extend(arns(&page)); - match page.get("LastEvaluatedBackupArn") { - Some(c) => cursor = Some(c.as_str().unwrap().to_owned()), - None => break, +/// Each reconciler duty that deletes a prefix or removes a tombstone, on a +/// root another catalog adopts right after the cycle's check read the +/// identity file: the fence re-reads it before the deletion or removal and +/// refuses. No `delete_prefix` is called, no row is moved or removed, and +/// the store is unhealthy at once. +#[tokio::test] +async fn each_reconciler_deletion_on_a_root_adopted_after_the_check_is_refused() { + let now = seconds_since_epoch(); + let settings = JobSettings { + failed_row_retention: Duration::from_secs(600), + ..job_settings() + }; + // (cell, status, heartbeat_at, completed_at, objects under the prefix) + for (cell, status, heartbeat_at, completed_at, objects) in [ + ("DELETING row", BackupStatus::Deleting, now, None, true), + ( + "stale FAILED row", + BackupStatus::Failed, + now - 1000.0, + Some(now), + true, + ), + ("DELETED row", BackupStatus::Deleted, now, Some(now), true), + ( + "tombstone", + BackupStatus::Failed, + now - 1000.0, + Some(now - 1000.0), + false, + ), + ] { + let mem = Arc::new(MemStore::default()); + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + let fake = Arc::new(Fake::default()); + let backup = arn(ACCOUNT); + let mut r = row(&backup, status); + r.heartbeat_at = heartbeat_at; + r.completed_at = completed_at; + fake.rows.lock().unwrap().insert(backup.clone(), r); + if objects { + let prefix = BackupArn::parse(&backup).unwrap().store_prefix(); + mem.insert(&format!("{prefix}data/000001.json.gz"), b"x"); } - assert!(pages < 10, "paging did not terminate"); + let runtime = runtime_on(&mem, store()).with_job_settings(settings); + *mem.identity_after_read.lock().unwrap() = Some(adopted_file()); + let report = cycle_on(&fake, &runtime).await; + assert_eq!(report.store_healthy, Some(true), "{cell}: the check passed"); + assert!(report.deletions_finished.is_empty(), "{cell}"); + assert!(report.failed_prefixes_deleted.is_empty(), "{cell}"); + assert!(report.deleted_prefixes_swept.is_empty(), "{cell}"); + assert!(report.tombstones_removed.is_empty(), "{cell}"); + assert!(mem.deletes.lock().unwrap().is_empty(), "{cell}"); + assert_eq!( + objects_under(&mem, &backup).len(), + usize::from(objects), + "{cell}" + ); + assert_eq!(fake.rows.lock().unwrap()[&backup].status, status, "{cell}"); + assert!(fake.set_deleted.lock().unwrap().is_empty(), "{cell}"); + assert!(fake.removed_rows.lock().unwrap().is_empty(), "{cell}"); + assert_eq!( + mem.gets() + .iter() + .filter(|k| *k == STORE_IDENTITY_KEY) + .count(), + 2, + "{cell}: the check's read and the fence's" + ); + assert_fenced(&runtime); } - assert_eq!(seen, expected); - assert_eq!(pages, 3); } +/// CreateBackup on a store whose last check passed, after another catalog +/// adopted the root: the fence before the row and the first put refuses. +/// No row is claimed, nothing is put, and the store is unhealthy. #[tokio::test] -async fn list_backups_last_page_has_no_cursor() { - let fake = fake_with_summaries((0..4).map(|n| summary(n, n)).collect()); - let first = list(&fake, json!({ "Limit": 2 })).await.unwrap(); - let cursor = first["LastEvaluatedBackupArn"].as_str().unwrap().to_owned(); - let second = list( +async fn create_backup_on_a_root_another_catalog_adopted_writes_nothing() { + let mem = Arc::new(MemStore::default()); + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + let fake = Arc::new(Fake::default()); + snapshot_fixture(&fake, &music_table(), songs(3), true, None); + let runtime = job_runtime(&mem); + mem.insert(STORE_IDENTITY_KEY, &adopted_file()); + let err = create_music_backup(&fake, &runtime).await.unwrap_err(); + assert!( + validation_message(err).contains("unhealthy (owned by another catalog)"), + "the refusal names the store's problem" + ); + wait_for_jobs(&runtime).await; + assert!(fake.claims.lock().unwrap().is_empty()); + assert!(fake.rows.lock().unwrap().is_empty()); + assert!(mem.puts.lock().unwrap().is_empty()); + assert_eq!(runtime.free_slots(), 4, "the slot is released"); + assert_fenced(&runtime); +} + +/// The backup job's failure path on a root another catalog adopted while +/// the job ran: FAILED is written, the fence before the deletion refuses, +/// and no `delete_prefix` is called; the store is unhealthy. +#[tokio::test] +async fn the_job_failure_path_on_a_root_adopted_while_the_job_ran_deletes_nothing() { + let mem = Arc::new(MemStore::default()); + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + let fake = Arc::new(Fake::default()); + let gate = closed_gate(); + snapshot_fixture( &fake, - json!({ "Limit": 2, "ExclusiveStartBackupArn": cursor }), + &music_table(), + songs(3), + true, + Some(Arc::clone(&gate)), + ); + let runtime = job_runtime(&mem); + let details = create_music_backup(&fake, &runtime).await.unwrap(); + mem.insert(STORE_IDENTITY_KEY, &adopted_file()); + mem.fail_puts + .store(true, std::sync::atomic::Ordering::Release); + gate.add_permits(3); + wait_for_jobs(&runtime).await; + assert_eq!( + fake.rows.lock().unwrap()[&details.backup_arn].status, + BackupStatus::Failed + ); + assert!(mem.deletes.lock().unwrap().is_empty()); + assert_fenced(&runtime); +} + +/// Purge of a row on a root another catalog adopted after the command's +/// check: the fence before the first `delete_prefix` refuses, nothing is +/// deleted, the reason names the problem, and the store is unhealthy. +#[tokio::test] +async fn purge_on_a_root_another_catalog_adopted_deletes_nothing() { + let mem = Arc::new(MemStore::default()); + mem.insert(STORE_IDENTITY_KEY, &adopted_file()); + let fake = Fake::default(); + let backup = arn_n(1); + seed(&fake, &mem, &backup, BackupStatus::Available); + let configured = ConfiguredStore::new( + store().store_kind, + store().store_root, + Arc::clone(&mem) as Arc, + None, + Ok(store()), + ); + let read = fake.rows.lock().unwrap()[&backup].clone(); + let err = maintenance::purge_row( + &fake, + &*mem, + &configured.fence(), + &read, + Duration::from_secs(300), ) .await - .unwrap(); - assert_eq!(arns(&second).len(), 2); - assert!(second.get("LastEvaluatedBackupArn").is_none()); - - // Exactly the limit, from the start. - let all = list(&fake, json!({ "Limit": 4 })).await.unwrap(); - assert!(all.get("LastEvaluatedBackupArn").is_none()); - // Empty listing. - let none = list(&fake_with_summaries(vec![]), json!({})).await.unwrap(); - assert_eq!(none, json!({ "BackupSummaries": [] })); + .unwrap_err(); + assert!( + err.starts_with("the deletion was refused: ") && err.contains("names catalog adopter"), + "{err}" + ); + assert!(err.ends_with("the row is unchanged"), "{err}"); + assert!(mem.deletes.lock().unwrap().is_empty()); + assert_eq!(objects_under(&mem, &backup).len(), 2); + assert!(fake.removed_rows.lock().unwrap().is_empty()); + assert!( + fake.delete_starts_seen.lock().unwrap().is_empty(), + "no DELETING write" + ); + assert_eq!(fake.rows.lock().unwrap()[&backup], read); + assert_eq!( + configured.health(), + StoreHealth::Unhealthy(StoreProblem::ForeignCatalog { + file_catalog_id: "adopter".to_owned(), + catalog_id: "catalog-1".to_owned(), + }) + ); } +/// The marker follows the owner. A durable catalog's `--adopt-store` of a +/// root an in-memory catalog owned writes `ephemeral_catalog: false`; an +/// in-memory server's startup check then finds the root owned by a durable +/// catalog and writes nothing; and the re-check of an in-memory server +/// whose root lost its file (no rows for it) writes the file back with the +/// marker. #[tokio::test] -async fn list_backups_resumes_after_a_cursor_whose_backup_was_deleted() { - let all: Vec = (0..5).map(|n| summary(n, 100 * (n + 1))).collect(); - let deleted = all[2].backup_arn.clone(); - let remaining: Vec = all - .iter() - .filter(|s| s.backup_arn != deleted) - .cloned() - .collect(); - let fake = fake_with_summaries(remaining); - let page = list(&fake, json!({ "ExclusiveStartBackupArn": deleted })) - .await - .unwrap(); +async fn the_marker_is_the_owners_after_an_adoption_and_a_rewrite() { + use super::store_identity::{AdoptRule, Adoption, adopt}; + let mem = MemStore::default(); + mem.insert( + STORE_IDENTITY_KEY, + &identity_file_of("store-7", "in-memory-start", true), + ); + let durable = Fake::default(); + *durable.catalog_id.lock().unwrap() = Some("durable".to_owned()); + let adopted = adopt( + &mem, + StoreKind::Filesystem, + "/r", + &durable, + "postgres", + false, + AdoptRule::Command, + ) + .await + .unwrap(); + assert!(matches!(adopted, Adoption::Adopted { .. }), "{adopted:?}"); + let file = StoreIdentityFile::parse(&mem.objects.lock().unwrap()[STORE_IDENTITY_KEY]).unwrap(); + assert_eq!(file.catalog_id, "durable"); + assert!(!file.ephemeral_catalog, "the durable adopter's marker"); + + let puts = mem.puts.lock().unwrap().len(); + let in_memory = Fake::default(); + let err = store_identity::establish( + &mem, + StoreKind::Filesystem, + "/r", + &in_memory, + "sqlite", + true, + ) + .await + .unwrap_err(); assert_eq!( - arns(&page), - vec![all[3].backup_arn.clone(), all[4].backup_arn.clone()] + err, + StoreProblem::ForeignCatalog { + file_catalog_id: "durable".to_owned(), + catalog_id: "catalog-1".to_owned(), + } + ); + assert_eq!(mem.puts.lock().unwrap().len(), puts, "nothing written"); + assert_eq!( + StoreIdentityFile::parse(&mem.objects.lock().unwrap()[STORE_IDENTITY_KEY]).unwrap(), + file ); - // A deleted cursor after every remaining backup: an empty last page. - let fake = fake_with_summaries(all[..2].to_vec()); - let page = list(&fake, json!({ "ExclusiveStartBackupArn": deleted })) + let lost = MemStore::default(); + let known = StoreIdentity { + store_kind: StoreKind::Filesystem, + store_root: "/r".to_owned(), + store_id: "store-7".to_owned(), + }; + store_identity::recheck(&lost, &known, &in_memory, "sqlite", true) .await .unwrap(); - assert!(arns(&page).is_empty()); - assert!(page.get("LastEvaluatedBackupArn").is_none()); + let rewritten = + StoreIdentityFile::parse(&lost.objects.lock().unwrap()[STORE_IDENTITY_KEY]).unwrap(); + assert_eq!(rewritten.store_id, "store-7"); + assert_eq!(rewritten.catalog_id, "catalog-1"); + assert!(rewritten.ephemeral_catalog, "the in-memory owner's marker"); +} + +// --- the deletion fence when its read fails ----------------------------------- + +/// A healthy store whose identity file reads fail after the first `pass`. +fn failing_identity(mem: &MemStore, pass: usize, failure: IdentityFailure) { + *mem.identity_get_failure.lock().unwrap() = Some((pass, failure)); } +/// The fence's read of the identity file fails (the store answers every +/// other call) at each call site: DeleteBackup, CreateBackup, the job's +/// failure path and purge. Each call is refused as a failed store call +/// would be, nothing is deleted or put, and the store's health is left as +/// the last check set it. DeleteBackup, as for any store failure, goes on +/// to the DELETING write and answers DELETED, its deletion refused by the +/// second fence and the row left DELETING; the other calls move no row. #[tokio::test] -async fn list_backups_refuses_a_cursor_it_cannot_place() { - let fake = fake_with_summaries(vec![summary(1, 1)]); - // Well formed, ours, not listed, and its id (one of the two shapes - // earlier builds issued: 13 digits with the hex suffix, 13 bare digits) - // carries no time: `backup_id_epoch_millis` needs the 14-digit form. - for id in ["1791976431438-73d8d5bc", "1791976431438"] { - let legacy = format!("arn:aws:dynamodb:{REGION}:{ACCOUNT}:table/Music/backup/{id}"); - let msg = validation_message( - list(&fake, json!({ "ExclusiveStartBackupArn": legacy })) - .await - .unwrap_err(), - ); - assert!(msg.contains("ExclusiveStartBackupArn"), "{id}: {msg}"); - assert!(msg.contains("carries no creation time"), "{id}: {msg}"); - } +async fn a_fence_whose_read_fails_refuses_each_call_and_leaves_the_health() { + // DeleteBackup. + let mem = Arc::new(MemStore::default()); + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + let fake = Arc::new(Fake::default()); + seed(&fake, &mem, &arn(ACCOUNT), BackupStatus::Available); + let runtime = runtime_on(&mem, store()); + failing_identity(&mem, 0, IdentityFailure::Transport); + let desc = delete_backup(&*fake, &runtime, ACCOUNT, REGION, &own(&arn(ACCOUNT))) + .await + .unwrap(); + assert_eq!(desc.backup_details.backup_status, "DELETED"); + assert!(mem.deletes.lock().unwrap().is_empty(), "DeleteBackup"); + assert_eq!( + fake.rows.lock().unwrap()[&arn(ACCOUNT)].status, + BackupStatus::Deleting + ); + assert_eq!( + runtime.configured_store().unwrap().health(), + StoreHealth::Healthy + ); + // CreateBackup. + let mem = Arc::new(MemStore::default()); + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + let fake = Arc::new(Fake::default()); + snapshot_fixture(&fake, &music_table(), songs(3), true, None); + let runtime = job_runtime(&mem); + failing_identity(&mem, 0, IdentityFailure::Transport); + let err = create_music_backup(&fake, &runtime).await.unwrap_err(); + assert!( + matches!(err, DynamoDbError::InternalServerError(_)), + "{err:?}" + ); + wait_for_jobs(&runtime).await; + assert!(fake.claims.lock().unwrap().is_empty(), "CreateBackup"); + assert!(mem.puts.lock().unwrap().is_empty()); assert_eq!( - validation_message( - list(&fake, json!({ "ExclusiveStartBackupArn": "garbage" })) - .await - .unwrap_err() - ), - "Invalid Request: BackupArn is not valid" + runtime.configured_store().unwrap().health(), + StoreHealth::Healthy + ); + + // The job's failure path: FAILED is written, the fence's read fails, + // nothing is deleted. + let mem = Arc::new(MemStore::default()); + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + let fake = Arc::new(Fake::default()); + let gate = closed_gate(); + snapshot_fixture( + &fake, + &music_table(), + songs(3), + true, + Some(Arc::clone(&gate)), + ); + let runtime = job_runtime(&mem); + let details = create_music_backup(&fake, &runtime).await.unwrap(); + failing_identity(&mem, 0, IdentityFailure::Transport); + mem.fail_puts + .store(true, std::sync::atomic::Ordering::Release); + gate.add_permits(3); + wait_for_jobs(&runtime).await; + assert_eq!( + fake.rows.lock().unwrap()[&details.backup_arn].status, + BackupStatus::Failed ); + assert!(mem.deletes.lock().unwrap().is_empty(), "fail_row"); + assert_eq!( + runtime.configured_store().unwrap().health(), + StoreHealth::Healthy + ); + + // Purge, of an AVAILABLE and of a dead CREATING row. + for status in [BackupStatus::Available, BackupStatus::Creating] { + let mem = Arc::new(MemStore::default()); + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + let fake = Fake::default(); + let backup = arn_n(1); + seed(&fake, &mem, &backup, status); + let configured = ConfiguredStore::new( + store().store_kind, + store().store_root, + Arc::clone(&mem) as Arc, + None, + Ok(store()), + ); + failing_identity(&mem, 0, IdentityFailure::Transport); + let read = fake.rows.lock().unwrap()[&backup].clone(); + let err = maintenance::purge_row( + &fake, + &*mem, + &configured.fence(), + &read, + Duration::from_secs(300), + ) + .await + .unwrap_err(); + assert!(err.starts_with("the deletion was refused: "), "{err}"); + assert!(mem.deletes.lock().unwrap().is_empty(), "purge {status}"); + assert_eq!(fake.rows.lock().unwrap()[&backup], read, "purge {status}"); + assert!(fake.delete_starts_seen.lock().unwrap().is_empty()); + assert_eq!(configured.health(), StoreHealth::Healthy); + } } -/// Measured on DynamoDB: a cursor of another account, -/// another region, or a table other than `TableName`'s is a -/// `ValidationException`, in that order; the account text is the service's, -/// the other two complete the fragments captured ("does not match the -/// region", "does not match the table name predicate"). +/// The fence's read fails at each reconciler duty that deletes a prefix or +/// removes a tombstone (the cycle's check read the file first): nothing is +/// deleted, no row moves, and the store stays healthy. #[tokio::test] -async fn list_backups_cursor_must_match_the_account_region_and_table() { - let fake = fake_with_summaries(vec![summary(1, 1), summary(2, 2)]); - let id = "01791976431438-73d8d5bc"; - let cases = [ - ( - format!("arn:aws:dynamodb:{REGION}:999999999999:table/Music/backup/{id}"), - None, - "account", - ), - ( - format!("arn:aws:dynamodb:eu-west-1:{ACCOUNT}:table/Music/backup/{id}"), - None, - "region", - ), - ( - format!("arn:aws:dynamodb:{REGION}:{ACCOUNT}:table/Other/backup/{id}"), - Some("Music"), - "table name predicate", - ), - // Account before region before table. - ( - format!("arn:aws:dynamodb:eu-west-1:999999999999:table/Other/backup/{id}"), - Some("Music"), - "account", - ), +async fn each_reconciler_deletion_whose_fence_read_fails_is_refused_with_the_health_left() { + let now = seconds_since_epoch(); + let settings = JobSettings { + failed_row_retention: Duration::from_secs(600), + ..job_settings() + }; + for (cell, status, heartbeat_at, completed_at, objects) in [ + ("DELETING row", BackupStatus::Deleting, now, None, true), ( - format!("arn:aws:dynamodb:eu-west-1:{ACCOUNT}:table/Other/backup/{id}"), - Some("Music"), - "region", + "stale FAILED row", + BackupStatus::Failed, + now - 1000.0, + Some(now), + true, ), - ]; - for (cursor, table, what) in cases { - let mut body = json!({ "ExclusiveStartBackupArn": cursor }); - if let Some(table) = table { - body["TableName"] = json!(table); + ("DELETED row", BackupStatus::Deleted, now, Some(now), true), + ( + "tombstone", + BackupStatus::Failed, + now - 1000.0, + Some(now - 1000.0), + false, + ), + ] { + let mem = Arc::new(MemStore::default()); + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + let fake = Arc::new(Fake::default()); + let backup = arn(ACCOUNT); + let mut r = row(&backup, status); + r.heartbeat_at = heartbeat_at; + r.completed_at = completed_at; + fake.rows.lock().unwrap().insert(backup.clone(), r); + if objects { + let prefix = BackupArn::parse(&backup).unwrap().store_prefix(); + mem.insert(&format!("{prefix}data/000001.json.gz"), b"x"); } - let msg = validation_message(list(&fake, body).await.unwrap_err()); + let runtime = runtime_on(&mem, store()).with_job_settings(settings); + failing_identity(&mem, 1, IdentityFailure::Transport); + let report = cycle_on(&fake, &runtime).await; + assert_eq!(report.store_healthy, Some(true), "{cell}: the check passed"); + assert!(report.deletions_finished.is_empty(), "{cell}"); + assert!(report.failed_prefixes_deleted.is_empty(), "{cell}"); + assert!(report.deleted_prefixes_swept.is_empty(), "{cell}"); + assert!(report.tombstones_removed.is_empty(), "{cell}"); + assert!(mem.deletes.lock().unwrap().is_empty(), "{cell}"); + assert_eq!(fake.rows.lock().unwrap()[&backup].status, status, "{cell}"); + assert!(fake.set_deleted.lock().unwrap().is_empty(), "{cell}"); + assert!(fake.removed_rows.lock().unwrap().is_empty(), "{cell}"); assert_eq!( - msg, - format!( - "Invalid Request: The supplied ExclusiveStartBackupArn does not match the {what}" - ), - "{cursor}" + runtime.configured_store().unwrap().health(), + StoreHealth::Healthy, + "{cell}" ); } - // Another table's cursor without TableName is fine (the listing is the - // account's), and a trailing newline is trimmed. - let cursor = format!("arn:aws:dynamodb:{REGION}:{ACCOUNT}:table/Other/backup/{id}\n"); - list(&fake, json!({ "ExclusiveStartBackupArn": cursor })) +} + +/// A root with no identity file under the catalog's rows (removed since +/// the last check) refuses the fenced deletion: DeleteBackup, as for a store +/// failure, answers DELETED with nothing deleted and the row DELETING for +/// the reconciler, and the health is left for the next check to set. +#[tokio::test] +async fn a_fence_on_a_root_with_no_identity_file_refuses() { + let mem = Arc::new(MemStore::default()); + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + let fake = Arc::new(Fake::default()); + seed(&fake, &mem, &arn(ACCOUNT), BackupStatus::Available); + let runtime = runtime_on(&mem, store()); + mem.objects.lock().unwrap().remove(STORE_IDENTITY_KEY); + let desc = delete_backup(&*fake, &runtime, ACCOUNT, REGION, &own(&arn(ACCOUNT))) .await .unwrap(); - // The cursor of the listing itself resumes after it. - let first = summary(1, 1).backup_arn; - let page = list( - &fake, - json!({ "ExclusiveStartBackupArn": format!("{first}\n"), "TableName": "Music" }), + assert_eq!(desc.backup_details.backup_status, "DELETED"); + assert!(mem.deletes.lock().unwrap().is_empty()); + assert_eq!(objects_under(&mem, &arn(ACCOUNT)).len(), 2); + assert_eq!( + fake.rows.lock().unwrap()[&arn(ACCOUNT)].status, + BackupStatus::Deleting + ); + assert_eq!( + runtime.configured_store().unwrap().health(), + StoreHealth::Healthy + ); +} + +/// A fence whose read of the identity file never answers is refused at the +/// store's check bound: DeleteBackup answers DELETED once the bound passes +/// (the second fence is refused at once, the first read still parked), with +/// nothing deleted, the row DELETING and the health left. +#[tokio::test(start_paused = true)] +async fn a_fence_whose_read_stalls_is_refused_at_its_bound() { + let mem = Arc::new(MemStore::default()); + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + let fake = Arc::new(Fake::default()); + seed(&fake, &mem, &arn(ACCOUNT), BackupStatus::Available); + let runtime = runtime_on(&mem, store()); + let bound = runtime.configured_store().unwrap().check_timeout(); + failing_identity(&mem, 0, IdentityFailure::Stall); + let started = tokio::time::Instant::now(); + // A guard far past the bound, so a fence with no bound fails here + // instead of hanging. + let desc = tokio::time::timeout( + bound * 10, + delete_backup(&*fake, &runtime, ACCOUNT, REGION, &own(&arn(ACCOUNT))), ) .await + .expect("the fence's read is bounded") .unwrap(); - assert_eq!(arns(&page), vec![summary(2, 2).backup_arn]); + assert_eq!(desc.backup_details.backup_status, "DELETED"); + assert!(started.elapsed() >= bound, "{:?}", started.elapsed()); + assert!(started.elapsed() < bound + Duration::from_secs(1)); + assert!(mem.deletes.lock().unwrap().is_empty()); + assert_eq!( + fake.rows.lock().unwrap()[&arn(ACCOUNT)].status, + BackupStatus::Deleting + ); + assert_eq!( + runtime.configured_store().unwrap().health(), + StoreHealth::Healthy + ); } -/// A backup created under an earlier `server.region` keeps that region in its -/// stored ARN, and `ListBackups` hands that ARN out as the cursor; the next -/// page accepts it (trimmed). Another region's ARN that is not a listed -/// backup's is still refused. +/// A fence refusal in the cycle's DELETING duty makes the store unhealthy, +/// and the cycle runs no later store duty on the root: the dead-owner duty +/// reads nothing under the dead CREATING row's prefix and leaves the row, +/// and the FAILED-prefix, DELETED-prefix and tombstone duties list nothing +/// under a stale FAILED row's and a DELETED row's prefixes, both past their +/// windows. #[tokio::test] -async fn list_backups_pages_past_a_backup_stored_under_another_region() { - let mut old = summary(2, 2); - old.backup_arn = old.backup_arn.replace(REGION, "eu-west-1"); - let all = vec![summary(1, 1), old.clone(), summary(3, 3)]; - let fake = fake_with_summaries(all); - let mut seen = Vec::new(); - let mut body = json!({ "Limit": 2 }); - let first = list(&fake, body.clone()).await.unwrap(); - seen.extend(arns(&first)); - assert_eq!( - first["LastEvaluatedBackupArn"].as_str(), - Some(old.backup_arn.as_str()) - ); - body["ExclusiveStartBackupArn"] = json!(format!("{}\n", old.backup_arn)); - for table in [None, Some("Music")] { - if let Some(t) = table { - body["TableName"] = json!(t); - } - let second = list(&fake, body.clone()).await.unwrap(); - assert_eq!(arns(&second), vec![summary(3, 3).backup_arn], "{body}"); +async fn a_fence_refusal_in_the_deleting_duty_stops_the_dead_owner_duty() { + let mem = Arc::new(MemStore::default()); + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + let fake = Arc::new(Fake::default()); + let dead = seed_dead_creating_with_manifest(&fake, &mem, true); + let deleting = arn_n(1); + assert_ne!(dead, deleting); + seed(&fake, &mem, &deleting, BackupStatus::Deleting); + let stale_failed = arn_n(2); + let deleted = arn_n(3); + let now = seconds_since_epoch(); + for (a, status) in [ + (&stale_failed, BackupStatus::Failed), + (&deleted, BackupStatus::Deleted), + ] { + let mut r = row(a, status); + r.heartbeat_at = now - 1_000_000.0; + r.completed_at = Some(now - 1_000_000.0); + fake.rows.lock().unwrap().insert(a.clone(), r); + mem.insert(&format!("{}data/000001.json.gz", prefix_of(a)), b"x"); } - seen.extend(arns(&list(&fake, body.clone()).await.unwrap())); - assert_eq!(seen.len(), 3); - let stranger = summary(9, 9).backup_arn.replace(REGION, "eu-west-1"); - let msg = validation_message( - list(&fake, json!({ "ExclusiveStartBackupArn": stranger })) - .await - .unwrap_err(), + let runtime = runtime_on(&mem, store()); + *mem.identity_after_read.lock().unwrap() = Some(adopted_file()); + let report = cycle_on(&fake, &runtime).await; + assert_eq!(report.store_healthy, Some(true), "the check passed"); + assert!(report.deletions_finished.is_empty()); + assert!(report.creating_resolved.is_empty()); + assert_fenced(&runtime); + let dead_prefix = prefix_of(&dead); + assert!( + !mem.gets().iter().any(|k| k.starts_with(&dead_prefix)), + "{:?}", + mem.gets() ); assert_eq!( - msg, - "Invalid Request: The supplied ExclusiveStartBackupArn does not match the region" + fake.rows.lock().unwrap()[&dead].status, + BackupStatus::Creating ); + assert!(fake.takeovers.lock().unwrap().is_empty()); + assert!(mem.deletes.lock().unwrap().is_empty()); + for a in [&stale_failed, &deleted] { + let prefix = prefix_of(a); + assert!( + !mem.lists.lock().unwrap().contains(&prefix), + "{a}: listed after the refusal: {:?}", + mem.lists.lock().unwrap() + ); + assert!(fake.rows.lock().unwrap().contains_key(a.as_str())); + } } -/// Measured on DynamoDB: `TableName` on `ListBackups` is checked as the service -/// model constrains it. +/// A read of the row under the ARN's spellings that fails skips the prefix +/// as a catalog error, which a later run may adopt: no insert is sent, +/// since the read is what keeps a second row of the id out on Cassandra. +/// The next sync, with the read answering, adopts the prefix. #[tokio::test] -async fn list_backups_table_name_is_validated() { - let fake = fake_with_summaries(vec![summary(1, 1)]); - let msg = validation_message(list(&fake, json!({ "TableName": "a" })).await.unwrap_err()); - assert_eq!( - msg, - "1 validation error detected: Value 'a' at 'tableName' failed to satisfy constraint: \ - Member must have length greater than or equal to 3" +async fn sync_skips_a_prefix_whose_row_read_fails_and_inserts_nothing() { + use super::sync::SkipReason; + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + let identity = store(); + let backup = sync_arn(1); + put_backup(&mem, &backup); + *fake.read_row_error.lock().unwrap() = Some("read timed out".to_owned()); + let report = super::sync::sync(&*fake, sync_target(&mem, &identity)) + .await + .unwrap(); + assert!(report.adopted.is_empty(), "{:?}", report.adopted); + assert!( + matches!(&skipped_by_prefix(&report)[&prefix_of(&backup)], SkipReason::Catalog(e) if e.contains("read timed out")), + "{:?}", + report.skipped ); - assert!(fake.listed_for.lock().unwrap().is_empty()); - list(&fake, json!({ "TableName": "Music" })).await.unwrap(); + assert_eq!(report.transient_skips(), 1); + assert!(fake.insert_attempts.lock().unwrap().is_empty()); + assert!(fake.rows.lock().unwrap().is_empty()); + + *fake.read_row_error.lock().unwrap() = None; + let report = super::sync::sync(&*fake, sync_target(&mem, &identity)) + .await + .unwrap(); + assert_eq!(report.adopted.len(), 1); assert_eq!( - fake.listed_for.lock().unwrap().as_slice(), - [Some("Music".to_owned())] + fake.rows.lock().unwrap()[&backup].status, + BackupStatus::Available ); } +/// On a catalog whose rows are unique by ARN alone (Cassandra's rule), two +/// prefixes of one account and backup id under two tables, each with a +/// complete manifest naming its own path, are not both adopted in one run: +/// the first gets its row, and the second is skipped on the row check as +/// the id's row under the other table. Without the row check the insert +/// would accept the second, since no backup-id uniqueness refuses it. #[tokio::test] -async fn list_backups_passes_the_table_filter_through() { - let fake = fake_with_summaries(vec![]); - list(&fake, json!({ "TableName": "Music" })).await.unwrap(); - list(&fake, json!({})).await.unwrap(); +async fn one_run_adopts_one_prefix_of_a_backup_id_where_rows_are_unique_by_arn() { + use super::sync::SkipReason; + let mem = Arc::new(MemStore::default()); + let fake = Arc::new(Fake::default()); + fake.unique_by_arn + .store(true, std::sync::atomic::Ordering::Release); + let identity = store(); + let music = sync_arn(1); + let other = music.replace("table/Music", "table/Other"); + put_backup(&mem, &music); + put_backup(&mem, &other); + let report = super::sync::sync(&*fake, sync_target(&mem, &identity)) + .await + .unwrap(); + // `Music` sorts before `Other`, so the listing reaches it first. assert_eq!( - *fake.listed_for.lock().unwrap(), - vec![Some("Music".to_owned()), None] + report + .adopted + .iter() + .map(|a| a.backup_arn.clone()) + .collect::>(), + std::slice::from_ref(&music) + ); + assert_eq!( + skipped_by_prefix(&report)[&prefix_of(&other)], + SkipReason::HasRow { + status: BackupStatus::Available, + store_id: "store-1".to_owned(), + other_table: Some("Music".to_owned()), + } ); + assert_eq!(fake.insert_attempts.lock().unwrap().len(), 1); + assert!(!fake.rows.lock().unwrap().contains_key(&other)); } -#[tokio::test] -async fn list_backups_filters_on_backup_type() { - let mut system = summary(2, 200); - system.backup_type = "SYSTEM".to_owned(); - let mut aws_backup = summary(3, 300); - aws_backup.backup_type = "AWS_BACKUP".to_owned(); - let user = summary(1, 100); - let fake = fake_with_summaries(vec![user.clone(), system.clone(), aws_backup.clone()]); - for (filter, expected) in [ - (json!(null), vec![user.backup_arn.clone()]), - (json!("USER"), vec![user.backup_arn.clone()]), - (json!("SYSTEM"), vec![system.backup_arn.clone()]), - (json!("AWS_BACKUP"), vec![aws_backup.backup_arn.clone()]), - ( - json!("ALL"), - vec![ - user.backup_arn.clone(), - system.backup_arn.clone(), - aws_backup.backup_arn.clone(), - ], - ), - ] { - let page = list(&fake, json!({ "BackupType": filter })).await.unwrap(); - assert_eq!(arns(&page), expected, "{filter}"); - } - let page = list(&fake, json!({})).await.unwrap(); - assert_eq!(arns(&page), vec![user.backup_arn.clone()]); - - let msg = validation_message( - list(&fake, json!({ "BackupType": "user" })) - .await - .unwrap_err(), +/// A cycle whose startup sync meets a root listing that never answers +/// returns as soon as the shutdown token is cancelled, well before the +/// sync's own bound, with the sync left pending and no later duty run. +#[tokio::test(start_paused = true)] +async fn a_cycles_startup_sync_ends_at_the_shutdown_and_stays_pending() { + let mem = Arc::new(MemStore::default()); + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + let fake = Arc::new(Fake::default()); + let deleting = arn(ACCOUNT); + seed(&fake, &mem, &deleting, BackupStatus::Deleting); + put_backup(&mem, &sync_arn(1)); + let runtime = runtime().with_store( + ConfiguredStore::new( + store().store_kind, + store().store_root, + Arc::clone(&mem) as Arc, + None, + Ok(store()), + ) + .with_ephemeral_catalog(true), ); + mem.stall_root_list + .store(true, std::sync::atomic::Ordering::Release); + let shutdown = extenddb_storage::CancellationToken::new(); + let cancel = shutdown.clone(); + tokio::spawn(async move { + tokio::time::sleep(Duration::from_secs(5)).await; + cancel.cancel(); + }); + let started = tokio::time::Instant::now(); + // A guard past the bound, so a cycle that ignores the token fails here + // instead of hanging. + let report = tokio::time::timeout( + super::sync::STARTUP_SYNC_TIMEOUT * 5, + reconciler::cycle( + &fake, + &runtime, + Duration::from_secs(300), + "test", + &mut reconciler::ReconcilerState::with_shutdown(shutdown.clone()), + ), + ) + .await + .expect("the cycle ends") + .unwrap(); + let took = started.elapsed(); + assert!(shutdown.is_cancelled()); + assert!(took < super::sync::STARTUP_SYNC_TIMEOUT, "{took:?}"); + assert_eq!(report.store_healthy, Some(true)); + assert!(report.deletions_finished.is_empty(), "{report:?}"); assert_eq!( - msg, - "1 validation error detected: Value 'user' at 'backupType' failed to satisfy \ - constraint: Member must satisfy enum value set: [USER, SYSTEM, AWS_BACKUP, ALL]" + fake.rows.lock().unwrap()[&deleting].status, + BackupStatus::Deleting ); - assert!(matches!( - list(&fake, json!({ "BackupType": 1 })).await.unwrap_err(), - DynamoDbError::SerializationException(_) - )); + assert!( + runtime.configured_store().unwrap().startup_sync_pending(), + "left pending" + ); + assert!(fake.insert_attempts.lock().unwrap().is_empty()); } +/// The adoption under the in-memory rule refuses a root whose file was +/// written by a durable catalog, with nothing written; the check of an +/// in-memory catalog does not hand such a root to the adoption at all (one +/// read of the file) and leaves it owned by another catalog. #[tokio::test] -async fn list_backups_filters_on_the_time_range_before_paging() { - // Creation times 1.000, 2.000, 3.000, 4.000, 5.000 s. - let all: Vec = (1..=5).map(|n| summary(n, 1_000 * n)).collect(); - let arn_of = |n: usize| all[n - 1].backup_arn.clone(); - let fake = fake_with_summaries(all.clone()); - for (body, expected) in [ - // Lower bound inclusive, upper bound exclusive, at the millisecond. - (json!({ "TimeRangeLowerBound": 2.0 }), vec![2, 3, 4, 5]), - (json!({ "TimeRangeLowerBound": 2.001 }), vec![3, 4, 5]), - (json!({ "TimeRangeUpperBound": 4.0 }), vec![1, 2, 3]), - (json!({ "TimeRangeUpperBound": 4.001 }), vec![1, 2, 3, 4]), - ( - json!({ "TimeRangeLowerBound": 2, "TimeRangeUpperBound": 4 }), - vec![2, 3], - ), - ( - json!({ "TimeRangeLowerBound": 3, "TimeRangeUpperBound": 3 }), - vec![], - ), - ( - json!({ "TimeRangeLowerBound": null, "TimeRangeUpperBound": null }), - vec![1, 2, 3, 4, 5], - ), - ( - json!({ "TimeRangeLowerBound": 0, "TimeRangeUpperBound": 0 }), - vec![], - ), - ] { - let page = list(&fake, body.clone()).await.unwrap(); - let want: Vec = expected.into_iter().map(arn_of).collect(); - assert_eq!(arns(&page), want, "{body}"); - } - - // Measured on DynamoDB: a lower bound above the upper - // bound and a bound before the epoch are refused. - assert_eq!( - validation_message( - list( - &fake, - json!({ "TimeRangeLowerBound": 4, "TimeRangeUpperBound": 2 }) - ) - .await - .unwrap_err() - ), - "Time range lower bound must be less than or equal to upper bound" +async fn the_in_memory_adoption_refuses_a_durable_owners_root() { + use super::store_identity::{AdoptRefusal, AdoptRule, adopt}; + let mem = MemStore::default(); + mem.insert( + STORE_IDENTITY_KEY, + &identity_file_of("store-7", "durable", false), ); + let fake = Fake::default(); + let err = adopt( + &mem, + StoreKind::Filesystem, + "/r", + &fake, + "sqlite", + true, + AdoptRule::EphemeralOwner, + ) + .await + .unwrap_err(); assert_eq!( - validation_message( - list( - &fake, - json!({ "TimeRangeLowerBound": 2.5, "TimeRangeUpperBound": 2.4999 }) - ) - .await - .unwrap_err() - ), - "Time range lower bound must be less than or equal to upper bound" + err, + AdoptRefusal::DurableOwner { + file_catalog_id: "durable".to_owned(), + } ); - for member in ["TimeRangeLowerBound", "TimeRangeUpperBound"] { - for at in [-1, -1_000_000_000] { - let msg = validation_message(list(&fake, json!({ member: at })).await.unwrap_err()); - assert!(msg.contains(member), "{member} {at}: {msg}"); - assert!(msg.contains("1970-01-01T00:00:00Z"), "{member} {at}: {msg}"); + assert!(mem.puts.lock().unwrap().is_empty()); + + let err = store_identity::establish(&mem, StoreKind::Filesystem, "/r", &fake, "sqlite", true) + .await + .unwrap_err(); + assert_eq!( + err, + StoreProblem::ForeignCatalog { + file_catalog_id: "durable".to_owned(), + catalog_id: "catalog-1".to_owned(), } - } + ); + let file_reads = |mem: &MemStore| { + mem.gets() + .iter() + .filter(|k| *k == STORE_IDENTITY_KEY) + .count() + }; + assert_eq!(file_reads(&mem), 2, "the adoption's read, then the check's"); + assert!(mem.puts.lock().unwrap().is_empty()); +} - // Paging walks the filtered list: Limit counts only backups in range, and - // the last in-range page has no cursor although later backups exist. - let mut seen = Vec::new(); - let mut body = json!({ "TimeRangeLowerBound": 2, "TimeRangeUpperBound": 4.5, "Limit": 2 }); - let first = list(&fake, body.clone()).await.unwrap(); - seen.extend(arns(&first)); +/// The fence refuses a file that names this catalog under another store id +/// than the one the store's check settled (the root replaced by another +/// root of the same catalog), and records the mismatch on the store; a file +/// with the settled id passes. +#[tokio::test] +async fn a_fence_refuses_a_file_of_this_catalog_under_another_store_id() { + let mem = Arc::new(MemStore::default()); + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-1", "catalog-1")); + let fake = Arc::new(Fake::default()); + seed(&fake, &mem, &arn(ACCOUNT), BackupStatus::Available); + let runtime = runtime_on(&mem, store()); + let fence = runtime.configured_store().unwrap().fence(); + fence.check("catalog-1").await.unwrap(); + + mem.insert(STORE_IDENTITY_KEY, &identity_file("store-2", "catalog-1")); + let mismatch = StoreProblem::IdMismatch { + file_id: "store-2".to_owned(), + row_ids: vec!["store-1".to_owned()], + }; + assert_eq!(fence.check("catalog-1").await, Err(mismatch.clone())); assert_eq!( - first["LastEvaluatedBackupArn"].as_str(), - Some(arn_of(3).as_str()) + runtime.configured_store().unwrap().health(), + StoreHealth::Unhealthy(mismatch) ); - body["ExclusiveStartBackupArn"] = first["LastEvaluatedBackupArn"].clone(); - let second = list(&fake, body).await.unwrap(); - seen.extend(arns(&second)); + let err = delete_backup(&*fake, &runtime, ACCOUNT, REGION, &own(&arn(ACCOUNT))) + .await + .unwrap_err(); assert!( - second.get("LastEvaluatedBackupArn").is_none(), - "the last page carries a cursor" + matches!(&err, DynamoDbError::ValidationException(m) if m.contains("identity mismatch")), + "{err:?}" + ); + assert!(mem.deletes.lock().unwrap().is_empty()); + assert_eq!( + fake.rows.lock().unwrap()[&arn(ACCOUNT)].status, + BackupStatus::Available ); - assert_eq!(seen, vec![arn_of(2), arn_of(3), arn_of(4)]); - - for member in ["TimeRangeLowerBound", "TimeRangeUpperBound"] { - assert!(matches!( - list(&fake, json!({ member: "2026-01-01" })).await.unwrap_err(), - DynamoDbError::SerializationException(m) if m.contains(member) - )); - } } diff --git a/crates/engine/src/create_table.rs b/crates/engine/src/create_table.rs index 7b827cd3a..ff51fac17 100755 --- a/crates/engine/src/create_table.rs +++ b/crates/engine/src/create_table.rs @@ -150,6 +150,12 @@ pub fn storage_err_to_dynamo(e: extenddb_storage::error::StorageError) -> Dynamo StorageError::IndexAlreadyExists(name) => DynamoDbError::ValidationException(format!( "One or more parameter values were invalid: Index already exists: {name}" )), + // Raised by the backup catalog's claim alone, whose caller handles + // it; here it is a programming error, reported without the row. + StorageError::AlreadyExists(msg) => { + tracing::error!(internal_error = %msg, "unexpected already-exists storage error"); + DynamoDbError::InternalServerError("Internal server error".to_owned()) + } // The resource exists and the request is well formed; its current state // forbids the change. The backend owns the wording because it owns the // state: the documented delete-table sentence and the measured diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index b8e372742..501720e57 100755 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -8,6 +8,12 @@ //! `DynamoDB`-format error responses. mod backup; +pub use backup::{ + BackupRuntime, ConfiguredStore, HeldClaim, JobSettings, KillPoint, LOCAL_CHECK_TIMEOUT, + OTHER_STORE, RESTORE_REGISTRATION_TIMEOUT, S3_CHECK_TIMEOUT, STORE_IDENTITY_KEY, Spool, + StoreFence, StoreHealth, StoreHealthReport, StoreIdentityFile, StoreProblem, TARGET_NOT_EMPTY, + check_configured_store, maintenance, read_identity_bounded, reconciler, store_identity, sync, +}; mod batch_get_item; mod batch_write_item; pub mod capacity_helpers; @@ -342,6 +348,9 @@ pub type AuthorizedKeyInfo = std::collections::HashMap, + /// The process's job slots and configured backup store, which the backup + /// operations read. + pub backup: Arc, pub limits: Arc, pub region: Arc, pub account_id: Arc, diff --git a/crates/server/Cargo.toml b/crates/server/Cargo.toml index 00814e1a2..54bffcbcc 100755 --- a/crates/server/Cargo.toml +++ b/crates/server/Cargo.toml @@ -9,6 +9,15 @@ license.workspace = true [features] mongodb-test-hooks = [] +# The backup and restore jobs' kill-point pauses (`EXTENDDB_BACKUP_PAUSE`), +# for the binary-level crash tests. The engine's points are its own feature; +# the points inside the backends' writes (`restore_sqlite_mid_batch`, +# `restore_postgres_before_progress_delete`, +# `restore_mongodb_after_insert_many`, `restore_cassandra_before_progress`) +# are calls of `extenddb_storage::backup_catalog::restore_pause_point`, so +# `extenddb-storage/test-hooks` arms them in whichever backend the build +# selects without naming a backend crate here (the server depends on none). +backup-test-hooks = ["extenddb-engine/backup-test-hooks", "extenddb-storage/test-hooks"] [dependencies] extenddb-cache = { workspace = true } @@ -16,6 +25,7 @@ extenddb-core = { workspace = true } extenddb-config = { workspace = true } extenddb-engine = { workspace = true } extenddb-storage = { workspace = true } +extenddb-backup-store = { workspace = true } extenddb-auth = { workspace = true } futures = { workspace = true } axum = { workspace = true } @@ -50,3 +60,5 @@ libc = { workspace = true } [dev-dependencies] extenddb-cache = { workspace = true, features = ["test-util"] } +tempfile = "3" +tokio = { workspace = true, features = ["test-util"] } diff --git a/crates/server/src/backup_store.rs b/crates/server/src/backup_store.rs new file mode 100644 index 000000000..2743ea835 --- /dev/null +++ b/crates/server/src/backup_store.rs @@ -0,0 +1,1743 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Startup of the backup store: the `[backup.store]` resolution, `open`, +//! the spool and its lock, and, for a filesystem store, `validate` and the +//! identity check; the split between a configuration error, which stops +//! startup, and a store condition or identity failure, which starts the +//! server with the store unhealthy; and the rule that nothing here waits on +//! the network, so a store endpoint that stalls never holds the data plane. +//! +//! Configuration errors, each named with the path or bucket in the store's +//! own message, every one decided without a request to the store: `open` +//! refusing the configuration (a missing section, an empty or S3 Express +//! bucket name, a part size, prefix or key-length problem, an insecure +//! endpoint, a filesystem root that does not exist or fails the owner-only +//! rule), a filesystem store's `validate` refusing the root for a reason of +//! the same kind (a root another user can reach, a root replaced by a +//! symlink), a spool that breaks its placement rule, that another live +//! instance holds or that exists with a mode other than owner-only, and an +//! S3 store without a spool on a backend with no default. A filesystem +//! root's `validate` also writes a probe file, which is a request to the +//! store: an I/O failure there or in resolving the root (a full or read-only +//! disk, a stale network handle, a root that vanished after `open`) is a +//! store condition and degrades the store, as the same failure one cycle +//! later does, rather than stopping startup. +//! +//! The one exception to the configuration errors is the defaulted SQLite +//! root (`backups/` beside the database file when `[backup.store]` is +//! absent) while the catalog has no backup rows for it: a failure of its +//! creation, `open`, `validate` or identity check is a logged warning that +//! leaves the store unconfigured, so an upgrade, a fresh deployment or a +//! catalog re-created by `destroy` and `init` under the old root never stops +//! a server that never configured backups. A defaulted root whose startup +//! checks did not finish within their bound starts `checking` with the rule +//! kept ([`ConfiguredStore::defaulted`]): the reconciler applies it to the +//! first check's outcome, so a failure then with no rows is the same warning +//! and leaves the store unconfigured. Once the catalog has rows for +//! that root ([`extenddb_storage::BackupCatalog::read_store_ids`]), the root +//! has been used and its backups must not vanish behind a warning: the +//! defaulted root is then handled as an explicitly configured one, a +//! refusal of `open` or `validate` a startup error, and an identity failure +//! an unhealthy store (reported on `/health`, its rows' restores and deletes +//! refused naming the kind and the category, rechecked each cycle). A +//! command that found no store learns why from [`NotConfigured`]. `backup +//! sync` ([`Caller::SyncCommand`]) is the one exception to the leniency: a +//! defaulted root whose identity file names another catalog is configured +//! and unhealthy for it, so that `--adopt-store` can adopt the root a +//! catalog re-created by `destroy` and `init` finds. +//! +//! A filesystem store's `validate` and identity check run here, before the +//! listener, bounded by [`LOCAL_CHECK_TIMEOUT`]: a root on a local disk +//! answers at once, and a mount that stops answering after the root is +//! opened leaves the server starting with the store `checking`; a root that +//! does not answer at startup holds the server before the listener, as the +//! database directory would (the banner's resolution of the root, the +//! default root's creation and `open`'s own resolution run unbounded, before +//! the checks). The socket is bound before the banner, so a startup held +//! that way leaves the port accepting connections it never answers until +//! the mount answers or the process is killed. A check abandoned at the +//! bound leaves its filesystem call parked on a blocking thread until the +//! mount answers (the engine's runtime module says how the later checks +//! avoid parking another); the thread costs nothing at exit, where the +//! binary ends its runtime with a bounded shutdown. An S3 store's +//! `validate` and identity check are requests to the endpoint, and run after +//! the listener is up, in the reconciler's first cycle +//! ([`extenddb_engine::check_configured_store`]); the store is `checking` +//! until that cycle records its result, and the operations that touch the +//! store refuse meanwhile as they do while it is unhealthy. A `validate` +//! refusal that is not a transport failure (a missing or denied bucket) then +//! makes the store unhealthy rather than stopping startup. +//! +//! A SQLite catalog with no database file (`:memory:`, the `sqlite-memory` +//! build's default) is new at every start, `catalog_id` included. Its +//! store's checks write `ephemeral_catalog: true` in an identity file they +//! create and adopt a root whose file another in-memory catalog wrote +//! (`extenddb_engine::store_identity`'s module doc), and its startup sync +//! is pending: the server runs it before it serves when the filesystem +//! store's check passed here, and the reconciler runs it after the first +//! check that passes otherwise (`extenddb_engine::sync::startup_sync`). A +//! root a durable catalog owns stays unhealthy there as anywhere. +//! +//! Transport failures (the store unreachable: no region, no credentials, a +//! refused connection) and failures of the identity check or the manifest +//! probe (a denial the store reports while a check runs, say a directory +//! under the root that another user can reach, included) leave the server +//! running with the store unhealthy: logged once, reported on `/health`, +//! the operations that touch the store refused, and the check run again +//! each reconciler cycle until it passes. + +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use extenddb_backup_store::{BackupStore, BackupStoreKind, StoreError}; +use extenddb_config::backup::{BackupConfig, EffectiveStore, Unconfigured}; +pub use extenddb_engine::LOCAL_CHECK_TIMEOUT; +use extenddb_engine::{ + BackupRuntime, ConfiguredStore, JobSettings, Spool, StoreProblem, store_identity, +}; +use extenddb_storage::backup_catalog::{StoreIdentity, StoreKind}; +use extenddb_storage::{BackupCatalog, StorageEngine}; + +/// The lock on the spool directory, held for the process's life so a second +/// instance on this host refuses the same `spool_path`. +pub struct SpoolLock { + _file: std::fs::File, + path: PathBuf, +} + +impl std::fmt::Debug for SpoolLock { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("SpoolLock") + .field("path", &self.path) + .finish() + } +} + +/// The name of the lock file inside the spool. +pub const SPOOL_LOCK_FILE: &str = ".lock"; + +/// Who is configuring the store, which decides the spool and the checks. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Caller { + /// A server: create the spool and hold its lock for the process's life; + /// a filesystem store's checks run here (module doc). + Server, + /// A command that writes (`backup purge`): the spool rule is checked, + /// the directory neither created nor locked; the checks run as a + /// server's do. + Command, + /// `backup sync`: as `Command`, except that a defaulted root whose + /// identity file names another catalog stays configured, unhealthy, + /// while the catalog has no rows for it, where `Command` and `Server` + /// take the warning path (module doc): it is the root a catalog + /// re-created by `destroy` and `init` finds, and `--adopt-store` is the + /// command that adopts it. + SyncCommand, + /// A command that reads beside a server (`backup list`, the purge's dry + /// run): as `Command`, and no check of the store runs, so a root is left + /// as it is found (neither the default directory nor an identity file is + /// created; a default root that does not exist yet leaves the store + /// unconfigured, logged); the store is returned `checking` and the + /// command reads what it needs itself. + ReadOnlyCommand, +} + +/// Why [`configure_described`] left the store unconfigured; `Display` is the +/// sentence a command prints for it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum NotConfigured { + /// `[backup.store]` is absent and this backend has no default root. + Absent, + /// `kind = "disabled"`. + Disabled, + /// A read-only command found that the default root does not exist yet: + /// no server has run with this database file. + DefaultRootMissing(PathBuf), + /// The defaulted root could not be created, opened, validated or checked + /// while the catalog has no rows for it (module doc): the warning path. + DefaultRootUnusable { + /// The root, as the configuration or `open` spelled it. + root: String, + /// The failure, as the warning logged it. + problem: String, + }, +} + +impl std::fmt::Display for NotConfigured { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Absent => f.write_str( + "no backup store is configured: [backup.store] is absent and this backend has \ + no default root, so there are no store-backed backups to list, purge or sync", + ), + Self::Disabled => f.write_str( + "no backup store is configured: [backup.store] is disabled, so there are no \ + store-backed backups to list, purge or sync", + ), + Self::DefaultRootMissing(root) => write!( + f, + "the default root {} does not exist yet: no server has run with this database \ + file, so there are no store-backed backups to list, purge or sync", + root.display() + ), + Self::DefaultRootUnusable { root, problem } => write!( + f, + "the default root {root} failed its check ({problem}) and the catalog has no \ + rows for it, so there are no store-backed backups to list, purge or sync; the \ + warning above says how to fix the root" + ), + } + } +} + +/// What [`configure_described`] built. +pub struct StoreSetup { + /// The job slots and, when one is configured, the store with its health. + pub runtime: BackupRuntime, + /// The spool's lock, held by a server for the process's life. + pub spool_lock: Option, + /// Why the runtime has no store, when it has none. + pub not_configured: Option, +} + +impl StoreSetup { + fn without_store(runtime: BackupRuntime, why: NotConfigured) -> Self { + Self { + runtime, + spool_lock: None, + not_configured: Some(why), + } + } + + fn with_store( + runtime: BackupRuntime, + store: ConfiguredStore, + spool_lock: Option, + ) -> Self { + Self { + runtime: runtime.with_store(store), + spool_lock, + not_configured: None, + } + } +} + +/// Why CreateBackup is bound to the in-catalog path on `backend`, whatever +/// the store configuration, or `None` where the store-backed job runs. +/// Cassandra cannot pin a table snapshot. SQLite's job claim is an +/// in-process slot, sound only with one process per database file, and the +/// serve lock that enforces that is `flock`, taken on Unix only +/// (`serve_lock.rs`): two servers on one file on another platform would +/// each take the other's rows for dead, so there the create stays +/// in-catalog and the store serves restores and deletes. +#[must_use] +pub fn in_catalog_create_only(backend: &str) -> Option<&'static str> { + match backend { + "cassandra" => Some("this backend cannot pin a table snapshot"), + "sqlite" if !cfg!(unix) => Some( + "the serve lock that keeps one process per SQLite database file is taken on Unix \ + only, and the job's claim is sound only with one process per file", + ), + _ => None, + } +} + +/// Build the backup runtime for this process: the job slots and, when the +/// configuration and the backend select one, the store with its health +/// ([`configure_described`] without the reason for an absent store). +/// +/// # Errors +/// +/// A configuration error (module doc), with the key, path or bucket named. +pub async fn configure( + backup: &BackupConfig, + database_file: Option<&Path>, + backend: &str, + storage: &Arc, + caller: Caller, +) -> anyhow::Result<(BackupRuntime, Option)> { + let setup = configure_described(backup, database_file, backend, &**storage, caller).await?; + Ok((setup.runtime, setup.spool_lock)) +} + +/// The `[backup]` settings the jobs and the reconciler read, as the +/// configuration resolves them: the heartbeat period is +/// [`BackupConfig::effective_heartbeat_interval_secs`]. +#[must_use] +pub fn job_settings(backup: &BackupConfig) -> JobSettings { + JobSettings { + data_file_target_bytes: backup.data_file_target_bytes, + heartbeat_interval: std::time::Duration::from_secs( + backup.effective_heartbeat_interval_secs(), + ), + stale_job_after: std::time::Duration::from_secs(backup.stale_job_after_secs), + failed_row_retention: std::time::Duration::from_secs(backup.failed_row_retention_secs), + } +} + +/// Build the backup runtime for this process: the job slots, the job +/// settings and, when the configuration and the backend select one, the +/// store with its health and the names the DynamoDB-shaped manifests carry +/// for it; otherwise why there is none. `catalog` is the catalog the +/// identity check reads and the rows rule of the module doc consults. The +/// server adds the metrics collector, the shutdown token and the source +/// names to the runtime it gets back. +/// +/// # Errors +/// +/// A configuration error (module doc), with the key, path or bucket named. +pub async fn configure_described( + backup: &BackupConfig, + database_file: Option<&Path>, + backend: &str, + catalog: &C, + caller: Caller, +) -> anyhow::Result { + let in_catalog_only = in_catalog_create_only(backend); + if let Some(why) = in_catalog_only { + tracing::info!( + "CreateBackup on this server writes the in-catalog format whatever [backup.store] \ + says: {why}" + ); + } + let runtime = BackupRuntime::new(backup.max_concurrent_jobs) + .with_legacy_create_only(in_catalog_only.is_some()) + .with_job_settings(job_settings(backup)); + let resolved = backup + .resolve_store(database_file) + .map_err(|e| anyhow::anyhow!("[backup] configuration: {e}"))?; + let (config, defaulted) = match resolved.store { + EffectiveStore::Unconfigured(Unconfigured::Absent) => { + tracing::info!( + "backup store: none configured ([backup.store] is absent and this backend has \ + no default root); CreateBackup writes in-catalog backups, with a deprecation \ + warning on each call" + ); + return Ok(StoreSetup::without_store(runtime, NotConfigured::Absent)); + } + EffectiveStore::Unconfigured(Unconfigured::Disabled) => { + tracing::info!( + "backup store: disabled ([backup.store] kind = \"disabled\"); CreateBackup \ + writes in-catalog backups, with a deprecation warning on each call" + ); + return Ok(StoreSetup::without_store(runtime, NotConfigured::Disabled)); + } + EffectiveStore::Configured { config, defaulted } => (config, defaulted), + }; + + // The defaulted root beside the database file is created if missing, + // private to this user; an explicitly configured root must exist. A + // read-only command creates nothing: a defaulted root no server has + // created yet is reported as such, and the command has nothing to read. + if defaulted + && caller == Caller::ReadOnlyCommand + && let Some(fs) = &config.filesystem + && !fs.path.exists() + { + tracing::info!( + "backup store: the default root {} does not exist yet (no server has run with this \ + database file); there are no store-backed backups to list", + fs.path.display() + ); + return Ok(StoreSetup::without_store( + runtime, + NotConfigured::DefaultRootMissing(fs.path.clone()), + )); + } + if defaulted + && let Some(fs) = &config.filesystem + && let Err(e) = create_private_dir_if_missing(&fs.path) + { + // A root that could not be created does not exist, so no row can + // name it: the warning path, whatever the catalog holds. + let root = fs.path.display().to_string(); + let problem = format!("it could not be created: {e}"); + warn_default_root_unusable(&root, &problem); + return Ok(StoreSetup::without_store( + runtime, + NotConfigured::DefaultRootUnusable { root, problem }, + )); + } + + let opened = match extenddb_backup_store::open_described(&config).await { + Ok(opened) => opened, + Err(e) => { + // The label rows record is the root's canonical path, which the + // refused root may still have (a parent others can write to). + let label = config + .filesystem + .as_ref() + .and_then(|fs| std::fs::canonicalize(&fs.path).ok()) + .map(|p| p.display().to_string()); + if defaulted + && let Some(root) = label.as_deref() + && !default_root_has_rows(catalog, root).await + { + let problem = format!("it is not usable: {e}"); + warn_default_root_unusable(root, &problem); + return Ok(StoreSetup::without_store( + runtime, + NotConfigured::DefaultRootUnusable { + root: root.to_owned(), + problem, + }, + )); + } + anyhow::bail!( + "[backup.store] ({}) cannot be opened: {e}{}", + describe_target(&config), + rows_note(defaulted) + ); + } + }; + let kind = match opened.kind { + BackupStoreKind::Filesystem => StoreKind::Filesystem, + BackupStoreKind::S3 => StoreKind::S3, + BackupStoreKind::Disabled => unreachable!("open_described never returns Disabled"), + }; + // `manifest-summary.json`'s `s3Bucket` and the key prefix every key of + // the DynamoDB-shaped manifests starts with: the bucket and the + // configured prefix on S3, the root path and no prefix on a filesystem + // store. + let (bucket, key_prefix) = match (kind, &config.s3) { + (StoreKind::S3, Some(s3)) => (s3.bucket.clone(), s3.prefix.clone()), + _ => (opened.root.clone(), String::new()), + }; + // A SQLite database with no file is in memory: its catalog is new at + // every start, so the store's checks mark the identity file they write + // and adopt a root another in-memory catalog owned, and the startup + // sync is pending (module doc). + let ephemeral_catalog = backend == "sqlite" && database_file.is_none(); + let configured = |store: ConfiguredStore| { + store + .with_manifest_names(bucket, key_prefix) + .with_ephemeral_catalog(ephemeral_catalog) + }; + + // The spool and its lock: configuration errors on every backend. + let (spool, spool_lock) = match (resolved.spool_path, caller) { + (Some(path), Caller::Server) => { + let (spool, lock) = take_spool(&path)?; + (Some(spool), Some(lock)) + } + (Some(_), Caller::Command | Caller::SyncCommand | Caller::ReadOnlyCommand) | (None, _) => { + (None, None) + } + }; + let spool_line = spool + .as_ref() + .map(|s| format!(", spool {}", s.path.display())) + .unwrap_or_default(); + + // An S3 store's checks are requests to the endpoint: they run once the + // listener is up, in the reconciler's first cycle. A read-only command + // runs no check at all. + if caller == Caller::ReadOnlyCommand { + let configured = configured(ConfiguredStore::checking( + kind, + opened.root, + opened.store, + spool, + )); + return Ok(StoreSetup::with_store(runtime, configured, spool_lock)); + } + if kind == StoreKind::S3 { + if caller == Caller::Server { + tracing::info!( + "backup store: {kind} {}{spool_line}; its validation and identity check run \ + after the listener is up, and CreateBackup writes to it once a check passes", + opened.root + ); + } + let configured = configured(ConfiguredStore::checking( + kind, + opened.root, + opened.store, + spool, + )); + return Ok(StoreSetup::with_store(runtime, configured, spool_lock)); + } + + // A filesystem store: `validate` and the identity check read the root + // here (`run_local_checks`). A `validate` refusal is a configuration + // error; a store condition or the identity check's failure degrades the + // store; on the defaulted root without rows either is the warning. + let outcome = match run_local_checks( + &*opened.store, + kind, + &opened.root, + catalog, + backend, + ephemeral_catalog, + ) + .await + { + LocalChecks::Done(outcome) => outcome, + LocalChecks::Refused(e) => { + if defaulted && !default_root_has_rows(catalog, &opened.root).await { + let problem = format!("it failed its validation: {e}"); + warn_default_root_unusable(&opened.root, &problem); + return Ok(StoreSetup::without_store( + runtime, + NotConfigured::DefaultRootUnusable { + root: opened.root, + problem, + }, + )); + } + anyhow::bail!( + "[backup.store] ({}) failed its startup validation: {e}{}", + describe_target(&config), + rows_note(defaulted) + ); + } + LocalChecks::TimedOut => { + tracing::warn!( + "backup store: {kind} {} did not answer its startup checks within {} s (a mount \ + that has stopped answering?); the server starts with the store checking, and \ + the reconciler's first cycle runs the checks again{}", + opened.root, + LOCAL_CHECK_TIMEOUT.as_secs(), + if defaulted { + " (the default root beside the database file: a first check that fails \ + while the catalog has no backup rows for it leaves the store unconfigured, \ + as a startup failure would)" + } else { + "" + } + ); + // `defaulted` travels with the store, so the reconciler applies + // the rows rule to the first check's outcome. + let configured = configured( + ConfiguredStore::checking(kind, opened.root, opened.store, spool) + .defaulted(defaulted), + ); + return Ok(StoreSetup::with_store(runtime, configured, spool_lock)); + } + }; + match &outcome { + Ok(identity) => tracing::info!( + "backup store: {kind} {} (store id {}){spool_line}; CreateBackup writes to it, and \ + it serves restores and deletes of the backups it holds", + opened.root, + identity.store_id + ), + Err(problem) + if defaulted + && !(caller == Caller::SyncCommand + && matches!(problem, StoreProblem::ForeignCatalog { .. })) + && !default_root_has_rows(catalog, &opened.root).await => + { + let problem = problem.to_string(); + warn_default_root_unusable(&opened.root, &problem); + return Ok(StoreSetup::without_store( + runtime, + NotConfigured::DefaultRootUnusable { + root: opened.root, + problem, + }, + )); + } + Err(problem) => { + // The `backup sync` exception configures a defaulted root + // another catalog owns whether or not this catalog has rows for + // it; the clause says which. + let note = if defaulted + && caller == Caller::SyncCommand + && matches!(problem, StoreProblem::ForeignCatalog { .. }) + && !default_root_has_rows(catalog, &opened.root).await + { + SYNC_EXCEPTION_NOTE + } else { + rows_note(defaulted) + }; + if caller == Caller::Server { + tracing::error!( + "backup store: {kind} {} is unhealthy: {problem}; the server starts, restore \ + and delete of a backup in the store are refused, /health reports the \ + backup_store component unhealthy, and the check runs again each reconciler \ + cycle{note}", + opened.root, + ); + } else { + // A command's own line: no server starts, and the command + // says what it does with an unhealthy store. + tracing::error!( + "backup store: {kind} {} is unhealthy: {problem}{note}", + opened.root, + ); + } + } + } + let configured = configured(ConfiguredStore::new( + kind, + opened.root, + opened.store, + spool, + outcome, + )); + Ok(StoreSetup::with_store(runtime, configured, spool_lock)) +} + +/// What a filesystem store's startup checks decided. +enum LocalChecks { + /// `validate` passed and the identity check ran, or `validate` failed on + /// a store condition (module doc): the identity, or the problem that + /// makes the store unhealthy. + Done(Result), + /// `validate` refused the root for a configuration reason. + Refused(StoreError), + /// Nothing answered within [`LOCAL_CHECK_TIMEOUT`]. + TimedOut, +} + +/// A filesystem store's startup checks: `validate`, then +/// [`store_identity::establish`] (which, for an in-memory catalog, adopts a +/// root another in-memory catalog owned), bounded by [`LOCAL_CHECK_TIMEOUT`] so a +/// root on a mount that stops answering after `open` does not hold the +/// listener. The store's synchronous filesystem calls run on the blocking +/// pool, so the bound fires whichever call stalls; a timed-out check leaves +/// that thread parked until the mount answers (module doc). +async fn run_local_checks( + store: &dyn BackupStore, + kind: StoreKind, + root: &str, + catalog: &C, + backend: &str, + ephemeral_catalog: bool, +) -> LocalChecks { + let checks = async { + match store.validate().await { + Ok(()) => LocalChecks::Done( + store_identity::establish(store, kind, root, catalog, backend, ephemeral_catalog) + .await, + ), + // A store condition, not a configuration error: the probe write + // or the root's resolution failed (module doc). + Err(e @ (StoreError::Transport { .. } | StoreError::Io { .. })) => { + LocalChecks::Done(Err(StoreProblem::from(e))) + } + Err(e) => LocalChecks::Refused(e), + } + }; + tokio::time::timeout(LOCAL_CHECK_TIMEOUT, checks) + .await + .unwrap_or(LocalChecks::TimedOut) +} + +/// Whether the catalog has backup rows for the defaulted root at `root` +/// (the label rows record), which ends the root's leniency (module doc). A +/// catalog that cannot be read is answered as having rows, so a read +/// failure never makes a used root's backups vanish behind a warning. +async fn default_root_has_rows(catalog: &C, root: &str) -> bool { + match catalog.read_store_ids(StoreKind::Filesystem, root).await { + Ok(ids) => !ids.is_empty(), + Err(e) => { + tracing::warn!( + "backup store: could not read the catalog's rows for the default root {root} \ + ({e}); the root is handled as a configured one" + ); + true + } + } +} + +/// The clause a defaulted root's startup error or unhealthy line carries, +/// saying why it was not the warning. +fn rows_note(defaulted: bool) -> &'static str { + if defaulted { + " (the default root beside the database file; the catalog has backup rows for it, so \ + it is handled as a configured root: fix it, or configure [backup.store])" + } else { + "" + } +} + +/// The clause of the unhealthy line under the `backup sync` exception +/// ([`Caller::SyncCommand`]): a defaulted root another catalog owns, with +/// no rows of this catalog. +const SYNC_EXCEPTION_NOTE: &str = " (the default root beside the database file, which every \ + database file in that directory shares; the catalog has no backup rows for it, and it is \ + configured for `backup sync`, so --adopt-store can adopt it from the catalog its \ + extenddb-store.json names)"; + +/// The warning for a defaulted root whose creation, `open`, `validate` or +/// identity check failed while the catalog has no rows for it: the store is +/// left unconfigured, as the module doc says. +fn warn_default_root_unusable(root: &str, problem: &str) { + tracing::warn!( + "backup store: the default root {root} failed its check ({problem}); the catalog has no \ + backup rows for it, so the store is unconfigured and CreateBackup writes in-catalog \ + backups. Fix the root (it must be owned by this user with mode 0700, under directories \ + owned by root or this user that others cannot write to, and its extenddb-store.json, \ + if any, must name this catalog: a catalog re-created by `destroy` and `init` adopts the \ + root and its backups with `extenddb backup sync --adopt-store`, once no server of the \ + catalog the file names runs, and the root is shared by every database file in its \ + directory), configure [backup.store] to choose another, or set kind = \"disabled\" to \ + silence this" + ); +} + +/// The store line of the console banner: what `[backup.store]` and the +/// backend select, before the store is opened. The configuration's own +/// refusals are not applied here; `configure` reports them. A filesystem +/// root is shown in the spelling the rows record and the startup log line +/// prints (the canonical directory, `banner_target`), with the configured +/// spelling beside it only where the two differ. +#[must_use] +pub fn banner_line(backup: &BackupConfig, database_file: Option<&Path>) -> String { + let resolved = match backup.resolve_store(database_file) { + Ok(resolved) => resolved, + Err(e) => return format!("backup store: configuration error ({e})"), + }; + let spool = resolved + .spool_path + .as_ref() + .map(|p| format!(", spool {}", p.display())) + .unwrap_or_default(); + match resolved.store { + EffectiveStore::Unconfigured(Unconfigured::Absent) => { + "backup store: none ([backup.store] is absent); CreateBackup writes in-catalog \ + backups" + .to_owned() + } + EffectiveStore::Unconfigured(Unconfigured::Disabled) => { + "backup store: disabled; CreateBackup writes in-catalog backups".to_owned() + } + EffectiveStore::Configured { config, defaulted } => format!( + "backup store: {}{}{spool}", + banner_target(&config), + if defaulted { + " (the default beside the database file)" + } else { + "" + } + ), + } +} + +/// The root a configuration names, for a startup error: the bucket of an S3 +/// store, the path of a filesystem one as written. Neither is a secret, and +/// an operator reading the error wants to know which store it is about. +fn describe_target(config: &extenddb_backup_store::BackupStoreConfig) -> String { + match config.kind { + BackupStoreKind::S3 => config.s3.as_ref().map_or_else( + || "s3".to_owned(), + |s3| format!("s3 bucket {:?}", s3.bucket), + ), + BackupStoreKind::Filesystem => config.filesystem.as_ref().map_or_else( + || "filesystem".to_owned(), + |fs| format!("filesystem path {}", shown_path(&fs.path)), + ), + BackupStoreKind::Disabled => "disabled".to_owned(), + } +} + +/// The banner's spelling of a configuration's root: as [`describe_target`], +/// except that a filesystem root is the canonical directory the store opens +/// (the spelling its rows record, `backup list` prints and the startup log +/// line names), followed by the configured spelling when that differs (a +/// root reached through a symlink, or written relative). A root not created +/// yet (the default root before the server's first start) is its canonical +/// parent joined with its name; one whose parent does not resolve either is +/// shown as configured. +fn banner_target(config: &extenddb_backup_store::BackupStoreConfig) -> String { + let Some(fs) = config + .filesystem + .as_ref() + .filter(|_| config.kind == BackupStoreKind::Filesystem) + else { + return describe_target(config); + }; + let canonical = std::fs::canonicalize(&fs.path).ok().or_else(|| { + let name = fs.path.file_name()?; + let parent = fs.path.parent().filter(|p| !p.as_os_str().is_empty())?; + Some(std::fs::canonicalize(parent).ok()?.join(name)) + }); + match canonical { + Some(canonical) if canonical == fs.path => { + format!("filesystem {}", shown_path(&canonical)) + } + Some(canonical) => format!( + "filesystem {} (configured as {})", + shown_path(&canonical), + shown_path(&fs.path) + ), + None => format!("filesystem {}", shown_path(&fs.path)), + } +} + +/// A path for an operator's line: quoted when it is empty or begins or ends +/// with whitespace, so the line shows where it ends. +fn shown_path(path: &Path) -> String { + let path = path.to_string_lossy(); + if path.is_empty() { + "\"\" (empty)".to_owned() + } else if path.trim() != path { + format!("{path:?}") + } else { + path.into_owned() + } +} + +/// Create `path` with mode `0700` unless it exists. +fn create_private_dir_if_missing(path: &Path) -> std::io::Result<()> { + if path.exists() { + return Ok(()); + } + let mut builder = std::fs::DirBuilder::new(); + #[cfg(unix)] + { + use std::os::unix::fs::DirBuilderExt; + builder.mode(0o700); + } + builder.recursive(true).create(path) +} + +/// Create the spool directory (`0700`) if missing and take its `.lock` +/// (`0600`, `flock`), refusing a spool another live instance holds. An +/// existing spool must meet the root's owner-only rule (owned by this user, +/// no group or other permission bits): it will hold downloaded backup +/// files, and a readable lock file would let any local user hold the spool. +fn take_spool(path: &Path) -> anyhow::Result<(Spool, SpoolLock)> { + create_private_dir_if_missing(path).map_err(|e| { + anyhow::anyhow!( + "backup.spool_path {} cannot be created: {e}", + path.display() + ) + })?; + let canonical = path + .canonicalize() + .map_err(|e| anyhow::anyhow!("backup.spool_path {} is not usable: {e}", path.display()))?; + check_spool_mode(&canonical)?; + let lock_path = canonical.join(SPOOL_LOCK_FILE); + let mut options = std::fs::OpenOptions::new(); + options.read(true).write(true).create(true).truncate(false); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + let file = options + .open(&lock_path) + .map_err(|e| anyhow::anyhow!("cannot open the spool lock {}: {e}", lock_path.display()))?; + // `create` honours the umask and the file may predate this rule: flock + // needs only a read handle, so a readable lock file would let any local + // user hold the spool. Set the mode on every open. + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + file.set_permissions(std::fs::Permissions::from_mode(0o600)) + .map_err(|e| anyhow::anyhow!("cannot set the mode of {}: {e}", lock_path.display()))?; + } + try_lock_exclusive(&file).map_err(|e| { + if e.kind() == std::io::ErrorKind::WouldBlock { + anyhow::anyhow!( + "backup.spool_path {} is held by another running extenddb instance (lock {}); \ + every instance on a host needs a spool of its own", + canonical.display(), + lock_path.display() + ) + } else { + anyhow::anyhow!("cannot lock {}: {e}", lock_path.display()) + } + })?; + let spool = Spool { + spool_host: format!("{}:{}", hostname(), canonical.display()), + path: canonical, + }; + Ok(( + spool, + SpoolLock { + _file: file, + path: lock_path, + }, + )) +} + +/// The owner-only rule on the spool directory, the one the filesystem store +/// applies to its root (`extenddb_backup_store::check_private_directory`): +/// owned by this user with mode `0700`, under directories owned by root or +/// this user that others cannot write to. +fn check_spool_mode(spool: &Path) -> anyhow::Result<()> { + if !spool.is_dir() { + anyhow::bail!("backup.spool_path {} is not a directory", spool.display()); + } + extenddb_backup_store::check_private_directory(spool, "spool").map_err(|e| { + anyhow::anyhow!( + "backup.spool_path {} is not private: {e}; the spool must have mode 0700 under \ + directories others cannot write to", + spool.display() + ) + }) +} + +/// This host's name, for `spool_host` and a maintenance claim's holder; +/// `unknown-host` when the system does not say. +#[must_use] +pub fn hostname() -> String { + #[cfg(unix)] + { + let mut buf = [0u8; 256]; + // SAFETY: `gethostname` writes at most `len` bytes into the buffer it + // is given, which is valid for its whole length. + let rc = unsafe { libc::gethostname(buf.as_mut_ptr().cast(), buf.len()) }; + if rc == 0 { + let end = buf.iter().position(|b| *b == 0).unwrap_or(buf.len()); + if let Ok(name) = std::str::from_utf8(&buf[..end]) + && !name.is_empty() + { + return name.to_owned(); + } + } + } + "unknown-host".to_owned() +} + +#[cfg(unix)] +fn try_lock_exclusive(file: &std::fs::File) -> std::io::Result<()> { + use std::os::unix::io::AsRawFd; + // SAFETY: `flock` takes a file descriptor and flags and touches no memory + // the caller owns; the descriptor belongs to `file`, borrowed for the + // call. + let rc = unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) }; + if rc == 0 { + Ok(()) + } else { + Err(std::io::Error::last_os_error()) + } +} + +#[cfg(not(unix))] +fn try_lock_exclusive(_file: &std::fs::File) -> std::io::Result<()> { + tracing::warn!( + "no file lock is taken on the spool on this platform: nothing prevents a second \ + extenddb instance from using the same spool_path" + ); + Ok(()) +} + +#[cfg(all(test, unix))] +mod tests { + use super::*; + + fn mode_of(path: &Path) -> u32 { + use std::os::unix::fs::PermissionsExt; + std::fs::metadata(path).unwrap().permissions().mode() & 0o777 + } + + /// The spool is created private with a private lock file, a second take + /// of the same spool is refused while the first lock lives, and a spool + /// elsewhere is not. + #[test] + fn the_spool_lock_refuses_a_second_instance() { + let dir = tempfile::tempdir().unwrap(); + let spool = dir.path().join("spool"); + let (first, lock) = take_spool(&spool).unwrap(); + assert_eq!(mode_of(&spool), 0o700); + assert_eq!(mode_of(&spool.join(SPOOL_LOCK_FILE)), 0o600); + assert!( + first + .spool_host + .ends_with(&format!(":{}", spool.canonicalize().unwrap().display())) + ); + assert!(!first.spool_host.starts_with(':'), "the host name is set"); + let err = take_spool(&spool).unwrap_err().to_string(); + assert!( + err.contains("held by another running extenddb instance"), + "{err}" + ); + assert!(err.contains(&spool.display().to_string()), "{err}"); + take_spool(&dir.path().join("other")).unwrap(); + drop(lock); + take_spool(&spool).expect("free once the holder is gone"); + } + + /// An existing spool that group or others can reach, or that another + /// user owns, is refused naming the path and the mode; one with mode + /// 0700 is taken. + #[test] + fn an_existing_spool_must_be_owner_only() { + use std::os::unix::fs::PermissionsExt; + let dir = tempfile::tempdir().unwrap(); + let spool = dir.path().join("spool"); + std::fs::create_dir(&spool).unwrap(); + for mode in [0o755u32, 0o750, 0o707, 0o770] { + std::fs::set_permissions(&spool, std::fs::Permissions::from_mode(mode)).unwrap(); + let err = take_spool(&spool).unwrap_err().to_string(); + assert!( + err.contains(&format!("{mode:04o}")) && err.contains("mode 0700"), + "{mode:04o}: {err}" + ); + assert!(err.contains("not private"), "{err}"); + assert!(err.contains(&spool.display().to_string()), "{err}"); + assert!( + !spool.join(SPOOL_LOCK_FILE).exists(), + "no lock file is created" + ); + } + std::fs::set_permissions(&spool, std::fs::Permissions::from_mode(0o700)).unwrap(); + take_spool(&spool).expect("an owner-only spool is taken"); + // A spool under a parent others can write to is refused as the root + // would be: the parent could be swapped under it. + let parent = dir.path().join("open"); + std::fs::create_dir(&parent).unwrap(); + std::fs::set_permissions(&parent, std::fs::Permissions::from_mode(0o777)).unwrap(); + let under = parent.join("spool"); + std::fs::create_dir(&under).unwrap(); + std::fs::set_permissions(&under, std::fs::Permissions::from_mode(0o700)).unwrap(); + let err = take_spool(&under).unwrap_err().to_string(); + assert!( + err.contains("an ancestor of the spool") && err.contains("0777"), + "{err}" + ); + } + + /// A lock file that predates the mode rule is narrowed to 0600 on open. + #[test] + fn a_permissive_lock_file_is_narrowed() { + use std::os::unix::fs::PermissionsExt; + let dir = tempfile::tempdir().unwrap(); + let spool = dir.path().join("spool"); + std::fs::create_dir(&spool).unwrap(); + std::fs::set_permissions(&spool, std::fs::Permissions::from_mode(0o700)).unwrap(); + let lock = spool.join(SPOOL_LOCK_FILE); + std::fs::write(&lock, b"").unwrap(); + std::fs::set_permissions(&lock, std::fs::Permissions::from_mode(0o644)).unwrap(); + let _held = take_spool(&spool).unwrap(); + assert_eq!(mode_of(&lock), 0o600); + } + + // --- the startup checks against doubles of the catalog and the store --- + + use extenddb_backup_store::{ByteStream, ObjectMeta}; + use extenddb_core::backup_format::BackupArn; + use extenddb_storage::BoxedFuture; + use extenddb_storage::backup_catalog::{ + BackupKey, BackupRow, BackupStatus, DeleteStart, JobClaim, MaintenanceClaim, RestoreJobRow, + RestoreTargetKey, StoreIdentity, TimeBounds, + }; + use extenddb_storage::error::StorageError; + use futures::stream::BoxStream; + + /// A catalog whose rows for every root carry `ids`, or whose rows cannot + /// be read (`rows_unreadable`), with no backup row to list; every other + /// method is an error, since the startup checks never reach it. + #[derive(Default)] + struct StubCatalog { + ids: Vec, + rows_unreadable: bool, + } + + fn unused() -> BoxedFuture<'static, Result> { + Box::pin(async { + Err(StorageError::Internal( + "not reached by the startup checks".to_owned(), + )) + }) + } + + impl BackupCatalog for StubCatalog { + fn catalog_id(&self) -> BoxedFuture<'_, Result> { + Box::pin(async { Ok("catalog-1".to_owned()) }) + } + fn claim_backup<'a>( + &'a self, + _row: &'a BackupRow, + ) -> BoxedFuture<'a, Result, StorageError>> { + unused() + } + fn insert_backup_row<'a>( + &'a self, + _row: &'a BackupRow, + ) -> BoxedFuture<'a, Result> { + unused() + } + fn claim_dead_backup<'a>( + &'a self, + _key: &'a BackupKey, + _stale_after: std::time::Duration, + ) -> BoxedFuture<'a, Result>, StorageError>> { + unused() + } + fn claim_restore<'a>( + &'a self, + _account_id: &'a str, + _input: extenddb_core::types::CreateTableInput, + _registration: &'a extenddb_storage::backup_catalog::RestoreRegistrationRow, + ) -> BoxedFuture< + 'a, + Result<(extenddb_core::types::TableDescription, Box), StorageError>, + > { + unused() + } + fn claim_restore_dead<'a>( + &'a self, + _key: &'a RestoreTargetKey, + _stale_after: std::time::Duration, + _spool_host: Option<&'a str>, + ) -> BoxedFuture<'a, Result>, StorageError>> { + unused() + } + fn read_restore_job<'a>( + &'a self, + _key: &'a RestoreTargetKey, + ) -> BoxedFuture<'a, Result, StorageError>> { + unused() + } + fn list_restore_jobs<'a>( + &'a self, + _status: Option, + ) -> BoxedFuture<'a, Result, StorageError>> { + unused() + } + fn list_backup_rows<'a>( + &'a self, + _status: Option, + _store: &'a StoreIdentity, + ) -> BoxStream<'a, Result> { + Box::pin(futures::stream::empty()) + } + fn newest_available_backup_row<'a>( + &'a self, + _store: &'a StoreIdentity, + ) -> BoxedFuture<'a, Result, StorageError>> { + Box::pin(async { Ok(None) }) + } + fn list_running_restore_jobs( + &self, + ) -> BoxedFuture< + '_, + Result, StorageError>, + > { + unused() + } + fn list_account_backup_rows<'a>( + &'a self, + _account_id: &'a str, + _table_name: Option<&'a str>, + _bounds: &'a TimeBounds, + ) -> BoxStream<'a, Result> { + Box::pin(futures::stream::empty()) + } + fn read_backup_row<'a>( + &'a self, + _arn: &'a BackupArn, + ) -> BoxedFuture<'a, Result, StorageError>> { + unused() + } + fn start_backup_delete<'a>( + &'a self, + _key: &'a BackupKey, + ) -> BoxedFuture<'a, Result> { + unused() + } + fn set_backup_deleted<'a>( + &'a self, + _key: &'a BackupKey, + ) -> BoxedFuture<'a, Result> { + unused() + } + fn remove_backup_row<'a>( + &'a self, + _key: &'a BackupKey, + ) -> BoxedFuture<'a, Result> { + unused() + } + fn forget_available_backup_row<'a>( + &'a self, + _key: &'a BackupKey, + ) -> BoxedFuture<'a, Result> + { + unused() + } + fn remove_dead_creating_backup_row<'a>( + &'a self, + _key: &'a BackupKey, + _stale_after: std::time::Duration, + ) -> BoxedFuture<'a, Result> { + unused() + } + fn list_backup_rows_older_than<'a>( + &'a self, + _status: BackupStatus, + _store: &'a StoreIdentity, + _clock: extenddb_storage::backup_catalog::StaleClock, + _age: std::time::Duration, + ) -> BoxStream<'a, Result> { + Box::pin(futures::stream::empty()) + } + fn backup_in_use<'a>( + &'a self, + _key: &'a BackupKey, + ) -> BoxedFuture<'a, Result, StorageError>> { + unused() + } + fn remove_restore_target<'a>( + &'a self, + _key: &'a RestoreTargetKey, + ) -> BoxedFuture<'a, Result> { + unused() + } + fn list_dead_restore_jobs( + &self, + _stale_after: std::time::Duration, + ) -> BoxedFuture<'_, Result, StorageError>> { + unused() + } + fn fail_restore_job<'a>( + &'a self, + _key: &'a RestoreTargetKey, + _owner_generation: u64, + _heartbeat_at: f64, + _reason: &'a str, + ) -> BoxedFuture<'a, Result> { + unused() + } + fn read_store_ids<'a>( + &'a self, + _store_kind: StoreKind, + _store_root: &'a str, + ) -> BoxedFuture<'a, Result, StorageError>> { + if self.rows_unreadable { + return Box::pin(async { + Err(StorageError::Internal( + "the catalog is not answering".to_owned(), + )) + }); + } + let ids = self.ids.clone(); + Box::pin(async move { Ok(ids) }) + } + fn maintenance_claim<'a>( + &'a self, + _holder: &'a str, + _stale_after: std::time::Duration, + ) -> BoxedFuture<'a, Result, StorageError>> { + unused() + } + fn check_maintenance_claim( + &self, + _stale_after: std::time::Duration, + ) -> BoxedFuture<'_, Result> { + unused() + } + } + + /// A store whose `validate` answers as told: never (a mount that has + /// stopped answering), or with a given error; nothing else is reached. + enum Validate { + Never, + Fails(fn() -> StoreError), + } + + struct FakeStore(Validate); + + fn not_reached() -> futures::future::BoxFuture<'static, Result> + { + Box::pin(async { Err(StoreError::Other("not reached".to_owned())) }) + } + + impl BackupStore for FakeStore { + fn put<'a>( + &'a self, + _key: &'a str, + _body: ByteStream, + ) -> futures::future::BoxFuture<'a, Result> { + not_reached() + } + fn put_if_absent<'a>( + &'a self, + _key: &'a str, + _body: ByteStream, + ) -> futures::future::BoxFuture<'a, Result, StoreError>> { + not_reached() + } + fn get<'a>( + &'a self, + _key: &'a str, + ) -> futures::future::BoxFuture<'a, Result> { + not_reached() + } + fn head<'a>( + &'a self, + _key: &'a str, + ) -> futures::future::BoxFuture<'a, Result, StoreError>> { + not_reached() + } + fn list<'a>(&'a self, _prefix: &'a str) -> BoxStream<'a, Result> { + Box::pin(futures::stream::empty()) + } + fn delete_prefix<'a>( + &'a self, + _prefix: &'a str, + ) -> futures::future::BoxFuture<'a, Result> { + not_reached() + } + fn validate(&self) -> futures::future::BoxFuture<'_, Result<(), StoreError>> { + match &self.0 { + Validate::Never => Box::pin(std::future::pending()), + Validate::Fails(make) => Box::pin(std::future::ready(Err(make()))), + } + } + } + + /// A SQLite deployment's `[backup]` without a store, and the database + /// file whose directory the default root is resolved against. + fn defaulted(dir: &Path) -> (BackupConfig, PathBuf) { + (BackupConfig::default(), dir.join("extenddb.sqlite")) + } + + /// The defaulted root's leniency ends with the catalog's first row for + /// it. The root is created empty (no identity file) each time: with no + /// rows for it, a fresh root gets its file and the store is healthy; + /// with rows for it and no file, an explicit root would be unhealthy + /// (`identity file missing`), and so is the defaulted one, configured + /// and reported rather than left unconfigured behind a warning. + #[tokio::test] + async fn a_defaulted_root_with_rows_is_handled_as_a_configured_one() { + let dir = tempfile::tempdir().unwrap(); + let (backup, database) = defaulted(dir.path()); + let root = dir.path().join("backups"); + + // Rows for the root (seeded, as a server that used the root leaves + // them), no identity file: an explicit root's `identity file + // missing`, on `/health` and in the refusals. + let with_rows = StubCatalog { + ids: vec!["store-1".to_owned()], + ..StubCatalog::default() + }; + let setup = configure_described( + &backup, + Some(&database), + "sqlite", + &with_rows, + Caller::Server, + ) + .await + .unwrap(); + assert_eq!(setup.not_configured, None, "the store is configured"); + let store = setup + .runtime + .configured_store() + .expect("a defaulted root with rows is a configured store"); + assert_eq!(store.kind, StoreKind::Filesystem); + assert_eq!( + store.root, + root.canonicalize().unwrap().display().to_string() + ); + assert_eq!( + store.health().category(), + Some("identity file missing"), + "{:?}", + store.health() + ); + let report = setup.runtime.store_health().unwrap(); + assert_eq!( + (report.status, report.reason), + ("unhealthy", Some("identity file missing")) + ); + assert!( + !root.join(store_identity::STORE_IDENTITY_KEY).exists(), + "no file is created over rows that name another id" + ); + + // No rows: the fresh-root case, the file created, the store healthy. + let without_rows = StubCatalog::default(); + let setup = configure_described( + &backup, + Some(&database), + "sqlite", + &without_rows, + Caller::Server, + ) + .await + .unwrap(); + assert_eq!(setup.not_configured, None); + assert!(setup.runtime.configured_store().unwrap().is_healthy()); + assert!(root.join(store_identity::STORE_IDENTITY_KEY).exists()); + + // No rows and a failed check (the file names another catalog, as a + // catalog re-created by `destroy` and `init` leaves it): the + // warning path, with the reason carried out for the commands. + let foreign = store_identity::StoreIdentityFile::parse( + br#"{"format_version":1,"store_id":"s-9","catalog_id":"other","backend":"sqlite","created_at_ms":0}"#, + ) + .unwrap(); + std::fs::write( + root.join(store_identity::STORE_IDENTITY_KEY), + foreign.to_bytes(), + ) + .unwrap(); + let setup = configure_described( + &backup, + Some(&database), + "sqlite", + &without_rows, + Caller::Server, + ) + .await + .unwrap(); + assert!(setup.runtime.configured_store().is_none()); + let why = setup.not_configured.expect("the reason is carried out"); + assert!( + matches!(&why, NotConfigured::DefaultRootUnusable { root, problem } + if root.contains("backups") && problem.contains("names catalog other")), + "{why:?}" + ); + assert!(why.to_string().contains("no rows for it"), "{why}"); + // `backup sync` is the exception: the same root is a configured, + // unhealthy store for it, which `--adopt-store` adopts; a purge + // takes the warning path as the server does. + let setup = configure_described( + &backup, + Some(&database), + "sqlite", + &without_rows, + Caller::SyncCommand, + ) + .await + .unwrap(); + assert_eq!(setup.not_configured, None); + assert_eq!( + setup + .runtime + .configured_store() + .expect("configured for sync") + .health() + .category(), + Some("owned by another catalog") + ); + let setup = configure_described( + &backup, + Some(&database), + "sqlite", + &without_rows, + Caller::Command, + ) + .await + .unwrap(); + assert!(setup.runtime.configured_store().is_none()); + // The exception is for a root another catalog owns only: another + // failure of the defaulted root takes the warning path for sync too. + std::fs::write(root.join(store_identity::STORE_IDENTITY_KEY), b"not json").unwrap(); + let setup = configure_described( + &backup, + Some(&database), + "sqlite", + &without_rows, + Caller::SyncCommand, + ) + .await + .unwrap(); + assert!(setup.runtime.configured_store().is_none()); + assert!( + matches!(&setup.not_configured, Some(NotConfigured::DefaultRootUnusable { problem, .. }) + if problem.contains("could not be read")), + "{:?}", + setup.not_configured + ); + std::fs::write( + root.join(store_identity::STORE_IDENTITY_KEY), + foreign.to_bytes(), + ) + .unwrap(); + // The same file over rows for the root: a configured, unhealthy + // store (`owned by another catalog`), not the warning. + let setup = configure_described( + &backup, + Some(&database), + "sqlite", + &with_rows, + Caller::Server, + ) + .await + .unwrap(); + assert_eq!( + setup + .runtime + .configured_store() + .unwrap() + .health() + .category(), + Some("owned by another catalog") + ); + } + + /// A defaulted root `open` refuses (a parent others can write to) is the + /// warning while the catalog has no rows for it, and a startup error + /// naming the root once it has; an explicitly configured root is the + /// startup error either way. + #[tokio::test] + async fn a_defaulted_root_open_refuses_is_a_warning_only_without_rows() { + use std::os::unix::fs::PermissionsExt; + let dir = tempfile::tempdir().unwrap(); + std::fs::set_permissions(dir.path(), std::fs::Permissions::from_mode(0o777)).unwrap(); + let (backup, database) = defaulted(dir.path()); + let setup = configure_described( + &backup, + Some(&database), + "sqlite", + &StubCatalog::default(), + Caller::Server, + ) + .await + .unwrap(); + let why = setup.not_configured.expect("the warning path"); + assert!( + matches!(&why, NotConfigured::DefaultRootUnusable { problem, .. } + if problem.contains("writable by group or others")), + "{why:?}" + ); + let err = configure_described( + &backup, + Some(&database), + "sqlite", + &StubCatalog { + ids: vec!["store-1".to_owned()], + ..StubCatalog::default() + }, + Caller::Server, + ) + .await + .err() + .expect("with rows, the refusal stops startup") + .to_string(); + assert!(err.contains("cannot be opened"), "{err}"); + assert!(err.contains("backup rows for it"), "{err}"); + assert!(err.contains(&dir.path().display().to_string()), "{err}"); + } + + /// A catalog whose rows cannot be read is answered as having rows, so a + /// defaulted root whose check then fails is a configured, unhealthy + /// store and not the warning that leaves the store unconfigured: the + /// root may hold backups the read would have shown. Here the root + /// carries a foreign identity file; the read failure reaches `establish` + /// first, so the problem is the catalog's (`catalog unreadable`), and + /// the next cycle's check reads again. + #[tokio::test] + async fn an_unreadable_catalog_keeps_a_failing_default_root_configured() { + use std::os::unix::fs::PermissionsExt; + let dir = tempfile::tempdir().unwrap(); + let (backup, database) = defaulted(dir.path()); + let root = dir.path().join("backups"); + std::fs::create_dir(&root).unwrap(); + std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o700)).unwrap(); + let foreign = store_identity::StoreIdentityFile::parse( + br#"{"format_version":1,"store_id":"s-9","catalog_id":"other","backend":"sqlite","created_at_ms":0}"#, + ) + .unwrap(); + std::fs::write( + root.join(store_identity::STORE_IDENTITY_KEY), + foreign.to_bytes(), + ) + .unwrap(); + let unreadable = StubCatalog { + rows_unreadable: true, + ..StubCatalog::default() + }; + let setup = configure_described( + &backup, + Some(&database), + "sqlite", + &unreadable, + Caller::Server, + ) + .await + .unwrap(); + assert_eq!(setup.not_configured, None, "the store is configured"); + let store = setup + .runtime + .configured_store() + .expect("a root whose rows cannot be read is a configured store"); + assert_eq!( + store.health().category(), + Some("catalog unreadable"), + "{:?}", + store.health() + ); + } + + /// The timed startup checks: a store that never answers leaves the + /// server starting with the store `checking` after `LOCAL_CHECK_TIMEOUT` + /// (not sooner, since a store that answers must get its outcome; not + /// later, since the listener waits on this); `validate`'s I/O failure + /// is a store condition that degrades (`failed validation`), its denial + /// a configuration refusal. + #[tokio::test(start_paused = true)] + async fn the_startup_checks_are_bounded_and_split_store_conditions_from_refusals() { + let catalog = StubCatalog::default(); + // The bound's value is part of the rule (a local disk answers in + // milliseconds), so it is pinned here rather than read back. + assert_eq!(LOCAL_CHECK_TIMEOUT, std::time::Duration::from_secs(5)); + let started = tokio::time::Instant::now(); + let stalled = FakeStore(Validate::Never); + // The test's own guard, far past the bound: a check with no bound + // fails here instead of hanging. + let outcome = tokio::time::timeout( + std::time::Duration::from_secs(120), + run_local_checks( + &stalled, + StoreKind::Filesystem, + "/r", + &catalog, + "test", + false, + ), + ) + .await + .expect("the checks return at their bound"); + assert!(matches!(outcome, LocalChecks::TimedOut)); + assert_eq!( + started.elapsed(), + LOCAL_CHECK_TIMEOUT, + "the bound is the whole wait, no more and no less" + ); + + let io = FakeStore(Validate::Fails(|| StoreError::Io { + context: "write probe /r/.put-validate.tmp".to_owned(), + source: std::io::Error::from_raw_os_error(28), + })); + let outcome = + run_local_checks(&io, StoreKind::Filesystem, "/r", &catalog, "test", false).await; + let LocalChecks::Done(Err(problem)) = outcome else { + panic!("an I/O failure of the probe degrades the store"); + }; + assert_eq!(problem.category(), "failed validation"); + assert!(problem.to_string().contains("write probe"), "{problem}"); + + let denied = FakeStore(Validate::Fails(|| { + StoreError::PermissionDenied("/r has mode 0755".to_owned()) + })); + let outcome = run_local_checks( + &denied, + StoreKind::Filesystem, + "/r", + &catalog, + "test", + false, + ) + .await; + assert!(matches!( + outcome, + LocalChecks::Refused(StoreError::PermissionDenied(_)) + )); + } + + /// `NotConfigured`'s sentences: each says why there are no store-backed + /// backups, and the unusable root's names the root and the failure. + #[test] + fn not_configured_reads_as_the_commands_print_it() { + assert!( + NotConfigured::Absent + .to_string() + .contains("[backup.store] is absent") + ); + assert!(NotConfigured::Disabled.to_string().contains("is disabled")); + let missing = NotConfigured::DefaultRootMissing(PathBuf::from("/srv/db/backups")); + assert!( + missing + .to_string() + .contains("/srv/db/backups does not exist yet") + ); + let unusable = NotConfigured::DefaultRootUnusable { + root: "/srv/db/backups".to_owned(), + problem: "extenddb-store.json names catalog other".to_owned(), + } + .to_string(); + assert!( + unusable.contains("/srv/db/backups failed its check"), + "{unusable}" + ); + assert!(unusable.contains("names catalog other"), "{unusable}"); + assert!(unusable.contains("no rows for it"), "{unusable}"); + } + + /// The banner quotes a root that is empty or begins or ends with + /// whitespace so the line shows where the path ends (none of these + /// resolves to another spelling on the test host). + #[test] + fn a_root_with_edge_whitespace_is_quoted_in_the_banner() { + let line = |path: &str| { + let backup = BackupConfig { + store: Some(extenddb_backup_store::BackupStoreConfig { + kind: BackupStoreKind::Filesystem, + filesystem: Some(extenddb_backup_store::FilesystemStoreConfig { + path: PathBuf::from(path), + }), + s3: None, + }), + ..BackupConfig::default() + }; + banner_line(&backup, None) + }; + let missing = format!("/nonexistent-{}/backups", uuid::Uuid::new_v4().simple()); + assert!( + line(&format!("{missing} ")).contains(&format!("filesystem \"{missing} \"")), + "{}", + line(&format!("{missing} ")) + ); + assert!( + line(&format!(" {missing}")).contains(&format!("filesystem \" {missing}\"")), + "{}", + line(&format!(" {missing}")) + ); + assert!( + line(&missing).ends_with(&format!("filesystem {missing}")), + "{}", + line(&missing) + ); + assert!(line("").contains("filesystem \"\" (empty)"), "{}", line("")); + } + + /// The banner spells a filesystem root as the store's rows and the + /// startup log line do, the canonical directory, and names the + /// configured spelling beside it only where the two differ: a root + /// reached through a symlink, and a root not created yet under a + /// symlinked parent (the default root before the first start). + #[cfg(unix)] + #[test] + fn the_banner_shows_the_canonical_root_and_the_configured_spelling_where_they_differ() { + let dir = std::env::temp_dir().join(format!( + "extenddb-banner-root-{}", + uuid::Uuid::new_v4().simple() + )); + let real = dir.join("real"); + std::fs::create_dir_all(real.join("backups")).unwrap(); + let link = dir.join("link"); + std::os::unix::fs::symlink(&real, &link).unwrap(); + let canonical_real = std::fs::canonicalize(&real).unwrap(); + let line = |path: &Path| { + let backup = BackupConfig { + store: Some(extenddb_backup_store::BackupStoreConfig { + kind: BackupStoreKind::Filesystem, + filesystem: Some(extenddb_backup_store::FilesystemStoreConfig { + path: path.to_path_buf(), + }), + s3: None, + }), + ..BackupConfig::default() + }; + banner_line(&backup, None) + }; + let canonical = canonical_real.join("backups"); + assert_eq!( + line(&canonical), + format!("backup store: filesystem {}", canonical.display()), + "one spelling where the configured root is canonical" + ); + for configured in [link.join("backups"), link.join("not-yet")] { + let expected = canonical_real.join(configured.file_name().unwrap()); + assert_eq!( + line(&configured), + format!( + "backup store: filesystem {} (configured as {})", + expected.display(), + configured.display() + ), + ); + } + let _ = std::fs::remove_dir_all(&dir); + } +} diff --git a/crates/server/src/handler.rs b/crates/server/src/handler.rs index ef3765bda..aeef4ccd1 100755 --- a/crates/server/src/handler.rs +++ b/crates/server/src/handler.rs @@ -147,6 +147,7 @@ pub(crate) async fn handle_request( // Dispatch let ctx = OperationContext { storage: state.storage.clone(), + backup: state.backup_runtime.clone(), limits: state.limits.clone(), region: state.region.clone(), account_id, diff --git a/crates/server/src/lib.rs b/crates/server/src/lib.rs index 035c35996..7df7aba50 100755 --- a/crates/server/src/lib.rs +++ b/crates/server/src/lib.rs @@ -11,6 +11,7 @@ pub(crate) mod authorization; pub mod authz_cache; pub(crate) mod authz_resource; +pub mod backup_store; pub mod console; pub mod dev_credentials; mod handler; @@ -29,7 +30,8 @@ mod throttle_helpers; mod workers; pub use serve::{ - BuildInfo, CatalogRefusal, LogTarget, RegistrationLost, ServeParams, log_to_syslog_raw, serve, + BuildInfo, CatalogRefusal, JOB_DRAIN_TIMEOUT, LogTarget, RegistrationLost, ServeParams, + WORKER_STOP_TIMEOUT, catalog_refusal, log_to_syslog_raw, serve, }; use std::path::PathBuf; @@ -55,6 +57,8 @@ use tower_http::set_header::SetResponseHeaderLayer; /// Application state shared across all handlers. pub struct AppState { pub storage: Arc, + /// The job slots and configured backup store the backup operations read. + pub backup_runtime: Arc, pub auth: Arc, pub limits: Arc, // Fix #9: Use Arc to avoid per-request cloning @@ -111,6 +115,11 @@ pub struct ServerTlsConfig { pub key_path: PathBuf, } +/// How long the listener drains active connections after a stop: the bound +/// on HTTPS, and on plain HTTP the time after which the PID file is removed. +/// `extenddb stop` waits for the sum of the shutdown bounds. +pub const CONNECTION_DRAIN_TIMEOUT: Duration = Duration::from_secs(5); + /// Build and start the HTTP server on a pre-bound listener. /// /// The caller is responsible for binding the `TcpListener` before passing it in. @@ -262,7 +271,7 @@ pub async fn start_server( let stop = admission.clone(); tokio::spawn(async move { signal_or_stop(&stop).await; - shutdown_handle.graceful_shutdown(Some(Duration::from_secs(5))); + shutdown_handle.graceful_shutdown(Some(CONNECTION_DRAIN_TIMEOUT)); // P50: Log timeout but don't call std::process::exit — let the // runtime shut down normally so destructors (including ZeroizeOnDrop) run. tokio::time::sleep(Duration::from_secs(10)).await; @@ -285,12 +294,34 @@ pub async fn start_server( .await?; } else { tracing::info!("extenddb listening on {local_addr}"); - axum::serve( + // The plain-HTTP listener's drain is bounded as the TLS listener's + // is: axum's graceful shutdown waits for every open connection with + // no bound of its own, and `extenddb stop` waits for the sum of the + // server's bounds, which counts this drain as `CONNECTION_DRAIN_TIMEOUT`. + let stopping = extenddb_storage::CancellationToken::new(); + let serve = axum::serve( nodelay_listener(listener), app.into_make_service_with_connect_info::(), ) - .with_graceful_shutdown(graceful_shutdown(pid_file.clone(), admission)) - .await?; + .with_graceful_shutdown({ + let stopping = stopping.clone(); + let pid_file = pid_file.clone(); + async move { + graceful_shutdown(pid_file, admission).await; + stopping.cancel(); + } + }); + let drain_bound = async { + stopping.cancelled().await; + tokio::time::sleep(CONNECTION_DRAIN_TIMEOUT).await; + }; + tokio::select! { + served = serve => served?, + () = drain_bound => tracing::warn!( + "connections still open {} s after the stop; the listener is closed with them", + CONNECTION_DRAIN_TIMEOUT.as_secs() + ), + } } // Normal shutdown path — clean up PID file after connections drain. @@ -299,9 +330,39 @@ pub async fn start_server( Ok(()) } -/// GET /health — REQ-OBS-006 -async fn health_check() -> impl IntoResponse { - (StatusCode::OK, axum::Json(json!({"status": "healthy"}))) +/// GET /health — REQ-OBS-006. +/// +/// Always 200 while the server answers: the data plane is up. `status` is +/// `healthy`, or `degraded` when a component is unhealthy, and `components` +/// carries each component the server has. Today the one component is +/// `backup_store`, present when a store is configured: `status` `healthy`, +/// `checking` (an S3 store's first check has not returned; the server's +/// `status` stays `healthy`) or `unhealthy`, `kind` (`filesystem` or `s3`) +/// and, when unhealthy, `reason`, the problem's category in one or two +/// words. The endpoint is unauthenticated, so the reason carries no id, ARN, +/// table name or root; the operator's log line does. A store problem never +/// takes the data plane out of rotation, which is why the HTTP status stays +/// 200. +async fn health_check( + axum::extract::State(state): axum::extract::State>, +) -> impl IntoResponse { + let mut components = serde_json::Map::new(); + let mut status = "healthy"; + if let Some(store) = state.backup_runtime.store_health() { + let mut component = json!({ + "status": store.status, + "kind": store.kind, + }); + if let Some(reason) = store.reason { + component["reason"] = serde_json::Value::String(reason.to_owned()); + status = "degraded"; + } + components.insert("backup_store".to_owned(), component); + } + ( + StatusCode::OK, + axum::Json(json!({ "status": status, "components": components })), + ) } async fn graceful_shutdown(pid_file: Option, admission: registration::Admission) { @@ -310,8 +371,11 @@ async fn graceful_shutdown(pid_file: Option, admission: registration::A // don't drain. Does NOT call std::process::exit — the runtime shuts down // normally so destructors (including ZeroizeOnDrop) run. tokio::spawn(async move { - tokio::time::sleep(Duration::from_secs(5)).await; - tracing::warn!("Graceful shutdown timed out after 5s, forcing PID file cleanup"); + tokio::time::sleep(CONNECTION_DRAIN_TIMEOUT).await; + tracing::warn!( + "Graceful shutdown timed out after {}s, forcing PID file cleanup", + CONNECTION_DRAIN_TIMEOUT.as_secs() + ); cleanup_pid_file(pid_file.as_deref()); }); } diff --git a/crates/server/src/metrics_endpoint.rs b/crates/server/src/metrics_endpoint.rs index 0ec7b06fd..0c24473bf 100755 --- a/crates/server/src/metrics_endpoint.rs +++ b/crates/server/src/metrics_endpoint.rs @@ -364,4 +364,48 @@ mod tests { "GSI-name dimension must be retained, got {dims:?}" ); } + + /// The backup job's metric names round-trip through the store's string + /// column: a row written under each name renders as that name and not + /// as the fallback `SystemErrors`, with its outcome or store-call + /// dimension kept. + #[test] + fn backup_job_metric_names_render_from_the_store() { + use extenddb_core::metrics::MetricName; + let names = [ + MetricName::BackupJobs, + MetricName::BackupJobDuration, + MetricName::BackupBytesUploaded, + MetricName::BackupSnapshotAge, + MetricName::BackupStoreErrors, + MetricName::RestoreJobs, + MetricName::RestoreJobDuration, + MetricName::RestoreBytesDownloaded, + ]; + let rows: Vec = names + .iter() + .map(|name| MetricsRow { + bucket: time::OffsetDateTime::UNIX_EPOCH, + metric: name.to_string(), + table_name: None, + index_name: None, + operation: Some("succeeded".to_owned()), + sum: 1.0, + count: 1, + min: 1.0, + max: 1.0, + }) + .collect(); + let (snapshots, _buckets) = aggregate_rows(&rows, 60, TimeWindow::Last5Minutes); + let mut rendered: Vec = snapshots.iter().map(|s| s.metric).collect(); + rendered.sort_by_key(|m| m.to_string()); + let mut expected = names.to_vec(); + expected.sort_by_key(|m| m.to_string()); + assert_eq!(rendered, expected); + assert!(snapshots.iter().all(|s| { + s.dimensions + .iter() + .any(|d| matches!(d, Dimension::Operation(o) if o == "succeeded")) + })); + } } diff --git a/crates/server/src/registration.rs b/crates/server/src/registration.rs index a763f774b..7bcc563ec 100644 --- a/crates/server/src/registration.rs +++ b/crates/server/src/registration.rs @@ -9,7 +9,11 @@ //! `ServiceUnavailable` (HTTP 503), which the AWS SDKs retry, and `/health` //! answers the same. It also stops its background workers and the tasks //! they started: their token fires, and the ones still running after 5 s are -//! aborted. What requests started goes on until it ends or the process +//! aborted; and it cancels the store-backed backup jobs running then, each +//! of which fails its row with reason `shutdown` and deletes its objects, +//! since a job's heartbeats and status writes would otherwise go on beside +//! the migration (the jobs started under a new registration run on a fresh +//! token). What requests started goes on until it ends or the process //! exits: the requests already running, and the work they left running in //! the background, which is a `RestoreTableFromBackup`'s item copy on every //! backend, an `UpdateTimeToLive` backfill on Cassandra and PostgreSQL, and diff --git a/crates/server/src/serve.rs b/crates/server/src/serve.rs index 57bfab1bd..ea4293596 100644 --- a/crates/server/src/serve.rs +++ b/crates/server/src/serve.rs @@ -176,6 +176,43 @@ impl std::fmt::Display for CatalogRefusal { impl std::error::Error for CatalogRefusal {} +/// The refusal for a catalog error the backend's factory returned, for +/// `serve` and for every command that opens the catalog through the factory +/// (`extenddb backup list` and `purge`): the message, with the command that +/// resolves the status, and the exit code `migrate` and `verify` use for it +/// (3 for a catalog behind this binary, 6 while a migration runs). +pub async fn catalog_refusal( + error: &extenddb_storage::migrations::MigrationError, + release: &'static str, + app_config: &config::AppConfig, + config_path: Option<&str>, +) -> CatalogRefusal { + use extenddb_storage::migrations::{Target, advice, installed}; + // A server started without a config file (a dev server configured + // by `EXTENDDB__` variables) advises `migrate` without one, which + // reads the same variables. The backup clause names the data + // store the catalog records. + let target = Target::at(app_config.storage.connection_config()); + let named = match installed() { + Some(migrations) => Target { + data_store: migrations.data_store_name(&target).await, + ..target + }, + None => target, + }; + let advice = advice::AdviceFor::new( + release, + config::redact_password(app_config.storage.connection_config()), + installed(), + &named, + config_path, + ); + CatalogRefusal { + exit_code: error.exit_code(), + message: advice::error_message(error, &advice.advice()), + } +} + /// `serve` stopped because the backend's serving registration was lost and /// could not be taken again. It stopped through the shutdown path a signal /// takes: the listener drained, the workers stopped and the PID file was @@ -341,6 +378,7 @@ async fn serve_inner(params: ServeParams, port: u16) -> anyhow::Result<()> { component_options.bootstrap_if_uninitialized = dev_mode; component_options.max_item_size_bytes = app_config.limits.max_item_size_bytes; component_options.release = build.version; + component_options.max_concurrent_jobs = app_config.backup.max_concurrent_jobs; // Wire the node's own bind address as the instance ID so each ExtendDB node // has a distinct node ID (to support logical clocks for sequences, etc.). @@ -357,30 +395,13 @@ async fn serve_inner(params: ServeParams, port: u16) -> anyhow::Result<()> { { Ok(components) => components, Err(extenddb_storage::server_components::BackendError::Catalog(error)) => { - use extenddb_storage::migrations::{Target, advice, installed}; - // A server started without a config file (a dev server configured - // by `EXTENDDB__` variables) advises `migrate` without one, which - // reads the same variables. The backup clause names the data - // store the catalog records. - let target = Target::at(app_config.storage.connection_config()); - let named = match installed() { - Some(migrations) => Target { - data_store: migrations.data_store_name(&target).await, - ..target - }, - None => target, - }; - let advice = advice::AdviceFor::new( + return Err(catalog_refusal( + &error, build.version, - config::redact_password(app_config.storage.connection_config()), - installed(), - &named, + &app_config, config_path.as_deref(), - ); - return Err(CatalogRefusal { - exit_code: error.exit_code(), - message: advice::error_message(&error, &advice.advice()), - } + ) + .await .into()); } Err(other) => return Err(other.into()), @@ -640,8 +661,49 @@ async fn serve_inner(params: ServeParams, port: u16) -> anyhow::Result<()> { initial_throttling, )); + // The backup store: resolved per backend, opened, validated, its identity + // checked. A configuration error stops startup here; a transport or + // identity failure starts the server with the store unhealthy + // (`backup_store` module doc). The spool lock lives until shutdown. + // Workers and backup jobs run until `shutdown` is cancelled, which + // happens after the HTTP server stops accepting; the jobs are cancelled + // on a child token first, so their failure paths run and record their + // metrics before the workers (the metrics flush among them) are drained. + let shutdown = CancellationToken::new(); + let backend_name = app_config.storage.backend.clone(); + let (backup_runtime, _spool_lock) = crate::backup_store::configure( + &app_config.backup, + app_config.storage.database_file().as_deref(), + &backend_name, + &storage, + crate::backup_store::Caller::Server, + ) + .await?; + let backup_runtime = Arc::new( + backup_runtime + .with_metrics(metrics.clone()) + .with_shutdown(shutdown.child_token()) + .with_source(&backend_name, build.version) + .with_limits(Arc::clone(&limits)) + .with_region(&app_config.server.region), + ); + // An in-memory catalog is new at every start: the server syncs it with + // the store before it serves, when the store's check passed here (a + // filesystem store), so the backups of the previous starts are served + // from the first request. The sync is bounded + // (`extenddb_engine::sync::STARTUP_SYNC_TIMEOUT`, 60 s); one that does + // not finish, or that met a store or catalog error on a prefix, is left + // to the reconciler's next cycle and logged (`backup_store` module doc). + if backup_runtime + .configured_store() + .is_some_and(extenddb_engine::ConfiguredStore::startup_sync_pending) + { + extenddb_engine::sync::startup_sync(&*storage, &backup_runtime).await; + } + let backup_runtime_for_reconciler = Arc::clone(&backup_runtime); let state = AppState { - storage, + storage: storage.clone(), + backup_runtime, auth, limits, region: Arc::from(app_config.server.region.as_str()), @@ -669,19 +731,29 @@ async fn serve_inner(params: ServeParams, port: u16) -> anyhow::Result<()> { }; // The server's own workers and the backend's run as one set, each set on - // a child token of `shutdown`, which is cancelled after the HTTP server - // stops accepting. The registration supervisor stops the set while the - // server holds no registration and starts a new one once it holds one. - // Handles are kept so a stop can drain them (the metrics flush worker - // persists its final bucket on the way out) instead of leaving the work - // to a runtime drop. - let shutdown = CancellationToken::new(); + // a child token of `shutdown`. The registration supervisor stops the set + // while the server holds no registration and starts a new one once it + // holds one. Handles are kept so a stop can drain them (the metrics flush + // worker persists its final bucket on the way out) instead of leaving the + // work to a runtime drop. let hooks: Option> = runtime_hooks.map(Arc::from); + let reconciler_settings = extenddb_engine::reconciler::ReconcilerSettings { + interval: std::time::Duration::from_secs(app_config.backup.reconcile_interval_secs), + stale_job_after: std::time::Duration::from_secs(app_config.backup.stale_job_after_secs), + }; + // The reconciler's once-per-record log set lives here, outside the + // workers, so a worker restart (each time the serving registration is + // taken again) does not log the same dead manifest record again. + let reconciler_state = Arc::new(tokio::sync::Mutex::new( + extenddb_engine::reconciler::ReconcilerState::default(), + )); let spawn: SpawnWorkers = { let catalog_store = catalog_store.clone(); let reload_handle = reload_handle.clone(); let config_log_level = app_config.logging.level.clone(); let metrics = metrics.clone(); + let storage = storage.clone(); + let backup_runtime = Arc::clone(&backup_runtime_for_reconciler); Box::new(move |token: CancellationToken| { let catalog_store = catalog_store.clone(); let reload_handle = reload_handle.clone(); @@ -689,6 +761,10 @@ async fn serve_inner(params: ServeParams, port: u16) -> anyhow::Result<()> { let throttle = throttle.clone(); let metrics = metrics.clone(); let hooks = hooks.clone(); + let storage = storage.clone(); + let reconciler_state = reconciler_state.clone(); + let backup_runtime = Arc::clone(&backup_runtime); + let backend_name = backend_name.clone(); futures::FutureExt::boxed(async move { let mut handles = vec![ // D-22: Poll log_level from the settings table. @@ -722,6 +798,19 @@ async fn serve_inner(params: ServeParams, port: u16) -> anyhow::Result<()> { )), // Phase 11a: Warn about approximate consumed capacity. tokio::spawn(workers::capacity_warning_worker(token.clone())), + // The backup reconciler: backend-agnostic, so it runs + // here beside the backends' workers rather than in each + // backend's `spawn_workers`, and stops and starts with + // them. + tokio::spawn(extenddb_engine::reconciler::run( + storage, + backup_runtime, + reconciler_settings, + backend_name, + metrics.clone(), + reconciler_state, + token.clone(), + )), ]; // Backend-specific workers via runtime hooks. if let Some(hooks) = hooks { @@ -738,7 +827,10 @@ async fn serve_inner(params: ServeParams, port: u16) -> anyhow::Result<()> { }) }) }; - let background = Arc::new(BackgroundWorkers::new(spawn, &shutdown)); + let background = Arc::new( + BackgroundWorkers::new(spawn, &shutdown) + .with_jobs(Arc::clone(&backup_runtime_for_reconciler)), + ); background.start().await; // A backend whose registration the database can end: the supervisor // refuses requests and stops the workers while it is lost, and asks for @@ -765,22 +857,16 @@ async fn serve_inner(params: ServeParams, port: u16) -> anyhow::Result<()> { let server_result = crate::start_server(listener, state, pid_file, tls_config).await; - // The HTTP server has stopped accepting. The supervisor returns once the - // token fires; then the workers stop, so in-flight cycles finish and the - // metrics flush worker writes its final bucket, and the serving - // registration is given up once every one of them has ended. - shutdown.cancel(); - if let Some(supervisor) = supervisor - && tokio::time::timeout(SUPERVISOR_TIMEOUT, supervisor) - .await - .is_err() - { - tracing::warn!( - "The registration supervisor did not return within {}s", - SUPERVISOR_TIMEOUT.as_secs() - ); - } - stop_then_release(&background, held.as_deref()).await; + // The HTTP server has stopped accepting. + drain_then_release( + &backup_runtime_for_reconciler, + JOB_DRAIN_TIMEOUT, + &shutdown, + supervisor, + &background, + held.as_deref(), + ) + .await; server_result?; @@ -796,6 +882,44 @@ async fn serve_inner(params: ServeParams, port: u16) -> anyhow::Result<()> { Ok(()) } +/// The shutdown sequence once the listener has stopped. The backup jobs are +/// cancelled and awaited first, within `job_drain` (a cancelled job's +/// failure path deletes its prefix, which on S3 is what sweeps an upload +/// still in flight), so their failure paths record their metrics before the +/// workers, the metrics flush among them, stop. Then the supervisor returns +/// once the token fires, the workers stop, so in-flight cycles finish and +/// the metrics flush worker writes its final bucket, and the serving +/// registration is given up once every job and worker has ended: a job +/// still running at the drain's bound keeps the registration, which expires +/// by itself ([`stop_then_release`]). +async fn drain_then_release( + jobs: &extenddb_engine::BackupRuntime, + job_drain: Duration, + shutdown: &CancellationToken, + supervisor: Option>, + workers: &BackgroundWorkers, + registrar: Option<&dyn extenddb_storage::hooks::ServingRegistrar>, +) { + let jobs_left = jobs.drain_jobs(job_drain).await; + shutdown.cancel(); + if let Some(supervisor) = supervisor + && tokio::time::timeout(SUPERVISOR_TIMEOUT, supervisor) + .await + .is_err() + { + tracing::warn!( + "The registration supervisor did not return within {}s", + SUPERVISOR_TIMEOUT.as_secs() + ); + } + stop_then_release(workers, registrar, jobs_left == 0).await; +} + +/// How long shutdown waits for the backup jobs' failure paths, before the +/// supervisor and the workers: a `delete_prefix` of a large S3 prefix takes +/// longer than the workers' drain allows. +pub const JOB_DRAIN_TIMEOUT: Duration = Duration::from_secs(30); + /// How long `serve` waits for the registration supervisor once the shutdown /// token fires: longer than a stop of the workers, which the supervisor may /// be running then. @@ -808,6 +932,23 @@ const DRAIN_TIMEOUT: Duration = Duration::from_secs(5); /// How long a stop waits for the aborted workers to end. const ABORT_TIMEOUT: Duration = Duration::from_secs(1); +/// How long shutdown's stop of the workers waits: the drain and the abort, +/// plus a supervisor's own stop it may queue behind (one that did not +/// return within [`SUPERVISOR_TIMEOUT`]). +const STOP_TIMEOUT: Duration = Duration::from_secs( + DRAIN_TIMEOUT.as_secs() + ABORT_TIMEOUT.as_secs() + SUPERVISOR_TIMEOUT.as_secs(), +); + +/// How long shutdown waits for the serving registration's release. +const RELEASE_TIMEOUT: Duration = Duration::from_secs(10); + +/// The longest shutdown takes after the job drain: the supervisor's return, +/// the workers' stop and the registration's release, each under its own +/// bound. `extenddb stop` waits for the sum of the shutdown bounds. +pub const WORKER_STOP_TIMEOUT: Duration = Duration::from_secs( + SUPERVISOR_TIMEOUT.as_secs() + STOP_TIMEOUT.as_secs() + RELEASE_TIMEOUT.as_secs(), +); + /// Starts one set of background workers on the token it is given, and /// returns their handles. type SpawnWorkers = Box< @@ -828,7 +969,11 @@ struct WorkerSet { /// and stop as one set. `serve` starts the first set and stops the one that /// runs at shutdown; the registration supervisor stops the set while the /// server holds no registration and starts a new one once it holds one. -/// Each set runs on a child token of the server's shutdown token. +/// Each set runs on a child token of the server's shutdown token. A stop +/// also cancels the backup jobs running then ([`Self::with_jobs`]): a +/// migration can take the catalog once a lost registration expires, and a +/// job's heartbeats and status writes would otherwise go on beside it; the +/// jobs started under a new registration run on a fresh token. struct BackgroundWorkers { spawn: SpawnWorkers, shutdown: CancellationToken, @@ -837,6 +982,8 @@ struct BackgroundWorkers { left_running: AtomicBool, drain: Duration, abort: Duration, + /// The runtime whose jobs a stop cancels, when the server has one. + jobs: Option>, } impl BackgroundWorkers { @@ -857,8 +1004,15 @@ impl BackgroundWorkers { left_running: AtomicBool::new(false), drain, abort, + jobs: None, } } + + /// The same workers with the backup jobs a stop cancels (type doc). + fn with_jobs(mut self, jobs: Arc) -> Self { + self.jobs = Some(jobs); + self + } } #[async_trait::async_trait] @@ -869,6 +1023,15 @@ impl WorkerControl for BackgroundWorkers { /// outlives its abort (one that blocks its thread) is reported once and /// for the rest of the process. async fn stop(&self) -> bool { + if let Some(jobs) = &self.jobs { + let cancelled = jobs.cancel_jobs(); + if cancelled > 0 { + tracing::warn!( + "{cancelled} backup job(s) cancelled with the workers' stop; each fails its \ + row with reason shutdown and deletes its objects" + ); + } + } let mut running = self.running.lock().await; if let Some(set) = running.take() && !stop_set(set, self.drain, self.abort).await @@ -931,16 +1094,16 @@ async fn stop_set(set: WorkerSet, drain: Duration, abort: Duration) -> bool { /// Stop the workers, then release the serving registration: a migration /// that waits for the servers starts once the registration is gone, so -/// releasing it while a worker runs would let the migration run beside a -/// worker that still writes to the catalog. When a worker could not be -/// stopped, the registration is not released and expires by itself. The +/// releasing it while a worker or a backup job runs would let the migration +/// run beside work that still writes to the catalog. When a worker could +/// not be stopped, or `jobs_ended` is false (a backup job outlived its +/// drain), the registration is not released and expires by itself. The /// stop and the release are bounded. async fn stop_then_release( workers: &BackgroundWorkers, registrar: Option<&dyn extenddb_storage::hooks::ServingRegistrar>, + jobs_ended: bool, ) { - const RELEASE_TIMEOUT: Duration = Duration::from_secs(10); - let bound = workers.drain + workers.abort + SUPERVISOR_TIMEOUT; let ended = tokio::time::timeout(bound, workers.stop()) .await @@ -955,6 +1118,13 @@ async fn stop_then_release( ); return; } + if !jobs_ended { + tracing::warn!( + "A backup job may still run, so the serving registration is not released; it expires \ + by itself" + ); + return; + } if tokio::time::timeout(RELEASE_TIMEOUT, registrar.release()) .await .is_err() @@ -1066,7 +1236,7 @@ mod tests { workers.start().await; let registrar = Recorder(events.clone()); shutdown.cancel(); - super::stop_then_release(&workers, Some(®istrar)).await; + super::stop_then_release(&workers, Some(®istrar), true).await; assert_eq!(*events.lock().unwrap(), ["worker stopped", "released"]); } @@ -1092,7 +1262,7 @@ mod tests { workers.start().await; let registrar = Recorder(events.clone()); shutdown.cancel(); - super::stop_then_release(&workers, Some(®istrar)).await; + super::stop_then_release(&workers, Some(®istrar), true).await; assert_eq!(*events.lock().unwrap(), ["released"]); let at_release = ticks.load(Ordering::SeqCst); tokio::time::sleep(Duration::from_millis(50)).await; @@ -1118,11 +1288,115 @@ mod tests { tokio::time::sleep(Duration::from_millis(50)).await; let registrar = Recorder(events.clone()); shutdown.cancel(); - super::stop_then_release(&workers, Some(®istrar)).await; + super::stop_then_release(&workers, Some(®istrar), true).await; assert!(events.lock().unwrap().is_empty(), "released"); assert!(!workers.stop().await, "a later stop forgot the worker"); } + /// A backup job still running after its drain keeps the registration + /// as a worker that could not be stopped does: the workers stop, and + /// the registration is left to expire. + #[tokio::test] + async fn a_job_left_running_keeps_the_registration() { + let events: Events = Arc::new(Mutex::new(Vec::new())); + let shutdown = CancellationToken::new(); + let workers = { + let events = events.clone(); + workers(&shutdown, move |token| { + let events = events.clone(); + async move { + token.cancelled().await; + events.lock().unwrap().push("worker stopped"); + } + }) + }; + workers.start().await; + let registrar = Recorder(events.clone()); + shutdown.cancel(); + super::stop_then_release(&workers, Some(®istrar), false).await; + assert_eq!(*events.lock().unwrap(), ["worker stopped"]); + } + + /// The shutdown sequence `serve` runs after its listener stops, with a + /// job that outlives a short drain bound: the job's token is cancelled, + /// the drain gives up at its bound, the server's token fires, the + /// workers stop, and the registration is not released. The same + /// sequence with the job ending at its cancellation releases it, after + /// the worker stopped. + #[tokio::test] + async fn the_shutdown_sequence_keeps_the_registration_while_a_job_outlives_the_drain() { + for (ends_on_cancel, expected) in [ + (false, vec!["worker stopped"]), + (true, vec!["job ended", "worker stopped", "released"]), + ] { + let events: Events = Arc::new(Mutex::new(Vec::new())); + let shutdown = CancellationToken::new(); + let runtime = + extenddb_engine::BackupRuntime::new(4).with_shutdown(shutdown.child_token()); + let token = runtime.shutdown(); + runtime.spawn_job({ + let events = events.clone(); + async move { + if ends_on_cancel { + token.cancelled().await; + events.lock().unwrap().push("job ended"); + } else { + // A job in a store call that does not return. + std::future::pending::<()>().await; + } + } + }); + let workers = { + let events = events.clone(); + workers(&shutdown, move |token| { + let events = events.clone(); + async move { + token.cancelled().await; + events.lock().unwrap().push("worker stopped"); + } + }) + }; + workers.start().await; + let registrar = Recorder(events.clone()); + super::drain_then_release( + &runtime, + Duration::from_millis(100), + &shutdown, + None, + &workers, + Some(®istrar), + ) + .await; + assert!(shutdown.is_cancelled()); + assert_eq!(*events.lock().unwrap(), expected, "{ends_on_cancel}"); + assert_eq!(runtime.running_jobs(), usize::from(!ends_on_cancel)); + } + } + + /// A stop of the workers (the supervisor's, at a registration loss) + /// cancels the jobs running then and leaves a fresh token for the jobs + /// that start under the new registration. + #[tokio::test] + async fn a_stop_of_the_workers_cancels_the_jobs_and_renews_their_token() { + let shutdown = CancellationToken::new(); + let runtime = + Arc::new(extenddb_engine::BackupRuntime::new(4).with_shutdown(shutdown.child_token())); + let before = runtime.shutdown(); + let workers = workers(&shutdown, |token| async move { token.cancelled().await }) + .with_jobs(Arc::clone(&runtime)); + workers.start().await; + assert!(workers.stop().await); + assert!(before.is_cancelled(), "the jobs' token is cancelled"); + let after = runtime.shutdown(); + assert!(!after.is_cancelled(), "a fresh token for the next jobs"); + assert!(!shutdown.is_cancelled(), "the server's token is untouched"); + shutdown.cancel(); + assert!( + after.is_cancelled(), + "the fresh token is the server's child" + ); + } + /// A stopped set starts again on a new token: the supervisor stops the /// workers at a loss and starts them after a new registration. #[tokio::test] diff --git a/crates/storage-cassandra/Cargo.toml b/crates/storage-cassandra/Cargo.toml index 886fece9b..ad9bb4b7e 100644 --- a/crates/storage-cassandra/Cargo.toml +++ b/crates/storage-cassandra/Cargo.toml @@ -32,6 +32,11 @@ chrono = { workspace = true } [features] # Entry points the integration tests build catalogs with. test-util = ["extenddb-storage/test-util"] +# Coordination hooks used only by this crate's integration test binary +# (`src/test_hooks.rs`). Production builds do not compile them. Enabling it +# also arms `extenddb_storage::backup_catalog::restore_pause_point`, the +# `bulk_load` kill point the restore kill-point harness drives. +test-hooks = ["extenddb-storage/test-hooks"] [dev-dependencies] extenddb-storage = { workspace = true, features = ["test-util"] } diff --git a/crates/storage-cassandra/migrations/catalog/0006_backup_catalog_index.cql b/crates/storage-cassandra/migrations/catalog/0006_backup_catalog_index.cql new file mode 100644 index 000000000..cad42d0cb --- /dev/null +++ b/crates/storage-cassandra/migrations/catalog/0006_backup_catalog_index.cql @@ -0,0 +1,72 @@ +-- Copyright 2026 ExtendDB contributors +-- SPDX-License-Identifier: Apache-2.0 +-- +-- Migration 6, backup_catalog_index, Cassandra catalog keyspace: the +-- statements of the catalog index of backups and restore jobs. The code body +-- in src/migrations/m0006_backup_catalog_index.rs executes them and then +-- backfills the restore registrations that exist and writes the catalog id, +-- which CQL cannot express. Every statement is IF NOT EXISTS, so a second +-- run changes nothing. The runner writes the catalog keyspace in front of +-- each table. +-- +-- backups_by_arn is reused for the index rows of backups whose bytes live in +-- a backup store. A row with store_kind set is such a row and belongs to the +-- catalog index. A row with store_kind unset is an in-catalog backup written +-- by this or an earlier release and belongs to the legacy reader. Neither +-- path sees the other's rows. owner_generation and heartbeat_at are the claim +-- of the job that owns the row (both kinds). backup_size_bytes and item_count, +-- already present, are the index row's size_bytes and item_count. +-- +-- backup_index_by_status gives the status-keyed read backups_by_arn cannot: +-- one partition per (store root, status), maintained beside every status +-- write of an index row. +-- +-- tables.restore_* gain the restore job record: status, kind, the store the +-- restore reads, the claim (restore_owner_generation and restore_heartbeat_at, +-- beside the restore_owner and restore_heartbeat columns migration 1 +-- creates, which stop being written), spool and progress fields, the failure +-- reason and the completion time. +-- +-- backup_maintenance holds the catalog-level maintenance claim, one row. + +ALTER TABLE backups_by_arn ADD IF NOT EXISTS backup_id text; +ALTER TABLE backups_by_arn ADD IF NOT EXISTS store_kind text; +ALTER TABLE backups_by_arn ADD IF NOT EXISTS store_root text; +ALTER TABLE backups_by_arn ADD IF NOT EXISTS store_id text; +ALTER TABLE backups_by_arn ADD IF NOT EXISTS owner_generation bigint; +ALTER TABLE backups_by_arn ADD IF NOT EXISTS heartbeat_at timestamp; +ALTER TABLE backups_by_arn ADD IF NOT EXISTS completed_at timestamp; +ALTER TABLE backups_by_arn ADD IF NOT EXISTS failure_reason text; +ALTER TABLE backups_by_arn ADD IF NOT EXISTS source_table_details_json text; + +CREATE TABLE IF NOT EXISTS backup_index_by_status ( + store_root text, + backup_status text, + backup_arn text, + account_id text, + store_kind text, + store_id text, + PRIMARY KEY ((store_root, backup_status), backup_arn) +); + +ALTER TABLE tables ADD IF NOT EXISTS restore_status text; +ALTER TABLE tables ADD IF NOT EXISTS restore_kind text; +ALTER TABLE tables ADD IF NOT EXISTS restore_store_kind text; +ALTER TABLE tables ADD IF NOT EXISTS restore_store_root text; +ALTER TABLE tables ADD IF NOT EXISTS restore_store_id text; +ALTER TABLE tables ADD IF NOT EXISTS restore_owner_generation bigint; +ALTER TABLE tables ADD IF NOT EXISTS restore_heartbeat_at timestamp; +ALTER TABLE tables ADD IF NOT EXISTS restore_spool_host text; +ALTER TABLE tables ADD IF NOT EXISTS restore_file_index int; +ALTER TABLE tables ADD IF NOT EXISTS restore_items_applied bigint; +ALTER TABLE tables ADD IF NOT EXISTS restore_failure_reason text; +ALTER TABLE tables ADD IF NOT EXISTS restore_completed_at timestamp; + +CREATE TABLE IF NOT EXISTS backup_maintenance ( + claim text, + holder text, + owner_generation bigint, + claimed_at timestamp, + heartbeat_at timestamp, + PRIMARY KEY (claim) +); diff --git a/crates/storage-cassandra/src/backup_catalog.rs b/crates/storage-cassandra/src/backup_catalog.rs new file mode 100644 index 000000000..61e7ee05e --- /dev/null +++ b/crates/storage-cassandra/src/backup_catalog.rs @@ -0,0 +1,2756 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The catalog index on Cassandra: the index rows in the reused +//! `backups_by_arn`, `backups_by_account` and `backups_by_table` tables, the +//! status-keyed `backup_index_by_status`, the restore job record on the +//! target's `tables` row, `backup_maintenance` and `settings.catalog_id`. +//! +//! An index row is a `backups_by_arn` row with `store_kind` set; a row without +//! it is an in-catalog backup of the legacy reader (`backup_engine.rs`), and +//! each path skips the other's rows. `backups_by_arn`, partitioned by +//! `(account_id, backup_arn)`, is authoritative and is written first by every +//! writer; the list rows and the `backup_index_by_status` row follow, and a +//! reader of those re-reads the authoritative row and skips an entry it no +//! longer matches, so a writer that dies between the writes leaves nothing a +//! reader believes. An index row's list rows exist for as long as the row +//! does, of whatever status (the legacy create writes its list rows only when +//! its row becomes AVAILABLE); `remove_backup_row` takes them with the row. +//! +//! The claim is the row: `claim_backup` inserts it `IF NOT EXISTS` with a +//! fresh `owner_generation` and `heartbeat_at = toTimestamp(now())`, every +//! later write by the job is a lightweight transaction `IF owner_generation = +//! `, and a heartbeat that does not apply means another owner has +//! the row. Times are the coordinator's clock; staleness is measured against +//! it (`coordinator_now_millis`), never an application host's. A dead +//! owner's row (`claim_dead_backup`) is taken over by an LWT on the +//! generation, the status and the heartbeat as read at `LOCAL_SERIAL`, the +//! heartbeat left as the owner wrote it. `sweep_restore_registrations` +//! removes, entry by entry under an LWT, the `active_restores` entries whose +//! target is no longer a CREATING restore target of the backup. +//! +//! DeleteBackup's lock is the lightweight transaction on the `backups_by_arn` +//! row, `IF backup_status = 'AVAILABLE' AND active_restores = null`, on the +//! same partition a restore registers on, so the two order wholly one before +//! the other, as the in-catalog paths do. +//! +//! A status change moves the row's `backup_index_by_status` entry with a +//! plain batch after the lightweight transaction on the row. A process that +//! dies between the two leaves an entry whose status the row no longer +//! matches; every reader of the status table re-reads the row and skips such +//! an entry, one wasted read each. `remove_backup_row` deletes the entries +//! under the status it read, so an entry under an earlier status stays until +//! a sweep of orphan entries exists, and no sweep removes them yet. One path +//! writes such an entry by design: `insert_backup_row`'s row is AVAILABLE +//! from its lightweight transaction on, so a DeleteBackup between that +//! transaction and the insert's entry batch moves a status entry the batch +//! has not written yet, and the batch then writes an AVAILABLE entry that no +//! later write removes (an insert completed by a later sync has the same +//! window). +//! +//! A conditional write whose error is the driver's `Io` or `Timeout`, or the +//! server's `WriteTimeout` or `WriteFailure`, has an unknown outcome +//! (`WriteOutcome::Unknown`: the Paxos round may still commit); every other +//! error is `Err`. +//! +//! The maintenance claim is the one row of `backup_maintenance`, taken with +//! `INSERT ... IF NOT EXISTS`; when a row exists, one whose heartbeat is older +//! than the stale window on the coordinator's clock is taken over with +//! `IF owner_generation = `, so two commands racing for a dead holder's +//! row see one win. An insert or a takeover whose answer was lost is settled +//! by a re-read at `LOCAL_SERIAL`: a row carrying the call's own generation is +//! its claim. Heartbeats and the +//! release are lightweight transactions on the generation. The two reads of +//! the claim row that decide whether another process's claim is live +//! (`check_maintenance_claim`, the row before a takeover) run at +//! `LOCAL_SERIAL`, so a heartbeat an LWT just wrote is seen; the RUNNING +//! restore records `list_dead_restore_jobs` scans are a range read, where a +//! serial level is a local quorum with no Paxos read, so the scan reads at +//! quorum and the coordinator clock after it. + +use std::time::Duration; + +use cdrs_tokio::consistency::Consistency; +use cdrs_tokio::error::Error as DriverError; +use cdrs_tokio::frame::message_error::ErrorType; +use cdrs_tokio::query::{BatchQueryBuilder, QueryValues}; +use cdrs_tokio::statement::StatementParamsBuilder; +use cdrs_tokio::types::IntoRustByName; +use cdrs_tokio::types::rows::Row; +use cdrs_tokio::types::value::Value; +use extenddb_core::backup_format::BackupArn; +use extenddb_core::types::CreateTableInput; +use extenddb_storage::backup_catalog::{ + BackupCatalog, BackupKey, BackupRow, BackupStatus, DeleteStart, ForgetRow, Heartbeat, JobClaim, + JobRow, JobStatus, MaintenanceClaim, RestoreJobRow, RestoreKind, RestoreRegistered, + RestoreRegistrationRow, RestoreTargetKey, StaleClock, StatusFields, StoreIdentity, StoreKind, + TimeBounds, UNOWNED_GENERATION, WriteOutcome, billing_mode_name, new_generation, +}; +use extenddb_storage::error::StorageError; +use futures::future::BoxFuture; +use futures::stream::BoxStream; + +use crate::CassandraEngine; +use crate::engine::CassandraSession; + +/// Attempts `start_backup_delete` and `forget_available_backup_row` make +/// when a restore registers or unregisters between their in-use check and +/// their write. +const DELETE_CLAIM_ATTEMPTS: usize = 3; + +/// The condition of the DELETING write and of the outright removal: the row +/// is AVAILABLE and no restore is registered on it. +const DELETE_CLAIM_CONDITION: &str = "IF backup_status = 'AVAILABLE' AND active_restores = null"; + +/// The `owner_generation` column value of a generation: the column is a +/// signed 64-bit integer and `new_generation` never exceeds `i64::MAX`. +pub(crate) fn generation_column(generation: u64) -> i64 { + i64::try_from(generation).unwrap_or(i64::MAX) +} + +/// The generation a column holds; a negative value no release writes reads +/// as the unowned sentinel. +fn generation_from_column(column: Option) -> u64 { + column + .and_then(|g| u64::try_from(g).ok()) + .unwrap_or_default() +} + +fn not_found(backup_arn: &str) -> StorageError { + extenddb_storage::backup_catalog::backup_not_found(backup_arn) +} + +fn millis_to_seconds(ms: i64) -> f64 { + crate::cassandra_util::millis_to_seconds_f64(ms) +} + +#[allow(clippy::cast_possible_truncation)] +fn seconds_to_millis(seconds: f64) -> i64 { + (seconds * 1_000.0).round() as i64 +} + +/// The coordinator's clock, in epoch milliseconds: the clock every heartbeat +/// is stamped with, so a stale window is measured on it. +pub(crate) async fn coordinator_now_millis(engine: &CassandraEngine) -> Result { + let rows = crate::cassandra_util::query_rows::( + &engine.session, + "SELECT toTimestamp(now()) AS now FROM system.local", + cdrs_tokio::query_values!(), + "coordinator clock", + ) + .await?; + let row = rows + .first() + .ok_or_else(|| StorageError::Internal("coordinator clock: no row".to_owned()))?; + crate::cassandra_util::get_column(row, "now", "coordinator clock") +} + +/// Refresh `heartbeat_at` of a `backups_by_arn` row on the coordinator's +/// clock, conditioned on the claim's generation; `false` when another owner +/// holds the row. Shared by the index claim and the in-catalog create. +#[doc(hidden)] +pub async fn heartbeat_backup_row( + engine: &CassandraEngine, + account_id: &str, + backup_arn: &str, + generation: u64, +) -> Result { + let query = format!( + "UPDATE {}.backups_by_arn SET heartbeat_at = toTimestamp(now()) \ + WHERE account_id = ? AND backup_arn = ? IF owner_generation = ?", + engine.catalog_keyspace() + ); + let result = crate::cassandra_util::query_lwt( + &engine.session, + &query, + cdrs_tokio::query_values!(account_id, backup_arn, generation_column(generation)), + ) + .await?; + crate::cassandra_util::lwt_applied(&result) +} + +/// Whether a write's outcome cannot be told from its error: the statement +/// may have reached a replica before the answer was lost. A write timeout +/// or failure on a lightweight transaction is the Paxos round not completing +/// within the coordinator's view; the proposal may still commit. +fn outcome_unknown(error: &DriverError) -> bool { + match error { + DriverError::Io(_) | DriverError::Timeout(_) => true, + DriverError::Server { body, .. } => matches!( + body.ty, + ErrorType::WriteTimeout(_) | ErrorType::WriteFailure(_) + ), + _ => false, + } +} + +/// Run a conditional write and report what it did. +pub(crate) async fn conditional_write( + session: &CassandraSession, + query: &str, + values: QueryValues, +) -> Result { + let params = StatementParamsBuilder::new() + .with_consistency(Consistency::LocalQuorum) + .with_serial_consistency(Consistency::LocalSerial) + .with_values(values) + .build(); + match session.query_with_params(query, params).await { + Ok(result) => Ok(if crate::cassandra_util::lwt_applied(&result)? { + WriteOutcome::Matched + } else { + WriteOutcome::NotMatched + }), + Err(error) if outcome_unknown(&error) => { + tracing::warn!("conditional catalog write with an unknown outcome: {error}"); + Ok(WriteOutcome::Unknown) + } + Err(error) => Err(StorageError::Internal(format!("LWT query failed: {error}"))), + } +} + +/// The index row a `backups_by_arn` row carries, or `None` for an in-catalog +/// backup (no `store_kind`). +fn index_row_from_row(row: &Row) -> Result, StorageError> { + let store_kind: Option = row.get_by_name("store_kind").ok().flatten(); + let Some(store_kind) = store_kind else { + return Ok(None); + }; + let column = |name: &str| -> Result { + crate::cassandra_util::get_column(row, name, "backup index row") + }; + let optional = |name: &str| -> Option { row.get_by_name(name).ok().flatten() }; + let backup_arn = column("backup_arn")?; + let store_kind = StoreKind::parse(&store_kind).ok_or_else(|| { + StorageError::Internal(format!( + "backup {backup_arn} carries an unknown store kind {store_kind:?}" + )) + })?; + let status: String = column("backup_status")?; + let backup_id = match optional("backup_id") { + Some(id) => id, + None => BackupArn::parse(&backup_arn) + .map(|arn| arn.backup_id().to_owned()) + .map_err(|e| { + StorageError::Internal(format!("backup {backup_arn} carries no backup id: {e}")) + })?, + }; + let source_table_details_json = match optional("source_table_details_json") { + Some(text) => serde_json::from_str(&text).map_err(|e| { + StorageError::Internal(format!( + "backup {backup_arn} carries a source table this build cannot read: {e}" + )) + })?, + None => serde_json::Value::Null, + }; + let created_at: i64 = crate::cassandra_util::get_column(row, "created_at", "backup index row")?; + let completed_at: Option = row.get_by_name("completed_at").ok().flatten(); + let heartbeat_at: Option = row.get_by_name("heartbeat_at").ok().flatten(); + let owner_generation: Option = row.get_by_name("owner_generation").ok().flatten(); + Ok(Some(BackupRow { + backup_id, + account_id: column("account_id")?, + table_name: column("table_name")?, + backup_arn, + backup_name: column("backup_name")?, + status: BackupStatus::parse(&status)?, + store: StoreIdentity { + store_kind, + store_root: column("store_root")?, + store_id: column("store_id")?, + }, + size_bytes: row + .get_by_name("backup_size_bytes") + .ok() + .flatten() + .unwrap_or(0), + item_count: row.get_by_name("item_count").ok().flatten().unwrap_or(0), + created_at: millis_to_seconds(created_at), + completed_at: completed_at.map(millis_to_seconds), + failure_reason: optional("failure_reason"), + source_table_details_json, + owner_generation: generation_from_column(owner_generation), + heartbeat_at: heartbeat_at.map_or(0.0, millis_to_seconds), + })) +} + +/// Whether `read`, the row a serial read found, is the row +/// `insert_backup_row(row)` writes: unowned, with every column the insert +/// takes from `row` equal (`created_at` to the stored millisecond). +fn is_unowned_insert_of(read: &BackupRow, row: &BackupRow) -> bool { + read.owner_generation == UNOWNED_GENERATION + && read.backup_id == row.backup_id + && read.table_name == row.table_name + && read.backup_name == row.backup_name + && read.status == row.status + && read.store == row.store + && read.size_bytes == row.size_bytes + && read.item_count == row.item_count + && seconds_to_millis(read.created_at) == seconds_to_millis(row.created_at) + && read.failure_reason == row.failure_reason + && read.source_table_details_json == row.source_table_details_json +} + +/// The rows `read` fetches, as a stream: the read runs once, on the first +/// poll, and a failed read is one `Err` item. +fn rows_stream<'a, F>(read: F) -> BoxStream<'a, Result> +where + F: std::future::Future, StorageError>> + Send + 'a, +{ + use futures::StreamExt; + Box::pin( + futures::stream::once(async move { + match read.await { + Ok(rows) => rows.into_iter().map(Ok).collect::>(), + Err(e) => vec![Err(e)], + } + }) + .flat_map(futures::stream::iter), + ) +} + +/// `(created_at, backup_arn)` ascending, the order every listing returns. +fn sort_listing(rows: &mut [BackupRow]) { + rows.sort_by(|a, b| { + a.created_at + .partial_cmp(&b.created_at) + .unwrap_or(std::cmp::Ordering::Equal) + .then_with(|| a.backup_arn.cmp(&b.backup_arn)) + }); +} + +/// A backup job's handle on its `backups_by_arn` row. +struct CassandraBackupClaim { + engine: CassandraEngine, + key: BackupKey, + store: StoreIdentity, + generation: u64, +} + +impl JobClaim for CassandraBackupClaim { + fn generation(&self) -> u64 { + self.generation + } + + fn heartbeat(&self) -> BoxFuture<'_, Result> { + Box::pin(async move { + let confirmed = heartbeat_backup_row( + &self.engine, + &self.key.account_id, + &self.key.backup_arn, + self.generation, + ) + .await?; + Ok(if confirmed { + Heartbeat::Confirmed + } else { + Heartbeat::Lost + }) + }) + } + + fn set_status<'a>( + &'a self, + from: JobStatus, + to: JobStatus, + fields: &'a StatusFields, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + let mut assignments = vec!["backup_status = ?".to_owned()]; + let mut values = vec![Value::from(to.as_str())]; + if let Some(size) = fields.size_bytes { + assignments.push("backup_size_bytes = ?".to_owned()); + values.push(Value::from(size)); + } + if let Some(count) = fields.item_count { + assignments.push("item_count = ?".to_owned()); + values.push(Value::from(count)); + } + if let Some(reason) = fields.failure_reason.as_deref() { + assignments.push("failure_reason = ?".to_owned()); + values.push(Value::from(reason)); + } + if fields.completed { + assignments.push("completed_at = toTimestamp(now())".to_owned()); + } + if let Some(source_table) = fields.source_table_details_json.as_ref() { + assignments.push("source_table_details_json = ?".to_owned()); + values.push(Value::from(source_table.to_string())); + } + values.extend([ + Value::from(self.key.account_id.as_str()), + Value::from(self.key.backup_arn.as_str()), + Value::from(generation_column(self.generation)), + Value::from(from.as_str()), + ]); + let query = format!( + "UPDATE {}.backups_by_arn SET {} WHERE account_id = ? AND backup_arn = ? \ + IF owner_generation = ? AND backup_status = ?", + self.engine.catalog_keyspace(), + assignments.join(", ") + ); + let outcome = conditional_write( + &self.engine.session, + &query, + QueryValues::SimpleValues(values), + ) + .await?; + if outcome == WriteOutcome::Matched { + self.engine + .move_status_index_row(&self.key, &self.store, from.as_str(), to.as_str()) + .await; + } + Ok(outcome) + }) + } + + fn read(&self) -> BoxFuture<'_, Result> { + Box::pin(async move { + let query = format!( + "SELECT backup_status, owner_generation, heartbeat_at, failure_reason \ + FROM {}.backups_by_arn WHERE account_id = ? AND backup_arn = ?", + self.engine.catalog_keyspace() + ); + let rows = crate::cassandra_util::query_rows_quorum::( + &self.engine.session, + &query, + cdrs_tokio::query_values!( + self.key.account_id.as_str(), + self.key.backup_arn.as_str() + ), + "job row", + ) + .await?; + let row = rows + .first() + .ok_or_else(|| not_found(&self.key.backup_arn))?; + let status: String = + crate::cassandra_util::get_column(row, "backup_status", "job row")?; + let generation: Option = row.get_by_name("owner_generation").ok().flatten(); + let heartbeat_at: Option = row.get_by_name("heartbeat_at").ok().flatten(); + Ok(JobRow { + status: JobStatus::parse(&status)?, + owner_generation: generation_from_column(generation), + heartbeat_at: heartbeat_at.map_or(0.0, millis_to_seconds), + failure_reason: row.get_by_name("failure_reason").ok().flatten(), + }) + }) + } +} + +impl CassandraEngine { + /// The index row `backup_arn` names in `account_id`, if there is one; an + /// in-catalog row is `None`. + /// Remove the three index entries of a `backups_by_arn` row this process + /// has deleted: its status partition, its account listing and its table + /// listing, in one logged batch. + async fn remove_index_entries(&self, row: &BackupRow) -> Result<(), StorageError> { + let catalog = self.catalog_keyspace(); + let created_at = seconds_to_millis(row.created_at); + if let Some(error) = crate::test_hooks::index_entry_removal_error(&row.backup_arn) { + return Err(error); + } + let batch = BatchQueryBuilder::new() + .with_consistency(Consistency::LocalQuorum) + .add_query( + format!( + "DELETE FROM {catalog}.backup_index_by_status \ + WHERE store_root = ? AND backup_status = ? AND backup_arn = ?" + ), + cdrs_tokio::query_values!( + row.store.store_root.as_str(), + row.status.as_str(), + row.backup_arn.as_str() + ), + ) + .add_query( + format!( + "DELETE FROM {catalog}.backups_by_account \ + WHERE account_id = ? AND created_at = ? AND backup_arn = ?" + ), + cdrs_tokio::query_values!( + row.account_id.as_str(), + created_at, + row.backup_arn.as_str() + ), + ) + .add_query( + format!( + "DELETE FROM {catalog}.backups_by_table \ + WHERE account_id = ? AND table_name = ? AND created_at = ? AND backup_arn = ?" + ), + cdrs_tokio::query_values!( + row.account_id.as_str(), + row.table_name.as_str(), + created_at, + row.backup_arn.as_str() + ), + ); + self.session + .batch( + batch + .build() + .map_err(|e| StorageError::Internal(e.to_string()))?, + ) + .await + .map_err(|e| StorageError::Internal(format!("Remove backup index rows: {e}")))?; + Ok(()) + } + + /// Write the three index entries of a `backups_by_arn` row this process + /// has just inserted, after it: its account listing, its table listing + /// and its status partition under `status`, in one logged batch. + async fn write_index_entries( + &self, + row: &BackupRow, + table_id: &str, + table_arn: &str, + status: BackupStatus, + ) -> Result<(), StorageError> { + let catalog = self.catalog_keyspace(); + let created_at = seconds_to_millis(row.created_at); + if let Some(error) = crate::test_hooks::index_entry_batch_error(&row.backup_arn) { + return Err(error); + } + let batch = BatchQueryBuilder::new() + .with_consistency(Consistency::LocalQuorum) + .add_query( + format!( + "INSERT INTO {catalog}.backups_by_account (account_id, created_at, \ + backup_arn, backup_name, table_id, table_name, table_arn, \ + backup_size_bytes) VALUES (?, ?, ?, ?, ?, ?, ?, ?)" + ), + cdrs_tokio::query_values!( + row.account_id.as_str(), + created_at, + row.backup_arn.as_str(), + row.backup_name.as_str(), + table_id, + row.table_name.as_str(), + table_arn, + row.size_bytes + ), + ) + .add_query( + format!( + "INSERT INTO {catalog}.backups_by_table (account_id, table_name, \ + created_at, backup_arn, backup_name, table_id, table_arn, \ + backup_size_bytes) VALUES (?, ?, ?, ?, ?, ?, ?, ?)" + ), + cdrs_tokio::query_values!( + row.account_id.as_str(), + row.table_name.as_str(), + created_at, + row.backup_arn.as_str(), + row.backup_name.as_str(), + table_id, + table_arn, + row.size_bytes + ), + ) + .add_query( + format!( + "INSERT INTO {catalog}.backup_index_by_status \ + (store_root, backup_status, backup_arn, account_id, store_kind, store_id) \ + VALUES (?, ?, ?, ?, ?, ?)" + ), + cdrs_tokio::query_values!( + row.store.store_root.as_str(), + status.as_str(), + row.backup_arn.as_str(), + row.account_id.as_str(), + row.store.store_kind.as_str(), + row.store.store_id.as_str() + ), + ); + self.session + .batch( + batch + .build() + .map_err(|e| StorageError::Internal(e.to_string()))?, + ) + .await + .map_err(|e| StorageError::Internal(format!("Backup index rows: {e}")))?; + Ok(()) + } + + /// [`Self::read_index_row`] at `LOCAL_SERIAL`, so a row a lightweight + /// transaction committed, or one whose round is in progress, is seen. + async fn read_index_row_serial( + &self, + account_id: &str, + backup_arn: &str, + ) -> Result, StorageError> { + let query = format!( + "SELECT * FROM {}.backups_by_arn WHERE account_id = ? AND backup_arn = ?", + self.catalog_keyspace() + ); + let rows = crate::cassandra_util::query_rows_serial::( + &self.session, + &query, + cdrs_tokio::query_values!(account_id, backup_arn), + "backup index row", + ) + .await?; + rows.first() + .map(index_row_from_row) + .transpose() + .map(|row| row.flatten().filter(|r| r.account_id == account_id)) + } + + async fn read_index_row( + &self, + account_id: &str, + backup_arn: &str, + ) -> Result, StorageError> { + let query = format!( + "SELECT * FROM {}.backups_by_arn WHERE account_id = ? AND backup_arn = ?", + self.catalog_keyspace() + ); + let rows = crate::cassandra_util::query_rows_quorum::( + &self.session, + &query, + cdrs_tokio::query_values!(account_id, backup_arn), + "backup index row", + ) + .await?; + rows.first() + .map(index_row_from_row) + .transpose() + .map(|row| row.flatten().filter(|r| r.account_id == account_id)) + } + + /// Move the `backup_index_by_status` entry of a row from one status + /// partition to another, after the authoritative row changed. A failure + /// is logged: a reader of the status index re-reads the authoritative + /// row and skips an entry whose status it no longer matches. + async fn move_status_index_row( + &self, + key: &BackupKey, + store: &StoreIdentity, + from: &str, + to: &str, + ) { + let catalog = self.catalog_keyspace(); + let batch = BatchQueryBuilder::new() + .with_consistency(Consistency::LocalQuorum) + .add_query( + format!( + "DELETE FROM {catalog}.backup_index_by_status \ + WHERE store_root = ? AND backup_status = ? AND backup_arn = ?" + ), + cdrs_tokio::query_values!(store.store_root.as_str(), from, key.backup_arn.as_str()), + ) + .add_query( + format!( + "INSERT INTO {catalog}.backup_index_by_status \ + (store_root, backup_status, backup_arn, account_id, store_kind, store_id) \ + VALUES (?, ?, ?, ?, ?, ?)" + ), + cdrs_tokio::query_values!( + store.store_root.as_str(), + to, + key.backup_arn.as_str(), + key.account_id.as_str(), + store.store_kind.as_str(), + store.store_id.as_str() + ), + ); + let sent = match batch.build() { + Ok(batch) => self.session.batch(batch).await.map(|_| ()), + Err(error) => Err(error), + }; + if let Err(error) = sent { + tracing::warn!( + "backup {}: its status index entry could not be moved from {from} to {to} \ + ({error}); readers verify against the row", + key.backup_arn + ); + } + } + + /// DeleteBackup's first step, and with `remove` the outright removal of + /// `forget_available_backup_row`, as one body: read the row, check its + /// status, run the in-use rule, then one lightweight transaction under + /// [`DELETE_CLAIM_CONDITION`] that writes DELETING or, with `remove`, + /// deletes the row (`Deleting` then means removed); then the row's index + /// entries. Every writer of `active_restores` is conditional, so the + /// condition orders the write wholly before or after a registration. + /// + /// A lost answer of the removal is settled by a serial read: no row + /// means removed, a row means the loop runs again (a lost answer on + /// every attempt is an error). A lost answer of the DELETING write is an + /// error (another delete may have written the same status). After the + /// removal a failed entry batch is logged: the row is gone, and every + /// reader of an entry re-reads the row and skips it. + async fn start_backup_delete_lwt( + &self, + key: &BackupKey, + remove: bool, + ) -> Result { + let catalog = self.catalog_keyspace(); + let write = if remove { + format!( + "DELETE FROM {catalog}.backups_by_arn WHERE account_id = ? AND backup_arn = ? \ + {DELETE_CLAIM_CONDITION}" + ) + } else { + format!( + "UPDATE {catalog}.backups_by_arn SET backup_status = 'DELETING' \ + WHERE account_id = ? AND backup_arn = ? {DELETE_CLAIM_CONDITION}" + ) + }; + let mut attempt = 0; + loop { + let stored = self + .load_stored_backup(&key.account_id, &key.backup_arn) + .await? + .filter(|stored| !stored.is_in_catalog()) + .ok_or_else(|| not_found(&key.backup_arn))?; + let status = BackupStatus::parse(&stored.backup_status)?; + if status != BackupStatus::Available { + return Ok(DeleteStart::Status(status)); + } + // The in-use rule reads the target's status; a registration whose + // target is no longer being restored is cleared here so it cannot + // hold the row's lock. + if let Some(target) = self.creating_restore_target(&stored).await? { + return Ok(DeleteStart::InUse(target)); + } + // The removal's entries are keyed by the row's columns, so the + // row is read before it goes. + let removed_row = if remove { + Some( + self.read_index_row(&key.account_id, &key.backup_arn) + .await? + .ok_or_else(|| not_found(&key.backup_arn))?, + ) + } else { + None + }; + crate::test_hooks::before_delete_claim(&key.backup_arn).await; + let values = + || cdrs_tokio::query_values!(key.account_id.as_str(), key.backup_arn.as_str()); + let fault = if remove { + crate::test_hooks::forget_write_fault(&key.backup_arn) + } else { + None + }; + let outcome = if fault == Some(crate::test_hooks::ProgressWriteFault::Lost) { + WriteOutcome::Unknown + } else { + let outcome = conditional_write(&self.session, &write, values()).await?; + if fault.is_some() { + WriteOutcome::Unknown + } else { + outcome + } + }; + let applied = match outcome { + WriteOutcome::Matched => true, + WriteOutcome::NotMatched => false, + WriteOutcome::Unknown if remove => self + .read_index_row_serial(&key.account_id, &key.backup_arn) + .await? + .is_none(), + WriteOutcome::Unknown => { + return Err(StorageError::Internal(format!( + "the DELETING write of backup {} has an unknown outcome", + key.backup_arn + ))); + } + }; + let lost = outcome == WriteOutcome::Unknown; + if applied { + if let Some(row) = removed_row { + if let Err(error) = self.remove_index_entries(&row).await { + tracing::warn!( + "backup {}: the row is removed and its list and status entries \ + are not ({error}); readers verify against the row and skip them", + key.backup_arn + ); + } + } else { + let store = StoreIdentity { + store_kind: StoreKind::parse(stored.store_kind.as_deref().unwrap_or("")) + .unwrap_or(StoreKind::Filesystem), + store_root: stored.store_root.clone().unwrap_or_default(), + store_id: stored.store_id.clone().unwrap_or_default(), + }; + self.move_status_index_row(key, &store, "AVAILABLE", "DELETING") + .await; + } + return Ok(DeleteStart::Deleting); + } + attempt += 1; + if attempt >= DELETE_CLAIM_ATTEMPTS { + if lost { + return Err(StorageError::Internal(format!( + "the removal of backup {} has an unknown outcome on every attempt", + key.backup_arn + ))); + } + // A registration or another writer landed between the check + // and the write on every attempt. + let stored = self + .load_stored_backup(&key.account_id, &key.backup_arn) + .await? + .ok_or_else(|| not_found(&key.backup_arn))?; + let status = BackupStatus::parse(&stored.backup_status)?; + if status != BackupStatus::Available { + return Ok(DeleteStart::Status(status)); + } + let target = stored + .active_restores + .values() + .next() + .cloned() + .unwrap_or_default(); + return Ok(DeleteStart::InUse(target)); + } + } + } + + /// The ARNs the status index lists for `store` under `status`. + async fn status_index_arns( + &self, + store: &StoreIdentity, + status: BackupStatus, + ) -> Result, StorageError> { + let query = format!( + "SELECT backup_arn, account_id, store_kind, store_id FROM {}.backup_index_by_status \ + WHERE store_root = ? AND backup_status = ?", + self.catalog_keyspace() + ); + let rows = crate::cassandra_util::query_rows_quorum::( + &self.session, + &query, + cdrs_tokio::query_values!(store.store_root.as_str(), status.as_str()), + "status index", + ) + .await?; + let mut arns = Vec::with_capacity(rows.len()); + for row in rows { + let store_kind: Option = row.get_by_name("store_kind").ok().flatten(); + let store_id: Option = row.get_by_name("store_id").ok().flatten(); + if store_kind.as_deref() != Some(store.store_kind.as_str()) + || store_id.as_deref() != Some(store.store_id.as_str()) + { + continue; + } + let backup_arn: String = + crate::cassandra_util::get_column(&row, "backup_arn", "status index")?; + let account_id: String = + crate::cassandra_util::get_column(&row, "account_id", "status index")?; + arns.push((account_id, backup_arn)); + } + Ok(arns) + } + + /// Re-apply the target's `active_restores[table_id] = table_name` entry + /// on the backup's row when it is absent, under the takeover of a dead + /// manifest record (the trait doc of `claim_restore_dead`): one serial + /// read of the map, then a lightweight transaction on the entry + /// conditioned on the backup still AVAILABLE. A backup no longer + /// AVAILABLE (a delete won while nothing held it) keeps no entry: the + /// resumed load fails on the missing prefix and the target is removed. + async fn reapply_restore_registration( + &self, + key: &RestoreTargetKey, + backup_arn: &str, + ) -> Result<(), StorageError> { + use cdrs_tokio::types::AsRust; + use cdrs_tokio::types::map::Map; + let catalog = self.catalog_keyspace(); + let rows = crate::cassandra_util::query_rows_serial::( + &self.session, + &format!( + "SELECT active_restores FROM {catalog}.backups_by_arn \ + WHERE account_id = ? AND backup_arn = ?" + ), + cdrs_tokio::query_values!(key.account_id.as_str(), backup_arn), + "restore registration", + ) + .await?; + let Some(row) = rows.first() else { + return Ok(()); + }; + let map: Option = row.get_by_name("active_restores").ok().flatten(); + let entries: std::collections::HashMap = match map { + Some(map) => map + .as_rust() + .map_err(|e| StorageError::Internal(format!("Parse active_restores: {e}")))? + .unwrap_or_default(), + None => std::collections::HashMap::new(), + }; + if entries.contains_key(&key.table_id) { + return Ok(()); + } + let applied = conditional_write( + &self.session, + &format!( + "UPDATE {catalog}.backups_by_arn SET active_restores[?] = ? \ + WHERE account_id = ? AND backup_arn = ? IF backup_status = 'AVAILABLE'" + ), + QueryValues::SimpleValues(vec![ + Value::from(key.table_id.as_str()), + Value::from(key.table_name.as_str()), + Value::from(key.account_id.as_str()), + Value::from(backup_arn), + ]), + ) + .await?; + match applied { + WriteOutcome::Matched => tracing::warn!( + "restore of table {}: the backup {backup_arn} did not list the target as in \ + use (its owner died between the target's creation and the registration); the \ + entry was re-applied by the takeover", + key.table_name + ), + WriteOutcome::NotMatched => tracing::warn!( + "restore of table {}: the backup {backup_arn} is no longer AVAILABLE and does \ + not list the target; the resumed load fails on the missing prefix", + key.table_name + ), + WriteOutcome::Unknown => tracing::warn!( + "restore of table {}: re-applying the backup's in-use entry had no known \ + outcome; the registration sweep removes a stale one", + key.table_name + ), + } + Ok(()) + } + + /// The `tables` row of a restore target, read at quorum. + async fn restore_target_row( + &self, + account_id: &str, + table_name: &str, + ) -> Result, StorageError> { + let query = format!( + "SELECT table_id, table_status, restore_source_backup_arn, restore_status \ + FROM {}.tables WHERE account_id = ? AND table_name = ?", + self.catalog_keyspace() + ); + let rows = crate::cassandra_util::query_rows_quorum::( + &self.session, + &query, + cdrs_tokio::query_values!(account_id, table_name), + "restore target", + ) + .await?; + rows.first() + .map(|row| { + Ok(RestoreTargetRow { + table_id: crate::cassandra_util::get_column(row, "table_id", "restore target")?, + table_status: row.get_by_name("table_status").ok().flatten(), + source_backup_arn: row.get_by_name("restore_source_backup_arn").ok().flatten(), + job_status: row.get_by_name("restore_status").ok().flatten(), + }) + }) + .transpose() + } +} + +/// The columns of a `tables` row `remove_restore_target` decides on. +struct RestoreTargetRow { + table_id: String, + table_status: Option, + source_backup_arn: Option, + job_status: Option, +} + +impl RestoreTargetRow { + /// Whether this row is still the CREATING restore target the removal + /// claims first. + fn is_creating_target(&self) -> bool { + self.table_status.as_deref() == Some("CREATING") + } + + /// Whether this row is a target whose removal was claimed (CREATING to + /// DELETING) and did not finish: DELETING with its registration and a + /// FAILED record, the state `remove_restore_target` resumes from the + /// drop. A DELETING row without a registration is a client's DeleteTable + /// in flight, and one whose record is not FAILED (a restored table a + /// client is deleting) belongs to that delete. + fn is_claimed_for_removal(&self) -> bool { + self.table_status.as_deref() == Some("DELETING") + && self.source_backup_arn.is_some() + && self.job_status.as_deref() == Some("FAILED") + } +} + +/// The `tables` columns one restore job record fills, in the order +/// [`restore_job_row_from`] reads them. +const RESTORE_JOB_COLUMNS: &str = "account_id, table_name, table_id, table_status, \ + restore_source_backup_arn, restore_kind, restore_status, restore_owner_generation, \ + restore_heartbeat_at, restore_store_kind, restore_store_root, restore_store_id, \ + restore_spool_host, restore_file_index, restore_items_applied, restore_failure_reason, \ + restore_completed_at"; + +/// One [`RestoreJobRow`] from a `tables` row that holds a restore record +/// (every member [`RESTORE_JOB_COLUMNS`] selects): the store the restore +/// reads, the claimant's spool, the recorded progress, the failure reason +/// and the completion time, beyond the six the reconciler's legacy and dead +/// arms read. +fn restore_job_row_from(row: &Row) -> Result { + let column = |name: &str| -> Result { + crate::cassandra_util::get_column(row, name, "restore job") + }; + let optional = |name: &str| -> Option { row.get_by_name(name).ok().flatten() }; + let store = optional("restore_store_kind") + .as_deref() + .and_then(StoreKind::parse) + .map(|store_kind| StoreIdentity { + store_kind, + store_root: optional("restore_store_root").unwrap_or_default(), + store_id: optional("restore_store_id").unwrap_or_default(), + }); + let generation: Option = row.get_by_name("restore_owner_generation").ok().flatten(); + let heartbeat_at: Option = row.get_by_name("restore_heartbeat_at").ok().flatten(); + let completed_at: Option = row.get_by_name("restore_completed_at").ok().flatten(); + let file_index: Option = row.get_by_name("restore_file_index").ok().flatten(); + let items_applied: Option = row.get_by_name("restore_items_applied").ok().flatten(); + Ok(RestoreJobRow { + target: RestoreTargetKey { + account_id: column("account_id")?, + table_name: column("table_name")?, + table_id: column("table_id")?, + }, + source_backup_arn: column("restore_source_backup_arn")?, + kind: RestoreKind::parse(&column("restore_kind")?)?, + status: JobStatus::parse(&column("restore_status")?)?, + owner_generation: generation_from_column(generation), + heartbeat_at: heartbeat_at.map_or(0.0, millis_to_seconds), + store, + spool_host: optional("restore_spool_host"), + file_index: file_index.and_then(|i| u32::try_from(i).ok()), + items_applied: items_applied.and_then(|i| u64::try_from(i).ok()), + failure_reason: optional("restore_failure_reason"), + completed_at: completed_at.map(millis_to_seconds), + }) +} + +/// A store-backed restore job's handle on its target's `tables` row. Unlike +/// the legacy copy's self-managed [`crate::backup_engine::JobHeartbeat`], +/// the engine drives this claim's heartbeat and status writes; every write +/// is a lightweight transaction conditioned on the restore generation and on +/// the record being RUNNING, and leaves `table_status` to +/// [`CassandraEngine::activate_restore_target_row`] (the ACTIVE flip) and to +/// `remove_restore_target` (the CREATING to DELETING step of a removal). +struct CassandraRestoreClaim { + engine: CassandraEngine, + key: RestoreTargetKey, + generation: u64, +} + +impl JobClaim for CassandraRestoreClaim { + fn generation(&self) -> u64 { + self.generation + } + + fn heartbeat(&self) -> BoxFuture<'_, Result> { + Box::pin(async move { + let query = format!( + "UPDATE {}.tables SET restore_heartbeat_at = toTimestamp(now()) \ + WHERE account_id = ? AND table_name = ? \ + IF table_id = ? AND restore_owner_generation = ? AND restore_status = 'RUNNING'", + self.engine.catalog_keyspace() + ); + let result = crate::cassandra_util::query_lwt( + &self.engine.session, + &query, + cdrs_tokio::query_values!( + self.key.account_id.as_str(), + self.key.table_name.as_str(), + self.key.table_id.as_str(), + generation_column(self.generation) + ), + ) + .await?; + Ok(if crate::cassandra_util::lwt_applied(&result)? { + Heartbeat::Confirmed + } else { + Heartbeat::Lost + }) + }) + } + + fn set_status<'a>( + &'a self, + from: JobStatus, + to: JobStatus, + fields: &'a StatusFields, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + // The record only; `table_status` is left to the ACTIVE flip and + // to the removal's CREATING to DELETING step (4(e)'s two-write + // rule). A restore record has no size/item/source-table columns, + // so those `StatusFields` are ignored. + let mut assignments = vec!["restore_status = ?".to_owned()]; + let mut values = vec![Value::from(to.as_str())]; + if let Some(reason) = fields.failure_reason.as_deref() { + assignments.push("restore_failure_reason = ?".to_owned()); + values.push(Value::from(reason)); + } + if fields.completed { + assignments.push("restore_completed_at = toTimestamp(now())".to_owned()); + } + values.extend([ + Value::from(self.key.account_id.as_str()), + Value::from(self.key.table_name.as_str()), + Value::from(self.key.table_id.as_str()), + Value::from(generation_column(self.generation)), + Value::from(from.as_str()), + ]); + let query = format!( + "UPDATE {}.tables SET {} WHERE account_id = ? AND table_name = ? \ + IF table_id = ? AND restore_owner_generation = ? AND restore_status = ?", + self.engine.catalog_keyspace(), + assignments.join(", ") + ); + conditional_write( + &self.engine.session, + &query, + QueryValues::SimpleValues(values), + ) + .await + }) + } + + fn read(&self) -> BoxFuture<'_, Result> { + Box::pin(async move { + let query = format!( + "SELECT restore_status, restore_owner_generation, restore_heartbeat_at, \ + restore_failure_reason FROM {}.tables WHERE account_id = ? AND table_name = ?", + self.engine.catalog_keyspace() + ); + let rows = crate::cassandra_util::query_rows_quorum::( + &self.engine.session, + &query, + cdrs_tokio::query_values!( + self.key.account_id.as_str(), + self.key.table_name.as_str() + ), + "restore job row", + ) + .await?; + let row = rows.first().ok_or_else(|| { + StorageError::TableNotFound(format!( + "restore target {} has no job record", + self.key.table_name + )) + })?; + let status: Option = row.get_by_name("restore_status").ok().flatten(); + let status = status.ok_or_else(|| { + StorageError::TableNotFound(format!( + "restore target {} has no job record", + self.key.table_name + )) + })?; + let generation: Option = + row.get_by_name("restore_owner_generation").ok().flatten(); + let heartbeat_at: Option = row.get_by_name("restore_heartbeat_at").ok().flatten(); + Ok(JobRow { + status: JobStatus::parse(&status)?, + owner_generation: generation_from_column(generation), + heartbeat_at: heartbeat_at.map_or(0.0, millis_to_seconds), + failure_reason: row.get_by_name("restore_failure_reason").ok().flatten(), + }) + }) + } +} + +impl BackupCatalog for CassandraEngine { + fn catalog_id(&self) -> BoxFuture<'_, Result> { + Box::pin(async move { + let query = format!( + "SELECT value FROM {}.settings WHERE key = 'catalog_id'", + self.catalog_keyspace() + ); + let rows = crate::cassandra_util::query_rows_quorum::( + &self.session, + &query, + cdrs_tokio::query_values!(), + "catalog id", + ) + .await?; + let row = rows.first().ok_or_else(|| { + StorageError::Internal( + "the catalog has no catalog_id; run extenddb migrate --yes".to_owned(), + ) + })?; + crate::cassandra_util::get_column(row, "value", "catalog id") + }) + } + + fn claim_backup<'a>( + &'a self, + row: &'a BackupRow, + ) -> BoxFuture<'a, Result, StorageError>> { + Box::pin(async move { + Self::validate_account_id(&row.account_id)?; + let source = row.source_table()?; + let key_schema = serde_json::to_string(&source.key_schema) + .map_err(|e| StorageError::Internal(format!("Serialize key schema: {e}")))?; + let attribute_definitions = serde_json::to_string(&source.attribute_definitions) + .map_err(|e| StorageError::Internal(format!("Serialize attributes: {e}")))?; + let generation = new_generation(); + let catalog = self.catalog_keyspace(); + // The row is the claim: it exists only owned. The legacy columns + // every `backups_by_arn` row carries are filled from the source + // table, so the row describes itself to either reader of it. + let insert = format!( + "INSERT INTO {catalog}.backups_by_arn (account_id, backup_arn, backup_id, \ + backup_name, table_id, table_name, table_arn, backup_status, backup_type, \ + backup_size_bytes, item_count, key_schema, attribute_definitions, billing_mode, \ + table_created_at, created_at, store_kind, store_root, store_id, \ + source_table_details_json, owner_generation, heartbeat_at) \ + VALUES (?, ?, ?, ?, ?, ?, ?, 'CREATING', 'USER', ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, \ + ?, ?, toTimestamp(now())) IF NOT EXISTS" + ); + let inserted = crate::cassandra_util::query_lwt( + &self.session, + &insert, + cdrs_tokio::query_values!( + row.account_id.as_str(), + row.backup_arn.as_str(), + row.backup_id.as_str(), + row.backup_name.as_str(), + source.table_id.as_str(), + row.table_name.as_str(), + source.table_arn.as_str(), + row.size_bytes, + row.item_count, + key_schema.as_str(), + attribute_definitions.as_str(), + billing_mode_name(source.billing_mode), + seconds_to_millis(source.table_creation_date_time), + seconds_to_millis(row.created_at), + row.store.store_kind.as_str(), + row.store.store_root.as_str(), + row.store.store_id.as_str(), + row.source_table_details_json.to_string().as_str(), + generation_column(generation) + ), + ) + .await?; + if !crate::cassandra_util::lwt_applied(&inserted)? { + return Err(StorageError::AlreadyExists(format!( + "backup {} already has a row", + row.backup_arn + ))); + } + // The index rows, after the authoritative row. + self.write_index_entries( + row, + &source.table_id, + &source.table_arn, + BackupStatus::Creating, + ) + .await?; + Ok(Box::new(CassandraBackupClaim { + engine: self.clone(), + key: row.key(), + store: row.store.clone(), + generation, + }) as Box) + }) + } + + fn insert_backup_row<'a>( + &'a self, + row: &'a BackupRow, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + Self::validate_account_id(&row.account_id)?; + let source = row.source_table()?; + let key_schema = serde_json::to_string(&source.key_schema) + .map_err(|e| StorageError::Internal(format!("Serialize key schema: {e}")))?; + let attribute_definitions = serde_json::to_string(&source.attribute_definitions) + .map_err(|e| StorageError::Internal(format!("Serialize attributes: {e}")))?; + let source_table_details_json = row.source_table_details_json.to_string(); + let catalog = self.catalog_keyspace(); + // `claim_backup`'s insert, unowned and complete. The partition is + // (`account_id`, `backup_arn`) and no table indexes the backup id, + // so a row for the id under another ARN spelling does not stop it. + let insert = format!( + "INSERT INTO {catalog}.backups_by_arn (account_id, backup_arn, backup_id, \ + backup_name, table_id, table_name, table_arn, backup_status, backup_type, \ + backup_size_bytes, item_count, key_schema, attribute_definitions, billing_mode, \ + table_created_at, created_at, completed_at, failure_reason, store_kind, \ + store_root, store_id, source_table_details_json, owner_generation, heartbeat_at) \ + VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'USER', ?, ?, ?, ?, ?, ?, ?, toTimestamp(now()), \ + ?, ?, ?, ?, ?, ?, toTimestamp(now())) IF NOT EXISTS" + ); + let values = || { + cdrs_tokio::query_values!( + row.account_id.as_str(), + row.backup_arn.as_str(), + row.backup_id.as_str(), + row.backup_name.as_str(), + source.table_id.as_str(), + row.table_name.as_str(), + source.table_arn.as_str(), + row.status.as_str(), + row.size_bytes, + row.item_count, + key_schema.as_str(), + attribute_definitions.as_str(), + billing_mode_name(source.billing_mode), + seconds_to_millis(source.table_creation_date_time), + seconds_to_millis(row.created_at), + row.failure_reason.as_deref(), + row.store.store_kind.as_str(), + row.store.store_root.as_str(), + row.store.store_id.as_str(), + source_table_details_json.as_str(), + generation_column(UNOWNED_GENERATION) + ) + }; + // A lost answer, and an insert that finds a row, are settled by a + // serial read. The row as this call writes it counts as inserted: + // another `insert_backup_row` of the same verified prefix writes + // the same row, either call may count it, and one that stopped + // (or whose batch failed) between the row and its index entries + // left it with no entries, which this call then writes. Any other + // row is one that existed. No row after a lost answer means the + // proposal did not commit, so the write is sent once more, as + // design 4(e) allows; no row after `NotMatched` is a legacy + // in-catalog row at the ARN, which `read_index_row_serial` does + // not return. + let mut inserted = None; + for _ in 0..2 { + let fault = crate::test_hooks::insert_write_fault(&row.backup_arn); + let outcome = if fault == Some(crate::test_hooks::ProgressWriteFault::Lost) { + WriteOutcome::Unknown + } else { + let outcome = conditional_write(&self.session, &insert, values()).await?; + if fault.is_some() { + WriteOutcome::Unknown + } else { + outcome + } + }; + let read = || self.read_index_row_serial(&row.account_id, &row.backup_arn); + match outcome { + WriteOutcome::Matched => inserted = Some(true), + WriteOutcome::NotMatched => { + inserted = Some( + read() + .await? + .is_some_and(|read| is_unowned_insert_of(&read, row)), + ); + } + WriteOutcome::Unknown => { + inserted = read().await?.map(|read| is_unowned_insert_of(&read, row)); + } + } + if inserted.is_some() { + break; + } + } + let Some(inserted) = inserted else { + return Err(StorageError::Internal(format!( + "the insert of backup {} has an unknown outcome", + row.backup_arn + ))); + }; + if inserted { + // The index entries, after the authoritative row, the status + // entry under the row's status. The batch is idempotent + // upserts; a failed one is sent once more, and one that fails + // twice is an error the next insert of the row repairs. + if let Err(first) = self + .write_index_entries(row, &source.table_id, &source.table_arn, row.status) + .await + { + tracing::warn!( + "backup {}: index entries not written, sending them once more: {first}", + row.backup_arn + ); + self.write_index_entries(row, &source.table_id, &source.table_arn, row.status) + .await?; + } + } + Ok(inserted) + }) + } + + fn list_backup_rows<'a>( + &'a self, + status: Option, + store: &'a StoreIdentity, + ) -> BoxStream<'a, Result> { + rows_stream(async move { + let statuses = match status { + Some(status) => vec![status], + None => vec![ + BackupStatus::Creating, + BackupStatus::Available, + BackupStatus::Deleting, + BackupStatus::Deleted, + BackupStatus::Failed, + ], + }; + let mut rows = Vec::new(); + for status in statuses { + for (account_id, backup_arn) in self.status_index_arns(store, status).await? { + // The entry is a hint: the authoritative row decides. + if let Some(row) = self.read_index_row(&account_id, &backup_arn).await? + && row.status == status + && row.store == *store + { + rows.push(row); + } + } + } + sort_listing(&mut rows); + Ok(rows) + }) + } + + fn newest_available_backup_row<'a>( + &'a self, + store: &'a StoreIdentity, + ) -> BoxFuture<'a, Result, StorageError>> { + Box::pin(async move { + // One partition read of the status index, then one row: the + // entries are ordered by the instant minted into the backup id + // (the row's `created_at` by the manifest rule), newest first, + // and the first entry whose row confirms the status and the + // store is the newest AVAILABLE row. An entry whose row moved + // (a hint the deleter's batch has not yet moved) costs one more + // read. + let mut entries = self + .status_index_arns(store, BackupStatus::Available) + .await?; + entries.sort_by(|(_, a), (_, b)| { + let minted = |arn: &str| { + extenddb_core::backup_format::backup_creation_date_time(arn).unwrap_or(0.0) + }; + minted(b) + .partial_cmp(&minted(a)) + .unwrap_or(std::cmp::Ordering::Equal) + .then_with(|| b.cmp(a)) + }); + for (account_id, backup_arn) in entries { + if let Some(row) = self.read_index_row(&account_id, &backup_arn).await? + && row.status == BackupStatus::Available + && row.store == *store + { + return Ok(Some(row)); + } + } + Ok(None) + }) + } + + fn list_account_backup_rows<'a>( + &'a self, + account_id: &'a str, + table_name: Option<&'a str>, + bounds: &'a TimeBounds, + ) -> BoxStream<'a, Result> { + rows_stream(async move { + Self::validate_account_id(account_id)?; + let catalog = self.catalog_keyspace(); + let mut query = match table_name { + Some(_) => format!( + "SELECT backup_arn FROM {catalog}.backups_by_table \ + WHERE account_id = ? AND table_name = ?" + ), + None => { + format!( + "SELECT backup_arn FROM {catalog}.backups_by_account WHERE account_id = ?" + ) + } + }; + let mut values = vec![Value::from(account_id)]; + if let Some(table_name) = table_name { + values.push(Value::from(table_name)); + } + // The clustering range narrows the read; `contains` is the exact + // rule (lower inclusive, upper exclusive) on the row's seconds. + #[allow(clippy::cast_possible_truncation)] + if let Some(lower) = bounds.lower { + query.push_str(" AND created_at >= ?"); + values.push(Value::from((lower * 1_000.0).floor() as i64)); + } + #[allow(clippy::cast_possible_truncation)] + if let Some(upper) = bounds.upper { + query.push_str(" AND created_at < ?"); + values.push(Value::from((upper * 1_000.0).ceil() as i64)); + } + let params = StatementParamsBuilder::new() + .with_consistency(Consistency::LocalQuorum) + .with_values(QueryValues::SimpleValues(values)) + .build(); + let listed = self + .session + .query_with_params(&query, params) + .await + .map_err(|e| StorageError::Internal(format!("List backup index: {e}")))? + .response_body() + .map_err(|e| StorageError::Internal(format!("Parse backup index list: {e}")))? + .into_rows() + .unwrap_or_default(); + let mut rows = Vec::with_capacity(listed.len()); + for entry in listed { + let backup_arn: String = + crate::cassandra_util::get_column(&entry, "backup_arn", "backup index list")?; + // An in-catalog backup's entry reads as `None` here and is the + // legacy reader's to list. + if let Some(row) = self.read_index_row(account_id, &backup_arn).await? + && bounds.contains(row.created_at) + { + rows.push(row); + } + } + sort_listing(&mut rows); + Ok(rows) + }) + } + + fn read_backup_row<'a>( + &'a self, + arn: &'a BackupArn, + ) -> BoxFuture<'a, Result, StorageError>> { + Box::pin(async move { + let row = self + .read_index_row(arn.account_id(), &arn.to_arn_string()) + .await?; + Ok(row.filter(|row| { + row.backup_id == arn.backup_id() + && row.table_name == arn.table_name() + && row.account_id == arn.account_id() + })) + }) + } + + fn start_backup_delete<'a>( + &'a self, + key: &'a BackupKey, + ) -> BoxFuture<'a, Result> { + Box::pin(self.start_backup_delete_lwt(key, false)) + } + + fn forget_available_backup_row<'a>( + &'a self, + key: &'a BackupKey, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + Ok(match self.start_backup_delete_lwt(key, true).await? { + DeleteStart::Deleting => ForgetRow::Removed, + DeleteStart::InUse(table) => ForgetRow::InUse(table), + DeleteStart::Status(status) => ForgetRow::Status(status), + }) + }) + } + + fn set_backup_deleted<'a>( + &'a self, + key: &'a BackupKey, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + let Some(row) = self + .read_index_row(&key.account_id, &key.backup_arn) + .await? + else { + return Ok(false); + }; + let query = format!( + "UPDATE {}.backups_by_arn SET backup_status = 'DELETED', \ + completed_at = toTimestamp(now()) \ + WHERE account_id = ? AND backup_arn = ? IF backup_status = 'DELETING'", + self.catalog_keyspace() + ); + let result = crate::cassandra_util::query_lwt( + &self.session, + &query, + cdrs_tokio::query_values!(key.account_id.as_str(), key.backup_arn.as_str()), + ) + .await?; + let applied = crate::cassandra_util::lwt_applied(&result)?; + if applied { + self.move_status_index_row(key, &row.store, "DELETING", "DELETED") + .await; + } + Ok(applied) + }) + } + + fn remove_backup_row<'a>( + &'a self, + key: &'a BackupKey, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + let Some(row) = self + .read_index_row(&key.account_id, &key.backup_arn) + .await? + else { + return Ok(false); + }; + if !matches!(row.status, BackupStatus::Deleted | BackupStatus::Failed) { + return Ok(false); + } + // The authoritative row first, conditioned on the tombstone + // status read; then its index entries. + let delete = format!( + "DELETE FROM {}.backups_by_arn WHERE account_id = ? AND backup_arn = ? \ + IF backup_status = ?", + self.catalog_keyspace() + ); + let result = crate::cassandra_util::query_lwt( + &self.session, + &delete, + cdrs_tokio::query_values!( + key.account_id.as_str(), + key.backup_arn.as_str(), + row.status.as_str() + ), + ) + .await?; + if !crate::cassandra_util::lwt_applied(&result)? { + return Ok(false); + } + self.remove_index_entries(&row).await?; + Ok(true) + }) + } + + fn remove_dead_creating_backup_row<'a>( + &'a self, + key: &'a BackupKey, + stale_after: Duration, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + let Some(row) = self + .read_index_row(&key.account_id, &key.backup_arn) + .await? + else { + return Ok(false); + }; + if row.status != BackupStatus::Creating { + return Ok(false); + } + let now = coordinator_now_millis(self).await?; + let stale_ms = i64::try_from(stale_after.as_millis()).unwrap_or(i64::MAX); + let heartbeat_at = seconds_to_millis(row.heartbeat_at); + if heartbeat_at >= now.saturating_sub(stale_ms) { + return Ok(false); + } + // Conditioned on the status and the heartbeat read: a heartbeat + // written since (the owner is alive after all) leaves the row. + let delete = format!( + "DELETE FROM {}.backups_by_arn WHERE account_id = ? AND backup_arn = ? \ + IF backup_status = 'CREATING' AND heartbeat_at = ?", + self.catalog_keyspace() + ); + let result = crate::cassandra_util::query_lwt( + &self.session, + &delete, + cdrs_tokio::query_values!( + key.account_id.as_str(), + key.backup_arn.as_str(), + heartbeat_at + ), + ) + .await?; + if !crate::cassandra_util::lwt_applied(&result)? { + return Ok(false); + } + self.remove_index_entries(&row).await?; + Ok(true) + }) + } + + fn claim_dead_backup<'a>( + &'a self, + key: &'a BackupKey, + stale_after: Duration, + ) -> BoxFuture<'a, Result>, StorageError>> { + Box::pin(async move { + // The owner is dead when the row's heartbeat is older than the + // window on the coordinator's clock. The row is read at + // LOCAL_SERIAL, so a heartbeat an LWT just wrote is seen, and the + // takeover is conditioned on the generation, the status and the + // heartbeat as read: a beat landing between the read and the + // write (the owner alive after all), or another taker's + // generation, leaves the row and answers `None`. The heartbeat is + // not refreshed: the FAILED-prefix window counts from the dead + // owner's last beat. + let catalog = self.catalog_keyspace(); + let query = format!( + "SELECT backup_status, owner_generation, heartbeat_at, store_kind, store_root, \ + store_id FROM {catalog}.backups_by_arn WHERE account_id = ? AND backup_arn = ?" + ); + let rows = crate::cassandra_util::query_rows_serial::( + &self.session, + &query, + cdrs_tokio::query_values!(key.account_id.as_str(), key.backup_arn.as_str()), + "dead backup row", + ) + .await?; + let Some(row) = rows.first() else { + return Ok(None); + }; + let status: Option = row.get_by_name("backup_status").ok().flatten(); + if status.as_deref() != Some(BackupStatus::Creating.as_str()) { + return Ok(None); + } + // An in-catalog row (no store) is the legacy duty's. + let store_kind: Option = row.get_by_name("store_kind").ok().flatten(); + let Some(store_kind) = store_kind.as_deref().and_then(StoreKind::parse) else { + return Ok(None); + }; + let store = StoreIdentity { + store_kind, + store_root: crate::cassandra_util::get_column( + row, + "store_root", + "dead backup row", + )?, + store_id: crate::cassandra_util::get_column(row, "store_id", "dead backup row")?, + }; + let owner_generation: Option = row.get_by_name("owner_generation").ok().flatten(); + let heartbeat_at: Option = row.get_by_name("heartbeat_at").ok().flatten(); + let now = coordinator_now_millis(self).await?; + let stale_ms = i64::try_from(stale_after.as_millis()).unwrap_or(i64::MAX); + if heartbeat_at.is_some_and(|at| at >= now.saturating_sub(stale_ms)) { + return Ok(None); + } + crate::test_hooks::before_takeover_write(&key.backup_arn).await; + let generation = new_generation(); + let take_over = format!( + "UPDATE {catalog}.backups_by_arn SET owner_generation = ? \ + WHERE account_id = ? AND backup_arn = ? \ + IF owner_generation = ? AND backup_status = 'CREATING' AND heartbeat_at = ?" + ); + let taken = conditional_write( + &self.session, + &take_over, + cdrs_tokio::query_values!( + generation_column(generation), + key.account_id.as_str(), + key.backup_arn.as_str(), + owner_generation, + heartbeat_at + ), + ) + .await?; + let claimed = match taken { + WriteOutcome::Matched => true, + WriteOutcome::NotMatched => false, + // A takeover whose answer was lost may have committed: a row + // carrying this call's generation is this call's claim. + WriteOutcome::Unknown => { + let rows = crate::cassandra_util::query_rows_serial::( + &self.session, + &format!( + "SELECT owner_generation FROM {catalog}.backups_by_arn \ + WHERE account_id = ? AND backup_arn = ?" + ), + cdrs_tokio::query_values!(key.account_id.as_str(), key.backup_arn.as_str()), + "dead backup row", + ) + .await?; + let current: Option = rows + .first() + .and_then(|r| r.get_by_name("owner_generation").ok().flatten()); + current == Some(generation_column(generation)) + } + }; + if !claimed { + return Ok(None); + } + Ok(Some(Box::new(CassandraBackupClaim { + engine: self.clone(), + key: key.clone(), + store, + generation, + }) as Box)) + }) + } + + fn list_backup_rows_older_than<'a>( + &'a self, + status: BackupStatus, + store: &'a StoreIdentity, + clock: StaleClock, + age: Duration, + ) -> BoxStream<'a, Result> { + rows_stream(async move { + // The coordinator's clock is read once, before the status + // partition is scanned, so a row's column is compared with the + // clock that stamped it and never with this host's; a heartbeat + // written while the rows are re-read is later than `now` and the + // row is not listed. A row with no `completed_at` has no age on + // that clock and is not listed. + let now = coordinator_now_millis(self).await?; + let age_ms = i64::try_from(age.as_millis()).unwrap_or(i64::MAX); + let cutoff = now.saturating_sub(age_ms); + let mut rows = Vec::new(); + for (account_id, backup_arn) in self.status_index_arns(store, status).await? { + let Some(row) = self.read_index_row(&account_id, &backup_arn).await? else { + continue; + }; + if row.status != status || row.store != *store { + continue; + } + let at = match clock { + StaleClock::Heartbeat => Some(seconds_to_millis(row.heartbeat_at)), + StaleClock::Completed => row.completed_at.map(seconds_to_millis), + }; + if at.is_some_and(|at| at < cutoff) { + rows.push(row); + } + } + sort_listing(&mut rows); + Ok(rows) + }) + } + + fn sweep_restore_registrations(&self) -> BoxFuture<'_, Result, StorageError>> { + Box::pin(async move { + // A registration is live while its target is a CREATING restore + // target with the registered table id restoring from this backup + // (the rule the in-use check applies); any other entry is one a + // restore left behind, its removal lost with its owner, and is + // removed by a lightweight transaction on the entry as read. The + // map has no index, so the scan is a range read of `backups_by_arn` + // at quorum; rows carrying an entry are few. + use cdrs_tokio::types::AsRust; + use cdrs_tokio::types::map::Map; + let catalog = self.catalog_keyspace(); + let scan = format!( + "SELECT account_id, backup_arn, active_restores FROM {catalog}.backups_by_arn" + ); + let rows = crate::cassandra_util::query_rows_quorum::( + &self.session, + &scan, + cdrs_tokio::query_values!(), + "restore registrations", + ) + .await?; + let remove = format!( + "DELETE active_restores[?] FROM {catalog}.backups_by_arn \ + WHERE account_id = ? AND backup_arn = ? IF active_restores[?] = ?" + ); + let mut swept = Vec::new(); + for row in rows { + let map: Option = row.get_by_name("active_restores").ok().flatten(); + let Some(map) = map else { + continue; + }; + let entries: std::collections::HashMap = map + .as_rust() + .map_err(|e| StorageError::Internal(format!("Parse active_restores: {e}")))? + .unwrap_or_default(); + if entries.is_empty() { + continue; + } + let account_id: String = + crate::cassandra_util::get_column(&row, "account_id", "restore registrations")?; + let backup_arn: String = + crate::cassandra_util::get_column(&row, "backup_arn", "restore registrations")?; + let mut touched = false; + for (table_id, target_name) in entries { + let live = self + .restore_target_row(&account_id, &target_name) + .await? + .is_some_and(|target| { + target.table_id == table_id + && target.is_creating_target() + && target.source_backup_arn.as_deref() == Some(backup_arn.as_str()) + }); + if live { + continue; + } + let removed = conditional_write( + &self.session, + &remove, + cdrs_tokio::query_values!( + table_id.as_str(), + account_id.as_str(), + backup_arn.as_str(), + table_id.as_str(), + target_name.as_str() + ), + ) + .await?; + if removed == WriteOutcome::Matched { + tracing::info!( + "backup {backup_arn}: removed the restore registration of table \ + {target_name} ({table_id}); the target is no longer being restored \ + from it" + ); + touched = true; + } + } + if touched { + swept.push(backup_arn); + } + } + swept.sort(); + Ok(swept) + }) + } + + fn backup_in_use<'a>( + &'a self, + key: &'a BackupKey, + ) -> BoxFuture<'a, Result, StorageError>> { + Box::pin(async move { + let Some(stored) = self + .load_stored_backup(&key.account_id, &key.backup_arn) + .await? + else { + return Ok(None); + }; + // The backup's `active_restores` map names the targets whose + // registration committed its second write. A store-backed + // registration creates the target row (`tables` with + // `restore_source_backup_arn`, CREATING) before it writes that + // map entry, so a second restore's in-use check in that window + // would miss the first and fall through to its own override + // validation (`ValidationException` naming the override's index + // where the other backends answer `BackupInUseException`). Read + // the target rows too: a + // CREATING restore target of this backup is in use whether or not + // its `active_restores` entry has landed yet. + if let Some(target) = self.creating_restore_target(&stored).await? { + return Ok(Some(target)); + } + self.creating_restore_target_of_backup(&key.account_id, &key.backup_arn) + .await + }) + } + + fn remove_restore_target<'a>( + &'a self, + key: &'a RestoreTargetKey, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + // The key must name a restore target with this table id; a table + // re-created under the name has another id and is not touched. + // The removal is two writes: a claim from CREATING to DELETING, + // one lightweight transaction on the row the ACTIVE flip + // conditions on, so the two exclude each other; then the drop. + // A row the claim already moved to DELETING, whose drop did not + // finish, is resumed from the drop (`is_claimed_for_removal`). + let Some(row) = self + .restore_target_row(&key.account_id, &key.table_name) + .await? + else { + return Ok(false); + }; + let Some(backup_arn) = row.source_backup_arn.clone() else { + return Ok(false); + }; + if row.table_id != key.table_id { + return Ok(false); + } + if row.is_creating_target() { + let claim = format!( + "UPDATE {}.tables SET table_status = 'DELETING' \ + WHERE account_id = ? AND table_name = ? \ + IF table_id = ? AND table_status = 'CREATING'", + self.catalog_keyspace() + ); + let result = crate::cassandra_util::query_lwt( + &self.session, + &claim, + cdrs_tokio::query_values!( + key.account_id.as_str(), + key.table_name.as_str(), + key.table_id.as_str() + ), + ) + .await?; + if !crate::cassandra_util::lwt_applied(&result)? { + return Ok(false); + } + } else if !row.is_claimed_for_removal() { + return Ok(false); + } + self.drop_restore_target(&key.account_id, &key.table_name, &key.table_id, &backup_arn) + .await?; + Ok(true) + }) + } + + fn list_dead_restore_jobs( + &self, + stale_after: Duration, + ) -> BoxFuture<'_, Result, StorageError>> { + Box::pin(async move { + // Two filtered scans over the tables (CQL takes one equality on a + // non-key column per scan): RUNNING records, dead when their + // heartbeat is older than the cutoff; and FAILED records whose + // table is still a restore target the removal has not finished + // (CREATING: the FAILED write landed and the removal did not; + // DELETING with its registration: the removal's claim landed and + // its drop did not), dead when their `restore_completed_at` (or, + // where none was stamped, their heartbeat) is that old. Both + // kinds are few. The clock is read after the scans, so no + // heartbeat a scan returned is later than `now` (with a zero + // window, every RUNNING record the scans saw is listed). A range + // scan has no Paxos read (a serial level resolves to a local + // quorum there), so it stays at quorum. + let mut scanned = Vec::new(); + for status in ["RUNNING", "FAILED"] { + let query = format!( + "SELECT {RESTORE_JOB_COLUMNS} FROM {}.tables WHERE restore_status = ? \ + ALLOW FILTERING", + self.catalog_keyspace() + ); + let rows = crate::cassandra_util::query_rows_quorum::( + &self.session, + &query, + cdrs_tokio::query_values!(status), + "dead restore jobs", + ) + .await?; + scanned.push((status, rows)); + } + let now = coordinator_now_millis(self).await?; + #[allow(clippy::cast_possible_truncation, clippy::cast_possible_wrap)] + let cutoff = now - stale_after.as_millis() as i64; + let mut dead = Vec::new(); + for (status, rows) in scanned { + for row in rows { + let heartbeat_at: Option = + row.get_by_name("restore_heartbeat_at").ok().flatten(); + let completed_at: Option = + row.get_by_name("restore_completed_at").ok().flatten(); + // A RUNNING record always carries a heartbeat (the insert + // or the backfill wrote it); one without is as dead as the + // epoch. + let age_of = if status == "FAILED" { + let target = RestoreTargetRow { + table_id: crate::cassandra_util::get_column( + &row, + "table_id", + "dead restore jobs", + )?, + table_status: row.get_by_name("table_status").ok().flatten(), + source_backup_arn: row + .get_by_name("restore_source_backup_arn") + .ok() + .flatten(), + job_status: Some(status.to_owned()), + }; + if !(target.is_creating_target() || target.is_claimed_for_removal()) { + continue; + } + completed_at.or(heartbeat_at) + } else { + heartbeat_at + }; + if age_of.is_some_and(|at| at >= cutoff) { + continue; + } + // A row this build cannot read (a kind or status no + // release writes) is logged and skipped, as ListBackups + // skips a row it cannot read, so one such row does not + // stop the cycle. + match restore_job_row_from(&row) { + Ok(job) => dead.push(job), + Err(e) => { + tracing::warn!( + "reconciler: restore job record of {} (account {}) skipped: {e}", + row.get_by_name("table_name") + .ok() + .flatten() + .unwrap_or_else(String::new), + row.get_by_name("account_id") + .ok() + .flatten() + .unwrap_or_else(String::new) + ); + continue; + } + } + } + } + Ok(dead) + }) + } + + fn list_running_restore_jobs(&self) -> BoxFuture<'_, Result, StorageError>> { + Box::pin(async move { + // The RUNNING scan of `list_dead_restore_jobs` with no cutoff: by + // status alone, so a record stamped in the scan's millisecond, or + // ahead of the coordinator's clock (skew between coordinators), + // is listed too. + let query = format!( + "SELECT {RESTORE_JOB_COLUMNS} FROM {}.tables WHERE restore_status = 'RUNNING' \ + ALLOW FILTERING", + self.catalog_keyspace() + ); + let rows = crate::cassandra_util::query_rows_quorum::( + &self.session, + &query, + cdrs_tokio::query_values!(), + "running restore jobs", + ) + .await?; + rows.iter().map(restore_job_row_from).collect() + }) + } + + fn claim_restore<'a>( + &'a self, + account_id: &'a str, + input: CreateTableInput, + registration: &'a RestoreRegistrationRow, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + Self::validate_account_id(account_id)?; + let (description, generation) = self + .register_manifest_restore_target(account_id, input, registration) + .await?; + let key = RestoreTargetKey { + account_id: account_id.to_owned(), + table_name: description.table_name.clone(), + table_id: description.table_id.clone(), + }; + Ok(( + description, + Box::new(CassandraRestoreClaim { + engine: self.clone(), + key, + generation, + }) as Box, + )) + }) + } + + fn claim_restore_dead<'a>( + &'a self, + key: &'a RestoreTargetKey, + stale_after: Duration, + spool_host: Option<&'a str>, + ) -> BoxFuture<'a, Result>, StorageError>> { + Box::pin(async move { + // The record is read at LOCAL_SERIAL, so a heartbeat an LWT just + // wrote is seen; dead is `restore_heartbeat_at` older than the + // window on the coordinator's clock. The takeover is conditioned + // on the generation, RUNNING, the manifest kind and the heartbeat + // as read: a beat landing between the read and the write (the + // owner alive after all), another taker's generation, or a record + // that left RUNNING leaves the row and answers `None`. The write + // sets the generation, the claimant's `spool_host` (`None` clears + // it) and `restore_heartbeat_at` to the coordinator's clock, so a + // second taker's read finds the record live at once (the trait + // doc says why a restore record is refreshed where a backup row + // is not). A legacy record is not a manifest restore's to take. + let catalog = self.catalog_keyspace(); + let read = format!( + "SELECT table_id, restore_status, restore_kind, restore_owner_generation, \ + restore_heartbeat_at, restore_source_backup_arn FROM {catalog}.tables \ + WHERE account_id = ? AND table_name = ?" + ); + let rows = crate::cassandra_util::query_rows_serial::( + &self.session, + &read, + cdrs_tokio::query_values!(key.account_id.as_str(), key.table_name.as_str()), + "dead restore job", + ) + .await?; + let Some(row) = rows.first() else { + return Ok(None); + }; + let table_id: Option = row.get_by_name("table_id").ok().flatten(); + if table_id.as_deref() != Some(key.table_id.as_str()) { + return Ok(None); + } + let status: Option = row.get_by_name("restore_status").ok().flatten(); + if status.as_deref() != Some(JobStatus::Running.as_str()) { + return Ok(None); + } + let owner_generation: Option = + row.get_by_name("restore_owner_generation").ok().flatten(); + let heartbeat_at: Option = row.get_by_name("restore_heartbeat_at").ok().flatten(); + let source_backup_arn: Option = + row.get_by_name("restore_source_backup_arn").ok().flatten(); + let now = coordinator_now_millis(self).await?; + let stale_ms = i64::try_from(stale_after.as_millis()).unwrap_or(i64::MAX); + if heartbeat_at.is_some_and(|at| at >= now.saturating_sub(stale_ms)) { + return Ok(None); + } + // A test may interpose a beat here, between the read and the + // write, to prove the heartbeat condition refuses the takeover. + crate::test_hooks::before_restore_takeover(&key.table_id).await; + let generation = new_generation(); + let take_over = format!( + "UPDATE {catalog}.tables SET restore_owner_generation = ?, \ + restore_spool_host = ?, restore_heartbeat_at = toTimestamp(now()) \ + WHERE account_id = ? AND table_name = ? \ + IF restore_owner_generation = ? AND restore_status = 'RUNNING' \ + AND restore_kind = 'manifest' AND restore_heartbeat_at = ?" + ); + let taken = conditional_write( + &self.session, + &take_over, + QueryValues::SimpleValues(vec![ + Value::from(generation_column(generation)), + match spool_host { + Some(host) => Value::from(host), + None => Value::from(Option::::None), + }, + Value::from(key.account_id.as_str()), + Value::from(key.table_name.as_str()), + Value::from(owner_generation), + Value::from(heartbeat_at), + ]), + ) + .await?; + let claimed = match taken { + WriteOutcome::Matched => true, + WriteOutcome::NotMatched => false, + // A takeover whose answer was lost may have committed: a + // record carrying this call's generation is this call's claim. + WriteOutcome::Unknown => { + let rows = crate::cassandra_util::query_rows_serial::( + &self.session, + &format!( + "SELECT restore_owner_generation FROM {catalog}.tables \ + WHERE account_id = ? AND table_name = ?" + ), + cdrs_tokio::query_values!(key.account_id.as_str(), key.table_name.as_str()), + "dead restore job", + ) + .await?; + let current: Option = rows + .first() + .and_then(|r| r.get_by_name("restore_owner_generation").ok().flatten()); + current == Some(generation_column(generation)) + } + }; + if !claimed { + return Ok(None); + } + // The backup's `active_restores` entry for the target, re-applied + // when it is absent: the registration writes the target's row and + // its DDL first and the entry second, so an owner that died + // between the two left a resumable record whose backup nothing + // held in use. Without the entry `start_backup_delete` would + // succeed under the resumed load. + if let Some(backup_arn) = source_backup_arn { + self.reapply_restore_registration(key, &backup_arn).await?; + } + Ok(Some(Box::new(CassandraRestoreClaim { + engine: self.clone(), + key: key.clone(), + generation, + }) as Box)) + }) + } + + fn read_restore_job<'a>( + &'a self, + key: &'a RestoreTargetKey, + ) -> BoxFuture<'a, Result, StorageError>> { + Box::pin(async move { + let query = format!( + "SELECT {RESTORE_JOB_COLUMNS} FROM {}.tables \ + WHERE account_id = ? AND table_name = ?", + self.catalog_keyspace() + ); + let rows = crate::cassandra_util::query_rows_quorum::( + &self.session, + &query, + cdrs_tokio::query_values!(key.account_id.as_str(), key.table_name.as_str()), + "restore job", + ) + .await?; + let Some(row) = rows.first() else { + return Ok(None); + }; + // No record, or a table re-created under the name with another id, + // is not this target's job. + let table_id: Option = row.get_by_name("table_id").ok().flatten(); + let status: Option = row.get_by_name("restore_status").ok().flatten(); + if table_id.as_deref() != Some(key.table_id.as_str()) || status.is_none() { + return Ok(None); + } + Ok(Some(restore_job_row_from(row)?)) + }) + } + + fn list_restore_jobs<'a>( + &'a self, + status: Option, + ) -> BoxFuture<'a, Result, StorageError>> { + Box::pin(async move { + // CQL filters one equality per scan, so a status filter is one + // scan and "every record" is a scan per restore status. + let statuses = match status { + Some(status) => vec![status], + None => vec![JobStatus::Running, JobStatus::Done, JobStatus::Failed], + }; + let mut jobs = Vec::new(); + for status in statuses { + let query = format!( + "SELECT {RESTORE_JOB_COLUMNS} FROM {}.tables WHERE restore_status = ? \ + ALLOW FILTERING", + self.catalog_keyspace() + ); + let rows = crate::cassandra_util::query_rows_quorum::( + &self.session, + &query, + cdrs_tokio::query_values!(status.as_str()), + "restore jobs", + ) + .await?; + for row in rows { + jobs.push(restore_job_row_from(&row)?); + } + } + Ok(jobs) + }) + } + + /// Conditioned on RUNNING, on the listed `owner_generation` and on the + /// listed `restore_heartbeat_at` (a millisecond timestamp, rebuilt from + /// the listed epoch seconds exactly), and written with a fresh generation + /// (design 4(e)), in one lightweight transaction. + fn fail_restore_job<'a>( + &'a self, + key: &'a RestoreTargetKey, + owner_generation: u64, + heartbeat_at: f64, + reason: &'a str, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + let query = format!( + "UPDATE {}.tables SET restore_status = 'FAILED', restore_failure_reason = ?, \ + restore_completed_at = toTimestamp(now()), restore_owner_generation = ? \ + WHERE account_id = ? AND table_name = ? \ + IF table_id = ? AND restore_status = 'RUNNING' AND restore_owner_generation = ? \ + AND restore_heartbeat_at = ?", + self.catalog_keyspace() + ); + let result = crate::cassandra_util::query_lwt( + &self.session, + &query, + cdrs_tokio::query_values!( + reason, + generation_column(new_generation()), + key.account_id.as_str(), + key.table_name.as_str(), + key.table_id.as_str(), + generation_column(owner_generation), + seconds_to_millis(heartbeat_at) + ), + ) + .await?; + crate::cassandra_util::lwt_applied(&result) + }) + } + + fn read_store_ids<'a>( + &'a self, + store_kind: StoreKind, + store_root: &'a str, + ) -> BoxFuture<'a, Result, StorageError>> { + Box::pin(async move { + // `backup_index_by_status` is partitioned by (root, status), so a + // root's rows are its five status partitions; the entry carries + // the kind and id, which is all this read needs (an orphan entry + // left by a crash names the same store its row did). + let query = format!( + "SELECT store_kind, store_id FROM {}.backup_index_by_status \ + WHERE store_root = ? AND backup_status = ?", + self.catalog_keyspace() + ); + let mut ids = Vec::new(); + for status in [ + BackupStatus::Creating, + BackupStatus::Available, + BackupStatus::Deleting, + BackupStatus::Deleted, + BackupStatus::Failed, + ] { + let rows = crate::cassandra_util::query_rows_quorum::( + &self.session, + &query, + cdrs_tokio::query_values!(store_root, status.as_str()), + "status index", + ) + .await?; + for row in rows { + let kind: Option = row.get_by_name("store_kind").ok().flatten(); + let id: Option = row.get_by_name("store_id").ok().flatten(); + if kind.as_deref() == Some(store_kind.as_str()) + && let Some(id) = id + && !ids.contains(&id) + { + ids.push(id); + } + } + } + ids.sort(); + Ok(ids) + }) + } + + fn maintenance_claim<'a>( + &'a self, + holder: &'a str, + stale_after: Duration, + ) -> BoxFuture<'a, Result, StorageError>> { + Box::pin(async move { + // The row is the claim: `INSERT ... IF NOT EXISTS` takes it when + // there is none; otherwise the row is read, and one whose + // heartbeat is older than `stale_after` on the coordinator's clock + // is taken over with `IF owner_generation = `, so two + // commands racing for a dead holder's row see one win. + let catalog = self.catalog_keyspace(); + let generation = new_generation(); + let insert = format!( + "INSERT INTO {catalog}.backup_maintenance \ + (claim, holder, owner_generation, claimed_at, heartbeat_at) \ + VALUES (?, ?, ?, toTimestamp(now()), toTimestamp(now())) IF NOT EXISTS" + ); + let inserted = conditional_write( + &self.session, + &insert, + cdrs_tokio::query_values!( + MAINTENANCE_CLAIM_KEY, + holder, + generation_column(generation) + ), + ) + .await?; + // A test's lost answer: the insert ran, the caller sees no result. + let inserted = if crate::test_hooks::maintenance_insert_answer_lost() { + WriteOutcome::Unknown + } else { + inserted + }; + if inserted != WriteOutcome::Matched { + let read = format!( + "SELECT holder, owner_generation, heartbeat_at FROM \ + {catalog}.backup_maintenance WHERE claim = ?" + ); + let rows = crate::cassandra_util::query_rows_serial::( + &self.session, + &read, + cdrs_tokio::query_values!(MAINTENANCE_CLAIM_KEY), + "maintenance claim", + ) + .await?; + let row = rows.first(); + let current_holder: Option = + row.and_then(|r| r.get_by_name("holder").ok().flatten()); + let current_generation: Option = + row.and_then(|r| r.get_by_name("owner_generation").ok().flatten()); + // An insert whose answer was lost (`Unknown`) may have + // committed: a row carrying this call's generation is this + // call's claim. + if inserted == WriteOutcome::Unknown + && current_generation == Some(generation_column(generation)) + { + return Ok(Box::new(CassandraMaintenanceClaim { + engine: self.clone(), + generation, + }) as Box); + } + let heartbeat_at: Option = + row.and_then(|r| r.get_by_name("heartbeat_at").ok().flatten()); + let now = coordinator_now_millis(self).await?; + let stale_ms = i64::try_from(stale_after.as_millis()).unwrap_or(i64::MAX); + let stale = heartbeat_at.is_none_or(|at| at < now.saturating_sub(stale_ms)); + let refused = || { + StorageError::AlreadyExists(format!( + "the backup maintenance claim is held by {}", + current_holder + .as_deref() + .unwrap_or("another maintenance command") + )) + }; + if !stale { + return Err(refused()); + } + let take_over = format!( + "UPDATE {catalog}.backup_maintenance SET holder = ?, owner_generation = ?, \ + claimed_at = toTimestamp(now()), heartbeat_at = toTimestamp(now()) \ + WHERE claim = ? IF owner_generation = ?" + ); + // A test's pause between the read and the takeover, so two + // takers both read the stale row before either writes. + crate::test_hooks::before_maintenance_takeover().await; + let taken = conditional_write( + &self.session, + &take_over, + cdrs_tokio::query_values!( + holder, + generation_column(generation), + MAINTENANCE_CLAIM_KEY, + current_generation + ), + ) + .await?; + match taken { + WriteOutcome::Matched => {} + WriteOutcome::NotMatched => return Err(refused()), + WriteOutcome::Unknown => { + // The takeover's answer was lost; the update may + // have committed. Re-read at `LOCAL_SERIAL`, as the + // insert path does: a row carrying this call's + // generation is this call's claim, and answering + // "held by" the old holder would leave a row with + // this generation and a fresh heartbeat that nobody + // renews, refusing every job until it goes stale. + let rows = crate::cassandra_util::query_rows_serial::( + &self.session, + &read, + cdrs_tokio::query_values!(MAINTENANCE_CLAIM_KEY), + "maintenance claim", + ) + .await?; + let now_generation: Option = rows + .first() + .and_then(|r| r.get_by_name("owner_generation").ok().flatten()); + if now_generation != Some(generation_column(generation)) { + return Err(refused()); + } + } + } + } + Ok(Box::new(CassandraMaintenanceClaim { + engine: self.clone(), + generation, + }) as Box) + }) + } + + fn check_maintenance_claim( + &self, + stale_after: Duration, + ) -> BoxFuture<'_, Result> { + Box::pin(async move { + let query = format!( + "SELECT heartbeat_at FROM {}.backup_maintenance WHERE claim = ?", + self.catalog_keyspace() + ); + let rows = crate::cassandra_util::query_rows_serial::( + &self.session, + &query, + cdrs_tokio::query_values!(MAINTENANCE_CLAIM_KEY), + "maintenance claim", + ) + .await?; + let Some(row) = rows.first() else { + return Ok(false); + }; + let heartbeat_at: Option = row.get_by_name("heartbeat_at").ok().flatten(); + let Some(heartbeat_at) = heartbeat_at else { + return Ok(false); + }; + let now = coordinator_now_millis(self).await?; + let stale_ms = i64::try_from(stale_after.as_millis()).unwrap_or(i64::MAX); + Ok(heartbeat_at >= now.saturating_sub(stale_ms)) + }) + } +} + +/// The partition key of the one maintenance claim row. +const MAINTENANCE_CLAIM_KEY: &str = "claim"; + +/// The holder's handle on the `backup_maintenance` row. +struct CassandraMaintenanceClaim { + engine: CassandraEngine, + generation: u64, +} + +impl MaintenanceClaim for CassandraMaintenanceClaim { + fn generation(&self) -> u64 { + self.generation + } + + fn heartbeat(&self) -> BoxFuture<'_, Result> { + Box::pin(async move { + let query = format!( + "UPDATE {}.backup_maintenance SET heartbeat_at = toTimestamp(now()) \ + WHERE claim = ? IF owner_generation = ?", + self.engine.catalog_keyspace() + ); + let result = crate::cassandra_util::query_lwt( + &self.engine.session, + &query, + cdrs_tokio::query_values!( + MAINTENANCE_CLAIM_KEY, + generation_column(self.generation) + ), + ) + .await?; + Ok(if crate::cassandra_util::lwt_applied(&result)? { + Heartbeat::Confirmed + } else { + Heartbeat::Lost + }) + }) + } + + fn release(&self) -> BoxFuture<'_, Result<(), StorageError>> { + Box::pin(async move { + let query = format!( + "DELETE FROM {}.backup_maintenance WHERE claim = ? IF owner_generation = ?", + self.engine.catalog_keyspace() + ); + // Not applied means another holder took the row over; it is + // theirs to release. + conditional_write( + &self.engine.session, + &query, + cdrs_tokio::query_values!( + MAINTENANCE_CLAIM_KEY, + generation_column(self.generation) + ), + ) + .await?; + Ok(()) + }) + } +} + +#[cfg(test)] +mod tests { + use std::net::{IpAddr, Ipv4Addr, SocketAddr}; + + use cdrs_tokio::frame::message_error::{ErrorBody, WriteTimeoutError, WriteType}; + + use super::*; + + fn server_error(ty: ErrorType) -> DriverError { + DriverError::Server { + body: ErrorBody { + message: "synthetic".to_owned(), + ty, + }, + addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 9042), + } + } + + /// Only a lost answer leaves a write unknown: an I/O or driver timeout, + /// or the server's write timeout on the Paxos round. A refused statement + /// (syntax, invalid) is an error with a known outcome. + #[test] + fn only_a_lost_answer_leaves_a_conditional_write_unknown() { + let cas_timeout = server_error(ErrorType::WriteTimeout(WriteTimeoutError { + cl: Consistency::LocalSerial, + received: 0, + block_for: 1, + write_type: WriteType::Cas, + contentions: Some(1), + })); + assert!(outcome_unknown(&cas_timeout)); + assert!(outcome_unknown(&DriverError::Io(std::io::Error::new( + std::io::ErrorKind::ConnectionReset, + "reset", + )))); + assert!(outcome_unknown(&DriverError::Timeout("waited".to_owned()))); + assert!(!outcome_unknown(&server_error(ErrorType::Invalid))); + assert!(!outcome_unknown(&server_error(ErrorType::Syntax))); + assert!(!outcome_unknown(&DriverError::General("x".to_owned()))); + } + + #[test] + fn generations_round_trip_through_the_column_and_a_bad_value_is_unowned() { + assert_eq!(generation_from_column(Some(generation_column(7))), 7); + assert_eq!(generation_from_column(None), 0); + assert_eq!(generation_from_column(Some(-1)), 0); + assert_eq!(generation_column(u64::MAX), i64::MAX); + } + + #[test] + fn listings_sort_by_created_at_then_arn() { + let row = |created_at: f64, arn: &str| BackupRow { + backup_id: String::new(), + account_id: String::new(), + table_name: String::new(), + backup_arn: arn.to_owned(), + backup_name: String::new(), + status: BackupStatus::Available, + store: StoreIdentity { + store_kind: StoreKind::Filesystem, + store_root: String::new(), + store_id: String::new(), + }, + size_bytes: 0, + item_count: 0, + created_at, + completed_at: None, + failure_reason: None, + source_table_details_json: serde_json::Value::Null, + owner_generation: 0, + heartbeat_at: 0.0, + }; + let mut rows = vec![row(2.0, "b"), row(1.0, "z"), row(2.0, "a")]; + sort_listing(&mut rows); + let arns: Vec<&str> = rows.iter().map(|r| r.backup_arn.as_str()).collect(); + assert_eq!(arns, ["z", "a", "b"]); + } + + /// The settle of an insert whose answer was lost: the row read back is + /// this insert only when it is unowned and every column the insert takes + /// from the row is equal; the columns the insert sets itself + /// (`completed_at`, `heartbeat_at`) and the ARN, which the read was + /// keyed by, do not count. + #[test] + fn a_lost_insert_is_settled_only_by_the_row_it_writes() { + let row = BackupRow { + backup_id: "01757500000123-0badcafe".to_owned(), + account_id: "123456789012".to_owned(), + table_name: "Music".to_owned(), + backup_arn: + "arn:aws:dynamodb:us-east-1:123456789012:table/Music/backup/01757500000123-0badcafe" + .to_owned(), + backup_name: "nightly".to_owned(), + status: BackupStatus::Available, + store: StoreIdentity { + store_kind: StoreKind::Filesystem, + store_root: "/backups".to_owned(), + store_id: "store-1".to_owned(), + }, + size_bytes: 1234, + item_count: 5, + created_at: 1_757_500_000.123, + completed_at: None, + failure_reason: None, + source_table_details_json: serde_json::json!({ "table_name": "Music" }), + owner_generation: 99, + heartbeat_at: 0.0, + }; + let read = BackupRow { + owner_generation: UNOWNED_GENERATION, + completed_at: Some(1_757_500_001.0), + heartbeat_at: 1_757_500_001.0, + created_at: 1_757_500_000.123_4, + ..row.clone() + }; + assert!(is_unowned_insert_of(&read, &row)); + let differing: [fn(&mut BackupRow); 11] = [ + |r| r.owner_generation = 7, + |r| r.backup_id = "01757500000123-00000000".to_owned(), + |r| r.table_name = "Other".to_owned(), + |r| r.backup_name = "other".to_owned(), + |r| r.status = BackupStatus::Creating, + |r| r.store.store_id = "store-2".to_owned(), + |r| r.size_bytes = 1, + |r| r.item_count = 1, + |r| r.created_at = 1_757_500_000.124, + |r| r.failure_reason = Some("x".to_owned()), + |r| r.source_table_details_json = serde_json::json!({ "table_name": "Other" }), + ]; + for (i, change) in differing.iter().enumerate() { + let mut other = read.clone(); + change(&mut other); + assert!(!is_unowned_insert_of(&other, &row), "change {i}"); + } + } +} diff --git a/crates/storage-cassandra/src/backup_engine.rs b/crates/storage-cassandra/src/backup_engine.rs index 970e95e88..ecb73e318 100755 --- a/crates/storage-cassandra/src/backup_engine.rs +++ b/crates/storage-cassandra/src/backup_engine.rs @@ -41,12 +41,29 @@ //! DeleteBackup removes stale registrations before claiming the backup, so one //! never blocks it. //! -//! From the insert of the target row until the copy ends, the restoring -//! process refreshes `tables.restore_heartbeat` every -//! [`RESTORE_HEARTBEAT_INTERVAL`] (first refresh immediately); -//! [`CassandraEngine::sweep_abandoned_restores`] removes a target whose -//! heartbeat is older than [`RESTORE_STALE_AFTER`]. The servers run it at -//! startup and every [`RESTORE_SWEEP_INTERVAL`] ([`poll_abandoned_restores`]). +//! The target's `tables` row is also its restore job record (`restore_status` +//! RUNNING, `restore_kind` legacy, `restore_owner_generation`, +//! `restore_heartbeat_at`), written by the one `INSERT ... IF NOT EXISTS`. +//! From that insert until the copy ends, the restoring process refreshes +//! `restore_heartbeat_at` every +//! [`extenddb_storage::backup_catalog::JOB_HEARTBEAT_INTERVAL`] (first refresh +//! immediately; the one period every backend's jobs heartbeat at, which +//! `[backup] heartbeat_interval_secs` reaches in a later release) with a +//! lightweight transaction conditioned on the generation, +//! the claim's fencing token; a refresh that does not apply means another +//! owner took the target, and the copy stops without touching it +//! ([`JobHeartbeat`] states when a copy stops). The +//! engine's reconciler fails and removes a target whose heartbeat is older +//! than its stale window, through the `BackupCatalog` methods of +//! `backup_catalog.rs`. The ACTIVE flip marks the record DONE in the same +//! lightweight transaction; a failed copy marks it FAILED, conditioned on the +//! generation and on RUNNING, before the target is removed. +//! +//! The in-catalog create holds the same kind of claim on its CREATING +//! `backups_by_arn` row (`owner_generation`, `heartbeat_at`) and heartbeats it +//! while it copies. [`LegacyBackupReader::abandoned_creating_backups`] reports +//! a CREATING row whose heartbeat is stale, fenced to DELETING, for the +//! reconciler to finish through [`LegacyBackupReader::delete`]. //! //! `restore_table_from_backup` spawns one task that registers and then //! copies; the request future waits only for the registration outcome and @@ -63,15 +80,20 @@ use cdrs_tokio::query::BatchQueryBuilder; use cdrs_tokio::types::map::Map; use cdrs_tokio::types::{AsRust, IntoRustByName, rows::Row}; use extenddb_core::types::{ - AttributeDefinition, BackupDescription, BackupDetails, BackupSummary, DeleteTableInput, - GsiInput, KeySchemaElement, LsiInput, ProvisionedThroughput, SourceTableDetails, - TableDescription, TableKeyInfo, + AttributeDefinition, BackupDescription, BackupDetails, BackupSummary, CreateTableInput, + DeleteTableInput, GsiInput, KeySchemaElement, LsiInput, ProvisionedThroughput, + SourceTableDetails, TableDescription, TableKeyInfo, +}; +use extenddb_storage::backup_catalog::{ + AbandonedLegacyBackup, RestoreRegistrationRow, new_generation, }; use extenddb_storage::backup_definition::{ BACKUP_DEFINITION_VERSION, BackupTableDefinition, throughput_from_catalog, }; use extenddb_storage::error::StorageError; -use extenddb_storage::{BackupEngine, BackupRestoreSource, MetadataEngine, RestoreRequest}; +use extenddb_storage::{ + BackupRestoreSource, JobPermit, LegacyBackupReader, MetadataEngine, RestoreRequest, +}; use futures::StreamExt; use futures::future::BoxFuture; @@ -86,18 +108,17 @@ use crate::delete_table::DeleteTableCaller; const BACKUP_ITEMS_PER_BUCKET: i64 = 64; const BACKUP_SCAN_PAGE_SIZE: i64 = 1_000; -/// Items of one bucket a restore copy writes concurrently. -const RESTORE_WRITE_CONCURRENCY: usize = 8; +/// Items of one bucket a restore copy writes concurrently. Shared by the +/// legacy copy and the store-backed `bulk_load`. +pub(crate) const RESTORE_WRITE_CONCURRENCY: usize = 8; -/// How often a running restore copy refreshes its target's heartbeat. -pub const RESTORE_HEARTBEAT_INTERVAL: Duration = Duration::from_secs(15); - -/// A restore target whose heartbeat is older than this is abandoned: the -/// process copying into it is gone. Four missed heartbeats. -pub const RESTORE_STALE_AFTER: Duration = Duration::from_secs(60); - -/// How often the abandoned-restore sweep runs. -pub const RESTORE_SWEEP_INTERVAL: Duration = Duration::from_secs(15); +/// Heartbeats that error in a row after which a job stops: three beats could +/// not confirm the claim, so the row may already be another owner's. One +/// error under a loaded node does not stop a job. A beat that hangs rather +/// than erroring is not counted (the loop waits on it); the bound then is +/// the per-bucket poll of [`JobHeartbeat::is_lost`], set by the beat that +/// returns once the row carries another generation. +const HEARTBEAT_ERRORS_TO_STOP: u32 = 3; /// Attempts a restore makes to register on its backup when a stale /// registration (cleared between attempts) kept the previous one from @@ -116,12 +137,23 @@ fn backup_not_found(backup_arn: &str) -> StorageError { StorageError::Validation(format!("Backup not found: {backup_arn}")) } -fn backup_in_use(table_name: &str, backup_arn: &str) -> StorageError { - StorageError::BackupInUse(format!( - "Backup is being used to restore table {table_name}: {backup_arn}" +/// The error of a copy whose row it can no longer call its own: a heartbeat +/// found another owner on it (the reconciler found the heartbeat stale), or +/// [`HEARTBEAT_ERRORS_TO_STOP`] heartbeats in a row could not confirm the +/// claim. Nothing of the row or its payload is touched: whoever claims the +/// row (the reconciler, once the heartbeat is stale) removes it. +fn claim_lost(backup_arn: &str) -> StorageError { + StorageError::Internal(format!( + "backup {backup_arn}: the claim on its row was lost or could not be confirmed while \ + the items copied; the row's owner removes it" )) } +/// The shared text, [`extenddb_storage::backup_catalog::backup_in_use`]. +fn backup_in_use(table_name: &str, backup_arn: &str) -> StorageError { + extenddb_storage::backup_catalog::backup_in_use(table_name, backup_arn) +} + /// The error of a restore whose [`REGISTRATION_ATTEMPTS`] all found the /// backup's registration taken by a restore that was just starting (or just /// stopping, its stale entry cleared between attempts). The backup is @@ -140,14 +172,14 @@ fn registration_exhausted( } #[derive(Debug)] -struct StoredBackup { +pub(crate) struct StoredBackup { backup_arn: String, account_id: String, backup_name: String, table_id: String, table_name: String, table_arn: String, - backup_status: String, + pub(crate) backup_status: String, backup_type: String, backup_size_bytes: i64, item_count: i64, @@ -160,7 +192,13 @@ struct StoredBackup { /// `BackupTableDefinition` JSON; `None` for a row written before V005. definition: Option, /// Restore target `table_id` -> target table name, while registered. - active_restores: HashMap, + pub(crate) active_restores: HashMap, + /// Set on a catalog index row (a backup whose bytes live in a store); + /// unset on an in-catalog backup. The two readers never see each other's + /// rows. + pub(crate) store_kind: Option, + pub(crate) store_root: Option, + pub(crate) store_id: Option, } impl StoredBackup { @@ -220,9 +258,18 @@ impl StoredBackup { .map_err(|e| StorageError::Internal(format!("Parse created_at: {e}")))?, definition: row.get_by_name("definition").ok().flatten(), active_restores, + store_kind: row.get_by_name("store_kind").ok().flatten(), + store_root: row.get_by_name("store_root").ok().flatten(), + store_id: row.get_by_name("store_id").ok().flatten(), }) } + /// Whether this row belongs to the in-catalog (legacy) format rather than + /// the catalog index. + pub(crate) fn is_in_catalog(&self) -> bool { + self.store_kind.is_none() + } + fn details(&self) -> BackupDetails { BackupDetails { backup_arn: self.backup_arn.clone(), @@ -310,12 +357,13 @@ struct RestoreJob { backup_size_bytes: i64, key_schema: Vec, attribute_definitions: Vec, - owner: uuid::Uuid, + /// The claim's fencing token on the target row. + generation: u64, } enum CopyError { - /// The target is no longer this copy's (the sweep claimed it). Whoever - /// claimed it removes it. + /// The row is no longer this copy's (another owner claimed it). Whoever + /// claimed it owns what happens to it. OwnershipLost, Failed(StorageError), } @@ -332,20 +380,57 @@ impl Drop for AbortOnDrop { } } -/// The heartbeat of one restore target, running from the moment the target -/// row is inserted until this value is dropped. -pub(crate) struct RestoreHeartbeat { +/// The heartbeat of one job's row (a restore target or an in-catalog +/// backup being created), running from the moment the row is inserted until +/// this value is dropped. +/// +/// When a copy stops, in one place: a heartbeat that does not apply +/// (`Ok(false)`: another owner holds the row) stops the heartbeat and sets +/// [`Self::is_lost`]; so does the [`HEARTBEAT_ERRORS_TO_STOP`]th heartbeat +/// that errors in a row (three beats could not confirm the claim, which may +/// be another owner's by now); a heartbeat error before that is logged and +/// the next period tries again, so one error under a loaded node does not +/// stop a job. A beat that hangs is not an error and is not counted: the +/// loop waits on it, and the copy's own writes, on the same session, hang +/// with it; the beat that returns is conditioned on the row's generation, so +/// a row the reconciler fenced meanwhile sets [`Self::is_lost`] and the +/// copy's per-bucket poll stops it after at most one more bucket, its row +/// writes (activation, abandonment) conditioned on the generation too. The +/// copy polls [`Self::is_lost`] between units of work and, once it is set, +/// touches neither the row nor its data again. +pub(crate) struct JobHeartbeat { _task: AbortOnDrop, lost: Arc, } -impl RestoreHeartbeat { - /// Run `beat` now and then every `interval` until this value is dropped. - /// `beat` returns whether the target is still this process's; the first - /// `Ok(false)` sets [`Self::is_lost`] and stops the heartbeat. An error - /// is logged and the next tick tries again (the sweep, not one missed - /// beat, decides that a target is abandoned). - fn spawn(interval: Duration, target_name: String, mut beat: F) -> Self +impl JobHeartbeat { + /// The heartbeat of a job of `account_id`: [`Self::spawn_every`] at the + /// period the jobs heartbeat at (`JOB_HEARTBEAT_INTERVAL`, or the period + /// a test hook set for the account), the beats failing where a test hook + /// asks them to. + fn spawn(account_id: &str, label: String, mut beat: F) -> Self + where + F: FnMut() -> Fut + Send + 'static, + Fut: std::future::Future> + Send, + { + let interval = crate::test_hooks::heartbeat_interval(account_id); + let account_id = account_id.to_owned(); + Self::spawn_every(interval, label, move || { + let injected = crate::test_hooks::heartbeat_error(&account_id); + let real = beat(); + async move { + match injected { + Some(error) => Err(error), + None => real.await, + } + } + }) + } + + /// Run `beat` now and then every `interval` until this value is dropped + /// or the heartbeat stops as the type's doc says. `beat` returns whether + /// the row is still this job's. + fn spawn_every(interval: Duration, label: String, mut beat: F) -> Self where F: FnMut() -> Fut + Send + 'static, Fut: std::future::Future> + Send, @@ -355,16 +440,26 @@ impl RestoreHeartbeat { let task = tokio::spawn(async move { // The first tick completes immediately. let mut ticker = tokio::time::interval(interval); + let mut errors_in_a_row = 0_u32; loop { ticker.tick().await; match beat().await { - Ok(true) => {} + Ok(true) => errors_in_a_row = 0, Ok(false) => { flag.store(true, Ordering::Release); break; } Err(error) => { - tracing::warn!("Restore of table {target_name}: heartbeat failed: {error}") + errors_in_a_row += 1; + if errors_in_a_row >= HEARTBEAT_ERRORS_TO_STOP { + tracing::error!( + "{label}: heartbeat failed {errors_in_a_row} times in a row \ + ({error}); the claim is unconfirmed and the copy stops" + ); + flag.store(true, Ordering::Release); + break; + } + tracing::warn!("{label}: heartbeat failed: {error}"); } } } @@ -375,7 +470,9 @@ impl RestoreHeartbeat { } } - /// Whether a heartbeat found the target no longer owned by this process. + /// Whether the heartbeat stopped: a beat found the row no longer owned by + /// this job, or [`HEARTBEAT_ERRORS_TO_STOP`] beats in a row could not + /// confirm it. fn is_lost(&self) -> bool { self.lost.load(Ordering::Acquire) } @@ -386,36 +483,45 @@ struct HeartbeatTarget { account_id: String, target_name: String, table_id: String, - owner: uuid::Uuid, + generation: u64, } impl CassandraEngine { - async fn load_backup( + /// The `backups_by_arn` row of either kind, read at quorum, if there is + /// one. + pub(crate) async fn load_stored_backup( &self, account_id: &str, backup_arn: &str, - ) -> Result { + ) -> Result, StorageError> { let query = format!( "SELECT * FROM {}.backups_by_arn WHERE account_id = ? AND backup_arn = ?", self.catalog_keyspace() ); - let result = self - .session - .query_with_values(&query, cdrs_tokio::query_values!(account_id, backup_arn)) - .await - .map_err(|e| StorageError::Internal(format!("Query backup: {e}")))?; - let rows = result - .response_body() - .map_err(|e| StorageError::Internal(format!("Parse backup response: {e}")))? - .into_rows() - .unwrap_or_default(); - let backup = rows - .first() + let rows = crate::cassandra_util::query_rows_quorum::( + &self.session, + &query, + cdrs_tokio::query_values!(account_id, backup_arn), + "backup", + ) + .await?; + rows.first() .map(StoredBackup::from_row) - .transpose()? - .filter(|backup| backup.account_id == account_id) - .ok_or_else(|| backup_not_found(backup_arn))?; - Ok(backup) + .transpose() + .map(|row| row.filter(|backup| backup.account_id == account_id)) + } + + /// The in-catalog backup `backup_arn` names; a catalog index row (one + /// with `store_kind` set) is not one, and is "not found" here. + async fn load_backup( + &self, + account_id: &str, + backup_arn: &str, + ) -> Result { + self.load_stored_backup(account_id, backup_arn) + .await? + .filter(StoredBackup::is_in_catalog) + .ok_or_else(|| backup_not_found(backup_arn)) } async fn load_available_backup( @@ -430,6 +536,31 @@ impl CassandraEngine { Ok(backup) } + /// The AVAILABLE catalog-index backup (`store_kind` set) `backup_arn` + /// names; an in-catalog row is "not found" here, the mirror of + /// [`Self::load_available_backup`]. A store-backed restore reads this + /// row, not the legacy one. + async fn load_available_index_backup( + &self, + account_id: &str, + backup_arn: &str, + ) -> Result { + let backup = self + .load_stored_backup(account_id, backup_arn) + .await? + .filter(|backup| !backup.is_in_catalog()) + .ok_or_else(|| backup_not_found(backup_arn))?; + if backup.backup_status != "AVAILABLE" { + return Err(backup_not_found(backup_arn)); + } + Ok(backup) + } + + /// Delete a backup's `backup_items` buckets: the `bucket_count(item_count)` + /// the row accounts for, then every further bucket until an empty one. + /// The create writes buckets in order and publishes `item_count` last, so + /// a create that died mid-copy leaves a row whose count is 0 and whose + /// buckets are contiguous from 0; the probe finds them all. async fn cleanup_backup_payload( &self, account_id: &str, @@ -437,18 +568,36 @@ impl CassandraEngine { item_count: i64, ) -> Result<(), StorageError> { let account_keyspace = self.account_keyspace(account_id); - let query = format!( + let delete = format!( "DELETE FROM {account_keyspace}.backup_items WHERE backup_arn = ? AND bucket = ?" ); - for bucket in 0..bucket_count(item_count) { + let probe = format!( + "SELECT item_index FROM {account_keyspace}.backup_items \ + WHERE backup_arn = ? AND bucket = ? LIMIT 1" + ); + let known = bucket_count(item_count); + let mut bucket = 0_i64; + loop { #[allow(clippy::cast_possible_truncation)] let bucket_i32 = bucket as i32; + if bucket >= known { + let rows = crate::cassandra_util::query_rows_quorum::( + &self.session, + &probe, + cdrs_tokio::query_values!(backup_arn, bucket_i32), + "probe backup payload", + ) + .await?; + if rows.is_empty() { + return Ok(()); + } + } self.session - .query_with_values(&query, cdrs_tokio::query_values!(backup_arn, bucket_i32)) + .query_with_values(&delete, cdrs_tokio::query_values!(backup_arn, bucket_i32)) .await .map_err(|e| StorageError::Internal(format!("Delete backup payload: {e}")))?; + bucket += 1; } - Ok(()) } async fn remove_backup_index_rows(&self, backup: &StoredBackup) -> Result<(), StorageError> { @@ -619,6 +768,24 @@ impl CassandraEngine { self.ensure_backup_not_restoring(&backup).await } + /// [`Self::registration_refusal`] for a store-backed (index) backup: it + /// re-reads the catalog-index row rather than the in-catalog one. + async fn registration_refusal_index( + &self, + account_id: &str, + backup_arn: &str, + ) -> Result<(), StorageError> { + let backup = self + .load_stored_backup(account_id, backup_arn) + .await? + .filter(|backup| !backup.is_in_catalog()) + .ok_or_else(|| backup_not_found(backup_arn))?; + if backup.backup_status != "AVAILABLE" { + return Err(backup_not_found(backup_arn)); + } + self.ensure_backup_not_restoring(&backup).await + } + async fn unregister_restore( &self, account_id: &str, @@ -706,10 +873,48 @@ impl CassandraEngine { }) } - /// BackupInUse if a registered restore of `backup` is still copying; - /// otherwise remove every stale registration so it cannot block the - /// DELETING claim. - async fn ensure_backup_not_restoring(&self, backup: &StoredBackup) -> Result<(), StorageError> { + /// The name of a CREATING restore target whose `restore_source_backup_arn` + /// is `backup_arn`, read from the `tables` rows directly rather than from + /// the backup's `active_restores` map: a store-backed registration writes + /// the target row before the map entry, so this catches an in-flight + /// registration the map does not yet name (the in-use check would + /// otherwise miss it and a second restore's own override validation + /// would answer first). A table + /// scan filtered on the two columns; the in-use check is a control-plane + /// path (DeleteBackup, a restore's registration), not a hot path. + pub(crate) async fn creating_restore_target_of_backup( + &self, + account_id: &str, + backup_arn: &str, + ) -> Result, StorageError> { + let query = format!( + "SELECT table_name FROM {}.tables \ + WHERE account_id = ? AND restore_source_backup_arn = ? AND table_status = 'CREATING' \ + ALLOW FILTERING", + self.catalog_keyspace() + ); + let rows = crate::cassandra_util::query_rows_quorum::( + &self.session, + &query, + cdrs_tokio::query_values!(account_id, backup_arn), + "restore target of backup", + ) + .await?; + Ok(rows.first().and_then(|row| { + let name: Option = row.get_by_name("table_name").ok().flatten(); + name + })) + } + + /// The name of a CREATING restore target registered on `backup` and still + /// being restored (the in-use rule: it reads the target's status, not the + /// registration alone); `None` when there is none. Every registration + /// whose target is no longer being restored is cleared on the way, so it + /// cannot hold the row's lock against a delete. + pub(crate) async fn creating_restore_target( + &self, + backup: &StoredBackup, + ) -> Result, StorageError> { for (table_id, target_name) in &backup.active_restores { if self .restore_target_in_progress( @@ -720,7 +925,7 @@ impl CassandraEngine { ) .await? { - return Err(backup_in_use(target_name, &backup.backup_arn)); + return Ok(Some(target_name.clone())); } tracing::info!( "Backup {}: clearing stale restore registration of table {target_name} \ @@ -730,7 +935,17 @@ impl CassandraEngine { self.unregister_restore(&backup.account_id, &backup.backup_arn, table_id) .await?; } - Ok(()) + Ok(None) + } + + /// BackupInUse naming the target if a registered restore of `backup` is + /// still copying; otherwise `Ok(())` with every stale registration + /// cleared. + async fn ensure_backup_not_restoring(&self, backup: &StoredBackup) -> Result<(), StorageError> { + match self.creating_restore_target(backup).await? { + Some(target_name) => Err(backup_in_use(&target_name, &backup.backup_arn)), + None => Ok(()), + } } // ── Restore copy (detached) ───────────────────────────────────────────── @@ -739,26 +954,26 @@ impl CassandraEngine { /// just been inserted. Called by the restore path of `create_table` right /// after the insert, before the target's data and index DDL, so the time /// that DDL takes (each statement waits for schema agreement) does not - /// count against [`RESTORE_STALE_AFTER`]. + /// count against the reconciler's stale window. pub(crate) fn start_restore_heartbeat( &self, account_id: &str, target_name: &str, table_id: &str, - owner: uuid::Uuid, - ) -> RestoreHeartbeat { + generation: u64, + ) -> JobHeartbeat { let state = Arc::new(( self.clone(), HeartbeatTarget { account_id: account_id.to_owned(), target_name: target_name.to_owned(), table_id: table_id.to_owned(), - owner, + generation, }, )); - RestoreHeartbeat::spawn( - RESTORE_HEARTBEAT_INTERVAL, - target_name.to_owned(), + JobHeartbeat::spawn( + account_id, + format!("Restore of table {target_name}"), move || { let state = Arc::clone(&state); async move { state.0.heartbeat_restore_target(&state.1).await } @@ -766,37 +981,75 @@ impl CassandraEngine { ) } + /// Refresh the target's `restore_heartbeat_at` on the coordinator's + /// clock, conditioned on the claim's generation, on the record still + /// RUNNING (a FAILED write, the reconciler's or the job's own, stops the + /// copy at its next heartbeat, as on the other three backends) and on the + /// target still CREATING (a target claimed to DELETING is the removal's); + /// `false` when any of those moved. async fn heartbeat_restore_target(&self, job: &HeartbeatTarget) -> Result { let query = format!( - "UPDATE {}.tables SET restore_heartbeat = ? WHERE account_id = ? AND table_name = ? \ - IF table_id = ? AND table_status = 'CREATING' AND restore_owner = ?", + "UPDATE {}.tables SET restore_heartbeat_at = toTimestamp(now()) \ + WHERE account_id = ? AND table_name = ? \ + IF table_id = ? AND table_status = 'CREATING' AND restore_status = 'RUNNING' \ + AND restore_owner_generation = ?", self.catalog_keyspace() ); let result = crate::cassandra_util::query_lwt( &self.session, &query, cdrs_tokio::query_values!( - crate::cassandra_util::now_millis(), job.account_id.as_str(), job.target_name.as_str(), job.table_id.as_str(), - job.owner + crate::backup_catalog::generation_column(job.generation) ), ) .await?; crate::cassandra_util::lwt_applied(&result) } + /// Flip the target ACTIVE and mark its job record DONE in one lightweight + /// transaction, conditioned on the table still being a CREATING target + /// whose record is RUNNING under this claim's generation. `false` when + /// another owner (the reconciler) claimed the target first. async fn activate_restore_target( &self, job: &RestoreJob, item_count: i64, table_size_bytes: i64, + ) -> Result { + self.activate_restore_target_row( + &job.account_id, + &job.target_name, + &job.table_id, + job.generation, + item_count, + table_size_bytes, + ) + .await + } + + /// The ACTIVE flip's lightweight transaction, by the target's address and + /// the claim's generation. Shared by the legacy copy + /// ([`Self::activate_restore_target`]) and the store-backed + /// [`extenddb_storage::restore_engine::RestoreEngine::finish_restore`]. + pub(crate) async fn activate_restore_target_row( + &self, + account_id: &str, + target_name: &str, + table_id: &str, + generation: u64, + item_count: i64, + table_size_bytes: i64, ) -> Result { let query = format!( "UPDATE {}.tables SET table_status = 'ACTIVE', status_transition_at = NULL, \ - item_count = ?, table_size_bytes = ? WHERE account_id = ? AND table_name = ? \ - IF table_id = ? AND table_status = 'CREATING' AND restore_owner = ?", + item_count = ?, table_size_bytes = ?, restore_status = 'DONE', \ + restore_completed_at = toTimestamp(now()) \ + WHERE account_id = ? AND table_name = ? \ + IF table_id = ? AND table_status = 'CREATING' AND restore_owner_generation = ? \ + AND restore_status = 'RUNNING'", self.catalog_keyspace() ); let result = crate::cassandra_util::query_lwt( @@ -805,10 +1058,10 @@ impl CassandraEngine { cdrs_tokio::query_values!( item_count, table_size_bytes, - job.account_id.as_str(), - job.target_name.as_str(), - job.table_id.as_str(), - job.owner + account_id, + target_name, + table_id, + crate::backup_catalog::generation_column(generation) ), ) .await?; @@ -817,9 +1070,9 @@ impl CassandraEngine { /// Remove a restore target this caller has claimed (or that never had a /// copy): unregister it from its backup first, so a crash between the two - /// steps leaves a target the sweep removes rather than a registration + /// steps leaves a target the reconciler removes rather than a registration /// that names nothing; then drop the incarnation with this `table_id`. - async fn remove_restore_target( + pub(crate) async fn drop_restore_target( &self, account_id: &str, target_name: &str, @@ -843,22 +1096,30 @@ impl CassandraEngine { } } - /// The copy failed while this process still owned the target: claim it - /// (CREATING -> DELETING, owner-fenced) and remove it. - async fn abandon_restore_target(&self, job: &RestoreJob) { + /// The copy failed while this process still owned the target: mark the + /// job record FAILED with the reason and claim the target for removal + /// (CREATING to DELETING) in one lightweight transaction conditioned on + /// the generation and on RUNNING, then remove it. A record another + /// generation owns means the claim holder owns the target, and nothing + /// further is touched. + async fn abandon_restore_target(&self, job: &RestoreJob, reason: &str) { let query = format!( - "UPDATE {}.tables SET table_status = 'DELETING' WHERE account_id = ? AND table_name = ? \ - IF table_id = ? AND table_status = 'CREATING' AND restore_owner = ?", + "UPDATE {}.tables SET table_status = 'DELETING', restore_status = 'FAILED', \ + restore_failure_reason = ?, restore_completed_at = toTimestamp(now()) \ + WHERE account_id = ? AND table_name = ? \ + IF table_id = ? AND table_status = 'CREATING' AND restore_owner_generation = ? \ + AND restore_status = 'RUNNING'", self.catalog_keyspace() ); let claimed = crate::cassandra_util::query_lwt( &self.session, &query, cdrs_tokio::query_values!( + reason, job.account_id.as_str(), job.target_name.as_str(), job.table_id.as_str(), - job.owner + crate::backup_catalog::generation_column(job.generation) ), ) .await @@ -876,8 +1137,8 @@ impl CassandraEngine { } Err(error) => { tracing::error!( - "Restore of table {} from {}: could not claim the failed target for \ - removal ({error}); the abandoned-restore sweep removes it", + "Restore of table {} from {}: could not mark the failed restore FAILED \ + ({error}); the reconciler removes the target once its heartbeat is stale", job.target_name, job.backup_arn ); @@ -885,7 +1146,7 @@ impl CassandraEngine { } } match self - .remove_restore_target( + .drop_restore_target( &job.account_id, &job.target_name, &job.table_id, @@ -900,7 +1161,7 @@ impl CassandraEngine { ), Err(error) => tracing::error!( "Restore of table {} from {}: removing the failed target failed ({error}); \ - the abandoned-restore sweep retries it", + the reconciler retries it", job.target_name, job.backup_arn ), @@ -910,7 +1171,7 @@ impl CassandraEngine { async fn copy_backup_into_target( &self, job: &RestoreJob, - heartbeat: &RestoreHeartbeat, + heartbeat: &JobHeartbeat, ) -> Result<(i64, i64), CopyError> { let key_info = TableKeyInfo { table_name: job.target_name.clone(), @@ -991,7 +1252,7 @@ impl CassandraEngine { /// was started when the target row was inserted; it is dropped (which /// aborts it) when the copy ends, and on every other exit from this /// function, a panic or cancellation included. - async fn run_restore_copy(self, job: RestoreJob, heartbeat: RestoreHeartbeat) { + async fn run_restore_copy(self, job: RestoreJob, heartbeat: JobHeartbeat) { let outcome = self.copy_backup_into_target(&job, &heartbeat).await; drop(heartbeat); match outcome { @@ -1026,12 +1287,12 @@ impl CassandraEngine { job.target_name, job.backup_arn ); - self.abandon_restore_target(&job).await; + self.abandon_restore_target(&job, &error.to_string()).await; } }, Err(CopyError::OwnershipLost) => tracing::warn!( - "Restore of table {} from {}: stopped; the target was claimed by the \ - abandoned-restore sweep", + "Restore of table {} from {}: stopped; the claim on the target was lost or \ + could not be confirmed (see JobHeartbeat), and its owner removes it", job.target_name, job.backup_arn ), @@ -1041,157 +1302,51 @@ impl CassandraEngine { job.target_name, job.backup_arn ); - self.abandon_restore_target(&job).await; + self.abandon_restore_target(&job, &error.to_string()).await; } } } - // ── Abandoned-restore sweep ───────────────────────────────────────────── - - /// Remove restore targets whose copy is gone: a CREATING or DELETING - /// target carrying a restore registration whose heartbeat is older than - /// `stale_after` (or absent). Returns the names removed. - /// - /// Each removal first claims the target with a lightweight transaction - /// conditioned on the heartbeat it observed, setting DELETING, a fresh - /// heartbeat, and this engine as owner, so a copy that heartbeats in the - /// meantime wins and keeps its target, a copy that comes back later finds - /// itself no longer owner and stops, and two sweepers do not both remove - /// it. The servers' periodic worker calls this with - /// [`RESTORE_STALE_AFTER`]. - /// - /// # Errors - /// - /// [`StorageError`] if the catalog cannot be read; a failure to remove one - /// target is logged and left for the next pass. - pub async fn sweep_abandoned_restores( - &self, - stale_after: Duration, - ) -> Result, StorageError> { - let catalog = self.catalog_keyspace(); - #[allow(clippy::cast_possible_truncation, clippy::cast_possible_wrap)] - let cutoff = crate::cassandra_util::now_millis() - stale_after.as_millis() as i64; - let mut removed = Vec::new(); - for status in ["CREATING", "DELETING"] { - // Cluster-wide filtered scan, as the control-plane poller does for - // CREATING tables; restore targets are few and short-lived. - let query = format!( - "SELECT account_id, table_name, table_id, table_status, \ - restore_source_backup_arn, restore_heartbeat FROM {catalog}.tables \ - WHERE table_status = ? ALLOW FILTERING" - ); - let rows = crate::cassandra_util::query_rows::( - &self.session, - &query, - cdrs_tokio::query_values!(status), - "restore sweep", - ) - .await?; - for row in rows { - let backup_arn: Option = - row.get_by_name("restore_source_backup_arn").ok().flatten(); - let Some(backup_arn) = backup_arn else { - continue; - }; - let heartbeat: Option = row.get_by_name("restore_heartbeat").ok().flatten(); - if heartbeat.is_some_and(|at| at >= cutoff) { - continue; - } - let account_id: String = crate::cassandra_util::get_column::<_, StorageError>( - &row, - "account_id", - "restore sweep", - )?; - let table_name: String = crate::cassandra_util::get_column::<_, StorageError>( - &row, - "table_name", - "restore sweep", - )?; - let table_id: String = crate::cassandra_util::get_column::<_, StorageError>( - &row, - "table_id", - "restore sweep", - )?; - match self - .claim_abandoned_restore(&account_id, &table_name, &table_id, status, heartbeat) - .await - { - Ok(true) => {} - Ok(false) => continue, - Err(error) => { - tracing::warn!("Restore sweep: claiming table {table_name}: {error}"); - continue; - } - } - match self - .remove_restore_target(&account_id, &table_name, &table_id, &backup_arn) - .await - { - Ok(()) => { - tracing::info!( - "Restore sweep: removed abandoned restore target {table_name} \ - (account {account_id}, restoring from {backup_arn}); its copy \ - stopped heartbeating" - ); - removed.push(table_name); - } - Err(error) => tracing::warn!( - "Restore sweep: removing abandoned restore target {table_name}: {error}; \ - retrying on a later pass" - ), - } - } - } - Ok(removed) - } - - async fn claim_abandoned_restore( + /// Move a CREATING in-catalog row whose heartbeat was observed stale to + /// DELETING under a fresh `owner_generation`, conditioned on the status + /// and the heartbeat observed (a `null` heartbeat for a row from before + /// migration 6). The fresh generation fences the dead create's generation out: + /// a late heartbeat or publish of that create no longer applies. `heartbeat_at` + /// is left as it was, so the row stays stale for the pass that finishes + /// it. `false` when the row moved in between: its create beat, published, + /// or removed it. + async fn fence_abandoned_create( &self, account_id: &str, - table_name: &str, - table_id: &str, - observed_status: &str, + backup_arn: &str, observed_heartbeat: Option, ) -> Result { let catalog = self.catalog_keyspace(); - let now = crate::cassandra_util::now_millis(); + let generation = crate::backup_catalog::generation_column(new_generation()); let result = if let Some(heartbeat) = observed_heartbeat { let query = format!( - "UPDATE {catalog}.tables SET table_status = 'DELETING', restore_owner = ?, \ - restore_heartbeat = ? WHERE account_id = ? AND table_name = ? \ - IF table_id = ? AND table_status = ? AND restore_heartbeat = ?" + "UPDATE {catalog}.backups_by_arn SET backup_status = 'DELETING', \ + owner_generation = ? \ + WHERE account_id = ? AND backup_arn = ? \ + IF backup_status = 'CREATING' AND heartbeat_at = ?" ); crate::cassandra_util::query_lwt( &self.session, &query, - cdrs_tokio::query_values!( - self.restore_owner, - now, - account_id, - table_name, - table_id, - observed_status, - heartbeat - ), + cdrs_tokio::query_values!(generation, account_id, backup_arn, heartbeat), ) .await? } else { let query = format!( - "UPDATE {catalog}.tables SET table_status = 'DELETING', restore_owner = ?, \ - restore_heartbeat = ? WHERE account_id = ? AND table_name = ? \ - IF table_id = ? AND table_status = ? AND restore_heartbeat = null" + "UPDATE {catalog}.backups_by_arn SET backup_status = 'DELETING', \ + owner_generation = ? \ + WHERE account_id = ? AND backup_arn = ? \ + IF backup_status = 'CREATING' AND heartbeat_at = null" ); crate::cassandra_util::query_lwt( &self.session, &query, - cdrs_tokio::query_values!( - self.restore_owner, - now, - account_id, - table_name, - table_id, - observed_status - ), + cdrs_tokio::query_values!(generation, account_id, backup_arn), ) .await? }; @@ -1216,15 +1371,14 @@ impl CassandraEngine { /// copy starts from a backup DeleteBackup has claimed. What is not /// atomic: between the two steps a concurrent DescribeTable can see a /// CREATING target of a restore that then fails, and a process exit - /// between them leaves a target that the abandoned-restore sweep removes - /// once its heartbeat (refreshed from the insert until the process - /// exited) is older than - /// [`RESTORE_STALE_AFTER`]. + /// between them leaves a target that the reconciler removes once its + /// heartbeat (refreshed from the insert until the process exited) is + /// older than the stale window. async fn register_restore_target( &self, account_id: &str, request: RestoreRequest, - ) -> Result<(TableDescription, RestoreJob, RestoreHeartbeat), StorageError> { + ) -> Result<(TableDescription, RestoreJob, JobHeartbeat), StorageError> { let RestoreRequest { backup_arn, source_table_arn, @@ -1258,12 +1412,18 @@ impl CassandraEngine { let attribute_definitions = input.attribute_definitions.clone(); #[allow(clippy::cast_possible_truncation)] let restore_date_time_ms = (restore_date_time * 1_000.0).round() as i64; + // The claim is the row itself: a fresh generation, written by the + // IF NOT EXISTS insert, fences every later write of this copy. + let generation = new_generation(); let columns = RestoreTargetColumns { table_id: uuid::Uuid::new_v4().to_string(), source_backup_arn: backup_arn.clone(), source_table_arn, restore_date_time_ms, - owner: self.restore_owner, + owner_generation: generation, + kind: extenddb_storage::backup_catalog::RestoreKind::Legacy, + store: None, + spool_host: None, }; let table_id = columns.table_id.clone(); @@ -1282,12 +1442,12 @@ impl CassandraEngine { Err(error) => { // Part of the target may exist under our table_id. if let Err(cleanup) = self - .remove_restore_target(account_id, &target_name, &table_id, &backup_arn) + .drop_restore_target(account_id, &target_name, &table_id, &backup_arn) .await { tracing::warn!( "Restore of table {target_name}: removing the partly created \ - target failed ({cleanup}); the abandoned-restore sweep removes it" + target failed ({cleanup}); the reconciler removes it" ); } return Err(error); @@ -1317,12 +1477,12 @@ impl CassandraEngine { if !matches!(registered, Ok(true)) { drop(heartbeat); if let Err(cleanup) = self - .remove_restore_target(account_id, &target_name, &table_id, &backup_arn) + .drop_restore_target(account_id, &target_name, &table_id, &backup_arn) .await { tracing::warn!( "Restore of table {target_name}: removing the unregistered target \ - failed ({cleanup}); the abandoned-restore sweep removes it" + failed ({cleanup}); the reconciler removes it" ); } return Err(registration_exhausted(registered, &backup_arn)); @@ -1338,40 +1498,241 @@ impl CassandraEngine { backup_size_bytes: backup.backup_size_bytes, key_schema, attribute_definitions, - owner: self.restore_owner, + generation, }; Ok((description, job, heartbeat)) } -} -/// The abandoned-restore sweep, for the server's worker set: one sweep at -/// startup, then one every [`RESTORE_SWEEP_INTERVAL`], until `shutdown` is -/// cancelled. A sweep already running when `shutdown` fires finishes first. -pub async fn poll_abandoned_restores( - engine: Arc, - shutdown: extenddb_storage::CancellationToken, -) { - loop { - if let Err(error) = engine.sweep_abandoned_restores(RESTORE_STALE_AFTER).await { - tracing::warn!("Abandoned-restore sweep failed: {error}"); + /// Claim the target a registration created for its own removal: CREATING + /// to DELETING and the record FAILED with `reason`, one lightweight + /// transaction conditioned on the table id, on CREATING, on the + /// registrant's generation and on RUNNING (the write + /// [`Self::abandon_restore_target`] makes). `false` when the record is no + /// longer this registration's: the registration ran no heartbeat during + /// the target's DDL, a reconciler whose stale window the DDL outlasted + /// took the record over, and the target is the successor's to load, so + /// the registrant removes nothing. The state a crash between the claim + /// and the drop leaves (DELETING with its registration and a FAILED + /// record) is the one `remove_restore_target` and the FAILED-target duty + /// resume from. + async fn claim_registered_target_for_removal( + &self, + account_id: &str, + target_name: &str, + table_id: &str, + generation: u64, + reason: &str, + ) -> Result { + let query = format!( + "UPDATE {}.tables SET table_status = 'DELETING', restore_status = 'FAILED', \ + restore_failure_reason = ?, restore_completed_at = toTimestamp(now()) \ + WHERE account_id = ? AND table_name = ? \ + IF table_id = ? AND table_status = 'CREATING' AND restore_owner_generation = ? \ + AND restore_status = 'RUNNING'", + self.catalog_keyspace() + ); + let result = crate::cassandra_util::query_lwt( + &self.session, + &query, + cdrs_tokio::query_values!( + reason, + account_id, + target_name, + table_id, + crate::backup_catalog::generation_column(generation) + ), + ) + .await?; + crate::cassandra_util::lwt_applied(&result) + } + + /// The error of a registration whose record another generation took over + /// while its target's DDL ran: the target is left to the record's owner. + fn registration_claim_lost(target_name: &str, backup_arn: &str) -> StorageError { + StorageError::Internal(format!( + "restore of table {target_name} from {backup_arn}: the registration lost its \ + record to a takeover during the target's creation (the record is not RUNNING \ + under the registration's generation); the target is left to the record's owner, \ + which finishes the restore" + )) + } + + /// Register a store-backed (manifest) restore target and return the + /// CREATING description and the claim's generation. The manifest analogue + /// of [`Self::register_restore_target`]: it reads the catalog-index backup + /// row (`store_kind` set) rather than the in-catalog one, writes the job + /// record with `restore_kind = 'manifest'` and the store columns, and + /// starts no self-managed heartbeat (the engine heartbeats through the + /// claim). The ordering against DeleteBackup is the same `active_restores` + /// lightweight transaction on the backup's partition, which + /// [`crate::backup_catalog`]'s `start_backup_delete` conflicts with. Two + /// concurrent registrations of one backup are ordered by that transaction + /// too: the in-use pre-read (`ensure_backup_not_restoring`) is not atomic + /// with it, so both may pass the pre-read and create their targets, but + /// its `active_restores = null` condition admits one; the other finds the + /// registered target in its re-read, is `BackupInUse` naming it, and + /// removes its own target, so one target is left. With no heartbeat + /// during the DDL, a registration whose DDL outlasts the stale window can + /// lose its record to a reconciler's takeover; its removal of the target + /// on a refusal is therefore conditioned on its own generation + /// ([`Self::claim_registered_target_for_removal`]) and removes nothing + /// then, answering the lost claim instead of the refusal. + pub(crate) async fn register_manifest_restore_target( + &self, + account_id: &str, + input: CreateTableInput, + registration: &RestoreRegistrationRow, + ) -> Result<(TableDescription, u64), StorageError> { + let backup_arn = registration.backup.backup_arn.clone(); + // The name is checked first (unfenced), so a restore into a CREATING + // restore target's name is refused as the name (TableBeingRestored), + // not as the backup, as on the other backends and the legacy path. + if let Some(source_backup_arn) = self + .creating_restore_target_under_name(account_id, &input.table_name) + .await? + { + return Err(StorageError::TableBeingRestored { + table_name: input.table_name.clone(), + backup_arn: source_backup_arn, + }); + } + let backup = self + .load_available_index_backup(account_id, &backup_arn) + .await?; + self.ensure_backup_not_restoring(&backup).await?; + + let target_name = input.table_name.clone(); + // The claim is the row itself: a fresh generation, written by the + // IF NOT EXISTS insert, fences every later write of this restore. + let generation = new_generation(); + #[allow(clippy::cast_possible_truncation)] + let restore_date_time_ms = (registration.restore_date_time * 1_000.0).round() as i64; + let columns = RestoreTargetColumns { + table_id: uuid::Uuid::new_v4().to_string(), + source_backup_arn: backup_arn.clone(), + source_table_arn: registration.source_table_arn.clone(), + restore_date_time_ms, + owner_generation: generation, + kind: extenddb_storage::backup_catalog::RestoreKind::Manifest, + store: Some(registration.store.clone()), + spool_host: registration.spool_host.clone(), + }; + let table_id = columns.table_id.clone(); + + // Step 1: the target row (IF NOT EXISTS reserves the name), its + // indexes and its data tables, exactly as `input` describes. + let description = match self + .create_table_for_restore_manifest(account_id, input, &columns) + .await + { + Ok(description) => description, + Err(StorageError::TableAlreadyExists(name)) => { + return Err(StorageError::TableAlreadyExists(name)); + } + Err(error) => { + if let Err(cleanup) = self + .drop_restore_target(account_id, &target_name, &table_id, &backup_arn) + .await + { + tracing::warn!( + "Restore of table {target_name}: removing the partly created \ + target failed ({cleanup}); the reconciler removes it" + ); + } + return Err(error); + } + }; + + // Step 2: register on the backup, fenced on AVAILABLE and on no other + // registration. A refusal caused only by stale registrations is + // retried once they are cleared, a bounded number of times. A test + // may hold here, between the target's creation and the fence, to + // move the backup to DELETING. + crate::test_hooks::before_restore_registration(&backup_arn).await; + let mut registered = Ok(false); + for _ in 0..REGISTRATION_ATTEMPTS { + registered = self + .register_restore(account_id, &backup_arn, &table_id, &target_name) + .await; + match registered { + Ok(true) => break, + Ok(false) => { + if let Err(refusal) = self + .registration_refusal_index(account_id, &backup_arn) + .await + { + registered = Err(refusal); + break; + } + } + Err(_) => break, + } } - if !extenddb_storage::sleep_or_shutdown(&shutdown, RESTORE_SWEEP_INTERVAL).await { - return; + if !matches!(registered, Ok(true)) { + // The removal is fenced on this registration's generation: a + // record a reconciler took over during the DDL is the + // successor's, and so is the target. + let refusal = registration_exhausted(registered, &backup_arn); + let owned = match self + .claim_registered_target_for_removal( + account_id, + &target_name, + &table_id, + generation, + &format!("the registration was refused: {refusal}"), + ) + .await + { + Ok(owned) => owned, + Err(error) => { + tracing::warn!( + "Restore of table {target_name}: claiming the unregistered target \ + for removal failed ({error}); the reconciler removes it" + ); + return Err(refusal); + } + }; + if !owned { + tracing::warn!( + "Restore of table {target_name} from {backup_arn}: the registration's \ + record was taken over during the target's creation; the target is left \ + to its new owner" + ); + return Err(Self::registration_claim_lost(&target_name, &backup_arn)); + } + if let Err(cleanup) = self + .drop_restore_target(account_id, &target_name, &table_id, &backup_arn) + .await + { + tracing::warn!( + "Restore of table {target_name}: removing the unregistered target \ + failed ({cleanup}); the reconciler removes it" + ); + } + return Err(refusal); } + + Ok((description, generation)) } } -impl BackupEngine for CassandraEngine { - fn create_backup( +impl LegacyBackupReader for CassandraEngine { + /// The in-catalog create: a CREATING `backups_by_arn` row owned by a + /// fresh generation and heartbeated while the items copy, the list rows, + /// then the AVAILABLE flip conditioned on the generation. The permit is + /// held until this future completes, which is when the copy ends. + fn create( &self, account_id: &str, table_name: &str, backup_name: &str, + permit: JobPermit, ) -> BoxFuture<'_, Result> { let account_id = account_id.to_owned(); let table_name = table_name.to_owned(); let backup_name = backup_name.to_owned(); Box::pin(async move { + let _permit = permit; Self::validate_account_id(&account_id)?; // fetch_table_key_info enforces the DynamoDB requirement that only @@ -1412,33 +1773,63 @@ impl BackupEngine for CassandraEngine { .transpose() .map_err(|e| StorageError::Internal(format!("Serialize stream: {e}")))?; + // The row is the claim: inserted CREATING under a fresh + // generation with a heartbeat on the coordinator's clock, and + // refreshed every JOB_HEARTBEAT_INTERVAL while the items copy. + let generation = new_generation(); let catalog = self.catalog_keyspace(); let insert_metadata = format!( - "INSERT INTO {catalog}.backups_by_arn (backup_arn, account_id, backup_name, table_id, table_name, table_arn, backup_status, backup_type, backup_size_bytes, item_count, key_schema, attribute_definitions, billing_mode, provisioned_throughput, stream_specification, table_created_at, created_at, definition) VALUES (?, ?, ?, ?, ?, ?, 'CREATING', 'USER', ?, 0, ?, ?, ?, ?, ?, ?, ?, ?) IF NOT EXISTS" + "INSERT INTO {catalog}.backups_by_arn (backup_arn, account_id, backup_name, table_id, table_name, table_arn, backup_status, backup_type, backup_size_bytes, item_count, key_schema, attribute_definitions, billing_mode, provisioned_throughput, stream_specification, table_created_at, created_at, definition, owner_generation, heartbeat_at) VALUES (?, ?, ?, ?, ?, ?, 'CREATING', 'USER', ?, 0, ?, ?, ?, ?, ?, ?, ?, ?, ?, toTimestamp(now())) IF NOT EXISTS" + ); + let inserted = crate::cassandra_util::query_lwt( + &self.session, + &insert_metadata, + cdrs_tokio::query_values!( + backup_arn.as_str(), + account_id.as_str(), + backup_name.as_str(), + table.table_id.as_str(), + table_name.as_str(), + table.table_arn.as_str(), + table.table_size_bytes, + key_schema.as_str(), + attribute_definitions.as_str(), + billing_mode, + provisioned_throughput.as_str(), + stream_specification.as_deref(), + table_created_at, + created_at, + definition_json.as_str(), + crate::backup_catalog::generation_column(generation) + ), + ) + .await + .map_err(|e| StorageError::Internal(format!("Create backup metadata: {e}")))?; + if !crate::cassandra_util::lwt_applied(&inserted)? { + return Err(StorageError::AlreadyExists(format!( + "backup {backup_arn} already has a row" + ))); + } + let heartbeat = { + let engine = self.clone(); + let (account, arn) = (account_id.clone(), backup_arn.clone()); + JobHeartbeat::spawn(&account_id, format!("Backup {backup_arn}"), move || { + let engine = engine.clone(); + let (account, arn) = (account.clone(), arn.clone()); + async move { + crate::backup_catalog::heartbeat_backup_row( + &engine, &account, &arn, generation, + ) + .await + } + }) + }; + // The row's generation-fenced removal, for a copy that fails + // while this process still owns the row. + let delete_metadata = format!( + "DELETE FROM {catalog}.backups_by_arn WHERE account_id = ? AND backup_arn = ? \ + IF owner_generation = ? AND backup_status = 'CREATING'" ); - self.session - .query_with_values( - &insert_metadata, - cdrs_tokio::query_values!( - backup_arn.as_str(), - account_id.as_str(), - backup_name.as_str(), - table.table_id.as_str(), - table_name.as_str(), - table.table_arn.as_str(), - table.table_size_bytes, - key_schema.as_str(), - attribute_definitions.as_str(), - billing_mode, - provisioned_throughput.as_str(), - stream_specification.as_deref(), - table_created_at, - created_at, - definition_json.as_str() - ), - ) - .await - .map_err(|e| StorageError::Internal(format!("Create backup metadata: {e}")))?; let account_keyspace = self.account_keyspace(&account_id); let insert_item = format!( @@ -1448,7 +1839,7 @@ impl BackupEngine for CassandraEngine { let mut payload_size = 0_i64; let mut start_key = None; - let snapshot_result: Result<(), StorageError> = async { + let snapshot_result: Result<(), CopyError> = async { loop { let (items, next_key) = self .scan_impl( @@ -1459,12 +1850,22 @@ impl BackupEngine for CassandraEngine { None, None, ) - .await?; + .await + .map_err(CopyError::Failed)?; for item in items { let item_data = serde_json::to_string(&item).map_err(|e| { - StorageError::Internal(format!("Serialize backup item: {e}")) + CopyError::Failed(StorageError::Internal(format!( + "Serialize backup item: {e}" + ))) })?; let bucket = item_count / BACKUP_ITEMS_PER_BUCKET; + // A bucket, not a page, is the unit of work between + // two polls: a copy that lost its row writes at most + // one bucket after the loss, into a payload whose + // owner is deleting bucket by bucket. + if item_count % BACKUP_ITEMS_PER_BUCKET == 0 && heartbeat.is_lost() { + return Err(CopyError::OwnershipLost); + } #[allow(clippy::cast_possible_truncation)] let bucket_i32 = bucket as i32; self.session @@ -1479,7 +1880,9 @@ impl BackupEngine for CassandraEngine { ) .await .map_err(|e| { - StorageError::Internal(format!("Write backup item: {e}")) + CopyError::Failed(StorageError::Internal(format!( + "Write backup item: {e}" + ))) })?; item_count += 1; #[allow(clippy::cast_possible_wrap)] @@ -1495,21 +1898,32 @@ impl BackupEngine for CassandraEngine { } .await; - if let Err(error) = snapshot_result { - let _ = self - .cleanup_backup_payload(&account_id, &backup_arn, item_count) + match snapshot_result { + Ok(()) => {} + Err(CopyError::OwnershipLost) => { + // The row is no longer this copy's to touch (see + // `claim_lost`); whoever owns it removes it and the + // payload. + drop(heartbeat); + return Err(claim_lost(&backup_arn)); + } + Err(CopyError::Failed(error)) => { + drop(heartbeat); + let _ = self + .cleanup_backup_payload(&account_id, &backup_arn, item_count) + .await; + let _ = crate::cassandra_util::query_lwt( + &self.session, + &delete_metadata, + cdrs_tokio::query_values!( + account_id.as_str(), + backup_arn.as_str(), + crate::backup_catalog::generation_column(generation) + ), + ) .await; - let delete_metadata = format!( - "DELETE FROM {catalog}.backups_by_arn \ - WHERE account_id = ? AND backup_arn = ? IF EXISTS" - ); - let _ = crate::cassandra_util::query_lwt( - &self.session, - &delete_metadata, - cdrs_tokio::query_values!(account_id.as_str(), backup_arn.as_str()), - ) - .await; - return Err(error); + return Err(error); + } } let backup_size_bytes = table.table_size_bytes.max(payload_size); @@ -1519,13 +1933,14 @@ impl BackupEngine for CassandraEngine { let insert_by_table = format!( "INSERT INTO {catalog}.backups_by_table (account_id, table_name, created_at, backup_arn, backup_name, table_id, table_arn, backup_size_bytes) VALUES (?, ?, ?, ?, ?, ?, ?, ?)" ); - let publish = format!( - "UPDATE {catalog}.backups_by_arn SET backup_status = 'AVAILABLE', backup_size_bytes = ?, item_count = ? WHERE account_id = ? AND backup_arn = ?" - ); - // Publication is a fixed three-statement operation. A logged batch - // prevents readers from observing list rows without the matching - // AVAILABLE authoritative row, or vice versa. - let publish_result: Result<(), StorageError> = async { + // The AVAILABLE flip is conditioned on the generation, so it is a + // lightweight transaction and cannot share a batch with the list + // rows (another partition). The list rows go first: a process + // that dies between the two leaves list rows naming a CREATING + // row, which `list` hides and the reconciler removes with the row + // once its heartbeat is stale, rather than an AVAILABLE backup no + // listing ever shows. + let list_rows: Result<(), StorageError> = async { let batch = BatchQueryBuilder::new() .with_consistency(Consistency::LocalQuorum) .add_query( @@ -1553,15 +1968,6 @@ impl BackupEngine for CassandraEngine { table.table_arn.as_str(), backup_size_bytes ), - ) - .add_query( - publish, - cdrs_tokio::query_values!( - backup_size_bytes, - item_count, - account_id.as_str(), - backup_arn.as_str() - ), ); self.session .batch( @@ -1574,25 +1980,60 @@ impl BackupEngine for CassandraEngine { Ok(()) } .await; - - if let Err(error) = publish_result { - if let Ok(backup) = self.load_backup(&account_id, &backup_arn).await { - let _ = self.remove_backup_index_rows(&backup).await; - } - let _ = self - .cleanup_backup_payload(&account_id, &backup_arn, item_count) - .await; - let delete_metadata = format!( - "DELETE FROM {catalog}.backups_by_arn \ - WHERE account_id = ? AND backup_arn = ? IF EXISTS" + let publish_result: Result<(), CopyError> = async { + list_rows.map_err(CopyError::Failed)?; + let publish = format!( + "UPDATE {catalog}.backups_by_arn SET backup_status = 'AVAILABLE', \ + backup_size_bytes = ?, item_count = ? WHERE account_id = ? AND backup_arn = ? \ + IF owner_generation = ? AND backup_status = 'CREATING'" ); - let _ = crate::cassandra_util::query_lwt( + let published = crate::cassandra_util::query_lwt( &self.session, - &delete_metadata, - cdrs_tokio::query_values!(account_id.as_str(), backup_arn.as_str()), + &publish, + cdrs_tokio::query_values!( + backup_size_bytes, + item_count, + account_id.as_str(), + backup_arn.as_str(), + crate::backup_catalog::generation_column(generation) + ), ) - .await; - return Err(error); + .await + .and_then(|result| crate::cassandra_util::lwt_applied(&result)) + .map_err(|e| { + CopyError::Failed(StorageError::Internal(format!("Publish backup: {e}"))) + })?; + if published { + Ok(()) + } else { + Err(CopyError::OwnershipLost) + } + } + .await; + drop(heartbeat); + + match publish_result { + Ok(()) => {} + Err(CopyError::OwnershipLost) => return Err(claim_lost(&backup_arn)), + Err(CopyError::Failed(error)) => { + if let Ok(backup) = self.load_backup(&account_id, &backup_arn).await { + let _ = self.remove_backup_index_rows(&backup).await; + } + let _ = self + .cleanup_backup_payload(&account_id, &backup_arn, item_count) + .await; + let _ = crate::cassandra_util::query_lwt( + &self.session, + &delete_metadata, + cdrs_tokio::query_values!( + account_id.as_str(), + backup_arn.as_str(), + crate::backup_catalog::generation_column(generation) + ), + ) + .await; + return Err(error); + } } Ok(BackupDetails { @@ -1606,7 +2047,7 @@ impl BackupEngine for CassandraEngine { }) } - fn describe_backup( + fn describe( &self, account_id: &str, backup_arn: &str, @@ -1620,7 +2061,7 @@ impl BackupEngine for CassandraEngine { }) } - fn list_backups( + fn list( &self, account_id: &str, table_name: Option<&str>, @@ -1673,7 +2114,7 @@ impl BackupEngine for CassandraEngine { }) } - fn delete_backup( + fn delete( &self, account_id: &str, backup_arn: &str, @@ -1692,9 +2133,13 @@ impl BackupEngine for CassandraEngine { let original_description = backup.description()?; // The claim is conditioned on the registration map being empty, on // the same partition the restore registers on (see the module - // docs), so a registered restore keeps the backup AVAILABLE. + // docs), so a registered restore keeps the backup AVAILABLE. The + // heartbeat is stamped with it, so a deletion this process does + // not finish is one `abandoned_creating_backups` reports once the + // stamp is stale, and a running one is left alone. let mark_deleting = format!( - "UPDATE {}.backups_by_arn SET backup_status = 'DELETING' \ + "UPDATE {}.backups_by_arn SET backup_status = 'DELETING', \ + heartbeat_at = toTimestamp(now()) \ WHERE account_id = ? AND backup_arn = ? \ IF backup_status = 'AVAILABLE' AND active_restores = null", self.catalog_keyspace() @@ -1702,6 +2147,7 @@ impl BackupEngine for CassandraEngine { let mut attempt = 0; while backup.backup_status == "AVAILABLE" { self.ensure_backup_not_restoring(&backup).await?; + crate::test_hooks::before_delete_claim(&backup_arn).await; let result = crate::cassandra_util::query_lwt( &self.session, &mark_deleting, @@ -1761,7 +2207,7 @@ impl BackupEngine for CassandraEngine { }) } - fn backup_restore_source( + fn restore_source( &self, account_id: &str, backup_arn: &str, @@ -1812,25 +2258,29 @@ impl BackupEngine for CassandraEngine { }) } - fn restore_table_from_backup( + fn restore( &self, account_id: &str, request: RestoreRequest, + permit: JobPermit, ) -> BoxFuture<'_, Result> { let account_id = account_id.to_owned(); Box::pin(async move { Self::validate_account_id(&account_id)?; // Registration and copy both run in one detached task that owns a - // clone of the engine and the request. The request future only - // waits for the registration outcome, so dropping it (a client - // timeout) at any await point after the first poll does not stop - // the restore; before the first poll nothing has been done. - // Contrast: if registration ran here and only the copy were + // clone of the engine, the request and the job permit (released + // when the task ends, which is when the copy ends). The request + // future only waits for the registration outcome, so dropping it + // (a client timeout) at any await point after the first poll does + // not stop the restore; before the first poll nothing has been + // done. Contrast: if registration ran here and only the copy were // spawned, a drop between creating the target and spawning would - // leave a CREATING target with no copy until the sweep removed it. + // leave a CREATING target with no copy until the reconciler + // removed it. let (registered_tx, registered_rx) = tokio::sync::oneshot::channel(); let engine = self.clone(); tokio::spawn(async move { + let _permit = permit; // Inside the task, like the registration, so the first poll // still spawns before any await. match engine.register_restore_target(&account_id, request).await { @@ -1850,6 +2300,78 @@ impl BackupEngine for CassandraEngine { })? }) } + + /// In-catalog rows whose owner is dead, for the reconciler's `delete`: + /// a DELETING row whose `heartbeat_at` is older than `stale_after` on the + /// coordinator's clock, a deletion its process did not finish; and a + /// CREATING row whose stamp is that old (or unset, a row from before the + /// heartbeat existed), fenced to DELETING here with a lightweight + /// transaction conditioned on the status and heartbeat observed and + /// setting a fresh `owner_generation`, so a create that beats in between + /// keeps its row and one that comes back later finds its claim lost at + /// its next heartbeat. DELETING is scanned first, so a row this pass + /// fences is reported once. `delete` finishes a DELETING row and refuses + /// a CREATING one, so a client's DeleteBackup of a backup still being + /// created stays "not found" while the reconciler's goes through. + fn abandoned_creating_backups( + &self, + stale_after: Duration, + ) -> BoxFuture<'_, Result, StorageError>> { + Box::pin(async move { + let catalog = self.catalog_keyspace(); + let now = crate::backup_catalog::coordinator_now_millis(self).await?; + #[allow(clippy::cast_possible_truncation, clippy::cast_possible_wrap)] + let cutoff = now - stale_after.as_millis() as i64; + let mut abandoned = Vec::new(); + for status in ["DELETING", "CREATING"] { + // A filtered scan of the backup rows: in-catalog backups + // mid-create or mid-delete are few and short-lived. + let query = format!( + "SELECT account_id, backup_arn, backup_status, store_kind, owner_generation, \ + heartbeat_at FROM {catalog}.backups_by_arn \ + WHERE backup_status = ? ALLOW FILTERING" + ); + let rows = crate::cassandra_util::query_rows_quorum::( + &self.session, + &query, + cdrs_tokio::query_values!(status), + "abandoned backups", + ) + .await?; + for row in rows { + let store_kind: Option = row.get_by_name("store_kind").ok().flatten(); + if store_kind.is_some() { + continue; + } + let heartbeat: Option = row.get_by_name("heartbeat_at").ok().flatten(); + if heartbeat.is_some_and(|at| at >= cutoff) { + continue; + } + let account_id: String = + crate::cassandra_util::get_column(&row, "account_id", "abandoned backups")?; + let backup_arn: String = + crate::cassandra_util::get_column(&row, "backup_arn", "abandoned backups")?; + if status == "CREATING" { + crate::test_hooks::before_abandoned_fence(&backup_arn).await; + let fenced = self + .fence_abandoned_create(&account_id, &backup_arn, heartbeat) + .await?; + if !fenced { + continue; + } + } + abandoned.push(AbandonedLegacyBackup { + account_id, + backup_arn, + // Reported, fenced to DELETING; the reconciler + // removes it through `delete`. + removed: false, + }); + } + } + Ok(abandoned) + }) + } } #[cfg(test)] @@ -1864,11 +2386,6 @@ mod tests { assert_eq!(bucket_count(BACKUP_ITEMS_PER_BUCKET + 1), 2); } - #[test] - fn a_restore_is_declared_abandoned_only_after_missed_heartbeats() { - assert!(RESTORE_STALE_AFTER >= RESTORE_HEARTBEAT_INTERVAL * 4); - } - /// Registration attempts that all lost to a restore just starting end in /// `BackupInUse` naming that, not in a missing backup; an error from the /// attempts themselves is reported as it is. @@ -1892,14 +2409,39 @@ mod tests { use std::sync::atomic::AtomicUsize; /// A heartbeat that counts its beats, every 10 ms, answering `owned`. - fn counting_heartbeat(beats: &Arc, owned: bool) -> RestoreHeartbeat { + fn counting_heartbeat(beats: &Arc, owned: bool) -> JobHeartbeat { let beats = Arc::clone(beats); - RestoreHeartbeat::spawn(Duration::from_millis(10), "T".to_owned(), move || { + JobHeartbeat::spawn_every(Duration::from_millis(10), "T".to_owned(), move || { beats.fetch_add(1, Ordering::SeqCst); async move { Ok(owned) } }) } + /// A heartbeat every 10 ms whose first `errors` beats fail and whose + /// later beats confirm the claim. + fn erring_heartbeat(beats: &Arc, errors: usize) -> JobHeartbeat { + let beats = Arc::clone(beats); + JobHeartbeat::spawn_every(Duration::from_millis(10), "T".to_owned(), move || { + let beat = beats.fetch_add(1, Ordering::SeqCst); + async move { + if beat < errors { + Err(StorageError::Internal("node unreachable".to_owned())) + } else { + Ok(true) + } + } + }) + } + + /// Wait until the heartbeat reports the claim lost; panic after 5 s. + async fn wait_for_loss(heartbeat: &JobHeartbeat) { + let deadline = std::time::Instant::now() + Duration::from_secs(5); + while !heartbeat.is_lost() { + assert!(std::time::Instant::now() < deadline, "loss never reported"); + tokio::time::sleep(Duration::from_millis(5)).await; + } + } + /// Wait until `beats` reaches `at_least`; panic after 5 s. async fn wait_for_beats(beats: &AtomicUsize, at_least: usize) { let deadline = std::time::Instant::now() + Duration::from_secs(5); @@ -1968,12 +2510,35 @@ mod tests { let beats = Arc::new(AtomicUsize::new(0)); let heartbeat = counting_heartbeat(&beats, false); wait_for_beats(&beats, 1).await; - let deadline = std::time::Instant::now() + Duration::from_secs(5); - while !heartbeat.is_lost() { - assert!(std::time::Instant::now() < deadline, "loss never reported"); - tokio::time::sleep(Duration::from_millis(5)).await; - } + wait_for_loss(&heartbeat).await; assert_stopped(&beats).await; assert_eq!(beats.load(Ordering::SeqCst), 1, "one beat, then stopped"); } + + /// Three errors in a row stop the heartbeat and report the claim lost; + /// two, followed by a confirmed beat, do not, and the heartbeat keeps + /// running. The literals pin the rule the design states (three), as the + /// direct test does, so a change to `HEARTBEAT_ERRORS_TO_STOP` fails here. + #[tokio::test] + async fn three_heartbeat_errors_in_a_row_stop_the_job_and_two_do_not() { + assert_eq!(HEARTBEAT_ERRORS_TO_STOP, 3); + + let beats = Arc::new(AtomicUsize::new(0)); + let heartbeat = erring_heartbeat(&beats, 3); + wait_for_loss(&heartbeat).await; + assert_stopped(&beats).await; + assert_eq!( + beats.load(Ordering::SeqCst), + 3, + "the third error in a row stopped the heartbeat" + ); + + let beats = Arc::new(AtomicUsize::new(0)); + let heartbeat = erring_heartbeat(&beats, 2); + wait_for_beats(&beats, 8).await; + assert!( + !heartbeat.is_lost(), + "two errors then a confirmed beat do not stop the job" + ); + } } diff --git a/crates/storage-cassandra/src/cassandra_util.rs b/crates/storage-cassandra/src/cassandra_util.rs index a4ebf5c0c..630f95bb7 100644 --- a/crates/storage-cassandra/src/cassandra_util.rs +++ b/crates/storage-cassandra/src/cassandra_util.rs @@ -147,6 +147,39 @@ pub async fn query_rows_quorum( Ok(body.into_rows().unwrap_or_default()) } +/// Execute a query at `LOCAL_SERIAL` and return all rows: the read sees +/// every Paxos round that committed, so a row an LWT wrote (a claim taken, +/// a heartbeat refreshed) is read as written. For the reads whose answer +/// decides whether another process's claim is live. +pub async fn query_rows_serial( + session: &Arc, + query: &str, + values: QueryValues, + context: &str, +) -> Result, E> { + use cdrs_tokio::consistency::Consistency; + use cdrs_tokio::statement::StatementParamsBuilder; + + let params = StatementParamsBuilder::new() + .with_consistency(Consistency::LocalSerial) + .with_values(values) + .build(); + let result = session + .query_with_params(query, params) + .await + .map_err(|e| { + tracing::error!("{context} serial query failed: {e}"); + E::db_error(format!("{context}: {e}")) + })?; + + let body = result.response_body().map_err(|e| { + tracing::error!("{context} response_body failed: {e}"); + E::db_error(format!("{context} response_body: {e}")) + })?; + + Ok(body.into_rows().unwrap_or_default()) +} + /// Execute a query and return the first row, if any. /// /// Similar to `sqlx::query().fetch_optional()`. diff --git a/crates/storage-cassandra/src/create_table.rs b/crates/storage-cassandra/src/create_table.rs index 024c74ce9..6758e71e8 100644 --- a/crates/storage-cassandra/src/create_table.rs +++ b/crates/storage-cassandra/src/create_table.rs @@ -9,11 +9,12 @@ use extenddb_core::types::{ ProvisionedThroughputDescription, RestoreSummary, SseDescription, SseType, TableDescription, TableStatus, }; +use extenddb_storage::backup_catalog::{RestoreKind, StoreIdentity}; use extenddb_storage::error::StorageError; use extenddb_storage::util::{index_arn, stream_arn, table_arn}; use crate::CassandraEngine; -use crate::backup_engine::RestoreHeartbeat; +use crate::backup_engine::JobHeartbeat; impl CassandraEngine { /// Initialize stream shards for a table atomically. @@ -101,8 +102,11 @@ impl CassandraEngine { } /// Create a restore target: CREATING, with no generic transition deadline, - /// carrying the restore registration columns. Only the restore copy (which - /// activates it) or the abandoned-restore sweep (which removes it) moves it + /// carrying the restore registration and its job record (RUNNING, kind + /// legacy, the claim's generation, a fresh heartbeat) in the one + /// `INSERT ... IF NOT EXISTS`, so no crash leaves a CREATING target + /// without a job record. Only the restore copy (which activates it) or + /// the reconciler (which removes it once its heartbeat is stale) moves it /// out of CREATING. Returns the target's heartbeat, started right after /// the row insert; dropping it stops it. pub(crate) async fn create_table_for_restore_impl( @@ -110,7 +114,7 @@ impl CassandraEngine { account_id: &str, input: CreateTableInput, restore: &RestoreTargetColumns, - ) -> Result<(TableDescription, RestoreHeartbeat), StorageError> { + ) -> Result<(TableDescription, JobHeartbeat), StorageError> { let (description, heartbeat) = self .create_table_impl_inner(account_id, input, Some(restore)) .await?; @@ -120,12 +124,31 @@ impl CassandraEngine { Ok((description, heartbeat)) } + /// Create a store-backed (manifest) restore target: CREATING, carrying + /// the `restore_kind = 'manifest'` job record and the store columns in + /// the one `INSERT ... IF NOT EXISTS`. No self-managed [`JobHeartbeat`] + /// is started: the engine owns the manifest restore's heartbeat through + /// the [`extenddb_storage::backup_catalog::JobClaim`] this registration + /// returns. The target's DDL that follows the insert is bounded by a + /// few schema statements, inside the engine's registration timeout. + pub(crate) async fn create_table_for_restore_manifest( + &self, + account_id: &str, + input: CreateTableInput, + restore: &RestoreTargetColumns, + ) -> Result { + let (description, _heartbeat) = self + .create_table_impl_inner(account_id, input, Some(restore)) + .await?; + Ok(description) + } + async fn create_table_impl_inner( &self, account_id: &str, input: CreateTableInput, restore: Option<&RestoreTargetColumns>, - ) -> Result<(TableDescription, Option), StorageError> { + ) -> Result<(TableDescription, Option), StorageError> { let schedule_activation = restore.is_none(); Self::validate_account_id(account_id)?; @@ -220,9 +243,9 @@ impl CassandraEngine { } // Insert table metadata with LWT (IF NOT EXISTS). Every catalog this - // release serves has the restore columns (catalog migration 1 creates - // them). Only a restore target has values for them, so only its insert - // names them, and an ordinary table's row leaves them null. + // release serves has the restore columns (catalog migrations 1 and 6 + // create them). Only a restore target has values for them, so only + // its insert names them, and an ordinary table's row leaves them null. let mut columns = String::from( "account_id, table_name, table_id, table_arn, key_schema, \ attribute_definitions, billing_mode, provisioned_throughput, stream_specification, \ @@ -262,20 +285,44 @@ impl CassandraEngine { Value::from(deletion_protection), Value::from(status_transition_at), ]; + let mut placeholders = vec!["?"; values.len()]; if let Some(r) = restore { + // The job record: RUNNING under the claim's generation, the + // restore kind, the heartbeat stamped on the coordinator's clock. columns.push_str( ", restore_source_backup_arn, restore_source_table_arn, restore_date_time, \ - restore_owner, restore_heartbeat", + restore_status, restore_kind, restore_owner_generation, restore_heartbeat_at", ); values.extend([ Value::from(r.source_backup_arn.as_str()), Value::from(r.source_table_arn.as_str()), Value::from(r.restore_date_time_ms), - Value::from(r.owner), - Value::from(creation_timestamp), + Value::from("RUNNING"), + Value::from(r.kind.as_str()), + Value::from(crate::backup_catalog::generation_column(r.owner_generation)), ]); + placeholders.extend(["?"; 6]); + placeholders.push("toTimestamp(now())"); + // A store-backed restore records the store it reads and the + // claimant's spool, so the reconciler can resume it. + if let Some(store) = &r.store { + columns.push_str( + ", restore_store_kind, restore_store_root, restore_store_id, \ + restore_spool_host", + ); + values.extend([ + Value::from(store.store_kind.as_str()), + Value::from(store.store_root.as_str()), + Value::from(store.store_id.as_str()), + match r.spool_host.as_deref() { + Some(host) => Value::from(host), + None => Value::NotSet, + }, + ]); + placeholders.extend(["?"; 4]); + } } - let placeholders = vec!["?"; values.len()].join(", "); + let placeholders = placeholders.join(", "); let insert_table_cql = format!( "INSERT INTO {catalog_keyspace}.tables ({columns}) VALUES ({placeholders}) IF NOT EXISTS" ); @@ -327,10 +374,17 @@ impl CassandraEngine { } } - // A restore target's heartbeat runs from here, before the DDL below. - // An error return from this function drops it, which stops it. - let heartbeat = restore.map(|r| { - self.start_restore_heartbeat(account_id, &input.table_name, &r.table_id, r.owner) + // A legacy restore target's heartbeat runs from here, before the DDL + // below, and is dropped (stopped) on an error return. A store-backed + // (manifest) restore target has no self-managed heartbeat: the engine + // heartbeats it through the claim the registration returns. + let heartbeat = restore.filter(|r| r.kind == RestoreKind::Legacy).map(|r| { + self.start_restore_heartbeat( + account_id, + &input.table_name, + &r.table_id, + r.owner_generation, + ) }); // Insert GSI metadata @@ -657,6 +711,19 @@ pub(crate) struct RestoreTargetColumns { pub(crate) source_table_arn: String, /// `RestoreSummary.RestoreDateTime`, epoch milliseconds. pub(crate) restore_date_time_ms: i64, - /// The engine instance running the copy; see `CassandraEngine::restore_owner`. - pub(crate) owner: uuid::Uuid, + /// The claim's fencing token (`tables.restore_owner_generation`): every + /// write the copy makes to the target row is conditioned on it. + pub(crate) owner_generation: u64, + /// `legacy` for an in-catalog restore (whose copy reads `backup_items`), + /// `manifest` for a store-backed one (whose copy the engine streams from + /// the store). A legacy target carries a self-managed heartbeat; a + /// manifest target is heartbeated by the engine's claim. + pub(crate) kind: RestoreKind, + /// The store a manifest restore reads its backup from; `None` on a legacy + /// restore, which reads the catalog. + pub(crate) store: Option, + /// `:` of the instance downloading the backup into + /// its spool; `None` for a filesystem store (read in place) and on a + /// legacy restore. + pub(crate) spool_host: Option, } diff --git a/crates/storage-cassandra/src/data/put_get_item.rs b/crates/storage-cassandra/src/data/put_get_item.rs index b74db89dc..c626383ef 100644 --- a/crates/storage-cassandra/src/data/put_get_item.rs +++ b/crates/storage-cassandra/src/data/put_get_item.rs @@ -80,6 +80,103 @@ impl CassandraEngine { .map(|_| ()) } + /// The write a store-backed restore makes for one backup item, at the + /// write timestamp `stamp` (microseconds since the epoch, the domain of + /// `version`): the base row and every GSI and LSI row of the item in one + /// logged batch carrying `USING TIMESTAMP stamp`, with no lightweight + /// transaction and no read before it. A CREATING restore target has no + /// writer but its restore job, whose writes are idempotent upserts of the + /// backup's items, so the put path's version fence has nothing to + /// serialize against; what the timestamp buys is the fence the restore + /// needs instead (`restore_engine`'s module doc): a job reads `stamp` + /// before it checks its claim, so a batch it lands late, after a takeover, + /// carries a timestamp below every write and delete the successor made, + /// and Cassandra keeps the later cell. `version` is set to `stamp`, so a + /// later ordinary write's `next_row_version` is above it and its index + /// rows resolve in order (`data::version`). `indexes` are the target's, + /// read once per data file by the caller, each with its propagation delay + /// zeroed so every GSI is written in the batch as LSIs always are. + pub(crate) async fn put_restored_item_at( + &self, + key_info: &TableKeyInfo, + indexes: &[super::index::IndexMeta], + item: &Item, + stamp: i64, + ) -> Result<(), StorageError> { + let data_keyspace = self.account_keyspace(&key_info.account_id); + let ddb_table = data_table_name(&key_info.table_id); + let pk_text = composite_pk_to_text(item, &key_info.key_schema)?; + let item_text = serde_json::to_value(item) + .map_err(|e| StorageError::Internal(e.to_string()))? + .to_string(); + if !indexes.is_empty() { + extenddb_core::validation::validate_index_keys( + item, + &super::index::index_key_refs(indexes), + &key_info.attribute_definitions, + ) + .map_err(|e| StorageError::Validation(e.to_string()))?; + } + let (cql, values) = if let Some((sk_name, sk_type)) = + sk_info(&key_info.key_schema, &key_info.attribute_definitions) + { + let sk_value = item + .get(sk_name) + .ok_or_else(|| StorageError::Internal("missing sort key".to_owned()))?; + let sk = parse_sk(sk_value, sk_type)?; + let sk_col = sk_column(sk_type); + ( + format!( + "UPDATE {data_keyspace}.{ddb_table} SET item_data = ?, version = ? \ + WHERE pk = ? AND {sk_col} = ?" + ), + vec![ + item_text.into(), + stamp.into(), + cdrs_tokio::types::value::Value::from(pk_text.as_str()), + super::index::sk_to_value(&sk), + ], + ) + } else { + ( + format!( + "UPDATE {data_keyspace}.{ddb_table} SET item_data = ?, version = ? WHERE pk = ?" + ), + vec![ + item_text.into(), + stamp.into(), + cdrs_tokio::types::value::Value::from(pk_text.as_str()), + ], + ) + }; + // The batch carries the timestamp, so the index statements pass none + // of their own (Cassandra refuses a batch with both). + let mut batch = BatchQueryBuilder::new() + .with_consistency(Consistency::LocalQuorum) + .with_timestamp(stamp) + .add_query(cql, cdrs_tokio::query::QueryValues::SimpleValues(values)); + if !indexes.is_empty() { + super::index::sync_indexes_at( + &mut batch, + &data_keyspace, + &key_info.key_schema, + &key_info.attribute_definitions, + indexes, + None, + Some(item), + 0, + None, + )?; + } + let built = batch + .build() + .map_err(|error| StorageError::Internal(error.to_string()))?; + self.session.batch(built).await.map_err(|error| { + StorageError::Internal(format!("put_restored_item_at batch: {error}")) + })?; + Ok(()) + } + #[allow(clippy::too_many_arguments)] async fn put_item_with_retry( &self, diff --git a/crates/storage-cassandra/src/delete_table.rs b/crates/storage-cassandra/src/delete_table.rs index 4a62507fc..052bc4e92 100644 --- a/crates/storage-cassandra/src/delete_table.rs +++ b/crates/storage-cassandra/src/delete_table.rs @@ -15,8 +15,7 @@ pub(crate) enum DeleteTableCaller<'a> { /// DeleteTable: refuses a deletion-protected table and a restore target /// still being restored. Client, - /// The restore copy or the abandoned-restore sweep removing a restore - /// target. Deletes only the incarnation with this `table_id`, so a target + /// The restore copy or the reconciler removing a restore target. Deletes only the incarnation with this `table_id`, so a target /// that was already removed and whose name was reused is left alone. RestoreCleanup { table_id: &'a str }, } @@ -92,7 +91,7 @@ impl CassandraEngine { } // A restore target is owned by its copy until the copy - // activates it (or the sweep removes it). Deleting it mid-copy + // activates it (or the reconciler removes it). Deleting it mid-copy // would race the copy's writes into dropped tables. let status: Option = table_row.get_by_name("table_status").ok().flatten(); let restoring_from: Option = table_row diff --git a/crates/storage-cassandra/src/engine.rs b/crates/storage-cassandra/src/engine.rs index ce410e752..474e32043 100644 --- a/crates/storage-cassandra/src/engine.rs +++ b/crates/storage-cassandra/src/engine.rs @@ -98,11 +98,6 @@ pub struct CassandraEngine { /// a restart merely restarts the traversal from the front. pub(crate) ttl_repair_scan_cursors: Arc>>, - - /// Identifies this engine instance as the owner of the restore copies it - /// runs (`tables.restore_owner`). Random per process start, so a restore - /// left behind by a crashed process is never mistaken for a live one. - pub(crate) restore_owner: uuid::Uuid, } /// Cap on concurrently in-flight detached TTL claim releases. @@ -159,7 +154,6 @@ impl CassandraEngine { ttl_repair_scan_cursors: Arc::new(std::sync::Mutex::new( std::collections::HashMap::new(), )), - restore_owner: uuid::Uuid::new_v4(), }) } diff --git a/crates/storage-cassandra/src/lib.rs b/crates/storage-cassandra/src/lib.rs index 46fe89cda..02e8092cc 100644 --- a/crates/storage-cassandra/src/lib.rs +++ b/crates/storage-cassandra/src/lib.rs @@ -14,6 +14,7 @@ pub(crate) const DEFAULT_CONTROL_PLANE_DELAY_SECONDS: f64 = 0.25; mod admin_store; mod authorization_store; +mod backup_catalog; mod backup_engine; pub mod bootstrapper; pub mod cassandra_util; @@ -30,17 +31,20 @@ mod metadata_engine; pub mod migrations; pub mod operations; pub(crate) mod propagation_hold; +mod restore_engine; mod snapshot_engine; mod stream_engine; pub mod stream_util; pub mod table_engine; mod table_helpers; +#[doc(hidden)] +pub mod test_hooks; pub mod ttl_worker; mod update_table; mod worker_store; pub mod workers; -pub use backup_engine::poll_abandoned_restores; +pub use backup_catalog::heartbeat_backup_row; pub use bootstrapper::CassandraBootstrapper; pub use catalog_store::CassandraCatalogStore; pub use config::CassandraStorageConfig; @@ -158,16 +162,6 @@ impl ServerRuntimeHooks for CassandraRuntimeHooks { .await }); - // Removes restore targets whose copy stopped heartbeating (the - // process running it died): once now, then periodically until - // shutdown. - let restore_sweep_engine = self.engine.clone(); - let restore_sweep_shutdown = ctx.shutdown.clone(); - let restore_sweep = tokio::spawn(async move { - backup_engine::poll_abandoned_restores(restore_sweep_engine, restore_sweep_shutdown) - .await; - }); - let mut handles = vec![ control_plane, transaction_recovery, @@ -176,7 +170,6 @@ impl ServerRuntimeHooks for CassandraRuntimeHooks { ttl_cleanup, ttl_repair, ttl_audit, - restore_sweep, ]; handles.extend(gsi_propagation); handles @@ -323,11 +316,14 @@ fn server_components_factory( .instance_id .clone() .unwrap_or_else(|| "unknown".to_owned()); - let registration = migrations::register_server( + // Every caller registers, a command too, so a migration does not run + // under one; a command's registration carries its own name. + let registration = migrations::register_server_as( status_session, &cassandra_config.keyspace_prefix, &host, options.release, + options.command.unwrap_or(migrations::SERVE_COMMAND), ) .await .map_err(BackendError::Catalog)?; @@ -347,14 +343,19 @@ fn server_components_factory( let control_plane_notify = engine.control_plane_notify.clone(); let engine = Arc::new(engine); - match engine.process_control_plane_transitions().await { - Ok(ref t) if t.is_empty() => {} - Ok(transitions) => { - for (name, transition) in &transitions { - tracing::info!("Recovered table '{name}': {transition}"); + // The startup recovery is a server's; a command reading beside one, + // or one that writes beside the live servers (`backup purge`, + // `options.startup_recovery` off), leaves it to the server. + if !options.beside_server && options.startup_recovery { + match engine.process_control_plane_transitions().await { + Ok(ref t) if t.is_empty() => {} + Ok(transitions) => { + for (name, transition) in &transitions { + tracing::info!("Recovered table '{name}': {transition}"); + } } + Err(e) => tracing::error!("Failed to recover control plane transitions: {e}"), } - Err(e) => tracing::error!("Failed to recover control plane transitions: {e}"), } // Load encryption key from catalog. diff --git a/crates/storage-cassandra/src/migrations/backend.rs b/crates/storage-cassandra/src/migrations/backend.rs index d30aee1aa..0d59590b5 100644 --- a/crates/storage-cassandra/src/migrations/backend.rs +++ b/crates/storage-cassandra/src/migrations/backend.rs @@ -41,11 +41,24 @@ use crate::catalog_store::CassandraCatalogStore; /// The handle code bodies receive: the session, the keyspace names and the /// catalog's management store (migration 2 rewrites policies through it), /// every call of which is bounded by the request timeout and, under the -/// lock, checked against the lease. +/// lock, checked against the lease. A body that sends requests on +/// [`Self::session`] itself checks [`Self::check`] before each write and +/// executes CQL files through [`Self::run_file`], as the runner does. pub struct CassandraStores { pub(crate) session: Arc, pub(crate) keyspace_prefix: String, management: LeasedManagementStore, + /// The lease of the lock the run holds, or works under. + lease: Option, + /// The test hook's request budget: `None` outside the tests. + stop_after: Option>, +} + +/// The error a write meets once the test hook's budget is spent. +fn stopped_by_hook() -> MigrationError { + MigrationError::Store( + "stopped by the test hook before a write, as a killed migrator stops".to_owned(), + ) } impl CassandraStores { @@ -57,16 +70,88 @@ impl CassandraStores { Self { management: LeasedManagementStore::new( CassandraCatalogStore::catalog_only(session.clone(), keyspace_prefix.to_owned()), - lease, + lease.clone(), ), session, keyspace_prefix: keyspace_prefix.to_owned(), + lease, + stop_after: None, } } pub(crate) fn catalog_keyspace(&self) -> String { format!("{}_catalog", self.keyspace_prefix) } + + /// Stop before a write when the lease's deadline has passed, or when + /// the test hook's budget is spent. + /// + /// # Errors + /// + /// The lease may have lapsed, or the hook stopped the run. + pub(crate) fn check(&self) -> Result<(), MigrationError> { + if stopped(self.stop_after.as_deref()) { + return Err(stopped_by_hook()); + } + self.lease.as_ref().map_or(Ok(()), LeaseHandle::check) + } + + /// Record that a request completed, which keeps the lease's renewal + /// going. + pub(crate) fn progress(&self) { + if let Some(lease) = &self.lease { + lease.progress(); + } + } + + /// The bound on a schema statement: shorter than the lease's margin when + /// the run holds or works under a lease (see [`ddl_bound`]). + pub(crate) fn ddl_timeout(&self) -> std::time::Duration { + ddl_bound(self.lease.is_some()) + } + + /// Wait for schema agreement, as progress of a leased run. + pub(crate) async fn agree(&self) -> Result<(), MigrationError> { + wait_for_agreement_with(&self.session, AGREEMENT_TIMEOUT, || self.progress()).await?; + self.progress(); + Ok(()) + } + + /// Execute file `name`'s statements in `keyspace`, each with its table + /// qualified by the keyspace, checking the lease before each one, + /// recording progress when each completes, and waiting for schema + /// agreement after each DDL statement: a file body's run, and a code + /// body's that executes a file. + /// + /// # Errors + /// + /// A statement the runner cannot qualify (before any runs), the lease + /// lapsed, or a statement or the agreement wait failed. + pub(crate) async fn run_file( + &self, + keyspace: &str, + name: &str, + text: &str, + ) -> Result<(), MigrationError> { + for statement in statements(keyspace, name, text)? { + self.check()?; + send_within( + &self.session, + &statement.text, + cdrs_tokio::query_values!(), + Consistency::Quorum, + None, + &format!("migration file {name} in {keyspace}"), + self.ddl_timeout(), + ) + .await?; + self.progress(); + if statement.ddl { + self.agree().await?; + } + } + Ok(()) + } } impl HasManagementStore for CassandraStores { @@ -149,9 +234,7 @@ impl CassandraMigrations<'_> { /// test hook's budget is spent. fn check(&self) -> Result<(), MigrationError> { if stopped(self.stop_after.as_deref()) { - return Err(MigrationError::Store( - "stopped by the test hook before a write, as a killed migrator stops".to_owned(), - )); + return Err(stopped_by_hook()); } match (self.lock, &self.lease) { (Some(lock), _) => lock.check(), @@ -201,30 +284,6 @@ impl CassandraMigrations<'_> { } } - /// Execute a file body's statements in `keyspace`, checking the lock - /// before each one, recording progress when each completes, and waiting - /// for schema agreement after each DDL statement. - async fn run_file(&self, keyspace: &str, name: &str, text: &str) -> Result<(), MigrationError> { - for statement in statements(keyspace, name, text)? { - self.check()?; - send_within( - self.session(), - &statement.text, - cdrs_tokio::query_values!(), - Consistency::Quorum, - None, - &format!("migration file {name} in {keyspace}"), - self.ddl_timeout(), - ) - .await?; - self.progress(); - if statement.ddl { - self.agree().await?; - } - } - Ok(()) - } - /// Create a missing account keyspace with the catalog keyspace's /// replication, read from `system_schema`: an `accounts` row whose /// keyspace a stopped CreateAccount never created (Cassandra item 5). @@ -461,7 +520,7 @@ impl MigrationBackend for CassandraMigrations<'_> { } let done = match step { Step::File { name, text } => { - self.run_file(&store.name, name, text).await?; + self.stores.run_file(&store.name, name, text).await?; Done::default() } Step::Code(body) => { diff --git a/crates/storage-cassandra/src/migrations/backend/constructors.rs b/crates/storage-cassandra/src/migrations/backend/constructors.rs index 65f175a1b..bb65652e6 100644 --- a/crates/storage-cassandra/src/migrations/backend/constructors.rs +++ b/crates/storage-cassandra/src/migrations/backend/constructors.rs @@ -110,13 +110,15 @@ impl<'l> CassandraMigrations<'l> { /// Stop this run before its `requests`-th write, as a migrator killed /// there stops: for the crash-and-rerun tests. Every write the backend /// sends counts (a file's statements, the ledger table and its rows, the - /// version row, an account keyspace, the start of a code body) and so does - /// each policy write of migration 2's body; reads do not. + /// version row, an account keyspace, the start of a code body), and so + /// do each policy write of migration 2's body and each write migration + /// 6's body sends; reads do not. #[cfg(any(test, feature = "test-util"))] #[must_use] pub fn stop_after(mut self, requests: usize) -> Self { let left = Arc::new(std::sync::atomic::AtomicUsize::new(requests)); self.stores.management.stop_after(Arc::clone(&left)); + self.stores.stop_after = Some(Arc::clone(&left)); self.stop_after = Some(left); self } diff --git a/crates/storage-cassandra/src/migrations/m0006_backup_catalog_index.rs b/crates/storage-cassandra/src/migrations/m0006_backup_catalog_index.rs new file mode 100644 index 000000000..3593d41a1 --- /dev/null +++ b/crates/storage-cassandra/src/migrations/m0006_backup_catalog_index.rs @@ -0,0 +1,200 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Migration 6, `backup_catalog_index`, Cassandra: the catalog index of +//! backups kept in a backup store (columns of `backups_by_arn` and the +//! `backup_index_by_status` table), the restore job record on the restore +//! registrations (`tables.restore_*`), the maintenance claim record +//! (`backup_maintenance`) and the catalog id (`settings.catalog_id`). +//! +//! The body executes `0006_backup_catalog_index.cql` as the runner executes +//! a file body, then backfills the registrations that exist and writes the +//! catalog id: CQL has no `UPDATE ... WHERE` on a non-key predicate, and the +//! id is a random value. Every statement of the file is `IF NOT EXISTS`, each +//! backfill write is a lightweight transaction conditioned on the job status +//! being unset, and the id is inserted `IF NOT EXISTS`, so a second run +//! changes nothing; a run stopped part way runs again from the start. +//! +//! The backfill: every registration that exists was written by a release +//! without the job record, so each is a legacy restore. One whose table is +//! not CREATING finished (DONE, completed when the table was created). One +//! whose table is CREATING is a restore the upgrade's stop interrupted: +//! RUNNING, with a heartbeat at the epoch (not null, so the staleness test +//! selects it) and an owner generation no job holds (0), so the first +//! reconciler pass fails it and removes its target. +//! +//! The checksum test pins this file together with the CQL file (`SOURCE`) +//! once the migration is released. The module is excluded from rustfmt and +//! clippy so neither can require an edit. + +use async_trait::async_trait; +use cdrs_tokio::consistency::Consistency; +use extenddb_storage::migrations::{CodeBody, Done, MigrationError}; + +use super::CassandraStores; +use super::connect::{column, nullable, rows, send}; + +/// The file this body executes first. +pub(crate) const FILE_NAME: &str = "0006_backup_catalog_index.cql"; + +/// The text of [`FILE_NAME`]. +pub(crate) const FILE: &str = + extenddb_storage::file_body!("migrations/catalog", "0006_backup_catalog_index.cql"); + +/// This module and the file it executes, whose SHA-256 the checksum test +/// pins. +pub(crate) const SOURCE: &str = concat!( + include_str!("m0006_backup_catalog_index.rs"), + extenddb_storage::file_body!("migrations/catalog", "0006_backup_catalog_index.cql") +); + +/// Recorded as this migration's checksum. Changes only with a new migration. +const FINGERPRINT: &str = "0006_backup_catalog_index/cassandra/1"; + +/// Migration 6's catalog body. +pub(crate) struct BackupCatalogIndex; + +#[async_trait] +impl CodeBody for BackupCatalogIndex { + fn fingerprint(&self) -> &'static str { + FINGERPRINT + } + + /// Nothing this migration finds stops it. + async fn detect(&self, _stores: &CassandraStores) -> Result, MigrationError> { + Ok(Vec::new()) + } + + async fn run(&self, stores: &CassandraStores) -> Result { + let catalog = stores.catalog_keyspace(); + stores.run_file(&catalog, FILE_NAME, FILE).await?; + let mut notes = Vec::new(); + let (done, running) = backfill(stores, &catalog).await?; + if done + running > 0 { + notes.push(format!( + "backfilled {} restore registration(s) as legacy restores, {running} of them \ + RUNNING with a dead owner for the reconciler to remove", + done + running + )); + } + if write_catalog_id(stores, &catalog).await? { + notes.push("wrote settings.catalog_id".to_owned()); + } + Ok(Done { notes }) + } +} + +/// Give every `tables` row that carries a restore registration without a job +/// status its job record; returns how many became DONE and how many RUNNING. +/// A row whose write does not apply (its status was set, or its table was +/// replaced, since the read) is left as it is. +async fn backfill(stores: &CassandraStores, catalog: &str) -> Result<(u32, u32), MigrationError> { + const CONTEXT: &str = "backfill the restore registrations"; + let found = rows( + &stores.session, + &format!( + "SELECT account_id, table_name, table_id, table_status, created_at, \ + restore_source_backup_arn, restore_status FROM {catalog}.tables" + ), + cdrs_tokio::query_values!(), + Consistency::Quorum, + CONTEXT, + ) + .await?; + stores.progress(); + let (mut done, mut running) = (0_u32, 0_u32); + for row in found { + let source: Option = nullable(&row, "restore_source_backup_arn", CONTEXT)?; + let status: Option = nullable(&row, "restore_status", CONTEXT)?; + if source.is_none() || status.is_some() { + continue; + } + let account_id: String = column(&row, "account_id", CONTEXT)?; + let table_name: String = column(&row, "table_name", CONTEXT)?; + let table_id: String = column(&row, "table_id", CONTEXT)?; + let table_status: Option = nullable(&row, "table_status", CONTEXT)?; + let created_at: i64 = nullable(&row, "created_at", CONTEXT)?.unwrap_or(0); + let interrupted = table_status.as_deref() == Some("CREATING"); + let (update, values) = if interrupted { + ( + format!( + "UPDATE {catalog}.tables SET restore_status = 'RUNNING', \ + restore_kind = 'legacy', restore_owner_generation = 0, \ + restore_heartbeat_at = 0 WHERE account_id = ? AND table_name = ? \ + IF table_id = ? AND restore_status = null" + ), + cdrs_tokio::query_values!(account_id.as_str(), table_name.as_str(), table_id.as_str()), + ) + } else { + ( + format!( + "UPDATE {catalog}.tables SET restore_status = 'DONE', \ + restore_kind = 'legacy', restore_owner_generation = 0, \ + restore_heartbeat_at = 0, restore_completed_at = ? \ + WHERE account_id = ? AND table_name = ? \ + IF table_id = ? AND restore_status = null" + ), + cdrs_tokio::query_values!( + created_at, + account_id.as_str(), + table_name.as_str(), + table_id.as_str() + ), + ) + }; + stores.check()?; + if applied(stores, &update, values, CONTEXT).await? { + if interrupted { + running += 1; + } else { + done += 1; + } + } + } + Ok((done, running)) +} + +/// Insert `settings.catalog_id`, a random id, unless the row exists; returns +/// whether this run wrote it. +async fn write_catalog_id(stores: &CassandraStores, catalog: &str) -> Result { + let id = uuid::Uuid::new_v4().to_string(); + stores.check()?; + applied( + stores, + &format!( + "INSERT INTO {catalog}.settings (key, value) VALUES ('catalog_id', ?) IF NOT EXISTS" + ), + cdrs_tokio::query_values!(id.as_str()), + "write settings.catalog_id", + ) + .await +} + +/// Send a lightweight transaction at `QUORUM` and `SERIAL` and return its +/// `[applied]` column. +async fn applied( + stores: &CassandraStores, + cql: &str, + values: cdrs_tokio::query::QueryValues, + context: &str, +) -> Result { + let envelope = send( + &stores.session, + cql, + values, + Consistency::Quorum, + Some(Consistency::Serial), + context, + ) + .await?; + stores.progress(); + let found = envelope + .response_body() + .map_err(|e| MigrationError::store(context, e))? + .into_rows() + .ok_or_else(|| MigrationError::Store(format!("{context}: the answer holds no rows")))?; + let row = found + .first() + .ok_or_else(|| MigrationError::Store(format!("{context}: no [applied] row")))?; + column(row, "[applied]", context) +} diff --git a/crates/storage-cassandra/src/migrations/mod.rs b/crates/storage-cassandra/src/migrations/mod.rs index aae33b64f..8818dcd3c 100644 --- a/crates/storage-cassandra/src/migrations/mod.rs +++ b/crates/storage-cassandra/src/migrations/mod.rs @@ -26,6 +26,11 @@ mod connect; mod cql; mod guarded; mod locks; +// Frozen once released: neither rustfmt nor a later clippy lint may require +// an edit that changes its checksum. +#[rustfmt::skip] +#[allow(clippy::all, deprecated)] +mod m0006_backup_catalog_index; pub mod postcheck; mod registrar; mod registrations; @@ -52,7 +57,7 @@ pub(crate) use connect::{LOCKED_DDL_TIMEOUT, REQUEST_TIMEOUT}; pub(crate) use locks::LeaseHandle; pub use locks::{LeaseTimes, MigrationLock}; pub use registrar::ServerRegistration; -pub use registrations::{RegistrationTimes, ServingRegistration}; +pub use registrations::{RegistrationTimes, SERVE_COMMAND, ServingRegistration}; use crate::cassandra_util::CassandraSession; @@ -127,6 +132,29 @@ pub fn migration(id: MigrationId) -> Migration { "0005_drop_continuous_backups.cql" )], }, + // Breaking: a binary without it neither reads the index nor writes + // the job record on the restore registrations it creates, so a + // restore it ran would be owned by nobody. Convergent: every + // statement is `IF NOT EXISTS`, the backfill writes only a + // registration whose job status is unset, which no registration + // this release writes is (each is inserted with its job record), and + // the catalog id is inserted only when absent; so a second run over + // data written since changes nothing but a registration a release + // before it wrote, which is the repair `--reconverge` is for. + MigrationId::BackupCatalogIndex => Migration::Applies { + kind: Kind::Breaking, + convergent: true, + bodies: vec![StoreBody::code( + StoreKind::Catalog, + &m0006_backup_catalog_index::BackupCatalogIndex, + m0006_backup_catalog_index::SOURCE, + )], + }, + MigrationId::RestoreProgress => Migration::NotApplicable( + "a store-backed restore records its progress on the target's catalog tables row \ + (restore_file_index, restore_items_applied, which migration 6 adds); the \ + restore_progress table is PostgreSQL's data database's", + ), } } @@ -273,13 +301,39 @@ pub async fn migrate_catalog_keyspace( keyspace_prefix: &str, release: &str, timeout: std::time::Duration, +) -> Result { + migrate_catalog_keyspace_through( + session, + keyspace_prefix, + release, + timeout, + MigrationId::LATEST, + ) + .await +} + +/// [`migrate_catalog_keyspace`] stopping after migration `through`: for +/// tests of a catalog a later migration has not reached. +/// +/// # Errors +/// +/// As [`migrate_catalog_keyspace`]. +#[cfg(any(test, feature = "test-util"))] +pub async fn migrate_catalog_keyspace_through( + session: Arc, + keyspace_prefix: &str, + release: &str, + timeout: std::time::Duration, + through: MigrationId, ) -> Result { let catalog = format!("{keyspace_prefix}_catalog"); create_lock_table(&session, &catalog).await?; let lock = MigrationLock::acquire(session, &catalog, LeaseTimes::DEFAULT, timeout).await?; let result = { let migrations = CassandraMigrations::locked_catalog_only(&lock, keyspace_prefix); - Runner::new(&migrations, release).migrate().await + Runner::new(&migrations, release) + .migrate_through(through) + .await }; lock.release().await; result @@ -671,11 +725,31 @@ pub async fn register_server( host: &str, release: &str, ) -> Result { - register_server_timed( + register_server_as(session, keyspace_prefix, host, release, SERVE_COMMAND).await +} + +/// [`register_server`] for the process `command` names: `extenddb serve` +/// ([`SERVE_COMMAND`]) or a command that opens the catalog as a server does +/// (`extenddb backup list`), whose row carries its name (the registrations +/// module doc says where) and whose refusal while a migration holds the +/// lock names the holder without the server's advice to start the server. +/// +/// # Errors +/// +/// As [`register_server`]. +pub async fn register_server_as( + session: Arc, + keyspace_prefix: &str, + host: &str, + release: &str, + command: &str, +) -> Result { + register_server_with( session, keyspace_prefix, host, release, + command, RegistrationTimes::DEFAULT, ) .await @@ -693,6 +767,26 @@ pub async fn register_server_timed( release: &str, times: RegistrationTimes, ) -> Result { + register_server_with( + session, + keyspace_prefix, + host, + release, + SERVE_COMMAND, + times, + ) + .await +} + +async fn register_server_with( + session: Arc, + keyspace_prefix: &str, + host: &str, + release: &str, + command: &str, + times: RegistrationTimes, +) -> Result { + let release = ®istrations::release_cell(release, command); let catalog = format!("{keyspace_prefix}_catalog"); agreement::wait_for_agreement(&session, AGREEMENT_TIMEOUT).await?; if !connect::table_exists(&session, &catalog, "extenddb_servers").await? { @@ -711,9 +805,15 @@ pub async fn register_server_timed( .await?; let checked = async { if let Some(holder) = locks::lock_holder(&session, &catalog).await? { + // A command's refusal ends at the holder; the command's caller + // adds that it is retried once the migration has finished. + let then = if command == SERVE_COMMAND { + "; start the server again when it finishes" + } else { + "" + }; return Err(MigrationError::Locked(format!( - "a migration holds the catalog's migration lock (holder {holder}); start the \ - server again when it finishes" + "a migration holds the catalog's migration lock (holder {holder}){then}" ))); } match catalog_status(session.clone(), keyspace_prefix).await? { diff --git a/crates/storage-cassandra/src/migrations/registrations.rs b/crates/storage-cassandra/src/migrations/registrations.rs index 33d011215..5b5b798e8 100644 --- a/crates/storage-cassandra/src/migrations/registrations.rs +++ b/crates/storage-cassandra/src/migrations/registrations.rs @@ -14,6 +14,15 @@ //! `migrate --yes`, which waits while a row exists, can see it gone. The server then stops admitting requests and stops its //! background workers, and registers again with a new row //! ([`super::registrar`]). +//! +//! A command that opens the catalog as a server does (`extenddb backup list`, +//! `extenddb backup purge`) registers too, so a migration waits for it. The +//! table has no column for the process's name, and a column would be a +//! catalog migration, so a command's row carries its name in `release`, +//! after the release and ` as ` (`1.0.0 as extenddb backup purge`; +//! [`release_cell`] writes it and [`registered_servers`] reads it back), and +//! the migration's wait prints the name. A row whose `release` has no +//! ` as ` is a server's. use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; @@ -58,9 +67,33 @@ impl RegistrationTimes { }; } -/// One running server, as `migrate --yes` names it. +/// The name a server's registration carries; a command's carries its own +/// (`extenddb backup list`). +pub const SERVE_COMMAND: &str = "extenddb serve"; + +/// What a registration writes in `release`: the release for a server, the +/// release, ` as ` and the command's name for a command. +pub(crate) fn release_cell(release: &str, command: &str) -> String { + if command == SERVE_COMMAND { + release.to_owned() + } else { + format!("{release} as {command}") + } +} + +/// The release and the process name a `release` cell carries. +fn split_release_cell(cell: &str) -> (String, String) { + match cell.split_once(" as ") { + Some((release, command)) => (release.to_owned(), command.to_owned()), + None => (cell.to_owned(), SERVE_COMMAND.to_owned()), + } +} + +/// One running server, or a command registered as one, as `migrate --yes` +/// names it. #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) struct Server { + pub(crate) command: String, pub(crate) host: String, pub(crate) pid: i32, pub(crate) release: String, @@ -71,8 +104,8 @@ impl std::fmt::Display for Server { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!( f, - "extenddb serve {} on {} (process {}, started {})", - self.release, self.host, self.pid, self.started_at + "{} {} on {} (process {}, started {})", + self.command, self.release, self.host, self.pid, self.started_at ) } } @@ -101,10 +134,14 @@ pub(crate) async fn registered_servers( found .iter() .map(|row| { + let (release, command) = split_release_cell( + &nullable::(row, "release", "read a server")?.unwrap_or_default(), + ); Ok(Server { + command, host: nullable::(row, "host", "read a server")?.unwrap_or_default(), pid: nullable::(row, "pid", "read a server")?.unwrap_or_default(), - release: nullable::(row, "release", "read a server")?.unwrap_or_default(), + release, started_at: nullable::(row, "started_at", "read a server")? .unwrap_or_default(), }) @@ -513,4 +550,31 @@ mod tests { } ); } + + /// A server's `release` cell is the release; a command's carries its + /// name after ` as `, and the read gives both back, so the wait names + /// the command where it names a server `extenddb serve`. + #[test] + fn a_commands_release_cell_carries_its_name() { + use super::{SERVE_COMMAND, Server, release_cell, split_release_cell}; + assert_eq!(release_cell("1.0.0", SERVE_COMMAND), "1.0.0"); + let cell = release_cell("1.0.0", "extenddb backup purge"); + assert_eq!(cell, "1.0.0 as extenddb backup purge"); + let (release, command) = split_release_cell(&cell); + let server = Server { + command, + host: "host-a".to_owned(), + pid: 7, + release, + started_at: "2026-11-03T00:00:00Z".to_owned(), + }; + assert_eq!( + server.to_string(), + "extenddb backup purge 1.0.0 on host-a (process 7, started 2026-11-03T00:00:00Z)" + ); + assert_eq!( + split_release_cell("1.0.0"), + ("1.0.0".to_owned(), SERVE_COMMAND.to_owned()) + ); + } } diff --git a/crates/storage-cassandra/src/migrations/tests.rs b/crates/storage-cassandra/src/migrations/tests.rs index 037323f88..fd75a11b4 100644 --- a/crates/storage-cassandra/src/migrations/tests.rs +++ b/crates/storage-cassandra/src/migrations/tests.rs @@ -69,6 +69,7 @@ fn every_migration_records_a_row_in_the_catalog_and_names_its_stores() { for id in [ MigrationId::SortKeyDefinitions, MigrationId::IndexTableBaseKey, + MigrationId::RestoreProgress, ] { assert!(matches!(migration(id), Migration::NotApplicable(_)), "{id}"); } diff --git a/crates/storage-cassandra/src/restore_engine.rs b/crates/storage-cassandra/src/restore_engine.rs new file mode 100644 index 000000000..0cc25ad5f --- /dev/null +++ b/crates/storage-cassandra/src/restore_engine.rs @@ -0,0 +1,599 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The data-plane side of a store-backed restore on Cassandra: the +//! [`RestoreEngine`] methods the engine's restore job calls. +//! +//! `bulk_load` writes one verified data file's items into the CREATING +//! target, each item's base row and every GSI and LSI row in one logged +//! batch ([`CassandraEngine::put_restored_item_at`], no lightweight +//! transaction: a CREATING restore target has no writer but its job), +//! `RESTORE_WRITE_CONCURRENCY` in flight, in batches of +//! at most [`RESTORE_BATCH_ITEMS`] items or [`RESTORE_BATCH_BYTES`] of +//! serialized item text (a batch closes at the first item that takes its +//! `item_data` text past the bound, so the batch term of a restore's memory +//! is bounded in bytes, not only in items). Before each batch's writes it +//! reads the coordinator's clock (the batch's write timestamp) and then +//! runs a lightweight transaction on the record (`confirm_restore_claim`: +//! the heartbeat refreshed `IF` the record is RUNNING under the claim's +//! generation) and ends the file as a lost claim when it is not applied, so +//! the check is linearized with a takeover's own transaction rather than a +//! quorum read beside it. The batch's writes into the account keyspace +//! carry no condition of their own (a Cassandra write cannot be conditioned +//! on another partition); the fence is the timestamp: every write of the +//! batch lands `USING TIMESTAMP` the reading taken before the check, so a +//! writer that stalls between the check and its writes and lands them after +//! a takeover lands them below every write and delete the successor made +//! (the successor's batches carry readings taken after the takeover, its +//! `finish_restore` and the deletes through the ACTIVE table are later +//! still), and Cassandra keeps the later cell or tombstone; the stalled +//! batch changes nothing the successor wrote or removed. The stamp is read +//! from the ExtendDB host's clock (`data::version::now_micros`, the clock +//! the item versions already use), while the writes it must lose to are +//! lightweight transactions and the deletes through the ACTIVE table, +//! whose write times the Cassandra coordinator sets from its own clock, so +//! the rule needs the ExtendDB hosts' clocks and the Cassandra nodes' clocks +//! to agree, as the version scheme already needs the hosts' to. After +//! each batch it records the file's progress +//! with a lightweight transaction conditioned on the restore generation and +//! on the record being RUNNING; `[applied] = false` there is the same lost +//! claim, and the load stops without touching the target again; an unknown +//! outcome of that transaction is decided by a LOCAL_SERIAL re-read of the +//! record (applied when it carries the values written, one retry when it +//! carries the earlier ones, the lost claim when it is not RUNNING under the +//! generation; `write_restore_progress`). The writes are idempotent upserts, +//! so a re-applied batch after a resume rewrites the same base and index rows +//! at a later timestamp (the base row's `version` is the timestamp, so a +//! later ordinary write's version is above it and its index rows resolve in +//! order, ADR-0027), and a key the backup +//! holds twice collapses into one row: a shortfall `count_restore_target` +//! finds against the manifest's total is the duplicate (the engine's count +//! check, 5.3 step 6). Progress is recorded after the batch's writes, so a +//! crash inside the first batch leaves items with no progress; the engine +//! resumes such a record with the first file re-applied (`reapplying`) when +//! the target holds no more than the backup's total, and fails it +//! `target_not_empty` above the total. An empty file records its progress +//! too (`items_applied = start_at`), so a resume does not re-read it. +//! +//! `finish_restore` is [`CassandraEngine::activate_restore_target_row`]'s +//! lightweight transaction (DONE, the table and its indexes ACTIVE, the +//! counts, conditioned on CREATING, RUNNING and the generation). The index +//! rows are created ACTIVE with the table, so Cassandra never returns +//! [`Finished::Pending`]. + +use cdrs_tokio::types::IntoRustByName; +use extenddb_core::types::{Item, TableKeyInfo}; +use extenddb_storage::backup_catalog::{ + JobClaim, JobStatus, RESTORE_CASSANDRA_BEFORE_PROGRESS, RestoreTargetKey, WriteOutcome, + restore_pause_point, +}; +use extenddb_storage::error::StorageError; +use extenddb_storage::restore_engine::{BulkLoadFile, Finished, RestoreEngine}; +pub(crate) use extenddb_storage::restore_engine::{RESTORE_BATCH_BYTES, RESTORE_BATCH_ITEMS}; +use futures::future::BoxFuture; +use futures::stream::BoxStream; +use futures::{StreamExt, TryStreamExt}; + +use crate::CassandraEngine; +use crate::backup_engine::RESTORE_WRITE_CONCURRENCY; +use crate::data::ddl::{data_table_name, index_table_name}; + +/// The error of a `bulk_load` whose progress write found the record no longer +/// this claim's: the reconciler took it over while the file was loading, and +/// whoever holds the record now owns the target. Nothing further is touched. +fn claim_lost(table_name: &str) -> StorageError { + StorageError::Internal(format!( + "restore of table {table_name}: the claim on the target was lost while a data file \ + loaded (the record is not RUNNING under this generation); the record's owner \ + finishes it" + )) +} + +impl CassandraEngine { + /// The key shape of a CREATING restore target, read from its `tables` + /// row by hand (the put path refuses a CREATING table through + /// `fetch_table_key_info`, so the restore builds the info itself, as the + /// legacy copy does). `TableNotFound` when `key` is not a CREATING + /// restore target with this table id, as the other backends answer. The + /// put path reads the target's indexes from the catalog by `table_id`, + /// so the routing fields are left empty. + async fn restore_target_key_info( + &self, + key: &RestoreTargetKey, + ) -> Result { + let query = format!( + "SELECT table_id, table_status, restore_source_backup_arn, key_schema, \ + attribute_definitions FROM {}.tables WHERE account_id = ? AND table_name = ?", + self.catalog_keyspace() + ); + let rows = crate::cassandra_util::query_rows_quorum::( + &self.session, + &query, + cdrs_tokio::query_values!(key.account_id.as_str(), key.table_name.as_str()), + "restore target key info", + ) + .await?; + let row = rows.first().filter(|row| { + let id: Option = row.get_by_name("table_id").ok().flatten(); + let status: Option = row.get_by_name("table_status").ok().flatten(); + let source: Option = + row.get_by_name("restore_source_backup_arn").ok().flatten(); + id.as_deref() == Some(&key.table_id) + && status.as_deref() == Some("CREATING") + && source.is_some() + }); + let row = row.ok_or_else(|| { + StorageError::TableNotFound(format!( + "{} is not a CREATING restore target with table id {}", + key.table_name, key.table_id + )) + })?; + let ks_text: String = + crate::cassandra_util::get_column(row, "key_schema", "restore target key info")?; + let ad_text: String = crate::cassandra_util::get_column( + row, + "attribute_definitions", + "restore target key info", + )?; + let key_schema: Vec = + serde_json::from_str(&ks_text) + .map_err(|e| StorageError::Internal(format!("key schema: {e}")))?; + let attribute_definitions = serde_json::from_str(&ad_text) + .map_err(|e| StorageError::Internal(format!("attribute definitions: {e}")))?; + Ok(TableKeyInfo { + table_name: key.table_name.clone(), + account_id: key.account_id.clone(), + table_id: key.table_id.clone(), + base_key_schema: key_schema.clone(), + key_schema, + attribute_definitions, + has_lsi: false, + global_secondary_indexes: Vec::new(), + local_secondary_indexes: Vec::new(), + stream_specification: None, + vector_indexes: Vec::new(), + }) + } + + /// The claim's liveness before each batch's writes: a lightweight + /// transaction on the record (`restore_heartbeat_at` refreshed, `IF` the + /// table id, the generation and RUNNING), linearized on the target's + /// partition with the takeover's own transaction, so the check and a + /// `claim_restore_dead` order wholly one before the other where a + /// quorum read could run beside one. `Err` (a lost claim) when the + /// transaction is not applied: the reconciler took the record over, and + /// whoever holds it owns the target, so nothing further is written. An + /// unknown outcome is decided by a LOCAL_SERIAL re-read of the record. + /// The writes themselves carry no condition on the record: a batch landed + /// after a takeover is fenced by its write timestamp, read before this + /// check (the module doc). + async fn confirm_restore_claim( + &self, + key: &RestoreTargetKey, + generation: u64, + ) -> Result<(), StorageError> { + let query = format!( + "UPDATE {}.tables SET restore_heartbeat_at = toTimestamp(now()) \ + WHERE account_id = ? AND table_name = ? \ + IF table_id = ? AND restore_owner_generation = ? AND restore_status = 'RUNNING'", + self.catalog_keyspace() + ); + let outcome = crate::backup_catalog::conditional_write( + &self.session, + &query, + cdrs_tokio::query::QueryValues::SimpleValues(vec![ + cdrs_tokio::types::value::Value::from(key.account_id.as_str()), + cdrs_tokio::types::value::Value::from(key.table_name.as_str()), + cdrs_tokio::types::value::Value::from(key.table_id.as_str()), + cdrs_tokio::types::value::Value::from(crate::backup_catalog::generation_column( + generation, + )), + ]), + ) + .await?; + match outcome { + WriteOutcome::Matched => return Ok(()), + WriteOutcome::NotMatched => return Err(claim_lost(&key.table_name)), + WriteOutcome::Unknown => {} + } + let query = format!( + "SELECT table_id, restore_status, restore_owner_generation FROM {}.tables \ + WHERE account_id = ? AND table_name = ?", + self.catalog_keyspace() + ); + let rows = crate::cassandra_util::query_rows_serial::( + &self.session, + &query, + cdrs_tokio::query_values!(key.account_id.as_str(), key.table_name.as_str()), + "restore claim", + ) + .await?; + let live = rows.first().is_some_and(|row| { + let table_id: Option = row.get_by_name("table_id").ok().flatten(); + let status: Option = row.get_by_name("restore_status").ok().flatten(); + let owner: Option = row.get_by_name("restore_owner_generation").ok().flatten(); + table_id.as_deref() == Some(key.table_id.as_str()) + && status.as_deref() == Some(JobStatus::Running.as_str()) + && owner == Some(crate::backup_catalog::generation_column(generation)) + }); + if live { + Ok(()) + } else { + Err(claim_lost(&key.table_name)) + } + } + + /// Write one batch of items into the target, each item's base and index + /// rows in its own logged batch at one write timestamp, + /// `RESTORE_WRITE_CONCURRENCY` in flight, after confirming the claim. + /// The timestamp is read before the claim check (the module doc's + /// fence). Drains `batch`. + async fn flush_restore_batch( + &self, + key: &RestoreTargetKey, + generation: u64, + key_info: &TableKeyInfo, + indexes: &[crate::data::index::IndexMeta], + batch: &mut Vec, + ) -> Result { + let stamp = crate::data::version::now_micros(); + self.confirm_restore_claim(key, generation).await?; + crate::test_hooks::after_restore_claim_check(&key.table_id).await; + let count = u64::try_from(batch.len()).unwrap_or(u64::MAX); + let mut writes = futures::stream::iter(batch.drain(..)) + .map(|item| async move { + self.put_restored_item_at(key_info, indexes, &item, stamp) + .await + }) + .buffer_unordered(RESTORE_WRITE_CONCURRENCY); + while let Some(result) = writes.next().await { + result?; + } + Ok(count) + } + + /// The progress lightweight transaction alone: `restore_file_index` and + /// `restore_items_applied`, conditioned on the table id, the restore + /// generation and the record being RUNNING. A test may make it answer + /// `Unknown`, with or without the transaction run. + async fn progress_write( + &self, + key: &RestoreTargetKey, + generation: u64, + file_index: u32, + items_applied: u64, + ) -> Result { + let fault = crate::test_hooks::progress_write_fault(&key.account_id); + if fault == Some(crate::test_hooks::ProgressWriteFault::Lost) { + return Ok(WriteOutcome::Unknown); + } + let query = format!( + "UPDATE {}.tables SET restore_file_index = ?, restore_items_applied = ? \ + WHERE account_id = ? AND table_name = ? \ + IF table_id = ? AND restore_owner_generation = ? AND restore_status = 'RUNNING'", + self.catalog_keyspace() + ); + let outcome = crate::backup_catalog::conditional_write( + &self.session, + &query, + cdrs_tokio::query::QueryValues::SimpleValues(vec![ + cdrs_tokio::types::value::Value::from( + i32::try_from(file_index).unwrap_or(i32::MAX), + ), + cdrs_tokio::types::value::Value::from( + i64::try_from(items_applied).unwrap_or(i64::MAX), + ), + cdrs_tokio::types::value::Value::from(key.account_id.as_str()), + cdrs_tokio::types::value::Value::from(key.table_name.as_str()), + cdrs_tokio::types::value::Value::from(key.table_id.as_str()), + cdrs_tokio::types::value::Value::from(crate::backup_catalog::generation_column( + generation, + )), + ]), + ) + .await?; + if fault == Some(crate::test_hooks::ProgressWriteFault::Landed) { + return Ok(WriteOutcome::Unknown); + } + Ok(outcome) + } + + /// What the record says after a progress write whose outcome is unknown, + /// read at LOCAL_SERIAL so a transaction that did commit is seen: RUNNING + /// under `generation` with this table id and carrying the values just + /// written is `Applied`; RUNNING under `generation` with other values + /// (the earlier progress: nothing but this job writes the record's + /// progress while it owns it) is `Pending`; anything else is `Lost`. + async fn progress_write_state( + &self, + key: &RestoreTargetKey, + generation: u64, + file_index: u32, + items_applied: u64, + ) -> Result { + let query = format!( + "SELECT table_id, restore_status, restore_owner_generation, restore_file_index, \ + restore_items_applied FROM {}.tables WHERE account_id = ? AND table_name = ?", + self.catalog_keyspace() + ); + let rows = crate::cassandra_util::query_rows_serial::( + &self.session, + &query, + cdrs_tokio::query_values!(key.account_id.as_str(), key.table_name.as_str()), + "restore progress", + ) + .await?; + let Some(row) = rows.first() else { + return Ok(ProgressWriteState::Lost); + }; + let table_id: Option = row.get_by_name("table_id").ok().flatten(); + let status: Option = row.get_by_name("restore_status").ok().flatten(); + let owner: Option = row.get_by_name("restore_owner_generation").ok().flatten(); + let owned = table_id.as_deref() == Some(key.table_id.as_str()) + && status.as_deref() == Some(JobStatus::Running.as_str()) + && owner == Some(crate::backup_catalog::generation_column(generation)); + if !owned { + return Ok(ProgressWriteState::Lost); + } + let recorded_file: Option = row.get_by_name("restore_file_index").ok().flatten(); + let recorded_items: Option = row.get_by_name("restore_items_applied").ok().flatten(); + let written = ( + Some(i32::try_from(file_index).unwrap_or(i32::MAX)), + Some(i64::try_from(items_applied).unwrap_or(i64::MAX)), + ); + if (recorded_file, recorded_items) == written { + Ok(ProgressWriteState::Applied) + } else { + Ok(ProgressWriteState::Pending) + } + } + + /// Record the file's progress (`restore_file_index`, the running total + /// `restore_items_applied`) with a lightweight transaction conditioned on + /// the restore generation and on the record being RUNNING. `[applied] = + /// false` is a lost claim. An unknown outcome (a write timeout or failure, + /// a lost connection: the Paxos round may still have committed) is + /// decided by re-reading the record at LOCAL_SERIAL: RUNNING under this + /// generation with the values just written is applied and the file goes + /// on; RUNNING under this generation with the earlier values is one + /// retry of the transaction, whose own unknown outcome is re-read the + /// same way, and a second unknown outcome that the record does not carry + /// fails the file (the error names it; the engine then fails the restore + /// and removes the target); anything else is the lost claim. + async fn write_restore_progress( + &self, + key: &RestoreTargetKey, + generation: u64, + file_index: u32, + items_applied: u64, + ) -> Result<(), StorageError> { + restore_pause_point(RESTORE_CASSANDRA_BEFORE_PROGRESS).await; + let mut retried = false; + loop { + match self + .progress_write(key, generation, file_index, items_applied) + .await? + { + WriteOutcome::Matched => return Ok(()), + WriteOutcome::NotMatched => return Err(claim_lost(&key.table_name)), + WriteOutcome::Unknown => {} + } + match self + .progress_write_state(key, generation, file_index, items_applied) + .await? + { + ProgressWriteState::Applied => return Ok(()), + ProgressWriteState::Lost => return Err(claim_lost(&key.table_name)), + ProgressWriteState::Pending if !retried => { + tracing::warn!( + "restore of table {}: the progress write for data file {file_index} \ + had no known outcome and the record does not carry it; written once \ + more", + key.table_name + ); + retried = true; + } + ProgressWriteState::Pending => { + return Err(StorageError::Internal(format!( + "restore of table {}: the progress write for data file {file_index} \ + had no known outcome twice and the record still carries the earlier \ + progress; the restore fails and its target is removed", + key.table_name + ))); + } + } + } + } +} + +/// What the record says about a progress write whose outcome was unknown. +#[derive(Debug, PartialEq, Eq)] +enum ProgressWriteState { + /// RUNNING under the claim's generation, the written values on it. + Applied, + /// RUNNING under the claim's generation, the earlier values on it. + Pending, + /// Not RUNNING under the claim's generation, or gone. + Lost, +} + +impl RestoreEngine for CassandraEngine { + fn bulk_load<'a>( + &'a self, + target: &'a RestoreTargetKey, + claim: &'a dyn JobClaim, + file: BulkLoadFile, + items: BoxStream<'a, Result>, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + let generation = claim.generation(); + let key_info = self.restore_target_key_info(target).await?; + // The target's indexes, once per file (a CREATING target's do not + // change), every GSI synchronous as the put path's + // `IndexWrites::AllSynchronous` makes them. + let mut indexes = crate::data::index::fetch_indexes_for_table( + &target.table_id, + &self.session, + &self.catalog_keyspace(), + ) + .await?; + for index in &mut indexes { + index.propagation_delay_ms = Some(0); + } + let batch_size = crate::test_hooks::restore_batch_size(&target.account_id).max(1); + let mut items = items; + let mut batch: Vec = Vec::with_capacity(batch_size.min(1024)); + let mut batch_bytes = 0usize; + let mut applied: u64 = 0; + let mut wrote_progress = false; + while let Some(item) = items.try_next().await? { + // The item's `item_data` text, the unit the byte bound is + // measured in (the put path serializes it again for the row). + let text_len = serde_json::to_string(&item) + .map_err(|e| StorageError::Internal(format!("item: {e}")))? + .len(); + batch_bytes = batch_bytes.saturating_add(text_len); + batch.push(item); + if batch.len() >= batch_size || batch_bytes >= RESTORE_BATCH_BYTES { + applied += self + .flush_restore_batch(target, generation, &key_info, &indexes, &mut batch) + .await?; + batch_bytes = 0; + self.write_restore_progress( + target, + generation, + file.file_index, + file.start_at.saturating_add(applied), + ) + .await?; + wrote_progress = true; + } + } + if !batch.is_empty() { + applied += self + .flush_restore_batch(target, generation, &key_info, &indexes, &mut batch) + .await?; + self.write_restore_progress( + target, + generation, + file.file_index, + file.start_at.saturating_add(applied), + ) + .await?; + wrote_progress = true; + } + // An empty file still records that it is complete, so a resume + // does not re-read it. + if !wrote_progress { + self.write_restore_progress(target, generation, file.file_index, file.start_at) + .await?; + } + Ok(applied) + }) + } + + fn finish_restore<'a>( + &'a self, + target: &'a RestoreTargetKey, + claim: &'a dyn JobClaim, + item_count: u64, + size_bytes: u64, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + let activated = self + .activate_restore_target_row( + &target.account_id, + &target.table_name, + &target.table_id, + claim.generation(), + i64::try_from(item_count).unwrap_or(i64::MAX), + i64::try_from(size_bytes).unwrap_or(i64::MAX), + ) + .await?; + if activated { + Ok(Finished::Active) + } else { + Ok(Finished::NotMatched(format!( + "the restore record of {} is not RUNNING under generation {}, or the table \ + is not a CREATING restore target", + target.table_name, + claim.generation() + ))) + } + }) + } + + fn count_restore_target<'a>( + &'a self, + key: &'a RestoreTargetKey, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + // A full COUNT(*) over the target's data table: a single scan of + // a CREATING table, run once per restore (the cost is stated in + // the limits doc). The id names the data table; the account + // keyspace scopes it to the target's account, and the catalog is + // read first so a key that is not a CREATING restore target with + // this id is `TableNotFound`, as on the other backends. + self.restore_target_key_info(key).await?; + let account_keyspace = self.account_keyspace(&key.account_id); + let data_table = data_table_name(&key.table_id); + let query = format!("SELECT COUNT(*) AS c FROM {account_keyspace}.{data_table}"); + let rows = crate::cassandra_util::query_rows_quorum::( + &self.session, + &query, + cdrs_tokio::query_values!(), + "count restore target", + ) + .await?; + let count: i64 = rows + .first() + .map(|row| crate::cassandra_util::get_column(row, "c", "count restore target")) + .transpose()? + .unwrap_or(0); + Ok(u64::try_from(count).unwrap_or(0)) + }) + } + + fn restore_target_ready<'a>( + &'a self, + key: &'a RestoreTargetKey, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + // The catalog first, so a bare id reads no other account's table; + // then the data table's row in `system_schema.tables` and one per + // GSI and LSI the catalog names (the registration LWT lands + // before the data DDL, which creates the data table and then + // each index table in its own statement, so a crash between them + // leaves a target missing an index table). Unquoted CQL + // identifiers are stored lowercased. + self.restore_target_key_info(key).await?; + let account_keyspace = self.account_keyspace(&key.account_id).to_lowercase(); + let indexes = crate::data::index::fetch_indexes_for_table( + &key.table_id, + &self.session, + &self.catalog_keyspace(), + ) + .await?; + let mut tables = vec![data_table_name(&key.table_id).to_lowercase()]; + tables.extend( + indexes + .iter() + .map(|index| index_table_name(&index.index_id).to_lowercase()), + ); + for table in tables { + let rows = crate::cassandra_util::query_rows::( + &self.session, + "SELECT table_name FROM system_schema.tables WHERE keyspace_name = ? AND \ + table_name = ?", + cdrs_tokio::query_values!(account_keyspace.clone(), table), + "restore target data table", + ) + .await?; + if rows.is_empty() { + return Ok(false); + } + } + Ok(true) + }) + } +} diff --git a/crates/storage-cassandra/src/test_hooks.rs b/crates/storage-cassandra/src/test_hooks.rs new file mode 100644 index 000000000..62f318c95 --- /dev/null +++ b/crates/storage-cassandra/src/test_hooks.rs @@ -0,0 +1,600 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Coordination hooks for this crate's integration tests (the +//! `direct_integration` binary), compiled only with the `test-hooks` feature. +//! Without the feature every hook is a no-op the compiler removes, so a +//! production build carries none of this. Each hook is keyed by the account +//! (or the backup ARN) a test owns, so the other tests running in the same +//! process are not affected. +//! +//! The hooks: +//! +//! * the heartbeat period of the jobs of one account, in place of +//! [`extenddb_storage::backup_catalog::JOB_HEARTBEAT_INTERVAL`], so a test +//! can watch a short create or restore beat; +//! * a count of heartbeats of one account's jobs that fail with an error +//! before reaching the catalog, to exercise the consecutive-error rule of +//! `JobHeartbeat`, and a read of how many are still to fire; +//! * a pause between the in-use read and the DELETING claim of a backup +//! delete (the legacy `delete` and `start_backup_delete`), so a test can +//! register a restore in that gap and see the claim's own +//! `active_restores = null` condition refuse the delete; +//! * a pause in the reconciler's scan of abandoned in-catalog creates, +//! between the read of a CREATING row and the fence that moves it to +//! DELETING, so a test can heartbeat the row in that gap and see the +//! fence's own `heartbeat_at = ?` condition leave it alone. +//! * a lost answer on the maintenance claim's insert, so a test can see the +//! re-read settle a claim whose `INSERT ... IF NOT EXISTS` committed; +//! * a pause between a maintenance claim takeover's read and its write, so +//! a test can hold two takers of one stale row there and see one win; +//! * a pause in `claim_dead_backup` between its `LOCAL_SERIAL` read and its +//! takeover LWT, so a test can land the owner's beat in that gap and see +//! the LWT's `heartbeat_at = ` condition refuse the takeover; +//! * a pause in a store-backed registration between the target's creation +//! and its `active_restores` lightweight transaction, so a test can move +//! the backup to DELETING in that gap and see the transaction's own +//! `backup_status = 'AVAILABLE'` condition refuse the registration; +//! * a pause in `claim_restore_dead` between its read and its takeover +//! lightweight transaction, and the batch size of a store-backed +//! `bulk_load`; +//! * a fault on the progress lightweight transaction of a store-backed +//! `bulk_load`, so a test can see the re-read that decides an unknown +//! outcome; +//! * a pause in a store-backed `bulk_load` between a batch's claim check and +//! its writes, so a test can let a successor write and delete in that gap +//! and see the stalled batch's timestamp lose to the successor's state; +//! * a fault on the lightweight transaction of `insert_backup_row`, and an +//! error on the logged batch that writes its index entries, so a test can +//! see the serial re-read settle a lost answer and an insert complete the +//! entries a stopped one left unwritten; +//! * a fault on the lightweight transaction of `forget_available_backup_row`, +//! and an error on the logged batch that removes a row's index entries, so +//! a test can see the serial re-read settle a lost removal and a removal +//! whose entries stay answer that the row is removed. + +use std::time::Duration; + +use extenddb_storage::error::StorageError; + +/// How a test makes a lightweight transaction (a restore's progress write, +/// the insert of `insert_backup_row` or the removal of +/// `forget_available_backup_row`) answer `WriteOutcome::Unknown`: +/// with the transaction run first (`Landed`: the answer was lost on its way +/// back) or not run at all (`Lost`: the proposal never reached a replica). +/// Each fault is taken by one write, in order. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ProgressWriteFault { + Landed, + Lost, +} + +#[cfg(feature = "test-hooks")] +mod armed { + use std::collections::{HashMap, VecDeque}; + use std::sync::{Arc, LazyLock, Mutex}; + use std::time::Duration; + + use tokio::sync::Notify; + + use super::ProgressWriteFault; + + /// A pause a test arms at one point of one operation: the operation + /// signals `reached` when it gets there and waits for `released`. + pub struct Pause { + reached: Notify, + released: Notify, + } + + impl Pause { + pub(super) fn new() -> Self { + Self { + reached: Notify::new(), + released: Notify::new(), + } + } + + /// Wait until the paused operation has reached the pause point. + pub async fn reached(&self) { + self.reached.notified().await; + } + + /// Let the paused operation continue. + pub fn release(&self) { + self.released.notify_one(); + } + + pub(super) async fn hold(&self) { + self.reached.notify_one(); + self.released.notified().await; + } + } + + #[derive(Default)] + pub(super) struct Hooks { + pub(super) heartbeat_interval: HashMap, + pub(super) heartbeat_errors: HashMap, + pub(super) before_delete_claim: HashMap>, + pub(super) before_abandoned_fence: HashMap>, + pub(super) lose_maintenance_insert_answer: bool, + pub(super) before_maintenance_takeover: Option>, + pub(super) before_takeover_write: HashMap>, + pub(super) before_restore_takeover: HashMap>, + pub(super) before_restore_registration: HashMap>, + pub(super) restore_batch_size: HashMap, + pub(super) progress_write_faults: HashMap>, + pub(super) after_restore_claim_check: HashMap>, + pub(super) insert_write_faults: HashMap>, + pub(super) index_entry_batch_errors: HashMap, + pub(super) forget_write_faults: HashMap>, + pub(super) index_entry_removal_errors: HashMap, + } + + pub(super) static HOOKS: LazyLock> = LazyLock::new(Mutex::default); + + pub(super) fn with_hooks(f: impl FnOnce(&mut Hooks) -> T) -> T { + let mut hooks = HOOKS + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + f(&mut hooks) + } +} + +#[cfg(feature = "test-hooks")] +pub use armed::Pause; + +/// Make the jobs of `account_id` heartbeat every `interval` instead of +/// `JOB_HEARTBEAT_INTERVAL`. +#[cfg(feature = "test-hooks")] +pub fn set_heartbeat_interval(account_id: &str, interval: Duration) { + armed::with_hooks(|hooks| { + hooks + .heartbeat_interval + .insert(account_id.to_owned(), interval); + }); +} + +/// Make the next `count` heartbeats of the jobs of `account_id` fail with an +/// error before reaching the catalog. +#[cfg(feature = "test-hooks")] +pub fn fail_heartbeats(account_id: &str, count: usize) { + armed::with_hooks(|hooks| { + hooks.heartbeat_errors.insert(account_id.to_owned(), count); + }); +} + +/// How many of the heartbeat errors [`fail_heartbeats`] armed for +/// `account_id` no beat has taken yet, so a test can see that the beats it +/// armed have fired. +#[cfg(feature = "test-hooks")] +pub fn pending_heartbeat_errors(account_id: &str) -> usize { + armed::with_hooks(|hooks| hooks.heartbeat_errors.get(account_id).copied().unwrap_or(0)) +} + +/// Arm a pause in the next delete of `backup_arn`, between its in-use read +/// and its DELETING claim. The pause is taken once and then disarmed. +#[cfg(feature = "test-hooks")] +pub fn pause_before_delete_claim(backup_arn: &str) -> std::sync::Arc { + let pause = std::sync::Arc::new(Pause::new()); + armed::with_hooks(|hooks| { + hooks + .before_delete_claim + .insert(backup_arn.to_owned(), std::sync::Arc::clone(&pause)); + }); + pause +} + +/// Arm a pause in the next abandoned-create scan that reaches the CREATING +/// row `backup_arn`, between the read of the row and the fence that moves it +/// to DELETING. The pause is taken once and then disarmed. +#[cfg(feature = "test-hooks")] +pub fn pause_before_abandoned_fence(backup_arn: &str) -> std::sync::Arc { + let pause = std::sync::Arc::new(Pause::new()); + armed::with_hooks(|hooks| { + hooks + .before_abandoned_fence + .insert(backup_arn.to_owned(), std::sync::Arc::clone(&pause)); + }); + pause +} + +/// Arm a pause in the next `claim_restore_dead` of the restore target +/// `table_id`, between its serial read (and staleness check) and its +/// takeover lightweight transaction. A test releases it after writing a +/// fresh heartbeat, to prove the takeover's `restore_heartbeat_at = ` +/// condition refuses a beat that lands in between. Taken once, then disarmed. +#[cfg(feature = "test-hooks")] +pub fn pause_before_restore_takeover(table_id: &str) -> std::sync::Arc { + let pause = std::sync::Arc::new(Pause::new()); + armed::with_hooks(|hooks| { + hooks + .before_restore_takeover + .insert(table_id.to_owned(), std::sync::Arc::clone(&pause)); + }); + pause +} + +/// Arm a pause in the next store-backed registration of a restore of +/// `backup_arn`, between the target's creation (its row and DDL) and the +/// `active_restores` lightweight transaction on the backup. A test moves the +/// backup to DELETING in the gap to prove the transaction's own +/// `backup_status = 'AVAILABLE'` condition, not the unfenced pre-read, +/// refuses the registration. Taken once, then disarmed. +#[cfg(feature = "test-hooks")] +pub fn pause_before_restore_registration(backup_arn: &str) -> std::sync::Arc { + let pause = std::sync::Arc::new(Pause::new()); + armed::with_hooks(|hooks| { + hooks + .before_restore_registration + .insert(backup_arn.to_owned(), std::sync::Arc::clone(&pause)); + }); + pause +} + +/// Arm a pause in the next `claim_dead_backup` of `backup_arn`, between its +/// read of the row and its takeover write. The pause is taken once and then +/// disarmed. +#[cfg(feature = "test-hooks")] +pub fn pause_before_takeover_write(backup_arn: &str) -> std::sync::Arc { + let pause = std::sync::Arc::new(Pause::new()); + armed::with_hooks(|hooks| { + hooks + .before_takeover_write + .insert(backup_arn.to_owned(), std::sync::Arc::clone(&pause)); + }); + pause +} + +/// Make the store-backed `bulk_load` of jobs of `account_id` flush and record +/// progress every `size` items instead of the production batch size, so a +/// test can cross a batch boundary without writing ten thousand items. +#[cfg(feature = "test-hooks")] +pub fn set_restore_batch_size(account_id: &str, size: usize) { + armed::with_hooks(|hooks| { + hooks.restore_batch_size.insert(account_id.to_owned(), size); + }); +} + +/// Arm a pause in the next batch of a store-backed `bulk_load` into the +/// restore target `table_id`, between the batch's claim check (its +/// lightweight transaction on the record, run after the batch's write +/// timestamp was read) and the batch's writes. A test takes the record over +/// in the gap, lets the successor write and delete, then releases the pause +/// to prove the stalled batch's writes, at the earlier timestamp, change +/// nothing the successor wrote or removed. Taken once, then disarmed. +#[cfg(feature = "test-hooks")] +pub fn pause_after_restore_claim_check(table_id: &str) -> std::sync::Arc { + let pause = std::sync::Arc::new(Pause::new()); + armed::with_hooks(|hooks| { + hooks + .after_restore_claim_check + .insert(table_id.to_owned(), std::sync::Arc::clone(&pause)); + }); + pause +} + +/// Hold here if a test armed a pause after the claim check of a batch into +/// the restore target `table_id`. +pub(crate) async fn after_restore_claim_check(table_id: &str) { + #[cfg(feature = "test-hooks")] + if let Some(pause) = armed::with_hooks(|hooks| hooks.after_restore_claim_check.remove(table_id)) + { + pause.hold().await; + } + let _ = table_id; +} + +/// Make the next progress lightweight transactions of the store-backed +/// `bulk_load` of jobs of `account_id` answer `WriteOutcome::Unknown`, one +/// fault per write in the order given: [`ProgressWriteFault::Landed`] runs +/// the transaction and drops its answer, [`ProgressWriteFault::Lost`] drops +/// the transaction. Writes after the last fault run as usual. +#[cfg(feature = "test-hooks")] +pub fn fail_progress_writes(account_id: &str, faults: &[ProgressWriteFault]) { + armed::with_hooks(|hooks| { + hooks + .progress_write_faults + .insert(account_id.to_owned(), faults.iter().copied().collect()); + }); +} + +/// Make the next lightweight transactions of `insert_backup_row` for +/// `backup_arn` answer `WriteOutcome::Unknown`, one fault per write in the +/// order given, as [`fail_progress_writes`] does for progress writes. Writes +/// after the last fault run as usual. +#[cfg(feature = "test-hooks")] +pub fn fail_insert_writes(backup_arn: &str, faults: &[ProgressWriteFault]) { + armed::with_hooks(|hooks| { + hooks + .insert_write_faults + .insert(backup_arn.to_owned(), faults.iter().copied().collect()); + }); +} + +/// Make the next `count` logged batches that write the index entries of +/// `backup_arn` fail with an error before reaching the catalog. +#[cfg(feature = "test-hooks")] +pub fn fail_index_entry_batches(backup_arn: &str, count: usize) { + armed::with_hooks(|hooks| { + hooks + .index_entry_batch_errors + .insert(backup_arn.to_owned(), count); + }); +} + +/// Make the next lightweight transactions of `forget_available_backup_row` +/// for `backup_arn` answer `WriteOutcome::Unknown`, one fault per write in +/// the order given, as [`fail_insert_writes`] does for the insert. +#[cfg(feature = "test-hooks")] +pub fn fail_forget_writes(backup_arn: &str, faults: &[ProgressWriteFault]) { + armed::with_hooks(|hooks| { + hooks + .forget_write_faults + .insert(backup_arn.to_owned(), faults.iter().copied().collect()); + }); +} + +/// Make the next `count` logged batches that remove the index entries of +/// `backup_arn` fail with an error before reaching the catalog. +#[cfg(feature = "test-hooks")] +pub fn fail_index_entry_removals(backup_arn: &str, count: usize) { + armed::with_hooks(|hooks| { + hooks + .index_entry_removal_errors + .insert(backup_arn.to_owned(), count); + }); +} + +/// The fault the next removal lightweight transaction of `backup_arn` takes, +/// if a test armed one. +pub(crate) fn forget_write_fault(backup_arn: &str) -> Option { + #[cfg(feature = "test-hooks")] + { + armed::with_hooks(|hooks| { + hooks + .forget_write_faults + .get_mut(backup_arn) + .and_then(std::collections::VecDeque::pop_front) + }) + } + #[cfg(not(feature = "test-hooks"))] + { + let _ = backup_arn; + None + } +} + +/// The error the next index-entry removal batch of `backup_arn` fails with, +/// if a test asked for one. +pub(crate) fn index_entry_removal_error(backup_arn: &str) -> Option { + #[cfg(feature = "test-hooks")] + { + let injected = + armed::with_hooks( + |hooks| match hooks.index_entry_removal_errors.get_mut(backup_arn) { + Some(left) if *left > 0 => { + *left -= 1; + true + } + _ => false, + }, + ); + if injected { + return Some(StorageError::Internal( + "test hook: index entry removal failed before reaching the catalog".to_owned(), + )); + } + } + let _ = backup_arn; + None +} + +/// The fault the next insert lightweight transaction of `backup_arn` takes, +/// if a test armed one. +pub(crate) fn insert_write_fault(backup_arn: &str) -> Option { + #[cfg(feature = "test-hooks")] + { + armed::with_hooks(|hooks| { + hooks + .insert_write_faults + .get_mut(backup_arn) + .and_then(std::collections::VecDeque::pop_front) + }) + } + #[cfg(not(feature = "test-hooks"))] + { + let _ = backup_arn; + None + } +} + +/// The error the next index-entry batch of `backup_arn` fails with, if a +/// test asked for one. +pub(crate) fn index_entry_batch_error(backup_arn: &str) -> Option { + #[cfg(feature = "test-hooks")] + { + let injected = + armed::with_hooks( + |hooks| match hooks.index_entry_batch_errors.get_mut(backup_arn) { + Some(left) if *left > 0 => { + *left -= 1; + true + } + _ => false, + }, + ); + if injected { + return Some(StorageError::Internal( + "test hook: index entry batch failed before reaching the catalog".to_owned(), + )); + } + } + let _ = backup_arn; + None +} + +/// The period the jobs of `account_id` heartbeat at. +pub(crate) fn heartbeat_interval(account_id: &str) -> Duration { + #[cfg(feature = "test-hooks")] + if let Some(interval) = + armed::with_hooks(|hooks| hooks.heartbeat_interval.get(account_id).copied()) + { + return interval; + } + let _ = account_id; + extenddb_storage::backup_catalog::JOB_HEARTBEAT_INTERVAL +} + +/// The error the next heartbeat of a job of `account_id` fails with, if a +/// test asked for one. +pub(crate) fn heartbeat_error(account_id: &str) -> Option { + #[cfg(feature = "test-hooks")] + { + let injected = + armed::with_hooks(|hooks| match hooks.heartbeat_errors.get_mut(account_id) { + Some(left) if *left > 0 => { + *left -= 1; + true + } + _ => false, + }); + if injected { + return Some(StorageError::Internal( + "test hook: heartbeat failed before reaching the catalog".to_owned(), + )); + } + } + let _ = account_id; + None +} + +/// Hold here if a test armed a pause for the delete of `backup_arn`. +pub(crate) async fn before_delete_claim(backup_arn: &str) { + #[cfg(feature = "test-hooks")] + if let Some(pause) = armed::with_hooks(|hooks| hooks.before_delete_claim.remove(backup_arn)) { + pause.hold().await; + } + let _ = backup_arn; +} + +/// Hold here if a test armed a pause for the abandoned-create fence of +/// `backup_arn`. +pub(crate) async fn before_abandoned_fence(backup_arn: &str) { + #[cfg(feature = "test-hooks")] + if let Some(pause) = armed::with_hooks(|hooks| hooks.before_abandoned_fence.remove(backup_arn)) + { + pause.hold().await; + } + let _ = backup_arn; +} + +/// Hold here if a test armed a pause for the takeover of restore target +/// `table_id`. +pub(crate) async fn before_restore_takeover(table_id: &str) { + #[cfg(feature = "test-hooks")] + if let Some(pause) = armed::with_hooks(|hooks| hooks.before_restore_takeover.remove(table_id)) { + pause.hold().await; + } + let _ = table_id; +} + +/// Hold here if a test armed a pause for the store-backed registration of a +/// restore of `backup_arn`. +pub(crate) async fn before_restore_registration(backup_arn: &str) { + #[cfg(feature = "test-hooks")] + if let Some(pause) = + armed::with_hooks(|hooks| hooks.before_restore_registration.remove(backup_arn)) + { + pause.hold().await; + } + let _ = backup_arn; +} + +/// Hold here if a test armed a pause for the takeover of `backup_arn`. +pub(crate) async fn before_takeover_write(backup_arn: &str) { + #[cfg(feature = "test-hooks")] + if let Some(pause) = armed::with_hooks(|hooks| hooks.before_takeover_write.remove(backup_arn)) { + pause.hold().await; + } + let _ = backup_arn; +} + +/// Make the next `maintenance_claim`'s `INSERT ... IF NOT EXISTS` lose its +/// answer: the statement runs and commits, and the caller sees +/// `WriteOutcome::Unknown`, as after a write timeout on the Paxos round. The +/// hook is taken once and then disarmed. +#[cfg(feature = "test-hooks")] +pub fn lose_next_maintenance_insert_answer() { + armed::with_hooks(|hooks| hooks.lose_maintenance_insert_answer = true); +} + +/// Arm a pause in every `maintenance_claim` takeover, between the read of +/// the stale row and the `UPDATE ... IF owner_generation = `, so a test +/// can hold two takers there and release both: each signals `reached` and +/// waits for one `release`. Disarmed by [`clear_maintenance_takeover_pause`]. +#[cfg(feature = "test-hooks")] +pub fn pause_before_maintenance_takeover() -> std::sync::Arc { + let pause = std::sync::Arc::new(Pause::new()); + armed::with_hooks(|hooks| { + hooks.before_maintenance_takeover = Some(std::sync::Arc::clone(&pause)); + }); + pause +} + +/// Disarm [`pause_before_maintenance_takeover`]. +#[cfg(feature = "test-hooks")] +pub fn clear_maintenance_takeover_pause() { + armed::with_hooks(|hooks| hooks.before_maintenance_takeover = None); +} + +/// Hold here if a test armed the takeover pause. +pub(crate) async fn before_maintenance_takeover() { + #[cfg(feature = "test-hooks")] + if let Some(pause) = armed::with_hooks(|hooks| hooks.before_maintenance_takeover.clone()) { + pause.hold().await; + } +} + +/// Whether a test asked for the maintenance claim insert's answer to be +/// lost ([`lose_next_maintenance_insert_answer`]); disarms the hook. +pub(crate) fn maintenance_insert_answer_lost() -> bool { + #[cfg(feature = "test-hooks")] + { + armed::with_hooks(|hooks| std::mem::take(&mut hooks.lose_maintenance_insert_answer)) + } + #[cfg(not(feature = "test-hooks"))] + false +} + +/// The number of items the store-backed `bulk_load` of jobs of `account_id` +/// applies before each progress write (the byte bound closes a batch +/// earlier). The production default is `RESTORE_BATCH_ITEMS`; a test may set +/// a small size with [`set_restore_batch_size`]. +pub(crate) fn restore_batch_size(account_id: &str) -> usize { + #[cfg(feature = "test-hooks")] + if let Some(size) = armed::with_hooks(|hooks| hooks.restore_batch_size.get(account_id).copied()) + { + return size; + } + let _ = account_id; + crate::restore_engine::RESTORE_BATCH_ITEMS +} + +/// The fault the next progress lightweight transaction of a job of +/// `account_id` takes, if a test armed one. +pub(crate) fn progress_write_fault(account_id: &str) -> Option { + #[cfg(feature = "test-hooks")] + { + armed::with_hooks(|hooks| { + hooks + .progress_write_faults + .get_mut(account_id) + .and_then(std::collections::VecDeque::pop_front) + }) + } + #[cfg(not(feature = "test-hooks"))] + { + let _ = account_id; + None + } +} diff --git a/crates/storage-cassandra/tests/direct/backup_engine.rs b/crates/storage-cassandra/tests/direct/backup_engine.rs index 3eb2735ac..ff08c8972 100644 --- a/crates/storage-cassandra/tests/direct/backup_engine.rs +++ b/crates/storage-cassandra/tests/direct/backup_engine.rs @@ -11,6 +11,7 @@ use std::collections::{BTreeMap, BTreeSet}; use std::time::{Duration, Instant}; +use extenddb_core::backup_format::BackupArn; use extenddb_core::expression::{Expr, ExpressionMaps, KeyCondition, PathElement}; use extenddb_core::types::{ AttributeDefinition, AttributeValue, BillingMode, CreateTableInput, DeleteTableInput, @@ -18,12 +19,19 @@ use extenddb_core::types::{ ProvisionedThroughput, ScalarAttributeType, TableDescription, TableKeyInfo, TableStatus, TimeToLiveStatus, }; +use extenddb_storage::backup_catalog::{ + BackupRow, BackupStatus, DeleteStart, Heartbeat, JobStatus, RestoreKind, + RestoreRegistrationRow, RestoreTargetKey, StaleClock, StatusFields, StoreIdentity, StoreKind, + TimeBounds, UNOWNED_GENERATION, WriteOutcome, backup_in_use_message, +}; use extenddb_storage::error::StorageError; +use extenddb_storage::migrations::{CatalogStatus, MigrationError, MigrationId}; use extenddb_storage::{ - BackupEngine, DataEngine, RestoreRequest, RestoreTableOverrides, TableEngine, + BackupCatalog, BulkLoadFile, DataEngine, Finished, JobPermit, LegacyBackupReader, + RestoreEngine, RestoreRequest, RestoreTableOverrides, TableEngine, }; use extenddb_storage_cassandra::CassandraEngine; -use futures::StreamExt; +use futures::{StreamExt, TryStreamExt}; use crate::helpers::{TestAccount, TestTable, setup_engine, unique_test_account}; @@ -140,11 +148,11 @@ async fn restore( target: &str, overrides: &RestoreTableOverrides, ) -> Result { - let source = engine.backup_restore_source(account_id, backup_arn).await?; + let source = engine.restore_source(account_id, backup_arn).await?; source.ensure_restorable()?; let input = source.restore_input(target, overrides); engine - .restore_table_from_backup( + .restore( account_id, RestoreRequest { backup_arn: backup_arn.to_owned(), @@ -152,6 +160,7 @@ async fn restore( restore_date_time: source.backup_creation_date_time, input, }, + JobPermit::detached(), ) .await } @@ -263,7 +272,12 @@ async fn backed_up_plain_table( ) .await; let backup = engine - .create_backup(&account.account_id, name, "copy-timing") + .create( + &account.account_id, + name, + "copy-timing", + JobPermit::detached(), + ) .await .expect("create backup"); (account, key_info, backup.backup_arn) @@ -293,10 +307,11 @@ async fn test_backup_describe_list_delete_and_account_scope() { } let details = engine - .create_backup( + .create( &table.key_info.account_id, &table.key_info.table_name, "daily", + JobPermit::detached(), ) .await .expect("create backup"); @@ -316,7 +331,7 @@ async fn test_backup_describe_list_delete_and_account_scope() { assert_eq!(details.backup_creation_date_time, embedded_seconds); let description = engine - .describe_backup(&table.key_info.account_id, &details.backup_arn) + .describe(&table.key_info.account_id, &details.backup_arn) .await .expect("describe backup"); assert_eq!( @@ -334,14 +349,14 @@ async fn test_backup_describe_list_delete_and_account_scope() { ); let table_backups = engine - .list_backups(&table.key_info.account_id, Some(&table.key_info.table_name)) + .list(&table.key_info.account_id, Some(&table.key_info.table_name)) .await .expect("list table backups"); assert_eq!(table_backups.len(), 1); assert_eq!(table_backups[0].backup_arn, details.backup_arn); assert_eq!(table_backups[0].table_id, table.key_info.table_id); let account_backups = engine - .list_backups(&table.key_info.account_id, None) + .list(&table.key_info.account_id, None) .await .expect("list account backups"); assert_eq!(account_backups.len(), 1); @@ -353,14 +368,14 @@ async fn test_backup_describe_list_delete_and_account_scope() { let foreign = unique_test_account(); let foreign_error = engine - .describe_backup(&foreign, &details.backup_arn) + .describe(&foreign, &details.backup_arn) .await .expect_err("another account must not resolve the backup"); assert!( matches!(foreign_error, StorageError::Validation(message) if message.contains("Backup not found")) ); let foreign_source = engine - .backup_restore_source(&foreign, &details.backup_arn) + .restore_source(&foreign, &details.backup_arn) .await .expect_err("another account must not resolve the restore source"); assert!( @@ -368,25 +383,25 @@ async fn test_backup_describe_list_delete_and_account_scope() { ); let deleted = engine - .delete_backup(&table.key_info.account_id, &details.backup_arn) + .delete(&table.key_info.account_id, &details.backup_arn) .await .expect("delete backup"); assert_eq!(deleted.backup_details.backup_status, "DELETED"); assert!( engine - .describe_backup(&table.key_info.account_id, &details.backup_arn) + .describe(&table.key_info.account_id, &details.backup_arn) .await .is_err() ); assert!( engine - .list_backups(&table.key_info.account_id, None) + .list(&table.key_info.account_id, None) .await .expect("list after delete") .is_empty() ); let gone = engine - .backup_restore_source(&table.key_info.account_id, &details.backup_arn) + .restore_source(&table.key_info.account_id, &details.backup_arn) .await .expect_err("a deleted backup is not a restore source"); assert!(matches!(gone, StorageError::Validation(m) if m.contains("Backup not found"))); @@ -412,7 +427,12 @@ async fn test_restore_uses_immutable_snapshot_with_sort_key() { .await; let backup = engine - .create_backup(&account_id, &table.key_info.table_name, "immutable") + .create( + &account_id, + &table.key_info.table_name, + "immutable", + JobPermit::detached(), + ) .await .expect("create backup"); @@ -488,7 +508,7 @@ async fn test_restore_uses_immutable_snapshot_with_sort_key() { .await .expect("delete restored table"); engine - .delete_backup(&account_id, &backup.backup_arn) + .delete(&account_id, &backup.backup_arn) .await .expect("delete backup"); } @@ -518,7 +538,12 @@ async fn test_table_name_filter_spans_table_recreation() { .await .expect("seed original table"); let backup = engine - .create_backup(&account.account_id, table_name, "before-recreate") + .create( + &account.account_id, + table_name, + "before-recreate", + JobPermit::detached(), + ) .await .expect("backup original table"); let original_table_id = source.key_info.table_id.clone(); @@ -542,7 +567,7 @@ async fn test_table_name_filter_spans_table_recreation() { assert_eq!(replacement.key_info.key_schema.len(), 2); let listed = engine - .list_backups(&account.account_id, Some(table_name)) + .list(&account.account_id, Some(table_name)) .await .expect("list backups across table recreation"); assert_eq!(listed.len(), 1); @@ -591,7 +616,7 @@ async fn test_table_name_filter_spans_table_recreation() { .await .expect("delete restored table"); engine - .delete_backup(&account.account_id, &backup.backup_arn) + .delete(&account.account_id, &backup.backup_arn) .await .expect("delete backup"); } @@ -664,11 +689,16 @@ async fn test_restore_reproduces_gsi_and_lsi_contents() { put_all(&engine, &source, items).await; let backup = engine - .create_backup(&account_id, "IndexedSource", "indexed") + .create( + &account_id, + "IndexedSource", + "indexed", + JobPermit::detached(), + ) .await .expect("create backup"); let recorded = engine - .backup_restore_source(&account_id, &backup.backup_arn) + .restore_source(&account_id, &backup.backup_arn) .await .expect("restore source"); let definition = recorded.definition.as_ref().expect("definition recorded"); @@ -781,7 +811,7 @@ async fn test_restore_applies_billing_overrides() { ) .await; let backup = engine - .create_backup(&account_id, "OnDemandSource", "od") + .create(&account_id, "OnDemandSource", "od", JobPermit::detached()) .await .expect("backup on-demand source"); let overrides = RestoreTableOverrides { @@ -840,11 +870,16 @@ async fn test_restore_applies_billing_overrides() { ) .await; let backup = engine - .create_backup(&account_id, "ProvisionedSource", "pv") + .create( + &account_id, + "ProvisionedSource", + "pv", + JobPermit::detached(), + ) .await .expect("backup provisioned source"); let source = engine - .backup_restore_source(&account_id, &backup.backup_arn) + .restore_source(&account_id, &backup.backup_arn) .await .expect("restore source"); assert_eq!(source.billing_mode, "PROVISIONED"); @@ -969,12 +1004,12 @@ async fn test_dropped_restore_request_still_reaches_active() { backed_up_plain_table(&engine, "DroppedSource", ITEMS).await; let account_id = account.account_id.clone(); let source = engine - .backup_restore_source(&account_id, &backup_arn) + .restore_source(&account_id, &backup_arn) .await .expect("restore source"); source.ensure_restorable().expect("restorable"); let input = source.restore_input("DroppedTarget", &RestoreTableOverrides::default()); - let request = engine.restore_table_from_backup( + let request = engine.restore( &account_id, RestoreRequest { backup_arn: backup_arn.clone(), @@ -982,6 +1017,7 @@ async fn test_dropped_restore_request_still_reaches_active() { restore_date_time: source.backup_creation_date_time, input, }, + JobPermit::detached(), ); assert!( request.now_or_never().is_none(), @@ -1042,7 +1078,7 @@ async fn test_backup_and_target_are_held_while_restoring() { let (account, source, backup_arn) = backed_up_plain_table(&engine, "HeldSource", ITEMS).await; let account_id = account.account_id.clone(); let source_backup = engine - .backup_restore_source(&account_id, &backup_arn) + .restore_source(&account_id, &backup_arn) .await .expect("restore source"); @@ -1057,11 +1093,11 @@ async fn test_backup_and_target_are_held_while_restoring() { .expect("restore"); let in_use = engine - .delete_backup(&account_id, &backup_arn) + .delete(&account_id, &backup_arn) .await .expect_err("DeleteBackup during the copy"); assert!( - matches!(in_use, StorageError::BackupInUse(ref m) if m.contains("HeldTarget")), + matches!(in_use, StorageError::BackupInUse(ref m) if m.contains(&backup_arn)), "got {in_use:?}" ); let refused = engine @@ -1130,7 +1166,7 @@ async fn test_backup_and_target_are_held_while_restoring() { .await .expect("DeleteTable after ACTIVE"); engine - .delete_backup(&account_id, &backup_arn) + .delete(&account_id, &backup_arn) .await .expect("DeleteBackup after ACTIVE"); } @@ -1156,7 +1192,12 @@ async fn test_a_restore_is_refused_while_a_target_of_its_backup_is_creating() { let (account, source, b1) = backed_up_plain_table(&engine, "InUseSource", ITEMS).await; let account_id = account.account_id.clone(); let b2 = engine - .create_backup(&account_id, &source.table_name, "second") + .create( + &account_id, + &source.table_name, + "second", + JobPermit::detached(), + ) .await .expect("second backup") .backup_arn; @@ -1167,7 +1208,7 @@ async fn test_a_restore_is_refused_while_a_target_of_its_backup_is_creating() { .expect("first restore"); let second = restore(&engine, &account_id, &b1, "InUseTarget2", &overrides).await; assert!( - matches!(second, Err(StorageError::BackupInUse(ref m)) if m.contains("InUseTarget")), + matches!(second, Err(StorageError::BackupInUse(ref m)) if m.contains(&b1)), "a second restore of a backup with a CREATING target: {second:?}" ); let gone = engine @@ -1209,11 +1250,11 @@ async fn test_a_restore_is_refused_while_a_target_of_its_backup_is_creating() { // The name is checked before the backup: a backup that does not exist, // restored into that name, is refused as the name too. let source = engine - .backup_restore_source(&account_id, &b2) + .restore_source(&account_id, &b2) .await .expect("source"); let missing = engine - .restore_table_from_backup( + .restore( &account_id, RestoreRequest { backup_arn: flip_last_hex(&b2), @@ -1221,6 +1262,7 @@ async fn test_a_restore_is_refused_while_a_target_of_its_backup_is_creating() { restore_date_time: source.backup_creation_date_time, input: source.restore_input("InUseTarget", &overrides), }, + JobPermit::detached(), ) .await; match missing { @@ -1352,7 +1394,7 @@ async fn test_backup_without_definition_restores_as_legacy() { ) .await; let backup = engine - .create_backup(&account_id, "LegacySource", "legacy") + .create(&account_id, "LegacySource", "legacy", JobPermit::detached()) .await .expect("create backup"); @@ -1371,7 +1413,7 @@ async fn test_backup_without_definition_restores_as_legacy() { .expect("strip definition"); let legacy = engine - .backup_restore_source(&account_id, &backup.backup_arn) + .restore_source(&account_id, &backup.backup_arn) .await .expect("legacy restore source"); assert!(legacy.definition.is_none()); @@ -1406,12 +1448,15 @@ async fn test_backup_without_definition_restores_as_legacy() { ); } -/// A restore whose copy stopped heartbeating is removed by the sweep, and -/// the backup it held becomes deletable. Ownership is taken away from the -/// running copy (as if its process had died and another had claimed it) by -/// rewriting the owner and back-dating the heartbeat. +/// A restore whose owner died is found by `list_dead_restore_jobs` only once +/// its heartbeat is older than the stale window, and is then failed and +/// removed through `fail_restore_job` and `remove_restore_target`, after +/// which the backup it held becomes deletable. Ownership is taken away from +/// the running copy (as if its process had died and another had claimed the +/// target) by rewriting the generation and back-dating the heartbeat; the +/// copy notices and does not resurrect the target. #[tokio::test] -async fn test_sweep_removes_abandoned_restore_target() { +async fn test_abandoned_restore_is_removed_only_when_its_heartbeat_is_stale() { if crate::helpers::skip_without_cassandra() { return; } @@ -1429,45 +1474,56 @@ async fn test_sweep_removes_abandoned_restore_target() { ) .await .expect("restore"); + orphan_restore_target(&engine, &account_id, "AbandonedTarget", &target.table_id).await; + let key = RestoreTargetKey { + account_id: account_id.clone(), + table_name: "AbandonedTarget".to_owned(), + table_id: target.table_id.clone(), + }; - let stale = extenddb_storage_cassandra::cassandra_util::now_millis() - 120_000; - let orphan = format!( - "UPDATE {}.tables SET restore_owner = ?, restore_heartbeat = ? \ - WHERE account_id = ? AND table_name = ? IF table_id = ? AND table_status = 'CREATING'", - engine.catalog_keyspace() - ); - let result = extenddb_storage_cassandra::cassandra_util::query_lwt( - engine.session(), - &orphan, - cdrs_tokio::query_values!( - uuid::Uuid::new_v4(), - stale, - account_id.as_str(), - "AbandonedTarget", - target.table_id.as_str() - ), - ) - .await - .expect("orphan the target"); - assert!( - extenddb_storage_cassandra::cassandra_util::lwt_applied(&result).unwrap(), - "the copy must still be running when the target is orphaned" - ); - - // A fresh heartbeat is not swept. + // A heartbeat two minutes old is not stale under a ten-minute window. let not_yet = engine - .sweep_abandoned_restores(Duration::from_secs(600)) + .list_dead_restore_jobs(Duration::from_secs(600)) .await - .expect("sweep"); - assert!(!not_yet.contains(&"AbandonedTarget".to_owned())); - - let removed = engine - .sweep_abandoned_restores(Duration::from_secs(60)) + .expect("list"); + assert!( + !not_yet.iter().any(|job| job.target == key), + "a heartbeat younger than the window is not dead: {not_yet:?}" + ); + let dead = engine + .list_dead_restore_jobs(Duration::from_secs(60)) .await - .expect("sweep"); + .expect("list"); + let job = dead + .iter() + .find(|job| job.target == key) + .unwrap_or_else(|| panic!("the orphaned target is dead under a 60 s window: {dead:?}")); + assert_eq!(job.source_backup_arn, backup_arn); + assert_eq!(job.status, JobStatus::Running); + assert!( + engine + .fail_restore_job(&key, job.owner_generation, job.heartbeat_at, "owner dead") + .await + .expect("fail"), + "the record was RUNNING" + ); + assert!( + !engine + .fail_restore_job(&key, job.owner_generation, job.heartbeat_at, "owner dead") + .await + .expect("fail again"), + "a FAILED record is not RUNNING" + ); assert!( - removed.contains(&"AbandonedTarget".to_owned()), - "removed {removed:?}" + engine.remove_restore_target(&key).await.expect("remove"), + "the target was a CREATING restore target" + ); + assert!( + !engine + .remove_restore_target(&key) + .await + .expect("remove again"), + "a removed target is not one" ); let gone = engine .describe_table( @@ -1477,7 +1533,7 @@ async fn test_sweep_removes_abandoned_restore_target() { }, ) .await - .expect_err("swept target is gone"); + .expect_err("removed target is gone"); assert!(matches!(gone, StorageError::TableNotFound(_))); // The orphaned copy notices it lost the target and does not resurrect it. tokio::time::sleep(Duration::from_secs(2)).await; @@ -1493,7 +1549,7 @@ async fn test_sweep_removes_abandoned_restore_target() { .is_err() ); engine - .delete_backup(&account_id, &backup_arn) + .delete(&account_id, &backup_arn) .await .expect("the abandoned restore no longer holds the backup"); } @@ -1508,16 +1564,16 @@ async fn test_restore_of_deleted_backup_is_not_found() { let engine = setup_engine().await; let (account, _source, backup_arn) = backed_up_plain_table(&engine, "GoneSource", 3).await; let source = engine - .backup_restore_source(&account.account_id, &backup_arn) + .restore_source(&account.account_id, &backup_arn) .await .expect("source"); let input = source.restore_input("GoneTarget", &RestoreTableOverrides::default()); engine - .delete_backup(&account.account_id, &backup_arn) + .delete(&account.account_id, &backup_arn) .await .expect("delete backup"); let error = engine - .restore_table_from_backup( + .restore( &account.account_id, RestoreRequest { backup_arn: backup_arn.clone(), @@ -1525,6 +1581,7 @@ async fn test_restore_of_deleted_backup_is_not_found() { restore_date_time: source.backup_creation_date_time, input, }, + JobPermit::detached(), ) .await .expect_err("restore of a deleted backup"); @@ -1544,10 +1601,15 @@ async fn test_restore_of_deleted_backup_is_not_found() { // ── Isolated catalogs (own keyspace prefix) ──────────────────────────────── -/// An engine on its own keyspace prefix (so its sweeps and workers touch no -/// other test's tables) with one provisioned account. +/// An engine on its own keyspace prefix (so its catalog scans see no other +/// test's tables) with one account. With `before_migration_6` the catalog +/// has run catalog migrations 1 to 5 only, as a catalog not yet migrated by +/// `extenddb migrate --yes`, and the account is not provisioned (see +/// [`provision_account`]); otherwise every migration has run and the account +/// is provisioned. async fn isolated_engine( tag: &str, + before_migration_6: bool, ) -> ( std::sync::Arc, String, @@ -1570,18 +1632,38 @@ async fn isolated_engine( .await .unwrap(); } - extenddb_storage_cassandra::migrations::migrate_catalog_keyspace( + let through = if before_migration_6 { + MigrationId::DropContinuousBackups + } else { + MigrationId::LATEST + }; + extenddb_storage_cassandra::migrations::migrate_catalog_keyspace_through( session.clone(), &config.keyspace_prefix, "test", std::time::Duration::from_secs(120), + through, ) .await .unwrap(); + if !before_migration_6 { + provision_account(&engine, &account_id).await; + } + (engine, account_id, drop_keyspaces) +} + +/// Run the account migrations on `account_id`'s keyspace, which exists, and +/// record the account in the catalog, as CreateAccount does. The account +/// run applies the catalog's pending migrations as well, so a test of a +/// catalog behind the binary provisions its account only once the catalog +/// is migrated. +async fn provision_account(engine: &CassandraEngine, account_id: &str) { + let session = engine.session_arc(); + let catalog_ks = engine.catalog_keyspace(); extenddb_storage_cassandra::migrations::migrate_account( session.clone(), - &config.keyspace_prefix, - &account_ks, + &prefix_of(engine), + &engine.account_keyspace(account_id), "test", ) .await @@ -1592,54 +1674,146 @@ async fn isolated_engine( "INSERT INTO {catalog_ks}.accounts (account_id, account_name, created_at) \ VALUES (?, ?, toTimestamp(now()))" ), - cdrs_tokio::query_values!(account_id.as_str(), "F4 Test"), + cdrs_tokio::query_values!(account_id, "F4 Test"), ) .await .unwrap(); - (engine, account_id, drop_keyspaces) } -/// Back-date the heartbeat of the CREATING restore target `name` by two -/// minutes and give it a foreign owner, as if the process copying into it had -/// died; asserts the copy was still running (the LWT applied). -async fn orphan_restore_target( - engine: &CassandraEngine, - account_id: &str, - name: &str, - table_id: &str, -) { - let stale = extenddb_storage_cassandra::cassandra_util::now_millis() - 120_000; - let orphan = format!( - "UPDATE {}.tables SET restore_owner = ?, restore_heartbeat = ? \ - WHERE account_id = ? AND table_name = ? IF table_id = ? AND table_status = 'CREATING'", - engine.catalog_keyspace() - ); - let result = extenddb_storage_cassandra::cassandra_util::query_lwt( - engine.session(), - &orphan, - cdrs_tokio::query_values!(uuid::Uuid::new_v4(), stale, account_id, name, table_id), +/// The columns and tables catalog migration 6 adds are absent from +/// `keyspace`, and migration 1's restore registration columns are present. +async fn assert_catalog_lacks_migration_6(engine: &CassandraEngine, keyspace: &str) { + let rows = extenddb_storage_cassandra::cassandra_util::query_rows::( + &engine.session_arc(), + "SELECT column_name FROM system_schema.columns WHERE keyspace_name = ?", + cdrs_tokio::query_values!(keyspace), + "columns", ) .await - .expect("orphan the target"); + .expect("read schema"); + let columns: Vec = rows + .iter() + .map(|row| { + extenddb_storage_cassandra::cassandra_util::get_column::( + row, + "column_name", + "columns", + ) + .unwrap() + }) + .collect(); + assert!(columns.iter().any(|c| c == "table_status"), "{columns:?}"); + for baseline in ["restore_source_backup_arn", "active_restores", "definition"] { + assert!(columns.iter().any(|c| c == baseline), "{baseline} absent"); + } + for added in [ + "restore_status", + "restore_owner_generation", + "restore_heartbeat_at", + "store_kind", + "source_table_details_json", + ] { + assert!(!columns.iter().any(|c| c == added), "{added} present"); + } + let tables = extenddb_storage_cassandra::cassandra_util::query_rows::( + &engine.session_arc(), + "SELECT table_name FROM system_schema.tables WHERE keyspace_name = ?", + cdrs_tokio::query_values!(keyspace), + "tables", + ) + .await + .expect("read schema"); + let names: Vec = tables + .iter() + .map(|row| { + extenddb_storage_cassandra::cassandra_util::get_column::( + row, + "table_name", + "tables", + ) + .unwrap() + }) + .collect(); + for added in ["backup_index_by_status", "backup_maintenance"] { + assert!(!names.iter().any(|n| n == added), "{added} present"); + } assert!( - extenddb_storage_cassandra::cassandra_util::lwt_applied(&result).unwrap(), - "the copy must still be running when the target is orphaned" + engine.catalog_id().await.is_err(), + "a catalog without migration 6 has no catalog id" ); } -/// The server's sweep worker sweeps once as soon as it starts (not one -/// interval later) and returns promptly once shutdown is signalled. +/// `serve` does not start on a catalog without migration 6: the serving +/// registration reads the ledgers and refuses a catalog behind the binary, +/// naming the pending migration, and leaves no registration behind. Once +/// `migrate --yes` has applied it, the same deployment registers, the +/// catalog has its id, and backups work. #[tokio::test] -async fn test_restore_sweep_worker_sweeps_at_startup_and_stops_on_shutdown() { +async fn test_serve_refuses_a_catalog_without_migration_6_until_it_is_migrated() { if crate::helpers::skip_without_cassandra() { return; } - let (engine, account_id, _drop_keyspaces) = isolated_engine("sw").await; - let source = create_active_table( + let (engine, account_id, _drop_keyspaces) = isolated_engine("v5", true).await; + let catalog = engine.catalog_keyspace(); + let prefix = prefix_of(&engine); + assert_catalog_lacks_migration_6(&engine, &catalog).await; + + let refused = extenddb_storage_cassandra::migrations::register_server( + engine.session_arc(), + &prefix, + "t", + "test", + ) + .await + .err() + .expect("serve refuses a catalog behind the binary"); + match &refused { + MigrationError::Status(CatalogStatus::Behind { version, pending }) => { + assert_eq!(*version, Some(MigrationId::DropContinuousBackups.number())); + assert!( + pending + .iter() + .any(|p| p.id == MigrationId::BackupCatalogIndex.number()), + "{refused}" + ); + } + other => panic!("expected a Behind refusal, got {other}"), + } + assert!( + refused.to_string().contains("backup_catalog_index"), + "{refused}" + ); + + // Liveness: once migration 6 is applied, as `migrate --yes` applies it, + // the deployment registers, the catalog has its id, and the same engine + // backs up without a restart. + extenddb_storage_cassandra::migrations::migrate_catalog_keyspace( + engine.session_arc(), + &prefix, + "test", + std::time::Duration::from_secs(120), + ) + .await + .expect("apply migration 6"); + provision_account(&engine, &account_id).await; + assert!( + uuid::Uuid::parse_str(&engine.catalog_id().await.expect("catalog id")).is_ok(), + "the migration writes a catalog id" + ); + let registration = extenddb_storage_cassandra::migrations::register_server( + engine.session_arc(), + &prefix, + "t", + "test", + ) + .await + .expect("serve accepts the migrated catalog"); + extenddb_storage::hooks::ServingRegistrar::release(®istration).await; + let created = create_active_table( &engine, &account_id, CreateTableInput { - table_name: "SweepSource".to_owned(), + table_name: "Migrated".to_owned(), key_schema: vec![ks("id", KeyType::Hash)], attribute_definitions: vec![attr("id", ScalarAttributeType::S)], billing_mode: Some(BillingMode::PayPerRequest), @@ -1647,63 +1821,50 @@ async fn test_restore_sweep_worker_sweeps_at_startup_and_stops_on_shutdown() { }, ) .await; - put_all( - &engine, - &source, - (0..1_500) - .map(|i| item(&format!("item-{i:05}"), None, &"x".repeat(200))) - .collect(), - ) - .await; + put_all(&engine, &created, vec![item("a", None, "v")]).await; let backup = engine - .create_backup(&account_id, "SweepSource", "sweep") + .create( + &account_id, + "Migrated", + "after-migration-6", + JobPermit::detached(), + ) .await - .expect("create backup"); - let target = restore( - &engine, - &account_id, - &backup.backup_arn, - "SweepTarget", - &RestoreTableOverrides::default(), + .expect("CreateBackup after migration 6"); + engine + .delete(&account_id, &backup.backup_arn) + .await + .expect("DeleteBackup after migration 6"); +} + +/// Back-date the heartbeat of the CREATING restore target `name` by two +/// minutes and give its job record another generation, as if the process +/// copying into it had died and another had claimed it; asserts the copy was +/// still running (the LWT applied). +async fn orphan_restore_target( + engine: &CassandraEngine, + account_id: &str, + name: &str, + table_id: &str, +) { + let stale = extenddb_storage_cassandra::cassandra_util::now_millis() - 120_000; + let orphan = format!( + "UPDATE {}.tables SET restore_owner_generation = ?, restore_heartbeat_at = ? \ + WHERE account_id = ? AND table_name = ? IF table_id = ? AND table_status = 'CREATING'", + engine.catalog_keyspace() + ); + let foreign_generation = 1_i64; + let result = extenddb_storage_cassandra::cassandra_util::query_lwt( + engine.session(), + &orphan, + cdrs_tokio::query_values!(foreign_generation, stale, account_id, name, table_id), ) .await - .expect("restore"); - orphan_restore_target(&engine, &account_id, "SweepTarget", &target.table_id).await; - - let shutdown = extenddb_storage::CancellationToken::new(); - let started = Instant::now(); - let worker = tokio::spawn(extenddb_storage_cassandra::poll_abandoned_restores( - std::sync::Arc::clone(&engine), - shutdown.clone(), - )); - // Well inside RESTORE_SWEEP_INTERVAL (15 s): only a sweep at startup - // removes the target this soon. - let deadline = started + Duration::from_secs(8); - loop { - let described = engine - .describe_table( - &account_id, - DescribeTableInput { - table_name: "SweepTarget".to_owned(), - }, - ) - .await; - if matches!(described, Err(StorageError::TableNotFound(_))) { - break; - } - assert!( - Instant::now() < deadline, - "the abandoned target was not swept within 8 s of the worker starting: {:?}", - described.map(|d| d.table_status) - ); - tokio::time::sleep(Duration::from_millis(100)).await; - } - - shutdown.cancel(); - tokio::time::timeout(Duration::from_secs(3), worker) - .await - .expect("the sweep worker did not stop within 3 s of shutdown") - .expect("sweep worker panicked"); + .expect("orphan the target"); + assert!( + extenddb_storage_cassandra::cassandra_util::lwt_applied(&result).unwrap(), + "the copy must still be running when the target is orphaned" + ); } /// The names of the tables in `keyspace`, read from `system_schema`. @@ -1727,9 +1888,9 @@ async fn table_names_in(engine: &CassandraEngine, keyspace: &str) -> BTreeSet = row.get_by_name("created_at").ok().flatten(); - let heartbeat: Option = - row.get_by_name("restore_heartbeat").ok().flatten(); - if heartbeat > created { + let beat: Option = row.get_by_name("beat").ok().flatten(); + let inserted: Option = row.get_by_name("inserted").ok().flatten(); + if let (Some(beat), Some(inserted)) = (beat, inserted) + && beat > inserted + { let tables = table_names_in(&engine, &account_keyspace).await; target_tables_at_first_beat = Some(tables.difference(&source_tables).count()); @@ -1885,3 +2047,7086 @@ async fn test_restore_heartbeat_runs_during_target_ddl() { ) .await; } + +// ── The catalog index ─────────────────────────────────────────────────────── + +/// A store identity no other test's rows carry, so a store-scoped listing +/// in the shared catalog keyspace sees only this test's rows. +fn unique_store() -> StoreIdentity { + StoreIdentity { + store_kind: StoreKind::Filesystem, + store_root: format!( + "/var/lib/extenddb/backups-{}", + crate::helpers::unique_test_id() + ), + store_id: "store-1".to_owned(), + } +} + +/// A row for a backup of `table` in `account` minted now, with the manifest +/// fixture's source table. +fn index_row(account: &str, table: &str, name: &str, store: &StoreIdentity) -> BackupRow { + let backup_id = extenddb_storage::util::new_backup_id(); + let backup_arn = extenddb_storage::util::backup_arn("us-east-1", account, table, &backup_id); + let manifest: serde_json::Value = serde_json::from_str(include_str!( + "../../../core/testdata/extenddb-manifest/v1.json" + )) + .expect("fixture"); + BackupRow { + backup_id: backup_id.clone(), + account_id: account.to_owned(), + table_name: table.to_owned(), + backup_arn, + backup_name: name.to_owned(), + status: BackupStatus::Creating, + store: store.clone(), + size_bytes: 0, + item_count: 0, + created_at: extenddb_storage::util::backup_creation_date_time(&backup_id).unwrap(), + completed_at: None, + failure_reason: None, + source_table_details_json: manifest["source_table"].clone(), + owner_generation: 0, + heartbeat_at: 0.0, + } +} + +fn arn_of(row: &BackupRow) -> BackupArn { + BackupArn::parse(&row.backup_arn).unwrap() +} + +/// Delete `rows` from `backups_by_arn` and their entries from the status +/// index and the two listings, whatever their status, so a test that wrote +/// rows outside a claim leaves none in the shared catalog keyspace. +async fn remove_index_rows(engine: &CassandraEngine, rows: &[&BackupRow]) { + let catalog = engine.catalog_keyspace(); + for row in rows { + let created_at = (row.created_at * 1_000.0).round() as i64; + for (cql, values) in [ + ( + format!( + "DELETE FROM {catalog}.backups_by_arn WHERE account_id = ? AND backup_arn = ?" + ), + cdrs_tokio::query_values!(row.account_id.as_str(), row.backup_arn.as_str()), + ), + ( + format!( + "DELETE FROM {catalog}.backups_by_account \ + WHERE account_id = ? AND created_at = ? AND backup_arn = ?" + ), + cdrs_tokio::query_values!( + row.account_id.as_str(), + created_at, + row.backup_arn.as_str() + ), + ), + ( + format!( + "DELETE FROM {catalog}.backups_by_table WHERE account_id = ? \ + AND table_name = ? AND created_at = ? AND backup_arn = ?" + ), + cdrs_tokio::query_values!( + row.account_id.as_str(), + row.table_name.as_str(), + created_at, + row.backup_arn.as_str() + ), + ), + ] { + engine + .session() + .query_with_values(&cql, values) + .await + .expect("remove a backup row"); + } + for status in [ + BackupStatus::Creating, + BackupStatus::Available, + BackupStatus::Deleting, + BackupStatus::Deleted, + BackupStatus::Failed, + ] { + engine + .session() + .query_with_values( + &format!( + "DELETE FROM {catalog}.backup_index_by_status \ + WHERE store_root = ? AND backup_status = ? AND backup_arn = ?" + ), + cdrs_tokio::query_values!( + row.store.store_root.as_str(), + status.as_str(), + row.backup_arn.as_str() + ), + ) + .await + .expect("remove a status entry"); + } + } +} + +/// Run `cql` with `values` as a lightweight transaction and assert it applied. +async fn apply(engine: &CassandraEngine, cql: &str, values: cdrs_tokio::query::QueryValues) { + let result = + extenddb_storage_cassandra::cassandra_util::query_lwt(engine.session(), cql, values) + .await + .expect("lwt"); + assert!( + extenddb_storage_cassandra::cassandra_util::lwt_applied(&result).unwrap(), + "{cql}" + ); +} + +/// The claim: a second claim of the same backup is refused while the row +/// exists, the heartbeat confirms the claim and refreshes the row, a status +/// write is conditioned on the generation and the `from` status, and the row +/// is read back by account, backup id, table and ARN. +#[tokio::test] +async fn test_claim_backup_excludes_a_second_owner_and_fences_its_writes() { + if crate::helpers::skip_without_cassandra() { + return; + } + let engine = setup_engine().await; + let account = TestAccount::new(&engine, "extenddb_ttl_test").await; + let account_id = account.account_id.clone(); + assert!( + uuid::Uuid::parse_str(&engine.catalog_id().await.expect("id")).is_ok(), + "the catalog has an id from the migration" + ); + let store = unique_store(); + let row = index_row(&account_id, "Music", "nightly", &store); + let claim = engine.claim_backup(&row).await.expect("claim"); + assert_ne!(claim.generation(), UNOWNED_GENERATION); + let second = engine.claim_backup(&row).await.err(); + assert!( + matches!(second, Some(StorageError::AlreadyExists(_))), + "{second:?}" + ); + let stored = engine + .read_backup_row(&arn_of(&row)) + .await + .expect("read") + .expect("the row exists"); + assert_eq!(stored.status, BackupStatus::Creating); + assert_eq!(stored.owner_generation, claim.generation()); + assert_eq!(stored.store, store); + assert_eq!(stored.created_at, row.created_at); + assert_eq!(stored.backup_id, row.backup_id); + assert_eq!( + stored.source_table_details_json, + row.source_table_details_json + ); + let before = stored.heartbeat_at; + assert!(before > 0.0, "the insert stamps the heartbeat"); + tokio::time::sleep(Duration::from_millis(20)).await; + assert_eq!( + claim.heartbeat().await.expect("heartbeat"), + Heartbeat::Confirmed + ); + let after = engine + .read_backup_row(&arn_of(&row)) + .await + .unwrap() + .unwrap() + .heartbeat_at; + assert!(after > before, "{after} > {before}"); + + // The same id under another account, table or region finds nothing; the + // legacy reader does not see the row either. + let other_account = unique_test_account(); + let other_account_arn = + BackupArn::new("aws", "us-east-1", &other_account, "Music", &row.backup_id).unwrap(); + assert!( + engine + .read_backup_row(&other_account_arn) + .await + .unwrap() + .is_none() + ); + let other_table = + BackupArn::new("aws", "us-east-1", &account_id, "Other", &row.backup_id).unwrap(); + assert!( + engine + .read_backup_row(&other_table) + .await + .unwrap() + .is_none() + ); + let other_region = + BackupArn::new("aws", "eu-west-1", &account_id, "Music", &row.backup_id).unwrap(); + assert!( + engine + .read_backup_row(&other_region) + .await + .unwrap() + .is_none() + ); + assert!( + matches!( + engine.describe(&account_id, &row.backup_arn).await, + Err(StorageError::Validation(ref m)) if m.contains("Backup not found") + ), + "an index row is not an in-catalog backup" + ); + + // A write from the wrong state matches nothing; the right one matches + // once, and the row then carries what it set. + let fields = StatusFields { + size_bytes: Some(1234), + item_count: Some(5), + failure_reason: None, + source_table_details_json: None, + completed: true, + }; + assert_eq!( + claim + .set_status(JobStatus::Running, JobStatus::Available, &fields) + .await + .unwrap(), + WriteOutcome::NotMatched + ); + assert_eq!( + claim + .set_status(JobStatus::Creating, JobStatus::Available, &fields) + .await + .unwrap(), + WriteOutcome::Matched + ); + assert_eq!( + claim + .set_status(JobStatus::Creating, JobStatus::Available, &fields) + .await + .unwrap(), + WriteOutcome::NotMatched + ); + let read = claim.read().await.expect("read"); + assert_eq!(read.status, JobStatus::Available); + assert_eq!(read.owner_generation, claim.generation()); + let stored = engine + .read_backup_row(&arn_of(&row)) + .await + .unwrap() + .unwrap(); + assert_eq!(stored.status, BackupStatus::Available); + assert_eq!(stored.size_bytes, 1234); + assert_eq!(stored.item_count, 5); + assert!(stored.completed_at.is_some()); + // The status index followed the row. + let available: Vec = engine + .list_backup_rows(Some(BackupStatus::Available), &store) + .try_collect() + .await + .unwrap(); + assert_eq!(available.len(), 1, "{available:?}"); + assert!( + engine + .list_backup_rows(Some(BackupStatus::Creating), &store) + .try_collect::>() + .await + .unwrap() + .is_empty() + ); + // The legacy listing does not show the index row. + assert!( + engine + .list(&account_id, None) + .await + .expect("legacy list") + .is_empty() + ); + + // A row another generation took over confirms nothing for this claim. + apply( + &engine, + &format!( + "UPDATE {}.backups_by_arn SET owner_generation = ? \ + WHERE account_id = ? AND backup_arn = ? IF owner_generation = ?", + engine.catalog_keyspace() + ), + cdrs_tokio::query_values!( + i64::try_from(claim.generation()).unwrap() + 1, + account_id.as_str(), + row.backup_arn.as_str(), + i64::try_from(claim.generation()).unwrap() + ), + ) + .await; + assert_eq!(claim.heartbeat().await.expect("heartbeat"), Heartbeat::Lost); + assert_eq!( + claim + .set_status( + JobStatus::Available, + JobStatus::Failed, + &StatusFields::default() + ) + .await + .unwrap(), + WriteOutcome::NotMatched + ); + drop(claim); + // The row is the claim: a successor can take it only through its insert, + // which the existing row refuses. + let third = engine.claim_backup(&row).await.err(); + assert!( + matches!(third, Some(StorageError::AlreadyExists(_))), + "{third:?}" + ); +} + +/// DeleteBackup's first step and the tombstone writes: only an AVAILABLE row +/// moves to DELETING; a CREATING row answers its status; a backup a CREATING +/// restore target reads answers the target's name; DELETING moves to DELETED +/// once; a DELETED row is removed once, with its list and status entries; a +/// missing row is "Backup not found". +#[tokio::test] +async fn test_start_backup_delete_checks_the_status_and_the_in_use_rule_under_the_lock() { + if crate::helpers::skip_without_cassandra() { + return; + } + let engine = setup_engine().await; + let account = TestAccount::new(&engine, "extenddb_ttl_test").await; + let account_id = account.account_id.clone(); + let store = unique_store(); + let row = index_row(&account_id, "Music", "nightly", &store); + let key = row.key(); + assert!(matches!( + engine.start_backup_delete(&key).await, + Err(StorageError::Validation(m)) if m.contains("Backup not found") + )); + let claim = engine.claim_backup(&row).await.expect("claim"); + assert_eq!( + engine.start_backup_delete(&key).await.unwrap(), + DeleteStart::Status(BackupStatus::Creating) + ); + assert_eq!( + claim + .set_status( + JobStatus::Creating, + JobStatus::Available, + &StatusFields::default() + ) + .await + .unwrap(), + WriteOutcome::Matched + ); + assert_eq!(engine.backup_in_use(&key).await.unwrap(), None); + + // A CREATING restore target registered from the backup: in use, naming + // the target; an ACTIVE one is not, and its registration is cleared. + let target = create_active_table( + &engine, + &account_id, + CreateTableInput { + table_name: "DelTarget".to_owned(), + key_schema: vec![ks("id", KeyType::Hash)], + attribute_definitions: vec![attr("id", ScalarAttributeType::S)], + billing_mode: Some(BillingMode::PayPerRequest), + ..Default::default() + }, + ) + .await; + let catalog = engine.catalog_keyspace(); + apply( + &engine, + &format!( + "UPDATE {catalog}.tables SET table_status = 'CREATING', status_transition_at = null, \ + restore_source_backup_arn = ? WHERE account_id = ? AND table_name = ? IF table_id = ?" + ), + cdrs_tokio::query_values!( + row.backup_arn.as_str(), + account_id.as_str(), + "DelTarget", + target.table_id.as_str() + ), + ) + .await; + apply( + &engine, + &format!( + "UPDATE {catalog}.backups_by_arn SET active_restores[?] = ? \ + WHERE account_id = ? AND backup_arn = ? IF EXISTS" + ), + cdrs_tokio::query_values!( + target.table_id.as_str(), + "DelTarget", + account_id.as_str(), + row.backup_arn.as_str() + ), + ) + .await; + assert_eq!( + engine.backup_in_use(&key).await.unwrap().as_deref(), + Some("DelTarget") + ); + assert_eq!( + engine.start_backup_delete(&key).await.unwrap(), + DeleteStart::InUse("DelTarget".to_owned()) + ); + assert_eq!( + engine.forget_available_backup_row(&key).await.unwrap(), + extenddb_storage::backup_catalog::ForgetRow::InUse("DelTarget".to_owned()), + "the outright removal keeps the in-use rule" + ); + assert_eq!( + engine + .read_backup_row(&arn_of(&row)) + .await + .unwrap() + .unwrap() + .status, + BackupStatus::Available, + "a refused delete leaves the row AVAILABLE" + ); + apply( + &engine, + &format!( + "UPDATE {catalog}.tables SET table_status = 'ACTIVE' \ + WHERE account_id = ? AND table_name = ? IF table_id = ?" + ), + cdrs_tokio::query_values!(account_id.as_str(), "DelTarget", target.table_id.as_str()), + ) + .await; + assert_eq!(engine.backup_in_use(&key).await.unwrap(), None); + + assert_eq!( + engine.start_backup_delete(&key).await.unwrap(), + DeleteStart::Deleting + ); + assert_eq!( + engine.start_backup_delete(&key).await.unwrap(), + DeleteStart::Status(BackupStatus::Deleting), + "a retry finds the row DELETING" + ); + assert_eq!( + engine.forget_available_backup_row(&key).await.unwrap(), + extenddb_storage::backup_catalog::ForgetRow::Status(BackupStatus::Deleting), + "only an AVAILABLE row is removed outright" + ); + // An AVAILABLE row goes outright, read and listed nowhere, with no + // tombstone; a second removal finds no row. + let mut adopted = index_row(&account_id, "Music", "adopted", &store); + adopted.status = BackupStatus::Available; + assert!(engine.insert_backup_row(&adopted).await.expect("insert")); + assert_eq!( + engine + .forget_available_backup_row(&adopted.key()) + .await + .unwrap(), + extenddb_storage::backup_catalog::ForgetRow::Removed + ); + assert!( + engine + .read_backup_row(&arn_of(&adopted)) + .await + .unwrap() + .is_none() + ); + for status in [None, Some(BackupStatus::Available)] { + let listed: Vec = engine + .list_backup_rows(status, &adopted.store) + .try_collect() + .await + .unwrap(); + assert!( + !listed.iter().any(|r| r.backup_arn == adopted.backup_arn), + "{status:?}" + ); + } + let by_account: Vec = engine + .list_account_backup_rows(&adopted.account_id, None, &TimeBounds::default()) + .try_collect() + .await + .unwrap(); + assert!( + !by_account + .iter() + .any(|r| r.backup_arn == adopted.backup_arn) + ); + assert_eq!( + raw_index_entries(&engine, &adopted).await, + 0, + "the list and status entries went with the row" + ); + assert!(matches!( + engine.forget_available_backup_row(&adopted.key()).await, + Err(StorageError::Validation(m)) if m.contains("Backup not found") + )); + assert!( + !engine.remove_backup_row(&key).await.unwrap(), + "DELETING is not a tombstone" + ); + assert!(engine.set_backup_deleted(&key).await.unwrap()); + assert!(!engine.set_backup_deleted(&key).await.unwrap()); + let stored = engine + .read_backup_row(&arn_of(&row)) + .await + .unwrap() + .unwrap(); + assert_eq!(stored.status, BackupStatus::Deleted); + assert!(stored.completed_at.is_some()); + let deleted: Vec = engine + .list_backup_rows(Some(BackupStatus::Deleted), &store) + .try_collect() + .await + .unwrap(); + assert_eq!(deleted.len(), 1, "the status index followed: {deleted:?}"); + assert!(engine.remove_backup_row(&key).await.unwrap()); + assert!(!engine.remove_backup_row(&key).await.unwrap()); + assert!( + engine + .read_backup_row(&arn_of(&row)) + .await + .unwrap() + .is_none() + ); + assert!( + engine + .list_backup_rows(None, &store) + .try_collect::>() + .await + .unwrap() + .is_empty(), + "the status entry went with the row" + ); + assert!( + engine + .list_account_backup_rows(&account_id, None, &TimeBounds::default()) + .try_collect::>() + .await + .unwrap() + .is_empty(), + "the list entries went with the row" + ); + drop(claim); + engine + .delete_table( + &account_id, + DeleteTableInput { + table_name: "DelTarget".to_owned(), + }, + ) + .await + .expect("delete the target"); +} + +/// The listings: by account with an optional table and the time bounds +/// (lower inclusive, upper exclusive) in (created_at, ARN) order, and by +/// store and status. +#[tokio::test] +async fn test_the_catalog_listings_are_scoped_and_ordered() { + if crate::helpers::skip_without_cassandra() { + return; + } + let engine = setup_engine().await; + let account = TestAccount::new(&engine, "extenddb_ttl_test").await; + let other = TestAccount::new(&engine, "extenddb_ttl_test").await; + let account_id = account.account_id.clone(); + let other_id = other.account_id.clone(); + let store = unique_store(); + let mut rows = Vec::new(); + for (account, table, name) in [ + (account_id.as_str(), "Music", "a"), + (account_id.as_str(), "Music", "b"), + (account_id.as_str(), "Other", "c"), + (other_id.as_str(), "Music", "d"), + ] { + tokio::time::sleep(Duration::from_millis(5)).await; + let row = index_row(account, table, name, &store); + let claim = engine.claim_backup(&row).await.expect("claim"); + if name == "b" { + assert_eq!( + claim + .set_status( + JobStatus::Creating, + JobStatus::Failed, + &StatusFields::default() + ) + .await + .unwrap(), + WriteOutcome::Matched + ); + } + rows.push((row, claim)); + } + let names = |listed: Vec| -> Vec { + listed.into_iter().map(|r| r.backup_name).collect() + }; + let all: Vec = engine + .list_account_backup_rows(&account_id, None, &TimeBounds::default()) + .try_collect() + .await + .unwrap(); + assert_eq!( + names(all.clone()), + ["a", "b", "c"], + "every status, in creation order" + ); + assert_eq!(all[1].status, BackupStatus::Failed); + let music: Vec = engine + .list_account_backup_rows(&account_id, Some("Music"), &TimeBounds::default()) + .try_collect() + .await + .unwrap(); + assert_eq!(names(music), ["a", "b"]); + let bounds = TimeBounds { + lower: Some(rows[1].0.created_at), + upper: Some(rows[2].0.created_at), + }; + let bounded: Vec = engine + .list_account_backup_rows(&account_id, None, &bounds) + .try_collect() + .await + .unwrap(); + assert_eq!(names(bounded), ["b"], "lower inclusive, upper exclusive"); + let others: Vec = engine + .list_account_backup_rows(&other_id, None, &TimeBounds::default()) + .try_collect() + .await + .unwrap(); + assert_eq!(names(others), ["d"]); + + let by_store: Vec = engine + .list_backup_rows(None, &store) + .try_collect() + .await + .unwrap(); + assert_eq!(names(by_store), ["a", "b", "c", "d"]); + let failed: Vec = engine + .list_backup_rows(Some(BackupStatus::Failed), &store) + .try_collect() + .await + .unwrap(); + assert_eq!(names(failed), ["b"]); + let elsewhere = StoreIdentity { + store_id: "store-2".to_owned(), + ..store.clone() + }; + let none: Vec = engine + .list_backup_rows(None, &elsewhere) + .try_collect() + .await + .unwrap(); + assert!(none.is_empty(), "another store's rows are not listed"); + drop(rows); +} + +/// The restore job record of a target, as `(status, kind, generation, +/// heartbeat_at ms, completed_at ms)`. +async fn job_record( + engine: &CassandraEngine, + account_id: &str, + name: &str, +) -> ( + Option, + Option, + Option, + Option, + Option, +) { + use cdrs_tokio::types::IntoRustByName; + let rows = extenddb_storage_cassandra::cassandra_util::query_rows_quorum::( + &engine.session_arc(), + &format!( + "SELECT restore_status, restore_kind, restore_owner_generation, restore_heartbeat_at, \ + restore_completed_at FROM {}.tables WHERE account_id = ? AND table_name = ?", + engine.catalog_keyspace() + ), + cdrs_tokio::query_values!(account_id, name), + "job record", + ) + .await + .expect("read the record"); + let row = rows.first().expect("the target exists"); + ( + row.get_by_name("restore_status").ok().flatten(), + row.get_by_name("restore_kind").ok().flatten(), + row.get_by_name("restore_owner_generation").ok().flatten(), + row.get_by_name("restore_heartbeat_at").ok().flatten(), + row.get_by_name("restore_completed_at").ok().flatten(), + ) +} + +/// A catalog at the shape before migration 6 holding a finished restore and a +/// restore the stop interrupted (its target CREATING) migrates with the +/// registrations backfilled: the finished one DONE, the interrupted one +/// RUNNING with the heartbeat at the epoch and the unowned generation, so the +/// reconciler's first pass fails it and removes its target. A second run of +/// the migration keeps the catalog id. +#[tokio::test] +async fn test_migration_6_backfills_the_registrations_and_the_reconciler_removes_the_interrupted_one() + { + if crate::helpers::skip_without_cassandra() { + return; + } + let (engine, account_id, _drop_keyspaces) = isolated_engine("bf", false).await; + let source = create_active_table( + &engine, + &account_id, + CreateTableInput { + table_name: "BfSource".to_owned(), + key_schema: vec![ks("id", KeyType::Hash)], + attribute_definitions: vec![attr("id", ScalarAttributeType::S)], + billing_mode: Some(BillingMode::PayPerRequest), + ..Default::default() + }, + ) + .await; + put_all( + &engine, + &source, + (0..40).map(|i| item(&format!("i{i}"), None, "v")).collect(), + ) + .await; + let backup = engine + .create(&account_id, "BfSource", "b", JobPermit::detached()) + .await + .expect("backup"); + let overrides = RestoreTableOverrides::default(); + let finished = restore( + &engine, + &account_id, + &backup.backup_arn, + "BfDone", + &overrides, + ) + .await + .expect("restore"); + wait_active(&engine, &account_id, "BfDone", Duration::from_secs(60)).await; + let interrupted = restore( + &engine, + &account_id, + &backup.backup_arn, + "BfCut", + &overrides, + ) + .await + .expect("restore"); + wait_active(&engine, &account_id, "BfCut", Duration::from_secs(60)).await; + let catalog_id = engine.catalog_id().await.expect("id"); + + // Back to the shape before migration 6: the registrations as the + // previous release wrote them (no job record), the interrupted target + // CREATING, and the migration forgotten. + let catalog = engine.catalog_keyspace(); + for (name, table_id) in [ + ("BfDone", finished.table_id.as_str()), + ("BfCut", interrupted.table_id.as_str()), + ] { + apply( + &engine, + &format!( + "UPDATE {catalog}.tables SET restore_status = null, restore_kind = null, \ + restore_owner_generation = null, restore_heartbeat_at = null, \ + restore_completed_at = null WHERE account_id = ? AND table_name = ? \ + IF table_id = ?" + ), + cdrs_tokio::query_values!(account_id.as_str(), name, table_id), + ) + .await; + } + apply( + &engine, + &format!( + "UPDATE {catalog}.tables SET table_status = 'CREATING', status_transition_at = null \ + WHERE account_id = ? AND table_name = ? IF table_id = ?" + ), + cdrs_tokio::query_values!(account_id.as_str(), "BfCut", interrupted.table_id.as_str()), + ) + .await; + let prefix = prefix_of(&engine); + forget_catalog_migration(&engine, &prefix, MigrationId::BackupCatalogIndex).await; + let report = extenddb_storage_cassandra::migrations::migrate_catalog_keyspace( + engine.session_arc(), + &prefix, + "test", + Duration::from_secs(120), + ) + .await + .expect("migrate again"); + let applied: Vec<(u32, Vec)> = report + .applied + .iter() + .map(|step| (step.id, step.notes.clone())) + .collect(); + assert_eq!( + applied, + [( + MigrationId::BackupCatalogIndex.number(), + vec![ + "backfilled 2 restore registration(s) as legacy restores, 1 of them RUNNING \ + with a dead owner for the reconciler to remove" + .to_owned() + ] + )], + "the catalog id is kept" + ); + + let (status, kind, generation, heartbeat, completed) = + job_record(&engine, &account_id, "BfCut").await; + assert_eq!(status.as_deref(), Some("RUNNING")); + assert_eq!(kind.as_deref(), Some("legacy")); + assert_eq!(generation, Some(i64::try_from(UNOWNED_GENERATION).unwrap())); + assert_eq!(heartbeat, Some(0), "the epoch, not null"); + assert_eq!(completed, None); + let (status, kind, _, _, completed) = job_record(&engine, &account_id, "BfDone").await; + assert_eq!(status.as_deref(), Some("DONE")); + assert_eq!(kind.as_deref(), Some("legacy")); + #[allow(clippy::cast_possible_truncation)] + let created_ms = (finished.creation_date_time * 1_000.0).round() as i64; + assert_eq!( + completed, + Some(created_ms), + "completed when the table was created" + ); + assert_eq!( + engine.catalog_id().await.expect("id"), + catalog_id, + "a second run keeps the catalog id" + ); + + // The first reconciler pass: the interrupted restore is dead. + let dead = engine + .list_dead_restore_jobs(Duration::from_secs(300)) + .await + .expect("list"); + assert_eq!(dead.len(), 1, "{dead:?}"); + assert_eq!( + dead[0].target, + RestoreTargetKey { + account_id: account_id.clone(), + table_name: "BfCut".to_owned(), + table_id: interrupted.table_id.clone(), + } + ); + assert_eq!(dead[0].owner_generation, UNOWNED_GENERATION); + assert_eq!(dead[0].source_backup_arn, backup.backup_arn); + assert!( + engine + .fail_restore_job( + &dead[0].target, + dead[0].owner_generation, + dead[0].heartbeat_at, + "owner dead" + ) + .await + .unwrap() + ); + assert!(engine.remove_restore_target(&dead[0].target).await.unwrap()); + assert!(matches!( + engine + .describe_table( + &account_id, + DescribeTableInput { + table_name: "BfCut".to_owned(), + }, + ) + .await, + Err(StorageError::TableNotFound(_)) + )); + assert!( + engine.table_key_info(&account_id, "BfCut").await.is_err(), + "the target's data went with it" + ); + assert_eq!( + describe(&engine, &account_id, "BfDone").await.table_status, + TableStatus::Active + ); + assert_eq!( + describe(&engine, &account_id, "BfSource") + .await + .table_status, + TableStatus::Active + ); + engine + .delete(&account_id, &backup.backup_arn) + .await + .expect("the removed target released the backup"); +} + +/// The legacy restore registers through the job record: RUNNING under the +/// claim's generation with a fresh heartbeat while the copy runs, DONE with +/// the ACTIVE flip. +#[tokio::test] +async fn test_a_legacy_restore_registers_a_running_job_record_and_finishes_it_done() { + if crate::helpers::skip_without_cassandra() { + return; + } + const ITEMS: usize = 1_500; + let engine = setup_engine().await; + let (account, _source, backup_arn) = backed_up_plain_table(&engine, "JrSource", ITEMS).await; + let account_id = account.account_id.clone(); + let target = restore( + &engine, + &account_id, + &backup_arn, + "JrTarget", + &RestoreTableOverrides::default(), + ) + .await + .expect("registers"); + let (status, kind, generation, heartbeat, completed) = + job_record(&engine, &account_id, "JrTarget").await; + assert_eq!(status.as_deref(), Some("RUNNING")); + assert_eq!(kind.as_deref(), Some("legacy")); + assert_ne!(generation, Some(i64::try_from(UNOWNED_GENERATION).unwrap())); + assert!(generation.is_some_and(|g| g > 0)); + let now = extenddb_storage_cassandra::cassandra_util::now_millis(); + let age = now - heartbeat.expect("a heartbeat"); + assert!(age.abs() < 60_000, "a fresh heartbeat: {age} ms old"); + assert_eq!(completed, None); + let key = RestoreTargetKey { + account_id: account_id.clone(), + table_name: "JrTarget".to_owned(), + table_id: target.table_id.clone(), + }; + assert!( + !engine + .list_dead_restore_jobs(Duration::from_secs(300)) + .await + .unwrap() + .iter() + .any(|job| job.target == key), + "a running restore is not dead" + ); + wait_active(&engine, &account_id, "JrTarget", Duration::from_secs(180)).await; + let (status, _, generation_after, _, completed) = + job_record(&engine, &account_id, "JrTarget").await; + assert_eq!(status.as_deref(), Some("DONE")); + assert_eq!(generation_after, generation, "DONE under the same claim"); + assert!(completed.is_some()); + assert!( + !engine + .fail_restore_job( + &key, + u64::try_from(generation_after.unwrap()).unwrap(), + extenddb_storage_cassandra::cassandra_util::millis_to_seconds_f64( + heartbeat.expect("a heartbeat") + ), + "late" + ) + .await + .unwrap(), + "a DONE record is not RUNNING" + ); + assert!( + !engine.remove_restore_target(&key).await.unwrap(), + "an ACTIVE table is not a CREATING restore target" + ); +} + +/// An in-catalog CreateBackup whose process died leaves a CREATING row whose +/// heartbeat goes stale. `abandoned_creating_backups` reports it only once +/// the heartbeat is older than the window, exactly once (the DELETING scan +/// runs before the CREATING one, so the row it just fenced is not read +/// again), fencing the row to DELETING under a fresh generation so the dead +/// create's heartbeat no longer applies; the reader's `delete` then removes +/// the row, its list entries and every payload bucket the row did not yet +/// account for. +#[tokio::test] +async fn test_an_abandoned_in_catalog_create_is_reported_once_stale_and_deleted() { + if crate::helpers::skip_without_cassandra() { + return; + } + const ITEMS: usize = 200; + const DEAD_GENERATION: u64 = 4242; + let engine = setup_engine().await; + let (account, _source, backup_arn) = + backed_up_plain_table(&engine, "AbandonedCreate", ITEMS).await; + let account_id = account.account_id.clone(); + let abandoned = extenddb_storage::backup_catalog::AbandonedLegacyBackup { + account_id: account_id.clone(), + backup_arn: backup_arn.clone(), + removed: false, + }; + let reported = |stale_after: Duration| { + let engine = &engine; + let abandoned = &abandoned; + async move { + engine + .abandoned_creating_backups(stale_after) + .await + .unwrap() + .into_iter() + .filter(|row| row == abandoned) + .count() + } + }; + // A live, finished backup is not reported. + assert_eq!(reported(Duration::from_secs(60)).await, 0); + // What a create that died mid-copy leaves: a CREATING row counting no + // items yet, owned by the dead create's generation, its payload buckets + // written, its heartbeat two minutes old. + let catalog = engine.catalog_keyspace(); + let stale = extenddb_storage_cassandra::cassandra_util::now_millis() - 120_000; + apply( + &engine, + &format!( + "UPDATE {catalog}.backups_by_arn SET backup_status = 'CREATING', item_count = 0, \ + owner_generation = ? WHERE account_id = ? AND backup_arn = ? \ + IF backup_status = 'AVAILABLE'" + ), + cdrs_tokio::query_values!( + i64::try_from(DEAD_GENERATION).unwrap(), + account_id.as_str(), + backup_arn.as_str() + ), + ) + .await; + assert!( + extenddb_storage_cassandra::heartbeat_backup_row( + &engine, + &account_id, + &backup_arn, + DEAD_GENERATION + ) + .await + .unwrap(), + "until the row is fenced, the create's own heartbeat applies" + ); + apply( + &engine, + &format!( + "UPDATE {catalog}.backups_by_arn SET heartbeat_at = ? \ + WHERE account_id = ? AND backup_arn = ? IF backup_status = 'CREATING'" + ), + cdrs_tokio::query_values!(stale, account_id.as_str(), backup_arn.as_str()), + ) + .await; + assert!( + matches!( + engine.delete(&account_id, &backup_arn).await, + Err(StorageError::Validation(ref m)) if m.contains("Backup not found") + ), + "a client's delete of a backup still being created is not found" + ); + assert_eq!( + reported(Duration::from_secs(600)).await, + 0, + "a heartbeat younger than the window is not dead" + ); + assert_eq!( + reported(Duration::from_secs(60)).await, + 1, + "a heartbeat older than the window is dead, reported once" + ); + assert!( + !extenddb_storage_cassandra::heartbeat_backup_row( + &engine, + &account_id, + &backup_arn, + DEAD_GENERATION + ) + .await + .unwrap(), + "the fence took the row under a fresh generation: the dead create's heartbeat is lost" + ); + let deleted = engine + .delete(&account_id, &backup_arn) + .await + .expect("the reconciler's delete finishes the fenced row"); + assert_eq!(deleted.backup_details.backup_status, "DELETED"); + assert_eq!( + reported(Duration::from_secs(60)).await, + 0, + "a deleted row is not reported again" + ); + assert!(engine.list(&account_id, None).await.unwrap().is_empty()); + // The payload is gone, bucket by bucket past the count the row carried. + let probe = format!( + "SELECT item_index FROM {}.backup_items WHERE backup_arn = ? AND bucket = ? LIMIT 1", + engine.account_keyspace(&account_id) + ); + for bucket in 0..4_i32 { + let rows = extenddb_storage_cassandra::cassandra_util::query_rows_quorum::( + &engine.session_arc(), + &probe, + cdrs_tokio::query_values!(backup_arn.as_str(), bucket), + "probe", + ) + .await + .unwrap(); + assert!(rows.is_empty(), "bucket {bucket} still holds items"); + } +} + +/// The `restore_completed_at` of a target's job record, in epoch ms. +async fn completed_at(engine: &CassandraEngine, account_id: &str, name: &str) -> Option { + job_record(engine, account_id, name).await.4 +} + +/// The dead jobs under `stale_after` that name `key`. +async fn dead_jobs_of( + engine: &CassandraEngine, + key: &RestoreTargetKey, + stale_after: Duration, +) -> Vec { + engine + .list_dead_restore_jobs(stale_after) + .await + .expect("list dead jobs") + .into_iter() + .filter(|job| &job.target == key) + .collect() +} + +/// A restore whose FAILED mark landed but whose target removal did not (the +/// target still CREATING) is listed by `list_dead_restore_jobs` as FAILED +/// once the record is stale: its `restore_completed_at`, or its heartbeat +/// where none was stamped, older than the window. While fresh it is left to +/// the writer still on its failure path. `remove_restore_target` then removes +/// the target, with the FAILED record as it was. A FAILED record whose table +/// is ACTIVE is not listed; one whose table is DELETING with its registration +/// is. +#[tokio::test] +async fn test_a_failed_record_whose_target_removal_did_not_land_is_listed_once_stale_and_removed() { + if crate::helpers::skip_without_cassandra() { + return; + } + const ITEMS: usize = 1_500; + let engine = setup_engine().await; + let (account, _source, backup_arn) = backed_up_plain_table(&engine, "FtSource", ITEMS).await; + let account_id = account.account_id.clone(); + let catalog = engine.catalog_keyspace(); + + // A finished restore: an ACTIVE table whose record is then marked FAILED + // and aged by hand. Not a CREATING target, so never listed. + let done = restore( + &engine, + &account_id, + &backup_arn, + "FtDone", + &RestoreTableOverrides::default(), + ) + .await + .expect("restore"); + wait_active(&engine, &account_id, "FtDone", Duration::from_secs(120)).await; + let done_key = RestoreTargetKey { + account_id: account_id.clone(), + table_name: "FtDone".to_owned(), + table_id: done.table_id.clone(), + }; + apply( + &engine, + &format!( + "UPDATE {catalog}.tables SET restore_status = 'FAILED', restore_completed_at = 0 \ + WHERE account_id = ? AND table_name = ? IF table_id = ?" + ), + cdrs_tokio::query_values!(account_id.as_str(), "FtDone", done.table_id.as_str()), + ) + .await; + assert!( + dead_jobs_of(&engine, &done_key, Duration::from_secs(1)) + .await + .is_empty(), + "a FAILED record of an ACTIVE table is not listed" + ); + + // The shape the duty exists for: the owner died, the reconciler's FAILED + // write landed, the removal did not. + let target = restore( + &engine, + &account_id, + &backup_arn, + "FtTarget", + &RestoreTableOverrides::default(), + ) + .await + .expect("restore"); + orphan_restore_target(&engine, &account_id, "FtTarget", &target.table_id).await; + let key = RestoreTargetKey { + account_id: account_id.clone(), + table_name: "FtTarget".to_owned(), + table_id: target.table_id.clone(), + }; + let dead = dead_jobs_of(&engine, &key, Duration::from_secs(60)).await; + assert_eq!(dead.len(), 1, "the orphaned record is listed: {dead:?}"); + assert!( + engine + .fail_restore_job( + &key, + dead[0].owner_generation, + dead[0].heartbeat_at, + "owner dead" + ) + .await + .expect("fail") + ); + let failed_at = completed_at(&engine, &account_id, "FtTarget") + .await + .expect("the FAILED write stamps completed_at"); + assert!( + dead_jobs_of(&engine, &key, Duration::from_secs(60)) + .await + .is_empty(), + "a fresh FAILED record is left to the writer on its failure path" + ); + + // Two minutes later (on the record's clock): stale under a 60 s window, + // not under a 10 minute one. + let stale = failed_at - 120_000; + let set_completed = |value: Option| { + let engine = &engine; + let catalog = &catalog; + let account_id = &account_id; + let table_id = &target.table_id; + async move { + apply( + engine, + &format!( + "UPDATE {catalog}.tables SET restore_completed_at = ? \ + WHERE account_id = ? AND table_name = ? IF table_id = ?" + ), + cdrs_tokio::query_values!( + value, + account_id.as_str(), + "FtTarget", + table_id.as_str() + ), + ) + .await; + } + }; + set_completed(Some(stale)).await; + assert!( + dead_jobs_of(&engine, &key, Duration::from_secs(600)) + .await + .is_empty(), + "not stale under a window longer than its age" + ); + let listed = dead_jobs_of(&engine, &key, Duration::from_secs(60)).await; + assert_eq!(listed.len(), 1, "{listed:?}"); + assert_eq!(listed[0].status, JobStatus::Failed); + assert_eq!( + listed[0].kind, + extenddb_storage::backup_catalog::RestoreKind::Legacy + ); + assert_eq!(listed[0].source_backup_arn, backup_arn); + + // No completion stamp: the record is aged by its heartbeat (two minutes + // old since it was orphaned). + set_completed(None).await; + assert!( + dead_jobs_of(&engine, &key, Duration::from_secs(600)) + .await + .is_empty(), + "aged by the heartbeat: not stale under 10 minutes" + ); + assert_eq!( + dead_jobs_of(&engine, &key, Duration::from_secs(60)) + .await + .len(), + 1, + "aged by the heartbeat: stale under 60 s" + ); + + // A DELETING table with its registration (the removal's first write + // landed) is listed too; the removal of that state is + // `test_a_failed_record_whose_target_is_deleting_is_listed_once_stale_and_its_removal_resumed`. + apply( + &engine, + &format!( + "UPDATE {catalog}.tables SET table_status = 'DELETING' \ + WHERE account_id = ? AND table_name = ? IF table_id = ?" + ), + cdrs_tokio::query_values!(account_id.as_str(), "FtTarget", target.table_id.as_str()), + ) + .await; + assert_eq!( + dead_jobs_of(&engine, &key, Duration::from_secs(60)) + .await + .len(), + 1, + "a FAILED record of a DELETING table with its registration is listed" + ); + apply( + &engine, + &format!( + "UPDATE {catalog}.tables SET table_status = 'CREATING' \ + WHERE account_id = ? AND table_name = ? IF table_id = ?" + ), + cdrs_tokio::query_values!(account_id.as_str(), "FtTarget", target.table_id.as_str()), + ) + .await; + + // The duty: the target removed, the FAILED record not written again. + assert!( + !engine + .fail_restore_job( + &key, + dead[0].owner_generation, + dead[0].heartbeat_at, + "again" + ) + .await + .expect("fail again"), + "a FAILED record is not RUNNING: no second FAILED write" + ); + assert!( + engine.remove_restore_target(&key).await.expect("remove"), + "the target was a CREATING restore target" + ); + assert!(matches!( + engine + .describe_table( + &account_id, + DescribeTableInput { + table_name: "FtTarget".to_owned(), + }, + ) + .await, + Err(StorageError::TableNotFound(_)) + )); + assert!( + dead_jobs_of(&engine, &key, Duration::from_secs(60)) + .await + .is_empty(), + "a removed target has no record" + ); + engine + .delete(&account_id, &backup_arn) + .await + .expect("the removed target released the backup"); + engine + .delete_table( + &account_id, + DeleteTableInput { + table_name: "FtDone".to_owned(), + }, + ) + .await + .expect("delete the finished target"); +} + +/// The removal of a FAILED record's target is two writes: the claim from +/// CREATING to DELETING, then the drop. A target the claim moved to DELETING +/// whose drop then failed (the row DELETING with its registration and FAILED +/// record) is listed by `list_dead_restore_jobs` as FAILED once the record is +/// stale, left alone while fresh, and `remove_restore_target` resumes the +/// removal from the drop: the table, its data table and its record are gone, +/// the backup is released and the name is free. Three DELETING rows the +/// resume rule must leave alone: a restored table with a DONE record that a +/// client is deleting (its registration still on the row), which is neither +/// listed nor removed and keeps its data; the DELETING+FAILED row named by a +/// key with another `table_id`, on which `remove_restore_target` drops +/// nothing; and a DELETING row without a registration (a client's +/// DeleteTable in flight), neither listed nor removed. +#[tokio::test] +async fn test_a_failed_record_whose_target_is_deleting_is_listed_once_stale_and_its_removal_resumed() + { + if crate::helpers::skip_without_cassandra() { + return; + } + const ITEMS: usize = 1_500; + let engine = setup_engine().await; + let (account, _source, backup_arn) = backed_up_plain_table(&engine, "PaSource", ITEMS).await; + let account_id = account.account_id.clone(); + let catalog = engine.catalog_keyspace(); + + // A restored table, ACTIVE with a DONE record, that a client's + // DeleteTable moved to DELETING: the registration is on the row and the + // record is not FAILED, so the row belongs to that delete. + let done = restore( + &engine, + &account_id, + &backup_arn, + "PaDone", + &RestoreTableOverrides::default(), + ) + .await + .expect("restore the table a client deletes"); + wait_active(&engine, &account_id, "PaDone", Duration::from_secs(60)).await; + let done_key = RestoreTargetKey { + account_id: account_id.clone(), + table_name: "PaDone".to_owned(), + table_id: done.table_id.clone(), + }; + assert_eq!( + job_record(&engine, &account_id, "PaDone") + .await + .0 + .as_deref(), + Some("DONE"), + "the activated restore's record is DONE" + ); + apply( + &engine, + &format!( + "UPDATE {catalog}.tables SET table_status = 'DELETING' \ + WHERE account_id = ? AND table_name = ? IF table_id = ? AND table_status = 'ACTIVE'" + ), + cdrs_tokio::query_values!(account_id.as_str(), "PaDone", done.table_id.as_str()), + ) + .await; + let done_data_table = format!("items_{}", done.table_id.replace('-', "_")); + assert!( + dead_jobs_of(&engine, &done_key, Duration::from_secs(1)) + .await + .is_empty(), + "a DONE record of a DELETING table is not listed" + ); + assert!( + !engine + .remove_restore_target(&done_key) + .await + .expect("remove a restored table a client is deleting"), + "a DELETING restored table whose record is DONE belongs to the client's delete" + ); + assert!( + data_table_exists(&engine, &account_id, &done_data_table).await, + "the client's delete keeps the restored table's data" + ); + apply( + &engine, + &format!( + "UPDATE {catalog}.tables SET table_status = 'ACTIVE' \ + WHERE account_id = ? AND table_name = ? IF table_id = ?" + ), + cdrs_tokio::query_values!(account_id.as_str(), "PaDone", done.table_id.as_str()), + ) + .await; + engine + .delete_table( + &account_id, + DeleteTableInput { + table_name: "PaDone".to_owned(), + }, + ) + .await + .expect("delete the restored table"); + + let target = restore( + &engine, + &account_id, + &backup_arn, + "PaTarget", + &RestoreTableOverrides::default(), + ) + .await + .expect("restore"); + orphan_restore_target(&engine, &account_id, "PaTarget", &target.table_id).await; + let key = RestoreTargetKey { + account_id: account_id.clone(), + table_name: "PaTarget".to_owned(), + table_id: target.table_id.clone(), + }; + let dead = dead_jobs_of(&engine, &key, Duration::from_secs(60)).await; + assert_eq!(dead.len(), 1, "the orphaned record is listed: {dead:?}"); + assert!( + engine + .fail_restore_job( + &key, + dead[0].owner_generation, + dead[0].heartbeat_at, + "owner dead" + ) + .await + .expect("fail") + ); + let failed_at = completed_at(&engine, &account_id, "PaTarget") + .await + .expect("the FAILED write stamps completed_at"); + // The removal's claim landed and its drop did not. + apply( + &engine, + &format!( + "UPDATE {catalog}.tables SET table_status = 'DELETING' \ + WHERE account_id = ? AND table_name = ? IF table_id = ? AND table_status = 'CREATING'" + ), + cdrs_tokio::query_values!(account_id.as_str(), "PaTarget", target.table_id.as_str()), + ) + .await; + assert!( + dead_jobs_of(&engine, &key, Duration::from_secs(60)) + .await + .is_empty(), + "a fresh FAILED record of a DELETING target is left to the remover on its path" + ); + + apply( + &engine, + &format!( + "UPDATE {catalog}.tables SET restore_completed_at = ? \ + WHERE account_id = ? AND table_name = ? IF table_id = ?" + ), + cdrs_tokio::query_values!( + failed_at - 120_000, + account_id.as_str(), + "PaTarget", + target.table_id.as_str() + ), + ) + .await; + assert!( + dead_jobs_of(&engine, &key, Duration::from_secs(600)) + .await + .is_empty(), + "not stale under a window longer than its age" + ); + let listed = dead_jobs_of(&engine, &key, Duration::from_secs(60)).await; + assert_eq!( + listed.len(), + 1, + "a stale FAILED record of a DELETING target with its registration is listed: {listed:?}" + ); + assert_eq!(listed[0].status, JobStatus::Failed); + assert_eq!(listed[0].source_backup_arn, backup_arn); + + // The duty resumes the drop, on the row's own table id only: a key with + // another id (a table re-created under the name) touches nothing. + let data_table = format!("items_{}", target.table_id.replace('-', "_")); + assert!( + data_table_exists(&engine, &account_id, &data_table).await, + "the target's data table exists before the removal" + ); + let other_id_key = RestoreTargetKey { + table_id: "not-this-id".to_owned(), + ..key.clone() + }; + assert!( + !engine + .remove_restore_target(&other_id_key) + .await + .expect("remove under another table id"), + "a key naming another table id removes nothing from a DELETING target" + ); + assert!( + data_table_exists(&engine, &account_id, &data_table).await, + "the data table is kept under a key with another table id" + ); + assert_eq!( + dead_jobs_of(&engine, &key, Duration::from_secs(60)) + .await + .len(), + 1, + "the record is still listed after the mismatched removal" + ); + assert!( + engine.remove_restore_target(&key).await.expect("remove"), + "a DELETING target with its registration and FAILED record is removed" + ); + assert!(matches!( + engine + .describe_table( + &account_id, + DescribeTableInput { + table_name: "PaTarget".to_owned(), + }, + ) + .await, + Err(StorageError::TableNotFound(_)) + )); + assert!( + !data_table_exists(&engine, &account_id, &data_table).await, + "the target's data table is dropped" + ); + assert!( + dead_jobs_of(&engine, &key, Duration::from_secs(60)) + .await + .is_empty(), + "a removed target has no record" + ); + assert!( + !engine + .remove_restore_target(&key) + .await + .expect("remove again"), + "nothing left to remove" + ); + // The name is free, and the backup released. + let reused = create_active_table( + &engine, + &account_id, + CreateTableInput { + table_name: "PaTarget".to_owned(), + key_schema: vec![ks("id", KeyType::Hash)], + attribute_definitions: vec![attr("id", ScalarAttributeType::S)], + billing_mode: Some(BillingMode::PayPerRequest), + ..Default::default() + }, + ) + .await; + assert_ne!(reused.table_id, target.table_id); + engine + .delete(&account_id, &backup_arn) + .await + .expect("the removed target released the backup"); + + // A DELETING row without a registration is a client's DeleteTable in + // flight: not a restore target, so not listed and not removed. + let plain_key = RestoreTargetKey { + account_id: account_id.clone(), + table_name: "PaTarget".to_owned(), + table_id: reused.table_id.clone(), + }; + apply( + &engine, + &format!( + "UPDATE {catalog}.tables SET table_status = 'DELETING' \ + WHERE account_id = ? AND table_name = ? IF table_id = ? AND table_status = 'ACTIVE'" + ), + cdrs_tokio::query_values!(account_id.as_str(), "PaTarget", reused.table_id.as_str()), + ) + .await; + assert!( + dead_jobs_of(&engine, &plain_key, Duration::from_secs(1)) + .await + .is_empty(), + "a DELETING table that is not a restore target is not listed" + ); + assert!( + !engine + .remove_restore_target(&plain_key) + .await + .expect("remove a plain table"), + "a DELETING table without a registration is not removed" + ); + assert!( + data_table_exists( + &engine, + &account_id, + &format!("items_{}", reused.table_id.replace('-', "_")) + ) + .await, + "the client's delete keeps its table" + ); + apply( + &engine, + &format!( + "UPDATE {catalog}.tables SET table_status = 'ACTIVE' \ + WHERE account_id = ? AND table_name = ? IF table_id = ?" + ), + cdrs_tokio::query_values!(account_id.as_str(), "PaTarget", reused.table_id.as_str()), + ) + .await; + engine + .delete_table( + &account_id, + DeleteTableInput { + table_name: "PaTarget".to_owned(), + }, + ) + .await + .expect("delete the re-created table"); +} + +/// Whether `table` exists in `account_id`'s keyspace. +async fn data_table_exists(engine: &CassandraEngine, account_id: &str, table: &str) -> bool { + !extenddb_storage_cassandra::cassandra_util::query_rows::( + &engine.session_arc(), + "SELECT table_name FROM system_schema.tables WHERE keyspace_name = ? AND table_name = ?", + cdrs_tokio::query_values!(engine.account_keyspace(account_id), table), + "schema", + ) + .await + .expect("read the schema") + .is_empty() +} + +#[cfg(feature = "test-hooks")] +/// The write times of a `backups_by_arn` row's `heartbeat_at` and of a column +/// only the insert writes (`backup_name`), as `(heartbeat, inserted)`, for +/// the one row of `account_id`, with the row's status. +async fn create_row_write_times( + engine: &CassandraEngine, + account_id: &str, +) -> Option<(Option, Option, String)> { + use cdrs_tokio::types::IntoRustByName; + let rows = extenddb_storage_cassandra::cassandra_util::query_rows_quorum::( + &engine.session_arc(), + &format!( + "SELECT WRITETIME(heartbeat_at) AS beat, WRITETIME(backup_name) AS inserted, \ + backup_status FROM {}.backups_by_arn WHERE account_id = ? ALLOW FILTERING", + engine.catalog_keyspace() + ), + cdrs_tokio::query_values!(account_id), + "create row", + ) + .await + .expect("read the create's row"); + let row = rows.first()?; + Some(( + row.get_by_name("beat").ok().flatten(), + row.get_by_name("inserted").ok().flatten(), + row.get_r_by_name("backup_status").expect("status"), + )) +} + +#[cfg(feature = "test-hooks")] +/// The in-catalog create heartbeats its CREATING row while the items copy: +/// `heartbeat_at` is written again after the insert, while the row is still +/// CREATING, and the published row carries the last beat. +#[tokio::test] +async fn test_the_in_catalog_create_heartbeats_its_row_while_the_items_copy() { + if crate::helpers::skip_without_cassandra() { + return; + } + const ITEMS: usize = 3_000; + let engine = std::sync::Arc::new(setup_engine().await); + let account = TestAccount::new(&engine, "extenddb_ttl_test").await; + let account_id = account.account_id.clone(); + extenddb_storage_cassandra::test_hooks::set_heartbeat_interval( + &account_id, + Duration::from_millis(100), + ); + let key_info = create_active_table( + &engine, + &account_id, + CreateTableInput { + table_name: "Beating".to_owned(), + key_schema: vec![ks("id", KeyType::Hash)], + attribute_definitions: vec![attr("id", ScalarAttributeType::S)], + billing_mode: Some(BillingMode::PayPerRequest), + ..Default::default() + }, + ) + .await; + put_all( + &engine, + &key_info, + (0..ITEMS) + .map(|i| item(&format!("item-{i:05}"), None, &"x".repeat(200))) + .collect(), + ) + .await; + + // Watch the row while the create runs: the first time its heartbeat's + // write time passes the insert's while the row is still CREATING. + let watching = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(true)); + let watcher = { + let engine = std::sync::Arc::clone(&engine); + let account_id = account_id.clone(); + let watching = std::sync::Arc::clone(&watching); + tokio::spawn(async move { + let mut refreshed_while_creating = false; + while watching.load(std::sync::atomic::Ordering::Acquire) && !refreshed_while_creating { + if let Some((Some(beat), Some(inserted), status)) = + create_row_write_times(&engine, &account_id).await + && beat > inserted + && status == "CREATING" + { + refreshed_while_creating = true; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + refreshed_while_creating + }) + }; + let created = engine + .create(&account_id, "Beating", "beats", JobPermit::detached()) + .await + .expect("create backup"); + watching.store(false, std::sync::atomic::Ordering::Release); + assert!( + watcher.await.expect("watcher"), + "heartbeat_at was written again while the row was CREATING" + ); + let (beat, inserted, status) = create_row_write_times(&engine, &account_id) + .await + .expect("the published row"); + assert_eq!(status, "AVAILABLE"); + assert!( + beat.zip(inserted) + .is_some_and(|(beat, inserted)| beat > inserted), + "the published row carries the last beat: {beat:?} vs {inserted:?}" + ); + engine + .delete(&account_id, &created.backup_arn) + .await + .expect("delete backup"); +} + +/// How many of `row`'s three index entries (`backups_by_account`, +/// `backups_by_table`, `backup_index_by_status` under its status) are in +/// the tables, read directly: every listing re-reads the authoritative row +/// and skips an entry whose row is gone, so only a direct read sees one. +async fn raw_index_entries(engine: &CassandraEngine, row: &BackupRow) -> usize { + let catalog = engine.catalog_keyspace(); + let created_at = (row.created_at * 1_000.0).round() as i64; + let mut found = 0; + for (cql, values) in [ + ( + format!( + "SELECT backup_arn FROM {catalog}.backups_by_account \ + WHERE account_id = ? AND created_at = ? AND backup_arn = ?" + ), + cdrs_tokio::query_values!(row.account_id.as_str(), created_at, row.backup_arn.as_str()), + ), + ( + format!( + "SELECT backup_arn FROM {catalog}.backups_by_table WHERE account_id = ? \ + AND table_name = ? AND created_at = ? AND backup_arn = ?" + ), + cdrs_tokio::query_values!( + row.account_id.as_str(), + row.table_name.as_str(), + created_at, + row.backup_arn.as_str() + ), + ), + ( + format!( + "SELECT backup_arn FROM {catalog}.backup_index_by_status \ + WHERE store_root = ? AND backup_status = ? AND backup_arn = ?" + ), + cdrs_tokio::query_values!( + row.store.store_root.as_str(), + row.status.as_str(), + row.backup_arn.as_str() + ), + ), + ] { + let rows = extenddb_storage_cassandra::cassandra_util::query_rows_quorum::( + &engine.session_arc(), + &cql, + values, + "index entry", + ) + .await + .expect("read an index entry"); + found += rows.len(); + } + found +} + +#[cfg(feature = "test-hooks")] +/// Items of `backup_arn`'s payload in `account_id`'s keyspace, all buckets. +async fn payload_item_count(engine: &CassandraEngine, account_id: &str, backup_arn: &str) -> i64 { + use cdrs_tokio::types::IntoRustByName; + let rows = extenddb_storage_cassandra::cassandra_util::query_rows_quorum::( + &engine.session_arc(), + &format!( + "SELECT COUNT(*) AS n FROM {}.backup_items WHERE backup_arn = ? ALLOW FILTERING", + engine.account_keyspace(account_id) + ), + cdrs_tokio::query_values!(backup_arn), + "payload count", + ) + .await + .expect("count the payload"); + rows.first() + .and_then(|row| row.get_by_name("n").ok().flatten()) + .unwrap_or(0) +} + +/// The `backup_status` of the `backups_by_arn` row `backup_arn`, if there is +/// one. +#[cfg(feature = "test-hooks")] +async fn backup_status_of( + engine: &CassandraEngine, + account_id: &str, + backup_arn: &str, +) -> Option { + use cdrs_tokio::types::IntoRustByName; + extenddb_storage_cassandra::cassandra_util::query_rows_quorum::( + &engine.session_arc(), + &format!( + "SELECT backup_status FROM {}.backups_by_arn WHERE account_id = ? AND backup_arn = ?", + engine.catalog_keyspace() + ), + cdrs_tokio::query_values!(account_id, backup_arn), + "backup status", + ) + .await + .expect("read the row") + .first() + .and_then(|row| row.get_by_name("backup_status").ok().flatten()) +} + +#[cfg(feature = "test-hooks")] +/// Three heartbeat errors in a row stop an in-catalog create within the +/// bucket it is writing: the create fails naming the unconfirmed claim, its +/// row stays CREATING for the reconciler, and the payload ends at a bucket +/// boundary short of the table. Two errors followed by a confirmed beat do +/// not stop it. +#[tokio::test] +async fn test_three_heartbeat_errors_stop_a_create_within_a_bucket_and_two_do_not() { + if crate::helpers::skip_without_cassandra() { + return; + } + const ITEMS: i64 = 3_000; + const BUCKET: i64 = 64; + let engine = std::sync::Arc::new(setup_engine().await); + let account = TestAccount::new(&engine, "extenddb_ttl_test").await; + let account_id = account.account_id.clone(); + extenddb_storage_cassandra::test_hooks::set_heartbeat_interval( + &account_id, + Duration::from_millis(10), + ); + let key_info = create_active_table( + &engine, + &account_id, + CreateTableInput { + table_name: "Erring".to_owned(), + key_schema: vec![ks("id", KeyType::Hash)], + attribute_definitions: vec![attr("id", ScalarAttributeType::S)], + billing_mode: Some(BillingMode::PayPerRequest), + ..Default::default() + }, + ) + .await; + put_all( + &engine, + &key_info, + (0..ITEMS) + .map(|i| item(&format!("item-{i:05}"), None, &"x".repeat(200))) + .collect(), + ) + .await; + + // The create starts with confirmed beats; once its first bucket is on + // disk, the next three beats fail. + let create = { + let engine = std::sync::Arc::clone(&engine); + let account_id = account_id.clone(); + tokio::spawn(async move { + engine + .create(&account_id, "Erring", "errs", JobPermit::detached()) + .await + }) + }; + let backup_arn = first_written_backup_arn(&engine, &account_id).await; + extenddb_storage_cassandra::test_hooks::fail_heartbeats(&account_id, 3); + let error = create + .await + .expect("create task") + .expect_err("three heartbeat errors in a row stop the create"); + assert!( + matches!(error, StorageError::Internal(ref m) if m.contains("could not be confirmed")), + "{error}" + ); + assert_eq!( + backup_status_of(&engine, &account_id, &backup_arn).await, + Some("CREATING".to_owned()), + "the row is left for the reconciler" + ); + let written = payload_item_count(&engine, &account_id, &backup_arn).await; + assert!( + written > 0 && written < ITEMS, + "the copy stopped part way: {written} of {ITEMS}" + ); + assert_eq!( + written % BUCKET, + 0, + "the copy polls its claim per bucket, so it stopped at a bucket boundary: {written}" + ); + // The reconciler finishes the row once its heartbeat is stale. + let stale = extenddb_storage_cassandra::cassandra_util::now_millis() - 120_000; + apply( + &engine, + &format!( + "UPDATE {}.backups_by_arn SET heartbeat_at = ? \ + WHERE account_id = ? AND backup_arn = ? IF backup_status = 'CREATING'", + engine.catalog_keyspace() + ), + cdrs_tokio::query_values!(stale, account_id.as_str(), backup_arn.as_str()), + ) + .await; + assert_eq!( + engine + .abandoned_creating_backups(Duration::from_secs(60)) + .await + .unwrap() + .iter() + .filter(|row| row.backup_arn == backup_arn) + .count(), + 1 + ); + engine + .delete(&account_id, &backup_arn) + .await + .expect("the reconciler's delete finishes the stopped create"); + assert_eq!( + payload_item_count(&engine, &account_id, &backup_arn).await, + 0 + ); + + // Two errors, then confirmed beats: the create finishes. The errors are + // armed once the copy is writing, and both are seen taken by a beat + // while the payload is still short of the table, so the create met two + // errors in a row where three would have stopped it. + let create = { + let engine = std::sync::Arc::clone(&engine); + let account_id = account_id.clone(); + tokio::spawn(async move { + engine + .create(&account_id, "Erring", "two-errors", JobPermit::detached()) + .await + }) + }; + let second_arn = first_written_backup_arn(&engine, &account_id).await; + extenddb_storage_cassandra::test_hooks::fail_heartbeats(&account_id, 2); + let deadline = Instant::now() + Duration::from_secs(30); + while extenddb_storage_cassandra::test_hooks::pending_heartbeat_errors(&account_id) > 0 { + assert!( + Instant::now() < deadline, + "the two erring beats never fired" + ); + tokio::time::sleep(Duration::from_millis(2)).await; + } + let written_at_second_error = payload_item_count(&engine, &account_id, &second_arn).await; + assert!( + written_at_second_error < ITEMS, + "both erring beats fired while the copy was still writing: {written_at_second_error} \ + of {ITEMS}" + ); + let created = create + .await + .expect("create task") + .expect("two heartbeat errors do not stop a create"); + assert_eq!(created.backup_arn, second_arn); + assert_eq!(created.backup_status, "AVAILABLE"); + assert_eq!( + payload_item_count(&engine, &account_id, &created.backup_arn).await, + ITEMS + ); + engine + .delete(&account_id, &created.backup_arn) + .await + .expect("delete backup"); +} + +#[cfg(feature = "test-hooks")] +/// Wait for the first `backup_items` row in `account_id`'s keyspace and +/// return its backup ARN: the create whose copy has begun writing. +async fn first_written_backup_arn(engine: &CassandraEngine, account_id: &str) -> String { + let deadline = Instant::now() + Duration::from_secs(30); + loop { + assert!( + Instant::now() < deadline, + "the create never started writing" + ); + if let Some(arn) = extenddb_storage_cassandra::cassandra_util::query_rows::( + &engine.session_arc(), + &format!( + "SELECT backup_arn FROM {}.backup_items LIMIT 1", + engine.account_keyspace(account_id) + ), + cdrs_tokio::query_values!(), + "first item", + ) + .await + .expect("probe") + .first() + .map(|row| { + extenddb_storage_cassandra::cassandra_util::get_column::( + row, + "backup_arn", + "first item", + ) + .unwrap() + }) { + return arn; + } + tokio::time::sleep(Duration::from_millis(5)).await; + } +} + +#[cfg(feature = "test-hooks")] +/// A restore that registers on the backup between DeleteBackup's in-use read +/// and its DELETING claim is what the claim's own `active_restores = null` +/// condition is for: the claim does not apply, the delete answers +/// BackupInUse naming the target, and the backup stays AVAILABLE; the same +/// for `start_backup_delete` on a catalog index row. +#[tokio::test] +async fn test_a_restore_registering_between_the_in_use_read_and_the_claim_keeps_the_backup() { + if crate::helpers::skip_without_cassandra() { + return; + } + const ITEMS: usize = 1_500; + let engine = std::sync::Arc::new(setup_engine().await); + let (account, _source, backup_arn) = backed_up_plain_table(&engine, "RaceSource", ITEMS).await; + let account_id = account.account_id.clone(); + + // The legacy delete, paused after its in-use read found no restore. + let pause = extenddb_storage_cassandra::test_hooks::pause_before_delete_claim(&backup_arn); + let delete = { + let engine = std::sync::Arc::clone(&engine); + let (account_id, backup_arn) = (account_id.clone(), backup_arn.clone()); + tokio::spawn(async move { engine.delete(&account_id, &backup_arn).await }) + }; + pause.reached().await; + let target = restore( + &engine, + &account_id, + &backup_arn, + "RaceTarget", + &RestoreTableOverrides::default(), + ) + .await + .expect("the restore registers while the delete is between its read and its claim"); + assert_eq!(target.table_status, TableStatus::Creating); + pause.release(); + let refused = delete + .await + .expect("delete task") + .expect_err("the claim's active_restores = null condition refuses the delete"); + assert!( + matches!(refused, StorageError::BackupInUse(ref m) if m.contains(&backup_arn)), + "{refused}" + ); + assert_eq!( + backup_status_of(&engine, &account_id, &backup_arn).await, + Some("AVAILABLE".to_owned()), + "a refused delete leaves the row" + ); + wait_active(&engine, &account_id, "RaceTarget", Duration::from_secs(120)).await; + engine + .delete(&account_id, &backup_arn) + .await + .expect("the finished restore released the backup"); + + // `start_backup_delete` on an index row: the registration lands in the + // same gap; the claim answers InUse and the row stays AVAILABLE. + let store = unique_store(); + let row = index_row(&account_id, "RaceSource", "nightly", &store); + let key = row.key(); + let claim = engine.claim_backup(&row).await.expect("claim"); + assert_eq!( + claim + .set_status( + JobStatus::Creating, + JobStatus::Available, + &StatusFields::default() + ) + .await + .unwrap(), + WriteOutcome::Matched + ); + let pause = extenddb_storage_cassandra::test_hooks::pause_before_delete_claim(&row.backup_arn); + let start = { + let engine = std::sync::Arc::clone(&engine); + let key = key.clone(); + tokio::spawn(async move { engine.start_backup_delete(&key).await }) + }; + pause.reached().await; + let catalog = engine.catalog_keyspace(); + let registered = create_active_table( + &engine, + &account_id, + CreateTableInput { + table_name: "RaceIndexTarget".to_owned(), + key_schema: vec![ks("id", KeyType::Hash)], + attribute_definitions: vec![attr("id", ScalarAttributeType::S)], + billing_mode: Some(BillingMode::PayPerRequest), + ..Default::default() + }, + ) + .await; + apply( + &engine, + &format!( + "UPDATE {catalog}.tables SET table_status = 'CREATING', status_transition_at = null, \ + restore_source_backup_arn = ? WHERE account_id = ? AND table_name = ? IF table_id = ?" + ), + cdrs_tokio::query_values!( + row.backup_arn.as_str(), + account_id.as_str(), + "RaceIndexTarget", + registered.table_id.as_str() + ), + ) + .await; + apply( + &engine, + &format!( + "UPDATE {catalog}.backups_by_arn SET active_restores[?] = ? \ + WHERE account_id = ? AND backup_arn = ? IF backup_status = 'AVAILABLE'" + ), + cdrs_tokio::query_values!( + registered.table_id.as_str(), + "RaceIndexTarget", + account_id.as_str(), + row.backup_arn.as_str() + ), + ) + .await; + pause.release(); + assert_eq!( + start.await.expect("start task").unwrap(), + DeleteStart::InUse("RaceIndexTarget".to_owned()) + ); + assert_eq!( + engine + .read_backup_row(&arn_of(&row)) + .await + .unwrap() + .unwrap() + .status, + BackupStatus::Available, + "a refused delete leaves the row AVAILABLE" + ); + apply( + &engine, + &format!( + "UPDATE {catalog}.tables SET table_status = 'ACTIVE' \ + WHERE account_id = ? AND table_name = ? IF table_id = ?" + ), + cdrs_tokio::query_values!( + account_id.as_str(), + "RaceIndexTarget", + registered.table_id.as_str() + ), + ) + .await; + assert_eq!( + engine.start_backup_delete(&key).await.unwrap(), + DeleteStart::Deleting + ); + assert!(engine.set_backup_deleted(&key).await.unwrap()); + assert!(engine.remove_backup_row(&key).await.unwrap()); + drop(claim); + engine + .delete_table( + &account_id, + DeleteTableInput { + table_name: "RaceIndexTarget".to_owned(), + }, + ) + .await + .expect("delete the target"); +} + +/// The keyspace prefix `engine` was built with. +fn prefix_of(engine: &CassandraEngine) -> String { + engine + .catalog_keyspace() + .strip_suffix("_catalog") + .expect("catalog keyspace name") + .to_owned() +} + +/// Remove migration `id`'s ledger row from the catalog keyspace under the +/// migration lock, as `migrate --forget` does, so the next run applies it +/// again. +async fn forget_catalog_migration(engine: &CassandraEngine, prefix: &str, id: MigrationId) { + use extenddb_storage_cassandra::migrations::{CassandraMigrations, LeaseTimes, MigrationLock}; + let lock = MigrationLock::acquire( + engine.session_arc(), + &engine.catalog_keyspace(), + LeaseTimes::DEFAULT, + Duration::from_secs(120), + ) + .await + .expect("take the lock"); + let forgotten = { + let migrations = CassandraMigrations::locked_catalog_only(&lock, prefix); + extenddb_storage::migrations::Runner::new(&migrations, "test") + .forget(id.number()) + .await + }; + lock.release().await; + forgotten.expect("forget the migration"); +} + +/// A catalog at migration 5 whose migration 6 fails in its backfill (a +/// `tables` row carrying a restore registration and no `table_id`, which the +/// backfill cannot key) stays behind with migration 6 unrecorded, so `serve` +/// refuses it and the next `migrate --yes` runs the body again; once the +/// cause is gone the second run records it and the catalog is current. A +/// fresh catalog is current at once. +#[tokio::test] +async fn test_a_migration_6_that_fails_in_its_backfill_stays_pending_and_resumes() { + if crate::helpers::skip_without_cassandra() { + return; + } + let current = CatalogStatus::Current { + version: MigrationId::LATEST.number(), + }; + let (fresh, _account, _drop_fresh) = isolated_engine("mv", false).await; + assert_eq!( + extenddb_storage_cassandra::migrations::catalog_status( + fresh.session_arc(), + &prefix_of(&fresh) + ) + .await + .expect("status"), + current, + "a fresh catalog has every migration recorded" + ); + + let (engine, account_id, _drop_keyspaces) = isolated_engine("mf", true).await; + let catalog = engine.catalog_keyspace(); + let prefix = prefix_of(&engine); + // Behind at migration 5 with migration 6 pending (and 7, which does not + // apply on Cassandra and is recorded after it). + let behind_at_5 = |status: CatalogStatus| match status { + CatalogStatus::Behind { version, pending } => { + version == Some(MigrationId::DropContinuousBackups.number()) + && pending + .iter() + .any(|p| p.id == MigrationId::BackupCatalogIndex.number()) + && pending.iter().all(|p| { + p.id == MigrationId::BackupCatalogIndex.number() + || p.id == MigrationId::RestoreProgress.number() + }) + } + _ => false, + }; + let status = || async { + extenddb_storage_cassandra::migrations::catalog_status(engine.session_arc(), &prefix) + .await + .expect("status") + }; + assert!(behind_at_5(status().await)); + // A registration the backfill cannot key. + engine + .session() + .query_with_values( + format!( + "INSERT INTO {catalog}.tables (account_id, table_name, table_status, \ + restore_source_backup_arn) VALUES (?, 'Unkeyed', 'ACTIVE', 'arn:aws:dynamodb:us-east-1:000000000000:table/T/backup/01700000000000-abc')" + ), + cdrs_tokio::query_values!(account_id.as_str()), + ) + .await + .expect("a row without a table_id"); + let migrate = || { + extenddb_storage_cassandra::migrations::migrate_catalog_keyspace( + engine.session_arc(), + &prefix, + "test", + Duration::from_secs(120), + ) + }; + let failed = migrate() + .await + .expect_err("the backfill fails on the unkeyed row"); + assert!(failed.to_string().contains("table_id"), "{failed}"); + assert!( + behind_at_5(status().await), + "a failed body leaves migration 6 unrecorded" + ); + assert!( + matches!( + extenddb_storage_cassandra::migrations::register_server( + engine.session_arc(), + &prefix, + "t", + "test" + ) + .await, + Err(MigrationError::Status(CatalogStatus::Behind { .. })) + ), + "serve refuses the half-migrated catalog" + ); + + // The cause gone, the next run applies the body again and records it. + engine + .session() + .query_with_values( + format!("DELETE FROM {catalog}.tables WHERE account_id = ? AND table_name = 'Unkeyed'"), + cdrs_tokio::query_values!(account_id.as_str()), + ) + .await + .expect("remove the row"); + migrate().await.expect("the second run succeeds"); + assert_eq!(status().await, current); + let registration = extenddb_storage_cassandra::migrations::register_server( + engine.session_arc(), + &prefix, + "t", + "test", + ) + .await + .expect("serve accepts the migrated catalog"); + extenddb_storage::hooks::ServingRegistrar::release(®istration).await; + assert!( + uuid::Uuid::parse_str(&engine.catalog_id().await.expect("catalog id")).is_ok(), + "the body wrote the catalog id" + ); +} + +/// Items in the data table of a restore target, whatever its status. +#[cfg(feature = "test-hooks")] +async fn target_item_count(engine: &CassandraEngine, account_id: &str, table_id: &str) -> i64 { + use cdrs_tokio::types::IntoRustByName; + let rows = extenddb_storage_cassandra::cassandra_util::query_rows_quorum::( + &engine.session_arc(), + &format!( + "SELECT COUNT(*) AS n FROM {}.items_{}", + engine.account_keyspace(account_id), + table_id.replace('-', "_") + ), + cdrs_tokio::query_values!(), + "target count", + ) + .await + .expect("count the target"); + rows.first() + .and_then(|row| row.get_by_name("n").ok().flatten()) + .unwrap_or(0) +} + +/// Start a restore of `backup_arn` whose jobs heartbeat every 100 ms into +/// `target`, and return its description once the copy has written something +/// but is far from done. +#[cfg(feature = "test-hooks")] +async fn restore_mid_copy( + engine: &CassandraEngine, + account_id: &str, + backup_arn: &str, + target: &str, + items: i64, +) -> TableDescription { + extenddb_storage_cassandra::test_hooks::set_heartbeat_interval( + account_id, + Duration::from_millis(100), + ); + let desc = restore( + engine, + account_id, + backup_arn, + target, + &RestoreTableOverrides::default(), + ) + .await + .expect("restore"); + let deadline = Instant::now() + Duration::from_secs(60); + loop { + let copied = target_item_count(engine, account_id, &desc.table_id).await; + if copied > 0 { + assert!( + copied < items / 2, + "the copy is already {copied} of {items} items in" + ); + break; + } + assert!(Instant::now() < deadline, "the copy did not start"); + tokio::time::sleep(Duration::from_millis(20)).await; + } + desc +} + +/// Wait until the target's item count stops growing, and return it. Panics +/// if it is still growing after 30 s. +#[cfg(feature = "test-hooks")] +async fn settled_item_count(engine: &CassandraEngine, account_id: &str, table_id: &str) -> i64 { + let deadline = Instant::now() + Duration::from_secs(30); + let mut last = target_item_count(engine, account_id, table_id).await; + loop { + tokio::time::sleep(Duration::from_millis(600)).await; + let now = target_item_count(engine, account_id, table_id).await; + if now == last { + return now; + } + assert!(Instant::now() < deadline, "the copy keeps writing"); + last = now; + } +} + +#[cfg(feature = "test-hooks")] +/// A FAILED write on a running restore's record stops the copy at its next +/// heartbeat, as on the other three backends: the heartbeat requires +/// `restore_status = 'RUNNING'`, so it is not applied, the copy stops within +/// a bucket, the target is left CREATING under the FAILED record with fewer +/// items than the backup, the copy's own cleanup touches neither, and the +/// FAILED-target duty then lists the pair and removes the target. The FAILED +/// is written under the record's own generation (the shape a job's failure +/// path writes, and what the reviewer wrote by hand), so the status term is +/// the one condition that stops the heartbeat; without it the heartbeat keeps +/// applying over the FAILED record and the copy writes every item. +#[tokio::test] +async fn test_a_failed_record_stops_the_restore_copy_at_its_next_heartbeat() { + if crate::helpers::skip_without_cassandra() { + return; + } + const ITEMS: usize = 20_000; + let engine = setup_engine().await; + let (account, _source, backup_arn) = backed_up_plain_table(&engine, "FcSource", ITEMS).await; + let account_id = account.account_id.clone(); + let target = restore_mid_copy( + &engine, + &account_id, + &backup_arn, + "FcTarget", + i64::try_from(ITEMS).unwrap(), + ) + .await; + let key = RestoreTargetKey { + account_id: account_id.clone(), + table_name: "FcTarget".to_owned(), + table_id: target.table_id.clone(), + }; + let (status, _, generation, _, _) = job_record(&engine, &account_id, "FcTarget").await; + assert_eq!(status.as_deref(), Some("RUNNING")); + // FAILED under the record's own generation: the status alone changes. + apply( + &engine, + &format!( + "UPDATE {}.tables SET restore_status = 'FAILED', restore_failure_reason = 'owner dead', \ + restore_completed_at = toTimestamp(now()) \ + WHERE account_id = ? AND table_name = ? IF table_id = ? AND restore_status = 'RUNNING'", + engine.catalog_keyspace() + ), + cdrs_tokio::query_values!( + account_id.as_str(), + "FcTarget", + target.table_id.as_str() + ), + ) + .await; + let copied = settled_item_count(&engine, &account_id, &target.table_id).await; + assert!( + copied < i64::try_from(ITEMS).unwrap(), + "the copy stopped short: {copied} of {ITEMS} items" + ); + let described = describe(&engine, &account_id, "FcTarget").await; + assert_eq!( + described.table_status, + TableStatus::Creating, + "the target is left to the reconciler, not flipped ACTIVE and not removed by the copy" + ); + let (status, _, after, _, completed) = job_record(&engine, &account_id, "FcTarget").await; + assert_eq!( + status.as_deref(), + Some("FAILED"), + "the record is left as written" + ); + assert_eq!( + after, generation, + "the stopped copy writes nothing on its row" + ); + assert!(completed.is_some()); + + // The FAILED-target duty: listed once the record is stale, removed. + let dead = dead_jobs_of(&engine, &key, Duration::ZERO).await; + assert_eq!(dead.len(), 1, "{dead:?}"); + assert_eq!(dead[0].status, JobStatus::Failed); + assert!(engine.remove_restore_target(&key).await.expect("remove")); + assert!( + matches!( + engine + .describe_table( + &account_id, + extenddb_core::types::DescribeTableInput { + table_name: "FcTarget".to_owned() + } + ) + .await, + Err(StorageError::TableNotFound(_)) + ), + "the target is gone" + ); + engine + .delete(&account_id, &backup_arn) + .await + .expect("the backup is free once the target is gone"); +} + +#[cfg(feature = "test-hooks")] +/// A target claimed to DELETING mid-copy (the removal's first write) stops +/// the copy at its next heartbeat: the heartbeat requires `table_status = +/// 'CREATING'`, so it is not applied, the copy stops within a bucket and +/// writes nothing more into a table whose owner is dropping it, and the copy's +/// own cleanup touches neither the row nor the data. Without the condition +/// the heartbeat keeps applying and the copy writes every item. +#[tokio::test] +async fn test_a_target_claimed_to_deleting_stops_the_restore_copy_at_its_next_heartbeat() { + if crate::helpers::skip_without_cassandra() { + return; + } + const ITEMS: usize = 20_000; + let engine = setup_engine().await; + let (account, _source, backup_arn) = backed_up_plain_table(&engine, "DcSource", ITEMS).await; + let account_id = account.account_id.clone(); + let target = restore_mid_copy( + &engine, + &account_id, + &backup_arn, + "DcTarget", + i64::try_from(ITEMS).unwrap(), + ) + .await; + let (status, _, generation, _, _) = job_record(&engine, &account_id, "DcTarget").await; + assert_eq!(status.as_deref(), Some("RUNNING")); + // The removal's claim: CREATING to DELETING, as `remove_restore_target`'s + // first write makes it. + apply( + &engine, + &format!( + "UPDATE {}.tables SET table_status = 'DELETING' \ + WHERE account_id = ? AND table_name = ? IF table_id = ? AND table_status = 'CREATING'", + engine.catalog_keyspace() + ), + cdrs_tokio::query_values!(account_id.as_str(), "DcTarget", target.table_id.as_str()), + ) + .await; + let copied = settled_item_count(&engine, &account_id, &target.table_id).await; + assert!( + copied < i64::try_from(ITEMS).unwrap(), + "the copy stopped short: {copied} of {ITEMS} items" + ); + let (status, _, after, _, _) = job_record(&engine, &account_id, "DcTarget").await; + assert_eq!( + status.as_deref(), + Some("RUNNING"), + "the stopped copy writes nothing on a row it no longer owns" + ); + assert_eq!(after, generation); + let (table_status,): (Option,) = { + use cdrs_tokio::types::IntoRustByName; + let rows = extenddb_storage_cassandra::cassandra_util::query_rows_quorum::( + &engine.session_arc(), + &format!( + "SELECT table_status FROM {}.tables WHERE account_id = ? AND table_name = ?", + engine.catalog_keyspace() + ), + cdrs_tokio::query_values!(account_id.as_str(), "DcTarget"), + "table status", + ) + .await + .expect("read"); + (rows + .first() + .and_then(|r| r.get_by_name("table_status").ok().flatten()),) + }; + assert_eq!( + table_status.as_deref(), + Some("DELETING"), + "the removal's claim stands" + ); + + // The removal resumes from the drop (the state the FAILED-target duty + // lists when the record is FAILED); here the record is RUNNING, so finish + // the removal by hand: mark the record FAILED under its claim, then let + // `remove_restore_target` resume from the drop. + let key = RestoreTargetKey { + account_id: account_id.clone(), + table_name: "DcTarget".to_owned(), + table_id: target.table_id.clone(), + }; + apply( + &engine, + &format!( + "UPDATE {}.tables SET restore_status = 'FAILED', \ + restore_completed_at = toTimestamp(now()) \ + WHERE account_id = ? AND table_name = ? IF table_id = ?", + engine.catalog_keyspace() + ), + cdrs_tokio::query_values!(account_id.as_str(), "DcTarget", target.table_id.as_str()), + ) + .await; + assert!(engine.remove_restore_target(&key).await.expect("remove")); + engine + .delete(&account_id, &backup_arn) + .await + .expect("the backup is free once the target is gone"); +} + +#[cfg(feature = "test-hooks")] +/// The abandoned-create fence is conditioned on the heartbeat the scan read: +/// a create whose heartbeat lands between the scan and the fence keeps its +/// CREATING row under its own generation, and is not reported. The scan is +/// paused after the read by a hook; the create's heartbeat lands in the gap; +/// the fence matches nothing. Without the `heartbeat_at = ?` condition the +/// fence would move a row whose owner just proved alive to DELETING. +#[tokio::test] +async fn test_a_create_that_heartbeats_between_the_scan_and_the_fence_keeps_its_row() { + if crate::helpers::skip_without_cassandra() { + return; + } + const ITEMS: usize = 100; + const LIVE_GENERATION: u64 = 4343; + let engine = std::sync::Arc::new(setup_engine().await); + let (account, _source, backup_arn) = backed_up_plain_table(&engine, "HfSource", ITEMS).await; + let account_id = account.account_id.clone(); + // A CREATING row whose heartbeat is two minutes old when the scan reads it. + let catalog = engine.catalog_keyspace(); + let stale = extenddb_storage_cassandra::cassandra_util::now_millis() - 120_000; + apply( + &engine, + &format!( + "UPDATE {catalog}.backups_by_arn SET backup_status = 'CREATING', item_count = 0, \ + owner_generation = ?, heartbeat_at = ? WHERE account_id = ? AND backup_arn = ? \ + IF backup_status = 'AVAILABLE'" + ), + cdrs_tokio::query_values!( + i64::try_from(LIVE_GENERATION).unwrap(), + stale, + account_id.as_str(), + backup_arn.as_str() + ), + ) + .await; + let pause = extenddb_storage_cassandra::test_hooks::pause_before_abandoned_fence(&backup_arn); + let scan = { + let engine = std::sync::Arc::clone(&engine); + tokio::spawn(async move { + engine + .abandoned_creating_backups(Duration::from_secs(60)) + .await + }) + }; + pause.reached().await; + // The create's heartbeat lands after the scan read the row. + assert!( + extenddb_storage_cassandra::heartbeat_backup_row( + &engine, + &account_id, + &backup_arn, + LIVE_GENERATION + ) + .await + .unwrap() + ); + pause.release(); + let reported = scan.await.unwrap().expect("scan"); + assert!( + !reported.iter().any(|a| a.backup_arn == backup_arn), + "a row whose heartbeat landed after the scan is not reported: {reported:?}" + ); + assert_eq!( + backup_status_of(&engine, &account_id, &backup_arn) + .await + .as_deref(), + Some("CREATING"), + "the fence matched nothing" + ); + assert!( + extenddb_storage_cassandra::heartbeat_backup_row( + &engine, + &account_id, + &backup_arn, + LIVE_GENERATION + ) + .await + .unwrap(), + "the row is still the create's: its heartbeat applies" + ); + // The same row, stale again with no heartbeat in the gap: fenced and + // reported, and the create's heartbeat is lost. + apply( + &engine, + &format!( + "UPDATE {catalog}.backups_by_arn SET heartbeat_at = ? \ + WHERE account_id = ? AND backup_arn = ? IF backup_status = 'CREATING'" + ), + cdrs_tokio::query_values!(stale, account_id.as_str(), backup_arn.as_str()), + ) + .await; + let reported = engine + .abandoned_creating_backups(Duration::from_secs(60)) + .await + .expect("scan"); + assert!(reported.iter().any(|a| a.backup_arn == backup_arn)); + assert!( + !extenddb_storage_cassandra::heartbeat_backup_row( + &engine, + &account_id, + &backup_arn, + LIVE_GENERATION + ) + .await + .unwrap() + ); + engine + .delete(&account_id, &backup_arn) + .await + .expect("the reconciler's delete finishes the fenced row"); +} + +/// A stale CREATING catalog index row (`store_kind` set: a store-backed +/// backup whose writer died, which a later release's reconciler resolves) is +/// skipped by the legacy scan: it is not reported, not fenced to DELETING, +/// and keeps its generation, so the legacy reader's `delete` never reaches a +/// row of the other format. Without the skip the scan would fence it and hand +/// it to `delete`. +#[tokio::test] +async fn test_a_stale_creating_index_row_is_skipped_by_the_abandoned_create_scan() { + if crate::helpers::skip_without_cassandra() { + return; + } + let engine = setup_engine().await; + let account = TestAccount::new(&engine, "extenddb_ttl_test").await; + let account_id = account.account_id.clone(); + let store = unique_store(); + let row = index_row(&account_id, "Music", "nightly", &store); + let claim = engine.claim_backup(&row).await.expect("claim"); + let generation = claim.generation(); + // The writer died: its heartbeat is two minutes old. + let catalog = engine.catalog_keyspace(); + let stale = extenddb_storage_cassandra::cassandra_util::now_millis() - 120_000; + apply( + &engine, + &format!( + "UPDATE {catalog}.backups_by_arn SET heartbeat_at = ? \ + WHERE account_id = ? AND backup_arn = ? IF owner_generation = ?" + ), + cdrs_tokio::query_values!( + stale, + account_id.as_str(), + row.backup_arn.as_str(), + i64::try_from(generation).unwrap() + ), + ) + .await; + let reported = engine + .abandoned_creating_backups(Duration::from_secs(60)) + .await + .expect("scan"); + assert!( + !reported.iter().any(|a| a.backup_arn == row.backup_arn), + "an index row is not the legacy scan's: {reported:?}" + ); + let stored = engine + .read_backup_row(&arn_of(&row)) + .await + .expect("read") + .expect("the row is still there"); + assert_eq!( + stored.status, + BackupStatus::Creating, + "not fenced to DELETING" + ); + assert_eq!( + stored.owner_generation, generation, + "the writer's claim stands" + ); + assert!( + matches!( + engine.delete(&account_id, &row.backup_arn).await, + Err(StorageError::Validation(ref m)) if m.contains("Backup not found") + ), + "the legacy reader does not see an index row" + ); + assert!(engine.remove_backup_row(&row.key()).await.is_ok()); +} + +/// `fail_restore_job` is conditioned on the `owner_generation` and +/// `restore_heartbeat_at` the listing read: a heartbeat that lands between +/// the listing and the write keeps the record its owner's, and with the +/// listed values current the write lands under a fresh generation. +#[tokio::test] +async fn test_the_failed_write_matches_only_the_listed_generation_and_heartbeat() { + if crate::helpers::skip_without_cassandra() { + return; + } + let engine = setup_engine().await; + let (account, _source, backup_arn) = backed_up_plain_table(&engine, "FwSource", 50).await; + let account_id = account.account_id.clone(); + let target = restore( + &engine, + &account_id, + &backup_arn, + "FwTarget", + &RestoreTableOverrides::default(), + ) + .await + .expect("restore"); + wait_active(&engine, &account_id, "FwTarget", Duration::from_secs(180)).await; + let key = RestoreTargetKey { + account_id: account_id.clone(), + table_name: "FwTarget".to_owned(), + table_id: target.table_id.clone(), + }; + let catalog = engine.catalog_keyspace(); + let crash = |heartbeat: i64| { + let engine = &engine; + let catalog = &catalog; + let account_id = &account_id; + let table_id = &target.table_id; + async move { + apply( + engine, + &format!( + "UPDATE {catalog}.tables SET table_status = 'CREATING', \ + restore_status = 'RUNNING', restore_completed_at = null, \ + restore_heartbeat_at = ? WHERE account_id = ? AND table_name = ? \ + IF table_id = ?" + ), + cdrs_tokio::query_values!( + heartbeat, + account_id.as_str(), + "FwTarget", + table_id.as_str() + ), + ) + .await; + } + }; + let stale = extenddb_storage_cassandra::cassandra_util::now_millis() - 600_000; + crash(stale).await; + let dead = dead_jobs_of(&engine, &key, Duration::from_secs(300)).await; + assert_eq!(dead.len(), 1, "{dead:?}"); + let (_, _, generation, _, _) = job_record(&engine, &account_id, "FwTarget").await; + assert_eq!( + i64::try_from(dead[0].owner_generation).unwrap(), + generation.expect("generation") + ); + + // The owner's heartbeat lands after the listing. + apply( + &engine, + &format!( + "UPDATE {catalog}.tables SET restore_heartbeat_at = toTimestamp(now()) \ + WHERE account_id = ? AND table_name = ? IF table_id = ?" + ), + cdrs_tokio::query_values!(account_id.as_str(), "FwTarget", target.table_id.as_str()), + ) + .await; + assert!( + !engine + .fail_restore_job( + &key, + dead[0].owner_generation, + dead[0].heartbeat_at, + "owner dead" + ) + .await + .expect("fail"), + "a heartbeat after the listing keeps the record its owner's" + ); + let (status, _, after, _, _) = job_record(&engine, &account_id, "FwTarget").await; + assert_eq!(status.as_deref(), Some("RUNNING")); + assert_eq!(after, generation, "the generation is untouched"); + assert!( + dead_jobs_of(&engine, &key, Duration::from_secs(300)) + .await + .is_empty() + ); + + // Listed values that are still current: the write lands, under a fresh + // generation. + crash(stale).await; + let dead = dead_jobs_of(&engine, &key, Duration::from_secs(300)).await; + assert_eq!(dead.len(), 1); + assert!( + engine + .fail_restore_job( + &key, + dead[0].owner_generation, + dead[0].heartbeat_at, + "owner dead" + ) + .await + .expect("fail") + ); + let (status, _, fenced, _, completed) = job_record(&engine, &account_id, "FwTarget").await; + assert_eq!(status.as_deref(), Some("FAILED")); + assert_ne!( + fenced, generation, + "the FAILED write carries a fresh generation" + ); + assert!(completed.is_some()); + assert!(engine.remove_restore_target(&key).await.expect("remove")); + engine + .delete(&account_id, &backup_arn) + .await + .expect("the backup is free"); +} + +/// `read_store_ids`: the distinct ids of the rows for a root, of every +/// status, scoped to the kind and the root, read from the status partitions. +#[tokio::test] +async fn test_read_store_ids_lists_the_distinct_ids_of_a_root() { + if crate::helpers::skip_without_cassandra() { + return; + } + let engine = setup_engine().await; + let account = TestAccount::new(&engine, "extenddb_ttl_test").await; + let account_id = account.account_id.clone(); + let store = unique_store(); + let root = store.store_root.clone(); + let ids = |kind: StoreKind| { + let engine = &engine; + let root = root.clone(); + async move { engine.read_store_ids(kind, &root).await.expect("ids") } + }; + assert!(ids(StoreKind::Filesystem).await.is_empty()); + let first = index_row(&account_id, "Music", "a", &store); + let _c1 = engine.claim_backup(&first).await.expect("claim"); + let second = index_row(&account_id, "Music", "b", &store); + let c2 = engine.claim_backup(&second).await.expect("claim"); + assert_eq!( + c2.set_status( + JobStatus::Creating, + JobStatus::Failed, + &StatusFields::default() + ) + .await + .expect("fail"), + WriteOutcome::Matched + ); + assert_eq!( + ids(StoreKind::Filesystem).await, + ["store-1"], + "two rows of one store, of two statuses, are one id" + ); + let mut other_store = store.clone(); + other_store.store_id = "store-2".to_owned(); + let other = index_row(&account_id, "Songs", "c", &other_store); + let _c3 = engine.claim_backup(&other).await.expect("claim"); + assert_eq!(ids(StoreKind::Filesystem).await, ["store-1", "store-2"]); + assert!( + ids(StoreKind::S3).await.is_empty(), + "another kind at the same root is another store" + ); + assert!( + engine + .read_store_ids(StoreKind::Filesystem, "/elsewhere") + .await + .expect("ids") + .is_empty() + ); +} + +/// `insert_backup_row`: the row lands as given, unowned, with `completed_at` +/// and `heartbeat_at` on the coordinator's clock, its status entry and both +/// list rows written (`list_backup_rows` reads the first, the account and +/// table listings the others); a second insert and an insert over a CREATING +/// row a claim holds answer `false` and change nothing; a claim over the +/// inserted row is refused. The rule is per ARN spelling: the same id under +/// another region is another row. +#[tokio::test] +async fn test_insert_backup_row_inserts_an_unowned_row_only_when_absent() { + if crate::helpers::skip_without_cassandra() { + return; + } + let engine = setup_engine().await; + let account = TestAccount::new(&engine, "extenddb_ttl_test").await; + let account_id = account.account_id.clone(); + let store = unique_store(); + let mut adopted = index_row(&account_id, "Music", "adopted", &store); + adopted.status = BackupStatus::Available; + adopted.size_bytes = 1234; + adopted.item_count = 5; + // Ignored: the insert sets these three itself. + adopted.owner_generation = 99; + adopted.completed_at = None; + adopted.heartbeat_at = 0.0; + assert!(engine.insert_backup_row(&adopted).await.expect("insert")); + let stored = engine + .read_backup_row(&arn_of(&adopted)) + .await + .expect("read") + .expect("the row exists"); + assert_eq!(stored.owner_generation, UNOWNED_GENERATION); + let completed_at = stored.completed_at.expect("completed_at is set"); + for at in [completed_at, stored.heartbeat_at] { + assert!((at - adopted.created_at).abs() < 300.0, "{at}"); + } + assert!( + (stored.created_at - adopted.created_at).abs() < 1e-3, + "{} is the id's instant {}", + stored.created_at, + adopted.created_at + ); + assert_eq!( + BackupRow { + owner_generation: 99, + completed_at: None, + heartbeat_at: 0.0, + created_at: adopted.created_at, + ..stored.clone() + }, + adopted, + "every other column is the row as given" + ); + + // A second insert of the key, with other values, writes nothing. + let mut again = adopted.clone(); + again.backup_name = "again".to_owned(); + again.size_bytes = 1; + again.status = BackupStatus::Failed; + assert!(!engine.insert_backup_row(&again).await.expect("insert")); + assert_eq!( + engine.read_backup_row(&arn_of(&adopted)).await.unwrap(), + Some(stored.clone()), + "the refused inserts changed nothing" + ); + let claimed = engine.claim_backup(&adopted).await.err(); + assert!( + matches!(claimed, Some(StorageError::AlreadyExists(_))), + "{claimed:?}" + ); + + // Over a CREATING row a live claim holds. + let creating = index_row(&account_id, "Music", "job", &store); + let claim = engine.claim_backup(&creating).await.expect("claim"); + let mut over = creating.clone(); + over.status = BackupStatus::Available; + assert!(!engine.insert_backup_row(&over).await.expect("insert")); + let job_row = engine + .read_backup_row(&arn_of(&creating)) + .await + .unwrap() + .unwrap(); + assert_eq!(job_row.status, BackupStatus::Creating); + assert_eq!(job_row.owner_generation, claim.generation()); + + // Read and listed like a created row. + let arns = + |rows: Vec| -> Vec { rows.into_iter().map(|r| r.backup_arn).collect() }; + let available: Vec = engine + .list_backup_rows(Some(BackupStatus::Available), &store) + .try_collect() + .await + .unwrap(); + assert_eq!(available, std::slice::from_ref(&stored)); + let creating_rows: Vec = engine + .list_backup_rows(Some(BackupStatus::Creating), &store) + .try_collect() + .await + .unwrap(); + assert_eq!( + arns(creating_rows), + std::slice::from_ref(&creating.backup_arn) + ); + for table in [None, Some("Music")] { + let listed: Vec = engine + .list_account_backup_rows(&account_id, table, &TimeBounds::default()) + .try_collect() + .await + .unwrap(); + assert!( + listed.contains(&stored), + "{table:?}: {:?}", + arns(listed.clone()) + ); + } + drop(claim); + + // Cassandra indexes no backup id: another spelling of the ARN inserts. + let mut respelled = adopted.clone(); + respelled.backup_arn = + BackupArn::new("aws", "eu-west-1", &account_id, "Music", &adopted.backup_id) + .unwrap() + .to_arn_string(); + assert!(engine.insert_backup_row(&respelled).await.expect("insert")); + assert!( + engine + .read_backup_row(&arn_of(&respelled)) + .await + .unwrap() + .is_some() + ); + remove_index_rows(&engine, &[&adopted, &respelled, &creating]).await; +} + +/// Delete the three index entries of `row` and keep its `backups_by_arn` +/// row: the state an insert leaves when it stops, or its batch fails, +/// between the two writes. +async fn drop_index_entries(engine: &CassandraEngine, row: &BackupRow) { + let catalog = engine.catalog_keyspace(); + let created_at = (row.created_at * 1_000.0).round() as i64; + for (cql, values) in [ + ( + format!( + "DELETE FROM {catalog}.backups_by_account \ + WHERE account_id = ? AND created_at = ? AND backup_arn = ?" + ), + cdrs_tokio::query_values!(row.account_id.as_str(), created_at, row.backup_arn.as_str()), + ), + ( + format!( + "DELETE FROM {catalog}.backups_by_table WHERE account_id = ? \ + AND table_name = ? AND created_at = ? AND backup_arn = ?" + ), + cdrs_tokio::query_values!( + row.account_id.as_str(), + row.table_name.as_str(), + created_at, + row.backup_arn.as_str() + ), + ), + ( + format!( + "DELETE FROM {catalog}.backup_index_by_status \ + WHERE store_root = ? AND backup_status = ? AND backup_arn = ?" + ), + cdrs_tokio::query_values!( + row.store.store_root.as_str(), + row.status.as_str(), + row.backup_arn.as_str() + ), + ), + ] { + engine + .session() + .query_with_values(&cql, values) + .await + .expect("remove an index entry"); + } +} + +/// Which of the three listings return `row`: the store's status listing +/// (`list_backup_rows`), the account listing and the table listing. +async fn listed_in(engine: &CassandraEngine, row: &BackupRow) -> [bool; 3] { + let by_status: Vec = engine + .list_backup_rows(Some(row.status), &row.store) + .try_collect() + .await + .expect("status listing"); + let mut listed = [by_status.iter().any(|r| r.backup_arn == row.backup_arn); 3]; + for (slot, table) in [(1, None), (2, Some(row.table_name.as_str()))] { + let rows: Vec = engine + .list_account_backup_rows(&row.account_id, table, &TimeBounds::default()) + .try_collect() + .await + .expect("account listing"); + listed[slot] = rows.iter().any(|r| r.backup_arn == row.backup_arn); + } + listed +} + +/// An insert that stopped between its row and its index entries leaves an +/// AVAILABLE row no listing returns. The next insert of the same row (the +/// next sync of the same verified prefix) finds the row its own, writes the +/// entries and answers `true`; an insert with other values still answers +/// `false` and writes nothing. +#[tokio::test] +async fn test_insert_backup_row_completes_an_insert_that_stopped_before_its_entries() { + if crate::helpers::skip_without_cassandra() { + return; + } + let engine = setup_engine().await; + let account = TestAccount::new(&engine, "extenddb_ttl_test").await; + let account_id = account.account_id.clone(); + let store = unique_store(); + let mut row = index_row(&account_id, "Music", "torn", &store); + row.status = BackupStatus::Available; + row.size_bytes = 77; + row.item_count = 3; + assert!(engine.insert_backup_row(&row).await.expect("insert")); + drop_index_entries(&engine, &row).await; + assert_eq!( + listed_in(&engine, &row).await, + [false; 3], + "the torn state: the row is listed nowhere" + ); + + assert!( + engine.insert_backup_row(&row).await.expect("insert"), + "the second insert of the same row completes the first" + ); + assert_eq!(listed_in(&engine, &row).await, [true; 3]); + let stored = engine + .read_backup_row(&arn_of(&row)) + .await + .unwrap() + .expect("the row exists"); + assert_eq!(stored.owner_generation, UNOWNED_GENERATION); + + let mut other = row.clone(); + other.size_bytes = 1; + drop_index_entries(&engine, &row).await; + assert!( + !engine.insert_backup_row(&other).await.expect("insert"), + "a row with other values is one that existed" + ); + assert_eq!(listed_in(&engine, &row).await, [false; 3]); + remove_index_rows(&engine, &[&row]).await; +} + +/// An insert at an ARN an earlier release's in-catalog backup holds (a +/// `backups_by_arn` row with no store kind, which the index read does not +/// return) answers `false` and writes no list or status entry: the +/// `NotMatched` arm's serial read finds no index row, and that is not this +/// insert's row. +#[tokio::test] +async fn test_an_insert_over_an_in_catalog_backup_answers_false_and_writes_no_entry() { + if crate::helpers::skip_without_cassandra() { + return; + } + let engine = setup_engine().await; + let account = TestAccount::new(&engine, "extenddb_ttl_test").await; + let account_id = account.account_id.clone(); + let store = unique_store(); + let mut row = index_row(&account_id, "Music", "legacy", &store); + row.status = BackupStatus::Available; + let catalog = engine.catalog_keyspace(); + engine + .session() + .query_with_values( + &format!( + "INSERT INTO {catalog}.backups_by_arn (account_id, backup_arn, backup_id, \ + backup_name, table_name, backup_status) VALUES (?, ?, ?, ?, ?, 'AVAILABLE')" + ), + cdrs_tokio::query_values!( + row.account_id.as_str(), + row.backup_arn.as_str(), + row.backup_id.as_str(), + "legacy", + "Music" + ), + ) + .await + .expect("an in-catalog backup's row"); + let inserted = engine.insert_backup_row(&row).await.expect("insert"); + let entries = raw_index_entries(&engine, &row).await; + remove_index_rows(&engine, &[&row]).await; + assert!( + !inserted, + "an insert over an in-catalog backup answered true" + ); + assert_eq!( + entries, 0, + "an insert over an in-catalog backup wrote entries" + ); +} + +/// An insert whose answer is lost is settled by a serial read: one whose +/// transaction ran finds its row and answers `true` with its entries +/// written; one whose transaction never ran finds no row, sends it once +/// more and answers `true`; one that finds a row a claim holds answers +/// `false` and changes nothing; two lost answers with no row are an error +/// and leave no row. +#[cfg(feature = "test-hooks")] +#[tokio::test] +async fn test_an_unknown_insert_outcome_is_settled_by_a_serial_read() { + use extenddb_storage_cassandra::test_hooks::{ProgressWriteFault, fail_insert_writes}; + if crate::helpers::skip_without_cassandra() { + return; + } + let engine = setup_engine().await; + let account = TestAccount::new(&engine, "extenddb_ttl_test").await; + let account_id = account.account_id.clone(); + let store = unique_store(); + let available = |name: &str| { + let mut row = index_row(&account_id, "Music", name, &store); + row.status = BackupStatus::Available; + row + }; + + // The transaction ran; its answer was lost. + let landed = available("landed"); + fail_insert_writes(&landed.backup_arn, &[ProgressWriteFault::Landed]); + assert!(engine.insert_backup_row(&landed).await.expect("insert")); + assert_eq!(listed_in(&engine, &landed).await, [true; 3]); + + // The transaction never ran: the write is sent once more. + let lost = available("lost"); + fail_insert_writes(&lost.backup_arn, &[ProgressWriteFault::Lost]); + assert!(engine.insert_backup_row(&lost).await.expect("insert")); + assert_eq!(listed_in(&engine, &lost).await, [true; 3]); + let stored = engine.read_backup_row(&arn_of(&lost)).await.unwrap(); + assert_eq!(stored.map(|r| r.owner_generation), Some(UNOWNED_GENERATION)); + + // A row a claim holds is there: the insert is refused and changes nothing. + let creating = index_row(&account_id, "Music", "job", &store); + let claim = engine.claim_backup(&creating).await.expect("claim"); + let mut over = creating.clone(); + over.status = BackupStatus::Available; + fail_insert_writes(&over.backup_arn, &[ProgressWriteFault::Lost]); + assert!(!engine.insert_backup_row(&over).await.expect("insert")); + let job_row = engine + .read_backup_row(&arn_of(&creating)) + .await + .unwrap() + .unwrap(); + assert_eq!( + (job_row.status, job_row.owner_generation), + (BackupStatus::Creating, claim.generation()) + ); + assert!( + !listed_in(&engine, &over).await[0], + "no AVAILABLE status entry" + ); + drop(claim); + + // Two lost answers and no row: an unknown outcome is an error. + let unknown = available("unknown"); + fail_insert_writes( + &unknown.backup_arn, + &[ProgressWriteFault::Lost, ProgressWriteFault::Lost], + ); + let failed = engine + .insert_backup_row(&unknown) + .await + .expect_err("two lost answers and no row"); + assert!( + matches!(&failed, StorageError::Internal(m) if m.contains("unknown outcome")), + "{failed}" + ); + assert_eq!( + engine.read_backup_row(&arn_of(&unknown)).await.unwrap(), + None + ); + remove_index_rows(&engine, &[&landed, &lost, &creating, &unknown]).await; +} + +/// A failed batch of index entries is sent once more; one that fails twice +/// is an error that leaves the row unlisted, and the next insert of the row +/// writes the entries. +#[cfg(feature = "test-hooks")] +#[tokio::test] +async fn test_a_failed_index_entry_batch_is_sent_once_more() { + use extenddb_storage_cassandra::test_hooks::fail_index_entry_batches; + if crate::helpers::skip_without_cassandra() { + return; + } + let engine = setup_engine().await; + let account = TestAccount::new(&engine, "extenddb_ttl_test").await; + let account_id = account.account_id.clone(); + let store = unique_store(); + let available = |name: &str| { + let mut row = index_row(&account_id, "Music", name, &store); + row.status = BackupStatus::Available; + row + }; + + let once = available("once"); + fail_index_entry_batches(&once.backup_arn, 1); + assert!(engine.insert_backup_row(&once).await.expect("insert")); + assert_eq!(listed_in(&engine, &once).await, [true; 3]); + + let twice = available("twice"); + fail_index_entry_batches(&twice.backup_arn, 2); + let failed = engine + .insert_backup_row(&twice) + .await + .expect_err("a batch that fails twice"); + assert!( + matches!(&failed, StorageError::Internal(m) if m.contains("index entry batch")), + "{failed}" + ); + assert!( + engine + .read_backup_row(&arn_of(&twice)) + .await + .unwrap() + .is_some() + ); + assert_eq!(listed_in(&engine, &twice).await, [false; 3]); + assert!(engine.insert_backup_row(&twice).await.expect("insert")); + assert_eq!(listed_in(&engine, &twice).await, [true; 3]); + remove_index_rows(&engine, &[&once, &twice]).await; +} + +/// A removal whose answer is lost is settled by a serial read: one whose +/// transaction ran finds no row and answers `Removed` with the entries +/// removed; one whose transaction never ran finds the row, runs once more and +/// answers `Removed`; a lost answer on every attempt is an error that leaves +/// the row AVAILABLE and listed. +#[cfg(feature = "test-hooks")] +#[tokio::test] +async fn test_an_unknown_forget_outcome_is_settled_by_a_serial_read() { + use extenddb_storage::backup_catalog::ForgetRow; + use extenddb_storage_cassandra::test_hooks::{ProgressWriteFault, fail_forget_writes}; + if crate::helpers::skip_without_cassandra() { + return; + } + let engine = setup_engine().await; + let account = TestAccount::new(&engine, "extenddb_ttl_test").await; + let account_id = account.account_id.clone(); + let store = unique_store(); + let adopted = |name: &str| { + let mut row = index_row(&account_id, "Music", name, &store); + row.status = BackupStatus::Available; + row + }; + + for (name, fault) in [ + ("landed", ProgressWriteFault::Landed), + ("lost", ProgressWriteFault::Lost), + ] { + let row = adopted(name); + assert!(engine.insert_backup_row(&row).await.expect("insert")); + fail_forget_writes(&row.backup_arn, &[fault]); + assert_eq!( + engine + .forget_available_backup_row(&row.key()) + .await + .unwrap(), + ForgetRow::Removed, + "{name}" + ); + assert_eq!( + engine.read_backup_row(&arn_of(&row)).await.unwrap(), + None, + "{name}" + ); + assert_eq!(raw_index_entries(&engine, &row).await, 0, "{name}"); + } + + let unknown = adopted("unknown"); + assert!(engine.insert_backup_row(&unknown).await.expect("insert")); + fail_forget_writes(&unknown.backup_arn, &[ProgressWriteFault::Lost; 3]); + let failed = engine + .forget_available_backup_row(&unknown.key()) + .await + .expect_err("a lost answer on every attempt"); + assert!( + matches!(&failed, StorageError::Internal(m) if m.contains("unknown outcome")), + "{failed}" + ); + assert_eq!( + engine + .read_backup_row(&arn_of(&unknown)) + .await + .unwrap() + .map(|r| r.status), + Some(BackupStatus::Available) + ); + assert_eq!(listed_in(&engine, &unknown).await, [true; 3]); + remove_index_rows(&engine, &[&unknown]).await; +} + +/// A removal whose entry batch fails after the row is gone answers `Removed`: +/// the row is read and listed nowhere, the entries it left are skipped by +/// every listing, and a second removal finds no row. +#[cfg(feature = "test-hooks")] +#[tokio::test] +async fn test_a_forget_whose_entry_removal_fails_answers_removed() { + use extenddb_storage::backup_catalog::ForgetRow; + use extenddb_storage_cassandra::test_hooks::fail_index_entry_removals; + if crate::helpers::skip_without_cassandra() { + return; + } + let engine = setup_engine().await; + let account = TestAccount::new(&engine, "extenddb_ttl_test").await; + let account_id = account.account_id.clone(); + let store = unique_store(); + let mut row = index_row(&account_id, "Music", "orphaned", &store); + row.status = BackupStatus::Available; + assert!(engine.insert_backup_row(&row).await.expect("insert")); + fail_index_entry_removals(&row.backup_arn, 1); + assert_eq!( + engine + .forget_available_backup_row(&row.key()) + .await + .unwrap(), + ForgetRow::Removed + ); + assert_eq!(engine.read_backup_row(&arn_of(&row)).await.unwrap(), None); + assert_eq!( + raw_index_entries(&engine, &row).await, + 3, + "the failed batch left the entries" + ); + assert_eq!( + listed_in(&engine, &row).await, + [false; 3], + "every listing skips an entry whose row is gone" + ); + assert!(matches!( + engine.forget_available_backup_row(&row.key()).await, + Err(StorageError::Validation(m)) if m.contains("Backup not found") + )); + remove_index_rows(&engine, &[&row]).await; +} + +/// The in-use rule runs before the removal's write and clears a registration +/// whose target is no longer being restored: a row whose `active_restores` +/// still names a finished (ACTIVE) target is removed at once. Without the +/// rule the write's `active_restores = null` condition refuses every attempt +/// and the removal answers `InUse` naming the finished table. +#[tokio::test] +async fn test_a_forget_clears_a_finished_restores_registration_and_removes_the_row() { + use extenddb_storage::backup_catalog::ForgetRow; + if crate::helpers::skip_without_cassandra() { + return; + } + let engine = setup_engine().await; + let account = TestAccount::new(&engine, "extenddb_ttl_test").await; + let account_id = account.account_id.clone(); + let store = unique_store(); + let mut row = index_row(&account_id, "Music", "finished", &store); + row.status = BackupStatus::Available; + assert!(engine.insert_backup_row(&row).await.expect("insert")); + let finished = create_active_table( + &engine, + &account_id, + CreateTableInput { + table_name: "FinishedTarget".to_owned(), + key_schema: vec![ks("id", KeyType::Hash)], + attribute_definitions: vec![attr("id", ScalarAttributeType::S)], + billing_mode: Some(BillingMode::PayPerRequest), + ..Default::default() + }, + ) + .await; + apply( + &engine, + &format!( + "UPDATE {}.backups_by_arn SET active_restores[?] = ? \ + WHERE account_id = ? AND backup_arn = ? IF EXISTS", + engine.catalog_keyspace() + ), + cdrs_tokio::query_values!( + finished.table_id.as_str(), + "FinishedTarget", + account_id.as_str(), + row.backup_arn.as_str() + ), + ) + .await; + assert_eq!( + engine + .forget_available_backup_row(&row.key()) + .await + .unwrap(), + ForgetRow::Removed + ); + assert_eq!(engine.read_backup_row(&arn_of(&row)).await.unwrap(), None); + assert_eq!(raw_index_entries(&engine, &row).await, 0); +} + +/// `remove_dead_creating_backup_row`: a CREATING row whose heartbeat is older +/// than the window is removed outright, its status-index entry with it; one +/// with a live owner, one that is not CREATING, and one that is gone are left, +/// answering `false`. +#[tokio::test] +async fn test_a_dead_creating_row_is_removed_and_a_live_or_moved_one_is_kept() { + if crate::helpers::skip_without_cassandra() { + return; + } + let engine = setup_engine().await; + let account = TestAccount::new(&engine, "extenddb_ttl_test").await; + let account_id = account.account_id.clone(); + let store = unique_store(); + let stale = Duration::from_secs(300); + let ten_minutes_ago = i64::try_from( + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_millis(), + ) + .unwrap() + - 600_000; + let age = |backup_arn: String, generation: u64| { + let engine = &engine; + let account_id = account_id.clone(); + async move { + apply( + engine, + &format!( + "UPDATE {}.backups_by_arn SET heartbeat_at = ? \ + WHERE account_id = ? AND backup_arn = ? IF owner_generation = ?", + engine.catalog_keyspace() + ), + cdrs_tokio::query_values!( + ten_minutes_ago, + account_id.as_str(), + backup_arn.as_str(), + i64::try_from(generation).unwrap() + ), + ) + .await; + } + }; + let creating = index_row(&account_id, "Music", "dead-a", &store); + let key = creating.key(); + let claim = engine.claim_backup(&creating).await.expect("claim"); + assert!( + !engine + .remove_dead_creating_backup_row(&key, stale) + .await + .unwrap(), + "a live owner keeps its row" + ); + age(creating.backup_arn.clone(), claim.generation()).await; + assert!( + engine + .remove_dead_creating_backup_row(&key, stale) + .await + .unwrap() + ); + assert!( + engine + .read_backup_row(&arn_of(&creating)) + .await + .unwrap() + .is_none() + ); + let creating_rows: Vec = engine + .list_backup_rows(Some(BackupStatus::Creating), &store) + .try_collect() + .await + .unwrap(); + assert!( + creating_rows.is_empty(), + "the status index entry went with the row: {creating_rows:?}" + ); + assert!( + !engine + .remove_dead_creating_backup_row(&key, stale) + .await + .unwrap(), + "gone" + ); + drop(claim); + + let available = index_row(&account_id, "Music", "dead-b", &store); + let claim = engine.claim_backup(&available).await.expect("claim"); + assert_eq!( + claim + .set_status( + JobStatus::Creating, + JobStatus::Available, + &StatusFields::default() + ) + .await + .expect("available"), + WriteOutcome::Matched + ); + age(available.backup_arn.clone(), claim.generation()).await; + assert!( + !engine + .remove_dead_creating_backup_row(&available.key(), stale) + .await + .unwrap(), + "only a CREATING row" + ); + assert_eq!( + engine + .read_backup_row(&arn_of(&available)) + .await + .unwrap() + .unwrap() + .status, + BackupStatus::Available + ); + drop(claim); +} + +/// `list_backup_rows_older_than`: the rows of a store with a status whose +/// named clock column is older than the age on the coordinator's clock. A +/// row FAILED after its heartbeat aged is listed on the heartbeat clock and +/// not on the completed clock until that window too has passed (waited out +/// on the coordinator's clock, not hand-dated); a row with no `completed_at` +/// is never listed on the completed clock; another status and another store +/// are not listed. +#[tokio::test] +async fn test_rows_older_than_are_listed_on_the_named_clock() { + if crate::helpers::skip_without_cassandra() { + return; + } + let engine = setup_engine().await; + let account = TestAccount::new(&engine, "extenddb_ttl_test").await; + let account_id = account.account_id.clone(); + let store = unique_store(); + let window = Duration::from_secs(1); + let listed = |status: BackupStatus, clock: StaleClock, age: Duration| { + let engine = &engine; + let store = &store; + async move { + let mut arns: Vec = engine + .list_backup_rows_older_than(status, store, clock, age) + .map_ok(|row| row.backup_arn) + .try_collect() + .await + .expect("listing"); + arns.sort(); + arns + } + }; + let completed_failure = StatusFields { + failure_reason: Some("owner dead".to_owned()), + completed: true, + ..StatusFields::default() + }; + let open_failure = StatusFields { + failure_reason: Some("owner dead".to_owned()), + completed: false, + ..StatusFields::default() + }; + let stamped = index_row(&account_id, "Music", "stamped", &store); + let unstamped = index_row(&account_id, "Music", "unstamped", &store); + let creating = index_row(&account_id, "Music", "creating", &store); + let mut other_store = store.clone(); + other_store.store_id = "store-2".to_owned(); + let elsewhere = index_row(&account_id, "Music", "elsewhere", &other_store); + let stamped_claim = engine.claim_backup(&stamped).await.expect("claim"); + let unstamped_claim = engine.claim_backup(&unstamped).await.expect("claim"); + let _creating_claim = engine.claim_backup(&creating).await.expect("claim"); + let elsewhere_claim = engine.claim_backup(&elsewhere).await.expect("claim"); + assert_eq!( + elsewhere_claim + .set_status(JobStatus::Creating, JobStatus::Failed, &completed_failure) + .await + .expect("write"), + WriteOutcome::Matched + ); + assert!( + listed(BackupStatus::Failed, StaleClock::Heartbeat, Duration::ZERO) + .await + .is_empty(), + "nothing of this store is FAILED yet" + ); + + // The heartbeats age past the window; then the two rows fail, one with a + // completion time and one without. + tokio::time::sleep(Duration::from_millis(1_300)).await; + assert_eq!( + stamped_claim + .set_status(JobStatus::Creating, JobStatus::Failed, &completed_failure) + .await + .expect("write"), + WriteOutcome::Matched + ); + assert_eq!( + unstamped_claim + .set_status(JobStatus::Creating, JobStatus::Failed, &open_failure) + .await + .expect("write"), + WriteOutcome::Matched + ); + let mut both = vec![stamped.backup_arn.clone(), unstamped.backup_arn.clone()]; + both.sort(); + assert_eq!( + listed(BackupStatus::Failed, StaleClock::Heartbeat, window).await, + both, + "a row just failed is listed on the heartbeat clock: its dead owner's beat is stale" + ); + assert!( + listed(BackupStatus::Failed, StaleClock::Completed, window) + .await + .is_empty(), + "the completion time is an instant old" + ); + tokio::time::sleep(Duration::from_millis(50)).await; + assert_eq!( + listed(BackupStatus::Failed, StaleClock::Completed, Duration::ZERO).await, + std::slice::from_ref(&stamped.backup_arn), + "a row with no completed_at is not listed on the completed clock" + ); + assert!( + listed( + BackupStatus::Failed, + StaleClock::Heartbeat, + Duration::from_secs(300) + ) + .await + .is_empty(), + "a long window lists nothing" + ); + assert_eq!( + listed(BackupStatus::Creating, StaleClock::Heartbeat, window).await, + std::slice::from_ref(&creating.backup_arn), + "the CREATING row is listed under its own status only" + ); + assert!( + listed( + BackupStatus::Creating, + StaleClock::Completed, + Duration::ZERO + ) + .await + .is_empty(), + "a CREATING row has no completion time" + ); + assert!( + listed(BackupStatus::Deleted, StaleClock::Completed, Duration::ZERO) + .await + .is_empty() + ); + + // The completed window crossed on the coordinator's clock. + tokio::time::sleep(Duration::from_millis(1_300)).await; + assert_eq!( + listed(BackupStatus::Failed, StaleClock::Completed, window).await, + std::slice::from_ref(&stamped.backup_arn) + ); + assert_eq!( + listed(BackupStatus::Failed, StaleClock::Heartbeat, window).await, + both, + "the other store's FAILED row is not listed" + ); +} + +/// The maintenance claim: taken once, refused to a second holder naming the +/// first, heartbeated on the coordinator's clock, released; a holder whose +/// heartbeat is older than the stale window is dead, so its claim is not +/// reported held and is taken over with a lightweight transaction on its +/// generation, and its handle finds the claim lost. The catalog is shared by +/// this binary's tests, so the test releases what it takes. +#[tokio::test] +async fn test_the_maintenance_claim_excludes_a_second_holder_and_expires_when_stale() { + if crate::helpers::skip_without_cassandra() { + return; + } + let engine = setup_engine().await; + let stale = Duration::from_secs(300); + let check = || engine.check_maintenance_claim(stale); + // Another test of this binary never holds the claim; a stale row an + // interrupted run left is taken over below like any dead holder's. + let first = engine + .maintenance_claim("purge on host-a", stale) + .await + .expect("claim"); + assert_ne!(first.generation(), UNOWNED_GENERATION); + assert!(check().await.expect("check"), "a live claim is held"); + match engine.maintenance_claim("purge on host-b", stale).await { + Err(StorageError::AlreadyExists(m)) => assert!(m.contains("purge on host-a"), "{m}"), + Err(other) => panic!("a second holder is refused with AlreadyExists: {other:?}"), + Ok(_) => panic!("a second holder is refused"), + } + assert_eq!( + first.heartbeat().await.expect("heartbeat"), + Heartbeat::Confirmed + ); + first.release().await.expect("release"); + assert!(!check().await.expect("check"), "released"); + first.release().await.expect("a second release is a no-op"); + + // A dead holder: its heartbeat aged past the window. + let dead = engine + .maintenance_claim("purge on host-c", stale) + .await + .expect("claim"); + let aged = format!( + "UPDATE {}.backup_maintenance SET heartbeat_at = ? WHERE claim = 'claim' \ + IF owner_generation = ?", + engine.catalog_keyspace() + ); + let ten_minutes_ago = i64::try_from( + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_millis(), + ) + .unwrap() + - 600_000; + apply( + &engine, + &aged, + cdrs_tokio::query_values!(ten_minutes_ago, i64::try_from(dead.generation()).unwrap()), + ) + .await; + assert!(!check().await.expect("check"), "a stale claim is not held"); + let taker = engine + .maintenance_claim("purge on host-d", stale) + .await + .expect("a dead holder's claim is taken over"); + assert_ne!(taker.generation(), dead.generation()); + assert!(check().await.expect("check")); + assert_eq!( + dead.heartbeat().await.expect("heartbeat"), + Heartbeat::Lost, + "the dead holder's handle finds the claim lost" + ); + dead.release() + .await + .expect("a release by a lost holder changes nothing"); + assert!(check().await.expect("check"), "the taker still holds it"); + taker.release().await.expect("release"); + assert!(!check().await.expect("check")); +} + +/// Two takers racing for one dead holder's row see one win: each reads the +/// stale row (the test hook holds both after their read, so neither sees +/// the other's write first) and runs `UPDATE ... IF owner_generation = `, +/// and the lightweight transaction matches once. With `IF EXISTS` in its +/// place both would win and two commands would hold one claim. The loser is +/// refused naming a holder, and the winner's handle heartbeats. +#[cfg(feature = "test-hooks")] +#[tokio::test] +async fn test_two_takers_of_one_stale_maintenance_claim_see_one_win() { + if crate::helpers::skip_without_cassandra() { + return; + } + let engine = setup_engine().await; + let stale = Duration::from_secs(300); + let dead = engine + .maintenance_claim("purge on host-dead", stale) + .await + .expect("claim"); + let aged = format!( + "UPDATE {}.backup_maintenance SET heartbeat_at = ? WHERE claim = 'claim' \ + IF owner_generation = ?", + engine.catalog_keyspace() + ); + let ten_minutes_ago = i64::try_from( + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_millis(), + ) + .unwrap() + - 600_000; + apply( + &engine, + &aged, + cdrs_tokio::query_values!(ten_minutes_ago, i64::try_from(dead.generation()).unwrap()), + ) + .await; + assert!( + !engine.check_maintenance_claim(stale).await.expect("check"), + "a stale claim is not held" + ); + let pause = extenddb_storage_cassandra::test_hooks::pause_before_maintenance_takeover(); + let (a, b, ()) = tokio::join!( + engine.maintenance_claim("purge on host-a", stale), + engine.maintenance_claim("purge on host-b", stale), + async { + pause.reached().await; + pause.reached().await; + extenddb_storage_cassandra::test_hooks::clear_maintenance_takeover_pause(); + pause.release(); + pause.release(); + }, + ); + let (winner, loser) = match (a, b) { + (Ok(w), Err(l)) | (Err(l), Ok(w)) => (w, l), + (Ok(_), Ok(_)) => panic!("two takers both won one stale claim"), + (Err(a), Err(b)) => panic!("neither taker won: {a:?} / {b:?}"), + }; + match loser { + StorageError::AlreadyExists(m) => assert!(m.contains("purge on host-"), "{m}"), + other => panic!("the loser is refused with AlreadyExists: {other:?}"), + } + assert_ne!(winner.generation(), dead.generation()); + assert_eq!( + winner.heartbeat().await.expect("heartbeat"), + Heartbeat::Confirmed + ); + assert_eq!( + dead.heartbeat().await.expect("heartbeat"), + Heartbeat::Lost, + "the dead holder's handle finds the claim lost" + ); + assert!(engine.check_maintenance_claim(stale).await.expect("check")); + winner.release().await.expect("release"); + assert!(!engine.check_maintenance_claim(stale).await.expect("check")); +} + +/// An insert whose answer is lost after it committed (the test hook turns +/// the applied answer into `Unknown`) is settled by the re-read: the row +/// carries the call's generation, so the call holds the claim, heartbeats it +/// and releases it. Without the re-read the command would be refused while +/// its own row, which nobody heartbeats, refused every job until it went +/// stale. +#[cfg(feature = "test-hooks")] +#[tokio::test] +async fn test_a_maintenance_insert_whose_answer_is_lost_is_settled_by_the_re_read() { + if crate::helpers::skip_without_cassandra() { + return; + } + let engine = setup_engine().await; + let stale = Duration::from_secs(300); + assert!( + !engine.check_maintenance_claim(stale).await.expect("check"), + "no claim is held before the test" + ); + extenddb_storage_cassandra::test_hooks::lose_next_maintenance_insert_answer(); + let claim = engine + .maintenance_claim("purge on host-lost", stale) + .await + .expect("an insert whose answer was lost is this call's claim once re-read"); + assert!( + engine.check_maintenance_claim(stale).await.expect("check"), + "the committed insert is held" + ); + let rows = extenddb_storage_cassandra::cassandra_util::query_rows::( + &engine.session_arc(), + &format!( + "SELECT holder, owner_generation FROM {}.backup_maintenance WHERE claim = 'claim'", + engine.catalog_keyspace() + ), + cdrs_tokio::query_values!(), + "maintenance claim", + ) + .await + .expect("read"); + use cdrs_tokio::types::IntoRustByName; + let holder: Option = rows + .first() + .and_then(|r| r.get_by_name("holder").ok().flatten()); + let generation: Option = rows + .first() + .and_then(|r| r.get_by_name("owner_generation").ok().flatten()); + assert_eq!(holder.as_deref(), Some("purge on host-lost")); + assert_eq!(generation, Some(i64::try_from(claim.generation()).unwrap())); + assert_eq!( + claim.heartbeat().await.expect("heartbeat"), + Heartbeat::Confirmed + ); + claim.release().await.expect("release"); + assert!(!engine.check_maintenance_claim(stale).await.expect("check")); +} + +/// `newest_available_backup_row` reads the status index's AVAILABLE +/// partition and one row: the newest AVAILABLE row in the listing's order, +/// none while the store has no AVAILABLE row, and never a row of another +/// store or status. `list_running_restore_jobs` lists a RUNNING record by +/// status alone, one whose heartbeat is ahead of the coordinator's clock +/// included, which the dead-job listing under a zero window does not list. +#[tokio::test] +async fn test_the_newest_available_row_and_the_running_restores_are_read_by_status() { + if crate::helpers::skip_without_cassandra() { + return; + } + let engine = setup_engine().await; + let account = TestAccount::new(&engine, "extenddb_ttl_test").await; + let account_id = account.account_id.clone(); + let store = unique_store(); + assert!( + engine + .newest_available_backup_row(&store) + .await + .expect("read") + .is_none(), + "no AVAILABLE row yet" + ); + let publish = |row: BackupRow| { + let engine = &engine; + async move { + let claim = engine.claim_backup(&row).await.expect("claim"); + assert_eq!( + claim + .set_status( + JobStatus::Creating, + JobStatus::Available, + &StatusFields::default() + ) + .await + .expect("publish"), + WriteOutcome::Matched + ); + } + }; + publish(index_row(&account_id, "Music", "older", &store)).await; + tokio::time::sleep(Duration::from_millis(5)).await; + publish(index_row(&account_id, "Music", "newer", &store)).await; + let creating = index_row(&account_id, "Music", "creating", &store); + let _creating = engine.claim_backup(&creating).await.expect("claim"); + let mut other_store = store.clone(); + other_store.store_id = "store-2".to_owned(); + publish(index_row(&account_id, "Songs", "elsewhere", &other_store)).await; + let newest = engine + .newest_available_backup_row(&store) + .await + .expect("read") + .expect("one AVAILABLE row"); + let listed: Vec = engine + .list_backup_rows(Some(BackupStatus::Available), &store) + .try_collect() + .await + .expect("list"); + assert_eq!(listed.len(), 2); + assert_eq!(newest.backup_arn, listed.last().unwrap().backup_arn); + assert_eq!(newest.backup_name, "newer"); + + // A RUNNING restore record stamped a minute ahead of the coordinator. + let table = TestTable::with_account(&engine, &account_id, "rr_target", false).await; + let table_id = table.key_info.table_id.clone(); + let ahead = i64::try_from( + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_millis(), + ) + .unwrap() + + 60_000; + apply( + &engine, + &format!( + "UPDATE {}.tables SET restore_status = 'RUNNING', restore_kind = 'legacy', \ + restore_owner_generation = 7, restore_heartbeat_at = ?, \ + restore_source_backup_arn = ? WHERE account_id = ? AND table_name = ? \ + IF table_id = ?", + engine.catalog_keyspace() + ), + cdrs_tokio::query_values!( + ahead, + newest.backup_arn.as_str(), + account_id.as_str(), + "rr_target", + table_id.as_str() + ), + ) + .await; + let running = engine + .list_running_restore_jobs() + .await + .expect("list running"); + let ours: Vec<&extenddb_storage::backup_catalog::RestoreJobRow> = running + .iter() + .filter(|job| job.target.account_id == account_id) + .collect(); + assert_eq!(ours.len(), 1, "{running:?}"); + assert_eq!(ours[0].target.table_name, "rr_target"); + assert_eq!(ours[0].target.table_id, table_id); + assert_eq!(ours[0].status, JobStatus::Running); + assert!( + !engine + .list_dead_restore_jobs(Duration::ZERO) + .await + .expect("list dead") + .iter() + .any(|job| job.target.account_id == account_id), + "the age-based listing misses a heartbeat ahead of the clock" + ); +} +/// The index row of `row`, read back through the catalog. +async fn stored_index_row(engine: &CassandraEngine, row: &BackupRow) -> BackupRow { + engine + .read_backup_row(&arn_of(row)) + .await + .expect("read") + .expect("the row exists") +} + +/// `claim_dead_backup`: refused while the owner's heartbeat is within the +/// window, on the coordinator's clock; once the window has passed (waited +/// out, not hand-dated) the row is taken over under a fresh generation with +/// its heartbeat left as the dead owner wrote it, the old claim finds its +/// heartbeat lost and its writes unmatched, and the successor's AVAILABLE +/// write carries the source table and a completion time and moves the status +/// index. A successor that heartbeats is a live owner to a third taker; a row +/// that is not CREATING, and a missing row, answer `None`. +#[tokio::test] +async fn test_claim_dead_backup_takes_over_a_stale_row_and_refuses_a_live_one() { + if crate::helpers::skip_without_cassandra() { + return; + } + let engine = setup_engine().await; + let account = TestAccount::new(&engine, "extenddb_ttl_test").await; + let account_id = account.account_id.clone(); + let store = unique_store(); + let row = index_row(&account_id, "Music", "dead-owner", &store); + let key = row.key(); + let window = Duration::from_secs(1); + assert!( + engine + .claim_dead_backup(&key, window) + .await + .expect("claim") + .is_none(), + "a missing row" + ); + let owner = engine.claim_backup(&row).await.expect("claim"); + let before = stored_index_row(&engine, &row).await; + assert!( + engine + .claim_dead_backup(&key, Duration::from_secs(300)) + .await + .expect("claim") + .is_none(), + "a live owner within a long window" + ); + assert!( + engine + .claim_dead_backup(&key, window) + .await + .expect("claim") + .is_none(), + "a heartbeat written an instant ago is within a one-second window" + ); + tokio::time::sleep(Duration::from_millis(1_300)).await; + let successor = engine + .claim_dead_backup(&key, window) + .await + .expect("claim") + .expect("the window passed on the coordinator's clock: taken over"); + assert_ne!(successor.generation(), owner.generation()); + assert_ne!(successor.generation(), UNOWNED_GENERATION); + let after = stored_index_row(&engine, &row).await; + assert_eq!(after.owner_generation, successor.generation()); + assert_eq!(after.status, BackupStatus::Creating); + assert_eq!( + after.heartbeat_at, before.heartbeat_at, + "the takeover left the heartbeat as the dead owner wrote it" + ); + assert_eq!( + owner.heartbeat().await.expect("heartbeat"), + Heartbeat::Lost, + "the old owner's heartbeat finds another generation" + ); + assert_eq!( + owner + .set_status( + JobStatus::Creating, + JobStatus::Failed, + &StatusFields::default() + ) + .await + .expect("write"), + WriteOutcome::NotMatched, + "the old generation's write is fenced" + ); + + // The successor heartbeats: a third taker finds a live owner. + assert_eq!( + successor.heartbeat().await.expect("heartbeat"), + Heartbeat::Confirmed + ); + assert!( + engine + .claim_dead_backup(&key, window) + .await + .expect("claim") + .is_none(), + "the successor's heartbeat is fresh" + ); + + // The successor's AVAILABLE write carries the manifest's source table. + let mut source_table = row.source_table_details_json.clone(); + source_table["item_count"] = serde_json::json!(41); + let fields = StatusFields { + size_bytes: Some(4_096), + item_count: Some(41), + failure_reason: None, + source_table_details_json: Some(source_table.clone()), + completed: true, + }; + assert_eq!( + successor + .set_status(JobStatus::Creating, JobStatus::Available, &fields) + .await + .expect("write"), + WriteOutcome::Matched + ); + let available = stored_index_row(&engine, &row).await; + assert_eq!(available.status, BackupStatus::Available); + assert_eq!(available.size_bytes, 4_096); + assert_eq!(available.item_count, 41); + assert_eq!( + available.source_table_details_json, source_table, + "the write applied the source table" + ); + assert!(available.completed_at.is_some(), "completed is stamped"); + let listed: Vec = engine + .list_backup_rows(Some(BackupStatus::Available), &store) + .try_collect() + .await + .unwrap(); + assert_eq!(listed.len(), 1, "the status index followed: {listed:?}"); + assert!( + engine + .claim_dead_backup(&key, Duration::ZERO) + .await + .expect("claim") + .is_none(), + "an AVAILABLE row is not taken over, whatever its heartbeat's age" + ); + + // A FAILED write without the member leaves the stored source table. + let failed = index_row(&account_id, "Music", "dead-owner-failed", &store); + let claim = engine.claim_backup(&failed).await.expect("claim"); + assert_eq!( + claim + .set_status( + JobStatus::Creating, + JobStatus::Failed, + &StatusFields { + failure_reason: Some("owner dead".to_owned()), + completed: true, + ..StatusFields::default() + } + ) + .await + .expect("write"), + WriteOutcome::Matched + ); + let stored = stored_index_row(&engine, &failed).await; + assert_eq!( + stored.source_table_details_json, + failed.source_table_details_json + ); + assert_eq!(stored.failure_reason.as_deref(), Some("owner dead")); +} + +/// The `active_restores` of a `backups_by_arn` row, as the map's entries. +async fn registrations( + engine: &CassandraEngine, + account_id: &str, + backup_arn: &str, +) -> std::collections::HashMap { + use cdrs_tokio::types::AsRust; + use cdrs_tokio::types::IntoRustByName; + let rows = extenddb_storage_cassandra::cassandra_util::query_rows_quorum::( + &engine.session_arc(), + &format!( + "SELECT active_restores FROM {}.backups_by_arn \ + WHERE account_id = ? AND backup_arn = ?", + engine.catalog_keyspace() + ), + cdrs_tokio::query_values!(account_id, backup_arn), + "registrations", + ) + .await + .expect("read the row"); + let map: Option = rows + .first() + .expect("the row exists") + .get_by_name("active_restores") + .ok() + .flatten(); + map.map(|map| map.as_rust().expect("map").unwrap_or_default()) + .unwrap_or_default() +} + +/// `sweep_restore_registrations`: an entry whose target is a CREATING +/// restore target of the backup with the registered table id is kept; one +/// whose target is ACTIVE, one whose target restores from another backup and +/// one whose target has no row are removed, and the swept backup's ARN is +/// returned once. A second sweep finds nothing to do. +#[tokio::test] +async fn test_the_registrations_sweep_removes_stale_entries_and_keeps_a_live_one() { + if crate::helpers::skip_without_cassandra() { + return; + } + let (engine, account_id, _drop_keyspaces) = isolated_engine("sw", false).await; + let store = unique_store(); + let catalog = engine.catalog_keyspace(); + let row = index_row(&account_id, "Music", "registered", &store); + let claim = engine.claim_backup(&row).await.expect("claim"); + assert_eq!( + claim + .set_status( + JobStatus::Creating, + JobStatus::Available, + &StatusFields::default() + ) + .await + .expect("write"), + WriteOutcome::Matched + ); + let other = index_row(&account_id, "Music", "other", &store); + let other_claim = engine.claim_backup(&other).await.expect("claim"); + assert_eq!( + other_claim + .set_status( + JobStatus::Creating, + JobStatus::Available, + &StatusFields::default() + ) + .await + .expect("write"), + WriteOutcome::Matched + ); + let mut targets = Vec::new(); + for name in ["SwLive", "SwActive", "SwForeign"] { + targets.push( + create_active_table( + &engine, + &account_id, + CreateTableInput { + table_name: name.to_owned(), + key_schema: vec![ks("id", KeyType::Hash)], + attribute_definitions: vec![attr("id", ScalarAttributeType::S)], + billing_mode: Some(BillingMode::PayPerRequest), + ..Default::default() + }, + ) + .await, + ); + } + let (live, active, foreign) = (&targets[0], &targets[1], &targets[2]); + // A CREATING target restoring from the backup; a CREATING target + // restoring from the other backup; an ACTIVE table; a name with no row. + for (target, source) in [(live, &row), (foreign, &other)] { + apply( + &engine, + &format!( + "UPDATE {catalog}.tables SET table_status = 'CREATING', \ + status_transition_at = null, restore_source_backup_arn = ? \ + WHERE account_id = ? AND table_name = ? IF table_id = ?" + ), + cdrs_tokio::query_values!( + source.backup_arn.as_str(), + account_id.as_str(), + target.table_name.as_str(), + target.table_id.as_str() + ), + ) + .await; + } + let missing_id = uuid::Uuid::new_v4().to_string(); + for (table_id, name) in [ + (live.table_id.as_str(), "SwLive"), + (active.table_id.as_str(), "SwActive"), + (foreign.table_id.as_str(), "SwForeign"), + (missing_id.as_str(), "SwMissing"), + ] { + apply( + &engine, + &format!( + "UPDATE {catalog}.backups_by_arn SET active_restores[?] = ? \ + WHERE account_id = ? AND backup_arn = ? IF EXISTS" + ), + cdrs_tokio::query_values!(table_id, name, account_id.as_str(), row.backup_arn.as_str()), + ) + .await; + } + // The other backup registers its own live restore: nothing to sweep. + apply( + &engine, + &format!( + "UPDATE {catalog}.backups_by_arn SET active_restores[?] = ? \ + WHERE account_id = ? AND backup_arn = ? IF EXISTS" + ), + cdrs_tokio::query_values!( + foreign.table_id.as_str(), + "SwForeign", + account_id.as_str(), + other.backup_arn.as_str() + ), + ) + .await; + assert_eq!( + registrations(&engine, &account_id, &row.backup_arn) + .await + .len(), + 4 + ); + + let swept = engine.sweep_restore_registrations().await.expect("sweep"); + assert_eq!(swept, std::slice::from_ref(&row.backup_arn)); + let left = registrations(&engine, &account_id, &row.backup_arn).await; + assert_eq!( + left, + std::collections::HashMap::from([(live.table_id.clone(), "SwLive".to_owned())]), + "the live entry is kept and the three stale ones are removed" + ); + assert_eq!( + registrations(&engine, &account_id, &other.backup_arn).await, + std::collections::HashMap::from([(foreign.table_id.clone(), "SwForeign".to_owned())]), + "the other backup's live entry is untouched" + ); + assert!( + engine + .sweep_restore_registrations() + .await + .expect("sweep") + .is_empty(), + "a second sweep finds nothing" + ); + assert_eq!( + engine.backup_in_use(&row.key()).await.expect("in use"), + Some("SwLive".to_owned()), + "the kept entry is the in-use rule's" + ); +} + +// ── Store-backed (manifest) restore ────────────────────────────────────────── + +/// An AVAILABLE catalog-index backup row (`store_kind` set) for `table`, the +/// row a store-backed restore registers against. +async fn available_index_backup( + engine: &CassandraEngine, + account_id: &str, + table: &str, + name: &str, + store: &StoreIdentity, +) -> BackupRow { + let row = index_row(account_id, table, name, store); + let claim = engine.claim_backup(&row).await.expect("claim backup"); + assert_eq!( + claim + .set_status( + JobStatus::Creating, + JobStatus::Available, + &StatusFields { + completed: true, + ..StatusFields::default() + } + ) + .await + .expect("make the backup AVAILABLE"), + WriteOutcome::Matched + ); + row +} + +/// A hash-only `CreateTableInput` for a restore target. +fn manifest_target_input(name: &str) -> CreateTableInput { + CreateTableInput { + table_name: name.to_owned(), + key_schema: vec![ks("id", KeyType::Hash)], + attribute_definitions: vec![attr("id", ScalarAttributeType::S)], + billing_mode: Some(BillingMode::PayPerRequest), + ..Default::default() + } +} + +/// The registration a store-backed restore of `row` writes, with `spool_host`. +fn manifest_registration(row: &BackupRow, spool_host: Option<&str>) -> RestoreRegistrationRow { + RestoreRegistrationRow { + backup: row.key(), + source_table_arn: format!( + "arn:aws:dynamodb:us-east-1:{}:table/{}", + row.account_id, row.table_name + ), + restore_date_time: row.created_at, + store: row.store.clone(), + spool_host: spool_host.map(str::to_owned), + owner_generation: UNOWNED_GENERATION, + } +} + +/// A hash-keyed item with key `k{i:05}`. +fn id_item(i: usize) -> Item { + let mut it = BTreeMap::new(); + it.insert("id".to_owned(), AttributeValue::S(format!("k{i:05}"))); + it.insert("v".to_owned(), AttributeValue::S(format!("val{i}"))); + it +} + +/// The items `lo..hi` as a `bulk_load` stream. +fn item_stream( + lo: usize, + hi: usize, +) -> futures::stream::BoxStream<'static, Result> { + let items: Vec = (lo..hi).map(id_item).collect(); + futures::stream::iter(items.into_iter().map(Ok)).boxed() +} + +/// `claim_restore` registers a CREATING target with a RUNNING `manifest` +/// record carrying the store, the spool host and the claim's generation, the +/// `active_restores` entry on the backup, and a `RestoreSummary` on the +/// description; the claim heartbeats and reads the record. +#[tokio::test] +async fn test_claim_restore_registers_a_manifest_target_with_its_record() { + if crate::helpers::skip_without_cassandra() { + return; + } + let engine = setup_engine().await; + let account = TestAccount::new(&engine, "extenddb_ttl_test").await; + let account_id = account.account_id.clone(); + let store = unique_store(); + let row = available_index_backup(&engine, &account_id, "Music", "nightly", &store).await; + let registration = manifest_registration(&row, Some("host-a:/var/spool")); + + let (desc, claim) = engine + .claim_restore( + &account_id, + manifest_target_input("RManifest"), + ®istration, + ) + .await + .expect("the registration creates the target and the record"); + assert_eq!(desc.table_status, TableStatus::Creating); + let summary = desc.restore_summary.as_ref().expect("a RestoreSummary"); + assert_eq!( + summary.source_backup_arn.as_deref(), + Some(row.backup_arn.as_str()) + ); + assert!(summary.restore_in_progress); + assert_ne!(claim.generation(), UNOWNED_GENERATION); + + let key = RestoreTargetKey { + account_id: account_id.clone(), + table_name: desc.table_name.clone(), + table_id: desc.table_id.clone(), + }; + let record = engine + .read_restore_job(&key) + .await + .expect("read") + .expect("the record exists"); + assert_eq!(record.kind, RestoreKind::Manifest); + assert_eq!(record.status, JobStatus::Running); + assert_eq!(record.owner_generation, claim.generation()); + assert_eq!(record.store.as_ref(), Some(&store)); + assert_eq!(record.spool_host.as_deref(), Some("host-a:/var/spool")); + assert_eq!(record.source_backup_arn, row.backup_arn); + + // The backup is in use by the target, and the heartbeat confirms and the + // read returns RUNNING. + assert_eq!( + engine.backup_in_use(&row.key()).await.unwrap().as_deref(), + Some("RManifest") + ); + assert_eq!( + claim.heartbeat().await.expect("heartbeat"), + Heartbeat::Confirmed + ); + assert_eq!(claim.read().await.expect("read").status, JobStatus::Running); +} + +/// The two orderings against `start_backup_delete` on the backup's +/// `backup_index` row: a registration that lands first leaves the backup in +/// use; a DeleteBackup that lands first leaves the registration with no +/// backup to read. +#[tokio::test] +async fn test_claim_restore_and_start_backup_delete_order_on_the_backup_index_row() { + if crate::helpers::skip_without_cassandra() { + return; + } + let engine = setup_engine().await; + let account = TestAccount::new(&engine, "extenddb_ttl_test").await; + let account_id = account.account_id.clone(); + + // Registration wins: DeleteBackup finds the backup in use by the target. + let store = unique_store(); + let won = available_index_backup(&engine, &account_id, "Music", "a", &store).await; + let (desc, _claim) = engine + .claim_restore( + &account_id, + manifest_target_input("WonTarget"), + &manifest_registration(&won, None), + ) + .await + .expect("register"); + assert_eq!(desc.table_status, TableStatus::Creating); + assert_eq!( + engine.start_backup_delete(&won.key()).await.unwrap(), + DeleteStart::InUse("WonTarget".to_owned()) + ); + + // DeleteBackup wins: the registration finds the backup gone. + let lost = available_index_backup(&engine, &account_id, "Music", "b", &store).await; + assert_eq!( + engine.start_backup_delete(&lost.key()).await.unwrap(), + DeleteStart::Deleting + ); + let refused = engine + .claim_restore( + &account_id, + manifest_target_input("LostTarget"), + &manifest_registration(&lost, None), + ) + .await + .err() + .expect("a registration against a DELETING backup is Backup not found"); + assert!( + matches!(refused, StorageError::Validation(ref m) if m.contains("Backup not found")), + "{refused}" + ); + // The lost registration left no target behind. + assert!( + engine + .describe_table( + &account_id, + DescribeTableInput { + table_name: "LostTarget".to_owned() + } + ) + .await + .is_err() + ); +} + +/// A second restore of a backup whose target is CREATING is `BackupInUse`, +/// and a restore into a taken name is `TableAlreadyExists`. +#[tokio::test] +async fn test_a_second_restore_is_in_use_and_a_taken_name_is_refused() { + if crate::helpers::skip_without_cassandra() { + return; + } + let engine = setup_engine().await; + let account = TestAccount::new(&engine, "extenddb_ttl_test").await; + let account_id = account.account_id.clone(); + let store = unique_store(); + let row = available_index_backup(&engine, &account_id, "Music", "nightly", &store).await; + + let (_desc, _claim) = engine + .claim_restore( + &account_id, + manifest_target_input("FirstTarget"), + &manifest_registration(&row, None), + ) + .await + .expect("the first restore registers"); + + // A second restore of the same backup, into a different name, is in use. + let in_use = engine + .claim_restore( + &account_id, + manifest_target_input("SecondTarget"), + &manifest_registration(&row, None), + ) + .await + .err() + .expect("the backup is in use by the first restore"); + // The service's text names the backup, not the target. + assert!( + matches!(in_use, StorageError::BackupInUse(ref m) if *m == backup_in_use_message(&row.backup_arn)), + "{in_use}" + ); + + // A restore into an existing table's name is TableAlreadyExists. + create_active_table( + &engine, + &account_id, + CreateTableInput { + table_name: "Taken".to_owned(), + key_schema: vec![ks("id", KeyType::Hash)], + attribute_definitions: vec![attr("id", ScalarAttributeType::S)], + billing_mode: Some(BillingMode::PayPerRequest), + ..Default::default() + }, + ) + .await; + let other = available_index_backup(&engine, &account_id, "Music", "other", &store).await; + let taken = engine + .claim_restore( + &account_id, + manifest_target_input("Taken"), + &manifest_registration(&other, None), + ) + .await + .err() + .expect("the name is taken"); + assert!( + matches!(taken, StorageError::TableAlreadyExists(ref name) if name == "Taken"), + "{taken}" + ); +} + +/// `claim_restore_dead`: a live owner, and a legacy record, are not taken +/// over; a stale manifest record is, under a fresh generation, with the +/// claimant's `restore_spool_host` written and `restore_heartbeat_at` +/// refreshed to the coordinator's clock in the same lightweight transaction, +/// so a second taker right after the first (before the successor's first +/// beat) finds the record live and is refused while the first taker's beat +/// confirms; the old claim's writes are then fenced. +#[tokio::test] +async fn test_claim_restore_dead_takes_over_a_stale_manifest_record() { + if crate::helpers::skip_without_cassandra() { + return; + } + let engine = setup_engine().await; + let account = TestAccount::new(&engine, "extenddb_ttl_test").await; + let account_id = account.account_id.clone(); + let store = unique_store(); + let row = available_index_backup(&engine, &account_id, "Music", "nightly", &store).await; + let (desc, owner) = engine + .claim_restore( + &account_id, + manifest_target_input("Takeover"), + &manifest_registration(&row, Some("owner:/s")), + ) + .await + .expect("register"); + let key = RestoreTargetKey { + account_id: account_id.clone(), + table_name: desc.table_name.clone(), + table_id: desc.table_id.clone(), + }; + let window = Duration::from_secs(1); + let before = engine + .read_restore_job(&key) + .await + .unwrap() + .unwrap() + .heartbeat_at; + assert!( + engine + .claim_restore_dead(&key, Duration::from_secs(300), Some("other:/s")) + .await + .expect("claim") + .is_none(), + "a live owner within a long window is not taken over" + ); + + tokio::time::sleep(Duration::from_millis(1_300)).await; + let successor = engine + .claim_restore_dead(&key, window, Some("successor:/spool")) + .await + .expect("claim") + .expect("the window passed: taken over"); + assert_ne!(successor.generation(), owner.generation()); + let after = engine.read_restore_job(&key).await.unwrap().unwrap(); + assert_eq!(after.owner_generation, successor.generation()); + assert_eq!(after.status, JobStatus::Running); + assert_eq!( + after.spool_host.as_deref(), + Some("successor:/spool"), + "the takeover wrote the claimant's spool host" + ); + // A second taker right after the first, before the successor's first + // beat: its serial read finds the fresh heartbeat and it is refused; the + // first taker's beat confirms. + let second = engine + .claim_restore_dead(&key, window, Some("second:/s")) + .await + .expect("a second takeover attempt"); + assert!( + second.is_none(), + "a second taker right after the first is refused: {:?}", + second.as_ref().map(|c| c.generation()) + ); + assert_eq!( + successor.heartbeat().await.expect("heartbeat"), + Heartbeat::Confirmed, + "the first taker keeps the record" + ); + assert!( + after.heartbeat_at > before + 1.0, + "the takeover refreshed the heartbeat: {} was {before}", + after.heartbeat_at + ); + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("epoch") + .as_secs_f64(); + assert!( + (now - after.heartbeat_at).abs() < 5.0, + "the refreshed heartbeat is the coordinator's now: {} vs {now}", + after.heartbeat_at + ); + assert_eq!(owner.heartbeat().await.expect("heartbeat"), Heartbeat::Lost); + assert_eq!( + owner + .set_status( + JobStatus::Running, + JobStatus::Failed, + &StatusFields::default() + ) + .await + .expect("write"), + WriteOutcome::NotMatched, + "the old generation's write is fenced" + ); + + // The successor heartbeats again: a third taker finds it live. + assert_eq!( + successor.heartbeat().await.expect("heartbeat"), + Heartbeat::Confirmed + ); + assert!( + engine + .claim_restore_dead(&key, window, Some("third:/s")) + .await + .expect("claim") + .is_none(), + "the successor's heartbeat is fresh" + ); + + // A legacy record is not a manifest restore's to take, whatever its age. + apply( + &engine, + &format!( + "UPDATE {}.tables SET restore_kind = 'legacy' \ + WHERE account_id = ? AND table_name = ? IF table_id = ?", + engine.catalog_keyspace() + ), + cdrs_tokio::query_values!( + account_id.as_str(), + key.table_name.as_str(), + key.table_id.as_str() + ), + ) + .await; + tokio::time::sleep(Duration::from_millis(1_300)).await; + assert!( + engine + .claim_restore_dead(&key, window, Some("x:/s")) + .await + .expect("claim") + .is_none(), + "a legacy record is not taken over by the manifest takeover" + ); +} + +/// `claim_restore_dead` refuses the takeover when a heartbeat lands between +/// its serial read and its lightweight transaction: the LWT conditions on the +/// heartbeat it read, which the beat has moved. +#[cfg(feature = "test-hooks")] +#[tokio::test] +async fn test_claim_restore_dead_refuses_a_beat_between_the_read_and_the_write() { + if crate::helpers::skip_without_cassandra() { + return; + } + let engine = setup_engine().await; + let account = TestAccount::new(&engine, "extenddb_ttl_test").await; + let account_id = account.account_id.clone(); + let store = unique_store(); + let row = available_index_backup(&engine, &account_id, "Music", "nightly", &store).await; + let (desc, owner) = engine + .claim_restore( + &account_id, + manifest_target_input("BeatRace"), + &manifest_registration(&row, Some("owner:/s")), + ) + .await + .expect("register"); + let key = RestoreTargetKey { + account_id: account_id.clone(), + table_name: desc.table_name.clone(), + table_id: desc.table_id.clone(), + }; + tokio::time::sleep(Duration::from_millis(1_300)).await; + + let pause = + extenddb_storage_cassandra::test_hooks::pause_before_restore_takeover(&key.table_id); + let takeover = { + let engine = engine.clone(); + let key = key.clone(); + tokio::spawn(async move { + engine + .claim_restore_dead(&key, Duration::from_secs(1), Some("successor:/s")) + .await + }) + }; + pause.reached().await; + // A beat lands after the takeover read its stale heartbeat. + apply( + &engine, + &format!( + "UPDATE {}.tables SET restore_heartbeat_at = toTimestamp(now()) \ + WHERE account_id = ? AND table_name = ? IF table_id = ?", + engine.catalog_keyspace() + ), + cdrs_tokio::query_values!( + account_id.as_str(), + key.table_name.as_str(), + key.table_id.as_str() + ), + ) + .await; + pause.release(); + assert!( + takeover + .await + .expect("takeover task") + .expect("claim") + .is_none(), + "the beat between the read and the write refuses the takeover" + ); + let record = engine.read_restore_job(&key).await.unwrap().unwrap(); + assert_eq!( + record.owner_generation, + owner.generation(), + "the record is still the original owner's" + ); + assert_eq!( + record.spool_host.as_deref(), + Some("owner:/s"), + "the refused takeover did not rewrite the spool host" + ); +} + +/// A record whose backup does not list its target (the owner died between +/// the target's creation and the `active_restores` registration): the +/// takeover re-applies the entry, so the backup is in use for the resumed +/// load and `start_backup_delete` is refused. +#[tokio::test] +async fn test_claim_restore_dead_reapplies_a_missing_active_restores_entry() { + if crate::helpers::skip_without_cassandra() { + return; + } + let engine = setup_engine().await; + let account = TestAccount::new(&engine, "extenddb_ttl_test").await; + let account_id = account.account_id.clone(); + let store = unique_store(); + let row = available_index_backup(&engine, &account_id, "Music", "nightly", &store).await; + let (desc, owner) = engine + .claim_restore( + &account_id, + manifest_target_input("Unlisted"), + &manifest_registration(&row, Some("owner:/s")), + ) + .await + .expect("register"); + let key = RestoreTargetKey { + account_id: account_id.clone(), + table_name: desc.table_name.clone(), + table_id: desc.table_id.clone(), + }; + // The crash's shape: the target and its RUNNING record exist, the + // backup's entry does not. + apply( + &engine, + &format!( + "DELETE active_restores[?] FROM {}.backups_by_arn \ + WHERE account_id = ? AND backup_arn = ? IF EXISTS", + engine.catalog_keyspace() + ), + cdrs_tokio::query_values!( + key.table_id.as_str(), + account_id.as_str(), + row.backup_arn.as_str() + ), + ) + .await; + assert!( + registrations(&engine, &account_id, &row.backup_arn) + .await + .is_empty(), + "the entry is gone" + ); + drop(owner); + tokio::time::sleep(Duration::from_millis(1_300)).await; + let successor = engine + .claim_restore_dead(&key, Duration::from_secs(1), Some("successor:/s")) + .await + .expect("claim") + .expect("the stale record is taken over"); + let entries = registrations(&engine, &account_id, &row.backup_arn).await; + assert_eq!( + entries.get(&key.table_id).map(String::as_str), + Some("Unlisted"), + "the takeover re-applied the backup's entry for the target: {entries:?}" + ); + assert_eq!( + engine.start_backup_delete(&row.key()).await.unwrap(), + DeleteStart::InUse("Unlisted".to_owned()), + "the backup is in use for the resumed load" + ); + assert_eq!( + engine.backup_in_use(&row.key()).await.unwrap().as_deref(), + Some("Unlisted") + ); + drop(successor); +} + +/// A DeleteBackup that lands between a registration's unfenced pre-read and +/// its `active_restores` lightweight transaction: the transaction's own +/// `backup_status = 'AVAILABLE'` condition refuses the registration, which +/// answers `Backup not found` and leaves no target and no entry. +#[cfg(feature = "test-hooks")] +#[tokio::test] +async fn test_a_delete_between_the_registrations_pre_read_and_its_fence_leaves_no_target() { + if crate::helpers::skip_without_cassandra() { + return; + } + let engine = setup_engine().await; + let account = TestAccount::new(&engine, "extenddb_ttl_test").await; + let account_id = account.account_id.clone(); + let store = unique_store(); + let row = available_index_backup(&engine, &account_id, "Music", "raced", &store).await; + let pause = + extenddb_storage_cassandra::test_hooks::pause_before_restore_registration(&row.backup_arn); + let registration = { + let engine = engine.clone(); + let account_id = account_id.clone(); + let registration = manifest_registration(&row, None); + tokio::spawn(async move { + engine + .claim_restore( + &account_id, + manifest_target_input("RacedTarget"), + ®istration, + ) + .await + }) + }; + // The registration has read the backup AVAILABLE and created the target; + // the delete lands before its fence. + pause.reached().await; + assert_eq!( + engine.start_backup_delete(&row.key()).await.unwrap(), + DeleteStart::Deleting, + "nothing holds the backup before the registration's fence" + ); + pause.release(); + let refused = registration + .await + .expect("registration task") + .err() + .expect("the fence refuses the registration"); + assert!( + matches!(refused, StorageError::Validation(ref m) if m.contains("Backup not found")), + "{refused}" + ); + assert!( + matches!( + engine + .describe_table( + &account_id, + DescribeTableInput { + table_name: "RacedTarget".to_owned() + } + ) + .await, + Err(StorageError::TableNotFound(_)) + ), + "the refused registration left no target" + ); + assert!( + registrations(&engine, &account_id, &row.backup_arn) + .await + .is_empty(), + "no entry on the DELETING backup" + ); +} + +/// Two concurrent registrations of one backup leave one target: the in-use +/// pre-read is not atomic with the `active_restores` lightweight +/// transaction, so both can pass it (the first held before its fence while +/// the second runs whole), but the fence's `active_restores = null` admits +/// one; the other finds the registered target in its re-read, is +/// `BackupInUse` naming it, and removes its own target. +#[cfg(feature = "test-hooks")] +#[tokio::test] +async fn test_two_concurrent_registrations_of_one_backup_leave_one_target() { + if crate::helpers::skip_without_cassandra() { + return; + } + let engine = setup_engine().await; + let account = TestAccount::new(&engine, "extenddb_ttl_test").await; + let account_id = account.account_id.clone(); + let store = unique_store(); + let row = available_index_backup(&engine, &account_id, "Music", "twice", &store).await; + let pause = + extenddb_storage_cassandra::test_hooks::pause_before_restore_registration(&row.backup_arn); + let first = { + let engine = engine.clone(); + let account_id = account_id.clone(); + let registration = manifest_registration(&row, None); + tokio::spawn(async move { + engine + .claim_restore( + &account_id, + manifest_target_input("FirstOfTwo"), + ®istration, + ) + .await + }) + }; + // The first registration has passed the pre-read (no restore registered) + // and created its target; the second runs whole before the first's fence. + pause.reached().await; + let (second_desc, second_claim) = engine + .claim_restore( + &account_id, + manifest_target_input("SecondOfTwo"), + &manifest_registration(&row, None), + ) + .await + .expect("the second registration, whole before the first's fence"); + pause.release(); + let refused = first + .await + .expect("registration task") + .err() + .expect("the first's fence finds the second registered"); + assert!( + matches!(refused, StorageError::BackupInUse(ref m) if *m == backup_in_use_message(&row.backup_arn)), + "{refused}" + ); + assert!( + matches!( + engine + .describe_table( + &account_id, + DescribeTableInput { + table_name: "FirstOfTwo".to_owned() + } + ) + .await, + Err(StorageError::TableNotFound(_)) + ), + "the refused registration left no target" + ); + assert_eq!( + registrations(&engine, &account_id, &row.backup_arn).await, + std::collections::HashMap::from([(second_desc.table_id.clone(), "SecondOfTwo".to_owned())]), + "one entry on the backup, the second's" + ); + assert_eq!( + engine + .backup_in_use(&row.key()) + .await + .expect("in use") + .as_deref(), + Some("SecondOfTwo") + ); + assert_eq!( + second_claim.heartbeat().await.expect("beat"), + Heartbeat::Confirmed, + "the second registration's claim is live" + ); +} + +/// A registration whose record a reconciler took over during its target's +/// DDL (no heartbeat runs there, so a DDL that outlasts the stale window +/// leaves the record dead) removes nothing: its `active_restores` transaction +/// finds the successor's entry, and its removal of the target is conditioned +/// on its own generation, which the takeover moved. The target stays, the +/// record is the successor's, and the registration answers an error naming +/// the lost claim rather than the refusal. +#[cfg(feature = "test-hooks")] +#[tokio::test] +async fn test_a_registration_taken_over_during_its_ddl_leaves_the_successors_target() { + if crate::helpers::skip_without_cassandra() { + return; + } + let engine = setup_engine().await; + let account = TestAccount::new(&engine, "extenddb_ttl_test").await; + let account_id = account.account_id.clone(); + let store = unique_store(); + let row = available_index_backup(&engine, &account_id, "Music", "slowddl", &store).await; + let pause = + extenddb_storage_cassandra::test_hooks::pause_before_restore_registration(&row.backup_arn); + let registration = { + let engine = engine.clone(); + let account_id = account_id.clone(); + let registration = manifest_registration(&row, Some("registrant:/s")); + tokio::spawn(async move { + engine + .claim_restore( + &account_id, + composite_target_input("SlowDdl"), + ®istration, + ) + .await + }) + }; + // The target's row and DDL exist; the registration is held before its + // `active_restores` transaction, as a DDL past the stale window would + // hold it. The record's heartbeat is the row insert's. + pause.reached().await; + let desc = describe(&engine, &account_id, "SlowDdl").await; + assert_eq!(desc.table_status, TableStatus::Creating); + let key = RestoreTargetKey { + account_id: account_id.clone(), + table_name: desc.table_name.clone(), + table_id: desc.table_id.clone(), + }; + let registrant_generation = engine + .read_restore_job(&key) + .await + .unwrap() + .unwrap() + .owner_generation; + tokio::time::sleep(Duration::from_millis(1_300)).await; + let successor = engine + .claim_restore_dead(&key, Duration::from_secs(1), Some("successor:/s")) + .await + .expect("claim") + .expect("the registration's record is stale and taken over"); + assert_ne!(successor.generation(), registrant_generation); + assert_eq!( + registrations(&engine, &account_id, &row.backup_arn).await, + std::collections::HashMap::from([(key.table_id.clone(), "SlowDdl".to_owned())]), + "the takeover re-applied the target's entry" + ); + + pause.release(); + let outcome = registration.await.expect("registration task"); + + // The successor's target, record and registration stand. + let target = engine + .describe_table( + &account_id, + DescribeTableInput { + table_name: "SlowDdl".to_owned(), + }, + ) + .await; + assert!( + matches!(&target, Ok(desc) if desc.table_status == TableStatus::Creating), + "the registrant's refusal path left the successor's target in place: {target:?}" + ); + let lost = outcome + .err() + .expect("the registration does not return a claim on a record another owns"); + assert!( + matches!(&lost, StorageError::Internal(m) if m.contains("lost its record to a takeover")), + "the registration names the lost claim, not the refusal: {lost}" + ); + let record = engine + .read_restore_job(&key) + .await + .expect("read") + .expect("the record stands"); + assert_eq!(record.status, JobStatus::Running); + assert_eq!(record.owner_generation, successor.generation()); + assert_eq!(record.spool_host.as_deref(), Some("successor:/s")); + assert_eq!( + registrations(&engine, &account_id, &row.backup_arn).await, + std::collections::HashMap::from([(key.table_id.clone(), "SlowDdl".to_owned())]), + "the successor's entry on the backup stands" + ); + assert_eq!( + successor.heartbeat().await.expect("beat"), + Heartbeat::Confirmed + ); + // The successor loads into the target the registrant left: its data and + // index tables are there. + assert_eq!( + engine + .bulk_load( + &key, + &*successor, + BulkLoadFile { + file_index: 0, + start_at: 0, + reapplying: false, + }, + futures::stream::iter((0..6).map(composite_item).map(Ok)).boxed(), + ) + .await + .expect("the successor's load"), + 6 + ); + assert_eq!(engine.count_restore_target(&key).await.expect("count"), 6); + assert_eq!( + index_rows(&engine, &account_id, &key.table_id, "gix").await, + 3 + ); +} + +/// A composite-key restore target with a GSI and an LSI. +fn composite_target_input(name: &str) -> CreateTableInput { + CreateTableInput { + table_name: name.to_owned(), + key_schema: vec![ks("id", KeyType::Hash), ks("sk", KeyType::Range)], + attribute_definitions: vec![ + attr("id", ScalarAttributeType::S), + attr("sk", ScalarAttributeType::N), + attr("g", ScalarAttributeType::S), + attr("l", ScalarAttributeType::S), + ], + billing_mode: Some(BillingMode::PayPerRequest), + global_secondary_indexes: Some(vec![GsiInput { + index_name: "gix".to_owned(), + key_schema: vec![ks("g", KeyType::Hash)], + projection: Projection { + projection_type: ProjectionType::KeysOnly, + non_key_attributes: None, + }, + provisioned_throughput: None, + }]), + local_secondary_indexes: Some(vec![LsiInput { + index_name: "lix".to_owned(), + key_schema: vec![ks("id", KeyType::Hash), ks("l", KeyType::Range)], + projection: Projection { + projection_type: ProjectionType::All, + non_key_attributes: None, + }, + }]), + ..Default::default() + } +} + +/// Item `i` of the composite fixture: `id` one of five, `sk` the index, +/// every second item carrying the GSI's key `g`, every third the LSI's `l`. +fn composite_item(i: usize) -> Item { + let mut it = BTreeMap::new(); + it.insert("id".to_owned(), AttributeValue::S(format!("p{}", i % 5))); + it.insert("sk".to_owned(), AttributeValue::N(i.to_string())); + if i.is_multiple_of(2) { + it.insert("g".to_owned(), AttributeValue::S(format!("g{}", i % 4))); + } + if i.is_multiple_of(3) { + it.insert("l".to_owned(), AttributeValue::S(format!("l{i}"))); + } + it +} + +/// The rows of the target's secondary index `index_name`. +async fn index_rows( + engine: &CassandraEngine, + account_id: &str, + table_id: &str, + index_name: &str, +) -> i64 { + use cdrs_tokio::types::IntoRustByName; + let rows = extenddb_storage_cassandra::cassandra_util::query_rows_quorum::( + &engine.session_arc(), + &format!( + "SELECT index_id FROM {}.indexes WHERE table_id = ? AND index_name = ?", + engine.catalog_keyspace() + ), + cdrs_tokio::query_values!(table_id, index_name), + "index id", + ) + .await + .expect("read the index"); + let index_id: String = rows + .first() + .expect("the index exists") + .get_by_name("index_id") + .ok() + .flatten() + .expect("index_id"); + let rows = extenddb_storage_cassandra::cassandra_util::query_rows_quorum::( + &engine.session_arc(), + &format!( + "SELECT COUNT(*) AS c FROM {}.{}", + engine.account_keyspace(account_id), + extenddb_storage_cassandra::data::ddl::index_table_name(&index_id) + ), + cdrs_tokio::query_values!(), + "index rows", + ) + .await + .expect("count the index rows"); + rows.first() + .expect("a count") + .get_by_name("c") + .ok() + .flatten() + .expect("c") +} + +/// Per item of the composite fixture (keyed by its `sk`), the base row's +/// `version` when `index_name` is `None`, otherwise the write timestamp of +/// the item's row in that secondary index (`WRITETIME(item_data)`). +#[cfg(feature = "test-hooks")] +async fn stamps_by_sk( + engine: &CassandraEngine, + account_id: &str, + table_id: &str, + index_name: Option<&str>, +) -> BTreeMap { + use cdrs_tokio::types::IntoRustByName; + let keyspace = engine.account_keyspace(account_id); + let cql = match index_name { + None => format!( + "SELECT item_data, version AS stamp FROM {keyspace}.items_{}", + table_id.replace('-', "_") + ), + Some(index_name) => { + let rows = + extenddb_storage_cassandra::cassandra_util::query_rows_quorum::( + &engine.session_arc(), + &format!( + "SELECT index_id FROM {}.indexes WHERE table_id = ? AND index_name = ?", + engine.catalog_keyspace() + ), + cdrs_tokio::query_values!(table_id, index_name), + "index id", + ) + .await + .expect("read the index"); + let index_id: String = rows + .first() + .expect("the index exists") + .get_by_name("index_id") + .ok() + .flatten() + .expect("index_id"); + format!( + "SELECT item_data, WRITETIME(item_data) AS stamp FROM {keyspace}.{}", + extenddb_storage_cassandra::data::ddl::index_table_name(&index_id) + ) + } + }; + let rows = extenddb_storage_cassandra::cassandra_util::query_rows_quorum::( + &engine.session_arc(), + &cql, + cdrs_tokio::query_values!(), + "stamps", + ) + .await + .expect("read the stamps"); + rows.into_iter() + .map(|row| { + let json: String = row.get_r_by_name("item_data").expect("item_data"); + let image: Item = serde_json::from_str(&json).expect("an item image"); + let Some(AttributeValue::N(sk)) = image.get("sk") else { + panic!("no sk in {image:?}"); + }; + let stamp: i64 = row.get_r_by_name("stamp").expect("a stamp"); + (sk.clone(), stamp) + }) + .collect() +} + +/// Every GSI and LSI row of the composite target carries its base row's +/// `version` as its write timestamp, the stamp the write path gives the +/// index rows of each version it stores (ADR-0027); returns the base +/// versions. +#[cfg(feature = "test-hooks")] +async fn assert_index_rows_carry_the_base_versions( + engine: &CassandraEngine, + account_id: &str, + table_id: &str, +) -> BTreeMap { + let versions = stamps_by_sk(engine, account_id, table_id, None).await; + for index_name in ["gix", "lix"] { + let stamps = stamps_by_sk(engine, account_id, table_id, Some(index_name)).await; + assert!(!stamps.is_empty(), "{index_name} has rows"); + for (sk, stamp) in &stamps { + assert_eq!( + Some(stamp), + versions.get(sk), + "the {index_name} row of item {sk} carries its base row's version" + ); + } + } + versions +} + +/// `bulk_load` into a composite-key target with a GSI and an LSI: a file +/// resumed after its first three items (`start_at: 3`) applies exactly the +/// rest and records `start_at` plus the items applied as the file's total; +/// the GSI and LSI rows are written with the base rows; an empty file, and a +/// file resumed at its end, record their progress. +#[tokio::test] +async fn test_bulk_load_resumes_mid_file_with_its_index_rows_and_records_an_empty_file() { + if crate::helpers::skip_without_cassandra() { + return; + } + let engine = setup_engine().await; + let account = TestAccount::new(&engine, "extenddb_ttl_test").await; + let account_id = account.account_id.clone(); + let store = unique_store(); + let row = available_index_backup(&engine, &account_id, "Music", "nightly", &store).await; + let (desc, claim) = engine + .claim_restore( + &account_id, + composite_target_input("Composite"), + &manifest_registration(&row, None), + ) + .await + .expect("register a composite target"); + let key = RestoreTargetKey { + account_id: account_id.clone(), + table_name: desc.table_name.clone(), + table_id: desc.table_id.clone(), + }; + // File 0 resumed after three of its ten items. + let rest: Vec = (3..10).map(composite_item).collect(); + let applied = engine + .bulk_load( + &key, + &*claim, + BulkLoadFile { + file_index: 0, + start_at: 3, + reapplying: false, + }, + futures::stream::iter(rest.into_iter().map(Ok)).boxed(), + ) + .await + .expect("the rest of file 0"); + assert_eq!(applied, 7, "exactly the rest"); + let record = engine.read_restore_job(&key).await.unwrap().unwrap(); + assert_eq!(record.file_index, Some(0)); + assert_eq!( + record.items_applied, + Some(10), + "the running total is start_at plus the items applied" + ); + assert_eq!(engine.count_restore_target(&key).await.expect("count"), 7); + assert_eq!( + index_rows(&engine, &account_id, &key.table_id, "gix").await, + 3, + "the GSI rows of items 4, 6 and 8" + ); + assert_eq!( + index_rows(&engine, &account_id, &key.table_id, "lix").await, + 3, + "the LSI rows of items 3, 6 and 9" + ); + + // An empty file records its progress as complete. + let applied = engine + .bulk_load( + &key, + &*claim, + BulkLoadFile { + file_index: 1, + start_at: 0, + reapplying: false, + }, + futures::stream::iter(Vec::>::new()).boxed(), + ) + .await + .expect("an empty file"); + assert_eq!(applied, 0); + let record = engine.read_restore_job(&key).await.unwrap().unwrap(); + assert_eq!( + (record.file_index, record.items_applied), + (Some(1), Some(0)), + "an empty file's progress is recorded" + ); + // A file resumed at its end: `start_at` is the recorded total. + let applied = engine + .bulk_load( + &key, + &*claim, + BulkLoadFile { + file_index: 2, + start_at: 4, + reapplying: false, + }, + futures::stream::iter(Vec::>::new()).boxed(), + ) + .await + .expect("a file resumed at its end"); + assert_eq!(applied, 0); + let record = engine.read_restore_job(&key).await.unwrap().unwrap(); + assert_eq!( + (record.file_index, record.items_applied), + (Some(2), Some(4)) + ); + + // Another account's key, and the target once ACTIVE, name no CREATING + // restore target: `count_restore_target` is `TableNotFound`. + let foreign = RestoreTargetKey { + account_id: "111111111111".to_owned(), + ..key.clone() + }; + assert!(matches!( + engine.count_restore_target(&foreign).await, + Err(StorageError::TableNotFound(_)) + )); + assert_eq!( + engine + .finish_restore(&key, &*claim, 7, 4_096) + .await + .expect("finish"), + Finished::Active + ); + assert!( + matches!( + engine.count_restore_target(&key).await, + Err(StorageError::TableNotFound(_)) + ), + "an ACTIVE table is not a CREATING restore target" + ); +} + +/// A file resumed mid-way (`start_at: 3`) with more than one batch left: +/// the progress written after each batch inside the loop is `start_at` plus +/// the items applied so far, as the write after the tail is, so a second +/// crash between two of the resumed file's batches resumes from a running +/// total that counts the items the first run applied. Three items per batch, +/// seven items: the record reads 6 after the first batch, 9 after the +/// second, 10 at the end. +#[cfg(feature = "test-hooks")] +#[tokio::test] +async fn test_a_mid_file_resume_records_start_at_in_every_batch_progress() { + if crate::helpers::skip_without_cassandra() { + return; + } + let engine = setup_engine().await; + let account = TestAccount::new(&engine, "extenddb_ttl_test").await; + let account_id = account.account_id.clone(); + extenddb_storage_cassandra::test_hooks::set_restore_batch_size(&account_id, 3); + let store = unique_store(); + let row = available_index_backup(&engine, &account_id, "Music", "nightly", &store).await; + let (desc, claim) = engine + .claim_restore( + &account_id, + manifest_target_input("ResumedBatches"), + &manifest_registration(&row, None), + ) + .await + .expect("register"); + let key = RestoreTargetKey { + account_id: account_id.clone(), + table_name: desc.table_name.clone(), + table_id: desc.table_id.clone(), + }; + // The stream reads the record when asked for the first item of the + // second and of the third batch: by then the previous batch's progress + // is written. + let seen = std::sync::Arc::new(std::sync::Mutex::new(Vec::new())); + let seen_by_stream = std::sync::Arc::clone(&seen); + let reader = engine.clone(); + let read_key = key.clone(); + let rest: futures::stream::BoxStream<'static, Result> = + Box::pin(futures::stream::unfold(3usize, move |i| { + let reader = reader.clone(); + let key = read_key.clone(); + let seen = std::sync::Arc::clone(&seen_by_stream); + async move { + if i >= 10 { + return None; + } + if i == 6 || i == 9 { + let record = reader.read_restore_job(&key).await.unwrap().unwrap(); + seen.lock() + .expect("seen") + .push((record.file_index, record.items_applied)); + } + Some((Ok(id_item(i)), i + 1)) + } + })); + let applied = engine + .bulk_load( + &key, + &*claim, + BulkLoadFile { + file_index: 0, + start_at: 3, + reapplying: false, + }, + rest, + ) + .await + .expect("the rest of file 0"); + assert_eq!(applied, 7, "exactly the rest"); + assert_eq!( + *seen.lock().expect("seen"), + vec![(Some(0), Some(6)), (Some(0), Some(9))], + "each batch's progress is start_at plus the items applied so far" + ); + let record = engine.read_restore_job(&key).await.unwrap().unwrap(); + assert_eq!( + (record.file_index, record.items_applied), + (Some(0), Some(10)), + "the tail's progress is the file's total" + ); + assert_eq!(engine.count_restore_target(&key).await.expect("count"), 7); +} + +/// A progress lightweight transaction whose outcome is unknown (a write +/// timeout, a lost connection) does not fail the file: the record is re-read +/// at LOCAL_SERIAL. One whose transaction committed (the answer alone was +/// lost) is found applied and the file goes on; one whose transaction never +/// ran is found carrying the earlier progress and is written once more; a +/// second unknown outcome the record does not carry fails the file with an +/// error naming it, and the batches before it stand with their progress. +#[cfg(feature = "test-hooks")] +#[tokio::test] +async fn test_an_unknown_progress_write_outcome_is_decided_by_a_re_read() { + use extenddb_storage_cassandra::test_hooks::ProgressWriteFault; + if crate::helpers::skip_without_cassandra() { + return; + } + let engine = setup_engine().await; + let account = TestAccount::new(&engine, "extenddb_ttl_test").await; + let account_id = account.account_id.clone(); + extenddb_storage_cassandra::test_hooks::set_restore_batch_size(&account_id, 3); + let store = unique_store(); + let row = available_index_backup(&engine, &account_id, "Music", "nightly", &store).await; + let (desc, claim) = engine + .claim_restore( + &account_id, + manifest_target_input("UnknownProgress"), + &manifest_registration(&row, None), + ) + .await + .expect("register"); + let key = RestoreTargetKey { + account_id: account_id.clone(), + table_name: desc.table_name.clone(), + table_id: desc.table_id.clone(), + }; + + // File 0, three batches: the first batch's write lands but its answer is + // lost (the re-read finds the values written), the second's never runs + // (the re-read finds the first batch's values; written once more), the + // tail's runs as usual. The file completes with every item and its + // progress. + extenddb_storage_cassandra::test_hooks::fail_progress_writes( + &account_id, + &[ProgressWriteFault::Landed, ProgressWriteFault::Lost], + ); + let applied = engine + .bulk_load( + &key, + &*claim, + BulkLoadFile { + file_index: 0, + start_at: 0, + reapplying: false, + }, + item_stream(0, 7), + ) + .await + .expect("an unknown outcome the record decides does not fail the file"); + assert_eq!(applied, 7); + let record = engine.read_restore_job(&key).await.unwrap().unwrap(); + assert_eq!( + (record.file_index, record.items_applied), + (Some(0), Some(7)), + "the file's progress is recorded" + ); + assert_eq!(engine.count_restore_target(&key).await.expect("count"), 7); + + // File 1: the first batch's write is lost twice over; the second unknown + // outcome fails the file. The batch's items landed (the writes are + // idempotent upserts a resume rewrites) and the record carries file 0's + // progress, so a resume re-applies file 1 from its start. + extenddb_storage_cassandra::test_hooks::fail_progress_writes( + &account_id, + &[ProgressWriteFault::Lost, ProgressWriteFault::Lost], + ); + let failed = engine + .bulk_load( + &key, + &*claim, + BulkLoadFile { + file_index: 1, + start_at: 0, + reapplying: false, + }, + item_stream(7, 14), + ) + .await + .expect_err("two unknown outcomes the record does not carry fail the file"); + assert!( + matches!(&failed, StorageError::Internal(m) if m.contains("had no known outcome twice")), + "{failed}" + ); + let record = engine.read_restore_job(&key).await.unwrap().unwrap(); + assert_eq!( + (record.file_index, record.items_applied), + (Some(0), Some(7)), + "file 1 recorded no progress" + ); + assert_eq!( + engine.count_restore_target(&key).await.expect("count"), + 10, + "the first batch of file 1 landed before its progress write" + ); + assert_eq!( + claim.heartbeat().await.expect("beat"), + Heartbeat::Confirmed, + "the claim is this job's throughout" + ); +} + +/// A batch closes at the first item whose text takes it past +/// `RESTORE_BATCH_BYTES`, long before the count cap: a file of 100 items of +/// about 350 KB each (35 MB of text against the 32 MiB bound) is written in +/// two batches, the first's progress recorded before the stream hands over +/// its last item. +#[tokio::test] +async fn test_a_batch_closes_at_the_byte_bound_before_the_count_cap() { + if crate::helpers::skip_without_cassandra() { + return; + } + const N: usize = 100; + const TEXT: usize = 350_000; + const { + assert!(N < extenddb_storage::restore_engine::RESTORE_BATCH_ITEMS); + // 100 items cross the byte bound once. + assert!( + N * TEXT > extenddb_storage::restore_engine::RESTORE_BATCH_BYTES + && (N - 2) * TEXT < 2 * extenddb_storage::restore_engine::RESTORE_BATCH_BYTES + ); + } + let engine = setup_engine().await; + let account = TestAccount::new(&engine, "extenddb_ttl_test").await; + let account_id = account.account_id.clone(); + let store = unique_store(); + let row = available_index_backup(&engine, &account_id, "Music", "nightly", &store).await; + let (desc, claim) = engine + .claim_restore( + &account_id, + manifest_target_input("BigItems"), + &manifest_registration(&row, None), + ) + .await + .expect("register"); + let key = RestoreTargetKey { + account_id: account_id.clone(), + table_name: desc.table_name.clone(), + table_id: desc.table_id.clone(), + }; + // The stream reads the record when asked for the last item: by then the + // byte bound has closed the first batch and recorded its progress. + let seen = std::sync::Arc::new(std::sync::Mutex::new(None)); + let seen_by_stream = std::sync::Arc::clone(&seen); + let reader = engine.clone(); + let read_key = key.clone(); + let big: futures::stream::BoxStream<'static, Result> = + Box::pin(futures::stream::unfold(0usize, move |i| { + let reader = reader.clone(); + let key = read_key.clone(); + let seen = std::sync::Arc::clone(&seen_by_stream); + async move { + if i >= N { + return None; + } + if i == N - 1 { + let record = reader.read_restore_job(&key).await.unwrap().unwrap(); + *seen.lock().expect("seen") = Some((record.file_index, record.items_applied)); + } + let mut item = id_item(i); + item.insert("v".to_owned(), AttributeValue::S("x".repeat(TEXT))); + Some((Ok(item), i + 1)) + } + })); + let applied = engine + .bulk_load( + &key, + &*claim, + BulkLoadFile { + file_index: 0, + start_at: 0, + reapplying: false, + }, + big, + ) + .await + .expect("bulk_load"); + assert_eq!(applied as usize, N); + let (file_index, items_applied) = seen + .lock() + .expect("seen") + .expect("the stream read the record at its last item"); + assert_eq!(file_index, Some(0), "the first batch recorded its progress"); + let first_batch = items_applied.expect("the first batch's total") as usize; + assert!( + 0 < first_batch && first_batch < N, + "the first batch closed at the byte bound with {first_batch} of {N} items" + ); + assert!( + first_batch * TEXT + < extenddb_storage::restore_engine::RESTORE_BATCH_BYTES + TEXT + 1_000 * N, + "the batch closed at the first item past the bound: {first_batch} items" + ); + let record = engine.read_restore_job(&key).await.unwrap().unwrap(); + assert_eq!( + (record.file_index, record.items_applied), + (Some(0), Some(N as u64)) + ); + assert_eq!( + engine.count_restore_target(&key).await.expect("count") as usize, + N + ); +} + +/// A claim lost between two batches of one file (the record's generation +/// moved by a takeover while the load ran): the read before the next batch +/// finds the record no longer this claim's, the file ends as a lost claim, +/// and the second batch's items never land. +#[cfg(feature = "test-hooks")] +#[tokio::test] +async fn test_bulk_load_stops_at_a_lost_claim_between_two_batches() { + if crate::helpers::skip_without_cassandra() { + return; + } + let engine = setup_engine().await; + let account = TestAccount::new(&engine, "extenddb_ttl_test").await; + let account_id = account.account_id.clone(); + extenddb_storage_cassandra::test_hooks::set_restore_batch_size(&account_id, 3); + let store = unique_store(); + let row = available_index_backup(&engine, &account_id, "Music", "nightly", &store).await; + let (desc, claim) = engine + .claim_restore( + &account_id, + manifest_target_input("LostMidFile"), + &manifest_registration(&row, None), + ) + .await + .expect("register"); + let key = RestoreTargetKey { + account_id: account_id.clone(), + table_name: desc.table_name.clone(), + table_id: desc.table_id.clone(), + }; + // The stream hands the first batch's three items, then moves the + // record's generation when asked for the first item of the second. + let mover = engine.clone(); + let moved_key = key.clone(); + let generation = i64::try_from(claim.generation()).unwrap(); + let stream: futures::stream::BoxStream<'static, Result> = + Box::pin(futures::stream::unfold(0usize, move |i| { + let mover = mover.clone(); + let key = moved_key.clone(); + async move { + if i >= 7 { + return None; + } + if i == 3 { + apply( + &mover, + &format!( + "UPDATE {}.tables SET restore_owner_generation = ? \ + WHERE account_id = ? AND table_name = ? \ + IF table_id = ? AND restore_owner_generation = ?", + mover.catalog_keyspace() + ), + cdrs_tokio::query_values!( + generation + 1, + key.account_id.as_str(), + key.table_name.as_str(), + key.table_id.as_str(), + generation + ), + ) + .await; + } + Some((Ok(id_item(i)), i + 1)) + } + })); + let out = engine + .bulk_load( + &key, + &*claim, + BulkLoadFile { + file_index: 0, + start_at: 0, + reapplying: false, + }, + stream, + ) + .await; + assert!( + matches!(&out, Err(StorageError::Internal(m)) if m.contains("claim on the target was lost")), + "the read before the second batch finds another owner: {out:?}" + ); + assert_eq!( + engine.count_restore_target(&key).await.expect("count"), + 3, + "the first batch's items alone landed" + ); + let record = engine.read_restore_job(&key).await.unwrap().unwrap(); + assert_eq!( + (record.file_index, record.items_applied), + (Some(0), Some(3)), + "the first batch's progress stands" + ); +} + +/// `bulk_load` crosses a batch boundary, recording the file's running total +/// after each batch, and a re-applied batch changes nothing observable: the +/// writes are idempotent upserts of the base rows and of each GSI and LSI row +/// through the ordinary put path (an index row whose key did not change is +/// rewritten by its `INSERT` alone, and each index row carries its base row's +/// `version` as its write timestamp, ADR-0027), so the base count and both +/// index row counts equal those of the first load, after the last batch alone +/// is re-applied (`start_at: 8`, a resume's shape) and after the whole file +/// is, and every index row's stamp follows its base row's version. +#[cfg(feature = "test-hooks")] +#[tokio::test] +async fn test_bulk_load_crosses_a_batch_boundary_and_a_reapply_changes_nothing() { + if crate::helpers::skip_without_cassandra() { + return; + } + let engine = setup_engine().await; + let account = TestAccount::new(&engine, "extenddb_ttl_test").await; + let account_id = account.account_id.clone(); + // Four items per batch, so twelve items cross two boundaries. + extenddb_storage_cassandra::test_hooks::set_restore_batch_size(&account_id, 4); + let store = unique_store(); + let row = available_index_backup(&engine, &account_id, "Music", "nightly", &store).await; + let (desc, claim) = engine + .claim_restore( + &account_id, + composite_target_input("BulkTarget"), + &manifest_registration(&row, None), + ) + .await + .expect("register"); + let key = RestoreTargetKey { + account_id: account_id.clone(), + table_name: desc.table_name.clone(), + table_id: desc.table_id.clone(), + }; + let composite_items = |range: std::ops::Range| { + futures::stream::iter(range.map(composite_item).map(Ok)).boxed() + }; + // Items 0, 2, 4, 6, 8 and 10 carry the GSI's key; 0, 3, 6 and 9 the LSI's. + let (gsi_rows, lsi_rows) = (6, 4); + + let applied = engine + .bulk_load( + &key, + &*claim, + BulkLoadFile { + file_index: 0, + start_at: 0, + reapplying: false, + }, + composite_items(0..12), + ) + .await + .expect("bulk_load"); + assert_eq!(applied, 12); + let record = engine.read_restore_job(&key).await.unwrap().unwrap(); + assert_eq!(record.file_index, Some(0)); + assert_eq!(record.items_applied, Some(12)); + assert_eq!(engine.count_restore_target(&key).await.expect("count"), 12); + assert_eq!( + index_rows(&engine, &account_id, &key.table_id, "gix").await, + gsi_rows + ); + assert_eq!( + index_rows(&engine, &account_id, &key.table_id, "lix").await, + lsi_rows + ); + let loaded = + assert_index_rows_carry_the_base_versions(&engine, &account_id, &key.table_id).await; + assert_eq!(loaded.len(), 12); + + // The last batch re-applied (a resume whose progress write was lost): + // the same base and index rows. + let reapplied = engine + .bulk_load( + &key, + &*claim, + BulkLoadFile { + file_index: 0, + start_at: 8, + reapplying: true, + }, + composite_items(8..12), + ) + .await + .expect("bulk_load re-apply of the last batch"); + assert_eq!(reapplied, 4); + assert_eq!(engine.count_restore_target(&key).await.expect("count"), 12); + assert_eq!( + index_rows(&engine, &account_id, &key.table_id, "gix").await, + gsi_rows, + "a re-applied batch keeps every GSI row" + ); + assert_eq!( + index_rows(&engine, &account_id, &key.table_id, "lix").await, + lsi_rows, + "a re-applied batch keeps every LSI row" + ); + // The re-applied items' base rows moved to a later version and their + // index rows with them; the others kept both. + let reapplied_versions = + assert_index_rows_carry_the_base_versions(&engine, &account_id, &key.table_id).await; + for (sk, version) in &reapplied_versions { + if sk.parse::().expect("an integer sk") >= 8 { + assert!(version > &loaded[sk], "item {sk} was written again"); + } else { + assert_eq!(version, &loaded[sk], "item {sk} was not written again"); + } + } + + // The whole file re-applied: the count stays 12 and the index rows stand. + let reapplied = engine + .bulk_load( + &key, + &*claim, + BulkLoadFile { + file_index: 0, + start_at: 0, + reapplying: true, + }, + composite_items(0..12), + ) + .await + .expect("bulk_load re-apply"); + assert_eq!(reapplied, 12); + assert_eq!( + engine.count_restore_target(&key).await.expect("count"), + 12, + "a re-applied file changes nothing" + ); + assert_eq!( + ( + index_rows(&engine, &account_id, &key.table_id, "gix").await, + index_rows(&engine, &account_id, &key.table_id, "lix").await + ), + (gsi_rows, lsi_rows), + "a re-applied file keeps every index row" + ); + assert_index_rows_carry_the_base_versions(&engine, &account_id, &key.table_id).await; +} + +/// An unordered backup carrying one key in two data files applies both items +/// (the second overwrites the first), so `count_restore_target` is one short +/// of the manifest's total: the shortfall is the duplicate, which the +/// engine's count check turns into a FAILED restore. +#[tokio::test] +async fn test_an_unordered_duplicate_key_is_caught_by_the_count_alone() { + if crate::helpers::skip_without_cassandra() { + return; + } + let engine = setup_engine().await; + let account = TestAccount::new(&engine, "extenddb_ttl_test").await; + let account_id = account.account_id.clone(); + let store = unique_store(); + let row = available_index_backup(&engine, &account_id, "Music", "nightly", &store).await; + let (desc, claim) = engine + .claim_restore( + &account_id, + manifest_target_input("DupTarget"), + &manifest_registration(&row, None), + ) + .await + .expect("register"); + let key = RestoreTargetKey { + account_id: account_id.clone(), + table_name: desc.table_name.clone(), + table_id: desc.table_id.clone(), + }; + // File 0: items 0..6. File 1: items 6..12 and item 0 again (the backup + // holds k00000 twice). The manifest's total is 13 items. + engine + .bulk_load( + &key, + &*claim, + BulkLoadFile { + file_index: 0, + start_at: 0, + reapplying: false, + }, + item_stream(0, 6), + ) + .await + .expect("file 0"); + let file_one: Vec = (6..12) + .map(id_item) + .chain(std::iter::once(id_item(0))) + .collect(); + engine + .bulk_load( + &key, + &*claim, + BulkLoadFile { + file_index: 1, + start_at: 0, + reapplying: false, + }, + futures::stream::iter(file_one.into_iter().map(Ok)).boxed(), + ) + .await + .expect("file 1"); + let count = engine.count_restore_target(&key).await.expect("count"); + assert_eq!(count, 12, "the duplicated key collapsed into one row"); + assert!(count < 13, "the count is one short of the manifest's 13"); +} + +/// `bulk_load` stops with a lost-claim error when the record's generation has +/// moved (the reconciler took it over): the progress lightweight transaction +/// matches no RUNNING record under the claim's generation. +#[tokio::test] +async fn test_bulk_load_is_refused_when_the_claim_was_lost() { + if crate::helpers::skip_without_cassandra() { + return; + } + let engine = setup_engine().await; + let account = TestAccount::new(&engine, "extenddb_ttl_test").await; + let account_id = account.account_id.clone(); + let store = unique_store(); + let row = available_index_backup(&engine, &account_id, "Music", "nightly", &store).await; + let (desc, claim) = engine + .claim_restore( + &account_id, + manifest_target_input("LostTarget"), + &manifest_registration(&row, None), + ) + .await + .expect("register"); + let key = RestoreTargetKey { + account_id: account_id.clone(), + table_name: desc.table_name.clone(), + table_id: desc.table_id.clone(), + }; + // Another owner bumps the generation. + apply( + &engine, + &format!( + "UPDATE {}.tables SET restore_owner_generation = ? \ + WHERE account_id = ? AND table_name = ? \ + IF table_id = ? AND restore_owner_generation = ?", + engine.catalog_keyspace() + ), + cdrs_tokio::query_values!( + i64::try_from(claim.generation()).unwrap() + 1, + account_id.as_str(), + key.table_name.as_str(), + key.table_id.as_str(), + i64::try_from(claim.generation()).unwrap() + ), + ) + .await; + let error = engine + .bulk_load( + &key, + &*claim, + BulkLoadFile { + file_index: 0, + start_at: 0, + reapplying: false, + }, + item_stream(0, 3), + ) + .await + .expect_err("the progress write finds another owner"); + assert!( + error.to_string().contains("claim on the target was lost"), + "{error}" + ); +} + +/// `finish_restore` flips the target ACTIVE and marks the record DONE, and a +/// stale generation's `finish_restore` is `NotMatched`. +#[tokio::test] +async fn test_finish_restore_flips_the_target_and_refuses_a_stale_generation() { + if crate::helpers::skip_without_cassandra() { + return; + } + let engine = setup_engine().await; + let account = TestAccount::new(&engine, "extenddb_ttl_test").await; + let account_id = account.account_id.clone(); + let store = unique_store(); + let row = available_index_backup(&engine, &account_id, "Music", "nightly", &store).await; + let (desc, claim) = engine + .claim_restore( + &account_id, + manifest_target_input("FinishTarget"), + &manifest_registration(&row, None), + ) + .await + .expect("register"); + let key = RestoreTargetKey { + account_id: account_id.clone(), + table_name: desc.table_name.clone(), + table_id: desc.table_id.clone(), + }; + engine + .bulk_load( + &key, + &*claim, + BulkLoadFile { + file_index: 0, + start_at: 0, + reapplying: false, + }, + item_stream(0, 5), + ) + .await + .expect("bulk_load"); + let count = engine.count_restore_target(&key).await.expect("count"); + assert_eq!(count, 5); + + assert_eq!( + engine + .finish_restore(&key, &*claim, count, 4_096) + .await + .expect("finish"), + Finished::Active + ); + let described = describe(&engine, &account_id, "FinishTarget").await; + assert_eq!(described.table_status, TableStatus::Active); + assert_eq!(described.item_count, 5); + + // A second flip, the record no longer RUNNING, matches nothing. + assert!(matches!( + engine + .finish_restore(&key, &*claim, count, 4_096) + .await + .expect("finish again"), + Finished::NotMatched(_) + )); +} + +/// After a claim writes the record FAILED (leaving `table_status` alone), +/// `remove_restore_target` performs the two-write removal (CREATING to +/// DELETING, then the drop) and the target is gone. +#[tokio::test] +async fn test_remove_restore_target_after_a_failed_write() { + if crate::helpers::skip_without_cassandra() { + return; + } + let engine = setup_engine().await; + let account = TestAccount::new(&engine, "extenddb_ttl_test").await; + let account_id = account.account_id.clone(); + let store = unique_store(); + let row = available_index_backup(&engine, &account_id, "Music", "nightly", &store).await; + let (desc, claim) = engine + .claim_restore( + &account_id, + manifest_target_input("FailTarget"), + &manifest_registration(&row, None), + ) + .await + .expect("register"); + let key = RestoreTargetKey { + account_id: account_id.clone(), + table_name: desc.table_name.clone(), + table_id: desc.table_id.clone(), + }; + assert_eq!( + claim + .set_status( + JobStatus::Running, + JobStatus::Failed, + &StatusFields { + failure_reason: Some("a data file failed".to_owned()), + completed: true, + ..StatusFields::default() + } + ) + .await + .expect("fail the record"), + WriteOutcome::Matched + ); + // The FAILED write left the table CREATING; the record carries the reason. + assert_eq!( + describe(&engine, &account_id, "FailTarget") + .await + .table_status, + TableStatus::Creating, + "the FAILED write leaves table_status alone for the removal's two-write step" + ); + let record = engine.read_restore_job(&key).await.unwrap().unwrap(); + assert_eq!(record.status, JobStatus::Failed); + assert_eq!(record.failure_reason.as_deref(), Some("a data file failed")); + + assert!( + engine.remove_restore_target(&key).await.expect("remove"), + "the two-write removal of a FAILED target succeeds" + ); + assert!( + matches!( + engine + .describe_table( + &account_id, + DescribeTableInput { + table_name: "FailTarget".to_owned() + } + ) + .await, + Err(StorageError::TableNotFound(_)) + ), + "the target is gone" + ); + assert!( + engine.read_restore_job(&key).await.expect("read").is_none(), + "the record went with the target" + ); +} + +/// The liveness test inside the takeover write: with the owner's beat +/// landing between `claim_dead_backup`'s `LOCAL_SERIAL` read (which found +/// the heartbeat stale) and its LWT, the LWT's `heartbeat_at = ` +/// condition fails and the takeover answers `None`; the row keeps the +/// owner's generation and the owner's next beat confirms. Without that +/// condition the staleness comparison alone would decide, and a live owner's +/// job would be failed by the reconciler. +#[cfg(feature = "test-hooks")] +#[tokio::test] +async fn test_a_beat_between_the_takeovers_read_and_its_write_refuses_the_takeover() { + if crate::helpers::skip_without_cassandra() { + return; + } + let engine = setup_engine().await; + let account = TestAccount::new(&engine, "extenddb_ttl_test").await; + let account_id = account.account_id.clone(); + let store = unique_store(); + let row = index_row(&account_id, "Music", "beat-in-the-gap", &store); + let key = row.key(); + let window = Duration::from_secs(1); + let owner = engine.claim_backup(&row).await.expect("claim"); + tokio::time::sleep(Duration::from_millis(1_300)).await; + let pause = + extenddb_storage_cassandra::test_hooks::pause_before_takeover_write(&row.backup_arn); + let taker = { + let engine = engine.clone(); + let key = key.clone(); + tokio::spawn(async move { engine.claim_dead_backup(&key, window).await }) + }; + pause.reached().await; + // The read found the beat stale; the owner beats now. + assert_eq!( + owner.heartbeat().await.expect("heartbeat"), + Heartbeat::Confirmed + ); + pause.release(); + let taken = taker.await.expect("the taker's task").expect("claim"); + assert!( + taken.is_none(), + "the LWT's heartbeat condition refuses a row whose owner beat in the gap" + ); + let after = stored_index_row(&engine, &row).await; + assert_eq!(after.owner_generation, owner.generation()); + assert_eq!(after.status, BackupStatus::Creating); + assert_eq!( + owner.heartbeat().await.expect("heartbeat"), + Heartbeat::Confirmed, + "the owner's claim is intact" + ); + // Without the beat, the same stale row is taken over. + tokio::time::sleep(Duration::from_millis(1_300)).await; + let successor = engine + .claim_dead_backup(&key, window) + .await + .expect("claim") + .expect("taken over once no beat lands in the gap"); + assert_ne!(successor.generation(), owner.generation()); +} + +// ── The batch's timestamp fence, the first batch resumed without progress, the +// ── readiness check's index tables, the in-use rule of a running restore ─────── + +/// The items of a target's data table (rows with an `item_data`: a delete +/// leaves the partition's static delete stamp behind, so `COUNT(*)` would +/// count a deleted item's partition), counted whatever the table's status +/// (`count_restore_target` refuses a target no longer CREATING). +#[cfg(feature = "test-hooks")] +async fn base_rows(engine: &CassandraEngine, account_id: &str, table_id: &str) -> i64 { + use cdrs_tokio::types::IntoRustByName; + let rows = extenddb_storage_cassandra::cassandra_util::query_rows_quorum::( + &engine.session_arc(), + &format!( + "SELECT COUNT(item_data) AS c FROM {}.items_{}", + engine.account_keyspace(account_id), + table_id.replace('-', "_") + ), + cdrs_tokio::query_values!(), + "base rows", + ) + .await + .expect("count the base rows"); + rows.first() + .expect("a count") + .get_by_name("c") + .ok() + .flatten() + .expect("c") +} + +/// Microseconds since the epoch on this host's clock, the domain of the +/// restore's write timestamps. +#[cfg(feature = "test-hooks")] +fn micros_now() -> i64 { + i64::try_from( + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("after the epoch") + .as_micros(), + ) + .expect("fits") +} + +/// The batch fence (the module doc of `restore_engine`): a writer paused +/// between a batch's claim check and the batch's writes, taken over, whose +/// successor re-applies the batch, flips the target ACTIVE and deletes two +/// of its items, lands its batch when released at the timestamp it read +/// before the check, below every write and delete the successor made: the +/// deleted items stay deleted, their GSI and LSI rows stay gone, and the +/// surviving rows carry the successor's versions, not the stalled writer's. +/// Without the timestamp (the writes stamped when they land) the stalled +/// batch would resurrect the deleted items and their index rows. +#[cfg(feature = "test-hooks")] +#[tokio::test] +async fn test_a_batch_landed_after_a_takeover_loses_to_the_successors_writes_and_deletes() { + if crate::helpers::skip_without_cassandra() { + return; + } + let engine = setup_engine().await; + let account = TestAccount::new(&engine, "extenddb_ttl_test").await; + let account_id = account.account_id.clone(); + extenddb_storage_cassandra::test_hooks::set_restore_batch_size(&account_id, 4); + let store = unique_store(); + let row = available_index_backup(&engine, &account_id, "Music", "nightly", &store).await; + let (desc, owner) = engine + .claim_restore( + &account_id, + composite_target_input("Fenced"), + &manifest_registration(&row, Some("owner:/s")), + ) + .await + .expect("register"); + let key = RestoreTargetKey { + account_id: account_id.clone(), + table_name: desc.table_name.clone(), + table_id: desc.table_id.clone(), + }; + let composite_items = |range: std::ops::Range| { + futures::stream::iter(range.map(composite_item).map(Ok)).boxed() + }; + + // The owner's one batch of four, held after its claim check. + let pause = + extenddb_storage_cassandra::test_hooks::pause_after_restore_claim_check(&key.table_id); + let stalled = { + let engine = engine.clone(); + let key = key.clone(); + tokio::spawn(async move { + engine + .bulk_load( + &key, + &*owner, + BulkLoadFile { + file_index: 0, + start_at: 0, + reapplying: false, + }, + composite_items(0..4), + ) + .await + }) + }; + pause.reached().await; + assert_eq!( + base_rows(&engine, &account_id, &key.table_id).await, + 0, + "nothing is written before the check" + ); + + // The owner's heartbeat (the check refreshed it) goes stale; the + // successor takes the record over, writes the batch itself at later + // timestamps, finishes, and two items are deleted through the ACTIVE + // table. + tokio::time::sleep(Duration::from_millis(1_300)).await; + let successor_from = micros_now(); + let successor = engine + .claim_restore_dead(&key, Duration::from_secs(1), Some("successor:/s")) + .await + .expect("claim") + .expect("the stale record is taken over"); + let applied = engine + .bulk_load( + &key, + &*successor, + BulkLoadFile { + file_index: 0, + start_at: 0, + reapplying: true, + }, + composite_items(0..4), + ) + .await + .expect("the successor's load"); + assert_eq!(applied, 4); + assert_eq!( + engine + .finish_restore(&key, &*successor, 4, 1_024) + .await + .expect("finish"), + Finished::Active + ); + let key_info = engine + .table_key_info(&account_id, "Fenced") + .await + .expect("the ACTIVE target's key info"); + // The deletes' GSI tombstones are written in the delete's own batch (no + // propagation worker runs in these tests). + make_gsis_synchronous(&engine, &key.table_id, &["gix"]).await; + // Items 0 and 2 carry the GSI's key `g`; item 0 the LSI's `l` as well. + for i in [0usize, 2] { + let mut k = BTreeMap::new(); + k.insert("id".to_owned(), AttributeValue::S(format!("p{}", i % 5))); + k.insert("sk".to_owned(), AttributeValue::N(i.to_string())); + engine + .delete_item(&key_info, &k, false, None, &ExpressionMaps::default(), None) + .await + .expect("delete through the ACTIVE table"); + } + assert_eq!(base_rows(&engine, &account_id, &key.table_id).await, 2); + assert_eq!( + index_rows(&engine, &account_id, &key.table_id, "gix").await, + 0 + ); + assert_eq!( + index_rows(&engine, &account_id, &key.table_id, "lix").await, + 1 + ); + + // The stalled writer lands its batch now. Its progress write then finds + // the record no longer its own (the lost claim); its writes carry the + // timestamp read before its check, so they change nothing. + pause.release(); + let out = stalled.await.expect("the owner's task"); + assert!( + matches!(&out, Err(StorageError::Internal(m)) if m.contains("claim on the target was lost")), + "the stalled writer's progress write reports the lost claim: {out:?}" + ); + assert_eq!( + base_rows(&engine, &account_id, &key.table_id).await, + 2, + "the deleted items stay deleted" + ); + assert_eq!( + index_rows(&engine, &account_id, &key.table_id, "gix").await, + 0, + "the deleted items' GSI rows stay gone" + ); + assert_eq!( + index_rows(&engine, &account_id, &key.table_id, "lix").await, + 1, + "the deleted item's LSI row stays gone" + ); + for i in [0usize, 2] { + let mut k = BTreeMap::new(); + k.insert("id".to_owned(), AttributeValue::S(format!("p{}", i % 5))); + k.insert("sk".to_owned(), AttributeValue::N(i.to_string())); + assert!( + engine.get_item(&key_info, &k).await.expect("get").is_none(), + "item {i} was not resurrected" + ); + } + let versions = live_versions_by_sk(&engine, &account_id, &key.table_id).await; + assert_eq!(versions.keys().collect::>(), ["1", "3"]); + for (sk, version) in &versions { + assert!( + *version >= successor_from, + "item {sk} carries the successor's version ({version}), not the stalled writer's \ + (before {successor_from})" + ); + } + // The surviving LSI row (item 3's) carries its base row's version. + assert_eq!( + stamps_by_sk(&engine, &account_id, &key.table_id, Some("lix")).await, + BTreeMap::from([("3".to_owned(), versions["3"])]) + ); +} + +/// Per live item of the composite fixture (keyed by its `sk`), the base +/// row's `version`; a deleted item's partition, left with its static delete +/// stamp alone, is skipped. +#[cfg(feature = "test-hooks")] +async fn live_versions_by_sk( + engine: &CassandraEngine, + account_id: &str, + table_id: &str, +) -> BTreeMap { + use cdrs_tokio::types::IntoRustByName; + let rows = extenddb_storage_cassandra::cassandra_util::query_rows_quorum::( + &engine.session_arc(), + &format!( + "SELECT item_data, version FROM {}.items_{}", + engine.account_keyspace(account_id), + table_id.replace('-', "_") + ), + cdrs_tokio::query_values!(), + "versions", + ) + .await + .expect("read the versions"); + rows.into_iter() + .filter_map(|row| { + let json: Option = row.get_by_name("item_data").ok().flatten(); + let image: Item = serde_json::from_str(&json?).expect("an item image"); + let Some(AttributeValue::N(sk)) = image.get("sk") else { + panic!("no sk in {image:?}"); + }; + let version: i64 = row.get_r_by_name("version").expect("a version"); + Some((sk.clone(), version)) + }) + .collect() +} + +/// A first batch whose writes landed and whose progress was never recorded +/// (here the progress lightweight transaction's outcome is unknown twice and +/// the load fails, the state a crash before the first progress record +/// leaves) is resumed with the first file re-applied (`reapplying`, from +/// `start_at: 0`): the upserts rewrite the landed rows and write the rest, +/// and the base count and both index-row counts are exact. The engine +/// takes this path for a target holding no more than the backup's total +/// (`a_half_applied_first_file_with_no_progress_is_resumed_with_upserts`). +#[cfg(feature = "test-hooks")] +#[tokio::test] +async fn test_a_first_batch_applied_without_progress_resumes_to_the_exact_count() { + if crate::helpers::skip_without_cassandra() { + return; + } + let engine = setup_engine().await; + let account = TestAccount::new(&engine, "extenddb_ttl_test").await; + let account_id = account.account_id.clone(); + extenddb_storage_cassandra::test_hooks::set_restore_batch_size(&account_id, 3); + let store = unique_store(); + let row = available_index_backup(&engine, &account_id, "Music", "nightly", &store).await; + let (desc, claim) = engine + .claim_restore( + &account_id, + composite_target_input("HalfFirst"), + &manifest_registration(&row, None), + ) + .await + .expect("register"); + let key = RestoreTargetKey { + account_id: account_id.clone(), + table_name: desc.table_name.clone(), + table_id: desc.table_id.clone(), + }; + let composite_items = |range: std::ops::Range| { + futures::stream::iter(range.map(composite_item).map(Ok)).boxed() + }; + // The first batch's writes land; its progress write is lost twice and the + // load fails with no progress recorded. + extenddb_storage_cassandra::test_hooks::fail_progress_writes( + &account_id, + &[ + extenddb_storage_cassandra::test_hooks::ProgressWriteFault::Lost, + extenddb_storage_cassandra::test_hooks::ProgressWriteFault::Lost, + ], + ); + let out = engine + .bulk_load( + &key, + &*claim, + BulkLoadFile { + file_index: 0, + start_at: 0, + reapplying: false, + }, + composite_items(0..7), + ) + .await; + assert!( + matches!(&out, Err(StorageError::Internal(m)) if m.contains("no known outcome twice")), + "{out:?}" + ); + let record = engine.read_restore_job(&key).await.unwrap().unwrap(); + assert_eq!( + (record.file_index, record.items_applied), + (None, None), + "no progress is recorded" + ); + assert_eq!( + engine.count_restore_target(&key).await.expect("count"), + 3, + "the first batch's items landed" + ); + + // The resume: the whole first file re-applied with upserts. + let applied = engine + .bulk_load( + &key, + &*claim, + BulkLoadFile { + file_index: 0, + start_at: 0, + reapplying: true, + }, + composite_items(0..7), + ) + .await + .expect("the resume's load"); + assert_eq!(applied, 7); + assert_eq!(engine.count_restore_target(&key).await.expect("count"), 7); + // Items 0, 2, 4 and 6 carry `g`; 0, 3 and 6 carry `l`. + assert_eq!( + index_rows(&engine, &account_id, &key.table_id, "gix").await, + 4 + ); + assert_eq!( + index_rows(&engine, &account_id, &key.table_id, "lix").await, + 3 + ); + let record = engine.read_restore_job(&key).await.unwrap().unwrap(); + assert_eq!( + (record.file_index, record.items_applied), + (Some(0), Some(7)) + ); + assert_index_rows_carry_the_base_versions(&engine, &account_id, &key.table_id).await; +} + +/// `restore_target_ready` reads the index tables as well as the data table: +/// a target whose GSI table is missing (the registration creates the data +/// table and then each index table in its own statement, so a crash between +/// them leaves this) is not ready, where the data table alone would pass. +#[tokio::test] +async fn test_restore_target_ready_requires_every_index_table() { + if crate::helpers::skip_without_cassandra() { + return; + } + let engine = setup_engine().await; + let account = TestAccount::new(&engine, "extenddb_ttl_test").await; + let account_id = account.account_id.clone(); + let store = unique_store(); + let row = available_index_backup(&engine, &account_id, "Music", "nightly", &store).await; + let (desc, _claim) = engine + .claim_restore( + &account_id, + composite_target_input("NoGsiTable"), + &manifest_registration(&row, None), + ) + .await + .expect("register"); + let key = RestoreTargetKey { + account_id: account_id.clone(), + table_name: desc.table_name.clone(), + table_id: desc.table_id.clone(), + }; + assert!( + engine.restore_target_ready(&key).await.expect("ready"), + "the registration created every table" + ); + // The GSI's table is dropped, as a crash between the data table's DDL + // and the index table's would leave it. + use cdrs_tokio::types::IntoRustByName; + let rows = extenddb_storage_cassandra::cassandra_util::query_rows_quorum::( + &engine.session_arc(), + &format!( + "SELECT index_id FROM {}.indexes WHERE table_id = ? AND index_name = ?", + engine.catalog_keyspace() + ), + cdrs_tokio::query_values!(key.table_id.as_str(), "gix"), + "index id", + ) + .await + .expect("read the index"); + let index_id: String = rows + .first() + .expect("the GSI exists") + .get_by_name("index_id") + .ok() + .flatten() + .expect("index_id"); + engine + .session() + .query(format!( + "DROP TABLE {}.{}", + engine.account_keyspace(&account_id), + extenddb_storage_cassandra::data::ddl::index_table_name(&index_id) + )) + .await + .expect("drop the GSI's table"); + assert!( + !engine.restore_target_ready(&key).await.expect("ready"), + "a target missing its GSI table is not ready" + ); +} + +/// The in-use rule's input on Cassandra while a store-backed restore of the +/// backup runs: `backup_in_use` names the CREATING target, twice (the +/// engine reads it before it validates a second restore's overrides, so an +/// unknown GSI override, a throughput override and a plain restore are each +/// `BackupInUseException`, as on the other backends); it names the target +/// still when the backup's `active_restores` entry has not landed (the +/// registration writes the target row first), through the target rows; and +/// it names nothing once the target is ACTIVE. +#[tokio::test] +async fn test_a_running_store_backed_restore_keeps_its_backup_in_use_until_active() { + if crate::helpers::skip_without_cassandra() { + return; + } + let engine = setup_engine().await; + let account = TestAccount::new(&engine, "extenddb_ttl_test").await; + let account_id = account.account_id.clone(); + let store = unique_store(); + let row = available_index_backup(&engine, &account_id, "Music", "nightly", &store).await; + assert_eq!( + engine.backup_in_use(&row.key()).await.expect("in use"), + None + ); + let (desc, claim) = engine + .claim_restore( + &account_id, + manifest_target_input("InUseTarget"), + &manifest_registration(&row, None), + ) + .await + .expect("register"); + let key = RestoreTargetKey { + account_id: account_id.clone(), + table_name: desc.table_name.clone(), + table_id: desc.table_id.clone(), + }; + for _ in 0..2 { + assert_eq!( + engine.backup_in_use(&row.key()).await.expect("in use"), + Some("InUseTarget".to_owned()), + "a running store-backed restore keeps the backup in use" + ); + let second = engine + .claim_restore( + &account_id, + manifest_target_input("SecondTarget"), + &manifest_registration(&row, None), + ) + .await + .err() + .expect("a second restore is refused"); + assert!( + matches!(second, StorageError::BackupInUse(ref m) if *m == backup_in_use_message(&row.backup_arn)), + "{second}" + ); + } + // The registration's `active_restores` entry removed: the target row + // alone keeps the backup in use. + apply( + &engine, + &format!( + "DELETE active_restores[?] FROM {}.backups_by_arn \ + WHERE account_id = ? AND backup_arn = ? IF EXISTS", + engine.catalog_keyspace() + ), + cdrs_tokio::query_values!( + key.table_id.as_str(), + account_id.as_str(), + row.backup_arn.as_str() + ), + ) + .await; + assert!( + registrations(&engine, &account_id, &row.backup_arn) + .await + .is_empty() + ); + assert_eq!( + engine.backup_in_use(&row.key()).await.expect("in use"), + Some("InUseTarget".to_owned()), + "the CREATING target row keeps the backup in use before the map entry lands" + ); + // ACTIVE: the backup is free. + assert_eq!( + engine + .finish_restore(&key, &*claim, 0, 0) + .await + .expect("finish"), + Finished::Active + ); + assert_eq!( + engine.backup_in_use(&row.key()).await.expect("in use"), + None + ); +} diff --git a/crates/storage-cassandra/tests/fixtures/0006_backup_catalog_index.expected.json b/crates/storage-cassandra/tests/fixtures/0006_backup_catalog_index.expected.json new file mode 100644 index 000000000..72e38d496 --- /dev/null +++ b/crates/storage-cassandra/tests/fixtures/0006_backup_catalog_index.expected.json @@ -0,0 +1,197 @@ +{ + "columns": { + "backup_index_by_status.account_id": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "backup_index_by_status.backup_arn": { + "kind": "clustering", + "position": 0, + "clustering_order": "asc", + "type": "text" + }, + "backup_index_by_status.backup_status": { + "kind": "partition_key", + "position": 1, + "clustering_order": "none", + "type": "text" + }, + "backup_index_by_status.store_id": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "backup_index_by_status.store_kind": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "backup_index_by_status.store_root": { + "kind": "partition_key", + "position": 0, + "clustering_order": "none", + "type": "text" + }, + "backup_maintenance.claim": { + "kind": "partition_key", + "position": 0, + "clustering_order": "none", + "type": "text" + }, + "backup_maintenance.claimed_at": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "timestamp" + }, + "backup_maintenance.heartbeat_at": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "timestamp" + }, + "backup_maintenance.holder": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "backup_maintenance.owner_generation": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "bigint" + }, + "backups_by_arn.backup_id": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "backups_by_arn.completed_at": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "timestamp" + }, + "backups_by_arn.failure_reason": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "backups_by_arn.heartbeat_at": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "timestamp" + }, + "backups_by_arn.owner_generation": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "bigint" + }, + "backups_by_arn.source_table_details_json": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "backups_by_arn.store_id": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "backups_by_arn.store_kind": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "backups_by_arn.store_root": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "tables.restore_completed_at": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "timestamp" + }, + "tables.restore_failure_reason": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "tables.restore_file_index": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "int" + }, + "tables.restore_heartbeat_at": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "timestamp" + }, + "tables.restore_items_applied": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "bigint" + }, + "tables.restore_kind": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "tables.restore_owner_generation": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "bigint" + }, + "tables.restore_spool_host": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "tables.restore_status": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "tables.restore_store_id": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "tables.restore_store_kind": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + }, + "tables.restore_store_root": { + "kind": "regular", + "position": -1, + "clustering_order": "none", + "type": "text" + } + }, + "indexes": {} +} diff --git a/crates/storage-cassandra/tests/fixtures/golden/0006_backup_catalog_index.txt b/crates/storage-cassandra/tests/fixtures/golden/0006_backup_catalog_index.txt new file mode 100644 index 000000000..50d1be01c --- /dev/null +++ b/crates/storage-cassandra/tests/fixtures/golden/0006_backup_catalog_index.txt @@ -0,0 +1,6 @@ +-- table_name, table_status, restore_status, restore_kind, restore_owner_generation, restore_heartbeat_at, restore_store_kind, restore_completed_at = created_at +cu_orders ACTIVE DONE legacy 0 0 null true +interrupted CREATING RUNNING legacy 0 0 null null +-- notes +backfilled 2 restore registration(s) as legacy restores, 1 of them RUNNING with a dead owner for the reconciler to remove +wrote settings.catalog_id diff --git a/crates/storage-cassandra/tests/migrations.rs b/crates/storage-cassandra/tests/migrations.rs index 3a2349632..3094558c2 100644 --- a/crates/storage-cassandra/tests/migrations.rs +++ b/crates/storage-cassandra/tests/migrations.rs @@ -106,9 +106,14 @@ async fn two_migrators_started_together_apply_a_legacy_catalog_once() { &format!("INSERT INTO {catalog}.settings (key, value) VALUES ('catalog_version', '0.0.4')"), ) .await; + // The second migrator waits for the first's whole run, every schema + // statement of migrations 1 to 6 with an agreement wait after each: + // about 16 s on an idle node, and past 60 s beside the rest of this + // binary's tests on one node. + let wait = 4 * TIMEOUT; let (first, other) = tokio::join!( - migrate_catalog_keyspace(first_node.clone(), &prefix, "test", TIMEOUT), - migrate_catalog_keyspace(second_node.clone(), &prefix, "test", TIMEOUT), + migrate_catalog_keyspace(first_node.clone(), &prefix, "test", wait), + migrate_catalog_keyspace(second_node.clone(), &prefix, "test", wait), ); let applied = [ first.expect("the first migrator").applied.len(), @@ -176,12 +181,7 @@ async fn migration_1_creates_the_expected_definitions() { std::fs::write(&path, text).expect("write the expected file"); continue; } - let mut expected: postcheck::Definitions = - serde_json::from_str(&std::fs::read_to_string(&path).expect("read")).expect("json"); - // Migration 5 drops continuous_backups, which migration 1 creates. - expected - .columns - .retain(|name, _| !name.starts_with("continuous_backups.")); + let expected = migrations_support::expected_definitions(dir); assert_eq!( postcheck::compare(&expected, &actual), Vec::::new(), diff --git a/crates/storage-cassandra/tests/migrations/lock.rs b/crates/storage-cassandra/tests/migrations/lock.rs index b56c2d441..eceac2106 100644 --- a/crates/storage-cassandra/tests/migrations/lock.rs +++ b/crates/storage-cassandra/tests/migrations/lock.rs @@ -12,7 +12,7 @@ use extenddb_storage::migrations::{CatalogMigrations, CatalogStatus, MigrationEr use extenddb_storage_cassandra::cassandra_util::CassandraSession; use extenddb_storage_cassandra::migrations::{ CassandraCatalogMigrations, LeaseTimes, MigrationLock, RegistrationTimes, ServerRegistration, - ServingRegistration, register_server, schema_statement_bounds, + ServingRegistration, register_server, register_server_as, schema_statement_bounds, }; use super::{deployment, target_of}; @@ -197,6 +197,96 @@ async fn a_server_and_a_migration_refuse_each_other() { assert!(rows.is_empty(), "a released server leaves no row"); } +/// The `release` cells of the catalog's `extenddb_servers` rows. +async fn server_releases(session: &CassandraSession, catalog: &str) -> Vec { + use cdrs_tokio::types::IntoRustByName; + session + .query(format!("SELECT release FROM {catalog}.extenddb_servers")) + .await + .expect("read") + .response_body() + .expect("body") + .into_rows() + .unwrap_or_default() + .iter() + .map(|row| { + let release: Option = row.get_by_name("release").expect("release"); + release.unwrap_or_default() + }) + .collect() +} + +/// A command that opens the catalog as a server does (`extenddb backup +/// list`) registers under its own name: the row's `release` carries it +/// after the release, a migration's wait names the command where it names +/// a server `extenddb serve`, the name is kept when the registration is taken +/// again, and the command's refusal while a migration holds the lock names +/// the holder without the server's advice to start the server. +#[tokio::test] +async fn a_command_registers_under_its_own_name() { + if helpers::skip_without_cassandra() { + return; + } + const COMMAND: &str = "extenddb backup list"; + let session = session().await; + let (prefix, _drops) = new_prefix("command"); + deployment(&session, &prefix).await; + let catalog = format!("{prefix}_catalog"); + let registration = register_server_as(session.clone(), &prefix, "127.0.0.1:1", "test", COMMAND) + .await + .expect("register"); + assert_eq!( + server_releases(&session, &catalog).await, + vec![format!("test as {COMMAND}")] + ); + let migrations = CassandraCatalogMigrations; + match migrations + .lock(&target_of(&prefix), Duration::from_secs(2)) + .await + { + Err(MigrationError::Locked(message)) => { + assert!( + message.contains(&format!("{COMMAND} test on 127.0.0.1:1")), + "{message}" + ); + assert!(!message.contains("extenddb serve"), "{message}"); + } + other => panic!("{other:?}"), + } + let again = registration.register_again().await; + assert!(matches!(again, Reregistration::Registered), "{again:?}"); + assert!( + server_releases(&session, &catalog) + .await + .contains(&format!("test as {COMMAND}")), + "taken again under the command's name" + ); + registration.release().await; + + let lock = migrations + .lock(&target_of(&prefix), Duration::from_secs(5)) + .await + .expect("no server is registered"); + let refused = register_server_as(session.clone(), &prefix, "127.0.0.1:2", "test", COMMAND) + .await + .err() + .expect("a migration holds the lock"); + assert_eq!(refused.exit_code(), 6); + let why = refused.to_string(); + assert!(why.contains("holds the catalog's migration lock"), "{why}"); + assert!(!why.contains("start the server"), "{why}"); + let served = register_server(session.clone(), &prefix, "127.0.0.1:3", "test") + .await + .err() + .expect("a server is refused too"); + assert!(served.to_string().contains("start the server"), "{served}"); + migrations.release(lock).await; + assert!( + server_releases(&session, &catalog).await.is_empty(), + "a refused command leaves no row" + ); +} + /// A code body's request to a node that stops answering fails at the request /// timeout, through the store the body is handed, rather than when the lease /// cuts the body 35 to 65 s later (SPEC.md A6: every request under the lock diff --git a/crates/storage-cassandra/tests/migrations_stopped.rs b/crates/storage-cassandra/tests/migrations_stopped.rs index a1df7802f..d0481e0dc 100644 --- a/crates/storage-cassandra/tests/migrations_stopped.rs +++ b/crates/storage-cassandra/tests/migrations_stopped.rs @@ -3,12 +3,14 @@ //! Runs stopped part way, as a migrator killed at that point stops, and the //! run after each, which finishes with the definitions of a fresh install; -//! and migration 2's rewrite of a released catalog's policies, compared with -//! a committed dump. Each test uses keyspaces of its own prefix and drops +//! and migration 2's rewrite of a released catalog's policies and migration +//! 6's backfill of its restore registrations, each compared with a committed +//! dump. Each test uses keyspaces of its own prefix and drops //! them. Skipped without a node unless `EXTENDDB_TEST_CASSANDRA=required`. //! -//! With `EXTENDDB_BLESS_EXPECTED=1`, the migration 2 test writes -//! `tests/fixtures/golden/0002_resource_tag_condition_keys.txt` instead of +//! With `EXTENDDB_BLESS_EXPECTED=1`, the migration 2 and migration 6 tests +//! write `tests/fixtures/golden/0002_resource_tag_condition_keys.txt` and +//! `tests/fixtures/golden/0006_backup_catalog_index.txt` instead of //! comparing. #[path = "common/mod.rs"] @@ -242,16 +244,7 @@ async fn assert_finished(session: &Arc, prefix: &str, released let actual = postcheck::describe(session, &keyspace) .await .expect("describe"); - let path = format!( - "{}/migrations/{dir}/0001_baseline.expected.json", - env!("CARGO_MANIFEST_DIR") - ); - let mut expected: postcheck::Definitions = - serde_json::from_str(&std::fs::read_to_string(&path).expect("read")).expect("json"); - // Migration 5 drops continuous_backups, which migration 1 creates. - expected - .columns - .retain(|name, _| !name.starts_with("continuous_backups.")); + let expected = migrations_support::expected_definitions(dir); assert_eq!(expected, actual, "{at}: {dir}"); } let store = CassandraCatalogStore::catalog_only(session.clone(), prefix.to_owned()); @@ -459,6 +452,171 @@ async fn migration_2_rewrites_a_released_catalogs_policies_in_every_place() { ); } +/// Migration 6 on a released catalog holding two restore registrations: one +/// whose table is ACTIVE (a finished restore) and one whose table is CREATING +/// (a restore the stop interrupted). The job record is backfilled, the +/// finished one DONE and completed at the table's creation, the interrupted +/// one RUNNING with the epoch heartbeat and generation 0; the catalog id is +/// written once; and a reconverge, which runs the body again, changes +/// nothing. +#[tokio::test] +async fn migration_6_backfills_a_released_catalogs_restore_registrations() { + if helpers::skip_without_cassandra() { + return; + } + let session = session().await; + let (prefix, _drops) = new_prefix("m6"); + released_catalog(&session, &prefix).await; + let catalog = format!("{prefix}_catalog"); + for (name, id, status) in [ + ( + "cu_orders", + "0b7c2f5e-3d1a-4e8b-9c6f-1a2b3c4d5e6f", + "ACTIVE", + ), + ( + "interrupted", + "4e1c0d2b-7a9f-4c3e-8b6d-2f5a1c9e7d30", + "CREATING", + ), + ] { + cql( + &session, + &format!( + "INSERT INTO {catalog}.tables (account_id, table_name, table_id, table_status, \ + created_at, restore_source_backup_arn, restore_source_table_arn, \ + restore_date_time) VALUES ('{ACCOUNT}', '{name}', '{id}', '{status}', \ + 1700000000000, \ + 'arn:aws:dynamodb:us-east-1:{ACCOUNT}:table/src/backup/01700000000000-0000000a', \ + 'arn:aws:dynamodb:us-east-1:{ACCOUNT}:table/src', 1700000000000)" + ), + ) + .await; + } + let (done, _) = + run_stopped_after(&session, &prefix, usize::MAX, false, MigrationId::LATEST).await; + done.expect("migrate"); + assert_finished(&session, &prefix, true, "migrated").await; + let registrations = || async { + let mut text = String::from( + "-- table_name, table_status, restore_status, restore_kind, \ + restore_owner_generation, restore_heartbeat_at, restore_store_kind, \ + restore_completed_at = created_at\n", + ); + let rows = session + .query(format!( + "SELECT table_name, table_status, created_at, restore_status, restore_kind, \ + restore_owner_generation, restore_heartbeat_at, restore_store_kind, \ + restore_completed_at FROM {catalog}.tables" + )) + .await + .expect("read") + .response_body() + .expect("body") + .into_rows() + .unwrap_or_default(); + let mut lines: Vec = rows + .iter() + .map(|row| { + let text = |name: &str| -> String { + let value: Option = row.get_by_name(name).expect(name); + value.unwrap_or_else(|| "null".to_owned()) + }; + let number = |name: &str| -> String { + let value: Option = row.get_by_name(name).expect(name); + value.map_or_else(|| "null".to_owned(), |v| v.to_string()) + }; + let created: Option = row.get_by_name("created_at").expect("created_at"); + let completed: Option = + row.get_by_name("restore_completed_at").expect("completed"); + let completed_at_creation = match completed { + None => "null".to_owned(), + Some(at) => (Some(at) == created).to_string(), + }; + [ + text("table_name"), + text("table_status"), + text("restore_status"), + text("restore_kind"), + number("restore_owner_generation"), + number("restore_heartbeat_at"), + text("restore_store_kind"), + completed_at_creation, + ] + .join("\t") + }) + .collect(); + lines.sort(); + for line in lines { + text.push_str(&line); + text.push('\n'); + } + text + }; + let mut text = registrations().await; + let store = CassandraCatalogStore::catalog_only(session.clone(), prefix.clone()); + let catalog_id = store + .get_setting("catalog_id") + .await + .expect("read") + .expect("a catalog id"); + assert!(uuid::Uuid::parse_str(&catalog_id).is_ok(), "{catalog_id}"); + let rows = session + .query(format!( + "SELECT notes FROM {catalog}.extenddb_migrations WHERE id = 6" + )) + .await + .expect("read") + .response_body() + .expect("body") + .into_rows() + .unwrap_or_default(); + let notes: String = rows + .first() + .expect("migration 6's row") + .get_r_by_name("notes") + .expect("notes"); + text.push_str("-- notes\n"); + text.push_str(¬es); + text.push('\n'); + let path = format!( + "{}/tests/fixtures/golden/0006_backup_catalog_index.txt", + env!("CARGO_MANIFEST_DIR") + ); + if std::env::var("EXTENDDB_BLESS_EXPECTED").as_deref() == Ok("1") { + std::fs::write(&path, &text).expect("write golden"); + } + let expected = std::fs::read_to_string(&path).unwrap_or_default(); + assert_eq!( + expected, text, + "differs from {path}; regenerate with EXTENDDB_BLESS_EXPECTED=1" + ); + + // A reconverge runs the body again and changes nothing. + let before = registrations().await; + let lock = MigrationLock::acquire(session.clone(), &catalog, LeaseTimes::DEFAULT, TIMEOUT) + .await + .expect("take the lock"); + let reconverged = { + let migrations = CassandraMigrations::locked(&lock, &prefix); + Runner::new(&migrations, "test").reconverge().await + }; + lock.release().await; + let reconverged = reconverged.expect("reconverge"); + assert!( + reconverged.applied.iter().any( + |step| step.id == MigrationId::BackupCatalogIndex.number() && step.notes.is_empty() + ), + "{reconverged:?}" + ); + assert_eq!(registrations().await, before); + assert_eq!( + store.get_setting("catalog_id").await.expect("read"), + Some(catalog_id), + "the catalog id is kept" + ); +} + /// Migration 2's ledger row names every policy it rewrote, so on a catalog /// with many tagged policies the row's notes pass the node's /// `batch_size_fail_threshold` (50 KB by default), and the row has to be diff --git a/crates/storage-cassandra/tests/migrations_support/mod.rs b/crates/storage-cassandra/tests/migrations_support/mod.rs index 11cd3485d..e3d33dcda 100644 --- a/crates/storage-cassandra/tests/migrations_support/mod.rs +++ b/crates/storage-cassandra/tests/migrations_support/mod.rs @@ -10,12 +10,43 @@ use std::time::Duration; use extenddb_storage_cassandra::cassandra_util::CassandraSession; use extenddb_storage_cassandra::engine::CassandraEngine; -use extenddb_storage_cassandra::migrations::wait_for_schema_agreement; +use extenddb_storage_cassandra::migrations::{postcheck, wait_for_schema_agreement}; use crate::helpers; pub const TIMEOUT: Duration = Duration::from_secs(60); +/// The columns and indexes migration 6 adds to the catalog keyspace. +pub const CATALOG_MIGRATION_6: &str = + include_str!("../fixtures/0006_backup_catalog_index.expected.json"); + +/// The definitions a keyspace of `dir` (`catalog` or `account`) holds after +/// every migration: migration 1's, without `continuous_backups`, which +/// migration 5 drops, and in the catalog with what migration 6 adds. +/// +/// # Panics +/// +/// A committed file cannot be read or is not a definitions file. +#[must_use] +pub fn expected_definitions(dir: &str) -> postcheck::Definitions { + let path = format!( + "{}/migrations/{dir}/0001_baseline.expected.json", + env!("CARGO_MANIFEST_DIR") + ); + let mut expected: postcheck::Definitions = + serde_json::from_str(&std::fs::read_to_string(&path).expect("read")).expect("json"); + expected + .columns + .retain(|name, _| !name.starts_with("continuous_backups.")); + if dir == "catalog" { + let added: postcheck::Definitions = + serde_json::from_str(CATALOG_MIGRATION_6).expect("migration 6"); + expected.columns.extend(added.columns); + expected.indexes.extend(added.indexes); + } + expected +} + pub async fn session() -> Arc { Arc::new( CassandraEngine::create_session(&helpers::test_config()) diff --git a/crates/storage-mongodb/Cargo.toml b/crates/storage-mongodb/Cargo.toml index 383f7e6ca..510805d92 100644 --- a/crates/storage-mongodb/Cargo.toml +++ b/crates/storage-mongodb/Cargo.toml @@ -10,9 +10,10 @@ license.workspace = true [features] # Enables deterministic coordination hooks used only by the MongoDB integration -# test binary (the mongodb-test-hooks server build). Production builds do not -# compile these hooks. -test-hooks = [] +# test binary (the mongodb-test-hooks server build), and the restore pause +# point of `extenddb-storage` the kill-point harness drives. Production builds +# do not compile these hooks. +test-hooks = ["extenddb-storage/test-hooks"] # Compiles the snapshot page-size setter, the open pause and the shared # snapshot conformance check the crate's snapshot tests use. Separate from # `test-hooks` so the crate's own tests build the production commit and diff --git a/crates/storage-mongodb/src/backup_catalog.rs b/crates/storage-mongodb/src/backup_catalog.rs new file mode 100644 index 000000000..4dda98fb4 --- /dev/null +++ b/crates/storage-mongodb/src/backup_catalog.rs @@ -0,0 +1,1758 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The catalog index on MongoDB: the `backup_index` collection, the restore +//! job record (`restore_job`) on a restore target's `tables` document, the +//! `backup_maintenance` collection and the `catalog_id` settings document. +//! +//! The claim of a backup job is the row itself: it is born owned by an +//! `update_one` with `upsert: true`, a filter on `owner_generation: {$exists: +//! false}` and an update pipeline that sets the generation and `heartbeat_at: +//! "$$NOW"`, so no row ever exists unowned and a second claim of the same +//! backup trips the unique index on (`account_id`, `backup_id`). Every write +//! the job makes carries its generation in the filter; a heartbeat that +//! matches other than exactly one document means the claim is lost. The +//! heartbeat is also the liveness test: the reconciler takes over a CREATING +//! row whose heartbeat is older than the stale window on the server's clock +//! with one conditional update that sets a fresh generation and leaves +//! `heartbeat_at` as the dead owner wrote it. +//! +//! The lock on a row for DeleteBackup is a session transaction that `$inc`s +//! `lock_seq` on the row before it reads the status and the in-use rule, so +//! two such transactions on one row conflict and order wholly one before the +//! other; a restore's registration over a `backup_index` row takes the same +//! lock. Staleness is measured on the server's clock (`$$NOW`). +//! +//! A conditional write whose error is the driver's network class +//! (`ErrorKind::Io`, `ConnectionPoolCleared`) has an unknown outcome +//! (`WriteOutcome::Unknown`: the statement may have reached the server); +//! a server error (a command or write error) is `Err`, as a statement error +//! is on PostgreSQL. +//! +//! The maintenance claim is one document (`_id: "claim"`) in +//! `backup_maintenance`: a record whose heartbeat is older than the stale +//! window is taken over by a conditional update, a missing one is inserted by +//! an upsert whose filter a live record never matches (its insert then trips +//! the `_id` index and the claim is refused), and heartbeats and the release +//! are conditioned on the generation. `$expr` is not allowed in an upsert's +//! filter, which is why the takeover and the insert are two writes. + +use std::time::Duration; + +use extenddb_core::backup_format::BackupArn; +use extenddb_core::types::{CreateTableInput, RestoreSummary, TableDescription}; +use extenddb_storage::backup_catalog::{ + BackupCatalog, BackupKey, BackupRow, BackupStatus, DeleteStart, ForgetRow, Heartbeat, JobClaim, + JobRow, JobStatus, MaintenanceClaim, RestoreJobRow, RestoreKind, RestoreRegistered, + RestoreRegistrationRow, RestoreTargetKey, StaleClock, StatusFields, StoreIdentity, StoreKind, + TimeBounds, UNOWNED_GENERATION, WriteOutcome, new_generation, +}; +use extenddb_storage::error::StorageError; +use futures::TryStreamExt; +use futures::future::BoxFuture; +use futures::stream::BoxStream; +use mongodb::bson::{self, Bson, Document, doc}; + +use crate::MongoEngine; +use crate::backup_engine::{ + RestoreRecord, RestoreRegistration, age_is_stale, restoring_target_filter, +}; +use crate::data_engine::{ + CommitOutcome, TRANSIENT_RETRY_ATTEMPTS, TxErr, backoff_sleep, is_duplicate_key, +}; + +/// The catalog collection of backup rows. +pub(crate) const BACKUP_INDEX: &str = "backup_index"; +/// The catalog collection of the maintenance claim record. +pub(crate) const BACKUP_MAINTENANCE: &str = "backup_maintenance"; +/// The settings document holding the catalog's random id. +pub(crate) const CATALOG_ID: &str = "catalog_id"; +/// The counter a row-locking transaction `$inc`s so two of them conflict. +pub(crate) const LOCK_SEQ: &str = "lock_seq"; + +/// The restore job record, a subdocument of a restore target's `tables` +/// document, with the field names the job record has on every backend. +pub(crate) const RESTORE_JOB: &str = "restore_job"; +pub(crate) const RESTORE_JOB_STATUS: &str = "restore_job.status"; +pub(crate) const RESTORE_JOB_KIND: &str = "restore_job.kind"; +pub(crate) const RESTORE_JOB_GENERATION: &str = "restore_job.owner_generation"; +pub(crate) const RESTORE_JOB_HEARTBEAT: &str = "restore_job.heartbeat_at"; +pub(crate) const RESTORE_JOB_FAILURE_REASON: &str = "restore_job.failure_reason"; +pub(crate) const RESTORE_JOB_COMPLETED_AT: &str = "restore_job.completed_at"; +pub(crate) const RESTORE_JOB_SPOOL_HOST: &str = "restore_job.spool_host"; +pub(crate) const RESTORE_JOB_FILE_INDEX: &str = "restore_job.file_index"; +/// Items of `file_index` applied so far, written with each batch's inserts +/// in one transaction (`restore_engine.rs`). +pub(crate) const RESTORE_JOB_ITEMS_APPLIED: &str = "restore_job.items_applied"; +/// Whether the registration's DDL finished: `false` from the record's +/// insert, `true` once the target's collections and index metadata exist; +/// the reconciler's resume refuses a record left `false`. +pub(crate) const RESTORE_JOB_DDL_DONE: &str = "restore_job.ddl_done"; + +fn internal(e: impl std::fmt::Display) -> StorageError { + StorageError::Internal(e.to_string()) +} + +fn not_found(backup_arn: &str) -> StorageError { + extenddb_storage::backup_catalog::backup_not_found(backup_arn) +} + +/// The error for a backup a CREATING restore target still reads: the shared +/// text, [`extenddb_storage::backup_catalog::backup_in_use`]. +pub(crate) fn backup_in_use(table: &str, backup_arn: &str) -> StorageError { + extenddb_storage::backup_catalog::backup_in_use(table, backup_arn) +} + +/// An `owner_generation` as the catalog stores it (a 64-bit integer); +/// generations are minted below `i64::MAX`. +pub(crate) fn generation_field(generation: u64) -> i64 { + i64::try_from(generation).unwrap_or(i64::MAX) +} + +fn generation_from_field(doc: &Document, key: &str) -> u64 { + match doc.get(key) { + Some(Bson::Int64(g)) => u64::try_from(*g).unwrap_or_default(), + Some(Bson::Int32(g)) => u64::try_from(*g).unwrap_or_default(), + Some(Bson::Double(g)) => { + #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)] + let g = *g as u64; + g + } + _ => 0, + } +} + +#[allow(clippy::cast_precision_loss)] +fn datetime_epoch_secs(dt: bson::DateTime) -> f64 { + dt.timestamp_millis() as f64 / 1000.0 +} + +#[allow(clippy::cast_possible_truncation)] +fn epoch_secs_datetime(secs: f64) -> bson::DateTime { + bson::DateTime::from_millis((secs * 1000.0).round() as i64) +} + +fn optional_datetime(doc: &Document, key: &str) -> Option { + doc.get_datetime(key) + .ok() + .map(|dt| datetime_epoch_secs(*dt)) +} + +fn required_str(doc: &Document, key: &str) -> Result { + doc.get_str(key) + .map(str::to_owned) + .map_err(|_| StorageError::Internal(format!("backup row is missing {key}"))) +} + +/// Whether a write's outcome cannot be told from its error: the statement +/// may have reached the server before the connection failed. +fn outcome_unknown(e: &mongodb::error::Error) -> bool { + matches!( + *e.kind, + mongodb::error::ErrorKind::Io(_) | mongodb::error::ErrorKind::ConnectionPoolCleared { .. } + ) +} + +/// One `backup_index` document as a [`BackupRow`]. +fn row_from_doc(doc: &Document) -> Result { + let backup_arn = required_str(doc, "backup_arn")?; + let store_kind = required_str(doc, "store_kind")?; + let store_kind = StoreKind::parse(&store_kind).ok_or_else(|| { + StorageError::Internal(format!( + "backup {backup_arn} carries an unknown store kind {store_kind:?}" + )) + })?; + let source_table_details_json: serde_json::Value = doc + .get("source_table_details_json") + .cloned() + .map(bson::from_bson) + .transpose() + .map_err(|e| StorageError::Internal(format!("parse source_table_details_json: {e}")))? + .ok_or_else(|| { + StorageError::Internal(format!( + "backup {backup_arn} is missing source_table_details_json" + )) + })?; + Ok(BackupRow { + backup_id: required_str(doc, "backup_id")?, + account_id: required_str(doc, "account_id")?, + table_name: required_str(doc, "table_name")?, + backup_arn, + backup_name: required_str(doc, "backup_name")?, + status: BackupStatus::parse(&required_str(doc, "status")?)?, + store: StoreIdentity { + store_kind, + store_root: required_str(doc, "store_root")?, + store_id: required_str(doc, "store_id")?, + }, + size_bytes: doc.get_i64("size_bytes").unwrap_or(0), + item_count: doc.get_i64("item_count").unwrap_or(0), + created_at: optional_datetime(doc, "created_at").unwrap_or(0.0), + completed_at: optional_datetime(doc, "completed_at"), + failure_reason: doc.get_str("failure_reason").ok().map(str::to_owned), + source_table_details_json, + owner_generation: generation_from_field(doc, "owner_generation"), + heartbeat_at: optional_datetime(doc, "heartbeat_at").unwrap_or(0.0), + }) +} + +/// The fields of `row` an insert sets, with `status` (the claim's CREATING, +/// or the row's own for `insert_backup_row`); the generation, the heartbeat +/// and any completion are added by the caller. +fn row_fields(row: &BackupRow, status: BackupStatus) -> Result { + let source = bson::to_bson(&row.source_table_details_json) + .map_err(|e| StorageError::Internal(format!("serialize source_table_details_json: {e}")))?; + Ok(doc! { + "table_name": &row.table_name, + "backup_arn": &row.backup_arn, + "backup_name": &row.backup_name, + "status": status.as_str(), + "store_kind": row.store.store_kind.as_str(), + "store_root": &row.store.store_root, + "store_id": &row.store.store_id, + "size_bytes": row.size_bytes, + "item_count": row.item_count, + "created_at": epoch_secs_datetime(row.created_at), + "source_table_details_json": source, + LOCK_SEQ: 0_i64, + }) +} + +/// The filter for the row `key` names. +fn key_filter(key: &BackupKey) -> Document { + doc! { "account_id": &key.account_id, "backup_id": &key.backup_id } +} + +/// `set` as the `$set` stage of a pipeline update: `$$NOW` stays the server's +/// clock and every other value is wrapped in `$literal`, so a string, or a +/// key inside the source table object, is never read as a field path. +pub(crate) fn literal_set(set: Document) -> Document { + set.into_iter() + .map(|(k, v)| { + let v = match v { + Bson::String(s) if s == "$$NOW" => Bson::String(s), + other => doc! { "$literal": other }.into(), + }; + (k, v) + }) + .collect() +} + +/// The filter for the rows of `store`. +fn store_filter(store: &StoreIdentity) -> Document { + doc! { + "store_kind": store.store_kind.as_str(), + "store_root": &store.store_root, + "store_id": &store.store_id, + } +} + +/// The `$expr` of [`BackupCatalog::list_backup_rows_older_than`]: the +/// `clock` column older than `age` on the server's clock. `age_is_stale` +/// reads a missing column as the epoch, which is the rule the dead-job +/// listing wants and not this one: a row with `completed_at` unset has not +/// reached its terminal status's clock, so the column must also be present. +fn older_than_filter(clock: StaleClock, age: Duration) -> Document { + let column = match clock { + StaleClock::Heartbeat => "heartbeat_at", + StaleClock::Completed => "completed_at", + }; + doc! { + column: { "$type": "date" }, + "$expr": age_is_stale(&[column], age), + } +} + +/// The sort every listing applies: `(created_at, backup_arn)`. +fn listing_sort() -> Document { + doc! { "created_at": 1, "backup_arn": 1 } +} + +/// The rows `read` fetches, as a stream: the read runs once, on the first +/// poll, and a failed read is one `Err` item. +fn rows_stream<'a, F>(read: F) -> BoxStream<'a, Result> +where + F: std::future::Future, StorageError>> + Send + 'a, +{ + use futures::StreamExt; + Box::pin( + futures::stream::once(async move { + match read.await { + Ok(docs) => docs.iter().map(row_from_doc).collect::>(), + Err(e) => vec![Err(e)], + } + }) + .flat_map(futures::stream::iter), + ) +} + +/// The filter of the restore job records the reconciler acts on: a RUNNING +/// record whose heartbeat is older than `stale_after` on the server's clock +/// (a dead owner), and a FAILED record whose table is still a CREATING +/// restore target once `completed_at` (or, where none was stamped, the +/// heartbeat) is that old: the FAILED write landed and the removal did not, +/// and the delay leaves a task still on its own failure path to finish it. +pub(crate) fn dead_restore_job_filter(stale_after: Duration) -> Document { + doc! { + "$or": [ + { + RESTORE_JOB_STATUS: JobStatus::Running.as_str(), + "$expr": age_is_stale(&[RESTORE_JOB_HEARTBEAT], stale_after), + }, + { + RESTORE_JOB_STATUS: JobStatus::Failed.as_str(), + "table_status": "CREATING", + "$expr": age_is_stale( + &[RESTORE_JOB_COMPLETED_AT, RESTORE_JOB_HEARTBEAT], + stale_after, + ), + }, + ] + } +} + +/// A backup job's handle on its `backup_index` document. +struct MongoBackupClaim { + index: mongodb::Collection, + key: BackupKey, + generation: u64, +} + +impl MongoBackupClaim { + /// The filter of the row while it carries this claim. + fn owned_filter(&self) -> Document { + let mut filter = key_filter(&self.key); + filter.insert("owner_generation", generation_field(self.generation)); + filter + } +} + +impl JobClaim for MongoBackupClaim { + fn generation(&self) -> u64 { + self.generation + } + + fn heartbeat(&self) -> BoxFuture<'_, Result> { + Box::pin(async move { + let result = self + .index + .update_one( + self.owned_filter(), + vec![doc! { "$set": { "heartbeat_at": "$$NOW" } }], + ) + .await + .map_err(|e| StorageError::Internal(format!("backup heartbeat: {e}")))?; + Ok(if result.matched_count == 1 { + Heartbeat::Confirmed + } else { + Heartbeat::Lost + }) + }) + } + + fn set_status<'a>( + &'a self, + from: JobStatus, + to: JobStatus, + fields: &'a StatusFields, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + let mut filter = self.owned_filter(); + filter.insert("status", from.as_str()); + let mut set = doc! { "status": to.as_str() }; + if let Some(size) = fields.size_bytes { + set.insert("size_bytes", size); + } + if let Some(count) = fields.item_count { + set.insert("item_count", count); + } + if let Some(reason) = &fields.failure_reason { + set.insert("failure_reason", reason.as_str()); + } + if let Some(source) = &fields.source_table_details_json { + let source = bson::to_bson(source).map_err(|e| { + StorageError::Internal(format!("serialize source_table_details_json: {e}")) + })?; + set.insert("source_table_details_json", source); + } + if fields.completed { + set.insert("completed_at", "$$NOW"); + } + // A pipeline update, so `$$NOW` is the server's clock; every other + // value is wrapped in `$literal` so the pipeline does not read a + // string, or a key of the source table object, as a field path. + let set = literal_set(set); + match self + .index + .update_one(filter, vec![doc! { "$set": set }]) + .await + { + Ok(result) if result.matched_count == 1 => Ok(WriteOutcome::Matched), + Ok(_) => Ok(WriteOutcome::NotMatched), + Err(e) if outcome_unknown(&e) => Ok(WriteOutcome::Unknown), + Err(e) => Err(internal(e)), + } + }) + } + + fn read(&self) -> BoxFuture<'_, Result> { + Box::pin(async move { + let doc = self + .index + .find_one(key_filter(&self.key)) + .await + .map_err(internal)? + .ok_or_else(|| not_found(&self.key.backup_arn))?; + Ok(JobRow { + status: JobStatus::parse(&required_str(&doc, "status")?)?, + owner_generation: generation_from_field(&doc, "owner_generation"), + heartbeat_at: optional_datetime(&doc, "heartbeat_at").unwrap_or(0.0), + failure_reason: doc.get_str("failure_reason").ok().map(str::to_owned), + }) + }) + } +} + +/// The filter of the `tables` document `key` names: the `_id` and the table +/// id, so a table re-created under the name is never the one addressed. +fn target_filter(key: &RestoreTargetKey) -> Document { + doc! { + "_id": { "account_id": &key.account_id, "table_name": &key.table_name }, + "table_id": &key.table_id, + } +} + +/// One restore target's `tables` document as a [`RestoreJobRow`]: the key, +/// the provenance and every member of the `restore_job` subdocument; the +/// store is read when all three of its fields are there. +pub(crate) fn restore_job_from_doc(doc: &Document) -> Result { + let id = doc + .get_document("_id") + .map_err(|_| StorageError::Internal("table is missing _id".to_owned()))?; + let job = doc.get_document(RESTORE_JOB).map_err(|_| { + StorageError::Internal("restore target is missing its job record".to_owned()) + })?; + let source_backup_arn = required_str(doc, "restore_source_backup_arn")?; + let store = match ( + job.get_str("store_kind"), + job.get_str("store_root"), + job.get_str("store_id"), + ) { + (Ok(kind), Ok(store_root), Ok(store_id)) => Some(StoreIdentity { + store_kind: StoreKind::parse(kind).ok_or_else(|| { + StorageError::Internal(format!( + "restore of {source_backup_arn} carries an unknown store kind {kind:?}" + )) + })?, + store_root: store_root.to_owned(), + store_id: store_id.to_owned(), + }), + _ => None, + }; + Ok(RestoreJobRow { + target: RestoreTargetKey { + account_id: required_str(id, "account_id")?, + table_name: required_str(id, "table_name")?, + table_id: required_str(doc, "table_id")?, + }, + source_backup_arn, + kind: RestoreKind::parse(&required_str(job, "kind")?)?, + status: JobStatus::parse(&required_str(job, "status")?)?, + owner_generation: generation_from_field(job, "owner_generation"), + heartbeat_at: optional_datetime(job, "heartbeat_at").unwrap_or(0.0), + store, + spool_host: job.get_str("spool_host").ok().map(str::to_owned), + file_index: match job.get("file_index") { + Some(Bson::Int32(i)) => u32::try_from(*i).ok(), + Some(Bson::Int64(i)) => u32::try_from(*i).ok(), + _ => None, + }, + items_applied: match job.get("items_applied") { + Some(Bson::Int32(i)) => u64::try_from(*i).ok(), + Some(Bson::Int64(i)) => u64::try_from(*i).ok(), + _ => None, + }, + failure_reason: job.get_str("failure_reason").ok().map(str::to_owned), + completed_at: optional_datetime(job, "completed_at"), + }) +} + +/// A store-backed restore job's handle on the job record of its target's +/// `tables` document: every write carries the target's key, its table id +/// and the generation, and the heartbeat and the status writes also require +/// the record RUNNING (or the `from` status). +struct MongoRestoreClaim { + tables: mongodb::Collection, + key: RestoreTargetKey, + generation: u64, +} + +impl MongoRestoreClaim { + /// The filter of the record while it is in `status` under this claim. + fn owned_filter(&self, status: JobStatus) -> Document { + let mut filter = target_filter(&self.key); + filter.insert(RESTORE_JOB_STATUS, status.as_str()); + filter.insert(RESTORE_JOB_GENERATION, generation_field(self.generation)); + filter + } +} + +impl JobClaim for MongoRestoreClaim { + fn generation(&self) -> u64 { + self.generation + } + + fn heartbeat(&self) -> BoxFuture<'_, Result> { + Box::pin(async move { + let result = self + .tables + .update_one( + self.owned_filter(JobStatus::Running), + vec![doc! { "$set": { RESTORE_JOB_HEARTBEAT: "$$NOW" } }], + ) + .await + .map_err(|e| StorageError::Internal(format!("restore heartbeat: {e}")))?; + Ok(if result.matched_count == 1 { + Heartbeat::Confirmed + } else { + Heartbeat::Lost + }) + }) + } + + fn set_status<'a>( + &'a self, + from: JobStatus, + to: JobStatus, + fields: &'a StatusFields, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + let mut set = doc! { RESTORE_JOB_STATUS: to.as_str() }; + if let Some(reason) = &fields.failure_reason { + set.insert(RESTORE_JOB_FAILURE_REASON, reason.as_str()); + } + if fields.completed { + set.insert(RESTORE_JOB_COMPLETED_AT, "$$NOW"); + } + let set = literal_set(set); + match self + .tables + .update_one(self.owned_filter(from), vec![doc! { "$set": set }]) + .await + { + Ok(result) if result.matched_count == 1 => Ok(WriteOutcome::Matched), + Ok(_) => Ok(WriteOutcome::NotMatched), + Err(e) if outcome_unknown(&e) => Ok(WriteOutcome::Unknown), + Err(e) => Err(internal(e)), + } + }) + } + + fn read(&self) -> BoxFuture<'_, Result> { + Box::pin(async move { + let doc = self + .tables + .find_one(target_filter(&self.key)) + .await + .map_err(internal)? + .ok_or_else(|| { + StorageError::TableNotFound(format!( + "restore target {} (table id {}) has no catalog document", + self.key.table_name, self.key.table_id + )) + })?; + let row = restore_job_from_doc(&doc)?; + Ok(JobRow { + status: row.status, + owner_generation: row.owner_generation, + heartbeat_at: row.heartbeat_at, + failure_reason: row.failure_reason, + }) + }) + } +} + +impl MongoEngine { + pub(crate) fn backup_index(&self) -> mongodb::Collection { + self.catalog_db.collection::(BACKUP_INDEX) + } + + fn tables(&self) -> mongodb::Collection { + self.catalog_db.collection::("tables") + } + + /// The name of a CREATING restore target registered from `backup_arn`, + /// if any: the one in-use rule DeleteBackup, a restore's registration and + /// `backup_in_use` share. Reads the target's status, not the job's. The + /// driver's error is returned as it is, so a read inside a transaction + /// keeps its transient label for the caller's retry. + pub(crate) async fn creating_restore_target( + &self, + account_id: &str, + backup_arn: &str, + session: Option<&mut mongodb::ClientSession>, + ) -> Result, mongodb::error::Error> { + let filter = restoring_target_filter(account_id, backup_arn); + let tables = self.tables(); + let find = tables.find_one(filter); + let target = match session { + Some(session) => find.session(session).await, + None => find.await, + }?; + Ok(target.map(|t| { + t.get_document("_id") + .ok() + .and_then(|id| id.get_str("table_name").ok()) + .unwrap_or("?") + .to_owned() + })) + } + + /// DeleteBackup's first step as one session transaction: `$inc` the + /// row's `lock_seq` (the lock), read the status, run the in-use rule, + /// write DELETING, or, with `remove`, delete the row + /// (`forget_available_backup_row`; `Deleting` then means removed). A + /// transaction that lost a write conflict is re-run. + async fn start_backup_delete_tx( + &self, + key: &BackupKey, + remove: bool, + ) -> Result { + let index = self.backup_index(); + let mut session = self.client.start_session().await.map_err(internal)?; + let tx_options = self.write_transaction_options(); + for attempt in 0..TRANSIENT_RETRY_ATTEMPTS { + session + .start_transaction() + .with_options(tx_options.clone()) + .await + .map_err(internal)?; + let body: Result = async { + let locked = index + .find_one_and_update(key_filter(key), doc! { "$inc": { LOCK_SEQ: 1_i64 } }) + .session(&mut session) + .await + .map_err(TxErr::from)? + .ok_or_else(|| TxErr::Fatal(not_found(&key.backup_arn)))?; + let status = BackupStatus::parse(&required_str(&locked, "status")?)?; + if status != BackupStatus::Available { + return Ok(DeleteStart::Status(status)); + } + if let Some(table) = self + .creating_restore_target(&key.account_id, &key.backup_arn, Some(&mut session)) + .await + .map_err(TxErr::from)? + { + return Ok(DeleteStart::InUse(table)); + } + let mut filter = key_filter(key); + filter.insert("status", BackupStatus::Available.as_str()); + if remove { + index + .delete_one(filter) + .session(&mut session) + .await + .map_err(TxErr::from)?; + } else { + index + .update_one( + filter, + doc! { "$set": { "status": BackupStatus::Deleting.as_str() } }, + ) + .session(&mut session) + .await + .map_err(TxErr::from)?; + } + Ok(DeleteStart::Deleting) + } + .await; + match body { + Ok(outcome) => match self.commit_write_transaction(&mut session, None).await? { + CommitOutcome::Committed => return Ok(outcome), + CommitOutcome::RerunBody => backoff_sleep(attempt).await, + }, + Err(TxErr::Transient) => { + let _ = session.abort_transaction().await; + backoff_sleep(attempt).await; + } + Err(TxErr::Fatal(e)) => { + let _ = session.abort_transaction().await; + return Err(e); + } + } + } + Err(StorageError::Internal(format!( + "{} of {} did not commit after {TRANSIENT_RETRY_ATTEMPTS} conflicting attempts", + if remove { + "The backup sync --prune-missing removal" + } else { + "DeleteBackup" + }, + key.backup_arn + ))) + } + + /// A store-backed restore's registration (the design's 5.3 step 4) as + /// one session transaction: `$inc` the `backup_index` row's `lock_seq` + /// while it is AVAILABLE (the lock `start_backup_delete` takes, so the + /// two order wholly one before the other), run the in-use rule, insert + /// the CREATING target with its provenance and its RUNNING `kind = + /// manifest` record under a fresh generation, the store and the + /// claimant's spool, with the server's clock on its heartbeat. The + /// transaction commits right after the insert, before the target's + /// collections and indexes are created; one that lost a write conflict + /// (a DeleteBackup or another registration of the backup) is re-run and + /// sees the other's result. + /// + /// Refusals: a missing or non-AVAILABLE row is `Backup not found`; a + /// CREATING target of the backup is `BackupInUse` naming it; a taken + /// name is what `create_table_for_restore` reports (`TableAlreadyExists`, + /// or `TableBeingRestored` when the name is a CREATING restore target). + async fn claim_restore_tx( + &self, + account_id: &str, + input: CreateTableInput, + registration: &RestoreRegistrationRow, + ) -> Result { + let target_table_name = input.table_name.clone(); + let backup_arn = registration.backup.backup_arn.as_str(); + let index = self.backup_index(); + let mut session = self.client.start_session().await.map_err(internal)?; + let tx_options = self.write_transaction_options(); + let generation = new_generation(); + for attempt in 0..TRANSIENT_RETRY_ATTEMPTS { + session + .start_transaction() + .with_options(tx_options.clone()) + .await + .map_err(internal)?; + let body: Result, TxErr> = async { + let mut lock = key_filter(®istration.backup); + lock.insert("status", BackupStatus::Available.as_str()); + let locked = index + .find_one_and_update(lock, doc! { "$inc": { LOCK_SEQ: 1_i64 } }) + .session(&mut session) + .await + .map_err(TxErr::from)?; + if locked.is_none() { + return Ok(Err(not_found(backup_arn))); + } + if let Some(restoring) = self + .creating_restore_target(account_id, backup_arn, Some(&mut session)) + .await + .map_err(TxErr::from)? + { + return Ok(Err(backup_in_use(&restoring, backup_arn))); + } + let mut target = RestoreRegistration { + backup_arn, + source_table_arn: ®istration.source_table_arn, + restore_date_time: epoch_secs_datetime(registration.restore_date_time), + owner_generation: generation, + // This process's clock, replaced by the server's inside + // the transaction (`stamp_server_clock`); test builds set + // it behind to prove the replacement. + first_heartbeat: self.local_now(), + record: RestoreRecord::Manifest { + store: ®istration.store, + spool_host: registration.spool_host.as_deref(), + }, + session: Some(&mut session), + }; + match self + .create_table_for_restore(account_id, input.clone(), &mut target) + .await + { + Ok(desc) => Ok(Ok(desc)), + Err(StorageError::Transient(_)) => Err(TxErr::Transient), + Err(e) => { + // A creation that failed after its insert committed + // (a collection or index could not be created) left a + // registered target under this generation; a name + // refusal left nothing, and the removal matches + // nothing. + if let Err(cleanup) = self + .remove_target_of_generation(account_id, &target_table_name, generation) + .await + { + tracing::error!( + "restore of {backup_arn} into {target_table_name} failed during \ + target creation ({e}), and cleanup failed: {cleanup}" + ); + } + Ok(Err(e)) + } + } + } + .await; + match body { + // The insert committed its transaction before the target's + // collections were created; nothing is left to commit. + Ok(Ok(mut desc)) => { + desc.restore_summary = Some(RestoreSummary { + source_backup_arn: Some(backup_arn.to_owned()), + source_table_arn: Some(registration.source_table_arn.clone()), + restore_date_time: registration.restore_date_time, + restore_in_progress: true, + }); + let key = RestoreTargetKey { + account_id: account_id.to_owned(), + table_name: target_table_name, + table_id: desc.table_id.clone(), + }; + return Ok(( + desc, + Box::new(MongoRestoreClaim { + tables: self.tables(), + key, + generation, + }) as Box, + )); + } + Ok(Err(refusal)) => { + let _ = session.abort_transaction().await; + return Err(refusal); + } + Err(TxErr::Transient) => { + let _ = session.abort_transaction().await; + backoff_sleep(attempt).await; + } + Err(TxErr::Fatal(e)) => { + let _ = session.abort_transaction().await; + return Err(e); + } + } + } + Err(StorageError::Internal(format!( + "the registration of the restore of {backup_arn} into {target_table_name} did not \ + commit after {TRANSIENT_RETRY_ATTEMPTS} conflicting attempts" + ))) + } +} + +impl BackupCatalog for MongoEngine { + fn catalog_id(&self) -> BoxFuture<'_, Result> { + Box::pin(async move { + let id = self + .catalog_db + .collection::("settings") + .find_one(doc! { "_id": CATALOG_ID }) + .await + .map_err(internal)? + .and_then(|d| d.get_str("value").ok().map(str::to_owned)); + id.ok_or_else(|| { + StorageError::Internal( + "the catalog has no catalog_id; run extenddb migrate --yes".to_owned(), + ) + }) + }) + } + + fn claim_backup<'a>( + &'a self, + row: &'a BackupRow, + ) -> BoxFuture<'a, Result, StorageError>> { + Box::pin(async move { + let generation = new_generation(); + let mut filter = key_filter(&row.key()); + filter.insert("owner_generation", doc! { "$exists": false }); + // The row is the claim. The upsert inserts it owned, with the + // server's clock on the heartbeat; an existing row never matches + // the filter (every row carries a generation), so the upsert's + // insert hits the unique index and the claim is refused. + let mut set = row_fields(row, BackupStatus::Creating)?; + set.insert("owner_generation", generation_field(generation)); + set.insert("heartbeat_at", "$$NOW"); + let set = literal_set(set); + match self + .backup_index() + .update_one(filter, vec![doc! { "$set": set }]) + .upsert(true) + .await + { + Ok(result) if result.upserted_id.is_some() => {} + Ok(_) => { + return Err(StorageError::AlreadyExists(format!( + "backup {} already has a row", + row.backup_arn + ))); + } + Err(e) if is_duplicate_key(&e) => { + return Err(StorageError::AlreadyExists(format!( + "a live job owns backup {}", + row.backup_arn + ))); + } + Err(e) => return Err(internal(e)), + } + Ok(Box::new(MongoBackupClaim { + index: self.backup_index(), + key: row.key(), + generation, + }) as Box) + }) + } + + fn insert_backup_row<'a>( + &'a self, + row: &'a BackupRow, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + // The claim's upsert, unowned: an existing row never matches the + // filter (every row carries a generation, this one 0), so the + // upsert's insert hits a unique index, (`account_id`, + // `backup_id`) or `backup_id`, and nothing is written. + let mut filter = key_filter(&row.key()); + filter.insert("owner_generation", doc! { "$exists": false }); + let mut set = row_fields(row, row.status)?; + if let Some(reason) = &row.failure_reason { + set.insert("failure_reason", reason); + } + set.insert("owner_generation", generation_field(UNOWNED_GENERATION)); + set.insert("heartbeat_at", "$$NOW"); + set.insert("completed_at", "$$NOW"); + let set = literal_set(set); + match self + .backup_index() + .update_one(filter, vec![doc! { "$set": set }]) + .upsert(true) + .await + { + Ok(result) => Ok(result.upserted_id.is_some()), + Err(e) if is_duplicate_key(&e) => Ok(false), + Err(e) => Err(internal(e)), + } + }) + } + + fn list_backup_rows<'a>( + &'a self, + status: Option, + store: &'a StoreIdentity, + ) -> BoxStream<'a, Result> { + rows_stream(async move { + let mut filter = store_filter(store); + if let Some(status) = status { + filter.insert("status", status.as_str()); + } + self.backup_index() + .find(filter) + .sort(listing_sort()) + .await + .map_err(internal)? + .try_collect() + .await + .map_err(internal) + }) + } + + fn newest_available_backup_row<'a>( + &'a self, + store: &'a StoreIdentity, + ) -> BoxFuture<'a, Result, StorageError>> { + Box::pin(async move { + let mut filter = store_filter(store); + filter.insert("status", BackupStatus::Available.as_str()); + self.backup_index() + .find_one(filter) + .sort(doc! { "created_at": -1, "backup_arn": -1 }) + .await + .map_err(internal)? + .as_ref() + .map(row_from_doc) + .transpose() + }) + } + + fn list_account_backup_rows<'a>( + &'a self, + account_id: &'a str, + table_name: Option<&'a str>, + bounds: &'a TimeBounds, + ) -> BoxStream<'a, Result> { + rows_stream(async move { + let mut filter = doc! { "account_id": account_id }; + if let Some(table_name) = table_name { + filter.insert("table_name", table_name); + } + let mut created = Document::new(); + if let Some(lower) = bounds.lower { + created.insert("$gte", epoch_secs_datetime(lower)); + } + if let Some(upper) = bounds.upper { + created.insert("$lt", epoch_secs_datetime(upper)); + } + if !created.is_empty() { + filter.insert("created_at", created); + } + self.backup_index() + .find(filter) + .sort(listing_sort()) + .await + .map_err(internal)? + .try_collect() + .await + .map_err(internal) + }) + } + + fn read_backup_row<'a>( + &'a self, + arn: &'a BackupArn, + ) -> BoxFuture<'a, Result, StorageError>> { + Box::pin(async move { + let doc = self + .backup_index() + .find_one(doc! { + "account_id": arn.account_id(), + "backup_id": arn.backup_id(), + "table_name": arn.table_name(), + "backup_arn": arn.to_arn_string(), + }) + .await + .map_err(internal)?; + doc.as_ref().map(row_from_doc).transpose() + }) + } + + fn start_backup_delete<'a>( + &'a self, + key: &'a BackupKey, + ) -> BoxFuture<'a, Result> { + Box::pin(self.start_backup_delete_tx(key, false)) + } + + fn forget_available_backup_row<'a>( + &'a self, + key: &'a BackupKey, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + Ok(match self.start_backup_delete_tx(key, true).await? { + DeleteStart::Deleting => ForgetRow::Removed, + DeleteStart::InUse(table) => ForgetRow::InUse(table), + DeleteStart::Status(status) => ForgetRow::Status(status), + }) + }) + } + + fn set_backup_deleted<'a>( + &'a self, + key: &'a BackupKey, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + let mut filter = key_filter(key); + filter.insert("status", BackupStatus::Deleting.as_str()); + let result = self + .backup_index() + .update_one( + filter, + vec![doc! { "$set": { + "status": { "$literal": BackupStatus::Deleted.as_str() }, + "completed_at": "$$NOW", + } }], + ) + .await + .map_err(internal)?; + Ok(result.matched_count == 1) + }) + } + + fn remove_backup_row<'a>( + &'a self, + key: &'a BackupKey, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + let mut filter = key_filter(key); + filter.insert( + "status", + doc! { "$in": [BackupStatus::Deleted.as_str(), BackupStatus::Failed.as_str()] }, + ); + let result = self + .backup_index() + .delete_one(filter) + .await + .map_err(internal)?; + Ok(result.deleted_count == 1) + }) + } + + fn remove_dead_creating_backup_row<'a>( + &'a self, + key: &'a BackupKey, + stale_after: Duration, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + let mut filter = key_filter(key); + filter.insert("status", BackupStatus::Creating.as_str()); + filter.insert("$expr", age_is_stale(&["heartbeat_at"], stale_after)); + let result = self + .backup_index() + .delete_one(filter) + .await + .map_err(internal)?; + Ok(result.deleted_count == 1) + }) + } + + fn claim_dead_backup<'a>( + &'a self, + key: &'a BackupKey, + stale_after: Duration, + ) -> BoxFuture<'a, Result>, StorageError>> { + Box::pin(async move { + // The heartbeat is the liveness test: a live owner refreshes it + // every period, so a CREATING row whose heartbeat is older than + // the window on the server's clock has no owner, and the one + // write below both tests that and takes the row. The filter + // binds no generation, so the write takes a row whatever + // generation it carries; `heartbeat_at` is left as the dead + // owner wrote it. A second taker before this one resolves the + // row (another process's reconciler; one process runs one + // reconciler) finds the same stale heartbeat and takes the row + // again under its own generation, which fences this claim's + // status write as any takeover fences the dead owner's. + let generation = new_generation(); + let mut filter = key_filter(key); + filter.insert("status", BackupStatus::Creating.as_str()); + filter.insert("$expr", age_is_stale(&["heartbeat_at"], stale_after)); + let update = vec![doc! { "$set": literal_set(doc! { + "owner_generation": generation_field(generation), + }) }]; + let taken = self + .backup_index() + .find_one_and_update(filter, update) + .await + .map_err(internal)?; + Ok(taken.map(|_| { + Box::new(MongoBackupClaim { + index: self.backup_index(), + key: key.clone(), + generation, + }) as Box + })) + }) + } + + fn list_backup_rows_older_than<'a>( + &'a self, + status: BackupStatus, + store: &'a StoreIdentity, + clock: StaleClock, + age: Duration, + ) -> BoxStream<'a, Result> { + rows_stream(async move { + let mut filter = store_filter(store); + filter.insert("status", status.as_str()); + filter.extend(older_than_filter(clock, age)); + self.backup_index() + .find(filter) + .sort(listing_sort()) + .await + .map_err(internal)? + .try_collect() + .await + .map_err(internal) + }) + } + + fn backup_in_use<'a>( + &'a self, + key: &'a BackupKey, + ) -> BoxFuture<'a, Result, StorageError>> { + Box::pin(async move { + self.creating_restore_target(&key.account_id, &key.backup_arn, None) + .await + .map_err(internal) + }) + } + + fn remove_restore_target<'a>( + &'a self, + key: &'a RestoreTargetKey, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + // The key must name a CREATING restore target with this table id; + // a table re-created under the name has another id and is not + // touched. The delete is one atomic step on the document, so it + // and the ACTIVE flip (conditioned on CREATING) exclude each + // other. The catalog document goes first: once it is gone no + // provenance keeps the source backup in use, whatever the + // collection drops do, and the target counts as removed: a drop + // that fails then is logged, naming the collections, and leaves + // them in the data database (no record remains for a later cycle + // to retry from, and no sweep removes them yet). + let removed = self + .tables() + .find_one_and_delete(doc! { + "_id": { "account_id": &key.account_id, "table_name": &key.table_name }, + "table_id": &key.table_id, + "table_status": "CREATING", + "restore_source_backup_arn": { "$exists": true }, + }) + .await + .map_err(internal)?; + if removed.is_none() { + return Ok(false); + } + let dropped = self.remove_target_collections(&key.table_id).await; + #[cfg(test)] + let dropped = if self.backup_liveness.target_drop_fails { + Err(StorageError::Internal( + "test: the reply to the target's collection drops was lost".to_owned(), + )) + } else { + dropped + }; + if let Err(e) = dropped { + tracing::error!( + "the restore target {} (account {}) was removed from the catalog, but \ + dropping its data collection {} or the index collections of table id {} \ + failed; they are left in the data database, where nothing removes them, \ + and can be dropped by hand: {e}", + key.table_name, + key.account_id, + crate::data::data_collection_name(&key.table_id), + key.table_id + ); + } + Ok(true) + }) + } + + fn list_dead_restore_jobs( + &self, + stale_after: Duration, + ) -> BoxFuture<'_, Result, StorageError>> { + Box::pin(async move { + let docs: Vec = self + .tables() + .find(dead_restore_job_filter(stale_after)) + .await + .map_err(internal)? + .try_collect() + .await + .map_err(internal)?; + // A document this build cannot read (a kind or status no release + // writes) is logged and skipped, as ListBackups skips a row it + // cannot read, so one such document does not stop the cycle. + Ok(docs + .iter() + .filter_map(|doc| match restore_job_from_doc(doc) { + Ok(row) => Some(row), + Err(e) => { + tracing::warn!( + "reconciler: restore job record {:?} skipped: {e}", + doc.get("_id") + ); + None + } + }) + .collect()) + }) + } + + fn claim_restore<'a>( + &'a self, + account_id: &'a str, + input: CreateTableInput, + registration: &'a RestoreRegistrationRow, + ) -> BoxFuture<'a, Result> { + Box::pin(self.claim_restore_tx(account_id, input, registration)) + } + + fn claim_restore_dead<'a>( + &'a self, + key: &'a RestoreTargetKey, + stale_after: Duration, + spool_host: Option<&'a str>, + ) -> BoxFuture<'a, Result>, StorageError>> { + Box::pin(async move { + // The heartbeat is the liveness test, as for a backup row: a + // RUNNING manifest record whose heartbeat is older than the window + // on the server's clock has no owner, and the one write below + // tests that and takes the record under a fresh generation with + // the claimant's spool and `heartbeat_at` at the server's clock, + // so a second taker (another instance's reconciler in the period + // before the successor's first beat) finds it live; the trait doc + // says why a restore record is refreshed where a backup row is + // not. A legacy record is never taken over (the reconciler fails + // it), and a record that is not RUNNING is its owner's or + // finished. + let generation = new_generation(); + let mut filter = target_filter(key); + filter.insert(RESTORE_JOB_STATUS, JobStatus::Running.as_str()); + filter.insert(RESTORE_JOB_KIND, RestoreKind::Manifest.as_str()); + filter.insert("$expr", age_is_stale(&[RESTORE_JOB_HEARTBEAT], stale_after)); + let mut update = vec![doc! { "$set": literal_set(doc! { + RESTORE_JOB_GENERATION: generation_field(generation), + RESTORE_JOB_HEARTBEAT: "$$NOW", + }) }]; + match spool_host { + Some(host) => update.push(doc! { "$set": literal_set(doc! { + RESTORE_JOB_SPOOL_HOST: host, + }) }), + None => update.push(doc! { "$unset": RESTORE_JOB_SPOOL_HOST }), + } + let taken = self + .tables() + .find_one_and_update(filter, update) + .await + .map_err(internal)?; + Ok(taken.map(|_| { + Box::new(MongoRestoreClaim { + tables: self.tables(), + key: key.clone(), + generation, + }) as Box + })) + }) + } + + fn read_restore_job<'a>( + &'a self, + key: &'a RestoreTargetKey, + ) -> BoxFuture<'a, Result, StorageError>> { + Box::pin(async move { + let mut filter = target_filter(key); + filter.insert(RESTORE_JOB, doc! { "$exists": true }); + let doc = self.tables().find_one(filter).await.map_err(internal)?; + doc.as_ref().map(restore_job_from_doc).transpose() + }) + } + + fn list_restore_jobs<'a>( + &'a self, + status: Option, + ) -> BoxFuture<'a, Result, StorageError>> { + Box::pin(async move { + let mut filter = doc! { RESTORE_JOB: { "$exists": true } }; + if let Some(status) = status { + filter.insert(RESTORE_JOB_STATUS, status.as_str()); + } + let docs: Vec = self + .tables() + .find(filter) + .sort(doc! { "_id": 1 }) + .await + .map_err(internal)? + .try_collect() + .await + .map_err(internal)?; + docs.iter().map(restore_job_from_doc).collect() + }) + } + + fn list_running_restore_jobs(&self) -> BoxFuture<'_, Result, StorageError>> { + Box::pin(async move { + // By status alone: a record stamped in this read's instant, or + // ahead of the server's clock, is RUNNING too. + let docs: Vec = self + .tables() + .find(doc! { RESTORE_JOB_STATUS: JobStatus::Running.as_str() }) + .await + .map_err(internal)? + .try_collect() + .await + .map_err(internal)?; + docs.iter().map(restore_job_from_doc).collect() + }) + } + + /// Conditioned on RUNNING, on the listed `owner_generation` and on the + /// listed `heartbeat_at` (a `Date`, millisecond precision, rebuilt from + /// the listed epoch seconds exactly), and written with a fresh generation + /// (design 4(e)). + fn fail_restore_job<'a>( + &'a self, + key: &'a RestoreTargetKey, + owner_generation: u64, + heartbeat_at: f64, + reason: &'a str, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + let result = self + .tables() + .update_one( + doc! { + "_id": { "account_id": &key.account_id, "table_name": &key.table_name }, + "table_id": &key.table_id, + RESTORE_JOB_STATUS: JobStatus::Running.as_str(), + RESTORE_JOB_GENERATION: generation_field(owner_generation), + RESTORE_JOB_HEARTBEAT: epoch_secs_datetime(heartbeat_at), + }, + vec![doc! { "$set": { + RESTORE_JOB_STATUS: { "$literal": JobStatus::Failed.as_str() }, + RESTORE_JOB_FAILURE_REASON: { "$literal": reason }, + RESTORE_JOB_COMPLETED_AT: "$$NOW", + RESTORE_JOB_GENERATION: { "$literal": generation_field(new_generation()) }, + } }], + ) + .await + .map_err(internal)?; + Ok(result.matched_count == 1) + }) + } + + fn read_store_ids<'a>( + &'a self, + store_kind: StoreKind, + store_root: &'a str, + ) -> BoxFuture<'a, Result, StorageError>> { + Box::pin(async move { + let ids = self + .backup_index() + .distinct( + "store_id", + doc! { "store_kind": store_kind.as_str(), "store_root": store_root }, + ) + .await + .map_err(internal)?; + let mut ids: Vec = ids + .into_iter() + .filter_map(|v| match v { + Bson::String(s) => Some(s), + _ => None, + }) + .collect(); + ids.sort(); + Ok(ids) + }) + } + + fn maintenance_claim<'a>( + &'a self, + holder: &'a str, + stale_after: Duration, + ) -> BoxFuture<'a, Result, StorageError>> { + Box::pin(async move { + // The record is the claim, as a backup row is: one document under + // a fixed `_id`. Two writes: the first takes over a record whose + // heartbeat is stale (a dead holder's), the second inserts the + // record when there is none. A live record matches neither + // filter, so the second write's upsert trips the `_id` index and + // the claim is refused; two commands racing for a stale record + // both run the first write, and the one whose filter no longer + // matches falls to the second and is refused the same way. + // (`$expr` is not allowed in an upsert's filter, which is why + // the two are not one write.) + let generation = new_generation(); + let collection = self.maintenance(); + let set = doc! { "$set": { + "holder": { "$literal": holder }, + "owner_generation": { "$literal": generation_field(generation) }, + "claimed_at": "$$NOW", + "heartbeat_at": "$$NOW", + } }; + let refused = |current: Option| { + StorageError::AlreadyExists(format!( + "the backup maintenance claim is held by {}", + current.as_deref().unwrap_or("another maintenance command") + )) + }; + let taken_over = collection + .update_one( + doc! { + "_id": MAINTENANCE_CLAIM_ID, + "$expr": age_is_stale(&["heartbeat_at"], stale_after), + }, + vec![set.clone()], + ) + .await + .map_err(internal)? + .matched_count + == 1; + if !taken_over { + match collection + .update_one( + doc! { + "_id": MAINTENANCE_CLAIM_ID, + "owner_generation": { "$exists": false }, + }, + vec![set], + ) + .upsert(true) + .await + { + // The upsert matched a document with no + // `owner_generation` (an operator's insert, say, not a + // release's) and overwrote it with this call's + // generation: the claim is this call's, and refusing here + // would leave a document nobody heartbeats until it goes + // stale. + Ok(result) if result.upserted_id.is_some() || result.matched_count == 1 => {} + Ok(_) => return Err(refused(None)), + Err(e) if is_duplicate_key(&e) => { + let current = collection + .find_one(doc! { "_id": MAINTENANCE_CLAIM_ID }) + .await + .map_err(internal)? + .and_then(|d| d.get_str("holder").ok().map(str::to_owned)); + return Err(refused(current)); + } + Err(e) => return Err(internal(e)), + } + } + Ok(Box::new(MongoMaintenanceClaim { + collection, + generation, + }) as Box) + }) + } + + fn check_maintenance_claim( + &self, + stale_after: Duration, + ) -> BoxFuture<'_, Result> { + Box::pin(async move { + let held = self + .maintenance() + .find_one(doc! { + "$nor": [ { "$expr": age_is_stale(&["heartbeat_at"], stale_after) } ], + }) + .await + .map_err(internal)?; + Ok(held.is_some()) + }) + } +} + +/// The `_id` of the one maintenance claim document. +const MAINTENANCE_CLAIM_ID: &str = "claim"; + +impl MongoEngine { + fn maintenance(&self) -> mongodb::Collection { + self.catalog_db.collection::(BACKUP_MAINTENANCE) + } +} + +/// The holder's handle on the maintenance claim document. +struct MongoMaintenanceClaim { + collection: mongodb::Collection, + generation: u64, +} + +impl MongoMaintenanceClaim { + fn owned_filter(&self) -> Document { + doc! { + "_id": MAINTENANCE_CLAIM_ID, + "owner_generation": generation_field(self.generation), + } + } +} + +impl MaintenanceClaim for MongoMaintenanceClaim { + fn generation(&self) -> u64 { + self.generation + } + + fn heartbeat(&self) -> BoxFuture<'_, Result> { + Box::pin(async move { + let result = self + .collection + .update_one( + self.owned_filter(), + vec![doc! { "$set": { "heartbeat_at": "$$NOW" } }], + ) + .await + .map_err(|e| StorageError::Internal(format!("maintenance heartbeat: {e}")))?; + Ok(if result.matched_count == 1 { + Heartbeat::Confirmed + } else { + Heartbeat::Lost + }) + }) + } + + fn release(&self) -> BoxFuture<'_, Result<(), StorageError>> { + Box::pin(async move { + self.collection + .delete_one(self.owned_filter()) + .await + .map_err(internal)?; + Ok(()) + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn fixture_source_table() -> serde_json::Value { + let manifest: serde_json::Value = serde_json::from_str(include_str!( + "../../core/testdata/extenddb-manifest/v1.json" + )) + .expect("fixture parses"); + manifest["source_table"].clone() + } + + fn row() -> BackupRow { + BackupRow { + backup_id: "01757500000123-0badcafe".to_owned(), + account_id: "123456789012".to_owned(), + table_name: "Music".to_owned(), + backup_arn: + "arn:aws:dynamodb:us-east-1:123456789012:table/Music/backup/01757500000123-0badcafe" + .to_owned(), + backup_name: "music-nightly".to_owned(), + status: BackupStatus::Creating, + store: StoreIdentity { + store_kind: StoreKind::S3, + store_root: "s3://bucket/prefix".to_owned(), + store_id: "store-1".to_owned(), + }, + size_bytes: 61828, + item_count: 100_000, + created_at: 1_757_500_000.123, + completed_at: None, + failure_reason: None, + source_table_details_json: fixture_source_table(), + owner_generation: 7, + heartbeat_at: 1_757_500_001.0, + } + } + + /// The document the claim writes reads back as the row it was built + /// from, the source table object included, with the times kept to the + /// millisecond. + #[test] + fn a_row_round_trips_through_its_document() { + let row = row(); + let mut doc = row_fields(&row, row.status).unwrap(); + doc.insert("account_id", &row.account_id); + doc.insert("backup_id", &row.backup_id); + doc.insert("owner_generation", generation_field(row.owner_generation)); + doc.insert("heartbeat_at", epoch_secs_datetime(row.heartbeat_at)); + let read = row_from_doc(&doc).unwrap(); + assert_eq!(read, row); + assert_eq!(read.source_table().unwrap().table_name, "Music"); + } + + #[test] + fn a_document_with_an_unknown_store_kind_or_status_is_internal() { + let row = row(); + let mut doc = row_fields(&row, row.status).unwrap(); + doc.insert("account_id", &row.account_id); + doc.insert("backup_id", &row.backup_id); + doc.insert("owner_generation", 7_i64); + doc.insert("heartbeat_at", epoch_secs_datetime(1.0)); + let mut bad_kind = doc.clone(); + bad_kind.insert("store_kind", "gcs"); + assert!(matches!( + row_from_doc(&bad_kind), + Err(StorageError::Internal(_)) + )); + let mut bad_status = doc; + bad_status.insert("status", "ACTIVE"); + assert!(matches!( + row_from_doc(&bad_status), + Err(StorageError::Internal(_)) + )); + } + + #[test] + fn only_a_connection_class_failure_leaves_a_write_unknown() { + let io: mongodb::error::Error = mongodb::error::ErrorKind::Io(std::sync::Arc::new( + std::io::Error::new(std::io::ErrorKind::ConnectionReset, "reset"), + )) + .into(); + assert!(outcome_unknown(&io)); + let command: mongodb::error::Error = mongodb::error::ErrorKind::Command( + bson::from_document(doc! { "code": 112, "codeName": "WriteConflict", "errmsg": "x" }) + .expect("a command error document"), + ) + .into(); + assert!(!outcome_unknown(&command)); + } + + /// The older-than filter reads the named column alone against the + /// server's clock and requires the column to be a date, so a row whose + /// `completed_at` was never stamped is not read as old as the epoch. + #[test] + fn the_older_than_filter_names_one_clock_column_and_requires_it_set() { + let heartbeat = older_than_filter(StaleClock::Heartbeat, Duration::from_secs(90)); + assert_eq!( + heartbeat.get_document("heartbeat_at").unwrap(), + &doc! { "$type": "date" } + ); + assert!(heartbeat.get("completed_at").is_none()); + let expr = heartbeat.get_document("$expr").unwrap().to_string(); + assert!(expr.contains("$heartbeat_at"), "{expr}"); + assert!(!expr.contains("completed_at"), "{expr}"); + assert!(expr.contains("$$NOW") && expr.contains("90000"), "{expr}"); + + let completed = older_than_filter(StaleClock::Completed, Duration::from_secs(90)); + assert_eq!( + completed.get_document("completed_at").unwrap(), + &doc! { "$type": "date" } + ); + assert!(completed.get("heartbeat_at").is_none()); + let expr = completed.get_document("$expr").unwrap().to_string(); + assert!(expr.contains("$completed_at"), "{expr}"); + assert!(!expr.contains("heartbeat_at"), "{expr}"); + } + + /// The dead-job filter selects, by the server's clock, RUNNING records + /// with a stale heartbeat and FAILED records of a CREATING table whose + /// `completed_at`, or heartbeat where none, is stale. + #[test] + fn the_dead_job_filter_reads_both_kinds_of_record_by_the_server_clock() { + let filter = dead_restore_job_filter(Duration::from_secs(300)); + let arms = filter.get_array("$or").expect("$or"); + assert_eq!(arms.len(), 2); + let running = arms[0].as_document().expect("arm"); + assert_eq!(running.get_str(RESTORE_JOB_STATUS), Ok("RUNNING")); + assert!(running.get("table_status").is_none()); + let expr = running.get_document("$expr").expect("$expr").to_string(); + assert!(expr.contains("$restore_job.heartbeat_at"), "{expr}"); + assert!(!expr.contains("completed_at"), "{expr}"); + assert!(expr.contains("$$NOW") && expr.contains("300000"), "{expr}"); + let failed = arms[1].as_document().expect("arm"); + assert_eq!(failed.get_str(RESTORE_JOB_STATUS), Ok("FAILED")); + assert_eq!(failed.get_str("table_status"), Ok("CREATING")); + let expr = failed.get_document("$expr").expect("$expr").to_string(); + let completed = expr + .find("$restore_job.completed_at") + .expect("completed_at"); + let heartbeat = expr + .find("$restore_job.heartbeat_at") + .expect("heartbeat_at"); + assert!(completed < heartbeat, "completed_at is read first: {expr}"); + assert!(expr.contains("$$NOW") && expr.contains("300000"), "{expr}"); + } + + #[test] + fn generations_round_trip_through_the_integer_field() { + let g = new_generation(); + let doc = doc! { "owner_generation": generation_field(g) }; + assert_eq!(generation_from_field(&doc, "owner_generation"), g); + assert_eq!(generation_from_field(&doc! {}, "owner_generation"), 0); + } + + /// A pipeline `$set` keeps `$$NOW` as the server's clock and wraps every + /// other value, the source table object included, so none is read as a + /// field path. + #[test] + fn a_pipeline_set_wraps_every_value_but_now_in_a_literal() { + let source = bson::to_bson(&fixture_source_table()).unwrap(); + let set = literal_set(doc! { + "status": "AVAILABLE", + "completed_at": "$$NOW", + "owner_generation": 7_i64, + "source_table_details_json": source.clone(), + }); + assert_eq!(set.get_str("completed_at"), Ok("$$NOW")); + assert_eq!( + set.get_document("status").unwrap(), + &doc! { "$literal": "AVAILABLE" } + ); + assert_eq!( + set.get_document("owner_generation").unwrap(), + &doc! { "$literal": 7_i64 } + ); + assert_eq!( + set.get_document("source_table_details_json").unwrap(), + &doc! { "$literal": source } + ); + } +} diff --git a/crates/storage-mongodb/src/backup_engine.rs b/crates/storage-mongodb/src/backup_engine.rs index b33a83d6a..100223ce5 100644 --- a/crates/storage-mongodb/src/backup_engine.rs +++ b/crates/storage-mongodb/src/backup_engine.rs @@ -1,12 +1,12 @@ // Copyright 2026 ExtendDB contributors // SPDX-License-Identifier: Apache-2.0 -//! `BackupEngine` implementation for `MongoDB`. +//! The in-catalog backup format on `MongoDB`: its `LegacyBackupReader`. //! //! Backups are stored as one MongoDB collection per backup, plus a `backups` //! metadata collection in the catalog. `CreateBackup` uses MongoDB's //! server-side aggregation `$out` stage to clone the source data collection -//! into `_backup_` in the data database — no per-item traffic +//! into `_backup_` in the data database, with no per-item traffic //! between the driver and the server. `RestoreTableFromBackup` uses the same //! stage in reverse, in a task detached from the request. `DeleteBackup` drops //! the collection. @@ -15,48 +15,72 @@ //! from that id so the `backup_arn` (which contains slashes and colons) never //! appears in a collection name. //! -//! # Claims, restore targets, and liveness +//! # The restore job record, DeleteBackup and the claim //! -//! MongoDB has no transaction spanning the claim and the target's collections, -//! so a restore and a DeleteBackup exclude each other through markers: +//! A restore registers through the job record on its target's `tables` +//! document (`restore_job`: `status`, `kind = legacy`, `owner_generation`, +//! `heartbeat_at`, [`crate::backup_catalog`]), beside the provenance fields +//! `DescribeTable` reports. The registration is one session transaction on +//! the catalog database: it `$inc`s `lock_seq` on the backup's `backups` +//! document while that document is AVAILABLE, checks that no CREATING target +//! is restoring the backup already, and inserts the CREATING target. The +//! legacy DeleteBackup takes the same lock in its own transaction before it +//! re-checks for a CREATING target and writes DELETING (on the delete's side +//! the DELETING write is itself a write to the document, so the `$inc` there +//! is what makes a refused delete, `Status` or `InUse`, conflict too), so +//! the two order wholly one before the other: a DeleteBackup that wins leaves the +//! registration a document that is not AVAILABLE (`Backup not found`, no +//! target), and a registration that wins makes the DeleteBackup `BackupInUse`. +//! From then on the CREATING target is what keeps DeleteBackup and other +//! restores of the backup out, until the target is ACTIVE, as the service +//! answers. Both in-use checks read at the transaction read concern +//! (`transaction_read_concern`, `snapshot` by default), which is +//! majority-committed data, while `DescribeTable` reads the catalog at +//! `local`: a client whose `DescribeTable` has just shown the target ACTIVE +//! can be answered `BackupInUse` by a RestoreTableFromBackup or DeleteBackup +//! issued within the majority-commit lag (the journal flush, and any +//! concurrent batch write still holding an earlier oplog slot), and the +//! retry succeeds. That lock replaces the claim markers earlier releases wrote on +//! the `backups` document (`restore_in_progress`, `delete_in_progress` and +//! their heartbeats): nothing writes or reads them now, and a marker a dead +//! process left no longer blocks the backup until a sweep clears it. Once +//! DeleteBackup has written DELETING the backup never reads AVAILABLE again: +//! the drop that follows may have run on the server even if the request +//! stops before its reply, and a later DeleteBackup of a DELETING backup +//! resumes the drop and the DELETED write instead of answering that the +//! backup is in use; `describe` and `list` leave DELETING out as they leave +//! DELETED out. //! -//! - a restore claims the backup document (`restore_in_progress`), checks that -//! no CREATING target is restoring the backup already, inserts its CREATING -//! target with provenance (`restore_source_backup_arn`, -//! `restore_operation_id`), then releases the claim; from then on the -//! CREATING target is what keeps DeleteBackup and other restores of the -//! backup out (`BackupInUse` until the target is ACTIVE, as the service -//! answers). A claim another restore released before inserting anything is -//! retried for up to [`RESTORE_CLAIM_RETRY_FOR`]. The exclusion is bounded -//! by the heartbeat: a restore that stalls between its claim and its insert -//! for longer than [`BACKUP_HEARTBEAT_STALE_AFTER`] has its claim swept, a -//! second restore of the backup can then claim and insert, and when the -//! first resumes it inserts too (its release of the claim returns `false` -//! and it goes on while the backup is AVAILABLE), leaving two CREATING -//! targets of one backup that both finish. The other three backends have no -//! such window: PostgreSQL and SQLite register a restore in one transaction, -//! and Cassandra's registration is a lightweight transaction on the -//! backup's row conditioned on no other registration; -//! - DeleteBackup claims the same document (`delete_in_progress`) under the -//! inverse predicate, re-checks for a CREATING target, and only then drops -//! the backup's data. Once the drop may have started its claim is never -//! released; if DeleteBackup stops after that, the sweep finishes it. -//! -//! Every marker carries its owner's operation id and a heartbeat timestamp -//! (`_heartbeat` on a claim, `restore_heartbeat_at` on a target) that -//! a companion task refreshes at once and then every -//! [`BACKUP_HEARTBEAT_INTERVAL`] with the server's clock (`$$NOW`); a restore -//! also refreshes its target's right after inserting it, replacing the insert's -//! local-clock value. [`MongoEngine::sweep_stale_backup_claims`], -//! run at startup and periodically by the GSI backfill worker, removes only -//! markers whose heartbeat is older than [`BACKUP_HEARTBEAT_STALE_AFTER`], so -//! several processes can share one catalog without clearing each other's live -//! work. A restore target stops heartbeating once its base copy is done and -//! `restore_backfill_pending` is set; from then the backfill worker owns it and -//! finishes it after a restart, so the sweep never touches it. - -use std::collections::HashMap; -use std::sync::Arc; +//! The restore's claim is the job record: the target is inserted RUNNING +//! under a fresh `owner_generation`, and the copy task refreshes +//! `heartbeat_at` with the server's clock (`$$NOW`) at once and then every +//! [`JOB_HEARTBEAT_INTERVAL`], the one constant every backend's heartbeat +//! loop reads and the stale floor is computed from (`[backup] +//! heartbeat_interval_secs` reaches the backends in a later release); a +//! refresh that matches other than exactly one document, or fails, means the +//! claim is lost and the copy stops without touching the target (the FAILED +//! write and the removal, each conditioned on the generation, are what +//! follow); the `$out` already sent runs to its end on the server, a write +//! stage the server does not interrupt for a client's disconnect, and then +//! fails rather than recreate the data collection the removal dropped, since +//! `$out` refuses to replace a target whose indexes changed while it ran (see +//! `copy_restore`), so a lost heartbeat leaves no orphan collection. The +//! copy's hand-over to the GSI backfill worker marks the +//! record DONE in the same write, conditioned on the generation and on +//! RUNNING; a failure marks it FAILED under the same condition and removes +//! the target only when that matched. A target whose record is RUNNING with +//! a stale heartbeat has a dead owner: the reconciler +//! (`BackupCatalog::list_dead_restore_jobs`) fails it and removes it; a +//! FAILED record whose target is still CREATING (the removal did not land) +//! is listed once the record is stale, and the reconciler removes the target +//! with no second FAILED write. The removal deletes the target's `tables` +//! document first and then drops its data and index collections; once the +//! document is gone the target is removed (the name and the backup are +//! free, and no record is left for a later cycle), so a drop that fails +//! after it is logged with the collections' names and leaves them in the +//! data database, where nothing in the catalog points at them and no sweep +//! removes them yet; the log line names them for an operator to drop. + use std::time::{Duration, Instant}; use futures::TryStreamExt; @@ -64,62 +88,80 @@ use futures::future::BoxFuture; use mongodb::bson::{self, Bson, Document, doc}; use serde::Serialize; use serde::de::DeserializeOwned; -use tokio::sync::{OwnedSemaphorePermit, Semaphore}; use extenddb_core::types::{ AttributeDefinition, BackupDescription, BackupDetails, BackupSummary, GsiInput, KeySchemaElement, LsiInput, Projection, ProvisionedThroughput, SourceTableDetails, StreamSpecification, TableDescription, TimeToLiveDescription, TimeToLiveStatus, }; +use extenddb_storage::backup_catalog::{ + AbandonedLegacyBackup, BackupCatalog, JOB_HEARTBEAT_INTERVAL, JobStatus, RestoreKind, + RestoreTargetKey, StoreIdentity, new_generation, +}; use extenddb_storage::backup_definition::{BACKUP_DEFINITION_VERSION, BackupTableDefinition}; use extenddb_storage::error::StorageError; -use extenddb_storage::{BackupEngine, BackupRestoreSource, RestoreRequest, util}; +use extenddb_storage::{BackupRestoreSource, JobPermit, LegacyBackupReader, RestoreRequest, util}; use crate::MongoEngine; +use crate::backup_catalog::{ + LOCK_SEQ, RESTORE_JOB, RESTORE_JOB_COMPLETED_AT, RESTORE_JOB_FAILURE_REASON, + RESTORE_JOB_GENERATION, RESTORE_JOB_HEARTBEAT, RESTORE_JOB_STATUS, backup_in_use, + generation_field, +}; use crate::data::data_collection_name; - -/// How often a live restore or DeleteBackup refreshes its heartbeats. -pub(crate) const BACKUP_HEARTBEAT_INTERVAL: Duration = Duration::from_secs(15); -/// A claim or restore target whose heartbeat is older than this belongs to an -/// operation that stopped (its process died); the sweep removes it. Four -/// missed refreshes, so a slow catalog write does not make live work look dead. -pub(crate) const BACKUP_HEARTBEAT_STALE_AFTER: Duration = Duration::from_secs(60); - -pub(crate) const RESTORE_CLAIM: &str = "restore_in_progress"; -pub(crate) const RESTORE_CLAIM_HEARTBEAT: &str = "restore_in_progress_heartbeat"; -pub(crate) const DELETE_CLAIM: &str = "delete_in_progress"; -pub(crate) const DELETE_CLAIM_HEARTBEAT: &str = "delete_in_progress_heartbeat"; -pub(crate) const TARGET_OPERATION: &str = "restore_operation_id"; -pub(crate) const TARGET_HEARTBEAT: &str = "restore_heartbeat_at"; +use crate::data_engine::{CommitOutcome, TRANSIENT_RETRY_ATTEMPTS, TxErr, backoff_sleep}; /// The definition version stored on a backup document written by this build. /// Backup documents written before it carry the same fields without the /// marker and are read as version 1. const DEFINITION_VERSION_FIELD: &str = "definition_version"; -/// Heartbeat timing for backup claims and restore targets. Production uses -/// the defaults; tests shorten both so liveness can be observed in seconds. +/// How long a request waits for its detached task to register the CREATING +/// target before it answers `Internal`. The bound is what turns a connection +/// pool that has stopped handing out connections into a 500 instead of a +/// request that never returns (the driver does not enforce +/// `waitQueueTimeoutMS`, so a checkout waits for as long as every connection +/// stays checked out). The task goes on: a request that hit the bound may +/// leave a CREATING target that appears after its error, holds the backup in +/// use (`BackupInUseException` to DeleteBackup) and takes the name until the +/// copy ends, and DescribeTable is where a client finds it. +pub(crate) const RESTORE_REGISTRATION_TIMEOUT: Duration = Duration::from_secs(60); + +/// Heartbeat timing of a restore's job record. Production uses the default, +/// [`JOB_HEARTBEAT_INTERVAL`]; tests shorten it so liveness can be observed +/// in seconds. #[derive(Debug, Clone)] pub(crate) struct BackupLiveness { pub(crate) heartbeat_interval: Duration, - pub(crate) stale_after: Duration, - /// How long a restore waits for its account slot and a server slot - /// together before it is refused with `LimitExceeded` - /// ([`RESTORE_SLOT_WAIT`] in production; tests shorten it). - pub(crate) slot_wait: Duration, + /// Test only: when set, a restore's task waits for one permit after the + /// name check and before it registers its target, so a test can order a + /// registration after a DeleteBackup of the same backup. + #[cfg(test)] + pub(crate) registration_gate: Option>, /// Test only: when set, a restore's copy task waits for one permit after /// registering its target and before its base copy, so a test can observe /// the CREATING window deterministically. #[cfg(test)] pub(crate) copy_gate: Option>, - /// Test only: what a DeleteBackup or sweep does right after it dropped a - /// backup's data collection. + /// Test only: what a DeleteBackup does right after it dropped a backup's + /// data collection. #[cfg(test)] pub(crate) after_backup_drop: Option, + /// Test only: when set, `remove_restore_target` treats the reply to the + /// target's collection drops as lost (an error after the drops ran). + #[cfg(test)] + pub(crate) target_drop_fails: bool, /// Test only: how far this process's clock is behind the server's, applied /// to the one heartbeat written with the local clock (the target insert). #[cfg(test)] pub(crate) local_clock_behind: Duration, + /// Test only: when set, the restore heartbeat fails at its next tick once + /// the test has armed the cell (an injected error, as a network fault on + /// the heartbeat's connection would be), and records that it did: + /// [`HeartbeatFault::ARMED`] is the test's request, + /// [`HeartbeatFault::INJECTED`] the loop's answer. + #[cfg(test)] + pub(crate) heartbeat_fault: Option>, } /// Test only: the two ways a backup drop can end after the server ran it. @@ -133,128 +175,40 @@ pub(crate) enum AfterBackupDrop { Hang, } +/// Test only: the states of [`BackupLiveness::heartbeat_fault`]. +#[cfg(test)] +pub(crate) struct HeartbeatFault; + +#[cfg(test)] +impl HeartbeatFault { + /// Nothing requested. + pub(crate) const IDLE: u8 = 0; + /// The next heartbeat tick fails. + pub(crate) const ARMED: u8 = 1; + /// A heartbeat failed as requested. + pub(crate) const INJECTED: u8 = 2; +} + impl Default for BackupLiveness { fn default() -> Self { Self { - heartbeat_interval: BACKUP_HEARTBEAT_INTERVAL, - stale_after: BACKUP_HEARTBEAT_STALE_AFTER, - slot_wait: RESTORE_SLOT_WAIT, + heartbeat_interval: JOB_HEARTBEAT_INTERVAL, + #[cfg(test)] + registration_gate: None, #[cfg(test)] copy_gate: None, #[cfg(test)] after_backup_drop: None, #[cfg(test)] + target_drop_fails: false, + #[cfg(test)] local_clock_behind: Duration::ZERO, + #[cfg(test)] + heartbeat_fault: None, } } } -/// Restores one account may have in flight at once in this process. A -/// restore beyond the limit waits for one of its own account's slots, so one -/// account cannot hold every other account's restores back. Same figure as -/// the PostgreSQL backend. -const RESTORE_SLOTS_PER_ACCOUNT: usize = 4; - -/// Upper limit on [`restore_server_slot_count`]. -const RESTORE_SERVER_SLOTS_MAX: usize = 16; - -/// How long a restore waits for its account slot and a server slot, -/// together, before it is refused with `LimitExceeded`. -pub(crate) const RESTORE_SLOT_WAIT: Duration = Duration::from_secs(30); - -/// How long a request waits for its detached task to register the CREATING -/// target before it answers `Internal`. The wait for a slot -/// ([`RESTORE_SLOT_WAIT`]) runs inside the task, so it counts against this -/// bound too. The task goes on; the bound is what turns a connection pool -/// that has stopped handing out connections into a 500 instead of a request -/// that never returns (the driver enforces no checkout timeout, see -/// [`crate::effective_pool_size`]). -pub(crate) const RESTORE_REGISTRATION_TIMEOUT: Duration = Duration::from_secs(60); - -/// Restores the whole process may have in flight at once, across accounts: -/// `min(16, pool_size / 2)`, at least 1, where `pool_size` is the driver pool -/// of the engine's client (its `maxPoolSize`, default 10). -/// -/// A restore's detached task holds up to one pooled connection at a time -/// (the `$out` copy, a catalog write) and its heartbeat task takes one for -/// each refresh, so without a process-wide bound enough concurrent restores -/// can hold every pooled connection and every other request waits on a -/// checkout the driver never times out. Half the pool stays free for them. -/// A restore takes its account slot first, then one of these; the two waits -/// share one deadline of [`BackupLiveness::slot_wait`]. -pub(crate) fn restore_server_slot_count(pool_size: u32) -> usize { - usize::try_from(pool_size / 2) - .unwrap_or(usize::MAX) - .clamp(1, RESTORE_SERVER_SLOTS_MAX) -} - -/// The two slots a running restore holds until its copy ends. -pub(crate) struct RestorePermits { - _account: OwnedSemaphorePermit, - _server: OwnedSemaphorePermit, -} - -/// Take an account slot, then a server slot, waiting at most `wait` for -/// both together. Running out of time on either is `LimitExceeded`, with a -/// message naming the limit that was full. -async fn acquire_restore_permits( - account: Arc, - server: Arc, - server_slots: usize, - wait: Duration, -) -> Result { - let deadline = tokio::time::Instant::now() + wait; - let closed = - |e: tokio::sync::AcquireError| StorageError::Internal(format!("restore slot: {e}")); - let account_permit = tokio::time::timeout_at(deadline, account.acquire_owned()) - .await - .map_err(|_| { - StorageError::LimitExceeded(format!( - "Too many restores are in progress for this account (at most \ - {RESTORE_SLOTS_PER_ACCOUNT} at once); retry later" - )) - })? - .map_err(closed)?; - let server_permit = tokio::time::timeout_at(deadline, server.acquire_owned()) - .await - .map_err(|_| { - StorageError::LimitExceeded(format!( - "Too many restores are in progress on this server (at most {server_slots} at \ - once); retry later" - )) - })? - .map_err(closed)?; - Ok(RestorePermits { - _account: account_permit, - _server: server_permit, - }) -} - -/// Per-account restore slots of one engine (shared by its clones, so one -/// per server process). An entry nobody holds (no permit, no waiter) is -/// pruned the next time any account takes a slot, so the map does not grow -/// with every account that ever restored. -#[derive(Clone, Default)] -pub(crate) struct RestoreAccountSlots(Arc>>>); - -impl RestoreAccountSlots { - /// The slot semaphore for `account_id`. - pub(crate) fn for_account(&self, account_id: &str) -> Arc { - let mut slots = self - .0 - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - // A permit and a waiter each hold a clone, so a count of one means - // the account has nothing in flight. - slots.retain(|_, s| Arc::strong_count(s) > 1); - Arc::clone( - slots - .entry(account_id.to_owned()) - .or_insert_with(|| Arc::new(Semaphore::new(RESTORE_SLOTS_PER_ACCOUNT))), - ) - } -} - /// Return the MongoDB collection name that holds items for a given backup. /// /// The collection lives in the data database. The name is derived from the @@ -368,33 +322,11 @@ fn restore_gsi_throughput( }) } -/// A restore first claims backup metadata, then transfers protection to the -/// provenance-bearing CREATING target. DeleteBackup claims the same metadata -/// before dropping data, so neither operation can pass the other between its -/// check and destructive action. The heartbeat fields do not take part in -/// either predicate: a stale claim still excludes the other operation until -/// the sweep removes it. -fn restore_claim_filter(account_id: &str, backup_arn: &str) -> Document { - doc! { - "_id": backup_arn, - "account_id": account_id, - "backup_status": "AVAILABLE", - "restore_in_progress": { "$exists": false }, - "delete_in_progress": { "$exists": false }, - } -} - -fn delete_claim_filter(account_id: &str, backup_arn: &str) -> Document { - doc! { - "_id": backup_arn, - "account_id": account_id, - "backup_status": "AVAILABLE", - "restore_in_progress": { "$exists": false }, - "delete_in_progress": { "$exists": false }, - } -} - -fn restoring_target_filter(account_id: &str, backup_arn: &str) -> Document { +/// The filter of a CREATING restore target registered from `backup_arn` in +/// `account_id`: the in-use rule, which reads the target's status, not the +/// job's, so it also covers the window between the job's DONE write and the +/// backfill worker's ACTIVE flip. +pub(crate) fn restoring_target_filter(account_id: &str, backup_arn: &str) -> Document { doc! { "_id.account_id": account_id, "restore_source_backup_arn": backup_arn, @@ -402,631 +334,139 @@ fn restoring_target_filter(account_id: &str, backup_arn: &str) -> Document { } } -/// How long a restore retries a backup claim held by another restore. -pub(crate) const RESTORE_CLAIM_RETRY_FOR: Duration = Duration::from_secs(2); - -/// Why a restore's claim found no backup document to claim, from a re-read of -/// the document (`None`: no document for this ARN and account). -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum RestoreClaimMiss { - /// Missing, or not AVAILABLE. - NotFound, - /// AVAILABLE and claimed by a DeleteBackup. - DeleteClaimed, - /// AVAILABLE and claimed by another restore only. - RestoreClaimed, - /// AVAILABLE and unclaimed: the other claim was released meanwhile. - Free, -} - -fn restore_claim_miss(doc: Option<&Document>) -> RestoreClaimMiss { - let Some(doc) = doc else { - return RestoreClaimMiss::NotFound; - }; - if doc.get_str("backup_status") != Ok("AVAILABLE") { - RestoreClaimMiss::NotFound - } else if doc.contains_key(DELETE_CLAIM) { - RestoreClaimMiss::DeleteClaimed - } else if doc.contains_key(RESTORE_CLAIM) { - RestoreClaimMiss::RestoreClaimed - } else { - RestoreClaimMiss::Free - } -} - -/// The error for a DeleteBackup whose claim found no document to claim. -/// `restoring`: a CREATING restore target of the backup exists; `miss`: the -/// re-read backup document, classified. A backup another DeleteBackup has -/// claimed is in use (it reads AVAILABLE until that delete finishes), not -/// missing. -fn delete_claim_miss_error( - restoring: bool, - miss: RestoreClaimMiss, - backup_arn: &str, -) -> StorageError { - match miss { - RestoreClaimMiss::DeleteClaimed => { - StorageError::BackupInUse(format!("Backup is being deleted: {backup_arn}")) - } - RestoreClaimMiss::RestoreClaimed => StorageError::BackupInUse(format!( - "Backup is being used to restore another table: {backup_arn}" - )), - RestoreClaimMiss::NotFound | RestoreClaimMiss::Free if restoring => { - StorageError::BackupInUse(format!( - "Backup is being used to restore another table: {backup_arn}" - )) - } - RestoreClaimMiss::NotFound | RestoreClaimMiss::Free => { - StorageError::Validation(format!("Backup not found: {backup_arn}")) - } - } -} - -/// Install `field = operation_id` and its heartbeat (server clock) in one -/// update. A pipeline update, so `$$NOW` is the server's time and processes -/// with skewed clocks agree on staleness. -fn claim_update(field: &str, heartbeat_field: &str, operation_id: &str) -> Vec { - let mut set = Document::new(); - set.insert(field, doc! { "$literal": operation_id }); - set.insert(heartbeat_field, "$$NOW"); - vec![doc! { "$set": set }] -} - -/// `$expr` that is true when `heartbeat_field` is missing (a marker written -/// before heartbeats existed) or older than `stale_after` by the server clock. -fn heartbeat_is_stale(heartbeat_field: &str, stale_after: Duration) -> Document { +/// `$expr` that is true when the first of `fields` that is set is older than +/// `stale_after` by the server clock; a document with none of them set is as +/// old as the epoch. The reconciler's two ages: a RUNNING record's heartbeat, +/// and a FAILED record's `completed_at` or, where none was stamped, its +/// heartbeat. +pub(crate) fn age_is_stale(fields: &[&str], stale_after: Duration) -> Document { let stale_ms = i64::try_from(stale_after.as_millis()).unwrap_or(i64::MAX); + let age = fields.iter().rev().fold( + Bson::DateTime(bson::DateTime::from_millis(0)), + |fallback, field| Bson::Document(doc! { "$ifNull": [ format!("${field}"), fallback ] }), + ); doc! { "$lt": [ - { "$ifNull": [ format!("${heartbeat_field}"), bson::DateTime::from_millis(0) ] }, + age, { "$subtract": [ "$$NOW", stale_ms ] }, ] } } -/// Filter for a stale claim of `field` on backup documents. -fn stale_claim_filter(field: &str, heartbeat_field: &str, stale_after: Duration) -> Document { - let mut filter = Document::new(); - filter.insert(field, doc! { "$exists": true }); - filter.insert("$expr", heartbeat_is_stale(heartbeat_field, stale_after)); - filter -} - -/// Filter for an abandoned restore target: CREATING, carrying restore -/// provenance, still before its base copy finished (no backfill flag; after -/// that the backfill worker owns it), and not heartbeating. -fn stale_target_filter(stale_after: Duration) -> Document { - doc! { - "table_status": "CREATING", - "restore_source_backup_arn": { "$exists": true }, - "restore_backfill_pending": { "$exists": false }, - "$expr": heartbeat_is_stale(TARGET_HEARTBEAT, stale_after), - } +/// How the job of a registration reads its backup, and what the record +/// carries for it beyond the members every record has. +pub(crate) enum RestoreRecord<'a> { + /// The in-catalog backup: `kind = legacy`, nothing more. + Legacy, + /// A backup in the store: `kind = manifest`, the store the backup row + /// names and the claimant's spool. + Manifest { + store: &'a StoreIdentity, + spool_host: Option<&'a str>, + }, } -fn claim_heartbeat_field(field: &'static str) -> &'static str { - if field == RESTORE_CLAIM { - RESTORE_CLAIM_HEARTBEAT - } else { - DELETE_CLAIM_HEARTBEAT - } -} - -/// Restore provenance written into the target's first catalog document. -pub(crate) struct RestoreProvenance<'a> { +/// What a restore's registration writes into the target's first catalog +/// document: the provenance `DescribeTable` reports and the job record. +pub(crate) struct RestoreRegistration<'a> { pub(crate) backup_arn: &'a str, pub(crate) source_table_arn: &'a str, pub(crate) restore_date_time: bson::DateTime, - pub(crate) operation_id: &'a str, - /// The target's first heartbeat. An insert cannot use `$$NOW`, so this is - /// the inserting process's clock; `register_restore` overwrites it with - /// the server's clock as soon as the insert returns. + /// The claim's fencing token, written as `restore_job.owner_generation`. + pub(crate) owner_generation: u64, + /// The record's first heartbeat. An insert cannot use `$$NOW`, so this is + /// the inserting process's clock; the copy task's first refresh replaces + /// it with the server's as soon as the task starts. A manifest record + /// has it rewritten with the server's clock inside the registration's + /// transaction ([`Self::stamp_server_clock`]). pub(crate) first_heartbeat: bson::DateTime, - /// Notified once the target document is inserted, while the rest of the - /// target (collections, indexes) is still being created. - pub(crate) inserted: Option<&'a tokio::sync::Notify>, + pub(crate) record: RestoreRecord<'a>, + /// The session whose transaction the target's insert runs in and which + /// the insert commits; `None` inserts and commits nothing beyond the + /// document itself. + pub(crate) session: Option<&'a mut mongodb::ClientSession>, } -impl RestoreProvenance<'_> { +impl RestoreRegistration<'_> { pub(crate) fn insert_into(&self, table_doc: &mut Document) { table_doc.insert("restore_source_backup_arn", self.backup_arn); table_doc.insert("restore_source_table_arn", self.source_table_arn); table_doc.insert("restore_date_time", self.restore_date_time); - table_doc.insert(TARGET_OPERATION, self.operation_id); - table_doc.insert(TARGET_HEARTBEAT, self.first_heartbeat); - } -} - -struct BackupMetadataClaim { - backups: mongodb::Collection, - account_id: String, - backup_arn: String, - field: &'static str, - operation_id: String, - armed: bool, -} - -impl BackupMetadataClaim { - fn new( - backups: mongodb::Collection, - account_id: &str, - backup_arn: &str, - field: &'static str, - operation_id: String, - ) -> Self { - Self { - backups, - account_id: account_id.to_owned(), - backup_arn: backup_arn.to_owned(), - field, - operation_id, - armed: true, - } - } - - fn filter(&self) -> Document { - let mut filter = doc! { - "_id": &self.backup_arn, - "account_id": &self.account_id, + let kind = match self.record { + RestoreRecord::Legacy => RestoreKind::Legacy, + RestoreRecord::Manifest { .. } => RestoreKind::Manifest, }; - filter.insert(self.field, &self.operation_id); - filter - } - - fn unset_update(field: &'static str) -> Document { - let mut unset = Document::new(); - unset.insert(field, ""); - unset.insert(claim_heartbeat_field(field), ""); - doc! { "$unset": unset } - } - - /// Release the claim. `Ok(false)` means it was no longer ours to release: - /// a sweep removed it because its heartbeat had lapsed. - async fn release(&mut self) -> Result { - if !self.armed { - return Ok(true); - } - let result = self - .backups - .update_one(self.filter(), Self::unset_update(self.field)) - .await - .map_err(internal)?; - self.armed = false; - Ok(result.matched_count == 1) - } - - fn disarm(&mut self) { - self.armed = false; - } -} - -impl Drop for BackupMetadataClaim { - /// Release a claim whose operation was dropped before its destructive - /// step (a DeleteBackup request cancelled between its claim and its drop; - /// DeleteBackup disarms the claim before dropping, so this never runs - /// after the data may be gone). This runs only when a Tokio runtime is - /// current and the spawned update can still reach the catalog; a process - /// exit, a drop outside a runtime, or a failed update leaves the marker in - /// place. The backstop for all of those is the heartbeat sweep - /// ([`MongoEngine::sweep_stale_backup_claims`]): the companion heartbeat - /// stops with the operation, and once the marker is older than - /// [`BACKUP_HEARTBEAT_STALE_AFTER`] the sweep clears a restore claim and - /// finishes the delete of a delete claim. - fn drop(&mut self) { - if !self.armed { - return; + let mut job = doc! { + "status": JobStatus::Running.as_str(), + "kind": kind.as_str(), + "owner_generation": generation_field(self.owner_generation), + "heartbeat_at": self.first_heartbeat, + // Set once the target's collections and index metadata exist + // (`create_table_impl_inner`); a resume refuses an unmarked + // record. + "ddl_done": false, + }; + if let RestoreRecord::Manifest { store, spool_host } = &self.record { + job.insert("store_kind", store.store_kind.as_str()); + job.insert("store_root", &store.store_root); + job.insert("store_id", &store.store_id); + if let Some(spool_host) = spool_host { + job.insert("spool_host", *spool_host); + } } - let backups = self.backups.clone(); - let filter = self.filter(); - let update = Self::unset_update(self.field); - let backup_arn = self.backup_arn.clone(); - if let Ok(runtime) = tokio::runtime::Handle::try_current() { - let _cleanup = runtime.spawn(async move { - if let Err(error) = backups.update_one(filter, update).await { - tracing::error!( - "could not clear cancelled backup metadata claim for {backup_arn}: {error}" - ); - } - }); + table_doc.insert(RESTORE_JOB, job); + } + + /// Rewrite the record's first heartbeat with the server's clock, inside + /// the registration's transaction: a manifest record is heartbeated by + /// the engine's timer, whose first beat comes one period after the + /// registration, so the record must not start on the inserting + /// process's clock. The driver's error is returned as it is, so a write + /// conflict keeps its transient label for the caller's retry. Nothing + /// for a legacy record, whose copy task beats at once. + pub(crate) async fn stamp_server_clock( + &mut self, + tables_coll: &mongodb::Collection, + table_id: &str, + ) -> Result<(), mongodb::error::Error> { + if !matches!(self.record, RestoreRecord::Manifest { .. }) { + return Ok(()); } - } -} - -/// The companion heartbeat of one restore or DeleteBackup operation. Aborted -/// when dropped, so the heartbeat stops exactly when its owner does. -pub(crate) struct Heartbeat(tokio::task::JoinHandle<()>); - -impl Drop for Heartbeat { - fn drop(&mut self) { - self.0.abort(); - } -} - -/// The updates that refresh every marker `operation_id` owns on `backup_arn` -/// (either claim) and on the restore target, if any, as (collection, marker, -/// filter, pipeline update). Each matches only the owner's marker, so a -/// refresh never revives a marker a sweep already removed. -fn heartbeat_updates( - operation_id: &str, - account_id: &str, - backup_arn: &str, - target_table: Option<&str>, -) -> Vec<(&'static str, &'static str, Document, Vec)> { - let mut updates = Vec::new(); - for (field, heartbeat_field) in [ - (RESTORE_CLAIM, RESTORE_CLAIM_HEARTBEAT), - (DELETE_CLAIM, DELETE_CLAIM_HEARTBEAT), - ] { - let mut filter = doc! { "_id": backup_arn, "account_id": account_id }; - filter.insert(field, operation_id); - let mut set = Document::new(); - set.insert(heartbeat_field, "$$NOW"); - updates.push(("backups", field, filter, vec![doc! { "$set": set }])); - } - if let Some(table_name) = target_table { - let mut filter = doc! { - "_id": { "account_id": account_id, "table_name": table_name }, - "table_status": "CREATING", + let Some(session) = self.session.as_deref_mut() else { + return Ok(()); }; - filter.insert(TARGET_OPERATION, operation_id); - let mut set = Document::new(); - set.insert(TARGET_HEARTBEAT, "$$NOW"); - updates.push(( - "tables", - TARGET_HEARTBEAT, - filter, - vec![doc! { "$set": set }], - )); - } - updates -} - -/// Run `run` on every item, also after one fails, and return every error. -async fn try_each(items: Vec, mut run: impl FnMut(T) -> Fut) -> Vec -where - Fut: std::future::Future>, -{ - let mut errors = Vec::new(); - for item in items { - if let Err(error) = run(item).await { - errors.push(error); - } + tables_coll + .update_one( + doc! { + "table_id": table_id, + RESTORE_JOB_GENERATION: generation_field(self.owner_generation), + }, + vec![doc! { "$set": { RESTORE_JOB_HEARTBEAT: "$$NOW" } }], + ) + .session(session) + .await + .map(|_| ()) } - errors } -/// Refresh every marker `operation_id` owns (see [`heartbeat_updates`]). A -/// failed update does not stop the others; every failure is returned. -async fn refresh_heartbeats( - catalog: &mongodb::Database, - operation_id: &str, - account_id: &str, - backup_arn: &str, - target_table: Option<&str>, -) -> Vec { - let updates = heartbeat_updates(operation_id, account_id, backup_arn, target_table); - try_each(updates, |(collection, marker, filter, update)| { - let collection_handle = catalog.collection::(collection); - async move { - collection_handle - .update_one(filter, update) - .await - .map(|_| ()) - .map_err(|e| StorageError::Internal(format!("{collection}.{marker}: {e}"))) - } - }) - .await -} - -/// What one sweep removed, by name, for the log. -#[derive(Debug, Default, Clone, PartialEq, Eq)] -pub struct BackupSweep { - /// Backup ARNs whose stale restore claim was cleared. - pub restore_claims_cleared: Vec, - /// `/
    ` of abandoned restore targets removed. - pub restore_targets_removed: Vec, - /// Backup ARNs whose interrupted DeleteBackup the sweep finished. - pub delete_claims_finished: Vec, - /// Backup ARNs whose stale delete claim the sweep took over but left held, - /// because a restore target from the backup appeared after its first - /// check; a later sweep finishes the delete. - pub delete_claims_waiting: Vec, +/// Why a legacy registration or DeleteBackup found no AVAILABLE backup +/// document to lock, from a re-read of the document. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum LockMiss { + /// Missing, another account's, DELETED, or in a state no release writes. + NotFound, + /// A DeleteBackup has written DELETING and is dropping the data. + Deleting, } -impl BackupSweep { - #[must_use] - pub fn is_empty(&self) -> bool { - self.restore_claims_cleared.is_empty() - && self.restore_targets_removed.is_empty() - && self.delete_claims_finished.is_empty() - && self.delete_claims_waiting.is_empty() - } -} - -impl std::fmt::Display for BackupSweep { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let parts = [ - ( - "cleared abandoned restore claim(s) on", - &self.restore_claims_cleared, - ), - ( - "removed abandoned restore target(s)", - &self.restore_targets_removed, - ), - ( - "finished interrupted DeleteBackup(s) of", - &self.delete_claims_finished, - ), - ( - "left interrupted DeleteBackup(s) to a later sweep (a restore target exists) of", - &self.delete_claims_waiting, - ), - ]; - let mut first = true; - for (label, names) in parts { - if names.is_empty() { - continue; - } - if !first { - f.write_str("; ")?; - } - first = false; - write!(f, "{label} [{}]", names.join(", "))?; - } - Ok(()) +fn lock_miss(doc: Option<&Document>) -> LockMiss { + match doc.and_then(|d| d.get_str("backup_status").ok()) { + Some("DELETING") => LockMiss::Deleting, + _ => LockMiss::NotFound, } } impl MongoEngine { - /// Start the heartbeat of `operation_id`: at once and then every - /// `heartbeat_interval` it refreshes the operation's claim on `backup_arn` - /// and, for a restore, its CREATING target. Refreshes that find no marker - /// do nothing; each failed update is logged. - pub(crate) fn start_backup_heartbeat( - &self, - operation_id: &str, - account_id: &str, - backup_arn: &str, - target_table: Option<&str>, - ) -> Heartbeat { - let catalog = self.catalog_db.clone(); - let interval = self.backup_liveness.heartbeat_interval; - let operation_id = operation_id.to_owned(); - let account_id = account_id.to_owned(); - let backup_arn = backup_arn.to_owned(); - let target_table = target_table.map(str::to_owned); - Heartbeat(tokio::spawn(async move { - loop { - for error in refresh_heartbeats( - &catalog, - &operation_id, - &account_id, - &backup_arn, - target_table.as_deref(), - ) - .await - { - tracing::warn!( - "backup heartbeat for operation {operation_id} on {backup_arn} failed: \ - {error}" - ); - } - tokio::time::sleep(interval).await; - } - })) - } - - /// Remove the backup markers and restore targets whose owner stopped - /// heartbeating. Safe while other processes share the catalog: every step - /// re-checks staleness in the same statement that acts, so a marker whose - /// owner refreshed it meanwhile is left alone. Run at startup and - /// periodically by the GSI backfill worker. - /// - /// - A stale restore claim is removed. If its restore had already created - /// its target, the target carries its own heartbeat and is judged by it. - /// - A stale CREATING restore target that has not finished its base copy - /// is removed completely: catalog document (which is its provenance), data - /// collection, and index collections and documents, so DeleteBackup of - /// the source and the target name are free again. - /// - A stale delete claim whose backup has a CREATING restore target is - /// left as it is; a later sweep finishes the delete once the target is - /// ACTIVE or removed. Otherwise the claim is taken over (re-claimed under - /// a new operation id and heartbeat) and the delete is finished: the - /// caller already received an answer or died waiting for one, the - /// collection drop is idempotent, and leaving the backup AVAILABLE could - /// point at data the crash already dropped. The sweep never releases a - /// delete claim: the stalled owner may still drop the data. - /// - /// Targets are swept before delete claims, so a delete blocked only by a - /// dead restore is finished in the same pass. - pub async fn sweep_stale_backup_claims(&self) -> Result { - let stale_after = self.backup_liveness.stale_after; - let backups_coll = self.catalog_db.collection::("backups"); - let tables_coll = self.catalog_db.collection::("tables"); - let mut sweep = BackupSweep::default(); - - // 1. Stale restore claims. - let stale: Vec = backups_coll - .find(stale_claim_filter( - RESTORE_CLAIM, - RESTORE_CLAIM_HEARTBEAT, - stale_after, - )) - .await - .map_err(internal)? - .try_collect() - .await - .map_err(internal)?; - for meta in stale { - let (Ok(arn), Some(owner)) = (meta.get_str("_id"), meta.get(RESTORE_CLAIM)) else { - continue; - }; - let mut filter = doc! { "_id": arn }; - filter.insert(RESTORE_CLAIM, owner.clone()); - filter.insert( - "$expr", - heartbeat_is_stale(RESTORE_CLAIM_HEARTBEAT, stale_after), - ); - let result = backups_coll - .update_one(filter, BackupMetadataClaim::unset_update(RESTORE_CLAIM)) - .await - .map_err(internal)?; - if result.modified_count == 1 { - tracing::warn!( - "backup sweep: cleared the restore claim on {arn} (operation {owner}); its \ - heartbeat stopped" - ); - sweep.restore_claims_cleared.push(arn.to_owned()); - } - } - - // 2. Abandoned restore targets. - let stale: Vec = tables_coll - .find(stale_target_filter(stale_after)) - .await - .map_err(internal)? - .try_collect() - .await - .map_err(internal)?; - for target in stale { - let (Ok(id), Ok(table_id)) = (target.get_document("_id"), target.get_str("table_id")) - else { - continue; - }; - let mut filter = stale_target_filter(stale_after); - filter.insert("_id", id.clone()); - filter.insert("table_id", table_id); - let removed = tables_coll - .find_one_and_delete(filter) - .await - .map_err(internal)?; - let Some(removed) = removed else { - continue; - }; - let name = format!( - "{}/{}", - id.get_str("account_id").unwrap_or("?"), - id.get_str("table_name").unwrap_or("?") - ); - let source = removed - .get_str("restore_source_backup_arn") - .unwrap_or("?") - .to_owned(); - match self.remove_target_collections(table_id).await { - Ok(()) => tracing::warn!( - "backup sweep: removed abandoned restore target {name} (table_id \ - {table_id}, restoring {source}); its restore stopped heartbeating before \ - its copy finished" - ), - Err(error) => tracing::error!( - "backup sweep: removed abandoned restore target {name} (table_id \ - {table_id}, restoring {source}) from the catalog, but dropping its \ - collections failed: {error}" - ), - } - sweep.restore_targets_removed.push(name); - } - - // 3. Stale delete claims. - let stale: Vec = backups_coll - .find(stale_claim_filter( - DELETE_CLAIM, - DELETE_CLAIM_HEARTBEAT, - stale_after, - )) - .await - .map_err(internal)? - .try_collect() - .await - .map_err(internal)?; - for meta in stale { - let (Ok(arn), Ok(account_id), Some(owner)) = ( - meta.get_str("_id"), - meta.get_str("account_id"), - meta.get(DELETE_CLAIM), - ) else { - continue; - }; - // A CREATING restore target of this backup: leave the claim for a - // later sweep, which finishes the delete once the target is - // ACTIVE or removed. Nothing is written, so the next sweep looks - // again. - if tables_coll - .find_one(restoring_target_filter(account_id, arn)) - .await - .map_err(internal)? - .is_some() - { - tracing::debug!( - "backup sweep: the stale DeleteBackup claim on {arn} waits for a running \ - restore from it" - ); - continue; - } - let sweeper = uuid::Uuid::new_v4().to_string(); - let mut filter = doc! { "_id": arn }; - filter.insert(DELETE_CLAIM, owner.clone()); - filter.insert( - "$expr", - heartbeat_is_stale(DELETE_CLAIM_HEARTBEAT, stale_after), - ); - let taken = backups_coll - .update_one( - filter, - claim_update(DELETE_CLAIM, DELETE_CLAIM_HEARTBEAT, &sweeper), - ) - .await - .map_err(internal)?; - if taken.modified_count != 1 { - continue; - } - // The sweep never releases a delete claim it took over: the - // stalled owner may still be about to drop the data, and only a - // held claim keeps the backup from reading as AVAILABLE once the - // data is gone. Any early return below leaves the claim held by - // `sweeper`; its heartbeat stops when this function returns, and a - // later sweep takes it over again. - let _heartbeat = self.start_backup_heartbeat(&sweeper, account_id, arn, None); - let mut sweeper_filter = doc! { "_id": arn, "account_id": account_id }; - sweeper_filter.insert(DELETE_CLAIM, &sweeper); - // A restore whose claim was swept while it stalled can insert its - // target after the check above; it aborts that target when it sees - // this claim, and a later sweep finishes the delete. - if tables_coll - .find_one(restoring_target_filter(account_id, arn)) - .await - .map_err(internal)? - .is_some() - { - tracing::warn!( - "backup sweep: took over the stale DeleteBackup claim on {arn}, but a \ - restore target from it exists; a later sweep finishes the delete" - ); - sweep.delete_claims_waiting.push(arn.to_owned()); - continue; - } - if let Ok(backup_id) = meta.get_str("backup_id") { - self.drop_backup_collection(backup_id).await?; - } - backups_coll - .update_one( - sweeper_filter, - doc! { - "$set": { "backup_status": "DELETED" }, - "$unset": { DELETE_CLAIM: "", DELETE_CLAIM_HEARTBEAT: "" }, - }, - ) - .await - .map_err(internal)?; - tracing::warn!( - "backup sweep: finished the interrupted DeleteBackup of {arn} (operation {owner})" - ); - sweep.delete_claims_finished.push(arn.to_owned()); - } - Ok(sweep) - } - /// Drop a backup's data collection (missing is success). In test builds /// [`BackupLiveness::after_backup_drop`] can make it fail or hang after the /// drop ran. @@ -1048,7 +488,10 @@ impl MongoEngine { /// Drop a removed restore target's data and index collections and its /// index documents. Tries every step and returns the first error. - async fn remove_target_collections(&self, table_id: &str) -> Result<(), StorageError> { + pub(crate) async fn remove_target_collections( + &self, + table_id: &str, + ) -> Result<(), StorageError> { let mut cleanup_error = None; if let Err(e) = self .drop_collection_if_exists(&data_collection_name(table_id)) @@ -1063,44 +506,9 @@ impl MongoEngine { cleanup_error.map_or(Ok(()), Err) } - /// Remove a partial restore target without allowing its provenance to keep - /// the source backup in use. The full predicate ensures cleanup can only - /// claim the target created by this restore. - async fn abort_backup_restore( - &self, - account_id: &str, - target_table_name: &str, - backup_arn: &str, - restore_operation_id: &str, - ) -> Result { - let removed = self - .catalog_db - .collection::("tables") - .find_one_and_delete(doc! { - "_id": { "account_id": account_id, "table_name": target_table_name }, - "restore_source_backup_arn": backup_arn, - "restore_operation_id": restore_operation_id, - "table_status": "CREATING", - "status_transition_at": Bson::Null, - }) - .await - .map_err(internal)?; - let Some(removed) = removed else { - return Ok(false); - }; - let table_id = removed - .get_str("table_id") - .map_err(|_| StorageError::Internal("restore target missing table_id".to_owned()))?; - - // The catalog row is already gone, so even a failed physical cleanup - // cannot leave provenance that blocks DeleteBackup. - self.remove_target_collections(table_id).await?; - Ok(true) - } - /// This process's clock, for the one heartbeat that cannot use the /// server's (the target insert). Test builds can set it behind. - fn local_now(&self) -> bson::DateTime { + pub(crate) fn local_now(&self) -> bson::DateTime { let now = bson::DateTime::now(); #[cfg(test)] let now = bson::DateTime::from_millis( @@ -1113,10 +521,9 @@ impl MongoEngine { /// `TableBeingRestored` when `target_table_name` is a CREATING restore /// target, naming its source backup; `Ok(())` otherwise. Unfenced: the - /// target insert is the fence. Checked before the restore takes a slot - /// and again under the claim, so a restore into such a name is refused as - /// the name (the service's TableInUseException) at once, whatever state - /// the backup is in, as on the other backends. + /// target insert is the fence. Checked first, so a restore into such a + /// name is refused as the name (the service's TableInUseException) at + /// once, whatever state the backup is in, as on the other backends. async fn refuse_a_name_being_restored( &self, account_id: &str, @@ -1142,345 +549,355 @@ impl MongoEngine { Ok(()) } - /// Phase one of a restore: install `restore_in_progress = operation_id` - /// atomically with the AVAILABLE and account check, excluding - /// DeleteBackup's claim, then check that no CREATING target is restoring - /// this backup already. Returns the backup document. + /// Register a restore: one session transaction that locks the backup + /// document (`$inc lock_seq` while AVAILABLE), checks that no CREATING + /// target is restoring the backup, and inserts the CREATING target with + /// its provenance and its RUNNING job record. The transaction commits + /// right after the insert, before the target's collections and indexes + /// are created; a transaction that lost a write conflict (a DeleteBackup + /// or another registration of the backup) is re-run and sees the other's + /// result. Returns the target's description and the backup's data + /// collection id. /// - /// Only the claim holder inserts a target, and it releases the claim only - /// after that insert, so the claim followed by the target check is what - /// makes "one restore of a backup at a time" hold against a simultaneous - /// request: the second either finds the claim held or finds the first's - /// target. Both are `BackupInUse`, as the service answers while a restore - /// from the backup is still CREATING. The guarantee holds for as long as - /// the holder's heartbeat does: a holder stalled past the stale window - /// between claim and insert loses the claim to the sweep and a second - /// restore can be admitted (see the module doc). A claim held by - /// DeleteBackup is `BackupInUse` too. Missing, another account's, or not - /// AVAILABLE: `Backup not found`. A claim that was released between the - /// claim attempt and the re-read (a restore that failed before its - /// insert) is retried for up to [`RESTORE_CLAIM_RETRY_FOR`]. - async fn claim_backup_for_restore( + /// Refusals: a missing, another account's, DELETED or DELETING backup is + /// `Backup not found`; a CREATING target of the backup is `BackupInUse` + /// naming it, or `TableBeingRestored` when that target is the name asked + /// for; a taken name is what `create_table_for_restore` reports. + async fn register_restore( &self, account_id: &str, - backup_arn: &str, - target_table_name: &str, - operation_id: &str, - ) -> Result { + request: &RestoreRequest, + owner_generation: u64, + ) -> Result<(TableDescription, String), StorageError> { + let RestoreRequest { + backup_arn, + source_table_arn, + restore_date_time, + input, + } = request; + let target_table_name = input.table_name.as_str(); let backups_coll = self.catalog_db.collection::("backups"); - // The name again, unfenced, under the claim: the state may have - // changed while this restore waited for a slot. The target insert - // stays the fence. - self.refuse_a_name_being_restored(account_id, target_table_name) - .await?; - let started = Instant::now(); - let mut delay = Duration::from_millis(10); - let doc = loop { - if let Some(doc) = backups_coll - .find_one_and_update( - restore_claim_filter(account_id, backup_arn), - claim_update(RESTORE_CLAIM, RESTORE_CLAIM_HEARTBEAT, operation_id), - ) - .await - .map_err(internal)? - { - break doc; - } - let current = backups_coll - .find_one(doc! { "_id": backup_arn, "account_id": account_id }) + let mut session = self.client.start_session().await.map_err(internal)?; + let tx_options = self.write_transaction_options(); + for attempt in 0..TRANSIENT_RETRY_ATTEMPTS { + session + .start_transaction() + .with_options(tx_options.clone()) .await .map_err(internal)?; - match restore_claim_miss(current.as_ref()) { - RestoreClaimMiss::NotFound => { - return Err(StorageError::Validation(format!( + let body: Result, TxErr> = async { + let locked = backups_coll + .find_one_and_update( + doc! { + "_id": backup_arn, + "account_id": account_id, + "backup_status": "AVAILABLE", + }, + doc! { "$inc": { LOCK_SEQ: 1_i64 } }, + ) + .session(&mut session) + .await + .map_err(TxErr::from)?; + let Some(backup_doc) = locked else { + // Missing, another account's, DELETED, or DELETING: the + // backup is not there to restore from (a DELETING one has + // its data dropped by the DeleteBackup that won). + return Ok(Err(StorageError::Validation(format!( "Backup not found: {backup_arn}" - ))); - } - RestoreClaimMiss::DeleteClaimed => { - return Err(StorageError::BackupInUse(format!( - "Backup is being deleted: {backup_arn}" - ))); + )))); + }; + let backup_id = + backup_doc + .get_str("backup_id") + .map(str::to_owned) + .map_err(|_| { + TxErr::Fatal(StorageError::Internal(format!( + "backup metadata for {backup_arn} has no backup_id" + ))) + })?; + // Under the lock: a CREATING target of this backup belongs to a + // restore that is still running (or one abandoned and not yet + // removed, which blocks DeleteBackup the same way). When that + // target is the name this restore asked for, the name is what + // is refused (TableBeingRestored, the service's + // TableInUseException), as on the other backends. + if let Some(restoring) = self + .creating_restore_target(account_id, backup_arn, Some(&mut session)) + .await + .map_err(TxErr::from)? + { + return Ok(Err(if restoring == target_table_name { + StorageError::TableBeingRestored { + table_name: target_table_name.to_owned(), + backup_arn: backup_arn.clone(), + } + } else { + backup_in_use(&restoring, backup_arn) + })); } - RestoreClaimMiss::RestoreClaimed => { - return Err(StorageError::BackupInUse(format!( - "Backup is being used to restore another table: {backup_arn}" - ))); + let mut registration = RestoreRegistration { + backup_arn, + source_table_arn, + restore_date_time: epoch_secs_datetime(*restore_date_time), + owner_generation, + first_heartbeat: self.local_now(), + record: RestoreRecord::Legacy, + session: Some(&mut session), + }; + match self + .create_table_for_restore(account_id, input.clone(), &mut registration) + .await + { + Ok(desc) => Ok(Ok((desc, backup_id))), + // The insert's transaction lost a write conflict and was + // aborted before anything outside it was created. + Err(StorageError::Transient(_)) => Err(TxErr::Transient), + Err(e) => { + // A creation that failed after its insert committed + // (a collection or index could not be created) left a + // registered target under this generation; a name + // refusal left nothing, and the removal matches + // nothing. + if let Err(cleanup) = self + .remove_target_of_generation( + account_id, + target_table_name, + owner_generation, + ) + .await + { + tracing::error!( + "restore of {backup_arn} into {target_table_name} failed during \ + target creation ({e}), and cleanup failed: {cleanup}" + ); + } + Ok(Err(e)) + } } - // Free: the other claim was released between the two reads. - RestoreClaimMiss::Free => {} } - if started.elapsed() + delay > RESTORE_CLAIM_RETRY_FOR { - return Err(StorageError::BackupInUse(format!( - "Backup is being used by another restore that is starting; retry: \ - {backup_arn}" - ))); - } - // Jitter, so restores that collided do not retry in step. - let jitter_ms = u64::try_from(uuid::Uuid::new_v4().as_u128() % 10).unwrap_or(0); - tokio::time::sleep(delay + Duration::from_millis(jitter_ms)).await; - delay = (delay * 2).min(Duration::from_millis(200)); - }; - // Under the claim: a CREATING target of this backup belongs to a - // restore that is still running (or one abandoned and not yet swept, - // which blocks DeleteBackup the same way). When that target is the - // name this restore asked for, the name is what is refused - // (TableBeingRestored, the service's TableInUseException), as on the - // other backends; a CREATING restore target of another backup under - // the name is caught by the insert. - let restoring = self - .catalog_db - .collection::("tables") - .find_one(restoring_target_filter(account_id, backup_arn)) .await; - match restoring { - Ok(None) => Ok(doc), - Ok(Some(target)) => { - let target_name = target - .get_document("_id") - .and_then(|id| id.get_str("table_name")) - .unwrap_or("?"); - let mut claim = BackupMetadataClaim::new( - backups_coll, - account_id, - backup_arn, - RESTORE_CLAIM, - operation_id.to_owned(), - ); - claim.release().await?; - if target_name == target_table_name { - return Err(StorageError::TableBeingRestored { - table_name: target_table_name.to_owned(), - backup_arn: backup_arn.to_owned(), + match body { + // The insert committed its transaction before the target's + // collections were created; nothing is left to commit. + Ok(Ok(registered)) => { + let (mut desc, backup_id) = registered; + desc.restore_summary = Some(extenddb_core::types::RestoreSummary { + source_backup_arn: Some(backup_arn.clone()), + source_table_arn: Some(source_table_arn.clone()), + restore_date_time: *restore_date_time, + restore_in_progress: true, }); + return Ok((desc, backup_id)); + } + Ok(Err(refusal)) => { + let _ = session.abort_transaction().await; + return Err(refusal); + } + Err(TxErr::Transient) => { + let _ = session.abort_transaction().await; + backoff_sleep(attempt).await; + } + Err(TxErr::Fatal(e)) => { + let _ = session.abort_transaction().await; + return Err(e); } - Err(StorageError::BackupInUse(format!( - "Backup is being used to restore table {target_name}: {backup_arn}" - ))) - } - Err(e) => { - let mut claim = BackupMetadataClaim::new( - backups_coll, - account_id, - backup_arn, - RESTORE_CLAIM, - operation_id.to_owned(), - ); - claim.release().await?; - Err(internal(e)) } } + Err(StorageError::Internal(format!( + "the registration of the restore of {backup_arn} into {target_table_name} did not \ + commit after {TRANSIENT_RETRY_ATTEMPTS} conflicting attempts" + ))) } - /// Phase two of a restore, once the target document exists: replace the - /// target's local-clock heartbeat with the server's, then release the - /// backup claim. `Err` means the claim had been swept and the backup is - /// gone or being deleted, so the caller must abort the target. - async fn hand_claim_to_target( + /// Remove the CREATING restore target `target_table_name` registered + /// under `owner_generation`, with its collections; `Ok(false)` when there + /// is none. + pub(crate) async fn remove_target_of_generation( &self, - restore_claim: &mut BackupMetadataClaim, account_id: &str, - backup_arn: &str, target_table_name: &str, - operation_id: &str, - ) -> Result<(), StorageError> { - // A process whose clock is behind the server's must not look stale to - // another process's sweep before its first periodic refresh. - for error in refresh_heartbeats( - &self.catalog_db, - operation_id, - account_id, - backup_arn, - Some(target_table_name), - ) - .await - { - tracing::warn!( - "restore of {backup_arn} into {target_table_name}: first heartbeat refresh \ - failed: {error}" - ); - } - match restore_claim.release().await? { - true => Ok(()), - // A sweep removed the claim because its heartbeat lapsed (this - // process stalled for longer than the stale window). A DeleteBackup - // may have claimed and dropped the backup in the gap. The target - // now exists, so any later DeleteBackup sees it; one that started - // earlier is visible as a delete claim or a DELETED status. - false => match self - .catalog_db - .collection::("backups") - .find_one(doc! { - "_id": backup_arn, - "account_id": account_id, - "backup_status": "AVAILABLE", - DELETE_CLAIM: { "$exists": false }, - }) - .await - .map_err(internal)? - { - Some(_) => Ok(()), - None => Err(StorageError::Validation(format!( - "Backup not found: {backup_arn}" - ))), - }, - } + owner_generation: u64, + ) -> Result { + let removed = self + .catalog_db + .collection::("tables") + .find_one_and_delete(doc! { + "_id": { "account_id": account_id, "table_name": target_table_name }, + "table_status": "CREATING", + RESTORE_JOB_GENERATION: generation_field(owner_generation), + }) + .await + .map_err(internal)?; + let Some(removed) = removed else { + return Ok(false); + }; + let table_id = removed + .get_str("table_id") + .map_err(|_| StorageError::Internal("restore target missing table_id".to_owned()))?; + self.remove_target_collections(table_id).await?; + Ok(true) } - /// Phase one and two of a restore: claim the backup, insert the CREATING - /// target with provenance, release the claim. Returns the target's - /// description and the backup's data collection id. - async fn register_restore( + /// The heartbeat of a running restore: refresh the job record's + /// `heartbeat_at` with the server's clock at once and then every + /// [`BackupLiveness::heartbeat_interval`], for as long as the record is + /// RUNNING under `owner_generation`. Returns when a refresh matches other + /// than exactly one document or fails: the claim is lost and the caller + /// stops the copy. + async fn restore_heartbeat_loop( &self, account_id: &str, - request: RestoreRequest, - operation_id: &str, - ) -> Result<(TableDescription, String), StorageError> { - let RestoreRequest { - backup_arn, - source_table_arn, - restore_date_time, - input, - } = request; - let target_table_name = input.table_name.clone(); - let backups_coll = self.catalog_db.collection::("backups"); - let backup_doc = self - .claim_backup_for_restore(account_id, &backup_arn, &target_table_name, operation_id) - .await?; - let mut restore_claim = BackupMetadataClaim::new( - backups_coll.clone(), - account_id, - &backup_arn, - RESTORE_CLAIM, - operation_id.to_owned(), - ); - let Ok(backup_id) = backup_doc.get_str("backup_id").map(str::to_owned) else { - let error = StorageError::Internal(format!( - "backup metadata for {backup_arn} has no backup_id" - )); - if let Err(cleanup) = restore_claim.release().await { - tracing::error!( - "restore of {backup_arn} failed ({error}) and its metadata claim could not \ - be cleared: {cleanup}" - ); - } - return Err(error); - }; - - // Provenance is part of the initial CREATING document. The claim is - // kept until that document exists, so protection overlaps and - // DeleteBackup can observe neither an unclaimed backup nor an - // ownerless target; it is released as soon as the document is - // inserted, while the target's collections and indexes are still - // being created. - // - // The creation and the hand-over are polled together until both have - // finished. An earlier shape selected the hand-over and only then - // awaited the rest of the creation; that left the creation's - // in-flight command (its `createCollection`, issued right after the - // insert) holding a pooled connection with its reply unread while - // the hand-over waited for connections of its own. Enough restores in - // that state at once held every pooled connection, and the driver's - // checkout never times out: the process wedged (20 concurrent - // restores of distinct backups on the default pool of 10). - let inserted = tokio::sync::Notify::new(); - let provenance = RestoreProvenance { - backup_arn: &backup_arn, - source_table_arn: &source_table_arn, - restore_date_time: epoch_secs_datetime(restore_date_time), - operation_id, - first_heartbeat: self.local_now(), - inserted: Some(&inserted), - }; - let (created, handed) = { - let create = self.create_table_for_restore(account_id, input, &provenance); - let hand = async { - inserted.notified().await; - self.hand_claim_to_target( - &mut restore_claim, - account_id, - &backup_arn, - &target_table_name, - operation_id, - ) - .await - }; - tokio::pin!(create); - tokio::pin!(hand); - let mut created: Option> = None; - let mut handed: Option> = None; - // A creation that failed may have failed before its insert, in - // which case the hand-over never starts; a successful one always - // notified, so its hand-over is awaited to the end. - while created.is_none() - || (handed.is_none() && created.as_ref().is_some_and(Result::is_ok)) - { - tokio::select! { - result = &mut create, if created.is_none() => created = Some(result), - result = &mut hand, if handed.is_none() => handed = Some(result), - } - } - (created.expect("creation polled to completion"), handed) - }; - let mut desc = match created { - Ok(desc) => desc, - Err(error) => { - if let Err(cleanup) = self - .abort_backup_restore(account_id, &target_table_name, &backup_arn, operation_id) - .await - { - tracing::error!( - "restore of {backup_arn} into {target_table_name} failed during target \ - creation ({error}), and cleanup failed: {cleanup}" - ); - } - if handed.is_none() - && let Err(cleanup) = restore_claim.release().await - { - tracing::error!( - "restore of {backup_arn} failed during target creation ({error}), and \ - its metadata claim could not be cleared: {cleanup}" - ); - } - return Err(error); - } + target_table_name: &str, + table_id: &str, + owner_generation: u64, + ) -> StorageError { + let tables_coll = self.catalog_db.collection::("tables"); + let filter = doc! { + "_id": { "account_id": account_id, "table_name": target_table_name }, + "table_id": table_id, + RESTORE_JOB_STATUS: JobStatus::Running.as_str(), + RESTORE_JOB_GENERATION: generation_field(owner_generation), }; - let still_ours = handed.expect("a created target was notified and handed over"); - if let Err(error) = still_ours { - if let Err(cleanup) = self - .abort_backup_restore(account_id, &target_table_name, &backup_arn, operation_id) - .await + loop { + #[cfg(test)] + if let Some(cell) = &self.backup_liveness.heartbeat_fault + && cell + .compare_exchange( + HeartbeatFault::ARMED, + HeartbeatFault::INJECTED, + std::sync::atomic::Ordering::SeqCst, + std::sync::atomic::Ordering::SeqCst, + ) + .is_ok() { - tracing::error!( - "restore of {backup_arn} lost its metadata claim ({error}) and target cleanup \ - failed: {cleanup}" + return StorageError::Internal( + "restore heartbeat: injected connection error".to_owned(), ); } - return Err(error); + match tables_coll + .update_one( + filter.clone(), + vec![doc! { "$set": { RESTORE_JOB_HEARTBEAT: "$$NOW" } }], + ) + .await + { + Ok(result) if result.matched_count == 1 => {} + Ok(result) => { + return StorageError::Internal(format!( + "the restore job record no longer carries this claim ({} document(s) \ + matched)", + result.matched_count + )); + } + Err(e) => return StorageError::Internal(format!("restore heartbeat: {e}")), + } + tokio::time::sleep(self.backup_liveness.heartbeat_interval).await; } + } - desc.restore_summary = Some(extenddb_core::types::RestoreSummary { - source_backup_arn: Some(backup_arn), - source_table_arn: Some(source_table_arn), - restore_date_time, - restore_in_progress: true, - }); - Ok((desc, backup_id)) + /// RUNNING to FAILED on the job record of the target, conditioned on + /// `owner_generation`: the write a failing job makes before it removes + /// its target. `Ok(false)` when the record is not RUNNING under that + /// generation. + async fn fail_restore_job_as( + &self, + account_id: &str, + target_table_name: &str, + table_id: &str, + owner_generation: u64, + reason: &str, + ) -> Result { + let result = self + .catalog_db + .collection::("tables") + .update_one( + doc! { + "_id": { "account_id": account_id, "table_name": target_table_name }, + "table_id": table_id, + RESTORE_JOB_STATUS: JobStatus::Running.as_str(), + RESTORE_JOB_GENERATION: generation_field(owner_generation), + }, + vec![doc! { "$set": { + RESTORE_JOB_STATUS: { "$literal": JobStatus::Failed.as_str() }, + RESTORE_JOB_FAILURE_REASON: { "$literal": reason }, + RESTORE_JOB_COMPLETED_AT: "$$NOW", + } }], + ) + .await + .map_err(internal)?; + Ok(result.matched_count == 1) } - /// Phase three, in the detached task: copy the backup's items into the - /// target with `$out`, then hand the target to the backfill worker. On - /// failure the target is removed. Logs the outcome either way. + /// The end of a restore's load (the design's 5.3 step 6 on this + /// backend): the job record DONE and the target handed to the GSI + /// backfill worker (`restore_backfill_pending`), with `item_count` and, + /// when given, `table_size_bytes`, in one write conditioned on the + /// generation, on the record RUNNING and on the table CREATING. From + /// then on the worker owns the target: it promotes each index and flips + /// the table ACTIVE, conditioned on the flag and on CREATING, and the + /// reconciler leaves a DONE record alone. `Ok(false)` when the record is + /// not RUNNING under `owner_generation` or the table is not CREATING. + pub(crate) async fn hand_over_to_backfill( + &self, + target: &RestoreTargetKey, + owner_generation: u64, + item_count: u64, + size_bytes: Option, + ) -> Result { + let mut set = doc! { + "item_count": { "$literal": i64::try_from(item_count).unwrap_or(i64::MAX) }, + "restore_backfill_pending": { "$literal": true }, + RESTORE_JOB_STATUS: { "$literal": JobStatus::Done.as_str() }, + RESTORE_JOB_COMPLETED_AT: "$$NOW", + }; + if let Some(size_bytes) = size_bytes { + set.insert( + "table_size_bytes", + doc! { "$literal": i64::try_from(size_bytes).unwrap_or(i64::MAX) }, + ); + } + let handed_over = self + .catalog_db + .collection::("tables") + .update_one( + doc! { + "_id": { "account_id": &target.account_id, "table_name": &target.table_name }, + "table_id": &target.table_id, + "table_status": "CREATING", + RESTORE_JOB_STATUS: JobStatus::Running.as_str(), + RESTORE_JOB_GENERATION: generation_field(owner_generation), + }, + vec![doc! { "$set": set }], + ) + .await + .map_err(internal)?; + Ok(handed_over.matched_count == 1) + } + + /// The detached copy of a restore: copy the backup's items into the + /// target with `$out` while the job record is heartbeated, then hand the + /// target to the backfill worker, marking the record DONE in the same + /// write. On failure the record is marked FAILED and the target removed, + /// each conditioned on the generation. A lost heartbeat takes that path + /// at once, dropping the copy future; the `$out` already sent is a write + /// stage, which the server does not interrupt when its client + /// disconnects, so it runs to its end there and then fails: the removal + /// dropped the target's data collection, which the registration created + /// with its indexes, and `$out` refuses to replace a target whose indexes + /// changed while it ran, so it leaves no collection behind (MongoDB 7.0, + /// error 125 "indexes of target collection changed during processing"). + /// Logs the outcome either way. async fn copy_restore( &self, account_id: &str, target_table_name: &str, backup_arn: &str, - operation_id: &str, + owner_generation: u64, backup_id: &str, table_id: &str, ) { let started = Instant::now(); - let copied = async { + let copy = async { #[cfg(test)] if let Some(gate) = &self.backup_liveness.copy_gate { gate.acquire().await.map_err(internal)?.forget(); @@ -1508,43 +925,52 @@ impl MongoEngine { // The base copy is complete, but restored secondary-index // collections are still empty. Mark the table pending restore // completion and leave its indexes CREATING so the shared worker - // can backfill them in bounded, restartable batches. Dropping the - // operation id and heartbeat hands the target to that worker; the - // sweep no longer judges it. - let mut filter = doc! { - "_id": { "account_id": account_id, "table_name": target_table_name }, - "table_id": table_id, - "table_status": "CREATING", - }; - filter.insert(TARGET_OPERATION, operation_id); + // can backfill them in bounded, restartable batches, and mark the + // job record DONE in the same write, conditioned on its generation + // and on RUNNING. From here the worker owns the target: it + // promotes each index and flips the table ACTIVE, conditioned on + // the flag and on CREATING, and the reconciler leaves a DONE + // record alone. let handed_over = self - .catalog_db - .collection::("tables") - .update_one( - filter, - doc! { - "$set": { - "item_count": i64::try_from(item_count).unwrap_or(i64::MAX), - "restore_backfill_pending": true, - }, - "$unset": { TARGET_OPERATION: "", TARGET_HEARTBEAT: "" }, + .hand_over_to_backfill( + &RestoreTargetKey { + account_id: account_id.to_owned(), + table_name: target_table_name.to_owned(), + table_id: table_id.to_owned(), }, + owner_generation, + item_count, + None, ) - .await - .map_err(internal)?; - Ok::<_, StorageError>((item_count, handed_over.matched_count == 1)) - } - .await; + .await?; + Ok::<_, StorageError>((item_count, handed_over)) + }; + let heartbeat = + self.restore_heartbeat_loop(account_id, target_table_name, table_id, owner_generation); + let outcome = tokio::select! { + copied = copy => copied, + lost = heartbeat => { + tracing::error!( + "restore of {backup_arn} into {target_table_name} (account {account_id}): the \ + job record's heartbeat failed ({lost}); stopping the copy" + ); + Err(StorageError::Internal(format!("claim lost: {lost}"))) + } + }; - match copied { + match outcome { Ok((items, true)) => tracing::info!( "restore of {backup_arn} into {target_table_name}: copied {items} item(s) in \ {:.1}s; index backfill pending", started.elapsed().as_secs_f64() ), Ok((items, false)) => { - // The target was removed while `$out` ran (a sweep judged it - // abandoned). `$out` may have recreated its data collection. + // The target was removed while `$out` ran (the reconciler + // judged it abandoned), or the record passed to another + // claim. `$out` does not recreate the dropped data collection + // (module doc); the drop here covers a removal that ran + // between `$out`'s rename and the hand-over write, after + // which the collection exists with no document naming it. let orphan = self .catalog_db .collection::("tables") @@ -1562,24 +988,188 @@ impl MongoEngine { {dropped:?})" ); } - Err(error) => match self - .abort_backup_restore(account_id, target_table_name, backup_arn, operation_id) - .await - { - Ok(true) => tracing::error!( - "restore of {backup_arn} into {target_table_name} failed and the partial \ - target was removed: {error}" - ), - Ok(false) => tracing::error!( - "restore of {backup_arn} into {target_table_name} failed after its target \ - was already removed: {error}" - ), - Err(cleanup) => tracing::error!( - "restore of {backup_arn} into {target_table_name} failed ({error}), and \ - physical cleanup also failed: {cleanup}" - ), + Err(error) => { + // The target must not stay behind: it is CREATING with no + // scheduled transition, so nothing else would move it on, and + // the name would stay taken. The job record is marked FAILED + // first, conditioned on the generation and on RUNNING; only if + // that matched is the target removed. A record another + // generation owns, or a failed write, means the claim holder + // owns the target, and nothing further is touched. + let failed = self + .fail_restore_job_as( + account_id, + target_table_name, + table_id, + owner_generation, + &error.to_string(), + ) + .await; + match failed { + Ok(true) => {} + Ok(false) => { + tracing::warn!( + "restore of {backup_arn} into {target_table_name} ended with \ + \"{error}\"; its job record was no longer RUNNING under this claim, \ + so the target ({table_id}) was left to its owner" + ); + return; + } + Err(write) => { + tracing::error!( + "restore of {backup_arn} into {target_table_name} failed ({error}), \ + and its job record could not be marked FAILED ({write}); the \ + reconciler removes the target once its heartbeat is stale" + ); + return; + } + } + let key = RestoreTargetKey { + account_id: account_id.to_owned(), + table_name: target_table_name.to_owned(), + table_id: table_id.to_owned(), + }; + match BackupCatalog::remove_restore_target(self, &key).await { + Ok(true) => tracing::error!( + "restore of {backup_arn} into {target_table_name} failed and the partial \ + target was removed: {error}" + ), + Ok(false) => tracing::error!( + "restore of {backup_arn} into {target_table_name} failed after its target \ + was already removed: {error}" + ), + Err(cleanup) => tracing::error!( + "restore of {backup_arn} into {target_table_name} failed ({error}), and \ + physical cleanup also failed; the reconciler retries it: {cleanup}" + ), + } + } + } + } + + /// The destructive half of the legacy DeleteBackup, from a document that + /// is DELETING: drop the backup's data collection, then write DELETED. + /// The drop may have run on the server even if this future is dropped + /// while waiting for its reply or the reply is an error; the document + /// stays DELETING either way, and a later DeleteBackup resumes here. + async fn finish_legacy_delete( + &self, + account_id: &str, + backup_arn: &str, + backup_doc: &Document, + ) -> Result { + // A backup_id is absent on a pre-$out backup, which has no physical + // collection to drop. + if let Ok(backup_id) = backup_doc.get_str("backup_id") { + self.drop_backup_collection(backup_id).await?; + } + self.catalog_db + .collection::("backups") + .update_one( + doc! { "_id": backup_arn, "account_id": account_id, "backup_status": "DELETING" }, + doc! { "$set": { "backup_status": "DELETED" } }, + ) + .await + .map_err(internal)?; + let desc = description_from_doc(backup_doc, backup_arn)?; + Ok(BackupDescription { + backup_details: BackupDetails { + backup_status: "DELETED".to_owned(), + ..desc.backup_details }, + ..desc + }) + } + + /// The legacy DeleteBackup's decision as one session transaction: lock + /// the backup document (`$inc lock_seq` while AVAILABLE), re-check for a + /// CREATING target, write DELETING. `Ok(Some(doc))` is the document as it + /// was, now DELETING; `Ok(None)` means a concurrent DeleteBackup has + /// written DELETING already and this one resumes its drop. + async fn start_legacy_delete( + &self, + account_id: &str, + backup_arn: &str, + ) -> Result, StorageError> { + let backups_coll = self.catalog_db.collection::("backups"); + let mut session = self.client.start_session().await.map_err(internal)?; + let tx_options = self.write_transaction_options(); + for attempt in 0..TRANSIENT_RETRY_ATTEMPTS { + session + .start_transaction() + .with_options(tx_options.clone()) + .await + .map_err(internal)?; + let body: Result, StorageError>, TxErr> = async { + let locked = backups_coll + .find_one_and_update( + doc! { + "_id": backup_arn, + "account_id": account_id, + "backup_status": "AVAILABLE", + }, + doc! { "$inc": { LOCK_SEQ: 1_i64 } }, + ) + .session(&mut session) + .await + .map_err(TxErr::from)?; + let Some(backup_doc) = locked else { + let current = backups_coll + .find_one(doc! { "_id": backup_arn, "account_id": account_id }) + .session(&mut session) + .await + .map_err(TxErr::from)?; + return Ok(match lock_miss(current.as_ref()) { + LockMiss::Deleting => Ok(None), + LockMiss::NotFound => Err(StorageError::Validation(format!( + "Backup not found: {backup_arn}" + ))), + }); + }; + // A restore can register its target between the unlocked + // first check and this lock; from here new registrations + // conflict with this transaction. + if let Some(target) = self + .creating_restore_target(account_id, backup_arn, Some(&mut session)) + .await + .map_err(TxErr::from)? + { + return Ok(Err(backup_in_use(&target, backup_arn))); + } + backups_coll + .update_one( + doc! { "_id": backup_arn, "account_id": account_id, "backup_status": "AVAILABLE" }, + doc! { "$set": { "backup_status": "DELETING" } }, + ) + .session(&mut session) + .await + .map_err(TxErr::from)?; + Ok(Ok(Some(backup_doc))) + } + .await; + match body { + Ok(Ok(outcome)) => match self.commit_write_transaction(&mut session, None).await? { + CommitOutcome::Committed => return Ok(outcome), + CommitOutcome::RerunBody => backoff_sleep(attempt).await, + }, + Ok(Err(refusal)) => { + let _ = session.abort_transaction().await; + return Err(refusal); + } + Err(TxErr::Transient) => { + let _ = session.abort_transaction().await; + backoff_sleep(attempt).await; + } + Err(TxErr::Fatal(e)) => { + let _ = session.abort_transaction().await; + return Err(e); + } + } } + Err(StorageError::Internal(format!( + "DeleteBackup of {backup_arn} did not commit after {TRANSIENT_RETRY_ATTEMPTS} \ + conflicting attempts" + ))) } } @@ -1686,17 +1276,90 @@ fn restore_source_from_doc( }) } -impl BackupEngine for MongoEngine { - fn create_backup( +/// The `DescribeBackup` description of a backup document, with the status as +/// stored. `SourceTableFeatureDetails` and the throughput are filled by the +/// engine from `restore_source` for an AVAILABLE backup. +fn description_from_doc( + backup_doc: &Document, + backup_arn: &str, +) -> Result { + let name = backup_doc + .get_str("backup_name") + .unwrap_or_default() + .to_owned(); + let status = backup_doc + .get_str("backup_status") + .unwrap_or("AVAILABLE") + .to_owned(); + let table_id = backup_doc + .get_str("table_id") + .unwrap_or_default() + .to_owned(); + let table_name = backup_doc + .get_str("table_name") + .unwrap_or_default() + .to_owned(); + let table_arn = backup_doc + .get_str("table_arn") + .unwrap_or_default() + .to_owned(); + let size = backup_doc.get_i64("backup_size_bytes").unwrap_or(0); + let count = backup_doc.get_i64("item_count").unwrap_or(0); + let billing = backup_doc + .get_str("billing_mode") + .unwrap_or("PAY_PER_REQUEST") + .to_owned(); + + let created_at = backup_created_at(backup_doc); + let table_created = backup_doc + .get_f64("table_creation_date_time") + .unwrap_or(created_at); + let key_schema: Vec = decode_required(backup_doc, "key_schema")?; + + Ok(BackupDescription { + backup_details: BackupDetails { + backup_arn: backup_arn.to_owned(), + backup_name: name, + backup_status: status, + backup_type: "USER".to_owned(), + backup_size_bytes: size, + backup_creation_date_time: created_at, + }, + source_table_details: SourceTableDetails { + table_name, + table_id, + table_arn, + key_schema, + item_count: count, + table_size_bytes: size, + billing_mode: Some(billing), + table_creation_date_time: table_created, + ..SourceTableDetails::default() + }, + source_table_feature_details: None, + }) +} + +/// The statuses of a backup document the legacy describe and list leave +/// out: a deleted backup and one whose DeleteBackup has started. +fn gone_statuses() -> Document { + doc! { "$nin": ["DELETED", "DELETING"] } +} + +impl LegacyBackupReader for MongoEngine { + fn create( &self, account_id: &str, table_name: &str, backup_name: &str, + permit: JobPermit, ) -> BoxFuture<'_, Result> { let account_id = account_id.to_string(); let table_name = table_name.to_string(); let backup_name = backup_name.to_string(); Box::pin(async move { + // The job's slot, held until the backup document is written. + let _permit = permit; let tables_coll = self.catalog_db.collection::("tables"); let table_doc = tables_coll .find_one(doc! { @@ -1926,7 +1589,7 @@ impl BackupEngine for MongoEngine { }) } - fn describe_backup( + fn describe( &self, account_id: &str, backup_arn: &str, @@ -1936,82 +1599,24 @@ impl BackupEngine for MongoEngine { Box::pin(async move { let backups_coll = self.catalog_db.collection::("backups"); // Scope the lookup to the calling account so a backup ARN cannot be - // read cross-account, and exclude DELETED backups so a deleted - // backup reads as BackupNotFoundException. Matches the postgres - // backend. + // read cross-account, and leave out a deleted backup and one being + // deleted, so both read as BackupNotFoundException. let backup_doc = backups_coll .find_one(doc! { "_id": &backup_arn, "account_id": &account_id, - "backup_status": { "$ne": "DELETED" }, + "backup_status": gone_statuses(), }) .await .map_err(internal)? .ok_or_else(|| { StorageError::Validation(format!("Backup not found: {backup_arn}")) })?; - - let name = backup_doc - .get_str("backup_name") - .unwrap_or_default() - .to_owned(); - let status = backup_doc - .get_str("backup_status") - .unwrap_or("AVAILABLE") - .to_owned(); - let table_id = backup_doc - .get_str("table_id") - .unwrap_or_default() - .to_owned(); - let table_name = backup_doc - .get_str("table_name") - .unwrap_or_default() - .to_owned(); - let table_arn = backup_doc - .get_str("table_arn") - .unwrap_or_default() - .to_owned(); - let size = backup_doc.get_i64("backup_size_bytes").unwrap_or(0); - let count = backup_doc.get_i64("item_count").unwrap_or(0); - let billing = backup_doc - .get_str("billing_mode") - .unwrap_or("PAY_PER_REQUEST") - .to_owned(); - - let created_at = backup_created_at(&backup_doc); - let table_created = backup_doc - .get_f64("table_creation_date_time") - .unwrap_or(created_at); - let key_schema: Vec = decode_required(&backup_doc, "key_schema")?; - - Ok(BackupDescription { - backup_details: BackupDetails { - backup_arn: backup_arn.clone(), - backup_name: name, - backup_status: status, - backup_type: "USER".to_owned(), - backup_size_bytes: size, - backup_creation_date_time: created_at, - }, - source_table_details: SourceTableDetails { - table_name, - table_id, - table_arn, - key_schema, - item_count: count, - table_size_bytes: size, - billing_mode: Some(billing), - table_creation_date_time: table_created, - // Filled by the engine from backup_restore_source for an - // AVAILABLE backup. - ..SourceTableDetails::default() - }, - source_table_feature_details: None, - }) + description_from_doc(&backup_doc, &backup_arn) }) } - fn list_backups( + fn list( &self, account_id: &str, table_name: Option<&str>, @@ -2023,7 +1628,7 @@ impl BackupEngine for MongoEngine { let mut filter = doc! { "account_id": &account_id, - "backup_status": { "$ne": "DELETED" }, + "backup_status": gone_statuses(), }; if let Some(tn) = &table_name { filter.insert("table_name", tn.as_str()); @@ -2057,7 +1662,7 @@ impl BackupEngine for MongoEngine { }) } - fn delete_backup( + fn delete( &self, account_id: &str, backup_arn: &str, @@ -2065,123 +1670,51 @@ impl BackupEngine for MongoEngine { let account_id = account_id.to_string(); let backup_arn = backup_arn.to_string(); Box::pin(async move { - let desc = self.describe_backup(&account_id, &backup_arn).await?; let backups_coll = self.catalog_db.collection::("backups"); - let tables_coll = self.catalog_db.collection::("tables"); - - // Fast-path the durable half of the two-phase restore claim. - if let Some(target) = tables_coll - .find_one(restoring_target_filter(&account_id, &backup_arn)) - .await - .map_err(internal)? - { - let name = target - .get_document("_id") - .ok() - .and_then(|id| id.get_str("table_name").ok()) - .unwrap_or("?"); - return Err(StorageError::BackupInUse(format!( - "Backup is being used to restore table {name}: {backup_arn}" - ))); - } - - // Atomically claim deletion only when restore has not claimed the - // metadata. Restore uses the inverse predicate, so once either - // marker is installed the other operation cannot start. The - // heartbeat runs from before the claim until this future ends - // (completes or is dropped). - let delete_operation_id = uuid::Uuid::new_v4().to_string(); - let _heartbeat = - self.start_backup_heartbeat(&delete_operation_id, &account_id, &backup_arn, None); - let meta = backups_coll - .find_one_and_update( - delete_claim_filter(&account_id, &backup_arn), - claim_update(DELETE_CLAIM, DELETE_CLAIM_HEARTBEAT, &delete_operation_id), - ) - .return_document(mongodb::options::ReturnDocument::Before) + let current = backups_coll + .find_one(doc! { "_id": &backup_arn, "account_id": &account_id }) .await .map_err(internal)?; - let Some(meta) = meta else { - let restoring = tables_coll - .find_one(restoring_target_filter(&account_id, &backup_arn)) - .await - .map_err(internal)? - .is_some(); - let reread = backups_coll - .find_one(doc! { "_id": &backup_arn, "account_id": &account_id }) - .await - .map_err(internal)?; - return Err(delete_claim_miss_error( - restoring, - restore_claim_miss(reread.as_ref()), - &backup_arn, - )); + let current = match current.as_ref().map(|d| d.get_str("backup_status")) { + None | Some(Ok("DELETED")) => { + return Err(StorageError::Validation(format!( + "Backup not found: {backup_arn}" + ))); + } + // A DeleteBackup that stopped after writing DELETING: resume + // its drop and its DELETED write. + Some(Ok("DELETING")) => { + let doc = current.expect("a status was read from a document"); + return self + .finish_legacy_delete(&account_id, &backup_arn, &doc) + .await; + } + Some(_) => current.expect("a status was read from a document"), }; - let mut delete_claim = BackupMetadataClaim::new( - backups_coll.clone(), - &account_id, - &backup_arn, - DELETE_CLAIM, - delete_operation_id, - ); - // A restore can transfer its marker to a target between the first - // target read and our metadata claim. Recheck after claiming; new - // restores are now excluded by delete_in_progress. - if let Some(target) = tables_coll - .find_one(restoring_target_filter(&account_id, &backup_arn)) + // The in-use rule before the lock, so a backup a running restore + // reads is refused without a transaction. + if let Some(target) = self + .creating_restore_target(&account_id, &backup_arn, None) .await .map_err(internal)? { - let name = target - .get_document("_id") - .ok() - .and_then(|id| id.get_str("table_name").ok()) - .unwrap_or("?") - .to_owned(); - delete_claim.release().await?; - return Err(StorageError::BackupInUse(format!( - "Backup is being used to restore table {name}: {backup_arn}" - ))); + return Err(backup_in_use(&target, &backup_arn)); } - // Drop the backup collection. If backup_id is absent (for an old - // pre-$out backup), there is no physical collection to drop. - // - // From here on the claim is never released. The drop may have run - // on the server even if this future is dropped while waiting for - // its reply or the reply is an error, and releasing would leave an - // AVAILABLE backup with no data, which a restore would copy as an - // empty table. A claim left behind (an error below, or this future - // dropped) stops heartbeating when this future ends, and the - // stale-claim sweep finishes the delete. - delete_claim.disarm(); - if let Ok(backup_id) = meta.get_str("backup_id") { - self.drop_backup_collection(backup_id).await?; - } - - backups_coll - .update_one( - delete_claim.filter(), - doc! { - "$set": { "backup_status": "DELETED" }, - "$unset": { DELETE_CLAIM: "", DELETE_CLAIM_HEARTBEAT: "" }, - }, - ) + // The decision under the lock: AVAILABLE and no CREATING target, + // then DELETING, in one transaction a restore's registration + // conflicts with. + let backup_doc = match self.start_legacy_delete(&account_id, &backup_arn).await? { + Some(doc) => doc, + None => current, + }; + self.finish_legacy_delete(&account_id, &backup_arn, &backup_doc) .await - .map_err(internal)?; - - Ok(BackupDescription { - backup_details: BackupDetails { - backup_status: "DELETED".to_owned(), - ..desc.backup_details - }, - ..desc - }) }) } - fn backup_restore_source( + fn restore_source( &self, account_id: &str, backup_arn: &str, @@ -2206,32 +1739,32 @@ impl BackupEngine for MongoEngine { }) } - fn restore_table_from_backup( + fn restore( &self, account_id: &str, request: RestoreRequest, + permit: JobPermit, ) -> BoxFuture<'_, Result> { let engine = self.clone(); let account_id = account_id.to_owned(); Box::pin(async move { - // The whole restore, admission included, runs in a task that owns - // its inputs, so the request future being dropped (a client - // timeout) stops neither the wait for a slot, nor the - // registration, nor the copy. At most RESTORE_SLOTS_PER_ACCOUNT - // restores per account and restore_server_slot_count per process - // run at once; the permits are released when the copy ends. A - // restore refused for want of a slot creates nothing. The request - // waits only until the CREATING target exists and the backup claim - // has passed to it. + // The whole restore, registration included, runs in a task that + // owns its inputs and the job permit, so the request future being + // dropped (a client timeout) stops neither the registration nor + // the copy, and the permit is released when the copy ends. A + // refused restore creates nothing. The request waits for the + // CREATING target to exist, at most `RESTORE_REGISTRATION_TIMEOUT` + // (its doc says what a request that hit the bound leaves behind). let (registered_tx, registered_rx) = tokio::sync::oneshot::channel(); tokio::spawn(async move { + let _permit = permit; let backup_arn = request.backup_arn.clone(); let target_table_name = request.input.table_name.clone(); // A refusal the request is no longer there to receive (its // client went away, or the registration bound fired) would // otherwise vanish, so it is logged. A refused restore removes // what it created; a cleanup that failed is logged on its own, - // and the heartbeat sweep finishes it. + // and the reconciler finishes it. let refuse = |registered: tokio::sync::oneshot::Sender<_>, e: StorageError| { if let Err(Err(e)) = registered.send(Err(e)) { tracing::warn!( @@ -2240,8 +1773,9 @@ impl BackupEngine for MongoEngine { ); } }; - // The name before a slot is taken, so a client retrying its - // own restore is answered at once instead of after the wait. + // The name first, unfenced, as on the other backends: a + // restore into a name that is a CREATING restore target is + // refused as the name at once, whatever state the backup is in. if let Err(error) = engine .refuse_a_name_being_restored(&account_id, &target_table_name) .await @@ -2249,31 +1783,19 @@ impl BackupEngine for MongoEngine { refuse(registered_tx, error); return; } - let _permits = match acquire_restore_permits( - engine.restore_account_slots.for_account(&account_id), - Arc::clone(&engine.restore_server_slots), - engine.restore_server_slot_count, - engine.backup_liveness.slot_wait, - ) - .await - { - Ok(permits) => permits, - Err(error) => { - refuse(registered_tx, error); - return; + #[cfg(test)] + if let Some(gate) = &engine.backup_liveness.registration_gate { + match gate.acquire().await { + Ok(permit) => permit.forget(), + Err(e) => { + refuse(registered_tx, internal(e)); + return; + } } - }; - let operation_id = uuid::Uuid::new_v4().to_string(); - // Started before the claim so the claim and the target are - // refreshed for as long as this task runs. - let _heartbeat = engine.start_backup_heartbeat( - &operation_id, - &account_id, - &backup_arn, - Some(&target_table_name), - ); + } + let owner_generation = new_generation(); let (desc, backup_id) = match engine - .register_restore(&account_id, request, &operation_id) + .register_restore(&account_id, &request, owner_generation) .await { Ok(registered) => registered, @@ -2290,7 +1812,7 @@ impl BackupEngine for MongoEngine { &account_id, &target_table_name, &backup_arn, - &operation_id, + owner_generation, &backup_id, &table_id, ) @@ -2310,15 +1832,20 @@ impl BackupEngine for MongoEngine { } }) } + + /// None: this backend's create commits its backup document AVAILABLE + /// after the copy and leaves no CREATING row behind. + fn abandoned_creating_backups( + &self, + _stale_after: Duration, + ) -> BoxFuture<'_, Result, StorageError>> { + Box::pin(async { Ok(Vec::new()) }) + } } #[cfg(test)] mod tests { - use super::{ - RESTORE_SLOTS_PER_ACCOUNT, RestoreAccountSlots, delete_claim_filter, - insert_non_empty_array, restore_claim_filter, restore_gsi_throughput, - restore_provisioned_throughput, restoring_target_filter, - }; + use super::{insert_non_empty_array, restore_gsi_throughput, restore_provisioned_throughput}; use extenddb_core::types::{GsiInput, ProvisionedThroughput}; use mongodb::bson::Document; @@ -2392,91 +1919,20 @@ mod tests { None ); } - - #[test] - fn restore_and_delete_claims_exclude_each_other() { - let restore = restore_claim_filter("123456789012", "backup"); - let delete = delete_claim_filter("123456789012", "backup"); - for filter in [&restore, &delete] { - assert_eq!(filter.get_str("backup_status"), Ok("AVAILABLE")); - assert_eq!( - filter - .get_document("restore_in_progress") - .and_then(|predicate| predicate.get_bool("$exists")), - Ok(false) - ); - assert_eq!( - filter - .get_document("delete_in_progress") - .and_then(|predicate| predicate.get_bool("$exists")), - Ok(false) - ); - } - } - - #[test] - fn durable_restore_target_predicate_is_account_scoped_and_creating() { - let filter = restoring_target_filter("123456789012", "backup"); - assert_eq!(filter.get_str("_id.account_id"), Ok("123456789012")); - assert_eq!(filter.get_str("restore_source_backup_arn"), Ok("backup")); - assert_eq!(filter.get_str("table_status"), Ok("CREATING")); - } - - /// One engine's account slots are shared by its clones and by nothing - /// else: a second engine in the same process (another test's fixture) - /// has its own full allowance for the same account. - #[test] - fn account_restore_slots_belong_to_one_engine() { - let slots = RestoreAccountSlots::default(); - let held = slots - .for_account("123456789012") - .try_acquire_owned() - .expect("a free slot"); - assert_eq!( - slots - .clone() - .for_account("123456789012") - .available_permits(), - RESTORE_SLOTS_PER_ACCOUNT - 1, - "a clone sees the same account map" - ); - assert_eq!( - RestoreAccountSlots::default() - .for_account("123456789012") - .available_permits(), - RESTORE_SLOTS_PER_ACCOUNT, - "another engine has its own" - ); - drop(held); - // Nothing in flight for the account: its entry is pruned on the next - // lookup of any account, and a fresh one has the full allowance. - slots.for_account("other"); - assert_eq!( - slots.0.lock().unwrap().len(), - 1, - "the idle account's entry is gone" - ); - assert_eq!( - slots.for_account("123456789012").available_permits(), - RESTORE_SLOTS_PER_ACCOUNT - ); - } } #[cfg(test)] mod definition_tests { use super::{ - BackupMetadataClaim, BackupSweep, DELETE_CLAIM, DELETE_CLAIM_HEARTBEAT, RESTORE_CLAIM, - RESTORE_CLAIM_HEARTBEAT, RestoreClaimMiss, backup_definition_from_doc, claim_update, - delete_claim_miss_error, heartbeat_updates, restore_claim_miss, restore_source_from_doc, - stale_claim_filter, stale_target_filter, try_each, + LockMiss, RestoreRecord, RestoreRegistration, backup_definition_from_doc, + description_from_doc, gone_statuses, lock_miss, restore_source_from_doc, + restoring_target_filter, }; use extenddb_core::types::{ OnDemandThroughput, ProvisionedThroughput, StreamViewType, TimeToLiveStatus, }; use extenddb_storage::error::StorageError; use mongodb::bson::{self, doc}; - use std::time::Duration; const ARN: &str = "arn:aws:dynamodb:us-east-1:123456789012:table/src/backup/01791976431438-0000abcd"; @@ -2523,59 +1979,6 @@ mod definition_tests { } } - #[test] - fn constant_keys_in_doc_macros_are_the_constant_values() { - // The update documents use `CONST: value` keys; this pins that the - // macro takes the constant's value, not its identifier. - let update = BackupMetadataClaim::unset_update(DELETE_CLAIM); - let unset = update.get_document("$unset").expect("$unset"); - assert!(unset.contains_key("delete_in_progress")); - assert!(unset.contains_key("delete_in_progress_heartbeat")); - let inline = doc! { DELETE_CLAIM_HEARTBEAT: "" }; - assert!(inline.contains_key("delete_in_progress_heartbeat")); - let claim = claim_update(RESTORE_CLAIM, RESTORE_CLAIM_HEARTBEAT, "op"); - let set = claim[0].get_document("$set").expect("$set"); - assert_eq!( - set.get_document("restore_in_progress") - .and_then(|d| d.get_str("$literal")), - Ok("op") - ); - assert_eq!(set.get_str("restore_in_progress_heartbeat"), Ok("$$NOW")); - } - - #[test] - fn stale_filters_compare_the_heartbeat_with_the_server_clock() { - let filter = stale_claim_filter( - RESTORE_CLAIM, - RESTORE_CLAIM_HEARTBEAT, - Duration::from_secs(60), - ); - assert_eq!( - filter - .get_document("restore_in_progress") - .and_then(|d| d.get_bool("$exists")), - Ok(true) - ); - let expr = filter.get_document("$expr").expect("$expr").to_string(); - assert!(expr.contains("$restore_in_progress_heartbeat"), "{expr}"); - assert!(expr.contains("$$NOW") && expr.contains("60000"), "{expr}"); - let target = stale_target_filter(Duration::from_secs(60)); - assert_eq!(target.get_str("table_status"), Ok("CREATING")); - assert_eq!( - target - .get_document("restore_backfill_pending") - .and_then(|d| d.get_bool("$exists")), - Ok(false) - ); - assert!( - target - .get_document("$expr") - .expect("$expr") - .to_string() - .contains("$restore_heartbeat_at") - ); - } - #[test] fn the_definition_is_read_from_the_backup_document() { let def = backup_definition_from_doc(&backup_doc(), "PROVISIONED", ARN).expect("def"); @@ -2701,129 +2104,168 @@ mod definition_tests { ); } - /// A failed heartbeat update does not stop the others. - #[tokio::test] - async fn try_each_runs_every_item_after_a_failure() { - let mut attempted = Vec::new(); - let errors = try_each(vec![1, 2, 3], |i| { - attempted.push(i); - async move { - if i == 1 { - Err(StorageError::Internal(format!("update {i}"))) - } else { - Ok(()) - } - } - }) - .await; - assert_eq!(attempted, [1, 2, 3]); - assert_eq!(errors.len(), 1); - assert!(matches!(&errors[0], StorageError::Internal(m) if m == "update 1")); + /// The description reports the document's status as stored, so a + /// resumed DeleteBackup can answer DELETED from the document it read. + #[test] + fn the_description_reports_the_documents_status_and_source_table() { + let desc = description_from_doc(&backup_doc(), ARN).expect("description"); + assert_eq!(desc.backup_details.backup_arn, ARN); + assert_eq!(desc.backup_details.backup_status, "AVAILABLE"); + assert_eq!(desc.backup_details.backup_size_bytes, 77); + assert!((desc.backup_details.backup_creation_date_time - 1_791_976_431.438).abs() < 1e-9); + assert_eq!(desc.source_table_details.table_name, "src"); + assert_eq!(desc.source_table_details.table_id, "tid"); + assert_eq!(desc.source_table_details.item_count, 3); + assert_eq!( + desc.source_table_details.billing_mode.as_deref(), + Some("PROVISIONED") + ); + assert!(desc.source_table_feature_details.is_none()); + let mut deleting = backup_doc(); + deleting.insert("backup_status", "DELETING"); + assert_eq!( + description_from_doc(&deleting, ARN) + .expect("description") + .backup_details + .backup_status, + "DELETING" + ); } + /// The legacy describe and list leave out a deleted backup and one whose + /// DeleteBackup has started; a restore's registration and a DeleteBackup + /// tell the two apart only to decide whether a delete is resumed. #[test] - fn heartbeat_updates_cover_both_claims_and_the_target() { - let updates = heartbeat_updates("op", "123456789012", ARN, Some("dst")); - let names: Vec<_> = updates.iter().map(|u| (u.0, u.1)).collect(); + fn a_deleting_or_deleted_document_is_gone_and_only_deleting_resumes_a_delete() { + let gone = gone_statuses(); + let excluded = gone.get_array("$nin").expect("$nin"); assert_eq!( - names, - [ - ("backups", "restore_in_progress"), - ("backups", "delete_in_progress"), - ("tables", "restore_heartbeat_at"), - ] + excluded, + &[bson::Bson::from("DELETED"), bson::Bson::from("DELETING")] ); - assert_eq!(updates[0].2.get_str("restore_in_progress"), Ok("op")); - assert_eq!(updates[2].2.get_str("restore_operation_id"), Ok("op")); - assert_eq!(heartbeat_updates("op", "123456789012", ARN, None).len(), 2); + assert_eq!(lock_miss(None), LockMiss::NotFound); + let mut d = doc! { "_id": ARN, "backup_status": "AVAILABLE" }; + assert_eq!(lock_miss(Some(&d)), LockMiss::NotFound); + d.insert("backup_status", "DELETED"); + assert_eq!(lock_miss(Some(&d)), LockMiss::NotFound); + d.insert("backup_status", "DELETING"); + assert_eq!(lock_miss(Some(&d)), LockMiss::Deleting); } - /// Why a restore claim missed decides between retrying, BackupInUse, - /// and Backup not found. #[test] - fn a_restore_claim_miss_is_classified_from_the_reread() { - let available = doc! { "_id": ARN, "backup_status": "AVAILABLE" }; - assert_eq!(restore_claim_miss(None), RestoreClaimMiss::NotFound); - let mut d = available.clone(); - d.insert("backup_status", "DELETED"); - assert_eq!(restore_claim_miss(Some(&d)), RestoreClaimMiss::NotFound); + fn the_in_use_filter_is_account_scoped_and_reads_the_targets_status() { + let filter = restoring_target_filter("123456789012", "backup"); + assert_eq!(filter.get_str("_id.account_id"), Ok("123456789012")); + assert_eq!(filter.get_str("restore_source_backup_arn"), Ok("backup")); + assert_eq!(filter.get_str("table_status"), Ok("CREATING")); + } + + /// The registration writes the provenance DescribeTable reports and the + /// job record RUNNING, as a legacy restore, under its generation. + #[test] + fn a_registration_writes_the_provenance_and_a_running_legacy_job_record() { + let registration = RestoreRegistration { + backup_arn: ARN, + source_table_arn: "arn:aws:dynamodb:us-east-1:123456789012:table/src", + restore_date_time: bson::DateTime::from_millis(1_791_976_431_438), + owner_generation: 7, + first_heartbeat: bson::DateTime::from_millis(1_791_976_500_000), + record: RestoreRecord::Legacy, + session: None, + }; + let mut table_doc = doc! { "table_status": "CREATING" }; + registration.insert_into(&mut table_doc); + assert_eq!(table_doc.get_str("restore_source_backup_arn"), Ok(ARN)); assert_eq!( - restore_claim_miss(Some(&doc! { "_id": ARN })), - RestoreClaimMiss::NotFound + table_doc.get_str("restore_source_table_arn"), + Ok("arn:aws:dynamodb:us-east-1:123456789012:table/src") ); - assert_eq!(restore_claim_miss(Some(&available)), RestoreClaimMiss::Free); - let mut d = available.clone(); - d.insert(RESTORE_CLAIM, "other"); assert_eq!( - restore_claim_miss(Some(&d)), - RestoreClaimMiss::RestoreClaimed + table_doc + .get_datetime("restore_date_time") + .map(|d| d.timestamp_millis()), + Ok(1_791_976_431_438) ); - d.insert(DELETE_CLAIM, "delete"); + let job = table_doc.get_document("restore_job").expect("job record"); + assert_eq!(job.get_str("status"), Ok("RUNNING")); + assert_eq!(job.get_str("kind"), Ok("legacy")); + assert_eq!(job.get_i64("owner_generation"), Ok(7)); assert_eq!( - restore_claim_miss(Some(&d)), - RestoreClaimMiss::DeleteClaimed + job.get_datetime("heartbeat_at") + .map(|d| d.timestamp_millis()), + Ok(1_791_976_500_000) ); + assert!(!table_doc.contains_key("restore_operation_id")); + assert!(!table_doc.contains_key("restore_heartbeat_at")); } - /// R4B-1: a DeleteBackup whose claim misses because another DeleteBackup - /// holds the backup answers BackupInUse, not Backup not found. + /// The manifest arm writes `kind = manifest`, the store the backup row + /// names and the claimant's spool; a filesystem store leaves + /// `spool_host` out. #[test] - fn a_delete_claim_miss_on_a_delete_claimed_backup_is_in_use() { - let in_use = |e: StorageError| match e { - StorageError::BackupInUse(m) => m, - other => panic!("expected BackupInUse, got {other:?}"), + fn a_manifest_registration_writes_the_kind_store_and_spool() { + use extenddb_storage::backup_catalog::{StoreIdentity, StoreKind}; + let store = StoreIdentity { + store_kind: StoreKind::S3, + store_root: "s3://bucket/prefix".to_owned(), + store_id: "store-7".to_owned(), }; - for restoring in [false, true] { - let m = in_use(delete_claim_miss_error( - restoring, - RestoreClaimMiss::DeleteClaimed, - ARN, - )); - assert!(m.starts_with("Backup is being deleted"), "{m}"); - let m = in_use(delete_claim_miss_error( - restoring, - RestoreClaimMiss::RestoreClaimed, - ARN, - )); - assert!( - m.starts_with("Backup is being used to restore another table"), - "{m}" - ); - } - for miss in [RestoreClaimMiss::NotFound, RestoreClaimMiss::Free] { - let m = in_use(delete_claim_miss_error(true, miss, ARN)); - assert!( - m.starts_with("Backup is being used to restore another table"), - "{m}" - ); - assert!(matches!( - delete_claim_miss_error(false, miss, ARN), - StorageError::Validation(m) if m.starts_with("Backup not found") - )); - } - } - - #[test] - fn the_sweep_summary_names_what_it_removed() { - assert!(BackupSweep::default().is_empty()); - assert_eq!(BackupSweep::default().to_string(), ""); - let sweep = BackupSweep { - restore_claims_cleared: vec!["a".to_owned()], - restore_targets_removed: vec!["acct/t1".to_owned(), "acct/t2".to_owned()], - delete_claims_finished: vec![], - delete_claims_waiting: vec!["b".to_owned()], + let registration = RestoreRegistration { + backup_arn: ARN, + source_table_arn: "arn:aws:dynamodb:us-east-1:123456789012:table/src", + restore_date_time: bson::DateTime::from_millis(1_791_976_431_438), + owner_generation: 9, + first_heartbeat: bson::DateTime::from_millis(1_791_976_500_000), + record: RestoreRecord::Manifest { + store: &store, + spool_host: Some("host-b:/var/spool"), + }, + session: None, + }; + let mut table_doc = doc! { "table_status": "CREATING" }; + registration.insert_into(&mut table_doc); + let job = table_doc.get_document("restore_job").expect("job record"); + assert_eq!(job.get_str("status"), Ok("RUNNING")); + assert_eq!(job.get_str("kind"), Ok("manifest")); + assert_eq!(job.get_str("store_kind"), Ok(StoreKind::S3.as_str())); + assert_eq!(job.get_str("store_root"), Ok("s3://bucket/prefix")); + assert_eq!(job.get_str("store_id"), Ok("store-7")); + assert_eq!(job.get_str("spool_host"), Ok("host-b:/var/spool")); + + // A filesystem store spools nothing. + let fs = StoreIdentity { + store_kind: StoreKind::Filesystem, + store_root: "/var/lib/extenddb/backups".to_owned(), + store_id: "store-1".to_owned(), }; - assert!(!sweep.is_empty()); + let registration = RestoreRegistration { + backup_arn: ARN, + source_table_arn: "x", + restore_date_time: bson::DateTime::from_millis(0), + owner_generation: 9, + first_heartbeat: bson::DateTime::from_millis(0), + record: RestoreRecord::Manifest { + store: &fs, + spool_host: None, + }, + session: None, + }; + let mut table_doc = doc! { "table_status": "CREATING" }; + registration.insert_into(&mut table_doc); + let job = table_doc.get_document("restore_job").expect("job record"); + assert_eq!(job.get_str("kind"), Ok("manifest")); assert_eq!( - sweep.to_string(), - "cleared abandoned restore claim(s) on [a]; removed abandoned restore target(s) \ - [acct/t1, acct/t2]; left interrupted DeleteBackup(s) to a later sweep (a \ - restore target exists) of [b]" + job.get_str("store_kind"), + Ok(StoreKind::Filesystem.as_str()) + ); + assert!( + job.get_str("spool_host").is_err(), + "no spool for a filesystem store" ); } } -/// The claim protocol model; see the module documentation. +/// The protocol model; see the module documentation. #[cfg(test)] #[path = "backup_protocol_model.rs"] mod protocol_model; diff --git a/crates/storage-mongodb/src/backup_live_tests.rs b/crates/storage-mongodb/src/backup_live_tests.rs index 6339e279f..6bb62bfa8 100644 --- a/crates/storage-mongodb/src/backup_live_tests.rs +++ b/crates/storage-mongodb/src/backup_live_tests.rs @@ -1,8 +1,10 @@ // Copyright 2026 ExtendDB contributors // SPDX-License-Identifier: Apache-2.0 -//! Live MongoDB tests for backup and restore: the detached copy, the -//! heartbeat sweep, and the DeleteBackup/DeleteTable refusals during a copy. +//! Live MongoDB tests for backup and restore: the catalog index and its +//! claim, the detached copy and its job record, the reconciler's duties over +//! an abandoned restore, and the DeleteBackup/DeleteTable refusals during a +//! copy. //! //! They need a replica set (the data path uses transactions) and read its URI //! from `EXTENDDB_TEST_MONGODB_URI`, for example @@ -11,35 +13,52 @@ //! `cargo test -p extenddb-storage-mongodb --lib -- --ignored live_`. //! A run that sets `--ignored` without the variable fails rather than skips. //! -//! Each test uses its own catalog and data databases (random names) and drops -//! them at the end, so tests run in parallel and leave nothing on success. Each -//! runs its own GSI backfill worker, as `serve` does, and every engine uses -//! shortened heartbeat timing (200 ms refresh, 1.5 s stale), which the tests -//! read from the engine rather than assume. +//! Each test uses its own catalog and data databases (random names), runs the +//! catalog migration step on them, and drops them at the end, so tests run in +//! parallel and leave nothing on success. Each runs its own GSI backfill +//! worker, as `serve` does, and every engine uses a shortened heartbeat +//! interval (200 ms), which the tests read from the engine rather than +//! assume; the stale window is the one the reconciler would pass. use std::sync::Arc; use std::time::{Duration, Instant}; +use extenddb_core::backup_format::BackupArn; use extenddb_core::types::{ AttributeDefinition, AttributeValue, BillingMode, CreateTableInput, DeleteTableInput, DescribeTableInput, GsiInput, Item, KeySchemaElement, KeyType, LsiInput, Projection, ProjectionType, ScalarAttributeType, TableStatus, }; +use extenddb_storage::backup_catalog::{ + BackupRow, BackupStatus, DeleteStart, Heartbeat, JobStatus, RestoreKind, + RestoreRegistrationRow, RestoreTargetKey, StaleClock, StatusFields, StoreIdentity, StoreKind, + TimeBounds, UNOWNED_GENERATION, WriteOutcome, +}; use extenddb_storage::error::StorageError; -use extenddb_storage::{BackupEngine, RestoreRequest, RestoreTableOverrides, TableEngine}; -use futures::FutureExt; +use extenddb_storage::restore_engine::{BulkLoadFile, Finished, RestoreEngine}; +use extenddb_storage::{ + BackupCatalog, JobPermit, LegacyBackupReader, RestoreRequest, RestoreTableOverrides, + TableEngine, +}; +use futures::stream::BoxStream; +use futures::{FutureExt, StreamExt, TryStreamExt}; use mongodb::bson::{self, Document, doc}; -use super::{ - AfterBackupDrop, BackupLiveness, DELETE_CLAIM, DELETE_CLAIM_HEARTBEAT, RESTORE_CLAIM, - RESTORE_CLAIM_HEARTBEAT, RestoreProvenance, TARGET_HEARTBEAT, claim_update, - restore_claim_filter, -}; +use super::{AfterBackupDrop, BackupLiveness, HeartbeatFault}; use crate::MongoEngine; -use crate::data::{data_collection_name, item_to_document}; +use crate::backup_catalog::{ + BACKUP_INDEX, RESTORE_JOB, RESTORE_JOB_GENERATION, RESTORE_JOB_HEARTBEAT, RESTORE_JOB_STATUS, +}; +use crate::data::{data_collection_name, document_to_item, item_to_document}; +use crate::migrations::{MongoTarget, run_backup_catalog_index}; +use crate::restore_engine::INSERT_MANY_DOCUMENTS; const ACCOUNT: &str = "123456789012"; +const OTHER_ACCOUNT: &str = "210987654321"; +const REGION: &str = "us-east-1"; const ITEMS: usize = 3_000; +/// The stale window the reconciler would pass (`stale_job_after_secs`). +const STALE_AFTER: Duration = Duration::from_secs(300); fn uri() -> String { std::env::var("EXTENDDB_TEST_MONGODB_URI").expect( @@ -48,14 +67,20 @@ fn uri() -> String { ) } +/// The deployment of the scratch databases `catalog` and `data`. +fn target(catalog: &str, data: &str) -> MongoTarget { + MongoTarget::with_databases(uri(), catalog, data) +} + fn test_liveness() -> BackupLiveness { BackupLiveness { heartbeat_interval: Duration::from_millis(200), - stale_after: Duration::from_millis(1_500), - slot_wait: Duration::from_secs(2), + registration_gate: None, copy_gate: None, after_backup_drop: None, + target_drop_fails: false, local_clock_behind: Duration::ZERO, + heartbeat_fault: None, } } @@ -78,6 +103,13 @@ impl Fixture { .insert_one(doc! { "_id": "control_plane_delay_seconds", "value": "0" }) .await .expect("settings"); + // Migration 6's catalog body alone, as `init` and `migrate --yes` + // run it: migration 1's collections and indexes are not created + // here, so this catalog is not `init`'s; the tests of this file need + // only what migration 6 creates. + run_backup_catalog_index(&target(&catalog, &data)) + .await + .expect("migration 6's body"); let worker = spawn_worker(&engine); Self { engine, @@ -102,20 +134,14 @@ impl Fixture { async fn engine_on(catalog: &str, data: &str, liveness: BackupLiveness) -> MongoEngine { let client = mongodb::Client::with_uri_str(uri()).await.expect("client"); - // The driver default pool (10) and the slot count production derives - // from it, so the tests see the same admission bound as `serve`. - let slots = super::restore_server_slot_count(10); MongoEngine { client: client.clone(), catalog_db: client.database(catalog), data_db: client.database(data), - region: "us-east-1".to_owned(), + region: REGION.to_owned(), gsi_cache: Arc::default(), tx_read_concern: mongodb::options::ReadConcern::snapshot(), backup_liveness: liveness, - restore_server_slot_count: slots, - restore_server_slots: Arc::new(tokio::sync::Semaphore::new(slots)), - restore_account_slots: super::RestoreAccountSlots::default(), } } @@ -229,7 +255,7 @@ async fn restore_request_for( target: &str, ) -> RestoreRequest { let source = engine - .backup_restore_source(account, backup_arn) + .restore_source(account, backup_arn) .await .expect("restore source"); source.ensure_restorable().expect("restorable"); @@ -242,13 +268,26 @@ async fn restore_request_for( } } +/// The `tables` collection read at `majority`: the state a registration's or +/// a DeleteBackup's transaction will see. The engine's transactions read at +/// its `tx_read_concern` (`snapshot` here), which is majority-committed data; +/// a read at the collection's default (`local`) sees the backfill worker's +/// ACTIVE flip as soon as it is applied on the primary, while the majority +/// commit point still waits for the journal and for any concurrent batch +/// write holding an earlier oplog slot (another test's 3,000-item seed), so +/// a test that observed ACTIVE at `local` and then issued a request could be +/// answered `BackupInUse` by a transaction that still saw CREATING. +fn tables_majority(engine: &MongoEngine) -> mongodb::Collection { + engine.catalog_db.collection_with_options( + "tables", + mongodb::options::CollectionOptions::builder() + .read_concern(mongodb::options::ReadConcern::majority()) + .build(), + ) +} + async fn table_doc(engine: &MongoEngine, name: &str) -> Option { - engine - .catalog_db - .collection::("tables") - .find_one(doc! { "_id": { "account_id": ACCOUNT, "table_name": name } }) - .await - .expect("read table") + table_doc_for(engine, ACCOUNT, name).await } async fn wait_for_status(engine: &MongoEngine, name: &str, status: &str) -> Document { @@ -342,10 +381,15 @@ async fn assert_fully_restored_with(engine: &MongoEngine, name: &str, items: usi "index {index_name}" ); } - assert!(!doc.contains_key(TARGET_HEARTBEAT)); - assert!(!doc.contains_key("restore_operation_id")); + // The job record went DONE with the hand-over and stayed with the table. + let job = doc.get_document(RESTORE_JOB).expect("job record"); + assert_eq!(job.get_str("status"), Ok("DONE")); + assert_eq!(job.get_str("kind"), Ok("legacy")); + assert!(job.get_datetime("completed_at").is_ok()); + assert!(!doc.contains_key("restore_backfill_pending")); } +/// An engine whose restores wait for one permit each before their copy. fn gated(engine: &MongoEngine) -> (MongoEngine, Arc) { let gate = Arc::new(tokio::sync::Semaphore::new(0)); let mut gated = engine.clone(); @@ -353,1527 +397,4602 @@ fn gated(engine: &MongoEngine) -> (MongoEngine, Arc) { (gated, gate) } -/// (a) The response is CREATING while the copy has not run; the target then -/// reaches ACTIVE with every item and index entry; the summary carries the -/// source table ARN and the backup's creation time. +/// An engine whose restores wait for one permit each before they register +/// their target, and one more before their copy. +fn gated_at_registration( + engine: &MongoEngine, +) -> ( + MongoEngine, + Arc, + Arc, +) { + let (mut gated, copy_gate) = gated(engine); + let registration_gate = Arc::new(tokio::sync::Semaphore::new(0)); + gated.backup_liveness.registration_gate = Some(registration_gate.clone()); + (gated, registration_gate, copy_gate) +} + +/// The names among `targets` that exist in the catalog: the catalog's account +/// of which restores registered, whatever their requests answered. +async fn registered_targets(engine: &MongoEngine, targets: &[String]) -> Vec { + let mut registered = Vec::new(); + for target in targets { + if table_doc(engine, target).await.is_some() { + registered.push(target.clone()); + } + } + registered +} + +/// The restore job record of `name`'s table document. +/// The server's clock (`$$NOW`), read through an aggregation. +async fn server_now(engine: &MongoEngine) -> bson::DateTime { + let mut cursor = engine + .catalog_db + .collection::("tables") + .aggregate(vec![ + doc! { "$limit": 1 }, + doc! { "$project": { "_id": 0, "now": "$$NOW" } }, + ]) + .await + .expect("aggregate"); + let doc = cursor + .next() + .await + .expect("one document") + .expect("the server's clock"); + *doc.get_datetime("now").expect("now") +} + +async fn job_record(engine: &MongoEngine, name: &str) -> Document { + table_doc(engine, name) + .await + .expect("target") + .get_document(RESTORE_JOB) + .expect("job record") + .clone() +} + +fn target_key(doc: &Document) -> RestoreTargetKey { + let id = doc.get_document("_id").expect("_id"); + RestoreTargetKey { + account_id: id.get_str("account_id").expect("account").to_owned(), + table_name: id.get_str("table_name").expect("name").to_owned(), + table_id: doc.get_str("table_id").expect("table_id").to_owned(), + } +} + +/// Leave the state a process killed mid-copy leaves: the target CREATING, its +/// job record RUNNING with a heartbeat `age` old, and no task. +async fn age_running_record(engine: &MongoEngine, name: &str, age: Duration) { + let heartbeat = bson::DateTime::from_millis( + bson::DateTime::now().timestamp_millis() - i64::try_from(age.as_millis()).expect("i64"), + ); + engine + .catalog_db + .collection::("tables") + .update_one( + doc! { "_id": { "account_id": ACCOUNT, "table_name": name } }, + doc! { "$set": { + "table_status": "CREATING", + "status_transition_at": bson::Bson::Null, + RESTORE_JOB_STATUS: "RUNNING", + RESTORE_JOB_HEARTBEAT: heartbeat, + }, "$unset": { "restore_job.completed_at": "", "restore_backfill_pending": "" } }, + ) + .await + .expect("age the record"); +} + +async fn backup_meta(engine: &MongoEngine, backup_arn: &str) -> Document { + engine + .catalog_db + .collection::("backups") + .find_one(doc! { "_id": backup_arn }) + .await + .expect("read backup") + .expect("backup document") +} + +fn with_drop_fault(engine: &MongoEngine, fault: AfterBackupDrop) -> MongoEngine { + let mut faulty = engine.clone(); + faulty.backup_liveness.after_backup_drop = Some(fault); + faulty +} + +fn store() -> StoreIdentity { + StoreIdentity { + store_kind: StoreKind::Filesystem, + store_root: "/var/lib/extenddb/backups".to_owned(), + store_id: "store-1".to_owned(), + } +} + +/// A `backup_index` row for a backup of `table` in `account` minted now, +/// with the manifest fixture's source table. +fn row(account: &str, table: &str, name: &str) -> BackupRow { + let backup_id = extenddb_storage::util::new_backup_id(); + let backup_arn = extenddb_storage::util::backup_arn(REGION, account, table, &backup_id); + let manifest: serde_json::Value = serde_json::from_str(include_str!( + "../../core/testdata/extenddb-manifest/v1.json" + )) + .expect("fixture"); + BackupRow { + backup_id: backup_id.clone(), + account_id: account.to_owned(), + table_name: table.to_owned(), + backup_arn, + backup_name: name.to_owned(), + status: BackupStatus::Creating, + store: store(), + size_bytes: 0, + item_count: 0, + created_at: extenddb_storage::util::backup_creation_date_time(&backup_id).unwrap(), + completed_at: None, + failure_reason: None, + source_table_details_json: manifest["source_table"].clone(), + owner_generation: 0, + heartbeat_at: 0.0, + } +} + +fn arn_of(row: &BackupRow) -> BackupArn { + BackupArn::parse(&row.backup_arn).unwrap() +} + +/// The claim: a second claim of the same backup is refused while the row +/// exists, the heartbeat confirms the claim and refreshes the row with the +/// server's clock, a status write is conditioned on the generation and the +/// `from` status, and the row is read back by account, backup id, table and +/// ARN. The catalog has an id and no maintenance claim until one is written. #[tokio::test(flavor = "multi_thread")] #[ignore = "needs a MongoDB replica set in EXTENDDB_TEST_MONGODB_URI"] -async fn live_restore_returns_creating_and_the_detached_copy_finishes() { +async fn live_claim_backup_excludes_a_second_owner_and_fences_its_writes() { let fx = Fixture::new().await; - let source_arn = source_table(&fx.engine, "src").await; - let backup = fx + assert!( + uuid::Uuid::parse_str(&fx.engine.catalog_id().await.expect("id")).is_ok(), + "the catalog has an id from the migration step" + ); + assert!( + !fx.engine + .check_maintenance_claim(std::time::Duration::from_secs(300)) + .await + .expect("check") + ); + let row = row(ACCOUNT, "Music", "nightly"); + let claim = fx.engine.claim_backup(&row).await.expect("claim"); + assert_ne!(claim.generation(), UNOWNED_GENERATION); + let second = fx.engine.claim_backup(&row).await.err(); + assert!( + matches!(second, Some(StorageError::AlreadyExists(_))), + "{second:?}" + ); + let stored = fx .engine - .create_backup(ACCOUNT, "src", "b") + .read_backup_row(&arn_of(&row)) .await - .expect("backup"); - let request = restore_request(&fx.engine, &backup.backup_arn, "dst").await; - let (gated, gate) = gated(&fx.engine); - - let desc = tokio::time::timeout( - Duration::from_secs(20), - gated.restore_table_from_backup(ACCOUNT, request), - ) - .await - .expect("the restore must return while its copy is held") - .expect("restore"); - assert_eq!(desc.table_status, TableStatus::Creating); - let doc = table_doc(&fx.engine, "dst").await.expect("target"); - assert_eq!(doc.get_str("table_status"), Ok("CREATING")); + .expect("read") + .expect("the row exists"); + assert_eq!(stored.status, BackupStatus::Creating); + assert_eq!(stored.owner_generation, claim.generation()); + assert_eq!(stored.store, store()); + assert_eq!(stored.created_at, row.created_at); assert_eq!( - count(&fx.engine, &data_collection_name(&desc.table_id)).await, - 0, - "no item may be copied before the response" + stored.source_table_details_json, + row.source_table_details_json ); - let summary = fx + assert!( + stored.heartbeat_at > 0.0, + "the heartbeat is the server's now" + ); + let before = stored.heartbeat_at; + tokio::time::sleep(Duration::from_millis(20)).await; + assert_eq!( + claim.heartbeat().await.expect("heartbeat"), + Heartbeat::Confirmed + ); + let after = fx .engine - .describe_table( - ACCOUNT, - DescribeTableInput { - table_name: "dst".to_owned(), - }, - ) + .read_backup_row(&arn_of(&row)) .await - .expect("describe") - .restore_summary - .expect("restore summary while CREATING"); + .unwrap() + .unwrap() + .heartbeat_at; + assert!(after > before, "{after} > {before}"); + + // The same id under another account, table or region finds nothing. + let other_account = + BackupArn::new("aws", REGION, OTHER_ACCOUNT, "Music", &row.backup_id).unwrap(); + assert!( + fx.engine + .read_backup_row(&other_account) + .await + .unwrap() + .is_none() + ); + let other_table = BackupArn::new("aws", REGION, ACCOUNT, "Other", &row.backup_id).unwrap(); + assert!( + fx.engine + .read_backup_row(&other_table) + .await + .unwrap() + .is_none() + ); + let other_region = + BackupArn::new("aws", "eu-west-1", ACCOUNT, "Music", &row.backup_id).unwrap(); + assert!( + fx.engine + .read_backup_row(&other_region) + .await + .unwrap() + .is_none() + ); + + // A write from the wrong state matches nothing; the right one matches + // once, and the row then carries what it set. + let fields = StatusFields { + size_bytes: Some(1234), + item_count: Some(5), + failure_reason: None, + source_table_details_json: None, + completed: true, + }; assert_eq!( - summary.source_backup_arn.as_deref(), - Some(backup.backup_arn.as_str()) + claim + .set_status(JobStatus::Running, JobStatus::Available, &fields) + .await + .unwrap(), + WriteOutcome::NotMatched ); assert_eq!( - summary.source_table_arn.as_deref(), - Some(source_arn.as_str()) + claim + .set_status(JobStatus::Creating, JobStatus::Available, &fields) + .await + .unwrap(), + WriteOutcome::Matched ); - assert_eq!(summary.restore_date_time, backup.backup_creation_date_time); - assert!(summary.restore_in_progress); - // The backup time is the shared id's instant, kept to the millisecond. assert_eq!( - Some(backup.backup_creation_date_time), - extenddb_storage::util::backup_creation_date_time(&backup.backup_arn) + claim + .set_status(JobStatus::Creating, JobStatus::Available, &fields) + .await + .unwrap(), + WriteOutcome::NotMatched + ); + let read = claim.read().await.expect("read"); + assert_eq!(read.status, JobStatus::Available); + assert_eq!(read.owner_generation, claim.generation()); + let stored = fx + .engine + .read_backup_row(&arn_of(&row)) + .await + .unwrap() + .unwrap(); + assert_eq!(stored.status, BackupStatus::Available); + assert_eq!(stored.size_bytes, 1234); + assert_eq!(stored.item_count, 5); + assert!(stored.completed_at.is_some()); + + // A row another generation took over confirms nothing for this claim. + fx.engine + .catalog_db + .collection::(BACKUP_INDEX) + .update_one( + doc! { "backup_id": &row.backup_id }, + doc! { "$inc": { "owner_generation": 1_i64 } }, + ) + .await + .expect("take over"); + assert_eq!(claim.heartbeat().await.expect("heartbeat"), Heartbeat::Lost); + assert_eq!( + claim + .set_status( + JobStatus::Available, + JobStatus::Failed, + &StatusFields::default() + ) + .await + .unwrap(), + WriteOutcome::NotMatched + ); + drop(claim); + // The row is the claim: a successor can take it only through its + // insert, which the existing row refuses. + let third = fx.engine.claim_backup(&row).await.err(); + assert!( + matches!(third, Some(StorageError::AlreadyExists(_))), + "{third:?}" ); - gate.add_permits(1); - assert_fully_restored(&fx.engine, "dst").await; fx.finish().await; } -/// (b) A request future dropped after its first poll still produces a -/// complete restore, whether the poll parked in the registration or in the -/// wait for an account slot. +/// `read_store_ids`: the distinct ids of the rows for a root, of every +/// status, scoped to the kind and the root. #[tokio::test(flavor = "multi_thread")] #[ignore = "needs a MongoDB replica set in EXTENDDB_TEST_MONGODB_URI"] -async fn live_a_dropped_restore_request_still_reaches_active() { +async fn live_read_store_ids_lists_the_distinct_ids_of_a_root() { let fx = Fixture::new().await; - source_table(&fx.engine, "src").await; - let backup = fx - .engine - .create_backup(ACCOUNT, "src", "b") - .await - .expect("backup"); - let request = restore_request(&fx.engine, &backup.backup_arn, "dst").await; - let mut future = fx.engine.restore_table_from_backup(ACCOUNT, request); - assert!( - (&mut future).now_or_never().is_none(), - "the first poll must not complete the restore" + let root = store().store_root.clone(); + let ids = |kind: StoreKind, root: String| { + let engine = &fx.engine; + async move { engine.read_store_ids(kind, &root).await.expect("ids") } + }; + assert!(ids(StoreKind::Filesystem, root.clone()).await.is_empty()); + let first = row(ACCOUNT, "Music", "a"); + let _c1 = fx.engine.claim_backup(&first).await.expect("claim"); + let second = row(ACCOUNT, "Music", "b"); + let c2 = fx.engine.claim_backup(&second).await.expect("claim"); + c2.set_status( + JobStatus::Creating, + JobStatus::Failed, + &StatusFields::default(), + ) + .await + .expect("fail"); + assert_eq!( + ids(StoreKind::Filesystem, root.clone()).await, + ["store-1"], + "two rows of one store, of two statuses, are one id" + ); + let mut other = row(ACCOUNT, "Songs", "c"); + other.store.store_id = "store-2".to_owned(); + let _c3 = fx.engine.claim_backup(&other).await.expect("claim"); + assert_eq!( + ids(StoreKind::Filesystem, root.clone()).await, + ["store-1", "store-2"] ); - drop(future); - assert_fully_restored(&fx.engine, "dst").await; - - // Every account slot is held, so the request's first poll parks in the - // admission wait; the slots are freed only after the future is gone. - let account_slots = fx.engine.restore_account_slots.for_account(ACCOUNT); - let held = Arc::clone(&account_slots) - .acquire_many_owned(u32::try_from(account_slots.available_permits()).expect("u32")) - .await - .expect("hold every account slot"); - let backup = fx - .engine - .create_backup(ACCOUNT, "src", "b2") - .await - .expect("backup"); - let request = restore_request(&fx.engine, &backup.backup_arn, "dst2").await; - let mut future = fx.engine.restore_table_from_backup(ACCOUNT, request); assert!( - (&mut future).now_or_never().is_none(), - "the first poll must not complete the restore" + ids(StoreKind::S3, root.clone()).await.is_empty(), + "another kind at the same root is another store" ); - drop(future); assert!( - table_doc(&fx.engine, "dst2").await.is_none(), - "nothing is created while the restore waits for a slot" + ids(StoreKind::Filesystem, "/elsewhere".to_owned()) + .await + .is_empty() ); - drop(held); - assert_fully_restored(&fx.engine, "dst2").await; fx.finish().await; } -/// Leave the state a process killed mid-restore leaves: the backup's restore -/// claim and a CREATING target with provenance, both with a heartbeat far in -/// the past, and no task. -async fn simulate_crashed_restore(engine: &MongoEngine, backup_arn: &str, target: &str) -> String { - let dead = "dead-operation"; - let backups = engine.catalog_db.collection::("backups"); - let claimed = backups - .find_one_and_update( - restore_claim_filter(ACCOUNT, backup_arn), - claim_update(RESTORE_CLAIM, RESTORE_CLAIM_HEARTBEAT, dead), - ) - .await - .expect("claim") - .expect("claimable"); - let request = restore_request(engine, backup_arn, target).await; - let desc = engine - .create_table_for_restore( - ACCOUNT, - request.input, - &RestoreProvenance { - backup_arn, - source_table_arn: &request.source_table_arn, - restore_date_time: bson::DateTime::now(), - operation_id: dead, - first_heartbeat: bson::DateTime::now(), - inserted: None, - }, - ) - .await - .expect("target"); - drop(claimed); - let long_ago = bson::DateTime::from_millis(bson::DateTime::now().timestamp_millis() - 600_000); - let mut set = Document::new(); - set.insert(RESTORE_CLAIM_HEARTBEAT, long_ago); - backups - .update_one(doc! { "_id": backup_arn }, doc! { "$set": set }) - .await - .expect("age claim"); - let mut set = Document::new(); - set.insert(TARGET_HEARTBEAT, long_ago); - engine - .catalog_db - .collection::("tables") - .update_one( - doc! { "_id": { "account_id": ACCOUNT, "table_name": target } }, - doc! { "$set": set }, - ) - .await - .expect("age target"); - desc.table_id -} - -/// (c) A crashed restore is swept completely; DeleteBackup of its source then -/// succeeds and the target name can be restored into. +/// `insert_backup_row`: the row lands as given, unowned (the document +/// carries `owner_generation: 0`, which the claim's upsert filter relies on), +/// with `completed_at` and `heartbeat_at` on the server's clock; a second +/// insert, an insert over a CREATING row a claim holds, and an insert of the +/// id under another account (the unique `backup_id`) answer `false` and +/// change nothing; a claim over the inserted row is refused; the row is read +/// and listed like a created one. #[tokio::test(flavor = "multi_thread")] #[ignore = "needs a MongoDB replica set in EXTENDDB_TEST_MONGODB_URI"] -async fn live_an_abandoned_restore_is_swept_and_frees_the_backup_and_the_name() { +async fn live_insert_backup_row_inserts_an_unowned_row_only_when_absent() { let fx = Fixture::new().await; - source_table(&fx.engine, "src").await; - let crashed = fx - .engine - .create_backup(ACCOUNT, "src", "b1") - .await - .expect("backup 1"); - let spare = fx + let mut adopted = row(ACCOUNT, "Music", "adopted"); + adopted.status = BackupStatus::Available; + adopted.size_bytes = 1234; + adopted.item_count = 5; + // Ignored: the insert sets these three itself. + adopted.owner_generation = 99; + adopted.completed_at = None; + adopted.heartbeat_at = 0.0; + assert!(fx.engine.insert_backup_row(&adopted).await.expect("insert")); + let stored = fx .engine - .create_backup(ACCOUNT, "src", "b2") + .read_backup_row(&arn_of(&adopted)) .await - .expect("backup 2"); - // The periodic sweep in the fixture's worker would race this test's own - // sweep; stop it so the assertions see exactly one sweep's result. - fx.worker.abort(); - let table_id = simulate_crashed_restore(&fx.engine, &crashed.backup_arn, "dst").await; - let indexes = index_ids(&fx.engine, &table_id).await; - assert_eq!(indexes.len(), 2); - assert!(collection_exists(&fx.engine, &data_collection_name(&table_id)).await); - - let before = fx.engine.delete_backup(ACCOUNT, &crashed.backup_arn).await; + .expect("read") + .expect("the row exists"); + assert_eq!(stored.owner_generation, UNOWNED_GENERATION); + let completed_at = stored.completed_at.expect("completed_at is set"); + for at in [completed_at, stored.heartbeat_at] { + assert!((at - adopted.created_at).abs() < 300.0, "{at}"); + } assert!( - matches!(before, Err(StorageError::BackupInUse(_))), - "the crashed restore blocks DeleteBackup before the sweep: {before:?}" + (stored.created_at - adopted.created_at).abs() < 1e-3, + "{} is the id's instant {}", + stored.created_at, + adopted.created_at ); - - let sweep = fx.engine.sweep_stale_backup_claims().await.expect("sweep"); assert_eq!( - sweep.restore_claims_cleared, - std::slice::from_ref(&crashed.backup_arn) + BackupRow { + owner_generation: 99, + completed_at: None, + heartbeat_at: 0.0, + created_at: adopted.created_at, + ..stored.clone() + }, + adopted, + "every other column is the row as given" + ); + + // A second insert of the key, with other values, writes nothing. + let mut again = adopted.clone(); + again.backup_name = "again".to_owned(); + again.size_bytes = 1; + again.status = BackupStatus::Failed; + assert!(!fx.engine.insert_backup_row(&again).await.expect("insert")); + // The id under another account: the unique `backup_id`. + let mut elsewhere = adopted.clone(); + elsewhere.account_id = OTHER_ACCOUNT.to_owned(); + elsewhere.backup_arn = + extenddb_storage::util::backup_arn(REGION, OTHER_ACCOUNT, "Music", &adopted.backup_id); + assert!( + !fx.engine + .insert_backup_row(&elsewhere) + .await + .expect("insert") + ); + assert!( + fx.engine + .read_backup_row(&arn_of(&elsewhere)) + .await + .unwrap() + .is_none() ); assert_eq!( - sweep.restore_targets_removed, - [format!("{ACCOUNT}/dst")], - "the abandoned target must be removed" + fx.engine.read_backup_row(&arn_of(&adopted)).await.unwrap(), + Some(stored.clone()), + "the refused inserts changed nothing" ); - assert!(table_doc(&fx.engine, "dst").await.is_none()); - assert!(!collection_exists(&fx.engine, &data_collection_name(&table_id)).await); - for (_, index_id, _) in &indexes { - assert!(!collection_exists(&fx.engine, &data_collection_name(index_id)).await); - } - assert!(index_ids(&fx.engine, &table_id).await.is_empty()); - let meta = fx + let claimed = fx.engine.claim_backup(&adopted).await.err(); + assert!( + matches!(claimed, Some(StorageError::AlreadyExists(_))), + "{claimed:?}" + ); + + // Over a CREATING row a live claim holds. + let creating = row(ACCOUNT, "Music", "job"); + let claim = fx.engine.claim_backup(&creating).await.expect("claim"); + let mut over = creating.clone(); + over.status = BackupStatus::Available; + assert!(!fx.engine.insert_backup_row(&over).await.expect("insert")); + let job_row = fx .engine - .catalog_db - .collection::("backups") - .find_one(doc! { "_id": &crashed.backup_arn }) + .read_backup_row(&arn_of(&creating)) .await - .expect("read") - .expect("backup"); - assert!(!meta.contains_key(RESTORE_CLAIM) && !meta.contains_key(RESTORE_CLAIM_HEARTBEAT)); + .unwrap() + .unwrap(); + assert_eq!(job_row.status, BackupStatus::Creating); + assert_eq!(job_row.owner_generation, claim.generation()); - let deleted = fx + // Read and listed like a created row. + let arns = + |rows: Vec| -> Vec { rows.into_iter().map(|r| r.backup_arn).collect() }; + let available: Vec = fx .engine - .delete_backup(ACCOUNT, &crashed.backup_arn) + .list_backup_rows(Some(BackupStatus::Available), &store()) + .try_collect() .await - .expect("DeleteBackup after the sweep"); - assert_eq!(deleted.backup_details.backup_status, "DELETED"); + .unwrap(); + assert_eq!(available, std::slice::from_ref(&stored)); + let creating_rows: Vec = fx + .engine + .list_backup_rows(Some(BackupStatus::Creating), &store()) + .try_collect() + .await + .unwrap(); + assert_eq!( + arns(creating_rows), + std::slice::from_ref(&creating.backup_arn) + ); + for table in [None, Some("Music")] { + let listed: Vec = fx + .engine + .list_account_backup_rows(ACCOUNT, table, &TimeBounds::default()) + .try_collect() + .await + .unwrap(); + assert!( + listed.contains(&stored), + "{table:?}: {:?}", + arns(listed.clone()) + ); + } + drop(claim); - let worker = spawn_worker(&fx.engine); - let request = restore_request(&fx.engine, &spare.backup_arn, "dst").await; - fx.engine - .restore_table_from_backup(ACCOUNT, request) + let document = fx + .engine + .backup_index() + .find_one(doc! { "backup_id": &adopted.backup_id }) .await - .expect("restore into the freed name"); - assert_fully_restored(&fx.engine, "dst").await; - worker.abort(); + .expect("read") + .expect("the document"); + assert_eq!(document.get_i64("owner_generation"), Ok(0), "{document}"); + assert_eq!( + fx.engine + .backup_index() + .count_documents(doc! { "backup_id": &adopted.backup_id }) + .await + .expect("count"), + 1 + ); fx.finish().await; } -/// (d) Claims and restore targets of another, live, instance on the same -/// catalog survive this instance's sweep for many stale windows, because their -/// heartbeats keep moving; once the owner stops, the sweep removes them. +/// `remove_dead_creating_backup_row`: a CREATING row whose heartbeat is older +/// than the window is removed outright; one with a live owner, one that is +/// not CREATING, and one that is gone are left, answering `false`. #[tokio::test(flavor = "multi_thread")] #[ignore = "needs a MongoDB replica set in EXTENDDB_TEST_MONGODB_URI"] -async fn live_a_live_claim_of_another_instance_survives_this_instances_sweep() { +async fn live_a_dead_creating_row_is_removed_and_a_live_or_moved_one_is_kept() { let fx = Fixture::new().await; - fx.worker.abort(); - source_table(&fx.engine, "src").await; - let b1 = fx - .engine - .create_backup(ACCOUNT, "src", "b1") - .await - .expect("backup 1"); - let b2 = fx - .engine - .create_backup(ACCOUNT, "src", "b2") - .await - .expect("backup 2"); - let other = fx.second_instance(test_liveness()).await; - let stale_after = fx.engine.backup_liveness.stale_after; - assert!(other.backup_liveness.heartbeat_interval * 3 < stale_after); - - // A live delete claim on b1, held the way delete_backup holds it. - let op = "live-delete"; - let heartbeat = other.start_backup_heartbeat(op, ACCOUNT, &b1.backup_arn, None); - other - .catalog_db - .collection::("backups") - .update_one( - doc! { "_id": &b1.backup_arn }, - claim_update(DELETE_CLAIM, DELETE_CLAIM_HEARTBEAT, op), - ) - .await - .expect("claim"); - // A live restore target restoring b2, held before its copy. - let (gated_other, gate) = gated(&other); - let request = restore_request(&other, &b2.backup_arn, "dst").await; - gated_other - .restore_table_from_backup(ACCOUNT, request) - .await - .expect("restore"); - - let backups = fx.engine.catalog_db.collection::("backups"); - let claim_heartbeat = || async { - backups - .find_one(doc! { "_id": &b1.backup_arn }) - .await - .expect("read") - .and_then(|d| d.get_datetime(DELETE_CLAIM_HEARTBEAT).ok().copied()) + let stale = std::time::Duration::from_secs(300); + let index = fx.engine.backup_index(); + let age = |backup_id: String| { + let index = index.clone(); + async move { + index + .update_one( + doc! { "backup_id": backup_id }, + vec![doc! { "$set": { "heartbeat_at": { "$subtract": [ "$$NOW", 600_000_i64 ] } } }], + ) + .await + .expect("age"); + } }; - let target_heartbeat = || async { - table_doc(&fx.engine, "dst") + let creating = row(ACCOUNT, "Music", "a"); + let key = creating.key(); + let claim = fx.engine.claim_backup(&creating).await.expect("claim"); + assert!( + !fx.engine + .remove_dead_creating_backup_row(&key, stale) .await - .and_then(|d| d.get_datetime(TARGET_HEARTBEAT).ok().copied()) - }; - let first_claim = claim_heartbeat().await.expect("claim heartbeat"); - let first_target = target_heartbeat().await.expect("target heartbeat"); - - // Sweep repeatedly across three stale windows. - let until = Instant::now() + stale_after * 3; - let mut sweeps = 0; - while Instant::now() < until { - let sweep = fx.engine.sweep_stale_backup_claims().await.expect("sweep"); - assert!(sweep.is_empty(), "a live owner's state was swept: {sweep}"); - sweeps += 1; - tokio::time::sleep(Duration::from_millis(250)).await; - } - assert!(sweeps >= 10); - let meta = backups - .find_one(doc! { "_id": &b1.backup_arn }) - .await - .expect("read") - .expect("backup"); - assert_eq!( - meta.get_str(DELETE_CLAIM), - Ok(op), - "the live claim must still be held" + .unwrap(), + "a live owner keeps its row" ); + age(creating.backup_id.clone()).await; assert!( - table_doc(&fx.engine, "dst").await.is_some(), - "the live target must remain" + fx.engine + .remove_dead_creating_backup_row(&key, stale) + .await + .unwrap() ); - // Liveness: both heartbeats moved. - assert!(claim_heartbeat().await.expect("claim heartbeat") > first_claim); - assert!(target_heartbeat().await.expect("target heartbeat") > first_target); - let in_use = fx.engine.delete_backup(ACCOUNT, &b2.backup_arn).await; assert!( - matches!(in_use, Err(StorageError::BackupInUse(_))), - "{in_use:?}" + fx.engine + .read_backup_row(&arn_of(&creating)) + .await + .unwrap() + .is_none() + ); + assert!( + !fx.engine + .remove_dead_creating_backup_row(&key, stale) + .await + .unwrap(), + "gone" ); + drop(claim); - // The owner of the delete claim stops: the sweep takes the claim over and - // finishes the delete once the heartbeat is stale. - drop(heartbeat); - tokio::time::sleep(stale_after + Duration::from_millis(500)).await; - let sweep = fx.engine.sweep_stale_backup_claims().await.expect("sweep"); + let available = row(ACCOUNT, "Music", "b"); + let claim = fx.engine.claim_backup(&available).await.expect("claim"); + claim + .set_status( + JobStatus::Creating, + JobStatus::Available, + &StatusFields::default(), + ) + .await + .expect("available"); + age(available.backup_id.clone()).await; + assert!( + !fx.engine + .remove_dead_creating_backup_row(&available.key(), stale) + .await + .unwrap(), + "only a CREATING row" + ); assert_eq!( - sweep.delete_claims_finished, - std::slice::from_ref(&b1.backup_arn) + fx.engine + .read_backup_row(&arn_of(&available)) + .await + .unwrap() + .unwrap() + .status, + BackupStatus::Available ); - assert!(sweep.restore_targets_removed.is_empty(), "{sweep}"); - - // The restore finishes normally. - let worker = spawn_worker(&fx.engine); - gate.add_permits(1); - assert_fully_restored(&fx.engine, "dst").await; - worker.abort(); + drop(claim); fx.finish().await; } -/// (e) While the copy runs, DeleteBackup of the source is BackupInUse and -/// DeleteTable of the target is refused; both succeed once it is ACTIVE. +/// The `backup_index` collection read at `majority`, for the observations +/// a test makes between the writes under test. +fn index_majority(engine: &MongoEngine) -> mongodb::Collection { + engine.catalog_db.collection_with_options( + BACKUP_INDEX, + mongodb::options::CollectionOptions::builder() + .read_concern(mongodb::options::ReadConcern::majority()) + .build(), + ) +} + +async fn index_doc(engine: &MongoEngine, row: &BackupRow) -> Document { + index_majority(engine) + .find_one(doc! { "account_id": &row.account_id, "backup_id": &row.backup_id }) + .await + .expect("read the row") + .expect("the row exists") +} + +/// `claim_dead_backup`: a CREATING row is refused while its owner's heartbeat +/// is within the window, and taken over once the window has passed on the +/// server's clock (no hand-dating: the test waits), under a fresh generation +/// and with `heartbeat_at` as the dead owner left it; the old claim then +/// finds its heartbeat lost and its status write unmatched; the successor's +/// heartbeat makes it a live owner a further taker is refused, and its +/// AVAILABLE write carries the source table object and stamps +/// `completed_at`. An AVAILABLE row and a missing row are refused. #[tokio::test(flavor = "multi_thread")] #[ignore = "needs a MongoDB replica set in EXTENDDB_TEST_MONGODB_URI"] -async fn live_backup_and_target_are_protected_while_the_copy_runs() { +async fn live_claim_dead_backup_takes_over_a_stale_row_and_refuses_a_live_one() { let fx = Fixture::new().await; - source_table(&fx.engine, "src").await; - let backup = fx - .engine - .create_backup(ACCOUNT, "src", "b") - .await - .expect("backup"); - let request = restore_request(&fx.engine, &backup.backup_arn, "dst").await; - let (gated, gate) = gated(&fx.engine); - gated - .restore_table_from_backup(ACCOUNT, request) - .await - .expect("restore"); - - let in_use = fx.engine.delete_backup(ACCOUNT, &backup.backup_arn).await; + let window = Duration::from_secs(3); + let dead = row(ACCOUNT, "Music", "a"); + let key = dead.key(); + let owner = fx.engine.claim_backup(&dead).await.expect("claim"); + assert_eq!(owner.heartbeat().await.unwrap(), Heartbeat::Confirmed); assert!( - matches!(in_use, Err(StorageError::BackupInUse(_))), - "the source backup is in use while the copy runs: {in_use:?}" + fx.engine + .claim_dead_backup(&key, STALE_AFTER) + .await + .unwrap() + .is_none(), + "a live owner" ); - let refused = fx - .engine - .delete_table( - ACCOUNT, - DeleteTableInput { - table_name: "dst".to_owned(), - }, - ) - .await; assert!( - matches!(&refused, Err(StorageError::IndexesInUse(m)) if m.contains("being restored")), - "DeleteTable of a target being restored is refused: {refused:?}" + fx.engine + .claim_dead_backup(&key, window) + .await + .unwrap() + .is_none(), + "a heartbeat within the window" + ); + let before = index_doc(&fx.engine, &dead).await; + let heartbeat_before = *before.get_datetime("heartbeat_at").expect("heartbeat_at"); + tokio::time::sleep(window + Duration::from_millis(300)).await; + let successor = fx + .engine + .claim_dead_backup(&key, window) + .await + .unwrap() + .expect("the window has passed on the server's clock"); + assert_ne!(successor.generation(), owner.generation()); + assert_ne!(successor.generation(), UNOWNED_GENERATION); + let after = index_doc(&fx.engine, &dead).await; + assert_eq!( + *after.get_datetime("heartbeat_at").expect("heartbeat_at"), + heartbeat_before, + "the takeover left the heartbeat as the dead owner wrote it" ); + assert_eq!(after.get_str("status"), Ok("CREATING")); assert_eq!( - table_doc(&fx.engine, "dst") + after.get_i64("owner_generation").unwrap(), + i64::try_from(successor.generation()).unwrap() + ); + assert_eq!(owner.heartbeat().await.unwrap(), Heartbeat::Lost); + assert_eq!( + owner + .set_status( + JobStatus::Creating, + JobStatus::Available, + &StatusFields::default() + ) .await - .expect("target") - .get_str("table_status"), - Ok("CREATING"), - "both calls ran inside the copy window" + .unwrap(), + WriteOutcome::NotMatched, + "the old generation is fenced" ); + assert_eq!(successor.heartbeat().await.unwrap(), Heartbeat::Confirmed); - gate.add_permits(1); - assert_fully_restored(&fx.engine, "dst").await; - fx.engine - .delete_table( - ACCOUNT, - DeleteTableInput { - table_name: "dst".to_owned(), - }, - ) - .await - .expect("DeleteTable after ACTIVE"); - let deleted = fx + // A second taker while the row is held: the heartbeat the successor + // just refreshed is within the window, so it is refused; another + // process's reconciler racing the takeover itself, before any + // heartbeat, would take the row again and fence the successor. + assert!( + fx.engine + .claim_dead_backup(&key, window) + .await + .unwrap() + .is_none(), + "a heartbeating successor is a live owner" + ); + + let mut source = dead.source_table_details_json.clone(); + source["item_count"] = serde_json::json!(4242); + let fields = StatusFields { + size_bytes: Some(777), + item_count: Some(4242), + failure_reason: None, + source_table_details_json: Some(source.clone()), + completed: true, + }; + assert_eq!( + successor + .set_status(JobStatus::Creating, JobStatus::Available, &fields) + .await + .unwrap(), + WriteOutcome::Matched + ); + let stored = fx .engine - .delete_backup(ACCOUNT, &backup.backup_arn) + .read_backup_row(&arn_of(&dead)) .await - .expect("DeleteBackup after ACTIVE"); - assert_eq!(deleted.backup_details.backup_status, "DELETED"); + .unwrap() + .unwrap(); + assert_eq!(stored.status, BackupStatus::Available); + assert_eq!(stored.size_bytes, 777); + assert_eq!(stored.item_count, 4242); + assert_eq!( + stored.source_table_details_json, source, + "the AVAILABLE write carries the source table object" + ); + assert_eq!(stored.source_table().unwrap().item_count, 4242); + assert!(stored.completed_at.is_some()); + assert_eq!(stored.owner_generation, successor.generation()); + + tokio::time::sleep(window + Duration::from_millis(300)).await; + assert!( + fx.engine + .claim_dead_backup(&key, window) + .await + .unwrap() + .is_none(), + "an AVAILABLE row with a stale heartbeat is not CREATING" + ); + let missing = row(ACCOUNT, "Music", "none").key(); + assert!( + fx.engine + .claim_dead_backup(&missing, window) + .await + .unwrap() + .is_none(), + "a missing row" + ); + drop(owner); + drop(successor); fx.finish().await; } -/// The definition captured at CreateBackup carries TTL and the stream -/// specification, and ListBackups reports the source table id. +/// `list_backup_rows_older_than`: the rows of the store with the status +/// whose named clock column is older than the age on the server's clock. A +/// row FAILED just now by a dead owner is listed on the heartbeat clock (its +/// heartbeat is the owner's last) and not yet on the completed clock, which +/// tells the two columns apart; the completed window is then crossed by +/// waiting. A FAILED row with no `completed_at` is never listed on the +/// completed clock. A CREATING row, and a FAILED row of another store, are +/// not listed under FAILED; a DELETED row is listed under DELETED on its +/// `completed_at`. The listing keeps the catalog's order. #[tokio::test(flavor = "multi_thread")] #[ignore = "needs a MongoDB replica set in EXTENDDB_TEST_MONGODB_URI"] -async fn live_create_backup_captures_ttl_stream_and_table_id() { - use extenddb_storage::MetadataEngine; +async fn live_rows_older_than_are_listed_on_the_named_clock() { let fx = Fixture::new().await; - source_table(&fx.engine, "src").await; - fx.engine - .update_ttl(ACCOUNT, "src", "expires", true) - .await - .expect("ttl"); - let table = table_doc(&fx.engine, "src").await.expect("src"); - let table_id = table.get_str("table_id").expect("id").to_owned(); - let mut backups = Vec::new(); - for i in 0..3 { - backups.push( - fx.engine - .create_backup(ACCOUNT, "src", &format!("b{i}")) + let window = Duration::from_secs(3); + let names = |listed: Vec| -> Vec { + listed.into_iter().map(|r| r.backup_name).collect() + }; + let older = |status: BackupStatus, store: StoreIdentity, clock: StaleClock| { + let engine = &fx.engine; + async move { + let rows: Vec = engine + .list_backup_rows_older_than(status, &store, clock, window) + .try_collect() .await - .expect("backup"), - ); - } - for b in &backups { - let source = fx - .engine - .backup_restore_source(ACCOUNT, &b.backup_arn) + .expect("listing"); + names(rows) + } + }; + let mut other_store = store(); + other_store.store_id = "store-2".to_owned(); + + let completed = row(ACCOUNT, "Music", "completed"); + let completed_claim = fx.engine.claim_backup(&completed).await.expect("claim"); + tokio::time::sleep(Duration::from_millis(5)).await; + let unstamped = row(ACCOUNT, "Music", "unstamped"); + let unstamped_claim = fx.engine.claim_backup(&unstamped).await.expect("claim"); + tokio::time::sleep(Duration::from_millis(5)).await; + let creating = row(ACCOUNT, "Music", "creating"); + let _creating_claim = fx.engine.claim_backup(&creating).await.expect("claim"); + let mut elsewhere = row(ACCOUNT, "Music", "elsewhere"); + elsewhere.store = other_store.clone(); + let elsewhere_claim = fx.engine.claim_backup(&elsewhere).await.expect("claim"); + let deleted = row(ACCOUNT, "Music", "deleted"); + let deleted_claim = fx.engine.claim_backup(&deleted).await.expect("claim"); + let done = StatusFields { + completed: true, + ..StatusFields::default() + }; + assert_eq!( + deleted_claim + .set_status(JobStatus::Creating, JobStatus::Available, &done) .await - .expect("source"); - assert_eq!( - Some(source.backup_creation_date_time), - extenddb_storage::util::backup_creation_date_time(&b.backup_arn) - ); - assert_eq!( - source.backup_creation_date_time, - b.backup_creation_date_time - ); - let def = source.definition.expect("definition"); - let ttl = def.time_to_live.expect("ttl captured"); - assert_eq!(ttl.attribute_name.as_deref(), Some("expires")); - assert!(def.stream_specification.is_none()); - assert_eq!(def.global_secondary_indexes.len(), 1); - assert_eq!(def.local_secondary_indexes.len(), 1); - let described = fx - .engine - .describe_backup(ACCOUNT, &b.backup_arn) + .unwrap(), + WriteOutcome::Matched + ); + assert_eq!( + fx.engine.start_backup_delete(&deleted.key()).await.unwrap(), + DeleteStart::Deleting + ); + assert!(fx.engine.set_backup_deleted(&deleted.key()).await.unwrap()); + assert!( + older(BackupStatus::Failed, store(), StaleClock::Heartbeat) .await - .expect("describe"); - assert_eq!( - described.backup_details.backup_creation_date_time, - b.backup_creation_date_time - ); - assert!(described.source_table_feature_details.is_none()); - } - let listed = fx.engine.list_backups(ACCOUNT, None).await.expect("list"); - assert_eq!(listed.len(), 3); - for summary in &listed { - assert_eq!(summary.table_id, table_id); + .is_empty(), + "no row is FAILED yet" + ); + + tokio::time::sleep(window + Duration::from_millis(300)).await; + // The owners die here: FAILED writes with the heartbeat left as it was. + for (claim, fields) in [ + (&completed_claim, done.clone()), + (&unstamped_claim, StatusFields::default()), + (&elsewhere_claim, done.clone()), + ] { assert_eq!( - Some(summary.backup_creation_date_time), - extenddb_storage::util::backup_creation_date_time(&summary.backup_arn) + claim + .set_status(JobStatus::Creating, JobStatus::Failed, &fields) + .await + .unwrap(), + WriteOutcome::Matched ); } - let foreign = fx - .engine - .backup_restore_source("999999999999", &backups[0].backup_arn) - .await; + let unstamped_doc = index_doc(&fx.engine, &unstamped).await; assert!( - matches!(&foreign, Err(StorageError::Validation(m)) if m.starts_with("Backup not found")), - "{foreign:?}" + unstamped_doc.get("completed_at").is_none(), + "a write with `completed: false` stamps nothing: {unstamped_doc}" + ); + assert_eq!( + older(BackupStatus::Failed, store(), StaleClock::Heartbeat).await, + ["completed", "unstamped"], + "rows FAILED just now are listed on their dead owners' heartbeats, in the catalog's order" + ); + assert!( + older(BackupStatus::Failed, store(), StaleClock::Completed) + .await + .is_empty(), + "a `completed_at` stamped just now is within the window; an unset one is never older" + ); + assert_eq!( + older(BackupStatus::Deleted, store(), StaleClock::Completed).await, + ["deleted"] + ); + assert_eq!( + older(BackupStatus::Creating, store(), StaleClock::Heartbeat).await, + ["creating"] + ); + assert_eq!( + older( + BackupStatus::Failed, + other_store.clone(), + StaleClock::Heartbeat + ) + .await, + ["elsewhere"] + ); + + tokio::time::sleep(window + Duration::from_millis(300)).await; + assert_eq!( + older(BackupStatus::Failed, store(), StaleClock::Completed).await, + ["completed"], + "the completed window crossed on the server's clock; no `completed_at` is never older" + ); + assert_eq!( + older(BackupStatus::Failed, store(), StaleClock::Heartbeat).await, + ["completed", "unstamped"] ); fx.finish().await; } -/// The GSI backfill worker's periodic sweep, with no explicit sweep call and -/// no restart, removes a crashed restore and finishes an interrupted -/// DeleteBackup. +/// The maintenance claim: taken once, refused to a second holder naming the +/// first, heartbeated on the server's clock, released; a holder whose +/// heartbeat is older than the stale window is dead, so its claim is not +/// reported held and is taken over, and its handle finds the claim lost. A +/// record without a heartbeat (no release of this server wrote one) is +/// stale. #[tokio::test(flavor = "multi_thread")] #[ignore = "needs a MongoDB replica set in EXTENDDB_TEST_MONGODB_URI"] -async fn live_the_worker_sweeps_abandoned_state_without_a_restart() { +async fn live_the_maintenance_claim_excludes_a_second_holder_and_expires_when_stale() { let fx = Fixture::new().await; - source_table(&fx.engine, "src").await; - let crashed = fx + let stale = std::time::Duration::from_secs(300); + let check = || fx.engine.check_maintenance_claim(stale); + let maintenance = fx .engine - .create_backup(ACCOUNT, "src", "b1") - .await - .expect("backup 1"); - let interrupted = fx + .catalog_db + .collection::(crate::backup_catalog::BACKUP_MAINTENANCE); + assert!(!check().await.expect("check")); + let first = fx .engine - .create_backup(ACCOUNT, "src", "b2") + .maintenance_claim("purge on host-a", stale) .await - .expect("backup 2"); - simulate_crashed_restore(&fx.engine, &crashed.backup_arn, "dst").await; - let mut set = doc! { DELETE_CLAIM: "dead-delete" }; - set.insert( - DELETE_CLAIM_HEARTBEAT, - bson::DateTime::from_millis(bson::DateTime::now().timestamp_millis() - 600_000), + .expect("claim"); + assert_ne!(first.generation(), UNOWNED_GENERATION); + assert!(check().await.expect("check"), "a live claim is held"); + match fx.engine.maintenance_claim("purge on host-b", stale).await { + Err(StorageError::AlreadyExists(m)) => assert!(m.contains("purge on host-a"), "{m}"), + Err(other) => panic!("a second holder is refused with AlreadyExists: {other:?}"), + Ok(_) => panic!("a second holder is refused"), + } + assert_eq!( + first.heartbeat().await.expect("heartbeat"), + Heartbeat::Confirmed ); - fx.engine - .catalog_db - .collection::("backups") + first.release().await.expect("release"); + assert!(!check().await.expect("check"), "released"); + assert_eq!( + maintenance.count_documents(doc! {}).await.expect("count"), + 0 + ); + first.release().await.expect("a second release is a no-op"); + + // A dead holder: its heartbeat aged past the window. + let dead = fx + .engine + .maintenance_claim("purge on host-c", stale) + .await + .expect("claim"); + maintenance .update_one( - doc! { "_id": &interrupted.backup_arn }, - doc! { "$set": set }, + doc! {}, + vec![doc! { "$set": { "heartbeat_at": { "$subtract": [ "$$NOW", 600_000_i64 ] } } }], ) .await - .expect("stale delete claim"); + .expect("age"); + assert!(!check().await.expect("check"), "a stale claim is not held"); + let taker = fx + .engine + .maintenance_claim("purge on host-d", stale) + .await + .expect("a dead holder's claim is taken over"); + assert_ne!(taker.generation(), dead.generation()); + assert!(check().await.expect("check")); + assert_eq!( + dead.heartbeat().await.expect("heartbeat"), + Heartbeat::Lost, + "the dead holder's handle finds the claim lost" + ); + dead.release() + .await + .expect("a release by a lost holder changes nothing"); + assert!(check().await.expect("check"), "the taker still holds it"); + let holder = maintenance + .find_one(doc! {}) + .await + .expect("find") + .expect("record"); + assert_eq!(holder.get_str("holder").unwrap(), "purge on host-d"); + taker.release().await.expect("release"); + assert!(!check().await.expect("check")); - // Worst case: one heartbeat interval until the sweep is due, plus the - // worker's 5 s tick; the state is already older than stale_after. - let deadline = - Instant::now() + fx.engine.backup_liveness.heartbeat_interval + Duration::from_secs(15); - loop { - let target_gone = table_doc(&fx.engine, "dst").await.is_none(); - let status = fx - .engine - .catalog_db - .collection::("backups") - .find_one(doc! { "_id": &interrupted.backup_arn }) - .await - .expect("read") - .and_then(|d| d.get_str("backup_status").ok().map(str::to_owned)); - if target_gone && status.as_deref() == Some("DELETED") { - break; - } - assert!( - Instant::now() < deadline, - "the worker did not sweep: target gone {target_gone}, b2 status {status:?}" - ); - tokio::time::sleep(Duration::from_millis(200)).await; - } - fx.engine - .delete_backup(ACCOUNT, &crashed.backup_arn) + // A record with no heartbeat at all is stale: taken over, never held. + maintenance + .insert_one( + doc! { "_id": "claim", "holder": "an older release", "owner_generation": 1_i64 }, + ) + .await + .expect("record"); + assert!(!check().await.expect("check")); + let over = fx + .engine + .maintenance_claim("purge on host-e", stale) .await - .expect("DeleteBackup after the worker's sweep"); + .expect("taken over"); + assert!(check().await.expect("check")); + over.release().await.expect("release"); fx.finish().await; } -async fn backup_meta(engine: &MongoEngine, backup_arn: &str) -> Document { - engine - .catalog_db - .collection::("backups") - .find_one(doc! { "_id": backup_arn }) - .await - .expect("read backup") - .expect("backup document") -} +/// DeleteBackup's first step and the tombstone writes: only an AVAILABLE +/// row moves to DELETING; a CREATING row answers its status; a backup a +/// CREATING restore target reads answers the target's name; DELETING moves +/// to DELETED once; a DELETED row is removed once; a missing row is "Backup +/// not found". +#[tokio::test(flavor = "multi_thread")] +#[ignore = "needs a MongoDB replica set in EXTENDDB_TEST_MONGODB_URI"] +async fn live_start_backup_delete_checks_the_status_and_the_in_use_rule_under_the_lock() { + let fx = Fixture::new().await; + let row = row(ACCOUNT, "Music", "nightly"); + let key = row.key(); + assert!(matches!( + fx.engine.start_backup_delete(&key).await, + Err(StorageError::Validation(m)) if m.contains("Backup not found") + )); + let claim = fx.engine.claim_backup(&row).await.expect("claim"); + assert_eq!( + fx.engine.start_backup_delete(&key).await.unwrap(), + DeleteStart::Status(BackupStatus::Creating) + ); + assert_eq!( + claim + .set_status( + JobStatus::Creating, + JobStatus::Available, + &StatusFields::default() + ) + .await + .unwrap(), + WriteOutcome::Matched + ); + assert_eq!(fx.engine.backup_in_use(&key).await.unwrap(), None); -fn with_drop_fault(engine: &MongoEngine, fault: AfterBackupDrop) -> MongoEngine { - let mut faulty = engine.clone(); - faulty.backup_liveness.after_backup_drop = Some(fault); - faulty -} + // A CREATING restore target registered from the backup: in use, naming + // the target; an ACTIVE one is not. + source_table_with(&fx.engine, "del_target", 1).await; + let tables = fx.engine.catalog_db.collection::("tables"); + let target_filter = doc! { "_id": { "account_id": ACCOUNT, "table_name": "del_target" } }; + tables + .update_one( + target_filter.clone(), + doc! { "$set": { + "table_status": "CREATING", + "status_transition_at": bson::Bson::Null, + "restore_source_backup_arn": &row.backup_arn, + "restore_source_table_arn": "arn:aws:dynamodb:us-east-1:123456789012:table/Music", + "restore_date_time": bson::DateTime::now(), + RESTORE_JOB: { + "status": "RUNNING", + "kind": "manifest", + "owner_generation": 7_i64, + "heartbeat_at": bson::DateTime::now(), + }, + } }, + ) + .await + .expect("register a target"); + assert_eq!( + fx.engine.backup_in_use(&key).await.unwrap().as_deref(), + Some("del_target") + ); + assert_eq!( + fx.engine.start_backup_delete(&key).await.unwrap(), + DeleteStart::InUse("del_target".to_owned()) + ); + assert_eq!( + fx.engine.forget_available_backup_row(&key).await.unwrap(), + extenddb_storage::backup_catalog::ForgetRow::InUse("del_target".to_owned()), + "the outright removal keeps the in-use rule" + ); + assert_eq!( + fx.engine + .read_backup_row(&arn_of(&row)) + .await + .unwrap() + .unwrap() + .status, + BackupStatus::Available, + "a refused delete leaves the row AVAILABLE" + ); + tables + .update_one(target_filter, doc! { "$set": { "table_status": "ACTIVE" } }) + .await + .expect("make it ACTIVE"); + assert_eq!(fx.engine.backup_in_use(&key).await.unwrap(), None); -/// After a DeleteBackup or sweep stopped after its drop: the claim is still -/// held, the backup's data is gone, the backup is not restorable, and once the -/// claim is stale the sweep finishes the delete. -async fn assert_claim_held_then_swept(engine: &MongoEngine, backup_arn: &str, backup_id: &str) { - // Give a claim's Drop (a spawned release) time to run if it were armed. - tokio::time::sleep(Duration::from_millis(500)).await; - let meta = backup_meta(engine, backup_arn).await; + assert_eq!( + fx.engine.start_backup_delete(&key).await.unwrap(), + DeleteStart::Deleting + ); + assert_eq!( + fx.engine.start_backup_delete(&key).await.unwrap(), + DeleteStart::Status(BackupStatus::Deleting), + "a retry finds the row DELETING" + ); + assert_eq!( + fx.engine.forget_available_backup_row(&key).await.unwrap(), + extenddb_storage::backup_catalog::ForgetRow::Status(BackupStatus::Deleting), + "only an AVAILABLE row is removed outright" + ); + // An AVAILABLE row goes outright, read and listed nowhere, with no + // tombstone; a second removal finds no row. + let mut adopted = self::row(ACCOUNT, "Music", "adopted"); + adopted.status = BackupStatus::Available; + assert!(fx.engine.insert_backup_row(&adopted).await.expect("insert")); + assert_eq!( + fx.engine + .forget_available_backup_row(&adopted.key()) + .await + .unwrap(), + extenddb_storage::backup_catalog::ForgetRow::Removed + ); assert!( - meta.contains_key(DELETE_CLAIM), - "the delete claim must not be released once the drop may have run: {meta:?}" + fx.engine + .read_backup_row(&arn_of(&adopted)) + .await + .unwrap() + .is_none() ); - assert_eq!(meta.get_str("backup_status"), Ok("AVAILABLE")); - assert!(!collection_exists(engine, &format!("_backup_{backup_id}")).await); - let request = restore_request(engine, backup_arn, "dst").await; - let refused = engine.restore_table_from_backup(ACCOUNT, request).await; + for status in [None, Some(BackupStatus::Available)] { + let listed: Vec = fx + .engine + .list_backup_rows(status, &adopted.store) + .try_collect() + .await + .unwrap(); + assert!( + !listed.iter().any(|r| r.backup_arn == adopted.backup_arn), + "{status:?}" + ); + } + let by_account: Vec = fx + .engine + .list_account_backup_rows(&adopted.account_id, None, &TimeBounds::default()) + .try_collect() + .await + .unwrap(); assert!( - matches!(&refused, Err(StorageError::BackupInUse(_))), - "a restore must not start from a backup whose data was dropped: {refused:?}" + !by_account + .iter() + .any(|r| r.backup_arn == adopted.backup_arn) ); - assert!(table_doc(engine, "dst").await.is_none()); - let operation = meta.get_str(DELETE_CLAIM).expect("claim id").to_owned(); - let second = engine.delete_backup(ACCOUNT, backup_arn).await; + assert!(matches!( + fx.engine.forget_available_backup_row(&adopted.key()).await, + Err(StorageError::Validation(m)) if m.contains("Backup not found") + )); assert!( - matches!(&second, Err(StorageError::BackupInUse(m)) if m.starts_with("Backup is being deleted")), - "a DeleteBackup during a held delete claim must answer BackupInUse: {second:?}" + !fx.engine.remove_backup_row(&key).await.unwrap(), + "DELETING is not a tombstone" ); - let meta = backup_meta(engine, backup_arn).await; - assert_eq!(meta.get_str(DELETE_CLAIM), Ok(operation.as_str())); - assert_eq!(meta.get_str("backup_status"), Ok("AVAILABLE")); - - tokio::time::sleep(engine.backup_liveness.stale_after + Duration::from_millis(500)).await; - let sweep = engine.sweep_stale_backup_claims().await.expect("sweep"); - assert_eq!(sweep.delete_claims_finished, [backup_arn.to_owned()]); - let meta = backup_meta(engine, backup_arn).await; - assert_eq!(meta.get_str("backup_status"), Ok("DELETED")); - assert!(!meta.contains_key(DELETE_CLAIM)); -} - -/// The reply to DeleteBackup's drop is an error after the server dropped -/// the collection. The request fails, but the claim stays until the sweep -/// finishes the delete. -#[tokio::test(flavor = "multi_thread")] -#[ignore = "needs a MongoDB replica set in EXTENDDB_TEST_MONGODB_URI"] -async fn live_a_delete_whose_drop_reply_fails_keeps_its_claim() { - let fx = Fixture::new().await; - fx.worker.abort(); - source_table(&fx.engine, "src").await; - let backup = fx + assert!(fx.engine.set_backup_deleted(&key).await.unwrap()); + assert!(!fx.engine.set_backup_deleted(&key).await.unwrap()); + let stored = fx .engine - .create_backup(ACCOUNT, "src", "b") - .await - .expect("backup"); - let backup_id = backup_meta(&fx.engine, &backup.backup_arn) + .read_backup_row(&arn_of(&row)) .await - .get_str("backup_id") - .expect("backup_id") - .to_owned(); - let faulty = with_drop_fault(&fx.engine, AfterBackupDrop::Fail); - let failed = faulty.delete_backup(ACCOUNT, &backup.backup_arn).await; + .unwrap() + .unwrap(); + assert_eq!(stored.status, BackupStatus::Deleted); + assert!(stored.completed_at.is_some()); + assert!(fx.engine.remove_backup_row(&key).await.unwrap()); + assert!(!fx.engine.remove_backup_row(&key).await.unwrap()); assert!( - matches!(&failed, Err(StorageError::Internal(_))), - "{failed:?}" + fx.engine + .read_backup_row(&arn_of(&row)) + .await + .unwrap() + .is_none() ); - assert_claim_held_then_swept(&fx.engine, &backup.backup_arn, &backup_id).await; + drop(claim); fx.finish().await; } -/// The DeleteBackup future is dropped (a client timeout) while it waits -/// for the reply to a drop the server already ran. +/// The listings: by account with an optional table and the time bounds +/// (lower inclusive, upper exclusive) in (created_at, ARN) order, and by +/// store and status. #[tokio::test(flavor = "multi_thread")] #[ignore = "needs a MongoDB replica set in EXTENDDB_TEST_MONGODB_URI"] -async fn live_a_delete_dropped_while_waiting_for_its_drop_keeps_its_claim() { +async fn live_the_listings_are_scoped_and_ordered() { let fx = Fixture::new().await; - fx.worker.abort(); - source_table(&fx.engine, "src").await; - let backup = fx + let mut rows = Vec::new(); + for (account, table, name) in [ + (ACCOUNT, "Music", "a"), + (ACCOUNT, "Music", "b"), + (ACCOUNT, "Other", "c"), + (OTHER_ACCOUNT, "Music", "d"), + ] { + tokio::time::sleep(Duration::from_millis(5)).await; + let row = row(account, table, name); + let claim = fx.engine.claim_backup(&row).await.expect("claim"); + if name == "b" { + assert_eq!( + claim + .set_status( + JobStatus::Creating, + JobStatus::Failed, + &StatusFields::default() + ) + .await + .unwrap(), + WriteOutcome::Matched + ); + } + rows.push((row, claim)); + } + let names = |listed: Vec| -> Vec { + listed.into_iter().map(|r| r.backup_name).collect() + }; + let all: Vec = fx .engine - .create_backup(ACCOUNT, "src", "b") + .list_account_backup_rows(ACCOUNT, None, &TimeBounds::default()) + .try_collect() .await - .expect("backup"); - let backup_id = backup_meta(&fx.engine, &backup.backup_arn) + .unwrap(); + assert_eq!( + names(all.clone()), + ["a", "b", "c"], + "every status, in creation order" + ); + assert_eq!(all[1].status, BackupStatus::Failed); + let music: Vec = fx + .engine + .list_account_backup_rows(ACCOUNT, Some("Music"), &TimeBounds::default()) + .try_collect() .await - .get_str("backup_id") - .expect("backup_id") - .to_owned(); - let faulty = with_drop_fault(&fx.engine, AfterBackupDrop::Hang); - let timed_out = tokio::time::timeout( - Duration::from_secs(3), - faulty.delete_backup(ACCOUNT, &backup.backup_arn), - ) - .await; - assert!(timed_out.is_err(), "the delete must hang after its drop"); - assert_claim_held_then_swept(&fx.engine, &backup.backup_arn, &backup_id).await; + .unwrap(); + assert_eq!(names(music), ["a", "b"]); + let bounds = TimeBounds { + lower: Some(rows[1].0.created_at), + upper: Some(rows[2].0.created_at), + }; + let bounded: Vec = fx + .engine + .list_account_backup_rows(ACCOUNT, None, &bounds) + .try_collect() + .await + .unwrap(); + assert_eq!(names(bounded), ["b"], "lower inclusive, upper exclusive"); + let other: Vec = fx + .engine + .list_account_backup_rows(OTHER_ACCOUNT, None, &TimeBounds::default()) + .try_collect() + .await + .unwrap(); + assert_eq!(names(other), ["d"]); + + let by_store: Vec = fx + .engine + .list_backup_rows(None, &store()) + .try_collect() + .await + .unwrap(); + assert_eq!(names(by_store), ["a", "b", "c", "d"]); + let failed: Vec = fx + .engine + .list_backup_rows(Some(BackupStatus::Failed), &store()) + .try_collect() + .await + .unwrap(); + assert_eq!(names(failed), ["b"]); + let elsewhere = StoreIdentity { + store_id: "store-2".to_owned(), + ..store() + }; + let none: Vec = fx + .engine + .list_backup_rows(None, &elsewhere) + .try_collect() + .await + .unwrap(); + assert!(none.is_empty(), "another store's rows are not listed"); + drop(rows); fx.finish().await; } -/// B1 addendum: the sweep's own delete takeover gets an error reply after its -/// drop ran. The claim it took over stays held, and the next sweep finishes. +/// A catalog at the shape before migration 6, holding a finished +/// restore, a restore handed to the GSI backfill worker at the stop (its +/// copy complete, the target CREATING with `restore_backfill_pending`) and a +/// restore the stop interrupted (its target CREATING, no flag), migrates with +/// the registrations backfilled: the finished and the handed-over ones DONE, +/// the interrupted one RUNNING with `heartbeat_at` at the epoch and the +/// unowned generation, so the reconciler's first pass fails it and removes +/// its target and leaves the handed-over one to the worker. The body keeps +/// the catalog id; a second run changes nothing. #[tokio::test(flavor = "multi_thread")] #[ignore = "needs a MongoDB replica set in EXTENDDB_TEST_MONGODB_URI"] -async fn live_a_sweep_whose_drop_reply_fails_keeps_the_claim() { +async fn live_migration_6_backfills_the_registrations_and_the_reconciler_removes_the_interrupted_one() + { + const FEW: usize = 300; let fx = Fixture::new().await; - fx.worker.abort(); - source_table(&fx.engine, "src").await; + source_table_with(&fx.engine, "src", FEW).await; let backup = fx .engine - .create_backup(ACCOUNT, "src", "b") + .create(ACCOUNT, "src", "b", JobPermit::detached()) .await .expect("backup"); - let backup_id = backup_meta(&fx.engine, &backup.backup_arn) + let request = restore_request(&fx.engine, &backup.backup_arn, "bf_done").await; + fx.engine + .restore(ACCOUNT, request, JobPermit::detached()) .await - .get_str("backup_id") - .expect("backup_id") - .to_owned(); - let mut set = doc! { DELETE_CLAIM: "dead-delete" }; - set.insert( - DELETE_CLAIM_HEARTBEAT, - bson::DateTime::from_millis(bson::DateTime::now().timestamp_millis() - 600_000), + .expect("restore"); + assert_fully_restored_with(&fx.engine, "bf_done", FEW).await; + // A restore the earlier release had handed to the GSI backfill worker at + // the stop: the copy complete, the table CREATING with + // `restore_backfill_pending`, one index still CREATING (so the worker + // waits rather than flipping it while the step runs). Its data is whole, + // so the step backfills it DONE, not RUNNING: marked RUNNING with the + // epoch heartbeat, the first reconciler pass would fail it and remove a + // target the worker is about to finish. From a backup of its own: a + // CREATING target keeps its backup in use, which would refuse the + // interrupted restore below. + let handed_backup = fx + .engine + .create(ACCOUNT, "src", "b2", JobPermit::detached()) + .await + .expect("second backup"); + let request = restore_request(&fx.engine, &handed_backup.backup_arn, "bf_handed").await; + let handed = fx + .engine + .restore(ACCOUNT, request, JobPermit::detached()) + .await + .expect("restore"); + assert_fully_restored_with(&fx.engine, "bf_handed", FEW).await; + let tables = fx.engine.catalog_db.collection::("tables"); + let index_docs = fx.engine.catalog_db.collection::("indexes"); + index_docs + .update_one( + doc! { "_id.table_id": &handed.table_id, "_id.index_name": "g1" }, + doc! { "$set": { "index_status": "CREATING" } }, + ) + .await + .expect("an index still being backfilled"); + tables + .update_one( + doc! { "_id": { "account_id": ACCOUNT, "table_name": "bf_handed" } }, + doc! { "$set": { + "table_status": "CREATING", + "status_transition_at": bson::Bson::Null, + "restore_backfill_pending": true, + } }, + ) + .await + .expect("handed over to the worker"); + let (gated, _gate) = gated(&fx.engine); + let request = restore_request(&fx.engine, &backup.backup_arn, "bf_cut").await; + let interrupted = gated + .restore(ACCOUNT, request, JobPermit::detached()) + .await + .expect("restore"); + let catalog_id = fx.engine.catalog_id().await.expect("id"); + + // Back to the shape before migration 6: the registrations as the + // earlier release wrote them. The interrupted restore's copy task loses + // its claim with the record and stops. + tables + .update_many( + doc! { "restore_source_backup_arn": { "$exists": true } }, + doc! { + "$unset": { RESTORE_JOB: "" }, + "$set": { + "restore_operation_id": "old-operation", + "restore_heartbeat_at": bson::DateTime::now(), + }, + }, + ) + .await + .expect("old shape"); + let report = run_backup_catalog_index(&target(&fx.catalog, &fx.data)) + .await + .expect("migrate again"); + assert_eq!( + report.notes, + [ + "backfilled 3 restore registration(s) as legacy restores, 1 of them RUNNING with a \ + dead owner for the reconciler to remove" + ], + "the catalog id is kept" ); - fx.engine - .catalog_db - .collection::("backups") - .update_one(doc! { "_id": &backup.backup_arn }, doc! { "$set": set }) + assert_eq!(fx.engine.catalog_id().await.expect("id"), catalog_id); + + let done = table_doc(&fx.engine, "bf_done").await.expect("bf_done"); + let job = done.get_document(RESTORE_JOB).expect("job record"); + assert_eq!(job.get_str("status"), Ok("DONE")); + assert_eq!(job.get_str("kind"), Ok("legacy")); + assert_eq!(job.get_i64("owner_generation"), Ok(0)); + assert_eq!( + job.get_datetime("heartbeat_at") + .map(|d| d.timestamp_millis()), + Ok(0), + "the epoch, not absent" + ); + assert_eq!( + job.get_datetime("completed_at"), + done.get_datetime("creation_date_time"), + "completed when the table was created" + ); + assert!(!done.contains_key("restore_operation_id")); + assert!(!done.contains_key("restore_heartbeat_at")); + let handed_doc = table_doc(&fx.engine, "bf_handed").await.expect("bf_handed"); + assert_eq!(handed_doc.get_str("table_status"), Ok("CREATING")); + assert_eq!(handed_doc.get_bool("restore_backfill_pending"), Ok(true)); + let job = handed_doc.get_document(RESTORE_JOB).expect("job record"); + assert_eq!( + job.get_str("status"), + Ok("DONE"), + "a target handed to the worker has its copy complete: DONE, not RUNNING" + ); + assert_eq!(job.get_str("kind"), Ok("legacy")); + assert_eq!(job.get_i64("owner_generation"), Ok(0)); + assert_eq!( + job.get_datetime("completed_at"), + handed_doc.get_datetime("creation_date_time") + ); + let cut = table_doc(&fx.engine, "bf_cut").await.expect("bf_cut"); + assert_eq!(cut.get_str("table_status"), Ok("CREATING")); + let job = cut.get_document(RESTORE_JOB).expect("job record"); + assert_eq!(job.get_str("status"), Ok("RUNNING")); + assert_eq!(job.get_str("kind"), Ok("legacy")); + assert_eq!(job.get_i64("owner_generation"), Ok(0)); + assert_eq!( + job.get_datetime("heartbeat_at") + .map(|d| d.timestamp_millis()), + Ok(0) + ); + assert!(!job.contains_key("completed_at")); + + // A second run changes nothing. + let again = run_backup_catalog_index(&target(&fx.catalog, &fx.data)) .await - .expect("stale delete claim"); + .expect("a second run"); + assert!(again.notes.is_empty(), "{:?}", again.notes); - let faulty = with_drop_fault(&fx.engine, AfterBackupDrop::Fail); - let failed = faulty.sweep_stale_backup_claims().await; + // The first reconciler pass: the interrupted restore is dead. + let dead = fx + .engine + .list_dead_restore_jobs(STALE_AFTER) + .await + .expect("list"); + assert_eq!( + dead.len(), + 1, + "the interrupted restore alone; the handed-over one is DONE: {dead:?}" + ); + assert_eq!( + dead[0].target, + RestoreTargetKey { + account_id: ACCOUNT.to_owned(), + table_name: "bf_cut".to_owned(), + table_id: interrupted.table_id.clone(), + } + ); + assert_eq!(dead[0].owner_generation, UNOWNED_GENERATION); + assert_eq!(dead[0].kind, RestoreKind::Legacy); + let indexes = index_ids(&fx.engine, &interrupted.table_id).await; + assert_eq!(indexes.len(), 2); assert!( - matches!(&failed, Err(StorageError::Internal(_))), - "{failed:?}" + fx.engine + .fail_restore_job( + &dead[0].target, + dead[0].owner_generation, + dead[0].heartbeat_at, + "owner dead" + ) + .await + .unwrap() ); - let meta = backup_meta(&fx.engine, &backup.backup_arn).await; - assert_ne!( - meta.get_str(DELETE_CLAIM), - Ok("dead-delete"), - "the sweep took the claim over before dropping" + assert!( + fx.engine + .remove_restore_target(&dead[0].target) + .await + .unwrap() + ); + assert!(table_doc(&fx.engine, "bf_cut").await.is_none()); + assert!(!collection_exists(&fx.engine, &data_collection_name(&interrupted.table_id)).await); + for (_, index_id, _) in &indexes { + assert!(!collection_exists(&fx.engine, &data_collection_name(index_id)).await); + } + assert!( + index_ids(&fx.engine, &interrupted.table_id) + .await + .is_empty() + ); + assert_eq!( + table_doc(&fx.engine, "bf_done") + .await + .expect("bf_done") + .get_str("table_status"), + Ok("ACTIVE") + ); + // The worker finishes the handed-over target once its index is done. + index_docs + .update_one( + doc! { "_id.table_id": &handed.table_id, "_id.index_name": "g1" }, + doc! { "$set": { "index_status": "ACTIVE" } }, + ) + .await + .expect("the index backfill done"); + let finished = wait_for_status(&fx.engine, "bf_handed", "ACTIVE").await; + assert!(!finished.contains_key("restore_backfill_pending")); + assert_eq!( + finished + .get_document(RESTORE_JOB) + .expect("job") + .get_str("status"), + Ok("DONE") ); - assert_claim_held_then_swept(&fx.engine, &backup.backup_arn, &backup_id).await; fx.finish().await; } -/// The sweep leaves a stale delete claim alone while a restore target of the -/// backup is CREATING (the DeleteBackup died between its claim and its -/// recheck), and finishes the delete once the restore is ACTIVE. +/// The legacy restore registers through the job record: RUNNING under the +/// claim's generation while the copy runs, with a heartbeat on the server's +/// clock, and DONE with the hand-over. #[tokio::test(flavor = "multi_thread")] #[ignore = "needs a MongoDB replica set in EXTENDDB_TEST_MONGODB_URI"] -async fn live_a_stale_delete_claim_waits_for_a_running_restore() { +async fn live_a_legacy_restore_registers_a_running_job_record_and_finishes_it_done() { + const FEW: usize = 300; let fx = Fixture::new().await; - fx.worker.abort(); - source_table(&fx.engine, "src").await; + source_table_with(&fx.engine, "src", FEW).await; let backup = fx .engine - .create_backup(ACCOUNT, "src", "b") + .create(ACCOUNT, "src", "b", JobPermit::detached()) .await .expect("backup"); - let request = restore_request(&fx.engine, &backup.backup_arn, "dst").await; let (gated, gate) = gated(&fx.engine); + let request = restore_request(&fx.engine, &backup.backup_arn, "dst").await; gated - .restore_table_from_backup(ACCOUNT, request) + .restore(ACCOUNT, request, JobPermit::detached()) .await - .expect("restore"); - let mut set = doc! { DELETE_CLAIM: "dead-delete" }; - set.insert( - DELETE_CLAIM_HEARTBEAT, - bson::DateTime::from_millis(bson::DateTime::now().timestamp_millis() - 600_000), + .expect("registers"); + let job = job_record(&fx.engine, "dst").await; + assert_eq!(job.get_str("status"), Ok("RUNNING")); + assert_eq!(job.get_str("kind"), Ok("legacy")); + let generation = job.get_i64("owner_generation").expect("generation"); + assert_ne!(generation, i64::try_from(UNOWNED_GENERATION).unwrap()); + let first = *job.get_datetime("heartbeat_at").expect("heartbeat"); + let age_ms = bson::DateTime::now().timestamp_millis() - first.timestamp_millis(); + assert!(age_ms.abs() < 60_000, "a fresh heartbeat: {age_ms} ms old"); + assert!( + fx.engine + .list_dead_restore_jobs(STALE_AFTER) + .await + .unwrap() + .is_empty(), + "a running restore is not dead" ); - fx.engine - .catalog_db - .collection::("backups") - .update_one(doc! { "_id": &backup.backup_arn }, doc! { "$set": set }) + // The heartbeat moves while the copy is held. + tokio::time::sleep(fx.engine.backup_liveness.heartbeat_interval * 3).await; + let later = *job_record(&fx.engine, "dst") .await - .expect("stale delete claim"); - - for _ in 0..3 { - let sweep = fx.engine.sweep_stale_backup_claims().await.expect("sweep"); - assert!(sweep.is_empty(), "{sweep}"); - tokio::time::sleep(Duration::from_millis(300)).await; - } - let meta = backup_meta(&fx.engine, &backup.backup_arn).await; - assert_eq!( - meta.get_str(DELETE_CLAIM), - Ok("dead-delete"), - "the claim must be left as it was: {meta:?}" - ); + .get_datetime("heartbeat_at") + .expect("heartbeat"); + assert!(later > first, "the heartbeat moved: {later:?} > {first:?}"); - let worker = spawn_worker(&fx.engine); gate.add_permits(1); - assert_fully_restored(&fx.engine, "dst").await; - let sweep = fx.engine.sweep_stale_backup_claims().await.expect("sweep"); - assert_eq!( - sweep.delete_claims_finished, - std::slice::from_ref(&backup.backup_arn) - ); - assert_eq!( - backup_meta(&fx.engine, &backup.backup_arn) - .await - .get_str("backup_status"), - Ok("DELETED") - ); - worker.abort(); + assert_fully_restored_with(&fx.engine, "dst", FEW).await; + let job = job_record(&fx.engine, "dst").await; + assert_eq!(job.get_str("status"), Ok("DONE")); + assert_eq!(job.get_i64("owner_generation"), Ok(generation)); + assert!(job.get_datetime("completed_at").is_ok()); fx.finish().await; } -/// The first heartbeat refresh happens at once, not one interval later. +/// A restore whose heartbeat fails while its `$out` runs on the server (a +/// fault on the heartbeat's connection, injected at the next tick once the +/// test has seen the `$out` in `currentOp`): the copy stops without touching +/// the target (the record FAILED and the target removed, each conditioned), +/// and the `$out` the server runs to its end leaves no orphan +/// `_data_`: the removal dropped the target collection the +/// registration created, and `$out` refuses to replace a target whose +/// indexes changed while it ran. The name is free and the backup no longer +/// in use afterwards. The copy is large enough that `$out` outlasts the +/// probe and several heartbeat periods, and the fault cell records that the +/// heartbeat failed as requested. #[tokio::test(flavor = "multi_thread")] #[ignore = "needs a MongoDB replica set in EXTENDDB_TEST_MONGODB_URI"] -async fn live_the_heartbeat_refreshes_at_once() { +async fn live_a_heartbeat_lost_while_out_runs_leaves_no_orphan_collection() { + const MANY: usize = 150_000; let fx = Fixture::new().await; - fx.worker.abort(); - source_table(&fx.engine, "src").await; + source_table_with(&fx.engine, "src", MANY).await; let backup = fx .engine - .create_backup(ACCOUNT, "src", "b") + .create(ACCOUNT, "src", "b", JobPermit::detached()) .await .expect("backup"); - let long_ago = bson::DateTime::from_millis(bson::DateTime::now().timestamp_millis() - 600_000); - let mut set = doc! { DELETE_CLAIM: "op" }; - set.insert(DELETE_CLAIM_HEARTBEAT, long_ago); - fx.engine - .catalog_db - .collection::("backups") - .update_one(doc! { "_id": &backup.backup_arn }, doc! { "$set": set }) + let backup_key = extenddb_storage::backup_catalog::BackupKey::from_arn( + &BackupArn::parse(&backup.backup_arn).expect("arn"), + ); + let fault = Arc::new(std::sync::atomic::AtomicU8::new(HeartbeatFault::IDLE)); + let mut faulty = fx.engine.clone(); + faulty.backup_liveness.heartbeat_interval = Duration::from_millis(50); + faulty.backup_liveness.heartbeat_fault = Some(fault.clone()); + let request = restore_request(&faulty, &backup.backup_arn, "dst").await; + let desc = faulty + .restore(ACCOUNT, request, JobPermit::detached()) .await - .expect("claim"); - let mut slow = fx.engine.clone(); - slow.backup_liveness.heartbeat_interval = Duration::from_secs(3_600); - let _heartbeat = slow.start_backup_heartbeat("op", ACCOUNT, &backup.backup_arn, None); - let deadline = Instant::now() + Duration::from_secs(5); - loop { - let beat = *backup_meta(&fx.engine, &backup.backup_arn) + .expect("restore registers"); + let data = data_collection_name(&desc.table_id); + + // The `$out` shows in `currentOp` as an insert into its temporary + // collection whose command is the aggregate over the backup collection + // (named by the legacy document's own `backup_id`, not the ARN's). + let legacy_id = backup_meta(&fx.engine, &backup.backup_arn) + .await + .get_str("backup_id") + .expect("backup_id") + .to_owned(); + let out_source = super::backup_collection_name(&legacy_id); + let out_running = || async { + !fx.engine + .client + .database("admin") + .run_command(doc! { + "currentOp": 1, + "$ownOps": false, + "command.aggregate": &out_source, + "command.pipeline.$out": { "$exists": true }, + }) .await - .get_datetime(DELETE_CLAIM_HEARTBEAT) - .expect("heartbeat"); - if beat > long_ago { + .expect("currentOp") + .get_array("inprog") + .map(|ops| ops.is_empty()) + .unwrap_or(true) + }; + let deadline = Instant::now() + Duration::from_secs(60); + while !out_running().await { + assert!(Instant::now() < deadline, "the copy's $out did not start"); + tokio::time::sleep(Duration::from_millis(20)).await; + } + fault.store(HeartbeatFault::ARMED, std::sync::atomic::Ordering::SeqCst); + + // The catalog side: the target is removed once the heartbeat fails, with + // `$out` still running. + let deadline = Instant::now() + Duration::from_secs(60); + while table_doc(&fx.engine, "dst").await.is_some() { + assert!( + Instant::now() < deadline, + "the target was not removed after the heartbeat failed" + ); + tokio::time::sleep(Duration::from_millis(20)).await; + } + assert_eq!( + fault.load(std::sync::atomic::Ordering::SeqCst), + HeartbeatFault::INJECTED, + "the heartbeat failed as requested" + ); + assert!( + out_running().await, + "the window was real: $out was still running when the target was removed" + ); + + // The server side: `$out` ends on its own and leaves no collection. + let deadline = Instant::now() + Duration::from_secs(60); + while out_running().await { + assert!(Instant::now() < deadline, "$out did not end"); + tokio::time::sleep(Duration::from_millis(100)).await; + } + tokio::time::sleep(Duration::from_millis(500)).await; + assert!( + !collection_exists(&fx.engine, &data).await, + "no orphan data collection after $out ended" + ); + assert!(index_ids(&fx.engine, &desc.table_id).await.is_empty()); + assert!(table_doc(&fx.engine, "dst").await.is_none()); + assert_eq!(fx.engine.backup_in_use(&backup_key).await.unwrap(), None); + + // Liveness: the name restores again (the backup is not in use and the + // name is free), through to the hand-over; the index backfill of a table + // this size is the worker's and is not waited for. + let request = restore_request(&fx.engine, &backup.backup_arn, "dst").await; + let again = fx + .engine + .restore(ACCOUNT, request, JobPermit::detached()) + .await + .expect("restore again"); + let deadline = Instant::now() + Duration::from_secs(60); + loop { + let doc = table_doc(&fx.engine, "dst") + .await + .expect("the second target"); + let handed = doc + .get_document(RESTORE_JOB) + .map(|job| job.get_str("status") == Ok("DONE")) + .unwrap_or(false); + if handed || doc.get_str("table_status") == Ok("ACTIVE") { break; } assert!( Instant::now() < deadline, - "the first refresh did not happen at once" + "the second restore did not hand over" ); - tokio::time::sleep(Duration::from_millis(50)).await; + tokio::time::sleep(Duration::from_millis(100)).await; } + assert_eq!( + count(&fx.engine, &data_collection_name(&again.table_id)).await, + MANY as u64 + ); fx.finish().await; } -/// A restore from a process whose clock is 10 minutes behind the server's -/// is not swept by another process (stale window 5 minutes), because the -/// target's first heartbeat is replaced with the server's clock at once. +/// A RUNNING restore record is dead by the age of its heartbeat on the +/// server's clock, whatever the table's age; a stale record is marked FAILED +/// and its target removed through `remove_restore_target`, with its data and +/// index collections, which frees the backup and the name; a fresh record, +/// and a record that is no longer RUNNING, are left alone. #[tokio::test(flavor = "multi_thread")] #[ignore = "needs a MongoDB replica set in EXTENDDB_TEST_MONGODB_URI"] -async fn live_a_restore_from_a_process_with_a_slow_clock_is_not_swept() { +async fn live_an_abandoned_restore_is_removed_only_when_its_heartbeat_is_stale() { + const FEW: usize = 300; let fx = Fixture::new().await; - fx.worker.abort(); - source_table(&fx.engine, "src").await; + source_table_with(&fx.engine, "src", FEW).await; let backup = fx .engine - .create_backup(ACCOUNT, "src", "b") + .create(ACCOUNT, "src", "b", JobPermit::detached()) .await .expect("backup"); - let liveness = BackupLiveness { - heartbeat_interval: Duration::from_secs(3_600), - stale_after: Duration::from_secs(300), - slot_wait: Duration::from_secs(2), - copy_gate: None, - after_backup_drop: None, - local_clock_behind: Duration::from_secs(600), - }; - let behind = fx.second_instance(liveness.clone()).await; - let (behind, gate) = gated(&behind); - let other = fx - .second_instance(BackupLiveness { - local_clock_behind: Duration::ZERO, - ..liveness - }) - .await; let request = restore_request(&fx.engine, &backup.backup_arn, "dst").await; - behind - .restore_table_from_backup(ACCOUNT, request) + fx.engine + .restore(ACCOUNT, request, JobPermit::detached()) .await .expect("restore"); - let beat = *table_doc(&fx.engine, "dst") - .await - .expect("target") - .get_datetime(TARGET_HEARTBEAT) - .expect("heartbeat"); - let age_ms = bson::DateTime::now().timestamp_millis() - beat.timestamp_millis(); + assert_fully_restored_with(&fx.engine, "dst", FEW).await; + let doc = table_doc(&fx.engine, "dst").await.expect("target"); + let key = target_key(&doc); + let indexes = index_ids(&fx.engine, &key.table_id).await; + assert_eq!(indexes.len(), 2); + let backup_key = extenddb_storage::backup_catalog::BackupKey::from_arn( + &BackupArn::parse(&backup.backup_arn).expect("arn"), + ); + + // An ACTIVE former target under the right key: no longer a CREATING + // restore target, so the removal is refused and the table, its data and + // its index collections survive (the `table_status: CREATING` condition + // is what excludes the removal from the ACTIVE flip). assert!( - age_ms.abs() < 60_000, - "the target heartbeat is {age_ms} ms old: the server clock must replace the local one" + !fx.engine.remove_restore_target(&key).await.expect("remove"), + "an ACTIVE table under the key is not a CREATING restore target" ); - let sweep = other.sweep_stale_backup_claims().await.expect("sweep"); - assert!(sweep.is_empty(), "{sweep}"); - assert!(table_doc(&fx.engine, "dst").await.is_some()); + let active = table_doc(&fx.engine, "dst") + .await + .expect("the ACTIVE table survives"); + assert_eq!(active.get_str("table_status"), Ok("ACTIVE")); + assert_eq!( + count(&fx.engine, &data_collection_name(&key.table_id)).await, + FEW as u64 + ); + for (index_name, index_id, _) in &indexes { + assert!( + collection_exists(&fx.engine, &data_collection_name(index_id)).await, + "index {index_name} survives" + ); + } - let worker = spawn_worker(&other); - gate.add_permits(1); - assert_fully_restored(&fx.engine, "dst").await; - worker.abort(); - fx.finish().await; -} + // A fresh heartbeat: a live restore, here or on another instance. + age_running_record(&fx.engine, "dst", Duration::from_secs(1)).await; + assert!( + fx.engine + .list_dead_restore_jobs(STALE_AFTER) + .await + .expect("list") + .is_empty() + ); + assert_eq!( + fx.engine + .backup_in_use(&backup_key) + .await + .unwrap() + .as_deref(), + Some("dst"), + "a CREATING target keeps its backup in use" + ); + let in_use = fx.engine.delete(ACCOUNT, &backup.backup_arn).await; + assert!( + matches!(in_use, Err(StorageError::BackupInUse(_))), + "{in_use:?}" + ); -async fn table_doc_for(engine: &MongoEngine, account: &str, name: &str) -> Option { - engine - .catalog_db - .collection::("tables") - .find_one(doc! { "_id": { "account_id": account, "table_name": name } }) + // A stale heartbeat: dead. + age_running_record(&fx.engine, "dst", Duration::from_secs(600)).await; + let dead = fx + .engine + .list_dead_restore_jobs(STALE_AFTER) .await - .expect("read table") -} - -async fn wait_for_status_for(engine: &MongoEngine, account: &str, name: &str, status: &str) { - let deadline = Instant::now() + Duration::from_secs(120); - loop { - if let Some(doc) = table_doc_for(engine, account, name).await - && doc.get_str("table_status") == Ok(status) - { - return; - } - // The account id is kept out of the message: the scanner reads an - // interpolated account as a logged credential. - let current = table_doc_for(engine, account, name) + .expect("list"); + assert_eq!(dead.len(), 1, "{dead:?}"); + assert_eq!(dead[0].target, key); + assert_eq!(dead[0].kind, RestoreKind::Legacy); + assert_eq!(dead[0].status, JobStatus::Running); + assert_eq!(dead[0].source_backup_arn, backup.backup_arn); + assert!( + fx.engine + .fail_restore_job( + &key, + dead[0].owner_generation, + dead[0].heartbeat_at, + "owner dead" + ) .await - .and_then(|doc| doc.get_str("table_status").ok().map(str::to_owned)); - assert!( - Instant::now() < deadline, - "table {name} did not reach {status}: {current:?}" - ); - tokio::time::sleep(Duration::from_millis(100)).await; + .expect("fail") + ); + let job = job_record(&fx.engine, "dst").await; + assert_eq!(job.get_str("status"), Ok("FAILED")); + assert_eq!(job.get_str("failure_reason"), Ok("owner dead")); + assert!(job.get_datetime("completed_at").is_ok()); + assert!( + !fx.engine + .fail_restore_job( + &key, + dead[0].owner_generation, + dead[0].heartbeat_at, + "owner dead" + ) + .await + .expect("fail again"), + "a record that is no longer RUNNING is not written twice" + ); + assert!( + fx.engine + .list_dead_restore_jobs(STALE_AFTER) + .await + .unwrap() + .is_empty(), + "a FAILED record is not dead" + ); + // A key with another table id names a table re-created under the name, + // not this target. + let other_id = RestoreTargetKey { + table_id: "another-table".to_owned(), + ..key.clone() + }; + assert!(!fx.engine.remove_restore_target(&other_id).await.unwrap()); + // The removal with the reply to its collection drops lost: the document + // is gone, so the target is removed (`true`), the name and the backup + // are free and no record is left for a later cycle; the collections, + // dropped before the reply was lost, are gone too. + let mut drops_fail = fx.engine.clone(); + drops_fail.backup_liveness.target_drop_fails = true; + assert!( + drops_fail + .remove_restore_target(&key) + .await + .expect("the target is removed although the reply to its collection drops was lost") + ); + assert!(table_doc(&fx.engine, "dst").await.is_none()); + assert!(!collection_exists(&fx.engine, &data_collection_name(&key.table_id)).await); + for (_, index_id, _) in &indexes { + assert!(!collection_exists(&fx.engine, &data_collection_name(index_id)).await); } + assert!(index_ids(&fx.engine, &key.table_id).await.is_empty()); + assert!( + !fx.engine.remove_restore_target(&key).await.unwrap(), + "a target that is gone is not removed twice" + ); + + // The backup and the name are free again. + assert_eq!(fx.engine.backup_in_use(&backup_key).await.unwrap(), None); + let request = restore_request(&fx.engine, &backup.backup_arn, "dst").await; + fx.engine + .restore(ACCOUNT, request, JobPermit::detached()) + .await + .expect("restore into the freed name"); + assert_fully_restored_with(&fx.engine, "dst", FEW).await; + let deleted = fx + .engine + .delete(ACCOUNT, &backup.backup_arn) + .await + .expect("DeleteBackup once the target is ACTIVE"); + assert_eq!(deleted.backup_details.backup_status, "DELETED"); + fx.finish().await; } -/// One restore of a backup at a time. Four restores of one backup -/// started at the same moment: exactly one is accepted and the other three -/// are `BackupInUse`; a restore started while the accepted one is still -/// CREATING (copy held) is `BackupInUse` as well; once the target is ACTIVE a -/// new restore of the backup is accepted. +/// The FAILED-target duty's listing: a FAILED record whose table is still a +/// CREATING restore target (the FAILED write landed and the removal did not) +/// is not listed while the record is fresh (the task that failed it may +/// still be removing the target), is listed once `completed_at` is older +/// than the stale window, as FAILED, and its target is then removed with no +/// second FAILED write; a FAILED record without a `completed_at` is aged by +/// its heartbeat; a FAILED record whose table is DELETING or ACTIVE is never +/// listed. #[tokio::test(flavor = "multi_thread")] #[ignore = "needs a MongoDB replica set in EXTENDDB_TEST_MONGODB_URI"] -async fn live_a_second_restore_of_a_backup_while_one_is_creating_is_backup_in_use() { +async fn live_a_failed_record_whose_target_removal_failed_is_listed_once_stale_and_removed() { const FEW: usize = 300; let fx = Fixture::new().await; source_table_with(&fx.engine, "src", FEW).await; let backup = fx .engine - .create_backup(ACCOUNT, "src", "b") + .create(ACCOUNT, "src", "b", JobPermit::detached()) .await .expect("backup"); - let (gated, gate) = gated(&fx.engine); - let mut requests = Vec::new(); - for i in 0..4 { - requests.push(restore_request(&fx.engine, &backup.backup_arn, &format!("dst{i}")).await); - } - let results = futures::future::join_all( - requests - .into_iter() - .map(|request| gated.restore_table_from_backup(ACCOUNT, request)), - ) - .await; - let accepted: Vec = results - .iter() - .enumerate() - .filter_map(|(i, r)| r.is_ok().then_some(i)) - .collect(); - assert_eq!( - accepted.len(), - 1, - "exactly one restore is accepted: {results:?}" - ); - for (i, result) in results.iter().enumerate() { - if i != accepted[0] { - match result { - Err(StorageError::BackupInUse(m)) => assert!( - m.starts_with("Backup is being used to restore table dst") - || m.starts_with("Backup is being used to restore another table: "), - "restore {i}: {m}" - ), - other => panic!("restore {i} must be BackupInUse: {other:?}"), - } + let request = restore_request(&fx.engine, &backup.backup_arn, "dst").await; + fx.engine + .restore(ACCOUNT, request, JobPermit::detached()) + .await + .expect("restore"); + assert_fully_restored_with(&fx.engine, "dst", FEW).await; + let key = target_key(&table_doc(&fx.engine, "dst").await.expect("target")); + let server_ago = |age: Duration| { + bson::DateTime::from_millis( + bson::DateTime::now().timestamp_millis() - i64::try_from(age.as_millis()).expect("i64"), + ) + }; + let set = |update: Document| { + let tables = fx.engine.catalog_db.collection::("tables"); + async move { + tables + .update_one( + doc! { "_id": { "account_id": ACCOUNT, "table_name": "dst" } }, + update, + ) + .await + .expect("set the record"); } - } - let winner = format!("dst{}", accepted[0]); + }; + let listed = || async { + fx.engine + .list_dead_restore_jobs(STALE_AFTER) + .await + .expect("list") + }; + + // The state a dead restore leaves once the reconciler failed its record + // and the target's removal did not land. + age_running_record(&fx.engine, "dst", Duration::from_secs(600)).await; + let dead = listed().await; + assert_eq!(dead.len(), 1, "{dead:?}"); + assert_eq!(dead[0].status, JobStatus::Running); + assert!( + fx.engine + .fail_restore_job( + &key, + dead[0].owner_generation, + dead[0].heartbeat_at, + "owner dead" + ) + .await + .expect("fail") + ); + let job = job_record(&fx.engine, "dst").await; + assert_eq!(job.get_str("status"), Ok("FAILED")); + assert!(job.get_datetime("completed_at").is_ok()); assert_eq!( - table_doc(&fx.engine, &winner) + table_doc(&fx.engine, "dst") .await .expect("target") .get_str("table_status"), Ok("CREATING") ); - // Sequential, while the winner's copy is held: still in use, and the - // refusal names the target with the wording the other backends use. - let request = restore_request(&fx.engine, &backup.backup_arn, "late").await; - match fx.engine.restore_table_from_backup(ACCOUNT, request).await { - Err(StorageError::BackupInUse(m)) => assert_eq!( - m, - format!( - "Backup is being used to restore table {winner}: {}", - backup.backup_arn + + // Fresh: `fail_restore_job` stamped `completed_at` with the server's + // clock a moment ago, and whoever failed it may still be removing the + // target. + assert!( + listed().await.is_empty(), + "a FAILED record within the stale window is left to its writer" + ); + // Stale: listed, as FAILED, for the FAILED-target duty. + set(doc! { "$set": { "restore_job.completed_at": server_ago(Duration::from_secs(600)) } }) + .await; + let dead = listed().await; + assert_eq!(dead.len(), 1, "{dead:?}"); + assert_eq!(dead[0].target, key); + assert_eq!(dead[0].status, JobStatus::Failed); + assert_eq!(dead[0].kind, RestoreKind::Legacy); + assert_eq!(dead[0].source_backup_arn, backup.backup_arn); + // A FAILED record without `completed_at` is aged by its heartbeat: fresh, + // not listed; stale, listed. + set(doc! { + "$set": { RESTORE_JOB_HEARTBEAT: bson::DateTime::now() }, + "$unset": { "restore_job.completed_at": "" }, + }) + .await; + assert!( + listed().await.is_empty(), + "a FAILED record with a fresh heartbeat and no completed_at is fresh" + ); + set(doc! { "$set": { RESTORE_JOB_HEARTBEAT: server_ago(Duration::from_secs(600)) } }).await; + assert_eq!(listed().await.len(), 1); + // A FAILED record whose table the removal already moved to DELETING is + // the control plane's row. + set(doc! { "$set": { "table_status": "DELETING" } }).await; + assert!( + listed().await.is_empty(), + "a DELETING table is not a target" + ); + set(doc! { "$set": { "table_status": "CREATING" } }).await; + + // The reconciler's FAILED-target duty: no second FAILED write, the target + // removed with its collections. + assert!( + !fx.engine + .fail_restore_job( + &key, + dead[0].owner_generation, + dead[0].heartbeat_at, + "owner dead" ) - ), - other => panic!("a restore during the CREATING window is BackupInUse: {other:?}"), - } - assert!(table_doc(&fx.engine, "late").await.is_none()); - // The refused restores left no claim behind: DeleteBackup is refused only - // because of the winner's CREATING target. - let meta = backup_meta(&fx.engine, &backup.backup_arn).await; - assert!(!meta.contains_key(RESTORE_CLAIM), "{meta:?}"); + .await + .expect("fail again"), + "a FAILED record is not RUNNING and is not written FAILED again" + ); + assert!(fx.engine.remove_restore_target(&key).await.expect("remove")); + assert!(table_doc(&fx.engine, "dst").await.is_none()); + assert!(!collection_exists(&fx.engine, &data_collection_name(&key.table_id)).await); + assert!(listed().await.is_empty()); + assert_eq!( + fx.engine + .backup_in_use(&extenddb_storage::backup_catalog::BackupKey::from_arn( + &BackupArn::parse(&backup.backup_arn).expect("arn"), + )) + .await + .unwrap(), + None, + "the backup is free once the target is gone" + ); - gate.add_permits(1); - assert_fully_restored_with(&fx.engine, &winner, FEW).await; - let request = restore_request(&fx.engine, &backup.backup_arn, "late").await; + // A FAILED record whose table is ACTIVE (a flip that won against a late + // FAILED write) is nobody's duty. + let request = restore_request(&fx.engine, &backup.backup_arn, "dst").await; fx.engine - .restore_table_from_backup(ACCOUNT, request) + .restore(ACCOUNT, request, JobPermit::detached()) .await - .expect("a restore after the first is ACTIVE is accepted"); - assert_fully_restored_with(&fx.engine, "late", FEW).await; + .expect("restore again"); + assert_fully_restored_with(&fx.engine, "dst", FEW).await; + set(doc! { "$set": { + RESTORE_JOB_STATUS: "FAILED", + "restore_job.completed_at": server_ago(Duration::from_secs(600)), + } }) + .await; + assert!( + listed().await.is_empty(), + "a FAILED record of an ACTIVE table is not listed" + ); fx.finish().await; } -/// A restore into a name that is a CREATING restore target is -/// `TableBeingRestored`, naming the target and the backup it is being restored -/// from; the same name taken by an ACTIVE table is `TableAlreadyExists`. +/// A restore target of another, live, instance on the same catalog is not +/// dead for many stale windows, because its heartbeat keeps moving, while a +/// record whose owner stopped is; DeleteBackup of the live one's source is +/// refused; the live restore then finishes. #[tokio::test(flavor = "multi_thread")] #[ignore = "needs a MongoDB replica set in EXTENDDB_TEST_MONGODB_URI"] -async fn live_a_restore_into_a_creating_restore_target_is_table_being_restored() { +async fn live_a_restore_of_another_instance_is_not_dead_while_it_heartbeats() { const FEW: usize = 300; let fx = Fixture::new().await; source_table_with(&fx.engine, "src", FEW).await; let b1 = fx .engine - .create_backup(ACCOUNT, "src", "b1") + .create(ACCOUNT, "src", "b1", JobPermit::detached()) .await .expect("backup 1"); let b2 = fx .engine - .create_backup(ACCOUNT, "src", "b2") + .create(ACCOUNT, "src", "b2", JobPermit::detached()) .await .expect("backup 2"); - let (gated, gate) = gated(&fx.engine); - let request = restore_request(&fx.engine, &b1.backup_arn, "dst").await; - gated - .restore_table_from_backup(ACCOUNT, request) + // A restore whose owner stopped: finished, then aged into a dead record. + let request = restore_request(&fx.engine, &b1.backup_arn, "stopped").await; + fx.engine + .restore(ACCOUNT, request, JobPermit::detached()) .await - .expect("first restore"); - let request = restore_request(&fx.engine, &b2.backup_arn, "dst").await; - match fx.engine.restore_table_from_backup(ACCOUNT, request).await { - Err(StorageError::TableBeingRestored { - table_name, - backup_arn, - }) => { - assert_eq!(table_name, "dst"); - assert_eq!(backup_arn, b1.backup_arn); - } - other => panic!("restore into a CREATING restore target: {other:?}"), - } - // The same backup into the same name: the name is what is refused, not - // the backup (the service answers TableInUseException), and b1's claim - // is released. - let request = restore_request(&fx.engine, &b1.backup_arn, "dst").await; - match fx.engine.restore_table_from_backup(ACCOUNT, request).await { - Err(StorageError::TableBeingRestored { - table_name, - backup_arn, - }) => { - assert_eq!(table_name, "dst"); - assert_eq!(backup_arn, b1.backup_arn); - } - other => panic!("the same backup into its CREATING target: {other:?}"), - } - // The name is checked before the backup: a backup that does not exist, - // restored into that name, is refused as the name too. - let mut missing = restore_request(&fx.engine, &b2.backup_arn, "dst").await; - missing.backup_arn = flip_last_hex(&b2.backup_arn); - match fx.engine.restore_table_from_backup(ACCOUNT, missing).await { - Err(StorageError::TableBeingRestored { - table_name, - backup_arn, - }) => { - assert_eq!(table_name, "dst"); - assert_eq!(backup_arn, b1.backup_arn); - } - other => panic!("a missing backup into a CREATING restore target: {other:?}"), + .expect("restore"); + assert_fully_restored_with(&fx.engine, "stopped", FEW).await; + age_running_record(&fx.engine, "stopped", Duration::from_secs(60)).await; + + // A live restore of another instance, held before its copy. + let other = fx.second_instance(test_liveness()).await; + let stale_after = other.backup_liveness.heartbeat_interval * 6; + let (gated_other, gate) = gated(&other); + let request = restore_request(&other, &b2.backup_arn, "dst").await; + gated_other + .restore(ACCOUNT, request, JobPermit::detached()) + .await + .expect("restore"); + let first = *job_record(&fx.engine, "dst") + .await + .get_datetime("heartbeat_at") + .expect("heartbeat"); + + // Across three stale windows, only the stopped one is dead. + let until = Instant::now() + stale_after * 3; + let mut checks = 0; + while Instant::now() < until { + let dead = fx + .engine + .list_dead_restore_jobs(stale_after) + .await + .expect("list"); + assert_eq!( + dead.iter() + .map(|j| j.target.table_name.as_str()) + .collect::>(), + ["stopped"], + "only the record whose owner stopped is dead" + ); + checks += 1; + tokio::time::sleep(Duration::from_millis(250)).await; } - let meta = backup_meta(&fx.engine, &b1.backup_arn).await; - assert!(!meta.contains_key(RESTORE_CLAIM), "{meta:?}"); - // b2 was not left claimed by the refused restore. - let meta = backup_meta(&fx.engine, &b2.backup_arn).await; - assert!(!meta.contains_key(RESTORE_CLAIM), "{meta:?}"); - // With every account slot held, a client retrying its own restore is - // still answered at once, not after the slot wait. - let account_slots = fx.engine.restore_account_slots.for_account(ACCOUNT); - let held = Arc::clone(&account_slots) - .acquire_many_owned(u32::try_from(account_slots.available_permits()).expect("u32")) - .await - .expect("hold every account slot"); - let request = restore_request(&fx.engine, &b1.backup_arn, "dst").await; - let started = Instant::now(); - let retried = fx.engine.restore_table_from_backup(ACCOUNT, request).await; - assert!( - started.elapsed() < fx.engine.backup_liveness.slot_wait, - "the retry waited for a slot: {:?}", - started.elapsed() - ); + assert!(checks >= 10); + // Liveness: the live heartbeat moved. + let later = *job_record(&fx.engine, "dst") + .await + .get_datetime("heartbeat_at") + .expect("heartbeat"); + assert!(later > first, "{later:?} > {first:?}"); + let in_use = fx.engine.delete(ACCOUNT, &b2.backup_arn).await; assert!( - matches!(&retried, Err(StorageError::TableBeingRestored { table_name, .. }) if table_name == "dst"), - "{retried:?}" + matches!(in_use, Err(StorageError::BackupInUse(_))), + "{in_use:?}" ); - drop(held); + gate.add_permits(1); assert_fully_restored_with(&fx.engine, "dst", FEW).await; - let request = restore_request(&fx.engine, &b2.backup_arn, "dst").await; - assert!( - matches!( - fx.engine.restore_table_from_backup(ACCOUNT, request).await, - Err(StorageError::TableAlreadyExists(_)) - ), - "an ACTIVE table of that name is TableAlreadyExists" - ); fx.finish().await; } -/// Twenty restores of twenty distinct backups (twenty accounts, so no -/// per-account bound and no shared claim serializes their registration) -/// against the driver's default pool of 10, with the server bound lifted, -/// all register, the engine answers a catalog read right after, and every -/// target reaches ACTIVE. Before the fix, the registration's hand-over left -/// each creation's in-flight command holding a pooled connection while it -/// waited for another, and this wedged the process for good. -#[tokio::test(flavor = "multi_thread")] -#[ignore = "needs a MongoDB replica set in EXTENDDB_TEST_MONGODB_URI"] -async fn live_twenty_concurrent_restores_of_distinct_backups_do_not_wedge_the_pool() { - const N: usize = 20; - const FEW: usize = 20; - let fx = Fixture::new().await; - let mut unbounded = fx.engine.clone(); - unbounded.restore_server_slot_count = 64; - unbounded.restore_server_slots = Arc::new(tokio::sync::Semaphore::new(64)); - let accounts: Vec = (0..N).map(|i| format!("9100000000{i:02}")).collect(); - let mut requests = Vec::new(); - for account in &accounts { - source_table_for(&fx.engine, account, "src", FEW).await; - let backup = fx - .engine - .create_backup(account, "src", "b") - .await - .expect("backup"); - requests.push(( - account.clone(), - restore_request_for(&fx.engine, account, &backup.backup_arn, "dst").await, - )); - } - let started = Instant::now(); - let results = futures::future::join_all(requests.into_iter().map(|(account, request)| { - let engine = unbounded.clone(); - async move { - tokio::time::timeout( - Duration::from_secs(60), - engine.restore_table_from_backup(&account, request), - ) +/// After a DeleteBackup stopped after its drop: the backup is DELETING, so +/// it is neither described, listed nor restorable, and a retry of +/// DeleteBackup resumes the drop and answers DELETED. +async fn assert_deleting_then_resumed(engine: &MongoEngine, backup_arn: &str, backup_id: &str) { + let meta = backup_meta(engine, backup_arn).await; + assert_eq!( + meta.get_str("backup_status"), + Ok("DELETING"), + "the backup must not read AVAILABLE once the drop may have run: {meta:?}" + ); + assert!(!collection_exists(engine, &format!("_backup_{backup_id}")).await); + let described = engine.describe(ACCOUNT, backup_arn).await; + assert!( + matches!(&described, Err(StorageError::Validation(m)) if m.starts_with("Backup not found")), + "{described:?}" + ); + assert!(engine.list(ACCOUNT, None).await.expect("list").is_empty()); + let source = engine.restore_source(ACCOUNT, backup_arn).await; + assert!( + matches!(&source, Err(StorageError::Validation(m)) if m.starts_with("Backup not found")), + "a restore must not start from a backup whose data was dropped: {source:?}" + ); + assert!(table_doc(engine, "dst").await.is_none()); + let resumed = engine + .delete(ACCOUNT, backup_arn) + .await + .expect("a retry resumes the delete"); + assert_eq!(resumed.backup_details.backup_status, "DELETED"); + assert_eq!( + backup_meta(engine, backup_arn) .await - } - })) - .await; - let elapsed = started.elapsed(); - for (i, result) in results.iter().enumerate() { - match result { - Ok(Ok(_)) => {} - Ok(Err(e)) => panic!("restore {i} failed: {e}"), - Err(_) => panic!("restore {i} did not return within 60 s: the pool is wedged"), - } - } - eprintln!("{N} concurrent restores of distinct backups registered in {elapsed:?}"); - // Liveness: the engine still answers a catalog read, promptly. - let probe = tokio::time::timeout( - Duration::from_secs(10), - fx.engine.describe_table( - &accounts[0], - DescribeTableInput { - table_name: "src".to_owned(), - }, - ), - ) - .await - .expect("a catalog read after the restores returns within 10 s") - .expect("describe"); - assert_eq!(probe.table_status, TableStatus::Active); - for account in &accounts { - wait_for_status_for(&fx.engine, account, "dst", "ACTIVE").await; - } - fx.finish().await; + .get_str("backup_status"), + Ok("DELETED") + ); + let again = engine.delete(ACCOUNT, backup_arn).await; + assert!( + matches!(&again, Err(StorageError::Validation(m)) if m.starts_with("Backup not found")), + "{again:?}" + ); } -/// With the default bound (`min(16, pool/2)` = 5 for the default pool -/// of 10), seven restores of seven backups from two accounts (four and three, -/// under the per-account bound of four) started while copies are held: five are -/// admitted and two are `LimitExceeded` naming the server bound after the -/// slot wait, the engine still answers meanwhile, the five finish once the -/// copies are released, and their slots are then free again. +/// The reply to DeleteBackup's drop is an error after the server dropped +/// the collection. The request fails, but the backup stays DELETING until a +/// retry finishes the delete. #[tokio::test(flavor = "multi_thread")] #[ignore = "needs a MongoDB replica set in EXTENDDB_TEST_MONGODB_URI"] -async fn live_restores_beyond_the_server_bound_are_limit_exceeded_and_slots_are_freed() { - const FEW: usize = 20; +async fn live_a_delete_whose_drop_reply_fails_leaves_the_backup_deleting_until_a_retry() { let fx = Fixture::new().await; - assert_eq!(fx.engine.restore_server_slot_count, 5); - let (gated, gate) = gated(&fx.engine); - let accounts = ["920000000001", "920000000002"]; - let mut requests = Vec::new(); - // One backup per restore: a second restore of one backup while the first - // is CREATING is BackupInUse, which is not what this test is about. - for (account, restores) in accounts.iter().zip([4, 3]) { - source_table_for(&fx.engine, account, "src", FEW).await; - for i in 0..restores { - let backup = fx - .engine - .create_backup(account, "src", &format!("b{i}")) - .await - .expect("backup"); - let target = format!("dst{i}"); - requests.push(( - (*account).to_owned(), - target.clone(), - restore_request_for(&fx.engine, account, &backup.backup_arn, &target).await, - )); - } - } - let started = Instant::now(); - let results = futures::future::join_all(requests.iter().map(|(account, _, request)| { - let engine = gated.clone(); - async move { - engine - .restore_table_from_backup(account, request.clone()) - .await - } - })) - .await; - let elapsed = started.elapsed(); - let admitted: Vec<&(String, String, RestoreRequest)> = requests - .iter() - .zip(&results) - .filter_map(|(r, result)| result.is_ok().then_some(r)) - .collect(); - assert_eq!(admitted.len(), 5, "five restores are admitted: {results:?}"); - for result in &results { - if let Err(e) = result { - match e { - StorageError::LimitExceeded(m) => { - assert!(m.contains("on this server (at most 5"), "{m}"); - } - other => panic!("a refused restore is LimitExceeded: {other:?}"), - } - } - } + source_table_with(&fx.engine, "src", 300).await; + let backup = fx + .engine + .create(ACCOUNT, "src", "b", JobPermit::detached()) + .await + .expect("backup"); + let backup_id = backup_meta(&fx.engine, &backup.backup_arn) + .await + .get_str("backup_id") + .expect("backup_id") + .to_owned(); + let faulty = with_drop_fault(&fx.engine, AfterBackupDrop::Fail); + let failed = faulty.delete(ACCOUNT, &backup.backup_arn).await; assert!( - elapsed >= fx.engine.backup_liveness.slot_wait, - "the refused restores waited the slot wait: {elapsed:?}" + matches!(&failed, Err(StorageError::Internal(_))), + "{failed:?}" ); - // The two refused restores never claimed their backup or inserted a - // target: the refusal came before any catalog write. - for (i, (account, target, _)) in requests.iter().enumerate() { - let doc = table_doc_for(&fx.engine, account, target).await; - let is_admitted = admitted.iter().any(|(a, t, _)| a == account && t == target); - assert_eq!(doc.is_some(), is_admitted, "request {i}: {target}"); - } - // Liveness while five copies are held. - tokio::time::timeout( - Duration::from_secs(10), - fx.engine.describe_table( - accounts[0], - DescribeTableInput { - table_name: "src".to_owned(), - }, - ), - ) - .await - .expect("a catalog read while copies are held returns within 10 s") - .expect("describe"); - assert_eq!(fx.engine.restore_server_slots.available_permits(), 0); - - gate.add_permits(5); - for (account, target, _) in &admitted { - wait_for_status_for(&fx.engine, account, target, "ACTIVE").await; - } - // Slots are released when the copy task ends, shortly after hand-over. - let deadline = Instant::now() + Duration::from_secs(10); - while fx.engine.restore_server_slots.available_permits() < 5 { - assert!(Instant::now() < deadline, "the five slots are freed"); - tokio::time::sleep(Duration::from_millis(50)).await; - } + assert_deleting_then_resumed(&fx.engine, &backup.backup_arn, &backup_id).await; fx.finish().await; } -/// The refusal paths of the admission bound, without a server. -#[tokio::test] -async fn restore_permits_wait_then_limit_exceeded() { - let wait = Duration::from_millis(50); - let account = Arc::new(tokio::sync::Semaphore::new(4)); - let server = Arc::new(tokio::sync::Semaphore::new(1)); - let held = super::acquire_restore_permits(Arc::clone(&account), Arc::clone(&server), 1, wait) +/// The DeleteBackup future is dropped (a client timeout) while it waits +/// for the reply to a drop the server already ran. +#[tokio::test(flavor = "multi_thread")] +#[ignore = "needs a MongoDB replica set in EXTENDDB_TEST_MONGODB_URI"] +async fn live_a_delete_dropped_while_waiting_for_its_drop_leaves_the_backup_deleting() { + let fx = Fixture::new().await; + source_table_with(&fx.engine, "src", 300).await; + let backup = fx + .engine + .create(ACCOUNT, "src", "b", JobPermit::detached()) .await - .expect("the first restore gets both slots"); - let other_account = Arc::new(tokio::sync::Semaphore::new(4)); - match super::acquire_restore_permits(Arc::clone(&other_account), Arc::clone(&server), 1, wait) + .expect("backup"); + let backup_id = backup_meta(&fx.engine, &backup.backup_arn) .await - { - Err(StorageError::LimitExceeded(m)) => { - assert!(m.contains("on this server (at most 1"), "{m}") - } - other => panic!("a full server bound must refuse: {:?}", other.err()), - } - assert_eq!(other_account.available_permits(), 4); - drop(held); - super::acquire_restore_permits(Arc::clone(&other_account), Arc::clone(&server), 1, wait) - .await - .expect("a freed server slot is granted"); - let full_account = Arc::new(tokio::sync::Semaphore::new(0)); - let roomy_server = Arc::new(tokio::sync::Semaphore::new(4)); - match super::acquire_restore_permits(full_account, Arc::clone(&roomy_server), 4, wait).await { - Err(StorageError::LimitExceeded(m)) => assert!(m.contains("for this account"), "{m}"), - other => panic!("a full account must refuse: {:?}", other.err()), - } - assert_eq!(roomy_server.available_permits(), 4); - assert_eq!(super::restore_server_slot_count(1), 1); - assert_eq!(super::restore_server_slot_count(10), 5); - assert_eq!(super::restore_server_slot_count(40), 16); + .get_str("backup_id") + .expect("backup_id") + .to_owned(); + let faulty = with_drop_fault(&fx.engine, AfterBackupDrop::Hang); + let timed_out = tokio::time::timeout( + Duration::from_secs(3), + faulty.delete(ACCOUNT, &backup.backup_arn), + ) + .await; + assert!(timed_out.is_err(), "the delete must hang after its drop"); + assert_deleting_then_resumed(&fx.engine, &backup.backup_arn, &backup_id).await; + fx.finish().await; } -/// UpdateTable on a CREATING restore target is `TableNotActive`, for -/// every kind of change (billing mode, deletion protection, a stream, a GSI), -/// and leaves the target as the backup describes; once ACTIVE the same -/// changes are accepted. A table that does not exist is `TableNotFound`. +/// A restore from a process whose clock is 10 minutes behind the server's +/// is not dead to another process (stale window 5 minutes): the record's +/// first heartbeat, written with the local clock at the insert, is replaced +/// by the server's clock at once, not one heartbeat interval later. #[tokio::test(flavor = "multi_thread")] #[ignore = "needs a MongoDB replica set in EXTENDDB_TEST_MONGODB_URI"] -async fn live_update_table_on_a_creating_restore_target_is_refused() { - use extenddb_core::types::{ - CreateGsiAction, GlobalSecondaryIndexUpdate, ProvisionedThroughput, StreamSpecification, - StreamViewType, UpdateTableInput, - }; +async fn live_a_restore_from_a_process_with_a_slow_clock_is_not_dead() { const FEW: usize = 300; let fx = Fixture::new().await; source_table_with(&fx.engine, "src", FEW).await; let backup = fx .engine - .create_backup(ACCOUNT, "src", "b") + .create(ACCOUNT, "src", "b", JobPermit::detached()) .await .expect("backup"); - let (gated, gate) = gated(&fx.engine); + let liveness = BackupLiveness { + heartbeat_interval: Duration::from_secs(3_600), + registration_gate: None, + copy_gate: None, + after_backup_drop: None, + target_drop_fails: false, + local_clock_behind: Duration::from_secs(600), + heartbeat_fault: None, + }; + let behind = fx.second_instance(liveness.clone()).await; + let (behind, gate) = gated(&behind); + let other = fx + .second_instance(BackupLiveness { + local_clock_behind: Duration::ZERO, + ..liveness + }) + .await; let request = restore_request(&fx.engine, &backup.backup_arn, "dst").await; - gated - .restore_table_from_backup(ACCOUNT, request) + behind + .restore(ACCOUNT, request, JobPermit::detached()) .await .expect("restore"); - let before = table_doc(&fx.engine, "dst").await.expect("target"); - assert_eq!(before.get_str("table_status"), Ok("CREATING")); + let deadline = Instant::now() + Duration::from_secs(5); + loop { + let beat = *job_record(&fx.engine, "dst") + .await + .get_datetime("heartbeat_at") + .expect("heartbeat"); + let age_ms = bson::DateTime::now().timestamp_millis() - beat.timestamp_millis(); + if age_ms.abs() < 60_000 { + break; + } + assert!( + Instant::now() < deadline, + "the record's heartbeat is {age_ms} ms old: the server clock must replace the \ + local one at once" + ); + tokio::time::sleep(Duration::from_millis(50)).await; + } + assert!( + other + .list_dead_restore_jobs(STALE_AFTER) + .await + .expect("list") + .is_empty() + ); + assert!(table_doc(&fx.engine, "dst").await.is_some()); - let blank = UpdateTableInput { - table_name: "dst".to_owned(), - billing_mode: None, - table_throughput_mode: None, - provisioned_throughput: None, - deletion_protection_enabled: None, - global_secondary_index_updates: None, - attribute_definitions: None, - stream_specification: None, - table_class: None, - on_demand_throughput: None, - vector_index_updates: None, - }; - let changes = vec![ - UpdateTableInput { - billing_mode: Some(BillingMode::Provisioned), - provisioned_throughput: Some(ProvisionedThroughput { - read_capacity_units: 5, - write_capacity_units: 5, - }), - ..blank.clone() - }, - UpdateTableInput { - deletion_protection_enabled: Some(true), - ..blank.clone() - }, - UpdateTableInput { - stream_specification: Some(StreamSpecification { - stream_enabled: true, - stream_view_type: Some(StreamViewType::NewAndOldImages), - }), - ..blank.clone() - }, - UpdateTableInput { - attribute_definitions: Some(vec![attr("gpk", ScalarAttributeType::S)]), - global_secondary_index_updates: Some(vec![GlobalSecondaryIndexUpdate { - create: Some(CreateGsiAction { - index_name: "g2".to_owned(), - key_schema: vec![key("gpk", KeyType::Hash)], - projection: Projection { - projection_type: ProjectionType::KeysOnly, - non_key_attributes: None, - }, - provisioned_throughput: None, - }), - update: None, - delete: None, - }]), - ..blank.clone() - }, - ]; - for change in &changes { - let refused = fx.engine.update_table(ACCOUNT, change.clone()).await; + gate.add_permits(1); + assert_fully_restored_with(&fx.engine, "dst", FEW).await; + fx.finish().await; +} + +/// A DeleteBackup and restores of one backup order wholly one before the +/// other: when the delete goes first, every restore is "Backup not found" +/// and no target is left; when a registration goes first, exactly one target +/// exists, the delete is BackupInUse, and so is every other restore. Even +/// rounds hold the three registrations behind a gate until the delete has +/// answered, so the delete wins; odd rounds release the registrations and +/// start the delete at the same moment, a race over the backup's lock that +/// the registrations have won on every run so far. Both orders must occur. +/// The restores are accounted by the +/// catalog (the targets that exist for the request names), not by what their +/// requests answered: a request that outlives the registration bound answers +/// an error while its target appears, and the copy of every registered +/// target is released and driven to ACTIVE before the backup is deleted. +#[tokio::test(flavor = "multi_thread")] +#[ignore = "needs a MongoDB replica set in EXTENDDB_TEST_MONGODB_URI"] +async fn live_a_delete_and_concurrent_restores_of_one_backup_serialize() { + const FEW: usize = 20; + const ROUNDS: usize = 6; + const RESTORES: usize = 3; + let fx = Fixture::new().await; + source_table_with(&fx.engine, "src", FEW).await; + // A backup that stays, so a restore request of a deleted backup can be + // built after the delete (the request is built from an AVAILABLE source). + let keep = fx + .engine + .create(ACCOUNT, "src", "keep", JobPermit::detached()) + .await + .expect("backup"); + let b_arn_for_request = keep.backup_arn.clone(); + let (mut delete_won, mut registration_won) = (0, 0); + for round in 0..ROUNDS { + let backup = fx + .engine + .create(ACCOUNT, "src", &format!("b{round}"), JobPermit::detached()) + .await + .expect("backup"); + let targets: Vec = (0..RESTORES).map(|i| format!("r{round}_{i}")).collect(); + let mut requests = Vec::new(); + for target in &targets { + requests.push(restore_request(&fx.engine, &backup.backup_arn, target).await); + } + let (gated, registration_gate, copy_gate) = gated_at_registration(&fx.engine); + let delete_first = round % 2 == 0; + let restores = futures::future::join_all( + requests + .into_iter() + .map(|request| gated.restore(ACCOUNT, request, JobPermit::detached())), + ); + let delete = async { + if delete_first { + let outcome = fx.engine.delete(ACCOUNT, &backup.backup_arn).await; + registration_gate.add_permits(RESTORES); + outcome + } else { + registration_gate.add_permits(RESTORES); + fx.engine.delete(ACCOUNT, &backup.backup_arn).await + } + }; + let (restores, delete) = tokio::join!(restores, delete); + // The catalog's account of the race, which every answer agrees with + // where the request saw one. + let registered = registered_targets(&fx.engine, &targets).await; + for (target, result) in targets.iter().zip(&restores) { + match result { + Ok(_) => assert!( + registered.contains(target), + "round {round}: {target} was accepted but has no target" + ), + Err(StorageError::BackupInUse(_)) => assert!( + !registered.contains(target), + "round {round}: {target} was refused but has a target" + ), + Err(StorageError::Validation(m)) if m.starts_with("Backup not found") => { + assert!( + !registered.contains(target), + "round {round}: {target} was refused but has a target" + ); + } + Err(StorageError::Internal(m)) if m.contains("was not registered within") => { + eprintln!( + "round {round}: {target}: the request outlived the registration bound" + ); + } + Err(other) => panic!("round {round}: {target}: {other}"), + } + } + match delete { + Ok(desc) => { + delete_won += 1; + assert_eq!(desc.backup_details.backup_status, "DELETED"); + assert!( + registered.is_empty(), + "round {round}: a target was registered after the delete won: \ + {registered:?} {restores:?}" + ); + for (target, result) in targets.iter().zip(&restores) { + assert!( + matches!(result, Err(StorageError::Validation(m)) if m.starts_with("Backup not found")), + "round {round}: {target}: {result:?}" + ); + } + } + Err(StorageError::BackupInUse(_)) => { + registration_won += 1; + assert_eq!( + registered.len(), + 1, + "round {round}: exactly one restore registers: {registered:?} {restores:?}" + ); + for (target, result) in targets.iter().zip(&restores) { + assert!( + !matches!(result, Err(StorageError::Validation(_))), + "round {round}: {target}: a registration won, so no restore saw the \ + backup gone: {result:?}" + ); + } + copy_gate.add_permits(registered.len()); + for target in ®istered { + assert_fully_restored_with(&fx.engine, target, FEW).await; + } + fx.engine + .delete(ACCOUNT, &backup.backup_arn) + .await + .expect("DeleteBackup once every target is ACTIVE"); + } + Err(other) => panic!("round {round}: the delete failed: {other}"), + } + assert_eq!( + backup_meta(&fx.engine, &backup.backup_arn) + .await + .get_str("backup_status"), + Ok("DELETED") + ); + // The registration's own check, whichever order the race took: a + // deleted backup is not there to restore from, and nothing is created. + let late = format!("r{round}_late"); + let mut request = restore_request(&fx.engine, &b_arn_for_request, &late).await; + request.backup_arn = backup.backup_arn.clone(); + let refused = fx + .engine + .restore(ACCOUNT, request, JobPermit::detached()) + .await; assert!( - matches!(&refused, Err(StorageError::TableNotActive(n)) if n == "dst"), - "UpdateTable on a CREATING restore target: {refused:?}" + matches!(&refused, Err(StorageError::Validation(m)) if m.starts_with("Backup not found")), + "round {round}: {refused:?}" ); + assert!(table_doc(&fx.engine, &late).await.is_none()); } - let mut after = table_doc(&fx.engine, "dst").await.expect("target"); - // The restore's heartbeat refreshes this field on its own schedule. - let mut before = before; - before.remove(TARGET_HEARTBEAT); - after.remove(TARGET_HEARTBEAT); - assert_eq!(after, before, "a refused UpdateTable changes nothing"); + eprintln!( + "the delete won {delete_won} and a registration won {registration_won} of {ROUNDS} rounds" + ); + assert!( + delete_won >= 1 && registration_won >= 1, + "both orders occur: the delete won {delete_won}, a registration won {registration_won}" + ); + fx.finish().await; +} + +/// (a) The response is CREATING while the copy has not run; the target then +/// reaches ACTIVE with every item and index entry; the summary carries the +/// source table ARN and the backup's creation time. +#[tokio::test(flavor = "multi_thread")] +#[ignore = "needs a MongoDB replica set in EXTENDDB_TEST_MONGODB_URI"] +async fn live_restore_returns_creating_and_the_detached_copy_finishes() { + let fx = Fixture::new().await; + let source_arn = source_table(&fx.engine, "src").await; + let backup = fx + .engine + .create(ACCOUNT, "src", "b", JobPermit::detached()) + .await + .expect("backup"); + let request = restore_request(&fx.engine, &backup.backup_arn, "dst").await; + let (gated, gate) = gated(&fx.engine); + + let desc = tokio::time::timeout( + Duration::from_secs(20), + gated.restore(ACCOUNT, request, JobPermit::detached()), + ) + .await + .expect("the restore must return while its copy is held") + .expect("restore"); + assert_eq!(desc.table_status, TableStatus::Creating); + let doc = table_doc(&fx.engine, "dst").await.expect("target"); + assert_eq!(doc.get_str("table_status"), Ok("CREATING")); assert_eq!( - index_ids(&fx.engine, before.get_str("table_id").expect("id")) + count(&fx.engine, &data_collection_name(&desc.table_id)).await, + 0, + "no item may be copied before the response" + ); + let summary = fx + .engine + .describe_table( + ACCOUNT, + DescribeTableInput { + table_name: "dst".to_owned(), + }, + ) + .await + .expect("describe") + .restore_summary + .expect("restore summary while CREATING"); + assert_eq!( + summary.source_backup_arn.as_deref(), + Some(backup.backup_arn.as_str()) + ); + assert_eq!( + summary.source_table_arn.as_deref(), + Some(source_arn.as_str()) + ); + assert_eq!(summary.restore_date_time, backup.backup_creation_date_time); + assert!(summary.restore_in_progress); + // The backup time is the shared id's instant, kept to the millisecond. + assert_eq!( + Some(backup.backup_creation_date_time), + extenddb_storage::util::backup_creation_date_time(&backup.backup_arn) + ); + + gate.add_permits(1); + assert_fully_restored(&fx.engine, "dst").await; + fx.finish().await; +} + +/// (b) A request future dropped after its first poll still produces a +/// complete restore. +#[tokio::test(flavor = "multi_thread")] +#[ignore = "needs a MongoDB replica set in EXTENDDB_TEST_MONGODB_URI"] +async fn live_a_dropped_restore_request_still_reaches_active() { + let fx = Fixture::new().await; + source_table(&fx.engine, "src").await; + let backup = fx + .engine + .create(ACCOUNT, "src", "b", JobPermit::detached()) + .await + .expect("backup"); + let request = restore_request(&fx.engine, &backup.backup_arn, "dst").await; + let mut future = fx.engine.restore(ACCOUNT, request, JobPermit::detached()); + assert!( + (&mut future).now_or_never().is_none(), + "the first poll must not complete the restore" + ); + drop(future); + assert_fully_restored(&fx.engine, "dst").await; + fx.finish().await; +} + +/// (e) While the copy runs, DeleteBackup of the source is BackupInUse and +/// DeleteTable of the target is refused; both succeed once it is ACTIVE. +#[tokio::test(flavor = "multi_thread")] +#[ignore = "needs a MongoDB replica set in EXTENDDB_TEST_MONGODB_URI"] +async fn live_backup_and_target_are_protected_while_the_copy_runs() { + let fx = Fixture::new().await; + source_table(&fx.engine, "src").await; + let backup = fx + .engine + .create(ACCOUNT, "src", "b", JobPermit::detached()) + .await + .expect("backup"); + let request = restore_request(&fx.engine, &backup.backup_arn, "dst").await; + let (gated, gate) = gated(&fx.engine); + gated + .restore(ACCOUNT, request, JobPermit::detached()) + .await + .expect("restore"); + + let in_use = fx.engine.delete(ACCOUNT, &backup.backup_arn).await; + assert!( + matches!(in_use, Err(StorageError::BackupInUse(_))), + "the source backup is in use while the copy runs: {in_use:?}" + ); + let refused = fx + .engine + .delete_table( + ACCOUNT, + DeleteTableInput { + table_name: "dst".to_owned(), + }, + ) + .await; + assert!( + matches!(&refused, Err(StorageError::IndexesInUse(m)) if m.contains("being restored")), + "DeleteTable of a target being restored is refused: {refused:?}" + ); + assert_eq!( + table_doc(&fx.engine, "dst") + .await + .expect("target") + .get_str("table_status"), + Ok("CREATING"), + "both calls ran inside the copy window" + ); + + gate.add_permits(1); + assert_fully_restored(&fx.engine, "dst").await; + fx.engine + .delete_table( + ACCOUNT, + DeleteTableInput { + table_name: "dst".to_owned(), + }, + ) + .await + .expect("DeleteTable after ACTIVE"); + let deleted = fx + .engine + .delete(ACCOUNT, &backup.backup_arn) + .await + .expect("DeleteBackup after ACTIVE"); + assert_eq!(deleted.backup_details.backup_status, "DELETED"); + fx.finish().await; +} + +/// The definition captured at CreateBackup carries TTL and the stream +/// specification, and ListBackups reports the source table id. +#[tokio::test(flavor = "multi_thread")] +#[ignore = "needs a MongoDB replica set in EXTENDDB_TEST_MONGODB_URI"] +async fn live_create_backup_captures_ttl_stream_and_table_id() { + use extenddb_storage::MetadataEngine; + let fx = Fixture::new().await; + source_table(&fx.engine, "src").await; + fx.engine + .update_ttl(ACCOUNT, "src", "expires", true) + .await + .expect("ttl"); + let table = table_doc(&fx.engine, "src").await.expect("src"); + let table_id = table.get_str("table_id").expect("id").to_owned(); + let mut backups = Vec::new(); + for i in 0..3 { + backups.push( + fx.engine + .create(ACCOUNT, "src", &format!("b{i}"), JobPermit::detached()) + .await + .expect("backup"), + ); + } + for b in &backups { + let source = fx + .engine + .restore_source(ACCOUNT, &b.backup_arn) + .await + .expect("source"); + assert_eq!( + Some(source.backup_creation_date_time), + extenddb_storage::util::backup_creation_date_time(&b.backup_arn) + ); + assert_eq!( + source.backup_creation_date_time, + b.backup_creation_date_time + ); + let def = source.definition.expect("definition"); + let ttl = def.time_to_live.expect("ttl captured"); + assert_eq!(ttl.attribute_name.as_deref(), Some("expires")); + assert!(def.stream_specification.is_none()); + assert_eq!(def.global_secondary_indexes.len(), 1); + assert_eq!(def.local_secondary_indexes.len(), 1); + let described = fx + .engine + .describe(ACCOUNT, &b.backup_arn) + .await + .expect("describe"); + assert_eq!( + described.backup_details.backup_creation_date_time, + b.backup_creation_date_time + ); + assert!(described.source_table_feature_details.is_none()); + } + let listed = fx.engine.list(ACCOUNT, None).await.expect("list"); + assert_eq!(listed.len(), 3); + for summary in &listed { + assert_eq!(summary.table_id, table_id); + assert_eq!( + Some(summary.backup_creation_date_time), + extenddb_storage::util::backup_creation_date_time(&summary.backup_arn) + ); + } + let foreign = fx + .engine + .restore_source("999999999999", &backups[0].backup_arn) + .await; + assert!( + matches!(&foreign, Err(StorageError::Validation(m)) if m.starts_with("Backup not found")), + "{foreign:?}" + ); + fx.finish().await; +} + +async fn table_doc_for(engine: &MongoEngine, account: &str, name: &str) -> Option { + tables_majority(engine) + .find_one(doc! { "_id": { "account_id": account, "table_name": name } }) + .await + .expect("read table") +} + +async fn wait_for_status_for(engine: &MongoEngine, account: &str, name: &str, status: &str) { + let deadline = Instant::now() + Duration::from_secs(120); + loop { + if let Some(doc) = table_doc_for(engine, account, name).await + && doc.get_str("table_status") == Ok(status) + { + return; + } + // The account id is kept out of the message: the scanner reads an + // interpolated account as a logged credential. + let current = table_doc_for(engine, account, name) + .await + .and_then(|doc| doc.get_str("table_status").ok().map(str::to_owned)); + assert!( + Instant::now() < deadline, + "table {name} did not reach {status}: {current:?}" + ); + tokio::time::sleep(Duration::from_millis(100)).await; + } +} + +/// One restore of a backup at a time. Four restores of one backup +/// started at the same moment: exactly one is accepted and the other three +/// are `BackupInUse`; a restore started while the accepted one is still +/// CREATING (copy held) is `BackupInUse` as well; once the target is ACTIVE a +/// new restore of the backup is accepted. Every refusal names the target the +/// backup is being restored into. +#[tokio::test(flavor = "multi_thread")] +#[ignore = "needs a MongoDB replica set in EXTENDDB_TEST_MONGODB_URI"] +async fn live_a_second_restore_of_a_backup_while_one_is_creating_is_backup_in_use() { + const FEW: usize = 300; + let fx = Fixture::new().await; + source_table_with(&fx.engine, "src", FEW).await; + let backup = fx + .engine + .create(ACCOUNT, "src", "b", JobPermit::detached()) + .await + .expect("backup"); + let (gated, gate) = gated(&fx.engine); + let mut requests = Vec::new(); + for i in 0..4 { + requests.push(restore_request(&fx.engine, &backup.backup_arn, &format!("dst{i}")).await); + } + let results = futures::future::join_all( + requests + .into_iter() + .map(|request| gated.restore(ACCOUNT, request, JobPermit::detached())), + ) + .await; + let accepted: Vec = results + .iter() + .enumerate() + .filter_map(|(i, r)| r.is_ok().then_some(i)) + .collect(); + assert_eq!( + accepted.len(), + 1, + "exactly one restore is accepted: {results:?}" + ); + for (i, result) in results.iter().enumerate() { + if i != accepted[0] { + match result { + Err(StorageError::BackupInUse(m)) => assert_eq!( + *m, + format!( + "Backup is being used to restore another table: {}", + backup.backup_arn + ), + "restore {i}: the service's text, naming the backup and not the target" + ), + other => panic!("restore {i} must be BackupInUse: {other:?}"), + } + } + } + let winner = format!("dst{}", accepted[0]); + assert_eq!( + table_doc(&fx.engine, &winner) + .await + .expect("target") + .get_str("table_status"), + Ok("CREATING") + ); + // Sequential, while the winner's copy is held: still in use, and the + // refusal names the target with the wording the other backends use. + let request = restore_request(&fx.engine, &backup.backup_arn, "late").await; + match fx + .engine + .restore(ACCOUNT, request, JobPermit::detached()) + .await + { + Err(StorageError::BackupInUse(m)) => assert_eq!( + m, + format!( + "Backup is being used to restore another table: {}", + backup.backup_arn + ) + ), + other => panic!("a restore during the CREATING window is BackupInUse: {other:?}"), + } + assert!(table_doc(&fx.engine, "late").await.is_none()); + + gate.add_permits(1); + assert_fully_restored_with(&fx.engine, &winner, FEW).await; + let request = restore_request(&fx.engine, &backup.backup_arn, "late").await; + fx.engine + .restore(ACCOUNT, request, JobPermit::detached()) + .await + .expect("a restore after the first is ACTIVE is accepted"); + assert_fully_restored_with(&fx.engine, "late", FEW).await; + fx.finish().await; +} + +/// A restore into a name that is a CREATING restore target is +/// `TableBeingRestored`, naming the target and the backup it is being restored +/// from; the same name taken by an ACTIVE table is `TableAlreadyExists`. +#[tokio::test(flavor = "multi_thread")] +#[ignore = "needs a MongoDB replica set in EXTENDDB_TEST_MONGODB_URI"] +async fn live_a_restore_into_a_creating_restore_target_is_table_being_restored() { + const FEW: usize = 300; + let fx = Fixture::new().await; + source_table_with(&fx.engine, "src", FEW).await; + let b1 = fx + .engine + .create(ACCOUNT, "src", "b1", JobPermit::detached()) + .await + .expect("backup 1"); + let b2 = fx + .engine + .create(ACCOUNT, "src", "b2", JobPermit::detached()) + .await + .expect("backup 2"); + let (gated, gate) = gated(&fx.engine); + let request = restore_request(&fx.engine, &b1.backup_arn, "dst").await; + gated + .restore(ACCOUNT, request, JobPermit::detached()) + .await + .expect("first restore"); + let request = restore_request(&fx.engine, &b2.backup_arn, "dst").await; + match fx + .engine + .restore(ACCOUNT, request, JobPermit::detached()) + .await + { + Err(StorageError::TableBeingRestored { + table_name, + backup_arn, + }) => { + assert_eq!(table_name, "dst"); + assert_eq!(backup_arn, b1.backup_arn); + } + other => panic!("restore into a CREATING restore target: {other:?}"), + } + // What the engine's own name check reads before its job permit: the + // target describes as CREATING with its source backup in the + // RestoreSummary, which is what it answers TableInUse with. + let described = fx + .engine + .describe_table( + ACCOUNT, + DescribeTableInput { + table_name: "dst".to_owned(), + }, + ) + .await + .expect("describe the CREATING target"); + assert_eq!(described.table_status, TableStatus::Creating); + assert_eq!( + described + .restore_summary + .as_ref() + .and_then(|r| r.source_backup_arn.as_deref()), + Some(b1.backup_arn.as_str()), + "the CREATING target names its source backup" + ); + // The same backup into the same name: the name is what is refused, not + // the backup (the service answers TableInUseException). + let request = restore_request(&fx.engine, &b1.backup_arn, "dst").await; + match fx + .engine + .restore(ACCOUNT, request, JobPermit::detached()) + .await + { + Err(StorageError::TableBeingRestored { + table_name, + backup_arn, + }) => { + assert_eq!(table_name, "dst"); + assert_eq!(backup_arn, b1.backup_arn); + } + other => panic!("the same backup into its CREATING target: {other:?}"), + } + // The name is checked before the backup: a backup that does not exist, + // restored into that name, is refused as the name too. + let mut missing = restore_request(&fx.engine, &b2.backup_arn, "dst").await; + missing.backup_arn = flip_last_hex(&b2.backup_arn); + match fx + .engine + .restore(ACCOUNT, missing, JobPermit::detached()) + .await + { + Err(StorageError::TableBeingRestored { + table_name, + backup_arn, + }) => { + assert_eq!(table_name, "dst"); + assert_eq!(backup_arn, b1.backup_arn); + } + other => panic!("a missing backup into a CREATING restore target: {other:?}"), + } + gate.add_permits(1); + assert_fully_restored_with(&fx.engine, "dst", FEW).await; + let request = restore_request(&fx.engine, &b2.backup_arn, "dst").await; + assert!( + matches!( + fx.engine + .restore(ACCOUNT, request, JobPermit::detached()) + .await, + Err(StorageError::TableAlreadyExists(_)) + ), + "an ACTIVE table of that name is TableAlreadyExists" + ); + // The former target describes as ACTIVE, so the engine's name check + // answers TableAlreadyExists for it too. + let former = fx + .engine + .describe_table( + ACCOUNT, + DescribeTableInput { + table_name: "dst".to_owned(), + }, + ) + .await + .expect("describe the former target"); + assert_eq!(former.table_status, TableStatus::Active); + fx.finish().await; +} + +/// Twenty restores of twenty distinct backups (twenty accounts, so no shared +/// lock serializes their registration) against the driver's default pool of +/// 10 all register, the engine answers a catalog read right after, and every +/// target reaches ACTIVE. Before the fix, the registration's hand-over left +/// each creation's in-flight command holding a pooled connection while it +/// waited for another, and this wedged the process for good. +#[tokio::test(flavor = "multi_thread")] +#[ignore = "needs a MongoDB replica set in EXTENDDB_TEST_MONGODB_URI"] +async fn live_twenty_concurrent_restores_of_distinct_backups_do_not_wedge_the_pool() { + const N: usize = 20; + const FEW: usize = 20; + let fx = Fixture::new().await; + let accounts: Vec = (0..N).map(|i| format!("9100000000{i:02}")).collect(); + let mut requests = Vec::new(); + for account in &accounts { + source_table_for(&fx.engine, account, "src", FEW).await; + let backup = fx + .engine + .create(account, "src", "b", JobPermit::detached()) + .await + .expect("backup"); + requests.push(( + account.clone(), + restore_request_for(&fx.engine, account, &backup.backup_arn, "dst").await, + )); + } + let started = Instant::now(); + let results = futures::future::join_all(requests.into_iter().map(|(account, request)| { + let engine = fx.engine.clone(); + async move { + tokio::time::timeout( + Duration::from_secs(60), + engine.restore(&account, request, JobPermit::detached()), + ) + .await + } + })) + .await; + let elapsed = started.elapsed(); + for (i, result) in results.iter().enumerate() { + match result { + Ok(Ok(_)) => {} + Ok(Err(e)) => panic!("restore {i} failed: {e}"), + Err(_) => panic!("restore {i} did not return within 60 s: the pool is wedged"), + } + } + eprintln!("{N} concurrent restores of distinct backups registered in {elapsed:?}"); + // Liveness: the engine still answers a catalog read, promptly. + let probe = tokio::time::timeout( + Duration::from_secs(10), + fx.engine.describe_table( + &accounts[0], + DescribeTableInput { + table_name: "src".to_owned(), + }, + ), + ) + .await + .expect("a catalog read after the restores returns within 10 s") + .expect("describe"); + assert_eq!(probe.table_status, TableStatus::Active); + for account in &accounts { + wait_for_status_for(&fx.engine, account, "dst", "ACTIVE").await; + } + fx.finish().await; +} + +/// UpdateTable on a CREATING restore target is `TableNotActive`, for +/// every kind of change (billing mode, deletion protection, a stream, a GSI), +/// and leaves the target as the backup describes; once ACTIVE the same +/// changes are accepted. A table that does not exist is `TableNotFound`. +#[tokio::test(flavor = "multi_thread")] +#[ignore = "needs a MongoDB replica set in EXTENDDB_TEST_MONGODB_URI"] +async fn live_update_table_on_a_creating_restore_target_is_refused() { + use extenddb_core::types::{ + CreateGsiAction, GlobalSecondaryIndexUpdate, ProvisionedThroughput, StreamSpecification, + StreamViewType, UpdateTableInput, + }; + const FEW: usize = 300; + let fx = Fixture::new().await; + source_table_with(&fx.engine, "src", FEW).await; + let backup = fx + .engine + .create(ACCOUNT, "src", "b", JobPermit::detached()) + .await + .expect("backup"); + let (gated, gate) = gated(&fx.engine); + let request = restore_request(&fx.engine, &backup.backup_arn, "dst").await; + gated + .restore(ACCOUNT, request, JobPermit::detached()) + .await + .expect("restore"); + let before = table_doc(&fx.engine, "dst").await.expect("target"); + assert_eq!(before.get_str("table_status"), Ok("CREATING")); + + let blank = UpdateTableInput { + table_name: "dst".to_owned(), + billing_mode: None, + table_throughput_mode: None, + provisioned_throughput: None, + deletion_protection_enabled: None, + global_secondary_index_updates: None, + attribute_definitions: None, + stream_specification: None, + table_class: None, + on_demand_throughput: None, + vector_index_updates: None, + }; + let changes = vec![ + UpdateTableInput { + billing_mode: Some(BillingMode::Provisioned), + provisioned_throughput: Some(ProvisionedThroughput { + read_capacity_units: 5, + write_capacity_units: 5, + }), + ..blank.clone() + }, + UpdateTableInput { + deletion_protection_enabled: Some(true), + ..blank.clone() + }, + UpdateTableInput { + stream_specification: Some(StreamSpecification { + stream_enabled: true, + stream_view_type: Some(StreamViewType::NewAndOldImages), + }), + ..blank.clone() + }, + UpdateTableInput { + attribute_definitions: Some(vec![attr("gpk", ScalarAttributeType::S)]), + global_secondary_index_updates: Some(vec![GlobalSecondaryIndexUpdate { + create: Some(CreateGsiAction { + index_name: "g2".to_owned(), + key_schema: vec![key("gpk", KeyType::Hash)], + projection: Projection { + projection_type: ProjectionType::KeysOnly, + non_key_attributes: None, + }, + provisioned_throughput: None, + }), + update: None, + delete: None, + }]), + ..blank.clone() + }, + ]; + for change in &changes { + let refused = fx.engine.update_table(ACCOUNT, change.clone()).await; + assert!( + matches!(&refused, Err(StorageError::TableNotActive(n)) if n == "dst"), + "UpdateTable on a CREATING restore target: {refused:?}" + ); + } + let mut after = table_doc(&fx.engine, "dst").await.expect("target"); + // The restore's heartbeat refreshes the job record on its own schedule; + // the record stays RUNNING. + assert_eq!( + after + .get_document(RESTORE_JOB) + .expect("job") + .get_str("status"), + Ok("RUNNING") + ); + let mut before = before; + before.remove(RESTORE_JOB); + after.remove(RESTORE_JOB); + assert_eq!(after, before, "a refused UpdateTable changes nothing"); + assert_eq!( + index_ids(&fx.engine, before.get_str("table_id").expect("id")) + .await + .len(), + 2, + "no GSI was added" + ); + let missing = fx + .engine + .update_table( + ACCOUNT, + UpdateTableInput { + table_name: "nope".to_owned(), + deletion_protection_enabled: Some(true), + ..blank.clone() + }, + ) + .await; + assert!( + matches!(missing, Err(StorageError::TableNotFound(_))), + "{missing:?}" + ); + + gate.add_permits(1); + assert_fully_restored_with(&fx.engine, "dst", FEW).await; + let desc = fx + .engine + .update_table(ACCOUNT, changes[1].clone()) + .await + .expect("UpdateTable once ACTIVE"); + assert!(desc.deletion_protection_enabled); + fx.finish().await; +} + +/// A stream record's `created_at` field and its own `ApproximateCreationDateTime` +/// come from one clock read on the live write path, for a plain write and for +/// a service-identity deletion (the TTL worker's shape). +#[tokio::test] +#[ignore = "needs EXTENDDB_TEST_MONGODB_URI"] +async fn live_stream_record_created_at_matches_the_record_timestamp() { + use extenddb_core::expression::ExpressionMaps; + use extenddb_core::types::{StreamSpecification, StreamViewType, UserIdentity}; + use extenddb_storage::{DataEngine, StreamCapture}; + + let fx = Fixture::new().await; + fx.engine + .create_table( + ACCOUNT, + CreateTableInput { + table_name: "clock".to_owned(), + key_schema: vec![key("pk", KeyType::Hash)], + attribute_definitions: vec![attr("pk", ScalarAttributeType::S)], + billing_mode: Some(BillingMode::PayPerRequest), + stream_specification: Some(StreamSpecification { + stream_enabled: true, + stream_view_type: Some(StreamViewType::NewAndOldImages), + }), + ..Default::default() + }, + ) + .await + .expect("create the table"); + let key_info = fx + .engine + .table_key_info(ACCOUNT, "clock") + .await + .expect("key info"); + let maps = ExpressionMaps::default(); + let capture = StreamCapture { + view_type: StreamViewType::NewAndOldImages, + user_identity: None, + region: Arc::from("us-east-1"), + }; + for v in ["one", "two"] { + let item: Item = [ + ("pk".to_owned(), AttributeValue::S("k".to_owned())), + ("v".to_owned(), AttributeValue::S(v.to_owned())), + ] + .into_iter() + .collect(); + fx.engine + .put_item(&key_info, item, false, None, &maps, Some(&capture)) + .await + .expect("put"); + } + let key: Item = [("pk".to_owned(), AttributeValue::S("k".to_owned()))] + .into_iter() + .collect(); + fx.engine + .delete_item( + &key_info, + &key, + false, + None, + &maps, + Some(&StreamCapture { + view_type: StreamViewType::NewAndOldImages, + user_identity: Some(UserIdentity { + identity_type: "Service".to_owned(), + principal_id: "dynamodb.amazonaws.com".to_owned(), + }), + region: Arc::from("us-east-1"), + }), + ) + .await + .expect("delete"); + + let mut cursor = fx + .engine + .data_db + .collection::("stream_records") + .find(doc! {}) + .await + .expect("read back"); + let mut seen = 0; + while cursor.advance().await.expect("cursor") { + let d = cursor.deserialize_current().expect("document"); + let created_at = d.get_datetime("created_at").expect("created_at"); + let approx = d + .get_document("record_data") + .and_then(|r| r.get_document("dynamodb")) + .expect("record_data.dynamodb") + .get_i64("ApproximateCreationDateTime") + .expect("ApproximateCreationDateTime"); + assert_eq!( + created_at.timestamp_millis().div_euclid(1_000), + approx, + "created_at {created_at} and ApproximateCreationDateTime {approx} disagree" + ); + seen += 1; + } + assert_eq!(seen, 3, "two puts and one delete"); + fx.finish().await; +} + +// ───────────────────────────────────────────────────────────────────────── +// Store-backed restore: claim_restore, the claim, claim_restore_dead, +// read/list_restore_jobs, bulk_load, finish_restore, count_restore_target. +// The registration takes the `backup_index` row's lock (as the legacy path +// takes the `backups` row's), writes the record `kind = manifest` with the +// store and the claimant's spool, and the record's heartbeat is the server's +// clock from the registration on; bulk_load writes base documents only, the +// GSI backfill worker fills the indexes and flips the table ACTIVE. +// ───────────────────────────────────────────────────────────────────────── + +/// A composite-key input with a GSI (ALL) and an LSI (KEYS_ONLY), the shape +/// `source_table_for` creates, for a restore target. +fn manifest_input(name: &str) -> CreateTableInput { + CreateTableInput { + table_name: name.to_owned(), + key_schema: vec![key("pk", KeyType::Hash), key("sk", KeyType::Range)], + attribute_definitions: vec![ + attr("pk", ScalarAttributeType::S), + attr("sk", ScalarAttributeType::N), + attr("gpk", ScalarAttributeType::S), + attr("lsk", ScalarAttributeType::N), + ], + billing_mode: Some(BillingMode::PayPerRequest), + global_secondary_indexes: Some(vec![GsiInput { + index_name: "g1".to_owned(), + key_schema: vec![key("gpk", KeyType::Hash)], + projection: Projection { + projection_type: ProjectionType::All, + non_key_attributes: None, + }, + provisioned_throughput: None, + }]), + local_secondary_indexes: Some(vec![LsiInput { + index_name: "l1".to_owned(), + key_schema: vec![key("pk", KeyType::Hash), key("lsk", KeyType::Range)], + projection: Projection { + projection_type: ProjectionType::KeysOnly, + non_key_attributes: None, + }, + }]), + ..Default::default() + } +} + +/// A `backup_index` row of a backup of `table`, left AVAILABLE: what a +/// store-backed restore registers against. Returns the row (its `key()` is +/// the registration's backup key). +async fn available_backup(engine: &MongoEngine, table: &str, name: &str) -> BackupRow { + let row = row(ACCOUNT, table, name); + let claim = engine.claim_backup(&row).await.expect("claim backup row"); + assert_eq!( + claim + .set_status( + JobStatus::Creating, + JobStatus::Available, + &StatusFields::default() + ) + .await + .expect("to AVAILABLE"), + WriteOutcome::Matched + ); + drop(claim); + row +} + +/// The registration row for `row`, with the claimant's `spool_host` +/// (`None` for a filesystem store). `owner_generation` is unowned: the +/// backend mints its own. +fn registration(row: &BackupRow, spool: Option<&str>) -> RestoreRegistrationRow { + RestoreRegistrationRow { + backup: row.key(), + source_table_arn: format!( + "arn:aws:dynamodb:{REGION}:{ACCOUNT}:table/{}", + row.table_name + ), + restore_date_time: row.created_at, + store: store(), + spool_host: spool.map(str::to_owned), + owner_generation: UNOWNED_GENERATION, + } +} + +/// Item `i` of a restore: every attribute the source shape carries, both +/// index keys, and a 38-digit non-key number that survives as a string. +fn manifest_item(i: usize) -> Item { + [ + ("pk".to_owned(), AttributeValue::S(format!("p{}", i % 50))), + ("sk".to_owned(), AttributeValue::N(i.to_string())), + ("gpk".to_owned(), AttributeValue::S(format!("g{}", i % 7))), + ("lsk".to_owned(), AttributeValue::N((i * 3).to_string())), + ( + "big".to_owned(), + AttributeValue::N("12345678901234567890123456789012345678".to_owned()), + ), + ] + .into_iter() + .collect() +} + +/// The items `lo..hi` as the stream `bulk_load` reads. +fn item_stream(lo: usize, hi: usize) -> BoxStream<'static, Result> { + let items: Vec<_> = (lo..hi).map(manifest_item).collect(); + futures::stream::iter(items.into_iter().map(Ok)).boxed() +} + +/// The registration writes the target CREATING with its `RestoreSummary` +/// and a RUNNING `kind = manifest` record carrying the store, the +/// claimant's `spool_host`, a fresh generation and the server's clock on +/// its heartbeat (the registering process's clock is an hour behind, and +/// the stored heartbeat is within seconds of the server's `$$NOW`); the +/// claim heartbeats and reads the record; `read_restore_job` and +/// `list_restore_jobs` fill the store and spool; `backup_in_use` names the +/// target. +#[tokio::test(flavor = "multi_thread")] +#[ignore = "needs a MongoDB replica set in EXTENDDB_TEST_MONGODB_URI"] +async fn live_claim_restore_registers_the_target_and_its_record() { + let fx = Fixture::new().await; + fx.worker.abort(); + let backup = available_backup(&fx.engine, "src", "b").await; + let reg = registration(&backup, Some("host-a:/var/spool/x")); + let behind = fx + .second_instance(BackupLiveness { + local_clock_behind: Duration::from_secs(3_600), + ..test_liveness() + }) + .await; + let (desc, claim) = behind + .claim_restore(ACCOUNT, manifest_input("dst"), ®) + .await + .expect("register"); + assert_eq!(desc.table_status, TableStatus::Creating); + let summary = desc.restore_summary.as_ref().expect("a RestoreSummary"); + assert_eq!( + summary.source_backup_arn.as_deref(), + Some(backup.backup_arn.as_str()) + ); + assert!(summary.restore_in_progress); + let generation = claim.generation(); + assert_ne!(generation, UNOWNED_GENERATION); + + let doc = table_doc(&fx.engine, "dst").await.expect("target"); + let job = doc.get_document(RESTORE_JOB).expect("job record"); + assert_eq!(job.get_str("status"), Ok("RUNNING")); + assert_eq!(job.get_str("kind"), Ok("manifest")); + assert_eq!( + u64::try_from(job.get_i64("owner_generation").unwrap()).unwrap(), + generation + ); + assert_eq!( + job.get_str("store_kind"), + Ok(StoreKind::Filesystem.as_str()) + ); + assert_eq!(job.get_str("store_root"), Ok(store().store_root.as_str())); + assert_eq!(job.get_str("store_id"), Ok(store().store_id.as_str())); + assert_eq!(job.get_str("spool_host"), Ok("host-a:/var/spool/x")); + // The heartbeat is the server's clock stamped inside the registration, + // not the inserting process's (an hour behind): within seconds of the + // server's `$$NOW`, read back through an aggregation. + let heartbeat = *job.get_datetime("heartbeat_at").expect("heartbeat"); + let now = server_now(&fx.engine).await; + let age_ms = now.timestamp_millis() - heartbeat.timestamp_millis(); + assert!( + (-2_000..=5_000).contains(&age_ms), + "the record's heartbeat is the server's clock, not the slow process's: {heartbeat:?} \ + is {age_ms} ms behind {now:?}" + ); + + let key = target_key(&doc); + let read = fx + .engine + .read_restore_job(&key) + .await + .expect("read") + .expect("a record"); + assert_eq!(read.kind, RestoreKind::Manifest); + assert_eq!(read.status, JobStatus::Running); + assert_eq!(read.owner_generation, generation); + assert_eq!( + read.store.as_ref().map(|s| s.store_kind), + Some(StoreKind::Filesystem) + ); + assert_eq!(read.spool_host.as_deref(), Some("host-a:/var/spool/x")); + assert_eq!(read.file_index, None); + + let listed = fx + .engine + .list_restore_jobs(Some(JobStatus::Running)) + .await + .expect("list"); + assert_eq!(listed.len(), 1); + assert_eq!(listed[0].target, key); + assert!(listed[0].store.is_some()); + assert_eq!(listed[0].spool_host.as_deref(), Some("host-a:/var/spool/x")); + + assert_eq!( + fx.engine + .backup_in_use(&backup.key()) + .await + .unwrap() + .as_deref(), + Some("dst") + ); + assert_eq!(claim.heartbeat().await.unwrap(), Heartbeat::Confirmed); + drop(claim); + fx.finish().await; +} + +/// A DeleteBackup and a store-backed registration of one backup order wholly +/// one before the other, on the `backup_index` row's lock: a delete that +/// wins leaves the registration `Backup not found` and no target; a +/// registration that wins makes the delete `BackupInUse`. A loop of +/// concurrent races holds the invariant that no target is ever registered +/// against a row that is not AVAILABLE. +#[tokio::test(flavor = "multi_thread")] +#[ignore = "needs a MongoDB replica set in EXTENDDB_TEST_MONGODB_URI"] +async fn live_the_two_orderings_against_delete_backup_on_the_backup_index_row() { + let fx = Fixture::new().await; + fx.worker.abort(); + + // Order A: the delete wins. + let a = available_backup(&fx.engine, "srcA", "ba").await; + assert_eq!( + fx.engine.start_backup_delete(&a.key()).await.unwrap(), + DeleteStart::Deleting + ); + let refused = fx + .engine + .claim_restore(ACCOUNT, manifest_input("dstA"), ®istration(&a, None)) + .await + .err(); + assert!( + matches!(&refused, Some(StorageError::Validation(m)) if m.contains("Backup not found")), + "the registration finds the backup gone: {refused:?}" + ); + assert!(table_doc(&fx.engine, "dstA").await.is_none()); + + // Order B: the registration wins. + let b = available_backup(&fx.engine, "srcB", "bb").await; + let (_, claim) = fx + .engine + .claim_restore(ACCOUNT, manifest_input("dstB"), ®istration(&b, None)) + .await + .expect("register"); + assert_eq!( + fx.engine.start_backup_delete(&b.key()).await.unwrap(), + DeleteStart::InUse("dstB".to_owned()) + ); + assert_eq!( + fx.engine + .read_backup_row(&arn_of(&b)) + .await + .unwrap() + .unwrap() + .status, + BackupStatus::Available, + "a refused delete leaves the row AVAILABLE" + ); + drop(claim); + + // Concurrent races: the lock keeps the two consistent every time, and + // both outcomes occur across the loop (no frozen answer). + let mut delete_won = 0; + let mut restore_won = 0; + for i in 0..16 { + let backup = available_backup(&fx.engine, &format!("srcC{i}"), "bc").await; + let name = format!("dstC{i}"); + let e1 = fx.engine.clone(); + let e2 = fx.engine.clone(); + let key = backup.key(); + let reg = registration(&backup, None); + let del = tokio::spawn(async move { e1.start_backup_delete(&key).await }); + let res = + tokio::spawn( + async move { e2.claim_restore(ACCOUNT, manifest_input(&name), ®).await }, + ); + let (del, res) = (del.await.unwrap().unwrap(), res.await.unwrap()); + let res_ok = res.is_ok(); + let res_err = res.err(); + let target = table_doc(&fx.engine, &format!("dstC{i}")).await; + let status = fx + .engine + .read_backup_row(&arn_of(&backup)) + .await + .unwrap() + .expect("the backup_index row") + .status; + if target.is_some() { + assert_eq!( + status, + BackupStatus::Available, + "a target was registered, so the delete must have lost" + ); + assert!( + matches!(del, DeleteStart::InUse(_)), + "iteration {i}: {del:?}" + ); + assert!(res_ok, "iteration {i}"); + restore_won += 1; + } else { + assert!( + matches!(&res_err, Some(StorageError::Validation(m)) if m.contains("Backup not found")), + "iteration {i}: no target but the registration did not refuse: {res_err:?}" + ); + assert_eq!(status, BackupStatus::Deleting, "iteration {i}"); + delete_won += 1; + } + } + assert!( + delete_won > 0 && restore_won > 0, + "both orders occur: {delete_won} / {restore_won}" + ); + fx.finish().await; +} + +/// Two concurrent registrations of one backup leave one CREATING target and +/// answer `BackupInUse` to the other (the in-use rule under the lock). +#[tokio::test(flavor = "multi_thread")] +#[ignore = "needs a MongoDB replica set in EXTENDDB_TEST_MONGODB_URI"] +async fn live_two_concurrent_registrations_of_one_backup_leave_one_target() { + let fx = Fixture::new().await; + fx.worker.abort(); + let backup = available_backup(&fx.engine, "src", "b").await; + let e1 = fx.engine.clone(); + let e2 = fx.engine.clone(); + let r1 = registration(&backup, None); + let r2 = registration(&backup, None); + let t1 = + tokio::spawn(async move { e1.claim_restore(ACCOUNT, manifest_input("r1"), &r1).await }); + let t2 = + tokio::spawn(async move { e2.claim_restore(ACCOUNT, manifest_input("r2"), &r2).await }); + let o1 = t1.await.unwrap(); + let o2 = t2.await.unwrap(); + let oks = [&o1, &o2].iter().filter(|o| o.is_ok()).count(); + let in_use = [&o1, &o2] + .iter() + .filter(|o| matches!(o, Err(StorageError::BackupInUse(_)))) + .count(); + let errs: Vec<_> = [&o1, &o2] + .iter() + .filter_map(|o| o.as_ref().err().map(ToString::to_string)) + .collect(); + assert_eq!( + (oks, in_use), + (1, 1), + "one target, one BackupInUse: oks={oks}, errs={errs:?}" + ); + let targets = registered_targets(&fx.engine, &["r1".to_owned(), "r2".to_owned()]).await; + assert_eq!( + targets.len(), + 1, + "exactly one target registered: {targets:?}" + ); + for (_, claim) in [o1, o2].into_iter().flatten() { + drop(claim); + } + fx.finish().await; +} + +/// A name a table or a CREATING restore target already holds is refused as +/// `TableAlreadyExists` or `TableBeingRestored`, with nothing registered. +#[tokio::test(flavor = "multi_thread")] +#[ignore = "needs a MongoDB replica set in EXTENDDB_TEST_MONGODB_URI"] +async fn live_claim_restore_refuses_a_taken_name() { + let fx = Fixture::new().await; + fx.worker.abort(); + // A plain ACTIVE table. + fx.engine + .create_table_impl(ACCOUNT, manifest_input("taken"), false) + .await + .expect("create a table"); + let backup = available_backup(&fx.engine, "src", "b").await; + let exists = fx + .engine + .claim_restore( + ACCOUNT, + manifest_input("taken"), + ®istration(&backup, None), + ) + .await + .err(); + assert!( + matches!(&exists, Some(StorageError::TableAlreadyExists(n)) if n == "taken"), + "{exists:?}" + ); + + // A CREATING restore target. + let (_, claim) = fx + .engine + .claim_restore( + ACCOUNT, + manifest_input("being"), + ®istration(&backup, None), + ) + .await + .expect("first registration"); + let b2 = available_backup(&fx.engine, "src2", "b2").await; + let restoring = fx + .engine + .claim_restore(ACCOUNT, manifest_input("being"), ®istration(&b2, None)) + .await + .err(); + assert!( + matches!( + &restoring, + Some(StorageError::TableBeingRestored { table_name, backup_arn }) + if table_name == "being" && backup_arn == &backup.backup_arn + ), + "{restoring:?}" + ); + drop(claim); + fx.finish().await; +} + +/// `claim_restore_dead` takes over a RUNNING manifest record whose heartbeat +/// is stale: a fresh generation, the claimant's `spool_host` and +/// `heartbeat_at` refreshed to the server's clock in one write, so a second +/// taker issued right after the first (before the successor's first beat) +/// finds the record live and is refused, and the first taker's beat +/// confirms; the old generation's writes are fenced out. A live record, and +/// a legacy record whatever its age, are left alone. +#[tokio::test(flavor = "multi_thread")] +#[ignore = "needs a MongoDB replica set in EXTENDDB_TEST_MONGODB_URI"] +async fn live_claim_restore_dead_takes_over_a_stale_manifest_record() { + let fx = Fixture::new().await; + fx.worker.abort(); + let backup = available_backup(&fx.engine, "src", "b").await; + let (doc, claim) = fx + .engine + .claim_restore( + ACCOUNT, + manifest_input("t"), + ®istration(&backup, Some("old:/s")), + ) + .await + .expect("register"); + let desc_doc = table_doc(&fx.engine, "t").await.expect("target"); + let key = target_key(&desc_doc); + let g1 = claim.generation(); + let _ = doc; + + // Live: fresh heartbeat, so not dead. + assert!( + fx.engine + .claim_restore_dead(&key, STALE_AFTER, Some("new:/s2")) + .await + .unwrap() + .is_none(), + "a live record is not taken over" + ); + + // Age it past the window and take it over. + age_running_record(&fx.engine, "t", Duration::from_secs(600)).await; + let aged = *job_record(&fx.engine, "t") + .await + .get_datetime("heartbeat_at") + .expect("heartbeat"); + let taken = fx + .engine + .claim_restore_dead(&key, STALE_AFTER, Some("new:/s2")) + .await + .expect("takeover") + .expect("a stale record is taken over"); + let g2 = taken.generation(); + assert_ne!(g2, g1, "a fresh generation"); + let after = job_record(&fx.engine, "t").await; + assert_eq!( + u64::try_from(after.get_i64("owner_generation").unwrap()).unwrap(), + g2 + ); + assert_eq!( + after.get_str("spool_host"), + Ok("new:/s2"), + "the claimant's spool is written" + ); + // A second taker right after the first, before the successor's first + // beat: the record is live to it, and the first taker's beat confirms. + let second = fx + .engine + .claim_restore_dead(&key, STALE_AFTER, Some("third:/s3")) + .await + .expect("a second takeover attempt"); + assert!( + second.is_none(), + "a second taker right after the first is refused: {:?}", + second.as_ref().map(|c| c.generation()) + ); + assert_eq!( + taken.heartbeat().await.unwrap(), + Heartbeat::Confirmed, + "the first taker keeps the record" + ); + let refreshed = *after.get_datetime("heartbeat_at").expect("heartbeat"); + assert!( + refreshed > aged, + "the takeover refreshed heartbeat_at: {refreshed:?} was {aged:?}" + ); + let now = server_now(&fx.engine).await; + let age_ms = now.timestamp_millis() - refreshed.timestamp_millis(); + assert!( + (-2_000..=5_000).contains(&age_ms), + "the refreshed heartbeat is the server's now: {refreshed:?} is {age_ms} ms behind {now:?}" + ); + assert!( + fx.engine + .list_dead_restore_jobs(STALE_AFTER) + .await + .unwrap() + .is_empty(), + "the taken-over record is live to the listing at once" + ); + + // The old generation is fenced out; the new one owns the record. + assert_eq!( + claim + .set_status( + JobStatus::Running, + JobStatus::Failed, + &StatusFields::default() + ) + .await + .unwrap(), + WriteOutcome::NotMatched + ); + assert!( + matches!( + fx.engine.finish_restore(&key, &*claim, 0, 0).await.unwrap(), + Finished::NotMatched(_) + ), + "the dead generation cannot finish the restore" + ); + drop(claim); + drop(taken); + fx.finish().await; +} + +/// A legacy record, and a record that is not RUNNING, are never taken over. +#[tokio::test(flavor = "multi_thread")] +#[ignore = "needs a MongoDB replica set in EXTENDDB_TEST_MONGODB_URI"] +async fn live_claim_restore_dead_leaves_a_legacy_or_finished_record() { + let fx = Fixture::new().await; + fx.worker.abort(); + let backup = available_backup(&fx.engine, "src", "b").await; + let (doc, claim) = fx + .engine + .claim_restore(ACCOUNT, manifest_input("leg"), ®istration(&backup, None)) + .await + .expect("register"); + let key = target_key(&table_doc(&fx.engine, "leg").await.expect("target")); + let _ = doc; + // Make the record legacy and stale. + fx.engine + .catalog_db + .collection::("tables") + .update_one( + doc! { "_id": { "account_id": ACCOUNT, "table_name": "leg" } }, + doc! { "$set": { "restore_job.kind": "legacy" } }, + ) + .await + .expect("to legacy"); + age_running_record(&fx.engine, "leg", Duration::from_secs(600)).await; + // age_running_record rewrites the record RUNNING; set kind legacy again. + fx.engine + .catalog_db + .collection::("tables") + .update_one( + doc! { "_id": { "account_id": ACCOUNT, "table_name": "leg" } }, + doc! { "$set": { "restore_job.kind": "legacy" } }, + ) + .await + .expect("to legacy"); + assert!( + fx.engine + .claim_restore_dead(&key, STALE_AFTER, Some("new:/s")) + .await + .unwrap() + .is_none(), + "a legacy record is not taken over" + ); + drop(claim); + fx.finish().await; +} + +/// `bulk_load` writes a file's base documents and records `(file_index, +/// items_applied)` in the same transaction as each batch, so a resume +/// continues from the exact item `items_applied` names and re-applies +/// nothing; a duplicate key on a file being loaded for the first time (a +/// backup that holds a key twice) fails the file. +#[tokio::test(flavor = "multi_thread")] +#[ignore = "needs a MongoDB replica set in EXTENDDB_TEST_MONGODB_URI"] +async fn live_bulk_load_records_each_batch_and_a_resume_continues_from_items_applied() { + let fx = Fixture::new().await; + fx.worker.abort(); + let backup = available_backup(&fx.engine, "src", "b").await; + let (doc, claim) = fx + .engine + .claim_restore(ACCOUNT, manifest_input("dst"), ®istration(&backup, None)) + .await + .expect("register"); + let key = target_key(&table_doc(&fx.engine, "dst").await.expect("target")); + let _ = doc; + + // File 0: 2500 items across three insert_many batches, each committed + // with its progress in one transaction. + let applied = fx + .engine + .bulk_load( + &key, + &*claim, + BulkLoadFile { + file_index: 0, + start_at: 0, + reapplying: false, + }, + item_stream(0, 2500), + ) + .await + .expect("file 0"); + assert_eq!(applied, 2500); + assert_eq!(fx.engine.count_restore_target(&key).await.unwrap(), 2500); + let record = fx.engine.read_restore_job(&key).await.unwrap().unwrap(); + assert_eq!(record.file_index, Some(0)); + assert_eq!(record.items_applied, Some(2500)); + + // File 1's first 1000 items commit (one batch), then the process dies: + // the record names file 1 with 1000 applied. + let applied = fx + .engine + .bulk_load( + &key, + &*claim, + BulkLoadFile { + file_index: 1, + start_at: 0, + reapplying: false, + }, + item_stream(2500, 3500), + ) + .await + .expect("file 1, first batch"); + assert_eq!(applied, 1000); + let record = fx.engine.read_restore_job(&key).await.unwrap().unwrap(); + assert_eq!(record.file_index, Some(1)); + assert_eq!(record.items_applied, Some(1000)); + + // The resume continues file 1 from item 1000: the engine skipped the + // 1000 already applied and streams only the rest, so nothing is + // re-applied and no duplicate arises (the `reapplying` flag the engine + // sets for the first resumed file is vestigial on this backend, since + // the progress is exact). + let resumed = fx + .engine + .bulk_load( + &key, + &*claim, + BulkLoadFile { + file_index: 1, + start_at: 1000, + reapplying: true, + }, + item_stream(3500, 4000), + ) + .await + .expect("resume file 1"); + assert_eq!(resumed, 500); + assert_eq!(fx.engine.count_restore_target(&key).await.unwrap(), 4000); + let record = fx.engine.read_restore_job(&key).await.unwrap().unwrap(); + assert_eq!(record.file_index, Some(1)); + assert_eq!(record.items_applied, Some(1500)); + + // A file whose items the target already holds (a backup that holds a + // key twice) fails the file; the unique `_id` raises E11000. + let dup = fx + .engine + .bulk_load( + &key, + &*claim, + BulkLoadFile { + file_index: 2, + start_at: 0, + reapplying: false, + }, + item_stream(2500, 3500), + ) + .await; + assert!( + matches!(&dup, Err(StorageError::Validation(m)) if m.contains("holds a key twice")), + "{dup:?}" + ); + assert_eq!(fx.engine.count_restore_target(&key).await.unwrap(), 4000); + drop(claim); + fx.finish().await; +} + +/// A claim lost between two `insert_many` calls of one file (the record's +/// generation moved by a takeover while the load ran): the read before the +/// next call finds the record no longer this claim's, the file ends as a +/// lost claim, and the second call's documents never land. +#[tokio::test(flavor = "multi_thread")] +#[ignore = "needs a MongoDB replica set in EXTENDDB_TEST_MONGODB_URI"] +async fn live_bulk_load_stops_at_a_lost_claim_between_two_insert_many_calls() { + let fx = Fixture::new().await; + fx.worker.abort(); + let backup = available_backup(&fx.engine, "src", "b").await; + let (_doc, claim) = fx + .engine + .claim_restore( + ACCOUNT, + manifest_input("lost"), + ®istration(&backup, None), + ) + .await + .expect("register"); + let key = target_key(&table_doc(&fx.engine, "lost").await.expect("target")); + // The stream hands the first call's documents, then moves the record's + // generation when asked for the first item of the second call. + let tables = fx.engine.catalog_db.collection::("tables"); + let moved_key = key.clone(); + let stream: BoxStream<'static, Result> = Box::pin(futures::stream::unfold( + 0usize, + move |i| { + let tables = tables.clone(); + let key = moved_key.clone(); + async move { + if i >= 2 * INSERT_MANY_DOCUMENTS + 500 { + return None; + } + if i == INSERT_MANY_DOCUMENTS { + tables + .update_one( + doc! { "_id": { "account_id": &key.account_id, "table_name": &key.table_name } }, + doc! { "$inc": { RESTORE_JOB_GENERATION: 1_i64 } }, + ) + .await + .expect("another owner takes the record"); + } + Some((Ok(manifest_item(i)), i + 1)) + } + }, + )); + let out = fx + .engine + .bulk_load( + &key, + &*claim, + BulkLoadFile { + file_index: 0, + start_at: 0, + reapplying: false, + }, + stream, + ) + .await; + assert!( + matches!(&out, Err(StorageError::Internal(m)) if m.contains("claim on the target was lost")), + "the second batch's fence finds another owner: {out:?}" + ); + assert_eq!( + count(&fx.engine, &data_collection_name(&key.table_id)).await, + INSERT_MANY_DOCUMENTS as u64, + "the first batch's documents alone landed; the second batch is fenced out" + ); + // The first batch committed its documents and its progress in one + // transaction, so the record carries that batch's progress; the second + // batch, fenced out, recorded nothing. + let record = fx.engine.read_restore_job(&key).await.unwrap().unwrap(); + assert_eq!( + record.file_index, + Some(0), + "the first batch's progress is recorded" + ); + assert_eq!(record.items_applied, Some(INSERT_MANY_DOCUMENTS as u64)); + drop(claim); + fx.finish().await; +} + +/// `bulk_load` refuses an `N` sort key beyond Decimal128's precision with +/// the write path's `ValidationException`, and a 38-digit non-key number +/// survives a restore as a string. +#[tokio::test(flavor = "multi_thread")] +#[ignore = "needs a MongoDB replica set in EXTENDDB_TEST_MONGODB_URI"] +async fn live_bulk_load_refuses_a_long_sort_key_and_keeps_a_long_number() { + let fx = Fixture::new().await; + fx.worker.abort(); + let backup = available_backup(&fx.engine, "src", "b").await; + let (_, claim) = fx + .engine + .claim_restore(ACCOUNT, manifest_input("dec"), ®istration(&backup, None)) + .await + .expect("register"); + let key = target_key(&table_doc(&fx.engine, "dec").await.expect("target")); + + let long_sk: Item = [ + ("pk".to_owned(), AttributeValue::S("p".to_owned())), + // 35 significant digits: beyond Decimal128. + ( + "sk".to_owned(), + AttributeValue::N("12345678901234567890123456789012345".to_owned()), + ), + ("gpk".to_owned(), AttributeValue::S("g".to_owned())), + ("lsk".to_owned(), AttributeValue::N("1".to_owned())), + ] + .into_iter() + .collect(); + let refused = fx + .engine + .bulk_load( + &key, + &*claim, + BulkLoadFile { + file_index: 0, + start_at: 0, + reapplying: false, + }, + futures::stream::iter(vec![Ok(long_sk)]).boxed(), + ) + .await; + assert!( + matches!(&refused, Err(StorageError::Validation(m)) if m.contains("Decimal128")), + "{refused:?}" + ); + + // A good item with a 38-digit non-key number restores intact. + fx.engine + .bulk_load( + &key, + &*claim, + BulkLoadFile { + file_index: 0, + start_at: 0, + reapplying: false, + }, + item_stream(0, 1), + ) + .await + .expect("a good item"); + let stored = fx + .engine + .data_db + .collection::(&data_collection_name(&key.table_id)) + .find_one(doc! {}) + .await + .unwrap() + .expect("the item"); + let item = document_to_item(&stored).expect("item"); + assert_eq!( + item.get("big"), + Some(&AttributeValue::N( + "12345678901234567890123456789012345678".to_owned() + )), + "the 38-digit non-key number survives" + ); + drop(claim); + fx.finish().await; +} + +/// `finish_restore` hands the target to the backfill worker (`Pending`, +/// DONE + `restore_backfill_pending`), the backup stays in use until the +/// worker flips the table ACTIVE, and `list_dead_restore_jobs` leaves the +/// DONE record alone. A stale generation's `finish_restore` is `NotMatched`. +#[tokio::test(flavor = "multi_thread")] +#[ignore = "needs a MongoDB replica set in EXTENDDB_TEST_MONGODB_URI"] +async fn live_finish_restore_hands_over_and_the_worker_flips_active() { + const FEW: usize = 300; + let fx = Fixture::new().await; + fx.worker.abort(); + let backup = available_backup(&fx.engine, "src", "b").await; + let (_, claim) = fx + .engine + .claim_restore(ACCOUNT, manifest_input("fin"), ®istration(&backup, None)) + .await + .expect("register"); + let key = target_key(&table_doc(&fx.engine, "fin").await.expect("target")); + fx.engine + .bulk_load( + &key, + &*claim, + BulkLoadFile { + file_index: 0, + start_at: 0, + reapplying: false, + }, + item_stream(0, FEW), + ) + .await + .expect("load"); + + // A stale generation cannot finish it. + age_running_record(&fx.engine, "fin", Duration::from_secs(600)).await; + let taken = fx + .engine + .claim_restore_dead(&key, STALE_AFTER, None) + .await + .unwrap() + .expect("takeover"); + assert_eq!( + fx.engine + .finish_restore(&key, &*claim, FEW as u64, 1) + .await + .unwrap(), + Finished::NotMatched(format!( + "the job record of fin is not RUNNING under generation {}, or the table is not \ + CREATING with table id {}", + claim.generation(), + key.table_id + )) + ); + + assert_eq!( + fx.engine + .finish_restore(&key, &*taken, FEW as u64, 1) + .await + .unwrap(), + Finished::Pending + ); + let doc = table_doc(&fx.engine, "fin").await.expect("target"); + assert_eq!(doc.get_str("table_status"), Ok("CREATING")); + assert!(doc.get_bool("restore_backfill_pending").unwrap_or(false)); + assert_eq!( + doc.get_document(RESTORE_JOB).unwrap().get_str("status"), + Ok("DONE") + ); + assert_eq!(doc.get_i64("item_count"), Ok(FEW as i64)); + + // Between the DONE write and the worker's flip the backup is in use and + // the DONE record is not dead. + assert_eq!( + fx.engine + .backup_in_use(&backup.key()) + .await + .unwrap() + .as_deref(), + Some("fin") + ); + assert!( + fx.engine + .list_dead_restore_jobs(STALE_AFTER) .await - .len(), - 2, - "no GSI was added" + .unwrap() + .iter() + .all(|j| j.target.table_name != "fin"), + "the DONE+pending record is not dead" ); - let missing = fx + + // The worker flips it ACTIVE with every index populated. + let worker = spawn_worker(&fx.engine); + let active = wait_for_status(&fx.engine, "fin", "ACTIVE").await; + let table_id = active.get_str("table_id").expect("table_id"); + assert_eq!(active.get_i64("item_count"), Ok(FEW as i64)); + assert_eq!( + count(&fx.engine, &data_collection_name(table_id)).await, + FEW as u64 + ); + for (index_name, index_id, status) in index_ids(&fx.engine, table_id).await { + assert_eq!(status, "ACTIVE", "{index_name}"); + assert_eq!( + count(&fx.engine, &data_collection_name(&index_id)).await, + FEW as u64, + "index {index_name}" + ); + } + let job = active.get_document(RESTORE_JOB).expect("job record"); + assert_eq!(job.get_str("status"), Ok("DONE")); + assert_eq!(job.get_str("kind"), Ok("manifest")); + assert!(!active.contains_key("restore_backfill_pending")); + assert_eq!(fx.engine.backup_in_use(&backup.key()).await.unwrap(), None); + worker.abort(); + drop(claim); + drop(taken); + fx.finish().await; +} + +/// The `restore_mongodb_after_insert_many` kill point and the file-index +/// write's fence: a claim taken over while `bulk_load` is paused between its +/// `insert_many` and the record write finds the write matched nothing (the +/// generation moved), so the lost claim's progress never lands. Runs only +/// under `test-hooks` (the pause point is compiled then) and must run alone +/// (`--test-threads=1`): the pause is driven by a process-wide environment +/// variable. +#[cfg(feature = "test-hooks")] +#[tokio::test(flavor = "multi_thread")] +#[ignore = "needs a MongoDB replica set and --features test-hooks, run with --test-threads=1"] +async fn a_takeover_during_bulk_load_fences_the_file_index_write() { + let fx = Fixture::new().await; + fx.worker.abort(); + let backup = available_backup(&fx.engine, "src", "b").await; + let (_, claim) = fx .engine - .update_table( + .claim_restore( ACCOUNT, - UpdateTableInput { - table_name: "nope".to_owned(), - deletion_protection_enabled: Some(true), - ..blank.clone() + manifest_input("fence"), + ®istration(&backup, None), + ) + .await + .expect("register"); + let key = target_key(&table_doc(&fx.engine, "fence").await.expect("target")); + + // Take the record over before the owner's batch runs: its heartbeat is + // aged and a second owner claims it under a fresh generation. + age_running_record(&fx.engine, "fence", Duration::from_secs(600)).await; + let taken = fx + .engine + .claim_restore_dead(&key, STALE_AFTER, None) + .await + .expect("takeover") + .expect("a stale record is taken over"); + + // The old claim's load is fenced inside the batch transaction: the + // `find_one_and_update` on the record (RUNNING under the old generation) + // matches nothing, so neither the documents nor the progress of the + // batch are written. + let out = fx + .engine + .bulk_load( + &key, + &*claim, + BulkLoadFile { + file_index: 0, + start_at: 0, + reapplying: false, }, + item_stream(0, 10), ) .await; assert!( - matches!(missing, Err(StorageError::TableNotFound(_))), - "{missing:?}" + matches!(&out, Err(StorageError::Internal(m)) if m.contains("claim on the target was lost")), + "the fenced-out load must not write the batch: {out:?}" + ); + assert_eq!( + count(&fx.engine, &data_collection_name(&key.table_id)).await, + 0, + "no document lands after the takeover" + ); + // The successor's record never advanced its file_index. + assert_eq!( + fx.engine + .read_restore_job(&key) + .await + .unwrap() + .unwrap() + .file_index, + None, + "the lost claim's batch wrote no progress" ); - gate.add_permits(1); - assert_fully_restored_with(&fx.engine, "dst", FEW).await; - let desc = fx + drop(taken); + fx.finish().await; +} + +/// The registration itself writes the record's first heartbeat on the +/// server's clock (`$$NOW`, in the registration's transaction): a target +/// registered with a first heartbeat ten minutes behind, as a host whose +/// clock is behind the database's would write it, carries the server's time +/// as soon as the registration commits, before any heartbeat loop runs, so +/// another server cannot list the new restore dead in the window before the +/// copy task's first refresh. No restore task runs here: the registration is +/// called directly. +#[tokio::test(flavor = "multi_thread")] +#[ignore = "needs a MongoDB replica set in EXTENDDB_TEST_MONGODB_URI"] +async fn live_a_registration_writes_its_first_heartbeat_on_the_server_clock() { + let fx = Fixture::new().await; + source_table_with(&fx.engine, "src", 10).await; + let backup = fx .engine - .update_table(ACCOUNT, changes[1].clone()) + .create(ACCOUNT, "src", "b", JobPermit::detached()) .await - .expect("UpdateTable once ACTIVE"); - assert!(desc.deletion_protection_enabled); + .expect("backup"); + let request = restore_request(&fx.engine, &backup.backup_arn, "dst").await; + let behind = bson::DateTime::from_millis(bson::DateTime::now().timestamp_millis() - 600_000); + let mut registration = crate::backup_engine::RestoreRegistration { + backup_arn: &request.backup_arn, + source_table_arn: &request.source_table_arn, + restore_date_time: bson::DateTime::now(), + owner_generation: 7, + first_heartbeat: behind, + record: crate::backup_engine::RestoreRecord::Legacy, + session: None, + }; + fx.engine + .create_table_for_restore(ACCOUNT, request.input, &mut registration) + .await + .expect("register"); + let job = job_record(&fx.engine, "dst").await; + let beat = job.get_datetime("heartbeat_at").expect("heartbeat"); + let age_ms = bson::DateTime::now().timestamp_millis() - beat.timestamp_millis(); + assert!( + age_ms.abs() < 60_000, + "the registration's heartbeat is {age_ms} ms old: the server clock replaces the \ + process's inside the registration" + ); + assert_eq!(job.get_str("status"), Ok("RUNNING")); + assert_eq!(job.get_i64("owner_generation"), Ok(7)); + assert!( + fx.engine + .list_dead_restore_jobs(STALE_AFTER) + .await + .expect("list") + .is_empty(), + "a record just registered by a slow-clock host is not dead" + ); + // The target this test registered has no task: remove it so the fixture + // can drop its databases cleanly. + let key = RestoreTargetKey { + account_id: ACCOUNT.to_owned(), + table_name: "dst".to_owned(), + table_id: table_doc(&fx.engine, "dst") + .await + .expect("target") + .get_str("table_id") + .expect("table_id") + .to_owned(), + }; + assert!(fx.engine.remove_restore_target(&key).await.expect("remove")); fx.finish().await; } -/// A stream record's `created_at` field and its own `ApproximateCreationDateTime` -/// come from one clock read on the live write path, for a plain write and for -/// a service-identity deletion (the TTL worker's shape). -#[tokio::test] -#[ignore = "needs EXTENDDB_TEST_MONGODB_URI"] -async fn live_stream_record_created_at_matches_the_record_timestamp() { - use extenddb_core::expression::ExpressionMaps; - use extenddb_core::types::{StreamSpecification, StreamViewType, UserIdentity}; - use extenddb_storage::{DataEngine, StreamCapture}; - +/// `fail_restore_job` is conditioned on the `owner_generation` and +/// `heartbeat_at` the listing read: a heartbeat that lands between the +/// listing and the write keeps the record its owner's (the write matches +/// nothing), as does a generation that is not the record's; with the listed +/// values current the write lands under a fresh generation. +#[tokio::test(flavor = "multi_thread")] +#[ignore = "needs a MongoDB replica set in EXTENDDB_TEST_MONGODB_URI"] +async fn live_the_failed_write_matches_only_the_listed_generation_and_heartbeat() { let fx = Fixture::new().await; + source_table_with(&fx.engine, "src", 10).await; + let backup = fx + .engine + .create(ACCOUNT, "src", "b", JobPermit::detached()) + .await + .expect("backup"); + let request = restore_request(&fx.engine, &backup.backup_arn, "dst").await; fx.engine - .create_table( - ACCOUNT, - CreateTableInput { - table_name: "clock".to_owned(), - key_schema: vec![key("pk", KeyType::Hash)], - attribute_definitions: vec![attr("pk", ScalarAttributeType::S)], - billing_mode: Some(BillingMode::PayPerRequest), - stream_specification: Some(StreamSpecification { - stream_enabled: true, - stream_view_type: Some(StreamViewType::NewAndOldImages), - }), - ..Default::default() - }, - ) + .restore(ACCOUNT, request, JobPermit::detached()) .await - .expect("create the table"); - let key_info = fx + .expect("restore"); + let done = wait_for_status(&fx.engine, "dst", "ACTIVE").await; + let key = RestoreTargetKey { + account_id: ACCOUNT.to_owned(), + table_name: "dst".to_owned(), + table_id: done.get_str("table_id").expect("table_id").to_owned(), + }; + let tables = fx.engine.catalog_db.collection::("tables"); + let set = |update: Document| { + let tables = tables.clone(); + async move { + tables + .update_one( + doc! { "_id": { "account_id": ACCOUNT, "table_name": "dst" } }, + update, + ) + .await + .expect("set the record"); + } + }; + let generation_of = || async { + job_record(&fx.engine, "dst") + .await + .get_i64("owner_generation") + .expect("generation") + }; + + set(doc! { "$set": { "table_status": "CREATING" } }).await; + age_running_record(&fx.engine, "dst", Duration::from_secs(600)).await; + let dead = fx .engine - .table_key_info(ACCOUNT, "clock") + .list_dead_restore_jobs(STALE_AFTER) .await - .expect("key info"); - let maps = ExpressionMaps::default(); - let capture = StreamCapture { - view_type: StreamViewType::NewAndOldImages, - user_identity: None, - region: Arc::from("us-east-1"), - }; - for v in ["one", "two"] { - let item: Item = [ - ("pk".to_owned(), AttributeValue::S("k".to_owned())), - ("v".to_owned(), AttributeValue::S(v.to_owned())), - ] - .into_iter() - .collect(); + .expect("list"); + assert_eq!(dead.len(), 1, "{dead:?}"); + let listed = dead[0].clone(); + let generation = generation_of().await; + assert_eq!(i64::try_from(listed.owner_generation).unwrap(), generation); + + // The owner's heartbeat lands after the listing. + set(doc! { "$set": { RESTORE_JOB_HEARTBEAT: bson::DateTime::now() } }).await; + assert!( + !fx.engine + .fail_restore_job( + &key, + listed.owner_generation, + listed.heartbeat_at, + "owner dead" + ) + .await + .expect("fail"), + "a heartbeat after the listing keeps the record its owner's" + ); + let job = job_record(&fx.engine, "dst").await; + assert_eq!(job.get_str("status"), Ok("RUNNING")); + assert_eq!(job.get_i64("owner_generation"), Ok(generation)); + assert!( fx.engine - .put_item(&key_info, item, false, None, &maps, Some(&capture)) + .list_dead_restore_jobs(STALE_AFTER) .await - .expect("put"); - } - let key: Item = [("pk".to_owned(), AttributeValue::S("k".to_owned()))] - .into_iter() - .collect(); - fx.engine - .delete_item( - &key_info, - &key, - false, - None, - &maps, - Some(&StreamCapture { - view_type: StreamViewType::NewAndOldImages, - user_identity: Some(UserIdentity { - identity_type: "Service".to_owned(), - principal_id: "dynamodb.amazonaws.com".to_owned(), - }), - region: Arc::from("us-east-1"), - }), - ) - .await - .expect("delete"); + .expect("list") + .is_empty() + ); + // Another generation with the current heartbeat: no match either. + let current = *job.get_datetime("heartbeat_at").expect("heartbeat"); + assert!( + !fx.engine + .fail_restore_job( + &key, + listed.owner_generation + 1, + current.timestamp_millis() as f64 / 1000.0, + "owner dead" + ) + .await + .expect("fail") + ); + assert_eq!( + job_record(&fx.engine, "dst").await.get_str("status"), + Ok("RUNNING") + ); - let mut cursor = fx + // Listed values that are still current: the write lands, under a fresh + // generation. + age_running_record(&fx.engine, "dst", Duration::from_secs(600)).await; + let dead = fx .engine - .data_db - .collection::("stream_records") - .find(doc! {}) + .list_dead_restore_jobs(STALE_AFTER) .await - .expect("read back"); - let mut seen = 0; - while cursor.advance().await.expect("cursor") { - let d = cursor.deserialize_current().expect("document"); - let created_at = d.get_datetime("created_at").expect("created_at"); - let approx = d - .get_document("record_data") - .and_then(|r| r.get_document("dynamodb")) - .expect("record_data.dynamodb") - .get_i64("ApproximateCreationDateTime") - .expect("ApproximateCreationDateTime"); + .expect("list"); + assert_eq!(dead.len(), 1); + assert!( + fx.engine + .fail_restore_job( + &dead[0].target, + dead[0].owner_generation, + dead[0].heartbeat_at, + "owner dead" + ) + .await + .expect("fail") + ); + let job = job_record(&fx.engine, "dst").await; + assert_eq!(job.get_str("status"), Ok("FAILED")); + assert_ne!( + job.get_i64("owner_generation"), + Ok(generation), + "the FAILED write carries a fresh generation" + ); + assert!(fx.engine.remove_restore_target(&key).await.expect("remove")); + fx.finish().await; +} +/// A claim document with a fresh `heartbeat_at` and no `owner_generation` +/// (an operator's insert, not a release's): the upsert's filter matches it +/// and overwrites it with this call's generation, so the claim is this +/// call's and is heartbeated and released by it, where refusing it would +/// leave a document nobody heartbeats until it went stale. A document with +/// a generation and a fresh heartbeat (a live holder's) is still refused. +#[tokio::test(flavor = "multi_thread")] +#[ignore = "needs a MongoDB replica set in EXTENDDB_TEST_MONGODB_URI"] +async fn live_a_claim_document_without_a_generation_is_taken_not_refused() { + let fx = Fixture::new().await; + let stale = std::time::Duration::from_secs(300); + let maintenance = fx + .engine + .catalog_db + .collection::(crate::backup_catalog::BACKUP_MAINTENANCE); + maintenance + .insert_one(doc! { "_id": "claim", "holder": "an operator's insert" }) + .await + .expect("insert"); + maintenance + .update_one(doc! {}, vec![doc! { "$set": { "heartbeat_at": "$$NOW" } }]) + .await + .expect("stamp"); + let claim = fx + .engine + .maintenance_claim("purge on host-a", stale) + .await + .expect("a document with no owner_generation is taken"); + let document = maintenance + .find_one(doc! { "_id": "claim" }) + .await + .expect("read") + .expect("one document"); + assert_eq!(document.get_str("holder").unwrap(), "purge on host-a"); + assert!( + document.get("owner_generation").is_some(), + "the document carries the taker's generation: {document:?}" + ); + assert!( + fx.engine + .check_maintenance_claim(stale) + .await + .expect("check") + ); + assert_eq!( + claim.heartbeat().await.expect("heartbeat"), + Heartbeat::Confirmed + ); + match fx.engine.maintenance_claim("purge on host-b", stale).await { + Err(StorageError::AlreadyExists(m)) => assert!(m.contains("purge on host-a"), "{m}"), + Err(other) => panic!("a live holder's document is refused with AlreadyExists: {other:?}"), + Ok(_) => panic!("a live holder's document is refused"), + } + claim.release().await.expect("release"); + assert!( + !fx.engine + .check_maintenance_claim(stale) + .await + .expect("check") + ); + fx.finish().await; +} + +/// `newest_available_backup_row` reads the last AVAILABLE row in the +/// listing's order, and none when the store has no AVAILABLE row; +/// `list_running_restore_jobs` lists a RUNNING record by status alone, one +/// whose heartbeat is ahead of the server's clock included, which the +/// dead-job listing under a zero window does not list. +#[tokio::test(flavor = "multi_thread")] +#[ignore = "needs a MongoDB replica set in EXTENDDB_TEST_MONGODB_URI"] +async fn live_the_newest_available_row_and_the_running_restores_are_read_by_status() { + let fx = Fixture::new().await; + let store = store(); + assert!( + fx.engine + .newest_available_backup_row(&store) + .await + .expect("read") + .is_none(), + "no AVAILABLE row yet" + ); + let older = row(ACCOUNT, "Music", "older"); + let newer = row(ACCOUNT, "Music", "newer"); + for row in [&older, &newer] { + let claim = fx.engine.claim_backup(row).await.expect("claim"); assert_eq!( - created_at.timestamp_millis().div_euclid(1_000), - approx, - "created_at {created_at} and ApproximateCreationDateTime {approx} disagree" + claim + .set_status( + JobStatus::Creating, + JobStatus::Available, + &StatusFields::default() + ) + .await + .expect("publish"), + WriteOutcome::Matched ); - seen += 1; } - assert_eq!(seen, 3, "two puts and one delete"); + let newest = fx + .engine + .newest_available_backup_row(&store) + .await + .expect("read") + .expect("one AVAILABLE row"); + let listed: Vec = fx + .engine + .list_backup_rows(Some(BackupStatus::Available), &store) + .try_collect() + .await + .expect("list"); + assert_eq!( + newest.backup_arn, + listed.last().unwrap().backup_arn, + "the newest row is the last in the listing's order" + ); + + // A RUNNING restore record stamped 60 s ahead of the server's clock. + source_table_with(&fx.engine, "rr_target", 1).await; + let table_id = table_doc(&fx.engine, "rr_target") + .await + .expect("the table") + .get_str("table_id") + .expect("table_id") + .to_owned(); + fx.engine + .catalog_db + .collection::("tables") + .update_one( + doc! { "_id": { "account_id": ACCOUNT, "table_name": "rr_target" } }, + vec![doc! { "$set": { + "table_status": { "$literal": "CREATING" }, + "status_transition_at": { "$literal": bson::Bson::Null }, + "restore_source_backup_arn": { "$literal": &older.backup_arn }, + "restore_source_table_arn": { + "$literal": "arn:aws:dynamodb:us-east-1:123456789012:table/Music" + }, + "restore_date_time": "$$NOW", + RESTORE_JOB: { + "kind": { "$literal": "legacy" }, + "status": { "$literal": "RUNNING" }, + "owner_generation": { "$literal": 7_i64 }, + "heartbeat_at": { "$add": [ "$$NOW", 60_000_i64 ] }, + }, + } }], + ) + .await + .expect("stamp a RUNNING record ahead of the clock"); + let running = fx + .engine + .list_running_restore_jobs() + .await + .expect("list running"); + assert_eq!(running.len(), 1, "{running:?}"); + assert_eq!(running[0].target.table_name, "rr_target"); + assert_eq!(running[0].target.table_id, table_id); + assert!( + fx.engine + .list_dead_restore_jobs(std::time::Duration::ZERO) + .await + .expect("list dead") + .is_empty(), + "the age-based listing misses a heartbeat ahead of the clock" + ); fx.finish().await; } diff --git a/crates/storage-mongodb/src/backup_protocol_model.rs b/crates/storage-mongodb/src/backup_protocol_model.rs index bada2d560..e7d546627 100644 --- a/crates/storage-mongodb/src/backup_protocol_model.rs +++ b/crates/storage-mongodb/src/backup_protocol_model.rs @@ -1,139 +1,94 @@ // Copyright 2026 ExtendDB contributors // SPDX-License-Identifier: Apache-2.0 -//! An exhaustive model of the restore / DeleteBackup claim protocol, with the -//! heartbeat sweep as an adversary. It is a model of the protocol, not a test -//! of the code: each step stands for one MongoDB operation in -//! `register_restore`, `copy_restore`, `delete_backup`, or -//! `sweep_stale_backup_claims`, except `RRelease`, and the model is kept in -//! step with them by hand. `RRelease` stands for up to three operations of -//! `register_restore` run as one atomic step: the claim release (an -//! `update_one`), the lost-claim recheck of the backup document (a `find_one`, -//! only when the release matched nothing), and the abort of the target -//! (`abort_backup_restore`, only when the recheck finds the backup gone or -//! delete-claimed). Interleavings between those three are not explored. +//! An exhaustive model of the legacy restore / DeleteBackup protocol, with +//! the reconciler as an adversary. It is a model of the protocol, not a test +//! of the code: each step stands for one MongoDB operation, or one +//! transaction, in `register_restore`, `copy_restore`, `delete` or the +//! reconciler's removal of a target, and the model is kept in step with them +//! by hand. //! //! Threads, each a fixed sequence of steps; every interleaving of them is //! checked: //! -//! - two restores of the same backup into different targets: claim (the second -//! restore retries once, as `register_restore` retries a claim held by another -//! restore), insert the CREATING target, release the claim (with the -//! lost-claim recheck), read the backup's data (`$out`, as one read), hand the -//! target over, and the backfill worker flipping it ACTIVE. The retry also -//! happens when the first claim found a delete claim, where the code ends the -//! restore with BackupInUse at once; that allows more interleavings than the -//! code has; -//! - DeleteBackup: target check, claim, recheck, drop the data, mark DELETED; +//! - two restores of the same backup into different targets: register (one +//! transaction: the backup must be AVAILABLE and no CREATING target of it +//! may exist, else the restore is refused), read the backup's data (`$out`, +//! as one read), hand the target over (the job record DONE), and the +//! backfill worker flipping it ACTIVE; +//! - DeleteBackup: the unlocked target check, the decision (one transaction: +//! AVAILABLE and no CREATING target, then DELETING), the drop of the data, +//! DELETED; //! - DeleteBackup stopping at any point (its request future dropped, or an -//! error returned): its claim is released only if it is still armed, and it -//! is disarmed before the drop; -//! - sweep events that act on a live owner whose heartbeat lapsed (a stalled -//! process): clearing the restore claim (twice), removing each CREATING -//! target that was not handed over; -//! - the sweep's delete takeover (take over a stale delete claim, check for a -//! restore target, drop, mark DELETED), and that takeover stopping at any -//! point. The sweep never releases a delete claim it took over. The code -//! also checks for a CREATING target before taking over and leaves the claim -//! alone when it finds one; the model has no such check and always takes -//! over, which allows more interleavings than the code has, so the model -//! does not exercise that check. +//! error returned, including an ambiguous error from the drop itself): +//! nothing is reverted; +//! - a retry of DeleteBackup, which resumes a DELETING backup: drops the data +//! again (idempotent) and writes DELETED; +//! - the reconciler removing each CREATING target that was not handed over, +//! as it does once the owner's heartbeat is stale; the step may fire at any +//! point, whether or not the owner is stalled, which over-approximates what +//! the reconciler can do. //! -//! Sweep events may fire at any point, whether or not the owner is stalled, -//! which over-approximates what the sweep can do. +//! Not modelled: `$out` as a non-atomic read, more than one restore of each +//! target, the timing of heartbeats (staleness is assumed possible at every +//! step), a failed write, and a registration's transaction retry (a write +//! conflict re-runs the registration, which the model's atomic step stands +//! for). //! -//! Not modelled: `$out` as a non-atomic read, more than one DeleteBackup, more -//! than one delete takeover, the timing of heartbeats (staleness is assumed -//! possible at every step), and a release that fails. In the model every -//! release succeeds. In the code: -//! -//! - DeleteBackup's `release()` after its recheck found a target can return an -//! error; the request then fails with the claim still armed, its `Drop` -//! spawns a second release, and if that fails too the claim stays until the -//! sweep finishes the delete, which it does only once no CREATING target of -//! the backup exists (so a DeleteBackup that returned an error can still -//! take effect later); -//! - a restore's release in `hand_claim_to_target` can return an error, and so -//! can the lost-claim recheck; the restore then aborts its target (if the -//! abort fails, the sweep removes the target once its heartbeat is stale) -//! and its still-armed claim is released by `Drop` or, if that fails, cleared -//! by the sweep; -//! - the releases on a restore's earlier failure paths (no `backup_id`, target -//! creation failed before the hand-over) log an error and leave the same -//! `Drop` / sweep backstop. -//! -//! Invariants, checked after every step: no restore claim and delete claim are -//! held together; no restore is handed over after reading dropped data; the -//! backup is never AVAILABLE with no claim after its data was dropped (a later -//! restore would copy nothing). Liveness: some interleaving completes a restore -//! and the delete, some completes both restores, and without stops and sweep -//! events no interleaving leaves a claim behind. +//! Invariants, checked after every step: no restore is handed over after +//! reading dropped data; the backup is never AVAILABLE after its data was +//! dropped (a later restore would copy nothing); a DELETING backup never +//! reads AVAILABLE again. Liveness: some interleaving completes a restore and +//! the delete, some completes both restores, and without a stop no +//! interleaving leaves the backup DELETING. //! //! The checker explores reachable states once each (depth first, memoised on //! the state and every thread's position) and counts the interleavings below //! each state, so the count is the number of interleavings covered without -//! enumerating them. A brute-force enumerator cross-checks it on smaller -//! configurations, including the 39,060 interleavings of the previous model. -//! -//! The four tests that explore the full model take about 25 s each in a debug -//! build and are `#[ignore]`d, so `cargo test` stays quick; run them with -//! `cargo test -p extenddb-storage-mongodb --lib protocol_model -- --include-ignored` -//! after any change to the claim protocol or to this model. The three quick -//! tests (the 20-ordering model, the brute-force cross-check, and the -//! no-stops liveness check) run by default. +//! enumerating them. A brute-force enumerator cross-checks it on a smaller +//! configuration. use std::collections::{HashMap, HashSet}; #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] enum Step { - // Restore `i` into target `i`. `RClaim(i, last)`: a failed claim ends the - // restore only when `last`. - RClaim(usize, bool), - RInsertTarget(usize), - RRelease(usize), + // Restore `i` into target `i`. + RRegister(usize), RCopyRead(usize), RHandOver(usize), RActivate(usize), // DeleteBackup DCheckTarget, - DClaim, - DRecheckTarget, + DDecide, DDropData, DMarkDeleted, - /// The DeleteBackup request stops: future dropped, or an error returned - /// (including an ambiguous error from the drop itself). + /// The DeleteBackup request stops: future dropped, or an error returned. DStop, - // Sweep acting on a stalled live owner - SClearRestoreClaim, + /// A retry of DeleteBackup finding the backup DELETING. + DResume, + /// The reconciler removes restore `i`'s target, judging its owner dead. SRemoveTarget(usize), - // Sweep step 3 - STakeOverDelete, - SCheckTarget, - SDropData, - SMarkDeleted, - /// The sweep's step 3 returns early (an error from any of its operations). - SStop, } #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] enum Target { None, Creating, - /// CREATING with `restore_backfill_pending`: still matched by - /// `restoring_target_filter`, no longer by the sweep. + /// CREATING with `restore_backfill_pending` and the job record DONE: + /// still in use, no longer the reconciler's. HandedOver, Active, } #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] -enum DeleteOwner { - Request, - Sweep, +enum Status { + Available, + Deleting, + Deleted, } #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] struct Restore { - claimed: bool, target: Target, copy_saw_data: Option, done: bool, @@ -141,39 +96,27 @@ struct Restore { #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] struct World { - available: bool, + status: Status, data_present: bool, - restore_claim: Option, - delete_claim: Option, restores: [Restore; 2], d_done: bool, - /// DeleteBackup's `BackupMetadataClaim` would release on drop/error. - d_armed: bool, - s_active: bool, - s_done: bool, - /// The sweep's taken-over claim would be released on an early return. - s_armed: bool, + /// Whether the DeleteBackup request has written DELETING. + d_decided: bool, } impl World { fn initial() -> Self { let restore = Restore { - claimed: false, target: Target::None, copy_saw_data: None, done: false, }; Self { - available: true, + status: Status::Available, data_present: true, - restore_claim: None, - delete_claim: None, restores: [restore; 2], d_done: false, - d_armed: false, - s_active: false, - s_done: false, - s_armed: false, + d_decided: false, } } @@ -185,49 +128,28 @@ impl World { } } -/// Earlier revisions of the code, to show the model notices each of them. +/// Earlier or weaker shapes of the protocol, to show the model notices each. #[derive(Clone, Copy, Debug, Default)] struct Variant { - /// B1 before the fix: DeleteBackup's claim (and the sweep's taken-over - /// claim) stays armed across the drop, so a stop after the drop releases it. - release_after_drop: bool, - /// The sweep before this fix: it released a taken-over delete claim when it - /// found a restore target, and its armed claim released on an early return. - sweep_releases: bool, - /// `register_restore` without its recheck after a lost claim. - no_lost_claim_recheck: bool, + /// The delete's decision without the in-use check under the lock: only + /// the unlocked first check guards a registered target. + decide_without_target_check: bool, + /// A registration that does not require the backup to be AVAILABLE. + register_without_status_check: bool, + /// A DeleteBackup that stops after its decision puts the backup back to + /// AVAILABLE (the shape of a claim released on error). + revert_on_stop: bool, } fn apply(w: &mut World, step: Step, v: Variant) -> Result<(), String> { match step { - Step::RClaim(i, last) => { - let r = &mut w.restores[i]; - if !r.done && !r.claimed { - if w.available && w.restore_claim.is_none() && w.delete_claim.is_none() { - w.restore_claim = Some(i); - r.claimed = true; - } else if last { - r.done = true; - } - } - } - Step::RInsertTarget(i) => { - if !w.restores[i].done { - w.restores[i].target = Target::Creating; - } - } - Step::RRelease(i) => { + Step::RRegister(i) => { if !w.restores[i].done { - if w.restore_claim == Some(i) { - w.restore_claim = None; - } else if !v.no_lost_claim_recheck && !(w.available && w.delete_claim.is_none()) { - // Claim lost and the backup is gone or being deleted: - // abort the target. - let r = &mut w.restores[i]; - if r.target == Target::Creating { - r.target = Target::None; - } - r.done = true; + let available = w.status == Status::Available || v.register_without_status_check; + if available && !w.restoring_target() { + w.restores[i].target = Target::Creating; + } else { + w.restores[i].done = true; } } } @@ -260,116 +182,62 @@ fn apply(w: &mut World, step: Step, v: Variant) -> Result<(), String> { w.d_done = true; } } - Step::DClaim => { + Step::DDecide => { if !w.d_done { - if w.available && w.restore_claim.is_none() && w.delete_claim.is_none() { - w.delete_claim = Some(DeleteOwner::Request); - w.d_armed = true; + let in_use = !v.decide_without_target_check && w.restoring_target(); + if w.status == Status::Available && !in_use { + w.status = Status::Deleting; + w.d_decided = true; } else { + // In use: the request ends here. Not AVAILABLE (a + // concurrent delete decided first): the code resumes that + // delete's drop, which the model gives its own step, + // `DResume`; this request's own steps end here. w.d_done = true; } } } - Step::DRecheckTarget => { - if !w.d_done && w.restoring_target() { - // release() matches the operation id. - if w.delete_claim == Some(DeleteOwner::Request) { - w.delete_claim = None; - } - w.d_armed = false; - w.d_done = true; - } - } Step::DDropData => { if !w.d_done { - if !v.release_after_drop { - w.d_armed = false; - } w.data_present = false; } } Step::DMarkDeleted => { if !w.d_done { - if w.delete_claim == Some(DeleteOwner::Request) { - w.available = false; - w.delete_claim = None; + if w.status == Status::Deleting { + w.status = Status::Deleted; } - w.d_armed = false; w.d_done = true; } } Step::DStop => { if !w.d_done { w.d_done = true; - if w.d_armed && w.delete_claim == Some(DeleteOwner::Request) { - w.delete_claim = None; - } - w.d_armed = false; - } - } - Step::SClearRestoreClaim => w.restore_claim = None, - Step::SRemoveTarget(i) => { - if w.restores[i].target == Target::Creating { - w.restores[i].target = Target::None; - } - } - Step::STakeOverDelete => { - if !w.s_done { - if w.delete_claim == Some(DeleteOwner::Request) { - w.delete_claim = Some(DeleteOwner::Sweep); - w.s_active = true; - w.s_armed = v.sweep_releases; - } else { - w.s_done = true; + if v.revert_on_stop && w.d_decided && w.status == Status::Deleting { + w.status = Status::Available; } } } - Step::SCheckTarget => { - if w.s_active && !w.s_done && w.restoring_target() { - if v.sweep_releases && w.delete_claim == Some(DeleteOwner::Sweep) { - w.delete_claim = None; - } - // Otherwise the claim is left for a later sweep. - w.s_armed = false; - w.s_done = true; - } - } - Step::SDropData => { - if w.s_active && !w.s_done { - if !v.release_after_drop { - w.s_armed = false; - } + Step::DResume => { + if w.status == Status::Deleting { w.data_present = false; + w.status = Status::Deleted; } } - Step::SMarkDeleted => { - if w.s_active && !w.s_done { - if w.delete_claim == Some(DeleteOwner::Sweep) { - w.available = false; - w.delete_claim = None; - } - w.s_armed = false; - w.s_done = true; - } - } - Step::SStop => { - if w.s_active && !w.s_done { - w.s_done = true; - if w.s_armed && w.delete_claim == Some(DeleteOwner::Sweep) { - w.delete_claim = None; - } - w.s_armed = false; + Step::SRemoveTarget(i) => { + if w.restores[i].target == Target::Creating { + w.restores[i].target = Target::None; } } } - if w.restore_claim.is_some() && w.delete_claim.is_some() { - return Err(format!("both claims held: {w:?}")); - } - if w.available && !w.data_present && w.delete_claim.is_none() { + if w.status == Status::Available && !w.data_present { return Err(format!( - "backup AVAILABLE and unclaimed after its data was dropped: {w:?}" + "backup AVAILABLE after its data was dropped: {w:?}" )); } + if w.d_decided && w.status == Status::Available { + return Err(format!("a DELETING backup read AVAILABLE again: {w:?}")); + } Ok(()) } @@ -485,67 +353,36 @@ fn multinomial(lengths: &[usize]) -> u128 { result } -/// The previous model's restore: no retry, no ACTIVE flip. -const RESTORE_0: &[Step] = &[ - Step::RClaim(0, true), - Step::RInsertTarget(0), - Step::RRelease(0), - Step::RCopyRead(0), - Step::RHandOver(0), -]; const RESTORE_A: &[Step] = &[ - Step::RClaim(0, true), - Step::RInsertTarget(0), - Step::RRelease(0), + Step::RRegister(0), Step::RCopyRead(0), Step::RHandOver(0), Step::RActivate(0), ]; const RESTORE_B: &[Step] = &[ - Step::RClaim(1, false), - Step::RClaim(1, true), - Step::RInsertTarget(1), - Step::RRelease(1), + Step::RRegister(1), Step::RCopyRead(1), Step::RHandOver(1), Step::RActivate(1), ]; const DELETE: &[Step] = &[ Step::DCheckTarget, - Step::DClaim, - Step::DRecheckTarget, + Step::DDecide, Step::DDropData, Step::DMarkDeleted, ]; -const SWEEP_DELETE: &[Step] = &[ - Step::STakeOverDelete, - Step::SCheckTarget, - Step::SDropData, - Step::SMarkDeleted, -]; -/// The previous model's sweep event sets. -const OLD_SWEEPS: &[&[Step]] = &[ - &[], - &[Step::SClearRestoreClaim], - &[Step::SRemoveTarget(0)], - &[Step::SClearRestoreClaim, Step::SRemoveTarget(0)], - &[Step::SRemoveTarget(0), Step::SClearRestoreClaim], -]; -/// Every thread: two restores, DeleteBackup and its stop, four sweep events, -/// the delete takeover and its stop. +/// Every thread: two restores, DeleteBackup, its stop, a retry, and the +/// reconciler's removal of each target. fn full_model() -> Vec<&'static [Step]> { vec![ RESTORE_A, RESTORE_B, DELETE, &[Step::DStop], - &[Step::SClearRestoreClaim], - &[Step::SClearRestoreClaim], + &[Step::DResume], &[Step::SRemoveTarget(0)], &[Step::SRemoveTarget(1)], - SWEEP_DELETE, - &[Step::SStop], ] } @@ -553,75 +390,65 @@ fn lengths(threads: &[&[Step]]) -> Vec { threads.iter().map(|t| t.len()).collect() } +/// A restore's registration and DeleteBackup's decision, as the two +/// transactions order: every interleaving of one restore and one delete, +/// without a stop or the reconciler, ends with the delete DELETED and the +/// restore refused, the restore ACTIVE and the delete refused, or the delete +/// DELETED after the restore reached ACTIVE; never a delete that proceeded +/// while the target was CREATING. #[test] -fn the_original_20_orderings_hold_without_a_sweep() { - // restore {claim, insert, release} x delete {claim, recheck, drop}. - let orders = interleavings(&[ - &[ - Step::RClaim(0, true), - Step::RInsertTarget(0), - Step::RRelease(0), - ], - &[Step::DClaim, Step::DRecheckTarget, Step::DDropData], - ]); - assert_eq!(orders.len(), 20); - for order in orders { - run(&order, Variant::default()).unwrap_or_else(|e| panic!("{order:?}: {e}")); +fn a_registration_and_a_delete_order_wholly_one_before_the_other() { + let threads: [&[Step]; 2] = [RESTORE_A, DELETE]; + let orders = interleavings(&threads); + assert_eq!(orders.len() as u128, multinomial(&[4, 4])); + let mut delete_won = 0; + let mut restore_won = 0; + let mut both = 0; + for order in &orders { + let w = run(order, Variant::default()).unwrap_or_else(|e| panic!("{order:?}: {e}")); + match (w.status, w.restores[0].target) { + (Status::Deleted, Target::None) => delete_won += 1, + (Status::Available, Target::Active) => restore_won += 1, + (Status::Deleted, Target::Active) => both += 1, + other => panic!("{order:?} ended in {other:?}"), + } } + assert!( + delete_won > 0 && restore_won > 0 && both > 0, + "{delete_won} / {restore_won} / {both}" + ); } #[test] fn the_explorer_agrees_with_brute_force_enumeration() { - // The previous model's 39,060 interleavings, checked both ways, for the - // fixed protocol and for a variant that the model must reject. for v in [ Variant::default(), Variant { - no_lost_claim_recheck: true, + decide_without_target_check: true, ..Variant::default() }, Variant { - release_after_drop: true, + revert_on_stop: true, ..Variant::default() }, ] { - let mut total = 0_u128; - for sweep in OLD_SWEEPS { - let threads = [RESTORE_0, DELETE, sweep]; - let explored = explore(&threads, v); - let orders = interleavings(&threads); - assert_eq!(orders.len() as u128, multinomial(&lengths(&threads))); - let brute_bad = orders.iter().filter(|o| run(o, v).is_err()).count(); - // The explorer stops a schedule at its first violation, so it - // counts exactly the schedules that never violate an invariant. - assert_eq!( - explored.interleavings, - (orders.len() - brute_bad) as u128, - "{v:?} {sweep:?}" - ); - assert_eq!( - explored.violations.is_empty(), - brute_bad == 0, - "{v:?} {sweep:?}" - ); - total += orders.len() as u128; - } - assert_eq!(total, 252 + 2 * 2772 + 2 * 16632); + let threads: [&[Step]; 4] = [RESTORE_A, DELETE, &[Step::DStop], &[Step::SRemoveTarget(0)]]; + let explored = explore(&threads, v); + let orders = interleavings(&threads); + assert_eq!(orders.len() as u128, multinomial(&lengths(&threads))); + let brute_bad = orders.iter().filter(|o| run(o, v).is_err()).count(); + // The explorer stops a schedule at its first violation, so it counts + // exactly the schedules that never violate an invariant. + assert_eq!( + explored.interleavings, + (orders.len() - brute_bad) as u128, + "{v:?}" + ); + assert_eq!(explored.violations.is_empty(), brute_bad == 0, "{v:?}"); } - // And with the DeleteBackup stop, where the B1 variant must fail both ways. - let threads: [&[Step]; 3] = [RESTORE_0, DELETE, &[Step::DStop]]; - let orders = interleavings(&threads); - let v = Variant { - release_after_drop: true, - ..Variant::default() - }; - assert!(orders.iter().any(|o| run(o, v).is_err())); - assert!(!explore(&threads, v).violations.is_empty()); - assert!(orders.iter().all(|o| run(o, Variant::default()).is_ok())); } #[test] -#[ignore = "explores the full model (about 25 s in a debug build); run with --include-ignored"] fn no_interleaving_of_the_full_model_breaks_the_protocol() { let threads = full_model(); let explored = explore(&threads, Variant::default()); @@ -631,14 +458,14 @@ fn no_interleaving_of_the_full_model_breaks_the_protocol() { explored.violations.len(), explored.violations[0] ); - // Every interleaving was covered: 28 steps in 10 threads. + // Every interleaving was covered: 16 steps in 7 threads. assert_eq!(explored.interleavings, multinomial(&lengths(&threads))); - assert_eq!(explored.interleavings, 29_173_282_778_255_616_000); + assert_eq!(explored.interleavings, 1_513_512_000); // Liveness: the protocol does make progress. let t = &explored.terminals; assert!( - t.iter() - .any(|w| !w.available && w.restores.iter().any(|r| r.target == Target::Active)), + t.iter().any(|w| w.status == Status::Deleted + && w.restores.iter().any(|r| r.target == Target::Active)), "no interleaving completes both a restore and the delete" ); assert!( @@ -649,86 +476,81 @@ fn no_interleaving_of_the_full_model_breaks_the_protocol() { assert!( t.iter().any(|w| w.restores[1].target == Target::Active && w.restores[0].target == Target::Active - && !w.available), + && w.status == Status::Deleted), "no interleaving completes both restores and then the delete" ); + // A DeleteBackup that stopped after writing DELETING leaves the backup + // DELETING only until a retry resumes it; with the retry in the model, a + // terminal state is DELETING only when the stop came after the retry. + assert!(t.iter().any(|w| w.status == Status::Deleting)); } #[test] -fn without_stops_or_sweeps_no_claim_is_left_behind() { - let threads: [&[Step]; 3] = [RESTORE_A, RESTORE_B, DELETE]; +fn without_a_stop_the_backup_is_never_left_deleting() { + let threads: [&[Step]; 5] = [ + RESTORE_A, + RESTORE_B, + DELETE, + &[Step::SRemoveTarget(0)], + &[Step::SRemoveTarget(1)], + ]; let explored = explore(&threads, Variant::default()); assert!(explored.violations.is_empty(), "{:?}", explored.violations); - assert_eq!(explored.interleavings, multinomial(&[6, 7, 5])); + assert_eq!(explored.interleavings, multinomial(&[4, 4, 4, 1, 1])); for w in &explored.terminals { - assert_eq!(w.restore_claim, None, "{w:?}"); - assert_eq!(w.delete_claim, None, "{w:?}"); - // Without a stall, a restore that claimed always completes. - for r in &w.restores { - assert!(!r.claimed || r.target == Target::Active, "{w:?}"); - } + assert_ne!(w.status, Status::Deleting, "{w:?}"); + // A delete that decided finished; one that did not left the backup + // AVAILABLE with its data. + assert_eq!(w.data_present, w.status == Status::Available, "{w:?}"); } - // The retry lets the second restore claim after the first released in - // some interleaving where its first attempt failed; without the retry - // step that schedule ends the restore. - assert!( - explored - .terminals - .iter() - .any(|w| w.restores.iter().all(|r| r.target == Target::Active)) - ); } #[test] -#[ignore = "explores the full model (about 25 s in a debug build); run with --include-ignored"] -fn a_delete_claim_released_after_the_drop_is_found() { - // The claim released by a stop after the drop. +fn a_delete_deciding_without_the_in_use_check_under_the_lock_is_found() { let explored = explore( &full_model(), Variant { - release_after_drop: true, + decide_without_target_check: true, ..Variant::default() }, ); assert!(!explored.violations.is_empty()); assert!( - explored.violations.iter().any(|e| e - .contains("AVAILABLE and unclaimed after its data was dropped") - && e.contains("DDropData, DStop")), + explored + .violations + .iter() + .any(|e| e.contains("dropped backup data")), "{}", explored.violations[0] ); } #[test] -#[ignore = "explores the full model (about 25 s in a debug build); run with --include-ignored"] -fn a_sweep_that_releases_a_taken_over_delete_claim_is_found() { - // The sweep before this fix: the stalled DeleteBackup it took over from - // can still drop the data after the sweep released the claim. +fn a_registration_without_the_status_check_is_found() { let explored = explore( &full_model(), Variant { - sweep_releases: true, + register_without_status_check: true, ..Variant::default() }, ); assert!(!explored.violations.is_empty()); assert!( - explored.violations.iter().all(|e| e - .contains("AVAILABLE and unclaimed after its data was dropped") - || e.contains("dropped backup data")), + explored + .violations + .iter() + .any(|e| e.contains("dropped backup data")), "{}", explored.violations[0] ); } #[test] -#[ignore = "explores the full model (about 25 s in a debug build); run with --include-ignored"] -fn the_lost_claim_recheck_is_what_keeps_a_stalled_restore_safe() { +fn a_delete_that_reverts_on_a_stop_is_found() { let explored = explore( &full_model(), Variant { - no_lost_claim_recheck: true, + revert_on_stop: true, ..Variant::default() }, ); @@ -737,21 +559,9 @@ fn the_lost_claim_recheck_is_what_keeps_a_stalled_restore_safe() { explored .violations .iter() - .any(|e| e.contains("dropped backup data")), + .any(|e| e.contains("AVAILABLE after its data was dropped") + && e.contains("DDropData, DStop")), "{}", explored.violations[0] ); - // Without a sweep (no lapse) the recheck never matters. - let threads: [&[Step]; 3] = [RESTORE_A, RESTORE_B, DELETE]; - assert!( - explore( - &threads, - Variant { - no_lost_claim_recheck: true, - ..Variant::default() - } - ) - .violations - .is_empty() - ); } diff --git a/crates/storage-mongodb/src/data_engine.rs b/crates/storage-mongodb/src/data_engine.rs index 249e5e33e..be8d9de03 100644 --- a/crates/storage-mongodb/src/data_engine.rs +++ b/crates/storage-mongodb/src/data_engine.rs @@ -379,7 +379,7 @@ impl MongoEngine { transaction_options(self.transaction_read_concern(), TransactionKind::ReadOnly) } - fn write_transaction_options(&self) -> mongodb::options::TransactionOptions { + pub(crate) fn write_transaction_options(&self) -> mongodb::options::TransactionOptions { transaction_options(self.transaction_read_concern(), TransactionKind::Write) } @@ -3524,7 +3524,7 @@ impl MongoEngine { /// `gate_table` names the table for the unknown-commit test gate and /// is only passed by UpdateItem, the operation the gate is specified /// for. It is ignored in builds without the `test-hooks` feature. - async fn commit_write_transaction( + pub(crate) async fn commit_write_transaction( &self, session: &mut mongodb::ClientSession, gate_table: Option<&str>, @@ -3637,7 +3637,7 @@ pub(crate) struct GsiBackfillProgress { /// sustained hot-key contention; large enough to absorb ordinary /// snapshot-isolation aborts. Matches the OCC retry ceiling elsewhere /// in this file. -const TRANSIENT_RETRY_ATTEMPTS: u32 = 50; +pub(crate) const TRANSIENT_RETRY_ATTEMPTS: u32 = 50; /// Return type of `try_build_native_update`. Distinguishes an /// operator-document update (`{$set, $unset, $inc}`) from an @@ -3662,7 +3662,7 @@ enum NativeUpdate { /// Error signal used inside per-attempt transaction bodies. Lets the /// body use `?` for control flow while distinguishing "retry this /// whole transaction" from "return this error to the caller." -enum TxErr { +pub(crate) enum TxErr { Transient, Fatal(StorageError), } @@ -3697,7 +3697,7 @@ impl From for TxErr { /// UnknownTransactionCommitResult label, under which the commit may /// already have applied. Commit errors are classified by /// `classify_commit_error` instead. -fn is_transient_write_conflict(e: &mongodb::error::Error) -> bool { +pub(crate) fn is_transient_write_conflict(e: &mongodb::error::Error) -> bool { if e.contains_label(mongodb::error::TRANSIENT_TRANSACTION_ERROR) { return true; } @@ -3743,7 +3743,7 @@ fn classify_commit_error(e: &mongodb::error::Error) -> CommitErrorClass { } /// Result of `commit_write_transaction` for the caller's retry loop. -enum CommitOutcome { +pub(crate) enum CommitOutcome { /// The transaction committed. Return the attempt's success value. Committed, /// The transaction aborted without committing. The caller may re-run @@ -3755,7 +3755,7 @@ enum CommitOutcome { /// conditional-insert sites — a duplicate is the manifestation of a /// conditional-put race, so it must be surfaced as /// `ConditionalCheckFailedException` rather than a 500. -fn is_duplicate_key(e: &mongodb::error::Error) -> bool { +pub(crate) fn is_duplicate_key(e: &mongodb::error::Error) -> bool { matches!( *e.kind, mongodb::error::ErrorKind::Write(mongodb::error::WriteFailure::WriteError( @@ -3787,7 +3787,7 @@ fn synthetic_error_with_labels(message: &str, labels: &[&str]) -> mongodb::error /// Exponential-backoff sleep with random jitter. Used inside the OCC /// / WriteConflict retry loops so competing writers don't lock-step /// re-retry into the same conflict window. -async fn backoff_sleep(attempt: u32) { +pub(crate) async fn backoff_sleep(attempt: u32) { let base_us = 50u64.saturating_mul(1u64 << attempt.min(8)); let jitter = rand::random_range(0..=base_us); tokio::time::sleep(std::time::Duration::from_micros(jitter)).await; @@ -4331,9 +4331,6 @@ mod tests { gsi_cache: Default::default(), tx_read_concern: mongodb::options::ReadConcern::snapshot(), backup_liveness: Default::default(), - restore_server_slot_count: 1, - restore_server_slots: std::sync::Arc::new(tokio::sync::Semaphore::new(1)), - restore_account_slots: Default::default(), }; let base_coll = engine.data_db.collection::("base"); let key_schema = vec![KeySchemaElement { diff --git a/crates/storage-mongodb/src/data_engine_key_repair.rs b/crates/storage-mongodb/src/data_engine_key_repair.rs index a3cb710cc..2d2dac083 100644 --- a/crates/storage-mongodb/src/data_engine_key_repair.rs +++ b/crates/storage-mongodb/src/data_engine_key_repair.rs @@ -28,11 +28,14 @@ //! write conflict and runs again. //! //! [`spawn_key_repair`] runs the whole pass as a background task, so `serve` -//! opens its listener without waiting for it (see `server_components_factory` -//! in `lib.rs`). Until the repair reaches a given document, requests on that -//! document's key answer as they did before this repair existed. Progress is -//! logged every [`PROGRESS_EVERY`] documents and once at the end. A failure -//! on one document is logged and skipped; the summary reports it. +//! opens its listener without waiting for it (see `build_components` in +//! `lib.rs`, which spawns it only once `migrations::register_and_check` has +//! accepted the catalog, and only for a server, not for a command that opens +//! the catalog beside one). Until the repair reaches a given document, +//! requests on that document's key answer as they did before this repair +//! existed. Progress is logged every [`PROGRESS_EVERY`] documents and once +//! at the end. A failure on one document is logged and skipped; the summary +//! reports it. //! //! A scan that stops on an error, as it does when a table is dropped while the //! pass runs, is logged, and the pass goes on to the next table. The summary diff --git a/crates/storage-mongodb/src/data_engine_live_tests_repair_pass.rs b/crates/storage-mongodb/src/data_engine_live_tests_repair_pass.rs index 3d9cd5d8d..85e91421d 100644 --- a/crates/storage-mongodb/src/data_engine_live_tests_repair_pass.rs +++ b/crates/storage-mongodb/src/data_engine_live_tests_repair_pass.rs @@ -11,6 +11,7 @@ use std::sync::Arc; use bson::{Bson, Document, doc}; use extenddb_core::types::TableKeyInfo; +use extenddb_storage::server_components::ServerComponentsOptions; use futures::TryStreamExt; use super::{ @@ -198,3 +199,182 @@ async fn live_the_spawned_key_repair_rewrites_documents_stored_without_their_key .expect("the finished task dropped its engine"); finish(engine).await; } + +/// A server that refuses its catalog has written nothing. `serve`'s factory +/// registers and reads the catalog's status (`register_and_check`) before it +/// builds the engine and starts the repair, so a refusal there leaves the +/// data alone. The factory opens the default databases, so this test calls +/// the same `register_and_check` on scratch databases and pins the refusal +/// that order relies on. With the catalog migrated through migration 5 only +/// and a data document stored without its key, `register_and_check` refuses +/// the catalog as behind, naming migration 6, leaves no registration, and +/// the document keeps the shape the pre-#436 upsert stored. Once the catalog +/// is migrated the same call accepts it, and a repair started after it, as +/// the factory starts one, rewrites the document well within the first +/// half's wait. +#[tokio::test] +#[ignore = "needs a MongoDB replica set in EXTENDDB_TEST_MONGODB_URI"] +async fn live_a_refused_catalog_starts_no_key_repair() { + use crate::migrations::{ + MongoTarget, SERVERS, Timing, migrate_through_for_tests, register_and_check, + }; + use extenddb_storage::hooks::ServingRegistrar; + use extenddb_storage::migrations::{CatalogStatus, MigrationError, MigrationId}; + use std::time::{Duration, Instant}; + + let engine = scratch_engine().await; + let target = MongoTarget::with_databases( + super::uri(), + engine.catalog_db.name(), + engine.data_db.name(), + ); + migrate_through_for_tests(&target, MigrationId::DropContinuousBackups) + .await + .expect("migrations 1 to 5"); + let key_info = table(&engine, "pass-refused", false); + let key = item_key(false); + engine + .catalog_db + .collection::("tables") + .insert_one(catalog_row("refused", &key_info.table_id, &key_info)) + .await + .expect("catalog row"); + store_without_key(&engine, &key_info, &key, Some("upd")).await; + let collection = engine + .data_db + .collection::(&data_collection_name(&key_info.table_id)); + let unrepaired = |document: &Document| document.get_object_id("_id").is_ok(); + + match register_and_check(&target, "test", Timing::PRODUCTION).await { + Ok(_) => panic!("a catalog without migration 6 is refused"), + Err(MigrationError::Status(CatalogStatus::Behind { version, pending })) => { + assert_eq!(version, Some(MigrationId::DropContinuousBackups.number())); + assert!( + pending + .iter() + .any(|p| p.id == MigrationId::BackupCatalogIndex.number()), + "{pending:?}" + ); + } + Err(e) => panic!("expected a Behind refusal, got {e}"), + } + assert_eq!( + engine + .catalog_db + .collection::(SERVERS) + .count_documents(doc! {}) + .await + .expect("count"), + 0, + "the refused server deleted its registration" + ); + // A repair of one document finishes in milliseconds (the second half + // measures it); two seconds is far past any such run. + tokio::time::sleep(Duration::from_secs(2)).await; + assert!( + unrepaired(&stored(&engine, &key_info, &key).await), + "the refused open rewrote a data document" + ); + assert_eq!( + collection.count_documents(doc! {}).await.expect("count"), + 1, + "the refused open wrote a data document" + ); + + migrate_through_for_tests(&target, MigrationId::LATEST) + .await + .expect("migration 6"); + let registration = register_and_check(&target, "test", Timing::PRODUCTION) + .await + .expect("a migrated catalog is served"); + let engine = Arc::new(engine); + let started = Instant::now(); + let repair = spawn_key_repair(Arc::clone(&engine)); + let deadline = started + Duration::from_secs(30); + while unrepaired(&stored(&engine, &key_info, &key).await) { + assert!( + Instant::now() < deadline, + "the repair started after the accepted check did not rewrite the document" + ); + tokio::time::sleep(Duration::from_millis(20)).await; + } + eprintln!( + "the repair rewrote the document {:?} after the check", + started.elapsed() + ); + let want = with(key.clone(), "w", s("upd")); + assert_eq!( + stored(&engine, &key_info, &key).await, + expected_document(&want, &key_info, 1), + "the started repair rebuilt the document" + ); + let _ = repair.await; + ServingRegistrar::release(®istration).await; + let engine = Arc::try_unwrap(engine) + .ok() + .expect("the finished task dropped its engine"); + finish(engine).await; +} + +/// A command that opens the catalog beside a server (`backup list`, the +/// purge's dry run: `beside_server`) starts no key repair. The factory's +/// `build_components` decides through `start_key_repair`, which this test +/// calls with the engine the factory would hand it (the factory opens the +/// default databases). With a data document stored without its key, the +/// call for a command starts no task and the document is still in the shape +/// the pre-#436 upsert stored two seconds later, far past a one document +/// repair (the refusal test measures one in milliseconds). The same call for +/// a server then starts the repair, which rewrites the document. +#[tokio::test] +#[ignore = "needs a MongoDB replica set in EXTENDDB_TEST_MONGODB_URI"] +async fn live_an_open_beside_a_server_starts_no_key_repair() { + use std::time::{Duration, Instant}; + + let engine = scratch_engine().await; + let key_info = table(&engine, "pass-beside", false); + let key = item_key(false); + engine + .catalog_db + .collection::("tables") + .insert_one(catalog_row("beside", &key_info.table_id, &key_info)) + .await + .expect("catalog row"); + store_without_key(&engine, &key_info, &key, Some("upd")).await; + let unrepaired = |document: &Document| document.get_object_id("_id").is_ok(); + let engine = Arc::new(engine); + + let mut beside = ServerComponentsOptions::default(); + beside.beside_server = true; + assert!( + crate::start_key_repair(&engine, &beside).is_none(), + "an open beside a server started a key repair" + ); + tokio::time::sleep(Duration::from_secs(2)).await; + assert!( + unrepaired(&stored(&engine, &key_info, &key).await), + "an open beside a server rewrote a data document" + ); + + let started = Instant::now(); + let repair = crate::start_key_repair(&engine, &ServerComponentsOptions::default()) + .expect("a server's open starts the repair"); + let deadline = started + Duration::from_secs(30); + while unrepaired(&stored(&engine, &key_info, &key).await) { + assert!( + Instant::now() < deadline, + "the repair the server's open started did not rewrite the document" + ); + tokio::time::sleep(Duration::from_millis(20)).await; + } + let want = with(key.clone(), "w", s("upd")); + assert_eq!( + stored(&engine, &key_info, &key).await, + expected_document(&want, &key_info, 1), + "the server's repair rebuilt the document" + ); + let _ = repair.await; + let engine = Arc::try_unwrap(engine) + .ok() + .expect("the finished task dropped its engine"); + finish(engine).await; +} diff --git a/crates/storage-mongodb/src/lib.rs b/crates/storage-mongodb/src/lib.rs index b7ebaa376..4c628a985 100644 --- a/crates/storage-mongodb/src/lib.rs +++ b/crates/storage-mongodb/src/lib.rs @@ -8,8 +8,8 @@ mod admin_store; mod authorization_store; +mod backup_catalog; mod backup_engine; -pub use backup_engine::BackupSweep; mod bootstrapper; mod catalog_store; pub mod condition; @@ -22,6 +22,7 @@ mod metadata_engine; pub mod migrations; mod operations; pub mod pushdown; +mod restore_engine; mod snapshot_engine; mod stream_engine; mod table_engine; @@ -132,9 +133,12 @@ fn server_components_factory( // Register as the catalog's server, then read the migration lock and // the catalog's status, before the engine writes anything: a server // that refuses has deleted its registration and written nothing else. - let registration = migrations::register_and_check( + // Every caller registers, a command too, so a migration does not run + // under one; a command's registration carries its own name. + let registration = migrations::register_and_check_as( &migrations::MongoTarget::new(&connection_string), options.release, + options.command.unwrap_or(migrations::SERVE_COMMAND), migrations::Timing::PRODUCTION, ) .await @@ -147,6 +151,7 @@ fn server_components_factory( max_catalog_connections, tx_read_concern, Arc::clone(®istration), + &options, ) .await; if built.is_err() { @@ -156,6 +161,33 @@ fn server_components_factory( }) } +/// Start the startup key repair, for a server only; returns its task. +/// +/// An UpdateItem on a build before the fix for issue #436 could create +/// an item whose document has no key. The repair rewrites any such +/// document, so that reads return the key and PutItem on it succeeds (see +/// `data_engine::key_repair`). It runs in the background: the pass costs +/// time in proportion to the number of affected documents, and the +/// listener must not wait for it. A key the pass has not reached yet +/// answers as it did before the repair existed, and each batch is its own +/// transaction, so running it while another process serves the same data +/// is safe. +/// +/// The repair is a server's: a command that opens the catalog beside a +/// server (`options.beside_server`: `extenddb backup list` and the purge's +/// dry run), or one that writes beside the live servers with +/// `options.startup_recovery` off (`purge --confirm`), starts none, as on +/// PostgreSQL, where such a command runs none of the startup recovery; it +/// would otherwise start a write pass over the data and exit while the pass +/// may be mid-batch. +fn start_key_repair( + engine: &Arc, + options: &extenddb_storage::server_components::ServerComponentsOptions, +) -> Option> { + (!options.beside_server && options.startup_recovery) + .then(|| data_engine::key_repair::spawn_key_repair(engine.clone())) +} + /// The engine and stores `serve` runs on, built once the server holds its /// registration. async fn build_components( @@ -165,6 +197,7 @@ async fn build_components( max_catalog_connections: Option, tx_read_concern: mongodb::options::ReadConcern, registration: Arc, + options: &extenddb_storage::server_components::ServerComponentsOptions, ) -> Result { // Create MongoEngine let engine = MongoEngine::new(connection_string, region, max_connections, tx_read_concern) @@ -174,29 +207,9 @@ async fn build_components( details: e.to_string(), })?; - // A crash can leave a backup's metadata claimed by a restore or a - // delete that will never finish, or a restore target nothing will - // finish. The sweep removes only state whose heartbeat has stopped, so - // it is safe while another process shares this catalog; the GSI - // backfill worker repeats it periodically. - match engine.sweep_stale_backup_claims().await { - Ok(sweep) if sweep.is_empty() => {} - Ok(sweep) => tracing::warn!("startup backup sweep: {sweep}"), - Err(e) => tracing::error!("Failed to sweep stale backup claims: {e}"), - } - let engine = Arc::new(engine); - // An UpdateItem on a build before the fix for issue #436 could create - // an item whose document has no key. Rewrite any such document, so - // that reads return the key and PutItem on it succeeds (see - // `data_engine::key_repair`). This runs in the background: the pass - // costs time in proportion to the number of affected documents, and - // the listener must not wait for it. A key the pass has not reached - // yet answers as it did before the repair existed, and each batch is - // its own transaction, so running this while another process serves - // the same data is safe. - data_engine::key_repair::spawn_key_repair(engine.clone()); + start_key_repair(&engine, options); // Create one shared catalog/management client. MongoDB Client clones // share the underlying pool, so max_catalog_connections limits the @@ -416,17 +429,8 @@ pub struct MongoEngine { /// [`crate::config::MongoStorageConfig::transaction_read_concern`] for /// MongoDB-wire-compatible backends that don't support snapshot reads. tx_read_concern: mongodb::options::ReadConcern, - /// Heartbeat timing for backup claims and restore targets - /// (see [`backup_engine`]). + /// Heartbeat timing of a restore's job record (see [`backup_engine`]). pub(crate) backup_liveness: backup_engine::BackupLiveness, - /// Restores this process may run at once across accounts, and the - /// semaphore that enforces it; see - /// [`backup_engine::restore_server_slot_count`]. - pub(crate) restore_server_slot_count: usize, - pub(crate) restore_server_slots: Arc, - /// Restores each account may run at once in this process; see - /// [`backup_engine::RestoreAccountSlots`]. - pub(crate) restore_account_slots: backup_engine::RestoreAccountSlots, } /// Build a MongoDB client from a connection string, applying the shared @@ -494,22 +498,6 @@ pub(crate) async fn guarded_client_options( Ok(options) } -/// The connection pool size the driver will use for `options`: the explicit -/// `maxPoolSize`, else the driver's default of 10 (`mongodb` 3.x). The driver -/// does not enforce `waitQueueTimeoutMS` (its `ClientOptions::wait_queue_timeout` -/// documents it as not supported), so a checkout waits for as long as every -/// connection stays checked out; the work this backend admits at once is -/// therefore bounded from the pool size instead (see -/// [`backup_engine::restore_server_slot_count`]). -pub(crate) fn effective_pool_size(options: &mongodb::options::ClientOptions) -> u32 { - options - .max_pool_size - .unwrap_or(DRIVER_DEFAULT_MAX_POOL_SIZE) -} - -/// `mongodb::cmap::DEFAULT_MAX_POOL_SIZE`, which the driver does not export. -const DRIVER_DEFAULT_MAX_POOL_SIZE: u32 = 10; - async fn client_options_with_pool_size( connection_string: &str, max_pool_size: Option, @@ -530,9 +518,10 @@ fn apply_max_pool_size(options: &mut mongodb::options::ClientOptions, max_pool_s /// Refuse a connection-string write concern weaker than `majority`. /// -/// The backup claim protocol (DeleteBackup, restores and the stale-claim -/// sweep, see [`backup_engine`]) treats an acknowledged claim or CREATING -/// target as permanent. A write acknowledged under `w=1` (or any node count or +/// The backup protocol (DeleteBackup's DELETING write, a restore's CREATING +/// target and its job record, the catalog index's claims, see +/// [`backup_engine`] and [`backup_catalog`]) treats an acknowledged write as +/// permanent. A write acknowledged under `w=1` (or any node count or /// tag, whose relation to a majority this cannot see) can be rolled back when /// the primary fails over, and `journal=false` lets a majority acknowledge a /// write none of them has journaled. Accepted: no write concern (the server @@ -560,7 +549,7 @@ fn check_write_concern(wc: Option<&mongodb::options::WriteConcern>) -> Result<() None => Ok(()), Some(found) => Err(StorageError::Connection(format!( "MongoDB connection string must leave the write concern at the server \ - default or set w=majority (found {found}). Backup claims and restore \ + default or set w=majority (found {found}). Backup rows and restore \ targets assume an acknowledged write survives a primary failover, \ which a weaker write concern does not guarantee." ))), @@ -587,10 +576,11 @@ pub(crate) fn majority_write_concern( /// whatever the client's options or the server default say (inside a /// transaction the transaction's own options apply). /// -/// The backup claim protocol depends on this: DeleteBackup, a restore, and -/// the stale-claim sweep each check the other's marker or CREATING target -/// with a `find_one` after writing their own, and that check must see every -/// write the primary has acknowledged. A secondary can lag, and a `majority` +/// The backup protocol depends on this: DeleteBackup and a restore's +/// registration each check for the other's CREATING target or status write +/// (under their transactions, and DeleteBackup's unlocked first check +/// before), and the reconciler reads the job records' heartbeats; every such +/// read must see what the primary has acknowledged. A secondary can lag, and a `majority` /// or `snapshot` read can miss a write acknowledged with `w: 1`; a write /// acknowledged with `w: 1` can also be rolled back on failover, after the /// other side has acted on it. @@ -624,14 +614,12 @@ impl MongoEngine { tx_read_concern: mongodb::options::ReadConcern, ) -> Result { let options = guarded_client_options(connection_string, max_connections, true).await?; - let pool_size = effective_pool_size(&options); let client = mongodb::Client::with_options(options) .map_err(|e| StorageError::Connection(e.to_string()))?; let catalog_db = catalog_database(&client); let data_db = client.database("extenddb_data"); - let restore_server_slot_count = backup_engine::restore_server_slot_count(pool_size); Ok(Self { client, catalog_db, @@ -640,9 +628,6 @@ impl MongoEngine { gsi_cache: Arc::default(), tx_read_concern, backup_liveness: backup_engine::BackupLiveness::default(), - restore_server_slot_count, - restore_server_slots: Arc::new(tokio::sync::Semaphore::new(restore_server_slot_count)), - restore_account_slots: backup_engine::RestoreAccountSlots::default(), }) } @@ -707,28 +692,8 @@ impl MongoEngine { mod tests { use super::GsiCache; use super::client_options_with_pool_size; - use super::effective_pool_size; use std::time::Duration; - /// The restore bound is sized from the pool the driver will really use: - /// the URI's maxPoolSize, the configured override, or the driver default. - #[tokio::test] - async fn effective_pool_size_follows_uri_override_and_default() { - let uri = client_options_with_pool_size("mongodb://localhost:27017/?maxPoolSize=30", None) - .await - .expect("parse"); - assert_eq!(effective_pool_size(&uri), 30); - let configured = - client_options_with_pool_size("mongodb://localhost:27017/?maxPoolSize=30", Some(8)) - .await - .expect("parse"); - assert_eq!(effective_pool_size(&configured), 8); - let default = client_options_with_pool_size("mongodb://localhost:27017", None) - .await - .expect("parse"); - assert_eq!(effective_pool_size(&default), 10); - } - #[tokio::test] async fn absent_pool_override_preserves_uri_max_pool_size() { let options = diff --git a/crates/storage-mongodb/src/migrations/m0006_backup_catalog_index.rs b/crates/storage-mongodb/src/migrations/m0006_backup_catalog_index.rs new file mode 100644 index 000000000..bbcdfb2d2 --- /dev/null +++ b/crates/storage-mongodb/src/migrations/m0006_backup_catalog_index.rs @@ -0,0 +1,178 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Migration 6, `backup_catalog_index`, MongoDB: the catalog index of +//! backups kept in a backup store (the `backup_index` collection and its +//! indexes), the restore job record on the restore registrations (the +//! `restore_job` field of a restore target's `tables` document, and the +//! index the reconciler reads it by), the maintenance claim record (the +//! `backup_maintenance` collection) and the catalog id (the `catalog_id` +//! settings document). +//! +//! Every request is safe to send again: creating a collection that exists +//! fails with NamespaceExists (code 48), which is accepted by its code; +//! creating an index that exists with the same key and options succeeds; the +//! backfill touches only registrations without a job record; and the +//! catalog id is written with `$setOnInsert`. A second run changes nothing, +//! and a migrator stopped part way runs the body again. +//! +//! The backfill: every registration that exists was written by a release +//! without the job record, so each is a legacy restore. One whose table is +//! not CREATING, or whose copy had finished and was handed to the index +//! backfill worker (`restore_backfill_pending`), is DONE, completed when the +//! table was created. One whose table is CREATING with no hand-over is a +//! restore the upgrade's stop interrupted: RUNNING, with a heartbeat at the +//! epoch (not absent, so the staleness test selects it) and an owner +//! generation no job holds (0), so the first reconciler pass fails it and +//! removes its target. The owner and heartbeat fields of the earlier +//! registration shape are removed; the job record replaces them. +//! +//! The checksum test pins this file once the migration is released. The +//! module is excluded from rustfmt and clippy so neither can require an +//! edit, and keeps its collection, field and index names here instead of +//! reading the crate's, which may change. + +use async_trait::async_trait; +use bson::{Document, doc}; +use extenddb_storage::migrations::{CodeBody, Done, MigrationError}; +use mongodb::IndexModel; +use mongodb::options::IndexOptions; + +use super::MongoStores; + +/// This module, whose SHA-256 the checksum test pins. +pub(crate) const SOURCE: &str = include_str!("m0006_backup_catalog_index.rs"); + +/// Recorded as this migration's checksum. Changes only with a new migration. +const FINGERPRINT: &str = "0006_backup_catalog_index/mongodb/1"; + +/// NamespaceExists: `create` on a collection that exists. +const NAMESPACE_EXISTS: i32 = 48; + +/// The collections this migration creates in the catalog. +const COLLECTIONS: [&str; 2] = ["backup_index", "backup_maintenance"]; + +/// The indexes this migration creates in the catalog: the collection, the +/// key, and whether it is unique. +fn indexes() -> Vec<(&'static str, Document, bool)> { + vec![ + ("backup_index", doc! { "account_id": 1, "backup_id": 1 }, true), + ("backup_index", doc! { "backup_id": 1 }, true), + ("backup_index", doc! { "account_id": 1, "table_name": 1, "created_at": 1 }, false), + ( + "backup_index", + doc! { "store_kind": 1, "store_root": 1, "store_id": 1, "status": 1 }, + false, + ), + ("tables", doc! { "restore_job.status": 1, "restore_job.heartbeat_at": 1 }, false), + ] +} + +/// Migration 6's catalog body. +pub(crate) struct BackupCatalogIndex; + +#[async_trait] +impl CodeBody for BackupCatalogIndex { + fn fingerprint(&self) -> &'static str { + FINGERPRINT + } + + /// Nothing this migration finds stops it. + async fn detect(&self, _stores: &MongoStores) -> Result, MigrationError> { + Ok(Vec::new()) + } + + async fn run(&self, stores: &MongoStores) -> Result { + let db = &stores.catalog; + for name in COLLECTIONS { + stores + .request(format!("create the collection {}.{name}", db.name()), async { + match db.create_collection(name).await { + Err(e) if command_code(&e) == Some(NAMESPACE_EXISTS) => Ok(()), + other => other, + } + }) + .await?; + } + for (collection, key, unique) in indexes() { + let what = format!("create the index {key:?} on {}.{collection}", db.name()); + let options = IndexOptions::builder().unique(unique.then_some(true)).build(); + let model = IndexModel::builder().keys(key).options(options).build(); + let collection = db.collection::(collection); + stores.request(what, async { collection.create_index(model).await.map(|_| ()) }).await?; + } + + let mut notes = Vec::new(); + let tables = db.collection::("tables"); + let interrupted = doc! { "$and": [ + { "$eq": ["$table_status", "CREATING"] }, + { "$ne": ["$restore_backfill_pending", true] }, + ] }; + let filter = doc! { + "restore_source_backup_arn": { "$exists": true }, + "restore_job": { "$exists": false }, + }; + let running = stores + .request(format!("count the interrupted restores in {}.tables", db.name()), async { + let mut running = filter.clone(); + running.insert("$expr", interrupted.clone()); + tables.count_documents(running).await + }) + .await?; + let backfilled = stores + .request(format!("backfill the restore job records in {}.tables", db.name()), async { + tables + .update_many( + filter.clone(), + vec![ + doc! { "$set": { "restore_job": { + "status": { "$cond": [interrupted.clone(), "RUNNING", "DONE"] }, + "kind": "legacy", + "owner_generation": 0_i64, + "heartbeat_at": bson::DateTime::from_millis(0), + "completed_at": { + "$cond": [interrupted.clone(), "$$REMOVE", "$creation_date_time"] + }, + } } }, + doc! { "$unset": ["restore_operation_id", "restore_heartbeat_at"] }, + ], + ) + .await + }) + .await? + .modified_count; + if backfilled > 0 { + notes.push(format!( + "backfilled {backfilled} restore registration(s) as legacy restores, {running} of \ + them RUNNING with a dead owner for the reconciler to remove" + )); + } + + let settings = db.collection::("settings"); + let written = stores + .request(format!("write the catalog id in {}.settings", db.name()), async { + settings + .update_one( + doc! { "_id": "catalog_id" }, + doc! { "$setOnInsert": { "value": uuid::Uuid::new_v4().to_string() } }, + ) + .upsert(true) + .await + }) + .await? + .upserted_id + .is_some(); + if written { + notes.push("wrote settings.catalog_id".to_owned()); + } + Ok(Done { notes }) + } +} + +/// The server's error code of a failed command, if it reported one. +fn command_code(error: &mongodb::error::Error) -> Option { + match error.kind.as_ref() { + mongodb::error::ErrorKind::Command(command) => Some(command.code), + _ => None, + } +} diff --git a/crates/storage-mongodb/src/migrations/mod.rs b/crates/storage-mongodb/src/migrations/mod.rs index 9a3c88598..1f2cc9aa7 100644 --- a/crates/storage-mongodb/src/migrations/mod.rs +++ b/crates/storage-mongodb/src/migrations/mod.rs @@ -38,6 +38,9 @@ mod m0001_baseline; #[rustfmt::skip] #[allow(clippy::all, deprecated)] mod m0005_drop_continuous_backups; +#[rustfmt::skip] +#[allow(clippy::all, deprecated)] +mod m0006_backup_catalog_index; mod postcheck; use std::time::Duration; @@ -59,7 +62,7 @@ pub use locks::{MigrationLock, Timing}; #[cfg(feature = "test-util")] pub use postcheck::{Collection, Definitions, Index, describe_store}; pub use registrar::ServerRegistration; -pub use registrations::ServingRegistration; +pub use registrations::{SERVE_COMMAND, ServingRegistration}; /// The catalog database every MongoDB deployment uses. pub const CATALOG_DATABASE: &str = "extenddb_catalog"; @@ -137,6 +140,31 @@ pub fn migration(id: MigrationId) -> Migration { m0005_drop_continuous_backups::SOURCE, )], }, + // Breaking: a binary without it neither reads the index nor writes + // the job record on the restore registrations it creates, so a + // restore it ran would be owned by nobody; and the index's unique + // indexes are what refuse a second claim of one backup. Convergent: + // the collections and indexes are created only when absent, the + // backfill touches only a registration without a job record, which + // no registration this release writes is (each is inserted with its + // record), and the catalog id is written only when absent; so a + // second run over data written since changes nothing but a + // registration a release before it wrote, which is the repair + // `--reconverge` is for. + MigrationId::BackupCatalogIndex => Migration::Applies { + kind: Kind::Breaking, + convergent: true, + bodies: vec![StoreBody::code( + StoreKind::Catalog, + &m0006_backup_catalog_index::BackupCatalogIndex, + m0006_backup_catalog_index::SOURCE, + )], + }, + MigrationId::RestoreProgress => Migration::NotApplicable( + "a store-backed restore records its progress on the target's restore job record \ + (restore_job.file_index); the restore_progress table is PostgreSQL's data \ + database's", + ), } } @@ -266,6 +294,50 @@ pub async fn migrate_holding( result } +/// Take the migration lock on `target`, apply every pending migration +/// through `id`, and release the lock: for the crate's live tests, which +/// build catalogs in databases of their own. +/// +/// # Errors +/// +/// As [`Runner::migrate_through`], or the lock could not be taken. +#[cfg(test)] +pub(crate) async fn migrate_through_for_tests( + target: &MongoTarget, + id: MigrationId, +) -> Result { + let (held, target) = lock(target, true, Duration::from_secs(10), Timing::PRODUCTION).await?; + let result = async { + let migrations = MongoMigrations::open(&target, Some(&held)).await?; + let result = Runner::new(&migrations, "test").migrate_through(id).await; + migrations.close().await; + result + } + .await; + held.release().await; + result +} + +/// Run migration 6's catalog body on `target`'s catalog alone, with no lock +/// and no ledger row, as a migrator stopped before it wrote the row leaves +/// it: for the live tests of the body's backfill. +/// +/// # Errors +/// +/// As the body. +#[cfg(test)] +pub(crate) async fn run_backup_catalog_index( + target: &MongoTarget, +) -> Result { + use extenddb_storage::migrations::{CodeBody, MigrationBackend}; + let migrations = MongoMigrations::open(target, None).await?; + let done = m0006_backup_catalog_index::BackupCatalogIndex + .run(migrations.stores()) + .await; + migrations.close().await; + done +} + /// Register as the catalog's server and read its status, before anything /// opens the engine: a server that refuses has written nothing but its /// registration, which it deletes. The registration is written before the @@ -281,7 +353,24 @@ pub async fn register_and_check( release: &str, timing: Timing, ) -> Result { - let registration = ServingRegistration::register(target, release, timing).await?; + register_and_check_as(target, release, SERVE_COMMAND, timing).await +} + +/// [`register_and_check`] for the process `command` names: `extenddb serve` +/// ([`SERVE_COMMAND`]) or a command that opens the catalog as a server does +/// (`extenddb backup list`), whose registration carries its name +/// ([`ServingRegistration::register_as`]). +/// +/// # Errors +/// +/// As [`register_and_check`]. +pub async fn register_and_check_as( + target: &MongoTarget, + release: &str, + command: &'static str, + timing: Timing, +) -> Result { + let registration = ServingRegistration::register_as(target, release, command, timing).await?; let status = match catalog_status(target).await { Ok(status) => status, Err(e) => { @@ -305,6 +394,7 @@ pub async fn register_and_check( Ok(ServerRegistration::new( target, release, + command, timing, status, registration, diff --git a/crates/storage-mongodb/src/migrations/registrar.rs b/crates/storage-mongodb/src/migrations/registrar.rs index 9ff1d5cf5..9cf857329 100644 --- a/crates/storage-mongodb/src/migrations/registrar.rs +++ b/crates/storage-mongodb/src/migrations/registrar.rs @@ -25,11 +25,12 @@ use super::registrations::ServingRegistration; use super::{MongoTarget, catalog_status}; /// The serving registration of a running server, with what it needs to -/// take it again: the catalog, the release it registers as, its timings and -/// the status the server started with. +/// take it again: the catalog, the release and the process name it +/// registers as, its timings and the status the server started with. pub struct ServerRegistration { target: MongoTarget, release: String, + command: &'static str, timing: Timing, started: CatalogStatus, current: Mutex>, @@ -39,6 +40,7 @@ impl ServerRegistration { pub(super) fn new( target: &MongoTarget, release: &str, + command: &'static str, timing: Timing, started: CatalogStatus, registration: ServingRegistration, @@ -46,6 +48,7 @@ impl ServerRegistration { Self { target: target.clone(), release: release.to_owned(), + command, timing, started, current: Mutex::new(Arc::new(registration)), @@ -86,12 +89,18 @@ impl ServingRegistrar for ServerRegistration { } async fn register_again(&self) -> Reregistration { - let registration = - match ServingRegistration::register(&self.target, &self.release, self.timing).await { - Ok(registration) => registration, - Err(MigrationError::Locked(why)) => return Reregistration::Refused(why), - Err(e) => return Reregistration::Unavailable(e.to_string()), - }; + let registration = match ServingRegistration::register_as( + &self.target, + &self.release, + self.command, + self.timing, + ) + .await + { + Ok(registration) => registration, + Err(MigrationError::Locked(why)) => return Reregistration::Refused(why), + Err(e) => return Reregistration::Unavailable(e.to_string()), + }; let status = match catalog_status(&self.target).await { Ok(status) => status, Err(e) => { diff --git a/crates/storage-mongodb/src/migrations/registrations.rs b/crates/storage-mongodb/src/migrations/registrations.rs index 8fd1c29e8..1d756a861 100644 --- a/crates/storage-mongodb/src/migrations/registrations.rs +++ b/crates/storage-mongodb/src/migrations/registrations.rs @@ -5,6 +5,12 @@ //! `extenddb_servers` while it serves, and the read of every live one that //! `migrate --yes` waits on. The lock it is paired with is in `locks.rs`, //! whose module doc describes both. +//! +//! A command that opens the catalog as a server does (`extenddb backup list`, +//! `extenddb backup purge`) registers too, so a migration waits for it, and +//! its document's `command` field carries its name, which the migration's +//! wait prints. A document without the field (an earlier build's) is a +//! server's. use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Instant; @@ -38,7 +44,8 @@ pub(crate) async fn live_registrations( .iter() .map(|d| { format!( - "pid {} on {} (release {}, registered {}, expires {})", + "{}, pid {} on {} (release {}, registered {}, expires {})", + d.get_str("command").unwrap_or(SERVE_COMMAND), d.get("pid") .map_or_else(|| "unknown".to_owned(), ToString::to_string), d.get_str("host").unwrap_or("an unknown host"), @@ -52,6 +59,10 @@ pub(crate) async fn live_registrations( .collect()) } +/// The name a server's registration carries; a command's carries its own +/// (`extenddb backup list`). +pub const SERVE_COMMAND: &str = "extenddb serve"; + /// This host's name, for the registration. fn host_name() -> String { std::fs::read_to_string("/proc/sys/kernel/hostname") @@ -85,6 +96,24 @@ impl ServingRegistration { target: &MongoTarget, release: &str, timing: Timing, + ) -> Result { + Self::register_as(target, release, SERVE_COMMAND, timing).await + } + + /// [`ServingRegistration::register`] for the process `command` names + /// ([`SERVE_COMMAND`], or a command such as `extenddb backup list`): the + /// document's `command` field carries the name, and a command's refusal + /// while a migration holds the lock names the holder without the + /// server's advice to start the server. + /// + /// # Errors + /// + /// As [`ServingRegistration::register`]. + pub async fn register_as( + target: &MongoTarget, + release: &str, + command: &str, + timing: Timing, ) -> Result { let client = connect(&target.connection_string).await?; let db = super::catalog_handle(&client, &target.catalog); @@ -94,6 +123,7 @@ impl ServingRegistration { .update_one( doc! { "_id": &token }, vec![doc! { "$set": { + "command": command, "host": host_name(), "pid": i64::from(std::process::id()), "release": release, @@ -135,9 +165,15 @@ impl ServingRegistration { }; if let Some(lock) = lock { registration.release().await; + // A command's refusal ends at the holder; the command's caller + // adds that it is retried once the migration has finished. + let then = if command == SERVE_COMMAND { + ", so the server does not start; start it again once the migration finishes" + } else { + "" + }; return Err(MigrationError::Locked(format!( - "{} holds the migration lock on this catalog, so the server does not start; \ - start it again once the migration finishes", + "{} holds the migration lock on this catalog{then}", holder(Some(&lock)) ))); } diff --git a/crates/storage-mongodb/src/migrations/tests.rs b/crates/storage-mongodb/src/migrations/tests.rs index 651ad34fe..7edae3bfb 100644 --- a/crates/storage-mongodb/src/migrations/tests.rs +++ b/crates/storage-mongodb/src/migrations/tests.rs @@ -39,7 +39,7 @@ fn every_migration_records_a_row_in_the_catalog_and_names_its_stores() { } #[test] -fn every_body_is_code_and_three_and_four_do_not_apply() { +fn every_body_is_code_and_three_four_and_seven_do_not_apply() { for id in MigrationId::ALL { for body in migration(id).bodies() { assert!(matches!(body.body, Body::Code { .. }), "{id}"); @@ -48,6 +48,7 @@ fn every_body_is_code_and_three_and_four_do_not_apply() { for id in [ MigrationId::SortKeyDefinitions, MigrationId::IndexTableBaseKey, + MigrationId::RestoreProgress, ] { let Migration::NotApplicable(reason) = migration(id) else { panic!("{id} applies on MongoDB"); diff --git a/crates/storage-mongodb/src/restore_engine.rs b/crates/storage-mongodb/src/restore_engine.rs new file mode 100644 index 000000000..8f5b06d7c --- /dev/null +++ b/crates/storage-mongodb/src/restore_engine.rs @@ -0,0 +1,372 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The data-plane side of a store-backed restore on `MongoDB` +//! ([`RestoreEngine`]): the items of one verified data file written into the +//! CREATING target's data collection, the per-file record, the hand-over to +//! the index backfill worker and the count. +//! +//! The target's documents are built by [`item_to_document`], the write +//! path's function, so a restored table is laid out as a natively created +//! one and refuses what the write path refuses (an `N` sort key beyond +//! Decimal128's 34 significant digits is the write path's `Validation` +//! error, and fails the file). Base documents only: the target's indexes +//! are created CREATING with it and the GSI backfill worker fills them from +//! the base collection and flips the table ACTIVE once every index has +//! left CREATING, so `finish_restore` is the hand-over +//! ([`crate::MongoEngine::hand_over_to_backfill`]) and answers +//! [`Finished::Pending`]. +//! +//! One file is `insert_many` calls of [`INSERT_MANY_DOCUMENTS`] documents, +//! each one session transaction with the record's progress write: the +//! transaction first writes the record (`find_one_and_update` on the +//! target's `_id`, `table_id`, `restore_job.status = RUNNING` and +//! `restore_job.owner_generation` equal to the claim's, setting +//! `restore_job.file_index`, `restore_job.items_applied` to the file's +//! running total and `heartbeat_at` to the server's clock), and ends the +//! file as a lost claim when that write matches nothing; then inserts the +//! batch in the same transaction and commits. The record write is the +//! fence: a takeover is a write on the same document, so a writer whose +//! record another instance took over while it stalled finds its transaction +//! aborted for the write conflict (re-run, it meets the other generation and +//! stops) or its record write matching nothing, and no batch of it lands +//! after the takeover, as PostgreSQL's and SQLite's loads condition their +//! writes. The progress is per batch and in the batch's transaction, so a +//! job that dies leaves no documents its record does not account for: a +//! resume starts after `items_applied`, and `BulkLoadFile::reapplying` is +//! not read (nothing of a batch lands without its record). A duplicate key +//! (E11000, the data collection's unique `_id`) is the backup's and fails +//! the file, as any other write error does. The record carries the +//! progress, so `read_restore_progress` and `sweep_restore_progress` keep +//! their defaults. + +use extenddb_core::types::{AttributeDefinition, Item, KeySchemaElement}; +use extenddb_storage::backup_catalog::{ + JobClaim, JobStatus, RESTORE_MONGODB_AFTER_INSERT_MANY, RestoreTargetKey, restore_pause_point, +}; +use extenddb_storage::error::StorageError; +use extenddb_storage::restore_engine::{BulkLoadFile, Finished, RestoreEngine}; +use futures::future::BoxFuture; +use futures::stream::BoxStream; +use mongodb::bson::{self, Document, doc}; + +use crate::MongoEngine; +use crate::backup_catalog::{ + RESTORE_JOB_FILE_INDEX, RESTORE_JOB_GENERATION, RESTORE_JOB_HEARTBEAT, + RESTORE_JOB_ITEMS_APPLIED, RESTORE_JOB_STATUS, generation_field, literal_set, +}; +use crate::data::{data_collection_name, item_to_document}; +use crate::data_engine::{CommitOutcome, TRANSIENT_RETRY_ATTEMPTS, TxErr, backoff_sleep}; + +/// Documents per `insert_many` call. Not a message bound: 1,000 documents at +/// the item size limit are about 390 MiB, and the driver splits one call by +/// the server's message and write-batch bounds on its own. It is the unit a +/// half-applied file is re-applied in, and the unit between two reads of the +/// record's claim. +pub(crate) const INSERT_MANY_DOCUMENTS: usize = 1000; + +/// The server's duplicate-key error code. +const DUPLICATE_KEY: i32 = 11000; + +fn internal(e: impl std::fmt::Display) -> StorageError { + StorageError::Internal(e.to_string()) +} + +/// The key shape of a CREATING restore target, read from its catalog +/// document. +struct TargetShape { + key_schema: Vec, + attribute_definitions: Vec, +} + +impl MongoEngine { + /// The target's catalog document, when `key` names a CREATING restore + /// target with this table id. + async fn restore_target_doc(&self, key: &RestoreTargetKey) -> Result { + self.catalog_db + .collection::("tables") + .find_one(doc! { + "_id": { "account_id": &key.account_id, "table_name": &key.table_name }, + "table_id": &key.table_id, + "table_status": "CREATING", + "restore_source_backup_arn": { "$exists": true }, + }) + .await + .map_err(internal)? + .ok_or_else(|| { + StorageError::TableNotFound(format!( + "{} is not a CREATING restore target with table id {}", + key.table_name, key.table_id + )) + }) + } + + /// Write `batch` into the target's data collection in one transaction + /// with the record's progress (module doc) and clear it; the documents + /// the server took. `applied_total` is the file's running total once + /// the batch lands (`start_at` plus every batch of this call). `Err` on + /// a lost claim (the record is not RUNNING under `generation`, or a + /// takeover's write conflicted with this transaction), on a duplicate + /// key (the backup's), and on any other error. + async fn insert_restore_batch( + &self, + collection: &mongodb::Collection, + target: &RestoreTargetKey, + generation: u64, + file_index: u32, + applied_total: u64, + batch: &mut Vec, + ) -> Result { + let tables = self.catalog_db.collection::("tables"); + let fence = doc! { + "_id": { "account_id": &target.account_id, "table_name": &target.table_name }, + "table_id": &target.table_id, + RESTORE_JOB_STATUS: JobStatus::Running.as_str(), + RESTORE_JOB_GENERATION: generation_field(generation), + }; + let progress = vec![doc! { "$set": literal_set(doc! { + RESTORE_JOB_FILE_INDEX: i64::from(file_index), + RESTORE_JOB_ITEMS_APPLIED: i64::try_from(applied_total).unwrap_or(i64::MAX), + RESTORE_JOB_HEARTBEAT: "$$NOW", + }) }]; + let lost = || { + StorageError::Internal(format!( + "restore of table {}: the claim on the target was lost while a data file \ + loaded (the record is not RUNNING under generation {generation}); the \ + record's owner finishes it", + target.table_name + )) + }; + let mut session = self.client.start_session().await.map_err(internal)?; + let tx_options = self.write_transaction_options(); + for attempt in 0..TRANSIENT_RETRY_ATTEMPTS { + session + .start_transaction() + .with_options(tx_options.clone()) + .await + .map_err(internal)?; + let body: Result<(), TxErr> = async { + let fenced = tables + .find_one_and_update(fence.clone(), progress.clone()) + .session(&mut session) + .await + .map_err(TxErr::from)?; + if fenced.is_none() { + return Err(TxErr::Fatal(lost())); + } + if batch.is_empty() { + return Ok(()); + } + collection + .insert_many(batch.iter().cloned()) + .session(&mut session) + .await + .map_err(|e| { + if is_duplicate_key_anywhere(&e) { + // The engine prefixes the file's key, so the + // file is not named here. + TxErr::Fatal(StorageError::Validation(format!( + "holds a primary key the target already has (the backup holds \ + a key twice): {e}" + ))) + } else { + TxErr::from(e) + } + })?; + Ok(()) + } + .await; + match body { + Ok(()) => match self.commit_write_transaction(&mut session, None).await? { + CommitOutcome::Committed => { + let sent = batch.len(); + batch.clear(); + return Ok(u64::try_from(sent).unwrap_or(u64::MAX)); + } + CommitOutcome::RerunBody => backoff_sleep(attempt).await, + }, + Err(TxErr::Transient) => { + let _ = session.abort_transaction().await; + backoff_sleep(attempt).await; + } + Err(TxErr::Fatal(e)) => { + let _ = session.abort_transaction().await; + return Err(e); + } + } + } + Err(StorageError::Internal(format!( + "restore of table {}: a batch of data file {file_index} did not commit after \ + {TRANSIENT_RETRY_ATTEMPTS} conflicting attempts", + target.table_name + ))) + } +} + +/// Whether `e` carries the server's duplicate-key code, as an insert's +/// write error, a bulk write's, or a command error inside a transaction. +fn is_duplicate_key_anywhere(e: &mongodb::error::Error) -> bool { + match &*e.kind { + mongodb::error::ErrorKind::InsertMany(failure) => failure + .write_errors + .as_ref() + .is_some_and(|errors| errors.iter().any(|w| w.code == DUPLICATE_KEY)), + mongodb::error::ErrorKind::Command(c) => c.code == DUPLICATE_KEY, + _ => crate::data_engine::is_duplicate_key(e), + } +} + +impl RestoreEngine for MongoEngine { + fn bulk_load<'a>( + &'a self, + target: &'a RestoreTargetKey, + claim: &'a dyn JobClaim, + file: BulkLoadFile, + mut items: BoxStream<'a, Result>, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + use futures::StreamExt; + + // The shape from the target's catalog document; each batch's + // transaction writes the record under the claim before its + // inserts (`insert_restore_batch`), which is the fence. + let generation = claim.generation(); + let table_doc = self.restore_target_doc(target).await?; + let shape = TargetShape { + key_schema: bson::from_bson( + table_doc + .get("key_schema") + .cloned() + .ok_or_else(|| internal("restore target is missing key_schema"))?, + ) + .map_err(|e| StorageError::Internal(format!("key schema: {e}")))?, + attribute_definitions: bson::from_bson( + table_doc + .get("attribute_definitions") + .cloned() + .ok_or_else(|| { + internal("restore target is missing attribute_definitions") + })?, + ) + .map_err(|e| StorageError::Internal(format!("attribute definitions: {e}")))?, + }; + let collection = self + .data_db + .collection::(&data_collection_name(&target.table_id)); + // The test-hook gate the in-catalog copy also honours: it holds + // the restore after the target's CREATING index metadata exists + // and before the first insert of its first file, so a test can + // run a backfill worker tick through that window and see that + // the worker completes no index of a restore whose base load has + // not started. Inert without the `test-hooks` feature and unless + // a test armed it. + if file.file_index == 0 && file.start_at == 0 { + self.wait_for_gsi_backfill_test_gate(&target.table_name) + .await?; + } + + let mut batch: Vec = Vec::with_capacity(INSERT_MANY_DOCUMENTS); + let mut applied: u64 = 0; + while let Some(item) = items.next().await { + let item = item?; + batch.push(item_to_document( + &item, + &shape.key_schema, + &shape.attribute_definitions, + )?); + if batch.len() == INSERT_MANY_DOCUMENTS { + let total = file.start_at + applied + batch.len() as u64; + applied += self + .insert_restore_batch( + &collection, + target, + generation, + file.file_index, + total, + &mut batch, + ) + .await?; + restore_pause_point(RESTORE_MONGODB_AFTER_INSERT_MANY).await; + } + } + // The file's last batch, or, for a file with no item left to + // apply, the file's record alone (the fence write with the + // running total and no insert). + let total = file.start_at + applied + batch.len() as u64; + applied += self + .insert_restore_batch( + &collection, + target, + generation, + file.file_index, + total, + &mut batch, + ) + .await?; + Ok(applied) + }) + } + + fn finish_restore<'a>( + &'a self, + target: &'a RestoreTargetKey, + claim: &'a dyn JobClaim, + item_count: u64, + size_bytes: u64, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + let handed_over = self + .hand_over_to_backfill(target, claim.generation(), item_count, Some(size_bytes)) + .await?; + if handed_over { + Ok(Finished::Pending) + } else { + Ok(Finished::NotMatched(format!( + "the job record of {} is not RUNNING under generation {}, or the table is \ + not CREATING with table id {}", + target.table_name, + claim.generation(), + target.table_id + ))) + } + }) + } + + fn count_restore_target<'a>( + &'a self, + key: &'a RestoreTargetKey, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + // The id names the data collection; the key's account and name + // are checked against the catalog so a bare id reads no other + // account's table. + self.restore_target_doc(key).await?; + self.data_db + .collection::(&data_collection_name(&key.table_id)) + .count_documents(doc! {}) + .await + .map_err(internal) + }) + } + + fn restore_target_ready<'a>( + &'a self, + key: &'a RestoreTargetKey, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + // The record's `ddl_done` marker, set after the last index + // metadata document and data collection were created + // (`create_table_impl_inner`); a record without it is one whose + // owner died between the registration's commit and the end of + // its DDL, and its collections and indexes are not the + // registration's. The catalog document is read first, so a bare + // id reads no other account's table. + let table_doc = self.restore_target_doc(key).await?; + Ok(table_doc + .get_document(crate::backup_catalog::RESTORE_JOB) + .ok() + .and_then(|job| job.get_bool("ddl_done").ok()) + .unwrap_or(false)) + }) + } +} diff --git a/crates/storage-mongodb/src/snapshot_engine.rs b/crates/storage-mongodb/src/snapshot_engine.rs index fc8eefa48..b1201580a 100644 --- a/crates/storage-mongodb/src/snapshot_engine.rs +++ b/crates/storage-mongodb/src/snapshot_engine.rs @@ -69,8 +69,25 @@ //! //! Memory: a batch is at most `batch_size` documents //! (`paging::DEFAULT_FETCH_BATCH`, 1000) and at most the server's 16 MiB -//! reply limit, whichever is reached first, so unlike the SQL backends' -//! row-bounded fetches one batch here is bounded in bytes too. +//! reply limit, whichever is reached first, so one batch here is bounded in +//! bytes by the server. The stream applies no cut of its own at +//! `paging::SNAPSHOT_FETCH_BYTES` (64 MiB): the reply limit is already under +//! that budget, and the byte cut the SQL backends make exists because their +//! row-bounded fetches have no such limit. +//! +//! A snapshot read serves the majority-committed snapshot, which can trail +//! what a `readConcern: local` read of the primary already returns (the +//! catalog handle's level, so `DescribeTable`'s): a table flipped ACTIVE a +//! few milliseconds earlier can be missing, or still CREATING, at the +//! snapshot's time, and a CreateBackup sent as soon as DescribeTable said +//! ACTIVE would be refused as a table that does not exist. So an open whose +//! snapshot finds the table missing or not ACTIVE reads the `tables` +//! document at `local`; when that read shows it ACTIVE, the open starts over +//! with a fresh session after a short sleep (5 ms, doubling to 100 ms), +//! for at most [`MAJORITY_CATCH_UP`], and answers the snapshot's refusal +//! once a local read no longer shows the table ACTIVE or the bound has +//! passed. The bound is far above the lag a w:majority write leaves (one +//! journal commit, 100 ms at most by default). use extenddb_core::types::Item; use extenddb_storage::error::StorageError; @@ -102,6 +119,10 @@ const STANDALONE_SERVER: &str = "MongoDB refused the first read at readConcern s standalone server cannot serve snapshot reads, and table snapshots need a replica set \ member"; +/// How long an open waits for the majority-committed snapshot to show a +/// table that a `local` read shows ACTIVE (module doc). +const MAJORITY_CATCH_UP: std::time::Duration = std::time::Duration::from_secs(2); + /// Why an open cannot proceed when the first read succeeded but the session /// reports no snapshot time. const NO_SNAPSHOT_TIME: &str = "MongoDB served the first read without a snapshot time: a read \ @@ -116,13 +137,51 @@ impl SnapshotEngine for MongoEngine { let account_id = account_id.to_owned(); let table_name = table_name.to_owned(); Box::pin(async move { - self.open_table_snapshot_impl(&account_id, &table_name) - .await + let deadline = tokio::time::Instant::now() + MAJORITY_CATCH_UP; + let mut delay = std::time::Duration::from_millis(5); + loop { + match self + .open_table_snapshot_impl(&account_id, &table_name) + .await + { + Err(StorageError::TableNotFound(_) | StorageError::TableNotActive(_)) + if tokio::time::Instant::now() < deadline + && self + .table_is_active_locally(&account_id, &table_name) + .await? => + { + tokio::time::sleep(delay).await; + delay = (delay * 2).min(std::time::Duration::from_millis(100)); + } + outcome => return outcome, + } + } }) } } impl MongoEngine { + /// Whether a `local` read of the `tables` document shows the table + /// ACTIVE: what `DescribeTable` reads, and the snapshot's time may trail. + async fn table_is_active_locally( + &self, + account_id: &str, + table_name: &str, + ) -> Result { + Ok(self + .catalog_db + .collection::("tables") + .find_one(doc! { + "_id": { "account_id": account_id, "table_name": table_name }, + "table_status": "ACTIVE", + }) + .read_concern(ReadConcern::local()) + .projection(doc! { "_id": 1 }) + .await + .map_err(|e| map_mongo_error(&e))? + .is_some()) + } + async fn open_table_snapshot_impl( &self, account_id: &str, @@ -470,7 +529,6 @@ mod tests { .insert_one(doc! { "_id": "control_plane_delay_seconds", "value": "0" }) .await .expect("setting"); - let slots = crate::backup_engine::restore_server_slot_count(10); let engine = MongoEngine { client: client.clone(), catalog_db, @@ -479,9 +537,6 @@ mod tests { gsi_cache: Arc::default(), tx_read_concern: ReadConcern::majority(), backup_liveness: crate::backup_engine::BackupLiveness::default(), - restore_server_slot_count: slots, - restore_server_slots: Arc::new(tokio::sync::Semaphore::new(slots)), - restore_account_slots: crate::backup_engine::RestoreAccountSlots::default(), }; let worker = tokio::spawn(crate::ttl_worker::gsi_backfill_worker( Arc::new(engine.clone()), @@ -648,6 +703,78 @@ mod tests { .await; } + /// A table a `local` read shows ACTIVE opens although the majority- + /// committed snapshot does not show it yet. The flip to ACTIVE is + /// written at `w: 1` without the journal, so the majority snapshot + /// trails it by up to one journal commit; the test observes that + /// precondition with a snapshot read of its own before each open, and + /// asserts that it saw it at least once. + #[tokio::test] + async fn an_open_waits_for_the_snapshot_to_show_a_table_a_local_read_shows_active() { + let Some(uri) = uri() else { + eprintln!( + "SKIP an_open_waits_for_the_snapshot_to_show_a_table_a_local_read_shows_active: \ + no MongoDB" + ); + return; + }; + let _hooks = hooks().await; + let engine = fixture(&uri).await; + engine + .create_table(ACCOUNT, snapshot_conformance::table_input("lagging")) + .await + .expect("create"); + snapshot_conformance::wait_active(&*engine, ACCOUNT, "lagging", || async { + let _ = + extenddb_storage::WorkerStore::process_control_plane_transitions(&*engine).await; + }) + .await; + let tables = engine.catalog_db.collection::("tables"); + let id = doc! { "_id": { "account_id": ACCOUNT, "table_name": "lagging" } }; + let unjournaled = mongodb::options::WriteConcern::builder() + .w(mongodb::options::Acknowledgment::Nodes(1)) + .journal(false) + .build(); + let mut trailing = 0; + for _ in 0..20 { + set_updating(&engine, "lagging").await; + tables + .update_one(id.clone(), doc! { "$set": { "table_status": "ACTIVE" } }) + .write_concern(unjournaled.clone()) + .await + .expect("flip to ACTIVE"); + let mut session = engine + .client + .start_session() + .snapshot(true) + .await + .expect("session"); + let at_snapshot = tables + .find_one(id.clone()) + .read_concern(ReadConcern::snapshot()) + .session(&mut session) + .await + .expect("snapshot read") + .and_then(|d| d.get_str("table_status").ok().map(str::to_owned)); + if at_snapshot.as_deref() != Some("ACTIVE") { + trailing += 1; + } + let snapshot = engine + .open_table_snapshot(ACCOUNT, "lagging") + .await + .expect("a table a local read shows ACTIVE opens"); + assert_eq!( + snapshot.description.table_status, + extenddb_core::types::TableStatus::Active + ); + } + eprintln!("the snapshot trailed the local read in {trailing} of 20 flips"); + assert!( + trailing > 0, + "precondition: no flip left the majority snapshot behind the local read" + ); + } + /// A document the decoder cannot read ends the stream at its error, as on /// the other backends: nothing after it, so a consumer cannot mistake the /// rest for the whole table. diff --git a/crates/storage-mongodb/src/stream_records_live_tests.rs b/crates/storage-mongodb/src/stream_records_live_tests.rs index bf769836a..ec5956c7a 100644 --- a/crates/storage-mongodb/src/stream_records_live_tests.rs +++ b/crates/storage-mongodb/src/stream_records_live_tests.rs @@ -42,7 +42,6 @@ impl Fixture { async fn new() -> Self { let tag = uuid::Uuid::new_v4().simple().to_string(); let client = mongodb::Client::with_uri_str(uri()).await.expect("client"); - let slots = crate::backup_engine::restore_server_slot_count(10); let engine = MongoEngine { client: client.clone(), catalog_db: client.database(&format!("s{}_cat", &tag[..12])), @@ -51,9 +50,6 @@ impl Fixture { gsi_cache: Arc::default(), tx_read_concern: mongodb::options::ReadConcern::snapshot(), backup_liveness: crate::backup_engine::BackupLiveness::default(), - restore_server_slot_count: slots, - restore_server_slots: Arc::new(tokio::sync::Semaphore::new(slots)), - restore_account_slots: crate::backup_engine::RestoreAccountSlots::default(), }; Self { engine } } diff --git a/crates/storage-mongodb/src/table_engine.rs b/crates/storage-mongodb/src/table_engine.rs index 77706c3c7..0ad4bd12b 100644 --- a/crates/storage-mongodb/src/table_engine.rs +++ b/crates/storage-mongodb/src/table_engine.rs @@ -129,15 +129,21 @@ impl MongoEngine { .await } - /// Create a restore target with provenance present in its first catalog - /// document, so backup deletion cannot observe an ownerless target. + /// Create a restore target with its provenance and its RUNNING job + /// record in its first catalog document, so nothing ever observes an + /// unregistered or unowned target. When the registration carries a + /// session, the document is inserted in that session's transaction and + /// the transaction is committed right after the insert, before the + /// target's collections and indexes are created; a transaction the + /// server aborted for a write conflict is reported as `Transient`, with + /// nothing created, so the caller re-runs it. pub(crate) async fn create_table_for_restore( &self, account_id: &str, input: CreateTableInput, - provenance: &crate::backup_engine::RestoreProvenance<'_>, + registration: &mut crate::backup_engine::RestoreRegistration<'_>, ) -> Result { - self.create_table_impl_inner(account_id, input, true, Some(provenance)) + self.create_table_impl_inner(account_id, input, true, Some(registration)) .await } @@ -146,7 +152,7 @@ impl MongoEngine { account_id: &str, input: CreateTableInput, defer_active: bool, - restore: Option<&crate::backup_engine::RestoreProvenance<'_>>, + restore: Option<&mut crate::backup_engine::RestoreRegistration<'_>>, ) -> Result { Self::validate_account_id(account_id)?; @@ -255,28 +261,100 @@ impl MongoEngine { "sse_specification": sse_spec_bson, "on_demand_throughput": on_demand_bson, }; - if let Some(provenance) = restore { - provenance.insert_into(&mut table_doc); + if let Some(registration) = restore.as_deref() { + registration.insert_into(&mut table_doc); } let tables_coll = self.catalog_db.collection::("tables"); - if let Err(e) = tables_coll.insert_one(table_doc).await { - if !e.to_string().contains("E11000") { + let for_restore = restore.is_some(); + let mut restore = restore; + let inserted = match restore + .as_deref_mut() + .and_then(|registration| registration.session.as_deref_mut()) + { + Some(session) => tables_coll.insert_one(table_doc).session(session).await, + None => tables_coll.insert_one(table_doc).await, + }; + // A manifest record's first heartbeat is rewritten with the server's + // clock in the same transaction (nothing for a legacy one). + let inserted = match inserted { + Ok(_) => match restore.as_deref_mut() { + Some(registration) => { + registration + .stamp_server_clock(&tables_coll, &table_id) + .await + } + None => Ok(()), + }, + Err(e) => Err(e), + }; + let mut session = restore.and_then(|registration| registration.session.as_deref_mut()); + if let Err(e) = inserted { + if let Some(session) = session { + let _ = session.abort_transaction().await; + } + if crate::data_engine::is_duplicate_key(&e) { + return Err(self + .name_taken_error(&tables_coll, account_id, &input.table_name, for_restore) + .await); + } + if crate::data_engine::is_transient_write_conflict(&e) { + return Err(StorageError::Transient(format!( + "the registration of {} lost a write conflict: {e}", + input.table_name + ))); + } + return Err(StorageError::Internal(e.to_string())); + } + // The registration's first heartbeat on the server's clock: an insert + // cannot carry `$$NOW`, so the inserted record holds the process's + // clock for the length of this transaction and this pipeline update, + // in the same transaction, replaces it before anything commits. Every + // heartbeat the reconciler ever reads is therefore on the database's + // clock, and a host whose clock is behind the server's by more than + // `stale_job_after_secs` cannot make another server list its new + // restore dead. + if for_restore { + let stamped = { + let update = tables_coll.update_one( + doc! { + "_id": { "account_id": account_id, "table_name": &input.table_name }, + "table_id": &table_id, + }, + vec![doc! { "$set": { + crate::backup_catalog::RESTORE_JOB_HEARTBEAT: "$$NOW", + } }], + ); + match session.as_deref_mut() { + Some(session) => update.session(session).await, + None => update.await, + } + }; + if let Err(e) = stamped { + if let Some(session) = session { + let _ = session.abort_transaction().await; + } + if crate::data_engine::is_transient_write_conflict(&e) { + return Err(StorageError::Transient(format!( + "the registration of {} lost a write conflict: {e}", + input.table_name + ))); + } return Err(StorageError::Internal(e.to_string())); } - return Err(self - .name_taken_error( - &tables_coll, - account_id, - &input.table_name, - restore.is_some(), - ) - .await); } - // A restore releases its backup claim as soon as the target document, - // which carries the provenance, exists. - if let Some(inserted) = restore.and_then(|provenance| provenance.inserted) { - inserted.notify_one(); + // The registration's transaction ends here, before any collection is + // created, so an aborted one leaves nothing behind. + if let Some(session) = session { + match self.commit_write_transaction(session, None).await? { + crate::data_engine::CommitOutcome::Committed => {} + crate::data_engine::CommitOutcome::RerunBody => { + return Err(StorageError::Transient(format!( + "the registration of {} was aborted at commit", + input.table_name + ))); + } + } } // Create the data collection with appropriate indexes @@ -504,6 +582,24 @@ impl MongoEngine { } } + // A restore target's DDL is complete here: the data collection, its + // unique key index and every GSI and LSI metadata document and data + // collection exist. The record was committed before any of them (the + // registration's transaction ends at the catalog insert), so the + // record carries `restore_job.ddl_done = false` until this write, + // and the reconciler's resume refuses a record left unmarked by an + // owner that died in between (`restore_target_ready`), rather than + // resuming into a target that is missing its indexes. + if for_restore { + tables_coll + .update_one( + doc! { "table_id": &table_id }, + doc! { "$set": { crate::backup_catalog::RESTORE_JOB_DDL_DONE: true } }, + ) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + } + // Derive the SSEDescription from the SSESpecification, mirroring the // postgres backend (table_helpers.rs:329-346). The specification's // `Enabled: true` becomes a KMS-status ENABLED description with a diff --git a/crates/storage-mongodb/src/ttl_worker.rs b/crates/storage-mongodb/src/ttl_worker.rs index 9d309f575..81de24c02 100644 --- a/crates/storage-mongodb/src/ttl_worker.rs +++ b/crates/storage-mongodb/src/ttl_worker.rs @@ -87,25 +87,14 @@ pub(crate) async fn stream_record_cleanup_worker( /// says the table has no indexes; an invalidated in-flight observation /// cannot publish that stale no-index result. /// -/// Every `heartbeat_interval` the loop also runs the backup sweep -/// ([`MongoEngine::sweep_stale_backup_claims`]), so a claim or restore target -/// left by a process that died is removed within about -/// `stale_after + heartbeat_interval + GSI_BACKFILL_INTERVAL` without a restart -/// of this one. The sweep removes only state whose heartbeat stopped, so it is -/// safe while other processes share the catalog. Returns once `shutdown` -/// fires, at its next pause between cycles. +/// A restore target whose load stopped before its hand-over is not this +/// worker's: its job record keeps a stale heartbeat, and the backup +/// reconciler acts on it, by its kind: a legacy (in-catalog) record is +/// failed and its target removed; a store-backed (`manifest`) record is +/// taken over and its load resumed, and the hand-over comes from the resume. +/// Returns once `shutdown` fires, at its next pause between cycles. pub(crate) async fn gsi_backfill_worker(storage: Arc, shutdown: CancellationToken) { - let mut last_backup_sweep = std::time::Instant::now(); while sleep_or_shutdown(&shutdown, GSI_BACKFILL_INTERVAL).await { - if last_backup_sweep.elapsed() >= storage.backup_liveness.heartbeat_interval { - last_backup_sweep = std::time::Instant::now(); - match storage.sweep_stale_backup_claims().await { - Ok(sweep) if sweep.is_empty() => {} - Ok(sweep) => tracing::warn!("periodic backup sweep: {sweep}"), - Err(e) => tracing::warn!("periodic backup sweep failed: {e}"), - } - } - let indexes_coll = storage.catalog_db.collection::("indexes"); let cursor = match indexes_coll .find(doc! { @@ -300,9 +289,11 @@ async fn restore_backfill_is_ready( } /// Complete restored tables after every restored GSI/LSI has been populated. -/// The restore request records `restore_backfill_pending` only after the base -/// `$out` copy succeeds, so this remains safe to retry after a process restart -/// or a client disconnect during index backfill. +/// `restore_backfill_pending` is set only once every base document is in +/// place: by the legacy copy after its `$out` succeeds, and by a store-backed +/// restore's `finish_restore` after its count check (`hand_over_to_backfill`), +/// so this remains safe to retry after a process restart or a client +/// disconnect during index backfill. async fn finish_ready_restores(storage: &MongoEngine) { let tables_coll = storage.catalog_db.collection::("tables"); let mut cursor = match tables_coll diff --git a/crates/storage-mongodb/tests/fixtures/0006_backup_catalog_index.expected.json b/crates/storage-mongodb/tests/fixtures/0006_backup_catalog_index.expected.json new file mode 100644 index 000000000..b37c62d4b --- /dev/null +++ b/crates/storage-mongodb/tests/fixtures/0006_backup_catalog_index.expected.json @@ -0,0 +1,115 @@ +{ + "backup_index": { + "type": "collection", + "options": {}, + "indexes": { + "_id_": { + "key": [ + [ + "_id", + 1 + ] + ], + "options": {} + }, + "account_id_1_backup_id_1": { + "key": [ + [ + "account_id", + 1 + ], + [ + "backup_id", + 1 + ] + ], + "options": { + "unique": true + } + }, + "backup_id_1": { + "key": [ + [ + "backup_id", + 1 + ] + ], + "options": { + "unique": true + } + }, + "account_id_1_table_name_1_created_at_1": { + "key": [ + [ + "account_id", + 1 + ], + [ + "table_name", + 1 + ], + [ + "created_at", + 1 + ] + ], + "options": {} + }, + "store_kind_1_store_root_1_store_id_1_status_1": { + "key": [ + [ + "store_kind", + 1 + ], + [ + "store_root", + 1 + ], + [ + "store_id", + 1 + ], + [ + "status", + 1 + ] + ], + "options": {} + } + } + }, + "backup_maintenance": { + "type": "collection", + "options": {}, + "indexes": { + "_id_": { + "key": [ + [ + "_id", + 1 + ] + ], + "options": {} + } + } + }, + "tables": { + "type": "collection", + "options": {}, + "indexes": { + "restore_job.status_1_restore_job.heartbeat_at_1": { + "key": [ + [ + "restore_job.status", + 1 + ], + [ + "restore_job.heartbeat_at", + 1 + ] + ], + "options": {} + } + } + } +} diff --git a/crates/storage-mongodb/tests/fixtures/golden/0006_backup_catalog_index.txt b/crates/storage-mongodb/tests/fixtures/golden/0006_backup_catalog_index.txt new file mode 100644 index 000000000..2f9b6bd30 --- /dev/null +++ b/crates/storage-mongodb/tests/fixtures/golden/0006_backup_catalog_index.txt @@ -0,0 +1,38 @@ +{ + "registrations": [ + { + "table": "golden6_done", + "table_status": "ACTIVE", + "status": "DONE", + "kind": "legacy", + "owner_generation": 0, + "heartbeat_at": 0, + "completed_at_creation": true, + "earlier_fields": [] + }, + { + "table": "golden6_handed", + "table_status": "CREATING", + "status": "DONE", + "kind": "legacy", + "owner_generation": 0, + "heartbeat_at": 0, + "completed_at_creation": true, + "earlier_fields": [] + }, + { + "table": "golden6_interrupted", + "table_status": "CREATING", + "status": "RUNNING", + "kind": "legacy", + "owner_generation": 0, + "heartbeat_at": 0, + "completed_at_creation": null, + "earlier_fields": [] + } + ], + "notes": [ + "backfilled 3 restore registration(s) as legacy restores, 1 of them RUNNING with a dead owner for the reconciler to remove", + "wrote settings.catalog_id" + ] +} diff --git a/crates/storage-mongodb/tests/migrations_fresh.rs b/crates/storage-mongodb/tests/migrations_fresh.rs index 12cf1838e..9b5270035 100644 --- a/crates/storage-mongodb/tests/migrations_fresh.rs +++ b/crates/storage-mongodb/tests/migrations_fresh.rs @@ -303,7 +303,10 @@ async fn a_fresh_store_gets_every_migration_and_the_expected_definitions() { let Some(uri) = server() else { return }; let d = Deployment::new(&uri, "fresh").await; let report = d.create().await; - assert_eq!((report.from, report.to), (None, Some(5))); + assert_eq!( + (report.from, report.to), + (None, Some(MigrationId::LATEST.number())) + ); let applied: Vec<(u32, String)> = report .applied .iter() @@ -318,6 +321,8 @@ async fn a_fresh_store_gets_every_migration_and_the_expected_definitions() { (3, d.target.catalog.clone()), (4, d.target.catalog.clone()), (5, d.target.catalog.clone()), + (6, d.target.catalog.clone()), + (7, d.target.catalog.clone()), ] ); let not_applicable: Vec = report @@ -326,8 +331,13 @@ async fn a_fresh_store_gets_every_migration_and_the_expected_definitions() { .filter(|s| s.outcome == Outcome::NotApplicable) .map(|s| s.id) .collect(); - assert_eq!(not_applicable, [3, 4], "{report:?}"); - assert_eq!(d.status().await, CatalogStatus::Current { version: 5 }); + assert_eq!(not_applicable, [3, 4, 7], "{report:?}"); + assert_eq!( + d.status().await, + CatalogStatus::Current { + version: MigrationId::LATEST.number() + } + ); let rows = d.catalog_ledger().await; let outcomes: Vec<(u32, Outcome)> = rows.iter().map(|r| (r.id, r.outcome)).collect(); assert_eq!( @@ -338,6 +348,8 @@ async fn a_fresh_store_gets_every_migration_and_the_expected_definitions() { (3, Outcome::NotApplicable), (4, Outcome::NotApplicable), (5, Outcome::Applied), + (6, Outcome::Applied), + (7, Outcome::NotApplicable), ] ); assert!( @@ -347,7 +359,10 @@ async fn a_fresh_store_gets_every_migration_and_the_expected_definitions() { let data_rows = Deployment::documents(&d.data(), "extenddb_migrations").await; assert_eq!(data_rows.len(), 1); assert_eq!(data_rows[0].get_i64("_id"), Ok(1)); - assert_eq!(d.setting("catalog_version").await.as_deref(), Some("5")); + assert_eq!( + d.setting("catalog_version").await, + Some(MigrationId::LATEST.number().to_string()) + ); assert_eq!(d.setting("resource_tag_migration").await, None); let history = Deployment::documents(&d.catalog(), "schema_history").await; assert_eq!(history.len(), 1); @@ -356,25 +371,55 @@ async fn a_fresh_store_gets_every_migration_and_the_expected_definitions() { let (catalog, data) = definitions(&d).await; if std::env::var("EXTENDDB_BLESS_EXPECTED").as_deref() == Ok("1") { let dir = env!("CARGO_MANIFEST_DIR"); - // Migration 1's file keeps the collection migration 5 drops. + // Migration 1's file keeps the collection migration 5 drops and + // leaves out what migration 6 adds, which its own file holds. let committed: Definitions = serde_json::from_str(CATALOG_EXPECTED).expect("catalog"); + let added: Definitions = + serde_json::from_str(migrations_support::CATALOG_MIGRATION_6).expect("migration 6"); let mut catalog_at_1 = catalog.clone(); + for (name, collection) in &added { + match committed.get(name) { + Some(before) => { + let mut kept = catalog_at_1[name].clone(); + kept.indexes + .retain(|index, _| !collection.indexes.contains_key(index)); + assert_eq!(&kept, before, "{name}: migration 1's definition"); + catalog_at_1.insert(name.clone(), kept); + } + None => { + catalog_at_1.remove(name); + } + } + } if let Some(dropped) = committed.get("continuous_backups") { catalog_at_1.insert("continuous_backups".to_owned(), dropped.clone()); } - for (path, defs) in [("catalog", &catalog_at_1), ("data", &data)] { + let mut at_6 = Definitions::new(); + for (name, collection) in &added { + let mut found = catalog[name].clone(); + if committed.contains_key(name) { + found + .indexes + .retain(|index, _| collection.indexes.contains_key(index)); + } + at_6.insert(name.clone(), found); + } + for (path, defs) in [ + ( + "migrations/catalog/0001_baseline.expected.json", + &catalog_at_1, + ), + ("migrations/data/0001_baseline.expected.json", &data), + ( + "tests/fixtures/0006_backup_catalog_index.expected.json", + &at_6, + ), + ] { let text = serde_json::to_string_pretty(defs).expect("json") + "\n"; - std::fs::write( - format!("{dir}/migrations/{path}/0001_baseline.expected.json"), - text, - ) - .expect("write"); + std::fs::write(format!("{dir}/{path}"), text).expect("write"); } } - let mut expected_catalog: Definitions = - serde_json::from_str(CATALOG_EXPECTED).expect("catalog"); - // Migration 5 drops continuous_backups, which migration 1 creates. - expected_catalog.remove("continuous_backups"); + let expected_catalog = migrations_support::expected_catalog(); let expected_data: Definitions = serde_json::from_str(DATA_EXPECTED).expect("data"); assert_eq!(catalog, expected_catalog); assert_eq!(data, expected_data); @@ -389,12 +434,18 @@ async fn a_second_run_applies_nothing_and_reconverge_keeps_the_rows() { let before = d.catalog_ledger().await; let again = d.migrate().await.expect("migrate"); assert!(again.applied.is_empty()); - assert_eq!((again.from, again.to), (Some(5), Some(5))); + assert_eq!( + (again.from, again.to), + ( + Some(MigrationId::LATEST.number()), + Some(MigrationId::LATEST.number()) + ) + ); let reconverged = d.reconverge().await.expect("reconverge"); - // Migration 1's steps in the catalog and the data database, and migration - // 5's in the catalog: 2 is not convergent, and 3 and 4 do not apply on - // MongoDB. + // Migration 1's steps in the catalog and the data database, and + // migrations 5's and 6's in the catalog: 2 is not convergent, and 3, 4 + // and 7 do not apply on MongoDB. let ran: Vec<(u32, String)> = reconverged .applied .iter() @@ -406,6 +457,7 @@ async fn a_second_run_applies_nothing_and_reconverge_keeps_the_rows() { (1, d.target.catalog.clone()), (1, d.target.data.clone()), (5, d.target.catalog.clone()), + (6, d.target.catalog.clone()), ], "{reconverged:?}" ); @@ -421,7 +473,12 @@ async fn a_second_run_applies_nothing_and_reconverge_keeps_the_rows() { before, "reconverge leaves the rows as they are" ); - assert_eq!(d.status().await, CatalogStatus::Current { version: 5 }); + assert_eq!( + d.status().await, + CatalogStatus::Current { + version: MigrationId::LATEST.number() + } + ); d.drop_databases().await; } @@ -456,7 +513,12 @@ async fn forget_clears_an_unreleased_id_and_migrate_applies_it_again() { assert_eq!(pending.iter().map(|p| p.id).collect::>(), [1, 1]); let report = d.migrate().await.expect("migrate"); assert_eq!(report.applied.len(), 2); - assert_eq!(d.status().await, CatalogStatus::Current { version: 5 }); + assert_eq!( + d.status().await, + CatalogStatus::Current { + version: MigrationId::LATEST.number() + } + ); d.drop_databases().await; } @@ -473,19 +535,19 @@ async fn a_changed_checksum_a_newer_catalog_and_an_older_release_are_reported() doc! { "_id": id, "name": "later", "kind": kind, "checksum": "c", "outcome": "applied", "notes": "", "applied_at": bson::DateTime::now(), "applied_by": "9.9.9" } }; - ledger.insert_one(row(6, "breaking")).await.expect("insert"); + ledger.insert_one(row(8, "breaking")).await.expect("insert"); let status = d.status().await; assert_eq!(status.label(), "AheadIncompatible"); assert_eq!(status.exit_code(), exit::AHEAD_INCOMPATIBLE); ledger - .delete_one(doc! { "_id": 6_i64 }) + .delete_one(doc! { "_id": 8_i64 }) .await .expect("delete"); - ledger.insert_one(row(6, "additive")).await.expect("insert"); + ledger.insert_one(row(8, "additive")).await.expect("insert"); assert_eq!(d.status().await.label(), "AheadCompatible"); assert!(d.migrate().await.expect("migrate").applied.is_empty()); ledger - .delete_one(doc! { "_id": 6_i64 }) + .delete_one(doc! { "_id": 8_i64 }) .await .expect("delete"); @@ -500,7 +562,12 @@ async fn a_changed_checksum_a_newer_catalog_and_an_older_release_are_reported() assert_eq!(refused.exit_code(), exit::ATTENTION); d.reconverge().await.expect("reconverge"); assert_eq!(d.setting("resource_tag_migration").await, None); - assert_eq!(d.status().await, CatalogStatus::Current { version: 5 }); + assert_eq!( + d.status().await, + CatalogStatus::Current { + version: MigrationId::LATEST.number() + } + ); settings .update_one( doc! { "_id": "catalog_version" }, @@ -510,7 +577,10 @@ async fn a_changed_checksum_a_newer_catalog_and_an_older_release_are_reported() .expect("version"); assert_eq!(d.status().await.label(), "WrittenByOlderRelease"); d.reconverge().await.expect("reconverge"); - assert_eq!(d.setting("catalog_version").await.as_deref(), Some("5")); + assert_eq!( + d.setting("catalog_version").await, + Some(MigrationId::LATEST.number().to_string()) + ); // A recorded checksum this binary's body does not have. ledger @@ -558,6 +628,6 @@ async fn a_ledger_document_this_binary_cannot_read_stops_the_status() { .expect_err("an error"); migrations.close().await; assert!(error.to_string().contains("\"sideways\""), "{error}"); - assert_eq!(MigrationId::LATEST.number(), 5); + assert_eq!(MigrationId::LATEST.number(), 7); d.drop_databases().await; } diff --git a/crates/storage-mongodb/tests/migrations_guards.rs b/crates/storage-mongodb/tests/migrations_guards.rs index 793f65361..8d68a404c 100644 --- a/crates/storage-mongodb/tests/migrations_guards.rs +++ b/crates/storage-mongodb/tests/migrations_guards.rs @@ -10,13 +10,12 @@ mod migrations_support; use std::time::Duration; use bson::{Document, doc}; -use extenddb_storage::migrations::{CatalogStatus, MigrationBackend, Runner}; +use extenddb_storage::migrations::{CatalogStatus, MigrationBackend, MigrationId, Runner}; use extenddb_storage_mongodb::migrations::{ Definitions, MigrationLock, MongoMigrations, Timing, describe_store, lock, }; use migrations_support::{Deployment, fixture, server}; -const CATALOG_EXPECTED: &str = include_str!("../migrations/catalog/0001_baseline.expected.json"); const DATA_EXPECTED: &str = include_str!("../migrations/data/0001_baseline.expected.json"); async fn index_names(db: &mongodb::Database, collection: &str) -> Vec { @@ -65,9 +64,7 @@ async fn assert_fresh_definitions(d: &Deployment) { .await .expect("data"); migrations.close().await; - let mut expected_catalog: Definitions = serde_json::from_str(CATALOG_EXPECTED).expect("json"); - // Migration 5 drops continuous_backups, which migration 1 creates. - expected_catalog.remove("continuous_backups"); + let expected_catalog = migrations_support::expected_catalog(); assert!(!catalog.contains_key("continuous_backups")); let expected_data: Definitions = serde_json::from_str(DATA_EXPECTED).expect("json"); for (name, want) in &expected_catalog { @@ -207,7 +204,12 @@ async fn migrate_creates_the_indexes_a_deployment_lacks() { .await .contains(&"table_id_1".to_owned()) ); - assert_eq!(d.status().await, CatalogStatus::Current { version: 5 }); + assert_eq!( + d.status().await, + CatalogStatus::Current { + version: MigrationId::LATEST.number() + } + ); assert_fresh_definitions(&d).await; d.drop_databases().await; } @@ -232,7 +234,9 @@ async fn stop_at_every_request(uri: &str, state: Option<&str>) -> usize { } assert_eq!( d.status().await, - CatalogStatus::Current { version: 5 }, + CatalogStatus::Current { + version: MigrationId::LATEST.number() + }, "{requests}" ); assert_fresh_definitions(&d).await; diff --git a/crates/storage-mongodb/tests/migrations_holders.rs b/crates/storage-mongodb/tests/migrations_holders.rs index ed0cd866c..37f8f520b 100644 --- a/crates/storage-mongodb/tests/migrations_holders.rs +++ b/crates/storage-mongodb/tests/migrations_holders.rs @@ -16,7 +16,7 @@ use bson::{Document, doc}; use extenddb_storage::hooks::ServingRegistrar; use extenddb_storage::management_store::OpError; use extenddb_storage::migrations::{ - CatalogStatus, HasManagementStore, HeldLock, MigrationBackend, MigrationError, + CatalogStatus, HasManagementStore, HeldLock, MigrationBackend, MigrationError, MigrationId, }; use extenddb_storage_mongodb::migrations::{ MigrationLock, MongoMigrations, MongoTarget, ServingRegistration, Timing, lock, @@ -113,7 +113,12 @@ async fn migration_2_keeps_the_lock_through_a_scan_longer_than_the_idle_limit() applied.expect("every migration applies"); // A shorter run would apply without the progress the scan records. assert!(took > timing.idle_limit * 4, "the run took {took:?}"); - assert_eq!(d.status().await, CatalogStatus::Current { version: 5 }); + assert_eq!( + d.status().await, + CatalogStatus::Current { + version: MigrationId::LATEST.number() + } + ); assert!(d.catalog_ledger().await.iter().any(|row| row.id == 2)); d.drop_databases().await; } @@ -151,7 +156,12 @@ async fn the_status_read_under_the_lock_keeps_it_through_a_long_scan() { "the status read took {took:?}" ); applied.expect("every migration applies"); - assert_eq!(d.status().await, CatalogStatus::Current { version: 5 }); + assert_eq!( + d.status().await, + CatalogStatus::Current { + version: MigrationId::LATEST.number() + } + ); d.drop_databases().await; } diff --git a/crates/storage-mongodb/tests/migrations_locks.rs b/crates/storage-mongodb/tests/migrations_locks.rs index 84727523a..64be558be 100644 --- a/crates/storage-mongodb/tests/migrations_locks.rs +++ b/crates/storage-mongodb/tests/migrations_locks.rs @@ -10,10 +10,10 @@ use std::time::{Duration, Instant}; use bson::{Document, doc}; use extenddb_storage::hooks::ServingRegistrar; -use extenddb_storage::migrations::{CatalogStatus, MigrationError, Runner}; +use extenddb_storage::migrations::{CatalogStatus, MigrationError, MigrationId, Runner}; use extenddb_storage_mongodb::migrations::{ MigrationLock, MongoMigrations, ServingRegistration, Timing, lock, migrate_holding, - register_and_check, + register_and_check, register_and_check_as, }; use migrations_support::{Deployment, fixture, server}; @@ -53,10 +53,15 @@ async fn two_migrators_started_together_apply_once_and_the_other_finds_nothing() applied.sort_unstable(); assert_eq!( applied, - [0, 6], + [0, 8], "one applies every step and the other finds nothing" ); - assert_eq!(d.status().await, CatalogStatus::Current { version: 5 }); + assert_eq!( + d.status().await, + CatalogStatus::Current { + version: MigrationId::LATEST.number() + } + ); d.drop_databases().await; } @@ -204,7 +209,12 @@ async fn migrate_waits_for_a_running_server_and_a_server_refuses_while_a_migrati .is_none() ); // The preview and verify read the status beside the server. - assert_eq!(d.status().await, CatalogStatus::Current { version: 5 }); + assert_eq!( + d.status().await, + CatalogStatus::Current { + version: MigrationId::LATEST.number() + } + ); serving.release().await; let (held, _) = lock(&d.target, false, Duration::from_secs(2), Timing::PRODUCTION) .await @@ -229,6 +239,74 @@ async fn migrate_waits_for_a_running_server_and_a_server_refuses_while_a_migrati d.drop_databases().await; } +/// A command that opens the catalog as a server does (`extenddb backup +/// list`) registers under its own name: the document carries it, a +/// migration's wait names it, the name is kept when the registration is +/// taken again, and its refusal while a migration holds the lock names the +/// holder without the server's advice to start the server. +#[tokio::test(flavor = "multi_thread")] +async fn a_command_registers_under_its_own_name() { + let Some(uri) = server() else { return }; + let d = Deployment::new(&uri, "command").await; + d.create().await; + const COMMAND: &str = "extenddb backup list"; + + let registered = register_and_check_as(&d.target, "test", COMMAND, Timing::PRODUCTION) + .await + .expect("registered"); + let documents = Deployment::documents(&d.catalog(), "extenddb_servers").await; + assert_eq!(documents.len(), 1, "{documents:?}"); + assert_eq!(documents[0].get_str("command"), Ok(COMMAND)); + let error = lock(&d.target, false, Duration::from_secs(2), Timing::PRODUCTION) + .await + .err() + .expect("the command's registration is waited for"); + assert_eq!(error.exit_code(), 6); + let message = error.to_string(); + assert!( + message.contains(&format!("{COMMAND}, pid {}", std::process::id())), + "{message}" + ); + let again = registered.register_again().await; + assert!( + matches!(again, extenddb_storage::hooks::Reregistration::Registered), + "{again:?}" + ); + let documents = Deployment::documents(&d.catalog(), "extenddb_servers").await; + assert!( + documents + .iter() + .any(|d| d.get_str("command") == Ok(COMMAND)), + "taken again under the command's name: {documents:?}" + ); + registered.release().await; + + let (held, _) = lock(&d.target, false, Duration::from_secs(2), Timing::PRODUCTION) + .await + .expect("no server runs"); + let refused = register_and_check_as(&d.target, "test", COMMAND, Timing::PRODUCTION) + .await + .err() + .expect("a migration holds the lock"); + assert_eq!(refused.exit_code(), 6); + let why = refused.to_string(); + assert!( + why.contains("holds the migration lock on this catalog"), + "{why}" + ); + assert!(!why.contains("the server does not start"), "{why}"); + let served = register_and_check(&d.target, "test", Timing::PRODUCTION) + .await + .err() + .expect("a server is refused too"); + assert!( + served.to_string().contains("the server does not start"), + "{served}" + ); + held.release().await; + d.drop_databases().await; +} + #[tokio::test(flavor = "multi_thread")] async fn a_server_refuses_a_catalog_it_cannot_serve_and_deletes_its_registration() { let Some(uri) = server() else { return }; diff --git a/crates/storage-mongodb/tests/migrations_released.rs b/crates/storage-mongodb/tests/migrations_released.rs index c636de7a9..d85f0342e 100644 --- a/crates/storage-mongodb/tests/migrations_released.rs +++ b/crates/storage-mongodb/tests/migrations_released.rs @@ -14,12 +14,11 @@ mod migrations_support; use std::collections::BTreeMap; use bson::{Document, doc}; -use extenddb_storage::migrations::{CatalogStatus, MigrationBackend}; +use extenddb_storage::migrations::{CatalogStatus, MigrationBackend, MigrationId}; use extenddb_storage_mongodb::migrations::{Definitions, MongoMigrations, describe_store}; use migrations_support::{Deployment, fixture, server}; use serde_json::Value; -const CATALOG_EXPECTED: &str = include_str!("../migrations/catalog/0001_baseline.expected.json"); const DATA_EXPECTED: &str = include_str!("../migrations/data/0001_baseline.expected.json"); const STATES: [&str; 4] = ["0.1.12", "0.1.13", "0.1.14", "dotted-ttl"]; @@ -136,9 +135,7 @@ fn golden(name: &str, value: &Value) { #[tokio::test(flavor = "multi_thread")] async fn every_released_catalog_upgrades_to_the_definitions_of_a_fresh_install() { let Some(uri) = server() else { return }; - let mut expected_catalog: Definitions = serde_json::from_str(CATALOG_EXPECTED).unwrap(); - // Migration 5 drops continuous_backups, which migration 1 creates. - expected_catalog.remove("continuous_backups"); + let expected_catalog = migrations_support::expected_catalog(); let expected_data: Definitions = serde_json::from_str(DATA_EXPECTED).unwrap(); let fresh = Deployment::new(&uri, "freshkeys").await; fresh.create().await; @@ -164,12 +161,18 @@ async fn every_released_catalog_upgrades_to_the_definitions_of_a_fresh_install() panic!("{state} is not behind"); }; assert_eq!(version, None, "{state} has no ledger"); - assert_eq!(pending.len(), 6, "{state}"); + assert_eq!(pending.len(), 8, "{state}"); let report = d.migrate().await.expect("migrate"); - assert_eq!((report.from, report.to), (None, Some(5)), "{state}"); + assert_eq!( + (report.from, report.to), + (None, Some(MigrationId::LATEST.number())), + "{state}" + ); assert_eq!( d.status().await, - CatalogStatus::Current { version: 5 }, + CatalogStatus::Current { + version: MigrationId::LATEST.number() + }, "{state}" ); @@ -289,6 +292,101 @@ async fn migration_2_rewrites_the_released_policies_as_the_golden_file_records() d.drop_databases().await; } +/// Migration 6's body on the oldest released catalog holding three restore +/// registrations: a finished restore (ACTIVE), one handed to the index +/// backfill worker at the stop (CREATING with `restore_backfill_pending`), +/// and one the stop interrupted (CREATING). Each gets its job record, the +/// first two DONE and completed at the table's creation, the third RUNNING +/// with the epoch heartbeat and generation 0, which the committed golden +/// file pins with the ledger row's notes. A reconverge runs the body again +/// and changes nothing. +#[tokio::test(flavor = "multi_thread")] +async fn migration_6_backfills_the_released_restore_registrations_as_the_golden_file_records() { + let Some(uri) = server() else { return }; + let d = Deployment::new(&uri, "golden6").await; + fixture::load(&d, &fixture::read("0.1.12")).await; + let tables = d.catalog().collection::("tables"); + let created = bson::DateTime::from_millis(1_700_000_000_000); + for (name, status, handed) in [ + ("golden6_done", "ACTIVE", false), + ("golden6_handed", "CREATING", true), + ("golden6_interrupted", "CREATING", false), + ] { + let mut table = doc! { + "_id": { "account_id": "123456789012", "table_name": name }, + "table_id": format!("golden6-{name}"), + "table_status": status, + "creation_date_time": created, + "restore_source_backup_arn": + "arn:aws:dynamodb:us-east-1:123456789012:table/src/backup/01700000000000-0000000a", + "restore_operation_id": "old-operation", + "restore_heartbeat_at": created, + }; + if handed { + table.insert("restore_backfill_pending", true); + } + tables.insert_one(table).await.unwrap(); + } + d.migrate().await.expect("migrate"); + let registrations = || async { + let mut found: Vec = Vec::new(); + for t in Deployment::documents(&d.catalog(), "tables").await { + if !t.contains_key("restore_source_backup_arn") { + continue; + } + let job = t.get_document("restore_job").expect("a job record"); + let earlier: Vec<&str> = ["restore_operation_id", "restore_heartbeat_at"] + .into_iter() + .filter(|f| t.contains_key(*f)) + .collect(); + found.push(serde_json::json!({ + "table": t.get_document("_id").unwrap().get_str("table_name").unwrap(), + "table_status": t.get_str("table_status").unwrap(), + "status": job.get_str("status").unwrap(), + "kind": job.get_str("kind").unwrap(), + "owner_generation": job.get_i64("owner_generation").unwrap(), + "heartbeat_at": job.get_datetime("heartbeat_at").unwrap().timestamp_millis(), + "completed_at_creation": job + .get_datetime("completed_at") + .ok() + .map(|at| Some(at) == t.get_datetime("creation_date_time").ok()), + "earlier_fields": earlier, + })); + } + found.sort_by_key(|v| v["table"].as_str().unwrap().to_owned()); + found + }; + let found = registrations().await; + let catalog_id = d.setting("catalog_id").await.expect("a catalog id"); + assert!(uuid::Uuid::parse_str(&catalog_id).is_ok(), "{catalog_id}"); + let rows = d.catalog_ledger().await; + let row = rows + .iter() + .find(|r| r.id == MigrationId::BackupCatalogIndex.number()) + .expect("migration 6's row"); + let notes: Vec<&str> = row.notes.lines().collect(); + golden( + "0006_backup_catalog_index", + &serde_json::json!({ "registrations": found, "notes": notes }), + ); + + let reconverged = d.reconverge().await.expect("reconverge"); + assert!( + reconverged + .applied + .iter() + .any(|step| step.id == row.id && step.notes.is_empty()), + "{reconverged:?}" + ); + assert_eq!(registrations().await, found); + assert_eq!( + d.setting("catalog_id").await, + Some(catalog_id), + "the catalog id is kept" + ); + d.drop_databases().await; +} + /// Migration 1's bodies on the oldest released catalog: both databases' /// definitions, the settings keys and the legacy ledger, which the /// committed golden file pins. @@ -401,7 +499,12 @@ async fn a_reconverge_leaves_policies_written_after_migration_2_as_they_are() { .any(|n| n.starts_with("Skipped 2 resource_tag_condition_keys:")), "{report:?}" ); - assert_eq!(d.status().await, CatalogStatus::Current { version: 5 }); + assert_eq!( + d.status().await, + CatalogStatus::Current { + version: MigrationId::LATEST.number() + } + ); assert_eq!( documents(Deployment::documents(&d.catalog(), "iam_policies").await), before diff --git a/crates/storage-mongodb/tests/migrations_support/mod.rs b/crates/storage-mongodb/tests/migrations_support/mod.rs index 06bddd927..24e970bd0 100644 --- a/crates/storage-mongodb/tests/migrations_support/mod.rs +++ b/crates/storage-mongodb/tests/migrations_support/mod.rs @@ -22,10 +22,44 @@ use extenddb_storage::migrations::{ CatalogStatus, LedgerRow, MigrationError, MigrationId, RunReport, Runner, }; use extenddb_storage_mongodb::migrations::{ - MigrationLock, MongoMigrations, MongoTarget, Timing, catalog_status, lock, migrate_holding, + Definitions, MigrationLock, MongoMigrations, MongoTarget, Timing, catalog_status, lock, + migrate_holding, }; use futures::TryStreamExt; +/// The definitions migration 1 creates in the catalog database. +pub const CATALOG_BASELINE: &str = + include_str!("../../migrations/catalog/0001_baseline.expected.json"); + +/// The collections and indexes migration 6 adds to the catalog database: +/// whole collections, and the indexes it adds to a collection migration 1 +/// creates (`tables`). +pub const CATALOG_MIGRATION_6: &str = + include_str!("../fixtures/0006_backup_catalog_index.expected.json"); + +/// The catalog database's definitions after every migration: migration 1's, +/// without `continuous_backups`, which migration 5 drops, and with what +/// migration 6 adds. +/// +/// # Panics +/// +/// A committed file is not a definitions file. +#[must_use] +pub fn expected_catalog() -> Definitions { + let mut expected: Definitions = serde_json::from_str(CATALOG_BASELINE).expect("migration 1"); + expected.remove("continuous_backups"); + let added: Definitions = serde_json::from_str(CATALOG_MIGRATION_6).expect("migration 6"); + for (name, collection) in added { + match expected.get_mut(&name) { + Some(existing) => existing.indexes.extend(collection.indexes), + None => { + expected.insert(name, collection); + } + } + } + expected +} + /// The server the tests run against, or `None` to skip. /// /// # Panics diff --git a/crates/storage-postgres/Cargo.toml b/crates/storage-postgres/Cargo.toml index 81b205617..b2c47b590 100755 --- a/crates/storage-postgres/Cargo.toml +++ b/crates/storage-postgres/Cargo.toml @@ -26,6 +26,11 @@ extenddb-storage = { workspace = true, features = ["test-util"] } # The crate's own tests need `test-hooks` and `test-util`; a dev-dependency on # itself turns both features on for test builds only. extenddb-storage-postgres = { path = ".", features = ["test-hooks", "test-util"] } +# The store identity tests run the engine's checks over two engines on one +# scratch catalog and two filesystem stores. +extenddb-engine = { workspace = true } +extenddb-backup-store = { workspace = true } +tempfile = "3" [dependencies] anyhow = { workspace = true } diff --git a/crates/storage-postgres/data_migrations/0007_restore_progress.sql b/crates/storage-postgres/data_migrations/0007_restore_progress.sql new file mode 100644 index 000000000..417530889 --- /dev/null +++ b/crates/storage-postgres/data_migrations/0007_restore_progress.sql @@ -0,0 +1,18 @@ +-- Copyright 2026 ExtendDB contributors +-- SPDX-License-Identifier: Apache-2.0 +-- +-- Migration 7, restore_progress, PostgreSQL data database. +-- +-- The progress record of a store-backed restore: one row per data file the +-- restore applied into its CREATING target, written in the data transaction +-- that wrote the file's rows, so a file is recorded exactly when its rows +-- are durable. The catalog's `table_restores` row names the target; this +-- table lives beside the target's rows because the two databases cannot +-- share a transaction. The restore's end deletes the target's rows, and the +-- reconciler removes rows of targets no RUNNING restore owns. +CREATE TABLE IF NOT EXISTS restore_progress ( + table_id TEXT NOT NULL, + file_index BIGINT NOT NULL, + items_applied BIGINT NOT NULL, + PRIMARY KEY (table_id, file_index) +); diff --git a/crates/storage-postgres/migrations/0006_backup_catalog_index.sql b/crates/storage-postgres/migrations/0006_backup_catalog_index.sql new file mode 100644 index 000000000..1e31c842e --- /dev/null +++ b/crates/storage-postgres/migrations/0006_backup_catalog_index.sql @@ -0,0 +1,98 @@ +-- Copyright 2026 ExtendDB contributors +-- SPDX-License-Identifier: Apache-2.0 +-- +-- Migration 6, backup_catalog_index, PostgreSQL catalog database: the catalog +-- index of backups kept in a backup store, the restore job fields on the +-- restore registrations, the maintenance claim record and the catalog id. +-- Every statement is guarded, and the backfill touches only rows the new +-- columns are still empty on, so a second run changes nothing. + +-- One row per backup in a backup store: the fast index for DescribeBackup +-- and ListBackups, and the state machine record of its job. Keyed by +-- (account_id, backup_id); the backup id is unique on its own. The backups +-- earlier releases wrote into the catalog stay in `backups`, which the +-- legacy reader alone consults. `backup_arn` is `COLLATE "C"` so the +-- listings' `ORDER BY created_at, backup_arn` is byte order, the order the +-- engine merges and pages by, whatever the database's collation. +CREATE TABLE IF NOT EXISTS backup_index ( + account_id TEXT NOT NULL, + backup_id TEXT NOT NULL, + table_name TEXT NOT NULL, + backup_arn TEXT COLLATE "C" NOT NULL, + backup_name TEXT NOT NULL, + status TEXT NOT NULL, + store_kind TEXT NOT NULL, + store_root TEXT NOT NULL, + store_id TEXT NOT NULL, + size_bytes BIGINT NOT NULL DEFAULT 0, + item_count BIGINT NOT NULL DEFAULT 0, + created_at TIMESTAMPTZ NOT NULL, + completed_at TIMESTAMPTZ, + failure_reason TEXT, + source_table_details_json JSONB NOT NULL, + owner_generation BIGINT NOT NULL, + heartbeat_at TIMESTAMPTZ NOT NULL, + PRIMARY KEY (account_id, backup_id) +); + +CREATE UNIQUE INDEX IF NOT EXISTS idx_backup_index_backup_id ON backup_index (backup_id); +CREATE INDEX IF NOT EXISTS idx_backup_index_account_table ON backup_index (account_id, table_name); +CREATE INDEX IF NOT EXISTS idx_backup_index_store_status + ON backup_index (store_kind, store_root, store_id, status); + +-- The restore registration becomes the restore job record: its status +-- (RUNNING, FAILED, DONE), how it reads its backup (legacy or manifest), the +-- store a manifest restore reads, the job claim (owner_generation, +-- heartbeat_at), the spool host, per-batch progress, and how it ended. +ALTER TABLE table_restores ADD COLUMN IF NOT EXISTS status TEXT; +ALTER TABLE table_restores ADD COLUMN IF NOT EXISTS kind TEXT; +ALTER TABLE table_restores ADD COLUMN IF NOT EXISTS store_kind TEXT; +ALTER TABLE table_restores ADD COLUMN IF NOT EXISTS store_root TEXT; +ALTER TABLE table_restores ADD COLUMN IF NOT EXISTS store_id TEXT; +ALTER TABLE table_restores ADD COLUMN IF NOT EXISTS owner_generation BIGINT; +ALTER TABLE table_restores ADD COLUMN IF NOT EXISTS heartbeat_at TIMESTAMPTZ; +ALTER TABLE table_restores ADD COLUMN IF NOT EXISTS spool_host TEXT; +ALTER TABLE table_restores ADD COLUMN IF NOT EXISTS file_index BIGINT; +ALTER TABLE table_restores ADD COLUMN IF NOT EXISTS items_applied BIGINT; +ALTER TABLE table_restores ADD COLUMN IF NOT EXISTS failure_reason TEXT; +ALTER TABLE table_restores ADD COLUMN IF NOT EXISTS completed_at TIMESTAMPTZ; + +-- The registrations that exist were written by a release without the job +-- record: every one is a legacy restore. One whose table is not CREATING +-- finished (DONE). One whose table is CREATING is a restore the upgrade's +-- stop interrupted: RUNNING, with a heartbeat at the epoch (not NULL, so the +-- staleness test selects it) and an owner generation no job holds, so the +-- first reconciler pass fails it and removes its target. +UPDATE table_restores r +SET kind = 'legacy', + owner_generation = 0, + heartbeat_at = '1970-01-01T00:00:00Z', + status = CASE WHEN t.table_status = 'CREATING' THEN 'RUNNING' ELSE 'DONE' END, + completed_at = CASE WHEN t.table_status = 'CREATING' THEN NULL ELSE t.creation_date_time END +FROM tables t +WHERE t.table_id = r.table_id AND r.status IS NULL; + +ALTER TABLE table_restores ALTER COLUMN status SET NOT NULL; +ALTER TABLE table_restores ALTER COLUMN kind SET NOT NULL; +ALTER TABLE table_restores ALTER COLUMN owner_generation SET NOT NULL; +ALTER TABLE table_restores ALTER COLUMN heartbeat_at SET NOT NULL; + +CREATE INDEX IF NOT EXISTS idx_table_restores_status ON table_restores (status, heartbeat_at); + +-- The catalog-level maintenance claim: at most one row, held by the +-- maintenance command that changes rows or bytes jobs may be using. Empty +-- until a command takes it. +CREATE TABLE IF NOT EXISTS backup_maintenance ( + singleton BOOLEAN PRIMARY KEY DEFAULT TRUE CHECK (singleton), + holder TEXT NOT NULL, + owner_generation BIGINT NOT NULL, + claimed_at TIMESTAMPTZ NOT NULL DEFAULT now(), + heartbeat_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +-- The catalog's random id, which a backup store's identity file names so a +-- store can tell which catalog owns it. Written once, here, for a new and +-- an existing catalog alike. +INSERT INTO settings (key, value) +SELECT 'catalog_id', gen_random_uuid()::text +WHERE NOT EXISTS (SELECT 1 FROM settings WHERE key = 'catalog_id'); diff --git a/crates/storage-postgres/src/backup_catalog.rs b/crates/storage-postgres/src/backup_catalog.rs new file mode 100644 index 000000000..83453dff2 --- /dev/null +++ b/crates/storage-postgres/src/backup_catalog.rs @@ -0,0 +1,1428 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The catalog index on `PostgreSQL`: `backup_index`, the restore job fields +//! of `table_restores`, and `backup_maintenance`. The data-plane side of a +//! store-backed restore is `restore_engine.rs`. +//! +//! The claim of a backup job is `pg_try_advisory_lock(hashtextextended( +//! 'backup:' || backup_id, RESTORE_LOCK_SEED))` on a dedicated, non-pooled +//! connection to the catalog database (a [`ClaimSession`]), taken before the +//! row is inserted and held for the job's life; the backup job's heartbeats +//! and status writes go through that connection and carry the claim's +//! `owner_generation`, so a session the server dropped surfaces as a failed +//! heartbeat and a row a successor took over matches no write (the legacy +//! restore heartbeats through its session too, and makes its DONE and FAILED +//! writes on pooled connections, each conditioned on RUNNING and the +//! generation). The lock on a row for DeleteBackup and a restore's +//! registration is `FOR UPDATE` on the `backup_index` row. Staleness is +//! measured on the catalog's clock. +//! +//! The maintenance claim is the same mechanism on the `backup_maintenance` +//! row: an advisory lock (`backup_maintenance` in the same key space) on a +//! dedicated connection is the proof of a live holder, so a second command +//! finds it taken and a holder whose session ended has released it; the row +//! carries the holder, the generation and the heartbeat. +//! `check_maintenance_claim` reports the claim held while a session holds +//! the lock (a transaction-scoped shared try-lock in one autocommit +//! statement fails while the exclusive lock is held, and holds nothing past +//! its statement, so a request dropped mid-check leaves no lock on the +//! pool), whatever the row's heartbeat says, and otherwise while the row's +//! heartbeat is younger than the stale window, so a dead holder's claim +//! expires for the jobs that read it without anyone taking it over. `release` deletes the row and ends the session. +//! +//! A conditional write whose error is of the connection class (an I/O or TLS +//! error, a pool or worker failure, SQLSTATE class 08, or 57P01 to 57P03) +//! has an unknown outcome (`WriteOutcome::Unknown`: the statement may have +//! reached the server); a statement error, a `statement_timeout` included, +//! is `Err`. The heartbeat period is +//! [`extenddb_storage::backup_catalog::JOB_HEARTBEAT_INTERVAL`] (15 s), the +//! constant every backend runs at until `[backup] heartbeat_interval_secs` +//! reaches the jobs in a later release. + +use std::time::Duration; + +use extenddb_core::backup_format::BackupArn; +use extenddb_core::types::{CreateTableInput, RestoreSummary}; +use extenddb_storage::backup_catalog::{ + BackupCatalog, BackupKey, BackupRow, BackupStatus, DeleteStart, ForgetRow, Heartbeat, JobClaim, + JobRow, JobStatus, MaintenanceClaim, RestoreJobRow, RestoreKind, RestoreRegistered, + RestoreRegistrationRow, RestoreTargetKey, StaleClock, StatusFields, StoreIdentity, StoreKind, + TimeBounds, UNOWNED_GENERATION, WriteOutcome, new_generation, +}; +use extenddb_storage::error::StorageError; +use futures::future::BoxFuture; +use futures::stream::BoxStream; +use sqlx::PgExecutor; +use tokio::sync::Mutex; + +use crate::PostgresEngine; +use crate::backup_engine::{ + ClaimSession, RESTORE_HEARTBEAT, backup_lock_key, generation_column, restore_lock_key, + try_claim, try_claim_with, +}; +use crate::create_table::RestoreSource; + +/// The name of a CREATING restore target registered from `backup_arn`, if +/// any: the one in-use rule DeleteBackup, a restore's registration and +/// `backup_in_use` share. Reads the target's status, not the job's. +pub(crate) async fn creating_restore_target<'e, E: PgExecutor<'e>>( + executor: E, + backup_arn: &str, +) -> Result, StorageError> { + sqlx::query_scalar( + "SELECT t.table_name FROM table_restores r JOIN tables t ON t.table_id = r.table_id \ + WHERE r.source_backup_arn = $1 AND t.table_status = 'CREATING' LIMIT 1", + ) + .bind(backup_arn) + .fetch_optional(executor) + .await + .map_err(db_err) +} + +/// The error for a backup a CREATING restore target still reads: the shared +/// text, [`extenddb_storage::backup_catalog::backup_in_use`]. +pub(crate) fn backup_in_use(table: &str, backup_arn: &str) -> StorageError { + extenddb_storage::backup_catalog::backup_in_use(table, backup_arn) +} + +fn db_err(e: sqlx::Error) -> StorageError { + StorageError::Internal(format!("Database error: {e}")) +} + +fn not_found(backup_arn: &str) -> StorageError { + extenddb_storage::backup_catalog::backup_not_found(backup_arn) +} + +/// The columns of one `backup_index` row, in the order every read selects +/// them. +const ROW_COLUMNS: &str = "backup_id, account_id, table_name, backup_arn, backup_name, status, \ + store_kind, store_root, store_id, size_bytes, item_count, \ + EXTRACT(EPOCH FROM created_at)::FLOAT8 AS created_at, \ + EXTRACT(EPOCH FROM completed_at)::FLOAT8 AS completed_at, \ + failure_reason, source_table_details_json, owner_generation, \ + EXTRACT(EPOCH FROM heartbeat_at)::FLOAT8 AS heartbeat_at"; + +/// One `backup_index` row as the driver reads it. +#[derive(sqlx::FromRow)] +struct RowTuple { + backup_id: String, + account_id: String, + table_name: String, + backup_arn: String, + backup_name: String, + status: String, + store_kind: String, + store_root: String, + store_id: String, + size_bytes: i64, + item_count: i64, + created_at: f64, + completed_at: Option, + failure_reason: Option, + source_table_details_json: serde_json::Value, + owner_generation: i64, + heartbeat_at: f64, +} + +fn row_from_tuple(t: RowTuple) -> Result { + let RowTuple { + backup_id, + account_id, + table_name, + backup_arn, + backup_name, + status, + store_kind, + store_root, + store_id, + size_bytes, + item_count, + created_at, + completed_at, + failure_reason, + source_table_details_json, + owner_generation, + heartbeat_at, + } = t; + let store_kind = StoreKind::parse(&store_kind).ok_or_else(|| { + StorageError::Internal(format!( + "backup {backup_arn} carries an unknown store kind {store_kind:?}" + )) + })?; + Ok(BackupRow { + backup_id, + account_id, + table_name, + backup_arn, + backup_name, + status: BackupStatus::parse(&status)?, + store: StoreIdentity { + store_kind, + store_root, + store_id, + }, + size_bytes, + item_count, + created_at, + completed_at, + failure_reason, + source_table_details_json, + owner_generation: u64::try_from(owner_generation).unwrap_or_default(), + heartbeat_at, + }) +} + +/// The rows `read` fetches, as a stream: the read runs once, on the first +/// poll, and a failed read is one `Err` item. +fn rows_stream<'a, F>(read: F) -> BoxStream<'a, Result> +where + F: std::future::Future, StorageError>> + Send + 'a, +{ + use futures::StreamExt; + Box::pin( + futures::stream::once(async move { + match read.await { + Ok(rows) => rows.into_iter().map(row_from_tuple).collect::>(), + Err(e) => vec![Err(e)], + } + }) + .flat_map(futures::stream::iter), + ) +} + +/// A backup job's handle on its `backup_index` row. +struct PostgresBackupClaim { + session: Mutex, + generation: u64, + key: BackupKey, + pool: sqlx::PgPool, +} + +/// The heartbeat of a backup job: refresh its row while it carries this +/// generation. +const BACKUP_HEARTBEAT: &str = "UPDATE backup_index SET heartbeat_at = now() \ + WHERE account_id = $1 AND backup_id = $2 AND owner_generation = $3"; + +/// Whether a write's outcome cannot be told from its error: the statement +/// may have reached the server before the connection failed. +fn outcome_unknown(e: &sqlx::Error) -> bool { + match e { + sqlx::Error::Io(_) + | sqlx::Error::Tls(_) + | sqlx::Error::PoolTimedOut + | sqlx::Error::PoolClosed + | sqlx::Error::WorkerCrashed => true, + sqlx::Error::Database(d) => d.code().is_some_and(|code| { + code.starts_with("08") || matches!(code.as_ref(), "57P01" | "57P02" | "57P03") + }), + _ => false, + } +} + +impl JobClaim for PostgresBackupClaim { + fn generation(&self) -> u64 { + self.generation + } + + fn heartbeat(&self) -> BoxFuture<'_, Result> { + Box::pin(async move { + let mut session = self.session.lock().await; + let confirmed = session + .confirm( + BACKUP_HEARTBEAT, + &[&self.key.account_id, &self.key.backup_id], + ) + .await + .map_err(|e| StorageError::Internal(format!("claim session: {e}")))?; + Ok(if confirmed { + Heartbeat::Confirmed + } else { + Heartbeat::Lost + }) + }) + } + + fn set_status<'a>( + &'a self, + from: JobStatus, + to: JobStatus, + fields: &'a StatusFields, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + let mut session = self.session.lock().await; + let written = sqlx::query( + "UPDATE backup_index SET status = $4, \ + size_bytes = COALESCE($5, size_bytes), \ + item_count = COALESCE($6, item_count), \ + failure_reason = COALESCE($7, failure_reason), \ + completed_at = CASE WHEN $8 THEN now() ELSE completed_at END, \ + source_table_details_json = COALESCE($10, source_table_details_json) \ + WHERE account_id = $1 AND backup_id = $2 AND owner_generation = $3 \ + AND status = $9", + ) + .bind(&self.key.account_id) + .bind(&self.key.backup_id) + .bind(generation_column(self.generation)) + .bind(to.as_str()) + .bind(fields.size_bytes) + .bind(fields.item_count) + .bind(fields.failure_reason.as_deref()) + .bind(fields.completed) + .bind(from.as_str()) + .bind(fields.source_table_details_json.as_ref()) + .execute(session.connection()) + .await; + match written { + Ok(result) if result.rows_affected() == 1 => Ok(WriteOutcome::Matched), + Ok(_) => Ok(WriteOutcome::NotMatched), + Err(e) if outcome_unknown(&e) => Ok(WriteOutcome::Unknown), + Err(e) => Err(db_err(e)), + } + }) + } + + fn read(&self) -> BoxFuture<'_, Result> { + Box::pin(async move { + let row: Option<(String, i64, f64, Option)> = sqlx::query_as( + "SELECT status, owner_generation, EXTRACT(EPOCH FROM heartbeat_at)::FLOAT8, \ + failure_reason FROM backup_index WHERE account_id = $1 AND backup_id = $2", + ) + .bind(&self.key.account_id) + .bind(&self.key.backup_id) + .fetch_optional(&self.pool) + .await + .map_err(db_err)?; + let (status, generation, heartbeat_at, failure_reason) = + row.ok_or_else(|| not_found(&self.key.backup_arn))?; + Ok(JobRow { + status: JobStatus::parse(&status)?, + owner_generation: u64::try_from(generation).unwrap_or_default(), + heartbeat_at, + failure_reason, + }) + }) + } +} + +/// A store-backed restore job's handle on its `table_restores` record: the +/// advisory lock on the target name (`restore_lock_key`) on the claim +/// session, the generation, and the pool for reads. The heartbeat and the +/// status writes go through the session, so a session the server dropped +/// surfaces as a lost heartbeat or an unknown write outcome. +struct PostgresRestoreClaim { + session: Mutex, + generation: u64, + table_id: String, + pool: sqlx::PgPool, +} + +impl JobClaim for PostgresRestoreClaim { + fn generation(&self) -> u64 { + self.generation + } + + fn heartbeat(&self) -> BoxFuture<'_, Result> { + Box::pin(async move { + let mut session = self.session.lock().await; + let confirmed = session + .confirm(RESTORE_HEARTBEAT, &[&self.table_id]) + .await + .map_err(|e| StorageError::Internal(format!("claim session: {e}")))?; + Ok(if confirmed { + Heartbeat::Confirmed + } else { + Heartbeat::Lost + }) + }) + } + + fn set_status<'a>( + &'a self, + from: JobStatus, + to: JobStatus, + fields: &'a StatusFields, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + let mut session = self.session.lock().await; + let written = sqlx::query( + "UPDATE table_restores SET status = $3, \ + failure_reason = COALESCE($4, failure_reason), \ + completed_at = CASE WHEN $5 THEN now() ELSE completed_at END \ + WHERE table_id = $1 AND owner_generation = $2 AND status = $6", + ) + .bind(&self.table_id) + .bind(generation_column(self.generation)) + .bind(to.as_str()) + .bind(fields.failure_reason.as_deref()) + .bind(fields.completed) + .bind(from.as_str()) + .execute(session.connection()) + .await; + match written { + Ok(result) if result.rows_affected() == 1 => Ok(WriteOutcome::Matched), + Ok(_) => Ok(WriteOutcome::NotMatched), + Err(e) if outcome_unknown(&e) => Ok(WriteOutcome::Unknown), + Err(e) => Err(db_err(e)), + } + }) + } + + fn read(&self) -> BoxFuture<'_, Result> { + Box::pin(async move { + let row: Option<(String, i64, f64, Option)> = sqlx::query_as( + "SELECT status, owner_generation, EXTRACT(EPOCH FROM heartbeat_at)::FLOAT8, \ + failure_reason FROM table_restores WHERE table_id = $1", + ) + .bind(&self.table_id) + .fetch_optional(&self.pool) + .await + .map_err(db_err)?; + let (status, generation, heartbeat_at, failure_reason) = row.ok_or_else(|| { + StorageError::TableNotFound(format!( + "restore target {} has no job record", + self.table_id + )) + })?; + Ok(JobRow { + status: JobStatus::parse(&status)?, + owner_generation: u64::try_from(generation).unwrap_or_default(), + heartbeat_at, + failure_reason, + }) + }) + } +} + +/// The columns of one restore job record, joined with its table, in the +/// order [`RestoreJobTuple`] reads them. +const RESTORE_ROW_COLUMNS: &str = "r.table_id, t.account_id, t.table_name, r.source_backup_arn, \ + r.kind, r.status, r.owner_generation, EXTRACT(EPOCH FROM r.heartbeat_at)::FLOAT8, \ + r.store_kind, r.store_root, r.store_id, r.spool_host, r.file_index, r.items_applied, \ + r.failure_reason, EXTRACT(EPOCH FROM r.completed_at)::FLOAT8"; + +/// One restore job record as the driver reads it. +#[allow(clippy::type_complexity)] +type RestoreJobTuple = ( + String, + String, + String, + String, + String, + String, + i64, + f64, + Option, + Option, + Option, + Option, + Option, + Option, + Option, + Option, +); + +fn restore_job_from_tuple(t: RestoreJobTuple) -> Result { + let ( + table_id, + account_id, + table_name, + source_backup_arn, + kind, + status, + owner_generation, + heartbeat_at, + store_kind, + store_root, + store_id, + spool_host, + file_index, + items_applied, + failure_reason, + completed_at, + ) = t; + let store = match (store_kind, store_root, store_id) { + (Some(kind), Some(store_root), Some(store_id)) => Some(StoreIdentity { + store_kind: StoreKind::parse(&kind).ok_or_else(|| { + StorageError::Internal(format!( + "restore of {source_backup_arn} carries an unknown store kind {kind:?}" + )) + })?, + store_root, + store_id, + }), + _ => None, + }; + Ok(RestoreJobRow { + target: RestoreTargetKey { + account_id, + table_name, + table_id, + }, + source_backup_arn, + kind: RestoreKind::parse(&kind)?, + status: JobStatus::parse(&status)?, + owner_generation: u64::try_from(owner_generation).unwrap_or_default(), + heartbeat_at, + store, + spool_host, + file_index: file_index.and_then(|i| u32::try_from(i).ok()), + items_applied: items_applied.and_then(|i| u64::try_from(i).ok()), + failure_reason, + completed_at, + }) +} + +impl BackupCatalog for PostgresEngine { + fn catalog_id(&self) -> BoxFuture<'_, Result> { + Box::pin(async move { + let id: Option = + sqlx::query_scalar("SELECT value FROM settings WHERE key = 'catalog_id'") + .fetch_optional(&self.pool) + .await + .map_err(db_err)?; + id.ok_or_else(|| { + StorageError::Internal( + "the catalog has no catalog_id; run extenddb migrate --yes".to_owned(), + ) + }) + }) + } + + fn claim_backup<'a>( + &'a self, + row: &'a BackupRow, + ) -> BoxFuture<'a, Result, StorageError>> { + Box::pin(async move { + let Some(mut session) = + try_claim(&self.pool, &backup_lock_key(&row.backup_id), None).await? + else { + return Err(StorageError::AlreadyExists(format!( + "a live job owns backup {}", + row.backup_arn + ))); + }; + let inserted = sqlx::query( + "INSERT INTO backup_index (backup_id, account_id, table_name, backup_arn, \ + backup_name, status, store_kind, store_root, store_id, size_bytes, item_count, \ + created_at, source_table_details_json, owner_generation, heartbeat_at) \ + VALUES ($1, $2, $3, $4, $5, 'CREATING', $6, $7, $8, $9, $10, to_timestamp($11), \ + $12, $13, now()) ON CONFLICT DO NOTHING", + ) + .bind(&row.backup_id) + .bind(&row.account_id) + .bind(&row.table_name) + .bind(&row.backup_arn) + .bind(&row.backup_name) + .bind(row.store.store_kind.as_str()) + .bind(&row.store.store_root) + .bind(&row.store.store_id) + .bind(row.size_bytes) + .bind(row.item_count) + .bind(row.created_at) + .bind(&row.source_table_details_json) + .bind(generation_column(session.generation())) + .execute(session.connection()) + .await + .map_err(db_err)? + .rows_affected(); + if inserted == 0 { + return Err(StorageError::AlreadyExists(format!( + "backup {} already has a row", + row.backup_arn + ))); + } + let generation = session.generation(); + Ok(Box::new(PostgresBackupClaim { + session: Mutex::new(session), + generation, + key: row.key(), + pool: self.pool.clone(), + }) as Box) + }) + } + + fn insert_backup_row<'a>( + &'a self, + row: &'a BackupRow, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + // No conflict target: the primary key (`account_id`, + // `backup_id`) and the unique `backup_id` both make it a no-op. + // No advisory lock: a job that holds the id's lock and has not + // inserted its row yet finds this one and is refused, as it is by + // any row with the key. + let inserted = sqlx::query( + "INSERT INTO backup_index (backup_id, account_id, table_name, backup_arn, \ + backup_name, status, store_kind, store_root, store_id, size_bytes, item_count, \ + created_at, completed_at, failure_reason, source_table_details_json, \ + owner_generation, heartbeat_at) \ + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, to_timestamp($12), now(), \ + $13, $14, $15, now()) ON CONFLICT DO NOTHING", + ) + .bind(&row.backup_id) + .bind(&row.account_id) + .bind(&row.table_name) + .bind(&row.backup_arn) + .bind(&row.backup_name) + .bind(row.status.as_str()) + .bind(row.store.store_kind.as_str()) + .bind(&row.store.store_root) + .bind(&row.store.store_id) + .bind(row.size_bytes) + .bind(row.item_count) + .bind(row.created_at) + .bind(&row.failure_reason) + .bind(&row.source_table_details_json) + .bind(generation_column(UNOWNED_GENERATION)) + .execute(&self.pool) + .await + .map_err(db_err)? + .rows_affected(); + Ok(inserted == 1) + }) + } + + fn list_backup_rows<'a>( + &'a self, + status: Option, + store: &'a StoreIdentity, + ) -> BoxStream<'a, Result> { + rows_stream(async move { + sqlx::query_as(&format!( + "SELECT {ROW_COLUMNS} FROM backup_index \ + WHERE store_kind = $1 AND store_root = $2 AND store_id = $3 \ + AND ($4::TEXT IS NULL OR status = $4) ORDER BY created_at, backup_arn" + )) + .bind(store.store_kind.as_str()) + .bind(&store.store_root) + .bind(&store.store_id) + .bind(status.map(BackupStatus::as_str)) + .fetch_all(&self.pool) + .await + .map_err(db_err) + }) + } + + fn newest_available_backup_row<'a>( + &'a self, + store: &'a StoreIdentity, + ) -> BoxFuture<'a, Result, StorageError>> { + Box::pin(async move { + let row: Option = sqlx::query_as(&format!( + "SELECT {ROW_COLUMNS} FROM backup_index \ + WHERE store_kind = $1 AND store_root = $2 AND store_id = $3 \ + AND status = 'AVAILABLE' ORDER BY created_at DESC, backup_arn DESC LIMIT 1" + )) + .bind(store.store_kind.as_str()) + .bind(&store.store_root) + .bind(&store.store_id) + .fetch_optional(&self.pool) + .await + .map_err(db_err)?; + row.map(row_from_tuple).transpose() + }) + } + + fn list_account_backup_rows<'a>( + &'a self, + account_id: &'a str, + table_name: Option<&'a str>, + bounds: &'a TimeBounds, + ) -> BoxStream<'a, Result> { + rows_stream(async move { + sqlx::query_as(&format!( + "SELECT {ROW_COLUMNS} FROM backup_index WHERE account_id = $1 \ + AND ($2::TEXT IS NULL OR table_name = $2) \ + AND ($3::FLOAT8 IS NULL OR created_at >= to_timestamp($3)) \ + AND ($4::FLOAT8 IS NULL OR created_at < to_timestamp($4)) \ + ORDER BY created_at, backup_arn" + )) + .bind(account_id) + .bind(table_name) + .bind(bounds.lower) + .bind(bounds.upper) + .fetch_all(&self.pool) + .await + .map_err(db_err) + }) + } + + fn read_backup_row<'a>( + &'a self, + arn: &'a BackupArn, + ) -> BoxFuture<'a, Result, StorageError>> { + Box::pin(async move { + let row: Option = sqlx::query_as(&format!( + "SELECT {ROW_COLUMNS} FROM backup_index WHERE account_id = $1 AND backup_id = $2 \ + AND table_name = $3 AND backup_arn = $4" + )) + .bind(arn.account_id()) + .bind(arn.backup_id()) + .bind(arn.table_name()) + .bind(arn.to_arn_string()) + .fetch_optional(&self.pool) + .await + .map_err(db_err)?; + row.map(row_from_tuple).transpose() + }) + } + + fn start_backup_delete<'a>( + &'a self, + key: &'a BackupKey, + ) -> BoxFuture<'a, Result> { + Box::pin(self.start_backup_delete_tx(key, false)) + } + + fn forget_available_backup_row<'a>( + &'a self, + key: &'a BackupKey, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + Ok(match self.start_backup_delete_tx(key, true).await? { + DeleteStart::Deleting => ForgetRow::Removed, + DeleteStart::InUse(table) => ForgetRow::InUse(table), + DeleteStart::Status(status) => ForgetRow::Status(status), + }) + }) + } + + fn set_backup_deleted<'a>( + &'a self, + key: &'a BackupKey, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + let affected = sqlx::query( + "UPDATE backup_index SET status = 'DELETED', completed_at = now() \ + WHERE account_id = $1 AND backup_id = $2 AND status = 'DELETING'", + ) + .bind(&key.account_id) + .bind(&key.backup_id) + .execute(&self.pool) + .await + .map_err(db_err)? + .rows_affected(); + Ok(affected == 1) + }) + } + + fn remove_backup_row<'a>( + &'a self, + key: &'a BackupKey, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + let affected = sqlx::query( + "DELETE FROM backup_index WHERE account_id = $1 AND backup_id = $2 \ + AND status IN ('DELETED', 'FAILED')", + ) + .bind(&key.account_id) + .bind(&key.backup_id) + .execute(&self.pool) + .await + .map_err(db_err)? + .rows_affected(); + Ok(affected == 1) + }) + } + + fn remove_dead_creating_backup_row<'a>( + &'a self, + key: &'a BackupKey, + stale_after: Duration, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + let affected = sqlx::query( + "DELETE FROM backup_index WHERE account_id = $1 AND backup_id = $2 \ + AND status = 'CREATING' \ + AND heartbeat_at < now() - make_interval(secs => $3)", + ) + .bind(&key.account_id) + .bind(&key.backup_id) + .bind(stale_after.as_secs_f64()) + .execute(&self.pool) + .await + .map_err(db_err)? + .rows_affected(); + Ok(affected == 1) + }) + } + + fn claim_dead_backup<'a>( + &'a self, + key: &'a BackupKey, + stale_after: Duration, + ) -> BoxFuture<'a, Result>, StorageError>> { + Box::pin(async move { + // The same advisory lock the owner took: a session that still + // holds it is alive whatever its heartbeat says (the lock goes + // with the session), and a second taker is refused here. + let Some(mut session) = + try_claim(&self.pool, &backup_lock_key(&key.backup_id), None).await? + else { + return Ok(None); + }; + let generation = session.generation(); + let taken = sqlx::query( + "UPDATE backup_index SET owner_generation = $3 \ + WHERE account_id = $1 AND backup_id = $2 AND status = 'CREATING' \ + AND heartbeat_at < now() - make_interval(secs => $4)", + ) + .bind(&key.account_id) + .bind(&key.backup_id) + .bind(generation_column(generation)) + .bind(stale_after.as_secs_f64()) + .execute(session.connection()) + .await + .map_err(db_err)? + .rows_affected(); + if taken != 1 { + session.close().await; + return Ok(None); + } + Ok(Some(Box::new(PostgresBackupClaim { + session: Mutex::new(session), + generation, + key: key.clone(), + pool: self.pool.clone(), + }) as Box)) + }) + } + + fn list_backup_rows_older_than<'a>( + &'a self, + status: BackupStatus, + store: &'a StoreIdentity, + clock: StaleClock, + age: Duration, + ) -> BoxStream<'a, Result> { + let column = match clock { + StaleClock::Heartbeat => "heartbeat_at", + StaleClock::Completed => "completed_at", + }; + rows_stream(async move { + sqlx::query_as(&format!( + "SELECT {ROW_COLUMNS} FROM backup_index \ + WHERE store_kind = $1 AND store_root = $2 AND store_id = $3 AND status = $4 \ + AND {column} < now() - make_interval(secs => $5) \ + ORDER BY created_at, backup_arn" + )) + .bind(store.store_kind.as_str()) + .bind(&store.store_root) + .bind(&store.store_id) + .bind(status.as_str()) + .bind(age.as_secs_f64()) + .fetch_all(&self.pool) + .await + .map_err(db_err) + }) + } + + fn backup_in_use<'a>( + &'a self, + key: &'a BackupKey, + ) -> BoxFuture<'a, Result, StorageError>> { + Box::pin(async move { creating_restore_target(&self.pool, &key.backup_arn).await }) + } + + fn remove_restore_target<'a>( + &'a self, + key: &'a RestoreTargetKey, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { self.remove_restore_target_locked(key).await }) + } + + fn remove_restore_target_exclusive<'a>( + &'a self, + key: &'a RestoreTargetKey, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + // The command path (`remove-target`) takes the target name's + // advisory lock first, the one the owner and `claim_restore_dead` + // take, so a removal and a takeover of the same target exclude + // each other, and a frozen owner whose claim session still holds + // the lock (its heartbeat stale but its session alive) keeps its + // target. The job's own failure path and the + // reconciler's resume call the plain `remove_restore_target` + // while they already hold this lock through their claim, so the + // lock lives only on this command-only variant. + let Some(session) = try_claim( + &self.pool, + &restore_lock_key(&key.account_id, &key.table_name), + None, + ) + .await? + else { + return Ok(false); + }; + let outcome = self.remove_restore_target_locked(key).await; + session.close().await; + outcome + }) + } + + fn list_dead_restore_jobs( + &self, + stale_after: Duration, + ) -> BoxFuture<'_, Result, StorageError>> { + Box::pin(async move { + // Two arms: a RUNNING record whose heartbeat is stale (a dead + // owner), and a FAILED record whose table is still a CREATING + // restore target once the record is stale (the FAILED write + // landed and the removal did not; `fail_restore_job` and the + // job's own failure path stamp `completed_at`, and a record + // without one is aged by its heartbeat). The delay is what + // leaves a job still on its own failure path to finish it. A row + // this build cannot read (a kind or status no release writes) is + // logged and skipped, as ListBackups skips a row it cannot read, + // so one such row does not stop the cycle. + let rows: Vec = sqlx::query_as(&format!( + "SELECT {RESTORE_ROW_COLUMNS} \ + FROM table_restores r JOIN tables t ON t.table_id = r.table_id \ + WHERE (r.status = 'RUNNING' \ + AND r.heartbeat_at < now() - make_interval(secs => $1)) \ + OR (r.status = 'FAILED' AND t.table_status = 'CREATING' \ + AND COALESCE(r.completed_at, r.heartbeat_at) \ + < now() - make_interval(secs => $1)) \ + ORDER BY t.account_id, t.table_name" + )) + .bind(stale_after.as_secs_f64()) + .fetch_all(&self.pool) + .await + .map_err(db_err)?; + Ok(rows + .into_iter() + .filter_map(|row| { + let (account_id, table_name) = (row.1.clone(), row.2.clone()); + match restore_job_from_tuple(row) { + Ok(job) => Some(job), + Err(e) => { + tracing::warn!( + "reconciler: restore job record of {table_name} (account \ + {account_id}) skipped: {e}" + ); + None + } + } + }) + .collect()) + }) + } + + fn list_running_restore_jobs(&self) -> BoxFuture<'_, Result, StorageError>> { + Box::pin(async move { + // By status alone: a record stamped in this statement's instant, + // or ahead of the server's clock, is RUNNING too. + let rows: Vec = sqlx::query_as(&format!( + "SELECT {RESTORE_ROW_COLUMNS} \ + FROM table_restores r JOIN tables t ON t.table_id = r.table_id \ + WHERE r.status = 'RUNNING' ORDER BY t.account_id, t.table_name" + )) + .fetch_all(&self.pool) + .await + .map_err(db_err)?; + rows.into_iter().map(restore_job_from_tuple).collect() + }) + } + + /// Conditioned on RUNNING, on the listed `owner_generation` and on the + /// listed `heartbeat_at` (compared through the same `EXTRACT(EPOCH ...)` + /// expression the listing read it with, so the comparison is exact), and + /// written with a fresh generation (design 4(e)). + fn fail_restore_job<'a>( + &'a self, + key: &'a RestoreTargetKey, + owner_generation: u64, + heartbeat_at: f64, + reason: &'a str, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + let affected = sqlx::query( + "UPDATE table_restores r SET status = 'FAILED', failure_reason = $4, \ + completed_at = now(), owner_generation = $7 FROM tables t \ + WHERE t.table_id = r.table_id AND r.table_id = $1 AND t.account_id = $2 \ + AND t.table_name = $3 AND r.status = 'RUNNING' AND r.owner_generation = $5 \ + AND EXTRACT(EPOCH FROM r.heartbeat_at)::FLOAT8 = $6", + ) + .bind(&key.table_id) + .bind(&key.account_id) + .bind(&key.table_name) + .bind(reason) + .bind(generation_column(owner_generation)) + .bind(heartbeat_at) + .bind(generation_column(new_generation())) + .execute(&self.pool) + .await + .map_err(db_err)? + .rows_affected(); + Ok(affected == 1) + }) + } + + fn claim_restore<'a>( + &'a self, + account_id: &'a str, + input: CreateTableInput, + registration: &'a RestoreRegistrationRow, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + let table_name = input.table_name.clone(); + // Ownership first, before the target exists, on the lock the + // legacy restore takes for a name (`restore_lock_key`), so a + // legacy and a store-backed restore into one name exclude each + // other too. A lock already held is another restore into this + // name in flight: TableBeingRestored naming its source once its + // CREATING row is visible, else the name is taken as CreateTable + // would report it. The record's `heartbeat_at` is `now()` of the + // registration transaction, its start (`create_table.rs`), so a + // registration that waited on the backup's row lock or whose data + // DDL took long returns a record already that old; the engine's + // first beat refreshes it before the reply. + let Some(session) = + try_claim(&self.pool, &restore_lock_key(account_id, &table_name), None).await? + else { + return Err(self.name_taken_error(account_id, &table_name, true).await); + }; + let generation = session.generation(); + let registration = RestoreRegistrationRow { + owner_generation: generation, + ..registration.clone() + }; + let mut desc = match self + .create_table_for_restore(account_id, input, RestoreSource::Manifest(®istration)) + .await + { + Ok(desc) => desc, + Err(e) => { + session.close().await; + return Err(e); + } + }; + desc.restore_summary = Some(RestoreSummary { + source_backup_arn: Some(registration.backup.backup_arn.clone()), + source_table_arn: Some(registration.source_table_arn.clone()), + restore_date_time: registration.restore_date_time, + restore_in_progress: true, + }); + let table_id = desc.table_id.clone(); + Ok(( + desc, + Box::new(PostgresRestoreClaim { + session: Mutex::new(session), + generation, + table_id, + pool: self.pool.clone(), + }) as Box, + )) + }) + } + + fn claim_restore_dead<'a>( + &'a self, + key: &'a RestoreTargetKey, + stale_after: Duration, + spool_host: Option<&'a str>, + ) -> BoxFuture<'a, Result>, StorageError>> { + Box::pin(async move { + // The same advisory lock the owner took on the target name: a + // session that still holds it is alive whatever its heartbeat + // says, and a second taker is refused here. + let Some(mut session) = try_claim( + &self.pool, + &restore_lock_key(&key.account_id, &key.table_name), + None, + ) + .await? + else { + return Ok(None); + }; + let generation = session.generation(); + // The generation, the claimant's spool and a fresh heartbeat in + // one write: a second taker, on this or another instance, finds + // the record live at once (the trait doc says why a restore + // record is refreshed where a backup row is not). + let taken = sqlx::query( + "UPDATE table_restores r SET owner_generation = $4, spool_host = $5, \ + heartbeat_at = now() \ + FROM tables t WHERE t.table_id = r.table_id AND r.table_id = $1 \ + AND t.account_id = $2 AND t.table_name = $3 \ + AND r.status = 'RUNNING' AND r.kind = 'manifest' \ + AND r.heartbeat_at < now() - make_interval(secs => $6)", + ) + .bind(&key.table_id) + .bind(&key.account_id) + .bind(&key.table_name) + .bind(generation_column(generation)) + .bind(spool_host) + .bind(stale_after.as_secs_f64()) + .execute(session.connection()) + .await + .map_err(db_err)? + .rows_affected(); + if taken != 1 { + session.close().await; + return Ok(None); + } + Ok(Some(Box::new(PostgresRestoreClaim { + session: Mutex::new(session), + generation, + table_id: key.table_id.clone(), + pool: self.pool.clone(), + }) as Box)) + }) + } + + fn read_restore_job<'a>( + &'a self, + key: &'a RestoreTargetKey, + ) -> BoxFuture<'a, Result, StorageError>> { + Box::pin(async move { + let row: Option = sqlx::query_as(&format!( + "SELECT {RESTORE_ROW_COLUMNS} \ + FROM table_restores r JOIN tables t ON t.table_id = r.table_id \ + WHERE r.table_id = $1 AND t.account_id = $2 AND t.table_name = $3" + )) + .bind(&key.table_id) + .bind(&key.account_id) + .bind(&key.table_name) + .fetch_optional(&self.pool) + .await + .map_err(db_err)?; + row.map(restore_job_from_tuple).transpose() + }) + } + + fn list_restore_jobs<'a>( + &'a self, + status: Option, + ) -> BoxFuture<'a, Result, StorageError>> { + Box::pin(async move { + let rows: Vec = sqlx::query_as(&format!( + "SELECT {RESTORE_ROW_COLUMNS} \ + FROM table_restores r JOIN tables t ON t.table_id = r.table_id \ + WHERE ($1::TEXT IS NULL OR r.status = $1) \ + ORDER BY t.account_id, t.table_name" + )) + .bind(status.map(JobStatus::as_str)) + .fetch_all(&self.pool) + .await + .map_err(db_err)?; + rows.into_iter().map(restore_job_from_tuple).collect() + }) + } + + fn read_store_ids<'a>( + &'a self, + store_kind: StoreKind, + store_root: &'a str, + ) -> BoxFuture<'a, Result, StorageError>> { + Box::pin(async move { + sqlx::query_scalar( + "SELECT DISTINCT store_id FROM backup_index \ + WHERE store_kind = $1 AND store_root = $2 ORDER BY store_id", + ) + .bind(store_kind.as_str()) + .bind(store_root) + .fetch_all(&self.pool) + .await + .map_err(db_err) + }) + } + + fn maintenance_claim<'a>( + &'a self, + holder: &'a str, + stale_after: Duration, + ) -> BoxFuture<'a, Result, StorageError>> { + Box::pin(async move { + // The advisory lock is the live-holder proof, as a job's is: a + // second command on any instance finds it taken while the first + // runs, and a holder whose session ended has released it. The + // row carries the generation the heartbeats and the release are + // conditioned on, and the heartbeat age jobs read. + let Some(mut session) = try_claim_with( + &self.pool, + MAINTENANCE_LOCK_KEY, + None, + MAINTENANCE_TAKE_TRIES, + MAINTENANCE_TAKE_PAUSE, + ) + .await? + else { + let current: Option = + sqlx::query_scalar("SELECT holder FROM backup_maintenance") + .fetch_optional(&self.pool) + .await + .map_err(db_err)?; + return Err(StorageError::AlreadyExists(format!( + "the backup maintenance claim is held by {}", + current.as_deref().unwrap_or("another maintenance command") + ))); + }; + // With the lock held, a row whose heartbeat is still young is a + // holder whose lock session ended within the stale window: dead + // by this backend's rule (4(e)), so the row is taken over + // whatever its age. `stale_after` is the other backends' rule and + // the one `check_maintenance_claim` applies on every backend. + let _ = stale_after; + let generation = session.generation(); + sqlx::query( + "INSERT INTO backup_maintenance (singleton, holder, owner_generation) \ + VALUES (TRUE, $1, $2) \ + ON CONFLICT (singleton) DO UPDATE SET holder = EXCLUDED.holder, \ + owner_generation = EXCLUDED.owner_generation, claimed_at = now(), \ + heartbeat_at = now()", + ) + .bind(holder) + .bind(generation_column(generation)) + .execute(session.connection()) + .await + .map_err(db_err)?; + Ok(Box::new(PostgresMaintenanceClaim { + session: Mutex::new(Some(session)), + generation, + }) as Box) + }) + } + + fn check_maintenance_claim( + &self, + stale_after: Duration, + ) -> BoxFuture<'_, Result> { + Box::pin(async move { + // A live holder's session holds the exclusive lock, so a shared + // try-lock fails; it succeeds when no session does. The lock is + // taken at transaction scope in one autocommit statement, so it + // ends with the statement and nothing is held on the pooled + // connection when the caller's future is dropped between awaits + // (this runs inside every CreateBackup and RestoreTableFromBackup + // request). Then the row a dead session left is held until its + // heartbeat ages past the window. + let unlocked: bool = sqlx::query_scalar( + "SELECT pg_try_advisory_xact_lock_shared(hashtextextended($1, $2))", + ) + .bind(MAINTENANCE_LOCK_KEY) + .bind(crate::backup_engine::RESTORE_LOCK_SEED) + .fetch_one(&self.pool) + .await + .map_err(db_err)?; + if !unlocked { + return Ok(true); + } + sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM backup_maintenance \ + WHERE heartbeat_at >= now() - make_interval(secs => $1))", + ) + .bind(stale_after.as_secs_f64()) + .fetch_one(&self.pool) + .await + .map_err(db_err) + }) + } +} + +/// The advisory-lock key of the maintenance claim, in the one lock key space +/// (`RESTORE_LOCK_SEED`) the job claims use. +const MAINTENANCE_LOCK_KEY: &str = "backup_maintenance"; + +/// How many times `maintenance_claim` tries the exclusive lock (on its one +/// claim connection), and the pause between tries. `check_maintenance_claim` holds the key's shared lock for +/// one statement, and every CreateBackup and RestoreTableFromBackup runs one, +/// so a take that lands inside such a statement fails with no holder to +/// name; the tries span about 10 ms, past any such statement, so only a live +/// holder's session refuses the take. +const MAINTENANCE_TAKE_TRIES: u32 = 6; +const MAINTENANCE_TAKE_PAUSE: Duration = Duration::from_millis(2); + +impl crate::PostgresEngine { + /// DeleteBackup's first step, and with `remove` the outright removal of + /// `forget_available_backup_row`, as one transaction: lock the row, read + /// its status, run the in-use rule, then write DELETING or, with + /// `remove`, delete the row (`Deleting` then means removed). One body, + /// so the two methods share the lock and the rule. + async fn start_backup_delete_tx( + &self, + key: &BackupKey, + remove: bool, + ) -> Result { + let mut tx = self.pool.begin().await.map_err(db_err)?; + // FOR UPDATE is the contract a restore's registration of this row + // must meet: it locks the same row before its own in-use check, so + // the two order wholly one before the other (a registration that + // commits first leaves this `InUse`; one that waits behind this + // finds the row DELETING, or no row). `claim_restore` + // (`create_table.rs`, the manifest arm) takes this lock; the legacy + // registration locks the legacy `backups` row instead. + let status: Option = sqlx::query_scalar( + "SELECT status FROM backup_index WHERE account_id = $1 AND backup_id = $2 \ + FOR UPDATE", + ) + .bind(&key.account_id) + .bind(&key.backup_id) + .fetch_optional(&mut *tx) + .await + .map_err(db_err)?; + let status = BackupStatus::parse(&status.ok_or_else(|| not_found(&key.backup_arn))?)?; + if status != BackupStatus::Available { + return Ok(DeleteStart::Status(status)); + } + if let Some(table) = creating_restore_target(&mut *tx, &key.backup_arn).await? { + return Ok(DeleteStart::InUse(table)); + } + let write = if remove { + "DELETE FROM backup_index \ + WHERE account_id = $1 AND backup_id = $2 AND status = 'AVAILABLE'" + } else { + "UPDATE backup_index SET status = 'DELETING' \ + WHERE account_id = $1 AND backup_id = $2 AND status = 'AVAILABLE'" + }; + sqlx::query(write) + .bind(&key.account_id) + .bind(&key.backup_id) + .execute(&mut *tx) + .await + .map_err(db_err)?; + tx.commit().await.map_err(db_err)?; + Ok(DeleteStart::Deleting) + } + + /// The lock step of `maintenance_claim` (the tries on one claim + /// connection) with `tries` and `pause` in place of the take's own, so a + /// test can hold the key's shared lock past the first try and release it + /// while the take still tries. `true` when the lock was taken (it is + /// released again as the session closes). + /// + /// # Errors + /// + /// As `maintenance_claim`'s connection and lock errors. + #[cfg(feature = "test-util")] + pub async fn maintenance_lock_for_tests( + &self, + tries: u32, + pause: Duration, + ) -> Result { + Ok( + try_claim_with(&self.pool, MAINTENANCE_LOCK_KEY, None, tries, pause) + .await? + .is_some(), + ) + } +} + +/// The holder's handle on the `backup_maintenance` row: the lock session +/// (`None` once released) and the generation its writes are conditioned on. +struct PostgresMaintenanceClaim { + session: Mutex>, + generation: u64, +} + +impl MaintenanceClaim for PostgresMaintenanceClaim { + fn generation(&self) -> u64 { + self.generation + } + + fn heartbeat(&self) -> BoxFuture<'_, Result> { + Box::pin(async move { + let mut session = self.session.lock().await; + let Some(session) = session.as_mut() else { + return Ok(Heartbeat::Lost); + }; + let confirmed = session + .confirm( + "UPDATE backup_maintenance SET heartbeat_at = now() WHERE owner_generation = $1", + &[], + ) + .await + .map_err(|e| StorageError::Internal(format!("claim session: {e}")))?; + Ok(if confirmed { + Heartbeat::Confirmed + } else { + Heartbeat::Lost + }) + }) + } + + /// Delete the row, release the advisory lock, then end the lock session. + /// The unlock is explicit: `PgConnection::close` returns before the + /// server backend exits and drops the session's locks, so a + /// `check_maintenance_claim` on another connection right after the + /// release would otherwise still find the lock held and answer held. A + /// second release finds no session and does nothing. + fn release(&self) -> BoxFuture<'_, Result<(), StorageError>> { + Box::pin(async move { + let mut slot = self.session.lock().await; + let Some(mut session) = slot.take() else { + return Ok(()); + }; + sqlx::query("DELETE FROM backup_maintenance WHERE owner_generation = $1") + .bind(generation_column(self.generation)) + .execute(session.connection()) + .await + .map_err(db_err)?; + sqlx::query("SELECT pg_advisory_unlock(hashtextextended($1, $2))") + .bind(MAINTENANCE_LOCK_KEY) + .bind(crate::backup_engine::RESTORE_LOCK_SEED) + .execute(session.connection()) + .await + .map_err(db_err)?; + session.close().await; + Ok(()) + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_row_tuple_with_an_unknown_store_kind_or_status_is_internal() { + let good = || RowTuple { + backup_id: "01757500000123-0badcafe".to_owned(), + account_id: "123456789012".to_owned(), + table_name: "Music".to_owned(), + backup_arn: "arn".to_owned(), + backup_name: "b".to_owned(), + status: "AVAILABLE".to_owned(), + store_kind: "s3".to_owned(), + store_root: "s3://bucket/prefix".to_owned(), + store_id: "store-1".to_owned(), + size_bytes: 1, + item_count: 2, + created_at: 3.0, + completed_at: None, + failure_reason: None, + source_table_details_json: serde_json::json!({}), + owner_generation: 7, + heartbeat_at: 4.0, + }; + let row = row_from_tuple(good()).unwrap(); + assert_eq!(row.store.store_kind, StoreKind::S3); + assert_eq!(row.owner_generation, 7); + let mut bad_kind = good(); + bad_kind.store_kind = "gcs".to_owned(); + assert!(matches!( + row_from_tuple(bad_kind), + Err(StorageError::Internal(_)) + )); + let mut bad_status = good(); + bad_status.status = "ACTIVE".to_owned(); + assert!(matches!( + row_from_tuple(bad_status), + Err(StorageError::Internal(_)) + )); + } + + #[test] + fn only_a_connection_class_failure_leaves_a_write_unknown() { + assert!(outcome_unknown(&sqlx::Error::PoolClosed)); + assert!(outcome_unknown(&sqlx::Error::Io(std::io::Error::new( + std::io::ErrorKind::ConnectionReset, + "reset", + )))); + assert!(!outcome_unknown(&sqlx::Error::Protocol("x".to_owned()))); + assert!(!outcome_unknown(&sqlx::Error::ColumnNotFound( + "status".to_owned() + ))); + } +} diff --git a/crates/storage-postgres/src/backup_engine.rs b/crates/storage-postgres/src/backup_engine.rs index 6d93da862..a37f1183e 100755 --- a/crates/storage-postgres/src/backup_engine.rs +++ b/crates/storage-postgres/src/backup_engine.rs @@ -1,16 +1,23 @@ // Copyright 2026 ExtendDB contributors // SPDX-License-Identifier: Apache-2.0 -//! Backup and restore implementation for `PostgreSQL` storage. +//! The in-catalog backup format on `PostgreSQL`: the create, describe, list, +//! delete, restore-source and restore paths of [`LegacyBackupReader`], frozen +//! apart from the restore's registration, which writes the restore job +//! fields and holds the job claim ([`ClaimSession`]) the catalog index's jobs +//! hold too (`backup_catalog.rs`). -use std::collections::HashMap; use std::sync::Arc; +use std::time::Duration; use extenddb_core::types::{ BackupDescription, BackupDetails, BackupSummary, GsiInput, Item, KeySchemaElement, LsiInput, Projection, RestoreSummary, ScalarAttributeType, SourceTableDetails, StreamSpecification, TableDescription, }; +use extenddb_storage::backup_catalog::{ + AbandonedLegacyBackup, JOB_HEARTBEAT_INTERVAL, JobBound, limit_exceeded_message, new_generation, +}; use extenddb_storage::backup_definition::{ BACKUP_DEFINITION_VERSION, BackupTableDefinition, COPY_BATCH_BYTES, COPY_BATCH_ITEMS, throughput_from_catalog, @@ -20,15 +27,14 @@ use extenddb_storage::util::{ SortKeyValue, backup_arn, backup_id_epoch_millis, composite_pk_to_text, new_backup_id, parse_sk, sk_column_n, }; -use extenddb_storage::{BackupEngine, BackupRestoreSource, RestoreRequest}; +use extenddb_storage::{BackupRestoreSource, JobPermit, LegacyBackupReader, RestoreRequest}; use futures::future::BoxFuture; -use tokio::sync::{OwnedSemaphorePermit, Semaphore}; use crate::PostgresEngine; -use crate::create_table::RestoreRegistration; +use crate::create_table::{RestoreRegistration, RestoreSource}; use crate::data::{ all_sort_key_info, data_table_name, index_table_name, insert_index_rows_multi, - item_has_index_keys, project_item_for_index, + item_has_index_keys, project_item_for_index, table_storage_relations, }; /// Convert a `PostgreSQL` `TIMESTAMPTZ` to epoch seconds as `f64`, keeping the @@ -48,202 +54,369 @@ fn pg_timestamp_to_epoch(ts: time::OffsetDateTime) -> f64 { } } -/// Seed for the 64-bit restore lock key. Restore locks use the single-`bigint` -/// advisory lock form, which PostgreSQL keeps in a key space separate from the -/// two-`int4` form the migration and vector-build locks use, so they cannot -/// meet; the seed keeps this key space distinct from any other `bigint` user. -const RESTORE_LOCK_SEED: i64 = 0x0045_4452; // 'E', 'D', 'R' - -/// Restores one account may have in flight at once in this process. Each -/// holds one dedicated connection for its lock on top of its pooled ones. A -/// restore beyond the limit waits for one of its own account's slots, so one -/// account cannot hold every other account's restores back. -const RESTORE_SLOTS_PER_ACCOUNT: usize = 4; +/// Seed for the 64-bit job lock keys: a restore's lock on its target name +/// and a backup job's lock on its backup id share this one key space. The +/// locks use the single-`bigint` advisory lock form, which PostgreSQL keeps +/// in a key space separate from the two-`int4` form the migration and +/// vector-build locks use, so they cannot meet; the seed keeps this key +/// space distinct from any other `bigint` user. Public for the tests that +/// hold or look up a lock; not an interface. +#[doc(hidden)] +pub const RESTORE_LOCK_SEED: i64 = 0x0045_4452; // 'E', 'D', 'R' -/// Upper limit on [`restore_server_slot_count`]. -const RESTORE_SERVER_SLOTS_MAX: usize = 16; - -/// Restores the whole process may have in flight at once, across accounts: -/// `min(16, pool_size / 2)`, at least 1. -/// -/// Each running copy holds one connection of the engine's catalog pool -/// (`pool_size`) for its whole duration, for the REPEATABLE READ snapshot it -/// reads the backup's items from. Without a process-wide bound, enough -/// accounts each running their per-account allowance would hold every -/// catalog connection and stall every other request that needs the catalog. -/// Half the pool stays free for them. A restore takes its account slot -/// first, then one of these; the two waits share one deadline of -/// [`RESTORE_SLOT_WAIT_SECS`]. -pub(crate) fn restore_server_slot_count(pool_size: u32) -> usize { - usize::try_from(pool_size / 2) - .unwrap_or(usize::MAX) - .clamp(1, RESTORE_SERVER_SLOTS_MAX) +/// A secondary index of a restore target, as the copy needs it. +pub(crate) struct RestoreIndex { + pub(crate) table: String, + pub(crate) key_schema: Vec, + pub(crate) projection: Projection, } -/// The two slots a running restore holds until its copy ends. -struct RestorePermits { - _account: OwnedSemaphorePermit, - _server: OwnedSemaphorePermit, +/// The secondary indexes of `table_id` as the catalog records them, each +/// with the data table its rows go into. +pub(crate) async fn restore_indexes( + pool: &sqlx::PgPool, + table_id: &str, +) -> Result, StorageError> { + let index_rows: Vec<(String, serde_json::Value, serde_json::Value)> = + sqlx::query_as("SELECT index_id, key_schema, projection FROM indexes WHERE table_id = $1") + .bind(table_id) + .fetch_all(pool) + .await + .map_err(db_err)?; + let mut indexes = Vec::with_capacity(index_rows.len()); + for (index_id, ks, proj) in index_rows { + indexes.push(RestoreIndex { + table: index_table_name(&index_id), + key_schema: serde_json::from_value(ks) + .map_err(|e| StorageError::Internal(format!("index key schema: {e}")))?, + projection: serde_json::from_value(proj) + .map_err(|e| StorageError::Internal(format!("index projection: {e}")))?, + }); + } + Ok(indexes) } -/// Take an account slot, then a server slot, waiting at most `wait` for -/// both together. Running out of time on either is `LimitExceeded`, with a -/// message naming the limit that was full. -async fn acquire_restore_permits( - account: Arc, - server: Arc, - server_slots: usize, - wait: std::time::Duration, -) -> Result { - let deadline = tokio::time::Instant::now() + wait; - let closed = - |e: tokio::sync::AcquireError| StorageError::Internal(format!("restore slot: {e}")); - let account_permit = tokio::time::timeout_at(deadline, account.acquire_owned()) - .await - .map_err(|_| { - StorageError::LimitExceeded(format!( - "Too many restores are in progress for this account (at most \ - {RESTORE_SLOTS_PER_ACCOUNT} at once); retry later" - )) - })? - .map_err(closed)?; - let server_permit = tokio::time::timeout_at(deadline, server.acquire_owned()) - .await - .map_err(|_| { - StorageError::LimitExceeded(format!( - "Too many restores are in progress on this server (at most {server_slots} at \ - once); retry later" - )) - })? - .map_err(closed)?; - Ok(RestorePermits { - _account: account_permit, - _server: server_permit, - }) +/// Why [`insert_restore_rows`] stopped: the base table refused a key it +/// already holds (the backup carries a key twice), or anything else. The +/// legacy copy reports both as the database error ([`From`]); the +/// store-backed load names the data file for the first. +pub(crate) enum RowInsertError { + DuplicateKey(sqlx::Error), + Storage(StorageError), } -/// Per-account restore slots of one engine (shared by its clones, so one -/// per server process). An entry nobody holds (no permit, no waiter) is -/// pruned the next time any account takes a slot, so the map does not grow -/// with every account that ever restored. -#[derive(Clone, Default)] -pub(crate) struct RestoreAccountSlots(Arc>>>); - -impl RestoreAccountSlots { - /// The slot semaphore for `account_id`. - pub(crate) fn for_account(&self, account_id: &str) -> Arc { - let mut slots = self - .0 - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - // A permit and a waiter each hold a clone, so a count of one means - // the account has nothing in flight. - slots.retain(|_, s| Arc::strong_count(s) > 1); - Arc::clone( - slots - .entry(account_id.to_owned()) - .or_insert_with(|| Arc::new(Semaphore::new(RESTORE_SLOTS_PER_ACCOUNT))), - ) +impl From for RowInsertError { + fn from(e: StorageError) -> Self { + Self::Storage(e) } } -/// How long a restore waits for its account slot and a server slot, -/// together, before it is refused with `LimitExceededException`. -const RESTORE_SLOT_WAIT_SECS: u64 = 30; +impl From for StorageError { + fn from(e: RowInsertError) -> Self { + match e { + RowInsertError::DuplicateKey(e) => db_err(e), + RowInsertError::Storage(e) => e, + } + } +} -/// How often the restore's lock session is exercised while its copy runs. -/// Shorter than the idle-session timeouts managed services apply (minutes), -/// and a quarter of [`ABANDONED_RESTORE_GRACE_SECS`]. -const RESTORE_HEARTBEAT_SECS: u64 = 15; +/// SQLSTATE `unique_violation`. +const SQLSTATE_UNIQUE_VIOLATION: &str = "23505"; + +/// A restore target as the row writes need it, built once per restore: the +/// base data table, the key shape, and the secondary indexes with their +/// data tables. `source` names the backup or data file in errors. +pub(crate) struct RestoreTarget<'a> { + pub(crate) ddb_table: String, + pub(crate) key_schema: &'a [KeySchemaElement], + pub(crate) attribute_definitions: &'a [extenddb_core::types::AttributeDefinition], + pub(crate) sort_keys: Vec<(&'a str, ScalarAttributeType)>, + pub(crate) indexes: Vec, + pub(crate) source: String, +} -/// How old an unowned restore target must be before it is treated as -/// abandoned. The owner takes its lock before creating the target, so this -/// only has to cover clock skew between the catalog and the sweep, with a -/// wide margin. -const ABANDONED_RESTORE_GRACE_SECS: i32 = 60; +impl<'a> RestoreTarget<'a> { + /// The target `table_id` with `key_schema` and `attribute_definitions`, + /// its indexes read from the catalog. + pub(crate) async fn load( + pool: &sqlx::PgPool, + table_id: &str, + key_schema: &'a [KeySchemaElement], + attribute_definitions: &'a [extenddb_core::types::AttributeDefinition], + source: String, + ) -> Result { + Ok(Self { + ddb_table: data_table_name(table_id), + key_schema, + attribute_definitions, + sort_keys: all_sort_key_info(key_schema, attribute_definitions), + indexes: restore_indexes(pool, table_id).await?, + source, + }) + } +} -/// A secondary index of a restore target, as the copy needs it. -struct RestoreIndex { - table: String, - key_schema: Vec, - projection: Projection, +/// Write one batch of restored items inside `tx`: one multi-row INSERT into +/// the base table, then one multi-row INSERT into each secondary index for +/// the batch's items that carry that index's keys. Plain INSERTs: two rows +/// for one key fail the statement rather than collapsing into one item. +pub(crate) async fn insert_restore_rows( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + target: &RestoreTarget<'_>, + batch: &[(Item, String)], +) -> Result<(), RowInsertError> { + if batch.is_empty() { + return Ok(()); + } + let RestoreTarget { + ddb_table, + key_schema, + attribute_definitions, + sort_keys, + indexes, + source, + } = target; + let mut cols = vec!["pk".to_owned()]; + cols.extend( + sort_keys + .iter() + .enumerate() + .map(|(i, &(_, t))| sk_column_n(i, t)), + ); + cols.push("item_data".to_owned()); + let width = cols.len(); + // The last column is the item, bound as text and cast in SQL. + let tuples: Vec = (0..batch.len()) + .map(|r| { + let ps: Vec = (1..=width) + .map(|c| { + if c == width { + format!("${}::jsonb", r * width + c) + } else { + format!("${}", r * width + c) + } + }) + .collect(); + format!("({})", ps.join(", ")) + }) + .collect(); + let sql = format!( + "INSERT INTO {ddb_table} ({}) VALUES {}", + cols.join(", "), + tuples.join(", ") + ); + let mut query = sqlx::query(&sql); + for (item, item_json) in batch { + query = query.bind(composite_pk_to_text(item, key_schema)?); + for &(name, sk_type) in sort_keys { + let value = item.get(name).ok_or_else(|| { + StorageError::Internal(format!("{source} has an item without sort key {name}")) + })?; + query = match parse_sk(value, sk_type)? { + SortKeyValue::S(v) => query.bind(v), + SortKeyValue::N(v) => query.bind(v), + SortKeyValue::B(v) => query.bind(v), + }; + } + query = query.bind(item_json.as_str()); + } + query.execute(&mut **tx).await.map_err(|e| { + let duplicate = e + .as_database_error() + .and_then(|d| d.code()) + .is_some_and(|code| code == SQLSTATE_UNIQUE_VIOLATION); + if duplicate { + RowInsertError::DuplicateKey(e) + } else { + RowInsertError::Storage(db_err(e)) + } + })?; + + for idx in indexes { + let idx_sks = all_sort_key_info(&idx.key_schema, attribute_definitions); + let projected: Vec<(&Item, Item)> = batch + .iter() + .filter(|(item, _)| item_has_index_keys(item, &idx.key_schema)) + .map(|(item, _)| { + ( + item, + project_item_for_index(item, &idx.key_schema, key_schema, &idx.projection), + ) + }) + .collect(); + let rows: Vec<(&Item, &Item)> = projected.iter().map(|(i, p)| (*i, p)).collect(); + insert_index_rows_multi( + tx, + &idx.table, + &rows, + &idx.key_schema, + key_schema, + &idx_sks, + sort_keys, + ) + .await?; + } + Ok(()) } -/// Session-scoped ownership of one restore, held for the life of the copy. -/// -/// A session-level `pg_try_advisory_lock` on a dedicated connection: it dies -/// with the session, so a crashed process's claim disappears on its own and -/// the abandoned-restore sweep can tell a dead restore from a running one. +/// Session-scoped ownership of one job, held for the job's life: a +/// session-level `pg_try_advisory_lock` on a dedicated, non-pooled connection +/// to the catalog database, the job's `owner_generation`, and the job permit +/// the engine handed it. /// -/// This needs the connection to be one server session for its whole life: -/// a direct connection to the catalog database, or one through a +/// The lock dies with the session, so a crashed process's claim disappears on +/// its own. This needs the connection to be one server session for its whole +/// life: a direct connection to the catalog database, or one through a /// session-mode pooler. Behind a transaction-mode pooler (PgBouncer /// `pool_mode = transaction`) the lock stays on whichever server session ran /// the statement, is not released when this connection closes, and can be -/// observed as free by the sweep on that same session; restore is not -/// supported there. +/// observed as free on that same session; jobs are not supported there. /// -/// The connection would otherwise sit idle for the whole copy, and some +/// The connection would otherwise sit idle for the whole job, and some /// servers end idle sessions (`idle_session_timeout`, or a managed service's -/// own reaping). [`RestoreOwner::heartbeat`] issues -/// a statement on it every [`RESTORE_HEARTBEAT_SECS`]; when one fails, the -/// copy is stopped and the target removed, rather than copying on into a -/// target the sweep may claim. The sweep's claim and the copy's flip to -/// ACTIVE are both conditional on CREATING, so if the two still race (the -/// session ends between heartbeats), exactly one wins. -struct RestoreOwner { +/// own reaping). The job's heartbeats go through this connection every +/// [`JOB_HEARTBEAT_INTERVAL`] (shorter than the idle-session timeouts managed +/// services apply, and a small fraction of the stale window the reconciler +/// waits): each refreshes the record's `heartbeat_at` +/// conditioned on the generation, so a session the server dropped, or a +/// record a successor took over, surfaces as a lost heartbeat and the job +/// stops rather than continuing unowned. A free lock shows that the owner's +/// catalog session ended, not that its process stopped, so the reconciler +/// acts on a record only once its heartbeat is older than the stale window. +pub(crate) struct ClaimSession { conn: sqlx::PgConnection, - _slots: Option, + generation: u64, + _permit: Option, } -impl RestoreOwner { - /// Exercise the lock session every [`RESTORE_HEARTBEAT_SECS`] until a - /// statement on it fails; returns that failure. Never returns while the - /// session is alive. - async fn heartbeat(&mut self) -> sqlx::Error { - let period = std::time::Duration::from_secs(RESTORE_HEARTBEAT_SECS); +impl ClaimSession { + /// The fencing token this claim was taken with. + pub(crate) fn generation(&self) -> u64 { + self.generation + } + + /// The claim connection, for the job's conditional writes. + pub(crate) fn connection(&mut self) -> &mut sqlx::PgConnection { + &mut self.conn + } + + /// End the session, releasing its advisory lock; a close that fails is + /// logged (the server reaps the connection either way). + pub(crate) async fn close(self) { + if let Err(e) = ::close(self.conn).await { + tracing::warn!("closing a claim session: {e}"); + } + } + + /// Run `statement`, bound with the record's key and this generation, on + /// the claim connection and report whether it refreshed exactly one row. + pub(crate) async fn confirm( + &mut self, + statement: &str, + key: &[&str], + ) -> Result { + let mut query = sqlx::query(statement); + for part in key { + query = query.bind(*part); + } + let affected = query + .bind(generation_column(self.generation)) + .execute(&mut self.conn) + .await? + .rows_affected(); + Ok(affected == 1) + } + + /// Heartbeat the record every [`JOB_HEARTBEAT_INTERVAL`] with + /// `statement` (as [`Self::confirm`] runs it) until a heartbeat fails or + /// refreshes no row; returns why. Never returns while the claim is + /// confirmed. + async fn heartbeat_loop(&mut self, statement: &str, key: &[&str]) -> StorageError { + let period = JOB_HEARTBEAT_INTERVAL; let mut tick = tokio::time::interval_at(tokio::time::Instant::now() + period, period); tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); loop { tick.tick().await; - if let Err(e) = sqlx::query("SELECT 1").execute(&mut self.conn).await { - return e; + match self.confirm(statement, key).await { + Ok(true) => {} + Ok(false) => { + return StorageError::Internal( + "the job record no longer carries this claim's generation".to_owned(), + ); + } + Err(e) => return StorageError::Internal(format!("claim session: {e}")), } } } } +/// An `owner_generation` as the catalog stores it (`BIGINT`); generations +/// are minted below `i64::MAX`. +pub(crate) fn generation_column(generation: u64) -> i64 { + i64::try_from(generation).unwrap_or(i64::MAX) +} + +/// The heartbeat of a restore: refresh its job record while it is RUNNING +/// under this generation. +pub(crate) const RESTORE_HEARTBEAT: &str = "UPDATE table_restores SET heartbeat_at = now() \ + WHERE table_id = $1 AND status = 'RUNNING' AND owner_generation = $2"; + /// The advisory-lock key for a restore into `table_name` in `account_id`. /// /// Keyed by name rather than table id so the restore can hold it before the -/// target exists: the sweep must never see an unowned target, including in -/// the window while `create_table_impl` is still running DDL. The key is a -/// 64-bit `hashtextextended`, so a collision is negligible, and one would -/// only make the sweep skip a table while the other name's lock is held, -/// never remove a table it should not. -fn restore_lock_key(account_id: &str, table_name: &str) -> String { +/// target exists: nothing must ever see an unowned target, including in the +/// window while `create_table_impl` is still running DDL. The key is a 64-bit +/// `hashtextextended`, so a collision is negligible, and one would only make +/// a second restore into the other name wait, never remove a table it should +/// not. +pub(crate) fn restore_lock_key(account_id: &str, table_name: &str) -> String { format!("{account_id}/{table_name}") } -/// Take the restore lock for `table_name` on a new dedicated connection. -/// `Ok(None)`: another session holds it. +/// The advisory-lock key of a backup job's claim on `backup_id`. +pub(crate) fn backup_lock_key(backup_id: &str) -> String { + format!("backup:{backup_id}") +} + +/// The TCP keepalive settings a claim session asks the server for, as +/// `(setting, seconds)`: idle 30 s before the first probe, then probes 10 s +/// apart, three unanswered before the connection is dropped. Public for the +/// test that reads them back with `SHOW`. +#[doc(hidden)] +pub const CLAIM_KEEPALIVES: [(&str, u32); 3] = [ + ("tcp_keepalives_idle", 30), + ("tcp_keepalives_interval", 10), + ("tcp_keepalives_count", 3), +]; + +/// Take the lock `key` on a new dedicated connection, with a fresh +/// generation. `Ok(None)`: another session holds it. /// /// A connection the server refuses for lack of connection slots is reported /// as `LimitExceeded`; any other failure to open it is `Internal` (see /// [`lock_connection_error`]). -async fn try_restore_lock( +pub(crate) async fn try_claim( pool: &sqlx::PgPool, - account_id: &str, - table_name: &str, - slots: Option, -) -> Result, StorageError> { + key: &str, + permit: Option, +) -> Result, StorageError> { + try_claim_with(pool, key, permit, 1, Duration::ZERO).await +} + +/// [`try_claim`] trying the lock up to `tries` times on the one connection, +/// `pause` apart, before answering `Ok(None)`. +pub(crate) async fn try_claim_with( + pool: &sqlx::PgPool, + key: &str, + permit: Option, + tries: u32, + pause: Duration, +) -> Result, StorageError> { let options = pool.connect_options(); let mut conn = ::connect_with(&options) .await .map_err(|e| { - tracing::warn!( - "could not open the restore lock connection for {account_id}/{table_name}: {e}" - ); + tracing::warn!("could not open the claim connection for {key}: {e}"); lock_connection_error(&e) })?; // A server-side idle_session_timeout would end this session between @@ -256,38 +429,110 @@ async fn try_restore_lock( .await { tracing::warn!( - "could not disable idle_session_timeout for the restore lock session; \ - continuing with the server setting: {e}" + "could not disable idle_session_timeout for the claim session; continuing with \ + the server setting: {e}" ); } - let taken: bool = sqlx::query_scalar("SELECT pg_try_advisory_lock(hashtextextended($1, $2))") - .bind(restore_lock_key(account_id, table_name)) - .bind(RESTORE_LOCK_SEED) - .fetch_one(&mut conn) - .await - .map_err(|e| StorageError::Internal(format!("restore lock: {e}")))?; - Ok(taken.then_some(RestoreOwner { + // The server's TCP keepalive probes toward this process, so the + // advisory lock of a claim whose host vanished (no FIN, no RST) is + // released about a minute after the loss (30 s idle, then three probes + // 10 s apart) instead of after the kernel's default two hours, and the + // row's successor can take the lock once the row's heartbeat is stale. + // The owner's own liveness check is the heartbeat, not these. Each is a + // session setting a managed service may refuse; a refusal is a warning + // and the server's defaults stay. + for (name, value) in CLAIM_KEEPALIVES { + if let Err(e) = sqlx::query(&format!("SET {name} = {value}")) + .execute(&mut conn) + .await + { + tracing::warn!( + "could not set {name} for the claim session; continuing with the server \ + setting: {e}" + ); + } + } + let mut taken = false; + for attempt in 1..=tries.max(1) { + taken = sqlx::query_scalar("SELECT pg_try_advisory_lock(hashtextextended($1, $2))") + .bind(key) + .bind(RESTORE_LOCK_SEED) + .fetch_one(&mut conn) + .await + .map_err(|e| StorageError::Internal(format!("claim lock: {e}")))?; + if taken || attempt == tries { + break; + } + tokio::time::sleep(pause).await; + } + Ok(taken.then(|| ClaimSession { conn, - _slots: slots, + generation: new_generation(), + _permit: permit, })) } +/// Open a claim session on `key` as a job would and read `settings` back +/// from it with `SHOW`, then end the session: for the test that checks the +/// session settings [`try_claim`] applies. `None` when another session holds +/// the lock. +/// +/// # Errors +/// +/// The claim's own errors, and `Internal` when a `SHOW` fails. +#[cfg(feature = "test-hooks")] +#[doc(hidden)] +pub async fn claim_session_settings( + pool: &sqlx::PgPool, + key: &str, + settings: &[&str], +) -> Result>, StorageError> { + let Some(mut session) = try_claim(pool, key, None).await? else { + return Ok(None); + }; + let mut values = Vec::with_capacity(settings.len()); + for name in settings { + let value: String = sqlx::query_scalar(&format!("SHOW {name}")) + .fetch_one(session.connection()) + .await + .map_err(|e| StorageError::Internal(format!("SHOW {name}: {e}")))?; + values.push(value); + } + session.close().await; + Ok(Some(values)) +} + +/// Take the restore lock for `table_name` on a new dedicated connection. +async fn try_restore_lock( + pool: &sqlx::PgPool, + account_id: &str, + table_name: &str, + permit: JobPermit, +) -> Result, StorageError> { + try_claim( + pool, + &restore_lock_key(account_id, table_name), + Some(permit), + ) + .await +} + /// SQLSTATE `too_many_connections`: the server (or the role or database) /// has no connection slot left. const SQLSTATE_TOO_MANY_CONNECTIONS: &str = "53300"; -/// Map a failure to open the restore lock connection. Only the server's +/// Map a failure to open a claim connection. Only the server's /// `too_many_connections` refusal is `LimitExceeded` (a client should back -/// off and retry); authentication, TLS, DNS, network, and every other -/// failure, including one that carries no SQLSTATE, is `Internal`. +/// off and retry), in the one job-refusal text +/// ([`limit_exceeded_message`]) naming the server's connection slots as the +/// bound; authentication, TLS, DNS, network, and every other failure, +/// including one that carries no SQLSTATE, is `Internal`. fn lock_connection_error(e: &sqlx::Error) -> StorageError { let code = e.as_database_error().and_then(|d| d.code()); if code.as_deref() == Some(SQLSTATE_TOO_MANY_CONNECTIONS) { - StorageError::LimitExceeded( - "Too many restores are in progress on this server; retry later".to_owned(), - ) + StorageError::LimitExceeded(limit_exceeded_message(JobBound::ServerConnections)) } else { - StorageError::Internal(format!("restore lock connection: {e}")) + StorageError::Internal(format!("claim connection: {e}")) } } @@ -452,7 +697,8 @@ impl PostgresEngine { } /// Copy a backup's items into a freshly created restore target, populate - /// its secondary indexes, and mark it ACTIVE. + /// its secondary indexes, and mark it ACTIVE, which marks the job record + /// DONE in the same catalog transaction, conditioned on `generation`. /// /// Every key column is derived from the item itself, so the copy does not /// depend on how the source table's columns were laid out. Rows are plain @@ -474,28 +720,19 @@ impl PostgresEngine { &self, desc: &TableDescription, backup_arn: &str, + generation: u64, ) -> Result<(), StorageError> { use futures::TryStreamExt; let ddb_table = data_table_name(&desc.table_id); - let sort_keys = all_sort_key_info(&desc.key_schema, &desc.attribute_definitions); - let index_rows: Vec<(String, serde_json::Value, serde_json::Value)> = sqlx::query_as( - "SELECT index_id, key_schema, projection FROM indexes WHERE table_id = $1", + let target = RestoreTarget::load( + &self.pool, + &desc.table_id, + &desc.key_schema, + &desc.attribute_definitions, + format!("backup {backup_arn}"), ) - .bind(&desc.table_id) - .fetch_all(&self.pool) - .await - .map_err(db_err)?; - let mut indexes = Vec::with_capacity(index_rows.len()); - for (index_id, ks, proj) in index_rows { - indexes.push(RestoreIndex { - table: index_table_name(&index_id), - key_schema: serde_json::from_value(ks) - .map_err(|e| StorageError::Internal(format!("index key schema: {e}")))?, - projection: serde_json::from_value(proj) - .map_err(|e| StorageError::Internal(format!("index projection: {e}")))?, - }); - } + .await?; // The backup's rows are read from one catalog snapshot in which the // backup is still AVAILABLE. DeleteBackup refuses while this restore @@ -537,10 +774,7 @@ impl PostgresEngine { batch_bytes += item_json.len(); batch.push((item, item_json)); if batch.len() == COPY_BATCH_ITEMS || batch_bytes >= COPY_BATCH_BYTES { - self.write_restore_batch( - &ddb_table, desc, &sort_keys, &indexes, &batch, backup_arn, - ) - .await?; + self.write_restore_batch(&target, &batch).await?; item_count += i64::try_from(batch.len()).unwrap_or(i64::MAX); batch.clear(); batch_bytes = 0; @@ -549,8 +783,7 @@ impl PostgresEngine { drop(rows); snapshot.commit().await.map_err(db_err)?; if !batch.is_empty() { - self.write_restore_batch(&ddb_table, desc, &sort_keys, &indexes, &batch, backup_arn) - .await?; + self.write_restore_batch(&target, &batch).await?; item_count += i64::try_from(batch.len()).unwrap_or(i64::MAX); } @@ -561,27 +794,16 @@ impl PostgresEngine { .await .map_err(db_err)?; - // Mark the restored table ACTIVE now that every batch has committed. - // The table was created with the transition deferred, so this is the - // first point it can become ACTIVE, by ordering rather than by timing. - // - // Only from CREATING. DeleteTable refuses a restore target, but the - // abandoned-restore sweep can claim one whose lock session was lost - // (moving it to DELETING); that claim wins, and flipping the row back - // to ACTIVE would leave a table whose data tables are being dropped. - let activated = sqlx::query( - "UPDATE tables SET item_count = $1, table_size_bytes = $2, table_status = 'ACTIVE', \ - status_transition_at = NULL WHERE table_id = $3 AND table_status = 'CREATING'", - ) - .bind(item_count) - .bind(table_size) - .bind(&desc.table_id) - .execute(&self.pool) - .await - .map_err(db_err)?; - if activated.rows_affected() == 0 { - // The target was claimed for removal while the copy ran; the - // copy's rows go with it. + // Mark the restored table ACTIVE now that every batch has committed, + // and its job record DONE, in one catalog transaction. The table was + // created with the transition deferred, so this is the first point it + // can become ACTIVE, by ordering rather than by timing. + if !self + .activate_restore_target(&desc.table_id, generation, item_count, table_size) + .await? + { + // The target was claimed for removal while the copy ran, or the + // job record passed to another claim; the copy's rows go with it. return Err(StorageError::TableNotFound(format!( "restore of {backup_arn} into {} did not complete: the table was deleted \ while it was being restored", @@ -591,96 +813,111 @@ impl PostgresEngine { Ok(()) } - /// Write and commit one batch of restored items: one multi-row INSERT - /// into the base table, then one multi-row INSERT into each secondary - /// index for the batch's items that carry that index's keys. + /// The end of a restore in one catalog transaction: the job record + /// RUNNING to DONE under `generation`, the table CREATING to ACTIVE with + /// its counts, and every secondary and vector index of the table ACTIVE + /// (the indexes are created ACTIVE with a restore target, so this is the + /// rule stated once rather than a change). `Ok(false)`, with nothing + /// written, when the record is not RUNNING under `generation` or the + /// table is not CREATING. + /// + /// Only from CREATING, and only while the job record is RUNNING under + /// this generation. DeleteTable refuses a restore target, but the + /// reconciler can claim one whose owner it found dead (moving it to + /// DELETING); that claim wins, and flipping the row back to ACTIVE + /// would leave a table whose data tables are being dropped. + pub(crate) async fn activate_restore_target( + &self, + table_id: &str, + generation: u64, + item_count: i64, + table_size: i64, + ) -> Result { + let mut tx = self.pool.begin().await.map_err(db_err)?; + let done = sqlx::query( + "UPDATE table_restores SET status = 'DONE', completed_at = now() \ + WHERE table_id = $1 AND status = 'RUNNING' AND owner_generation = $2", + ) + .bind(table_id) + .bind(generation_column(generation)) + .execute(&mut *tx) + .await + .map_err(db_err)? + .rows_affected(); + let activated = if done == 1 { + sqlx::query( + "UPDATE tables SET item_count = $1, table_size_bytes = $2, \ + table_status = 'ACTIVE', status_transition_at = NULL \ + WHERE table_id = $3 AND table_status = 'CREATING'", + ) + .bind(item_count) + .bind(table_size) + .bind(table_id) + .execute(&mut *tx) + .await + .map_err(db_err)? + .rows_affected() + } else { + 0 + }; + if activated == 0 { + tx.rollback().await.map_err(db_err)?; + return Ok(false); + } + sqlx::query( + "UPDATE indexes SET index_status = 'ACTIVE' \ + WHERE table_id = $1 AND index_status <> 'ACTIVE'", + ) + .bind(table_id) + .execute(&mut *tx) + .await + .map_err(db_err)?; + sqlx::query( + "UPDATE vector_indexes SET index_status = 'ACTIVE', backfilling = NULL \ + WHERE table_id = $1 AND index_status <> 'ACTIVE'", + ) + .bind(table_id) + .execute(&mut *tx) + .await + .map_err(db_err)?; + tx.commit().await.map_err(db_err)?; + Ok(true) + } + + /// RUNNING to FAILED on the job record of `table_id`, conditioned on + /// `generation`: the write a failing job makes before it removes its + /// target. `Ok(false)` when the record is not RUNNING under that + /// generation. + async fn fail_restore_job_as( + &self, + table_id: &str, + generation: u64, + reason: &str, + ) -> Result { + let affected = sqlx::query( + "UPDATE table_restores SET status = 'FAILED', failure_reason = $3, \ + completed_at = now() \ + WHERE table_id = $1 AND status = 'RUNNING' AND owner_generation = $2", + ) + .bind(table_id) + .bind(generation_column(generation)) + .bind(reason) + .execute(&self.pool) + .await + .map_err(db_err)? + .rows_affected(); + Ok(affected == 1) + } + + /// Write and commit one batch of restored items ([`insert_restore_rows`] + /// in one data transaction). async fn write_restore_batch( &self, - ddb_table: &str, - desc: &TableDescription, - sort_keys: &[(&str, ScalarAttributeType)], - indexes: &[RestoreIndex], + target: &RestoreTarget<'_>, batch: &[(Item, String)], - backup_arn: &str, ) -> Result<(), StorageError> { let mut tx = self.data_pool.begin().await.map_err(db_err)?; - let mut cols = vec!["pk".to_owned()]; - cols.extend( - sort_keys - .iter() - .enumerate() - .map(|(i, &(_, t))| sk_column_n(i, t)), - ); - cols.push("item_data".to_owned()); - let width = cols.len(); - // The last column is the item, bound as text and cast in SQL. - let tuples: Vec = (0..batch.len()) - .map(|r| { - let ps: Vec = (1..=width) - .map(|c| { - if c == width { - format!("${}::jsonb", r * width + c) - } else { - format!("${}", r * width + c) - } - }) - .collect(); - format!("({})", ps.join(", ")) - }) - .collect(); - let sql = format!( - "INSERT INTO {ddb_table} ({}) VALUES {}", - cols.join(", "), - tuples.join(", ") - ); - let mut query = sqlx::query(&sql); - for (item, item_json) in batch { - query = query.bind(composite_pk_to_text(item, &desc.key_schema)?); - for &(name, sk_type) in sort_keys { - let value = item.get(name).ok_or_else(|| { - StorageError::Internal(format!( - "backup {backup_arn} has an item without sort key {name}" - )) - })?; - query = match parse_sk(value, sk_type)? { - SortKeyValue::S(v) => query.bind(v), - SortKeyValue::N(v) => query.bind(v), - SortKeyValue::B(v) => query.bind(v), - }; - } - query = query.bind(item_json.as_str()); - } - query.execute(&mut *tx).await.map_err(db_err)?; - - for idx in indexes { - let idx_sks = all_sort_key_info(&idx.key_schema, &desc.attribute_definitions); - let projected: Vec<(&Item, Item)> = batch - .iter() - .filter(|(item, _)| item_has_index_keys(item, &idx.key_schema)) - .map(|(item, _)| { - ( - item, - project_item_for_index( - item, - &idx.key_schema, - &desc.key_schema, - &idx.projection, - ), - ) - }) - .collect(); - let rows: Vec<(&Item, &Item)> = projected.iter().map(|(i, p)| (*i, p)).collect(); - insert_index_rows_multi( - &mut tx, - &idx.table, - &rows, - &idx.key_schema, - &desc.key_schema, - &idx_sks, - sort_keys, - ) - .await?; - } + insert_restore_rows(&mut tx, target, batch).await?; tx.commit().await.map_err(db_err)?; Ok(()) } @@ -698,7 +935,17 @@ impl PostgresEngine { /// exactly one wins. The claim also schedules the row for the ordinary /// DELETING removal, so if a drop below fails, the row stays DELETING with /// its index rows and the control plane finishes the job later. - async fn abort_restore(&self, table_id: &str) -> Result { + /// + /// The relations dropped are the ones `DeleteTable` drops + /// (`table_storage_relations`): the data table, every secondary index's + /// table and every vector index's table, their ids read in the claiming + /// transaction while the catalog rows still exist (the `tables` row's + /// deletion cascades them away). + /// + /// Public for the test that runs it against an ACTIVE table; not an + /// interface. + #[doc(hidden)] + pub async fn abort_restore(&self, table_id: &str) -> Result { let mut tx = self.pool.begin().await.map_err(db_err)?; let claimed = sqlx::query( "UPDATE tables SET table_status = 'DELETING', status_transition_at = NOW() \ @@ -716,13 +963,17 @@ impl PostgresEngine { .fetch_all(&mut *tx) .await .map_err(db_err)?; + let vector_index_ids: Vec = + sqlx::query_scalar("SELECT index_id FROM vector_indexes WHERE table_id = $1") + .bind(table_id) + .fetch_all(&mut *tx) + .await + .map_err(db_err)?; tx.commit().await.map_err(db_err)?; if !claimed { return Ok(false); } - let mut drops = vec![data_table_name(table_id)]; - drops.extend(index_ids.iter().map(|id| index_table_name(id))); - for t in drops { + for t in table_storage_relations(table_id, &index_ids, &vector_index_ids) { sqlx::query(&format!("DROP TABLE IF EXISTS {t}")) .execute(&self.data_pool) .await @@ -736,61 +987,54 @@ impl PostgresEngine { Ok(true) } - /// Remove restore targets whose restore died with its process. - /// - /// A restore target is the only table that sits CREATING with no - /// scheduled transition. One older than the grace period whose restore - /// lock is free has no live owner: the process that created it crashed or - /// was killed mid-copy, or its lock session ended, and nothing else would - /// ever move it on. Removing it frees the name; the client sees the table - /// disappear, as it would after a failed restore. A target whose lock is - /// held is being copied by a live process, here or on another instance, - /// and is left alone. - /// - /// Each removal is logged at warn with the account, the table, and the - /// backup it was being restored from, so a table that vanishes from - /// ListTables has a server record of why. - /// - /// Returns the names of the tables removed. - pub(crate) async fn sweep_abandoned_restores(&self) -> Result, StorageError> { - let candidates: Vec<(String, String, String, Option)> = sqlx::query_as( - "SELECT t.table_id, t.account_id, t.table_name, r.source_backup_arn FROM tables t \ - LEFT JOIN table_restores r ON r.table_id = t.table_id \ - WHERE t.table_status = 'CREATING' AND t.status_transition_at IS NULL \ - AND t.creation_date_time < NOW() - make_interval(secs => $1)", + /// The body of `remove_restore_target` once its caller holds the target + /// name's advisory lock (`restore_lock_key`): the CREATING-target check, + /// the progress-row delete on the data pool (migration 7; a data + /// database from before it has no `restore_progress`), and the claim and + /// drop through [`Self::abort_restore`]. + pub async fn remove_restore_target_locked( + &self, + key: &extenddb_storage::backup_catalog::RestoreTargetKey, + ) -> Result { + let is_target: bool = sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM tables t JOIN table_restores r \ + ON r.table_id = t.table_id WHERE t.table_id = $1 AND t.account_id = $2 \ + AND t.table_name = $3 AND t.table_status = 'CREATING')", ) - .bind(f64::from(ABANDONED_RESTORE_GRACE_SECS)) - .fetch_all(&self.pool) + .bind(&key.table_id) + .bind(&key.account_id) + .bind(&key.table_name) + .fetch_one(&self.pool) .await .map_err(db_err)?; - let mut removed = Vec::new(); - for (table_id, account_id, table_name, source_backup_arn) in candidates { - let Some(_owner) = try_restore_lock(&self.pool, &account_id, &table_name, None).await? - else { - continue; - }; - if self.abort_restore(&table_id).await? { - tracing::warn!( - "removed abandoned restore target {table_name} ({table_id}) in account \ - {account_id}, restoring from {}: its restore did not finish and no \ - process owns it", - source_backup_arn - .as_deref() - .unwrap_or("") - ); - removed.push(table_name); - } + if !is_target { + return Ok(false); + } + let has_progress: bool = + sqlx::query_scalar("SELECT to_regclass('restore_progress') IS NOT NULL") + .fetch_one(&self.data_pool) + .await + .map_err(db_err)?; + if has_progress { + sqlx::query("DELETE FROM restore_progress WHERE table_id = $1") + .bind(&key.table_id) + .execute(&self.data_pool) + .await + .map_err(db_err)?; } - Ok(removed) + self.abort_restore(&key.table_id).await } - /// The detached task of RestoreTableFromBackup: wait for the account and - /// server slots, take the restore lock, register the CREATING target, - /// report the registration to the request (which may be gone), then copy. + /// The detached task of RestoreTableFromBackup: take the restore lock, + /// register the CREATING target with its job record, report the + /// registration to the request (which may be gone), then copy. `permit` + /// is the job slot the engine took, held by the claim until the copy + /// ends. async fn run_restore( self, account_id: String, request: RestoreRequest, + permit: JobPermit, registered: tokio::sync::oneshot::Sender>, ) { let RestoreRequest { @@ -803,7 +1047,8 @@ impl PostgresEngine { // A refusal the request is no longer there to receive (its client // went away) would otherwise vanish, so it is logged. A refused restore // removes what it created; a cleanup that failed is logged on its own, - // and the abandoned-restore sweep finishes it. + // and the reconciler finishes it (the FAILED-target duty, once the + // record is stale). let refuse = |registered: tokio::sync::oneshot::Sender<_>, e: StorageError| { if let Err(Err(e)) = registered.send(Err(e)) { tracing::warn!( @@ -812,15 +1057,13 @@ impl PostgresEngine { ); } }; - // The name first, unfenced, as on the other backends and before a - // slot is taken: a restore into a name that is a CREATING restore - // target is refused as the name (TableBeingRestored, the service's - // TableInUseException) at once, whatever state the backup is in and - // whether or not that target's restore is still running (an - // abandoned target holds no restore lock until the sweep removes - // it). A client retrying its own restore is answered here instead of - // waiting for a slot. The restore lock and the target insert stay - // the fence. + // The name first, unfenced, as on the other backends: a restore into + // a name that is a CREATING restore target is refused as the name + // (TableBeingRestored, the service's TableInUseException) at once, + // whatever state the backup is in and whether or not that target's + // restore is still running (an abandoned target holds no restore + // lock until the reconciler removes it). The restore lock and the + // target insert stay the fence. if let Some(source_backup_arn) = self.restoring_into(&account_id, &target).await { refuse( registered, @@ -831,32 +1074,15 @@ impl PostgresEngine { ); return; } - // Admission: at most RESTORE_SLOTS_PER_ACCOUNT restores per account - // and restore_server_slot_count per process at once. The permits - // travel with the lock owner and are released when the copy ends. - let slots = match acquire_restore_permits( - self.restore_account_slots.for_account(&account_id), - Arc::clone(&self.restore_server_slots), - self.restore_server_slot_count, - std::time::Duration::from_secs(RESTORE_SLOT_WAIT_SECS), - ) - .await - { - Ok(slots) => slots, - Err(e) => { - refuse(registered, e); - return; - } - }; - // Ownership first, before the target exists, so the abandoned-restore - // sweep never sees this target without an owner, including while the - // DDL runs. It is held until the copy ends; a crash anywhere releases - // it with the connection, and the sweep then removes the target. A - // lock already held means another restore into this name is in - // flight right now: TableBeingRestored naming its source backup once - // its CREATING row is visible, else the name is taken as CreateTable - // would report it. - let owner = match try_restore_lock(&self.pool, &account_id, &target, Some(slots)).await { + // Ownership first, before the target exists, so nothing ever sees + // this target without an owner, including while the DDL runs. It is + // held until the copy ends; a crash anywhere releases it with the + // connection, and the reconciler removes the target once the job + // record's heartbeat is stale. A lock already held means another + // restore into this name is in flight right now: TableBeingRestored + // naming its source backup once its CREATING row is visible, else + // the name is taken as CreateTable would report it. + let owner = match try_restore_lock(&self.pool, &account_id, &target, permit).await { Ok(Some(owner)) => owner, Ok(None) => { let error = self.name_taken_error(&account_id, &target, true).await; @@ -877,9 +1103,10 @@ impl PostgresEngine { backup_arn: &backup_arn, source_table_arn: &source_table_arn, restore_date_time, + owner_generation: owner.generation(), }; let mut desc = match self - .create_table_for_restore(&account_id, input, ®istration) + .create_table_for_restore(&account_id, input, RestoreSource::Legacy(®istration)) .await { Ok(desc) => desc, @@ -905,29 +1132,31 @@ impl PostgresEngine { } /// The detached half of RestoreTableFromBackup: copy the backup into the - /// registered CREATING target while heartbeating the lock session, then - /// flip the target ACTIVE, or remove it on failure. Owns everything it - /// uses; the request that spawned it may be gone. + /// registered CREATING target while heartbeating the job record through + /// the claim session, then flip the target ACTIVE (which marks the job + /// record DONE, conditioned on the generation), or remove it on failure. + /// Owns everything it uses; the request that spawned it may be gone. async fn run_restore_copy( self, - mut owner: RestoreOwner, + mut owner: ClaimSession, account_id: String, desc: TableDescription, backup_arn: String, ) { let table_name = desc.table_name.clone(); let outcome = { - let copy = self.copy_backup_items(&desc, &backup_arn); - let heartbeat = owner.heartbeat(); + let key = [desc.table_id.as_str()]; + let copy = self.copy_backup_items(&desc, &backup_arn, owner.generation()); + let heartbeat = owner.heartbeat_loop(RESTORE_HEARTBEAT, &key); tokio::select! { copied = copy => copied, lost = heartbeat => { tracing::error!( "restore of {backup_arn} into {table_name} (account {account_id}): the \ - restore lock session failed its heartbeat ({lost}); stopping the copy" + claim session failed its heartbeat ({lost}); stopping the copy" ); Err(StorageError::Internal(format!( - "restore lock session lost: {lost}" + "claim session lost: {lost}" ))) } } @@ -943,17 +1172,44 @@ impl PostgresEngine { }; // The target must not stay behind: it is CREATING with no scheduled // transition, so nothing else would move it on, and the name would - // stay taken. Removing it cascades its table_restores row. The lock - // is still held here (unless its session is what failed), so the - // sweep does not race this removal; both are conditional on CREATING - // in any case. + // stay taken. The job record is marked FAILED first, conditioned on + // the generation and on RUNNING; only if that matched is the target + // removed (which cascades the record). A record another generation + // owns, or a failed FAILED write (on a pooled connection, not the + // claim's), means the claim holder owns the target, and nothing + // further is touched. + let failed = self + .fail_restore_job_as(&desc.table_id, owner.generation(), &e.to_string()) + .await; + match failed { + Ok(true) => {} + Ok(false) => { + tracing::warn!( + "restore of {backup_arn} into {table_name} (account {account_id}) ended \ + with \"{e}\"; its job record was no longer RUNNING under this claim, so \ + the target ({}) was left to its owner", + desc.table_id + ); + drop(owner); + return; + } + Err(write) => { + tracing::error!( + "restore of {backup_arn} into {table_name} (account {account_id}) failed \ + ({e}), and its job record could not be marked FAILED ({write}); the \ + reconciler removes the target once its heartbeat is stale" + ); + drop(owner); + return; + } + } match self.abort_restore(&desc.table_id).await { Ok(true) => tracing::error!( "restore of {backup_arn} into {table_name} (account {account_id}) failed and \ the partial table was removed: {e}" ), - // Already claimed by the sweep (whose own removal is logged), or - // the activation committed before the copy was stopped. + // Already claimed by the reconciler (whose own removal is logged), + // or the activation committed before the copy was stopped. Ok(false) => tracing::warn!( "restore of {backup_arn} into {table_name} (account {account_id}) ended with \ \"{e}\"; the target ({}) was no longer CREATING, so it was left to its \ @@ -963,8 +1219,8 @@ impl PostgresEngine { Err(cleanup) => tracing::error!( "restore of {backup_arn} into {table_name} (account {account_id}) failed ({e}), \ and the partial table ({}) could not be removed yet; the control plane (if it \ - reached DELETING) or the abandoned-restore sweep (if still CREATING) finishes \ - the removal: {cleanup}", + reached DELETING) or the reconciler (if still CREATING, once the FAILED record \ + is older than stale_job_after_secs) finishes the removal: {cleanup}", desc.table_id ), } @@ -972,17 +1228,21 @@ impl PostgresEngine { } } -impl BackupEngine for PostgresEngine { - fn create_backup( +impl LegacyBackupReader for PostgresEngine { + fn create( &self, account_id: &str, table_name: &str, backup_name: &str, + permit: JobPermit, ) -> BoxFuture<'_, Result> { let account_id = account_id.to_string(); let table_name = table_name.to_string(); let backup_name = backup_name.to_string(); Box::pin(async move { + // The slot is the create's for its whole run; the copy is + // synchronous, so it is given back when this returns. + let _permit = permit; // The table's definition and its items must describe one instant. // They live in different databases, so no single snapshot covers // both. Instead the table row is held FOR SHARE from the first @@ -1265,7 +1525,7 @@ impl BackupEngine for PostgresEngine { }) } - fn describe_backup( + fn describe( &self, account_id: &str, backup_arn: &str, @@ -1349,7 +1609,7 @@ impl BackupEngine for PostgresEngine { }) } - fn list_backups( + fn list( &self, account_id: &str, table_name: Option<&str>, @@ -1425,7 +1685,7 @@ impl BackupEngine for PostgresEngine { }) } - fn delete_backup( + fn delete( &self, account_id: &str, backup_arn: &str, @@ -1435,7 +1695,7 @@ impl BackupEngine for PostgresEngine { Box::pin(async move { // Resolves account-scoped, so a backup owned by another account is // reported missing here and the writes below never run. - let desc = self.describe_backup(&account_id, &backup_arn).await?; + let desc = self.describe(&account_id, &backup_arn).await?; // One transaction, so a restore reading its snapshot sees either // the whole backup or a DELETED one, never AVAILABLE with its rows @@ -1455,18 +1715,10 @@ impl BackupEngine for PostgresEngine { .execute(&mut *tx) .await .map_err(db_err)?; - let restoring: Option = sqlx::query_scalar( - "SELECT t.table_name FROM table_restores r JOIN tables t ON t.table_id = r.table_id \ - WHERE r.source_backup_arn = $1 AND t.table_status = 'CREATING' LIMIT 1", - ) - .bind(&backup_arn) - .fetch_optional(&mut *tx) - .await - .map_err(db_err)?; - if let Some(table) = restoring { - return Err(StorageError::BackupInUse(format!( - "Backup is being used to restore table {table}: {backup_arn}" - ))); + if let Some(table) = + crate::backup_catalog::creating_restore_target(&mut *tx, &backup_arn).await? + { + return Err(crate::backup_catalog::backup_in_use(&table, &backup_arn)); } sqlx::query( "DELETE FROM backup_items WHERE backup_arn = $1 AND EXISTS (\ @@ -1510,7 +1762,7 @@ impl BackupEngine for PostgresEngine { }) } - fn backup_restore_source( + fn restore_source( &self, account_id: &str, backup_arn: &str, @@ -1628,37 +1880,44 @@ impl BackupEngine for PostgresEngine { /// runs in a detached task. /// /// The request hands everything to a `tokio::spawn`ed task that owns its - /// own clones: wait for a slot of its account's - /// ([`RESTORE_SLOTS_PER_ACCOUNT`]) and then a slot of the server's - /// ([`restore_server_slot_count`]), take the restore lock, create the - /// target and its `table_restores` row in one catalog transaction (which - /// re-checks the backup is AVAILABLE and owned by the account, holding it - /// FOR UPDATE), report that result back through a channel, then copy. - /// Dropping the request future at any point therefore changes nothing: - /// a restore refused for want of a slot creates nothing, the - /// registration still commits or fails on its own, and a registered - /// target is still copied and flipped ACTIVE, or removed on failure. - fn restore_table_from_backup( + /// own clones: take the restore lock, create the target and its + /// `table_restores` row (the job record, RUNNING under the claim's + /// generation) in one catalog transaction (which re-checks the backup is + /// AVAILABLE and owned by the account, holding it FOR UPDATE), report + /// that result back through a channel, then copy. Dropping the request + /// future at any point therefore changes nothing: the registration still + /// commits or fails on its own, and a registered target is still copied + /// and flipped ACTIVE, or removed on failure. + fn restore( &self, account_id: &str, request: RestoreRequest, + permit: JobPermit, ) -> BoxFuture<'_, Result> { let account_id = account_id.to_string(); Box::pin(async move { let (registered_tx, registered_rx) = tokio::sync::oneshot::channel(); let engine = self.clone(); - tokio::spawn(engine.run_restore(account_id, request, registered_tx)); + tokio::spawn(engine.run_restore(account_id, request, permit, registered_tx)); registered_rx.await.map_err(|_| { StorageError::Internal("the restore task ended before registering".to_owned()) })? }) } + + /// The in-catalog create commits its row AVAILABLE in one transaction + /// with the items, so it never leaves a CREATING row. + fn abandoned_creating_backups( + &self, + _stale_after: Duration, + ) -> BoxFuture<'_, Result, StorageError>> { + Box::pin(async { Ok(Vec::new()) }) + } } #[cfg(test)] mod tests { use std::borrow::Cow; - use std::time::Duration; use super::*; @@ -1703,10 +1962,14 @@ mod tests { #[test] fn only_too_many_connections_is_limit_exceeded() { - assert!(matches!( - lock_connection_error(&reply(Some("53300"))), - StorageError::LimitExceeded(_) - )); + match lock_connection_error(&reply(Some("53300"))) { + StorageError::LimitExceeded(m) => assert_eq!( + m, + limit_exceeded_message(JobBound::ServerConnections), + "the one job-refusal text, naming the server's connection slots" + ), + other => panic!("too_many_connections is LimitExceeded: {other:?}"), + } // invalid_password, invalid_catalog_name, cannot_connect_now, // insufficient_resources (53000, same class), and no SQLSTATE. for code in [ @@ -1792,91 +2055,4 @@ mod tests { ); } } - - #[test] - fn the_server_bound_is_half_the_pool_within_one_and_sixteen() { - assert_eq!(restore_server_slot_count(0), 1); - assert_eq!(restore_server_slot_count(1), 1); - assert_eq!(restore_server_slot_count(3), 1); - assert_eq!(restore_server_slot_count(10), 5); - assert_eq!(restore_server_slot_count(20), 10); - assert_eq!(restore_server_slot_count(32), 16); - assert_eq!(restore_server_slot_count(33), 16); - assert_eq!(restore_server_slot_count(1000), 16); - } - - /// A full server bound refuses with LimitExceeded naming the server, a - /// full account with LimitExceeded naming the account; a free slot of - /// each is granted, and releasing a held one lets the next restore in. - #[tokio::test] - async fn restore_permits_wait_then_limit_exceeded() { - let wait = Duration::from_millis(50); - let account = Arc::new(Semaphore::new(RESTORE_SLOTS_PER_ACCOUNT)); - let server = Arc::new(Semaphore::new(1)); - - let held = acquire_restore_permits(Arc::clone(&account), Arc::clone(&server), 1, wait) - .await - .expect("the first restore gets both slots"); - let other_account = Arc::new(Semaphore::new(RESTORE_SLOTS_PER_ACCOUNT)); - match acquire_restore_permits(Arc::clone(&other_account), Arc::clone(&server), 1, wait) - .await - { - Err(StorageError::LimitExceeded(m)) => { - assert!(m.contains("on this server (at most 1"), "{m}"); - } - other => panic!("a full server bound must refuse: {:?}", other.err()), - } - // The refused restore gave its account slot back. - assert_eq!(other_account.available_permits(), RESTORE_SLOTS_PER_ACCOUNT); - drop(held); - acquire_restore_permits(Arc::clone(&other_account), Arc::clone(&server), 1, wait) - .await - .expect("a freed server slot is granted"); - - let full_account = Arc::new(Semaphore::new(0)); - let roomy_server = Arc::new(Semaphore::new(4)); - match acquire_restore_permits(full_account, Arc::clone(&roomy_server), 4, wait).await { - Err(StorageError::LimitExceeded(m)) => { - assert!(m.contains("for this account"), "{m}"); - } - other => panic!("a full account must refuse: {:?}", other.err()), - } - assert_eq!(roomy_server.available_permits(), 4); - } - - /// One engine's account slots are shared by its clones and by nothing - /// else: a second engine in the same process has its own full allowance - /// for the same account. - #[test] - fn account_restore_slots_belong_to_one_engine() { - let slots = RestoreAccountSlots::default(); - let held = slots - .for_account("123456789012") - .try_acquire_owned() - .expect("a free slot"); - assert_eq!( - slots - .clone() - .for_account("123456789012") - .available_permits(), - RESTORE_SLOTS_PER_ACCOUNT - 1, - "a clone sees the same account map" - ); - assert_eq!( - RestoreAccountSlots::default() - .for_account("123456789012") - .available_permits(), - RESTORE_SLOTS_PER_ACCOUNT, - "another engine has its own" - ); - drop(held); - // Nothing in flight for the account: its entry is pruned on the next - // lookup of any account, and a fresh one has the full allowance. - slots.for_account("other"); - assert_eq!(slots.0.lock().unwrap().len(), 1); - assert_eq!( - slots.for_account("123456789012").available_permits(), - RESTORE_SLOTS_PER_ACCOUNT - ); - } } diff --git a/crates/storage-postgres/src/create_table.rs b/crates/storage-postgres/src/create_table.rs index daabe324e..61e8928c2 100755 --- a/crates/storage-postgres/src/create_table.rs +++ b/crates/storage-postgres/src/create_table.rs @@ -7,13 +7,14 @@ use extenddb_core::types::{ BillingMode, BillingModeSummary, CreateTableInput, GsiDescription, LsiDescription, ProvisionedThroughputDescription, SseDescription, SseType, TableDescription, TableStatus, }; +use extenddb_storage::backup_catalog::RestoreRegistrationRow; use extenddb_storage::error::StorageError; use extenddb_storage::util::{index_arn, stream_arn, table_arn}; use crate::PostgresEngine; -/// What a restore target records about where it came from, written in the -/// same catalog transaction that creates it. +/// What a legacy restore target records about where it came from, written +/// in the same catalog transaction that creates it. pub(crate) struct RestoreRegistration<'a> { pub(crate) backup_arn: &'a str, /// Reported as `RestoreSummary.SourceTableArn`. @@ -21,6 +22,21 @@ pub(crate) struct RestoreRegistration<'a> { /// Reported as `RestoreSummary.RestoreDateTime`, epoch seconds: the value /// the engine passes (the source backup's creation time), not "now". pub(crate) restore_date_time: f64, + /// The job claim's fencing token, written on the job record (RUNNING, + /// `kind = legacy`) with the target. + pub(crate) owner_generation: u64, +} + +/// Which backup a restore target is registered from, and so which row its +/// registration locks: a legacy restore locks the `backups` row of the +/// in-catalog backup, a store-backed one the `backup_index` row (the lock +/// DeleteBackup's `start_backup_delete` takes). The two paths keep their own +/// locks: a legacy and a store-backed restore are of two different backups +/// and never meet, and a DeleteBackup of a `backup_index` row meets only +/// store-backed registrations. +pub(crate) enum RestoreSource<'a> { + Legacy(&'a RestoreRegistration<'a>), + Manifest(&'a RestoreRegistrationRow), } impl PostgresEngine { @@ -51,7 +67,7 @@ impl PostgresEngine { &self, account_id: &str, input: CreateTableInput, - registration: &RestoreRegistration<'_>, + registration: RestoreSource<'_>, ) -> Result { self.create_table_impl_inner(account_id, input, true, Some(registration)) .await @@ -104,7 +120,7 @@ impl PostgresEngine { account_id: &str, input: CreateTableInput, defer_active: bool, - restore: Option<&RestoreRegistration<'_>>, + restore: Option>, ) -> Result { Self::validate_account_id(account_id)?; let table_id = uuid::Uuid::new_v4().to_string(); @@ -146,40 +162,67 @@ impl PostgresEngine { .await .map_err(|e| StorageError::Internal(e.to_string()))?; - if let Some(RestoreRegistration { backup_arn, .. }) = restore { - // FOR UPDATE, not FOR SHARE: two restores of one backup must not - // both pass the CREATING-target check below, so they serialize on - // the backup row (DeleteBackup locks it the same way). - let available: Option = sqlx::query_scalar( - "SELECT 1 FROM backups WHERE backup_arn = $1 AND account_id = $2 \ - AND backup_status = 'AVAILABLE' FOR UPDATE", - ) - .bind(backup_arn) - .bind(account_id) - .fetch_optional(&mut *tx) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - if available.is_none() { - return Err(StorageError::Validation(format!( - "Backup not found: {backup_arn}" - ))); + match &restore { + Some(RestoreSource::Legacy(RestoreRegistration { backup_arn, .. })) => { + // FOR UPDATE, not FOR SHARE: two restores of one backup must not + // both pass the CREATING-target check below, so they serialize on + // the backup row (DeleteBackup locks it the same way). + let available: Option = sqlx::query_scalar( + "SELECT 1 FROM backups WHERE backup_arn = $1 AND account_id = $2 \ + AND backup_status = 'AVAILABLE' FOR UPDATE", + ) + .bind(backup_arn) + .bind(account_id) + .fetch_optional(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + if available.is_none() { + return Err(StorageError::Validation(format!( + "Backup not found: {backup_arn}" + ))); + } + // One restore of a backup at a time, as the service: while a + // target restored from it is CREATING the backup is in use. Same + // predicate as DeleteBackup's. + if let Some(table) = + crate::backup_catalog::creating_restore_target(&mut *tx, backup_arn).await? + { + return Err(crate::backup_catalog::backup_in_use(&table, backup_arn)); + } } - // One restore of a backup at a time, as the service: while a - // target restored from it is CREATING the backup is in use. Same - // predicate as DeleteBackup's. - let restoring: Option = sqlx::query_scalar( - "SELECT t.table_name FROM table_restores r JOIN tables t ON t.table_id = r.table_id \ - WHERE r.source_backup_arn = $1 AND t.table_status = 'CREATING' LIMIT 1", - ) - .bind(backup_arn) - .fetch_optional(&mut *tx) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - if let Some(table) = restoring { - return Err(StorageError::BackupInUse(format!( - "Backup is being used to restore table {table}: {backup_arn}" - ))); + Some(RestoreSource::Manifest(registration)) => { + // The `backup_index` row FOR UPDATE: the lock + // `start_backup_delete` takes before its own in-use check, so + // a DeleteBackup and this registration order wholly one + // before the other, and two registrations of one backup + // serialize on the row before the CREATING-target check. + let status: Option = sqlx::query_scalar( + "SELECT status FROM backup_index WHERE account_id = $1 AND backup_id = $2 \ + FOR UPDATE", + ) + .bind(account_id) + .bind(®istration.backup.backup_id) + .fetch_optional(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + if status.as_deref() != Some("AVAILABLE") { + return Err(extenddb_storage::backup_catalog::backup_not_found( + ®istration.backup.backup_arn, + )); + } + if let Some(table) = crate::backup_catalog::creating_restore_target( + &mut *tx, + ®istration.backup.backup_arn, + ) + .await? + { + return Err(crate::backup_catalog::backup_in_use( + &table, + ®istration.backup.backup_arn, + )); + } } + None => {} } // Insert table metadata, returning creation timestamp and actual status. @@ -241,19 +284,53 @@ impl PostgresEngine { Err(e) => return Err(StorageError::Internal(e.to_string())), }; - if let Some(r) = restore { - sqlx::query( - "INSERT INTO table_restores \ - (table_id, source_backup_arn, source_table_arn, restore_date_time) \ - VALUES ($1, $2, $3, to_timestamp($4))", - ) - .bind(&table_id) - .bind(r.backup_arn) - .bind(r.source_table_arn) - .bind(r.restore_date_time) - .execute(&mut *tx) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; + // The job record: RUNNING under the claim's generation, with a + // heartbeat on the catalog's clock, so no CREATING target ever + // exists without an owned record. `now()` is this transaction's + // start, not the insert's instant: a registration that waited on the + // `backup_index` row's lock above commits a heartbeat that old, and + // the data DDL after the commit adds to the age the engine's first + // beat (sent before the registration's reply) then refreshes. + match &restore { + Some(RestoreSource::Legacy(r)) => { + sqlx::query( + "INSERT INTO table_restores \ + (table_id, source_backup_arn, source_table_arn, restore_date_time, \ + status, kind, owner_generation, heartbeat_at) \ + VALUES ($1, $2, $3, to_timestamp($4), 'RUNNING', 'legacy', $5, now())", + ) + .bind(&table_id) + .bind(r.backup_arn) + .bind(r.source_table_arn) + .bind(r.restore_date_time) + .bind(crate::backup_engine::generation_column(r.owner_generation)) + .execute(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + } + Some(RestoreSource::Manifest(r)) => { + sqlx::query( + "INSERT INTO table_restores \ + (table_id, source_backup_arn, source_table_arn, restore_date_time, \ + status, kind, store_kind, store_root, store_id, spool_host, \ + owner_generation, heartbeat_at) \ + VALUES ($1, $2, $3, to_timestamp($4), 'RUNNING', 'manifest', $5, $6, $7, \ + $8, $9, now())", + ) + .bind(&table_id) + .bind(&r.backup.backup_arn) + .bind(&r.source_table_arn) + .bind(r.restore_date_time) + .bind(r.store.store_kind.as_str()) + .bind(&r.store.store_root) + .bind(&r.store.store_id) + .bind(r.spool_host.as_deref()) + .bind(crate::backup_engine::generation_column(r.owner_generation)) + .execute(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + } + None => {} } // Insert GSI metadata diff --git a/crates/storage-postgres/src/delete_table.rs b/crates/storage-postgres/src/delete_table.rs index 81b75856b..6688b5caf 100755 --- a/crates/storage-postgres/src/delete_table.rs +++ b/crates/storage-postgres/src/delete_table.rs @@ -94,7 +94,7 @@ impl PostgresEngine { // filled by RestoreTableFromBackup. The service refuses to delete a // table it is still creating, and deleting this one would make the // restore fail part-way; refuse until the restore finishes. An - // abandoned target is removed by the restore sweep, not here. + // abandoned target is removed by the reconciler, not here. if row.table_status == "CREATING" { let restoring: bool = sqlx::query_scalar( "SELECT EXISTS(SELECT 1 FROM tables WHERE table_id = $1 \ diff --git a/crates/storage-postgres/src/lib.rs b/crates/storage-postgres/src/lib.rs index 81480973c..e916f9cb9 100755 --- a/crates/storage-postgres/src/lib.rs +++ b/crates/storage-postgres/src/lib.rs @@ -10,6 +10,7 @@ mod admin_store; mod authorization_store; +mod backup_catalog; mod backup_engine; mod bootstrapper; mod catalog_store; @@ -24,9 +25,12 @@ mod metadata_engine; pub mod migrations; mod operations; mod pg_util; +mod restore_engine; mod snapshot_engine; #[doc(hidden)] -pub use snapshot_engine::map_fetch_error; +pub use snapshot_engine::{ + MIN_NEXT_PAGE_ROWS, MIN_PAGE_ROWS, SNAPSHOT_SLOT_BOUND, map_fetch_error, page_sql_for, +}; mod stream_engine; #[doc(hidden)] pub mod stream_records_sql; @@ -39,6 +43,11 @@ mod vector_search; mod worker_store; mod workers; +#[cfg(feature = "test-hooks")] +#[doc(hidden)] +pub use backup_engine::claim_session_settings; +#[doc(hidden)] +pub use backup_engine::{CLAIM_KEEPALIVES, RESTORE_LOCK_SEED}; pub use bootstrapper::PostgresBootstrapper; pub use catalog_store::PostgresCatalogStore; pub use config::PostgresStorageConfig; @@ -96,6 +105,11 @@ pub use delete_table::test_pause::{ #[cfg(feature = "test-hooks")] #[doc(hidden)] pub use migrations::apply_all_migrations; +/// Pause `sweep_restore_progress` between its two reads, so a test can +/// commit a registration and a progress row in between. +#[cfg(feature = "test-hooks")] +#[doc(hidden)] +pub use restore_engine::sweep_pause; /// The `PostgreSQL` storage backend. /// @@ -261,15 +275,6 @@ pub struct PostgresEngine { /// the search path exists, so this backend still refuses vector operations /// over the wire; the flag is what that decision will read. pub(crate) vector_capable: bool, - /// Restores this engine (one per server process) may have in flight - /// across all accounts; see `backup_engine::restore_server_slot_count`. - pub(crate) restore_server_slots: Arc, - /// The number of permits `restore_server_slots` was created with, for - /// the refusal message. - pub(crate) restore_server_slot_count: usize, - /// Restores each account may run at once in this process; see - /// `backup_engine::RestoreAccountSlots`. - pub(crate) restore_account_slots: backup_engine::RestoreAccountSlots, /// The serving registration a server holds for its life, so `migrate /// --yes` refuses while it serves, and takes again after the database /// ended its session. `None` outside a server. @@ -290,14 +295,18 @@ pub struct PostgresEngine { /// /// What the permits do not prevent: request load alone exhausting a pool /// (an open then waits for its connection up to the acquire timeout like - /// any request), and restores, which take their own slots (half the - /// pool, at most 16: `backup_engine::restore_server_slot_count`), so - /// snapshots and restores together can hold the whole pool. On the separate layout requests that block on + /// any request), and restores, which hold no snapshot slot (their bound + /// is the engine's `max_concurrent_jobs`, which the pool is raised to + /// twice plus ten), so snapshots and restores together can hold the + /// whole pool. On the separate layout requests that block on /// the `tables` row an open holds `FOR SHARE` (`UpdateTable`, /// `DeleteTable` of that table) hold catalog connections while they /// wait; the open needs only a data connection by then, so they delay it /// no further. pub(crate) snapshot_open_slots: Arc, + /// The permits `snapshot_open_slots` was built with, for the refusal + /// that names the bound. + pub(crate) snapshot_slot_limit: usize, /// Whether `data_pool` is the catalog pool (no separate data database). /// The snapshot open uses one transaction on that layout, and /// `DeleteTable` runs its in-use probe and its synchronous drop on the @@ -413,7 +422,6 @@ impl PostgresEngine { ), } - let server_slots = backup_engine::restore_server_slot_count(pool_size); Ok(Self { pool, data_pool, @@ -425,13 +433,11 @@ impl PostgresEngine { initial_gsi_delay, )), vector_capable, - restore_server_slots: Arc::new(tokio::sync::Semaphore::new(server_slots)), - restore_server_slot_count: server_slots, - restore_account_slots: backup_engine::RestoreAccountSlots::default(), serving: None, snapshot_open_slots: Arc::new(tokio::sync::Semaphore::new(snapshot_slot_count( pool_size, ))), + snapshot_slot_limit: snapshot_slot_count(pool_size), shared_data_pool, #[cfg(feature = "test-hooks")] drop_attempts: Arc::default(), @@ -784,6 +790,24 @@ impl ServerRuntimeHooks for PostgresRuntimeHooks { } } +/// A pool size raised to what `max_concurrent_jobs` jobs need, when it is +/// smaller (`pool_size_for_jobs`: twice the jobs plus ten, so a snapshot +/// bound of half the pool admits every job the permit allows; the job's +/// claim connection is outside the pool). A raise is logged at warn, +/// naming both numbers, as the clamp to [`MIN_POOL_SIZE`] is; a size at or +/// above the floor is returned as configured, and the minimum clamp still +/// applies to it. +fn raise_pool_size_for_jobs(key: &str, configured: u32, floor: u32, jobs: usize) -> u32 { + if configured >= floor || floor <= MIN_POOL_SIZE { + return configured; + } + tracing::warn!( + "{key} = {configured} is below the {floor} connections backup.max_concurrent_jobs = \ + {jobs} needs (2 * max_concurrent_jobs + 10); raising it to {floor}" + ); + floor +} + /// Build server components for the Postgres backend (registered in [`register`]). fn server_components_factory( config: &(dyn extenddb_storage::config::StorageConfig + 'static), @@ -796,13 +820,32 @@ fn server_components_factory( Box> + Send>, > { let connection_string = config.connection_config().to_string(); - let max_connections = config.max_connections(); - let max_catalog_connections = config.max_catalog_connections(); + // Both pools admit the jobs the instance may run at once (the rule is + // `pool_size_for_jobs`'s). + let job_floor = + extenddb_storage::backup_catalog::pool_size_for_jobs(options.max_concurrent_jobs); + let max_connections = raise_pool_size_for_jobs( + "storage.postgres.pool_size", + config.max_connections(), + job_floor, + options.max_concurrent_jobs, + ); + let max_catalog_connections = raise_pool_size_for_jobs( + "storage.postgres.catalog_pool_size", + config.max_catalog_connections(), + job_floor, + options.max_concurrent_jobs, + ); let region = region.to_string(); Box::pin(async move { - let registration = migrations::ServerRegistration::register(&connection_string) - .await - .map_err(BackendError::Catalog)?; + // Every caller registers, a command too: a migration must not run + // under one. A command's session carries its own name. + let registration = migrations::ServerRegistration::register_as( + &connection_string, + options.command.unwrap_or(migrations::SERVE_APPLICATION), + ) + .await + .map_err(BackendError::Catalog)?; // Build PostgresConfig from extracted values let pg_config = PostgresConfig { @@ -820,23 +863,37 @@ fn server_components_factory( })?; engine.serving = Some(Arc::new(registration)); - // Rebuild any vector index a crash left CREATING, before serving: an index - // in that state is not searchable, and nothing else will repair it. - match crate::data::vector_index::reconcile_incomplete_vector_indexes(&engine).await { - Ok(n) if n > 0 => { - tracing::info!("Reconciled {n} incomplete vector index(es) at startup"); + // The startup recovery is a server's; a command reading beside one, + // or one that writes beside the live servers (`backup purge`, + // `options.startup_recovery` off), leaves it to the server. + let recover = !options.beside_server && options.startup_recovery; + if recover { + // Rebuild any vector index a crash left CREATING, before serving: + // an index in that state is not searchable, and nothing else will + // repair it. + match crate::data::vector_index::reconcile_incomplete_vector_indexes(&engine).await { + Ok(n) if n > 0 => { + tracing::info!("Reconciled {n} incomplete vector index(es) at startup"); + } + Ok(_) => {} + Err(e) => tracing::error!("Failed to reconcile incomplete vector indexes: {e}"), } - Ok(_) => {} - Err(e) => tracing::error!("Failed to reconcile incomplete vector indexes: {e}"), - } - // Recover control plane transitions (errors are logged, not fatal). - // A drop deferred by a session still holding the table wakes the - // poller, which `spawn_workers` starts later. - workers::recover_control_plane_at_startup(&engine).await; + // Recover control plane transitions (errors are logged, not fatal). + // A drop deferred by a session still holding the table wakes the + // poller, which `spawn_workers` starts later. + workers::recover_control_plane_at_startup(&engine).await; + } - // Start GSI workers - let engine = engine.start_gsi_workers(); + // Start GSI workers: a server's. A command (`backup list`, the + // purge's dry run, `purge --confirm`) leaves the queue to the + // server that owns the catalog, so the command applies no pending + // index write and exits with none half-applied. + let engine = if recover { + engine.start_gsi_workers() + } else { + engine + }; // Get data database name for logging (before wrapping in Arc) let data_db_name = engine @@ -915,3 +972,25 @@ fn server_components_factory( }) }) } + +#[cfg(test)] +mod pool_size_tests { + use super::{MIN_POOL_SIZE, raise_pool_size_for_jobs}; + use extenddb_storage::backup_catalog::pool_size_for_jobs; + + #[test] + fn a_pool_under_the_job_floor_is_raised_and_one_at_it_is_kept() { + let floor = pool_size_for_jobs(4); + assert_eq!(floor, 18); + assert_eq!(raise_pool_size_for_jobs("pool_size", 12, floor, 4), 18); + assert_eq!(raise_pool_size_for_jobs("pool_size", 18, floor, 4), 18); + assert_eq!(raise_pool_size_for_jobs("pool_size", 20, floor, 4), 20); + // No jobs asked for: the floor is the minimum, which the minimum + // clamp applies on its own. + assert_eq!( + raise_pool_size_for_jobs("pool_size", 5, pool_size_for_jobs(0), 0), + 5 + ); + assert!(pool_size_for_jobs(0) <= MIN_POOL_SIZE); + } +} diff --git a/crates/storage-postgres/src/migrations/locks.rs b/crates/storage-postgres/src/migrations/locks.rs index 3d0e1b6b9..d5ac4e2fc 100644 --- a/crates/storage-postgres/src/migrations/locks.rs +++ b/crates/storage-postgres/src/migrations/locks.rs @@ -54,6 +54,12 @@ pub(crate) const SERVING_KEY: i32 = 2; /// How often a holder runs `SELECT 1` on its lock connection. pub const KEEPALIVE_INTERVAL: Duration = Duration::from_secs(15); +/// The `application_name` of a server's serving session. A command that +/// opens the catalog as a server does (`extenddb backup list`) names its +/// own session after itself, so a migration waiting for the registrations +/// names the command rather than a server. +pub const SERVE_APPLICATION: &str = "extenddb serve"; + /// Server-side keepalives of the serving session: the first probe after 10 s /// of silence, then one every 5 s, and the session ends after 4 unanswered. const TCP_KEEPALIVES: [(&str, &str); 3] = [ @@ -413,16 +419,16 @@ pub struct ServingRegistration { } impl ServingRegistration { - /// Register, with the session named `extenddb serve`. The key is taken - /// with a try: a migration that holds or waits for it is never waited - /// for. + /// Register, with the session named `extenddb serve` + /// ([`SERVE_APPLICATION`]). The key is taken with a try: a migration + /// that holds or waits for it is never waited for. /// /// # Errors /// /// [`MigrationError::Locked`] (exit 6) when a migration is running, so a /// restart loop retries. pub async fn register(catalog: &PgConnectOptions) -> Result { - Self::register_keepalive(catalog, KEEPALIVE_INTERVAL).await + Self::register_keepalive(catalog, KEEPALIVE_INTERVAL, SERVE_APPLICATION).await } /// [`ServingRegistration::register`] with another keepalive interval, for @@ -436,15 +442,18 @@ impl ServingRegistration { catalog: &PgConnectOptions, keepalive: Duration, ) -> Result { - Self::register_keepalive(catalog, keepalive).await + Self::register_keepalive(catalog, keepalive, SERVE_APPLICATION).await } + /// Register with the session named `application`: `extenddb serve` for + /// a server, the command's name for a command. pub(crate) async fn register_keepalive( catalog: &PgConnectOptions, keepalive: Duration, + application: &str, ) -> Result { - let mut conn = connect(catalog, "extenddb serve").await?; - match register_on(&mut conn).await { + let mut conn = connect(catalog, application).await?; + match register_on(&mut conn, application).await { Ok(()) => {} Err(e) => { let _ = statement(&mut conn, "ROLLBACK").await; @@ -514,7 +523,15 @@ impl Drop for ServingRegistration { } } -async fn register_on(conn: &mut PgConnection) -> Result<(), MigrationError> { +async fn register_on(conn: &mut PgConnection, application: &str) -> Result<(), MigrationError> { + // What the refusal tells a server's operator to do once the migration is + // done; a command's refusal ends at the holders, and the command's + // caller adds that it is retried once the migration has finished. + let then = if application == SERVE_APPLICATION { + "; start the server when it has finished" + } else { + "" + }; let fail = |e: sqlx::Error| MigrationError::store("register the server", e); statement(conn, "BEGIN").await.map_err(fail)?; // Inside the transaction that holds the key, so a transaction-mode pooler @@ -532,15 +549,14 @@ async fn register_on(conn: &mut PgConnection) -> Result<(), MigrationError> { { let names = holders(conn, SERVING_KEY).await.unwrap_or_default(); return Err(MigrationError::Locked(format!( - "a migration is running on this catalog ({}); start the server when it has finished", + "a migration is running on this catalog ({}){then}", listed(&names) ))); } let migrators = holders(conn, MIGRATION_KEY).await.map_err(fail)?; if !migrators.is_empty() { return Err(MigrationError::Locked(format!( - "a migration holds the migration lock on this catalog ({}); start the server when \ - it has finished", + "a migration holds the migration lock on this catalog ({}){then}", listed(&migrators) ))); } diff --git a/crates/storage-postgres/src/migrations/mod.rs b/crates/storage-postgres/src/migrations/mod.rs index 740c3469a..319d03403 100644 --- a/crates/storage-postgres/src/migrations/mod.rs +++ b/crates/storage-postgres/src/migrations/mod.rs @@ -55,7 +55,7 @@ use sqlx::postgres::PgConnectOptions; pub use backend::{PostgresMigrations, PostgresStores}; pub use commands::{PostgresCatalogMigrations, resolve}; -pub use locks::{KEEPALIVE_INTERVAL, MigrationLock, ServingRegistration}; +pub use locks::{KEEPALIVE_INTERVAL, MigrationLock, SERVE_APPLICATION, ServingRegistration}; #[cfg(feature = "test-util")] pub use postcheck::{Column, Definitions, Index, compare, describe}; pub use registrar::ServerRegistration; @@ -127,6 +127,28 @@ pub fn migration(id: MigrationId) -> Migration { "0005_drop_continuous_backups.sql" )], }, + MigrationId::BackupCatalogIndex => Migration::Applies { + kind: Kind::Breaking, + convergent: true, + bodies: vec![file_body!( + StoreKind::Catalog, + "migrations", + "0006_backup_catalog_index.sql" + )], + }, + MigrationId::RestoreProgress => Migration::Applies { + // A new table is additive by CONTRIBUTING's step 4, but declaring + // `Kind::Additive` needs the old-binary job, which does not exist + // yet (the paragraph after step 4; `refuse_additive` in + // `crates/bin` pins it). + kind: Kind::Breaking, + convergent: true, + bodies: vec![file_body!( + StoreKind::Data, + "data_migrations", + "0007_restore_progress.sql" + )], + }, } } @@ -148,7 +170,9 @@ pub(crate) const fn runs_in(id: MigrationId) -> RunsIn { MigrationId::Baseline | MigrationId::ResourceTagConditionKeys | MigrationId::SortKeyDefinitions - | MigrationId::DropContinuousBackups => RunsIn::Transaction, + | MigrationId::DropContinuousBackups + | MigrationId::BackupCatalogIndex + | MigrationId::RestoreProgress => RunsIn::Transaction, MigrationId::IndexTableBaseKey => RunsIn::Autocommit, } } diff --git a/crates/storage-postgres/src/migrations/registrar.rs b/crates/storage-postgres/src/migrations/registrar.rs index 5bfbaeb52..f45e18fa0 100644 --- a/crates/storage-postgres/src/migrations/registrar.rs +++ b/crates/storage-postgres/src/migrations/registrar.rs @@ -23,16 +23,17 @@ use async_trait::async_trait; use extenddb_storage::hooks::{Reregistration, ServingRegistrar}; use extenddb_storage::migrations::{CatalogStatus, MigrationError, Target}; -use super::locks::{KEEPALIVE_INTERVAL, ServingRegistration}; +use super::locks::{KEEPALIVE_INTERVAL, SERVE_APPLICATION, ServingRegistration}; use super::{PostgresTarget, catalog_status, require_data_connection_row, resolve}; /// The serving registration of a running server, with what it needs to -/// take it again: the catalog's databases and the status the server started -/// with. +/// take it again: the catalog's databases, the status the server started +/// with, and the name its session carries. pub struct ServerRegistration { target: PostgresTarget, started: CatalogStatus, keepalive: Duration, + application: &'static str, current: Mutex, } @@ -48,7 +49,24 @@ impl ServerRegistration { /// [`MigrationError::Locked`] (exit 6) while a migration runs, or the /// error of a database that cannot be reached or read. pub async fn register(connection_string: &str) -> Result { - Self::register_keepalive(connection_string, KEEPALIVE_INTERVAL).await + Self::register_as(connection_string, SERVE_APPLICATION).await + } + + /// [`ServerRegistration::register`] for the process `application` names + /// (`extenddb serve`, or a command such as `extenddb backup list`): its + /// session carries the name as `application_name`, which a migration + /// waiting for the registrations prints, and a refusal while a + /// migration runs names the migration's sessions without the server's + /// advice to start the server. + /// + /// # Errors + /// + /// As [`ServerRegistration::register`]. + pub async fn register_as( + connection_string: &str, + application: &'static str, + ) -> Result { + Self::register_keepalive(connection_string, KEEPALIVE_INTERVAL, application).await } /// [`ServerRegistration::register`] with another keepalive interval, for @@ -62,18 +80,35 @@ impl ServerRegistration { connection_string: &str, keepalive: Duration, ) -> Result { - Self::register_keepalive(connection_string, keepalive).await + Self::register_keepalive(connection_string, keepalive, SERVE_APPLICATION).await + } + + /// [`ServerRegistration::register_as`] with another keepalive interval, + /// for tests. + /// + /// # Errors + /// + /// As [`ServerRegistration::register`]. + #[cfg(feature = "test-util")] + pub async fn register_as_with( + connection_string: &str, + application: &'static str, + keepalive: Duration, + ) -> Result { + Self::register_keepalive(connection_string, keepalive, application).await } async fn register_keepalive( connection_string: &str, keepalive: Duration, + application: &'static str, ) -> Result { let Some(target) = resolve(&Target::at(connection_string)).await? else { return Err(MigrationError::Status(CatalogStatus::Uninitialized)); }; let registration = - ServingRegistration::register_keepalive(&target.catalog, keepalive).await?; + ServingRegistration::register_keepalive(&target.catalog, keepalive, application) + .await?; let status = match catalog_status(&target).await { Ok(status) => status, Err(e) => { @@ -97,6 +132,7 @@ impl ServerRegistration { target, started: status, keepalive, + application, current: Mutex::new(registration), }) } @@ -130,14 +166,17 @@ impl ServingRegistrar for ServerRegistration { } async fn register_again(&self) -> Reregistration { - let registration = - match ServingRegistration::register_keepalive(&self.target.catalog, self.keepalive) - .await - { - Ok(registration) => registration, - Err(MigrationError::Locked(why)) => return Reregistration::Refused(why), - Err(e) => return Reregistration::Unavailable(e.to_string()), - }; + let registration = match ServingRegistration::register_keepalive( + &self.target.catalog, + self.keepalive, + self.application, + ) + .await + { + Ok(registration) => registration, + Err(MigrationError::Locked(why)) => return Reregistration::Refused(why), + Err(e) => return Reregistration::Unavailable(e.to_string()), + }; let status = match catalog_status(&self.target).await { Ok(status) => status, Err(e) => { diff --git a/crates/storage-postgres/src/restore_engine.rs b/crates/storage-postgres/src/restore_engine.rs new file mode 100644 index 000000000..c30666783 --- /dev/null +++ b/crates/storage-postgres/src/restore_engine.rs @@ -0,0 +1,438 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The data-plane side of a store-backed restore on `PostgreSQL` +//! ([`RestoreEngine`]): the items of one verified data file written into the +//! CREATING target, the progress record, the ACTIVE flip and the counts. +//! +//! One data transaction per file: the base rows in multi-row INSERTs of +//! `COPY_BATCH_ITEMS` rows or `COPY_BATCH_BYTES` of item text, each +//! secondary index's rows beside them (the `insert_restore_rows` the legacy +//! copy shares), each vector index's rows through the write path's +//! `VectorInsertPlan`, and the file's `restore_progress` row last, so a file +//! commits whole with its record or not at all. The data database has no +//! `tables` row and no job record, so nothing in `bulk_load` reads or +//! conditions on the claim's generation: the fence is the base table's +//! primary key, which refuses a file applied twice, and the claim's +//! liveness, which the engine confirms before each file. Plain INSERTs, so +//! a key the backup holds twice, within a file or across files, fails the +//! file (SQLSTATE 23505) instead of collapsing two items. The progress row +//! is a plain INSERT for a file applied from its start and an upsert +//! (`ON CONFLICT (table_id, file_index) DO UPDATE SET items_applied`) for a +//! file resumed past it, so a row left short of the file's count by a +//! decoder that died after the commit is completed by the resume. +//! +//! `finish_restore` is `activate_restore_target` (the catalog transaction +//! the legacy copy ends with: the record DONE, the table and every index +//! ACTIVE, conditioned on the generation and on CREATING), then the +//! progress rows deleted on the data pool. The delete runs after the commit +//! and cannot fail the restore: its error is logged at WARN and +//! `Finished::Active` returned, since the restore is complete once the flip +//! committed and the rows are the reconciler's `sweep_restore_progress` to +//! remove, as they are after a crash between the two. + +use std::collections::HashSet; + +use extenddb_core::types::{AttributeDefinition, Item, KeySchemaElement}; +use extenddb_storage::backup_catalog::{ + JobClaim, RESTORE_POSTGRES_BEFORE_PROGRESS_DELETE, RestoreTargetKey, restore_pause_point, +}; +use extenddb_storage::backup_definition::{COPY_BATCH_BYTES, COPY_BATCH_ITEMS}; +use extenddb_storage::error::StorageError; +use extenddb_storage::restore_engine::{BulkLoadFile, Finished, RestoreEngine}; +use futures::future::BoxFuture; +use futures::stream::BoxStream; + +use crate::PostgresEngine; +use crate::backup_engine::{RestoreTarget, RowInsertError, insert_restore_rows}; +use crate::data::data_table_name; +use crate::data::vector_index::{VectorInsertPlan, fetch_vector_indexes_for_table}; + +fn db_err(e: sqlx::Error) -> StorageError { + StorageError::Internal(format!("Database error: {e}")) +} + +/// The key shape of a CREATING restore target, read from its catalog row. +struct TargetShape { + key_schema: Vec, + attribute_definitions: Vec, +} + +impl PostgresEngine { + /// The target's key schema and attribute definitions, when `key` names + /// a CREATING restore target with this table id. + async fn restore_target_shape( + &self, + key: &RestoreTargetKey, + ) -> Result { + let row: Option<(serde_json::Value, serde_json::Value)> = sqlx::query_as( + "SELECT t.key_schema, t.attribute_definitions FROM tables t \ + JOIN table_restores r ON r.table_id = t.table_id \ + WHERE t.table_id = $1 AND t.account_id = $2 AND t.table_name = $3 \ + AND t.table_status = 'CREATING'", + ) + .bind(&key.table_id) + .bind(&key.account_id) + .bind(&key.table_name) + .fetch_optional(&self.pool) + .await + .map_err(db_err)?; + let (key_schema, attribute_definitions) = row.ok_or_else(|| { + StorageError::TableNotFound(format!( + "{} is not a CREATING restore target with table id {}", + key.table_name, key.table_id + )) + })?; + Ok(TargetShape { + key_schema: serde_json::from_value(key_schema) + .map_err(|e| StorageError::Internal(format!("key schema: {e}")))?, + attribute_definitions: serde_json::from_value(attribute_definitions) + .map_err(|e| StorageError::Internal(format!("attribute definitions: {e}")))?, + }) + } +} + +/// The error for a key the target already holds: the backup carries it +/// twice, within the file or in a file applied before it. The engine +/// prefixes the file's key, so the file is not named here. +fn duplicate_key_error(e: &sqlx::Error) -> StorageError { + StorageError::Validation(format!( + "holds a primary key the target already has (the backup holds a key twice): {e}" + )) +} + +/// Write `batch` into the base table, every secondary index and every vector +/// index inside the file's transaction, and clear it. +async fn flush_batch( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + target: &RestoreTarget<'_>, + vector_plans: &[VectorInsertPlan<'_>], + batch: &mut Vec<(Item, String)>, +) -> Result { + insert_restore_rows(tx, target, batch) + .await + .map_err(|e| match e { + RowInsertError::DuplicateKey(e) => duplicate_key_error(&e), + RowInsertError::Storage(e) => e, + })?; + for plan in vector_plans { + for (item, _) in batch.iter() { + plan.insert(tx, item).await?; + } + } + let written = u64::try_from(batch.len()).unwrap_or(u64::MAX); + batch.clear(); + Ok(written) +} + +impl RestoreEngine for PostgresEngine { + fn bulk_load<'a>( + &'a self, + target: &'a RestoreTargetKey, + _claim: &'a dyn JobClaim, + file: BulkLoadFile, + mut items: BoxStream<'a, Result>, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + use futures::StreamExt; + + let shape = self.restore_target_shape(target).await?; + let source = format!("data file {}", file.file_index); + let rows = RestoreTarget::load( + &self.pool, + &target.table_id, + &shape.key_schema, + &shape.attribute_definitions, + source.clone(), + ) + .await?; + let vector_metas = fetch_vector_indexes_for_table(&self.pool, &target.table_id).await?; + let vector_plans: Vec> = vector_metas + .iter() + .map(|(meta, _)| { + VectorInsertPlan::new(meta, &shape.key_schema, &shape.attribute_definitions) + }) + .collect(); + + let mut tx = self.data_pool.begin().await.map_err(db_err)?; + let mut batch: Vec<(Item, String)> = Vec::with_capacity(COPY_BATCH_ITEMS); + let mut batch_bytes = 0usize; + let mut applied: u64 = 0; + while let Some(item) = items.next().await { + let item = item?; + let item_json = serde_json::to_string(&item) + .map_err(|e| StorageError::Internal(format!("item: {e}")))?; + batch_bytes += item_json.len(); + batch.push((item, item_json)); + if batch.len() == COPY_BATCH_ITEMS || batch_bytes >= COPY_BATCH_BYTES { + applied += flush_batch(&mut tx, &rows, &vector_plans, &mut batch).await?; + batch_bytes = 0; + } + } + if !batch.is_empty() { + applied += flush_batch(&mut tx, &rows, &vector_plans, &mut batch).await?; + } + // The file's record, in the file's transaction. A file applied from + // its start writes a new row: a duplicate `(table_id, file_index)` + // is a file applied twice, which the base table's primary key has + // refused above for any file with items. A file resumed past its + // start (`start_at > 0`) updates the row the earlier run left + // (one with `items_applied` short of the file's count: the + // decoder died after the file's commit and the job's FAILED write + // never landed), so the resume completes the file instead of + // failing on the row's key. + let progress_row = if file.start_at > 0 { + "INSERT INTO restore_progress (table_id, file_index, items_applied) \ + VALUES ($1, $2, $3) \ + ON CONFLICT (table_id, file_index) DO UPDATE \ + SET items_applied = EXCLUDED.items_applied" + } else { + "INSERT INTO restore_progress (table_id, file_index, items_applied) \ + VALUES ($1, $2, $3)" + }; + sqlx::query(progress_row) + .bind(&target.table_id) + .bind(i64::from(file.file_index)) + .bind(i64::try_from(file.start_at.saturating_add(applied)).unwrap_or(i64::MAX)) + .execute(&mut *tx) + .await + .map_err(|e| { + let duplicate = e + .as_database_error() + .and_then(|d| d.code()) + .is_some_and(|code| code == "23505"); + if duplicate { + StorageError::Validation(format!( + "{source} was applied before and its progress recorded: {e}" + )) + } else { + db_err(e) + } + })?; + tx.commit().await.map_err(db_err)?; + Ok(applied) + }) + } + + fn finish_restore<'a>( + &'a self, + target: &'a RestoreTargetKey, + claim: &'a dyn JobClaim, + item_count: u64, + size_bytes: u64, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + // The key names the table; the flip itself is conditioned on the + // table id, CREATING, RUNNING and the generation. + let is_target: bool = sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM tables WHERE table_id = $1 AND account_id = $2 \ + AND table_name = $3)", + ) + .bind(&target.table_id) + .bind(&target.account_id) + .bind(&target.table_name) + .fetch_one(&self.pool) + .await + .map_err(db_err)?; + if !is_target { + return Ok(Finished::NotMatched(format!( + "{} is not a table with id {} in account {}", + target.table_name, target.table_id, target.account_id + ))); + } + let activated = self + .activate_restore_target( + &target.table_id, + claim.generation(), + i64::try_from(item_count).unwrap_or(i64::MAX), + i64::try_from(size_bytes).unwrap_or(i64::MAX), + ) + .await?; + if !activated { + return Ok(Finished::NotMatched(format!( + "the job record of {} is not RUNNING under generation {}, or the table is \ + not CREATING", + target.table_name, + claim.generation() + ))); + } + restore_pause_point(RESTORE_POSTGRES_BEFORE_PROGRESS_DELETE).await; + // The flip has committed: the table is ACTIVE and the record DONE, + // so the restore is complete whatever happens to the progress + // rows. A failure here is logged, not returned: an `Err` would make + // the engine fail a finished restore, and the rows are harmless + // (the data database reads them for nothing once the record is + // DONE) and the reconciler's `sweep_restore_progress` removes them. + if let Err(e) = sqlx::query("DELETE FROM restore_progress WHERE table_id = $1") + .bind(&target.table_id) + .execute(&self.data_pool) + .await + { + tracing::warn!( + "restore of table {}: the table is ACTIVE and its record DONE, but the \ + restore_progress rows of table id {} could not be deleted: {e}; the progress \ + rows are the reconciler's to sweep", + target.table_name, + target.table_id + ); + } + Ok(Finished::Active) + }) + } + + fn count_restore_target<'a>( + &'a self, + key: &'a RestoreTargetKey, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + // The id names the data table; the key's account and name are + // checked against the catalog so a bare id reads no other + // account's table. + self.restore_target_shape(key).await?; + let count: i64 = sqlx::query_scalar(&format!( + "SELECT count(*) FROM {}", + data_table_name(&key.table_id) + )) + .fetch_one(&self.data_pool) + .await + .map_err(db_err)?; + Ok(u64::try_from(count).unwrap_or_default()) + }) + } + + fn restore_target_ready<'a>( + &'a self, + key: &'a RestoreTargetKey, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + // The catalog row first, so a bare id reads no other account's + // table; then the data table's existence (the registration + // commits the catalog transaction before the data DDL). + self.restore_target_shape(key).await?; + let exists: bool = sqlx::query_scalar("SELECT to_regclass($1) IS NOT NULL") + .bind(data_table_name(&key.table_id)) + .fetch_one(&self.data_pool) + .await + .map_err(db_err)?; + Ok(exists) + }) + } + + fn read_restore_progress<'a>( + &'a self, + key: &'a RestoreTargetKey, + ) -> BoxFuture<'a, Result, StorageError>> { + Box::pin(async move { + let rows: Vec<(i64, i64)> = sqlx::query_as( + "SELECT file_index, items_applied FROM restore_progress WHERE table_id = $1 \ + ORDER BY file_index", + ) + .bind(&key.table_id) + .fetch_all(&self.data_pool) + .await + .map_err(db_err)?; + Ok(rows + .into_iter() + .map(|(index, applied)| { + ( + u32::try_from(index).unwrap_or(u32::MAX), + u64::try_from(applied).unwrap_or_default(), + ) + }) + .collect()) + }) + } + + fn sweep_restore_progress<'a>( + &'a self, + running: &'a [String], + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + // A positive list, candidates before the live set (the trait + // doc): the ids the progress rows name now, then the catalog's + // RUNNING records; a target registered after `running` was + // listed is RUNNING here, and one whose first row lands after + // the first read is not among the candidates. + let named: Vec = + sqlx::query_scalar("SELECT DISTINCT table_id FROM restore_progress") + .fetch_all(&self.data_pool) + .await + .map_err(db_err)?; + if named.is_empty() { + return Ok(0); + } + // A test may interpose here, after the candidate ids are read and + // before the live RUNNING set, to commit a registration and a + // progress row: a row of a record that is RUNNING at the read + // below is protected (the two reads order candidates before the + // live set, the trait doc). + #[cfg(feature = "test-hooks")] + sweep_pause::maybe_pause().await; + let live: Vec = + sqlx::query_scalar("SELECT table_id FROM table_restores WHERE status = 'RUNNING'") + .fetch_all(&self.pool) + .await + .map_err(db_err)?; + let protected: HashSet<&str> = running + .iter() + .chain(live.iter()) + .map(String::as_str) + .collect(); + let stale: Vec = named + .into_iter() + .filter(|id| !protected.contains(id.as_str())) + .collect(); + if stale.is_empty() { + return Ok(0); + } + let removed = + sqlx::query("DELETE FROM restore_progress WHERE table_id = ANY($1::TEXT[])") + .bind(&stale) + .execute(&self.data_pool) + .await + .map_err(db_err)? + .rows_affected(); + Ok(removed) + }) + } +} + +/// A rendezvous a test arms to pause `sweep_restore_progress` between its +/// two reads (the candidate progress-row ids, then the live RUNNING set), +/// so it can commit a registration and a progress row in between and prove +/// that row survives the sweep. `test-hooks` only. +#[cfg(feature = "test-hooks")] +#[doc(hidden)] +pub mod sweep_pause { + use std::sync::{LazyLock, Mutex}; + + use tokio::sync::Notify; + + struct Rendezvous { + arrived: std::sync::Arc, + proceed: std::sync::Arc, + } + + static PAUSE: LazyLock>> = LazyLock::new(|| Mutex::new(None)); + + /// Arm the pause once. Returns `(arrived, proceed)`: the sweep notifies + /// `arrived` when it reaches the point and waits on `proceed`. + #[must_use] + pub fn arm() -> (std::sync::Arc, std::sync::Arc) { + let arrived = std::sync::Arc::new(Notify::new()); + let proceed = std::sync::Arc::new(Notify::new()); + *PAUSE.lock().unwrap() = Some(Rendezvous { + arrived: arrived.clone(), + proceed: proceed.clone(), + }); + (arrived, proceed) + } + + pub(crate) async fn maybe_pause() { + let armed = PAUSE.lock().unwrap().take(); + if let Some(r) = armed { + r.arrived.notify_one(); + r.proceed.notified().await; + } + } +} diff --git a/crates/storage-postgres/src/snapshot_engine.rs b/crates/storage-postgres/src/snapshot_engine.rs index abb066cf3..1f6bbeb90 100644 --- a/crates/storage-postgres/src/snapshot_engine.rs +++ b/crates/storage-postgres/src/snapshot_engine.rs @@ -63,8 +63,8 @@ //! pool's 30 s acquire timeout). The `snapshot_open_slots` field doc says //! what the permits do not prevent. An open over the bound is refused with //! `LimitExceeded` rather than queued. The backup design's -//! `max_concurrent_jobs` is the caller-side bound; the restore path's slots -//! (`restore_server_slot_count`) do not count snapshots. +//! `max_concurrent_jobs` is the caller-side bound on jobs; a restore holds no +//! snapshot slot. //! //! `ACCESS SHARE` on the data table is held until the stream's last fetch //! (the transaction commits as soon as a short page shows the end), not @@ -91,46 +91,164 @@ //! layout the first catalog read already did. Its text is the manifest //! marker: the `xmin:xmax:xip` of the snapshot, recorded so a later //! change-log archive can tell which commits the snapshot includes; nothing -//! reopens a snapshot from it. Items are then read through a server-side -//! cursor in batches. +//! reopens a snapshot from it. Items are then read in keyset pages on the +//! same transaction: each page is one query that seeks past the last row's +//! primary key (`WHERE (pk, sk) > ($1, $2) ORDER BY pk, sk LIMIT n`, an +//! index scan on the primary key) and cuts itself at the byte budget +//! ([`PageQuery`]). //! -//! Memory: one `FETCH` is `paging::DEFAULT_FETCH_BATCH` (1000) rows, all of -//! them held as item text at once. At the default 400 KB item limit one -//! row's text can reach 27.7 MB (the measured worst case in #394's line-cap -//! table), so a fetch is bounded by rows, not bytes. A byte budget (keyset -//! paging with a byte cutoff) is required before the stream gets a caller; -//! the backup design assigns it to the backup job PR. +//! Memory: a page holds at most `paging::fetch_batch()` rows (1000) and +//! stops after the row that takes its item text to or past +//! `paging::SNAPSHOT_FETCH_BYTES` (64 MiB), so the stream holds at most the +//! budget plus one row of text in decoded form, plus the one row's text +//! being parsed: the rows are pulled from the connection one at a time and +//! parsed as they arrive (`fetch_page`), so the page's text is never held +//! whole on this side. The cut is made in the page query, so the server +//! returns no row past it; the rows it reads past the cut before `LIMIT n` +//! stops it are at most `n - 1`, and each of them costs the server one +//! serialization of its item text and two spills of it (the two window +//! functions buffer the `n` rows before the outer `WHERE` drops them), so a +//! page costs the server `n` rows of text, not the rows it returns. `n` is +//! [`MIN_PAGE_ROWS`] for the first page and is then sized from the largest +//! row of the last pages, those holding at least a row bound's worth of rows +//! ([`RowBoundMemory`], [`next_page_rows`]: the budget divided by that row, +//! never below [`MIN_NEXT_PAGE_ROWS`]), so a table of large rows pages at +//! the rows the budget holds plus one, and the over-read that remains is the +//! second page, and then the page that follows a row bound's worth of small +//! rows: each asks for the rows the small rows before it justified (the +//! first page's 16, then a row bound's worth) and may meet large ones, so it +//! reads up to its `n` rows of up to the line cap (32 MiB) each; `n` then +//! stays at the floor while large rows keep arriving within a row bound's +//! worth of rows and returns to the row bound a row bound's worth of small +//! rows after the last large one, so such a page recurs at most once per row +//! bound's worth of small rows, on a table whose large rows are rarer than +//! that. The bound is the rows the page reads, `n` of up to the line cap +//! each (measured on a 20,000-row table with 30 such rows every 33rd key +//! among the first 1,000: `docs/design/15-backup-restore-bench-pr8.md`); +//! nothing in this form stops the server at the budget, since a window +//! function runs over the `n` rows before the outer `WHERE` drops them. The +//! floor has a cost of its own: after a page holding a row at the line cap +//! the bound stays at the floor for a row bound's worth of rows, 250 pages of +//! four, whatever those rows hold, so a table whose large rows are rarer +//! than one per row bound pays 250 round trips per large row where one page +//! of memory would page it at the row bound (measured on a 20,000-row table +//! with a 27 MB row at every 1,100th key: 2,585 page queries against 38 with +//! one page of memory, 25.6 against 25.1 s, 1.06 against 1.09 GB of temp; +//! the record). //! -//! Order: the cursor orders by the primary key (`key_order_clause`) only when -//! the text key columns (`pk`, and `sk_s` for an `S` range key) have a -//! collation named "C" or "POSIX", read from `pg_attribute` inside the -//! snapshot transaction. The rule is the name: any other collation, -//! including "default" on a database whose own collation is C, streams -//! unordered, which is complete. A table created before the key columns were -//! declared `COLLATE "C"` keeps the database collation, and ordering it -//! bytewise is a Seq Scan and an external sort of the whole table before the -//! first FETCH (7 MB on disk for 100k rows at `work_mem` 4 MB), with the -//! lock, the connection and the permit held throughout. Such a table is -//! streamed with a plain `SELECT`, reported `key_ordered = false`, and a -//! warning names it. +//! The plan is the planner's choice, not the query's promise. With the +//! planner's estimate of the rows after the seek above `n`, the page is +//! `Limit` over an `Index Scan` on the primary key index with no sort (the +//! 100k-row test table's first page, a page in its middle and a page ten +//! rows from its end, `the_page_query_is_an_index_scan` and +//! `the_last_page_sorts_the_rows_after_the_seek`). On a table of about a +//! thousand rows, a page near its end whose `n` is above the rows left +//! (1,060 rows, 57 left, `n` = 810) gets a `Bitmap Heap Scan` driven by a +//! `Bitmap Index Scan` on the same index and a `Sort` of those rows under +//! the `Limit` (PostgreSQL 16, the same test), so such a page sorts the rows +//! after the seek, at most the rows left, with their text; the seek is +//! served by the primary key index either way. +//! +//! Order: the page orders by the primary key in the key columns' own +//! collation, which is the order the primary key index has, so every table +//! pages by its index. The stream is reported `key_ordered` only when the +//! key shape has a format-defined order (`key_schema_is_orderable`) and the +//! text key columns (`pk`, `sk_s`) have a collation named "C" or "POSIX", +//! read from `pg_attribute` inside the snapshot transaction; the rule is the +//! name, so "default" on a database whose own collation is C streams +//! unordered, which is complete (`open_stream` says why the table is not +//! sorted bytewise instead). use std::sync::Arc; use extenddb_core::types::ScalarAttributeType; use extenddb_core::types::{Item, TableDescription}; +use extenddb_storage::backup_catalog::{JobBound, limit_exceeded_message}; use extenddb_storage::error::StorageError; use extenddb_storage::snapshot::{drop_deleting_indexes, key_schema_is_orderable, paging}; -use extenddb_storage::util::sk_column; +use extenddb_storage::util::sk_column_n; use extenddb_storage::{SnapshotEngine, SnapshotInfo, TableSnapshot}; use futures::future::BoxFuture; use futures::stream::BoxStream; +use sqlx::Row as _; +use sqlx::postgres::PgRow; use sqlx::{PgConnection, Postgres, Transaction}; use tokio::sync::{Notify, OwnedSemaphorePermit}; use crate::PostgresEngine; use crate::data::{all_sort_key_info, data_table_name, index_table_name}; -const CURSOR: &str = "extenddb_snapshot"; +/// Rows the first page asks for, before anything is known about the row +/// size. The server serializes every row the inner `LIMIT` reads, so this +/// bounds the first page's cost at 16 rows of up to the line cap (32 MiB) +/// each; a table of small rows pays one short round trip for it, since the +/// second page is sized from the first ([`next_page_rows`]). +#[doc(hidden)] +pub const MIN_PAGE_ROWS: usize = 16; + +/// The fewest rows a page after the first asks for. The budget divided by +/// the largest row is two for 27 MB rows, while such a page keeps three (the +/// row that reaches the budget is kept), so without a floor every page of +/// such a table would be two rows and no page would reach its cut; at four, +/// a page of such rows reads one row past its cut. +#[doc(hidden)] +pub const MIN_NEXT_PAGE_ROWS: usize = 4; + +/// The largest rows of the last pages, which size the next page's row bound +/// ([`next_page_rows`]): the memory keeps the pages that together hold at +/// least [`RowBoundMemory::rows`] rows (the row bound, 1000 in production), +/// so a large row keeps the bound low for at least a row bound's worth of +/// rows after it rather than for one page. With one page of memory a large +/// row sets the next bound to 4, four small rows set the one after back to +/// 1000, and on a table whose large rows are scattered among small ones the +/// 1000-row page meets large rows again and the cycle repeats every other +/// page (5,555 rows read for 1,000 kept on the measured shape); with the +/// memory the bound stays at 4 while large rows keep arriving within a row +/// bound's worth of rows, so that table pages at 4 after its first 1000-row +/// page (1,900 read for 1,000 kept). +#[derive(Debug, Clone)] +struct RowBoundMemory { + /// How many rows the kept pages must hold at least. + rows: usize, + /// The largest row and the row count of each kept page, oldest first. + pages: std::collections::VecDeque<(usize, usize)>, +} + +impl RowBoundMemory { + fn new(rows: usize) -> Self { + Self { + rows, + pages: std::collections::VecDeque::new(), + } + } + + /// Record a page's largest row (`max_row`, 0 for an empty page) and its + /// row count, and forget the oldest pages the memory no longer needs: + /// the oldest page goes while the pages after it hold at least `rows`. + fn record(&mut self, max_row: usize, rows: usize) { + self.pages.push_back((max_row, rows)); + while self.pages.len() > 1 { + let after_oldest: usize = self.pages.iter().skip(1).map(|&(_, n)| n).sum(); + if after_oldest < self.rows { + break; + } + self.pages.pop_front(); + } + } + + /// The largest row among the kept pages; `None` when none held a row. + fn largest_row(&self) -> Option { + self.pages + .iter() + .map(|&(max_row, _)| max_row) + .max() + .filter(|&b| b > 0) + } +} + +/// The name the snapshot-slot refusal gives its bound: the setting the slot +/// count is derived from (half of `pool_size`). +pub const SNAPSHOT_SLOT_BOUND: &str = "[storage.postgres] pool_size snapshot-slot"; const BEGIN_SNAPSHOT: &str = "BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY"; @@ -239,17 +357,17 @@ impl PostgresEngine { let slot = Arc::clone(&self.snapshot_open_slots) .try_acquire_owned() .map_err(|_| { - StorageError::LimitExceeded( - "too many table snapshots are open on this server; retry when one ends" - .to_owned(), - ) + StorageError::LimitExceeded(limit_exceeded_message(JobBound::Counted { + name: SNAPSHOT_SLOT_BOUND, + limit: self.snapshot_slot_limit, + })) })?; let pinned = if self.shared_data_pool { self.pin_on_one_transaction(account_id, table_name).await? } else { self.pin_behind_barrier(account_id, table_name).await? }; - open_cursor(pinned, slot, table_name).await + open_stream(pinned, slot, table_name).await } /// Shared layout (module doc): one `REPEATABLE READ READ ONLY` @@ -462,53 +580,60 @@ async fn leave_out_gsis_without_data_table( Ok(()) } -/// The common half of the open: decide the order, then `DECLARE` the cursor -/// on the held transaction. -async fn open_cursor( +/// The common half of the open: decide the order and the collation, then +/// hand the held transaction to the keyset stream. +async fn open_stream( mut pinned: Pinned, slot: OwnedSemaphorePermit, table_name: &str, ) -> Result { let ddb_table = data_table_name(&pinned.table_id); - // Ordered only when the text key columns already sort bytewise: on a - // table whose key columns keep the database collation the ORDER BY would - // be a Seq Scan and a full sort before the first FETCH, with the lock, - // the connection and the permit held throughout (module doc). - let mut order_by = key_order_clause(&pinned.description); - if order_by.is_some() { - let not_bytewise: Vec = sqlx::query_scalar( - "SELECT a.attname::text FROM pg_attribute a \ - LEFT JOIN pg_collation c ON c.oid = a.attcollation \ - WHERE a.attrelid = to_regclass($1) AND a.attname = ANY($2) \ - AND NOT a.attisdropped \ - AND COALESCE(c.collname, '') NOT IN ('C', 'POSIX')", - ) - .bind(&ddb_table) - .bind(key_text_columns(&pinned.description)) - .fetch_all(pinned.held.open_conn()?) - .await - .map_err(|e| db_err(&e))?; - if !not_bytewise.is_empty() { - tracing::warn!( - table = table_name, - table_id = %pinned.table_id, - columns = ?not_bytewise, - "table snapshot: key columns do not have collation \"C\"; streaming the \ - table unordered (key_ordered = false) rather than sorting it" - ); - order_by = None; - } + let columns = key_columns(&pinned.description); + // Whether the text key columns already sort bytewise: a collation named + // "C" or "POSIX" in `pg_attribute`, read inside the snapshot transaction. + // The page orders by the primary key in the columns' own collation either + // way, which is the order the primary key index has, so a table created + // before the key columns were declared `TEXT COLLATE "C"` (it keeps the + // database collation, `en_US.utf8` in the default setups, where `a < B`) + // pages by its index and is reported unordered rather than sorted + // bytewise (a Seq Scan and an external sort of the whole table before the + // first row, with the lock, the connection and the permit held + // throughout). `COLLATE "C"` is written into the page query only when the + // columns have it, where it changes nothing and states the order the + // stream relies on (`pk` and `sk_s` bytewise, `sk_n` and `sk_b` by value: + // the order `extenddb_core::backup_format::key_order` defines), and the + // stream is reported ordered only then. + let text_columns: Vec = columns + .iter() + .filter(|c| c.kind == KeyKind::Text) + .map(|c| c.name.clone()) + .collect(); + let not_bytewise: Vec = sqlx::query_scalar( + "SELECT a.attname::text FROM pg_attribute a \ + LEFT JOIN pg_collation c ON c.oid = a.attcollation \ + WHERE a.attrelid = to_regclass($1) AND a.attname = ANY($2) \ + AND NOT a.attisdropped \ + AND COALESCE(c.collname, '') NOT IN ('C', 'POSIX')", + ) + .bind(&ddb_table) + .bind(&text_columns) + .fetch_all(pinned.held.open_conn()?) + .await + .map_err(|e| db_err(&e))?; + let collate_c = not_bytewise.is_empty(); + let orderable = key_schema_is_orderable(&pinned.description); + if orderable && !collate_c { + tracing::warn!( + table = table_name, + table_id = %pinned.table_id, + columns = ?not_bytewise, + "table snapshot: key columns do not have collation \"C\"; streaming the \ + table unordered (key_ordered = false) rather than sorting it" + ); } - let key_ordered = order_by.is_some(); - let select = match &order_by { - Some(cols) => format!("SELECT item_data::text FROM {ddb_table} ORDER BY {cols}"), - None => format!("SELECT item_data::text FROM {ddb_table}"), - }; + let key_ordered = orderable && collate_c; + let query = PageQuery::new(&ddb_table, columns, collate_c); paging::open_pause(paging::OpenStage::BeforeItems).await; - sqlx::query(&format!("DECLARE {CURSOR} NO SCROLL CURSOR FOR {select}")) - .execute(pinned.held.open_conn()?) - .await - .map_err(|e| db_err(&e))?; Ok(TableSnapshot { description: pinned.description, @@ -517,21 +642,10 @@ async fn open_cursor( marker: pinned.marker, }, key_ordered, - items: cursor_stream(pinned.held, slot), + items: keyset_stream(pinned.held, slot, query), }) } -/// The text key columns whose collation decides the order: `pk`, and `sk_s` -/// for an `S` range key (`sk_n` is `NUMERIC` and `sk_b` is `bytea`). -fn key_text_columns(description: &TableDescription) -> Vec { - let sks = all_sort_key_info(&description.key_schema, &description.attribute_definitions); - let mut columns = vec!["pk".to_owned()]; - if matches!(sks.as_slice(), [(_, ScalarAttributeType::S)]) { - columns.push("sk_s".to_owned()); - } - columns -} - /// Drop attribute definitions no key schema (table, GSI, LSI) names, after /// an index has been left out of a description. `CreateTable` refuses a /// definition no key uses. @@ -553,45 +667,334 @@ fn prune_unused_attribute_definitions(description: &mut TableDescription) { .retain(|d| used.contains(&d.attribute_name)); } -/// `ORDER BY` columns for the format's key order, or `None` when the key -/// shape has none (`key_schema_is_orderable`). The open uses the clause only -/// after checking that the text key columns (`pk`, and `sk_s` for an `S` -/// range key) have collation "C" or "POSIX" (`pg_attribute.attcollation`); -/// a table created before the key columns were declared `TEXT COLLATE "C"` -/// keeps the database collation (`en_US.utf8` in the default setups, where -/// `a < B`) and is streamed unordered instead, because `ORDER BY ... COLLATE -/// "C"` on it is a full sort before the first row. On the tables that reach -/// the clause the explicit `COLLATE "C"` therefore does not change the order; -/// it is kept as defence in depth (no test can tell it from its absence), so -/// the query states the order it relies on. `pk` and `sk_s` sort bytewise and -/// `sk_n` (`NUMERIC`) and `sk_b` (`BYTEA`) sort by value: the order -/// `extenddb_core::backup_format::key_order` defines. -fn key_order_clause(description: &TableDescription) -> Option { - if !key_schema_is_orderable(description) { - return None; - } +/// The type a primary-key column of the data table has, which decides how +/// the seek binds its value. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum KeyKind { + /// `pk`, `sk_s`, `sk_s`: `TEXT`. + Text, + /// `sk_n`, `sk_n`: `NUMERIC`, carried as its text. + Numeric, + /// `sk_b`, `sk_b`: `BYTEA`. + Bytes, +} + +/// A primary-key column of the data table. +#[derive(Debug, Clone)] +struct KeyColumn { + name: String, + kind: KeyKind, +} + +/// The primary-key columns of a table's data table, in the index's order: +/// `pk`, then one typed column per range key (`sk_column_n`). +fn key_columns(description: &TableDescription) -> Vec { let sks = all_sort_key_info(&description.key_schema, &description.attribute_definitions); - match sks.as_slice() { - [] => Some("pk COLLATE \"C\"".to_owned()), - [(_, ScalarAttributeType::S)] => Some("pk COLLATE \"C\", sk_s COLLATE \"C\"".to_owned()), - [(_, sk_type)] => Some(format!("pk COLLATE \"C\", {}", sk_column(*sk_type))), - _ => None, + let mut columns = vec![KeyColumn { + name: "pk".to_owned(), + kind: KeyKind::Text, + }]; + for (i, &(_, sk_type)) in sks.iter().enumerate() { + columns.push(KeyColumn { + name: sk_column_n(i, sk_type), + kind: match sk_type { + ScalarAttributeType::S => KeyKind::Text, + ScalarAttributeType::N => KeyKind::Numeric, + ScalarAttributeType::B => KeyKind::Bytes, + }, + }); } + columns } -/// Stream items from the open cursor, one `FETCH` per batch. The held +/// One key column's value from the last row of a page: where the next page +/// starts. +#[derive(Debug, Clone)] +enum KeyValue { + Text(String), + Numeric(String), + Bytes(Vec), +} + +/// The page query of one table: the first page's text and a following +/// page's, built once per open. +/// +/// Three layers. The innermost reads the rows after the seek in primary-key +/// order and stops at `LIMIT n`; the middle computes, over those `n` rows +/// only, the running total of item text in the same order and the count of +/// rows the limit returned (`count(*) OVER ()`, the end-of-stream signal); +/// the outer keeps every row whose predecessors' total is under the byte +/// budget and orders the result, since a window function's output order is +/// not promised to the client. The layering matters: a window function runs +/// before `LIMIT`, so a running total written beside the `LIMIT` would be +/// computed over the whole rest of the table. The innermost `ORDER BY ... +/// LIMIT` behind a row comparison on the primary-key columns is an index +/// scan on the primary key; `item_data::text` is computed once, in the +/// innermost layer, and the other layers read the alias. +#[derive(Debug, Clone)] +struct PageQuery { + first: String, + after: String, + columns: Vec, +} + +impl PageQuery { + fn new(ddb_table: &str, columns: Vec, collate_c: bool) -> Self { + Self { + first: page_sql(ddb_table, &columns, collate_c, false), + after: page_sql(ddb_table, &columns, collate_c, true), + columns, + } + } +} + +/// The text of a page query (see [`PageQuery`]). With `after_key` the query +/// seeks past a key bound as `$1..$k` (text, numeric as text, bytes) and +/// takes the row bound and the byte budget as `$k+1` and `$k+2`; without, as +/// `$1` and `$2`. +fn page_sql(ddb_table: &str, columns: &[KeyColumn], collate_c: bool, after_key: bool) -> String { + let order = |prefix: &str| -> String { + columns + .iter() + .enumerate() + .map(|(i, c)| { + let name = if prefix.is_empty() { + format!("t.{}", c.name) + } else { + format!("{prefix}.k{i}") + }; + if collate_c && c.kind == KeyKind::Text { + format!("{name} COLLATE \"C\"") + } else { + name + } + }) + .collect::>() + .join(", ") + }; + let seek = if after_key { + let left = columns + .iter() + .map(|c| format!("t.{}", c.name)) + .collect::>() + .join(", "); + let right = columns + .iter() + .enumerate() + .map(|(i, c)| { + let p = i + 1; + match c.kind { + KeyKind::Text => format!("${p}::text"), + KeyKind::Numeric => format!("${p}::text::numeric"), + KeyKind::Bytes => format!("${p}::bytea"), + } + }) + .collect::>() + .join(", "); + format!("WHERE ({left}) > ({right}) ") + } else { + String::new() + }; + let k = if after_key { columns.len() } else { 0 }; + let n_param = k + 1; + let budget_param = k + 2; + let inner_select = columns + .iter() + .enumerate() + .map(|(i, c)| format!("t.{} AS k{i}", c.name)) + .collect::>() + .join(", "); + let keys = (0..columns.len()) + .map(|i| format!("k{i}")) + .collect::>() + .join(", "); + let shown = columns + .iter() + .enumerate() + .map(|(i, c)| match c.kind { + KeyKind::Numeric => format!("page.k{i}::text AS c{i}"), + KeyKind::Text | KeyKind::Bytes => format!("page.k{i} AS c{i}"), + }) + .collect::>() + .join(", "); + format!( + "SELECT {shown}, page.item, page.n_total FROM (\ + SELECT {keys}, item, \ + sum(octet_length(item)) OVER (ORDER BY {lim_order} \ + ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS running, \ + count(*) OVER () AS n_total \ + FROM (\ + SELECT {inner_select}, t.item_data::text AS item FROM {ddb_table} t \ + {seek}ORDER BY {t_order} LIMIT ${n_param}\ + ) lim\ + ) page \ + WHERE page.running - octet_length(page.item) < ${budget_param} \ + ORDER BY {page_order}", + lim_order = order("lim"), + t_order = order(""), + page_order = order("page"), + ) +} + +/// The page query of `description`'s table, for the crate's tests: the +/// text [`PageQuery`] runs for a page after a key when `after_key`, else the +/// first page's, with `COLLATE "C"` on the text key columns when +/// `collate_c` (what `open_stream` decides from `pg_attribute`). +#[doc(hidden)] +#[must_use] +pub fn page_sql_for( + description: &TableDescription, + table_id: &str, + collate_c: bool, + after_key: bool, +) -> String { + page_sql( + &data_table_name(table_id), + &key_columns(description), + collate_c, + after_key, + ) +} + +/// One page as the database returned it, decoded. +struct Page { + /// The items the byte cut kept, in key order. + items: Vec, + /// The last kept row's key: where the next page starts. `None` for an + /// empty page. + last_key: Option>, + /// Each kept row's text, in bytes, in key order: what the next page's + /// row bound is sized from. + row_bytes: Vec, + /// Rows the innermost `LIMIT` returned, cut or not; fewer than the row + /// bound means the table is exhausted. + n_total: usize, +} + +/// Run the page query and decode its rows as they arrive: the rows are +/// pulled from the connection one at a time (`fetch`), each row's text is +/// parsed into its `Item` and dropped, so the page's text is never held +/// whole; what the page holds is its items. A row whose text does not parse +/// ends the page with `Internal`; a database error is mapped by +/// [`map_fetch_error`]. +async fn fetch_page( + conn: &mut PgConnection, + query: &PageQuery, + after: Option<&[KeyValue]>, + n: usize, + budget: usize, +) -> Result { + use futures::TryStreamExt as _; + let sql = if after.is_some() { + &query.after + } else { + &query.first + }; + let mut q = sqlx::query(sql); + for value in after.into_iter().flatten() { + q = match value { + KeyValue::Text(s) | KeyValue::Numeric(s) => q.bind(s.as_str()), + KeyValue::Bytes(b) => q.bind(b.as_slice()), + }; + } + let mut rows = q + .bind(i64::try_from(n).unwrap_or(i64::MAX)) + .bind(i64::try_from(budget).unwrap_or(i64::MAX)) + .fetch(conn); + let mut page = Page { + items: Vec::new(), + last_key: None, + row_bytes: Vec::new(), + n_total: 0, + }; + while let Some(row) = rows.try_next().await.map_err(|e| map_fetch_error(&e))? { + let row: PgRow = row; + let n_total: i64 = row.try_get("n_total").map_err(|e| db_err(&e))?; + page.n_total = usize::try_from(n_total).unwrap_or(0); + let mut key = Vec::with_capacity(query.columns.len()); + for (i, column) in query.columns.iter().enumerate() { + let name = format!("c{i}"); + key.push(match column.kind { + KeyKind::Text => { + KeyValue::Text(row.try_get(name.as_str()).map_err(|e| db_err(&e))?) + } + KeyKind::Numeric => { + KeyValue::Numeric(row.try_get(name.as_str()).map_err(|e| db_err(&e))?) + } + KeyKind::Bytes => { + KeyValue::Bytes(row.try_get(name.as_str()).map_err(|e| db_err(&e))?) + } + }); + } + let data: String = row.try_get("item").map_err(|e| db_err(&e))?; + drop(row); + let item = serde_json::from_str::(&data) + .map_err(|e| StorageError::Internal(format!("Parse item: {e}")))?; + page.row_bytes.push(data.len()); + drop(data); + page.items.push(item); + page.last_key = Some(key); + } + Ok(page) +} + +/// Rows the next page asks for, from the largest row of the last pages +/// (`memory`, [`RowBoundMemory`]): the rows the budget holds at that row +/// (`budget / largest`), between [`MIN_NEXT_PAGE_ROWS`] and the row bound. +/// The largest row and not the average, because the server serializes every +/// row the inner `LIMIT` reads: after a page of small rows with a few large +/// ones among them the average is small and a bound from it asks the next +/// page for hundreds of rows of a table that may now be large rows only (one +/// such page measured at 5.9 GB of server temp files for the three rows it +/// returned), where the largest row asks for the rows the budget holds of +/// it. A table of small rows pages at the row bound; a table of 27 MB rows +/// at [`MIN_NEXT_PAGE_ROWS`], of which one row per page is read past the cut +/// and dropped by the outer `WHERE`; a table whose large rows are scattered +/// among small ones pages at the row bound until the first page that meets +/// a large row, which reads up to its `n` rows of up to the line cap, and at +/// [`MIN_NEXT_PAGE_ROWS`] from there while large rows keep arriving within a +/// row bound's worth of rows, the bound returning to the row bound a row +/// bound's worth of small rows after the last large one; so every page that +/// follows a row bound's worth of small rows can read `n` rows of up to the +/// line cap, and no page can read more. The floor's cost: after a page +/// holding a row at the line cap the bound stays at the floor for a row +/// bound's worth of rows, 250 pages of four, whatever those rows hold, so a +/// table whose large rows are rarer than one per row bound pays 250 round +/// trips per large row (2,585 page queries against 38 with one page of +/// memory on a 20,000-row table with a 27 MB row at every 1,100th key, 25.6 +/// against 25.1 s, the record). A memory that held no row (an empty page) +/// gives the row bound. +fn next_page_rows(memory: &RowBoundMemory, budget: usize, batch: usize) -> usize { + let Some(max_row) = memory.largest_row() else { + return batch; + }; + (budget / max_row).max(MIN_NEXT_PAGE_ROWS).min(batch) +} + +/// Stream items by keyset pages on the held transaction. The held /// transaction lives in the stream's state: the last page commits it (waking /// the poller first), an error rolls it back, and dropping the stream before /// its end rolls it back through `HeldSnapshot`'s drop; each wakes the -/// control-plane poller. -fn cursor_stream( +/// control-plane poller. The row bound and the byte budget are read once +/// here, when the stream is built. +fn keyset_stream( held: HeldSnapshot, slot: OwnedSemaphorePermit, + query: PageQuery, ) -> BoxStream<'static, Result> { struct State { held: HeldSnapshot, /// Released with the transaction: the slot counts the connection. slot: Option, + query: PageQuery, + /// The last row's key; `None` before the first page. + after: Option>, + /// Rows the next page asks for. + n: usize, + batch: usize, + budget: usize, + /// The largest rows of the last pages, which size `n`. + memory: RowBoundMemory, buffer: std::vec::IntoIter, done: bool, } @@ -604,13 +1007,19 @@ fn cursor_stream( self.slot = None; } } + let batch = paging::fetch_batch(); let state = State { held, slot: Some(slot), + query, + after: None, + n: MIN_PAGE_ROWS.min(batch), + batch, + budget: paging::fetch_bytes(), + memory: RowBoundMemory::new(batch), buffer: Vec::new().into_iter(), done: false, }; - let batch = paging::fetch_batch(); Box::pin(futures::stream::unfold( state, move |mut state| async move { @@ -623,38 +1032,43 @@ fn cursor_stream( } let conn = state.held.conn()?; paging::record_fetch(); - let rows: Result, sqlx::Error> = - sqlx::query_as(&format!("FETCH {batch} FROM {CURSOR}")) - .fetch_all(conn) - .await; - let rows = match rows { - Ok(rows) => rows, + let page = fetch_page( + conn, + &state.query, + state.after.as_deref(), + state.n, + state.budget, + ) + .await; + let page = match page { + Ok(page) => page, Err(e) => { state.abandon(); - return Some((Err(map_fetch_error(&e)), state)); + return Some((Err(e), state)); } }; - if rows.len() < batch { + let returned = page.items.len(); + paging::record_page_bytes(page.row_bytes.iter().sum()); + paging::record_rows_past_cut( + u64::try_from(page.n_total.saturating_sub(returned)).unwrap_or(u64::MAX), + ); + // The table is exhausted when the limit returned fewer rows + // than asked and the cut dropped none of them; a short page + // the cut shortened still has rows past its last one. + if page.n_total < state.n && returned == page.n_total { state.done = true; // Close the transaction now rather than at drop so vacuum is // released as soon as the last row is out. let _ = state.held.finish().await; state.slot = None; + } else { + state.after = page.last_key; + state + .memory + .record(page.row_bytes.iter().copied().max().unwrap_or(0), returned); + state.n = next_page_rows(&state.memory, state.budget, state.batch); } - let mut items = Vec::with_capacity(rows.len()); - for (data,) in rows { - match serde_json::from_str::(&data) { - Ok(item) => items.push(item), - Err(e) => { - state.abandon(); - return Some(( - Err(StorageError::Internal(format!("Parse item: {e}"))), - state, - )); - } - } - } - state.buffer = items.into_iter(); + state.buffer = page.items.into_iter(); } }, )) @@ -679,3 +1093,140 @@ pub fn map_fetch_error(e: &sqlx::Error) -> StorageError { } db_err(e) } + +#[cfg(test)] +mod tests { + use super::*; + + fn columns(kinds: &[(&str, KeyKind)]) -> Vec { + kinds + .iter() + .map(|(name, kind)| KeyColumn { + name: (*name).to_owned(), + kind: *kind, + }) + .collect() + } + + fn memory_of(pages: &[(usize, usize)], rows: usize) -> RowBoundMemory { + let mut memory = RowBoundMemory::new(rows); + for &(max_row, n) in pages { + memory.record(max_row, n); + } + memory + } + + /// The row bound follows the largest row of the remembered pages within + /// its clamps: small rows page at the row bound, 27 MB rows at the floor + /// (the budget holds two, the floor is four), a memory that held no row + /// (an empty page) at the row bound, a row bound under the floor (a + /// test's small page) is the bound; and a page of 984 small rows and 3 + /// large ones is sized from the large row, not from the average (which + /// is 82 KB and would ask the next page for about 800 rows of a table + /// that is now large rows only). + #[test] + fn the_row_bound_adapts_within_its_clamps() { + let budget = 64 * 1024 * 1024; + assert_eq!( + next_page_rows(&memory_of(&[(300, 16)], 1000), budget, 1000), + 1000 + ); + // The floor as its value: the budget holds two such rows, a page + // keeps three, and four reads one past the cut. + assert_eq!( + next_page_rows(&memory_of(&[(27_000_000, 3)], 1000), budget, 1000), + 4 + ); + assert_eq!( + next_page_rows(&memory_of(&[(1024 * 1024, 8)], 1000), budget, 1000), + 64 + ); + assert_eq!( + next_page_rows(&memory_of(&[(0, 0)], 1000), budget, 1000), + 1000 + ); + assert_eq!(next_page_rows(&memory_of(&[(300, 3)], 3), budget, 3), 3); + // 984 rows of 300 bytes and 3 of 27 MB: the page's largest row sizes + // the next page. + let next = next_page_rows(&memory_of(&[(27_000_000, 987)], 1000), budget, 1000); + assert!( + next < 8, + "the next page asks for {next} rows after a page of 984 small rows and 3 large ones" + ); + assert!(next >= MIN_NEXT_PAGE_ROWS); + } + + /// The memory keeps the pages that together hold at least the row bound: + /// after a page with a 27 MB row, pages of four small rows keep the bound + /// at the floor until a row bound's worth of them have passed (250 pages + /// of four at a row bound of 1000), and the page after that is sized from + /// the small rows alone; a large row met within that span starts the + /// count again; the memory holds at most the pages it needs (the oldest + /// goes once the pages after it hold the row bound), so it does not grow + /// with the table. + #[test] + fn the_memory_covers_a_row_bound_of_rows() { + let budget = 64 * 1024 * 1024; + let mut memory = RowBoundMemory::new(1000); + memory.record(27_000_000, 83); + for _ in 0..249 { + memory.record(300, 4); + assert_eq!(next_page_rows(&memory, budget, 1000), MIN_NEXT_PAGE_ROWS); + } + // 249 pages of four hold 996 rows after the large page: still kept. + assert_eq!(memory.pages.len(), 250); + memory.record(300, 4); + // 1000 rows after it: the large page is forgotten. + assert_eq!(next_page_rows(&memory, budget, 1000), 1000); + assert_eq!(memory.pages.len(), 250); + // A large row within the span starts the count again. + memory.record(27_000_000, 4); + for _ in 0..249 { + memory.record(300, 4); + } + assert_eq!(next_page_rows(&memory, budget, 1000), MIN_NEXT_PAGE_ROWS); + memory.record(300, 4); + assert_eq!(next_page_rows(&memory, budget, 1000), 1000); + // Pages of the row bound: one page of memory is enough. + let mut memory = RowBoundMemory::new(1000); + memory.record(27_000_000, 3); + memory.record(300, 1000); + assert_eq!(memory.pages.len(), 1); + assert_eq!(next_page_rows(&memory, budget, 1000), 1000); + } + + /// The seek compares the key columns as their own types (numeric through + /// text, bytes as bytea), the first page has no seek, the text columns + /// carry `COLLATE "C"` only when asked, the row bound and the budget + /// take the parameters after the key, and `item_data::text` is computed + /// once in the innermost layer. + #[test] + fn the_page_query_binds_the_key_by_type() { + let cols = columns(&[("pk", KeyKind::Text), ("sk_n", KeyKind::Numeric)]); + let after = page_sql("\"_ddb_t\"", &cols, true, true); + assert!( + after.contains("WHERE (t.pk, t.sk_n) > ($1::text, $2::text::numeric) ORDER BY t.pk COLLATE \"C\", t.sk_n LIMIT $3"), + "{after}" + ); + assert!( + after.contains("< $4 ORDER BY page.k0 COLLATE \"C\", page.k1"), + "{after}" + ); + assert!(after.contains("page.k1::text AS c1"), "{after}"); + assert_eq!(after.matches("item_data::text").count(), 1, "{after}"); + let first = page_sql("\"_ddb_t\"", &cols, false, false); + assert!(!first.contains("WHERE (t.pk"), "{first}"); + assert!(first.contains("ORDER BY t.pk, t.sk_n LIMIT $1"), "{first}"); + assert!(first.contains("< $2 ORDER BY page.k0, page.k1"), "{first}"); + let bytes = columns(&[ + ("pk", KeyKind::Text), + ("sk_b", KeyKind::Bytes), + ("sk2_s", KeyKind::Text), + ]); + let after = page_sql("\"_ddb_t\"", &bytes, true, true); + assert!( + after.contains("> ($1::text, $2::bytea, $3::text) ORDER BY t.pk COLLATE \"C\", t.sk_b, t.sk2_s COLLATE \"C\" LIMIT $4"), + "{after}" + ); + } +} diff --git a/crates/storage-postgres/src/worker_store.rs b/crates/storage-postgres/src/worker_store.rs index e24f8f38e..01145fb60 100755 --- a/crates/storage-postgres/src/worker_store.rs +++ b/crates/storage-postgres/src/worker_store.rs @@ -176,24 +176,6 @@ impl PostgresEngine { .await .map_err(|e| StorageError::Internal(e.to_string()))?; - // CREATING with no owner → removed. A restore whose process died - // mid-copy leaves its target CREATING with no scheduled transition. - // The poller is notification-driven with a 60-second idle timeout, so - // the 60-second grace means an idle abandoned target is removed about - // 60–120 seconds after ownership is lost. Every pass opens a fresh - // out-of-pool connection for each old candidate to probe its advisory - // lock; a still-owned restore closes that probe and is checked again on - // the next pass. Failures are logged and skipped, so they cannot hold - // up the transitions above. - match self.sweep_abandoned_restores().await { - Ok(names) => { - for name in names { - transitions.push((name, "abandoned restore → removed")); - } - } - Err(e) => tracing::warn!("abandoned restore sweep failed: {e}"), - } - Ok(ControlPlanePass { transitions, deferred_drops, diff --git a/crates/storage-postgres/tests/backup_restore.rs b/crates/storage-postgres/tests/backup_restore.rs index 5c0a6f588..e57347186 100644 --- a/crates/storage-postgres/tests/backup_restore.rs +++ b/crates/storage-postgres/tests/backup_restore.rs @@ -14,7 +14,9 @@ //! - a backup with two rows for one key must fail rather than collapse them; //! - a backup of a multi-part key table (a preview gated by //! `enable_multipart_keys`) must be refused, not restored wrongly; -//! - the abandoned-restore sweep, which needs a crash to reach. +//! - the reconciler's duties on a dead or failed restore, which need a crash +//! to reach through the server and are driven here through the +//! `BackupCatalog` methods the reconciler calls. //! //! Each test builds a throwaway catalog and data database and drops them when //! it ends, passing or failing. The tests run one at a time: each opens its @@ -31,7 +33,7 @@ use extenddb_core::validation::validate_create_table; use extenddb_storage::error::StorageError; use extenddb_storage::migrations::Runner; use extenddb_storage::{ - BackupEngine, DataEngine, RestoreRequest, RestoreTableOverrides, TableEngine, + DataEngine, JobPermit, LegacyBackupReader, RestoreRequest, RestoreTableOverrides, TableEngine, }; use extenddb_storage_postgres::migrations::{PostgresMigrations, PostgresTarget}; use extenddb_storage_postgres::{PostgresConfig, PostgresEngine}; @@ -40,13 +42,10 @@ use sqlx::PgPool; use sqlx::postgres::PgPoolOptions; const ACCOUNT: &str = "123456789012"; -/// A second account, for the per-account restore slots. +/// A second account, for the cross-account checks. const OTHER_ACCOUNT: &str = "210987654321"; const REGION: &str = "us-east-1"; -/// The seed of the restore advisory-lock key (`backup_engine.rs`). -const RESTORE_LOCK_SEED: i64 = 0x0045_4452; -/// Restore slots per account (`backup_engine.rs`). -const RESTORE_SLOTS_PER_ACCOUNT: usize = 4; +use extenddb_storage_postgres::RESTORE_LOCK_SEED; /// A catalog database and a separate data database, the layout `extenddb /// init` creates. They must be separate here: the catalog schema carries a @@ -58,6 +57,8 @@ struct Scratch { db: PgPool, catalog: PgPool, admin: PgPool, + /// The catalog's URL, for a second engine on the same catalog. + catalog_url: String, db_names: [String; 2], _guard: DbGuard, _serial: tokio::sync::MutexGuard<'static, ()>, @@ -217,7 +218,7 @@ async fn scratch_with_pool(pool_size: u32) -> Scratch { } let engine = PostgresEngine::new( &PostgresConfig { - connection_string: catalog_url, + connection_string: catalog_url.clone(), pool_size, max_item_size_bytes: 400_000, }, @@ -230,6 +231,7 @@ async fn scratch_with_pool(pool_size: u32) -> Scratch { db, catalog, admin, + catalog_url, db_names: [catalog_name, data_name], _guard: guard, _serial: serial, @@ -276,7 +278,7 @@ async fn legacy_backup_of( .expect("create the source table"); let backup = s .engine - .create_backup(ACCOUNT, &source_name, "legacy") + .create(ACCOUNT, &source_name, "legacy", JobPermit::detached()) .await .expect("back up the empty source"); for item in rows { @@ -327,7 +329,7 @@ async fn restore_request( target: &str, backup_arn: &str, ) -> Result { - let source = s.engine.backup_restore_source(account, backup_arn).await?; + let source = s.engine.restore_source(account, backup_arn).await?; source.ensure_restorable()?; let input = source.restore_input(target, &RestoreTableOverrides::default()); validate_create_table(&input, &LimitsConfig::default()) @@ -349,7 +351,10 @@ async fn start_restore_in( backup_arn: &str, ) -> Result { let request = restore_request(s, account, target, backup_arn).await?; - let desc = s.engine.restore_table_from_backup(account, request).await?; + let desc = s + .engine + .restore(account, request, JobPermit::detached()) + .await?; assert_eq!( desc.table_status, TableStatus::Creating, @@ -437,16 +442,28 @@ async fn await_copies_waiting(s: &Scratch, n: i64) { /// The backend pid holding the restore lock for `account`/`table`, if any. async fn restore_lock_holder(s: &Scratch, account: &str, table: &str) -> Option { + advisory_lock_holder(s, &format!("{account}/{table}")).await +} + +/// The backend pid holding the claim lock of backup `backup_id` +/// (`backup:` under the shared seed), if any. +async fn backup_lock_holder(s: &Scratch, backup_id: &str) -> Option { + advisory_lock_holder(s, &format!("backup:{backup_id}")).await +} + +/// The backend pid holding the advisory lock `hashtextextended(key, +/// RESTORE_LOCK_SEED)` on the catalog database, if any. +async fn advisory_lock_holder(s: &Scratch, key: &str) -> Option { sqlx::query_scalar( "SELECT pid FROM pg_locks WHERE locktype = 'advisory' AND granted \ AND database = (SELECT oid FROM pg_database WHERE datname = current_database()) \ AND ((classid::bigint << 32) | objid::bigint) = hashtextextended($1, $2)", ) - .bind(format!("{account}/{table}")) + .bind(key) .bind(RESTORE_LOCK_SEED) .fetch_optional(&s.catalog) .await - .expect("find the restore lock holder") + .expect("find the advisory lock holder") } /// Rows of `table_restores` for a target: (source backup ARN, source table @@ -704,6 +721,14 @@ async fn indexed_source_with(s: &Scratch, name: &str, items: usize) -> String { .create_table(ACCOUNT, input) .await .expect("create the source table"); + put_indexed_items(s, name, &desc, items).await; + desc.table_id +} + +/// Write `items` items through PutItem into the table `indexed_source` +/// created (`desc` its description): every item has `pk` and `sk`, every +/// second one `g` (the GSI's key), every third one `l` (the LSI's). +async fn put_indexed_items(s: &Scratch, name: &str, desc: &TableDescription, items: usize) { let key_info = TableKeyInfo { table_name: name.to_owned(), account_id: ACCOUNT.to_owned(), @@ -735,7 +760,6 @@ async fn indexed_source_with(s: &Scratch, name: &str, items: usize) -> String { .await .expect("put"); } - desc.table_id } /// `(index_name, index_id)` of a table's secondary indexes, by name. @@ -826,7 +850,7 @@ async fn restore_recreates_indexes_and_throughput() { let src = indexed_source(&s, "idx_src").await; let backup = s .engine - .create_backup(ACCOUNT, "idx_src", "b") + .create(ACCOUNT, "idx_src", "b", JobPermit::detached()) .await .expect("backup"); let desc = restore(&s, "idx_dst", &backup.backup_arn) @@ -871,89 +895,128 @@ async fn restore_recreates_indexes_and_throughput() { s.cleanup().await; } +/// The reconciler's duty for a legacy restore whose owner died: a RUNNING +/// job record is dead once its heartbeat is older than the stale window on +/// the catalog's clock, whatever the table's age and whether or not the +/// advisory lock is free; a stale record is marked FAILED and its target +/// removed through `remove_restore_target`, with its data and index tables; +/// a fresh record, and a record that is no longer RUNNING, are left alone. #[tokio::test] -async fn abandoned_restore_is_removed_only_when_unowned_and_old() { +async fn abandoned_restore_is_removed_only_when_its_heartbeat_is_stale() { + use extenddb_storage::BackupCatalog; + use extenddb_storage::backup_catalog::{RestoreKind, RestoreTargetKey}; if base_conn().is_none() { - eprintln!("SKIP abandoned_restore_is_removed_only_when_unowned_and_old: no PostgreSQL"); + eprintln!( + "SKIP abandoned_restore_is_removed_only_when_its_heartbeat_is_stale: no PostgreSQL" + ); return; } + let stale = std::time::Duration::from_secs(300); let s = scratch().await; indexed_source(&s, "ab_src").await; let backup = s .engine - .create_backup(ACCOUNT, "ab_src", "b") + .create(ACCOUNT, "ab_src", "b", JobPermit::detached()) .await .expect("backup"); let desc = restore(&s, "ab_dst", &backup.backup_arn) .await .expect("restore"); assert_eq!(table_status(&s, "ab_dst").await.as_deref(), Some("ACTIVE")); + assert_eq!( + job_status(&s, &desc.table_id).await.as_deref(), + Some("DONE"), + "a finished restore's record is DONE" + ); let indexes = index_ids(&s, &desc.table_id).await; + let key = RestoreTargetKey { + account_id: ACCOUNT.to_owned(), + table_name: "ab_dst".to_owned(), + table_id: desc.table_id.clone(), + }; // The state a process killed mid-copy leaves: the target CREATING with no - // scheduled transition, some of its copy's batches committed. + // scheduled transition, its job record RUNNING with a heartbeat `age` + // old, some of its copy's batches committed. let crash = |age: &'static str| { let catalog = s.catalog.clone(); let id = desc.table_id.clone(); async move { + sqlx::query( + "UPDATE tables SET table_status = 'CREATING', status_transition_at = NULL \ + WHERE table_id = $1", + ) + .bind(&id) + .execute(&catalog) + .await + .expect("simulate a crash"); sqlx::query(&format!( - "UPDATE tables SET table_status = 'CREATING', status_transition_at = NULL, \ - creation_date_time = NOW() - INTERVAL '{age}' WHERE table_id = $1" + "UPDATE table_restores SET status = 'RUNNING', completed_at = NULL, \ + heartbeat_at = NOW() - INTERVAL '{age}' WHERE table_id = $1" )) .bind(&id) .execute(&catalog) .await - .expect("simulate a crash"); + .expect("age the heartbeat"); } }; - // Inside the grace period: a restore that has not taken its lock yet. + // A fresh heartbeat: a live restore, here or on another instance, even + // though the table itself is old. crash("1 second").await; - s.engine - .process_control_plane_transitions() + sqlx::query("UPDATE tables SET creation_date_time = NOW() - INTERVAL '10 minutes'") + .execute(&s.catalog) .await - .expect("sweep"); + .expect("age every table"); + assert!( + s.engine + .list_dead_restore_jobs(stale) + .await + .expect("list") + .is_empty() + ); assert_eq!( table_status(&s, "ab_dst").await.as_deref(), Some("CREATING") ); - // Old, but its lock is held: a live restore on another instance. The key - // is the account and name, so the lock covers the target before it exists. + // A stale heartbeat: dead, whether or not its lock session is still + // around (a free lock shows the session ended, not that the process did, + // and the record's heartbeat is what the reconciler waits on). crash("10 minutes").await; - let mut owner = s.catalog.acquire().await.expect("connection"); - sqlx::query("SELECT pg_advisory_lock(hashtextextended($2, $1))") - .bind(0x0045_4452_i64) - .bind(format!("{ACCOUNT}/ab_dst")) - .execute(&mut *owner) - .await - .expect("hold the restore lock"); - s.engine - .process_control_plane_transitions() - .await - .expect("sweep"); + let dead = s.engine.list_dead_restore_jobs(stale).await.expect("list"); + assert_eq!(dead.len(), 1, "{dead:?}"); + assert_eq!(dead[0].target, key); + assert_eq!(dead[0].kind, RestoreKind::Legacy); + assert_eq!(dead[0].source_backup_arn, backup.backup_arn); + assert!( + s.engine + .fail_restore_job( + &key, + dead[0].owner_generation, + dead[0].heartbeat_at, + "owner dead" + ) + .await + .expect("fail") + ); assert_eq!( - table_status(&s, "ab_dst").await.as_deref(), - Some("CREATING") + job_status(&s, &desc.table_id).await.as_deref(), + Some("FAILED") ); - sqlx::query("SELECT pg_advisory_unlock(hashtextextended($2, $1))") - .bind(0x0045_4452_i64) - .bind(format!("{ACCOUNT}/ab_dst")) - .execute(&mut *owner) - .await - .expect("release the restore lock"); - drop(owner); - - // Old and unowned: removed, with its data and index tables. - let transitions = s - .engine - .process_control_plane_transitions() - .await - .expect("sweep"); assert!( - transitions.iter().any(|(n, _)| n == "ab_dst"), - "{transitions:?}" + !s.engine + .fail_restore_job( + &key, + dead[0].owner_generation, + dead[0].heartbeat_at, + "owner dead" + ) + .await + .expect("fail again"), + "a record that is no longer RUNNING is not written twice" ); + assert!(s.engine.remove_restore_target(&key).await.expect("remove")); assert_eq!(table_status(&s, "ab_dst").await, None); assert!(!data_table_exists(&s, &desc.table_id).await); for (_, id) in &indexes { @@ -962,24 +1025,64 @@ async fn abandoned_restore_is_removed_only_when_unowned_and_old() { "index table {id} left behind" ); } + assert!( + !s.engine + .remove_restore_target(&key) + .await + .expect("remove again"), + "a removed target is removed once" + ); - // An ACTIVE table is never a candidate, and the name is free again. - restore(&s, "ab_dst", &backup.backup_arn) + // An ACTIVE table is never a candidate, the name is free again, and a + // key with the wrong table id (a table re-created under the name) does + // not remove the new table. + let again = restore(&s, "ab_dst", &backup.backup_arn) .await .expect("restore again"); assert_eq!(table_status(&s, "ab_dst").await.as_deref(), Some("ACTIVE")); - sqlx::query("UPDATE tables SET creation_date_time = NOW() - INTERVAL '10 minutes'") - .execute(&s.catalog) - .await - .expect("age every table"); - s.engine - .process_control_plane_transitions() - .await - .expect("sweep"); - assert_eq!(table_status(&s, "ab_dst").await.as_deref(), Some("ACTIVE")); + assert!( + s.engine + .list_dead_restore_jobs(stale) + .await + .expect("list") + .is_empty() + ); + assert!( + !s.engine + .remove_restore_target(&key) + .await + .expect("stale key") + ); + sqlx::query( + "UPDATE tables SET table_status = 'CREATING', status_transition_at = NULL WHERE table_id = $1", + ) + .bind(&again.table_id) + .execute(&s.catalog) + .await + .expect("make it a target again"); + assert!( + !s.engine + .remove_restore_target(&key) + .await + .expect("stale key"), + "the old table id never removes the new table" + ); + assert_eq!( + table_status(&s, "ab_dst").await.as_deref(), + Some("CREATING") + ); s.cleanup().await; } +/// The job record's status column for a target. +async fn job_status(s: &Scratch, table_id: &str) -> Option { + sqlx::query_scalar("SELECT status FROM table_restores WHERE table_id = $1") + .bind(table_id) + .fetch_optional(&s.catalog) + .await + .expect("read the job status") +} + #[tokio::test] async fn failed_restore_with_indexes_leaves_no_tables() { if base_conn().is_none() { @@ -990,7 +1093,7 @@ async fn failed_restore_with_indexes_leaves_no_tables() { indexed_source(&s, "fi_src").await; let backup = s .engine - .create_backup(ACCOUNT, "fi_src", "b") + .create(ACCOUNT, "fi_src", "b", JobPermit::detached()) .await .expect("backup"); sqlx::query("INSERT INTO backup_items (backup_arn, pk, item_data) VALUES ($1, 'x', $2)") @@ -1049,7 +1152,7 @@ async fn restore_keeps_sse_and_on_demand_limits_but_not_table_class() { let src = s.engine.create_table(ACCOUNT, input).await.expect("create"); let backup = s .engine - .create_backup(ACCOUNT, "cls_src", "b") + .create(ACCOUNT, "cls_src", "b", JobPermit::detached()) .await .expect("backup"); restore(&s, "cls_dst", &backup.backup_arn) @@ -1113,7 +1216,7 @@ async fn restore_crosses_batch_boundaries() { .collect(); let backup = s .engine - .create_backup(ACCOUNT, "many_src", "b") + .create(ACCOUNT, "many_src", "b", JobPermit::detached()) .await .expect("backup"); sqlx::query( @@ -1156,12 +1259,12 @@ async fn restore_into_a_name_whose_restore_lock_is_held_is_refused() { indexed_source(&s, "lk_src").await; let backup = s .engine - .create_backup(ACCOUNT, "lk_src", "b") + .create(ACCOUNT, "lk_src", "b", JobPermit::detached()) .await .expect("backup"); let mut other = s.catalog.acquire().await.expect("connection"); sqlx::query("SELECT pg_advisory_lock(hashtextextended($2, $1))") - .bind(0x0045_4452_i64) + .bind(RESTORE_LOCK_SEED) .bind(format!("{ACCOUNT}/lk_dst")) .execute(&mut *other) .await @@ -1202,21 +1305,22 @@ async fn a_restore_is_refused_while_a_target_of_its_backup_is_creating() { indexed_source(&s, "iu_src").await; let b1 = s .engine - .create_backup(ACCOUNT, "iu_src", "b1") + .create(ACCOUNT, "iu_src", "b1", JobPermit::detached()) .await .expect("backup 1"); let b2 = s .engine - .create_backup(ACCOUNT, "iu_src", "b2") + .create(ACCOUNT, "iu_src", "b2", JobPermit::detached()) .await .expect("backup 2"); - // Backups for the restores that fill the account's other slots, taken - // before the gate (a backup writes `backup_items`, which the gate locks). + // Backups for the other restores that run alongside, taken before the + // gate (a backup writes `backup_items`, which the gate locks). + const MORE_RESTORES: usize = 4; let mut more_backups = Vec::new(); - for i in 3..=(RESTORE_SLOTS_PER_ACCOUNT + 1) { + for i in 3..=(MORE_RESTORES + 1) { more_backups.push( s.engine - .create_backup(ACCOUNT, "iu_src", &format!("b{i}")) + .create(ACCOUNT, "iu_src", &format!("b{i}"), JobPermit::detached()) .await .expect("backup") .backup_arn, @@ -1234,8 +1338,10 @@ async fn a_restore_is_refused_while_a_target_of_its_backup_is_creating() { let second = start_restore(&s, "iu_dst2", &b1.backup_arn).await; assert!( - matches!(&second, Err(StorageError::BackupInUse(m)) if m.contains("iu_dst")), - "a second restore of a backup with a CREATING target: {second:?}" + matches!(&second, Err(StorageError::BackupInUse(m)) + if *m == format!("Backup is being used to restore another table: {}", b1.backup_arn)), + "a second restore of a backup with a CREATING target answers the service's text: \ + {second:?}" ); assert_eq!(table_status(&s, "iu_dst2").await, None); let into_target = start_restore(&s, "iu_dst", &b2.backup_arn).await; @@ -1249,9 +1355,31 @@ async fn a_restore_is_refused_while_a_target_of_its_backup_is_creating() { } other => panic!("a restore into a CREATING restore target: {other:?}"), } - // The name is checked before the backup and before a slot is taken: the - // same backup into the same name, and a backup that does not exist into - // it, are refused as the name, naming the target's source backup. + // What the engine's own name check reads before its job permit: the + // target describes as CREATING with its source backup in the + // RestoreSummary, which is what it answers TableInUse with. + let described = s + .engine + .describe_table( + ACCOUNT, + DescribeTableInput { + table_name: "iu_dst".to_owned(), + }, + ) + .await + .expect("describe the CREATING target"); + assert_eq!(described.table_status, TableStatus::Creating); + assert_eq!( + described + .restore_summary + .as_ref() + .and_then(|r| r.source_backup_arn.as_deref()), + Some(b1.backup_arn.as_str()), + "the CREATING target names its source backup" + ); + // The name is checked before the backup: the same backup into the same + // name, and a backup that does not exist into it, are refused as the + // name, naming the target's source backup. let missing_arn = flip_last_hex(&b2.backup_arn); let mut missing = restore_request(&s, ACCOUNT, "iu_dst", &b2.backup_arn) .await @@ -1264,7 +1392,9 @@ async fn a_restore_is_refused_while_a_target_of_its_backup_is_creating() { ), ( "a missing backup into a CREATING restore target", - s.engine.restore_table_from_backup(ACCOUNT, missing).await, + s.engine + .restore(ACCOUNT, missing, JobPermit::detached()) + .await, ), ] { match result { @@ -1280,23 +1410,23 @@ async fn a_restore_is_refused_while_a_target_of_its_backup_is_creating() { } // Only the first copy is waiting: the refused restores never started one. assert_eq!(copies_waiting(&s).await, 1); - // With every account slot taken, a client retrying its own restore is - // still answered at once, not after the 30 s slot wait. + // With several copies running, a client retrying its own restore is + // still answered at once. let mut more = Vec::new(); for (i, backup) in more_backups.iter().enumerate() { let target = format!("iu_more{i}"); start_restore(&s, &target, backup) .await - .expect("a restore within the account's slots"); + .expect("a restore alongside the first"); more.push(target); } - await_copies_waiting(&s, i64::try_from(RESTORE_SLOTS_PER_ACCOUNT).unwrap()).await; + await_copies_waiting(&s, i64::try_from(MORE_RESTORES).unwrap()).await; let retried = tokio::time::timeout( std::time::Duration::from_secs(5), start_restore(&s, "iu_dst", &b1.backup_arn), ) .await - .expect("a retry into a CREATING restore target does not wait for a slot"); + .expect("a retry into a CREATING restore target is answered at once"); assert!( matches!(&retried, Err(StorageError::TableBeingRestored { table_name, .. }) if table_name == "iu_dst"), "{retried:?}" @@ -1318,6 +1448,19 @@ async fn a_restore_is_refused_while_a_target_of_its_backup_is_creating() { matches!(&taken, Err(StorageError::TableAlreadyExists(n)) if n == "iu_dst"), "{taken:?}" ); + // The former target describes as ACTIVE, so the engine's name check + // answers TableAlreadyExists for it too. + let former = s + .engine + .describe_table( + ACCOUNT, + DescribeTableInput { + table_name: "iu_dst".to_owned(), + }, + ) + .await + .expect("describe the former target"); + assert_eq!(former.table_status, TableStatus::Active); s.cleanup().await; } @@ -1334,7 +1477,7 @@ async fn simultaneous_restores_of_one_backup_admit_exactly_one() { indexed_source(&s, "sim_src").await; let backup = s .engine - .create_backup(ACCOUNT, "sim_src", "b") + .create(ACCOUNT, "sim_src", "b", JobPermit::detached()) .await .expect("backup"); let gate = gate_copies(&s).await; @@ -1349,7 +1492,7 @@ async fn simultaneous_restores_of_one_backup_admit_exactly_one() { let results = futures::future::join_all( requests .into_iter() - .map(|request| s.engine.restore_table_from_backup(ACCOUNT, request)), + .map(|request| s.engine.restore(ACCOUNT, request, JobPermit::detached())), ) .await; let accepted: Vec = results @@ -1388,7 +1531,7 @@ async fn pre_0_0_4_backup_restores_after_migrating() { indexed_source(&s, "old_src").await; let backup = s .engine - .create_backup(ACCOUNT, "old_src", "b") + .create(ACCOUNT, "old_src", "b", JobPermit::detached()) .await .expect("backup"); sqlx::query("DROP TABLE backup_definitions") @@ -1416,7 +1559,7 @@ async fn pre_0_0_4_backup_restores_after_migrating() { // definitions; otherwise validate_create_table would refuse it. let source = s .engine - .backup_restore_source(ACCOUNT, &backup.backup_arn) + .restore_source(ACCOUNT, &backup.backup_arn) .await .expect("legacy source"); assert!(source.definition.is_none()); @@ -1474,7 +1617,9 @@ async fn production_create_backup_excludes_item_committed_after_snapshot_pin() { .await .expect("block CreateBackup's catalog insert"); - let mut backup_fut = s.engine.create_backup(ACCOUNT, "prod_snap", "b"); + let mut backup_fut = s + .engine + .create(ACCOUNT, "prod_snap", "b", JobPermit::detached()); let early = tokio::time::timeout(std::time::Duration::from_secs(2), &mut backup_fut).await; assert!( early.is_err(), @@ -1526,7 +1671,7 @@ async fn delete_backup_winning_before_restore_leaves_no_target() { snapshot_source(&s, "delete_first_src").await; let backup = s .engine - .create_backup(ACCOUNT, "delete_first_src", "b") + .create(ACCOUNT, "delete_first_src", "b", JobPermit::detached()) .await .expect("backup"); @@ -1537,7 +1682,7 @@ async fn delete_backup_winning_before_restore_leaves_no_target() { .execute(&mut *gate) .await .expect("block DeleteBackup after its row lock"); - let mut delete_fut = s.engine.delete_backup(ACCOUNT, &backup.backup_arn); + let mut delete_fut = s.engine.delete(ACCOUNT, &backup.backup_arn); let early_delete = tokio::time::timeout(std::time::Duration::from_secs(2), &mut delete_fut).await; assert!(early_delete.is_err(), "DeleteBackup did not reach the gate"); @@ -1593,7 +1738,7 @@ async fn restore_registration_winning_before_delete_returns_backup_in_use() { snapshot_source(&s, "restore_first_src").await; let backup = s .engine - .create_backup(ACCOUNT, "restore_first_src", "b") + .create(ACCOUNT, "restore_first_src", "b", JobPermit::detached()) .await .expect("backup"); @@ -1625,7 +1770,7 @@ async fn restore_registration_winning_before_delete_returns_backup_in_use() { let err = s .engine - .delete_backup(ACCOUNT, &backup.backup_arn) + .delete(ACCOUNT, &backup.backup_arn) .await .expect_err("a registered in-progress restore keeps its backup in use"); assert!(matches!(err, StorageError::BackupInUse(_)), "{err:?}"); @@ -1806,7 +1951,9 @@ async fn backup_definition_and_items_share_one_instant() { // The backup must wait for the writer rather than read around it. let arn = { - let backup = s.engine.create_backup(ACCOUNT, "bar_src", "b"); + let backup = s + .engine + .create(ACCOUNT, "bar_src", "b", JobPermit::detached()); tokio::pin!(backup); let early = tokio::time::timeout(std::time::Duration::from_millis(500), &mut backup).await; assert!( @@ -1851,7 +1998,7 @@ async fn backup_leaves_out_a_gsi_still_being_built() { .expect("make the GSI look half-built"); let backup = s .engine - .create_backup(ACCOUNT, "unb_src", "b") + .create(ACCOUNT, "unb_src", "b", JobPermit::detached()) .await .expect("backup"); let def: serde_json::Value = @@ -1903,7 +2050,7 @@ async fn delete_table_and_delete_backup_are_refused_during_the_copy() { indexed_source(&s, "dt_src").await; let backup = s .engine - .create_backup(ACCOUNT, "dt_src", "b") + .create(ACCOUNT, "dt_src", "b", JobPermit::detached()) .await .expect("backup"); let gate = gate_copies(&s).await; @@ -1925,7 +2072,7 @@ async fn delete_table_and_delete_backup_are_refused_during_the_copy() { assert!(matches!(err, StorageError::IndexesInUse(_)), "{err:?}"); let err = s .engine - .delete_backup(ACCOUNT, &backup.backup_arn) + .delete(ACCOUNT, &backup.backup_arn) .await .expect_err("DeleteBackup of a backup being restored from is refused"); assert!(matches!(err, StorageError::BackupInUse(_)), "{err:?}"); @@ -1943,7 +2090,7 @@ async fn delete_table_and_delete_backup_are_refused_during_the_copy() { ); assert_eq!(data_rows(&s, &desc.table_id).await, 30); s.engine - .delete_backup(ACCOUNT, &backup.backup_arn) + .delete(ACCOUNT, &backup.backup_arn) .await .expect("deletable once the restore is done"); s.engine @@ -1972,7 +2119,7 @@ async fn restore_summary_and_backup_in_use() { indexed_source(&s, "rs_src").await; let backup = s .engine - .create_backup(ACCOUNT, "rs_src", "b") + .create(ACCOUNT, "rs_src", "b", JobPermit::detached()) .await .expect("backup"); let gate = gate_copies(&s).await; @@ -1996,7 +2143,7 @@ async fn restore_summary_and_backup_in_use() { }; let source = s .engine - .backup_restore_source(ACCOUNT, &backup.backup_arn) + .restore_source(ACCOUNT, &backup.backup_arn) .await .expect("source"); // (g) The registration records what the engine passed: the source table's @@ -2023,7 +2170,7 @@ async fn restore_summary_and_backup_in_use() { assert!(describe("rs_src").await.restore_summary.is_none()); let err = s .engine - .delete_backup(ACCOUNT, &backup.backup_arn) + .delete(ACCOUNT, &backup.backup_arn) .await .expect_err("in use"); assert!(matches!(err, StorageError::BackupInUse(_)), "{err:?}"); @@ -2036,7 +2183,7 @@ async fn restore_summary_and_backup_in_use() { let done = describe("rs_dst").await.restore_summary.expect("summary"); assert!(!done.restore_in_progress); s.engine - .delete_backup(ACCOUNT, &backup.backup_arn) + .delete(ACCOUNT, &backup.backup_arn) .await .expect("deletable once the restore is done"); let after = describe("rs_dst").await.restore_summary.expect("summary"); @@ -2111,7 +2258,7 @@ async fn restore_returns_creating_and_the_detached_copy_finishes() { let src = indexed_source(&s, "cr_src").await; let backup = s .engine - .create_backup(ACCOUNT, "cr_src", "b") + .create(ACCOUNT, "cr_src", "b", JobPermit::detached()) .await .expect("backup"); add_backup_rows(&s, &backup.backup_arn, &bulk_rows(1_200)).await; @@ -2188,14 +2335,14 @@ async fn a_dropped_restore_request_still_reaches_active() { indexed_source(&s, "drop_src").await; let backup = s .engine - .create_backup(ACCOUNT, "drop_src", "b") + .create(ACCOUNT, "drop_src", "b", JobPermit::detached()) .await .expect("backup"); add_backup_rows(&s, &backup.backup_arn, &bulk_rows(1_200)).await; let request = restore_request(&s, ACCOUNT, "drop_dst", &backup.backup_arn) .await .expect("request"); - let fut = s.engine.restore_table_from_backup(ACCOUNT, request); + let fut = s.engine.restore(ACCOUNT, request, JobPermit::detached()); assert!( fut.now_or_never().is_none(), "the request cannot have finished on its first poll" @@ -2218,211 +2365,13 @@ async fn a_dropped_restore_request_still_reaches_active() { s.cleanup().await; } -/// (c) Restore slots are per account: an account with all of its slots held -/// by running copies makes its own next restore wait, and does not hold back -/// another account's. -#[tokio::test] -async fn one_accounts_restores_do_not_block_another_account() { - if base_conn().is_none() { - eprintln!("SKIP one_accounts_restores_do_not_block_another_account: no PostgreSQL"); - return; - } - // Each blocked copy holds a pooled catalog connection for its snapshot. - let s = scratch_with_pool(20).await; - indexed_source(&s, "sl_src").await; - // One backup per restore: a second restore of one backup while the - // first is CREATING is BackupInUse, not what is measured here. All are - // taken before the gate, which a CreateBackup would wait on too. - let mut backups_a = Vec::new(); - for i in 0..=RESTORE_SLOTS_PER_ACCOUNT + 1 { - backups_a.push( - s.engine - .create_backup(ACCOUNT, "sl_src", &format!("b{i}")) - .await - .expect("backup in A") - .backup_arn, - ); - } - let mut b_input = composite_table("sl_src_b"); - b_input.key_schema.truncate(1); - b_input.attribute_definitions.truncate(1); - s.engine - .create_table(OTHER_ACCOUNT, b_input) - .await - .expect("source in B"); - let backup_b = s - .engine - .create_backup(OTHER_ACCOUNT, "sl_src_b", "b") - .await - .expect("backup in B"); - - let gate = gate_copies(&s).await; - for (i, backup) in backups_a.iter().enumerate().take(RESTORE_SLOTS_PER_ACCOUNT) { - start_restore(&s, &format!("sl_a{i}"), backup) - .await - .expect("A restore within its slots"); - } - await_copies_waiting(&s, 4).await; - - // A's fifth waits for one of A's slots (the limit is real) ... - let mut fifth = Box::pin(start_restore(&s, "sl_a4", &backups_a[4])); - assert!( - tokio::time::timeout(std::time::Duration::from_secs(2), &mut fifth) - .await - .is_err(), - "account A's fifth restore must wait for one of A's slots" - ); - assert_eq!(table_status(&s, "sl_a4").await, None); - - // ... a sixth whose request is dropped while it waits for a slot (a - // client that gave up during the wait) still restores once a slot - // frees, and creates nothing before that ... - let request = restore_request(&s, ACCOUNT, "sl_a5", &backups_a[5]) - .await - .expect("request"); - let sixth = s.engine.restore_table_from_backup(ACCOUNT, request); - assert!( - sixth.now_or_never().is_none(), - "the sixth request cannot have finished on its first poll" - ); - tokio::time::sleep(std::time::Duration::from_millis(200)).await; - assert_eq!(table_status(&s, "sl_a5").await, None); - - // ... and B's does not. - tokio::time::timeout( - std::time::Duration::from_secs(10), - start_restore_in(&s, OTHER_ACCOUNT, "sl_b", &backup_b.backup_arn), - ) - .await - .expect("account A's full slots must not block account B") - .expect("B restore registers"); - await_copies_waiting(&s, 5).await; - - gate.commit().await.expect("release the copies"); - fifth.await.expect("A's fifth gets a slot once one frees"); - let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(30); - while table_status(&s, "sl_a5").await.is_none() { - assert!( - tokio::time::Instant::now() < deadline, - "the dropped sixth request's restore must still create its target" - ); - tokio::time::sleep(std::time::Duration::from_millis(10)).await; - } - for i in 0..=RESTORE_SLOTS_PER_ACCOUNT + 1 { - assert_eq!( - settled_status(&s, &format!("sl_a{i}")).await.as_deref(), - Some("ACTIVE"), - "sl_a{i}" - ); - } - assert_eq!( - settled_status_in(&s, OTHER_ACCOUNT, "sl_b") - .await - .as_deref(), - Some("ACTIVE") - ); - s.cleanup().await; -} - -/// (A1) The process-wide bound: with a pool of 10, at most 5 restores run at -/// once across all accounts, so a third account's restore waits (creating -/// nothing) while A holds 4 and B holds 1, and proceeds once they finish. -#[tokio::test] -async fn restores_across_accounts_are_bounded_per_server() { - if base_conn().is_none() { - eprintln!("SKIP restores_across_accounts_are_bounded_per_server: no PostgreSQL"); - return; - } - const THIRD_ACCOUNT: &str = "333333333333"; - let s = scratch_with_pool(10).await; - sqlx::query("INSERT INTO accounts (account_id, account_name) VALUES ($1, $2)") - .bind(THIRD_ACCOUNT) - .bind(format!("acct-{}-c", s.db_names[0])) - .execute(&s.catalog) - .await - .expect("seed the third account"); - let mut backups = Vec::new(); - for account in [ACCOUNT, OTHER_ACCOUNT, THIRD_ACCOUNT] { - let mut input = composite_table("gb_src"); - input.key_schema.truncate(1); - input.attribute_definitions.truncate(1); - s.engine - .create_table(account, input) - .await - .expect("source table"); - backups.push( - s.engine - .create_backup(account, "gb_src", "b") - .await - .expect("backup") - .backup_arn, - ); - } - // A's four restores need four backups (one restore of a backup at a - // time). - let mut backups_a = vec![backups[0].clone()]; - for i in 1..RESTORE_SLOTS_PER_ACCOUNT { - backups_a.push( - s.engine - .create_backup(ACCOUNT, "gb_src", &format!("b{i}")) - .await - .expect("backup") - .backup_arn, - ); - } - - let gate = gate_copies(&s).await; - for (i, backup) in backups_a.iter().enumerate() { - start_restore_in(&s, ACCOUNT, &format!("gb_a{i}"), backup) - .await - .expect("A restore within its slots"); - } - start_restore_in(&s, OTHER_ACCOUNT, "gb_b", &backups[1]) - .await - .expect("B restore takes the last server slot"); - await_copies_waiting(&s, 5).await; - - let mut third = Box::pin(start_restore_in(&s, THIRD_ACCOUNT, "gb_c", &backups[2])); - assert!( - tokio::time::timeout(std::time::Duration::from_secs(2), &mut third) - .await - .is_err(), - "the third account's restore must wait for a server slot" - ); - assert_eq!(table_status_in(&s, THIRD_ACCOUNT, "gb_c").await, None); - - gate.commit().await.expect("release the copies"); - tokio::time::timeout(std::time::Duration::from_secs(25), third) - .await - .expect("the third restore gets a server slot once one frees") - .expect("the third restore registers"); - for i in 0..RESTORE_SLOTS_PER_ACCOUNT { - assert_eq!( - settled_status(&s, &format!("gb_a{i}")).await.as_deref(), - Some("ACTIVE") - ); - } - assert_eq!( - settled_status_in(&s, OTHER_ACCOUNT, "gb_b") - .await - .as_deref(), - Some("ACTIVE") - ); - assert_eq!( - settled_status_in(&s, THIRD_ACCOUNT, "gb_c") - .await - .as_deref(), - Some("ACTIVE") - ); - s.cleanup().await; -} - /// (d) The restore's lock session is terminated mid-copy. Either the /// heartbeat notices and the copy stops and removes its target, or the -/// sweep claims the unowned target first; in both cases there is one -/// outcome, nothing is left behind, and the name restores again. +/// reconciler finds the record's heartbeat stale first; in both cases there +/// is one outcome, nothing is left behind, and the name restores again. #[tokio::test] async fn losing_the_lock_session_mid_copy_has_one_outcome() { + use extenddb_storage::BackupCatalog; if base_conn().is_none() { eprintln!("SKIP losing_the_lock_session_mid_copy_has_one_outcome: no PostgreSQL"); return; @@ -2431,7 +2380,7 @@ async fn losing_the_lock_session_mid_copy_has_one_outcome() { indexed_source(&s, "hb_src").await; let backup = s .engine - .create_backup(ACCOUNT, "hb_src", "b") + .create(ACCOUNT, "hb_src", "b", JobPermit::detached()) .await .expect("backup"); let tables_before = data_table_count(&s).await; @@ -2473,9 +2422,10 @@ async fn losing_the_lock_session_mid_copy_has_one_outcome() { assert_eq!(data_table_count(&s).await, tables_before); gate.commit().await.expect("release the gate"); - // Sweep path: the session is lost and the sweep claims the old, unowned - // target before the heartbeat would; the copy, released afterwards, must - // not bring it back. + // Reconciler path: the session is lost and the reconciler finds the + // record's heartbeat stale before the copy's own heartbeat would notice; + // it fails the record and removes the target, and the copy, released + // afterwards, must not bring it back. let gate = gate_copies(&s).await; let second = start_restore(&s, "hb_dst", &backup.backup_arn) .await @@ -2486,20 +2436,34 @@ async fn losing_the_lock_session_mid_copy_has_one_outcome() { .expect("lock held"); terminate(pid).await; sqlx::query( - "UPDATE tables SET creation_date_time = NOW() - INTERVAL '10 minutes' WHERE table_id = $1", + "UPDATE table_restores SET heartbeat_at = NOW() - INTERVAL '10 minutes' WHERE table_id = $1", ) .bind(&second.table_id) .execute(&s.catalog) .await - .expect("age the target past the grace period"); - let transitions = s + .expect("age the heartbeat past the stale window"); + let dead = s .engine - .process_control_plane_transitions() + .list_dead_restore_jobs(std::time::Duration::from_secs(300)) .await - .expect("sweep"); + .expect("list"); + assert_eq!(dead.len(), 1, "{dead:?}"); + assert!( + s.engine + .fail_restore_job( + &dead[0].target, + dead[0].owner_generation, + dead[0].heartbeat_at, + "owner dead" + ) + .await + .expect("fail") + ); assert!( - transitions.iter().any(|(n, _)| n == "hb_dst"), - "{transitions:?}" + s.engine + .remove_restore_target(&dead[0].target) + .await + .expect("remove") ); gate.commit().await.expect("release the copy"); // Past the next heartbeat, so the stopped copy has run its cleanup. @@ -2521,63 +2485,204 @@ async fn losing_the_lock_session_mid_copy_has_one_outcome() { /// (e) After a multi-batch restore, every base and index row is exactly the /// row the write path (PutItem) wrote for the source: same key columns, same /// projection, same membership. +/// (e) The restore heartbeat is conditioned on `status = 'RUNNING'`: when the +/// reconciler has failed a record whose copy is still alive (its heartbeat +/// found stale through a slow catalog, and its removal of the target then +/// errored, so the target is CREATING under a FAILED record), the copy's next +/// heartbeat refreshes nothing and the copy stops there, performing no +/// further store or table operation (design 4(e): "the owner itself aborts at +/// its next heartbeat"). Without the condition the heartbeat keeps confirming +/// on the FAILED record and the copy runs on. The copy is held at the gate +/// through the heartbeat, the record is written FAILED by hand (the state the +/// reconciler's write leaves), and the claim session's end is awaited before +/// the gate opens: the target is still CREATING, the record FAILED with its +/// heartbeat not refreshed, the data table empty, and the pair is what the +/// reconciler's FAILED-target duty then lists and removes. #[tokio::test] -async fn restored_index_rows_match_the_write_path() { +async fn a_copy_whose_record_the_reconciler_failed_stops_at_its_next_heartbeat() { + use extenddb_storage::BackupCatalog; if base_conn().is_none() { - eprintln!("SKIP restored_index_rows_match_the_write_path: no PostgreSQL"); + eprintln!( + "SKIP a_copy_whose_record_the_reconciler_failed_stops_at_its_next_heartbeat: no \ + PostgreSQL" + ); return; } let s = scratch().await; - let src = indexed_source_with(&s, "wp_src", 1_100).await; + indexed_source(&s, "hf_src").await; let backup = s .engine - .create_backup(ACCOUNT, "wp_src", "b") + .create(ACCOUNT, "hf_src", "b", JobPermit::detached()) .await .expect("backup"); - let desc = restore(&s, "wp_dst", &backup.backup_arn) + let gate = gate_copies(&s).await; + let desc = start_restore(&s, "hf_dst", &backup.backup_arn) .await - .expect("restore"); - assert_eq!(table_status(&s, "wp_dst").await.as_deref(), Some("ACTIVE")); - let rows = |id: String| { - let db = s.db.clone(); + .expect("registers"); + await_copies_waiting(&s, 1).await; + assert!( + restore_lock_holder(&s, ACCOUNT, "hf_dst").await.is_some(), + "the copy holds its claim session while it waits at the gate" + ); + let record = |s: &Scratch| { + let catalog = s.catalog.clone(); + let table_id = desc.table_id.clone(); async move { - sqlx::query_scalar::<_, String>(&format!( - "SELECT to_jsonb(t)::text FROM \"_ddb_{id}\" t ORDER BY 1" - )) - .fetch_all(&db) + let row: (String, f64, Option) = sqlx::query_as( + "SELECT status, EXTRACT(EPOCH FROM heartbeat_at)::FLOAT8, \ + EXTRACT(EPOCH FROM completed_at)::FLOAT8 FROM table_restores WHERE table_id = $1", + ) + .bind(&table_id) + .fetch_one(&catalog) .await - .expect("rows") + .expect("the job record"); + row } }; - let (src_gi, src_li) = gi_li(&s, &src).await; - let (dst_gi, dst_li) = gi_li(&s, &desc.table_id).await; - let base = rows(desc.table_id.clone()).await; - assert_eq!(base.len(), 1_100); - assert_eq!(base, rows(src.clone()).await); - for (a, b) in [(src_gi, dst_gi), (src_li, dst_li)] { - let want = rows(a).await; - assert!(!want.is_empty() && want.len() < 1_100, "partial membership"); - assert_eq!(rows(b).await, want); - } - assert!(xmins(&s, &desc.table_id).await.len() >= 3); - s.cleanup().await; -} + let (status, heartbeat_before, _) = record(&s).await; + assert_eq!(status, "RUNNING"); -/// The definition read reports a lost connection as `Transient`, not -/// `Internal`: DescribeBackup answers without feature details only for a -/// definition that does not decode, so a failover must not reach it as -/// `Internal`. The engine's catalog sessions are terminated, as a failover or -/// an administrator restart would; the next read runs on one of them (the -/// pool does not test a connection before handing it out). -#[tokio::test] -async fn a_definition_read_that_loses_its_connection_is_transient() { - if base_conn().is_none() { - eprintln!("SKIP a_definition_read_that_loses_its_connection_is_transient: no PostgreSQL"); - return; - } + // The reconciler's FAILED write, with its removal not landed. + let written = sqlx::query( + "UPDATE table_restores SET status = 'FAILED', completed_at = now(), \ + failure_reason = 'owner dead' WHERE table_id = $1 AND status = 'RUNNING'", + ) + .bind(&desc.table_id) + .execute(&s.catalog) + .await + .expect("fail the record") + .rows_affected(); + assert_eq!(written, 1); + + // The copy's next heartbeat matches no row; the copy stops and its claim + // session ends, with the gate still held. Observed through the advisory + // lock the session holds rather than a sleep; the deadline is well past + // one heartbeat period. + let started = std::time::Instant::now(); + let deadline = tokio::time::Instant::now() + + extenddb_storage::backup_catalog::JOB_HEARTBEAT_INTERVAL + + std::time::Duration::from_secs(25); + while restore_lock_holder(&s, ACCOUNT, "hf_dst").await.is_some() { + assert!( + tokio::time::Instant::now() < deadline, + "the copy did not stop at its next heartbeat after its record was failed" + ); + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + } + assert!( + started.elapsed() >= std::time::Duration::from_secs(5), + "the copy stopped at a heartbeat, not at the write: {:?}", + started.elapsed() + ); + let (status, heartbeat_after, completed_at) = record(&s).await; + assert_eq!( + status, "FAILED", + "the record is left as the reconciler wrote it" + ); + assert_eq!( + heartbeat_after, heartbeat_before, + "a heartbeat on a FAILED record refreshes nothing" + ); + assert!(completed_at.is_some()); + assert_eq!( + table_status(&s, "hf_dst").await.as_deref(), + Some("CREATING"), + "the target is left to the reconciler, not removed by the copy" + ); + gate.commit().await.expect("release the gate"); + tokio::time::sleep(std::time::Duration::from_secs(2)).await; + assert_eq!( + data_rows(&s, &desc.table_id).await, + 0, + "the stopped copy wrote nothing once the gate opened" + ); + assert_eq!( + table_status(&s, "hf_dst").await.as_deref(), + Some("CREATING") + ); + assert!(restore_lock_holder(&s, ACCOUNT, "hf_dst").await.is_none()); + + // The pair is the FAILED-target duty's: listed once the record is stale, + // and the removal lands. + tokio::time::sleep(std::time::Duration::from_millis(1_100)).await; + let dead = s + .engine + .list_dead_restore_jobs(std::time::Duration::from_secs(1)) + .await + .expect("list"); + assert_eq!(dead.len(), 1, "{dead:?}"); + assert_eq!( + dead[0].status, + extenddb_storage::backup_catalog::JobStatus::Failed + ); + assert!( + s.engine + .remove_restore_target(&dead[0].target) + .await + .expect("remove") + ); + assert_eq!(table_status(&s, "hf_dst").await, None); + assert!(!data_table_exists(&s, &desc.table_id).await); + s.cleanup().await; +} + +#[tokio::test] +async fn restored_index_rows_match_the_write_path() { + if base_conn().is_none() { + eprintln!("SKIP restored_index_rows_match_the_write_path: no PostgreSQL"); + return; + } + let s = scratch().await; + let src = indexed_source_with(&s, "wp_src", 1_100).await; + let backup = s + .engine + .create(ACCOUNT, "wp_src", "b", JobPermit::detached()) + .await + .expect("backup"); + let desc = restore(&s, "wp_dst", &backup.backup_arn) + .await + .expect("restore"); + assert_eq!(table_status(&s, "wp_dst").await.as_deref(), Some("ACTIVE")); + let rows = |id: String| { + let db = s.db.clone(); + async move { + sqlx::query_scalar::<_, String>(&format!( + "SELECT to_jsonb(t)::text FROM \"_ddb_{id}\" t ORDER BY 1" + )) + .fetch_all(&db) + .await + .expect("rows") + } + }; + let (src_gi, src_li) = gi_li(&s, &src).await; + let (dst_gi, dst_li) = gi_li(&s, &desc.table_id).await; + let base = rows(desc.table_id.clone()).await; + assert_eq!(base.len(), 1_100); + assert_eq!(base, rows(src.clone()).await); + for (a, b) in [(src_gi, dst_gi), (src_li, dst_li)] { + let want = rows(a).await; + assert!(!want.is_empty() && want.len() < 1_100, "partial membership"); + assert_eq!(rows(b).await, want); + } + assert!(xmins(&s, &desc.table_id).await.len() >= 3); + s.cleanup().await; +} + +/// The definition read reports a lost connection as `Transient`, not +/// `Internal`: DescribeBackup answers without feature details only for a +/// definition that does not decode, so a failover must not reach it as +/// `Internal`. The engine's catalog sessions are terminated, as a failover or +/// an administrator restart would; the next read runs on one of them (the +/// pool does not test a connection before handing it out). +#[tokio::test] +async fn a_definition_read_that_loses_its_connection_is_transient() { + if base_conn().is_none() { + eprintln!("SKIP a_definition_read_that_loses_its_connection_is_transient: no PostgreSQL"); + return; + } let s = scratch().await; let missing = format!("arn:aws:dynamodb:{REGION}:{ACCOUNT}:table/t/backup/01"); - match s.engine.backup_restore_source(ACCOUNT, &missing).await { + match s.engine.restore_source(ACCOUNT, &missing).await { Err(StorageError::Validation(m)) => assert!(m.contains("Backup not found"), "{m}"), other => panic!("a missing backup before the failover: {:?}", other.err()), } @@ -2609,16 +2714,14 @@ async fn a_definition_read_that_loses_its_connection_is_transient() { tokio::time::sleep(std::time::Duration::from_millis(50)).await; } - match s.engine.backup_restore_source(ACCOUNT, &missing).await { + match s.engine.restore_source(ACCOUNT, &missing).await { Err(StorageError::Transient(m)) => assert!(m.contains("Database error"), "{m}"), other => panic!("a read on a terminated session must be Transient: {other:?}"), } // And the engine recovers: a later read reconnects. let mut recovered = false; for _ in 0..10 { - if let Err(StorageError::Validation(_)) = - s.engine.backup_restore_source(ACCOUNT, &missing).await - { + if let Err(StorageError::Validation(_)) = s.engine.restore_source(ACCOUNT, &missing).await { recovered = true; break; } @@ -2626,3 +2729,3198 @@ async fn a_definition_read_that_loses_its_connection_is_transient() { assert!(recovered, "the pool reconnects after the failover"); s.cleanup().await; } + +// --- the catalog index ----------------------------------------------------------- + +/// A catalog row for a backup of `table` in `account` minted now, with the +/// manifest fixture's source table, in the `catalog_index` module's store. +fn catalog_index_row( + account: &str, + table: &str, + name: &str, +) -> extenddb_storage::backup_catalog::BackupRow { + catalog_index::row(account, table, name) +} + +mod catalog_index { + use extenddb_core::backup_format::BackupArn; + use extenddb_storage::BackupCatalog; + use extenddb_storage::backup_catalog::{ + BackupRow, BackupStatus, DeleteStart, Heartbeat, JobStatus, RestoreTargetKey, StaleClock, + StatusFields, StoreIdentity, StoreKind, TimeBounds, UNOWNED_GENERATION, WriteOutcome, + }; + use futures::TryStreamExt; + + use super::*; + + pub(super) fn store() -> StoreIdentity { + StoreIdentity { + store_kind: StoreKind::Filesystem, + store_root: "/var/lib/extenddb/backups".to_owned(), + store_id: "store-1".to_owned(), + } + } + + /// A row for a backup of `table` in `account` minted now, with the + /// manifest fixture's source table. + pub(super) fn row(account: &str, table: &str, name: &str) -> BackupRow { + let backup_id = extenddb_storage::util::new_backup_id(); + let backup_arn = extenddb_storage::util::backup_arn(REGION, account, table, &backup_id); + let manifest: serde_json::Value = serde_json::from_str(include_str!( + "../../core/testdata/extenddb-manifest/v1.json" + )) + .expect("fixture"); + BackupRow { + backup_id: backup_id.clone(), + account_id: account.to_owned(), + table_name: table.to_owned(), + backup_arn: backup_arn.clone(), + backup_name: name.to_owned(), + status: BackupStatus::Creating, + store: store(), + size_bytes: 0, + item_count: 0, + created_at: extenddb_storage::util::backup_creation_date_time(&backup_id).unwrap(), + completed_at: None, + failure_reason: None, + source_table_details_json: manifest["source_table"].clone(), + owner_generation: 0, + heartbeat_at: 0.0, + } + } + + fn arn_of(row: &BackupRow) -> BackupArn { + BackupArn::parse(&row.backup_arn).unwrap() + } + + /// The claim: a second claim of the same backup is refused while the + /// first is held, the heartbeat confirms the claim and refreshes the + /// row, a status write is conditioned on the generation and the `from` + /// status, and the row is read back by account, backup id, table and ARN. + #[tokio::test] + async fn claim_backup_excludes_a_second_owner_and_fences_its_writes() { + if base_conn().is_none() { + eprintln!( + "SKIP claim_backup_excludes_a_second_owner_and_fences_its_writes: no PostgreSQL" + ); + return; + } + let s = scratch().await; + assert!( + uuid::Uuid::parse_str(&s.engine.catalog_id().await.expect("id")).is_ok(), + "the catalog has an id from the migration" + ); + assert!( + !s.engine + .check_maintenance_claim(std::time::Duration::from_secs(300)) + .await + .expect("check") + ); + let row = row(ACCOUNT, "Music", "nightly"); + assert_eq!(backup_lock_holder(&s, &row.backup_id).await, None); + let claim = s.engine.claim_backup(&row).await.expect("claim"); + assert_ne!(claim.generation(), UNOWNED_GENERATION); + // The claim is the advisory lock on the claim session, held for the + // claim's life: what excludes a second owner before any row exists. + let holder = backup_lock_holder(&s, &row.backup_id) + .await + .expect("the claim session holds the backup's advisory lock"); + let second = s.engine.claim_backup(&row).await.err(); + assert!( + matches!(second, Some(StorageError::AlreadyExists(_))), + "{second:?}" + ); + assert_eq!( + backup_lock_holder(&s, &row.backup_id).await, + Some(holder), + "the refused claim took nothing" + ); + let stored = s + .engine + .read_backup_row(&arn_of(&row)) + .await + .expect("read") + .expect("the row exists"); + assert_eq!(stored.status, BackupStatus::Creating); + assert_eq!(stored.owner_generation, claim.generation()); + assert_eq!(stored.store, store()); + assert_eq!(stored.created_at, row.created_at); + assert_eq!( + stored.source_table_details_json, + row.source_table_details_json + ); + let before = stored.heartbeat_at; + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + assert_eq!( + claim.heartbeat().await.expect("heartbeat"), + Heartbeat::Confirmed + ); + let after = s + .engine + .read_backup_row(&arn_of(&row)) + .await + .unwrap() + .unwrap() + .heartbeat_at; + assert!(after > before, "{after} > {before}"); + + // The same id under another account, table or region finds nothing. + let other_account = + BackupArn::new("aws", REGION, OTHER_ACCOUNT, "Music", &row.backup_id).unwrap(); + assert!( + s.engine + .read_backup_row(&other_account) + .await + .unwrap() + .is_none() + ); + let other_table = BackupArn::new("aws", REGION, ACCOUNT, "Other", &row.backup_id).unwrap(); + assert!( + s.engine + .read_backup_row(&other_table) + .await + .unwrap() + .is_none() + ); + let other_region = + BackupArn::new("aws", "eu-west-1", ACCOUNT, "Music", &row.backup_id).unwrap(); + assert!( + s.engine + .read_backup_row(&other_region) + .await + .unwrap() + .is_none() + ); + + // A write from the wrong state matches nothing; the right one matches + // once, and the row then carries what it set. + let fields = StatusFields { + size_bytes: Some(1234), + item_count: Some(5), + failure_reason: None, + source_table_details_json: None, + completed: true, + }; + assert_eq!( + claim + .set_status(JobStatus::Running, JobStatus::Available, &fields) + .await + .unwrap(), + WriteOutcome::NotMatched + ); + assert_eq!( + claim + .set_status(JobStatus::Creating, JobStatus::Available, &fields) + .await + .unwrap(), + WriteOutcome::Matched + ); + assert_eq!( + claim + .set_status(JobStatus::Creating, JobStatus::Available, &fields) + .await + .unwrap(), + WriteOutcome::NotMatched + ); + let read = claim.read().await.expect("read"); + assert_eq!(read.status, JobStatus::Available); + assert_eq!(read.owner_generation, claim.generation()); + let stored = s + .engine + .read_backup_row(&arn_of(&row)) + .await + .unwrap() + .unwrap(); + assert_eq!(stored.status, BackupStatus::Available); + assert_eq!(stored.size_bytes, 1234); + assert_eq!(stored.item_count, 5); + assert!(stored.completed_at.is_some()); + + // A row another generation took over confirms nothing for this claim. + sqlx::query( + "UPDATE backup_index SET owner_generation = owner_generation + 1 WHERE backup_id = $1", + ) + .bind(&row.backup_id) + .execute(&s.catalog) + .await + .expect("take over"); + assert_eq!(claim.heartbeat().await.expect("heartbeat"), Heartbeat::Lost); + assert_eq!( + claim + .set_status( + JobStatus::Available, + JobStatus::Failed, + &StatusFields::default() + ) + .await + .unwrap(), + WriteOutcome::NotMatched + ); + drop(claim); + // The lock went with the claim's session (the server releases it when + // the connection closes, a moment after the drop): the row can be + // claimed by a successor only through its insert, which the existing + // row refuses. + let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(10); + while backup_lock_holder(&s, &row.backup_id).await.is_some() { + assert!( + tokio::time::Instant::now() < deadline, + "the advisory lock outlived the dropped claim" + ); + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + } + let third = s.engine.claim_backup(&row).await.err(); + assert!( + matches!(third, Some(StorageError::AlreadyExists(_))), + "{third:?}" + ); + s.cleanup().await; + } + + /// `start_backup_delete` runs under the `backup_index` row's lock: a + /// transaction holding the row `FOR UPDATE` while it registers a CREATING + /// target makes the delete wait, and when it commits the delete sees the + /// registration and answers `InUse`, with the row left AVAILABLE. Without + /// the `FOR UPDATE` the delete reads the row and the in-use rule past the + /// uncommitted registration and then waits on the row in its DELETING + /// write instead, so a wait alone cannot tell the two apart: the + /// `pg_stat_activity` probe below is scoped to the `SELECT ... FOR + /// UPDATE` text, and a delete that waits in its `UPDATE` never satisfies + /// it and fails the probe's deadline. That probe is what pins the lock; + /// the `InUse` answer after the commit is the second check. + #[tokio::test] + async fn start_backup_delete_waits_for_the_row_lock_and_then_sees_the_registration() { + if base_conn().is_none() { + eprintln!( + "SKIP start_backup_delete_waits_for_the_row_lock_and_then_sees_the_registration: no PostgreSQL" + ); + return; + } + waits_for_the_row_lock_and_then_sees_the_registration(false).await; + } + + /// `forget_available_backup_row` runs under the same lock: the outright + /// removal waits in the `SELECT ... FOR UPDATE` while a registration + /// holds the row, then answers `InUse` and leaves the row AVAILABLE. + /// Without the lock it would wait in its `DELETE` instead, and then + /// remove the row the committed registration reads. + #[tokio::test] + async fn forget_available_backup_row_waits_for_the_row_lock_and_then_sees_the_registration() { + if base_conn().is_none() { + eprintln!( + "SKIP forget_available_backup_row_waits_for_the_row_lock_and_then_sees_the_registration: no PostgreSQL" + ); + return; + } + waits_for_the_row_lock_and_then_sees_the_registration(true).await; + } + + /// The two cells above: `start_backup_delete`, or with `remove` + /// `forget_available_backup_row`, against a registration holding the + /// row's lock. + async fn waits_for_the_row_lock_and_then_sees_the_registration(remove: bool) { + let s = scratch().await; + let row = row(ACCOUNT, "Music", "nightly"); + let key = row.key(); + let claim = s.engine.claim_backup(&row).await.expect("claim"); + assert_eq!( + claim + .set_status( + JobStatus::Creating, + JobStatus::Available, + &StatusFields::default() + ) + .await + .unwrap(), + WriteOutcome::Matched + ); + indexed_source(&s, "lk_target").await; + let target_id = table_id_of(&s, "lk_target").await.unwrap(); + + // The registration's side: lock the row, then register the target, + // in one transaction that stays open. + let mut registration = s.catalog.begin().await.expect("begin"); + let status: String = sqlx::query_scalar( + "SELECT status FROM backup_index WHERE account_id = $1 AND backup_id = $2 FOR UPDATE", + ) + .bind(&key.account_id) + .bind(&key.backup_id) + .fetch_one(&mut *registration) + .await + .expect("lock the row"); + assert_eq!(status, "AVAILABLE"); + sqlx::query( + "INSERT INTO table_restores (table_id, source_backup_arn, source_table_arn, \ + restore_date_time, status, kind, owner_generation, heartbeat_at) \ + VALUES ($1, $2, 'arn:aws:dynamodb:us-east-1:123456789012:table/Music', now(), \ + 'RUNNING', 'manifest', 7, now())", + ) + .bind(&target_id) + .bind(&row.backup_arn) + .execute(&mut *registration) + .await + .expect("register a target"); + sqlx::query( + "UPDATE tables SET table_status = 'CREATING', status_transition_at = NULL \ + WHERE table_id = $1", + ) + .bind(&target_id) + .execute(&mut *registration) + .await + .expect("make it CREATING"); + + let engine = s.engine.clone(); + let deleting = tokio::spawn(async move { + let key = key; + if remove { + engine + .forget_available_backup_row(&key) + .await + .map(|answer| match answer { + extenddb_storage::backup_catalog::ForgetRow::Removed => { + DeleteStart::Deleting + } + extenddb_storage::backup_catalog::ForgetRow::InUse(table) => { + DeleteStart::InUse(table) + } + extenddb_storage::backup_catalog::ForgetRow::Status(status) => { + DeleteStart::Status(status) + } + }) + } else { + engine.start_backup_delete(&key).await + } + }); + // The delete is waiting on the registration's transaction, in its + // `SELECT ... FOR UPDATE` (scoped to this database and that statement: + // other tests of this binary produce row-lock waits of their own, and + // without the lock the wait would be the DELETING write's instead). + let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(30); + loop { + let waiting: i64 = sqlx::query_scalar( + "SELECT count(*) FROM pg_stat_activity WHERE datname = current_database() \ + AND wait_event_type = 'Lock' \ + AND query LIKE 'SELECT status FROM backup_index%FOR UPDATE'", + ) + .fetch_one(&s.catalog) + .await + .expect("count waiters"); + if waiting >= 1 { + break; + } + assert!( + tokio::time::Instant::now() < deadline, + "the delete step (remove: {remove}) did not wait for the row lock" + ); + assert!( + !deleting.is_finished(), + "the delete answered under the lock" + ); + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + } + assert!( + !deleting.is_finished(), + "the delete answered under the lock" + ); + registration + .commit() + .await + .expect("commit the registration"); + let answer = tokio::time::timeout(std::time::Duration::from_secs(30), deleting) + .await + .expect("the delete finishes once the lock is released") + .expect("task") + .expect("the delete step"); + assert_eq!(answer, DeleteStart::InUse("lk_target".to_owned())); + assert_eq!( + s.engine + .read_backup_row(&arn_of(&row)) + .await + .unwrap() + .unwrap() + .status, + BackupStatus::Available, + "the refused delete leaves the row AVAILABLE" + ); + drop(claim); + s.cleanup().await; + } + + /// The FAILED-target duty's listing: a FAILED record whose table is + /// still a CREATING restore target (the FAILED write landed and the + /// removal did not) is not listed while the record is fresh (the job + /// may still be on its own failure path), is listed once `completed_at` + /// is older than the stale window, with its status, and its target is + /// then removed with no second FAILED write; a FAILED record without a + /// `completed_at` is aged by its heartbeat; a FAILED record whose table + /// is ACTIVE is never listed. While the target is CREATING under the + /// FAILED record the backup is in use (the rule reads the target's + /// status, not the job's) and DeleteBackup is refused; the removal frees + /// it. + #[tokio::test] + async fn a_failed_record_whose_target_removal_failed_is_listed_once_stale_and_removed() { + use extenddb_storage::backup_catalog::RestoreKind; + if base_conn().is_none() { + eprintln!( + "SKIP a_failed_record_whose_target_removal_failed_is_listed_once_stale_and_removed: no PostgreSQL" + ); + return; + } + let stale = std::time::Duration::from_secs(300); + let s = scratch().await; + indexed_source(&s, "ft_src").await; + let backup = s + .engine + .create(ACCOUNT, "ft_src", "b", JobPermit::detached()) + .await + .expect("backup"); + let desc = restore(&s, "ft_dst", &backup.backup_arn) + .await + .expect("restore"); + let key = RestoreTargetKey { + account_id: ACCOUNT.to_owned(), + table_name: "ft_dst".to_owned(), + table_id: desc.table_id.clone(), + }; + // The state the probe left: a dead restore, failed by the reconciler + // (or by the job's own failure path), whose target removal failed. + sqlx::query( + "UPDATE tables SET table_status = 'CREATING', status_transition_at = NULL \ + WHERE table_id = $1", + ) + .bind(&desc.table_id) + .execute(&s.catalog) + .await + .expect("interrupt"); + sqlx::query( + "UPDATE table_restores SET status = 'RUNNING', completed_at = NULL, \ + heartbeat_at = NOW() - INTERVAL '10 minutes' WHERE table_id = $1", + ) + .bind(&desc.table_id) + .execute(&s.catalog) + .await + .expect("age the heartbeat"); + let dead = s.engine.list_dead_restore_jobs(stale).await.expect("list"); + assert_eq!(dead.len(), 1); + assert_eq!(dead[0].status, JobStatus::Running); + assert!( + s.engine + .fail_restore_job( + &key, + dead[0].owner_generation, + dead[0].heartbeat_at, + "owner dead" + ) + .await + .expect("fail") + ); + assert_eq!( + job_status(&s, &desc.table_id).await.as_deref(), + Some("FAILED") + ); + assert_eq!( + table_status(&s, "ft_dst").await.as_deref(), + Some("CREATING") + ); + + // Fresh: the record was just failed; whoever failed it may still be + // removing the target. + assert!( + s.engine + .list_dead_restore_jobs(stale) + .await + .expect("list") + .is_empty(), + "a FAILED record within the stale window is left to its writer" + ); + // Stale: listed, as FAILED, for the FAILED-target duty. + sqlx::query( + "UPDATE table_restores SET completed_at = NOW() - INTERVAL '10 minutes' \ + WHERE table_id = $1", + ) + .bind(&desc.table_id) + .execute(&s.catalog) + .await + .expect("age the FAILED record"); + let dead = s.engine.list_dead_restore_jobs(stale).await.expect("list"); + assert_eq!(dead.len(), 1, "{dead:?}"); + assert_eq!(dead[0].target, key); + assert_eq!(dead[0].status, JobStatus::Failed); + assert_eq!(dead[0].kind, RestoreKind::Legacy); + assert_eq!(dead[0].source_backup_arn, backup.backup_arn); + // A FAILED record without `completed_at` (a failure path that did + // not stamp one) is aged by its heartbeat: fresh, not listed; stale, + // listed. + sqlx::query( + "UPDATE table_restores SET completed_at = NULL, heartbeat_at = NOW() \ + WHERE table_id = $1", + ) + .bind(&desc.table_id) + .execute(&s.catalog) + .await + .expect("clear completed_at, fresh heartbeat"); + assert!( + s.engine + .list_dead_restore_jobs(stale) + .await + .expect("list") + .is_empty() + ); + sqlx::query( + "UPDATE table_restores SET heartbeat_at = NOW() - INTERVAL '10 minutes' \ + WHERE table_id = $1", + ) + .bind(&desc.table_id) + .execute(&s.catalog) + .await + .expect("age the heartbeat"); + assert_eq!( + s.engine + .list_dead_restore_jobs(stale) + .await + .expect("list") + .len(), + 1 + ); + // A FAILED record whose table the removal already moved to DELETING + // (the claim landed, the drop did not) is the control plane's row. + sqlx::query("UPDATE tables SET table_status = 'DELETING' WHERE table_id = $1") + .bind(&desc.table_id) + .execute(&s.catalog) + .await + .expect("DELETING"); + assert!( + s.engine + .list_dead_restore_jobs(stale) + .await + .expect("list") + .is_empty() + ); + sqlx::query("UPDATE tables SET table_status = 'CREATING' WHERE table_id = $1") + .bind(&desc.table_id) + .execute(&s.catalog) + .await + .expect("CREATING again"); + // The in-use rule reads the target's status, not the job's: with the + // record FAILED and the target still CREATING, the backup is in use + // (naming the target) and the legacy delete is refused. + let backup_key = extenddb_storage::backup_catalog::BackupKey::from_arn( + &extenddb_core::backup_format::BackupArn::parse(&backup.backup_arn).expect("arn"), + ); + assert_eq!( + s.engine + .backup_in_use(&backup_key) + .await + .expect("in use") + .as_deref(), + Some("ft_dst"), + "a CREATING target under a FAILED record keeps its backup in use" + ); + let refused = s.engine.delete(ACCOUNT, &backup.backup_arn).await; + assert!( + matches!(refused, Err(StorageError::BackupInUse(_))), + "DeleteBackup while the target is CREATING: {refused:?}" + ); + // The reconciler's FAILED-target duty: no second FAILED write, the + // target removed. + assert!( + !s.engine + .fail_restore_job( + &key, + dead[0].owner_generation, + dead[0].heartbeat_at, + "owner dead" + ) + .await + .expect("fail again"), + "a FAILED record is not RUNNING" + ); + assert!(s.engine.remove_restore_target(&key).await.expect("remove")); + assert_eq!(table_status(&s, "ft_dst").await, None); + assert!(!data_table_exists(&s, &desc.table_id).await); + // The removal frees the backup: no longer in use, and deletable. + assert_eq!( + s.engine.backup_in_use(&backup_key).await.expect("in use"), + None + ); + assert!( + s.engine + .list_dead_restore_jobs(stale) + .await + .expect("list") + .is_empty() + ); + + // A FAILED record whose table is ACTIVE (a flip that won against a + // late FAILED write) is nobody's duty. + let again = restore(&s, "ft_dst", &backup.backup_arn) + .await + .expect("restore again"); + assert_eq!(table_status(&s, "ft_dst").await.as_deref(), Some("ACTIVE")); + sqlx::query( + "UPDATE table_restores SET status = 'FAILED', \ + completed_at = NOW() - INTERVAL '10 minutes' WHERE table_id = $1", + ) + .bind(&again.table_id) + .execute(&s.catalog) + .await + .expect("fail the record of an ACTIVE table"); + assert!( + s.engine + .list_dead_restore_jobs(stale) + .await + .expect("list") + .is_empty(), + "an ACTIVE table is never a target to remove" + ); + s.engine + .delete(ACCOUNT, &backup.backup_arn) + .await + .expect("DeleteBackup once no CREATING target reads the backup"); + s.cleanup().await; + } + + /// The two conditions that make a target's removal exclusive with its + /// ACTIVE flip and with every other table: `abort_restore`'s claim is + /// conditional on CREATING, so against an ACTIVE table with the right id + /// it does nothing (as does `remove_restore_target`); and + /// `remove_restore_target` matches the account and name as well as the + /// id, so the right id under another account or name removes nothing. + #[tokio::test] + async fn a_target_is_removed_only_while_creating_and_only_under_its_full_key() { + if base_conn().is_none() { + eprintln!( + "SKIP a_target_is_removed_only_while_creating_and_only_under_its_full_key: no PostgreSQL" + ); + return; + } + let s = scratch().await; + indexed_source(&s, "ex_src").await; + let backup = s + .engine + .create(ACCOUNT, "ex_src", "b", JobPermit::detached()) + .await + .expect("backup"); + let desc = restore(&s, "ex_dst", &backup.backup_arn) + .await + .expect("restore"); + assert_eq!(table_status(&s, "ex_dst").await.as_deref(), Some("ACTIVE")); + let key = RestoreTargetKey { + account_id: ACCOUNT.to_owned(), + table_name: "ex_dst".to_owned(), + table_id: desc.table_id.clone(), + }; + + // ACTIVE with the right id: the ordering where the copy's flip won. + assert!( + !s.engine + .abort_restore(&desc.table_id) + .await + .expect("abort against ACTIVE"), + "abort_restore claims only a CREATING table" + ); + assert_eq!(table_status(&s, "ex_dst").await.as_deref(), Some("ACTIVE")); + assert!(data_table_exists(&s, &desc.table_id).await); + assert!( + !s.engine + .remove_restore_target(&key) + .await + .expect("remove against ACTIVE") + ); + assert_eq!(table_status(&s, "ex_dst").await.as_deref(), Some("ACTIVE")); + + // CREATING, the right id, the wrong account or name: nothing. + sqlx::query( + "UPDATE tables SET table_status = 'CREATING', status_transition_at = NULL \ + WHERE table_id = $1", + ) + .bind(&desc.table_id) + .execute(&s.catalog) + .await + .expect("make it a target"); + let other_account = RestoreTargetKey { + account_id: OTHER_ACCOUNT.to_owned(), + ..key.clone() + }; + let other_name = RestoreTargetKey { + table_name: "ex_other".to_owned(), + ..key.clone() + }; + for (wrong, differs_in) in [(&other_account, "account"), (&other_name, "name")] { + assert!( + !s.engine + .remove_restore_target(wrong) + .await + .expect("remove under the wrong key"), + "a key differing in its {differs_in} removes nothing" + ); + assert_eq!( + table_status(&s, "ex_dst").await.as_deref(), + Some("CREATING") + ); + assert!(data_table_exists(&s, &desc.table_id).await); + } + // The full key removes it. + assert!(s.engine.remove_restore_target(&key).await.expect("remove")); + assert_eq!(table_status(&s, "ex_dst").await, None); + assert!(!data_table_exists(&s, &desc.table_id).await); + s.cleanup().await; + } + + /// DeleteBackup's first step and the tombstone writes: only an AVAILABLE + /// row moves to DELETING; a CREATING row answers its status; a backup a + /// CREATING restore target reads answers the target's name; DELETING + /// moves to DELETED once; a DELETED row is removed once; a missing row is + /// "Backup not found". + #[tokio::test] + async fn start_backup_delete_checks_the_status_and_the_in_use_rule_under_the_lock() { + if base_conn().is_none() { + eprintln!( + "SKIP start_backup_delete_checks_the_status_and_the_in_use_rule_under_the_lock: no PostgreSQL" + ); + return; + } + let s = scratch().await; + let row = row(ACCOUNT, "Music", "nightly"); + let key = row.key(); + assert!(matches!( + s.engine.start_backup_delete(&key).await, + Err(StorageError::Validation(m)) if m.contains("Backup not found") + )); + let claim = s.engine.claim_backup(&row).await.expect("claim"); + assert_eq!( + s.engine.start_backup_delete(&key).await.unwrap(), + DeleteStart::Status(BackupStatus::Creating) + ); + assert_eq!( + claim + .set_status( + JobStatus::Creating, + JobStatus::Available, + &StatusFields::default() + ) + .await + .unwrap(), + WriteOutcome::Matched + ); + assert_eq!(s.engine.backup_in_use(&key).await.unwrap(), None); + + // A CREATING restore target registered from the backup: in use, + // naming the target; an ACTIVE one is not. + indexed_source(&s, "del_target").await; + let target_id = table_id_of(&s, "del_target").await.unwrap(); + sqlx::query( + "INSERT INTO table_restores (table_id, source_backup_arn, source_table_arn, \ + restore_date_time, status, kind, owner_generation, heartbeat_at) \ + VALUES ($1, $2, 'arn:aws:dynamodb:us-east-1:123456789012:table/Music', now(), \ + 'RUNNING', 'manifest', 7, now())", + ) + .bind(&target_id) + .bind(&row.backup_arn) + .execute(&s.catalog) + .await + .expect("register a target"); + sqlx::query("UPDATE tables SET table_status = 'CREATING', status_transition_at = NULL WHERE table_id = $1") + .bind(&target_id) + .execute(&s.catalog) + .await + .expect("make it CREATING"); + assert_eq!( + s.engine.backup_in_use(&key).await.unwrap().as_deref(), + Some("del_target") + ); + assert_eq!( + s.engine.start_backup_delete(&key).await.unwrap(), + DeleteStart::InUse("del_target".to_owned()) + ); + assert_eq!( + s.engine.forget_available_backup_row(&key).await.unwrap(), + extenddb_storage::backup_catalog::ForgetRow::InUse("del_target".to_owned()), + "the outright removal keeps the in-use rule" + ); + assert_eq!( + s.engine + .read_backup_row(&arn_of(&row)) + .await + .unwrap() + .unwrap() + .status, + BackupStatus::Available, + "a refused delete leaves the row AVAILABLE" + ); + sqlx::query("UPDATE tables SET table_status = 'ACTIVE' WHERE table_id = $1") + .bind(&target_id) + .execute(&s.catalog) + .await + .expect("make it ACTIVE"); + assert_eq!(s.engine.backup_in_use(&key).await.unwrap(), None); + + assert_eq!( + s.engine.start_backup_delete(&key).await.unwrap(), + DeleteStart::Deleting + ); + assert_eq!( + s.engine.start_backup_delete(&key).await.unwrap(), + DeleteStart::Status(BackupStatus::Deleting), + "a retry finds the row DELETING" + ); + assert_eq!( + s.engine.forget_available_backup_row(&key).await.unwrap(), + extenddb_storage::backup_catalog::ForgetRow::Status(BackupStatus::Deleting), + "only an AVAILABLE row is removed outright" + ); + // An AVAILABLE row goes outright, read and listed nowhere, with no + // tombstone; a second removal finds no row. + let mut adopted = catalog_index_row(ACCOUNT, "Music", "adopted"); + adopted.status = BackupStatus::Available; + assert!(s.engine.insert_backup_row(&adopted).await.expect("insert")); + assert_eq!( + s.engine + .forget_available_backup_row(&adopted.key()) + .await + .unwrap(), + extenddb_storage::backup_catalog::ForgetRow::Removed + ); + assert!( + s.engine + .read_backup_row(&arn_of(&adopted)) + .await + .unwrap() + .is_none() + ); + for status in [None, Some(BackupStatus::Available)] { + let listed: Vec = s + .engine + .list_backup_rows(status, &adopted.store) + .try_collect() + .await + .unwrap(); + assert!( + !listed.iter().any(|r| r.backup_arn == adopted.backup_arn), + "{status:?}" + ); + } + let by_account: Vec = s + .engine + .list_account_backup_rows(&adopted.account_id, None, &TimeBounds::default()) + .try_collect() + .await + .unwrap(); + assert!( + !by_account + .iter() + .any(|r| r.backup_arn == adopted.backup_arn) + ); + assert!(matches!( + s.engine.forget_available_backup_row(&adopted.key()).await, + Err(StorageError::Validation(m)) if m.contains("Backup not found") + )); + assert!( + !s.engine.remove_backup_row(&key).await.unwrap(), + "DELETING is not a tombstone" + ); + assert!(s.engine.set_backup_deleted(&key).await.unwrap()); + assert!(!s.engine.set_backup_deleted(&key).await.unwrap()); + let stored = s + .engine + .read_backup_row(&arn_of(&row)) + .await + .unwrap() + .unwrap(); + assert_eq!(stored.status, BackupStatus::Deleted); + assert!(stored.completed_at.is_some()); + assert!(s.engine.remove_backup_row(&key).await.unwrap()); + assert!(!s.engine.remove_backup_row(&key).await.unwrap()); + assert!( + s.engine + .read_backup_row(&arn_of(&row)) + .await + .unwrap() + .is_none() + ); + drop(claim); + s.cleanup().await; + } + + /// The listings: by account with an optional table and the time bounds + /// (lower inclusive, upper exclusive) in (created_at, ARN) order, and by + /// store and status. + #[tokio::test] + async fn the_listings_are_scoped_and_ordered() { + if base_conn().is_none() { + eprintln!("SKIP the_listings_are_scoped_and_ordered: no PostgreSQL"); + return; + } + let s = scratch().await; + let mut rows = Vec::new(); + for (account, table, name) in [ + (ACCOUNT, "Music", "a"), + (ACCOUNT, "Music", "b"), + (ACCOUNT, "Other", "c"), + (OTHER_ACCOUNT, "Music", "d"), + ] { + tokio::time::sleep(std::time::Duration::from_millis(5)).await; + let row = row(account, table, name); + let claim = s.engine.claim_backup(&row).await.expect("claim"); + if name == "b" { + assert_eq!( + claim + .set_status( + JobStatus::Creating, + JobStatus::Failed, + &StatusFields::default() + ) + .await + .unwrap(), + WriteOutcome::Matched + ); + } + rows.push((row, claim)); + } + let names = |listed: Vec| -> Vec { + listed.into_iter().map(|r| r.backup_name).collect() + }; + let all: Vec = s + .engine + .list_account_backup_rows(ACCOUNT, None, &TimeBounds::default()) + .try_collect() + .await + .unwrap(); + assert_eq!( + names(all.clone()), + ["a", "b", "c"], + "every status, in creation order" + ); + assert_eq!(all[1].status, BackupStatus::Failed); + let music: Vec = s + .engine + .list_account_backup_rows(ACCOUNT, Some("Music"), &TimeBounds::default()) + .try_collect() + .await + .unwrap(); + assert_eq!(names(music), ["a", "b"]); + let bounds = TimeBounds { + lower: Some(rows[1].0.created_at), + upper: Some(rows[2].0.created_at), + }; + let bounded: Vec = s + .engine + .list_account_backup_rows(ACCOUNT, None, &bounds) + .try_collect() + .await + .unwrap(); + assert_eq!(names(bounded), ["b"], "lower inclusive, upper exclusive"); + let other: Vec = s + .engine + .list_account_backup_rows(OTHER_ACCOUNT, None, &TimeBounds::default()) + .try_collect() + .await + .unwrap(); + assert_eq!(names(other), ["d"]); + + let by_store: Vec = s + .engine + .list_backup_rows(None, &store()) + .try_collect() + .await + .unwrap(); + assert_eq!(names(by_store), ["a", "b", "c", "d"]); + let failed: Vec = s + .engine + .list_backup_rows(Some(BackupStatus::Failed), &store()) + .try_collect() + .await + .unwrap(); + assert_eq!(names(failed), ["b"]); + let elsewhere = StoreIdentity { + store_id: "store-2".to_owned(), + ..store() + }; + let none: Vec = s + .engine + .list_backup_rows(None, &elsewhere) + .try_collect() + .await + .unwrap(); + assert!(none.is_empty(), "another store's rows are not listed"); + + // Two rows with one `created_at` order by the ARN in byte order + // (`backup_arn` is `COLLATE "C"`), the order the engine merges and + // pages by, not the database's collation: under `en_US` "a" sorts + // before "B", in byte order "B" (0x42) before "a" (0x61). The + // column's collation is asserted first, so the cell has teeth on a + // cluster whose default collation is already `C` (where the two + // orders agree and the rows alone would show nothing). + let collation: String = sqlx::query_scalar( + "SELECT c.collname FROM pg_attribute a JOIN pg_collation c ON c.oid = a.attcollation \ + WHERE a.attrelid = 'backup_index'::regclass AND a.attname = 'backup_arn'", + ) + .fetch_one(&s.catalog) + .await + .expect("the column's collation"); + assert_eq!(collation, "C", "backup_index.backup_arn is COLLATE \"C\""); + const CASE_ACCOUNT: &str = "333333333333"; + let mut same_instant = Vec::new(); + for table in ["a", "B"] { + let mut r = row(CASE_ACCOUNT, table, table); + r.created_at = 1_700_000_000.0; + let claim = s.engine.claim_backup(&r).await.expect("claim"); + same_instant.push((r, claim)); + } + let listed: Vec = s + .engine + .list_account_backup_rows(CASE_ACCOUNT, None, &TimeBounds::default()) + .try_collect() + .await + .unwrap(); + assert_eq!( + names(listed), + ["B", "a"], + "equal created_at: byte order of the ARN, whatever the database's collation" + ); + drop(same_instant); + drop(rows); + s.cleanup().await; + } + + /// A catalog at the shape before migration 6 holding a finished restore + /// and a restore the stop interrupted (its target CREATING) migrates with + /// the registrations backfilled: the finished one DONE, the interrupted + /// one RUNNING with `heartbeat_at` at the epoch and the unowned + /// generation, so the reconciler's first pass fails it and removes its + /// target. + #[tokio::test] + async fn migration_6_backfills_the_registrations_and_the_reconciler_removes_the_interrupted_one() + { + if base_conn().is_none() { + eprintln!( + "SKIP migration_6_backfills_the_registrations_and_the_reconciler_removes_the_interrupted_one: no PostgreSQL" + ); + return; + } + let s = scratch().await; + indexed_source(&s, "bf_src").await; + let backup = s + .engine + .create(ACCOUNT, "bf_src", "b", JobPermit::detached()) + .await + .expect("backup"); + let finished = restore(&s, "bf_done", &backup.backup_arn) + .await + .expect("restore"); + let interrupted = restore(&s, "bf_cut", &backup.backup_arn) + .await + .expect("restore"); + let catalog_id = s.engine.catalog_id().await.expect("id"); + + // Back to the shape before migration 6: the job record's columns and + // index gone, the registrations as the old release wrote them, the + // interrupted target CREATING, and the migration forgotten. + sqlx::raw_sql( + "DROP INDEX IF EXISTS idx_table_restores_status; \ + ALTER TABLE table_restores DROP COLUMN status, DROP COLUMN kind, \ + DROP COLUMN store_kind, DROP COLUMN store_root, DROP COLUMN store_id, \ + DROP COLUMN owner_generation, DROP COLUMN heartbeat_at, DROP COLUMN spool_host, \ + DROP COLUMN file_index, DROP COLUMN items_applied, DROP COLUMN failure_reason, \ + DROP COLUMN completed_at;", + ) + .execute(&s.catalog) + .await + .expect("drop the job record's columns"); + sqlx::query( + "UPDATE tables SET table_status = 'CREATING', status_transition_at = NULL WHERE table_id = $1", + ) + .bind(&interrupted.table_id) + .execute(&s.catalog) + .await + .expect("interrupt the second restore"); + let target = + PostgresTarget::new(s.catalog.connect_options().as_ref().clone(), &s.db_names[1]); + let runner = PostgresMigrations::open_unlocked(&target) + .await + .expect("open the runner"); + let migrations = Runner::new(&runner, "test"); + migrations.forget(6).await.expect("forget migration 6"); + let report = migrations.migrate().await.expect("migrate again"); + runner.close().await; + assert_eq!(report.applied.iter().map(|a| a.id).collect::>(), [6]); + + #[allow(clippy::type_complexity)] + let records: Vec<(String, String, String, i64, f64, Option)> = sqlx::query_as( + "SELECT t.table_name, r.status, r.kind, r.owner_generation, \ + EXTRACT(EPOCH FROM r.heartbeat_at)::FLOAT8, \ + EXTRACT(EPOCH FROM r.completed_at - t.creation_date_time)::FLOAT8 \ + FROM table_restores r JOIN tables t ON t.table_id = r.table_id ORDER BY t.table_name", + ) + .fetch_all(&s.catalog) + .await + .expect("read the records"); + assert_eq!(records.len(), 2, "{records:?}"); + let (name, status, kind, generation, heartbeat, completed) = &records[0]; + assert_eq!(name, "bf_cut"); + assert_eq!(status, "RUNNING"); + assert_eq!(kind, "legacy"); + assert_eq!(*generation, i64::try_from(UNOWNED_GENERATION).unwrap()); + assert_eq!(*heartbeat, 0.0, "the epoch, not NULL"); + assert_eq!(*completed, None); + let (name, status, kind, _, _, completed) = &records[1]; + assert_eq!(name, "bf_done"); + assert_eq!(status, "DONE"); + assert_eq!(kind, "legacy"); + assert_eq!( + *completed, + Some(0.0), + "completed when the table was created" + ); + assert_eq!( + s.engine.catalog_id().await.expect("id"), + catalog_id, + "a second run keeps the catalog id" + ); + + // The first reconciler pass: the interrupted restore is dead. + let dead = s + .engine + .list_dead_restore_jobs(std::time::Duration::from_secs(300)) + .await + .expect("list"); + assert_eq!(dead.len(), 1, "{dead:?}"); + assert_eq!( + dead[0].target, + RestoreTargetKey { + account_id: ACCOUNT.to_owned(), + table_name: "bf_cut".to_owned(), + table_id: interrupted.table_id.clone(), + } + ); + assert_eq!(dead[0].owner_generation, UNOWNED_GENERATION); + assert!( + s.engine + .fail_restore_job( + &dead[0].target, + dead[0].owner_generation, + dead[0].heartbeat_at, + "owner dead" + ) + .await + .unwrap() + ); + assert!( + s.engine + .remove_restore_target(&dead[0].target) + .await + .unwrap() + ); + assert_eq!(table_status(&s, "bf_cut").await, None); + assert!(!data_table_exists(&s, &interrupted.table_id).await); + assert_eq!(table_status(&s, "bf_done").await.as_deref(), Some("ACTIVE")); + assert_eq!(table_status(&s, "bf_src").await.as_deref(), Some("ACTIVE")); + let _ = finished; + s.cleanup().await; + } + + /// The legacy restore registers through the job record: RUNNING under + /// the claim's generation while the copy runs, DONE with the ACTIVE flip. + #[tokio::test] + async fn a_legacy_restore_registers_a_running_job_record_and_finishes_it_done() { + if base_conn().is_none() { + eprintln!( + "SKIP a_legacy_restore_registers_a_running_job_record_and_finishes_it_done: no PostgreSQL" + ); + return; + } + let s = scratch().await; + indexed_source(&s, "jr_src").await; + let backup = s + .engine + .create(ACCOUNT, "jr_src", "b", JobPermit::detached()) + .await + .expect("backup"); + let gate = gate_copies(&s).await; + let desc = start_restore(&s, "jr_dst", &backup.backup_arn) + .await + .expect("registers"); + await_copies_waiting(&s, 1).await; + let record: (String, String, i64, f64) = sqlx::query_as( + "SELECT status, kind, owner_generation, EXTRACT(EPOCH FROM now() - heartbeat_at)::FLOAT8 \ + FROM table_restores WHERE table_id = $1", + ) + .bind(&desc.table_id) + .fetch_one(&s.catalog) + .await + .expect("record"); + assert_eq!(record.0, "RUNNING"); + assert_eq!(record.1, "legacy"); + assert_ne!(record.2, i64::try_from(UNOWNED_GENERATION).unwrap()); + assert!(record.3 < 60.0, "a fresh heartbeat: {} s old", record.3); + assert!( + s.engine + .list_dead_restore_jobs(std::time::Duration::from_secs(300)) + .await + .unwrap() + .is_empty(), + "a running restore is not dead" + ); + gate.commit().await.expect("release"); + assert_eq!( + settled_status(&s, "jr_dst").await.as_deref(), + Some("ACTIVE") + ); + let record: (String, Option) = sqlx::query_as( + "SELECT status, EXTRACT(EPOCH FROM completed_at)::FLOAT8 FROM table_restores WHERE table_id = $1", + ) + .bind(&desc.table_id) + .fetch_one(&s.catalog) + .await + .expect("record"); + assert_eq!(record.0, "DONE"); + assert!(record.1.is_some()); + s.cleanup().await; + } + + /// `read_store_ids`: the distinct ids of the rows for a root, of every + /// status, scoped to the kind and the root. + #[tokio::test] + async fn read_store_ids_lists_the_distinct_ids_of_a_root() { + if base_conn().is_none() { + eprintln!("SKIP read_store_ids_lists_the_distinct_ids_of_a_root: no PostgreSQL"); + return; + } + let s = scratch().await; + let root = &store().store_root; + assert!( + s.engine + .read_store_ids(StoreKind::Filesystem, root) + .await + .expect("ids") + .is_empty() + ); + let first = row(ACCOUNT, "Music", "a"); + let _c1 = s.engine.claim_backup(&first).await.expect("claim"); + let second = row(ACCOUNT, "Music", "b"); + let c2 = s.engine.claim_backup(&second).await.expect("claim"); + c2.set_status( + JobStatus::Creating, + JobStatus::Failed, + &StatusFields::default(), + ) + .await + .expect("fail"); + assert_eq!( + s.engine + .read_store_ids(StoreKind::Filesystem, root) + .await + .expect("ids"), + ["store-1"], + "two rows of one store, of two statuses, are one id" + ); + let mut other = row(ACCOUNT, "Songs", "c"); + other.store.store_id = "store-2".to_owned(); + let _c3 = s.engine.claim_backup(&other).await.expect("claim"); + assert_eq!( + s.engine + .read_store_ids(StoreKind::Filesystem, root) + .await + .expect("ids"), + ["store-1", "store-2"] + ); + assert!( + s.engine + .read_store_ids(StoreKind::S3, root) + .await + .expect("ids") + .is_empty(), + "another kind at the same root is another store" + ); + assert!( + s.engine + .read_store_ids(StoreKind::Filesystem, "/elsewhere") + .await + .expect("ids") + .is_empty() + ); + s.cleanup().await; + } + + /// `insert_backup_row`: the row lands as given, unowned, with + /// `completed_at` and `heartbeat_at` on the database's clock; a second + /// insert, an insert over a CREATING row a claim holds, and an insert of + /// the id under another account (the unique `backup_id`) answer `false` + /// and change nothing; a claim over the inserted row is refused; the row + /// is read and listed like a created one. + #[tokio::test] + async fn insert_backup_row_inserts_an_unowned_row_only_when_absent() { + if base_conn().is_none() { + eprintln!( + "SKIP insert_backup_row_inserts_an_unowned_row_only_when_absent: no PostgreSQL" + ); + return; + } + let s = scratch().await; + let mut adopted = catalog_index_row(ACCOUNT, "Music", "adopted"); + adopted.status = BackupStatus::Available; + adopted.size_bytes = 1234; + adopted.item_count = 5; + // Ignored: the insert sets these three itself. + adopted.owner_generation = 99; + adopted.completed_at = None; + adopted.heartbeat_at = 0.0; + assert!(s.engine.insert_backup_row(&adopted).await.expect("insert")); + let stored = s + .engine + .read_backup_row(&arn_of(&adopted)) + .await + .expect("read") + .expect("the row exists"); + assert_eq!(stored.owner_generation, UNOWNED_GENERATION); + let completed_at = stored.completed_at.expect("completed_at is set"); + for at in [completed_at, stored.heartbeat_at] { + assert!((at - adopted.created_at).abs() < 300.0, "{at}"); + } + assert_eq!( + BackupRow { + owner_generation: 99, + completed_at: None, + heartbeat_at: 0.0, + ..stored.clone() + }, + adopted, + "every other column is the row as given" + ); + + // A second insert of the key, with other values, writes nothing. + let mut again = adopted.clone(); + again.backup_name = "again".to_owned(); + again.size_bytes = 1; + again.status = BackupStatus::Failed; + assert!(!s.engine.insert_backup_row(&again).await.expect("insert")); + // The id under another account: the unique `backup_id`. + let mut elsewhere = adopted.clone(); + elsewhere.account_id = OTHER_ACCOUNT.to_owned(); + elsewhere.backup_arn = + extenddb_storage::util::backup_arn(REGION, OTHER_ACCOUNT, "Music", &adopted.backup_id); + assert!( + !s.engine + .insert_backup_row(&elsewhere) + .await + .expect("insert") + ); + assert!( + s.engine + .read_backup_row(&arn_of(&elsewhere)) + .await + .unwrap() + .is_none() + ); + assert_eq!( + s.engine.read_backup_row(&arn_of(&adopted)).await.unwrap(), + Some(stored.clone()), + "the refused inserts changed nothing" + ); + let claimed = s.engine.claim_backup(&adopted).await.err(); + assert!( + matches!(claimed, Some(StorageError::AlreadyExists(_))), + "{claimed:?}" + ); + + // Over a CREATING row a live claim holds. + let creating = row(ACCOUNT, "Music", "job"); + let claim = s.engine.claim_backup(&creating).await.expect("claim"); + let mut over = creating.clone(); + over.status = BackupStatus::Available; + assert!(!s.engine.insert_backup_row(&over).await.expect("insert")); + let job_row = s + .engine + .read_backup_row(&arn_of(&creating)) + .await + .unwrap() + .unwrap(); + assert_eq!(job_row.status, BackupStatus::Creating); + assert_eq!(job_row.owner_generation, claim.generation()); + + // Read and listed like a created row. + let arns = |rows: Vec| -> Vec { + rows.into_iter().map(|r| r.backup_arn).collect() + }; + let available: Vec = s + .engine + .list_backup_rows(Some(BackupStatus::Available), &store()) + .try_collect() + .await + .unwrap(); + assert_eq!(available, std::slice::from_ref(&stored)); + let creating_rows: Vec = s + .engine + .list_backup_rows(Some(BackupStatus::Creating), &store()) + .try_collect() + .await + .unwrap(); + assert_eq!( + arns(creating_rows), + std::slice::from_ref(&creating.backup_arn) + ); + for table in [None, Some("Music")] { + let listed: Vec = s + .engine + .list_account_backup_rows(ACCOUNT, table, &TimeBounds::default()) + .try_collect() + .await + .unwrap(); + assert!( + listed.contains(&stored), + "{table:?}: {:?}", + arns(listed.clone()) + ); + } + drop(claim); + s.cleanup().await; + } + + /// The maintenance claim: taken once, refused to a second holder naming + /// the first, heartbeated, released; a holder whose session ended and + /// whose heartbeat is older than the stale window is dead, so its claim + /// is not reported held and is taken over, and its handle finds the claim + /// lost. + /// `remove_dead_creating_backup_row`: a CREATING row whose heartbeat is + /// older than the window is removed outright; one with a live owner, one + /// that is not CREATING, and one that is gone are left, answering + /// `false`. + #[tokio::test] + async fn a_dead_creating_row_is_removed_and_a_live_or_moved_one_is_kept() { + if base_conn().is_none() { + eprintln!( + "SKIP a_dead_creating_row_is_removed_and_a_live_or_moved_one_is_kept: no PostgreSQL" + ); + return; + } + let s = scratch().await; + let stale = std::time::Duration::from_secs(300); + let age = |backup_id: String| { + let catalog = s.catalog.clone(); + async move { + sqlx::query( + "UPDATE backup_index SET heartbeat_at = now() - interval '10 minutes' \ + WHERE backup_id = $1", + ) + .bind(backup_id) + .execute(&catalog) + .await + .expect("age"); + } + }; + let creating = row(ACCOUNT, "Music", "a"); + let key = creating.key(); + let claim = s.engine.claim_backup(&creating).await.expect("claim"); + assert!( + !s.engine + .remove_dead_creating_backup_row(&key, stale) + .await + .unwrap(), + "a live owner keeps its row" + ); + age(creating.backup_id.clone()).await; + assert!( + s.engine + .remove_dead_creating_backup_row(&key, stale) + .await + .unwrap() + ); + assert!( + s.engine + .read_backup_row(&arn_of(&creating)) + .await + .unwrap() + .is_none() + ); + assert!( + !s.engine + .remove_dead_creating_backup_row(&key, stale) + .await + .unwrap(), + "gone" + ); + drop(claim); + + let available = row(ACCOUNT, "Music", "b"); + let claim = s.engine.claim_backup(&available).await.expect("claim"); + claim + .set_status( + JobStatus::Creating, + JobStatus::Available, + &StatusFields::default(), + ) + .await + .expect("available"); + age(available.backup_id.clone()).await; + assert!( + !s.engine + .remove_dead_creating_backup_row(&available.key(), stale) + .await + .unwrap(), + "only a CREATING row" + ); + assert_eq!( + s.engine + .read_backup_row(&arn_of(&available)) + .await + .unwrap() + .unwrap() + .status, + BackupStatus::Available + ); + drop(claim); + s.cleanup().await; + } + + /// `list_backup_rows_older_than` compares the named column with the + /// database's `now()`: a FAILED row whose heartbeat is older than the age + /// but whose completion is not is listed on the heartbeat clock and not + /// on the completed clock (the two columns are told apart by waiting + /// between the last heartbeat and the FAILED write, on the database's + /// clock); a row crosses the window by waiting; a row of another store, + /// another status, or with `completed_at` unset on the completed clock + /// is not listed. + #[tokio::test] + async fn rows_older_than_are_listed_on_the_named_clock() { + if base_conn().is_none() { + eprintln!("SKIP rows_older_than_are_listed_on_the_named_clock: no PostgreSQL"); + return; + } + let s = scratch().await; + let age = std::time::Duration::from_secs(1); + let engine = &s.engine; + let list = move |status: BackupStatus, store: StoreIdentity, clock: StaleClock| async move { + let rows: Vec = engine + .list_backup_rows_older_than(status, &store, clock, age) + .try_collect() + .await + .expect("list"); + rows.into_iter().map(|r| r.backup_name).collect::>() + }; + // `aged`: heartbeat now, then FAILED after the age has passed, so its + // heartbeat is older than the age and its completion is not. + let aged = row(ACCOUNT, "Music", "aged"); + let aged_claim = s.engine.claim_backup(&aged).await.expect("claim"); + // `fresh`: FAILED at once; both columns within the age. + let fresh = row(ACCOUNT, "Music", "fresh"); + let fresh_claim = s.engine.claim_backup(&fresh).await.expect("claim"); + let failed = StatusFields { + failure_reason: Some("the test's".to_owned()), + completed: true, + ..StatusFields::default() + }; + assert_eq!( + fresh_claim + .set_status(JobStatus::Creating, JobStatus::Failed, &failed) + .await + .unwrap(), + WriteOutcome::Matched + ); + // `open`: FAILED without a completion stamp (`completed: false`). + let open = row(ACCOUNT, "Music", "open"); + let open_claim = s.engine.claim_backup(&open).await.expect("claim"); + assert_eq!( + open_claim + .set_status( + JobStatus::Creating, + JobStatus::Failed, + &StatusFields { + failure_reason: Some("the test's".to_owned()), + ..StatusFields::default() + } + ) + .await + .unwrap(), + WriteOutcome::Matched + ); + // `creating`: stays CREATING with its heartbeat ageing. + let creating = row(ACCOUNT, "Music", "creating"); + let _creating_claim = s.engine.claim_backup(&creating).await.expect("claim"); + // `elsewhere`: FAILED at once, in another store. + let mut elsewhere = row(ACCOUNT, "Music", "elsewhere"); + elsewhere.store.store_id = "store-2".to_owned(); + let elsewhere_claim = s.engine.claim_backup(&elsewhere).await.expect("claim"); + assert_eq!( + elsewhere_claim + .set_status(JobStatus::Creating, JobStatus::Failed, &failed) + .await + .unwrap(), + WriteOutcome::Matched + ); + + // Nothing is older than the age yet, on either clock. + assert_eq!( + list(BackupStatus::Failed, store(), StaleClock::Heartbeat).await, + Vec::::new() + ); + assert_eq!( + list(BackupStatus::Failed, store(), StaleClock::Completed).await, + Vec::::new() + ); + + tokio::time::sleep(age + std::time::Duration::from_millis(300)).await; + assert_eq!( + aged_claim + .set_status(JobStatus::Creating, JobStatus::Failed, &failed) + .await + .unwrap(), + WriteOutcome::Matched + ); + // Heartbeat clock: every FAILED row of the store heartbeated before + // the wait (`open` has no completion but has a heartbeat); not the + // CREATING row, not the other store's. + assert_eq!( + list(BackupStatus::Failed, store(), StaleClock::Heartbeat).await, + ["aged", "fresh", "open"] + ); + // Completed clock: `aged` completed just now, `open` never. + assert_eq!( + list(BackupStatus::Failed, store(), StaleClock::Completed).await, + ["fresh"] + ); + assert_eq!( + list(BackupStatus::Creating, store(), StaleClock::Heartbeat).await, + ["creating"] + ); + let mut other = store(); + other.store_id = "store-2".to_owned(); + assert_eq!( + list(BackupStatus::Failed, other, StaleClock::Heartbeat).await, + ["elsewhere"] + ); + // `aged` crosses the completed window by waiting. + tokio::time::sleep(age + std::time::Duration::from_millis(300)).await; + assert_eq!( + list(BackupStatus::Failed, store(), StaleClock::Completed).await, + ["aged", "fresh"] + ); + s.cleanup().await; + } + + #[tokio::test] + async fn the_maintenance_claim_excludes_a_second_holder_and_expires_when_stale() { + if base_conn().is_none() { + eprintln!( + "SKIP the_maintenance_claim_excludes_a_second_holder_and_expires_when_stale: \ + no PostgreSQL" + ); + return; + } + let s = scratch().await; + let stale = std::time::Duration::from_secs(300); + let check = || s.engine.check_maintenance_claim(stale); + assert!(!check().await.expect("check")); + let first = s + .engine + .maintenance_claim("purge on host-a", stale) + .await + .expect("claim"); + assert_ne!(first.generation(), UNOWNED_GENERATION); + assert!(check().await.expect("check"), "a live claim is held"); + match s.engine.maintenance_claim("purge on host-b", stale).await { + Err(StorageError::AlreadyExists(m)) => { + assert!(m.contains("purge on host-a"), "{m}"); + } + Err(other) => panic!("a second holder is refused with AlreadyExists: {other:?}"), + Ok(_) => panic!("a second holder is refused"), + } + assert_eq!( + first.heartbeat().await.expect("heartbeat"), + Heartbeat::Confirmed + ); + first.release().await.expect("release"); + assert!(!check().await.expect("check"), "released"); + first.release().await.expect("a second release is a no-op"); + + // A dead holder: its session ended (the handle dropped) and its + // heartbeat aged past the window. + let dead = s + .engine + .maintenance_claim("purge on host-c", stale) + .await + .expect("claim"); + let dead_generation = dead.generation(); + drop(dead); + // The drop closes the holder's session; the backend reaps it a + // moment later. Wait for the exclusive lock to be gone, so the check + // below is answered by the heartbeat arm alone and not by a lock + // still being released. + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + loop { + let exclusive: i64 = sqlx::query_scalar( + "SELECT count(*) FROM pg_locks WHERE locktype = 'advisory' AND mode = \ + 'ExclusiveLock' AND database = (SELECT oid FROM pg_database WHERE datname = \ + current_database())", + ) + .fetch_one(&s.catalog) + .await + .expect("pg_locks"); + if exclusive == 0 { + break; + } + assert!( + std::time::Instant::now() < deadline, + "the dropped holder's session still holds the lock" + ); + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + } + // The heartbeat arm: the lock is free (the session ended with the + // handle) and the row's heartbeat is young, so the jobs still see + // the claim held, as they must for a command that died a moment ago + // or lost its claim while its row is young. + assert!( + check().await.expect("check"), + "a dead session's row is held by its heartbeat until it ages" + ); + sqlx::query("UPDATE backup_maintenance SET heartbeat_at = now() - interval '10 minutes'") + .execute(&s.catalog) + .await + .expect("age"); + assert!(!check().await.expect("check"), "a stale claim is not held"); + let taker = s + .engine + .maintenance_claim("purge on host-d", stale) + .await + .expect("a dead holder's claim is taken over"); + assert_ne!(taker.generation(), dead_generation); + assert!(check().await.expect("check")); + let holder: String = sqlx::query_scalar("SELECT holder FROM backup_maintenance") + .fetch_one(&s.catalog) + .await + .expect("holder"); + assert_eq!(holder, "purge on host-d"); + // A holder whose session is still open is alive whatever its + // heartbeat says: the lock is the proof of life on this backend. + sqlx::query("UPDATE backup_maintenance SET heartbeat_at = now() - interval '10 minutes'") + .execute(&s.catalog) + .await + .expect("age"); + assert!( + matches!( + s.engine.maintenance_claim("purge on host-e", stale).await, + Err(StorageError::AlreadyExists(_)) + ), + "a live session keeps the claim" + ); + assert!( + check().await.expect("check"), + "the jobs' check sees a live session's claim held through the lock, whatever the \ + row's heartbeat says" + ); + // The check holds no lock past its statement: after it, the only + // advisory lock in the catalog is the holder's exclusive one, so no + // pooled connection carries a shared lock a dropped request left. + let modes: Vec = sqlx::query_scalar( + "SELECT mode FROM pg_locks WHERE locktype = 'advisory' AND database = \ + (SELECT oid FROM pg_database WHERE datname = current_database()) ORDER BY mode", + ) + .fetch_all(&s.catalog) + .await + .expect("pg_locks"); + assert_eq!( + modes, + ["ExclusiveLock"], + "the holder's lock and nothing else" + ); + assert_eq!( + taker.heartbeat().await.expect("heartbeat"), + Heartbeat::Confirmed + ); + assert!(check().await.expect("check"), "the heartbeat refreshed it"); + // The row's generation moved under the live holder (a takeover by + // another writer): the holder's next heartbeat matches no row and + // reports the claim lost, and its release leaves the other's row. + sqlx::query("UPDATE backup_maintenance SET owner_generation = owner_generation + 1") + .execute(&s.catalog) + .await + .expect("move the generation"); + assert_eq!( + taker.heartbeat().await.expect("heartbeat"), + Heartbeat::Lost, + "a heartbeat under a moved generation reports the claim lost" + ); + taker.release().await.expect("release"); + let rows: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM backup_maintenance") + .fetch_one(&s.catalog) + .await + .expect("count"); + assert_eq!( + rows, 1, + "a release by a holder that lost the claim leaves the row" + ); + assert!( + check().await.expect("check"), + "the lock is free and the row young: held by the heartbeat arm" + ); + sqlx::query("DELETE FROM backup_maintenance") + .execute(&s.catalog) + .await + .expect("clear"); + assert!(!check().await.expect("check")); + s.cleanup().await; + } + + /// `check_maintenance_claim` holds the claim's key shared for one + /// statement, and a take that lands inside it finds the exclusive lock + /// refused with no holder, so the take tries again on its claim + /// connection. Held across every try, the shared lock makes the take + /// refuse, naming no holder (there is no row). With the lock step given + /// a long span, a shared lock released once the claim connection has + /// made a failed try (seen in `pg_stat_activity`) is taken by a later + /// try. + #[tokio::test] + async fn a_maintenance_take_tries_again_past_a_checks_shared_lock() { + if base_conn().is_none() { + eprintln!( + "SKIP a_maintenance_take_tries_again_past_a_checks_shared_lock: no PostgreSQL" + ); + return; + } + let s = scratch().await; + let stale = std::time::Duration::from_secs(300); + const SHARED: &str = + "SELECT pg_advisory_xact_lock_shared(hashtextextended('backup_maintenance', $1))"; + + let mut check = s.catalog.begin().await.expect("begin"); + sqlx::query(SHARED) + .bind(RESTORE_LOCK_SEED) + .execute(&mut *check) + .await + .expect("the check's shared lock"); + let started = std::time::Instant::now(); + match s.engine.maintenance_claim("purge on host-a", stale).await { + Err(StorageError::AlreadyExists(m)) => { + assert!(m.contains("another maintenance command"), "{m}"); + } + Err(other) => panic!("refused otherwise: {other:?}"), + Ok(_) => panic!("taken beside a held shared lock"), + } + assert!( + started.elapsed() >= std::time::Duration::from_millis(10), + "the take tried once: {:?}", + started.elapsed() + ); + check.rollback().await.expect("rollback"); + + let mut check = s.catalog.begin().await.expect("begin"); + sqlx::query(SHARED) + .bind(RESTORE_LOCK_SEED) + .execute(&mut *check) + .await + .expect("the check's shared lock"); + let tried = std::sync::atomic::AtomicBool::new(false); + let (taken, ()) = tokio::join!( + async { + let taken = s + .engine + .maintenance_lock_for_tests(10_000, std::time::Duration::from_millis(1)) + .await; + tried.store(true, std::sync::atomic::Ordering::SeqCst); + taken + }, + async { + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(30); + loop { + let failed_try: bool = sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM pg_stat_activity \ + WHERE pid <> pg_backend_pid() AND datname = current_database() \ + AND query LIKE 'SELECT pg_try_advisory_lock(%')", + ) + .fetch_one(&s.catalog) + .await + .expect("activity"); + if failed_try + || tried.load(std::sync::atomic::Ordering::SeqCst) + || std::time::Instant::now() > deadline + { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(1)).await; + } + check.rollback().await.expect("rollback"); + } + ); + assert!( + taken.expect("the lock step"), + "a later try took the lock once the check's statement ended" + ); + s.cleanup().await; + } + + /// Purge's wait (`wait_for_quiet`) over this catalog, on a real claim. + /// A FAILED restore record whose target is still CREATING (the state a + /// stop between the FAILED write and the target's removal leaves) is + /// returned by the zero-window dead-job listing with the RUNNING + /// records; it is the reconciler's duty, not a running job, so the wait + /// is quiet at its first read and runs no duty itself: the target is + /// left CREATING for the server. A FAILED backup row whose heartbeat is + /// within the window (a job that has not yet seen its claim lost may + /// still write under its prefix) holds the wait to its limit, which + /// names it; once the heartbeat has aged past the window it does not. + #[tokio::test] + async fn the_wait_passes_a_failed_record_and_holds_on_a_failed_row_with_a_live_heartbeat() { + use extenddb_engine::HeldClaim; + use extenddb_engine::maintenance::{Interrupt, Wait, WaitOutcome, wait_for_quiet}; + if base_conn().is_none() { + eprintln!( + "SKIP the_wait_passes_a_failed_record_and_holds_on_a_failed_row_with_a_live_heartbeat: \ + no PostgreSQL" + ); + return; + } + let stale = std::time::Duration::from_secs(300); + let s = scratch().await; + indexed_source(&s, "wq_src").await; + let backup = s + .engine + .create(ACCOUNT, "wq_src", "b", JobPermit::detached()) + .await + .expect("backup"); + let desc = restore(&s, "wq_dst", &backup.backup_arn) + .await + .expect("restore"); + sqlx::query( + "UPDATE tables SET table_status = 'CREATING', status_transition_at = NULL \ + WHERE table_id = $1", + ) + .bind(&desc.table_id) + .execute(&s.catalog) + .await + .expect("the target back to CREATING"); + sqlx::query( + "UPDATE table_restores SET status = 'FAILED', completed_at = NOW() - INTERVAL '1 hour' \ + WHERE table_id = $1", + ) + .bind(&desc.table_id) + .execute(&s.catalog) + .await + .expect("the record FAILED an hour ago"); + let listed = s + .engine + .list_dead_restore_jobs(std::time::Duration::ZERO) + .await + .expect("list"); + assert_eq!(listed.len(), 1, "{listed:?}"); + assert_eq!( + listed[0].status, + JobStatus::Failed, + "the zero-window listing returns the FAILED record the wait reads" + ); + + let claim = s + .engine + .maintenance_claim("purge on host-a", stale) + .await + .expect("claim"); + let held = HeldClaim::start(claim, std::time::Duration::from_secs(1)); + let limit = std::time::Duration::from_secs(3); + let wait = Wait { + owner_window: stale, + limit, + poll: std::time::Duration::from_millis(200), + }; + let announced: std::sync::Mutex>> = std::sync::Mutex::new(Vec::new()); + let announce = |jobs: &[String]| announced.lock().unwrap().push(jobs.to_vec()); + let (_tx, mut interrupt) = Interrupt::new(); + let started = std::time::Instant::now(); + assert_eq!( + wait_for_quiet(&s.engine, &store(), &held, wait, &mut interrupt, announce).await, + WaitOutcome::Quiet, + "a FAILED restore record is the reconciler's duty, not a running job" + ); + assert!( + started.elapsed() < std::time::Duration::from_secs(1), + "quiet at the first read: {:?}", + started.elapsed() + ); + assert!(announced.lock().unwrap().is_empty()); + assert_eq!( + table_status(&s, "wq_dst").await.as_deref(), + Some("CREATING"), + "the wait runs no duty: the target is left for the server's reconciler" + ); + assert_eq!( + job_status(&s, &desc.table_id).await.as_deref(), + Some("FAILED") + ); + + // A FAILED backup row of the store whose heartbeat is this moment's. + let failed = row(ACCOUNT, "Music", "f"); + let job = s.engine.claim_backup(&failed).await.expect("claim"); + job.set_status( + JobStatus::Creating, + JobStatus::Failed, + &StatusFields { + failure_reason: Some("the test's".to_owned()), + ..StatusFields::default() + }, + ) + .await + .expect("failed"); + drop(job); + let started = std::time::Instant::now(); + let expected = vec![format!( + "the FAILED backup {} has a live owner (heartbeat within stale_job_after_secs)", + failed.backup_arn + )]; + assert_eq!( + wait_for_quiet(&s.engine, &store(), &held, wait, &mut interrupt, announce).await, + WaitOutcome::Busy(expected.clone()), + "a FAILED row with a live heartbeat holds the wait and is named" + ); + assert!(started.elapsed() >= limit, "{:?}", started.elapsed()); + assert_eq!(*announced.lock().unwrap(), vec![expected]); + // Aged past the window on the catalog's clock: a dead owner. + sqlx::query( + "UPDATE backup_index SET heartbeat_at = now() - interval '10 minutes' \ + WHERE backup_id = $1", + ) + .bind(&failed.backup_id) + .execute(&s.catalog) + .await + .expect("age"); + assert_eq!( + wait_for_quiet(&s.engine, &store(), &held, wait, &mut interrupt, announce).await, + WaitOutcome::Quiet, + "a FAILED row whose heartbeat has aged is not waited on" + ); + held.release().await.expect("release"); + assert!( + !s.engine + .check_maintenance_claim(stale) + .await + .expect("check") + ); + s.cleanup().await; + } + + /// The factory (`create_server_components`) starts the GSI queue workers + /// for a server only. Components opened `beside_server` (`backup list`, + /// the purge's dry run) leave a pending index write where it is, for + /// the server that owns the catalog; components opened as a server + /// apply it. + #[tokio::test] + async fn components_opened_beside_a_server_apply_no_pending_index_write() { + use extenddb_storage::server_components::ServerComponentsOptions; + if base_conn().is_none() { + eprintln!( + "SKIP components_opened_beside_a_server_apply_no_pending_index_write: no PostgreSQL" + ); + return; + } + let s = scratch().await; + indexed_source_with(&s, "gq_src", 0).await; + // An asynchronous index: from here a write queues its index update + // in `gsi_pending` for a worker, instead of applying it in its own + // transaction. + sqlx::query("UPDATE settings SET value = '50' WHERE key = 'index_propagation_delay_ms'") + .execute(&s.catalog) + .await + .expect("an asynchronous index"); + let pending = || async { + sqlx::query_scalar::<_, i64>("SELECT count(*) FROM gsi_pending") + .fetch_one(&s.db) + .await + .expect("count gsi_pending") + }; + assert_eq!(pending().await, 0); + let desc = s + .engine + .describe_table( + ACCOUNT, + DescribeTableInput { + table_name: "gq_src".to_owned(), + }, + ) + .await + .expect("describe"); + put_indexed_items(&s, "gq_src", &desc, 1).await; + assert_eq!(pending().await, 1, "the write queued its index update"); + + // The factory reads the encryption key `init` writes. + sqlx::query( + "INSERT INTO settings (key, value) VALUES ('encryption_key', 'test-key') \ + ON CONFLICT (key) DO NOTHING", + ) + .execute(&s.catalog) + .await + .expect("encryption key"); + // Installed once per process; a second install is refused and that + // is fine. + let _ = extenddb_storage::set_backend(extenddb_storage_postgres::backend()); + let config = extenddb_storage_postgres::PostgresStorageConfig { + connection_string: s.catalog_url.clone(), + pool_size: 10, + catalog_pool_size: Some(10), + }; + let open = |beside_server: bool| { + let mut options = ServerComponentsOptions::default(); + options.release = "test"; + options.beside_server = beside_server; + extenddb_storage::create_server_components(&config, REGION, options) + }; + + let beside = open(true).await.expect("open beside a server"); + tokio::time::sleep(std::time::Duration::from_secs(3)).await; + assert_eq!( + pending().await, + 1, + "components opened beside a server start no worker: the row is left for the server" + ); + drop(beside); + + let server = open(false).await.expect("open as a server"); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + while pending().await != 0 { + assert!( + std::time::Instant::now() < deadline, + "a server's workers apply the pending row" + ); + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + } + drop(server); + s.cleanup().await; + } + + /// `maintenance_claim` itself (its six tries 2 ms apart) takes the lock + /// past a check's shared lock that ends before the tries do: the shared + /// lock is held until the take's first try has failed (seen in + /// `pg_stat_activity`) and then released, and a later try takes the lock. + /// With one try in place of six the take would be refused with no holder + /// to name. + #[tokio::test] + async fn a_maintenance_take_passes_a_shared_lock_shorter_than_its_span() { + if base_conn().is_none() { + eprintln!( + "SKIP a_maintenance_take_passes_a_shared_lock_shorter_than_its_span: no PostgreSQL" + ); + return; + } + let s = scratch().await; + let stale = std::time::Duration::from_secs(300); + let mut check = s.catalog.begin().await.expect("begin"); + sqlx::query( + "SELECT pg_advisory_xact_lock_shared(hashtextextended('backup_maintenance', $1))", + ) + .bind(RESTORE_LOCK_SEED) + .execute(&mut *check) + .await + .expect("the check's shared lock"); + let tried = std::sync::atomic::AtomicBool::new(false); + let (taken, ()) = tokio::join!( + async { + let taken = s.engine.maintenance_claim("purge on host-a", stale).await; + tried.store(true, std::sync::atomic::Ordering::SeqCst); + taken + }, + async { + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(30); + loop { + let failed_try: bool = sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM pg_stat_activity \ + WHERE pid <> pg_backend_pid() AND datname = current_database() \ + AND query LIKE 'SELECT pg_try_advisory_lock(%')", + ) + .fetch_one(&s.catalog) + .await + .expect("activity"); + if failed_try + || tried.load(std::sync::atomic::Ordering::SeqCst) + || std::time::Instant::now() > deadline + { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(1)).await; + } + check.rollback().await.expect("rollback"); + } + ); + let claim = taken.expect("a later try of the take passed the check's shared lock"); + assert!( + s.engine + .check_maintenance_claim(stale) + .await + .expect("check"), + "the take holds the claim" + ); + claim.release().await.expect("release"); + s.cleanup().await; + } + + /// `release` unlocks the advisory lock before it closes the session, so a + /// `check_maintenance_claim` on another connection right after the + /// release reads the claim free, every time: a CreateBackup sent the + /// moment a purge exits is not refused. Twenty rounds, with no wait + /// between the release and the check. + #[tokio::test] + async fn a_released_claim_reads_free_at_once() { + if base_conn().is_none() { + eprintln!("SKIP a_released_claim_reads_free_at_once: no PostgreSQL"); + return; + } + let s = scratch().await; + let stale = std::time::Duration::from_secs(300); + for round in 0..20 { + let claim = s + .engine + .maintenance_claim("purge on host-a", stale) + .await + .expect("claim"); + assert!( + s.engine + .check_maintenance_claim(stale) + .await + .expect("check") + ); + claim.release().await.expect("release"); + assert!( + !s.engine + .check_maintenance_claim(stale) + .await + .expect("check"), + "round {round}: the claim reads held right after its release" + ); + } + s.cleanup().await; + } + + /// The purge's wait set reads RUNNING restore records by status alone: + /// a record whose heartbeat is ahead of the server's clock (coordinator + /// skew, a backward clock step) or in this statement's instant is + /// listed by `list_running_restore_jobs`, where the dead-job listing + /// under a zero window, which compares the heartbeat with `now()`, does + /// not list it. + #[tokio::test] + async fn running_restores_are_listed_by_status_whatever_their_heartbeat() { + if base_conn().is_none() { + eprintln!( + "SKIP running_restores_are_listed_by_status_whatever_their_heartbeat: no PostgreSQL" + ); + return; + } + let s = scratch().await; + indexed_source(&s, "rr_target").await; + let target_id = table_id_of(&s, "rr_target").await.unwrap(); + sqlx::query( + "INSERT INTO table_restores (table_id, source_backup_arn, source_table_arn, \ + restore_date_time, status, kind, owner_generation, heartbeat_at) \ + VALUES ($1, $2, 'arn:aws:dynamodb:us-east-1:123456789012:table/Music', now(), \ + 'RUNNING', 'legacy', 7, now() + INTERVAL '60 seconds')", + ) + .bind(&target_id) + .bind(&row(ACCOUNT, "Music", "nightly").backup_arn) + .execute(&s.catalog) + .await + .expect("a RUNNING record stamped ahead of the clock"); + let running = s + .engine + .list_running_restore_jobs() + .await + .expect("list running"); + assert_eq!(running.len(), 1, "{running:?}"); + assert_eq!(running[0].target.table_name, "rr_target"); + assert_eq!(running[0].status, JobStatus::Running); + assert!( + s.engine + .list_dead_restore_jobs(std::time::Duration::ZERO) + .await + .expect("list dead") + .is_empty(), + "the age-based listing misses a heartbeat ahead of the clock" + ); + sqlx::query("UPDATE table_restores SET status = 'DONE' WHERE table_id = $1") + .bind(&target_id) + .execute(&s.catalog) + .await + .expect("settle"); + assert!( + s.engine + .list_running_restore_jobs() + .await + .expect("list running") + .is_empty() + ); + s.cleanup().await; + } +} + +// --- the store identity file, over two engines on one catalog ---------------------- + +/// The identity checks of a configured store (`extenddb_engine::store_identity`) +/// run against this catalog and real filesystem stores: two instances of one +/// catalog starting against one empty root, and two instances with +/// host-local roots under one `store_root` label. +/// The catalog methods the store-backed backup job and the reconciler call: +/// the takeover of a dead owner's row, the age listings on the database's +/// clock, the AVAILABLE write's `source_table_details_json`, the claim +/// session's settings. +mod backup_job { + use std::time::Duration; + + use extenddb_core::backup_format::BackupArn; + use extenddb_storage::BackupCatalog; + use extenddb_storage::backup_catalog::{ + BackupRow, BackupStatus, JobStatus, StatusFields, WriteOutcome, + }; + + use super::catalog_index::row as new_row; + use super::*; + + fn arn_of(row: &BackupRow) -> BackupArn { + BackupArn::parse(&row.backup_arn).unwrap() + } + + async fn stored(s: &Scratch, row: &BackupRow) -> BackupRow { + s.engine + .read_backup_row(&arn_of(row)) + .await + .expect("read") + .expect("the row exists") + } + + /// Wait until no session holds the backup's advisory lock (the server + /// releases it a moment after the holding connection closes). + async fn await_lock_free(s: &Scratch, backup_id: &str) { + let deadline = tokio::time::Instant::now() + Duration::from_secs(10); + while backup_lock_holder(s, backup_id).await.is_some() { + assert!( + tokio::time::Instant::now() < deadline, + "the advisory lock outlived its session" + ); + tokio::time::sleep(Duration::from_millis(20)).await; + } + } + + /// `claim_dead_backup`: refused while the owner's session holds the + /// lock, refused while the row's heartbeat is within the stale window + /// though the lock is free (an owner that died a moment ago), taken over + /// once the heartbeat is older than the window on the database's clock + /// (the window is crossed by waiting, not by dating the row), with a + /// fresh generation and `heartbeat_at` as the dead owner left it; the + /// successor's AVAILABLE write carries the manifest's `source_table` + /// into the row; a row that is no longer CREATING is not taken over. + #[tokio::test] + async fn claim_dead_backup_takes_over_a_stale_row_and_refuses_a_live_one() { + if base_conn().is_none() { + eprintln!( + "SKIP claim_dead_backup_takes_over_a_stale_row_and_refuses_a_live_one: no \ + PostgreSQL" + ); + return; + } + let s = scratch().await; + let window = Duration::from_secs(1); + let row = new_row(ACCOUNT, "Music", "nightly"); + let key = row.key(); + let owner = s.engine.claim_backup(&row).await.expect("claim"); + // Alive: its session holds the lock. + assert!( + s.engine + .claim_dead_backup(&key, window) + .await + .expect("takeover attempt") + .is_none(), + "a row whose owner holds the lock was taken over" + ); + assert_eq!(stored(&s, &row).await.owner_generation, owner.generation()); + // The owner's session ends (a crash): the lock goes, the heartbeat + // stays as written. Within the window the row is not yet dead. + drop(owner); + await_lock_free(&s, &row.backup_id).await; + let left = stored(&s, &row).await; + assert!( + s.engine + .claim_dead_backup(&key, window) + .await + .expect("takeover attempt") + .is_none(), + "a row whose heartbeat is within the stale window was taken over" + ); + assert_eq!( + backup_lock_holder(&s, &row.backup_id).await, + None, + "a refused takeover left its lock session behind" + ); + tokio::time::sleep(window + Duration::from_millis(300)).await; + let successor = s + .engine + .claim_dead_backup(&key, window) + .await + .expect("takeover") + .expect("a stale CREATING row is taken over"); + assert_ne!(successor.generation(), left.owner_generation); + let taken = stored(&s, &row).await; + assert_eq!(taken.status, BackupStatus::Creating); + assert_eq!(taken.owner_generation, successor.generation()); + assert_eq!( + taken.heartbeat_at, left.heartbeat_at, + "the takeover refreshed heartbeat_at" + ); + assert!( + backup_lock_holder(&s, &row.backup_id).await.is_some(), + "the successor holds the lock" + ); + // A second taker is refused by the lock while the successor lives. + assert!( + s.engine + .claim_dead_backup(&key, window) + .await + .expect("second takeover attempt") + .is_none() + ); + // The successor's writes go through: the manifest's source table + // replaces the row's, the FAILED write's `None` leaves it. + let manifest_source = serde_json::json!({"table_name": "Music", "item_count": 42}); + let fields = StatusFields { + size_bytes: Some(10), + item_count: Some(42), + source_table_details_json: Some(manifest_source.clone()), + completed: true, + ..StatusFields::default() + }; + assert_eq!( + successor + .set_status(JobStatus::Creating, JobStatus::Available, &fields) + .await + .expect("write"), + WriteOutcome::Matched + ); + let available = stored(&s, &row).await; + assert_eq!(available.status, BackupStatus::Available); + assert_eq!(available.source_table_details_json, manifest_source); + assert_eq!(available.item_count, 42); + drop(successor); + await_lock_free(&s, &row.backup_id).await; + tokio::time::sleep(window + Duration::from_millis(300)).await; + assert!( + s.engine + .claim_dead_backup(&key, window) + .await + .expect("takeover attempt") + .is_none(), + "an AVAILABLE row was taken over" + ); + // A row that is gone. + assert!( + s.engine + .claim_dead_backup(&new_row(ACCOUNT, "Music", "other").key(), window) + .await + .expect("takeover attempt") + .is_none() + ); + s.cleanup().await; + } + + /// An owner whose claim session the server ended (`pg_terminate_backend`) + /// gets `Unknown` from its next status write, not an error, and its + /// re-read shows the row as it is: the fence the job and the reconciler + /// act on. Once the heartbeat is stale the row is a successor's. + #[tokio::test] + async fn a_terminated_claim_session_reports_unknown_and_the_row_is_reread() { + if base_conn().is_none() { + eprintln!( + "SKIP a_terminated_claim_session_reports_unknown_and_the_row_is_reread: no \ + PostgreSQL" + ); + return; + } + let s = scratch().await; + let row = new_row(ACCOUNT, "Music", "nightly"); + let owner = s.engine.claim_backup(&row).await.expect("claim"); + let pid = backup_lock_holder(&s, &row.backup_id) + .await + .expect("the claim session holds the lock"); + let terminated: bool = sqlx::query_scalar("SELECT pg_terminate_backend($1)") + .bind(pid) + .fetch_one(&s.catalog) + .await + .expect("terminate"); + assert!(terminated); + await_lock_free(&s, &row.backup_id).await; + let outcome = owner + .set_status( + JobStatus::Creating, + JobStatus::Available, + &StatusFields { + size_bytes: Some(1), + completed: true, + ..StatusFields::default() + }, + ) + .await + .expect("a lost connection is an outcome, not an error"); + assert_eq!(outcome, WriteOutcome::Unknown); + // The re-read runs on the pool, not the dead session. + let read = owner.read().await.expect("re-read"); + assert_eq!(read.status, JobStatus::Creating); + assert_eq!(read.owner_generation, owner.generation()); + // The heartbeat on the dead session is an error the caller treats as + // `Lost`. + assert!(owner.heartbeat().await.is_err()); + s.cleanup().await; + } + + /// The claim session asks the server for TCP keepalives + /// (`CLAIM_KEEPALIVES`), read back from the session with `SHOW`: idle + /// 30 s, then a probe every 10 s, three unanswered, so a vanished host + /// loses its lock after about a minute (design 4(e)). The literals, not + /// the constant, so a constant at the server's defaults (7200 s idle) + /// fails here. + #[tokio::test] + async fn the_claim_session_sets_tcp_keepalives() { + if base_conn().is_none() { + eprintln!("SKIP the_claim_session_sets_tcp_keepalives: no PostgreSQL"); + return; + } + let s = scratch().await; + let names = [ + "tcp_keepalives_idle", + "tcp_keepalives_interval", + "tcp_keepalives_count", + ]; + let values = extenddb_storage_postgres::claim_session_settings( + &s.catalog, + "backup:keepalive-test", + &names, + ) + .await + .expect("open a claim session") + .expect("the lock is free"); + assert_eq!(values, ["30", "10", "3"]); + assert_eq!( + extenddb_storage_postgres::CLAIM_KEEPALIVES, + [ + ("tcp_keepalives_idle", 30), + ("tcp_keepalives_interval", 10), + ("tcp_keepalives_count", 3), + ] + ); + // A pooled connection has the server's defaults, not the claim's. + let pooled: String = sqlx::query_scalar("SHOW tcp_keepalives_idle") + .fetch_one(&s.catalog) + .await + .expect("show"); + assert_ne!( + pooled, "30", + "the test server's default is the claim's value" + ); + s.cleanup().await; + } +} + +mod store_identity { + use std::sync::Arc; + + use extenddb_backup_store::{BackupStore, FilesystemStore, byte_stream_from}; + use extenddb_core::backup_format::BackupArn; + use extenddb_engine::StoreIdentityFile; + use extenddb_engine::store_identity::{STORE_IDENTITY_KEY, StoreProblem, establish, recheck}; + use extenddb_storage::BackupCatalog; + use extenddb_storage::backup_catalog::{JobStatus, StatusFields, StoreKind}; + + use super::*; + + /// A private root: `tempfile` creates directories with the umask applied + /// to 0777, which the store refuses. + fn private_root() -> tempfile::TempDir { + use std::os::unix::fs::PermissionsExt; + let dir = tempfile::tempdir().expect("tempdir"); + std::fs::set_permissions(dir.path(), std::fs::Permissions::from_mode(0o700)) + .expect("chmod"); + dir + } + + async fn second_engine(s: &Scratch) -> PostgresEngine { + PostgresEngine::new( + &PostgresConfig { + connection_string: s.catalog_url.clone(), + pool_size: 4, + max_item_size_bytes: 400_000, + }, + REGION, + ) + .await + .expect("a second engine on the same catalog") + } + + async fn read_file(store: &dyn BackupStore) -> StoreIdentityFile { + use futures::TryStreamExt; + let bytes: Vec = store + .get(STORE_IDENTITY_KEY) + .await + .expect("the file") + .try_fold(Vec::new(), |mut acc, chunk| async move { + acc.extend_from_slice(&chunk); + Ok(acc) + }) + .await + .expect("read"); + StoreIdentityFile::parse(&bytes).expect("parses") + } + + /// Two instances of one catalog started against one empty root end with + /// one id in the file, and a row either writes carries it. + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn two_instances_on_one_empty_root_end_with_one_id() { + if base_conn().is_none() { + eprintln!("SKIP two_instances_on_one_empty_root_end_with_one_id: no PostgreSQL"); + return; + } + let s = scratch().await; + let other = second_engine(&s).await; + let root = private_root(); + let store_a: Arc = + Arc::new(FilesystemStore::open(root.path()).await.unwrap()); + let store_b: Arc = + Arc::new(FilesystemStore::open(root.path()).await.unwrap()); + let label = root.path().canonicalize().unwrap().display().to_string(); + let (a, b) = tokio::join!( + establish( + &*store_a, + StoreKind::Filesystem, + &label, + &s.engine, + "postgres", + false + ), + establish( + &*store_b, + StoreKind::Filesystem, + &label, + &other, + "postgres", + false + ), + ); + let (a, b) = (a.expect("instance a"), b.expect("instance b")); + assert_eq!(a, b, "both instances use one id"); + let file = read_file(&*store_a).await; + assert_eq!(file.store_id, a.store_id); + assert_eq!(file.catalog_id, s.engine.catalog_id().await.unwrap()); + assert_eq!(file.backend, "postgres"); + // A row written under the identity carries it, and both instances + // read it back as the root's one id. + let mut row = super::catalog_index_row(ACCOUNT, "Music", "nightly"); + row.store = a.clone(); + let claim = s.engine.claim_backup(&row).await.expect("claim"); + drop(claim); + for engine in [&s.engine, &other] { + assert_eq!( + engine + .read_store_ids(StoreKind::Filesystem, &label) + .await + .unwrap(), + std::slice::from_ref(&a.store_id) + ); + recheck(&*store_a, &a, engine, "postgres", false) + .await + .expect("still healthy"); + } + s.cleanup().await; + } + + /// Two instances of one catalog with host-local filesystem roots under + /// one `store_root` label: the second, whose root has no file while the + /// catalog has rows for the label, is unhealthy; with the first's file + /// copied to it the manifest probe marks it unhealthy, because the + /// newest AVAILABLE backup's manifest is not in its root. + #[tokio::test] + async fn host_local_roots_under_one_label_leave_the_second_instance_unhealthy() { + if base_conn().is_none() { + eprintln!( + "SKIP host_local_roots_under_one_label_leave_the_second_instance_unhealthy: \ + no PostgreSQL" + ); + return; + } + let s = scratch().await; + let other = second_engine(&s).await; + let root_a = private_root(); + let root_b = private_root(); + let store_a: Arc = + Arc::new(FilesystemStore::open(root_a.path()).await.unwrap()); + let store_b: Arc = + Arc::new(FilesystemStore::open(root_b.path()).await.unwrap()); + // One label for both: the path both hosts configured. + let label = "/var/lib/extenddb/backups"; + + // The first instance creates the file and an AVAILABLE backup with its + // manifest in its own root. + let a = establish( + &*store_a, + StoreKind::Filesystem, + label, + &s.engine, + "postgres", + false, + ) + .await + .expect("instance a"); + let mut row = super::catalog_index_row(ACCOUNT, "Music", "nightly"); + row.store = a.clone(); + let claim = s.engine.claim_backup(&row).await.expect("claim"); + let arn = BackupArn::parse(&row.backup_arn).unwrap(); + store_a + .put( + &format!("{}extenddb-manifest.json", arn.store_prefix()), + byte_stream_from(b"{}".to_vec()), + ) + .await + .expect("manifest"); + claim + .set_status( + JobStatus::Creating, + JobStatus::Available, + &StatusFields::default(), + ) + .await + .expect("available"); + recheck(&*store_a, &a, &s.engine, "postgres", false) + .await + .expect("a is healthy"); + + // The second instance: its root has no file and the catalog has rows + // for the label. + let err = establish( + &*store_b, + StoreKind::Filesystem, + label, + &other, + "postgres", + false, + ) + .await + .unwrap_err(); + assert_eq!( + err, + StoreProblem::FileMissingWithRows(vec![a.store_id.clone()]) + ); + assert!( + store_b.head(STORE_IDENTITY_KEY).await.unwrap().is_none(), + "nothing was created over the rows" + ); + + // The operator copies the first's file to the second root: the ids + // agree and the probe finds the manifest missing. + let file = read_file(&*store_a).await; + store_b + .put(STORE_IDENTITY_KEY, byte_stream_from(file.to_bytes())) + .await + .unwrap(); + let err = establish( + &*store_b, + StoreKind::Filesystem, + label, + &other, + "postgres", + false, + ) + .await + .unwrap_err(); + assert_eq!( + err, + StoreProblem::ManifestMissing { + backup_arn: row.backup_arn.clone(), + } + ); + // The first instance is unaffected. + recheck(&*store_a, &a, &s.engine, "postgres", false) + .await + .expect("a stays healthy"); + // A shared root (the second instance pointed at the first's + // directory) passes. + let shared: Arc = + Arc::new(FilesystemStore::open(root_a.path()).await.unwrap()); + let b = establish( + &*shared, + StoreKind::Filesystem, + label, + &other, + "postgres", + false, + ) + .await + .expect("a shared root is healthy"); + assert_eq!(b, a); + s.cleanup().await; + } + + /// A root whose file names another catalog is unhealthy for this one + /// until an operator changes the file; the first catalog's instance is + /// unaffected. + #[tokio::test] + async fn a_root_owned_by_another_catalog_is_unhealthy() { + if base_conn().is_none() { + eprintln!("SKIP a_root_owned_by_another_catalog_is_unhealthy: no PostgreSQL"); + return; + } + let s = scratch().await; + let root = private_root(); + let store: Arc = + Arc::new(FilesystemStore::open(root.path()).await.unwrap()); + let label = root.path().canonicalize().unwrap().display().to_string(); + let foreign = StoreIdentityFile { + format_version: 1, + store_id: "s-foreign".to_owned(), + catalog_id: "another-catalog".to_owned(), + backend: "sqlite".to_owned(), + created_at_ms: 0, + ephemeral_catalog: false, + }; + store + .put(STORE_IDENTITY_KEY, byte_stream_from(foreign.to_bytes())) + .await + .unwrap(); + let err = establish( + &*store, + StoreKind::Filesystem, + &label, + &s.engine, + "postgres", + false, + ) + .await + .unwrap_err(); + assert_eq!( + err, + StoreProblem::ForeignCatalog { + file_catalog_id: "another-catalog".to_owned(), + catalog_id: s.engine.catalog_id().await.unwrap(), + } + ); + s.cleanup().await; + } +} + +/// The reconciler's fencing (design 4(e)): the FAILED write conditioned on +/// the listed claim, the ACTIVE flip guarded by the record, and a record this +/// build cannot read skipped by the listing. +mod reconciler_fencing { + use extenddb_storage::BackupCatalog; + use extenddb_storage::backup_catalog::{JobStatus, RestoreKind, RestoreTargetKey}; + + use super::*; + + const STALE: std::time::Duration = std::time::Duration::from_secs(300); + + /// The record's `(status, owner_generation, heartbeat_at)`, the heartbeat + /// as the listing reports it. + async fn record_claim(s: &Scratch, table_id: &str) -> (String, u64, f64) { + let (status, generation, heartbeat): (String, i64, f64) = sqlx::query_as( + "SELECT status, owner_generation, EXTRACT(EPOCH FROM heartbeat_at)::FLOAT8 \ + FROM table_restores WHERE table_id = $1", + ) + .bind(table_id) + .fetch_one(&s.catalog) + .await + .expect("the record"); + (status, u64::try_from(generation).unwrap(), heartbeat) + } + + fn key_of(desc: &TableDescription) -> RestoreTargetKey { + RestoreTargetKey { + account_id: ACCOUNT.to_owned(), + table_name: desc.table_name.clone(), + table_id: desc.table_id.clone(), + } + } + + /// The state a stopped process leaves: the target CREATING, the record + /// RUNNING with a heartbeat `age` old. + async fn crash(s: &Scratch, table_id: &str, age: &str) { + sqlx::query( + "UPDATE tables SET table_status = 'CREATING', status_transition_at = NULL \ + WHERE table_id = $1", + ) + .bind(table_id) + .execute(&s.catalog) + .await + .expect("crash"); + sqlx::query(&format!( + "UPDATE table_restores SET status = 'RUNNING', completed_at = NULL, \ + heartbeat_at = now() - interval '{age}' WHERE table_id = $1" + )) + .bind(table_id) + .execute(&s.catalog) + .await + .expect("age"); + } + + /// `fail_restore_job` is conditioned on the `owner_generation` and + /// `heartbeat_at` the listing read: a heartbeat that lands between the + /// listing and the write (the owner is alive after all; measured by the + /// reviewer with two servers on one catalog, where an 11.6 s old heartbeat + /// was overwritten and the live target removed) makes the write match + /// nothing, as does a generation that is not the record's; the record + /// stays RUNNING under its owner's generation. With the listed values + /// current, the write lands and sets a fresh generation. + #[tokio::test] + async fn the_failed_write_matches_only_the_listed_generation_and_heartbeat() { + if base_conn().is_none() { + eprintln!( + "SKIP the_failed_write_matches_only_the_listed_generation_and_heartbeat: no \ + PostgreSQL" + ); + return; + } + let s = scratch().await; + indexed_source(&s, "fw_src").await; + let backup = s + .engine + .create(ACCOUNT, "fw_src", "b", JobPermit::detached()) + .await + .expect("backup"); + let desc = restore(&s, "fw_dst", &backup.backup_arn) + .await + .expect("restore"); + let key = key_of(&desc); + crash(&s, &desc.table_id, "10 minutes").await; + let dead = s.engine.list_dead_restore_jobs(STALE).await.expect("list"); + assert_eq!(dead.len(), 1, "{dead:?}"); + let listed = dead[0].clone(); + let (_, generation, heartbeat) = record_claim(&s, &desc.table_id).await; + assert_eq!( + (listed.owner_generation, listed.heartbeat_at), + (generation, heartbeat) + ); + + // The owner's heartbeat lands after the listing. + sqlx::query("UPDATE table_restores SET heartbeat_at = now() WHERE table_id = $1") + .bind(&desc.table_id) + .execute(&s.catalog) + .await + .expect("heartbeat"); + assert!( + !s.engine + .fail_restore_job( + &key, + listed.owner_generation, + listed.heartbeat_at, + "owner dead" + ) + .await + .expect("fail"), + "a heartbeat after the listing keeps the record its owner's" + ); + let (status, after, current_heartbeat) = record_claim(&s, &desc.table_id).await; + assert_eq!(status, "RUNNING"); + assert_eq!(after, generation, "the generation is untouched"); + assert!( + s.engine + .list_dead_restore_jobs(STALE) + .await + .expect("list") + .is_empty(), + "a record with a fresh heartbeat is not dead" + ); + // Another generation with the current heartbeat: no match either. + assert!( + !s.engine + .fail_restore_job(&key, generation + 1, current_heartbeat, "owner dead") + .await + .expect("fail") + ); + assert_eq!(record_claim(&s, &desc.table_id).await.0, "RUNNING"); + + // Listed values that are still current: the write lands, under a + // fresh generation. + crash(&s, &desc.table_id, "10 minutes").await; + let dead = s.engine.list_dead_restore_jobs(STALE).await.expect("list"); + assert_eq!(dead.len(), 1); + assert!( + s.engine + .fail_restore_job( + &dead[0].target, + dead[0].owner_generation, + dead[0].heartbeat_at, + "owner dead" + ) + .await + .expect("fail") + ); + let (status, fenced, _) = record_claim(&s, &desc.table_id).await; + assert_eq!(status, "FAILED"); + assert_ne!( + fenced, dead[0].owner_generation, + "the FAILED write carries a fresh generation" + ); + assert_ne!(fenced, extenddb_storage::backup_catalog::UNOWNED_GENERATION); + assert!(s.engine.remove_restore_target(&key).await.expect("remove")); + s.cleanup().await; + } + + /// The ACTIVE flip is guarded by the job record: a record failed (by the + /// reconciler's write, with a fresh generation) after the copy's last + /// batch and before the flip leaves the target CREATING under the FAILED + /// record, with every item written, and the copy's own FAILED write and + /// removal match nothing; the FAILED-target duty then lists the pair once + /// stale and removes the target. The copy is held between its last batch + /// and the flip by a row lock on its record, which the reconciler's write + /// is made under; without the `done == 1` guard the flip would make a + /// table ACTIVE whose record says FAILED. + #[tokio::test] + async fn a_record_failed_after_the_last_batch_leaves_the_target_to_the_reconciler() { + if base_conn().is_none() { + eprintln!( + "SKIP a_record_failed_after_the_last_batch_leaves_the_target_to_the_reconciler: \ + no PostgreSQL" + ); + return; + } + let s = scratch().await; + indexed_source(&s, "lb_src").await; + let backup = s + .engine + .create(ACCOUNT, "lb_src", "b", JobPermit::detached()) + .await + .expect("backup"); + let gate = gate_copies(&s).await; + let desc = start_restore(&s, "lb_dst", &backup.backup_arn) + .await + .expect("registers"); + await_copies_waiting(&s, 1).await; + let key = key_of(&desc); + let (_, generation, _) = record_claim(&s, &desc.table_id).await; + + // Hold the record's row: the copy's DONE write after its last batch + // waits on it. Then let the copy run its batches. + let mut holder = s.catalog.begin().await.expect("begin"); + sqlx::query("SELECT 1 FROM table_restores WHERE table_id = $1 FOR UPDATE") + .bind(&desc.table_id) + .execute(&mut *holder) + .await + .expect("lock the record"); + gate.commit().await.expect("release the copy"); + let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(30); + loop { + let waiting: i64 = sqlx::query_scalar( + "SELECT count(*) FROM pg_stat_activity WHERE datname = current_database() \ + AND wait_event_type = 'Lock' \ + AND query LIKE 'UPDATE table_restores SET status = ''DONE''%'", + ) + .fetch_one(&s.catalog) + .await + .expect("waiters"); + if waiting == 1 { + break; + } + assert!( + tokio::time::Instant::now() < deadline, + "the copy did not reach its DONE write" + ); + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + } + assert_eq!( + data_rows(&s, &desc.table_id).await, + 30, + "every batch was written before the flip" + ); + + // The reconciler's write, under the row lock, then the lock released: + // the copy's DONE write re-evaluates against the FAILED row and + // matches nothing. + let fenced = extenddb_storage::backup_catalog::new_generation(); + sqlx::query( + "UPDATE table_restores SET status = 'FAILED', failure_reason = 'owner dead', \ + completed_at = now(), owner_generation = $2 WHERE table_id = $1 \ + AND status = 'RUNNING' AND owner_generation = $3", + ) + .bind(&desc.table_id) + .bind(i64::try_from(fenced).unwrap()) + .bind(i64::try_from(generation).unwrap()) + .execute(&mut *holder) + .await + .expect("fail the record"); + holder.commit().await.expect("release the record"); + + // The copy stops: its claim session ends without flipping the table. + let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(30); + while restore_lock_holder(&s, ACCOUNT, "lb_dst").await.is_some() { + assert!( + tokio::time::Instant::now() < deadline, + "the copy did not end after its record was failed" + ); + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + } + assert_eq!( + table_status(&s, "lb_dst").await.as_deref(), + Some("CREATING"), + "a record that is not RUNNING under the copy's generation does not flip the table" + ); + let (status, after, _) = record_claim(&s, &desc.table_id).await; + assert_eq!( + status, "FAILED", + "the copy's own FAILED write matched nothing" + ); + assert_eq!( + after, fenced, + "the record carries the reconciler's generation" + ); + assert_eq!(data_rows(&s, &desc.table_id).await, 30); + + // The FAILED-target duty: listed once the record is stale, removed. + sqlx::query( + "UPDATE table_restores SET completed_at = now() - interval '10 minutes' \ + WHERE table_id = $1", + ) + .bind(&desc.table_id) + .execute(&s.catalog) + .await + .expect("age"); + let dead = s.engine.list_dead_restore_jobs(STALE).await.expect("list"); + assert_eq!(dead.len(), 1, "{dead:?}"); + assert_eq!(dead[0].status, JobStatus::Failed); + assert_eq!(dead[0].target, key); + assert!(s.engine.remove_restore_target(&key).await.expect("remove")); + assert_eq!(table_status(&s, "lb_dst").await, None); + assert!(!data_table_exists(&s, &desc.table_id).await); + s.engine + .delete(ACCOUNT, &backup.backup_arn) + .await + .expect("the backup is free"); + s.cleanup().await; + } + + /// A record this build cannot read (a `kind` no release writes; the + /// reviewer planted `manifest-v2` and every cycle failed) is logged and + /// skipped by the dead listing, so the readable dead record beside it is + /// listed and the cycle's other duties run. + #[tokio::test] + async fn an_unreadable_record_is_skipped_by_the_dead_listing() { + if base_conn().is_none() { + eprintln!("SKIP an_unreadable_record_is_skipped_by_the_dead_listing: no PostgreSQL"); + return; + } + let s = scratch().await; + indexed_source(&s, "ur_src").await; + let backup = s + .engine + .create(ACCOUNT, "ur_src", "b", JobPermit::detached()) + .await + .expect("backup"); + let unreadable = restore(&s, "ur_unreadable", &backup.backup_arn) + .await + .expect("restore"); + let readable = restore(&s, "ur_readable", &backup.backup_arn) + .await + .expect("restore"); + for (desc, kind) in [(&unreadable, "manifest-v2"), (&readable, "legacy")] { + crash(&s, &desc.table_id, "10 minutes").await; + sqlx::query("UPDATE table_restores SET kind = $2 WHERE table_id = $1") + .bind(&desc.table_id) + .bind(kind) + .execute(&s.catalog) + .await + .expect("kind"); + } + let dead = s + .engine + .list_dead_restore_jobs(STALE) + .await + .expect("one unreadable row does not fail the listing"); + assert_eq!(dead.len(), 1, "{dead:?}"); + assert_eq!(dead[0].target, key_of(&readable)); + assert_eq!(dead[0].kind, RestoreKind::Legacy); + s.cleanup().await; + } +} diff --git a/crates/storage-postgres/tests/migrations_fresh.rs b/crates/storage-postgres/tests/migrations_fresh.rs index c088c8d84..9cbea0c8f 100644 --- a/crates/storage-postgres/tests/migrations_fresh.rs +++ b/crates/storage-postgres/tests/migrations_fresh.rs @@ -99,8 +99,11 @@ async fn a_fresh_install_is_current_and_a_second_run_changes_nothing() { settings_rows.get("data_database_connection_string"), Some(&s.data_url()) ); - // No baseline seeds a setting: the version and the connection rows only. - assert_eq!(settings_rows.len(), 3, "{settings_rows:?}"); + // No baseline seeds a setting: the version, the connection rows, and the + // catalog id migration 6 writes once. + assert_eq!(settings_rows.len(), 4, "{settings_rows:?}"); + let catalog_id = settings_rows.get("catalog_id").expect("a catalog id"); + assert!(uuid::Uuid::parse_str(catalog_id).is_ok(), "{catalog_id}"); assert_eq!( legacy_rows(&catalog).await, [ diff --git a/crates/storage-postgres/tests/migrations_serving.rs b/crates/storage-postgres/tests/migrations_serving.rs index e3697db51..8a3b933a7 100644 --- a/crates/storage-postgres/tests/migrations_serving.rs +++ b/crates/storage-postgres/tests/migrations_serving.rs @@ -13,7 +13,7 @@ mod migrations_support; use std::time::Duration; use extenddb_storage::hooks::{Reregistration, ServingRegistrar}; -use extenddb_storage::migrations::{CatalogStatus, MigrationId, Runner}; +use extenddb_storage::migrations::{CatalogStatus, MigrationError, MigrationId, Runner}; use extenddb_storage_postgres::migrations::{ MigrationLock, PostgresMigrations, ServerRegistration, migrate, }; @@ -309,3 +309,64 @@ async fn an_attempt_dropped_while_its_status_read_waits_leaves_no_key() { assert_eq!(held, [], "a dropped attempt left the serving key held"); admin.close().await; } + +/// A command that opens the catalog as a server does registers under its own +/// name: the serving session's `application_name` is the command's, a +/// migration that waits for the registrations names it, a registration +/// taken again keeps it, and a registration refused while a migration holds +/// the catalog names the migration without telling the operator to start a +/// server. +#[tokio::test] +async fn a_command_registers_under_its_own_name() { + let Some(base) = base_conn() else { + return skip("a command's registration"); + }; + let s = installed(&base, "pgs_command").await; + let admin = pool(&format!("{base}/postgres"), 1).await; + let registration = + ServerRegistration::register_as_with(&s.catalog_url(), "extenddb backup list", KEEPALIVE) + .await + .expect("register"); + let sessions = serving_sessions(&admin, &s).await; + assert_eq!(sessions.len(), 1, "{sessions:?}"); + assert_eq!(sessions[0].1, "extenddb backup list"); + let err = migrate(&s.target(), RELEASE, Duration::from_secs(1)) + .await + .expect_err("a migration while the command is registered"); + assert_eq!(err.exit_code(), 6, "{err}"); + assert!( + err.to_string().contains("\"extenddb backup list\""), + "the migration names the command: {err}" + ); + + end_the_serving_session(&admin, &s).await; + lost(®istration).await; + assert_eq!( + registration.register_again().await, + Reregistration::Registered + ); + let sessions = serving_sessions(&admin, &s).await; + assert_eq!(sessions.len(), 1, "{sessions:?}"); + assert_eq!(sessions[0].1, "extenddb backup list", "taken again"); + drop(registration); + + let lock = MigrationLock::acquire(&s.options(), Duration::from_secs(5)) + .await + .expect("the migration lock, once the command's key is gone"); + match ServerRegistration::register_as(&s.catalog_url(), "extenddb backup purge").await { + Err(e @ MigrationError::Locked(_)) => { + assert_eq!(e.exit_code(), 6, "{e}"); + assert!( + e.to_string() + .contains("a migration is running on this catalog") + && !e.to_string().contains("start the server"), + "{e}" + ); + } + Err(other) => panic!("refused otherwise: {other}"), + Ok(_) => panic!("registered beside a migration"), + } + assert_eq!(serving_sessions(&admin, &s).await, []); + lock.release().await; + admin.close().await; +} diff --git a/crates/storage-postgres/tests/restore_job.rs b/crates/storage-postgres/tests/restore_job.rs new file mode 100644 index 000000000..703c265fa --- /dev/null +++ b/crates/storage-postgres/tests/restore_job.rs @@ -0,0 +1,1797 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 +//! Storage-level tests of the store-backed restore on PostgreSQL: the +//! registration under the `backup_index` row's lock (`claim_restore`), the +//! takeover of a dead record (`claim_restore_dead`), the per-file load with +//! its progress row (`bulk_load`), the ACTIVE flip (`finish_restore`), the +//! counts and the progress sweep. The engine's restore job drives these +//! through `BackupCatalog` and `RestoreEngine`; what each must guarantee on +//! this backend is what a test here pins. +//! +//! Each test builds a throwaway catalog and data database and drops them +//! when it ends, passing or failing, as `backup_restore.rs` does; the tests +//! run one at a time. Requires `EXTENDDB_TEST_PG_CONNECTION_STRING`; +//! without it every test reports a skip and passes, unless +//! `EXTENDDB_TEST_PG_REQUIRED` says the job must reach a server. + +use std::time::Duration; + +use extenddb_core::types::{ + CreateTableInput, DistanceFunction, Item, Projection, ProjectionType, TableDescription, + TableStatus, VectorAttribute, VectorIndexSpecification, +}; +use extenddb_storage::backup_catalog::{ + BackupRow, BackupStatus, DeleteStart, Heartbeat, JobClaim, JobStatus, RestoreJobRow, + RestoreKind, RestoreRegistrationRow, RestoreTargetKey, StatusFields, StoreIdentity, StoreKind, + WriteOutcome, backup_in_use_message, +}; +use extenddb_storage::error::StorageError; +use extenddb_storage::migrations::Runner; +use extenddb_storage::restore_engine::{BulkLoadFile, Finished, RestoreEngine}; +use extenddb_storage::{BackupCatalog, TableEngine}; +use extenddb_storage_postgres::RESTORE_LOCK_SEED; +use extenddb_storage_postgres::migrations::{PostgresMigrations, PostgresTarget}; +use extenddb_storage_postgres::{PostgresConfig, PostgresEngine}; +use sqlx::PgPool; +use sqlx::postgres::PgPoolOptions; + +const ACCOUNT: &str = "123456789012"; +const REGION: &str = "us-east-1"; +/// A stale window no test's heartbeat crosses on its own. +const STALE: Duration = Duration::from_secs(300); + +// --- the scratch databases -------------------------------------------------------- + +/// A catalog database and a separate data database, the layout `extenddb +/// init` creates, with pgvector installed in the data database when the +/// server offers it. +struct Scratch { + engine: PostgresEngine, + /// The data database. + db: PgPool, + catalog: PgPool, + admin: PgPool, + db_names: [String; 2], + vector: bool, + _guard: DbGuard, + _serial: tokio::sync::MutexGuard<'static, ()>, +} + +/// Drops the scratch databases if the test panics, from the moment they are +/// created, including during setup. +struct DbGuard { + names: Vec, +} + +/// One test at a time: each opens its own pools and claim sessions. +static SERIAL: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); + +impl Scratch { + async fn cleanup(self) { + self.db.close().await; + self.catalog.close().await; + drop_databases(&self.admin, &self.db_names).await; + self.admin.close().await; + } +} + +async fn drop_databases(admin: &PgPool, names: &[String]) { + for name in names { + let _ = sqlx::query(&format!("DROP DATABASE IF EXISTS \"{name}\" WITH (FORCE)")) + .execute(admin) + .await; + } +} + +impl Drop for DbGuard { + fn drop(&mut self) { + if !std::thread::panicking() { + return; + } + let (Some(base), names) = (base_conn(), self.names.clone()) else { + return; + }; + let _ = std::thread::spawn(move || { + let Ok(rt) = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + else { + return; + }; + rt.block_on(async { + if let Ok(admin) = PgPoolOptions::new() + .max_connections(1) + .connect(&format!("{base}/postgres")) + .await + { + drop_databases(&admin, &names).await; + admin.close().await; + } + }); + }) + .join(); + } +} + +/// The server's base URL, or `None` to skip. With `EXTENDDB_TEST_PG_REQUIRED` +/// set to any value but `0`, a job that must reach a server fails here +/// instead. +fn base_conn() -> Option { + let conn = std::env::var("EXTENDDB_TEST_PG_CONNECTION_STRING").unwrap_or_default(); + if conn.trim().is_empty() { + let required = + std::env::var("EXTENDDB_TEST_PG_REQUIRED").is_ok_and(|v| !v.is_empty() && v != "0"); + assert!( + !required, + "EXTENDDB_TEST_PG_CONNECTION_STRING is not set, and EXTENDDB_TEST_PG_REQUIRED says \ + this job must reach a PostgreSQL server" + ); + return None; + } + Some(conn.trim_end_matches('/').to_owned()) +} + +/// `true` when the test can run; prints the skip line otherwise. +fn pg_available(test: &str) -> bool { + if base_conn().is_none() { + eprintln!("SKIP {test}: no PostgreSQL"); + return false; + } + true +} + +async fn connect(url: &str) -> PgPool { + PgPoolOptions::new() + .max_connections(3) + .connect(url) + .await + .expect("connect to a scratch database") +} + +async fn scratch() -> Scratch { + let serial = SERIAL.lock().await; + let base = base_conn().expect("caller checks base_conn() first"); + let stem = format!("eddb_rstr_{}", uuid::Uuid::new_v4().simple())[..24].to_owned(); + let (catalog_name, data_name) = (format!("{stem}_c"), format!("{stem}_d")); + let admin = PgPoolOptions::new() + .max_connections(1) + .connect(&format!("{base}/postgres")) + .await + .expect("connect to the postgres maintenance database"); + let guard = DbGuard { + names: vec![catalog_name.clone(), data_name.clone()], + }; + for name in [&catalog_name, &data_name] { + sqlx::query(&format!("CREATE DATABASE \"{name}\"")) + .execute(&admin) + .await + .expect("create a scratch database"); + } + let catalog_url = format!("{base}/{catalog_name}"); + let data_url = format!("{base}/{data_name}"); + // A fresh install, migrated by the runner as `init` does: migration 7 + // creates `restore_progress` in the data database. + let target = PostgresTarget::new(catalog_url.parse().expect("a postgres URL"), &data_name) + .with_data_connection_row(&data_url); + let runner = PostgresMigrations::open_unlocked(&target) + .await + .expect("open the migration runner"); + Runner::new(&runner, "test") + .migrate() + .await + .expect("migrate the scratch databases"); + runner.close().await; + let catalog = connect(&catalog_url).await; + let db = connect(&data_url).await; + let vector: bool = sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM pg_available_extensions WHERE name = 'vector')", + ) + .fetch_one(&db) + .await + .expect("read the available extensions"); + if vector { + sqlx::query("CREATE EXTENSION IF NOT EXISTS vector") + .execute(&db) + .await + .expect("create the pgvector extension"); + } else { + eprintln!("NOTE: no pgvector on this server; the vector index cells are skipped"); + } + for key in ["control_plane_delay_seconds", "index_propagation_delay_ms"] { + sqlx::query( + "INSERT INTO settings (key, value) VALUES ($1, '0') \ + ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value", + ) + .bind(key) + .execute(&catalog) + .await + .expect("set a delay to zero"); + } + sqlx::query( + "INSERT INTO settings (key, value) VALUES ('data_database_connection_string', $1) \ + ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value", + ) + .bind(&data_url) + .execute(&catalog) + .await + .expect("point the catalog at the data database"); + sqlx::query("INSERT INTO accounts (account_id, account_name) VALUES ($1, $2)") + .bind(ACCOUNT) + .bind(format!("acct-{stem}")) + .execute(&catalog) + .await + .expect("seed the account row"); + let engine = PostgresEngine::new( + &PostgresConfig { + connection_string: catalog_url.clone(), + pool_size: 6, + max_item_size_bytes: 400_000, + }, + REGION, + ) + .await + .expect("open a PostgresEngine on the scratch databases"); + Scratch { + engine, + db, + catalog, + admin, + db_names: [catalog_name, data_name], + vector, + _guard: guard, + _serial: serial, + } +} + +// --- rows, inputs and items ----------------------------------------------------- + +fn store() -> StoreIdentity { + StoreIdentity { + store_kind: StoreKind::S3, + store_root: "s3://bucket/extenddb".to_owned(), + store_id: "store-1".to_owned(), + } +} + +/// An AVAILABLE `backup_index` row of a backup of `table`, written through +/// the claim as a backup job writes it. +async fn available_backup(s: &Scratch, table: &str) -> BackupRow { + let backup_id = extenddb_storage::util::new_backup_id(); + let backup_arn = extenddb_storage::util::backup_arn(REGION, ACCOUNT, table, &backup_id); + let manifest: serde_json::Value = serde_json::from_str(include_str!( + "../../core/testdata/extenddb-manifest/v1.json" + )) + .expect("fixture"); + let row = BackupRow { + backup_id: backup_id.clone(), + account_id: ACCOUNT.to_owned(), + table_name: table.to_owned(), + backup_arn: backup_arn.clone(), + backup_name: "nightly".to_owned(), + status: BackupStatus::Creating, + store: store(), + size_bytes: 0, + item_count: 0, + created_at: extenddb_storage::util::backup_creation_date_time(&backup_id).unwrap(), + completed_at: None, + failure_reason: None, + source_table_details_json: manifest["source_table"].clone(), + owner_generation: 0, + heartbeat_at: 0.0, + }; + let claim = s.engine.claim_backup(&row).await.expect("claim the row"); + let written = claim + .set_status( + JobStatus::Creating, + JobStatus::Available, + &StatusFields { + item_count: Some(3), + size_bytes: Some(1234), + completed: true, + ..StatusFields::default() + }, + ) + .await + .expect("the AVAILABLE write"); + assert_eq!(written, WriteOutcome::Matched); + drop(claim); + BackupRow { + status: BackupStatus::Available, + ..row + } +} + +fn registration(row: &BackupRow, spool_host: Option<&str>) -> RestoreRegistrationRow { + RestoreRegistrationRow { + backup: row.key(), + source_table_arn: format!( + "arn:aws:dynamodb:{REGION}:{ACCOUNT}:table/{}", + row.table_name + ), + restore_date_time: row.created_at, + store: row.store.clone(), + spool_host: spool_host.map(str::to_owned), + owner_generation: 0, + } +} + +/// The target: a composite-key table with a GSI on `g`, an LSI on `l`, and +/// a vector index on `emb` (three dimensions) when the server has pgvector. +fn target_input(name: &str, vector: bool) -> CreateTableInput { + let mut input: CreateTableInput = serde_json::from_value(serde_json::json!({ + "TableName": name, + "KeySchema": [ + {"AttributeName": "pk", "KeyType": "HASH"}, + {"AttributeName": "sk", "KeyType": "RANGE"} + ], + "AttributeDefinitions": [ + {"AttributeName": "pk", "AttributeType": "S"}, + {"AttributeName": "sk", "AttributeType": "N"}, + {"AttributeName": "g", "AttributeType": "S"}, + {"AttributeName": "l", "AttributeType": "S"} + ], + "BillingMode": "PAY_PER_REQUEST", + "GlobalSecondaryIndexes": [{ + "IndexName": "gix", + "KeySchema": [{"AttributeName": "g", "KeyType": "HASH"}], + "Projection": {"ProjectionType": "KEYS_ONLY"} + }], + "LocalSecondaryIndexes": [{ + "IndexName": "lix", + "KeySchema": [ + {"AttributeName": "pk", "KeyType": "HASH"}, + {"AttributeName": "l", "KeyType": "RANGE"} + ], + "Projection": {"ProjectionType": "ALL"} + }] + })) + .expect("input"); + if vector { + input.vector_indexes = Some(vec![VectorIndexSpecification { + index_name: "vix".to_owned(), + dimensions: 3, + distance_function: DistanceFunction::Cosine, + vector_attribute: VectorAttribute { + attribute_name: "emb".to_owned(), + }, + search_schema: None, + projection: Some(Projection { + projection_type: ProjectionType::All, + non_key_attributes: None, + }), + }]); + } + input +} + +/// Item `i` of the fixture: every item has `pk` and `sk`, every second one +/// `g` (the GSI's key), every third one `l` (the LSI's), every fifth one +/// `emb` (the vector). +fn item(i: usize) -> Item { + let mut item = + serde_json::json!({"pk": {"S": format!("p{}", i % 7)}, "sk": {"N": i.to_string()}}); + if i.is_multiple_of(2) { + item["g"] = serde_json::json!({"S": format!("g{}", i % 4)}); + } + if i.is_multiple_of(3) { + item["l"] = serde_json::json!({"S": format!("l{i}")}); + } + if i.is_multiple_of(5) { + item["emb"] = serde_json::json!({"L": [{"N": "0.1"}, {"N": "0.2"}, {"N": "0.3"}]}); + } + serde_json::from_value(item).expect("item") +} + +fn items(range: std::ops::Range) -> Vec { + range.map(item).collect() +} + +fn stream(items: Vec) -> futures::stream::BoxStream<'static, Result> { + Box::pin(futures::stream::iter(items.into_iter().map(Ok))) +} + +fn key_of(desc: &TableDescription) -> RestoreTargetKey { + RestoreTargetKey { + account_id: ACCOUNT.to_owned(), + table_name: desc.table_name.clone(), + table_id: desc.table_id.clone(), + } +} + +/// Register a restore of `row` into `target` and return the description, its +/// key and the claim. +async fn register( + s: &Scratch, + row: &BackupRow, + target: &str, + spool_host: Option<&str>, +) -> (TableDescription, RestoreTargetKey, Box) { + let (desc, claim) = s + .engine + .claim_restore( + ACCOUNT, + target_input(target, s.vector), + ®istration(row, spool_host), + ) + .await + .expect("register the restore"); + let key = key_of(&desc); + (desc, key, claim) +} + +// --- reads -------------------------------------------------------------------- + +async fn table_status(s: &Scratch, name: &str) -> Option { + sqlx::query_scalar("SELECT table_status FROM tables WHERE account_id = $1 AND table_name = $2") + .bind(ACCOUNT) + .bind(name) + .fetch_optional(&s.catalog) + .await + .expect("table status") +} + +async fn record(s: &Scratch, key: &RestoreTargetKey) -> RestoreJobRow { + s.engine + .read_restore_job(key) + .await + .expect("read the record") + .expect("the record exists") +} + +/// `(status, kind, owner_generation, heartbeat_at, spool_host, completed_at)` +/// of the record, straight from the table. +async fn record_columns( + s: &Scratch, + table_id: &str, +) -> (String, String, i64, f64, Option, Option) { + sqlx::query_as( + "SELECT status, kind, owner_generation, EXTRACT(EPOCH FROM heartbeat_at)::FLOAT8, \ + spool_host, EXTRACT(EPOCH FROM completed_at)::FLOAT8 FROM table_restores \ + WHERE table_id = $1", + ) + .bind(table_id) + .fetch_one(&s.catalog) + .await + .expect("read table_restores") +} + +async fn rows_in(db: &PgPool, physical: &str) -> i64 { + sqlx::query_scalar(&format!("SELECT count(*) FROM {physical}")) + .fetch_one(db) + .await + .expect("count rows") +} + +async fn base_rows(s: &Scratch, table_id: &str) -> i64 { + rows_in(&s.db, &format!("\"_ddb_{table_id}\"")).await +} + +/// `(index_name, index_id, index_status)` of the table's secondary indexes. +async fn indexes(s: &Scratch, table_id: &str) -> Vec<(String, String, String)> { + sqlx::query_as( + "SELECT index_name, index_id, index_status FROM indexes WHERE table_id = $1 \ + ORDER BY index_name", + ) + .bind(table_id) + .fetch_all(&s.catalog) + .await + .expect("read indexes") +} + +async fn index_rows(s: &Scratch, table_id: &str, name: &str) -> i64 { + let id = indexes(s, table_id) + .await + .into_iter() + .find(|(n, _, _)| n == name) + .map(|(_, id, _)| id) + .expect("the index exists"); + rows_in(&s.db, &format!("\"_ddb_{id}\"")).await +} + +/// `(index_id, index_status)` of the table's vector indexes. +async fn vector_indexes(s: &Scratch, table_id: &str) -> Vec<(String, String)> { + sqlx::query_as("SELECT index_id, index_status FROM vector_indexes WHERE table_id = $1") + .bind(table_id) + .fetch_all(&s.catalog) + .await + .expect("read vector_indexes") +} + +async fn vector_rows(s: &Scratch, table_id: &str) -> i64 { + let (id, _) = vector_indexes(s, table_id) + .await + .pop() + .expect("the vector index exists"); + rows_in(&s.db, &format!("\"_ddb_vec_{id}\"")).await +} + +/// Whether the data database has a relation named `relation` (the physical +/// name without quotes), by `to_regclass`. +async fn relation_exists(s: &Scratch, relation: &str) -> bool { + let regclass: Option = sqlx::query_scalar("SELECT to_regclass($1)::TEXT") + .bind(format!("\"{relation}\"")) + .fetch_one(&s.db) + .await + .expect("probe the relation"); + regclass.is_some() +} + +async fn progress_rows(s: &Scratch, table_id: &str) -> Vec<(i64, i64)> { + sqlx::query_as( + "SELECT file_index, items_applied FROM restore_progress WHERE table_id = $1 \ + ORDER BY file_index", + ) + .bind(table_id) + .fetch_all(&s.db) + .await + .expect("read restore_progress") +} + +/// The backend pid holding the restore lock on `table`, if any. +async fn lock_holder(s: &Scratch, table: &str) -> Option { + sqlx::query_scalar( + "SELECT pid FROM pg_locks WHERE locktype = 'advisory' AND granted \ + AND database = (SELECT oid FROM pg_database WHERE datname = current_database()) \ + AND ((classid::bigint << 32) | objid::bigint) = hashtextextended($1, $2)", + ) + .bind(format!("{ACCOUNT}/{table}")) + .bind(RESTORE_LOCK_SEED) + .fetch_optional(&s.catalog) + .await + .expect("find the advisory lock holder") +} + +/// Date the record's heartbeat `age` into the past by hand. This proves the +/// takeover's predicate on a stale row, not the window boundary on the +/// database's clock, which `abandoned_restore_is_removed_only_when_its_heartbeat_is_stale` +/// in `backup_restore.rs` crosses. +async fn age_heartbeat(s: &Scratch, table_id: &str, age: Duration) -> f64 { + sqlx::query_scalar( + "UPDATE table_restores SET heartbeat_at = now() - make_interval(secs => $2) \ + WHERE table_id = $1 RETURNING EXTRACT(EPOCH FROM heartbeat_at)::FLOAT8", + ) + .bind(table_id) + .bind(age.as_secs_f64()) + .fetch_one(&s.catalog) + .await + .expect("age the heartbeat") +} + +// --- the registration ----------------------------------------------------------- + +/// `claim_restore` creates the CREATING target with no scheduled transition +/// and its RUNNING `manifest` record carrying the store, the spool host and +/// the claim's generation, in one transaction; the description carries the +/// `RestoreSummary`; the record reads back through `read_restore_job` and +/// `list_restore_jobs`; the claim holds the name's advisory lock and its +/// heartbeat confirms. +#[tokio::test] +async fn claim_restore_registers_the_target_with_its_record() { + if !pg_available("claim_restore_registers_the_target_with_its_record") { + return; + } + let s = scratch().await; + let row = available_backup(&s, "Music").await; + assert_eq!(lock_holder(&s, "Restored").await, None); + let (desc, key, claim) = register(&s, &row, "Restored", Some("host-a:/spool")).await; + assert_eq!(desc.table_status, TableStatus::Creating); + let summary = desc.restore_summary.as_ref().expect("a RestoreSummary"); + assert_eq!( + summary.source_backup_arn.as_deref(), + Some(row.backup_arn.as_str()) + ); + assert_eq!( + summary.source_table_arn.as_deref(), + Some(format!("arn:aws:dynamodb:{REGION}:{ACCOUNT}:table/Music").as_str()) + ); + assert_eq!(summary.restore_date_time, row.created_at); + assert!(summary.restore_in_progress); + assert!( + lock_holder(&s, "Restored").await.is_some(), + "the claim holds the name lock" + ); + + let (status, kind, generation, heartbeat_at, spool_host, completed_at) = + record_columns(&s, &key.table_id).await; + assert_eq!((status.as_str(), kind.as_str()), ("RUNNING", "manifest")); + assert_eq!(generation, i64::try_from(claim.generation()).unwrap()); + assert!(heartbeat_at > 0.0); + assert_eq!(spool_host.as_deref(), Some("host-a:/spool")); + assert_eq!(completed_at, None); + let transition: Option = sqlx::query_scalar( + "SELECT EXTRACT(EPOCH FROM status_transition_at)::FLOAT8 FROM tables WHERE table_id = $1", + ) + .bind(&key.table_id) + .fetch_one(&s.catalog) + .await + .expect("read the transition"); + assert_eq!( + transition, None, + "no scheduled transition: the job flips the table" + ); + + let job = record(&s, &key).await; + assert_eq!(job.target, key); + assert_eq!(job.source_backup_arn, row.backup_arn); + assert_eq!(job.kind, RestoreKind::Manifest); + assert_eq!(job.status, JobStatus::Running); + assert_eq!(job.owner_generation, claim.generation()); + assert_eq!(job.store, Some(store())); + assert_eq!(job.spool_host.as_deref(), Some("host-a:/spool")); + assert_eq!((job.file_index, job.items_applied), (None, None)); + assert_eq!( + (job.failure_reason.as_deref(), job.completed_at), + (None, None) + ); + let running = s + .engine + .list_restore_jobs(Some(JobStatus::Running)) + .await + .expect("list"); + assert_eq!(running, vec![job.clone()]); + assert!( + s.engine + .list_restore_jobs(Some(JobStatus::Done)) + .await + .expect("list") + .is_empty() + ); + assert_eq!( + s.engine.backup_in_use(&row.key()).await.expect("in use"), + Some("Restored".to_owned()) + ); + assert_eq!(claim.heartbeat().await.expect("beat"), Heartbeat::Confirmed); + let heartbeat_now = record_columns(&s, &key.table_id).await.3; + assert!( + heartbeat_now >= heartbeat_at, + "the heartbeat refreshed the row" + ); + + // The indexes a restore target is created with are ACTIVE from the start + // (the flip has nothing to move), and the GSI and LSI data tables exist. + let index_rows = indexes(&s, &key.table_id).await; + assert_eq!(index_rows.len(), 2); + assert!(index_rows.iter().all(|(_, _, status)| status == "ACTIVE")); + assert_eq!(base_rows(&s, &key.table_id).await, 0); + drop(claim); + s.cleanup().await; +} + +/// The ordering DeleteBackup wins: with `start_backup_delete` holding the +/// `backup_index` row FOR UPDATE, a registration waits behind it and, once +/// the row is DELETING, answers `Backup not found` and creates no target. +#[tokio::test] +async fn delete_backup_winning_before_the_registration_leaves_no_target() { + if !pg_available("delete_backup_winning_before_the_registration_leaves_no_target") { + return; + } + let s = scratch().await; + let row = available_backup(&s, "Music").await; + // `start_backup_delete` locks the row FOR UPDATE and then reads + // `table_restores`; a gate on that table holds it with the row lock + // taken. + let mut gate = s.catalog.begin().await.expect("begin the gate"); + sqlx::query("LOCK TABLE table_restores IN ACCESS EXCLUSIVE MODE") + .execute(&mut *gate) + .await + .expect("hold DeleteBackup after its row lock"); + let key = row.key(); + let mut delete = Box::pin(s.engine.start_backup_delete(&key)); + assert!( + tokio::time::timeout(Duration::from_secs(2), &mut delete) + .await + .is_err(), + "DeleteBackup did not reach the gate" + ); + let delete_waits: bool = sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM pg_locks \ + WHERE relation = 'table_restores'::regclass AND NOT granted)", + ) + .fetch_one(&s.catalog) + .await + .expect("observe the blocked DeleteBackup"); + assert!( + delete_waits, + "DeleteBackup was not paused after its row lock" + ); + + let input = target_input("Late", s.vector); + let reg = registration(&row, None); + let mut restore = Box::pin(s.engine.claim_restore(ACCOUNT, input, ®)); + assert!( + tokio::time::timeout(Duration::from_millis(700), &mut restore) + .await + .is_err(), + "the registration did not wait for DeleteBackup's row lock" + ); + assert_eq!( + table_status(&s, "Late").await, + None, + "the registration created its target before the row lock" + ); + + gate.commit().await.expect("let DeleteBackup finish"); + assert_eq!( + delete.await.expect("DeleteBackup wins"), + DeleteStart::Deleting + ); + let err = restore + .await + .err() + .expect("the registration finds the backup gone"); + assert!( + extenddb_storage::backup_catalog::is_backup_not_found(&err), + "{err:?}" + ); + assert_eq!( + table_status(&s, "Late").await, + None, + "no target was created" + ); + assert_eq!( + lock_holder(&s, "Late").await, + None, + "the refused registration released the name lock" + ); + assert!( + s.engine + .list_restore_jobs(None) + .await + .expect("list") + .is_empty() + ); + s.cleanup().await; +} + +/// The other ordering: a registration that commits first leaves +/// `start_backup_delete` with `InUse` naming the target, and a second +/// registration of the same backup is `BackupInUse` while the target is +/// CREATING. +#[tokio::test] +async fn a_registration_winning_before_delete_backup_keeps_the_backup_in_use() { + if !pg_available("a_registration_winning_before_delete_backup_keeps_the_backup_in_use") { + return; + } + let s = scratch().await; + let row = available_backup(&s, "Music").await; + let (_desc, key, claim) = register(&s, &row, "First", None).await; + assert_eq!( + s.engine + .start_backup_delete(&row.key()) + .await + .expect("DeleteBackup answers"), + DeleteStart::InUse("First".to_owned()) + ); + let second = s + .engine + .claim_restore( + ACCOUNT, + target_input("Second", s.vector), + ®istration(&row, None), + ) + .await + .err() + .expect("a second restore of the backup is refused"); + // The service's text names the backup, not the target. + assert!( + matches!(&second, StorageError::BackupInUse(m) if *m == backup_in_use_message(&row.backup_arn)), + "{second:?}" + ); + assert_eq!(table_status(&s, "Second").await, None); + assert_eq!(lock_holder(&s, "Second").await, None); + // The row stayed AVAILABLE through both refusals. + let status: String = sqlx::query_scalar("SELECT status FROM backup_index WHERE backup_id = $1") + .bind(&row.backup_id) + .fetch_one(&s.catalog) + .await + .expect("status"); + assert_eq!(status, "AVAILABLE"); + drop(claim); + let _ = key; + s.cleanup().await; +} + +/// A taken name: an ACTIVE table of the name is `TableAlreadyExists`; a +/// CREATING restore target of the name is `TableBeingRestored` naming its +/// source backup, whether the first registration's lock is still held or +/// its owner is gone. +#[tokio::test] +async fn a_taken_name_is_refused_as_create_table_or_as_a_restore_in_progress() { + if !pg_available("a_taken_name_is_refused_as_create_table_or_as_a_restore_in_progress") { + return; + } + let s = scratch().await; + let row = available_backup(&s, "Music").await; + s.engine + .create_table(ACCOUNT, target_input("Taken", false)) + .await + .expect("create a table of the name"); + let err = s + .engine + .claim_restore( + ACCOUNT, + target_input("Taken", s.vector), + ®istration(&row, None), + ) + .await + .err() + .expect("refused"); + assert!( + matches!(&err, StorageError::TableAlreadyExists(n) if n == "Taken"), + "{err:?}" + ); + + let (_desc, _key, claim) = register(&s, &row, "Restoring", None).await; + let other = available_backup(&s, "Other").await; + // With the first claim's lock held, the name lock refuses first. + let err = s + .engine + .claim_restore( + ACCOUNT, + target_input("Restoring", s.vector), + ®istration(&other, None), + ) + .await + .err() + .expect("refused"); + assert!( + matches!(&err, StorageError::TableBeingRestored { table_name, backup_arn } + if table_name == "Restoring" && *backup_arn == row.backup_arn), + "{err:?}" + ); + // With the lock gone (a dead owner), the unique index refuses the same way. + drop(claim); + let deadline = tokio::time::Instant::now() + Duration::from_secs(10); + while lock_holder(&s, "Restoring").await.is_some() { + assert!( + tokio::time::Instant::now() < deadline, + "the lock session did not end" + ); + tokio::time::sleep(Duration::from_millis(20)).await; + } + let err = s + .engine + .claim_restore( + ACCOUNT, + target_input("Restoring", s.vector), + ®istration(&other, None), + ) + .await + .err() + .expect("refused"); + assert!( + matches!(&err, StorageError::TableBeingRestored { table_name, .. } if table_name == "Restoring"), + "{err:?}" + ); + s.cleanup().await; +} + +// --- the takeover ------------------------------------------------------------- + +/// `claim_restore_dead` is refused while the owner's lock session lives and +/// while the heartbeat is young, and taken once both are gone: a fresh +/// generation, the claimant's `spool_host` and a refreshed `heartbeat_at` +/// (the database's clock) in one write, so a second taker finds the record +/// live at once; `list_dead_restore_jobs` lists the record with its members +/// only once stale; the old generation's writes are `NotMatched`. +#[tokio::test] +async fn a_dead_record_is_taken_over_with_the_claimants_spool_and_a_fresh_heartbeat() { + if !pg_available("a_dead_record_is_taken_over_with_the_claimants_spool_and_a_fresh_heartbeat") { + return; + } + let s = scratch().await; + let row = available_backup(&s, "Music").await; + let (_desc, key, claim) = register(&s, &row, "Target", Some("host-a:/spool")).await; + let old_generation = claim.generation(); + // A live owner: the lock is held. + assert!( + s.engine + .claim_restore_dead(&key, STALE, Some("host-b:/spool")) + .await + .expect("takeover") + .is_none(), + "a held lock is a live owner" + ); + // The lock gone (the session ended) but the heartbeat young. + let holder = lock_holder(&s, "Target") + .await + .expect("the owner's session"); + sqlx::query("SELECT pg_terminate_backend($1)") + .bind(holder) + .execute(&s.catalog) + .await + .expect("end the owner's session"); + let deadline = tokio::time::Instant::now() + Duration::from_secs(10); + while lock_holder(&s, "Target").await.is_some() { + assert!( + tokio::time::Instant::now() < deadline, + "the lock did not go" + ); + tokio::time::sleep(Duration::from_millis(20)).await; + } + // The owner's next heartbeat is lost: its session is gone. + assert!( + !matches!(claim.heartbeat().await, Ok(Heartbeat::Confirmed)), + "a terminated claim session cannot confirm" + ); + assert!( + s.engine + .claim_restore_dead(&key, STALE, Some("host-b:/spool")) + .await + .expect("takeover") + .is_none(), + "a young heartbeat is a live owner" + ); + assert!( + s.engine + .list_dead_restore_jobs(STALE) + .await + .expect("list") + .is_empty(), + "a young record is not listed" + ); + assert_eq!( + record(&s, &key).await.spool_host.as_deref(), + Some("host-a:/spool") + ); + + // Stale: listed with its members, and taken over. + let aged = age_heartbeat(&s, &key.table_id, Duration::from_secs(3600)).await; + let dead = s.engine.list_dead_restore_jobs(STALE).await.expect("list"); + assert_eq!(dead.len(), 1); + assert_eq!(dead[0].target, key); + assert_eq!(dead[0].kind, RestoreKind::Manifest); + assert_eq!(dead[0].status, JobStatus::Running); + assert_eq!(dead[0].store, Some(store())); + assert_eq!(dead[0].spool_host.as_deref(), Some("host-a:/spool")); + assert_eq!(dead[0].owner_generation, old_generation); + let successor = s + .engine + .claim_restore_dead(&key, STALE, Some("host-b:/spool")) + .await + .expect("takeover") + .expect("a stale record with no lock is dead"); + assert_ne!(successor.generation(), old_generation); + let (status, _, generation, heartbeat_at, spool_host, _) = + record_columns(&s, &key.table_id).await; + assert_eq!(status, "RUNNING"); + assert_eq!(generation, i64::try_from(successor.generation()).unwrap()); + assert_eq!( + spool_host.as_deref(), + Some("host-b:/spool"), + "the claimant's spool" + ); + assert!( + heartbeat_at > aged + 3_000.0, + "the takeover refreshed heartbeat_at: {heartbeat_at} vs the aged {aged}" + ); + let db_now: f64 = sqlx::query_scalar("SELECT EXTRACT(EPOCH FROM now())::FLOAT8") + .fetch_one(&s.catalog) + .await + .expect("the database's clock"); + assert!( + (db_now - heartbeat_at).abs() < 5.0, + "the refreshed heartbeat is the database's now: {heartbeat_at} vs {db_now}" + ); + assert!( + lock_holder(&s, "Target").await.is_some(), + "the successor holds the lock" + ); + // A second taker is refused while the successor lives: the lock, and + // the fresh heartbeat (the window a second instance's reconciler would + // otherwise find the record stale in). + assert!( + s.engine + .claim_restore_dead(&key, STALE, Some("host-c:/spool")) + .await + .expect("takeover") + .is_none() + ); + assert!( + s.engine + .list_dead_restore_jobs(STALE) + .await + .expect("list") + .is_empty(), + "the taken-over record is live to the listing at once" + ); + // The old generation's writes match nothing: the flip through the old + // claim, and the old claim's status write (its session is gone, so the + // write's outcome is unknown and the re-read shows the successor). + assert!(matches!( + s.engine + .finish_restore(&key, &*claim, 0, 0) + .await + .expect("finish"), + Finished::NotMatched(_) + )); + assert_eq!(record(&s, &key).await.status, JobStatus::Running); + let old_write = claim + .set_status( + JobStatus::Running, + JobStatus::Failed, + &StatusFields { + failure_reason: Some("stale".to_owned()), + completed: true, + ..StatusFields::default() + }, + ) + .await; + assert!( + matches!(old_write, Ok(WriteOutcome::Unknown) | Err(_)), + "{old_write:?}" + ); + let reread = claim.read().await.expect("the pool re-read"); + assert_eq!(reread.owner_generation, successor.generation()); + assert_eq!(reread.status, JobStatus::Running); + // The successor's heartbeat confirms and its status write matches; after + // its FAILED write a second write from it is NotMatched (not RUNNING). + assert_eq!( + successor.heartbeat().await.expect("beat"), + Heartbeat::Confirmed + ); + assert_eq!( + successor + .set_status( + JobStatus::Running, + JobStatus::Failed, + &StatusFields { + failure_reason: Some("count".to_owned()), + completed: true, + ..StatusFields::default() + }, + ) + .await + .expect("write"), + WriteOutcome::Matched + ); + assert_eq!( + successor + .set_status( + JobStatus::Running, + JobStatus::Done, + &StatusFields::default() + ) + .await + .expect("write"), + WriteOutcome::NotMatched + ); + let job = record(&s, &key).await; + assert_eq!(job.status, JobStatus::Failed); + assert_eq!(job.failure_reason.as_deref(), Some("count")); + assert!(job.completed_at.is_some()); + assert!( + s.engine + .list_dead_restore_jobs(STALE) + .await + .expect("list") + .is_empty(), + "a fresh FAILED record is left to its job's removal" + ); + drop(successor); + s.cleanup().await; +} + +// --- the load ------------------------------------------------------------------- + +/// `bulk_load` writes the base rows, the GSI, LSI and vector rows and the +/// file's progress row in one data transaction, across the 500-row batch +/// boundary; `start_at` is added to the recorded total; a duplicate key +/// within a file or against an earlier file fails the file with nothing of +/// it committed, no progress row included; a file whose progress is already +/// recorded fails whole, its index rows included. +#[tokio::test] +async fn bulk_load_writes_each_file_with_its_indexes_and_progress_in_one_transaction() { + if !pg_available("bulk_load_writes_each_file_with_its_indexes_and_progress_in_one_transaction") + { + return; + } + let s = scratch().await; + let row = available_backup(&s, "Music").await; + let (_desc, key, claim) = register(&s, &row, "Loaded", None).await; + + // File 0: 1200 items, past the 500-row batch. + let applied = s + .engine + .bulk_load( + &key, + &*claim, + BulkLoadFile { + file_index: 0, + start_at: 0, + reapplying: false, + }, + stream(items(0..1200)), + ) + .await + .expect("load file 0"); + assert_eq!(applied, 1200); + assert_eq!(base_rows(&s, &key.table_id).await, 1200); + assert_eq!( + index_rows(&s, &key.table_id, "gix").await, + 600, + "every second item has g" + ); + assert_eq!( + index_rows(&s, &key.table_id, "lix").await, + 400, + "every third item has l" + ); + if s.vector { + assert_eq!( + vector_rows(&s, &key.table_id).await, + 240, + "every fifth item has emb" + ); + } + assert_eq!(progress_rows(&s, &key.table_id).await, vec![(0, 1200)]); + assert_eq!( + s.engine.count_restore_target(&key).await.expect("count"), + 1200 + ); + + // File 1 resumed at item 5 of its 8: the record carries the running total. + let applied = s + .engine + .bulk_load( + &key, + &*claim, + BulkLoadFile { + file_index: 1, + start_at: 5, + reapplying: false, + }, + stream(items(1205..1208)), + ) + .await + .expect("load file 1"); + assert_eq!(applied, 3); + assert_eq!( + progress_rows(&s, &key.table_id).await, + vec![(0, 1200), (1, 8)] + ); + assert_eq!(base_rows(&s, &key.table_id).await, 1203); + + // File 2 holds a key twice: nothing of it lands, no progress row. + let mut twice = items(2000..2010); + twice.push(item(2003)); + let err = s + .engine + .bulk_load( + &key, + &*claim, + BulkLoadFile { + file_index: 2, + start_at: 0, + reapplying: false, + }, + stream(twice), + ) + .await + .expect_err("a duplicate key fails the file"); + assert!( + matches!(&err, StorageError::Validation(m) + if m.starts_with("holds a primary key the target already has") + && m.contains("the backup holds a key twice")), + "the backend names no file (the engine prefixes the file's key): {err:?}" + ); + assert_eq!(base_rows(&s, &key.table_id).await, 1203); + assert_eq!( + progress_rows(&s, &key.table_id).await, + vec![(0, 1200), (1, 8)] + ); + + // File 3 repeats a key file 0 wrote: the same. + let mut across = items(3000..3004); + across.push(item(7)); + let err = s + .engine + .bulk_load( + &key, + &*claim, + BulkLoadFile { + file_index: 3, + start_at: 0, + reapplying: false, + }, + stream(across), + ) + .await + .expect_err("a key of an earlier file fails the file"); + assert!( + matches!(&err, StorageError::Validation(m) + if m.starts_with("holds a primary key the target already has")), + "{err:?}" + ); + assert_eq!(base_rows(&s, &key.table_id).await, 1203); + assert_eq!( + index_rows(&s, &key.table_id, "gix").await, + 600 + 1, + "1206 has g" + ); + assert_eq!( + progress_rows(&s, &key.table_id).await, + vec![(0, 1200), (1, 8)] + ); + + // A decode or validation failure the engine found mid-stream: the items + // before it are not committed either. + let failing: futures::stream::BoxStream<'static, Result> = + Box::pin(futures::stream::iter( + items(4000..4003) + .into_iter() + .map(Ok) + .chain(std::iter::once(Err(StorageError::Validation( + "key {\"pk\": \"p1\"}, attribute n: 1E126 is not a number".to_owned(), + )))), + )); + let err = s + .engine + .bulk_load( + &key, + &*claim, + BulkLoadFile { + file_index: 4, + start_at: 0, + reapplying: false, + }, + failing, + ) + .await + .expect_err("an Err item fails the file"); + assert!( + matches!(&err, StorageError::Validation(m) if m.contains("1E126")), + "{err:?}" + ); + assert_eq!(base_rows(&s, &key.table_id).await, 1203); + assert_eq!( + progress_rows(&s, &key.table_id).await, + vec![(0, 1200), (1, 8)] + ); + + // File 0 again with keys the target lacks: its rows and index rows land + // in the file's transaction, and the progress row's primary key rolls + // the whole file back, index rows included. + let err = s + .engine + .bulk_load( + &key, + &*claim, + BulkLoadFile { + file_index: 0, + start_at: 0, + reapplying: false, + }, + stream(items(5000..5004)), + ) + .await + .expect_err("a recorded file is not applied twice"); + assert!( + matches!(&err, StorageError::Validation(m) if m.contains("data file 0")), + "{err:?}" + ); + assert_eq!(base_rows(&s, &key.table_id).await, 1203); + assert_eq!(index_rows(&s, &key.table_id, "gix").await, 601); + assert_eq!( + index_rows(&s, &key.table_id, "lix").await, + 400 + 1, + "1206 has l" + ); + if s.vector { + assert_eq!( + vector_rows(&s, &key.table_id).await, + 240 + 1, + "1205 has emb" + ); + } + assert_eq!( + progress_rows(&s, &key.table_id).await, + vec![(0, 1200), (1, 8)] + ); + + // File 1's row says 8 of its items were applied and the file holds 11 + // (the decoder died after the file's commit and the job's FAILED write + // never landed): the resume starts the file at 8, and the rest completes + // it, the row updated in place instead of refused on its key. + let applied = s + .engine + .bulk_load( + &key, + &*claim, + BulkLoadFile { + file_index: 1, + start_at: 8, + reapplying: false, + }, + stream(items(1208..1211)), + ) + .await + .expect("a file resumed from a short progress row completes"); + assert_eq!(applied, 3); + assert_eq!( + progress_rows(&s, &key.table_id).await, + vec![(0, 1200), (1, 11)], + "the short row is updated to the file's total" + ); + assert_eq!(base_rows(&s, &key.table_id).await, 1206); + assert_eq!(index_rows(&s, &key.table_id, "gix").await, 603); + assert_eq!(index_rows(&s, &key.table_id, "lix").await, 402); + if s.vector { + assert_eq!(vector_rows(&s, &key.table_id).await, 242); + } + + // Another account's key, or a wrong table id, reaches no table. + let foreign = RestoreTargetKey { + account_id: "210987654321".to_owned(), + ..key.clone() + }; + assert!(matches!( + s.engine.count_restore_target(&foreign).await, + Err(StorageError::TableNotFound(_)) + )); + assert!(matches!( + s.engine + .bulk_load( + &foreign, + &*claim, + BulkLoadFile { + file_index: 9, + start_at: 0, + reapplying: false, + }, + stream(items(9000..9001)), + ) + .await, + Err(StorageError::TableNotFound(_)) + )); + drop(claim); + s.cleanup().await; +} + +/// `finish_restore` marks the record DONE and the table and every index +/// ACTIVE in one transaction, conditioned on the claim's generation and on +/// CREATING, then removes the progress rows; a second call, or one under a +/// stale generation, is `NotMatched` and writes nothing. The progress sweep +/// keeps a RUNNING target's rows and removes any other's; +/// `remove_restore_target` deletes a target's rows. +#[tokio::test] +async fn finish_restore_flips_the_table_and_every_index_and_clears_the_progress() { + if !pg_available("finish_restore_flips_the_table_and_every_index_and_clears_the_progress") { + return; + } + let s = scratch().await; + let row = available_backup(&s, "Music").await; + let (_desc, key, claim) = register(&s, &row, "Done", None).await; + let other_row = available_backup(&s, "Other").await; + let (_desc2, other, other_claim) = register(&s, &other_row, "Still", None).await; + for (k, c) in [(&key, &claim), (&other, &other_claim)] { + s.engine + .bulk_load( + k, + &**c, + BulkLoadFile { + file_index: 0, + start_at: 0, + reapplying: false, + }, + stream(items(0..30)), + ) + .await + .expect("load"); + } + // An index a backend left CREATING (none does on this backend; the flip + // states the rule once) is moved with the table. + sqlx::query( + "UPDATE indexes SET index_status = 'CREATING' WHERE table_id = $1 AND index_name = 'gix'", + ) + .bind(&key.table_id) + .execute(&s.catalog) + .await + .expect("set an index CREATING"); + if s.vector { + sqlx::query("UPDATE vector_indexes SET index_status = 'CREATING' WHERE table_id = $1") + .bind(&key.table_id) + .execute(&s.catalog) + .await + .expect("set the vector index CREATING"); + } + + // The sweep: `Still`'s rows stay while it is RUNNING; a row of a table + // that is gone goes. + sqlx::query( + "INSERT INTO restore_progress (table_id, file_index, items_applied) VALUES ('gone', 0, 5)", + ) + .execute(&s.db) + .await + .expect("an orphan row"); + // A caller whose RUNNING list predates both registrations (the list is + // read before the sweep; a registration and its first file can land in + // between): the sweep protects the RUNNING records it reads itself after + // the candidates, so neither live row goes, only the orphan. + assert_eq!( + s.engine.sweep_restore_progress(&[]).await.expect("sweep"), + 1, + "the orphan row alone, with an empty caller list" + ); + assert_eq!(progress_rows(&s, &key.table_id).await, vec![(0, 30)]); + assert_eq!(progress_rows(&s, &other.table_id).await, vec![(0, 30)]); + sqlx::query( + "INSERT INTO restore_progress (table_id, file_index, items_applied) VALUES ('gone', 0, 5)", + ) + .execute(&s.db) + .await + .expect("the orphan row again"); + let running: Vec = s + .engine + .list_restore_jobs(Some(JobStatus::Running)) + .await + .expect("list") + .into_iter() + .map(|j| j.target.table_id) + .collect(); + assert_eq!(running.len(), 2); + assert_eq!( + s.engine + .sweep_restore_progress(&running) + .await + .expect("sweep"), + 1 + ); + assert_eq!(progress_rows(&s, &key.table_id).await, vec![(0, 30)]); + assert_eq!(progress_rows(&s, &other.table_id).await, vec![(0, 30)]); + assert_eq!( + s.engine.read_restore_progress(&key).await.expect("read"), + vec![(0, 30)] + ); + + // A stale generation flips nothing. + let stale_row = available_backup(&s, "Stale").await; + let (_d, stale_key, stale_claim) = register(&s, &stale_row, "StaleOwner", None).await; + let _ = stale_key; + assert!(matches!( + s.engine + .finish_restore(&key, &*stale_claim, 30, 4096) + .await + .expect("finish"), + Finished::NotMatched(_) + )); + assert_eq!(table_status(&s, "Done").await.as_deref(), Some("CREATING")); + assert_eq!(record(&s, &key).await.status, JobStatus::Running); + assert_eq!(progress_rows(&s, &key.table_id).await, vec![(0, 30)]); + drop(stale_claim); + + assert_eq!( + s.engine + .finish_restore(&key, &*claim, 30, 4096) + .await + .expect("finish"), + Finished::Active + ); + assert_eq!(table_status(&s, "Done").await.as_deref(), Some("ACTIVE")); + let (status, _, _, _, _, completed_at) = record_columns(&s, &key.table_id).await; + assert_eq!(status, "DONE"); + assert!(completed_at.is_some()); + let counts: (i64, i64) = + sqlx::query_as("SELECT item_count, table_size_bytes FROM tables WHERE table_id = $1") + .bind(&key.table_id) + .fetch_one(&s.catalog) + .await + .expect("counts"); + assert_eq!(counts, (30, 4096)); + assert!( + indexes(&s, &key.table_id) + .await + .iter() + .all(|(_, _, status)| status == "ACTIVE"), + "every secondary index is ACTIVE" + ); + if s.vector { + assert!( + vector_indexes(&s, &key.table_id) + .await + .iter() + .all(|(_, status)| status == "ACTIVE"), + "the vector index is ACTIVE" + ); + } + assert!( + progress_rows(&s, &key.table_id).await.is_empty(), + "the progress rows are gone" + ); + assert_eq!(progress_rows(&s, &other.table_id).await, vec![(0, 30)]); + let job = record(&s, &key).await; + assert_eq!(job.status, JobStatus::Done); + assert!(job.completed_at.is_some()); + // The job record's heartbeat is no longer refreshed: not RUNNING. + assert_eq!(claim.heartbeat().await.expect("beat"), Heartbeat::Lost); + // A second flip matches nothing. + assert!(matches!( + s.engine + .finish_restore(&key, &*claim, 30, 4096) + .await + .expect("finish"), + Finished::NotMatched(_) + )); + // The restored table reads through the data plane as any ACTIVE table. + let desc = s + .engine + .describe_table( + ACCOUNT, + extenddb_core::types::DescribeTableInput { + table_name: "Done".to_owned(), + }, + ) + .await + .expect("describe"); + assert_eq!(desc.table_status, TableStatus::Active); + assert_eq!(desc.item_count, 30); + + // `remove_restore_target` on the other target deletes its progress rows + // with the target. + assert!( + s.engine + .remove_restore_target(&other) + .await + .expect("remove") + ); + assert!(progress_rows(&s, &other.table_id).await.is_empty()); + assert_eq!(table_status(&s, "Still").await, None); + drop(other_claim); + drop(claim); + s.cleanup().await; +} + +/// `remove_restore_target` of a target with a GSI, an LSI and (with pgvector) +/// a vector index drops every relation the target's rows named: the data +/// table, each secondary index's table and the vector index's `_ddb_vec_` +/// table, so a failed vector restore leaves no relation holding its items +/// with no catalog row naming it. +#[tokio::test] +async fn remove_restore_target_drops_the_vector_relation_with_the_data_table() { + if !pg_available("remove_restore_target_drops_the_vector_relation_with_the_data_table") { + return; + } + let s = scratch().await; + let row = available_backup(&s, "Music").await; + let (_desc, key, claim) = register(&s, &row, "Vectors", None).await; + s.engine + .bulk_load( + &key, + &*claim, + BulkLoadFile { + file_index: 0, + start_at: 0, + reapplying: false, + }, + stream(items(0..30)), + ) + .await + .expect("load"); + let mut relations: Vec = vec![format!("_ddb_{}", key.table_id)]; + relations.extend( + indexes(&s, &key.table_id) + .await + .into_iter() + .map(|(_, id, _)| format!("_ddb_{id}")), + ); + if s.vector { + assert_eq!( + vector_rows(&s, &key.table_id).await, + 6, + "every fifth item has emb" + ); + relations.extend( + vector_indexes(&s, &key.table_id) + .await + .into_iter() + .map(|(id, _)| format!("_ddb_vec_{id}")), + ); + assert_eq!( + relations.len(), + 4, + "the data table, two index tables, one vector table" + ); + } + for relation in &relations { + assert!( + relation_exists(&s, relation).await, + "{relation} exists before the removal" + ); + } + + assert!(s.engine.remove_restore_target(&key).await.expect("remove")); + assert_eq!(table_status(&s, "Vectors").await, None); + if s.vector { + assert!(vector_indexes(&s, &key.table_id).await.is_empty()); + } + for relation in &relations { + assert!( + !relation_exists(&s, relation).await, + "{relation} is dropped with the target" + ); + } + drop(claim); + s.cleanup().await; +} + +/// A `finish_restore` whose progress delete fails after the flip committed +/// (the data database cannot run the `DELETE`: the `restore_progress` table +/// is renamed under it) is `Active`: the table is ACTIVE and the record DONE, +/// the failure is logged and not returned, the rows are left, and the +/// reconciler's `sweep_restore_progress` removes them once the record is no +/// longer RUNNING. +#[tokio::test] +async fn a_progress_delete_failing_after_the_flip_leaves_the_restore_finished() { + if !pg_available("a_progress_delete_failing_after_the_flip_leaves_the_restore_finished") { + return; + } + let s = scratch().await; + let row = available_backup(&s, "Music").await; + let (_desc, key, claim) = register(&s, &row, "Flipped", None).await; + s.engine + .bulk_load( + &key, + &*claim, + BulkLoadFile { + file_index: 0, + start_at: 0, + reapplying: false, + }, + stream(items(0..30)), + ) + .await + .expect("load"); + assert_eq!(progress_rows(&s, &key.table_id).await, vec![(0, 30)]); + + sqlx::query("ALTER TABLE restore_progress RENAME TO restore_progress_away") + .execute(&s.db) + .await + .expect("take the progress table away"); + let finished = s.engine.finish_restore(&key, &*claim, 30, 4096).await; + sqlx::query("ALTER TABLE restore_progress_away RENAME TO restore_progress") + .execute(&s.db) + .await + .expect("bring the progress table back"); + assert!( + matches!(finished, Ok(Finished::Active)), + "a progress delete failing after the flip is not the restore's failure: {finished:?}" + ); + assert_eq!(table_status(&s, "Flipped").await.as_deref(), Some("ACTIVE")); + assert_eq!(record(&s, &key).await.status, JobStatus::Done); + assert_eq!( + progress_rows(&s, &key.table_id).await, + vec![(0, 30)], + "the rows the delete could not remove are left" + ); + assert_eq!( + s.engine.sweep_restore_progress(&[]).await.expect("sweep"), + 1, + "the reconciler's sweep removes the DONE target's rows" + ); + assert!(progress_rows(&s, &key.table_id).await.is_empty()); + drop(claim); + s.cleanup().await; +} + +/// Terminating the claim session: the next heartbeat is lost, the status +/// write's outcome is unknown, and the record stays RUNNING for the +/// reconciler once stale. +#[tokio::test] +async fn a_terminated_claim_session_loses_its_next_heartbeat() { + if !pg_available("a_terminated_claim_session_loses_its_next_heartbeat") { + return; + } + let s = scratch().await; + let row = available_backup(&s, "Music").await; + let (_desc, key, claim) = register(&s, &row, "Cut", None).await; + assert_eq!(claim.heartbeat().await.expect("beat"), Heartbeat::Confirmed); + let holder = lock_holder(&s, "Cut").await.expect("the claim session"); + sqlx::query("SELECT pg_terminate_backend($1)") + .bind(holder) + .execute(&s.catalog) + .await + .expect("terminate"); + let beat = claim.heartbeat().await; + assert!(!matches!(beat, Ok(Heartbeat::Confirmed)), "{beat:?}"); + let write = claim + .set_status( + JobStatus::Running, + JobStatus::Done, + &StatusFields::default(), + ) + .await; + assert!( + matches!(write, Ok(WriteOutcome::Unknown) | Err(_)), + "{write:?}" + ); + assert_eq!(record(&s, &key).await.status, JobStatus::Running); + assert_eq!(table_status(&s, "Cut").await.as_deref(), Some("CREATING")); + s.cleanup().await; +} + +/// A legacy restore record is not a takeover candidate: `claim_restore_dead` +/// leaves a stale `kind = legacy` record to the reconciler's legacy duty. +#[tokio::test] +async fn a_legacy_record_is_not_taken_over() { + if !pg_available("a_legacy_record_is_not_taken_over") { + return; + } + let s = scratch().await; + let row = available_backup(&s, "Music").await; + let (_desc, key, claim) = register(&s, &row, "Legacyish", None).await; + drop(claim); + let deadline = tokio::time::Instant::now() + Duration::from_secs(10); + while lock_holder(&s, "Legacyish").await.is_some() { + assert!(tokio::time::Instant::now() < deadline); + tokio::time::sleep(Duration::from_millis(20)).await; + } + sqlx::query("UPDATE table_restores SET kind = 'legacy' WHERE table_id = $1") + .bind(&key.table_id) + .execute(&s.catalog) + .await + .expect("make the record legacy"); + age_heartbeat(&s, &key.table_id, Duration::from_secs(3600)).await; + assert!( + s.engine + .claim_restore_dead(&key, STALE, None) + .await + .expect("takeover") + .is_none() + ); + assert_eq!(record(&s, &key).await.kind, RestoreKind::Legacy); + assert_eq!(lock_holder(&s, "Legacyish").await, None); + s.cleanup().await; +} + +/// `sweep_restore_progress` reads the candidate progress-row ids before the +/// live RUNNING set, so a registration that commits and writes a progress +/// row between the two reads keeps that row: the row is not among the +/// candidates read first, and the record is RUNNING in the live set read +/// after. An orphan row with no record is swept in the same call, so the +/// sweep is seen to run. With the two reads swapped the registration's row +/// would be a candidate and not live, and would be deleted. +#[tokio::test] +async fn sweep_keeps_a_progress_row_registered_between_the_two_reads() { + if !pg_available("sweep_keeps_a_progress_row_registered_between_the_two_reads") { + return; + } + let s = scratch().await; + // An orphan progress row (no record names it): a candidate the sweep + // removes, so the sweep is doing work in this cycle. + sqlx::query("INSERT INTO restore_progress (table_id, file_index, items_applied) VALUES ('orphan-tid', 0, 5)") + .execute(&s.db) + .await + .expect("seed an orphan progress row"); + + let (arrived, proceed) = extenddb_storage_postgres::sweep_pause::arm(); + let engine = s.engine.clone(); + let sweep = tokio::spawn(async move { engine.sweep_restore_progress(&[]).await }); + + // The sweep has read the candidate ids and is paused before the live + // set: register a RUNNING target and write its progress row now. + arrived.notified().await; + let row = available_backup(&s, "Music").await; + let (_desc, key, claim) = register(&s, &row, "Late", None).await; + sqlx::query( + "INSERT INTO restore_progress (table_id, file_index, items_applied) VALUES ($1, 0, 7)", + ) + .bind(&key.table_id) + .execute(&s.db) + .await + .expect("the registration's progress row"); + proceed.notify_one(); + + let removed = sweep.await.expect("join").expect("sweep"); + assert_eq!(removed, 1, "only the orphan progress row was swept"); + assert_eq!( + progress_rows(&s, &key.table_id).await, + vec![(0, 7)], + "the live registration's progress row stayed" + ); + assert!( + progress_rows(&s, "orphan-tid").await.is_empty(), + "the orphan row went" + ); + drop(claim); + s.cleanup().await; +} diff --git a/crates/storage-postgres/tests/table_snapshot.rs b/crates/storage-postgres/tests/table_snapshot.rs index 7a985c88d..e0a99d5ab 100644 --- a/crates/storage-postgres/tests/table_snapshot.rs +++ b/crates/storage-postgres/tests/table_snapshot.rs @@ -555,9 +555,18 @@ async fn half_the_pool_holds_snapshots(shared: bool) { .await .expect("an open over the bound is refused at once, not queued") .expect_err("the sixth open is over the bound"); - assert!( - matches!(err, extenddb_storage::error::StorageError::LimitExceeded(_)), - "{err}" + // The one refusal text every job bound answers, naming this one. + assert_eq!( + err.to_string(), + extenddb_storage::error::StorageError::LimitExceeded( + extenddb_storage::backup_catalog::limit_exceeded_message( + extenddb_storage::backup_catalog::JobBound::Counted { + name: extenddb_storage_postgres::SNAPSHOT_SLOT_BOUND, + limit: 5, + } + ) + ) + .to_string() ); let key: Item = [ ("pk".to_owned(), AttributeValue::S("p000".to_owned())), @@ -1077,6 +1086,662 @@ async fn snapshot_marker_and_key_order(shared: bool) { } on_both_layouts!(snapshot_marker_and_key_order); +/// The keyset pages respect the byte budget on both layouts +/// (`snapshot_conformance::run_page_bounds`), and the rows the server reads +/// past each page's cut are those the row bound lets it: with a budget of +/// eight rows the first page asks for `MIN_PAGE_ROWS` (16) and keeps eight, +/// so it reads eight it drops; every later page is sized from the largest +/// row of the page before (the budget divided by it, eight here), so it +/// reads exactly the rows it keeps and drops none. The figure is asserted +/// exactly: a counter that never moves would pass a bound (lesson 25), and a +/// row bound sized from the average again reads the same eight, so the +/// mixed-page case is pinned by `next_page_rows`'s unit test and by +/// `the_row_bound_remembers_large_rows_among_small_ones`. +async fn pages_are_cut_at_the_byte_budget(shared: bool) { + let s = scratch_with(shared).await; + let bounds = snapshot_conformance::run_page_bounds( + &s.hooks, + &s.engine, + ACCOUNT, + "page_bounds", + || async { + s.engine + .process_control_plane_transitions() + .await + .expect("pump"); + }, + ) + .await; + let past_cut = bounds.rows_past_cut; + let first_page = u64::try_from( + extenddb_storage_postgres::MIN_PAGE_ROWS - snapshot_conformance::PAGE_BOUNDS_ROWS, + ) + .unwrap(); + // The figure the memory record cites beside the RSS numbers. + eprintln!( + "rows read past the cut: {past_cut} over {} pages of {} rows ({first_page} expected, all \ + on the first page)", + bounds.pages, + bounds.budget / bounds.row_bytes + ); + assert_eq!( + past_cut, first_page, + "{past_cut} rows read past the cut over {} pages: the first page reads MIN_PAGE_ROWS and \ + drops {first_page}, and every later page is sized to the rows it keeps", + bounds.pages + ); +} +on_both_layouts!(pages_are_cut_at_the_byte_budget); + +/// Every plan node type under `node`, in depth-first order. +fn plan_node_types(node: &serde_json::Value, into: &mut Vec) { + if let Some(t) = node.get("Node Type").and_then(serde_json::Value::as_str) { + into.push(t.to_owned()); + } + for child in node + .get("Plans") + .and_then(serde_json::Value::as_array) + .into_iter() + .flatten() + { + plan_node_types(child, into); + } +} + +/// The index names every scan node under `node` uses. +fn plan_index_names(node: &serde_json::Value, into: &mut Vec) { + if let Some(name) = node.get("Index Name").and_then(serde_json::Value::as_str) { + into.push(name.to_owned()); + } + for child in node + .get("Plans") + .and_then(serde_json::Value::as_array) + .into_iter() + .flatten() + { + plan_index_names(child, into); + } +} + +/// A 100k-row composite-key table (`pk` = `p00000`..`p02000`, 50 rows each, +/// `sk_n` = 1..100000) filled straight into the data table and analyzed, +/// with its description and a data pool; the engine's `put_item` would take +/// minutes for 100k rows. +async fn hundred_k_table(s: &Scratch) -> (extenddb_core::types::TableDescription, PgPool) { + filled_table( + s, + "hundred_k", + "SELECT 'p' || lpad((i / 50)::text, 5, '0'), i, \ + jsonb_build_object('pk', jsonb_build_object('S', 'p' || lpad((i / 50)::text, 5, '0')), \ + 'sk', jsonb_build_object('N', i::text), \ + 'v', jsonb_build_object('S', repeat('x', 100))) \ + FROM generate_series(1, 100000) AS i", + ) + .await +} + +/// One key value bound to a page query's seek: text (numeric keys travel +/// as text) or bytes. +enum Bind { + Text(&'static str), + Bytes(Vec), +} + +/// `EXPLAIN (FORMAT JSON)` of the page query (with `COLLATE "C"` on its text +/// key columns when `collate_c`, as the engine writes it for a table whose +/// columns have that collation) after `after` (`None` for the first page) at +/// row bound `n`: the plan's node types in depth-first order and the index +/// names its scans use, with the plan for the message. +async fn explain_page( + data: &PgPool, + desc: &extenddb_core::types::TableDescription, + collate_c: bool, + after: Option>, + n: i64, +) -> (Vec, Vec, serde_json::Value) { + let budget = i64::try_from(extenddb_storage::snapshot::paging::SNAPSHOT_FETCH_BYTES).unwrap(); + let sql = + extenddb_storage_postgres::page_sql_for(desc, &desc.table_id, collate_c, after.is_some()); + let explain_sql = format!("EXPLAIN (FORMAT JSON) {sql}"); + let mut explain = sqlx::query_scalar::<_, serde_json::Value>(&explain_sql); + let seek = after.is_some(); + for value in after.into_iter().flatten() { + explain = match value { + Bind::Text(t) => explain.bind(t), + Bind::Bytes(b) => explain.bind(b), + }; + } + let plan = explain + .bind(n) + .bind(budget) + .fetch_one(data) + .await + .unwrap_or_else(|e| panic!("explain (after a key: {seek}) at n = {n}: {e}")); + let root = &plan[0]["Plan"]; + let mut types = Vec::new(); + plan_node_types(root, &mut types); + let mut indexes = Vec::new(); + plan_index_names(root, &mut indexes); + (types, indexes, plan) +} + +/// Assert that a page query's plan is `Limit` over one `Index Scan` on the +/// primary key index with no sort: the seek and the order both follow the +/// index, so the page costs the server the rows it reads, up to `n`, each +/// serialized once, and not the rows before them. +fn assert_index_scan( + label: &str, + types: &[String], + indexes: &[String], + plan: &serde_json::Value, + pkey: &str, +) { + assert!( + !types.iter().any(|t| t == "Sort"), + "{label}: the plan sorts: {types:?}\n{plan}" + ); + let scans: Vec<&str> = types + .iter() + .map(String::as_str) + .filter(|t| t.ends_with("Scan") && *t != "Subquery Scan") + .collect(); + assert_eq!( + scans, + vec!["Index Scan"], + "{label}: the relation is not read by one index scan: {types:?}\n{plan}" + ); + assert_eq!( + indexes, + vec![pkey.to_owned()], + "{label}: the scan does not use the primary key index\n{plan}" + ); +} + +/// A key shape the page query is run on: the `CreateTable` input, the data +/// table's key columns, the `SELECT` filling 20,000 rows of it, and a key in +/// its middle to seek past. +struct KeyShape { + label: &'static str, + input: extenddb_core::types::CreateTableInput, + columns: &'static str, + rows: &'static str, + middle: Vec, + /// Whether the text key columns have collation "C", so the engine writes + /// `COLLATE "C"` into the query (every table but the legacy one). + collate_c: bool, + /// Whether the stream reports the table ordered (a multi-part range key + /// has no format order; a legacy table's key column has the database + /// collation). + ordered: bool, +} + +fn key_shape_input(name: &str, range: &[(&str, &str)]) -> extenddb_core::types::CreateTableInput { + let mut key_schema = vec![serde_json::json!({"AttributeName": "pk", "KeyType": "HASH"})]; + let mut defs = vec![serde_json::json!({"AttributeName": "pk", "AttributeType": "S"})]; + for (attr, ty) in range { + key_schema.push(serde_json::json!({"AttributeName": attr, "KeyType": "RANGE"})); + defs.push(serde_json::json!({"AttributeName": attr, "AttributeType": ty})); + } + serde_json::from_value(serde_json::json!({ + "TableName": name, + "KeySchema": key_schema, + "AttributeDefinitions": defs, + "BillingMode": "PAY_PER_REQUEST" + })) + .expect("create input") +} + +/// The six key shapes a data table's primary key can have: hash only; an +/// `S`, `N` or `B` range key; a multi-part range key; and a legacy table +/// whose text key column carries the database collation (recollated after +/// the fill, as `legacy_collation_table_streams_unordered_and_complete` +/// does). +fn key_shapes() -> Vec { + vec![ + KeyShape { + label: "hash only", + input: key_shape_input("hash_only", &[]), + columns: "pk, item_data", + rows: "SELECT 'p' || lpad(i::text, 5, '0'), \ + jsonb_build_object('pk', jsonb_build_object('S', 'p' || lpad(i::text, 5, '0')), \ + 'v', jsonb_build_object('S', repeat('x', 100))) \ + FROM generate_series(1, 20000) AS i", + middle: vec![Bind::Text("p10000")], + collate_c: true, + ordered: true, + }, + KeyShape { + label: "S range key", + input: key_shape_input("range_s", &[("sk", "S")]), + columns: "pk, sk_s, item_data", + rows: "SELECT 'p' || lpad((i / 50)::text, 5, '0'), 'k' || lpad(i::text, 5, '0'), \ + jsonb_build_object('pk', jsonb_build_object('S', 'p' || lpad((i / 50)::text, 5, '0')), \ + 'sk', jsonb_build_object('S', 'k' || lpad(i::text, 5, '0')), \ + 'v', jsonb_build_object('S', repeat('x', 100))) \ + FROM generate_series(1, 20000) AS i", + middle: vec![Bind::Text("p00200"), Bind::Text("k10000")], + collate_c: true, + ordered: true, + }, + KeyShape { + label: "N range key", + input: key_shape_input("range_n", &[("sk", "N")]), + columns: "pk, sk_n, item_data", + rows: "SELECT 'p' || lpad((i / 50)::text, 5, '0'), i, \ + jsonb_build_object('pk', jsonb_build_object('S', 'p' || lpad((i / 50)::text, 5, '0')), \ + 'sk', jsonb_build_object('N', i::text), \ + 'v', jsonb_build_object('S', repeat('x', 100))) \ + FROM generate_series(1, 20000) AS i", + middle: vec![Bind::Text("p00200"), Bind::Text("10000")], + collate_c: true, + ordered: true, + }, + KeyShape { + label: "B range key", + input: key_shape_input("range_b", &[("sk", "B")]), + columns: "pk, sk_b, item_data", + rows: "SELECT 'p' || lpad((i / 50)::text, 5, '0'), decode(lpad(to_hex(i), 8, '0'), 'hex'), \ + jsonb_build_object('pk', jsonb_build_object('S', 'p' || lpad((i / 50)::text, 5, '0')), \ + 'sk', jsonb_build_object('B', encode(decode(lpad(to_hex(i), 8, '0'), 'hex'), 'base64')), \ + 'v', jsonb_build_object('S', repeat('x', 100))) \ + FROM generate_series(1, 20000) AS i", + middle: vec![ + Bind::Text("p00200"), + Bind::Bytes(10000_u32.to_be_bytes().to_vec()), + ], + collate_c: true, + ordered: true, + }, + KeyShape { + label: "multi-part range key (S, N)", + input: key_shape_input("multi_part", &[("sk", "S"), ("sk2", "N")]), + columns: "pk, sk_s, sk2_n, item_data", + rows: "SELECT 'p' || lpad((i / 50)::text, 5, '0'), 'k' || lpad((i / 10)::text, 4, '0'), i, \ + jsonb_build_object('pk', jsonb_build_object('S', 'p' || lpad((i / 50)::text, 5, '0')), \ + 'sk', jsonb_build_object('S', 'k' || lpad((i / 10)::text, 4, '0')), \ + 'sk2', jsonb_build_object('N', i::text), \ + 'v', jsonb_build_object('S', repeat('x', 100))) \ + FROM generate_series(1, 20000) AS i", + middle: vec![ + Bind::Text("p00200"), + Bind::Text("k1000"), + Bind::Text("10000"), + ], + collate_c: true, + ordered: false, + }, + KeyShape { + label: "legacy collation (pk on the database collation)", + input: key_shape_input("legacy", &[("sk", "N")]), + columns: "pk, sk_n, item_data", + rows: "SELECT 'p' || lpad((i / 50)::text, 5, '0'), i, \ + jsonb_build_object('pk', jsonb_build_object('S', 'p' || lpad((i / 50)::text, 5, '0')), \ + 'sk', jsonb_build_object('N', i::text), \ + 'v', jsonb_build_object('S', repeat('x', 100))) \ + FROM generate_series(1, 20000) AS i", + middle: vec![Bind::Text("p00200"), Bind::Text("10000")], + collate_c: false, + ordered: false, + }, + ] +} + +/// On every key shape a table can have, the page query (first page and a +/// page after a key in the table's middle, at the first page's row bound of +/// 16 and at 1000) is `Limit` over one `Index Scan` on the primary key index +/// with no sort, on a 20,000-row table; and the stream over the rows is +/// complete, ordered where the shape has a format order, in 21 round trips +/// (16 rows, then nineteen pages of 1000 and a last of 984). The legacy +/// table carries no `COLLATE "C"` in its query and pages in its index's +/// order. Near the end of a small table the plan differs: +/// `the_last_page_sorts_the_rows_after_the_seek`. +async fn the_page_query_is_an_index_scan_on_every_key_shape(shared: bool) { + let s = scratch_with(shared).await; + for shape in key_shapes() { + let name = shape.input.table_name.clone(); + create_active(&s, shape.input).await; + let desc = s + .engine + .describe_table( + ACCOUNT, + extenddb_core::types::DescribeTableInput { + table_name: name.clone(), + }, + ) + .await + .expect("describe"); + let ddb_table = data_table_name(&desc.table_id); + let data = s.data_pool().await; + sqlx::query(&format!( + "INSERT INTO {ddb_table} ({}) {}", + shape.columns, shape.rows + )) + .execute(&data) + .await + .unwrap_or_else(|e| panic!("{}: fill the table: {e}", shape.label)); + if shape.label.starts_with("legacy") { + sqlx::query(&format!( + "ALTER TABLE {ddb_table} ALTER COLUMN pk TYPE TEXT COLLATE \"default\"" + )) + .execute(&data) + .await + .expect("recollate"); + } + sqlx::query(&format!("ANALYZE {ddb_table}")) + .execute(&data) + .await + .expect("analyze"); + let pkey = format!("{}_pkey", ddb_table.trim_matches('"')); + for n in [ + i64::try_from(extenddb_storage_postgres::MIN_PAGE_ROWS).unwrap(), + 1000, + ] { + let (types, indexes, plan) = explain_page(&data, &desc, shape.collate_c, None, n).await; + assert_index_scan( + &format!("{}, first page at n = {n}", shape.label), + &types, + &indexes, + &plan, + &pkey, + ); + let middle: Vec = shape + .middle + .iter() + .map(|b| match b { + Bind::Text(t) => Bind::Text(t), + Bind::Bytes(v) => Bind::Bytes(v.clone()), + }) + .collect(); + let (types, indexes, plan) = + explain_page(&data, &desc, shape.collate_c, Some(middle), n).await; + assert_index_scan( + &format!("{}, page after a key at n = {n}", shape.label), + &types, + &indexes, + &plan, + &pkey, + ); + } + let sql = + extenddb_storage_postgres::page_sql_for(&desc, &desc.table_id, shape.collate_c, true); + assert_eq!( + sql.contains("COLLATE \"C\""), + shape.collate_c, + "{}: {sql}", + shape.label + ); + data.close().await; + + let fetches_before = extenddb_storage::snapshot::paging::fetches(); + let snapshot = s + .engine + .open_table_snapshot(ACCOUNT, &name) + .await + .expect("open"); + assert_eq!(snapshot.key_ordered, shape.ordered, "{}", shape.label); + let order = extenddb_core::backup_format::KeyOrder::resolve( + &snapshot.description.key_schema, + &snapshot.description.attribute_definitions, + ) + .ok(); + let items: Vec = { + use futures::StreamExt as _; + snapshot + .items + .map(|item| item.expect("item")) + .collect() + .await + }; + assert_eq!(items.len(), 20_000, "{}: every row streams", shape.label); + if shape.ordered { + let order = order.expect("an ordered shape has a format order"); + for pair in items.windows(2) { + assert_eq!( + order.compare(&pair[0], &pair[1]).expect("comparable keys"), + std::cmp::Ordering::Less, + "{}: {:?} is not before {:?}", + shape.label, + pair[0], + pair[1] + ); + } + } + let fetches = extenddb_storage::snapshot::paging::fetches() - fetches_before; + assert_eq!( + fetches, 21, + "{}: a first page of 16, nineteen of 1000 and a last of 984", + shape.label + ); + } +} +on_both_layouts!(the_page_query_is_an_index_scan_on_every_key_shape); + +/// Bytes of item text in a 27 MB row of `scattered_table`: the size of +/// #394's worst-expansion line, so three such rows fill the 64 MiB budget +/// (0, 27, 54 MB before each) and the fourth is the next page's. +const LARGE_ROW_PAD: usize = 27_000_000; + +/// A table of `small` rows of 50-byte text under keys `a0000`.., with a +/// 27 MB row at every `every`th key (`i % every == every - 1`), filled +/// straight into the data table. Key order does not sort items by size, so +/// a table whose large items are a minority is this shape rather than the +/// grouped one (`put-worst`'s `zz` keys after the `a` keys). +async fn scattered_table( + s: &Scratch, + name: &str, + small: usize, + every: usize, +) -> (extenddb_core::types::TableDescription, PgPool) { + filled_table( + s, + name, + &format!( + "SELECT 'a' || lpad(i::text, 4, '0'), i, \ + jsonb_build_object('pk', jsonb_build_object('S', 'a' || lpad(i::text, 4, '0')), \ + 'sk', jsonb_build_object('N', i::text), \ + 'v', jsonb_build_object('S', CASE WHEN i % {every} = {} \ + THEN repeat('x', {LARGE_ROW_PAD}) ELSE repeat('x', 50) END)) \ + FROM generate_series(0, {}) AS i", + every - 1, + small - 1 + ), + ) + .await +} + +/// On a table whose 27 MB rows are scattered among small ones (every 33rd +/// key of 300, nine large rows) the row bound's memory holds: the first +/// page reads 16 small rows, the second asks for the row bound (1000), meets +/// the large rows and reads every row left (284) for the 83 it keeps (up to +/// and including the third large one, whose predecessors hold 54 MB), and +/// every page after it asks for the floor (4) until the table ends, since +/// the memory covers a row bound's worth of rows and the table has fewer: +/// 201 rows read past the cut, all on the second page, over 53 round trips, +/// every row once in key order. With one page of memory the bound returns to +/// 1000 after four small rows and the over-read recurs (306 rows past the +/// cut over 7 pages on this table), which is what the rule corrects; the +/// bound that remains is the second page, and then the page that follows a +/// row bound's worth of small rows, each reading up to `n` rows of up to the +/// line cap: the 201 rows here are that second page's. +async fn the_row_bound_remembers_large_rows_among_small_ones(shared: bool) { + let s = scratch_with(shared).await; + let (_desc, data) = scattered_table(&s, "scattered", 300, 33).await; + data.close().await; + let budget = extenddb_storage::snapshot::paging::SNAPSHOT_FETCH_BYTES; + extenddb_storage::snapshot::paging::reset_page_counters(); + let fetches_before = extenddb_storage::snapshot::paging::fetches(); + let snapshot = s + .engine + .open_table_snapshot(ACCOUNT, "scattered") + .await + .expect("open"); + assert!(snapshot.key_ordered); + let keys: Vec = { + use futures::StreamExt as _; + snapshot + .items + .map(|item| match item.expect("item").get("pk") { + Some(extenddb_core::types::AttributeValue::S(v)) => v.clone(), + other => panic!("pk is {other:?}"), + }) + .collect() + .await + }; + let expected: Vec = (0..300).map(|i| format!("a{i:04}")).collect(); + assert_eq!(keys, expected, "every row once, in key order"); + let fetches = extenddb_storage::snapshot::paging::fetches() - fetches_before; + let past_cut = extenddb_storage::snapshot::paging::rows_past_cut(); + let first = extenddb_storage_postgres::MIN_PAGE_ROWS; + let second_read = 300 - first; + let second_kept = 99 - first; + let rest: usize = 300 - 99; + eprintln!( + "scattered rows: {past_cut} rows read past the cut over {fetches} pages ({} expected, all on \ + the second page)", + second_read - second_kept + ); + assert_eq!( + past_cut, + u64::try_from(second_read - second_kept).unwrap(), + "rows read past the cut: the second page reads the {second_read} rows left for the \ + {second_kept} it keeps, and no later page asks for more than the floor" + ); + // The floor is written as its value: a floor of 2 (the budget divided by + // a 27 MB row) would page a table of such rows two to a page with no page + // reaching its cut, and show here as 103 round trips. + assert_eq!( + fetches, + u64::try_from(2 + rest.div_ceil(4)).unwrap(), + "round trips: two pages, then {rest} rows at the floor of four" + ); + let largest = extenddb_storage::snapshot::paging::max_page_bytes(); + assert!( + largest >= budget && largest < budget + LARGE_ROW_PAD, + "the largest page holds {largest} bytes" + ); +} +on_both_layouts!(the_row_bound_remembers_large_rows_among_small_ones); + +/// A table filled with `rows` straight into its data table and analyzed, as +/// `hundred_k_table` does, with its description and a data pool; `rows` is +/// the `SELECT` producing `(pk, sk_n, item_data)`. +async fn filled_table( + s: &Scratch, + name: &str, + rows: &str, +) -> (extenddb_core::types::TableDescription, PgPool) { + create_active(s, snapshot_conformance::table_input(name)).await; + let desc = s + .engine + .describe_table( + ACCOUNT, + extenddb_core::types::DescribeTableInput { + table_name: name.to_owned(), + }, + ) + .await + .expect("describe"); + let ddb_table = data_table_name(&desc.table_id); + let data = s.data_pool().await; + sqlx::query(&format!( + "INSERT INTO {ddb_table} (pk, sk_n, item_data) {rows}" + )) + .execute(&data) + .await + .expect("fill the table"); + sqlx::query(&format!("ANALYZE {ddb_table}")) + .execute(&data) + .await + .expect("analyze"); + (desc, data) +} + +/// The plan near the end of a table, where the planner's estimate of the +/// rows after the seek is under the row bound, is the planner's choice and +/// not always the index scan: on the 100k-row table a page after the +/// 99,990th row at `n` = 1000 (ten rows left) is still `Limit` over the +/// `Index Scan`, while on a table of 1,060 rows (1,000 under one key prefix, +/// 60 under another) a page after the third +/// of the last 60 at `n` = 810 is a `Bitmap Heap Scan` driven by a `Bitmap +/// Index Scan` on the primary key index and a `Sort` of the 57 rows after +/// the seek under the `Limit`, so that page sorts the rows it reads, with +/// their text (PostgreSQL 16). The cell pins both so the module doc's +/// statement of them stays measured; the seek uses the primary key index in +/// both, and the sorted page returns the rows after the seek in key order. +async fn the_last_page_sorts_the_rows_after_the_seek(shared: bool) { + let s = scratch_with(shared).await; + let (desc, data) = hundred_k_table(&s).await; + let ddb_table = data_table_name(&desc.table_id); + let pkey = vec![format!("{}_pkey", ddb_table.trim_matches('"'))]; + let (types, indexes, plan) = explain_page( + &data, + &desc, + true, + Some(vec![Bind::Text("p01999"), Bind::Text("99990")]), + 1000, + ) + .await; + eprintln!("100k rows, a page after ('p01999', 99990) at n = 1000: {types:?}"); + assert_eq!( + indexes, pkey, + "100k: the seek does not use the primary key index\n{plan}" + ); + assert!( + types.iter().any(|t| t == "Index Scan") && !types.iter().any(|t| t == "Sort"), + "100k rows: the page near the end is not the index scan the module doc says it is \ + (correct the doc if the planner changed): {types:?}\n{plan}" + ); + data.close().await; + + let (desc, data) = filled_table( + &s, + "small", + "SELECT pk, i, jsonb_build_object('pk', jsonb_build_object('S', pk), \ + 'sk', jsonb_build_object('N', i::text), \ + 'v', jsonb_build_object('S', repeat('x', 50))) \ + FROM (SELECT 'a' || lpad(i::text, 4, '0') AS pk, i FROM generate_series(0, 999) AS i \ + UNION ALL \ + SELECT 'zz' || lpad(i::text, 3, '0'), i FROM generate_series(0, 59) AS i) rows", + ) + .await; + let ddb_table = data_table_name(&desc.table_id); + let pkey = vec![format!("{}_pkey", ddb_table.trim_matches('"'))]; + let (types, indexes, plan) = explain_page( + &data, + &desc, + true, + Some(vec![Bind::Text("zz002"), Bind::Text("2")]), + 810, + ) + .await; + eprintln!("1,060 rows, a page after ('zz002', 2) at n = 810: {types:?}"); + assert_eq!( + indexes, pkey, + "1,060: the seek does not use the primary key index\n{plan}" + ); + assert!( + types.iter().any(|t| t == "Sort") && types.iter().any(|t| t == "Bitmap Heap Scan"), + "1,060 rows: the page near the end is not the bitmap scan and Sort the module doc says \ + PostgreSQL 16 chooses there (correct the doc if the planner changed): {types:?}\n{plan}" + ); + // The sorted page's rows: the 57 after the seek, in key order. + let sql = extenddb_storage_postgres::page_sql_for(&desc, &desc.table_id, true, true); + let rows: Vec<(String, String)> = sqlx::query_as(&format!("SELECT c0, c1 FROM ({sql}) q")) + .bind("zz002") + .bind("2") + .bind(810_i64) + .bind(i64::try_from(extenddb_storage::snapshot::paging::SNAPSHOT_FETCH_BYTES).unwrap()) + .fetch_all(&data) + .await + .expect("the last page"); + let expected: Vec<(String, String)> = (3..60) + .map(|i| (format!("zz{i:03}"), i.to_string())) + .collect(); + assert_eq!(rows, expected); + data.close().await; +} +on_both_layouts!(the_last_page_sorts_the_rows_after_the_seek); + /// A hash-only table with no index, for tests that need many of them. fn plain_input(name: &str) -> extenddb_core::types::CreateTableInput { serde_json::from_value(serde_json::json!({ @@ -1227,16 +1892,18 @@ on_both_layouts!(an_open_held_with_deletes_queued_completes); /// A restored table's snapshot description carries the same /// `restore_summary` as `describe_table`'s (both read the `table_restores` -/// row). The record is written directly, as `RestoreTableFromBackup` leaves -/// it. +/// row). The record is written directly, as a finished +/// `RestoreTableFromBackup` leaves it: a DONE legacy job record. async fn a_restored_table_describes_its_restore(shared: bool) { let s = scratch_with(shared).await; create_active(&s, snapshot_conformance::table_input("restored")).await; snapshot_conformance::populate(&s.engine, ACCOUNT, "restored", 5).await; sqlx::query( "INSERT INTO table_restores \ - (table_id, source_backup_arn, source_table_arn, restore_date_time) \ - SELECT table_id, $1, $2, '2026-10-01T12:00:00.250Z' FROM tables \ + (table_id, source_backup_arn, source_table_arn, restore_date_time, \ + status, kind, owner_generation, heartbeat_at, completed_at) \ + SELECT table_id, $1, $2, '2026-10-01T12:00:00.250Z', \ + 'DONE', 'legacy', 0, now(), now() FROM tables \ WHERE table_name = 'restored'", ) .bind("arn:aws:dynamodb:us-east-1:123456789012:table/src/backup/01759320000250-0a1b2c3d") @@ -1251,16 +1918,17 @@ async fn a_restored_table_describes_its_restore(shared: bool) { } on_both_layouts!(a_restored_table_describes_its_restore); -/// The pid of the one backend whose last statement was the stream's `FETCH`. +/// The pid of the one backend whose last statement was the stream's page +/// query (its projection starts with the first key column's alias). async fn stream_backend(s: &Scratch, data: &PgPool) -> i32 { sqlx::query_scalar( "SELECT pid FROM pg_stat_activity WHERE datname = $1 \ - AND state = 'idle in transaction' AND query LIKE 'FETCH%'", + AND state = 'idle in transaction' AND query LIKE 'SELECT page.k0 AS c0%'", ) .bind(s.data_name()) .fetch_one(data) .await - .expect("the stream's backend is idle in its transaction after a FETCH") + .expect("the stream's backend is idle in its transaction after a page query") } /// End the stream's backend from another session, as a server restart or @@ -1434,7 +2102,7 @@ async fn release_wakes_the_active_poller(shared: bool, how: Release) { match snapshot.items.next().await { Some(Ok(_)) => {} Some(Err(_)) => break, - None => panic!("the stream ended without the FETCH error"), + None => panic!("the stream ended without the page query error"), } } kept = Some(snapshot); @@ -1569,7 +2237,7 @@ on_both_layouts!(cancelling_an_open_wakes_the_active_poller); /// first drop attempt, and its end wakes the poller, which drops the table /// within 3 s rather than at its next timed attempt about 9.5 s later. async fn the_in_catalog_backup_wakes_the_active_poller(shared: bool) { - use extenddb_storage::BackupEngine as _; + use extenddb_storage::{JobPermit, LegacyBackupReader as _}; let s = scratch_with(shared).await; create_active(&s, snapshot_conformance::table_input("backed")).await; snapshot_conformance::populate(&s.engine, ACCOUNT, "backed", 20).await; @@ -1585,7 +2253,9 @@ async fn the_in_catalog_backup_wakes_the_active_poller(shared: bool) { .await .expect("hold backups"); - let backup = s.engine.create_backup(ACCOUNT, "backed", "held"); + let backup = s + .engine + .create(ACCOUNT, "backed", "held", JobPermit::detached()); let controller = async { let mut watcher = PgConnection::connect(&format!("{}/postgres", s.base)) .await @@ -1645,7 +2315,7 @@ async fn the_in_catalog_backup_wakes_the_active_poller(shared: bool) { } on_both_layouts!(the_in_catalog_backup_wakes_the_active_poller); -/// A `FETCH` that fails (the backend ended under the stream) ends the stream +/// A page query that fails (the backend ended under the stream) ends the stream /// at the error and frees its permit and connection at once: with the /// errored stream still held a full bound of five opens succeeds. The /// `SnapshotExpired` codes take this branch. @@ -1667,10 +2337,10 @@ async fn a_fetch_error_ends_the_stream_and_frees_its_permit(shared: bool) { match snapshot.items.next().await { Some(Ok(_)) => items += 1, Some(Err(e)) => { - eprintln!("FETCH error after {items} items: {e}"); + eprintln!("page query error after {items} items: {e}"); break; } - None => panic!("the stream ended without reporting the FETCH error"), + None => panic!("the stream ended without reporting the page query error"), } } assert_eq!(items, 5, "the first page, then the error"); diff --git a/crates/storage-postgres/tests/vector_control_plane.rs b/crates/storage-postgres/tests/vector_control_plane.rs index 6ff14fd35..1f6fc3213 100644 --- a/crates/storage-postgres/tests/vector_control_plane.rs +++ b/crates/storage-postgres/tests/vector_control_plane.rs @@ -33,7 +33,7 @@ use extenddb_core::types::{ }; use extenddb_storage::error::StorageError; use extenddb_storage::{ - BackupEngine, DataEngine, RestoreRequest, RestoreTableOverrides, TableEngine, + DataEngine, JobPermit, LegacyBackupReader, RestoreRequest, RestoreTableOverrides, TableEngine, }; use extenddb_storage_postgres::{PostgresConfig, PostgresEngine}; use sqlx::PgPool; @@ -415,11 +415,11 @@ async fn restore( target: &str, backup_arn: &str, ) -> Result, StorageError> { - let source = s.engine.backup_restore_source(ACCOUNT, backup_arn).await?; + let source = s.engine.restore_source(ACCOUNT, backup_arn).await?; source.ensure_restorable()?; let input = source.restore_input(target, &RestoreTableOverrides::default()); s.engine - .restore_table_from_backup( + .restore( ACCOUNT, RestoreRequest { backup_arn: backup_arn.to_owned(), @@ -427,6 +427,7 @@ async fn restore( restore_date_time: source.backup_creation_date_time, input, }, + JobPermit::detached(), ) .await?; let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(60); @@ -990,7 +991,7 @@ async fn restoring_a_backup_that_carries_vector_indexes_is_refused() { .expect("create a table with a vector index"); let details = s .engine - .create_backup(ACCOUNT, "t_source", "b_vector") + .create(ACCOUNT, "t_source", "b_vector", JobPermit::detached()) .await .expect("back up a table with a vector index"); @@ -1019,7 +1020,7 @@ async fn restoring_a_backup_that_carries_vector_indexes_is_refused() { .expect("drop the definition row"); let legacy = s .engine - .backup_restore_source(ACCOUNT, &details.backup_arn) + .restore_source(ACCOUNT, &details.backup_arn) .await .expect("legacy source"); assert!(legacy.definition.is_none()); @@ -1047,7 +1048,7 @@ async fn restoring_a_backup_that_carries_vector_indexes_is_refused() { .expect("create a plain table"); let plain = s .engine - .create_backup(ACCOUNT, "t_plain_source", "b_plain") + .create(ACCOUNT, "t_plain_source", "b_plain", JobPermit::detached()) .await .expect("back up a plain table"); let status = restore(&s, "t_plain_restored", &plain.backup_arn) @@ -1302,7 +1303,7 @@ async fn a_backup_snapshot_carries_the_wire_shape_behind_a_version() { .expect("create a table with a vector index"); let details = s .engine - .create_backup(ACCOUNT, "t_snapshot", "b_snapshot") + .create(ACCOUNT, "t_snapshot", "b_snapshot", JobPermit::detached()) .await .expect("back up the table"); @@ -1356,7 +1357,7 @@ async fn a_backup_taken_before_the_snapshot_column_existed_still_restores() { .expect("create a plain table"); let details = s .engine - .create_backup(ACCOUNT, "t_legacy", "b_legacy") + .create(ACCOUNT, "t_legacy", "b_legacy", JobPermit::detached()) .await .expect("back up the table"); @@ -1378,7 +1379,7 @@ async fn a_backup_taken_before_the_snapshot_column_existed_still_restores() { .expect("drop the definition row"); let source = s .engine - .backup_restore_source(ACCOUNT, &details.backup_arn) + .restore_source(ACCOUNT, &details.backup_arn) .await .expect("source"); assert!(source.definition.is_none()); diff --git a/crates/storage-sqlite/Cargo.toml b/crates/storage-sqlite/Cargo.toml index 589030ade..96a432a1a 100644 --- a/crates/storage-sqlite/Cargo.toml +++ b/crates/storage-sqlite/Cargo.toml @@ -42,6 +42,11 @@ libsqlite3-sys = { workspace = true } [dev-dependencies] extenddb-storage = { workspace = true, features = ["test-util"] } +# `tests/backup_job_panic.rs` drives the engine's CreateBackup job and +# reconciler against a `SqliteEngine` (the engine depends on `extenddb-storage`, +# not on this crate, so the edge is acyclic). +extenddb-engine = { workspace = true } +extenddb-backup-store = { workspace = true } # The slow-statement test sets sqlx's threshold, a `log` level, and prints # every tracing event of a child process. log = "0.4" diff --git a/crates/storage-sqlite/src/backup.rs b/crates/storage-sqlite/src/backup.rs index 8c9bdc8ab..5216386c4 100644 --- a/crates/storage-sqlite/src/backup.rs +++ b/crates/storage-sqlite/src/backup.rs @@ -1,7 +1,7 @@ // Copyright 2026 ExtendDB contributors // SPDX-License-Identifier: Apache-2.0 -//! `BackupEngine` for the SQLite backend. +//! The in-catalog backup format on SQLite: its `LegacyBackupReader`. //! //! A backup snapshots every item's `item_data` into `backup_items` and the //! table's definition (secondary indexes, billing mode and throughput, table @@ -10,26 +10,69 @@ //! //! RestoreTableFromBackup receives the target's `CreateTableInput` already //! resolved and validated by the engine. It creates the target CREATING and -//! registers the restore in one write transaction, returns, and copies the +//! registers the restore, with its job record (RUNNING, `kind = legacy`, the +//! claim's generation), in one write transaction, returns, and copies the //! snapshot in a detached task, filling the secondary indexes as it goes in -//! bounded batches. The task flips the target ACTIVE, or removes it if the -//! copy fails; the startup sweep removes a target a process exit left -//! CREATING. +//! bounded batches while a heartbeat refreshes the record. The task flips +//! the target ACTIVE and the record DONE in one transaction, or marks the +//! record FAILED and removes the target if the copy fails; a target a +//! process exit left CREATING is removed by the engine's reconciler once its +//! record's heartbeat is stale, and a restart ages every record of the +//! stopped process at startup so that happens on the first cycle. +//! +//! A create commits a CREATING `backups` row before its copy and holds an +//! in-process slot for the row's life; a CREATING row whose slot no task +//! holds (a create that died, or one of a process that stopped) is what +//! `abandoned_creating_backups` reports for the reconciler to remove. Its +//! `stale_after` plays no part on this backend: a slot no task holds is +//! proof of death, and `backups` has no heartbeat column to age. The +//! reader's `delete` of such a row removes the row whole, with its items and +//! definition, and leaves no DELETED tombstone: a backup that never finished +//! was never visible as one, and a DELETED row would describe one. The +//! engine's DeleteBackup reaches the same `delete` for a legacy ARN, so in +//! the window before the reconciler's next cycle a client's DeleteBackup of +//! a backup ListBackups shows CREATING answers DELETED (the row is removed), +//! where main answered BackupNotFound; the service's BackupInUse for a +//! CREATING backup is the catalog-index row's rule, not this reader's. +//! +//! A restore's heartbeat runs every +//! [`JOB_HEARTBEAT_INTERVAL`](extenddb_storage::backup_catalog::JOB_HEARTBEAT_INTERVAL), +//! the one constant every backend's heartbeat loop reads and the stale floor +//! is computed from (`[backup] heartbeat_interval_secs` reaches the backends +//! in a later release), under the engine write lock, as every writer does, +//! so a long write transaction delays it; the stale window the reconciler +//! waits (`stale_job_after_secs`, at least three heartbeats) is what absorbs +//! that. +//! +//! A create holds one of [`backup_server_slot_count`] server slots for its +//! snapshot's life, besides the engine's `max_concurrent_jobs` permit it +//! arrives with. The slot stays until a later release folds it into +//! `max_concurrent_jobs`: it is what bounds the pooled connections a flood +//! of snapshot opens holds, and the two bounds can differ. A create whose +//! slot is not free is refused at once with the one `LimitExceededException` +//! text ([`limit_exceeded_message`]), as the engine's permit refuses, rather +//! than waiting for one: a queued create would hold nothing yet, but the +//! caller would learn nothing for as long as it waited. use extenddb_core::types::{ AttributeDefinition, BackupDescription, BackupDetails, BackupSummary, GsiInput, Item, KeySchemaElement, LsiInput, Projection, SourceTableDetails, StreamSpecification, TableDescription, TimeToLiveDescription, TimeToLiveStatus, }; +use extenddb_storage::backup_catalog::{ + AbandonedLegacyBackup, JOB_HEARTBEAT_INTERVAL, JobBound, JobClaim, JobStatus, StatusFields, + WriteOutcome, limit_exceeded_message, new_generation, +}; use extenddb_storage::backup_definition::{ BACKUP_DEFINITION_VERSION, BackupTableDefinition, COPY_BATCH_BYTES, COPY_BATCH_ITEMS, throughput_from_catalog, }; use extenddb_storage::error::StorageError; use extenddb_storage::util::{composite_pk_to_text, parse_sk, sk_column_n}; -use extenddb_storage::{BackupEngine, BackupRestoreSource, RestoreRequest}; +use extenddb_storage::{BackupRestoreSource, JobPermit, LegacyBackupReader, RestoreRequest}; use futures::future::BoxFuture; +use crate::backup_catalog::{JobSlots, SqliteJobClaim, generation_column, legacy_backup_slot_key}; use crate::create_table::RestoreRegistration; use crate::data::{ BoundValue, all_sort_key_info, data_table_name, index_table_name, insert_index_row_multi, @@ -41,10 +84,6 @@ use crate::store::SqliteEngine; /// Upper limit on [`backup_server_slot_count`]. const BACKUP_SERVER_SLOTS_MAX: usize = 16; -/// How long a CreateBackup waits for a server slot before it is refused with -/// `LimitExceeded`. -pub(crate) const BACKUP_SLOT_WAIT: std::time::Duration = std::time::Duration::from_secs(30); - /// Backups and table snapshots the whole process may run at once, across /// accounts. The formula is stated once, on `SqliteConfig::pool_size` /// (`config.rs`); this computes it. @@ -65,28 +104,77 @@ pub(crate) fn backup_server_slot_count(pool_size: u32) -> usize { .clamp(1, BACKUP_SERVER_SLOTS_MAX) } -/// Take a server slot for a backup, waiting at most `wait`; `LimitExceeded` -/// when none frees up in time. -pub(crate) async fn acquire_backup_permit( - slots: std::sync::Arc, - slot_count: usize, - wait: std::time::Duration, -) -> Result { - tokio::time::timeout(wait, slots.acquire_owned()) - .await - .map_err(|_| slots_exhausted(slot_count))? - .map_err(|e| StorageError::Internal(format!("backup slot: {e}"))) +/// The pool a server needs so that [`backup_server_slot_count`] admits every +/// job `backup.max_concurrent_jobs` admits: `pool_size` when its slot count +/// already meets the jobs, else the smallest pool whose slot count does, the +/// slot cap included (the rule is stated once, on `SqliteConfig::pool_size`). +/// The in-memory pool is not sized by this: it has one connection by nature. +pub(crate) fn pool_size_for_jobs(pool_size: u32, max_concurrent_jobs: usize) -> u32 { + let wanted = max_concurrent_jobs.min(BACKUP_SERVER_SLOTS_MAX); + if backup_server_slot_count(pool_size) >= wanted { + return pool_size; + } + let slots = u32::try_from(wanted).unwrap_or(u32::MAX); + slots.saturating_add(1).saturating_mul(2) +} + +/// The warning a server logs when [`pool_size_for_jobs`] raised its pool: +/// both numbers, the raised pool's slot count, and what the slots admit. +/// With the jobs within the slot cap the slots admit every job the +/// `max_concurrent_jobs` permit admits; over it the slots are capped at +/// [`BACKUP_SERVER_SLOTS_MAX`], and the line says which jobs the slot bound +/// refuses (the limits document's "above 16 jobs a 17th concurrent job is +/// refused by the slot"). +pub(crate) fn pool_raise_message( + pool_size: u32, + for_jobs: u32, + max_concurrent_jobs: usize, +) -> String { + let slots = backup_server_slot_count(for_jobs); + let admits = if slots < max_concurrent_jobs { + format!( + "the slots are capped at {BACKUP_SERVER_SLOTS_MAX}, so jobs {} to \ + {max_concurrent_jobs} are refused by the slot bound", + slots + 1 + ) + } else { + "the slot bound refuses no job the max_concurrent_jobs permit admitted".to_owned() + }; + format!( + "SQLite pool_size {pool_size} raised to {for_jobs} for backup.max_concurrent_jobs = \ + {max_concurrent_jobs}: a pool of {for_jobs} has {slots} server slots, one per running \ + backup job, so {admits}" + ) } -/// The refusal when every server slot is held. Backups and table snapshots -/// (`snapshot_engine.rs`) share the slots, so the message names both: with -/// snapshots holding every slot a `CreateBackup` is refused with no backup -/// running. -pub(crate) fn slots_exhausted(slot_count: usize) -> StorageError { - StorageError::LimitExceeded(format!( - "Too many backups or table snapshots are in progress on this server (at most \ - {slot_count} at once); retry later" - )) +/// How the slot refusal names its bound, beside the number: the engine's +/// `max_concurrent_jobs` refusal carries the same text with its own name, so +/// an operator can tell which bound a request met. The shared text reads +/// "the bound", so the name is the setting the count derives from and +/// a qualifier, with no comma: "the [storage.sqlite] pool_size snapshot-slot +/// bound". +pub(crate) const BACKUP_SERVER_SLOTS_NAME: &str = "[storage.sqlite] pool_size snapshot-slot"; + +/// Take a server slot for a backup or a table snapshot (`snapshot_engine.rs`) +/// without waiting; `LimitExceeded`, in the one text every job refusal +/// carries, naming this bound, when none is free. The slots are shared, so with snapshots +/// holding every slot a `CreateBackup` is refused with no backup running. +pub(crate) fn acquire_backup_permit( + slots: &std::sync::Arc, + slot_count: usize, +) -> Result { + use tokio::sync::TryAcquireError; + std::sync::Arc::clone(slots) + .try_acquire_owned() + .map_err(|e| match e { + TryAcquireError::NoPermits => { + StorageError::LimitExceeded(limit_exceeded_message(JobBound::Counted { + name: BACKUP_SERVER_SLOTS_NAME, + limit: slot_count, + })) + } + TryAcquireError::Closed => StorageError::Internal(format!("backup slot: {e}")), + }) } /// Read the next batch of `(cursor, item_data)` from `table` after @@ -271,12 +359,14 @@ impl SqliteEngine { /// each RANGE part into its typed column), and each row is a plain /// INSERT, so two backup rows for one key fail the restore instead of /// collapsing into one item. Items are read and written in byte-bounded - /// batches, each in its own write transaction, and the target is flipped - /// to ACTIVE in the last one. Returns the number of items copied. + /// batches, each in its own write transaction, and in the last one the + /// job record is marked DONE, conditioned on `generation`, and the + /// target flipped to ACTIVE. Returns the number of items copied. async fn copy_backup_items( &self, desc: &TableDescription, backup_arn: &str, + generation: u64, ) -> Result { let ddb_table = data_table_name(&desc.table_id); let sort_keys = all_sort_key_info(&desc.key_schema, &desc.attribute_definitions); @@ -320,14 +410,14 @@ impl SqliteEngine { // no one sees it part-filled. Each batch first re-checks, under // the write lock, that the backup is still AVAILABLE (DeleteBackup // removes its rows and marks it DELETED in one transaction) and - // that the target is still this restore's. + // that the target is still this restore's, under this generation. let _writer = self.write_lock.lock().await; let mut tx = self .pool .begin_with("BEGIN IMMEDIATE") .await .map_err(db_err)?; - Self::ensure_restore_can_continue(&mut tx, desc, backup_arn).await?; + Self::ensure_restore_can_continue(&mut tx, desc, backup_arn, generation).await?; let batch = next_batch( &mut tx, "backup_items", @@ -343,19 +433,27 @@ impl SqliteEngine { .fetch_one(&mut *tx) .await .map_err(db_err)?; - // Only from CREATING, which the check above confirmed under - // the same lock. - sqlx::query( - "UPDATE tables SET item_count = ?, table_size_bytes = ?, \ - table_status = 'ACTIVE', status_transition_at = NULL \ - WHERE table_id = ? AND table_status = 'CREATING'", + // The job record DONE, only while it is RUNNING under this + // generation, and the table ACTIVE only from CREATING, which + // the check above confirmed under the same lock. A record + // another claim took over leaves the table to that owner. + let activated = crate::restore_engine::activate_restore_target( + &mut tx, + &desc.table_id, + generation, + count, + table_size, + Some(count), ) - .bind(count) - .bind(table_size) - .bind(&desc.table_id) - .execute(&mut *tx) - .await - .map_err(db_err)?; + .await?; + if !activated.record { + tx.rollback().await.map_err(db_err)?; + return Err(StorageError::TableNotFound(format!( + "restore of {backup_arn} into {} did not complete: its job record \ + passed to another owner", + desc.table_name + ))); + } tx.commit().await.map_err(db_err)?; return Ok(count); }; @@ -410,12 +508,14 @@ impl SqliteEngine { } } - /// Fail a restore whose backup was deleted or whose target is no longer - /// the one it created. + /// Fail a restore whose backup was deleted, whose target is no longer the + /// one it created, or whose job record is no longer RUNNING under + /// `generation`. async fn ensure_restore_can_continue( tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>, desc: &TableDescription, backup_arn: &str, + generation: u64, ) -> Result<(), StorageError> { let available: bool = sqlx::query_scalar( "SELECT EXISTS(SELECT 1 FROM backups WHERE backup_arn = ? \ @@ -434,10 +534,12 @@ impl SqliteEngine { "SELECT EXISTS(SELECT 1 FROM tables t \ JOIN table_restores r ON r.table_id = t.table_id \ WHERE t.table_id = ? AND r.source_backup_arn = ? \ - AND t.table_status = 'CREATING' AND t.status_transition_at IS NULL)", + AND t.table_status = 'CREATING' AND t.status_transition_at IS NULL \ + AND r.status = 'RUNNING' AND r.owner_generation = ?)", ) .bind(&desc.table_id) .bind(backup_arn) + .bind(generation_column(generation)) .fetch_one(&mut **tx) .await .map_err(db_err)?; @@ -452,38 +554,95 @@ impl SqliteEngine { } /// The detached half of RestoreTableFromBackup: copy the backup into the - /// registered CREATING target, which flips it ACTIVE in the copy's last - /// batch, or remove the target if the copy fails. Logs either outcome; a - /// process exit part-way is cleaned up by the startup sweep. - async fn finish_restore(&self, desc: &TableDescription, backup_arn: &str) { + /// registered CREATING target while the claim heartbeats the job record, + /// which flips it ACTIVE and the record DONE in the copy's last batch, or + /// mark the record FAILED and remove the target if the copy fails. Logs + /// either outcome. + async fn finish_restore( + &self, + desc: &TableDescription, + backup_arn: &str, + claim: &SqliteJobClaim, + ) { let target = &desc.table_name; let started = std::time::Instant::now(); - match self.copy_backup_items(desc, backup_arn).await { - Ok(items) => tracing::info!( - "restore of {backup_arn} into table '{target}' finished: {items} items in \ - {:.3}s; the table is ACTIVE", - started.elapsed().as_secs_f64() - ), - Err(e) => { + let outcome = { + let copy = self.copy_backup_items(desc, backup_arn, claim.generation()); + let heartbeat = claim.heartbeat_loop(JOB_HEARTBEAT_INTERVAL); + tokio::select! { + copied = copy => copied, + lost = heartbeat => { + tracing::error!( + "restore of {backup_arn} into table '{target}': the claim lost its \ + heartbeat ({lost}); stopping the copy" + ); + Err(StorageError::Internal(format!("claim lost: {lost}"))) + } + } + }; + let e = match outcome { + Ok(items) => { + tracing::info!( + "restore of {backup_arn} into table '{target}' finished: {items} items in \ + {:.3}s; the table is ACTIVE", + started.elapsed().as_secs_f64() + ); + return; + } + Err(e) => e, + }; + // The target must not stay behind: it is CREATING with no scheduled + // transition, so nothing else would move it on, and the name would + // stay taken. The job record is marked FAILED first, conditioned on + // the generation and on RUNNING; only if that matched is the target + // removed (which cascades the record). A record another generation + // owns means its holder owns the target, and nothing further is + // touched. + let failed = StatusFields { + failure_reason: Some(e.to_string()), + completed: true, + ..StatusFields::default() + }; + match claim + .set_status(JobStatus::Running, JobStatus::Failed, &failed) + .await + { + Ok(WriteOutcome::Matched) => {} + Ok(_) => { + tracing::warn!( + "restore of {backup_arn} into table '{target}' ended with \"{e}\"; its job \ + record was no longer RUNNING under this claim, so the target ({}) was \ + left to its owner", + desc.table_id + ); + return; + } + Err(write) => { tracing::error!( - "restore of {backup_arn} into table '{target}' failed, removing the \ - partial table: {e}" + "restore of {backup_arn} into table '{target}' failed ({e}), and its job \ + record could not be marked FAILED ({write}); the reconciler removes the \ + target once its heartbeat is stale" ); - match self.abort_restore(&desc.table_id).await { - Ok(true) => tracing::info!( - "removed partially restored table '{target}' ({})", - desc.table_id - ), - // Deleted or already gone: nothing of this restore remains. - Ok(false) => {} - Err(cleanup) => tracing::error!( - "could not remove partially restored table '{target}' ({}); the next \ - startup will: {cleanup}", - desc.table_id - ), - } + return; } } + tracing::error!( + "restore of {backup_arn} into table '{target}' failed, removing the partial \ + table: {e}" + ); + match self.abort_restore(&desc.table_id).await { + Ok(true) => tracing::info!( + "removed partially restored table '{target}' ({})", + desc.table_id + ), + // Deleted or already gone: nothing of this restore remains. + Ok(false) => {} + Err(cleanup) => tracing::error!( + "could not remove partially restored table '{target}' ({}); the reconciler \ + will: {cleanup}", + desc.table_id + ), + } } /// Remove a restore target whose copy failed or was abandoned: its @@ -491,7 +650,7 @@ impl SqliteEngine { /// target still CREATING with no scheduled transition is removed, and by /// table id, so this can never touch a table the restore did not create /// or one a client already deleted. - async fn abort_restore(&self, table_id: &str) -> Result { + pub(crate) async fn abort_restore(&self, table_id: &str) -> Result { let _writer = self.write_lock.lock().await; let mut tx = self .pool @@ -814,62 +973,15 @@ impl SqliteEngine { tx.commit().await.map_err(db_err)?; Ok(removed) } - - /// Remove backups left CREATING by a process that died during its copy. - /// The backups delete cascades to definitions and any copied item rows. - pub async fn sweep_incomplete_backups(&self) -> Result, StorageError> { - let _writer = self.write_lock.lock().await; - let mut tx = self - .pool - .begin_with("BEGIN IMMEDIATE") - .await - .map_err(db_err)?; - let removed: Vec = sqlx::query_scalar( - "DELETE FROM backups WHERE backup_status = 'CREATING' RETURNING backup_arn", - ) - .fetch_all(&mut *tx) - .await - .map_err(db_err)?; - tx.commit().await.map_err(db_err)?; - Ok(removed) - } - - /// Remove restore targets left by a process that died mid-restore. - /// - /// A restore target is the only table that sits CREATING with no - /// scheduled transition. Called once at startup, before any request is - /// served, so no restore of this process can be in flight and every such - /// table is abandoned. - /// - /// This assumes one server process per database file, which the backend - /// already depends on: writers are serialized by an in-process lock, so a - /// second process writing the same file is unsupported regardless. A - /// second process started against the file while the first is mid-restore - /// would remove that restore's target. - pub(crate) async fn sweep_abandoned_restores(&self) -> Result, StorageError> { - let candidates: Vec<(String, String)> = sqlx::query_as( - "SELECT table_id, table_name FROM tables \ - WHERE table_status = 'CREATING' AND status_transition_at IS NULL", - ) - .fetch_all(&self.pool) - .await - .map_err(db_err)?; - let mut removed = Vec::new(); - for (table_id, table_name) in candidates { - if self.abort_restore(&table_id).await? { - removed.push(table_name); - } - } - Ok(removed) - } } -impl BackupEngine for SqliteEngine { - fn create_backup( +impl LegacyBackupReader for SqliteEngine { + fn create( &self, account_id: &str, table_name: &str, backup_name: &str, + job_permit: JobPermit, ) -> BoxFuture<'_, Result> { let engine = self.clone(); let account_id = account_id.to_owned(); @@ -879,20 +991,20 @@ impl BackupEngine for SqliteEngine { // Admission: at most backup_server_slot_count backups run at once, // so a flood of CreateBackups cannot hold every pooled connection // (each holds its snapshot reader for its whole duration). The - // permit travels with the detached task and is released when the - // copy ends, whatever happened to the request. + // slot is tried, not waited for. The permit travels with the + // detached task and is released when the copy ends, whatever + // happened to the request. let permit = acquire_backup_permit( - std::sync::Arc::clone(&engine.backup_server_slots), + &engine.backup_server_slots, engine.backup_server_slot_count, - BACKUP_SLOT_WAIT, - ) - .await?; + )?; // Detach the operation from the request future. Once the CREATING // row commits, dropping an HTTP request must not drop the copy and // strand wire-visible state. A dropped JoinHandle leaves its Tokio // task running, so the copy reaches AVAILABLE or executes cleanup. let task = tokio::spawn(async move { let _permit = permit; + let _job_permit = job_permit; // One id gives the ARN and the creation time, so the ARN, the // reported time, and the ListBackups cursor agree. let backup_id = extenddb_storage::util::new_backup_id(); @@ -906,6 +1018,15 @@ impl BackupEngine for SqliteEngine { .ok_or_else(|| { StorageError::Internal(format!("backup id {backup_id} has no timestamp")) })?; + // The in-process slot that marks the CREATING row as owned, + // taken before the row exists and held until the row is + // AVAILABLE or removed. The id is fresh, so the slot is free. + let _slot = JobSlots::take(&engine.job_slots, legacy_backup_slot_key(&backup_arn)) + .ok_or_else(|| { + StorageError::Internal(format!( + "backup {backup_arn} is already being created" + )) + })?; let created = if engine.in_memory { engine .create_backup_single_connection( @@ -931,7 +1052,7 @@ impl BackupEngine for SqliteEngine { if let Err(cleanup) = engine.remove_incomplete_backup(&backup_arn).await { tracing::error!( "backup {backup_arn} failed ({error}); could not remove its CREATING \ - row: {cleanup}; the next startup will retry cleanup" + row: {cleanup}; the reconciler removes it once this task is gone" ); } return Err(error); @@ -944,7 +1065,7 @@ impl BackupEngine for SqliteEngine { }) } - fn describe_backup( + fn describe( &self, account_id: &str, backup_arn: &str, @@ -1006,7 +1127,7 @@ impl BackupEngine for SqliteEngine { }) } - fn list_backups( + fn list( &self, account_id: &str, table_name: Option<&str>, @@ -1065,7 +1186,7 @@ impl BackupEngine for SqliteEngine { }) } - fn delete_backup( + fn delete( &self, account_id: &str, backup_arn: &str, @@ -1075,8 +1196,14 @@ impl BackupEngine for SqliteEngine { Box::pin(async move { // Resolves account-scoped, so a backup owned by another account is // reported missing here and the writes below never run. - let desc = self.describe_backup(&account_id, &backup_arn).await?; - if desc.backup_details.backup_status != "AVAILABLE" { + let desc = self.describe(&account_id, &backup_arn).await?; + // A CREATING row whose create is dead (no task of this process + // holds its slot) is deletable: nothing will ever finish it, and + // the reconciler removes it through this path. One whose create + // is running is not a backup yet. + let dead_creating = desc.backup_details.backup_status == "CREATING" + && !self.job_slots.holds(&legacy_backup_slot_key(&backup_arn)); + if desc.backup_details.backup_status != "AVAILABLE" && !dead_creating { return Err(StorageError::Validation(format!( "Backup not found: {backup_arn}" ))); @@ -1100,18 +1227,10 @@ impl BackupEngine for SqliteEngine { // Refuse while a restore from this backup is still running, as the // service does. Under the write lock, which the restore also takes // to record itself, so the check cannot miss one. - let restoring: Option = sqlx::query_scalar( - "SELECT t.table_name FROM table_restores r JOIN tables t ON t.table_id = r.table_id \ - WHERE r.source_backup_arn = ? AND t.table_status = 'CREATING' LIMIT 1", - ) - .bind(&backup_arn) - .fetch_optional(&mut *tx) - .await - .map_err(db_err)?; - if let Some(table) = restoring { - return Err(StorageError::BackupInUse(format!( - "Backup is being used to restore table {table}: {backup_arn}" - ))); + if let Some(table) = + crate::backup_catalog::creating_restore_target(&mut *tx, &backup_arn).await? + { + return Err(crate::backup_catalog::backup_in_use(&table, &backup_arn)); } sqlx::query( "DELETE FROM backup_items WHERE backup_arn = ?1 AND EXISTS (\ @@ -1131,15 +1250,29 @@ impl BackupEngine for SqliteEngine { .execute(&mut *tx) .await .map_err(|e| StorageError::Internal(e.to_string()))?; - sqlx::query( - "UPDATE backups SET backup_status = 'DELETED' \ - WHERE backup_arn = ? AND account_id = ?", - ) - .bind(&backup_arn) - .bind(&account_id) - .execute(&mut *tx) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; + if dead_creating { + // Never a backup: no tombstone, as the failed create would + // have left none. + sqlx::query( + "DELETE FROM backups WHERE backup_arn = ? AND account_id = ? \ + AND backup_status = 'CREATING'", + ) + .bind(&backup_arn) + .bind(&account_id) + .execute(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + } else { + sqlx::query( + "UPDATE backups SET backup_status = 'DELETED' \ + WHERE backup_arn = ? AND account_id = ?", + ) + .bind(&backup_arn) + .bind(&account_id) + .execute(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + } tx.commit().await.map_err(db_err)?; Ok(BackupDescription { backup_details: BackupDetails { @@ -1152,7 +1285,7 @@ impl BackupEngine for SqliteEngine { }) } - fn backup_restore_source( + fn restore_source( &self, account_id: &str, backup_arn: &str, @@ -1251,10 +1384,11 @@ impl BackupEngine for SqliteEngine { }) } - fn restore_table_from_backup( + fn restore( &self, account_id: &str, request: RestoreRequest, + permit: JobPermit, ) -> BoxFuture<'_, Result> { let engine = self.clone(); let account_id = account_id.to_owned(); @@ -1262,7 +1396,8 @@ impl BackupEngine for SqliteEngine { // The whole restore runs in a task that owns its inputs, so the // request future being dropped (a client timeout) stops neither the // registration nor the copy. The request waits only until the - // CREATING target and its table_restores row have committed. + // CREATING target and its job record have committed. The job + // permit lives with the claim and is freed when the copy ends. let (registered_tx, registered_rx) = tokio::sync::oneshot::channel(); tokio::spawn(async move { let RestoreRequest { @@ -1273,12 +1408,16 @@ impl BackupEngine for SqliteEngine { } = request; // One write transaction: re-check the backup is AVAILABLE and // owned by the account, insert the CREATING target, record the - // restore. DeleteBackup takes the same lock, so it sees either - // no target or a CREATING target that names this backup. + // restore with its job record under a fresh generation (the + // insert is the claim). DeleteBackup takes the same lock, so + // it sees either no target or a CREATING target that names + // this backup. + let generation = new_generation(); let registration = RestoreRegistration { backup_arn: &backup_arn, source_table_arn: &source_table_arn, restore_date_time, + owner_generation: generation, }; let registered = engine .create_table_for_restore(&account_id, input, ®istration) @@ -1299,9 +1438,11 @@ impl BackupEngine for SqliteEngine { return; } }; + let claim = + SqliteJobClaim::for_restore(&engine, &desc.table_id, generation, permit); // A closed receiver means the request is gone; the copy goes on. let _ = registered_tx.send(Ok(desc.clone())); - engine.finish_restore(&desc, &backup_arn).await; + engine.finish_restore(&desc, &backup_arn, &claim).await; }); registered_rx.await.map_err(|_| { StorageError::Internal( @@ -1310,6 +1451,78 @@ impl BackupEngine for SqliteEngine { })? }) } + + /// CREATING `backups` rows no running create holds: a create commits its + /// row before the copy and holds an in-process slot for it until the row + /// is AVAILABLE or removed, so a row with no slot was left by a create + /// that died or by a process that stopped (a restart frees every slot). + /// A slot no task holds is proof of death here, so `stale_after` plays no + /// part. + /// + /// The removal is this call's (`removed: true`), one write transaction + /// under the engine write lock: the CREATING rows are read, the slot + /// check runs and the row, its items and its definition are deleted + /// `WHERE backup_status = 'CREATING'`, all while no other writer can + /// commit. A create commits AVAILABLE under the same lock and frees its + /// slot only after that commit, so a create that finishes between a read + /// and a later removal (the window a report-then-`delete` pair leaves, + /// where `delete` also removes an AVAILABLE backup) cannot lose its + /// backup: either its slot is still held when this write runs, or its + /// row is AVAILABLE and the status predicate excludes it. No tombstone + /// is left: a backup that never finished was never visible as one. + fn abandoned_creating_backups( + &self, + _stale_after: std::time::Duration, + ) -> BoxFuture<'_, Result, StorageError>> { + Box::pin(async move { + let _writer = self.write_lock.lock().await; + let mut tx = self + .pool + .begin_with("BEGIN IMMEDIATE") + .await + .map_err(db_err)?; + let creating: Vec<(String, String)> = sqlx::query_as( + "SELECT account_id, backup_arn FROM backups WHERE backup_status = 'CREATING' \ + ORDER BY backup_arn", + ) + .fetch_all(&mut *tx) + .await + .map_err(db_err)?; + let mut removed = Vec::new(); + for (account_id, backup_arn) in creating { + if self.job_slots.holds(&legacy_backup_slot_key(&backup_arn)) { + continue; + } + let gone = sqlx::query( + "DELETE FROM backups WHERE backup_arn = ? AND account_id = ? \ + AND backup_status = 'CREATING'", + ) + .bind(&backup_arn) + .bind(&account_id) + .execute(&mut *tx) + .await + .map_err(db_err)? + .rows_affected(); + if gone != 1 { + continue; + } + for table in ["backup_items", "backup_definitions"] { + sqlx::query(&format!("DELETE FROM {table} WHERE backup_arn = ?")) + .bind(&backup_arn) + .execute(&mut *tx) + .await + .map_err(db_err)?; + } + removed.push(AbandonedLegacyBackup { + account_id, + backup_arn, + removed: true, + }); + } + tx.commit().await.map_err(db_err)?; + Ok(removed) + }) + } } #[cfg(test)] @@ -1318,10 +1531,15 @@ mod restore_tests { use extenddb_core::types::{Item, TableDescription, TableKeyInfo}; use extenddb_storage::error::StorageError; - use extenddb_storage::{BackupEngine, DataEngine, RestoreRequest, RestoreTableOverrides}; + use extenddb_storage::{ + DataEngine, JobPermit, LegacyBackupReader, RestoreRequest, RestoreTableOverrides, + }; use serde_json::json; - use super::{acquire_backup_permit, backup_server_slot_count}; + use super::{ + BACKUP_SERVER_SLOTS_NAME, JobBound, acquire_backup_permit, backup_server_slot_count, + limit_exceeded_message, + }; use crate::SqliteEngine; use crate::create_table::RestoreRegistration; @@ -1334,7 +1552,7 @@ mod restore_tests { target: &str, backup_arn: &str, ) -> Result { - let source = engine.backup_restore_source(ACCOUNT, backup_arn).await?; + let source = engine.restore_source(ACCOUNT, backup_arn).await?; source.ensure_restorable()?; let input = source.restore_input(target, &RestoreTableOverrides::default()); Ok(RestoreRequest { @@ -1360,7 +1578,9 @@ mod restore_tests { backup_arn: &str, ) -> Result { let request = restore_request(engine, target, backup_arn).await?; - engine.restore_table_from_backup(ACCOUNT, request).await + engine + .restore(ACCOUNT, request, JobPermit::detached()) + .await } /// Poll the target's catalog row until its detached copy is done: `true` @@ -1519,7 +1739,7 @@ mod restore_tests { let engine = engine().await; let src = indexed_table(&engine, "src").await; let backup = engine - .create_backup(ACCOUNT, "src", "bkp") + .create(ACCOUNT, "src", "bkp", JobPermit::detached()) .await .expect("backup"); let desc = restore(&engine, "dst", &backup.backup_arn) @@ -1563,41 +1783,143 @@ mod restore_tests { ); } + /// The job record's status column for a target. + async fn job_status(engine: &SqliteEngine, table_id: &str) -> Option { + sqlx::query_scalar("SELECT status FROM table_restores WHERE table_id = ?") + .bind(table_id) + .fetch_optional(&engine.pool) + .await + .expect("read the job status") + } + + /// A restore target whose owner died is found by the reconciler only once + /// its job record's heartbeat is older than the stale window, whatever + /// the table's age; failing the record and removing the target each + /// happen once; a key with the wrong table id never removes a table + /// re-created under the name. #[tokio::test] - async fn startup_sweep_removes_an_abandoned_restore() { + async fn abandoned_restore_is_removed_only_when_its_heartbeat_is_stale() { + use extenddb_storage::BackupCatalog; + use extenddb_storage::backup_catalog::{RestoreKind, RestoreTargetKey}; + let stale = Duration::from_secs(300); let engine = engine().await; indexed_table(&engine, "src").await; let backup = engine - .create_backup(ACCOUNT, "src", "bkp") + .create(ACCOUNT, "src", "bkp", JobPermit::detached()) .await .expect("backup"); let desc = restore(&engine, "dst", &backup.backup_arn) .await .expect("restore"); + assert_eq!( + job_status(&engine, &desc.table_id).await.as_deref(), + Some("DONE"), + "a finished restore's record is DONE" + ); let index_ids: Vec = index_tables(&engine, &desc.table_id) .await .into_iter() .map(|(_, id)| id) .collect(); + let key = RestoreTargetKey { + account_id: ACCOUNT.to_owned(), + table_name: "dst".to_owned(), + table_id: desc.table_id.clone(), + }; // The state a process killed mid-copy leaves: the target CREATING with - // no scheduled transition (its copy transaction rolled back). + // no scheduled transition, its job record RUNNING with a heartbeat + // `age_secs` old. + let crash = |age_secs: i64| { + let pool = engine.pool.clone(); + let id = desc.table_id.clone(); + async move { + sqlx::query( + "UPDATE tables SET table_status = 'CREATING', status_transition_at = NULL \ + WHERE table_id = ?", + ) + .bind(&id) + .execute(&pool) + .await + .expect("simulate a crash"); + sqlx::query( + "UPDATE table_restores SET status = 'RUNNING', completed_at = NULL, \ + heartbeat_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now', ? || ' seconds') \ + WHERE table_id = ?", + ) + .bind(-age_secs) + .bind(&id) + .execute(&pool) + .await + .expect("age the heartbeat"); + } + }; + + // A fresh heartbeat: a live restore, even though the table is old. + crash(1).await; sqlx::query( - "UPDATE tables SET table_status = 'CREATING', status_transition_at = NULL \ - WHERE table_id = ?", + "UPDATE tables SET creation_date_time = strftime('%Y-%m-%dT%H:%M:%fZ', 'now', \ + '-10 minutes')", ) - .bind(&desc.table_id) .execute(&engine.pool) .await - .expect("simulate crash"); + .expect("age every table"); + assert!( + engine + .list_dead_restore_jobs(stale) + .await + .expect("list") + .is_empty() + ); + assert_eq!( + status_of(&engine, &desc.table_id).await.as_deref(), + Some("CREATING") + ); - let removed = engine.sweep_abandoned_restores().await.expect("sweep"); - assert_eq!(removed, vec!["dst".to_owned()]); - let rows: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM tables WHERE table_name = 'dst'") - .fetch_one(&engine.pool) - .await - .expect("rows"); - assert_eq!(rows, 0); + // A stale heartbeat: dead. + crash(600).await; + let dead = engine.list_dead_restore_jobs(stale).await.expect("list"); + assert_eq!(dead.len(), 1, "{dead:?}"); + assert_eq!(dead[0].target, key); + assert_eq!(dead[0].kind, RestoreKind::Legacy); + assert_eq!(dead[0].source_backup_arn, backup.backup_arn); + assert!(dead[0].heartbeat_at > 0.0); + assert!( + engine + .fail_restore_job( + &key, + dead[0].owner_generation, + dead[0].heartbeat_at, + "owner dead" + ) + .await + .expect("fail") + ); + assert_eq!( + job_status(&engine, &desc.table_id).await.as_deref(), + Some("FAILED") + ); + let reason: Option = + sqlx::query_scalar("SELECT failure_reason FROM table_restores WHERE table_id = ?") + .bind(&desc.table_id) + .fetch_one(&engine.pool) + .await + .expect("reason"); + assert_eq!(reason.as_deref(), Some("owner dead")); + assert!( + !engine + .fail_restore_job( + &key, + dead[0].owner_generation, + dead[0].heartbeat_at, + "owner dead" + ) + .await + .expect("fail again"), + "a record that is no longer RUNNING is not written twice" + ); + assert!(engine.remove_restore_target(&key).await.expect("remove")); + assert_eq!(status_of(&engine, &desc.table_id).await, None); let mut leftover = vec![desc.table_id.clone()]; leftover.extend(index_ids); for id in leftover { @@ -1610,86 +1932,539 @@ mod restore_tests { .expect("exists"); assert!(!exists, "data table _ddb_{id} left behind"); } + let registrations: i64 = + sqlx::query_scalar("SELECT COUNT(*) FROM table_restores WHERE table_id = ?") + .bind(&desc.table_id) + .fetch_one(&engine.pool) + .await + .expect("registrations"); + assert_eq!(registrations, 0, "the record goes with its target"); + assert!( + !engine + .remove_restore_target(&key) + .await + .expect("remove again"), + "a removed target is removed once" + ); - // A live table is never a candidate, and the name is free again. + // An ACTIVE table is never a candidate, the name is free again, and a + // key with the wrong table id (a table re-created under the name) + // does not remove the new table. + let again = restore(&engine, "dst", &backup.backup_arn) + .await + .expect("restore again"); assert!( engine - .sweep_abandoned_restores() + .list_dead_restore_jobs(stale) .await - .expect("sweep") + .expect("list") .is_empty() ); - restore(&engine, "dst", &backup.backup_arn) - .await - .expect("restore again"); + assert!(!engine.remove_restore_target(&key).await.expect("stale key")); + sqlx::query( + "UPDATE tables SET table_status = 'CREATING', status_transition_at = NULL \ + WHERE table_id = ?", + ) + .bind(&again.table_id) + .execute(&engine.pool) + .await + .expect("make it a target again"); + assert!( + !engine.remove_restore_target(&key).await.expect("stale key"), + "the old table id never removes the new table" + ); + assert_eq!( + status_of(&engine, &again.table_id).await.as_deref(), + Some("CREATING") + ); } + /// `remove_restore_target` acts only on the table its full key names: an + /// ACTIVE table is never removed, and a CREATING target is left alone by a + /// key with its table id under another account or another name; the full + /// key removes it. #[tokio::test] - async fn startup_sweep_removes_an_incomplete_backup_and_its_rows() { + async fn a_target_is_removed_only_while_creating_and_only_under_its_full_key() { + use extenddb_storage::BackupCatalog; + use extenddb_storage::backup_catalog::RestoreTargetKey; + const OTHER_ACCOUNT: &str = "111111111111"; let engine = engine().await; indexed_table(&engine, "src").await; let backup = engine - .create_backup(ACCOUNT, "src", "bkp") + .create(ACCOUNT, "src", "bkp", JobPermit::detached()) .await .expect("backup"); - sqlx::query("UPDATE backups SET backup_status = 'CREATING' WHERE backup_arn = ?") - .bind(&backup.backup_arn) - .execute(&engine.pool) - .await - .expect("simulate crash"); - let listed = engine - .list_backups(ACCOUNT, Some("src")) - .await - .expect("list CREATING backup"); - assert_eq!(listed.len(), 1); - assert_eq!(listed[0].backup_status, "CREATING"); - let described = engine - .describe_backup(ACCOUNT, &backup.backup_arn) - .await - .expect("describe CREATING backup"); - assert_eq!(described.backup_details.backup_status, "CREATING"); - engine - .delete_backup(ACCOUNT, &backup.backup_arn) + let desc = restore(&engine, "dst", &backup.backup_arn) .await - .expect_err("DeleteBackup only accepts AVAILABLE backups"); + .expect("restore"); + let data_table = crate::data::data_table_name(&desc.table_id); + let key = RestoreTargetKey { + account_id: ACCOUNT.to_owned(), + table_name: "dst".to_owned(), + table_id: desc.table_id.clone(), + }; - let removed = engine - .sweep_incomplete_backups() - .await - .expect("backup sweep"); - assert_eq!(removed, vec![backup.backup_arn.clone()]); - let backup_rows: i64 = - sqlx::query_scalar("SELECT COUNT(*) FROM backups WHERE backup_arn = ?") - .bind(&backup.backup_arn) - .fetch_one(&engine.pool) - .await - .expect("backup rows"); - let item_rows: i64 = - sqlx::query_scalar("SELECT COUNT(*) FROM backup_items WHERE backup_arn = ?") - .bind(&backup.backup_arn) - .fetch_one(&engine.pool) - .await - .expect("item rows"); - let definition_rows: i64 = - sqlx::query_scalar("SELECT COUNT(*) FROM backup_definitions WHERE backup_arn = ?") - .bind(&backup.backup_arn) - .fetch_one(&engine.pool) - .await - .expect("definition rows"); - assert_eq!((backup_rows, item_rows, definition_rows), (0, 0, 0)); - assert!( - engine - .sweep_incomplete_backups() - .await - .expect("second sweep") - .is_empty() + // ACTIVE with the full key: the ordering where the copy's flip won. + assert!(!engine.remove_restore_target(&key).await.expect("remove")); + assert_eq!( + status_of(&engine, &desc.table_id).await.as_deref(), + Some("ACTIVE") ); - } - #[tokio::test] - async fn failed_backup_copy_removes_its_creating_row() { - let (engine, path) = file_engine().await; - let source = plain_table(&engine, "source").await; + // CREATING, the right id, the wrong account or name: nothing. + sqlx::query( + "UPDATE tables SET table_status = 'CREATING', status_transition_at = NULL \ + WHERE table_id = ?", + ) + .bind(&desc.table_id) + .execute(&engine.pool) + .await + .expect("make it a target"); + let other_account = RestoreTargetKey { + account_id: OTHER_ACCOUNT.to_owned(), + ..key.clone() + }; + let other_name = RestoreTargetKey { + table_name: "other".to_owned(), + ..key.clone() + }; + for wrong in [&other_account, &other_name] { + assert!( + !engine + .remove_restore_target(wrong) + .await + .expect("remove under the wrong key"), + "{wrong:?}" + ); + assert_eq!( + status_of(&engine, &desc.table_id).await.as_deref(), + Some("CREATING"), + "{wrong:?}" + ); + assert_eq!(count(&engine, &data_table).await, 30, "{wrong:?}"); + } + + // The full key removes it. + assert!(engine.remove_restore_target(&key).await.expect("remove")); + assert_eq!(status_of(&engine, &desc.table_id).await, None); + } + + /// The FAILED-target duty's listing: a FAILED record whose table is still + /// a CREATING restore target (the FAILED write landed and the removal did + /// not) is not listed while the record is fresh (the task that failed it + /// may still be removing the target), is listed once `completed_at` is + /// older than the stale window, as FAILED, and its target is then removed + /// with no second FAILED write; a FAILED record without a `completed_at` + /// is aged by its heartbeat; a FAILED record whose table is DELETING or + /// ACTIVE is never listed. + #[tokio::test] + async fn a_failed_record_whose_target_removal_failed_is_listed_once_stale_and_removed() { + use extenddb_storage::BackupCatalog; + use extenddb_storage::backup_catalog::{JobStatus, RestoreKind, RestoreTargetKey}; + let stale = Duration::from_secs(300); + let engine = engine().await; + indexed_table(&engine, "src").await; + let backup = engine + .create(ACCOUNT, "src", "bkp", JobPermit::detached()) + .await + .expect("backup"); + let desc = restore(&engine, "dst", &backup.backup_arn) + .await + .expect("restore"); + let key = RestoreTargetKey { + account_id: ACCOUNT.to_owned(), + table_name: "dst".to_owned(), + table_id: desc.table_id.clone(), + }; + let set_record = |sql: &'static str| { + let pool = engine.pool.clone(); + let id = desc.table_id.clone(); + async move { + sqlx::query(sql) + .bind(&id) + .execute(&pool) + .await + .expect("set the record"); + } + }; + let set_table_status = |status: &'static str| { + let pool = engine.pool.clone(); + let id = desc.table_id.clone(); + async move { + sqlx::query( + "UPDATE tables SET table_status = ?, status_transition_at = NULL \ + WHERE table_id = ?", + ) + .bind(status) + .bind(&id) + .execute(&pool) + .await + .expect("set the table status"); + } + }; + let listed = || async { engine.list_dead_restore_jobs(stale).await.expect("list") }; + + // The state a dead restore leaves once the reconciler (or the copy's + // own failure path) failed its record and the target's removal did + // not land. + set_table_status("CREATING").await; + set_record( + "UPDATE table_restores SET status = 'RUNNING', completed_at = NULL, \ + heartbeat_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now', '-10 minutes') \ + WHERE table_id = ?", + ) + .await; + let dead = listed().await; + assert_eq!(dead.len(), 1, "{dead:?}"); + assert_eq!(dead[0].status, JobStatus::Running); + assert!( + engine + .fail_restore_job( + &key, + dead[0].owner_generation, + dead[0].heartbeat_at, + "owner dead" + ) + .await + .expect("fail") + ); + assert_eq!( + job_status(&engine, &desc.table_id).await.as_deref(), + Some("FAILED") + ); + assert_eq!( + status_of(&engine, &desc.table_id).await.as_deref(), + Some("CREATING") + ); + + // Fresh: `fail_restore_job` stamped `completed_at` a moment ago, and + // whoever failed it may still be removing the target. + assert!( + listed().await.is_empty(), + "a FAILED record within the stale window is left to its writer" + ); + // Stale: listed, as FAILED, for the FAILED-target duty. + set_record( + "UPDATE table_restores SET completed_at = \ + strftime('%Y-%m-%dT%H:%M:%fZ', 'now', '-10 minutes') WHERE table_id = ?", + ) + .await; + let dead = listed().await; + assert_eq!(dead.len(), 1, "{dead:?}"); + assert_eq!(dead[0].target, key); + assert_eq!(dead[0].status, JobStatus::Failed); + assert_eq!(dead[0].kind, RestoreKind::Legacy); + assert_eq!(dead[0].source_backup_arn, backup.backup_arn); + // A FAILED record without `completed_at` is aged by its heartbeat: + // fresh, not listed; stale, listed. + set_record( + "UPDATE table_restores SET completed_at = NULL, \ + heartbeat_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now') WHERE table_id = ?", + ) + .await; + assert!( + listed().await.is_empty(), + "a FAILED record with a fresh heartbeat and no completed_at is fresh" + ); + set_record( + "UPDATE table_restores SET heartbeat_at = \ + strftime('%Y-%m-%dT%H:%M:%fZ', 'now', '-10 minutes') WHERE table_id = ?", + ) + .await; + assert_eq!(listed().await.len(), 1); + // A FAILED record whose table the removal already moved to DELETING + // (the claim landed, the drop did not) is the control plane's row. + set_table_status("DELETING").await; + assert!( + listed().await.is_empty(), + "a DELETING table is not a target" + ); + set_table_status("CREATING").await; + + // The reconciler's FAILED-target duty: no second FAILED write, the + // target removed, with its record and its data tables. + assert!( + !engine + .fail_restore_job( + &key, + dead[0].owner_generation, + dead[0].heartbeat_at, + "owner dead" + ) + .await + .expect("fail again"), + "a FAILED record is not RUNNING and is not written FAILED again" + ); + assert!(engine.remove_restore_target(&key).await.expect("remove")); + assert_eq!(status_of(&engine, &desc.table_id).await, None); + let data_table: bool = sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?)", + ) + .bind(format!("_ddb_{}", desc.table_id)) + .fetch_one(&engine.pool) + .await + .expect("exists"); + assert!(!data_table, "data table left behind"); + let registrations: i64 = + sqlx::query_scalar("SELECT COUNT(*) FROM table_restores WHERE table_id = ?") + .bind(&desc.table_id) + .fetch_one(&engine.pool) + .await + .expect("registrations"); + assert_eq!(registrations, 0, "the record goes with its target"); + assert!(listed().await.is_empty()); + engine + .delete(ACCOUNT, &backup.backup_arn) + .await + .expect("the backup is free once the target is gone"); + + // A FAILED record whose table is ACTIVE (a flip that won against a + // late FAILED write) is nobody's duty. + let backup = engine + .create(ACCOUNT, "src", "bkp2", JobPermit::detached()) + .await + .expect("backup again"); + let again = restore(&engine, "dst", &backup.backup_arn) + .await + .expect("restore again"); + assert_eq!( + status_of(&engine, &again.table_id).await.as_deref(), + Some("ACTIVE") + ); + sqlx::query( + "UPDATE table_restores SET status = 'FAILED', completed_at = \ + strftime('%Y-%m-%dT%H:%M:%fZ', 'now', '-10 minutes') WHERE table_id = ?", + ) + .bind(&again.table_id) + .execute(&engine.pool) + .await + .expect("fail the record of an ACTIVE table"); + assert!( + listed().await.is_empty(), + "a FAILED record of an ACTIVE table is not listed" + ); + } + + /// A CREATING backup row is abandoned only while no task of this process + /// holds its slot: a running create is left alone and its row is not + /// deletable; a dead one (a crash, or the previous run) is removed with + /// its rows by the call that reports it (`removed: true`), with no + /// tombstone, and the reconciler then only logs it. A finished backup is + /// never reported. + #[tokio::test] + async fn an_abandoned_creating_backup_is_removed_by_the_call_that_reports_it() { + use crate::backup_catalog::{JobSlots, legacy_backup_slot_key}; + let stale = Duration::from_secs(300); + let engine = engine().await; + indexed_table(&engine, "src").await; + let backup = engine + .create(ACCOUNT, "src", "bkp", JobPermit::detached()) + .await + .expect("backup"); + assert!( + engine + .abandoned_creating_backups(stale) + .await + .expect("list") + .is_empty(), + "an AVAILABLE backup is not abandoned" + ); + sqlx::query("UPDATE backups SET backup_status = 'CREATING' WHERE backup_arn = ?") + .bind(&backup.backup_arn) + .execute(&engine.pool) + .await + .expect("simulate a crash"); + let listed = engine + .list(ACCOUNT, Some("src")) + .await + .expect("list CREATING backup"); + assert_eq!(listed.len(), 1); + assert_eq!(listed[0].backup_status, "CREATING"); + let described = engine + .describe(ACCOUNT, &backup.backup_arn) + .await + .expect("describe CREATING backup"); + assert_eq!(described.backup_details.backup_status, "CREATING"); + + // The slot a running create holds: not abandoned, not deletable. + let held = JobSlots::take( + &engine.job_slots, + legacy_backup_slot_key(&backup.backup_arn), + ) + .expect("free slot"); + assert!( + engine + .abandoned_creating_backups(stale) + .await + .expect("list") + .is_empty(), + "a CREATING row whose create is running is not abandoned" + ); + let err = engine + .delete(ACCOUNT, &backup.backup_arn) + .await + .expect_err("a backup still being created is not deletable"); + assert!( + matches!(&err, StorageError::Validation(m) if m.starts_with("Backup not found")), + "{err:?}" + ); + assert_eq!( + engine + .describe(ACCOUNT, &backup.backup_arn) + .await + .expect("still there") + .backup_details + .backup_status, + "CREATING" + ); + drop(held); + + // No slot: dead. Removed with its rows by this call. + let abandoned = engine + .abandoned_creating_backups(stale) + .await + .expect("list"); + assert_eq!(abandoned.len(), 1, "{abandoned:?}"); + assert_eq!(abandoned[0].account_id, ACCOUNT); + assert_eq!(abandoned[0].backup_arn, backup.backup_arn); + assert!( + abandoned[0].removed, + "SQLite removes the row in the reporting call" + ); + let leftovers: (i64, i64, i64) = sqlx::query_as( + "SELECT \ + (SELECT COUNT(*) FROM backups WHERE backup_arn = ?1), \ + (SELECT COUNT(*) FROM backup_items WHERE backup_arn = ?1), \ + (SELECT COUNT(*) FROM backup_definitions WHERE backup_arn = ?1)", + ) + .bind(&backup.backup_arn) + .fetch_one(&engine.pool) + .await + .expect("leftovers"); + assert_eq!( + leftovers, + (0, 0, 0), + "a backup that never finished leaves no tombstone" + ); + assert!( + engine + .abandoned_creating_backups(stale) + .await + .expect("second list") + .is_empty() + ); + let err = engine + .describe(ACCOUNT, &backup.backup_arn) + .await + .expect_err("gone"); + assert!( + matches!(&err, StorageError::Validation(m) if m.starts_with("Backup not found")), + "{err:?}" + ); + } + + /// The window between a read of the CREATING rows and the removal: a + /// create that commits AVAILABLE and frees its slot in between keeps its + /// backup. The removal runs under the engine write lock and deletes only + /// `backup_status = 'CREATING'` rows, so with the lock held by the test + /// (as a committing create holds it) the reconciler's call waits, and + /// once the row is AVAILABLE and the lock free it reports nothing and + /// removes nothing: the backup and its items are intact. A second + /// CREATING row with no slot beside it is removed by the same call, so the + /// write ran and its status predicate is what spared the first. + #[tokio::test] + async fn a_create_that_commits_available_during_the_abandoned_scan_keeps_its_backup() { + use crate::backup_catalog::{JobSlots, legacy_backup_slot_key}; + let stale = Duration::from_secs(300); + let engine = engine().await; + indexed_table(&engine, "src").await; + let finishing = engine + .create(ACCOUNT, "src", "finishing", JobPermit::detached()) + .await + .expect("backup"); + let dead = engine + .create(ACCOUNT, "src", "dead", JobPermit::detached()) + .await + .expect("backup"); + let items_of = |arn: String| { + let pool = engine.pool.clone(); + async move { + sqlx::query_scalar::<_, i64>( + "SELECT COUNT(*) FROM backup_items WHERE backup_arn = ?", + ) + .bind(arn) + .fetch_one(&pool) + .await + .expect("count") + } + }; + let item_count = items_of(finishing.backup_arn.clone()).await; + assert!(item_count > 0); + for arn in [&finishing.backup_arn, &dead.backup_arn] { + sqlx::query("UPDATE backups SET backup_status = 'CREATING' WHERE backup_arn = ?") + .bind(arn) + .execute(&engine.pool) + .await + .expect("simulate two creates mid-copy"); + } + // `finishing` is a running create: its slot is held until its row is + // AVAILABLE. `dead` has no slot. + let slot = JobSlots::take( + &engine.job_slots, + legacy_backup_slot_key(&finishing.backup_arn), + ) + .expect("free slot"); + + // The reconciler's scan arrives while a writer holds the lock; the + // create commits AVAILABLE and frees its slot before the lock is + // released. + let writer = engine.write_lock.lock().await; + let scan = { + let engine = engine.clone(); + tokio::spawn(async move { engine.abandoned_creating_backups(stale).await }) + }; + tokio::time::sleep(Duration::from_millis(50)).await; + assert!(!scan.is_finished(), "the scan waits for the write lock"); + sqlx::query("UPDATE backups SET backup_status = 'AVAILABLE' WHERE backup_arn = ?") + .bind(&finishing.backup_arn) + .execute(&engine.pool) + .await + .expect("the create commits AVAILABLE"); + drop(slot); + drop(writer); + let abandoned = scan.await.unwrap().expect("scan"); + assert_eq!( + abandoned + .iter() + .map(|a| a.backup_arn.as_str()) + .collect::>(), + [dead.backup_arn.as_str()], + "only the row with no slot and still CREATING is removed" + ); + assert_eq!( + engine + .describe(ACCOUNT, &finishing.backup_arn) + .await + .expect("the finished backup is intact") + .backup_details + .backup_status, + "AVAILABLE" + ); + assert_eq!(items_of(finishing.backup_arn.clone()).await, item_count); + assert_eq!(items_of(dead.backup_arn.clone()).await, 0); + assert!( + engine + .abandoned_creating_backups(stale) + .await + .expect("second scan") + .is_empty() + ); + } + + #[tokio::test] + async fn failed_backup_copy_removes_its_creating_row() { + let (engine, path) = file_engine().await; + let source = plain_table(&engine, "source").await; let table = crate::data::data_table_name(&source.table_id); sqlx::query(&format!( "INSERT INTO {table} (pk, item_data) VALUES ('broken', 'not json')" @@ -1699,7 +2474,7 @@ mod restore_tests { .expect("corrupt source row"); engine - .create_backup(ACCOUNT, "source", "broken") + .create(ACCOUNT, "source", "broken", JobPermit::detached()) .await .expect_err("invalid source item must fail the backup"); let leftovers: (i64, i64, i64) = sqlx::query_as( @@ -1722,7 +2497,7 @@ mod restore_tests { let engine = engine().await; indexed_table(&engine, "src").await; let backup = engine - .create_backup(ACCOUNT, "src", "bkp") + .create(ACCOUNT, "src", "bkp", JobPermit::detached()) .await .expect("backup"); // Corrupt one backup row so the copy fails part-way. @@ -1769,7 +2544,7 @@ mod restore_tests { let engine = engine().await; indexed_table(&engine, "src").await; let backup = engine - .create_backup(ACCOUNT, "src", "bkp") + .create(ACCOUNT, "src", "bkp", JobPermit::detached()) .await .expect("backup"); let input = serde_json::from_value(json!({ @@ -1791,6 +2566,7 @@ mod restore_tests { backup_arn: &backup.backup_arn, source_table_arn: "arn:aws:dynamodb:us-east-1:000000000000:table/src", restore_date_time: backup.backup_creation_date_time, + owner_generation: extenddb_storage::backup_catalog::new_generation(), }, ) .await @@ -1812,7 +2588,7 @@ mod restore_tests { assert_eq!(state, (true, true)); let err = engine - .delete_backup(ACCOUNT, &backup.backup_arn) + .delete(ACCOUNT, &backup.backup_arn) .await .expect_err("restore intent must block deletion"); assert!( @@ -1829,7 +2605,7 @@ mod restore_tests { .expect("intent count"); assert_eq!(intents, 0, "target cleanup must remove restore intent"); engine - .delete_backup(ACCOUNT, &backup.backup_arn) + .delete(ACCOUNT, &backup.backup_arn) .await .expect("backup is deletable after abort"); } @@ -1864,7 +2640,7 @@ mod restore_tests { ) .await; let backup = engine - .create_backup(ACCOUNT, "m", "bkp") + .create(ACCOUNT, "m", "bkp", JobPermit::detached()) .await .expect("backup"); // The refusal is the engine's (`ensure_restorable` in `restore`); this @@ -1903,7 +2679,7 @@ mod restore_tests { ) .await; let backup = engine - .create_backup(ACCOUNT, "c", "bkp") + .create(ACCOUNT, "c", "bkp", JobPermit::detached()) .await .expect("backup"); restore(&engine, "c2", &backup.backup_arn) @@ -1987,7 +2763,7 @@ mod restore_tests { .expect("put"); } let backup = engine - .create_backup(ACCOUNT, "src", "bkp") + .create(ACCOUNT, "src", "bkp", JobPermit::detached()) .await .expect("backup"); let desc = restore(&engine, "dst", &backup.backup_arn) @@ -2198,7 +2974,7 @@ mod restore_tests { let engine = engine().await; indexed_table(&engine, "src").await; let backup = engine - .create_backup(ACCOUNT, "src", "bkp") + .create(ACCOUNT, "src", "bkp", JobPermit::detached()) .await .expect("backup"); let desc = restore(&engine, "dst", &backup.backup_arn) @@ -2234,7 +3010,7 @@ mod restore_tests { let engine = engine().await; indexed_table(&engine, "src").await; let backup = engine - .create_backup(ACCOUNT, "src", "bkp") + .create(ACCOUNT, "src", "bkp", JobPermit::detached()) .await .expect("backup"); let desc = restore(&engine, "dst", &backup.backup_arn) @@ -2283,7 +3059,7 @@ mod restore_tests { .restore_in_progress ); let err = engine - .delete_backup(ACCOUNT, &backup.backup_arn) + .delete(ACCOUNT, &backup.backup_arn) .await .expect_err("in use"); assert!( @@ -2297,7 +3073,7 @@ mod restore_tests { .await .expect("finished"); engine - .delete_backup(ACCOUNT, &backup.backup_arn) + .delete(ACCOUNT, &backup.backup_arn) .await .expect("deletable once the restore is done"); let after = describe("dst").await.restore_summary.expect("summary"); @@ -2459,7 +3235,7 @@ mod restore_tests { let started = Instant::now(); engine - .create_backup(ACCOUNT, "source", "concurrent") + .create(ACCOUNT, "source", "concurrent", JobPermit::detached()) .await .expect("backup"); let finished = Instant::now(); @@ -2504,7 +3280,7 @@ mod restore_tests { let backup_engine = engine.clone(); let backup = tokio::spawn(async move { backup_engine - .create_backup(ACCOUNT, "source", "snapshot") + .create(ACCOUNT, "source", "snapshot", JobPermit::detached()) .await }); loop { @@ -2580,7 +3356,12 @@ mod restore_tests { let request_engine = engine.clone(); let request = tokio::spawn(async move { request_engine - .create_backup(ACCOUNT, "source", "cancelled-request") + .create( + ACCOUNT, + "source", + "cancelled-request", + JobPermit::detached(), + ) .await }); tokio::time::timeout(std::time::Duration::from_secs(10), async { @@ -2734,7 +3515,7 @@ mod restore_tests { let (engine, path) = file_engine().await; bulk_indexed_table(&engine, "src", BULK_ITEMS).await; let backup = engine - .create_backup(ACCOUNT, "src", "bkp") + .create(ACCOUNT, "src", "bkp", JobPermit::detached()) .await .expect("backup"); assert!( @@ -2791,7 +3572,7 @@ mod restore_tests { let (engine, path) = file_engine().await; bulk_indexed_table(&engine, "src", BULK_ITEMS).await; let backup = engine - .create_backup(ACCOUNT, "src", "bkp") + .create(ACCOUNT, "src", "bkp", JobPermit::detached()) .await .expect("backup"); let request = restore_request(&engine, "dst", &backup.backup_arn) @@ -2799,7 +3580,7 @@ mod restore_tests { .expect("request"); let polled_once = engine - .restore_table_from_backup(ACCOUNT, request) + .restore(ACCOUNT, request, JobPermit::detached()) .now_or_never(); assert!( polled_once.is_none(), @@ -2843,7 +3624,7 @@ mod restore_tests { let (engine, path) = file_engine().await; bulk_indexed_table(&engine, "src", BULK_ITEMS).await; let backup = engine - .create_backup(ACCOUNT, "src", "bkp") + .create(ACCOUNT, "src", "bkp", JobPermit::detached()) .await .expect("backup"); let desc = start_restore(&engine, "dst", &backup.backup_arn) @@ -2851,7 +3632,7 @@ mod restore_tests { .expect("restore"); let err = engine - .delete_backup(ACCOUNT, &backup.backup_arn) + .delete(ACCOUNT, &backup.backup_arn) .await .expect_err("the source backup is in use while the copy runs"); assert!(matches!(err, StorageError::BackupInUse(_)), "{err:?}"); @@ -2885,7 +3666,7 @@ mod restore_tests { BULK_ITEMS ); engine - .delete_backup(ACCOUNT, &backup.backup_arn) + .delete(ACCOUNT, &backup.backup_arn) .await .expect("deletable once the restore is done"); delete_target() @@ -2895,19 +3676,24 @@ mod restore_tests { } /// A target registered but never copied (a process that exited between - /// registration and copy) is removed by the startup sweep, its restore - /// registration with it, which frees the source backup. + /// registration and copy) holds its backup in use. At startup every + /// RUNNING record's heartbeat is aged to the epoch, so the reconciler's + /// duties find the restore dead on their first run, fail it and remove + /// the target, its registration with it, which frees the source backup. #[tokio::test] - async fn startup_sweep_removes_a_registered_target_that_was_never_copied() { + async fn a_restart_ages_an_interrupted_restore_so_the_reconciler_removes_it() { + use extenddb_storage::BackupCatalog; + use extenddb_storage::backup_catalog::{RestoreKind, RestoreTargetKey, UNOWNED_GENERATION}; let engine = engine().await; indexed_table(&engine, "src").await; let backup = engine - .create_backup(ACCOUNT, "src", "bkp") + .create(ACCOUNT, "src", "bkp", JobPermit::detached()) .await .expect("backup"); let request = restore_request(&engine, "dst", &backup.backup_arn) .await .expect("request"); + let generation = extenddb_storage::backup_catalog::new_generation(); let desc = engine .create_table_for_restore( ACCOUNT, @@ -2916,18 +3702,77 @@ mod restore_tests { backup_arn: &request.backup_arn, source_table_arn: &request.source_table_arn, restore_date_time: request.restore_date_time, + owner_generation: generation, }, ) .await .expect("register without a copy"); let err = engine - .delete_backup(ACCOUNT, &backup.backup_arn) + .delete(ACCOUNT, &backup.backup_arn) .await .expect_err("registered restore holds the backup"); assert!(matches!(err, StorageError::BackupInUse(_)), "{err:?}"); + let stale = Duration::from_secs(300); + assert!( + engine + .list_dead_restore_jobs(stale) + .await + .expect("list") + .is_empty(), + "a record registered a moment ago is not stale" + ); + + // The startup step: the previous process owned every RUNNING record. + let aged = engine + .expire_restore_job_heartbeats() + .await + .expect("expire"); + assert_eq!(aged, vec![("dst".to_owned(), RestoreKind::Legacy)]); + let (heartbeat, record_generation): (String, i64) = sqlx::query_as( + "SELECT heartbeat_at, owner_generation FROM table_restores WHERE table_id = ?", + ) + .bind(&desc.table_id) + .fetch_one(&engine.pool) + .await + .expect("record"); + assert_eq!(heartbeat, "1970-01-01T00:00:00.000Z"); + assert_eq!( + record_generation, + i64::try_from(generation).unwrap(), + "aging touches the heartbeat alone" + ); + assert!( + engine + .expire_restore_job_heartbeats() + .await + .expect("expire again") + .is_empty(), + "a record is reported once" + ); - let removed = engine.sweep_abandoned_restores().await.expect("sweep"); - assert_eq!(removed, vec!["dst".to_owned()]); + // The reconciler's duties on the first cycle. + let dead = engine.list_dead_restore_jobs(stale).await.expect("list"); + assert_eq!(dead.len(), 1, "{dead:?}"); + let target = RestoreTargetKey { + account_id: ACCOUNT.to_owned(), + table_name: "dst".to_owned(), + table_id: desc.table_id.clone(), + }; + assert_eq!(dead[0].target, target); + assert_eq!(dead[0].heartbeat_at, 0.0); + assert_ne!(dead[0].owner_generation, UNOWNED_GENERATION); + assert!( + engine + .fail_restore_job( + &target, + dead[0].owner_generation, + dead[0].heartbeat_at, + "owner dead" + ) + .await + .expect("fail") + ); + assert!(engine.remove_restore_target(&target).await.expect("remove")); assert_eq!(status_of(&engine, &desc.table_id).await, None); let registrations: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM table_restores WHERE table_id = ?") @@ -2937,9 +3782,9 @@ mod restore_tests { .expect("registrations"); assert_eq!(registrations, 0); engine - .delete_backup(ACCOUNT, &backup.backup_arn) + .delete(ACCOUNT, &backup.backup_arn) .await - .expect("the backup is free after the sweep"); + .expect("the backup is free once the target is gone"); } /// table_restores records the source table ARN and the restore time the @@ -2950,7 +3795,7 @@ mod restore_tests { let engine = engine().await; indexed_table(&engine, "src").await; let backup = engine - .create_backup(ACCOUNT, "src", "bkp") + .create(ACCOUNT, "src", "bkp", JobPermit::detached()) .await .expect("backup"); let mut request = restore_request(&engine, "dst", &backup.backup_arn) @@ -2961,7 +3806,7 @@ mod restore_tests { request.restore_date_time = 1_791_976_431.438; request.source_table_arn = "arn:aws:dynamodb:us-east-1:000000000000:table/elsewhere".into(); let desc = engine - .restore_table_from_backup(ACCOUNT, request) + .restore(ACCOUNT, request, JobPermit::detached()) .await .expect("restore"); assert!(wait_for_copy(&engine, &desc.table_id).await); @@ -3012,7 +3857,7 @@ mod restore_tests { let mut fractional = None; for i in 0..20 { let backup = engine - .create_backup(ACCOUNT, "src", &format!("bkp{i}")) + .create(ACCOUNT, "src", &format!("bkp{i}"), JobPermit::detached()) .await .expect("backup"); let id = backup.backup_arn.rsplit('/').next().expect("id").to_owned(); @@ -3026,12 +3871,12 @@ mod restore_tests { extenddb_storage::util::backup_arn("us-east-1", ACCOUNT, "src", &id) ); let described = engine - .describe_backup(ACCOUNT, &backup.backup_arn) + .describe(ACCOUNT, &backup.backup_arn) .await .expect("describe"); assert_eq!(described.backup_details.backup_creation_date_time, from_id); let source = engine - .backup_restore_source(ACCOUNT, &backup.backup_arn) + .restore_source(ACCOUNT, &backup.backup_arn) .await .expect("source"); assert_eq!(source.backup_creation_date_time, from_id); @@ -3042,10 +3887,7 @@ mod restore_tests { tokio::time::sleep(Duration::from_millis(1)).await; } let fractional = fractional.expect("some backup with a non-zero millisecond"); - let listed = engine - .list_backups(ACCOUNT, Some("src")) - .await - .expect("list"); + let listed = engine.list(ACCOUNT, Some("src")).await.expect("list"); let summary = listed .iter() .find(|b| b.backup_arn == fractional) @@ -3078,11 +3920,11 @@ mod restore_tests { .await .expect("stream"); let backup = engine - .create_backup(ACCOUNT, "src", "bkp") + .create(ACCOUNT, "src", "bkp", JobPermit::detached()) .await .expect("backup"); let source = engine - .backup_restore_source(ACCOUNT, &backup.backup_arn) + .restore_source(ACCOUNT, &backup.backup_arn) .await .expect("source"); let table = engine @@ -3123,7 +3965,7 @@ mod restore_tests { ), ] { let err = engine - .backup_restore_source(account, arn) + .restore_source(account, arn) .await .expect_err("not found"); assert!( @@ -3154,25 +3996,142 @@ mod restore_tests { assert!(raised[0].contains("pool_size >= 3"), "{raised:?}"); } - /// A flood of CreateBackups from one account on the default pool of 10 - /// does not starve another account's reads: at most - /// `backup_server_slot_count` backups run at once (observed as CREATING - /// rows), every read of the other account succeeds and none waits for - /// the pool's acquire timeout, and every backup either completes or is - /// refused with `LimitExceeded`. Before the bound, 16 concurrent backups - /// each held a reader connection and the other account's reads answered - /// an internal error after the 30 s acquire timeout. - #[tokio::test(flavor = "multi_thread", worker_threads = 4)] - async fn a_backup_flood_does_not_starve_another_accounts_reads() { - use extenddb_storage::TableEngine; - use std::sync::atomic::{AtomicBool, Ordering}; - use std::sync::{Arc, Mutex}; - use std::time::Instant; - - const FLOOD: usize = 16; - let (engine, path) = file_engine_with_pool_size(10).await; - assert_eq!(engine.backup_server_slot_count, 4); - let source = plain_table(&engine, "source").await; + /// A server whose `backup.max_concurrent_jobs` is over the slot count of + /// its configured pool raises the pool, once, with a warning naming both + /// numbers, so the slot count meets the jobs; a pool whose slots already + /// meet them, and an engine opened with no jobs, are left as configured + /// and log nothing; the jobs over the slot cap raise the pool to the cap. + #[tokio::test] + async fn a_pool_raised_for_the_jobs_is_logged_and_its_slots_meet_them() { + use super::pool_size_for_jobs; + assert_eq!(pool_size_for_jobs(10, 4), 10, "the defaults need no raise"); + assert_eq!(pool_size_for_jobs(4, 4), 10); + assert_eq!(pool_size_for_jobs(4, 1), 4); + assert_eq!(pool_size_for_jobs(4, 2), 6); + assert_eq!(pool_size_for_jobs(2, 0), 2); + // An odd pool whose slots are exactly the jobs is kept: 11 holds 4 + // slots, and a strict comparison would lower it to 10. + assert_eq!(pool_size_for_jobs(11, 4), 11); + assert_eq!(pool_size_for_jobs(9, 4), 10, "9 holds 3 slots"); + assert_eq!(pool_size_for_jobs(10, 16), 34); + assert_eq!( + pool_size_for_jobs(10, 1024), + 34, + "the slot cap bounds the raise" + ); + for (pool, jobs) in [(1, 0), (4, 1), (4, 4), (10, 4), (3, 16), (10, 1024)] { + assert!( + backup_server_slot_count(pool_size_for_jobs(pool, jobs)) >= jobs.min(16), + "pool {pool}, jobs {jobs}" + ); + } + + let target = std::env::var_os("CARGO_TARGET_DIR") + .map(std::path::PathBuf::from) + .unwrap_or_else(|| { + std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../target") + }); + let path = target.join(format!("backup-test-{}.sqlite", uuid::Uuid::new_v4())); + let path = path.to_str().expect("UTF-8 path").to_owned(); + let (guard, logs) = crate::schema::log_capture::capture(); + let raised = SqliteEngine::new_for_server(&path, 4, 4, "us-east-1", 409_600) + .await + .expect("engine"); + assert_eq!( + raised.backup_server_slot_count, 4, + "the slots meet the jobs" + ); + assert!( + raised.backup_server_slots.available_permits() == 4, + "the semaphore has the raised count" + ); + close_file_engine(raised, std::path::Path::new(&path)).await; + let kept = SqliteEngine::new_for_server(&path, 10, 4, "us-east-1", 409_600) + .await + .expect("engine"); + assert_eq!(kept.backup_server_slot_count, 4); + close_file_engine(kept, std::path::Path::new(&path)).await; + let memory = SqliteEngine::new_for_server(":memory:", 10, 4, "us-east-1", 409_600) + .await + .expect("engine"); + assert_eq!( + memory.backup_server_slot_count, 1, + "an in-memory database has one connection and one slot whatever the jobs" + ); + drop(memory); + drop(guard); + let logs = logs.lock().expect("logs").clone(); + let raised: Vec<&String> = logs + .iter() + .filter(|l| l.contains("pool_size") && l.contains("raised")) + .collect(); + assert_eq!(raised.len(), 1, "{logs:#?}"); + assert!(raised[0].starts_with("WARN"), "{raised:?}"); + assert!( + raised[0].contains("pool_size 4 raised to 10 for backup.max_concurrent_jobs = 4"), + "{raised:?}" + ); + assert!(raised[0].contains("has 4 server slots"), "{raised:?}"); + } + + /// The pool-raise warning says what the raised pool's slots admit: with + /// the jobs within the slot cap, every job the permit admits; over the + /// cap (16), the jobs the slot bound refuses, 17 to `max_concurrent_jobs`, + /// as the limits document states it (a line saying the slot refused no + /// admitted job at 20 jobs was wrong: it refuses the 17th). + #[test] + fn the_pool_raise_warning_names_the_jobs_the_slot_cap_refuses() { + use super::{pool_raise_message, pool_size_for_jobs}; + let at_8 = pool_raise_message(10, pool_size_for_jobs(10, 8), 8); + assert!( + at_8.starts_with( + "SQLite pool_size 10 raised to 18 for backup.max_concurrent_jobs = 8: a pool of \ + 18 has 8 server slots, one per running backup job, so " + ), + "{at_8}" + ); + assert!( + at_8.ends_with( + "so the slot bound refuses no job the max_concurrent_jobs permit admitted" + ), + "{at_8}" + ); + let at_20 = pool_raise_message(10, pool_size_for_jobs(10, 20), 20); + assert!( + at_20.starts_with( + "SQLite pool_size 10 raised to 34 for backup.max_concurrent_jobs = 20: a pool of \ + 34 has 16 server slots, one per running backup job, so " + ), + "{at_20}" + ); + assert!( + at_20.ends_with( + "so the slots are capped at 16, so jobs 17 to 20 are refused by the slot bound" + ), + "{at_20}" + ); + assert!(!at_20.contains("refuses no job"), "{at_20}"); + } + + /// A flood of CreateBackups from one account on the default pool of 10 + /// does not starve another account's reads: at most + /// `backup_server_slot_count` backups run at once (observed as CREATING + /// rows), every read of the other account succeeds and none waits for + /// the pool's acquire timeout, and every backup either completes or is + /// refused with `LimitExceeded`. Before the bound, 16 concurrent backups + /// each held a reader connection and the other account's reads answered + /// an internal error after the 30 s acquire timeout. + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn a_backup_flood_does_not_starve_another_accounts_reads() { + use extenddb_storage::TableEngine; + use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::{Arc, Mutex}; + use std::time::Instant; + + const FLOOD: usize = 16; + let (engine, path) = file_engine_with_pool_size(10).await; + assert_eq!(engine.backup_server_slot_count, 4); + let source = plain_table(&engine, "source").await; fill_table(&engine, &source, 3_000).await; const OTHER: &str = "111111111111"; sqlx::query("INSERT INTO accounts (account_id, account_name) VALUES (?, 'other')") @@ -3246,7 +4205,12 @@ mod restore_tests { let engine = engine.clone(); async move { engine - .create_backup(ACCOUNT, "source", &format!("flood{i}")) + .create( + ACCOUNT, + "source", + &format!("flood{i}"), + JobPermit::detached(), + ) .await } })) @@ -3303,11 +4267,11 @@ mod restore_tests { let engine = engine().await; indexed_table(&engine, "src").await; let b1 = engine - .create_backup(ACCOUNT, "src", "b1") + .create(ACCOUNT, "src", "b1", JobPermit::detached()) .await .expect("backup 1"); let b2 = engine - .create_backup(ACCOUNT, "src", "b2") + .create(ACCOUNT, "src", "b2", JobPermit::detached()) .await .expect("backup 2"); let desc = restore(&engine, "dst", &b1.backup_arn) @@ -3325,8 +4289,10 @@ mod restore_tests { let second = start_restore(&engine, "dst2", &b1.backup_arn).await; assert!( - matches!(&second, Err(StorageError::BackupInUse(m)) if m.contains("dst")), - "a second restore of a backup with a CREATING target: {second:?}" + matches!(&second, Err(StorageError::BackupInUse(m)) + if *m == format!("Backup is being used to restore another table: {}", b1.backup_arn)), + "a second restore of a backup with a CREATING target answers the service's text: \ + {second:?}" ); let none: i64 = sqlx::query_scalar( "SELECT COUNT(*) FROM tables WHERE account_id = ? AND table_name = 'dst2'", @@ -3366,7 +4332,10 @@ mod restore_tests { .await .expect("request"); missing.backup_arn = flip_last_hex(&b2.backup_arn); - match engine.restore_table_from_backup(ACCOUNT, missing).await { + match engine + .restore(ACCOUNT, missing, JobPermit::detached()) + .await + { Err(StorageError::TableBeingRestored { table_name, backup_arn, @@ -3382,6 +4351,19 @@ mod restore_tests { .execute(&engine.pool) .await .expect("finished"); + // The in-use rule reads the target's status, not the registration: + // an ACTIVE former target with nothing else CREATING holds the backup + // no longer, for the rule's own answer and for a second restore. + let b1_key = extenddb_storage::backup_catalog::BackupKey::from_arn( + &extenddb_core::backup_format::BackupArn::parse(&b1.backup_arn).expect("arn"), + ); + assert_eq!( + extenddb_storage::BackupCatalog::backup_in_use(&engine, &b1_key) + .await + .expect("in use"), + None, + "an ACTIVE former target does not hold its backup in use" + ); restore(&engine, "dst2", &b1.backup_arn) .await .expect("a restore once the first target is ACTIVE"); @@ -3392,9 +4374,10 @@ mod restore_tests { ); } - /// The bound's arithmetic and its refusal, without a database. + /// The bound's arithmetic and its refusal, without a database: a full + /// bound refuses with the one job-refusal text and does not wait. #[tokio::test] - async fn backup_slots_follow_the_pool_and_refuse_after_the_wait() { + async fn backup_slots_follow_the_pool_and_refuse_without_waiting() { assert_eq!(backup_server_slot_count(2), 1); assert_eq!(backup_server_slot_count(3), 1); assert_eq!(backup_server_slot_count(4), 1); @@ -3403,18 +4386,2123 @@ mod restore_tests { assert_eq!(backup_server_slot_count(34), 16); assert_eq!(backup_server_slot_count(1_000), 16); let slots = std::sync::Arc::new(tokio::sync::Semaphore::new(1)); - let held = acquire_backup_permit(slots.clone(), 1, Duration::from_millis(50)) - .await - .expect("first"); - match acquire_backup_permit(slots.clone(), 1, Duration::from_millis(50)).await { + let held = acquire_backup_permit(&slots, 1).expect("first"); + match acquire_backup_permit(&slots, 1) { + Err(StorageError::LimitExceeded(m)) => { + assert_eq!( + m, + limit_exceeded_message(JobBound::Counted { + name: BACKUP_SERVER_SLOTS_NAME, + limit: 1, + }) + ); + assert!( + m.contains( + "(at most 1 at once, the [storage.sqlite] pool_size snapshot-slot bound)" + ), + "{m}" + ); + } + other => panic!("a full bound refuses: {:?}", other.err()), + } + drop(held); + let _freed = acquire_backup_permit(&slots, 1).expect("freed"); + } + + /// With every server slot held, CreateBackup is refused at once, with the + /// `LimitExceededException` text the engine's `max_concurrent_jobs` permit + /// answers, rather than waiting for a slot to free up; the refusal writes + /// no row, and a freed slot admits the next create. + #[tokio::test] + async fn a_create_with_no_free_slot_is_refused_at_once_with_the_one_text() { + use std::time::Instant; + let engine = engine().await; + indexed_table(&engine, "src").await; + let held: Vec = (0..engine.backup_server_slot_count) + .map(|_| { + std::sync::Arc::clone(&engine.backup_server_slots) + .try_acquire_owned() + .expect("a free slot") + }) + .collect(); + assert_eq!(engine.backup_server_slots.available_permits(), 0); + + let started = Instant::now(); + let refused = tokio::time::timeout( + Duration::from_secs(5), + engine.create(ACCOUNT, "src", "bkp", JobPermit::detached()), + ) + .await + .expect("a create with no free slot is refused at once, not after a wait"); + match refused { Err(StorageError::LimitExceeded(m)) => { - assert!(m.contains("in progress on this server (at most 1"), "{m}"); + assert_eq!( + m, + limit_exceeded_message(JobBound::Counted { + name: BACKUP_SERVER_SLOTS_NAME, + limit: engine.backup_server_slot_count, + }) + ); } other => panic!("a full bound refuses: {:?}", other.err()), } + assert!( + started.elapsed() < Duration::from_secs(1), + "refused in {:?}", + started.elapsed() + ); + let rows: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM backups") + .fetch_one(&engine.pool) + .await + .expect("rows"); + assert_eq!(rows, 0, "a refused create writes nothing"); + drop(held); - let _freed = acquire_backup_permit(slots, 1, Duration::from_millis(50)) + engine + .create(ACCOUNT, "src", "bkp", JobPermit::detached()) + .await + .expect("admitted once a slot is free"); + } + + /// The catalog index: `backup_index` rows, their claim, the restore job + /// record and the migration's backfill. + mod catalog_index { + use extenddb_core::backup_format::BackupArn; + use extenddb_storage::BackupCatalog; + use extenddb_storage::backup_catalog::{ + BackupRow, BackupStatus, DeleteStart, Heartbeat, JobStatus, RestoreTargetKey, + StaleClock, StatusFields, StoreIdentity, StoreKind, TimeBounds, UNOWNED_GENERATION, + WriteOutcome, + }; + use futures::TryStreamExt; + + use super::*; + + const REGION: &str = "us-east-1"; + const OTHER_ACCOUNT: &str = "111111111111"; + + fn store() -> StoreIdentity { + StoreIdentity { + store_kind: StoreKind::Filesystem, + store_root: "/var/lib/extenddb/backups".to_owned(), + store_id: "store-1".to_owned(), + } + } + + /// A row for a backup of `table` in `account` minted now, with the + /// manifest fixture's source table. + fn row(account: &str, table: &str, name: &str) -> BackupRow { + let backup_id = extenddb_storage::util::new_backup_id(); + let backup_arn = extenddb_storage::util::backup_arn(REGION, account, table, &backup_id); + let manifest: serde_json::Value = serde_json::from_str(include_str!( + "../../core/testdata/extenddb-manifest/v1.json" + )) + .expect("fixture"); + BackupRow { + backup_id: backup_id.clone(), + account_id: account.to_owned(), + table_name: table.to_owned(), + backup_arn: backup_arn.clone(), + backup_name: name.to_owned(), + status: BackupStatus::Creating, + store: store(), + size_bytes: 0, + item_count: 0, + created_at: extenddb_storage::util::backup_creation_date_time(&backup_id).unwrap(), + completed_at: None, + failure_reason: None, + source_table_details_json: manifest["source_table"].clone(), + owner_generation: 0, + heartbeat_at: 0.0, + } + } + + fn arn_of(row: &BackupRow) -> BackupArn { + BackupArn::parse(&row.backup_arn).unwrap() + } + + /// The claim: a second claim of the same backup is refused while the + /// first is held, the heartbeat confirms the claim and refreshes the + /// row, a status write is conditioned on the generation and the + /// `from` status, and the row is read back by account, backup id, + /// table and ARN. + #[tokio::test] + async fn claim_backup_excludes_a_second_owner_and_fences_its_writes() { + let engine = engine().await; + assert!( + uuid::Uuid::parse_str(&engine.catalog_id().await.expect("id")).is_ok(), + "the catalog has an id from the migration" + ); + assert!( + !engine + .check_maintenance_claim(std::time::Duration::from_secs(300)) + .await + .expect("check") + ); + let row = row(ACCOUNT, "Music", "nightly"); + let claim = engine.claim_backup(&row).await.expect("claim"); + assert_ne!(claim.generation(), UNOWNED_GENERATION); + let second = engine.claim_backup(&row).await.err(); + assert!( + matches!(second, Some(StorageError::AlreadyExists(_))), + "{second:?}" + ); + let stored = engine + .read_backup_row(&arn_of(&row)) + .await + .expect("read") + .expect("the row exists"); + assert_eq!(stored.status, BackupStatus::Creating); + assert_eq!(stored.owner_generation, claim.generation()); + assert_eq!(stored.store, store()); + assert!( + (stored.created_at - row.created_at).abs() < 1e-6, + "{} is the id's instant {}", + stored.created_at, + row.created_at + ); + assert_eq!( + stored.source_table_details_json, + row.source_table_details_json + ); + let before = stored.heartbeat_at; + assert!(before > 0.0); + tokio::time::sleep(Duration::from_millis(20)).await; + assert_eq!( + claim.heartbeat().await.expect("heartbeat"), + Heartbeat::Confirmed + ); + let after = engine + .read_backup_row(&arn_of(&row)) + .await + .unwrap() + .unwrap() + .heartbeat_at; + assert!(after > before, "{after} > {before}"); + + // The same id under another account, table or region finds nothing. + let other_account = + BackupArn::new("aws", REGION, OTHER_ACCOUNT, "Music", &row.backup_id).unwrap(); + assert!( + engine + .read_backup_row(&other_account) + .await + .unwrap() + .is_none() + ); + let other_table = + BackupArn::new("aws", REGION, ACCOUNT, "Other", &row.backup_id).unwrap(); + assert!( + engine + .read_backup_row(&other_table) + .await + .unwrap() + .is_none() + ); + let other_region = + BackupArn::new("aws", "eu-west-1", ACCOUNT, "Music", &row.backup_id).unwrap(); + assert!( + engine + .read_backup_row(&other_region) + .await + .unwrap() + .is_none() + ); + + // A write from the wrong state matches nothing; the right one + // matches once, and the row then carries what it set. + let fields = StatusFields { + size_bytes: Some(1234), + item_count: Some(5), + failure_reason: None, + source_table_details_json: None, + completed: true, + }; + assert_eq!( + claim + .set_status(JobStatus::Running, JobStatus::Available, &fields) + .await + .unwrap(), + WriteOutcome::NotMatched + ); + assert_eq!( + claim + .set_status(JobStatus::Creating, JobStatus::Available, &fields) + .await + .unwrap(), + WriteOutcome::Matched + ); + assert_eq!( + claim + .set_status(JobStatus::Creating, JobStatus::Available, &fields) + .await + .unwrap(), + WriteOutcome::NotMatched + ); + let read = claim.read().await.expect("read"); + assert_eq!(read.status, JobStatus::Available); + assert_eq!(read.owner_generation, claim.generation()); + assert!(read.heartbeat_at >= after); + let stored = engine + .read_backup_row(&arn_of(&row)) + .await + .unwrap() + .unwrap(); + assert_eq!(stored.status, BackupStatus::Available); + assert_eq!(stored.size_bytes, 1234); + assert_eq!(stored.item_count, 5); + assert!(stored.completed_at.is_some()); + + // A row another generation took over confirms nothing for this + // claim. + sqlx::query( + "UPDATE backup_index SET owner_generation = owner_generation + 1 \ + WHERE backup_id = ?", + ) + .bind(&row.backup_id) + .execute(&engine.pool) + .await + .expect("take over"); + assert_eq!(claim.heartbeat().await.expect("heartbeat"), Heartbeat::Lost); + assert_eq!( + claim + .set_status( + JobStatus::Available, + JobStatus::Failed, + &StatusFields::default() + ) + .await + .unwrap(), + WriteOutcome::NotMatched + ); + drop(claim); + // The slot went with the claim: the row can be claimed by a + // successor only through its insert, which the existing row + // refuses. + let third = engine.claim_backup(&row).await.err(); + assert!( + matches!(third, Some(StorageError::AlreadyExists(_))), + "{third:?}" + ); + } + + /// DeleteBackup's first step and the tombstone writes: only an + /// AVAILABLE row moves to DELETING; a CREATING row answers its status; + /// a backup a CREATING restore target reads answers the target's + /// name; DELETING moves to DELETED once; a DELETED row is removed + /// once; a missing row is "Backup not found". + #[tokio::test] + async fn start_backup_delete_checks_the_status_and_the_in_use_rule_under_the_lock() { + let engine = engine().await; + let row = row(ACCOUNT, "Music", "nightly"); + let key = row.key(); + assert!(matches!( + engine.start_backup_delete(&key).await, + Err(StorageError::Validation(m)) if m.contains("Backup not found") + )); + let claim = engine.claim_backup(&row).await.expect("claim"); + assert_eq!( + engine.start_backup_delete(&key).await.unwrap(), + DeleteStart::Status(BackupStatus::Creating) + ); + assert_eq!( + claim + .set_status( + JobStatus::Creating, + JobStatus::Available, + &StatusFields::default() + ) + .await + .unwrap(), + WriteOutcome::Matched + ); + assert_eq!(engine.backup_in_use(&key).await.unwrap(), None); + + // A CREATING restore target registered from the backup: in use, + // naming the target; an ACTIVE one is not. + let target_id = indexed_table(&engine, "del_target").await; + sqlx::query(&format!( + "INSERT INTO table_restores (table_id, source_backup_arn, source_table_arn, \ + restore_date_time, status, kind, owner_generation, heartbeat_at) \ + VALUES (?, ?, 'arn:aws:dynamodb:us-east-1:123456789012:table/Music', {NOW}, \ + 'RUNNING', 'manifest', 7, {NOW})", + NOW = crate::backup_catalog::NOW + )) + .bind(&target_id) + .bind(&row.backup_arn) + .execute(&engine.pool) + .await + .expect("register a target"); + sqlx::query( + "UPDATE tables SET table_status = 'CREATING', status_transition_at = NULL \ + WHERE table_id = ?", + ) + .bind(&target_id) + .execute(&engine.pool) .await - .expect("freed"); + .expect("make it CREATING"); + assert_eq!( + engine.backup_in_use(&key).await.unwrap().as_deref(), + Some("del_target") + ); + assert_eq!( + engine.start_backup_delete(&key).await.unwrap(), + DeleteStart::InUse("del_target".to_owned()) + ); + assert_eq!( + engine.forget_available_backup_row(&key).await.unwrap(), + extenddb_storage::backup_catalog::ForgetRow::InUse("del_target".to_owned()), + "the outright removal keeps the in-use rule" + ); + assert_eq!( + engine + .read_backup_row(&arn_of(&row)) + .await + .unwrap() + .unwrap() + .status, + BackupStatus::Available, + "a refused delete leaves the row AVAILABLE" + ); + sqlx::query("UPDATE tables SET table_status = 'ACTIVE' WHERE table_id = ?") + .bind(&target_id) + .execute(&engine.pool) + .await + .expect("make it ACTIVE"); + assert_eq!(engine.backup_in_use(&key).await.unwrap(), None); + + assert_eq!( + engine.start_backup_delete(&key).await.unwrap(), + DeleteStart::Deleting + ); + assert_eq!( + engine.start_backup_delete(&key).await.unwrap(), + DeleteStart::Status(BackupStatus::Deleting), + "a retry finds the row DELETING" + ); + assert_eq!( + engine.forget_available_backup_row(&key).await.unwrap(), + extenddb_storage::backup_catalog::ForgetRow::Status(BackupStatus::Deleting), + "only an AVAILABLE row is removed outright" + ); + // An AVAILABLE row goes outright, read and listed nowhere, with no + // tombstone; a second removal finds no row. + let mut adopted = self::row(ACCOUNT, "Music", "adopted"); + adopted.status = BackupStatus::Available; + assert!(engine.insert_backup_row(&adopted).await.expect("insert")); + assert_eq!( + engine + .forget_available_backup_row(&adopted.key()) + .await + .unwrap(), + extenddb_storage::backup_catalog::ForgetRow::Removed + ); + assert!( + engine + .read_backup_row(&arn_of(&adopted)) + .await + .unwrap() + .is_none() + ); + for status in [None, Some(BackupStatus::Available)] { + let listed: Vec = engine + .list_backup_rows(status, &adopted.store) + .try_collect() + .await + .unwrap(); + assert!( + !listed.iter().any(|r| r.backup_arn == adopted.backup_arn), + "{status:?}" + ); + } + let by_account: Vec = engine + .list_account_backup_rows(&adopted.account_id, None, &TimeBounds::default()) + .try_collect() + .await + .unwrap(); + assert!( + !by_account + .iter() + .any(|r| r.backup_arn == adopted.backup_arn) + ); + assert!(matches!( + engine.forget_available_backup_row(&adopted.key()).await, + Err(StorageError::Validation(m)) if m.contains("Backup not found") + )); + assert!( + !engine.remove_backup_row(&key).await.unwrap(), + "DELETING is not a tombstone" + ); + assert!(engine.set_backup_deleted(&key).await.unwrap()); + assert!(!engine.set_backup_deleted(&key).await.unwrap()); + let stored = engine + .read_backup_row(&arn_of(&row)) + .await + .unwrap() + .unwrap(); + assert_eq!(stored.status, BackupStatus::Deleted); + assert!(stored.completed_at.is_some()); + assert!(engine.remove_backup_row(&key).await.unwrap()); + assert!(!engine.remove_backup_row(&key).await.unwrap()); + assert!( + engine + .read_backup_row(&arn_of(&row)) + .await + .unwrap() + .is_none() + ); + drop(claim); + } + + /// The listings: by account with an optional table and the time + /// bounds (lower inclusive, upper exclusive) in (created_at, ARN) + /// order, and by store and status. + #[tokio::test] + async fn the_listings_are_scoped_and_ordered() { + let engine = engine().await; + let mut rows = Vec::new(); + for (account, table, name) in [ + (ACCOUNT, "Music", "a"), + (ACCOUNT, "Music", "b"), + (ACCOUNT, "Other", "c"), + (OTHER_ACCOUNT, "Music", "d"), + ] { + tokio::time::sleep(Duration::from_millis(5)).await; + let row = row(account, table, name); + let claim = engine.claim_backup(&row).await.expect("claim"); + if name == "b" { + assert_eq!( + claim + .set_status( + JobStatus::Creating, + JobStatus::Failed, + &StatusFields::default() + ) + .await + .unwrap(), + WriteOutcome::Matched + ); + } + rows.push((row, claim)); + } + let names = |listed: Vec| -> Vec { + listed.into_iter().map(|r| r.backup_name).collect() + }; + let all: Vec = engine + .list_account_backup_rows(ACCOUNT, None, &TimeBounds::default()) + .try_collect() + .await + .unwrap(); + assert_eq!( + names(all.clone()), + ["a", "b", "c"], + "every status, in creation order" + ); + assert_eq!(all[1].status, BackupStatus::Failed); + let music: Vec = engine + .list_account_backup_rows(ACCOUNT, Some("Music"), &TimeBounds::default()) + .try_collect() + .await + .unwrap(); + assert_eq!(names(music), ["a", "b"]); + let bounds = TimeBounds { + lower: Some(rows[1].0.created_at), + upper: Some(rows[2].0.created_at), + }; + let bounded: Vec = engine + .list_account_backup_rows(ACCOUNT, None, &bounds) + .try_collect() + .await + .unwrap(); + assert_eq!(names(bounded), ["b"], "lower inclusive, upper exclusive"); + let other: Vec = engine + .list_account_backup_rows(OTHER_ACCOUNT, None, &TimeBounds::default()) + .try_collect() + .await + .unwrap(); + assert_eq!(names(other), ["d"]); + + let by_store: Vec = engine + .list_backup_rows(None, &store()) + .try_collect() + .await + .unwrap(); + assert_eq!(names(by_store), ["a", "b", "c", "d"]); + let failed: Vec = engine + .list_backup_rows(Some(BackupStatus::Failed), &store()) + .try_collect() + .await + .unwrap(); + assert_eq!(names(failed), ["b"]); + let elsewhere = StoreIdentity { + store_id: "store-2".to_owned(), + ..store() + }; + let none: Vec = engine + .list_backup_rows(None, &elsewhere) + .try_collect() + .await + .unwrap(); + assert!(none.is_empty(), "another store's rows are not listed"); + drop(rows); + } + + /// A catalog at the shape before migration 6 holding a finished + /// restore and a restore the stop interrupted (its target CREATING) + /// migrates with the registrations backfilled: the finished one DONE, + /// the interrupted one RUNNING with `heartbeat_at` at the epoch and + /// the unowned generation, so the reconciler's first pass fails it + /// and removes its target. + #[tokio::test] + async fn migration_6_backfills_the_registrations_and_the_reconciler_removes_the_interrupted_one() + { + let engine = engine().await; + indexed_table(&engine, "bf_src").await; + let backup = engine + .create(ACCOUNT, "bf_src", "b", JobPermit::detached()) + .await + .expect("backup"); + let finished = restore(&engine, "bf_done", &backup.backup_arn) + .await + .expect("restore"); + let interrupted = restore(&engine, "bf_cut", &backup.backup_arn) + .await + .expect("restore"); + let catalog_id = engine.catalog_id().await.expect("id"); + + // Back to the shape before migration 6: the job record's columns + // and index gone, the registrations as the old release wrote + // them, the interrupted target CREATING, and the migration + // forgotten. + let mut undo = String::from("DROP INDEX IF EXISTS idx_table_restores_status;"); + for column in [ + "status", + "kind", + "store_kind", + "store_root", + "store_id", + "owner_generation", + "heartbeat_at", + "spool_host", + "file_index", + "items_applied", + "failure_reason", + "completed_at", + ] { + undo.push_str(&format!( + " ALTER TABLE table_restores DROP COLUMN {column};" + )); + } + sqlx::raw_sql(&undo) + .execute(&engine.pool) + .await + .expect("drop the job record's columns"); + sqlx::query( + "UPDATE tables SET table_status = 'CREATING', status_transition_at = NULL \ + WHERE table_id = ?", + ) + .bind(&interrupted.table_id) + .execute(&engine.pool) + .await + .expect("interrupt the second restore"); + let on_pool = + crate::migrations::SqliteMigrations::on_pool(engine.pool.clone(), ":memory:") + .expect("one connection"); + let runner = extenddb_storage::migrations::Runner::new(&on_pool, "test"); + runner.forget(6).await.expect("forget migration 6"); + let report = runner.migrate().await.expect("migrate again"); + assert_eq!(report.applied.iter().map(|a| a.id).collect::>(), [6]); + + #[allow(clippy::type_complexity)] + let records: Vec<(String, String, String, i64, String, Option)> = sqlx::query_as( + "SELECT t.table_name, r.status, r.kind, r.owner_generation, r.heartbeat_at, \ + r.completed_at = t.creation_date_time \ + FROM table_restores r JOIN tables t ON t.table_id = r.table_id \ + ORDER BY t.table_name", + ) + .fetch_all(&engine.pool) + .await + .expect("read the records"); + assert_eq!(records.len(), 2, "{records:?}"); + let (name, status, kind, generation, heartbeat, completed) = &records[0]; + assert_eq!(name, "bf_cut"); + assert_eq!(status, "RUNNING"); + assert_eq!(kind, "legacy"); + assert_eq!(*generation, i64::try_from(UNOWNED_GENERATION).unwrap()); + assert_eq!(heartbeat, "1970-01-01T00:00:00.000Z", "the epoch, not NULL"); + assert_eq!(*completed, None); + let (name, status, kind, _, _, completed) = &records[1]; + assert_eq!(name, "bf_done"); + assert_eq!(status, "DONE"); + assert_eq!(kind, "legacy"); + assert_eq!( + *completed, + Some(true), + "completed when the table was created" + ); + assert_eq!( + engine.catalog_id().await.expect("id"), + catalog_id, + "a second run keeps the catalog id" + ); + + // The first reconciler pass: the interrupted restore is dead. + let dead = engine + .list_dead_restore_jobs(Duration::from_secs(300)) + .await + .expect("list"); + assert_eq!(dead.len(), 1, "{dead:?}"); + assert_eq!( + dead[0].target, + RestoreTargetKey { + account_id: ACCOUNT.to_owned(), + table_name: "bf_cut".to_owned(), + table_id: interrupted.table_id.clone(), + } + ); + assert_eq!(dead[0].owner_generation, UNOWNED_GENERATION); + assert_eq!(dead[0].heartbeat_at, 0.0); + assert!( + engine + .fail_restore_job( + &dead[0].target, + dead[0].owner_generation, + dead[0].heartbeat_at, + "owner dead" + ) + .await + .unwrap() + ); + assert!(engine.remove_restore_target(&dead[0].target).await.unwrap()); + assert_eq!(status_of(&engine, &interrupted.table_id).await, None); + let exists: bool = sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?)", + ) + .bind(crate::data::data_table_name(&interrupted.table_id)) + .fetch_one(&engine.pool) + .await + .expect("exists"); + assert!(!exists, "the interrupted target's data table is dropped"); + assert_eq!( + status_of(&engine, &finished.table_id).await.as_deref(), + Some("ACTIVE") + ); + } + + /// The legacy restore registers through the job record: RUNNING under + /// the claim's generation while the copy runs, DONE with the ACTIVE + /// flip, and never dead while it runs. + #[tokio::test] + async fn a_legacy_restore_registers_a_running_job_record_and_finishes_it_done() { + let (engine, path) = file_engine().await; + bulk_indexed_table(&engine, "jr_src", BULK_ITEMS).await; + let backup = engine + .create(ACCOUNT, "jr_src", "b", JobPermit::detached()) + .await + .expect("backup"); + let desc = start_restore(&engine, "jr_dst", &backup.backup_arn) + .await + .expect("registers"); + // At return the copy has not finished (several batches remain), + // so the record is read while the restore runs. + assert_eq!( + status_of(&engine, &desc.table_id).await.as_deref(), + Some("CREATING"), + "the restore must return before its copy completes" + ); + let record: (String, String, i64, f64) = sqlx::query_as( + "SELECT status, kind, owner_generation, \ + (julianday('now') - julianday(heartbeat_at)) * 86400.0 \ + FROM table_restores WHERE table_id = ?", + ) + .bind(&desc.table_id) + .fetch_one(&engine.pool) + .await + .expect("record"); + assert_eq!(record.0, "RUNNING"); + assert_eq!(record.1, "legacy"); + assert_ne!(record.2, i64::try_from(UNOWNED_GENERATION).unwrap()); + assert!(record.3 < 60.0, "a fresh heartbeat: {} s old", record.3); + assert!( + engine + .list_dead_restore_jobs(Duration::from_secs(300)) + .await + .unwrap() + .is_empty(), + "a running restore is not dead" + ); + assert!( + wait_for_copy(&engine, &desc.table_id).await, + "target removed" + ); + let record: (String, Option, Option) = sqlx::query_as( + "SELECT status, completed_at, items_applied FROM table_restores \ + WHERE table_id = ?", + ) + .bind(&desc.table_id) + .fetch_one(&engine.pool) + .await + .expect("record"); + assert_eq!(record.0, "DONE"); + assert!(record.1.is_some()); + assert_eq!(record.2, Some(BULK_ITEMS)); + close_file_engine(engine, &path).await; + } + + /// A copy that fails marks its record FAILED with the reason before + /// the target is removed; the removal cascades the record. + #[tokio::test] + async fn a_failed_legacy_restore_marks_its_record_failed_before_removing_the_target() { + let engine = engine().await; + indexed_table(&engine, "src").await; + let backup = engine + .create(ACCOUNT, "src", "bkp", JobPermit::detached()) + .await + .expect("backup"); + // Corrupt one backup row so the copy fails part-way. + sqlx::query( + "UPDATE backup_items SET item_data = '{\"pk\": {\"S\": \"x\"}}' \ + WHERE id = (SELECT MAX(id) FROM backup_items WHERE backup_arn = ?)", + ) + .bind(&backup.backup_arn) + .execute(&engine.pool) + .await + .expect("corrupt"); + let (logs_guard, logs) = crate::schema::log_capture::capture(); + let desc = start_restore(&engine, "dst", &backup.backup_arn) + .await + .expect("registration"); + assert!( + !wait_for_copy(&engine, &desc.table_id).await, + "an item without its sort key cannot restore" + ); + drop(logs_guard); + let registrations: i64 = + sqlx::query_scalar("SELECT COUNT(*) FROM table_restores WHERE table_id = ?") + .bind(&desc.table_id) + .fetch_one(&engine.pool) + .await + .expect("registrations"); + assert_eq!(registrations, 0, "the record went with the target"); + let logs = logs.lock().expect("logs").clone(); + assert!( + logs.iter() + .any(|l| l.contains("removing the partial table")), + "the FAILED write matched and the removal followed: {logs:#?}" + ); + assert!( + !logs.iter().any(|l| l.contains("left to its owner")), + "{logs:#?}" + ); + engine + .delete(ACCOUNT, &backup.backup_arn) + .await + .expect("the backup is free"); + } + + /// `read_store_ids`: the distinct ids of the rows for a root, of + /// every status, scoped to the kind and the root. + #[tokio::test] + async fn read_store_ids_lists_the_distinct_ids_of_a_root() { + let engine = engine().await; + let root = store().store_root.clone(); + assert!( + engine + .read_store_ids(StoreKind::Filesystem, &root) + .await + .expect("ids") + .is_empty() + ); + let first = row(ACCOUNT, "Music", "a"); + let _c1 = engine.claim_backup(&first).await.expect("claim"); + let second = row(ACCOUNT, "Music", "b"); + let c2 = engine.claim_backup(&second).await.expect("claim"); + c2.set_status( + JobStatus::Creating, + JobStatus::Failed, + &StatusFields::default(), + ) + .await + .expect("fail"); + assert_eq!( + engine + .read_store_ids(StoreKind::Filesystem, &root) + .await + .expect("ids"), + ["store-1"], + "two rows of one store, of two statuses, are one id" + ); + let mut other = row(ACCOUNT, "Songs", "c"); + other.store.store_id = "store-2".to_owned(); + let _c3 = engine.claim_backup(&other).await.expect("claim"); + assert_eq!( + engine + .read_store_ids(StoreKind::Filesystem, &root) + .await + .expect("ids"), + ["store-1", "store-2"] + ); + assert!( + engine + .read_store_ids(StoreKind::S3, &root) + .await + .expect("ids") + .is_empty(), + "another kind at the same root is another store" + ); + assert!( + engine + .read_store_ids(StoreKind::Filesystem, "/elsewhere") + .await + .expect("ids") + .is_empty() + ); + } + + /// `insert_backup_row`: the row lands as given, unowned, with + /// `completed_at` and `heartbeat_at` on the database's clock; a + /// second insert, an insert over a CREATING row a claim holds, and an + /// insert of the id under another account (the unique `backup_id`) + /// answer `false` and change nothing; a claim over the inserted row + /// is refused; the row is read and listed like a created one. + #[tokio::test] + async fn insert_backup_row_inserts_an_unowned_row_only_when_absent() { + let engine = engine().await; + let mut adopted = row(ACCOUNT, "Music", "adopted"); + adopted.status = BackupStatus::Available; + adopted.size_bytes = 1234; + adopted.item_count = 5; + // Ignored: the insert sets these three itself. + adopted.owner_generation = 99; + adopted.completed_at = None; + adopted.heartbeat_at = 0.0; + assert!(engine.insert_backup_row(&adopted).await.expect("insert")); + let stored = engine + .read_backup_row(&arn_of(&adopted)) + .await + .expect("read") + .expect("the row exists"); + assert_eq!(stored.owner_generation, UNOWNED_GENERATION); + let completed_at = stored.completed_at.expect("completed_at is set"); + for at in [completed_at, stored.heartbeat_at] { + assert!((at - adopted.created_at).abs() < 300.0, "{at}"); + } + assert!( + (stored.created_at - adopted.created_at).abs() < 1e-3, + "{} is the id's instant {}", + stored.created_at, + adopted.created_at + ); + assert_eq!( + BackupRow { + owner_generation: 99, + completed_at: None, + heartbeat_at: 0.0, + created_at: adopted.created_at, + ..stored.clone() + }, + adopted, + "every other column is the row as given" + ); + + // A second insert of the key, with other values, writes nothing. + let mut again = adopted.clone(); + again.backup_name = "again".to_owned(); + again.size_bytes = 1; + again.status = BackupStatus::Failed; + assert!(!engine.insert_backup_row(&again).await.expect("insert")); + // The id under another account: the unique `backup_id`. + let mut elsewhere = adopted.clone(); + elsewhere.account_id = OTHER_ACCOUNT.to_owned(); + elsewhere.backup_arn = extenddb_storage::util::backup_arn( + REGION, + OTHER_ACCOUNT, + "Music", + &adopted.backup_id, + ); + assert!(!engine.insert_backup_row(&elsewhere).await.expect("insert")); + assert!( + engine + .read_backup_row(&arn_of(&elsewhere)) + .await + .unwrap() + .is_none() + ); + assert_eq!( + engine.read_backup_row(&arn_of(&adopted)).await.unwrap(), + Some(stored.clone()), + "the refused inserts changed nothing" + ); + let claimed = engine.claim_backup(&adopted).await.err(); + assert!( + matches!(claimed, Some(StorageError::AlreadyExists(_))), + "{claimed:?}" + ); + + // Over a CREATING row a live claim holds. + let creating = row(ACCOUNT, "Music", "job"); + let claim = engine.claim_backup(&creating).await.expect("claim"); + let mut over = creating.clone(); + over.status = BackupStatus::Available; + assert!(!engine.insert_backup_row(&over).await.expect("insert")); + let job_row = engine + .read_backup_row(&arn_of(&creating)) + .await + .unwrap() + .unwrap(); + assert_eq!(job_row.status, BackupStatus::Creating); + assert_eq!(job_row.owner_generation, claim.generation()); + + // Read and listed like a created row. + let arns = |rows: Vec| -> Vec { + rows.into_iter().map(|r| r.backup_arn).collect() + }; + let available: Vec = engine + .list_backup_rows(Some(BackupStatus::Available), &store()) + .try_collect() + .await + .unwrap(); + assert_eq!(available, std::slice::from_ref(&stored)); + let creating_rows: Vec = engine + .list_backup_rows(Some(BackupStatus::Creating), &store()) + .try_collect() + .await + .unwrap(); + assert_eq!( + arns(creating_rows), + std::slice::from_ref(&creating.backup_arn) + ); + for table in [None, Some("Music")] { + let listed: Vec = engine + .list_account_backup_rows(ACCOUNT, table, &TimeBounds::default()) + .try_collect() + .await + .unwrap(); + assert!( + listed.contains(&stored), + "{table:?}: {:?}", + arns(listed.clone()) + ); + } + drop(claim); + } + + /// `remove_dead_creating_backup_row`: a CREATING row whose heartbeat + /// is older than the window is removed outright; one with a live + /// owner, one that is not CREATING, and one that is gone are left, + /// answering `false`. + #[tokio::test] + async fn a_dead_creating_row_is_removed_and_a_live_or_moved_one_is_kept() { + let engine = engine().await; + let stale = Duration::from_secs(300); + let age = |backup_id: String| { + let pool = engine.pool.clone(); + async move { + sqlx::query( + "UPDATE backup_index SET heartbeat_at = \ + strftime('%Y-%m-%dT%H:%M:%fZ', 'now', '-10 minutes') WHERE backup_id = ?", + ) + .bind(backup_id) + .execute(&pool) + .await + .expect("age"); + } + }; + let creating = row(ACCOUNT, "Music", "a"); + let key = creating.key(); + let claim = engine.claim_backup(&creating).await.expect("claim"); + assert!( + !engine + .remove_dead_creating_backup_row(&key, stale) + .await + .unwrap(), + "a live owner keeps its row" + ); + age(creating.backup_id.clone()).await; + assert!( + engine + .remove_dead_creating_backup_row(&key, stale) + .await + .unwrap() + ); + assert!( + engine + .read_backup_row(&arn_of(&creating)) + .await + .unwrap() + .is_none() + ); + assert!( + !engine + .remove_dead_creating_backup_row(&key, stale) + .await + .unwrap(), + "gone" + ); + drop(claim); + + let available = row(ACCOUNT, "Music", "b"); + let claim = engine.claim_backup(&available).await.expect("claim"); + claim + .set_status( + JobStatus::Creating, + JobStatus::Available, + &StatusFields::default(), + ) + .await + .expect("available"); + age(available.backup_id.clone()).await; + assert!( + !engine + .remove_dead_creating_backup_row(&available.key(), stale) + .await + .unwrap(), + "only a CREATING row" + ); + assert_eq!( + engine + .read_backup_row(&arn_of(&available)) + .await + .unwrap() + .unwrap() + .status, + BackupStatus::Available + ); + } + + /// `claim_dead_backup` by the slot rule: a CREATING row whose slot no + /// task of this process holds is dead at once, whatever its heartbeat + /// and whatever the stale window (a row the last process left, here + /// inserted with a fresh heartbeat and tested with a window of an + /// hour); a row whose claim a task holds is live and refused; a row + /// whose claim dropped (the job's task ended by any route) is taken + /// over at once. The successor holds the slot: a second taker and a + /// `claim_backup` of the row are refused until the successor's claim + /// drops. The takeover mints a fresh generation and leaves + /// `heartbeat_at` as the dead owner wrote it. The successor's + /// AVAILABLE write carries `source_table_details_json` when `Some`, + /// leaves it when `None`, and stamps `completed_at`. An AVAILABLE row + /// and a missing row are refused, and the slot taken for the test is + /// given back. + #[tokio::test] + async fn claim_dead_backup_takes_over_a_row_whose_slot_no_task_holds() { + let engine = engine().await; + let stale = Duration::from_secs(3600); + let slot_key = |row: &BackupRow| format!("backup:{}", row.backup_id); + + // A row the last process left: inserted directly, no slot held, + // its heartbeat fresh. Dead at once. + let left = row(ACCOUNT, "Music", "left"); + let source_table = serde_json::to_string(&left.source_table_details_json).unwrap(); + sqlx::query(&format!( + "INSERT INTO backup_index (backup_id, account_id, table_name, backup_arn, \ + backup_name, status, store_kind, store_root, store_id, size_bytes, item_count, \ + created_at, source_table_details_json, owner_generation, heartbeat_at) \ + VALUES (?, ?, ?, ?, ?, 'CREATING', ?, ?, ?, 0, 0, ?, ?, 7, {NOW})", + NOW = crate::backup_catalog::NOW + )) + .bind(&left.backup_id) + .bind(&left.account_id) + .bind(&left.table_name) + .bind(&left.backup_arn) + .bind(&left.backup_name) + .bind(left.store.store_kind.as_str()) + .bind(&left.store.store_root) + .bind(&left.store.store_id) + .bind(crate::sqlite_util::format_epoch_seconds(left.created_at)) + .bind(&source_table) + .execute(&engine.pool) + .await + .expect("a row of the last process"); + let before = engine + .read_backup_row(&arn_of(&left)) + .await + .unwrap() + .unwrap(); + assert_eq!(before.owner_generation, 7); + assert!(!engine.job_slots.holds(&slot_key(&left))); + let successor = engine + .claim_dead_backup(&left.key(), stale) + .await + .expect("claim") + .expect("a row with no slot holder is dead at once, fresh heartbeat or not"); + assert_ne!(successor.generation(), 7); + assert_ne!(successor.generation(), UNOWNED_GENERATION); + let after = engine + .read_backup_row(&arn_of(&left)) + .await + .unwrap() + .unwrap(); + assert_eq!(after.owner_generation, successor.generation()); + assert_eq!(after.status, BackupStatus::Creating); + assert_eq!( + after.heartbeat_at, before.heartbeat_at, + "the takeover left the heartbeat as the dead owner wrote it" + ); + assert!( + engine.job_slots.holds(&slot_key(&left)), + "the successor holds the slot" + ); + assert!( + engine + .claim_dead_backup(&left.key(), stale) + .await + .expect("claim") + .is_none(), + "a row another task holds is refused" + ); + assert!( + matches!( + engine.claim_backup(&left).await.err(), + Some(StorageError::AlreadyExists(m)) if m.contains("a live job owns") + ), + "a job's claim of a taken-over row meets the successor's slot" + ); + assert_eq!( + successor.heartbeat().await.expect("heartbeat"), + Heartbeat::Confirmed + ); + // The AVAILABLE write with the manifest's source table: the row + // carries it; a later write with `None` leaves it. + let mut details = left.source_table_details_json.clone(); + details["item_count"] = serde_json::json!(4242); + let fields = StatusFields { + size_bytes: Some(99), + item_count: Some(4242), + failure_reason: None, + source_table_details_json: Some(details.clone()), + completed: true, + }; + assert_eq!( + successor + .set_status(JobStatus::Creating, JobStatus::Available, &fields) + .await + .unwrap(), + WriteOutcome::Matched + ); + let stored = engine + .read_backup_row(&arn_of(&left)) + .await + .unwrap() + .unwrap(); + assert_eq!(stored.status, BackupStatus::Available); + assert_eq!(stored.size_bytes, 99); + assert_eq!(stored.item_count, 4242); + assert_eq!( + stored.source_table_details_json, details, + "the AVAILABLE write carries the source table it was given" + ); + assert_ne!( + stored.source_table_details_json, + left.source_table_details_json + ); + assert!(stored.completed_at.is_some(), "completed_at is stamped"); + assert_eq!( + successor + .set_status( + JobStatus::Available, + JobStatus::Failed, + &StatusFields { + failure_reason: Some("later".to_owned()), + ..StatusFields::default() + } + ) + .await + .unwrap(), + WriteOutcome::Matched + ); + let stored = engine + .read_backup_row(&arn_of(&left)) + .await + .unwrap() + .unwrap(); + assert_eq!( + stored.source_table_details_json, details, + "a write without the member leaves the column" + ); + // Not CREATING any more: refused, and the slot taken for the test + // is given back. + drop(successor); + assert!( + !engine.job_slots.holds(&slot_key(&left)), + "the claim's drop frees the slot" + ); + assert!( + engine + .claim_dead_backup(&left.key(), stale) + .await + .expect("claim") + .is_none(), + "a row that is not CREATING is refused" + ); + assert!( + !engine.job_slots.holds(&slot_key(&left)), + "a refusal leaves no slot behind" + ); + + // A live owner: its claim holds the slot, so it is refused + // whatever the heartbeat; once the claim drops (the task ended), + // the row is dead at once. + let live = row(ACCOUNT, "Music", "live"); + let owner = engine.claim_backup(&live).await.expect("claim"); + sqlx::query( + "UPDATE backup_index SET heartbeat_at = \ + strftime('%Y-%m-%dT%H:%M:%fZ', 'now', '-1 day') WHERE backup_id = ?", + ) + .bind(&live.backup_id) + .execute(&engine.pool) + .await + .expect("age the heartbeat"); + assert!( + engine + .claim_dead_backup(&live.key(), Duration::from_secs(1)) + .await + .expect("claim") + .is_none(), + "a held slot is a live owner, a day-old heartbeat or not" + ); + assert_eq!( + engine + .read_backup_row(&arn_of(&live)) + .await + .unwrap() + .unwrap() + .owner_generation, + owner.generation(), + "a refusal leaves the generation" + ); + drop(owner); + let successor = engine + .claim_dead_backup(&live.key(), stale) + .await + .expect("claim") + .expect("the owner's task ended: dead at once"); + assert_ne!(successor.generation(), UNOWNED_GENERATION); + drop(successor); + + // A missing row: refused, no slot left behind. + let missing = row(ACCOUNT, "Music", "missing"); + assert!( + engine + .claim_dead_backup(&missing.key(), stale) + .await + .expect("claim") + .is_none() + ); + assert!(!engine.job_slots.holds(&slot_key(&missing))); + } + + /// `list_backup_rows_older_than`: the rows of a store with a status + /// whose clock column is older than the age on the database's clock. + /// The two clocks are told apart by a row heartbeated at the start + /// and FAILED after the window passed (as the reconciler fails a dead + /// owner's row): listed on the heartbeat clock at once, not on the + /// completed clock until its own window passes. A FAILED row without + /// `completed_at` is never listed on the completed clock. A CREATING + /// row and another store's row are not listed. The window is crossed + /// by waiting, not by dating rows. + #[tokio::test] + async fn rows_older_than_are_listed_on_the_named_clock() { + let engine = engine().await; + let window = Duration::from_secs(1); + // Names in the listing's order; two rows minted in one millisecond + // share `created_at` and fall to ARN order, so the sets compare + // sorted. + let names = |listed: Vec| -> Vec { + let mut names: Vec = listed.into_iter().map(|r| r.backup_name).collect(); + names.sort(); + names + }; + let older = |status: BackupStatus, clock: StaleClock| { + let engine = engine.clone(); + async move { + let rows: Vec = engine + .list_backup_rows_older_than(status, &store(), clock, window) + .try_collect() + .await + .expect("list"); + names(rows) + } + }; + async fn fail(claim: &dyn extenddb_storage::backup_catalog::JobClaim, completed: bool) { + let fields = StatusFields { + failure_reason: Some("test".to_owned()), + completed, + ..StatusFields::default() + }; + assert_eq!( + claim + .set_status(JobStatus::Creating, JobStatus::Failed, &fields) + .await + .unwrap(), + WriteOutcome::Matched + ); + } + + // All heartbeated now. `late` stays CREATING until the window has + // passed; `unfinished` is FAILED without `completed_at`; + // `creating` stays CREATING; `elsewhere` is another store's; + // `gone` is DELETED now. + let late = row(ACCOUNT, "Music", "late"); + let late_claim = engine.claim_backup(&late).await.expect("claim"); + let unfinished = row(ACCOUNT, "Music", "unfinished"); + let unfinished_claim = engine.claim_backup(&unfinished).await.expect("claim"); + fail(unfinished_claim.as_ref(), false).await; + let creating = row(ACCOUNT, "Music", "creating"); + let _creating_claim = engine.claim_backup(&creating).await.expect("claim"); + let mut elsewhere = row(ACCOUNT, "Music", "elsewhere"); + elsewhere.store.store_id = "store-2".to_owned(); + let elsewhere_claim = engine.claim_backup(&elsewhere).await.expect("claim"); + fail(elsewhere_claim.as_ref(), true).await; + let gone = row(ACCOUNT, "Music", "gone"); + let gone_claim = engine.claim_backup(&gone).await.expect("claim"); + assert_eq!( + gone_claim + .set_status( + JobStatus::Creating, + JobStatus::Available, + &StatusFields::default() + ) + .await + .unwrap(), + WriteOutcome::Matched + ); + assert_eq!( + engine.start_backup_delete(&gone.key()).await.unwrap(), + DeleteStart::Deleting + ); + assert!(engine.set_backup_deleted(&gone.key()).await.unwrap()); + + // Inside the window on every clock: nothing is older. + assert!( + older(BackupStatus::Failed, StaleClock::Heartbeat) + .await + .is_empty() + ); + assert!( + older(BackupStatus::Failed, StaleClock::Completed) + .await + .is_empty() + ); + assert!( + older(BackupStatus::Deleted, StaleClock::Completed) + .await + .is_empty() + ); + + tokio::time::sleep(window + Duration::from_millis(300)).await; + // The reconciler's shape: the owner's last beat is past the + // window, the FAILED write is now. + fail(late_claim.as_ref(), true).await; + assert_eq!( + older(BackupStatus::Failed, StaleClock::Heartbeat).await, + ["late", "unfinished"], + "both heartbeats are past the window; the just-failed row is listed on the \ + heartbeat clock at once" + ); + assert!( + older(BackupStatus::Failed, StaleClock::Completed) + .await + .is_empty(), + "completed just now, or never" + ); + assert_eq!( + older(BackupStatus::Deleted, StaleClock::Completed).await, + ["gone"] + ); + let elsewhere_rows: Vec = engine + .list_backup_rows_older_than( + BackupStatus::Failed, + &elsewhere.store, + StaleClock::Heartbeat, + window, + ) + .try_collect() + .await + .expect("list"); + assert_eq!(names(elsewhere_rows), ["elsewhere"], "scoped to the store"); + + tokio::time::sleep(window + Duration::from_millis(300)).await; + assert_eq!( + older(BackupStatus::Failed, StaleClock::Completed).await, + ["late"], + "the completed window crossed on the database's clock; a row without \ + completed_at is never listed on this clock" + ); + drop(late_claim); + drop(unfinished_claim); + drop(elsewhere_claim); + drop(gone_claim); + } + + /// The maintenance claim: taken once, refused to a second holder + /// naming the first, heartbeated, released; a holder whose heartbeat + /// is older than the stale window is dead, so its claim is not + /// reported held and is taken over, and its handle finds the claim + /// lost. + #[tokio::test] + async fn the_maintenance_claim_excludes_a_second_holder_and_expires_when_stale() { + let engine = engine().await; + let stale = Duration::from_secs(300); + let check = || engine.check_maintenance_claim(stale); + assert!(!check().await.expect("check")); + let first = engine + .maintenance_claim("purge on host-a", stale) + .await + .expect("claim"); + assert_ne!(first.generation(), UNOWNED_GENERATION); + assert!(check().await.expect("check"), "a live claim is held"); + match engine.maintenance_claim("purge on host-b", stale).await { + Err(StorageError::AlreadyExists(m)) => { + assert!(m.contains("purge on host-a"), "{m}"); + } + Err(other) => panic!("a second holder is refused with AlreadyExists: {other:?}"), + Ok(_) => panic!("a second holder is refused"), + } + assert_eq!( + first.heartbeat().await.expect("heartbeat"), + Heartbeat::Confirmed + ); + first.release().await.expect("release"); + assert!(!check().await.expect("check"), "released"); + first.release().await.expect("a second release is a no-op"); + + // A dead holder: its heartbeat aged past the window. + let dead = engine + .maintenance_claim("purge on host-c", stale) + .await + .expect("claim"); + sqlx::query( + "UPDATE backup_maintenance SET heartbeat_at = \ + strftime('%Y-%m-%dT%H:%M:%fZ', 'now', '-10 minutes')", + ) + .execute(&engine.pool) + .await + .expect("age"); + assert!(!check().await.expect("check"), "a stale claim is not held"); + let taker = engine + .maintenance_claim("purge on host-d", stale) + .await + .expect("a dead holder's claim is taken over"); + assert_ne!(taker.generation(), dead.generation()); + assert!(check().await.expect("check")); + assert_eq!( + dead.heartbeat().await.expect("heartbeat"), + Heartbeat::Lost, + "the dead holder's handle finds the claim lost" + ); + dead.release() + .await + .expect("a release by a lost holder changes nothing"); + assert!(check().await.expect("check"), "the taker still holds it"); + let holder: String = sqlx::query_scalar("SELECT holder FROM backup_maintenance") + .fetch_one(&engine.pool) + .await + .expect("holder"); + assert_eq!(holder, "purge on host-d"); + taker.release().await.expect("release"); + assert!(!check().await.expect("check")); + } + + /// An engine that holds the serve lock (a command that opened the + /// file as `serve` does: `backup purge --confirm`) takes over a + /// maintenance claim whose heartbeat is young: under the lock the + /// row's holder is dead whatever its heartbeat says. The holder that + /// was taken over finds its claim lost. The same take on an engine + /// without the lock is refused naming the holder. + #[cfg(unix)] + #[tokio::test] + async fn under_the_serve_lock_a_live_looking_claim_is_taken_over() { + let stale = Duration::from_secs(300); + let unlocked = engine().await; + let killed = unlocked + .maintenance_claim("extenddb backup purge on host-a (pid 1)", stale) + .await + .expect("claim"); + let refused = unlocked + .maintenance_claim("extenddb backup purge on host-a (pid 2)", stale) + .await + .err() + .expect("a live claim is refused without the serve lock"); + assert!(refused.to_string().contains("(pid 1)"), "{refused}"); + + let dir = std::env::temp_dir().join(format!( + "extenddb-maintenance-serve-lock-{}", + uuid::Uuid::new_v4().simple() + )); + std::fs::create_dir_all(&dir).expect("dir"); + let mut locked = unlocked.clone(); + locked.serve_lock = Some(std::sync::Arc::new( + crate::serve_lock::ServeLock::try_acquire(&dir.join("db.sqlite")) + .unwrap_or_else(|_| panic!("the serve lock")), + )); + assert!( + locked.check_maintenance_claim(stale).await.expect("check"), + "the first claim's heartbeat is young" + ); + let taken = locked + .maintenance_claim("extenddb backup purge on host-a (pid 2)", stale) + .await + .expect("taken over under the serve lock"); + assert_ne!(taken.generation(), killed.generation()); + assert_eq!(killed.heartbeat().await.expect("beat"), Heartbeat::Lost); + assert_eq!(taken.heartbeat().await.expect("beat"), Heartbeat::Confirmed); + taken.release().await.expect("release"); + drop(locked); + let _ = std::fs::remove_dir_all(&dir); + } + + /// A zero window selects by status alone: under the serve lock every + /// owner is dead, so a FAILED or CREATING row whose heartbeat is + /// stamped ahead of the database's clock (a backward clock step) is + /// listed as dead and removed, where the age comparison would read it + /// as a live owner and `backup purge` would wait on it and refuse. + /// Under a non-zero window the same rows read as live. The RUNNING + /// restore listing follows the same rule, and the wait set's listing + /// by status sees such a record too. + #[tokio::test] + async fn a_zero_window_selects_dead_owners_by_status_alone() { + let engine = engine().await; + let failed = row(ACCOUNT, "Music", "failed-ahead"); + let failed_claim = engine.claim_backup(&failed).await.expect("claim"); + let fields = StatusFields { + failure_reason: Some("test".to_owned()), + completed: true, + ..StatusFields::default() + }; + assert_eq!( + failed_claim + .set_status(JobStatus::Creating, JobStatus::Failed, &fields) + .await + .unwrap(), + WriteOutcome::Matched + ); + let creating = row(ACCOUNT, "Music", "creating-ahead"); + let _creating_claim = engine.claim_backup(&creating).await.expect("claim"); + // Both heartbeats a day ahead of the clock. + let ahead: i64 = sqlx::query( + "UPDATE backup_index SET heartbeat_at = strftime('%Y-%m-%dT%H:%M:%fZ', \ + 'now', '+1 day') WHERE backup_id IN (?, ?)", + ) + .bind(&failed.backup_id) + .bind(&creating.backup_id) + .execute(&engine.pool) + .await + .expect("stamp ahead") + .rows_affected() + .try_into() + .unwrap(); + assert_eq!(ahead, 2); + let listed = |status: BackupStatus, window: Duration| { + let engine = engine.clone(); + async move { + let rows: Vec = engine + .list_backup_rows_older_than( + status, + &store(), + StaleClock::Heartbeat, + window, + ) + .try_collect() + .await + .expect("list"); + rows.into_iter().map(|r| r.backup_name).collect::>() + } + }; + let window = Duration::from_secs(300); + assert!( + listed(BackupStatus::Failed, window).await.is_empty() + && listed(BackupStatus::Creating, window).await.is_empty(), + "under a window a heartbeat ahead of the clock reads as live" + ); + assert_eq!( + listed(BackupStatus::Failed, Duration::ZERO).await, + ["failed-ahead"] + ); + assert_eq!( + listed(BackupStatus::Creating, Duration::ZERO).await, + ["creating-ahead"] + ); + assert!( + !engine + .remove_dead_creating_backup_row(&creating.key(), window) + .await + .unwrap(), + "kept under a window" + ); + assert!( + engine + .remove_dead_creating_backup_row(&creating.key(), Duration::ZERO) + .await + .unwrap(), + "removed under the zero window whatever the heartbeat says" + ); + assert!( + engine + .read_backup_row(&arn_of(&creating)) + .await + .unwrap() + .is_none() + ); + drop(failed_claim); + } + } + + /// The reconciler's fencing on SQLite (design 4(e)): the restore slot a + /// running copy holds, the FAILED write conditioned on the listed claim, + /// a copy stopping once its record is failed, and a record this build + /// cannot read skipped by the listing. + mod reconciler_fencing { + use extenddb_storage::BackupCatalog; + use extenddb_storage::backup_catalog::{JobStatus, RestoreKind, RestoreTargetKey}; + + use super::*; + use crate::backup_catalog::legacy_restore_slot_key; + + const STALE: Duration = Duration::from_secs(300); + + /// The claim a record carries: `(owner_generation, heartbeat_at)` as + /// the listing reports them. + async fn claim_of(engine: &SqliteEngine, table_id: &str) -> (u64, f64) { + let (generation, heartbeat): (i64, String) = sqlx::query_as( + "SELECT owner_generation, heartbeat_at FROM table_restores WHERE table_id = ?", + ) + .bind(table_id) + .fetch_one(&engine.pool) + .await + .expect("the record"); + ( + u64::try_from(generation).unwrap(), + crate::sqlite_util::timestamp_epoch_seconds(&heartbeat), + ) + } + + fn key_of(desc: &TableDescription) -> RestoreTargetKey { + RestoreTargetKey { + account_id: ACCOUNT.to_owned(), + table_name: desc.table_name.clone(), + table_id: desc.table_id.clone(), + } + } + + /// A running restore holds its in-process slot for the copy's life, + /// and the dead listing skips a RUNNING record whose slot is held + /// whatever its heartbeat says: the heartbeat queues behind every + /// writer holding the write lock (here the test, as an UpdateTable + /// adding a GSI would), so it is not the liveness rule of one + /// process. Once the copy ends the slot is freed, and a record aged + /// with no slot is listed. + #[tokio::test] + async fn a_running_restore_holds_its_slot_and_is_not_listed_dead_by_its_heartbeat() { + let (engine, path) = file_engine().await; + bulk_indexed_table(&engine, "src", BULK_ITEMS).await; + let backup = engine + .create(ACCOUNT, "src", "bkp", JobPermit::detached()) + .await + .expect("backup"); + let desc = start_restore(&engine, "dst", &backup.backup_arn) + .await + .expect("registers"); + let slot = legacy_restore_slot_key(&desc.table_id); + assert!( + engine.job_slots.holds(&slot), + "the restore holds its slot from registration" + ); + + // The copy waits for the write lock the test holds; its heartbeat + // is aged past the window meanwhile. + let gate = engine.write_lock.lock().await; + sqlx::query( + "UPDATE table_restores SET heartbeat_at = \ + strftime('%Y-%m-%dT%H:%M:%fZ', 'now', '-10 minutes') WHERE table_id = ?", + ) + .bind(&desc.table_id) + .execute(&engine.pool) + .await + .expect("age the heartbeat"); + assert_eq!( + status_of(&engine, &desc.table_id).await.as_deref(), + Some("CREATING") + ); + assert!( + engine + .list_dead_restore_jobs(STALE) + .await + .expect("list") + .is_empty(), + "a RUNNING record whose slot a task holds is alive, however old its heartbeat" + ); + drop(gate); + assert!( + wait_for_copy(&engine, &desc.table_id).await, + "the copy finishes" + ); + assert!( + !engine.job_slots.holds(&slot), + "the slot is freed with the claim when the copy ends" + ); + + // The state a stopped process leaves: no slot, an aged RUNNING + // record. Listed. + sqlx::query( + "UPDATE tables SET table_status = 'CREATING', status_transition_at = NULL \ + WHERE table_id = ?", + ) + .bind(&desc.table_id) + .execute(&engine.pool) + .await + .expect("simulate a crash"); + sqlx::query( + "UPDATE table_restores SET status = 'RUNNING', completed_at = NULL, \ + heartbeat_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now', '-10 minutes') \ + WHERE table_id = ?", + ) + .bind(&desc.table_id) + .execute(&engine.pool) + .await + .expect("age the record"); + let dead = engine.list_dead_restore_jobs(STALE).await.expect("list"); + assert_eq!(dead.len(), 1, "{dead:?}"); + assert_eq!(dead[0].target, key_of(&desc)); + close_file_engine(engine, &path).await; + } + + /// `fail_restore_job` is conditioned on the `owner_generation` and + /// `heartbeat_at` the listing read: a heartbeat that lands between + /// the listing and the write (the owner is alive after all) makes the + /// write match nothing, and so does a generation that is not the + /// record's; the record stays RUNNING under its owner's generation. + /// With the listed values current, the write lands and sets a fresh + /// generation, fencing the dead owner's claim out. + #[tokio::test] + async fn the_failed_write_matches_only_the_listed_generation_and_heartbeat() { + let engine = engine().await; + indexed_table(&engine, "src").await; + let backup = engine + .create(ACCOUNT, "src", "bkp", JobPermit::detached()) + .await + .expect("backup"); + let desc = restore(&engine, "dst", &backup.backup_arn) + .await + .expect("restore"); + let key = key_of(&desc); + let age = |seconds: i64| { + let pool = engine.pool.clone(); + let id = desc.table_id.clone(); + async move { + sqlx::query( + "UPDATE tables SET table_status = 'CREATING', \ + status_transition_at = NULL WHERE table_id = ?", + ) + .bind(&id) + .execute(&pool) + .await + .expect("crash"); + sqlx::query( + "UPDATE table_restores SET status = 'RUNNING', completed_at = NULL, \ + heartbeat_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now', ? || ' seconds') \ + WHERE table_id = ?", + ) + .bind(-seconds) + .bind(&id) + .execute(&pool) + .await + .expect("age"); + } + }; + age(600).await; + let dead = engine.list_dead_restore_jobs(STALE).await.expect("list"); + assert_eq!(dead.len(), 1, "{dead:?}"); + let listed = &dead[0]; + let (generation, heartbeat) = claim_of(&engine, &desc.table_id).await; + assert_eq!( + (listed.owner_generation, listed.heartbeat_at), + (generation, heartbeat) + ); + + // The owner's heartbeat lands after the listing. + sqlx::query( + "UPDATE table_restores SET heartbeat_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now') \ + WHERE table_id = ?", + ) + .bind(&desc.table_id) + .execute(&engine.pool) + .await + .expect("heartbeat"); + assert!( + !engine + .fail_restore_job( + &key, + listed.owner_generation, + listed.heartbeat_at, + "owner dead" + ) + .await + .expect("fail"), + "a heartbeat after the listing keeps the record its owner's" + ); + assert_eq!( + job_status(&engine, &desc.table_id).await.as_deref(), + Some("RUNNING") + ); + assert_eq!( + claim_of(&engine, &desc.table_id).await.0, + generation, + "the generation is untouched" + ); + assert!( + engine + .list_dead_restore_jobs(STALE) + .await + .expect("list") + .is_empty(), + "a record with a fresh heartbeat is not dead" + ); + + // Another generation with the current heartbeat: no match either. + let (_, current_heartbeat) = claim_of(&engine, &desc.table_id).await; + assert!( + !engine + .fail_restore_job(&key, generation + 1, current_heartbeat, "owner dead") + .await + .expect("fail") + ); + assert_eq!( + job_status(&engine, &desc.table_id).await.as_deref(), + Some("RUNNING") + ); + + // Listed values that are still current: the write lands, under a + // fresh generation. + age(600).await; + let dead = engine.list_dead_restore_jobs(STALE).await.expect("list"); + assert_eq!(dead.len(), 1); + assert!( + engine + .fail_restore_job( + &dead[0].target, + dead[0].owner_generation, + dead[0].heartbeat_at, + "owner dead" + ) + .await + .expect("fail") + ); + assert_eq!( + job_status(&engine, &desc.table_id).await.as_deref(), + Some("FAILED") + ); + let (after, _) = claim_of(&engine, &desc.table_id).await; + assert_ne!( + after, dead[0].owner_generation, + "the FAILED write carries a fresh generation" + ); + assert_ne!(after, extenddb_storage::backup_catalog::UNOWNED_GENERATION); + assert!(engine.remove_restore_target(&key).await.expect("remove")); + } + + /// A copy whose record was written FAILED stops at its next batch: the + /// per-batch check finds the record no longer RUNNING under the copy's + /// generation, the copy's own FAILED write matches nothing, so it + /// removes nothing and leaves the CREATING target with the FAILED + /// record for the FAILED-target duty, which removes it once stale. + /// Fewer items than the backup holds were copied. The FAILED is + /// written under the record's own generation (the shape a job's own + /// failure path writes; the reconciler's `fail_restore_job` also moves + /// the generation, which the check catches on its own), so the status + /// term of the per-batch check is what this test pins. + #[tokio::test] + async fn a_copy_whose_record_was_failed_stops_at_its_next_batch() { + let (engine, path) = file_engine().await; + bulk_indexed_table(&engine, "src", BULK_ITEMS).await; + let backup = engine + .create(ACCOUNT, "src", "bkp", JobPermit::detached()) + .await + .expect("backup"); + let desc = start_restore(&engine, "dst", &backup.backup_arn) + .await + .expect("registers"); + let key = key_of(&desc); + let slot = legacy_restore_slot_key(&desc.table_id); + + // Hold the copy between two batches and write the record FAILED + // under its own generation (the write lock is the test's, as a + // writer's would be). + let gate = engine.write_lock.lock().await; + let (base_at_gate, _) = target_counts(&engine, &desc.table_id).await; + assert!(base_at_gate < BULK_ITEMS, "the copy is not done"); + assert_eq!( + job_status(&engine, &desc.table_id).await.as_deref(), + Some("RUNNING") + ); + let (generation, _) = claim_of(&engine, &desc.table_id).await; + let written = sqlx::query(&format!( + "UPDATE table_restores SET status = 'FAILED', failure_reason = 'owner dead', \ + completed_at = {} WHERE table_id = ? AND status = 'RUNNING'", + crate::backup_catalog::NOW + )) + .bind(&desc.table_id) + .execute(&engine.pool) + .await + .expect("fail the record") + .rows_affected(); + assert_eq!(written, 1); + drop(gate); + + // The copy stops: its slot is freed with its claim. + let deadline = tokio::time::Instant::now() + Duration::from_secs(30); + while engine.job_slots.holds(&slot) { + assert!( + tokio::time::Instant::now() < deadline, + "the copy did not stop after its record was failed" + ); + tokio::time::sleep(Duration::from_millis(10)).await; + } + let (copied, _) = target_counts(&engine, &desc.table_id).await; + assert!( + copied < BULK_ITEMS, + "the copy stopped short: {copied} of {BULK_ITEMS} items" + ); + assert_eq!( + status_of(&engine, &desc.table_id).await.as_deref(), + Some("CREATING"), + "the target is left to the reconciler, not removed by the copy" + ); + let (status, reason): (String, Option) = sqlx::query_as( + "SELECT status, failure_reason FROM table_restores WHERE table_id = ?", + ) + .bind(&desc.table_id) + .fetch_one(&engine.pool) + .await + .expect("record"); + assert_eq!(status, "FAILED"); + assert_eq!( + reason.as_deref(), + Some("owner dead"), + "the record is left as written" + ); + assert_eq!( + claim_of(&engine, &desc.table_id).await.0, + generation, + "the stopped copy writes nothing on its row" + ); + tokio::time::sleep(Duration::from_millis(200)).await; + assert_eq!( + target_counts(&engine, &desc.table_id).await.0, + copied, + "the stopped copy writes nothing more" + ); + + // The FAILED-target duty: listed once the record is stale, removed. + sqlx::query( + "UPDATE table_restores SET completed_at = \ + strftime('%Y-%m-%dT%H:%M:%fZ', 'now', '-10 minutes') WHERE table_id = ?", + ) + .bind(&desc.table_id) + .execute(&engine.pool) + .await + .expect("age"); + let dead = engine.list_dead_restore_jobs(STALE).await.expect("list"); + assert_eq!(dead.len(), 1, "{dead:?}"); + assert_eq!(dead[0].status, JobStatus::Failed); + assert_eq!(dead[0].target, key); + assert!(engine.remove_restore_target(&key).await.expect("remove")); + assert_eq!(status_of(&engine, &desc.table_id).await, None); + engine + .delete(ACCOUNT, &backup.backup_arn) + .await + .expect("the backup is free"); + close_file_engine(engine, &path).await; + } + + /// A record this build cannot read (a `kind` no release writes) is + /// logged and skipped by the dead listing, as ListBackups skips a row + /// it cannot read, so the cycle's other duties run: the readable dead + /// record beside it is listed. + #[tokio::test] + async fn an_unreadable_record_is_skipped_by_the_dead_listing() { + let engine = engine().await; + indexed_table(&engine, "src").await; + let backup = engine + .create(ACCOUNT, "src", "bkp", JobPermit::detached()) + .await + .expect("backup"); + let unreadable = restore(&engine, "unreadable", &backup.backup_arn) + .await + .expect("restore"); + let readable = restore(&engine, "readable", &backup.backup_arn) + .await + .expect("restore"); + for (desc, kind) in [(&unreadable, "manifest-v2"), (&readable, "legacy")] { + sqlx::query( + "UPDATE tables SET table_status = 'CREATING', status_transition_at = NULL \ + WHERE table_id = ?", + ) + .bind(&desc.table_id) + .execute(&engine.pool) + .await + .expect("crash"); + sqlx::query( + "UPDATE table_restores SET status = 'RUNNING', completed_at = NULL, kind = ?, \ + heartbeat_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now', '-10 minutes') \ + WHERE table_id = ?", + ) + .bind(kind) + .bind(&desc.table_id) + .execute(&engine.pool) + .await + .expect("age"); + } + let dead = engine + .list_dead_restore_jobs(STALE) + .await + .expect("one unreadable row does not fail the listing"); + assert_eq!(dead.len(), 1, "{dead:?}"); + assert_eq!(dead[0].target, key_of(&readable)); + assert_eq!(dead[0].kind, RestoreKind::Legacy); + } + + /// The restore heartbeat applies only to a RUNNING record under the + /// claim's generation: on a DONE or FAILED record it is Lost, so a + /// copy still beating on a record someone moved stops at that beat. + #[tokio::test] + async fn the_restore_heartbeat_requires_a_running_record() { + use extenddb_storage::backup_catalog::{Heartbeat, JobClaim}; + let engine = engine().await; + indexed_table(&engine, "src").await; + let backup = engine + .create(ACCOUNT, "src", "bkp", JobPermit::detached()) + .await + .expect("backup"); + let desc = restore(&engine, "dst", &backup.backup_arn) + .await + .expect("restore"); + let (generation, _) = claim_of(&engine, &desc.table_id).await; + let claim = crate::backup_catalog::SqliteJobClaim::for_restore( + &engine, + &desc.table_id, + generation, + JobPermit::detached(), + ); + assert_eq!( + claim.heartbeat().await.expect("beat"), + Heartbeat::Lost, + "a DONE record is not RUNNING" + ); + for status in ["RUNNING", "FAILED"] { + sqlx::query("UPDATE table_restores SET status = ? WHERE table_id = ?") + .bind(status) + .bind(&desc.table_id) + .execute(&engine.pool) + .await + .expect("set"); + let expected = if status == "RUNNING" { + Heartbeat::Confirmed + } else { + Heartbeat::Lost + }; + assert_eq!(claim.heartbeat().await.expect("beat"), expected, "{status}"); + } + } } } diff --git a/crates/storage-sqlite/src/backup_catalog.rs b/crates/storage-sqlite/src/backup_catalog.rs new file mode 100644 index 000000000..a91a34734 --- /dev/null +++ b/crates/storage-sqlite/src/backup_catalog.rs @@ -0,0 +1,2059 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The catalog index on SQLite: `backup_index`, the restore job fields of +//! `table_restores`, `backup_maintenance` and `settings.catalog_id`. +//! +//! One process serves a database file (the serve lock enforces it, on Unix; +//! on a platform where no lock is taken the server binds CreateBackup to the +//! in-catalog path, `extenddb_server::backup_store::in_catalog_create_only`, +//! since two servers on one file would take each other's rows for dead), so +//! a job's claim is an in-process slot ([`JobSlots`]): a key held in a set for +//! the job's life and released by a drop guard, so a task that ends by any +//! route, a panic included, frees it. A restart frees every slot at once, +//! which is the proof of death this backend needs: a CREATING row (a +//! `backup_index` row or an in-catalog `backups` row) whose slot no task +//! holds is dead, whatever its heartbeat says, so the reconciler's takeover +//! ([`BackupCatalog::claim_dead_backup`]) reads no stale window here, and +//! every RUNNING restore job record of the stopped process is aged at startup +//! so the reconciler's first cycle finds it +//! ([`SqliteEngine::expire_restore_job_heartbeats`]). A store-backed +//! restore's claim holds the slot of its target name +//! ([`restore_slot_key`]) the same way: the registration takes it before the +//! target exists, so two registrations into one name exclude each other in +//! this process, and a RUNNING `manifest` record whose slot no task holds is +//! taken over at once by [`BackupCatalog::claim_restore_dead`]. +//! +//! Every write a job makes to its row goes through its claim and carries the +//! claim's `owner_generation`, so a row a successor took over matches no +//! write. The lock on a row for DeleteBackup and a restore's registration is +//! the engine write lock with a `BEGIN IMMEDIATE` transaction, as for every +//! other writer (design decision D1). Times are RFC 3339 text on the +//! database's clock (`strftime('now')`), and staleness is measured there. +//! A conditional write never has an unknown outcome here (one process, one +//! file: a statement either ran or errored), so `WriteOutcome::Unknown` is +//! never returned. +//! +//! The maintenance claim is the `backup_maintenance` row, taken under the +//! engine write lock: refused while a row with a heartbeat younger than the +//! stale window exists, taken over (replaced) from a dead holder, heartbeated +//! and released by generation. One process serves a file, so a holder is a +//! command run while the server is stopped. Such a command passes a zero +//! stale window, which the dead-owner listings and removals read as "every +//! owner is dead" and select by status alone ([`zero_window_clause`]), so a +//! heartbeat stamped ahead of the clock is not read as a live owner. + +use std::collections::HashSet; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use extenddb_core::backup_format::BackupArn; +use extenddb_core::types::{CreateTableInput, RestoreSummary}; +use extenddb_storage::JobPermit; +use extenddb_storage::backup_catalog::{ + BackupCatalog, BackupKey, BackupRow, BackupStatus, DeleteStart, ForgetRow, Heartbeat, JobClaim, + JobRow, JobStatus, MaintenanceClaim, RestoreJobRow, RestoreKind, RestoreRegistered, + RestoreRegistrationRow, RestoreTargetKey, StaleClock, StatusFields, StoreIdentity, StoreKind, + TimeBounds, UNOWNED_GENERATION, WriteOutcome, new_generation, +}; +use extenddb_storage::error::StorageError; +use futures::future::BoxFuture; +use futures::stream::BoxStream; + +use crate::sqlite_util::{format_epoch_seconds, timestamp_epoch_seconds}; +use crate::store::SqliteEngine; + +/// The database's clock as a catalog timestamp, millisecond precision. +pub(crate) const NOW: &str = "strftime('%Y-%m-%dT%H:%M:%fZ', 'now')"; + +/// The in-process job slots of one engine: the key of every row a job of +/// this process owns. A key is held from before the row exists until the job +/// ends, so a row whose key is absent has no live owner. +#[derive(Debug, Default)] +pub(crate) struct JobSlots(Mutex>); + +impl JobSlots { + /// Take `key`, or `None` when a job of this process holds it. + pub(crate) fn take(slots: &Arc, key: String) -> Option { + let mut held = slots + .0 + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + held.insert(key.clone()).then(|| JobSlot { + slots: Arc::clone(slots), + key, + }) + } + + /// Whether a job of this process holds `key`. + pub(crate) fn holds(&self, key: &str) -> bool { + self.0 + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .contains(key) + } +} + +/// A held slot; dropping it frees the key. +#[derive(Debug)] +pub(crate) struct JobSlot { + slots: Arc, + key: String, +} + +impl Drop for JobSlot { + fn drop(&mut self) { + self.slots + .0 + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .remove(&self.key); + } +} + +/// The slot key of a backup job's claim on a `backup_index` row. +fn backup_slot_key(backup_id: &str) -> String { + format!("backup:{backup_id}") +} + +/// The slot key of a legacy create's hold on its CREATING `backups` row. +pub(crate) fn legacy_backup_slot_key(backup_arn: &str) -> String { + format!("legacy-backup:{backup_arn}") +} + +/// The slot key of an in-catalog restore's hold on its job record, by +/// target table id. Held for the restore's life +/// ([`SqliteJobClaim::for_restore`]), so the reconciler never judges a +/// running restore of this process by its heartbeat, which queues behind +/// every writer holding the write lock. +pub(crate) fn legacy_restore_slot_key(table_id: &str) -> String { + format!("legacy-restore:{table_id}") +} + +/// The slot key of a store-backed restore's claim on its target name: taken +/// before the target exists, so two registrations into one name in this +/// process exclude each other before either reaches the catalog, and held +/// by the claim for the job's life. +pub(crate) fn restore_slot_key(account_id: &str, table_name: &str) -> String { + format!("restore:{account_id}/{table_name}") +} + +/// An `owner_generation` as the catalog stores it (`INTEGER`); generations +/// are minted below `i64::MAX`. +pub(crate) fn generation_column(generation: u64) -> i64 { + i64::try_from(generation).unwrap_or(i64::MAX) +} + +/// The row a claim writes through. +#[derive(Debug, Clone)] +pub(crate) enum JobRecord { + /// A `backup_index` row. + Backup(BackupKey), + /// The `table_restores` record of a restore target. + Restore { table_id: String }, +} + +/// A job's handle on its record: the in-process slot, the generation, and +/// the engine the conditional writes go through. Held by a backup job on its +/// `backup_index` row and by a legacy restore on its job record. +pub(crate) struct SqliteJobClaim { + engine: SqliteEngine, + _slot: Option, + _permit: Option, + generation: u64, + record: JobRecord, +} + +impl SqliteJobClaim { + /// A claim on a restore job record, born with the registration that + /// writes `generation` on the row. The permit is the job's slot in the + /// engine's `max_concurrent_jobs`, freed with the claim. The in-process + /// slot ([`legacy_restore_slot_key`]) is the SQLite rule of design 4(e): a + /// RUNNING record whose slot a task of this process holds is alive + /// whatever its heartbeat says, so a heartbeat delayed behind a long + /// writer (an UpdateTable adding a GSI to a large table holds the write + /// lock for minutes) cannot make the reconciler remove a live restore; + /// one whose slot no task holds (a restart frees every slot) is judged by + /// its heartbeat, which the restart ages to the epoch. The id is fresh, + /// so the slot is free. + pub(crate) fn for_restore( + engine: &SqliteEngine, + table_id: &str, + generation: u64, + permit: JobPermit, + ) -> Self { + Self { + engine: engine.clone(), + _slot: JobSlots::take(&engine.job_slots, legacy_restore_slot_key(table_id)), + _permit: Some(permit), + generation, + record: JobRecord::Restore { + table_id: table_id.to_owned(), + }, + } + } + + /// A claim on a store-backed restore's job record, born with the + /// registration that writes `generation` on the row, holding the + /// target name's slot ([`restore_slot_key`]) for the job's life. The + /// engine's job holds the permit; none is held here. + pub(crate) fn for_manifest_restore( + engine: &SqliteEngine, + table_id: &str, + generation: u64, + slot: JobSlot, + ) -> Self { + Self { + engine: engine.clone(), + _slot: Some(slot), + _permit: None, + generation, + record: JobRecord::Restore { + table_id: table_id.to_owned(), + }, + } + } + + /// Heartbeat the record every `period` until a heartbeat is not + /// confirmed; returns why. Never returns while the claim is confirmed. + pub(crate) async fn heartbeat_loop(&self, period: Duration) -> StorageError { + let mut tick = tokio::time::interval_at(tokio::time::Instant::now() + period, period); + tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + loop { + tick.tick().await; + match self.heartbeat().await { + Ok(Heartbeat::Confirmed) => {} + Ok(Heartbeat::Lost) => { + return StorageError::Internal( + "the job record no longer carries this claim's generation".to_owned(), + ); + } + Err(e) => return e, + } + } + } + + /// The beat's `UPDATE`, run by a caller that holds the engine write + /// lock. + async fn beat_under_the_lock(&self) -> Result { + let affected = match &self.record { + JobRecord::Backup(key) => { + sqlx::query(&format!( + "UPDATE backup_index SET heartbeat_at = {NOW} \ + WHERE account_id = ? AND backup_id = ? AND owner_generation = ?" + )) + .bind(&key.account_id) + .bind(&key.backup_id) + .bind(generation_column(self.generation)) + .execute(&self.engine.pool) + .await + } + JobRecord::Restore { table_id } => { + sqlx::query(&format!( + "UPDATE table_restores SET heartbeat_at = {NOW} \ + WHERE table_id = ? AND status = 'RUNNING' AND owner_generation = ?" + )) + .bind(table_id) + .bind(generation_column(self.generation)) + .execute(&self.engine.pool) + .await + } + } + .map_err(db_err)? + .rows_affected(); + Ok(if affected == 1 { + Heartbeat::Confirmed + } else { + Heartbeat::Lost + }) + } +} + +impl JobClaim for SqliteJobClaim { + fn generation(&self) -> u64 { + self.generation + } + + /// The beat is an `UPDATE` under the engine write lock, so it queues + /// behind every writer that holds or waits for that lock (item writes, + /// a restore's copy batches, the GSI backfill, the TTL sweep, a table's + /// drop) and returns only once they have; a busy writer delays the beat + /// by as long as it holds the lock. + fn heartbeat(&self) -> BoxFuture<'_, Result> { + Box::pin(async move { + let _writer = self.engine.write_lock.lock().await; + self.beat_under_the_lock().await + }) + } + + /// The bound starts once the beat holds the engine write lock: the wait + /// for the lock is this process's own queue (a writer holding it past a + /// heartbeat period, a GSI backfill say, delays the beat and does not end + /// the job), and only the `UPDATE` itself is held to `bound`. + fn heartbeat_within( + &self, + bound: Duration, + ) -> BoxFuture<'_, Option>> { + Box::pin(async move { + let _writer = self.engine.write_lock.lock().await; + tokio::time::timeout(bound, self.beat_under_the_lock()) + .await + .ok() + }) + } + + /// The wait for the engine write lock is outside the bound (above): a + /// beat parked there confirms the claim for as long as it waits. + fn heartbeat_parks_before_its_bound(&self) -> bool { + true + } + + fn set_status<'a>( + &'a self, + from: JobStatus, + to: JobStatus, + fields: &'a StatusFields, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + let _writer = self.engine.write_lock.lock().await; + let affected = match &self.record { + JobRecord::Backup(key) => { + let source_table = fields + .source_table_details_json + .as_ref() + .map(serde_json::to_string) + .transpose() + .map_err(|e| { + StorageError::Internal(format!("source table details: {e}")) + })?; + sqlx::query(&format!( + "UPDATE backup_index SET status = ?, \ + size_bytes = COALESCE(?, size_bytes), \ + item_count = COALESCE(?, item_count), \ + failure_reason = COALESCE(?, failure_reason), \ + source_table_details_json = COALESCE(?, source_table_details_json), \ + completed_at = CASE WHEN ? THEN {NOW} ELSE completed_at END \ + WHERE account_id = ? AND backup_id = ? AND owner_generation = ? \ + AND status = ?" + )) + .bind(to.as_str()) + .bind(fields.size_bytes) + .bind(fields.item_count) + .bind(fields.failure_reason.as_deref()) + .bind(source_table) + .bind(fields.completed) + .bind(&key.account_id) + .bind(&key.backup_id) + .bind(generation_column(self.generation)) + .bind(from.as_str()) + .execute(&self.engine.pool) + .await + } + JobRecord::Restore { table_id } => { + // The record's progress (`file_index`, `items_applied`) + // is `bulk_load`'s write; a status write leaves it. + sqlx::query(&format!( + "UPDATE table_restores SET status = ?, \ + failure_reason = COALESCE(?, failure_reason), \ + completed_at = CASE WHEN ? THEN {NOW} ELSE completed_at END \ + WHERE table_id = ? AND owner_generation = ? AND status = ?" + )) + .bind(to.as_str()) + .bind(fields.failure_reason.as_deref()) + .bind(fields.completed) + .bind(table_id) + .bind(generation_column(self.generation)) + .bind(from.as_str()) + .execute(&self.engine.pool) + .await + } + } + .map_err(db_err)? + .rows_affected(); + // One process and one file: a statement either ran or errored, + // so no outcome is ever unknown. + Ok(if affected == 1 { + WriteOutcome::Matched + } else { + WriteOutcome::NotMatched + }) + }) + } + + fn read(&self) -> BoxFuture<'_, Result> { + Box::pin(async move { + let row: Option<(String, i64, String, Option)> = match &self.record { + JobRecord::Backup(key) => { + sqlx::query_as( + "SELECT status, owner_generation, heartbeat_at, failure_reason \ + FROM backup_index WHERE account_id = ? AND backup_id = ?", + ) + .bind(&key.account_id) + .bind(&key.backup_id) + .fetch_optional(&self.engine.pool) + .await + } + JobRecord::Restore { table_id } => { + sqlx::query_as( + "SELECT status, owner_generation, heartbeat_at, failure_reason \ + FROM table_restores WHERE table_id = ?", + ) + .bind(table_id) + .fetch_optional(&self.engine.pool) + .await + } + } + .map_err(db_err)?; + let (status, generation, heartbeat_at, failure_reason) = + row.ok_or_else(|| match &self.record { + JobRecord::Backup(key) => not_found(&key.backup_arn), + JobRecord::Restore { table_id } => StorageError::Internal(format!( + "restore job record of table {table_id} is gone" + )), + })?; + Ok(JobRow { + status: JobStatus::parse(&status)?, + owner_generation: u64::try_from(generation).unwrap_or_default(), + heartbeat_at: timestamp_epoch_seconds(&heartbeat_at), + failure_reason, + }) + }) + } +} + +/// The name of a CREATING restore target registered from `backup_arn`, if +/// any: the one in-use rule DeleteBackup, a restore's registration and +/// `backup_in_use` share. Reads the target's status, not the job's. +pub(crate) async fn creating_restore_target<'e, E: sqlx::SqliteExecutor<'e>>( + executor: E, + backup_arn: &str, +) -> Result, StorageError> { + sqlx::query_scalar( + "SELECT t.table_name FROM table_restores r JOIN tables t ON t.table_id = r.table_id \ + WHERE r.source_backup_arn = ? AND t.table_status = 'CREATING' LIMIT 1", + ) + .bind(backup_arn) + .fetch_optional(executor) + .await + .map_err(db_err) +} + +/// The error for a backup a CREATING restore target still reads: the shared +/// text, [`extenddb_storage::backup_catalog::backup_in_use`]. +pub(crate) fn backup_in_use(table: &str, backup_arn: &str) -> StorageError { + extenddb_storage::backup_catalog::backup_in_use(table, backup_arn) +} + +fn db_err(e: sqlx::Error) -> StorageError { + StorageError::Internal(format!("Database error: {e}")) +} + +fn not_found(backup_arn: &str) -> StorageError { + extenddb_storage::backup_catalog::backup_not_found(backup_arn) +} + +/// The columns of one `backup_index` row, in the order every read selects +/// them. +const ROW_COLUMNS: &str = "backup_id, account_id, table_name, backup_arn, backup_name, status, \ + store_kind, store_root, store_id, size_bytes, item_count, created_at, completed_at, \ + failure_reason, source_table_details_json, owner_generation, heartbeat_at"; + +/// One `backup_index` row as the driver reads it. +#[derive(sqlx::FromRow)] +struct RowTuple { + backup_id: String, + account_id: String, + table_name: String, + backup_arn: String, + backup_name: String, + status: String, + store_kind: String, + store_root: String, + store_id: String, + size_bytes: i64, + item_count: i64, + created_at: String, + completed_at: Option, + failure_reason: Option, + source_table_details_json: String, + owner_generation: i64, + heartbeat_at: String, +} + +fn row_from_tuple(t: RowTuple) -> Result { + let RowTuple { + backup_id, + account_id, + table_name, + backup_arn, + backup_name, + status, + store_kind, + store_root, + store_id, + size_bytes, + item_count, + created_at, + completed_at, + failure_reason, + source_table_details_json, + owner_generation, + heartbeat_at, + } = t; + let store_kind = StoreKind::parse(&store_kind).ok_or_else(|| { + StorageError::Internal(format!( + "backup {backup_arn} carries an unknown store kind {store_kind:?}" + )) + })?; + let source_table_details_json = + serde_json::from_str(&source_table_details_json).map_err(|e| { + StorageError::Internal(format!( + "backup {backup_arn} carries a source table that is not JSON: {e}" + )) + })?; + Ok(BackupRow { + backup_id, + account_id, + table_name, + backup_arn, + backup_name, + status: BackupStatus::parse(&status)?, + store: StoreIdentity { + store_kind, + store_root, + store_id, + }, + size_bytes, + item_count, + created_at: timestamp_epoch_seconds(&created_at), + completed_at: completed_at.as_deref().map(timestamp_epoch_seconds), + failure_reason, + source_table_details_json, + owner_generation: u64::try_from(owner_generation).unwrap_or_default(), + heartbeat_at: timestamp_epoch_seconds(&heartbeat_at), + }) +} + +/// The rows `read` fetches, as a stream: the read runs once, on the first +/// poll, and a failed read is one `Err` item. +fn rows_stream<'a, F>(read: F) -> BoxStream<'a, Result> +where + F: std::future::Future, StorageError>> + Send + 'a, +{ + use futures::StreamExt; + Box::pin( + futures::stream::once(async move { + match read.await { + Ok(rows) => rows.into_iter().map(row_from_tuple).collect::>(), + Err(e) => vec![Err(e)], + } + }) + .flat_map(futures::stream::iter), + ) +} + +/// The age test of a dead-owner listing or removal, for one bound +/// parameter (`window` in seconds): `column` older than the window on the +/// database's clock, or, for a zero window, nothing (the bound is still +/// consumed). A zero window is `backup purge`'s under the serve lock: one +/// process has the file, so every owner is dead whatever its heartbeat says, +/// a heartbeat stamped ahead of the clock (a backward clock step) included, +/// where `julianday(x) < julianday('now')` would count it live. +fn zero_window_clause(window: Duration, column: &str) -> String { + if window.is_zero() { + "? = 0.0".to_owned() + } else { + format!("julianday({column}) < julianday('now') - ? / 86400.0") + } +} + +/// The columns of one restore job record joined with its table, in the +/// order [`RestoreJobTuple`] reads them; every read of a record selects +/// them. +const RESTORE_ROW_COLUMNS: &str = "r.table_id, t.account_id, t.table_name, r.source_backup_arn, \ + r.kind, r.status, r.owner_generation, r.heartbeat_at, r.store_kind, r.store_root, \ + r.store_id, r.spool_host, r.file_index, r.items_applied, r.failure_reason, r.completed_at"; + +/// One restore job record as the driver reads it. +#[allow(clippy::type_complexity)] +type RestoreJobTuple = ( + String, + String, + String, + String, + String, + String, + i64, + String, + Option, + Option, + Option, + Option, + Option, + Option, + Option, + Option, +); + +fn restore_job_from_tuple(t: RestoreJobTuple) -> Result { + let ( + table_id, + account_id, + table_name, + source_backup_arn, + kind, + status, + owner_generation, + heartbeat_at, + store_kind, + store_root, + store_id, + spool_host, + file_index, + items_applied, + failure_reason, + completed_at, + ) = t; + let store = match (store_kind, store_root, store_id) { + (Some(kind), Some(store_root), Some(store_id)) => Some(StoreIdentity { + store_kind: StoreKind::parse(&kind).ok_or_else(|| { + StorageError::Internal(format!( + "restore of {source_backup_arn} carries an unknown store kind {kind:?}" + )) + })?, + store_root, + store_id, + }), + _ => None, + }; + Ok(RestoreJobRow { + target: RestoreTargetKey { + account_id, + table_name, + table_id, + }, + source_backup_arn, + kind: RestoreKind::parse(&kind)?, + status: JobStatus::parse(&status)?, + owner_generation: u64::try_from(owner_generation).unwrap_or_default(), + heartbeat_at: timestamp_epoch_seconds(&heartbeat_at), + store, + spool_host, + file_index: file_index.and_then(|i| u32::try_from(i).ok()), + items_applied: items_applied.and_then(|i| u64::try_from(i).ok()), + failure_reason, + completed_at: completed_at.as_deref().map(timestamp_epoch_seconds), + }) +} + +impl SqliteEngine { + /// Every RUNNING restore job record at startup was owned by the process + /// that stopped: one process serves a file, so a restart is proof of + /// death. Set each record's heartbeat to the epoch, so the reconciler's + /// first cycle acts on it (a store-backed record is taken over and + /// resumed, a legacy one failed and its target removed) without waiting + /// the stale window. Returns the name and kind of each target whose + /// record was aged (one already at the epoch is not reported again). + /// Runs before the server takes requests, so no restore of this process + /// can be in flight. + pub(crate) async fn expire_restore_job_heartbeats( + &self, + ) -> Result, StorageError> { + let _writer = self.write_lock.lock().await; + let mut tx = self + .pool + .begin_with("BEGIN IMMEDIATE") + .await + .map_err(db_err)?; + let rows: Vec<(String, String)> = sqlx::query_as( + "UPDATE table_restores SET heartbeat_at = '1970-01-01T00:00:00.000Z' \ + WHERE status = 'RUNNING' AND heartbeat_at > '1970-01-01T00:00:00.000Z' \ + RETURNING COALESCE((SELECT table_name FROM tables t \ + WHERE t.table_id = table_restores.table_id), table_id), kind", + ) + .fetch_all(&mut *tx) + .await + .map_err(db_err)?; + tx.commit().await.map_err(db_err)?; + rows.into_iter() + .map(|(name, kind)| Ok((name, RestoreKind::parse(&kind)?))) + .collect() + } + + /// Remove the maintenance claim, whoever holds it: the server's startup + /// step under the serve lock, where one process has the file and so no + /// holder can be alive (a `backup purge` that died, or was killed, left + /// the row with a heartbeat the jobs' check would otherwise honour for + /// the whole stale window). Returns the holder's label when a row was + /// removed. + pub(crate) async fn clear_maintenance_claim(&self) -> Result, StorageError> { + let _writer = self.write_lock.lock().await; + sqlx::query_scalar("DELETE FROM backup_maintenance RETURNING holder") + .fetch_optional(&self.pool) + .await + .map_err(db_err) + } + + /// DeleteBackup's first step, and with `remove` the outright removal of + /// `forget_available_backup_row`, as one write transaction under the + /// write lock: read the status, run the in-use rule, then write DELETING + /// or, with `remove`, delete the row (`Deleting` then means removed). A + /// restore's registration runs its in-use check the same way, so it + /// orders wholly before or after either. One body, so the two methods + /// share the lock and the rule. + async fn start_backup_delete_tx( + &self, + key: &BackupKey, + remove: bool, + ) -> Result { + let _writer = self.write_lock.lock().await; + let mut tx = self + .pool + .begin_with("BEGIN IMMEDIATE") + .await + .map_err(db_err)?; + let status: Option = sqlx::query_scalar( + "SELECT status FROM backup_index WHERE account_id = ? AND backup_id = ?", + ) + .bind(&key.account_id) + .bind(&key.backup_id) + .fetch_optional(&mut *tx) + .await + .map_err(db_err)?; + let status = BackupStatus::parse(&status.ok_or_else(|| not_found(&key.backup_arn))?)?; + if status != BackupStatus::Available { + return Ok(DeleteStart::Status(status)); + } + if let Some(table) = creating_restore_target(&mut *tx, &key.backup_arn).await? { + return Ok(DeleteStart::InUse(table)); + } + let write = if remove { + "DELETE FROM backup_index \ + WHERE account_id = ? AND backup_id = ? AND status = 'AVAILABLE'" + } else { + "UPDATE backup_index SET status = 'DELETING' \ + WHERE account_id = ? AND backup_id = ? AND status = 'AVAILABLE'" + }; + sqlx::query(write) + .bind(&key.account_id) + .bind(&key.backup_id) + .execute(&mut *tx) + .await + .map_err(db_err)?; + tx.commit().await.map_err(db_err)?; + Ok(DeleteStart::Deleting) + } +} + +impl BackupCatalog for SqliteEngine { + fn catalog_id(&self) -> BoxFuture<'_, Result> { + Box::pin(async move { + let id: Option = + sqlx::query_scalar("SELECT value FROM settings WHERE key = 'catalog_id'") + .fetch_optional(&self.pool) + .await + .map_err(db_err)?; + id.ok_or_else(|| { + StorageError::Internal( + "the catalog has no catalog_id; run extenddb migrate --yes".to_owned(), + ) + }) + }) + } + + fn claim_backup<'a>( + &'a self, + row: &'a BackupRow, + ) -> BoxFuture<'a, Result, StorageError>> { + Box::pin(async move { + // The slot first, so no row ever exists unowned. + let Some(slot) = JobSlots::take(&self.job_slots, backup_slot_key(&row.backup_id)) + else { + return Err(StorageError::AlreadyExists(format!( + "a live job owns backup {}", + row.backup_arn + ))); + }; + let generation = new_generation(); + let source_table = serde_json::to_string(&row.source_table_details_json) + .map_err(|e| StorageError::Internal(format!("source table details: {e}")))?; + let _writer = self.write_lock.lock().await; + let mut tx = self + .pool + .begin_with("BEGIN IMMEDIATE") + .await + .map_err(db_err)?; + // OR IGNORE: typed values fail no NOT NULL or CHECK, so only a key conflict is ignored. + let inserted = sqlx::query(&format!( + "INSERT OR IGNORE INTO backup_index (backup_id, account_id, table_name, \ + backup_arn, backup_name, status, store_kind, store_root, store_id, size_bytes, \ + item_count, created_at, source_table_details_json, owner_generation, \ + heartbeat_at) \ + VALUES (?, ?, ?, ?, ?, 'CREATING', ?, ?, ?, ?, ?, ?, ?, ?, {NOW})" + )) + .bind(&row.backup_id) + .bind(&row.account_id) + .bind(&row.table_name) + .bind(&row.backup_arn) + .bind(&row.backup_name) + .bind(row.store.store_kind.as_str()) + .bind(&row.store.store_root) + .bind(&row.store.store_id) + .bind(row.size_bytes) + .bind(row.item_count) + .bind(format_epoch_seconds(row.created_at)) + .bind(&source_table) + .bind(generation_column(generation)) + .execute(&mut *tx) + .await + .map_err(db_err)? + .rows_affected(); + tx.commit().await.map_err(db_err)?; + if inserted == 0 { + drop(slot); + return Err(StorageError::AlreadyExists(format!( + "backup {} already has a row", + row.backup_arn + ))); + } + Ok(Box::new(SqliteJobClaim { + engine: self.clone(), + _slot: Some(slot), + _permit: None, + generation, + record: JobRecord::Backup(row.key()), + }) as Box) + }) + } + + fn insert_backup_row<'a>( + &'a self, + row: &'a BackupRow, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + // No slot: an unowned row has no job. A job that holds the id's + // slot and has not inserted its row yet finds this one and is + // refused, as it is by any row with the key. + let source_table = serde_json::to_string(&row.source_table_details_json) + .map_err(|e| StorageError::Internal(format!("source table details: {e}")))?; + let _writer = self.write_lock.lock().await; + let mut tx = self + .pool + .begin_with("BEGIN IMMEDIATE") + .await + .map_err(db_err)?; + // `ON CONFLICT DO NOTHING` covers the primary key (`account_id`, + // `backup_id`) and the unique `backup_id`, and no other + // constraint: a NOT NULL or CHECK violation is an error. + let inserted = sqlx::query(&format!( + "INSERT INTO backup_index (backup_id, account_id, table_name, \ + backup_arn, backup_name, status, store_kind, store_root, store_id, size_bytes, \ + item_count, created_at, completed_at, failure_reason, \ + source_table_details_json, owner_generation, heartbeat_at) \ + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, {NOW}, ?, ?, ?, {NOW}) \ + ON CONFLICT DO NOTHING" + )) + .bind(&row.backup_id) + .bind(&row.account_id) + .bind(&row.table_name) + .bind(&row.backup_arn) + .bind(&row.backup_name) + .bind(row.status.as_str()) + .bind(row.store.store_kind.as_str()) + .bind(&row.store.store_root) + .bind(&row.store.store_id) + .bind(row.size_bytes) + .bind(row.item_count) + .bind(format_epoch_seconds(row.created_at)) + .bind(&row.failure_reason) + .bind(&source_table) + .bind(generation_column(UNOWNED_GENERATION)) + .execute(&mut *tx) + .await + .map_err(db_err)? + .rows_affected(); + tx.commit().await.map_err(db_err)?; + Ok(inserted == 1) + }) + } + + fn list_backup_rows<'a>( + &'a self, + status: Option, + store: &'a StoreIdentity, + ) -> BoxStream<'a, Result> { + rows_stream(async move { + sqlx::query_as(&format!( + "SELECT {ROW_COLUMNS} FROM backup_index \ + WHERE store_kind = ?1 AND store_root = ?2 AND store_id = ?3 \ + AND (?4 IS NULL OR status = ?4) ORDER BY created_at, backup_arn" + )) + .bind(store.store_kind.as_str()) + .bind(&store.store_root) + .bind(&store.store_id) + .bind(status.map(BackupStatus::as_str)) + .fetch_all(&self.pool) + .await + .map_err(db_err) + }) + } + + fn newest_available_backup_row<'a>( + &'a self, + store: &'a StoreIdentity, + ) -> BoxFuture<'a, Result, StorageError>> { + Box::pin(async move { + let row: Option = sqlx::query_as(&format!( + "SELECT {ROW_COLUMNS} FROM backup_index \ + WHERE store_kind = ? AND store_root = ? AND store_id = ? \ + AND status = 'AVAILABLE' ORDER BY created_at DESC, backup_arn DESC LIMIT 1" + )) + .bind(store.store_kind.as_str()) + .bind(&store.store_root) + .bind(&store.store_id) + .fetch_optional(&self.pool) + .await + .map_err(db_err)?; + row.map(row_from_tuple).transpose() + }) + } + + fn list_account_backup_rows<'a>( + &'a self, + account_id: &'a str, + table_name: Option<&'a str>, + bounds: &'a TimeBounds, + ) -> BoxStream<'a, Result> { + rows_stream(async move { + // `created_at` is written by `format_epoch_seconds`, fixed width, + // so the bounds compare as text in time order. + sqlx::query_as(&format!( + "SELECT {ROW_COLUMNS} FROM backup_index WHERE account_id = ?1 \ + AND (?2 IS NULL OR table_name = ?2) \ + AND (?3 IS NULL OR created_at >= ?3) \ + AND (?4 IS NULL OR created_at < ?4) \ + ORDER BY created_at, backup_arn" + )) + .bind(account_id) + .bind(table_name) + .bind(bounds.lower.map(format_epoch_seconds)) + .bind(bounds.upper.map(format_epoch_seconds)) + .fetch_all(&self.pool) + .await + .map_err(db_err) + }) + } + + fn read_backup_row<'a>( + &'a self, + arn: &'a BackupArn, + ) -> BoxFuture<'a, Result, StorageError>> { + Box::pin(async move { + let row: Option = sqlx::query_as(&format!( + "SELECT {ROW_COLUMNS} FROM backup_index WHERE account_id = ? AND backup_id = ? \ + AND table_name = ? AND backup_arn = ?" + )) + .bind(arn.account_id()) + .bind(arn.backup_id()) + .bind(arn.table_name()) + .bind(arn.to_arn_string()) + .fetch_optional(&self.pool) + .await + .map_err(db_err)?; + row.map(row_from_tuple).transpose() + }) + } + + fn start_backup_delete<'a>( + &'a self, + key: &'a BackupKey, + ) -> BoxFuture<'a, Result> { + Box::pin(self.start_backup_delete_tx(key, false)) + } + + fn forget_available_backup_row<'a>( + &'a self, + key: &'a BackupKey, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + Ok(match self.start_backup_delete_tx(key, true).await? { + DeleteStart::Deleting => ForgetRow::Removed, + DeleteStart::InUse(table) => ForgetRow::InUse(table), + DeleteStart::Status(status) => ForgetRow::Status(status), + }) + }) + } + + fn set_backup_deleted<'a>( + &'a self, + key: &'a BackupKey, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + let _writer = self.write_lock.lock().await; + let affected = sqlx::query(&format!( + "UPDATE backup_index SET status = 'DELETED', completed_at = {NOW} \ + WHERE account_id = ? AND backup_id = ? AND status = 'DELETING'" + )) + .bind(&key.account_id) + .bind(&key.backup_id) + .execute(&self.pool) + .await + .map_err(db_err)? + .rows_affected(); + Ok(affected == 1) + }) + } + + fn remove_backup_row<'a>( + &'a self, + key: &'a BackupKey, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + let _writer = self.write_lock.lock().await; + let affected = sqlx::query( + "DELETE FROM backup_index WHERE account_id = ? AND backup_id = ? \ + AND status IN ('DELETED', 'FAILED')", + ) + .bind(&key.account_id) + .bind(&key.backup_id) + .execute(&self.pool) + .await + .map_err(db_err)? + .rows_affected(); + Ok(affected == 1) + }) + } + + fn remove_dead_creating_backup_row<'a>( + &'a self, + key: &'a BackupKey, + stale_after: Duration, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + let _writer = self.write_lock.lock().await; + // A zero window selects by status alone (`zero_window_clause`): + // the caller holds the serve lock and every owner is dead, + // whatever the heartbeat says, a heartbeat ahead of the clock + // (a backward clock step) included. + let affected = sqlx::query(&format!( + "DELETE FROM backup_index WHERE account_id = ? AND backup_id = ? \ + AND status = 'CREATING' \ + AND {}", + zero_window_clause(stale_after, "heartbeat_at") + )) + .bind(&key.account_id) + .bind(&key.backup_id) + .bind(stale_after.as_secs_f64()) + .execute(&self.pool) + .await + .map_err(db_err)? + .rows_affected(); + Ok(affected == 1) + }) + } + + fn claim_dead_backup<'a>( + &'a self, + key: &'a BackupKey, + _stale_after: Duration, + ) -> BoxFuture<'a, Result>, StorageError>> { + Box::pin(async move { + // The slot is the liveness test (module doc): a job of this + // process that holds it is alive whatever its heartbeat says, and + // a row without a holder is dead at once, so the stale window is + // not read. Taking the slot is the test and the exclusion of a + // second taker in one step. + let Some(slot) = JobSlots::take(&self.job_slots, backup_slot_key(&key.backup_id)) + else { + return Ok(None); + }; + let generation = new_generation(); + let _writer = self.write_lock.lock().await; + let mut tx = self + .pool + .begin_with("BEGIN IMMEDIATE") + .await + .map_err(db_err)?; + // `heartbeat_at` stays as the dead owner wrote it (trait doc). + let taken = sqlx::query( + "UPDATE backup_index SET owner_generation = ? \ + WHERE account_id = ? AND backup_id = ? AND status = 'CREATING'", + ) + .bind(generation_column(generation)) + .bind(&key.account_id) + .bind(&key.backup_id) + .execute(&mut *tx) + .await + .map_err(db_err)? + .rows_affected(); + tx.commit().await.map_err(db_err)?; + if taken != 1 { + drop(slot); + return Ok(None); + } + Ok(Some(Box::new(SqliteJobClaim { + engine: self.clone(), + _slot: Some(slot), + _permit: None, + generation, + record: JobRecord::Backup(key.clone()), + }) as Box)) + }) + } + + fn list_backup_rows_older_than<'a>( + &'a self, + status: BackupStatus, + store: &'a StoreIdentity, + clock: StaleClock, + age: Duration, + ) -> BoxStream<'a, Result> { + let column = match clock { + StaleClock::Heartbeat => "heartbeat_at", + StaleClock::Completed => "completed_at", + }; + rows_stream(async move { + // On the database's clock, as `remove_dead_creating_backup_row` + // measures; `julianday(NULL)` is NULL, so a row with + // `completed_at` unset is never older than any age on that clock. + // A zero window selects by status alone (`zero_window_clause`). + sqlx::query_as(&format!( + "SELECT {ROW_COLUMNS} FROM backup_index \ + WHERE store_kind = ? AND store_root = ? AND store_id = ? AND status = ? \ + AND {} \ + ORDER BY created_at, backup_arn", + zero_window_clause(age, column) + )) + .bind(store.store_kind.as_str()) + .bind(&store.store_root) + .bind(&store.store_id) + .bind(status.as_str()) + .bind(age.as_secs_f64()) + .fetch_all(&self.pool) + .await + .map_err(db_err) + }) + } + + fn backup_in_use<'a>( + &'a self, + key: &'a BackupKey, + ) -> BoxFuture<'a, Result, StorageError>> { + Box::pin(async move { creating_restore_target(&self.pool, &key.backup_arn).await }) + } + + fn remove_restore_target<'a>( + &'a self, + key: &'a RestoreTargetKey, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + // The key must name a CREATING restore target with this table id; + // a table re-created under the name has another id and is not + // touched. `abort_restore` then claims it from CREATING in one + // statement under the write lock, so it and the ACTIVE flip + // exclude each other. + let is_target: bool = sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM tables t JOIN table_restores r \ + ON r.table_id = t.table_id WHERE t.table_id = ? AND t.account_id = ? \ + AND t.table_name = ? AND t.table_status = 'CREATING')", + ) + .bind(&key.table_id) + .bind(&key.account_id) + .bind(&key.table_name) + .fetch_one(&self.pool) + .await + .map_err(db_err)?; + if !is_target { + return Ok(false); + } + // A store-backed restore's progress lives on the `table_restores` + // row (`file_index`, `items_applied`); SQLite has + // no `restore_progress` table (migration 7 is `NotApplicable` + // here), so the record goes with the target below and there are + // no rows to delete. + self.abort_restore(&key.table_id).await + }) + } + + fn list_dead_restore_jobs( + &self, + stale_after: Duration, + ) -> BoxFuture<'_, Result, StorageError>> { + Box::pin(async move { + // Two arms, as on PostgreSQL: a RUNNING record whose heartbeat is + // stale (a dead owner), and a FAILED record whose table is still + // a CREATING restore target once the record is stale (the FAILED + // write landed and the removal did not; `fail_restore_job` and + // the copy's own failure path stamp `completed_at`, and a record + // without one is aged by its heartbeat). The delay is what leaves + // a task still on its own failure path to finish it. Both ages + // are measured on the database's clock (`julianday('now')`); a + // zero window selects by status alone (`zero_window_clause`). + let rows: Vec = sqlx::query_as(&format!( + "SELECT {RESTORE_ROW_COLUMNS} \ + FROM table_restores r JOIN tables t ON t.table_id = r.table_id \ + WHERE (r.status = 'RUNNING' AND {}) \ + OR (r.status = 'FAILED' AND t.table_status = 'CREATING' \ + AND {}) \ + ORDER BY t.account_id, t.table_name", + zero_window_clause(stale_after, "r.heartbeat_at"), + zero_window_clause(stale_after, "COALESCE(r.completed_at, r.heartbeat_at)"), + )) + .bind(stale_after.as_secs_f64()) + .bind(stale_after.as_secs_f64()) + .fetch_all(&self.pool) + .await + .map_err(db_err)?; + // A RUNNING record whose in-process slot a task holds is skipped, + // however old its heartbeat: one process serves the file, so the + // slot is the liveness rule and the heartbeat only what a restart + // ages (design 4(e)); an in-catalog restore holds its table id's + // slot, a store-backed one its target name's. A row this build + // cannot read (a kind or status no release writes) is logged and + // skipped, as ListBackups skips a row it cannot read, so one such + // row does not stop the cycle. + Ok(rows + .into_iter() + .filter_map(|row| { + let (table_id, account_id, table_name, status) = + (row.0.clone(), row.1.clone(), row.2.clone(), row.5.clone()); + if status == JobStatus::Running.as_str() + && (self.job_slots.holds(&legacy_restore_slot_key(&table_id)) + || self + .job_slots + .holds(&restore_slot_key(&account_id, &table_name))) + { + return None; + } + match restore_job_from_tuple(row) { + Ok(job) => Some(job), + Err(e) => { + tracing::warn!( + "reconciler: restore job record of {table_name} (account \ + {account_id}) skipped: {e}" + ); + None + } + } + }) + .collect()) + }) + } + + fn list_running_restore_jobs(&self) -> BoxFuture<'_, Result, StorageError>> { + Box::pin(async move { + // By status alone: a record stamped in this statement's instant, + // or ahead of the database's clock, is RUNNING too. + let rows: Vec = sqlx::query_as(&format!( + "SELECT {RESTORE_ROW_COLUMNS} \ + FROM table_restores r JOIN tables t ON t.table_id = r.table_id \ + WHERE r.status = 'RUNNING' ORDER BY t.account_id, t.table_name" + )) + .fetch_all(&self.pool) + .await + .map_err(db_err)?; + rows.into_iter().map(restore_job_from_tuple).collect() + }) + } + + fn claim_restore<'a>( + &'a self, + account_id: &'a str, + input: CreateTableInput, + registration: &'a RestoreRegistrationRow, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + let table_name = input.table_name.clone(); + // The name's slot first, before the target exists (module doc): + // a slot already held is another store-backed restore into this + // name in flight in this process. It is refused as the name is + // taken: TableBeingRestored naming its source once its CREATING + // row is visible, else as CreateTable would report it. + let Some(slot) = + JobSlots::take(&self.job_slots, restore_slot_key(account_id, &table_name)) + else { + return Err(self.name_taken_error(account_id, &table_name).await); + }; + let generation = new_generation(); + let registration = RestoreRegistrationRow { + owner_generation: generation, + ..registration.clone() + }; + // The registration transaction (`BEGIN IMMEDIATE` under the + // engine write lock): the `backup_index` row's status, the + // in-use check, the CREATING target and its RUNNING record. + let mut desc = match self + .create_table_for_manifest_restore(account_id, input, ®istration) + .await + { + Ok(desc) => desc, + Err(e) => { + drop(slot); + return Err(e); + } + }; + desc.restore_summary = Some(RestoreSummary { + source_backup_arn: Some(registration.backup.backup_arn.clone()), + source_table_arn: Some(registration.source_table_arn.clone()), + restore_date_time: registration.restore_date_time, + restore_in_progress: true, + }); + let claim = + SqliteJobClaim::for_manifest_restore(self, &desc.table_id, generation, slot); + Ok((desc, Box::new(claim) as Box)) + }) + } + + fn claim_restore_dead<'a>( + &'a self, + key: &'a RestoreTargetKey, + _stale_after: Duration, + spool_host: Option<&'a str>, + ) -> BoxFuture<'a, Result>, StorageError>> { + Box::pin(async move { + // The slot is the liveness test, as for `claim_dead_backup`: a + // job of this process that holds the name's slot is alive + // whatever its heartbeat says, and a record without a holder is + // dead at once (a restart frees every slot, and ages every + // RUNNING record so the reconciler lists it). Taking the slot is + // the test and the exclusion of a second taker in one step. + let Some(slot) = JobSlots::take( + &self.job_slots, + restore_slot_key(&key.account_id, &key.table_name), + ) else { + return Ok(None); + }; + let generation = new_generation(); + let _writer = self.write_lock.lock().await; + let mut tx = self + .pool + .begin_with("BEGIN IMMEDIATE") + .await + .map_err(db_err)?; + // The generation, the claimant's spool and a fresh heartbeat in + // one write (the trait doc says why a restore record is refreshed + // where a backup row is not; here the slot already excludes a + // second taker of this process, and the beat keeps the record + // out of the reconciler's dead listing). A legacy record is + // never taken over: the reconciler fails and removes it. + let taken = sqlx::query(&format!( + "UPDATE table_restores SET owner_generation = ?, spool_host = ?, \ + heartbeat_at = {NOW} \ + WHERE table_id = ? AND status = 'RUNNING' AND kind = 'manifest' \ + AND EXISTS(SELECT 1 FROM tables t WHERE t.table_id = table_restores.table_id \ + AND t.account_id = ? AND t.table_name = ?)" + )) + .bind(generation_column(generation)) + .bind(spool_host) + .bind(&key.table_id) + .bind(&key.account_id) + .bind(&key.table_name) + .execute(&mut *tx) + .await + .map_err(db_err)? + .rows_affected(); + tx.commit().await.map_err(db_err)?; + if taken != 1 { + drop(slot); + return Ok(None); + } + Ok(Some(Box::new(SqliteJobClaim::for_manifest_restore( + self, + &key.table_id, + generation, + slot, + )) as Box)) + }) + } + + fn read_restore_job<'a>( + &'a self, + key: &'a RestoreTargetKey, + ) -> BoxFuture<'a, Result, StorageError>> { + Box::pin(async move { + let row: Option = sqlx::query_as(&format!( + "SELECT {RESTORE_ROW_COLUMNS} \ + FROM table_restores r JOIN tables t ON t.table_id = r.table_id \ + WHERE r.table_id = ? AND t.account_id = ? AND t.table_name = ?" + )) + .bind(&key.table_id) + .bind(&key.account_id) + .bind(&key.table_name) + .fetch_optional(&self.pool) + .await + .map_err(db_err)?; + row.map(restore_job_from_tuple).transpose() + }) + } + + fn list_restore_jobs<'a>( + &'a self, + status: Option, + ) -> BoxFuture<'a, Result, StorageError>> { + Box::pin(async move { + let rows: Vec = sqlx::query_as(&format!( + "SELECT {RESTORE_ROW_COLUMNS} \ + FROM table_restores r JOIN tables t ON t.table_id = r.table_id \ + WHERE (?1 IS NULL OR r.status = ?1) \ + ORDER BY t.account_id, t.table_name" + )) + .bind(status.map(JobStatus::as_str)) + .fetch_all(&self.pool) + .await + .map_err(db_err)?; + rows.into_iter().map(restore_job_from_tuple).collect() + }) + } + + /// Conditioned on RUNNING, on the listed `owner_generation` and on the + /// listed `heartbeat_at` (compared as instants through `julianday`, so + /// the text shape of the stored value does not matter), and written with + /// a fresh generation (design 4(e)). Under the engine write lock, as every + /// writer is. + fn fail_restore_job<'a>( + &'a self, + key: &'a RestoreTargetKey, + owner_generation: u64, + heartbeat_at: f64, + reason: &'a str, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + let _writer = self.write_lock.lock().await; + #[allow(clippy::cast_possible_truncation)] // epoch millis fit i64 + let listed_heartbeat = + crate::sqlite_util::format_epoch_millis((heartbeat_at * 1000.0).round() as i64); + let affected = sqlx::query(&format!( + "UPDATE table_restores SET status = 'FAILED', failure_reason = ?, \ + completed_at = {NOW}, owner_generation = ? \ + WHERE table_id = ? AND status = 'RUNNING' AND owner_generation = ? \ + AND julianday(heartbeat_at) = julianday(?) AND EXISTS(\ + SELECT 1 FROM tables t WHERE t.table_id = table_restores.table_id \ + AND t.account_id = ? AND t.table_name = ?)" + )) + .bind(reason) + .bind(generation_column(new_generation())) + .bind(&key.table_id) + .bind(generation_column(owner_generation)) + .bind(listed_heartbeat) + .bind(&key.account_id) + .bind(&key.table_name) + .execute(&self.pool) + .await + .map_err(db_err)? + .rows_affected(); + Ok(affected == 1) + }) + } + + fn read_store_ids<'a>( + &'a self, + store_kind: StoreKind, + store_root: &'a str, + ) -> BoxFuture<'a, Result, StorageError>> { + Box::pin(async move { + sqlx::query_scalar( + "SELECT DISTINCT store_id FROM backup_index \ + WHERE store_kind = ? AND store_root = ? ORDER BY store_id", + ) + .bind(store_kind.as_str()) + .bind(store_root) + .fetch_all(&self.pool) + .await + .map_err(db_err) + }) + } + + fn maintenance_claim<'a>( + &'a self, + holder: &'a str, + stale_after: Duration, + ) -> BoxFuture<'a, Result, StorageError>> { + Box::pin(async move { + // One process serves a file (the serve lock), so the row is the + // claim: taken under the engine write lock, refused while a row + // with a live heartbeat exists, taken over from a dead one. An + // engine that holds the serve lock (a command that opened the + // file as `serve` does) is the only process on the file, so the + // holder of any row is dead whatever its heartbeat says, as the + // server's startup decides (`clear_maintenance_claim`): the row + // is taken over. + let _writer = self.write_lock.lock().await; + let live: Option = if self.serve_lock.is_some() { + None + } else { + sqlx::query_scalar( + "SELECT holder FROM backup_maintenance \ + WHERE julianday(heartbeat_at) >= julianday('now') - ? / 86400.0", + ) + .bind(stale_after.as_secs_f64()) + .fetch_optional(&self.pool) + .await + .map_err(db_err)? + }; + if let Some(current) = live { + return Err(StorageError::AlreadyExists(format!( + "the backup maintenance claim is held by {current}" + ))); + } + let generation = new_generation(); + sqlx::query(&format!( + "INSERT OR REPLACE INTO backup_maintenance \ + (singleton, holder, owner_generation, claimed_at, heartbeat_at) \ + VALUES (1, ?, ?, {NOW}, {NOW})" + )) + .bind(holder) + .bind(generation_column(generation)) + .execute(&self.pool) + .await + .map_err(db_err)?; + Ok(Box::new(SqliteMaintenanceClaim { + engine: self.clone(), + generation, + }) as Box) + }) + } + + fn check_maintenance_claim( + &self, + stale_after: Duration, + ) -> BoxFuture<'_, Result> { + Box::pin(async move { + sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM backup_maintenance \ + WHERE julianday(heartbeat_at) >= julianday('now') - ? / 86400.0)", + ) + .bind(stale_after.as_secs_f64()) + .fetch_one(&self.pool) + .await + .map_err(db_err) + }) + } +} + +/// The holder's handle on the `backup_maintenance` row. +struct SqliteMaintenanceClaim { + engine: SqliteEngine, + generation: u64, +} + +impl MaintenanceClaim for SqliteMaintenanceClaim { + fn generation(&self) -> u64 { + self.generation + } + + fn heartbeat(&self) -> BoxFuture<'_, Result> { + Box::pin(async move { + let _writer = self.engine.write_lock.lock().await; + let affected = sqlx::query(&format!( + "UPDATE backup_maintenance SET heartbeat_at = {NOW} WHERE owner_generation = ?" + )) + .bind(generation_column(self.generation)) + .execute(&self.engine.pool) + .await + .map_err(db_err)? + .rows_affected(); + Ok(if affected == 1 { + Heartbeat::Confirmed + } else { + Heartbeat::Lost + }) + }) + } + + fn release(&self) -> BoxFuture<'_, Result<(), StorageError>> { + Box::pin(async move { + let _writer = self.engine.write_lock.lock().await; + sqlx::query("DELETE FROM backup_maintenance WHERE owner_generation = ?") + .bind(generation_column(self.generation)) + .execute(&self.engine.pool) + .await + .map_err(db_err)?; + Ok(()) + }) + } +} + +/// The `backup_catalog` module's unit tests that need no database. +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_row_tuple_with_an_unknown_store_kind_status_or_source_table_is_internal() { + let good = || RowTuple { + backup_id: "01757500000123-0badcafe".to_owned(), + account_id: "123456789012".to_owned(), + table_name: "Music".to_owned(), + backup_arn: "arn".to_owned(), + backup_name: "b".to_owned(), + status: "AVAILABLE".to_owned(), + store_kind: "s3".to_owned(), + store_root: "s3://bucket/prefix".to_owned(), + store_id: "store-1".to_owned(), + size_bytes: 1, + item_count: 2, + created_at: "2025-09-10T10:26:40.123000Z".to_owned(), + completed_at: None, + failure_reason: None, + source_table_details_json: "{}".to_owned(), + owner_generation: 7, + heartbeat_at: "2025-09-10T10:26:41.000Z".to_owned(), + }; + let row = row_from_tuple(good()).unwrap(); + assert_eq!(row.store.store_kind, StoreKind::S3); + assert_eq!(row.owner_generation, 7); + assert!((row.created_at - 1_757_500_000.123).abs() < 1e-6); + assert_eq!(row.heartbeat_at, 1_757_500_001.0); + let mut bad_kind = good(); + bad_kind.store_kind = "gcs".to_owned(); + assert!(matches!( + row_from_tuple(bad_kind), + Err(StorageError::Internal(_)) + )); + let mut bad_status = good(); + bad_status.status = "ACTIVE".to_owned(); + assert!(matches!( + row_from_tuple(bad_status), + Err(StorageError::Internal(_)) + )); + let mut bad_source = good(); + bad_source.source_table_details_json = "not json".to_owned(); + assert!(matches!( + row_from_tuple(bad_source), + Err(StorageError::Internal(_)) + )); + } + + #[test] + fn a_slot_is_held_once_and_freed_by_its_guard() { + let slots = Arc::new(JobSlots::default()); + let held = JobSlots::take(&slots, "backup:1".to_owned()).expect("free"); + assert!(slots.holds("backup:1")); + assert!(JobSlots::take(&slots, "backup:1".to_owned()).is_none()); + assert!(JobSlots::take(&slots, "backup:2".to_owned()).is_some()); + drop(held); + assert!(!slots.holds("backup:1")); + assert!(JobSlots::take(&slots, "backup:1".to_owned()).is_some()); + } + + #[test] + fn a_generation_fits_the_integer_column() { + assert_eq!(generation_column(7), 7); + assert_eq!(generation_column(u64::MAX), i64::MAX); + assert_ne!( + generation_column(new_generation()), + generation_column(extenddb_storage::backup_catalog::UNOWNED_GENERATION) + ); + } +} + +/// The job under the engine write lock (the maintainer's blocking item on +/// PR 8): a writer that holds the lock past two heartbeat periods delays the +/// beat and the claim re-check and ends no job, and a put that comes while +/// the beat is parked on the lock is not refused. +#[cfg(test)] +mod write_lock_tests { + use std::os::unix::fs::DirBuilderExt; + use std::sync::Arc; + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::time::Duration; + + use extenddb_backup_store::{BackupStore, ByteStream, FilesystemStore, ObjectMeta, StoreError}; + use extenddb_core::backup_format::BackupArn; + use extenddb_core::limits::LimitsConfig; + use extenddb_engine::{ + BackupRuntime, ConfiguredStore, JobSettings, OperationContext, check_configured_store, + dispatch, + }; + use extenddb_storage::backup_catalog::{BackupCatalog, BackupStatus, StoreKind}; + use futures::future::BoxFuture; + use futures::stream::BoxStream; + use serde_json::json; + + use crate::store::SqliteEngine; + + const ACCOUNT: &str = "123456789012"; + + /// A scratch directory under the cargo target directory, removed on + /// drop with the database files beside it. + struct Scratch(std::path::PathBuf); + + impl Drop for Scratch { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } + } + + async fn call( + ctx: &OperationContext, + operation: &str, + body: serde_json::Value, + ) -> serde_json::Value { + dispatch(operation, body, ctx, "127.0.0.1:0") + .await + .unwrap_or_else(|e| panic!("{operation}: {e:?}")) + .body + } + + /// A store whose first `put` waits `delay` before it starts, counting + /// the puts started: the job's second data file then comes after the + /// delay, while the test still holds the write lock. + struct SlowFirstPut { + inner: Arc, + delay: Duration, + puts_started: AtomicUsize, + } + + impl BackupStore for SlowFirstPut { + fn put<'a>( + &'a self, + key: &'a str, + body: ByteStream, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + if self.puts_started.fetch_add(1, Ordering::AcqRel) == 0 { + tokio::time::sleep(self.delay).await; + } + self.inner.put(key, body).await + }) + } + + fn put_if_absent<'a>( + &'a self, + key: &'a str, + body: ByteStream, + ) -> BoxFuture<'a, Result, StoreError>> { + self.inner.put_if_absent(key, body) + } + + fn get<'a>(&'a self, key: &'a str) -> BoxFuture<'a, Result> { + self.inner.get(key) + } + + fn head<'a>( + &'a self, + key: &'a str, + ) -> BoxFuture<'a, Result, StoreError>> { + self.inner.head(key) + } + + fn list<'a>(&'a self, prefix: &'a str) -> BoxStream<'a, Result> { + self.inner.list(prefix) + } + + fn delete_prefix<'a>(&'a self, prefix: &'a str) -> BoxFuture<'a, Result> { + self.inner.delete_prefix(prefix) + } + + fn validate(&self) -> BoxFuture<'_, Result<(), StoreError>> { + self.inner.validate() + } + } + + /// A file-backed engine with a healthy filesystem store under a scratch + /// root, a runtime with `settings`, and a `Music` table of three items. + struct Fixture { + _scratch: Scratch, + engine: SqliteEngine, + root: std::path::PathBuf, + runtime: Arc, + ctx: OperationContext, + } + + async fn fixture( + settings: JobSettings, + wrap: impl FnOnce(Arc) -> Arc, + ) -> Fixture { + let target = std::env::var_os("CARGO_TARGET_DIR") + .map(std::path::PathBuf::from) + .unwrap_or_else(|| { + std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../target") + }); + let scratch = Scratch(target.join(format!("backup-write-lock-{}", uuid::Uuid::new_v4()))); + std::fs::create_dir_all(&scratch.0).expect("scratch directory"); + let db_path = scratch.0.join("catalog.sqlite"); + let root = scratch.0.join("backups"); + std::fs::DirBuilder::new() + .mode(0o700) + .create(&root) + .expect("root"); + let engine = SqliteEngine::new( + db_path.to_str().expect("UTF-8 path"), + 4, + "us-east-1", + 409_600, + ) + .await + .expect("file engine"); + crate::migrations::apply_for_tests(&engine.pool).await; + for (key, value) in [ + ("control_plane_delay_seconds", "0"), + ("index_propagation_delay_ms", "0"), + ] { + sqlx::query( + "INSERT INTO settings (key, value) VALUES (?, ?) \ + ON CONFLICT(key) DO UPDATE SET value = excluded.value", + ) + .bind(key) + .bind(value) + .execute(&engine.pool) + .await + .expect("setting"); + } + sqlx::query("INSERT INTO accounts (account_id, account_name) VALUES (?, 'default')") + .bind(ACCOUNT) + .execute(&engine.pool) + .await + .expect("account"); + let storage: Arc = Arc::new(engine.clone()); + + let store: Arc = Arc::new( + FilesystemStore::open(&root) + .await + .expect("filesystem store"), + ); + let configured = ConfiguredStore::checking( + StoreKind::Filesystem, + root.to_str().expect("UTF-8 path").to_owned(), + wrap(store), + None, + ); + let outcome = check_configured_store(&configured, &*storage, "sqlite").await; + assert!(outcome.is_ok(), "the store's first check: {outcome:?}"); + configured.record_check(outcome); + let runtime = Arc::new( + BackupRuntime::new(4) + .with_job_settings(settings) + .with_store(configured) + .with_source("sqlite", "test"), + ); + let ctx = OperationContext { + storage: storage.clone(), + backup: runtime.clone(), + limits: Arc::new(LimitsConfig::default()), + region: Arc::from("us-east-1"), + account_id: Arc::from(ACCOUNT), + import_paths: Arc::from(Vec::new()), + export_paths: Arc::from(Vec::new()), + authorized_key_info: Default::default(), + auth_cache: extenddb_auth::AuthCacheRegistry::empty(), + table_key_info_lookup: None, + }; + call( + &ctx, + "CreateTable", + json!({ + "TableName": "Music", + "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}], + "AttributeDefinitions": [{"AttributeName": "pk", "AttributeType": "S"}], + "BillingMode": "PAY_PER_REQUEST" + }), + ) + .await; + for i in 0..3 { + call( + &ctx, + "PutItem", + json!({"TableName": "Music", "Item": {"pk": {"S": format!("k{i}")}}}), + ) + .await; + } + Fixture { + _scratch: scratch, + engine, + root, + runtime, + ctx, + } + } + + impl Fixture { + /// CreateBackup of `Music`, answered CREATING; the backup's ARN. + async fn create_backup(&self) -> BackupArn { + let details = call( + &self.ctx, + "CreateBackup", + json!({"TableName": "Music", "BackupName": "nightly"}), + ) + .await; + assert_eq!( + details["BackupDetails"]["BackupStatus"], "CREATING", + "{details}" + ); + BackupArn::parse( + details["BackupDetails"]["BackupArn"] + .as_str() + .expect("an ARN"), + ) + .expect("a backup ARN") + } + + /// Wait for the job to end, then read its row. + async fn ended_row(&self, arn: &BackupArn) -> extenddb_storage::backup_catalog::BackupRow { + let deadline = std::time::Instant::now() + Duration::from_secs(10); + while self.runtime.running_jobs() > 0 { + assert!( + std::time::Instant::now() < deadline, + "the job did not end after the lock was released" + ); + tokio::time::sleep(Duration::from_millis(20)).await; + } + self.engine + .read_backup_row(arn) + .await + .expect("read the row") + .expect("the row exists") + } + } + + /// With a heartbeat period of 300 ms, the test takes the engine write + /// lock the moment CreateBackup answers CREATING and holds it for three + /// periods. The timer's first beat and the job's re-check before the + /// manifest queue behind it; their bound starts once they hold the lock + /// (`heartbeat_within`), so both confirm when the lock is released and + /// the backup ends AVAILABLE with its manifest in the store. With the + /// bound over the whole call the first beat ends the job as a lost claim + /// ("the heartbeat did not return within 0 s") and the row stays + /// CREATING. + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn a_writer_holding_the_lock_past_two_periods_delays_the_beat_and_ends_no_job() { + let period = Duration::from_millis(300); + let f = fixture( + JobSettings { + heartbeat_interval: period, + ..JobSettings::default() + }, + |store| store, + ) + .await; + let arn = f.create_backup().await; + // The writer: the lock taken as the row exists, held for three + // periods, past the first beat's tick and its whole-call bound. + let held = f.engine.write_lock.lock().await; + tokio::time::sleep(3 * period).await; + assert_eq!(f.runtime.running_jobs(), 1, "the job waits on the lock"); + drop(held); + + let row = f.ended_row(&arn).await; + assert_eq!( + (row.status, row.failure_reason.clone()), + (BackupStatus::Available, None), + "{row:?}" + ); + assert_eq!(row.item_count, 3); + assert!( + f.root + .join(arn.store_prefix()) + .join("extenddb-manifest.json") + .is_file(), + "the manifest is in the store" + ); + assert!( + row.heartbeat_at > row.created_at, + "a beat landed after the insert ({} > {})", + row.heartbeat_at, + row.created_at + ); + f.engine.pool.close().await; + } + + /// The maintainer's second-round case: a put that comes while the beat + /// is parked on the lock. A data file per item (`data_file_target_bytes` + /// of one byte) and a store whose first put takes four periods: the + /// first data file's check passes on the claim's insert, the timer's + /// first beat parks on the lock the test took as the row appeared, and + /// the second data file's check comes four periods after the insert + /// with no beat confirmed since. The check passes on the beat in flight + /// (`confirms_the_claim`'s third clause), the second put starts under + /// the hold, and once the lock is released the beat lands and the backup + /// ends AVAILABLE with its three data files and its manifest. With the + /// in-flight clause removed the second data file's check refuses the + /// put ("no heartbeat confirmed the claim recently enough for data file + /// 2") and the job stops: this cell then sees one put started and the + /// row left CREATING, since no reconciler runs here (on a server the + /// reconciler fails the row and deletes its prefix, which is what the + /// maintainer measured at the third or fourth data file beside a GSI + /// build). + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn a_put_during_a_writers_hold_passes_on_the_beat_parked_behind_it() { + let period = Duration::from_millis(300); + let slow = Arc::new(std::sync::Mutex::new(None::>)); + let f = fixture( + JobSettings { + heartbeat_interval: period, + data_file_target_bytes: 1, + ..JobSettings::default() + }, + |store| { + let wrapped = Arc::new(SlowFirstPut { + inner: store, + delay: 4 * period, + puts_started: AtomicUsize::new(0), + }); + *slow.lock().unwrap() = Some(Arc::clone(&wrapped)); + wrapped + }, + ) + .await; + let slow = slow.lock().unwrap().take().expect("the wrapped store"); + let arn = f.create_backup().await; + // The writer: the lock taken as the row exists, held until the + // second data file's put has started, which needs the pre-put check + // to pass with the first beat parked behind this hold. + let held = f.engine.write_lock.lock().await; + let at_the_hold = f + .engine + .read_backup_row(&arn) + .await + .expect("read the row") + .expect("the row exists"); + let deadline = std::time::Instant::now() + Duration::from_secs(10); + while slow.puts_started.load(Ordering::Acquire) < 2 { + if std::time::Instant::now() >= deadline || f.runtime.running_jobs() == 0 { + drop(held); + let row = f.ended_row(&arn).await; + panic!( + "the second data file was not put under the hold: {} put(s) started, row {row:?}", + slow.puts_started.load(Ordering::Acquire) + ); + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + let parked = f + .engine + .read_backup_row(&arn) + .await + .expect("read the row") + .expect("the row exists"); + assert_eq!( + parked.heartbeat_at, at_the_hold.heartbeat_at, + "no beat landed under the hold: the first beat is parked on the lock" + ); + assert_eq!(parked.status, BackupStatus::Creating); + drop(held); + + let row = f.ended_row(&arn).await; + assert_eq!( + (row.status, row.failure_reason.clone()), + (BackupStatus::Available, None), + "{row:?}" + ); + assert_eq!(row.item_count, 3); + let prefix = f.root.join(arn.store_prefix()); + for name in [ + "data/000001.json.gz", + "data/000002.json.gz", + "data/000003.json.gz", + "extenddb-manifest.json", + ] { + assert!(prefix.join(name).is_file(), "{name} is in the store"); + } + assert!( + row.heartbeat_at > at_the_hold.heartbeat_at, + "the parked beat landed once the lock was released ({} > {})", + row.heartbeat_at, + at_the_hold.heartbeat_at + ); + f.engine.pool.close().await; + } +} diff --git a/crates/storage-sqlite/src/config.rs b/crates/storage-sqlite/src/config.rs index 01f69c0e2..f10c05cf2 100644 --- a/crates/storage-sqlite/src/config.rs +++ b/crates/storage-sqlite/src/config.rs @@ -21,14 +21,20 @@ use serde::Deserialize; /// of 10, and 1 for every `pool_size` of 5 or less; this is the one statement /// of the formula, `backup::backup_server_slot_count` computes it), shared by /// backups and table snapshots: each running backup and each open snapshot -/// stream (file-backed or in-memory) holds one. A snapshot that finds no slot -/// free is refused at once; a `CreateBackup` waits 30 s for one and then -/// answers `LimitExceededException`, with no backup running if snapshots hold -/// every slot. With one slot, one held snapshot blocks every `CreateBackup` -/// for as long as it is open. Running backups hold one connection each for -/// their read snapshot plus one shared writer (`n` backups hold `n + 1`), so -/// from `pool_size` 4 up a flood of backups leaves at least half the pool to -/// other requests; with 2 or 3 the one admitted backup holds two of them. +/// stream (file-backed or in-memory) holds one. A snapshot or a `CreateBackup` +/// that finds no slot free is refused at once with `LimitExceededException`, +/// so no backup runs while snapshots hold every slot. With one slot, one held +/// snapshot refuses every `CreateBackup` for as long as it is open. A server +/// with `backup.max_concurrent_jobs` over the slot count raises a file-backed +/// `pool_size` to `2 * (max_concurrent_jobs + 1)` (10 at the default of 4, +/// which is the default pool; 34 at most, where the 16-slot cap is reached), +/// logging a warning that names both numbers, so the slot never refuses a job +/// the engine's permit admitted (`backup::pool_size_for_jobs` computes it); +/// an in-memory database has one connection and one slot whatever the jobs. +/// Running backups hold one connection each for their read snapshot plus one +/// shared writer (`n` backups hold `n + 1`), so from `pool_size` 4 up a flood +/// of backups leaves at least half the pool to other requests; with 2 or 3 +/// the one admitted backup holds two of them. #[derive(Debug, Clone, Deserialize)] #[serde(deny_unknown_fields)] pub struct SqliteConfig { @@ -69,6 +75,14 @@ impl StorageConfig for SqliteConfig { &self.path } + fn database_file(&self) -> Option { + crate::serve_lock::database_file(&self.path).ok().flatten() + } + + fn database_location(&self) -> Result, String> { + crate::serve_lock::database_file(&self.path) + } + fn max_connections(&self) -> u32 { self.pool_size } diff --git a/crates/storage-sqlite/src/create_table.rs b/crates/storage-sqlite/src/create_table.rs index ce068463a..1b183037a 100644 --- a/crates/storage-sqlite/src/create_table.rs +++ b/crates/storage-sqlite/src/create_table.rs @@ -16,21 +16,77 @@ use extenddb_core::types::{ BillingMode, BillingModeSummary, CreateTableInput, GsiDescription, LsiDescription, ProvisionedThroughputDescription, SseDescription, SseType, TableDescription, TableStatus, }; +use extenddb_storage::backup_catalog::RestoreRegistrationRow; use extenddb_storage::error::StorageError; use extenddb_storage::util::{index_arn, stream_arn, table_arn}; use crate::sqlite_util::{format_epoch_seconds, format_timestamp, is_unique_violation}; use crate::store::SqliteEngine; -/// What a restore records about itself when its target is created. +/// What a restore records about itself when its target is created: the +/// registration and the restore job record it is born with (`RUNNING`, +/// `kind = legacy`, the claim's generation, a fresh heartbeat). pub(crate) struct RestoreRegistration<'a> { pub backup_arn: &'a str, pub source_table_arn: &'a str, /// Epoch seconds: the source backup's creation time. pub restore_date_time: f64, + /// The fencing token of the restore's claim; every later write by the + /// restore to its record carries it. + pub owner_generation: u64, +} + +/// Which backup a restore target is created for, and so which row the +/// registration re-checks in its write transaction: an in-catalog `backups` +/// row (the legacy reader's restore, `kind = legacy`) or a `backup_index` row +/// of a backup in the store (`kind = manifest`, with the store, the spool +/// host and the claim's generation on the record). +pub(crate) enum RestoreSource<'a> { + Legacy(&'a RestoreRegistration<'a>), + Manifest(&'a RestoreRegistrationRow), +} + +impl RestoreSource<'_> { + fn backup_arn(&self) -> &str { + match self { + Self::Legacy(r) => r.backup_arn, + Self::Manifest(r) => &r.backup.backup_arn, + } + } } impl SqliteEngine { + /// The error for a restore into a name another restore holds the slot + /// of: `TableBeingRestored` naming that restore's backup once its + /// CREATING row is committed, else `TableAlreadyExists`, as CreateTable + /// reports a taken name. Reads committed state through the pool; a read + /// failure is "not a restore target": an error is being produced + /// already. + pub(crate) async fn name_taken_error( + &self, + account_id: &str, + table_name: &str, + ) -> StorageError { + let restoring: Option = sqlx::query_scalar( + "SELECT r.source_backup_arn FROM tables t \ + JOIN table_restores r ON r.table_id = t.table_id \ + WHERE t.account_id = ? AND t.table_name = ? AND t.table_status = 'CREATING'", + ) + .bind(account_id) + .bind(table_name) + .fetch_optional(&self.pool) + .await + .ok() + .flatten(); + match restoring { + Some(backup_arn) => StorageError::TableBeingRestored { + table_name: table_name.to_owned(), + backup_arn, + }, + None => StorageError::TableAlreadyExists(table_name.to_owned()), + } + } + /// Create a table. When `defer_active` is set (the restore path), the row /// is written `CREATING` with **no** scheduled transition, so the /// background control-plane worker cannot flip it to `ACTIVE` while the @@ -56,8 +112,33 @@ impl SqliteEngine { input: CreateTableInput, registration: &RestoreRegistration<'_>, ) -> Result { - self.create_table_impl_inner(account_id, input, true, Some(registration)) - .await + self.create_table_impl_inner( + account_id, + input, + true, + Some(RestoreSource::Legacy(registration)), + ) + .await + } + + /// Create a store-backed restore's target and register the restore in + /// the write transaction that also re-checks, on the backup's + /// `backup_index` row, that it is AVAILABLE and has no CREATING target. + /// The job record is born RUNNING, `kind = manifest`, with the + /// registration's store, spool host and generation. + pub(crate) async fn create_table_for_manifest_restore( + &self, + account_id: &str, + input: CreateTableInput, + registration: &RestoreRegistrationRow, + ) -> Result { + self.create_table_impl_inner( + account_id, + input, + true, + Some(RestoreSource::Manifest(registration)), + ) + .await } async fn create_table_impl_inner( @@ -65,7 +146,7 @@ impl SqliteEngine { account_id: &str, input: CreateTableInput, defer_active: bool, - restore: Option<&RestoreRegistration<'_>>, + restore: Option>, ) -> Result { Self::validate_account_id(account_id)?; // D1: every writer holds the engine write lock. This method runs two @@ -148,7 +229,8 @@ impl SqliteEngine { .await .map_err(|e| StorageError::Internal(e.to_string()))?; - if let Some(RestoreRegistration { backup_arn, .. }) = restore { + if let Some(source) = &restore { + let backup_arn = source.backup_arn(); // A restore into a name that is itself a CREATING restore target // is refused as such before the backup is checked, so a second // restore of the same backup into the same name is @@ -170,37 +252,55 @@ impl SqliteEngine { backup_arn: source_backup_arn, }); } - let available: bool = sqlx::query_scalar( - "SELECT EXISTS(SELECT 1 FROM backups WHERE backup_arn = ? \ - AND account_id = ? AND backup_status = 'AVAILABLE')", - ) - .bind(backup_arn) - .bind(account_id) - .fetch_one(&mut *tx) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - if !available { - return Err(StorageError::Validation(format!( - "Backup not found: {backup_arn}" - ))); + match source { + RestoreSource::Legacy(_) => { + let available: bool = sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM backups WHERE backup_arn = ? \ + AND account_id = ? AND backup_status = 'AVAILABLE')", + ) + .bind(backup_arn) + .bind(account_id) + .fetch_one(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + if !available { + return Err(StorageError::Validation(format!( + "Backup not found: {backup_arn}" + ))); + } + } + RestoreSource::Manifest(registration) => { + // The `backup_index` row's status, in the write + // transaction `start_backup_delete` reads it in: the + // engine write lock and `BEGIN IMMEDIATE` are the lock + // on the row, so a DeleteBackup and this registration + // order wholly one before the other, and two + // registrations of one backup serialize before the + // CREATING-target check below. + let status: Option = sqlx::query_scalar( + "SELECT status FROM backup_index WHERE account_id = ? AND backup_id = ?", + ) + .bind(account_id) + .bind(®istration.backup.backup_id) + .fetch_optional(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + if status.as_deref() != Some("AVAILABLE") { + return Err(extenddb_storage::backup_catalog::backup_not_found( + backup_arn, + )); + } + } } // One restore of a backup at a time, as the service: while a // target restored from it is CREATING the backup is in use. Same // predicate as DeleteBackup's, in the same kind of write // transaction under the write lock, so two simultaneous restores // cannot both pass. - let restoring: Option = sqlx::query_scalar( - "SELECT t.table_name FROM table_restores r JOIN tables t ON t.table_id = r.table_id \ - WHERE r.source_backup_arn = ? AND t.table_status = 'CREATING' LIMIT 1", - ) - .bind(backup_arn) - .fetch_optional(&mut *tx) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - if let Some(table) = restoring { - return Err(StorageError::BackupInUse(format!( - "Backup is being used to restore table {table}: {backup_arn}" - ))); + if let Some(table) = + crate::backup_catalog::creating_restore_target(&mut *tx, backup_arn).await? + { + return Err(crate::backup_catalog::backup_in_use(&table, backup_arn)); } } @@ -261,19 +361,51 @@ impl SqliteEngine { return Err(StorageError::TableAlreadyExists(input.table_name.clone())); } - if let Some(registration) = restore { - sqlx::query( - "INSERT INTO table_restores \ - (table_id, source_backup_arn, source_table_arn, restore_date_time) \ - VALUES (?, ?, ?, ?)", - ) - .bind(&table_id) - .bind(registration.backup_arn) - .bind(registration.source_table_arn) - .bind(format_epoch_seconds(registration.restore_date_time)) - .execute(&mut *tx) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; + match &restore { + Some(RestoreSource::Legacy(registration)) => { + sqlx::query(&format!( + "INSERT INTO table_restores \ + (table_id, source_backup_arn, source_table_arn, restore_date_time, \ + status, kind, owner_generation, heartbeat_at) \ + VALUES (?, ?, ?, ?, 'RUNNING', 'legacy', ?, {})", + crate::backup_catalog::NOW + )) + .bind(&table_id) + .bind(registration.backup_arn) + .bind(registration.source_table_arn) + .bind(format_epoch_seconds(registration.restore_date_time)) + .bind(crate::backup_catalog::generation_column( + registration.owner_generation, + )) + .execute(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + } + Some(RestoreSource::Manifest(registration)) => { + sqlx::query(&format!( + "INSERT INTO table_restores \ + (table_id, source_backup_arn, source_table_arn, restore_date_time, \ + status, kind, store_kind, store_root, store_id, spool_host, \ + owner_generation, heartbeat_at) \ + VALUES (?, ?, ?, ?, 'RUNNING', 'manifest', ?, ?, ?, ?, ?, {})", + crate::backup_catalog::NOW + )) + .bind(&table_id) + .bind(®istration.backup.backup_arn) + .bind(®istration.source_table_arn) + .bind(format_epoch_seconds(registration.restore_date_time)) + .bind(registration.store.store_kind.as_str()) + .bind(®istration.store.store_root) + .bind(®istration.store.store_id) + .bind(registration.spool_host.as_deref()) + .bind(crate::backup_catalog::generation_column( + registration.owner_generation, + )) + .execute(&mut *tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + } + None => {} } // GSI / LSI metadata. diff --git a/crates/storage-sqlite/src/data/vector_index.rs b/crates/storage-sqlite/src/data/vector_index.rs index 02bc09836..80acca8cd 100644 --- a/crates/storage-sqlite/src/data/vector_index.rs +++ b/crates/storage-sqlite/src/data/vector_index.rs @@ -126,7 +126,7 @@ fn base_key_binds( } /// Column names for the base key, in key-schema order. -fn base_key_columns( +pub(crate) fn base_key_columns( base_key_schema: &[KeySchemaElement], attr_defs: &[AttributeDefinition], ) -> Vec { diff --git a/crates/storage-sqlite/src/lib.rs b/crates/storage-sqlite/src/lib.rs index 8730e8b21..ae0af24ea 100644 --- a/crates/storage-sqlite/src/lib.rs +++ b/crates/storage-sqlite/src/lib.rs @@ -23,6 +23,7 @@ mod admin_store; mod authorization_store; mod backup; +mod backup_catalog; mod bootstrapper; mod catalog_store; pub mod config; @@ -36,6 +37,9 @@ mod metadata; pub mod migrations; mod number_key; mod operations; +mod restore_engine; +#[cfg(test)] +mod restore_job_tests; mod schema; mod serve_lock; mod snapshot_engine; @@ -210,18 +214,39 @@ fn sqlite_server_components_factory( // One server per database file, taken before the database is opened: // startup recovery below assumes no other server is using it, and a // migration holds the same lock. A file database only; an in-memory - // one belongs to this process alone. - let serve_lock = serve_lock::ServeLock::try_acquire_for_location(&db_path) - .map_err(|failure| { - BackendError::Catalog(match failure { - serve_lock::LockFailure::Held(m) => MigrationError::Locked(m), - serve_lock::LockFailure::Failed(m) => MigrationError::Store(m), + // one belongs to this process alone. A command reading beside a + // server (`options.beside_server`) takes no lock and runs none of + // the recovery: the file is a served database and the server owns + // both. + let serve_lock = if options.beside_server { + // The catalog's status is still read, on a read-only + // connection: a catalog this binary cannot serve (behind, or + // ahead with a migration that is not additive) is reported as + // the server would report it, not as a missing table. + match migrations::catalog_status(&db_path) + .await + .map_err(BackendError::Catalog)? + { + extenddb_storage::migrations::CatalogStatus::Current { .. } + | extenddb_storage::migrations::CatalogStatus::AheadCompatible { .. } => {} + status => { + return Err(BackendError::Catalog(MigrationError::Status(status))); + } + } + None + } else { + serve_lock::ServeLock::try_acquire_for_location(&db_path) + .map_err(|failure| { + BackendError::Catalog(match failure { + serve_lock::LockFailure::Held(m) => MigrationError::Locked(m), + serve_lock::LockFailure::Failed(m) => MigrationError::Store(m), + }) + })? + .map(|lock| { + tracing::debug!("holding serve lock {}", lock.path().display()); + std::sync::Arc::new(lock) }) - })? - .map(|lock| { - tracing::debug!("holding serve lock {}", lock.path().display()); - std::sync::Arc::new(lock) - }); + }; // The catalog's status is read before the engine opens the file, so // a server that refuses has written nothing. A dev build migrates // instead, as `migrate --yes` would, copy included. @@ -231,13 +256,21 @@ fn sqlite_server_components_factory( .await .map_err(BackendError::Catalog)?; } - let mut engine = - SqliteEngine::new(&db_path, pool_size, ®ion, options.max_item_size_bytes) - .await - .map_err(|e| BackendError::ConnectionFailed { - backend: "sqlite".to_owned(), - details: e.to_string(), - })?; + let mut engine = SqliteEngine::new_for_server( + &db_path, + pool_size, + options.max_concurrent_jobs, + ®ion, + options.max_item_size_bytes, + ) + .await + .map_err(|e| BackendError::ConnectionFailed { + backend: "sqlite".to_owned(), + details: e.to_string(), + })?; + if options.beside_server { + return components_of(engine, &db_path).await; + } // An in-memory database exists only on the engine's single // connection, so its catalog is created there, by the runner, every @@ -274,6 +307,17 @@ fn sqlite_server_components_factory( } } + // A command that writes under the serve lock (`backup purge + // --confirm`, `options.startup_recovery` off) holds the file alone + // and resolves what it needs itself (the reconciler's duties with a + // zero window; its take of the maintenance claim takes over any row + // under the lock): none of the server's recovery below runs, so the + // command exits with none of it half done. + if !options.startup_recovery { + engine.serve_lock = serve_lock; + return components_of(engine, &db_path).await; + } + // Recover any in-flight control-plane transitions from a prior run. match engine.process_control_plane_transitions().await { Ok(transitions) => { @@ -284,31 +328,47 @@ fn sqlite_server_components_factory( Err(e) => tracing::error!("Failed to recover control plane transitions: {e}"), } - // Remove any restore target a prior crash left mid-copy. Before the - // server takes requests, so no restore can be in flight. - match engine.sweep_abandoned_restores().await { - Ok(names) => { - for name in &names { + // A restore the previous run left mid-copy: one process serves this + // file, so its owner is gone. Age every RUNNING job record so the + // engine's reconciler acts on it on its first cycle rather than + // after the stale window: a store-backed (`manifest`) record is + // taken over and resumed, an in-catalog (`legacy`) one is failed and + // its target removed. Before the server takes requests, so no + // restore of this process can be in flight. A backup the previous + // run left CREATING is reported to the reconciler by + // `abandoned_creating_backups` without a step here: no task of this + // process holds its slot. + match engine.expire_restore_job_heartbeats().await { + Ok(aged) => { + for (name, kind) in &aged { + let duty = match kind { + extenddb_storage::backup_catalog::RestoreKind::Manifest => { + "the reconciler resumes it (a store-backed restore)" + } + extenddb_storage::backup_catalog::RestoreKind::Legacy => { + "the reconciler removes it (an in-catalog restore)" + } + }; tracing::warn!( - "removed table '{name}': its restore did not finish before the last \ - shutdown" + "table '{name}': its restore did not finish before the last shutdown; \ + {duty}" ); } } - Err(e) => tracing::error!("Failed to sweep abandoned restores: {e}"), + Err(e) => tracing::error!("Failed to mark interrupted restores: {e}"), } - // Likewise a backup a prior crash left CREATING: its items were only - // partly copied, and nothing will finish them. - match engine.sweep_incomplete_backups().await { - Ok(arns) => { - for arn in &arns { - tracing::warn!( - "removed backup {arn}: its copy did not finish before the last shutdown" - ); - } - } - Err(e) => tracing::error!("Failed to sweep incomplete backups: {e}"), + // A maintenance claim (`backup purge`) the previous holder left: one + // process has this file, so the holder is dead whatever the row's + // heartbeat says, and the row would otherwise make the jobs refuse + // for the whole stale window. + match engine.clear_maintenance_claim().await { + Ok(Some(holder)) => tracing::warn!( + "the maintenance claim held by {holder} was left behind; cleared (one process \ + serves this file, so its holder is gone)" + ), + Ok(None) => {} + Err(e) => tracing::error!("Failed to clear a left-behind maintenance claim: {e}"), } // Rebuild any GSI left mid-backfill (status CREATING) by a prior crash. @@ -330,59 +390,72 @@ fn sqlite_server_components_factory( } engine.serve_lock = serve_lock; - let control_plane_notify = engine.control_plane_notify(); - let engine = Arc::new(engine); - - // Catalog + auth queries: a file-backed database opens a dedicated - // catalog pool over the same file; an in-memory database must reuse the - // engine's single shared connection (its data lives only there). - let catalog_pool = if in_memory { - engine.pool.clone() - } else { - catalog_pool(&db_path) - .await - .map_err(|e| BackendError::ConnectionFailed { - backend: "sqlite".to_owned(), - details: format!("catalog pool: {e}"), - })? - }; + components_of(engine, &db_path).await + }) +} - let enc_key: Option = - sqlx::query_scalar("SELECT value FROM settings WHERE key = 'encryption_key'") - .fetch_optional(&catalog_pool) - .await - .map_err(|e| { - BackendError::InitializationFailed(format!("fetch encryption key: {e}")) - })?; - let enc_key = enc_key.ok_or(BackendError::MissingEncryptionKey)?; - - // The catalog store writes through the second pool over the same - // file, so its writers must hold the SAME lock instance as the - // engine's (design decision D1); an unshared lock would leave the two - // pools contending at the SQLite level, where a slow commit exhausts - // a concurrent writer's busy_timeout ("database is locked" -> 500). - // For an in-memory database the pools are one and the sharing is - // harmless. The credential store only reads and needs no lock. - let catalog_store: Arc = - Arc::new(SqliteCatalogStore::with_encryption_key( - catalog_pool.clone(), - enc_key.clone(), - Arc::clone(&engine.write_lock), - )); - let credential_store: Arc = - Arc::new(SqliteCredentialStore::new(catalog_pool, enc_key)); - - let runtime_hooks = Box::new(SqliteRuntimeHooks { - engine: engine.clone(), - control_plane_notify, - }); - - Ok(ServerComponents { - engine, - catalog_store, - credential_store, - runtime_hooks: Some(runtime_hooks), - }) +/// The server components around an opened engine: the catalog and +/// credential stores over a second pool on the same file (the engine's own +/// connection for an in-memory database, whose data lives only there) and +/// the runtime hooks. +async fn components_of( + engine: SqliteEngine, + db_path: &str, +) -> Result { + let in_memory = engine.in_memory; + let control_plane_notify = engine.control_plane_notify(); + let engine = Arc::new(engine); + + // Catalog + auth queries: a file-backed database opens a dedicated + // catalog pool over the same file; an in-memory database must reuse the + // engine's single shared connection (its data lives only there). + + let catalog_pool = if in_memory { + engine.pool.clone() + } else { + catalog_pool(db_path) + .await + .map_err(|e| BackendError::ConnectionFailed { + backend: "sqlite".to_owned(), + details: format!("catalog pool: {e}"), + })? + }; + + let enc_key: Option = + sqlx::query_scalar("SELECT value FROM settings WHERE key = 'encryption_key'") + .fetch_optional(&catalog_pool) + .await + .map_err(|e| { + BackendError::InitializationFailed(format!("fetch encryption key: {e}")) + })?; + let enc_key = enc_key.ok_or(BackendError::MissingEncryptionKey)?; + + // The catalog store writes through the second pool over the same + // file, so its writers must hold the SAME lock instance as the + // engine's (design decision D1); an unshared lock would leave the two + // pools contending at the SQLite level, where a slow commit exhausts + // a concurrent writer's busy_timeout ("database is locked" -> 500). + // For an in-memory database the pools are one and the sharing is + // harmless. The credential store only reads and needs no lock. + let catalog_store: Arc = + Arc::new(SqliteCatalogStore::with_encryption_key( + catalog_pool.clone(), + enc_key.clone(), + Arc::clone(&engine.write_lock), + )); + let credential_store: Arc = + Arc::new(SqliteCredentialStore::new(catalog_pool, enc_key)); + + let runtime_hooks = Box::new(SqliteRuntimeHooks { + engine: engine.clone(), + control_plane_notify, + }); + + Ok(ServerComponents { + engine, + catalog_store, + credential_store, + runtime_hooks: Some(runtime_hooks), }) } @@ -421,4 +494,63 @@ mod serve_lock_tests { drop(again); let _ = std::fs::remove_dir_all(&dir); } + + /// A maintenance claim a killed `backup purge` left (its row's heartbeat + /// young) is cleared when a server starts on the file: under the serve + /// lock no holder can be alive, so the jobs' check answers free at once + /// rather than after the stale window. + #[tokio::test] + async fn a_server_start_clears_a_left_behind_maintenance_claim() { + let dir = std::env::temp_dir().join(format!( + "extenddb-serve-clears-claim-{}", + uuid::Uuid::new_v4().simple() + )); + std::fs::create_dir_all(&dir).expect("dir"); + let config = crate::config::SqliteConfig { + path: dir.join("db.sqlite").to_string_lossy().into_owned(), + pool_size: 2, + }; + let mut options = extenddb_storage::server_components::ServerComponentsOptions::default(); + options.bootstrap_if_uninitialized = true; + let stale = std::time::Duration::from_secs(300); + let first = super::sqlite_server_components_factory(&config, "us-east-1", options) + .await + .expect("first server"); + let claim = first + .engine + .maintenance_claim("purge on this host (pid 1)", stale) + .await + .expect("claim"); + assert!( + first + .engine + .check_maintenance_claim(stale) + .await + .expect("check") + ); + // The holder dies without releasing: the row stays, heartbeat young. + drop(claim); + drop(first); + + let second = super::sqlite_server_components_factory(&config, "us-east-1", options) + .await + .expect("starts once the first is gone"); + assert!( + !second + .engine + .check_maintenance_claim(stale) + .await + .expect("check"), + "the claim a dead holder left is cleared at startup, not honoured for the window" + ); + // The claim is free to a new holder at once. + let taken = second + .engine + .maintenance_claim("purge on this host (pid 2)", stale) + .await + .expect("a new holder takes the cleared claim"); + taken.release().await.expect("release"); + drop(second); + let _ = std::fs::remove_dir_all(&dir); + } } diff --git a/crates/storage-sqlite/src/migrations/golden_tests.rs b/crates/storage-sqlite/src/migrations/golden_tests.rs index a2e458ebb..53eed8028 100644 --- a/crates/storage-sqlite/src/migrations/golden_tests.rs +++ b/crates/storage-sqlite/src/migrations/golden_tests.rs @@ -1,7 +1,7 @@ // Copyright 2026 ExtendDB contributors // SPDX-License-Identifier: Apache-2.0 -//! Golden tests for SQLite migrations 2 to 4, each run on a released catalog +//! Golden tests for SQLite migrations 2 to 5, each run on a released catalog //! with the damage or the content its body handles, and migration 1's //! accepted and converged pre-release shapes. @@ -627,3 +627,106 @@ async fn migration_4_adds_the_base_key_index_to_index_tables_created_before_v0_1 text.push_str(&dump(&db, "SELECT notes FROM extenddb_migrations WHERE id = 4").await); golden("0004_index_table_base_key", &text); } + +/// Migration 6 on a released catalog holding two restore registrations: one +/// whose table is ACTIVE (a finished restore) and one whose table is CREATING +/// (a restore the stop interrupted). The job record is added and backfilled, +/// the interrupted one RUNNING with the epoch heartbeat and generation 0, the +/// index and maintenance tables exist, the catalog id is written once, and a +/// second run changes nothing. +#[tokio::test] +async fn migration_6_adds_the_catalog_index_and_backfills_the_registrations() { + let db = temp_db("m6"); + load_dump(&db, DEV_0_1_14).await; + // The dump predates `table_restores`; create it as that release did. + exec( + &db, + "CREATE TABLE table_restores (\ + table_id TEXT PRIMARY KEY REFERENCES tables(table_id) ON DELETE CASCADE, \ + source_backup_arn TEXT NOT NULL, \ + source_table_arn TEXT, \ + restore_date_time TEXT NOT NULL); \ + CREATE TEMP TABLE interrupted AS SELECT * FROM tables; \ + UPDATE interrupted SET table_id = '4e1c0d2b-7a9f-4c3e-8b6d-2f5a1c9e7d30', \ + table_name = 'interrupted', table_status = 'CREATING', status_transition_at = NULL; \ + INSERT INTO tables SELECT * FROM interrupted; \ + INSERT INTO table_restores (table_id, source_backup_arn, source_table_arn, restore_date_time) \ + SELECT table_id, 'arn:aws:dynamodb:us-east-1:000000000000:table/cu_orders/backup/01700000000000-0000000a', \ + 'arn:aws:dynamodb:us-east-1:000000000000:table/cu_orders', '2026-01-01T00:00:00.000Z' \ + FROM tables WHERE table_name IN ('cu_orders', 'interrupted');", + ) + .await; + run(&db).await.expect("migrate"); + let catalog_id = scalar(&db, "SELECT value FROM settings WHERE key = 'catalog_id'") + .await + .expect("a catalog id"); + assert!(uuid::Uuid::parse_str(&catalog_id).is_ok(), "{catalog_id}"); + assert_eq!(count(&db, "SELECT COUNT(*) FROM backup_index").await, 0); + assert_eq!( + count(&db, "SELECT COUNT(*) FROM backup_maintenance").await, + 0 + ); + let notes = scalar(&db, "SELECT notes FROM extenddb_migrations WHERE id = 6") + .await + .expect("row"); + for expected in [ + "added table_restores.status", + "added table_restores.completed_at", + "backfilled 2 restore registration(s) as legacy restores, 1 of them RUNNING", + "wrote settings.catalog_id", + ] { + assert!(notes.contains(expected), "{expected} not in {notes}"); + } + let mut text = dump( + &db, + "SELECT t.table_name, t.table_status, r.status, r.kind, r.owner_generation, \ + r.heartbeat_at, r.store_kind, r.completed_at = t.creation_date_time \ + FROM table_restores r JOIN tables t ON t.table_id = r.table_id ORDER BY t.table_name", + ) + .await; + text.push_str( + &dump( + &db, + "SELECT name FROM pragma_table_info('table_restores') ORDER BY cid", + ) + .await, + ); + text.push_str( + &dump( + &db, + "SELECT name, type FROM sqlite_master WHERE name LIKE 'backup_%' OR name LIKE \ + 'idx_backup_%' OR name = 'idx_table_restores_status' ORDER BY name", + ) + .await, + ); + text.push_str(&dump(&db, "SELECT notes FROM extenddb_migrations WHERE id = 6").await); + golden("0006_backup_catalog_index", &text); + + // A second run (through reconverge, which runs every convergent body + // again) adds nothing and keeps the catalog id. + reconverge(&db).await.expect("reconverge"); + assert_eq!( + scalar(&db, "SELECT value FROM settings WHERE key = 'catalog_id'") + .await + .as_deref(), + Some(catalog_id.as_str()) + ); + assert_eq!( + count( + &db, + "SELECT COUNT(*) FROM pragma_table_info('table_restores')" + ) + .await, + 16 + ); + assert_eq!( + scalar( + &db, + "SELECT status FROM table_restores r JOIN tables t ON t.table_id = r.table_id \ + WHERE t.table_name = 'interrupted'" + ) + .await + .as_deref(), + Some("RUNNING") + ); +} diff --git a/crates/storage-sqlite/src/migrations/m0006_backup_catalog_index.rs b/crates/storage-sqlite/src/migrations/m0006_backup_catalog_index.rs new file mode 100644 index 000000000..9d7d5380c --- /dev/null +++ b/crates/storage-sqlite/src/migrations/m0006_backup_catalog_index.rs @@ -0,0 +1,185 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Migration 6, `backup_catalog_index`, SQLite: the catalog index of backups +//! kept in a backup store (`backup_index`), the restore job fields on the +//! restore registrations (`table_restores`), the maintenance claim record +//! (`backup_maintenance`) and the catalog id (`settings.catalog_id`). +//! +//! SQLite has no `ADD COLUMN IF NOT EXISTS`, so the body reads the columns of +//! `table_restores` first and adds each one only when it is missing; the +//! tables and indexes are created `IF NOT EXISTS`; the backfill touches only +//! registrations whose `status` is still NULL; and the catalog id is written +//! only when the row is absent. A second run changes nothing, which makes +//! the migration convergent. +//! +//! The backfill: every registration that exists was written by a release +//! without the job record, so each is a legacy restore. One whose table is +//! not CREATING finished (DONE). One whose table is CREATING is a restore the +//! upgrade's stop interrupted: RUNNING, with a heartbeat at the epoch (not +//! NULL, so the staleness test selects it) and an owner generation no job +//! holds (0), so the first reconciler pass fails it and removes its target. +//! +//! This file's checksum is pinned once the migration is released. It is +//! excluded from rustfmt and clippy so neither can require an edit, and it +//! keeps its own column check instead of calling the crate's helpers, which +//! may change. + +use async_trait::async_trait; +use extenddb_storage::migrations::{CodeBody, Done, MigrationError}; +use sqlx::SqlitePool; + +use super::SqliteStores; + +/// This file, whose SHA-256 the checksum test pins. +pub(crate) const SOURCE: &str = include_str!("m0006_backup_catalog_index.rs"); + +/// Recorded as this migration's checksum. Changes only with a new migration. +const FINGERPRINT: &str = "0006_backup_catalog_index/sqlite/1"; + +/// The columns the job record adds to `table_restores`, with their SQLite +/// declarations. None carries NOT NULL: `ADD COLUMN` on a populated table +/// cannot, and the application writes every one of them. +const RESTORE_JOB_COLUMNS: [(&str, &str); 12] = [ + ("status", "TEXT"), + ("kind", "TEXT"), + ("store_kind", "TEXT"), + ("store_root", "TEXT"), + ("store_id", "TEXT"), + ("owner_generation", "INTEGER"), + ("heartbeat_at", "TEXT"), + ("spool_host", "TEXT"), + ("file_index", "INTEGER"), + ("items_applied", "INTEGER"), + ("failure_reason", "TEXT"), + ("completed_at", "TEXT"), +]; + +/// The statements that need no column check. +const GUARDED_SQL: &str = "\ +CREATE TABLE IF NOT EXISTS backup_index ( + account_id TEXT NOT NULL, + backup_id TEXT NOT NULL, + table_name TEXT NOT NULL, + backup_arn TEXT NOT NULL, + backup_name TEXT NOT NULL, + status TEXT NOT NULL, + store_kind TEXT NOT NULL, + store_root TEXT NOT NULL, + store_id TEXT NOT NULL, + size_bytes INTEGER NOT NULL DEFAULT 0, + item_count INTEGER NOT NULL DEFAULT 0, + created_at TEXT NOT NULL, + completed_at TEXT, + failure_reason TEXT, + source_table_details_json TEXT NOT NULL, + owner_generation INTEGER NOT NULL, + heartbeat_at TEXT NOT NULL, + PRIMARY KEY (account_id, backup_id) +); +CREATE UNIQUE INDEX IF NOT EXISTS idx_backup_index_backup_id ON backup_index (backup_id); +CREATE INDEX IF NOT EXISTS idx_backup_index_account_table ON backup_index (account_id, table_name); +CREATE INDEX IF NOT EXISTS idx_backup_index_store_status + ON backup_index (store_kind, store_root, store_id, status); +CREATE TABLE IF NOT EXISTS backup_maintenance ( + singleton INTEGER PRIMARY KEY CHECK (singleton = 1), + holder TEXT NOT NULL, + owner_generation INTEGER NOT NULL, + claimed_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + heartbeat_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) +); +"; + +/// The backfill of the registrations that exist, scoped to rows the job +/// record is still empty on. +const BACKFILL_SQL: &str = "\ +UPDATE table_restores SET + kind = 'legacy', + owner_generation = 0, + heartbeat_at = '1970-01-01T00:00:00.000Z', + status = CASE WHEN (SELECT table_status FROM tables t WHERE t.table_id = table_restores.table_id) = 'CREATING' + THEN 'RUNNING' ELSE 'DONE' END, + completed_at = CASE WHEN (SELECT table_status FROM tables t WHERE t.table_id = table_restores.table_id) = 'CREATING' + THEN NULL + ELSE (SELECT creation_date_time FROM tables t WHERE t.table_id = table_restores.table_id) END +WHERE status IS NULL; +CREATE INDEX IF NOT EXISTS idx_table_restores_status ON table_restores (status, heartbeat_at); +"; + +pub(crate) struct BackupCatalogIndex; + +#[async_trait] +impl CodeBody for BackupCatalogIndex { + fn fingerprint(&self) -> &'static str { + FINGERPRINT + } + + /// Nothing this migration finds stops it. + async fn detect(&self, _stores: &SqliteStores) -> Result, MigrationError> { + Ok(Vec::new()) + } + + async fn run(&self, stores: &SqliteStores) -> Result { + let pool = &stores.pool; + sqlx::raw_sql(GUARDED_SQL) + .execute(pool) + .await + .map_err(|e| MigrationError::store("create backup_index and backup_maintenance", e))?; + let mut notes = Vec::new(); + let present = columns_of(pool, "table_restores").await?; + for (name, declaration) in RESTORE_JOB_COLUMNS { + if present.iter().any(|c| c == name) { + continue; + } + sqlx::query(&format!("ALTER TABLE table_restores ADD COLUMN {name} {declaration}")) + .execute(pool) + .await + .map_err(|e| MigrationError::store(&format!("add table_restores.{name}"), e))?; + notes.push(format!("added table_restores.{name}")); + } + let running: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM table_restores r JOIN tables t ON t.table_id = r.table_id \ + WHERE r.status IS NULL AND t.table_status = 'CREATING'", + ) + .fetch_one(pool) + .await + .map_err(|e| MigrationError::store("count interrupted restores", e))?; + let backfilled: i64 = + sqlx::query_scalar("SELECT COUNT(*) FROM table_restores WHERE status IS NULL") + .fetch_one(pool) + .await + .map_err(|e| MigrationError::store("count registrations", e))?; + sqlx::raw_sql(BACKFILL_SQL) + .execute(pool) + .await + .map_err(|e| MigrationError::store("backfill table_restores", e))?; + if backfilled > 0 { + notes.push(format!( + "backfilled {backfilled} restore registration(s) as legacy restores, {running} of \ + them RUNNING with a dead owner for the reconciler to remove" + )); + } + let wrote_id = sqlx::query( + "INSERT INTO settings (key, value) SELECT 'catalog_id', ? \ + WHERE NOT EXISTS (SELECT 1 FROM settings WHERE key = 'catalog_id')", + ) + .bind(uuid::Uuid::new_v4().to_string()) + .execute(pool) + .await + .map_err(|e| MigrationError::store("write catalog_id", e))? + .rows_affected(); + if wrote_id > 0 { + notes.push("wrote settings.catalog_id".to_owned()); + } + Ok(Done { notes }) + } +} + +/// The column names of `table`, from `pragma_table_info`. +async fn columns_of(pool: &SqlitePool, table: &str) -> Result, MigrationError> { + sqlx::query_scalar("SELECT name FROM pragma_table_info(?)") + .bind(table) + .fetch_all(pool) + .await + .map_err(|e| MigrationError::store(&format!("read the columns of {table}"), e)) +} diff --git a/crates/storage-sqlite/src/migrations/mod.rs b/crates/storage-sqlite/src/migrations/mod.rs index 06647fbb4..e94ab8fe5 100644 --- a/crates/storage-sqlite/src/migrations/mod.rs +++ b/crates/storage-sqlite/src/migrations/mod.rs @@ -29,6 +29,9 @@ mod m0003_sort_key_definitions; #[rustfmt::skip] #[allow(clippy::all, deprecated)] mod m0004_index_table_base_key; +#[rustfmt::skip] +#[allow(clippy::all, deprecated)] +mod m0006_backup_catalog_index; mod postcheck; use std::time::Duration; @@ -106,6 +109,19 @@ pub fn migration(id: MigrationId) -> Migration { "0005_drop_continuous_backups.sql" )], }, + MigrationId::BackupCatalogIndex => Migration::Applies { + kind: Kind::Breaking, + convergent: true, + bodies: vec![StoreBody::code( + StoreKind::Catalog, + &m0006_backup_catalog_index::BackupCatalogIndex, + m0006_backup_catalog_index::SOURCE, + )], + }, + MigrationId::RestoreProgress => Migration::NotApplicable( + "the restore progress of a store-backed restore is on table_restores (file_index, \ + items_applied) on SQLite; the table is PostgreSQL's data database's", + ), } } diff --git a/crates/storage-sqlite/src/migrations/released_tests.rs b/crates/storage-sqlite/src/migrations/released_tests.rs index 71dcee691..321d2340d 100644 --- a/crates/storage-sqlite/src/migrations/released_tests.rs +++ b/crates/storage-sqlite/src/migrations/released_tests.rs @@ -316,7 +316,16 @@ async fn every_released_catalog_upgrades_to_the_fresh_schema() { for (key, value) in state.settings { expected_settings.insert((*key).to_owned(), (*value).to_owned()); } - assert_eq!(settings(&db).await, expected_settings, "{name}: settings"); + // Migration 6 writes a random catalog id on every catalog. + let mut actual_settings = settings(&db).await; + let catalog_id = actual_settings + .remove("catalog_id") + .unwrap_or_else(|| panic!("{name}: no catalog_id")); + assert!( + uuid::Uuid::parse_str(&catalog_id).is_ok(), + "{name}: {catalog_id}" + ); + assert_eq!(actual_settings, expected_settings, "{name}: settings"); // No post-check note: no released state has a pre-release shape. let notes = scalar(&db, "SELECT notes FROM extenddb_migrations WHERE id = 1").await; @@ -390,11 +399,13 @@ async fn a_catalog_an_older_release_wrote_to_is_reported_and_reconverged() { next_second().await; let report = reconverge(&db).await.expect("reconverge"); - // Every convergent migration ran again; migration 2, which is not - // convergent, is named as skipped. + // Every convergent migration that applies on SQLite ran again (7, + // not applicable here, has nothing to run); migration 2, which is + // not convergent, is named as skipped. let convergent = MigrationId::ALL .into_iter() - .filter(|id| super::migration(*id).convergent()) + .map(super::migration) + .filter(|m| m.convergent() && !m.bodies().is_empty()) .count(); assert_eq!(report.applied.len(), convergent, "{trace}: {report:?}"); let skipped = format!("Skipped {}:", MigrationId::ResourceTagConditionKeys); diff --git a/crates/storage-sqlite/src/migrations/tests.rs b/crates/storage-sqlite/src/migrations/tests.rs index a837e802e..90efc0d67 100644 --- a/crates/storage-sqlite/src/migrations/tests.rs +++ b/crates/storage-sqlite/src/migrations/tests.rs @@ -270,14 +270,25 @@ async fn a_fresh_database_is_current_after_one_run_and_seeds_nothing() { "SELECT COUNT(*) FROM extenddb_migrations WHERE outcome = 'applied'" ) .await, - i64::from(LATEST) + i64::from(LATEST) - 1, + "every id but 7 (the PostgreSQL progress table) applies on SQLite" + ); + assert_eq!( + scalar(&db, "SELECT outcome FROM extenddb_migrations WHERE id = 7") + .await + .as_deref(), + Some("not_applicable") ); assert_eq!(version_row(&db).await, Some(LATEST.to_string())); assert_eq!( count(&db, "SELECT COUNT(*) FROM settings").await, - 1, - "only the version row" + 2, + "the version row and the catalog id migration 6 writes" ); + let catalog_id = scalar(&db, "SELECT value FROM settings WHERE key = 'catalog_id'") + .await + .expect("a catalog id"); + assert!(uuid::Uuid::parse_str(&catalog_id).is_ok(), "{catalog_id}"); assert!(run(&db).await.expect("again").applied.is_empty()); } diff --git a/crates/storage-sqlite/src/restore_engine.rs b/crates/storage-sqlite/src/restore_engine.rs new file mode 100644 index 000000000..0b11818da --- /dev/null +++ b/crates/storage-sqlite/src/restore_engine.rs @@ -0,0 +1,562 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The data-plane side of a store-backed restore on SQLite +//! ([`RestoreEngine`]): the items of one verified data file written into the +//! CREATING target, the progress record, the ACTIVE flip and the count. +//! +//! One write transaction (`BEGIN IMMEDIATE` under the engine write lock) per +//! batch of at most [`RESTORE_BATCH_ITEMS`] items or [`RESTORE_BATCH_BYTES`] +//! of serialized item text (a batch closes at the first item that takes its +//! `item_data` text past the bound, so the batch term of a restore's memory +//! is bounded in bytes, not only in items): the base rows (plain INSERTs, so +//! a key the backup holds twice, within a file or across files, fails the +//! file instead of collapsing two items), each secondary index's rows and +//! each vector index's rows beside them, and last the record's progress, +//! `UPDATE table_restores SET file_index, items_applied` conditioned on the +//! claim's generation and on RUNNING: a batch commits whole with its +//! progress or not at all, and a batch whose record another owner took +//! rolls back and ends the file as a lost claim. Other writers on the server +//! wait for a batch rather than the whole file. A resume reads the record's +//! `(file_index, items_applied)` and the engine skips that many items of the +//! file before the stream starts. An empty file records its progress too +//! (`items_applied = start_at`), so a resume does not re-read it. +//! +//! The restore's own buffer is this one batch of `(Item, String)`, bounded +//! by [`RESTORE_BATCH_BYTES`] whatever the table's indexes are. A table with +//! secondary or vector indexes nonetheless restores in more resident memory +//! than one without (about 90 MiB over idle at 100,000 200-byte items with +//! two full-projection GSIs against 43 MiB with none), and the excess is +//! SQLite's own: its per-connection page cache (`PRAGMA cache_size = +//! -32000`, 32 MiB) holds the index tables' b-tree pages, which a load of +//! scattered index keys dirties across the whole tree, and glibc's +//! per-arena free lists hold freed blocks (the figure falls by about a third +//! under `MALLOC_ARENA_MAX=1`). The excess is bounded by `cache_size` and +//! grows only sub-linearly with the table (about 90 MiB at 100,000 items, +//! 175 MiB at 300,000), not with the restore's buffers; a smaller batch does +//! not lower it (measured: the peak is unchanged at a third of the item +//! cap), because the cache persists across batches. The published restore +//! figures are for a table with no secondary index and state that shape. +//! +//! A job's heartbeat on SQLite (`SqliteJobClaim::heartbeat`) is an `UPDATE` +//! under the same engine write lock, so a beat waits for the batch that +//! holds the lock and for every writer queued ahead of it: other jobs' +//! batches, item writes, the GSI backfill, the TTL sweep, a table's drop. +//! The bench record (`docs/design/15-backup-restore-bench-pr9.md`) puts one +//! batch at about 1.1 s for items of 27.7 MB (24 of them, one per batch, +//! restored in 26.8 s with their download and decode) and about 0.27 s for +//! 10,000 items of 200 bytes (1,000,000 in 26.8 s). A beat that has not +//! returned within one heartbeat period (15 s at the least) ends the job as +//! a lost claim: it stops with its record RUNNING and makes no further +//! table access, its slot is freed, and the reconciler's next cycle takes +//! the record over and resumes it from the last committed batch. At the +//! default four jobs a beat waits a few seconds at most; many jobs of large +//! items, or the drop of a large table, can hold it past a period, which +//! costs a restart of the file from its last batch and a lost-claim log +//! line, not data. +//! +//! `finish_restore` is [`activate_restore_target`], the one transaction the +//! legacy copy ends with too: the record DONE conditioned on the generation +//! and on RUNNING, the table ACTIVE conditioned on CREATING, and every +//! secondary and vector index of the table ACTIVE. + +use extenddb_core::types::{AttributeDefinition, Item, KeySchemaElement, Projection}; +use extenddb_storage::backup_catalog::{ + JobClaim, RESTORE_SQLITE_MID_BATCH, RestoreTargetKey, restore_pause_point, +}; +use extenddb_storage::error::StorageError; +use extenddb_storage::restore_engine::{BulkLoadFile, Finished, RestoreEngine}; +pub(crate) use extenddb_storage::restore_engine::{RESTORE_BATCH_BYTES, RESTORE_BATCH_ITEMS}; +use extenddb_storage::util::{composite_pk_to_text, parse_sk, sk_column_n}; +use extenddb_storage::vector_lifecycle::VectorIndexMeta; +use futures::future::BoxFuture; +use futures::stream::BoxStream; + +use crate::SqliteEngine; +use crate::backup_catalog::{NOW, generation_column}; +use crate::data::vector_index::{ + VectorRowConflict, base_key_columns, fetch_vector_indexes_for_table, insert_vector_row, +}; +use crate::data::{ + BoundValue, all_sort_key_info, data_table_name, index_table_name, insert_index_row_multi, + item_has_index_keys, project_item_for_index, sk_bound, +}; +use crate::sqlite_util::is_unique_violation; + +fn db_err(e: sqlx::Error) -> StorageError { + StorageError::Internal(format!("Database error: {e}")) +} + +/// The error for a key the target already holds: the backup carries it +/// twice, within the file or in a file applied before it. The engine +/// prefixes the file's key, so the file is not named here. +fn duplicate_key_error(e: &sqlx::Error) -> StorageError { + StorageError::Validation(format!( + "holds a primary key the target already has (the backup holds a key twice): {e}" + )) +} + +/// The key shape of a CREATING restore target, read from its catalog row. +struct TargetShape { + key_schema: Vec, + attribute_definitions: Vec, +} + +/// A secondary index of the target, as the load writes it. +struct TargetIndex { + table: String, + key_schema: Vec, + projection: Projection, +} + +/// What `bulk_load` writes per item and per batch, read once per file. +struct LoadPlan { + insert_sql: String, + shape: TargetShape, + indexes: Vec, + vector_key_columns: Vec, + file: BulkLoadFile, + generation: u64, + /// The file as an error names it. + source: String, +} + +/// What [`activate_restore_target`] matched. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct Activated { + /// The record was RUNNING under the generation and is DONE. + pub record: bool, + /// The table was CREATING and is ACTIVE. + pub table: bool, +} + +/// The end of a restore, inside the caller's write transaction: the record +/// DONE (`completed_at` now; `items_applied` left as the progress stands +/// unless the caller gives a total) only while it is RUNNING under +/// `generation`; the table ACTIVE with its counts only from CREATING; every +/// secondary and vector index of the table ACTIVE (`backfilling` cleared on +/// the vector rows, which a restore target creates CREATING with the table). +/// Reports what each conditional write matched; the caller rolls back when +/// it needed a match it did not get. +pub(crate) async fn activate_restore_target( + tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>, + table_id: &str, + generation: u64, + item_count: i64, + table_size_bytes: i64, + items_applied: Option, +) -> Result { + let record = sqlx::query(&format!( + "UPDATE table_restores SET status = 'DONE', \ + items_applied = COALESCE(?, items_applied), completed_at = {NOW} \ + WHERE table_id = ? AND status = 'RUNNING' AND owner_generation = ?" + )) + .bind(items_applied) + .bind(table_id) + .bind(generation_column(generation)) + .execute(&mut **tx) + .await + .map_err(db_err)? + .rows_affected() + == 1; + if !record { + return Ok(Activated { + record, + table: false, + }); + } + let table = sqlx::query( + "UPDATE tables SET item_count = ?, table_size_bytes = ?, \ + table_status = 'ACTIVE', status_transition_at = NULL \ + WHERE table_id = ? AND table_status = 'CREATING'", + ) + .bind(item_count) + .bind(table_size_bytes) + .bind(table_id) + .execute(&mut **tx) + .await + .map_err(db_err)? + .rows_affected() + == 1; + sqlx::query("UPDATE indexes SET index_status = 'ACTIVE' WHERE table_id = ? AND index_status <> 'ACTIVE'") + .bind(table_id) + .execute(&mut **tx) + .await + .map_err(db_err)?; + sqlx::query( + "UPDATE vector_indexes SET index_status = 'ACTIVE', backfilling = NULL \ + WHERE table_id = ? AND index_status <> 'ACTIVE'", + ) + .bind(table_id) + .execute(&mut **tx) + .await + .map_err(db_err)?; + Ok(Activated { record, table }) +} + +impl SqliteEngine { + /// The target's key schema and attribute definitions, when `key` names + /// a CREATING restore target with this table id. + async fn restore_target_shape( + &self, + key: &RestoreTargetKey, + ) -> Result { + let row: Option<(String, String)> = sqlx::query_as( + "SELECT t.key_schema, t.attribute_definitions FROM tables t \ + JOIN table_restores r ON r.table_id = t.table_id \ + WHERE t.table_id = ? AND t.account_id = ? AND t.table_name = ? \ + AND t.table_status = 'CREATING'", + ) + .bind(&key.table_id) + .bind(&key.account_id) + .bind(&key.table_name) + .fetch_optional(&self.pool) + .await + .map_err(db_err)?; + let (key_schema, attribute_definitions) = row.ok_or_else(|| { + StorageError::TableNotFound(format!( + "{} is not a CREATING restore target with table id {}", + key.table_name, key.table_id + )) + })?; + Ok(TargetShape { + key_schema: serde_json::from_str(&key_schema) + .map_err(|e| StorageError::Internal(format!("key schema: {e}")))?, + attribute_definitions: serde_json::from_str(&attribute_definitions) + .map_err(|e| StorageError::Internal(format!("attribute definitions: {e}")))?, + }) + } + + /// Everything a file's writes need that does not change between batches. + async fn load_plan( + &self, + key: &RestoreTargetKey, + file: BulkLoadFile, + generation: u64, + ) -> Result { + let shape = self.restore_target_shape(key).await?; + let sort_keys = all_sort_key_info(&shape.key_schema, &shape.attribute_definitions); + let mut cols = vec!["pk".to_owned()]; + cols.extend( + sort_keys + .iter() + .enumerate() + .map(|(i, &(_, t))| sk_column_n(i, t)), + ); + cols.push("item_data".to_owned()); + let data_table = data_table_name(&key.table_id); + let insert_sql = format!( + "INSERT INTO {data_table} ({}) VALUES ({})", + cols.join(", "), + vec!["?"; cols.len()].join(", ") + ); + let index_rows: Vec<(String, String, String)> = sqlx::query_as( + "SELECT index_id, key_schema, projection FROM indexes WHERE table_id = ?", + ) + .bind(&key.table_id) + .fetch_all(&self.pool) + .await + .map_err(db_err)?; + let mut indexes = Vec::with_capacity(index_rows.len()); + for (index_id, ks, proj) in index_rows { + indexes.push(TargetIndex { + table: index_table_name(&index_id), + key_schema: serde_json::from_str(&ks) + .map_err(|e| StorageError::Internal(format!("index key schema: {e}")))?, + projection: serde_json::from_str(&proj) + .map_err(|e| StorageError::Internal(format!("index projection: {e}")))?, + }); + } + let vector_key_columns = base_key_columns(&shape.key_schema, &shape.attribute_definitions); + Ok(LoadPlan { + insert_sql, + shape, + indexes, + vector_key_columns, + file, + generation, + source: format!("data file {}", file.file_index), + }) + } + + /// One batch: the base rows, the index rows, the vector rows and the + /// record's progress in one write transaction under the write lock. + /// `applied` is the file's running total before this batch, `start_at` + /// included; each item comes with its `item_data` text, serialized when + /// the batch was collected. `Ok(false)` when the record was not RUNNING + /// under the plan's generation: nothing of the batch is committed. + async fn write_restore_batch( + &self, + key: &RestoreTargetKey, + plan: &LoadPlan, + applied: u64, + batch: &mut Vec<(Item, String)>, + ) -> Result { + let batch_len = batch.len(); + let source = plan.source.as_str(); + let _writer = self.write_lock.lock().await; + let mut tx = self + .pool + .begin_with("BEGIN IMMEDIATE") + .await + .map_err(db_err)?; + let shape = &plan.shape; + let sort_keys = all_sort_key_info(&shape.key_schema, &shape.attribute_definitions); + // The vector indexes, read inside the transaction as the write path + // does (the id names the data table). + let vector_metas: Vec = + fetch_vector_indexes_for_table(&mut tx, &key.table_id).await?; + for (item, item_json) in batch.drain(..) { + let mut values = vec![BoundValue::Text(composite_pk_to_text( + &item, + &shape.key_schema, + )?)]; + for &(name, sk_type) in &sort_keys { + let value = item.get(name).ok_or_else(|| { + StorageError::Validation(format!( + "{source} holds an item without its sort key {name}" + )) + })?; + values.push(sk_bound(&parse_sk(value, sk_type)?)); + } + values.push(BoundValue::Text(item_json)); + let mut q = sqlx::query(&plan.insert_sql); + for v in values { + q = crate::data::bind_bound!(q, v); + } + q.execute(&mut *tx).await.map_err(|e| { + if is_unique_violation(&e) { + duplicate_key_error(&e) + } else { + db_err(e) + } + })?; + for idx in &plan.indexes { + if !item_has_index_keys(&item, &idx.key_schema) { + continue; + } + let idx_sks = all_sort_key_info(&idx.key_schema, &shape.attribute_definitions); + let projected = project_item_for_index( + &item, + &idx.key_schema, + &shape.key_schema, + &idx.projection, + ); + insert_index_row_multi( + &mut tx, + &idx.table, + &item, + &projected, + &idx.key_schema, + &shape.key_schema, + &idx_sks, + &sort_keys, + ) + .await?; + } + for meta in &vector_metas { + insert_vector_row( + &mut tx, + &key.table_id, + meta, + &item, + &shape.key_schema, + &shape.attribute_definitions, + &plan.vector_key_columns, + VectorRowConflict::Fail, + ) + .await?; + } + } + // The progress, last and in the same transaction, conditioned on the + // generation and on RUNNING: a record another owner took rolls the + // batch back with it. + let total = applied.saturating_add(u64::try_from(batch_len).unwrap_or(u64::MAX)); + let matched = sqlx::query( + "UPDATE table_restores SET file_index = ?, items_applied = ? \ + WHERE table_id = ? AND owner_generation = ? AND status = 'RUNNING'", + ) + .bind(i64::from(plan.file.file_index)) + .bind(i64::try_from(total).unwrap_or(i64::MAX)) + .bind(&key.table_id) + .bind(generation_column(plan.generation)) + .execute(&mut *tx) + .await + .map_err(db_err)? + .rows_affected(); + if matched != 1 { + tx.rollback().await.map_err(db_err)?; + return Ok(false); + } + tx.commit().await.map_err(db_err)?; + Ok(true) + } +} + +impl RestoreEngine for SqliteEngine { + fn bulk_load<'a>( + &'a self, + target: &'a RestoreTargetKey, + claim: &'a dyn JobClaim, + file: BulkLoadFile, + mut items: BoxStream<'a, Result>, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + use futures::StreamExt; + + let generation = claim.generation(); + let plan = self.load_plan(target, file, generation).await?; + let mut batch: Vec<(Item, String)> = Vec::with_capacity(RESTORE_BATCH_ITEMS.min(1024)); + let mut batch_bytes = 0usize; + let mut applied: u64 = 0; + let mut batches_written = 0u32; + let mut ended = false; + while !ended { + // Collect a batch, closed at the count cap or at the first + // item whose text takes it past the byte bound; an `Err` + // item ends the file with that error, and the batch in hand + // is never written. + while batch.len() < RESTORE_BATCH_ITEMS && batch_bytes < RESTORE_BATCH_BYTES { + match items.next().await { + Some(item) => { + let item = item?; + let item_json = serde_json::to_string(&item) + .map_err(|e| StorageError::Internal(format!("item: {e}")))?; + batch_bytes = batch_bytes.saturating_add(item_json.len()); + batch.push((item, item_json)); + } + None => { + ended = true; + break; + } + } + } + if batch.is_empty() && batches_written > 0 { + break; + } + if batches_written > 0 { + restore_pause_point(RESTORE_SQLITE_MID_BATCH).await; + } + let batch_len = batch.len(); + let written = self + .write_restore_batch( + target, + &plan, + file.start_at.saturating_add(applied), + &mut batch, + ) + .await?; + if !written { + return Err(StorageError::Internal(format!( + "claim lost: the job record of {} is not RUNNING under generation \ + {generation}; the batch was rolled back", + target.table_name + ))); + } + applied += u64::try_from(batch_len).unwrap_or(u64::MAX); + batches_written += 1; + batch_bytes = 0; + } + Ok(applied) + }) + } + + fn finish_restore<'a>( + &'a self, + target: &'a RestoreTargetKey, + claim: &'a dyn JobClaim, + item_count: u64, + size_bytes: u64, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + // The key names the table; the flip itself is conditioned on the + // table id, CREATING, RUNNING and the generation. + let is_target: bool = sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM tables WHERE table_id = ? AND account_id = ? \ + AND table_name = ?)", + ) + .bind(&target.table_id) + .bind(&target.account_id) + .bind(&target.table_name) + .fetch_one(&self.pool) + .await + .map_err(db_err)?; + if !is_target { + return Ok(Finished::NotMatched(format!( + "{} is not a table with id {} in account {}", + target.table_name, target.table_id, target.account_id + ))); + } + let _writer = self.write_lock.lock().await; + let mut tx = self + .pool + .begin_with("BEGIN IMMEDIATE") + .await + .map_err(db_err)?; + let activated = activate_restore_target( + &mut tx, + &target.table_id, + claim.generation(), + i64::try_from(item_count).unwrap_or(i64::MAX), + i64::try_from(size_bytes).unwrap_or(i64::MAX), + None, + ) + .await?; + if !(activated.record && activated.table) { + tx.rollback().await.map_err(db_err)?; + return Ok(Finished::NotMatched(format!( + "the job record of {} is not RUNNING under generation {}, or the table is \ + not CREATING", + target.table_name, + claim.generation() + ))); + } + tx.commit().await.map_err(db_err)?; + Ok(Finished::Active) + }) + } + + fn count_restore_target<'a>( + &'a self, + key: &'a RestoreTargetKey, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + // The id names the data table; the key's account and name are + // checked against the catalog so a bare id reads no other + // account's table. + self.restore_target_shape(key).await?; + let count: i64 = sqlx::query_scalar(&format!( + "SELECT COUNT(*) FROM {}", + data_table_name(&key.table_id) + )) + .fetch_one(&self.pool) + .await + .map_err(db_err)?; + Ok(u64::try_from(count).unwrap_or_default()) + }) + } + + fn restore_target_ready<'a>( + &'a self, + key: &'a RestoreTargetKey, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + // The catalog row first, so a bare id reads no other account's + // table; then the data table's existence (the registration + // commits the catalog transaction before the data DDL). + // `data_table_name` is a quoted identifier; `sqlite_master.name` + // holds it unquoted. + self.restore_target_shape(key).await?; + let quoted = data_table_name(&key.table_id); + let name = quoted.trim_matches('"'); + let exists: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = ?", + ) + .bind(name) + .fetch_one(&self.pool) + .await + .map_err(db_err)?; + Ok(exists > 0) + }) + } +} diff --git a/crates/storage-sqlite/src/restore_job_tests.rs b/crates/storage-sqlite/src/restore_job_tests.rs new file mode 100644 index 000000000..146904e27 --- /dev/null +++ b/crates/storage-sqlite/src/restore_job_tests.rs @@ -0,0 +1,1537 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Storage-level tests of the store-backed restore on SQLite: the +//! registration under the write lock (`claim_restore`), the takeover of a +//! record whose slot no task holds (`claim_restore_dead`), the per-batch load +//! with its progress write (`bulk_load`), the ACTIVE flip (`finish_restore`) +//! and the count. The engine's restore job drives these through +//! `BackupCatalog` and `RestoreEngine`; what each must guarantee on this +//! backend is what a test here pins. Every test builds its own in-memory +//! engine. + +use std::sync::Arc; +use std::time::Duration; + +use extenddb_core::types::{ + AttributeValue, CreateTableInput, DistanceFunction, Item, Projection, ProjectionType, + TableDescription, TableStatus, VectorAttribute, VectorIndexSpecification, +}; +use extenddb_storage::backup_catalog::{ + BackupRow, BackupStatus, DeleteStart, Heartbeat, JobClaim, JobStatus, RestoreJobRow, + RestoreKind, RestoreRegistrationRow, RestoreTargetKey, StatusFields, StoreIdentity, StoreKind, + WriteOutcome, backup_in_use_message, is_backup_not_found, +}; +use extenddb_storage::error::StorageError; +use extenddb_storage::restore_engine::{BulkLoadFile, Finished, RestoreEngine}; +use extenddb_storage::{BackupCatalog, JobPermit, TableEngine}; +use futures::StreamExt; +use futures::stream::BoxStream; + +use crate::SqliteEngine; +use crate::backup_catalog::{SqliteJobClaim, restore_slot_key}; +use crate::restore_engine::{RESTORE_BATCH_BYTES, RESTORE_BATCH_ITEMS}; + +const ACCOUNT: &str = "123456789012"; +const REGION: &str = "us-east-1"; +/// A stale window no test's heartbeat crosses on its own. +const STALE: Duration = Duration::from_secs(300); + +// --- the engine, rows, inputs and items ------------------------------------------- + +async fn engine() -> SqliteEngine { + let engine = SqliteEngine::new(":memory:", 2, REGION, 409_600) + .await + .expect("engine"); + crate::migrations::apply_for_tests(&engine.pool).await; + for (key, value) in [ + ("control_plane_delay_seconds", "0"), + ("index_propagation_delay_ms", "0"), + ] { + sqlx::query( + "INSERT INTO settings (key, value) VALUES (?, ?) \ + ON CONFLICT(key) DO UPDATE SET value = excluded.value", + ) + .bind(key) + .bind(value) + .execute(&engine.pool) + .await + .expect("setting"); + } + sqlx::query("INSERT INTO accounts (account_id, account_name) VALUES (?, 'default')") + .bind(ACCOUNT) + .execute(&engine.pool) + .await + .expect("account"); + engine +} + +fn store() -> StoreIdentity { + StoreIdentity { + store_kind: StoreKind::Filesystem, + store_root: "/var/lib/extenddb/backups".to_owned(), + store_id: "store-1".to_owned(), + } +} + +/// An AVAILABLE `backup_index` row of a backup of `table`, written through +/// the claim as a backup job writes it. +async fn available_backup(engine: &SqliteEngine, table: &str) -> BackupRow { + let backup_id = extenddb_storage::util::new_backup_id(); + let backup_arn = extenddb_storage::util::backup_arn(REGION, ACCOUNT, table, &backup_id); + let manifest: serde_json::Value = serde_json::from_str(include_str!( + "../../core/testdata/extenddb-manifest/v1.json" + )) + .expect("fixture"); + let row = BackupRow { + backup_id: backup_id.clone(), + account_id: ACCOUNT.to_owned(), + table_name: table.to_owned(), + backup_arn, + backup_name: "nightly".to_owned(), + status: BackupStatus::Creating, + store: store(), + size_bytes: 0, + item_count: 0, + created_at: extenddb_storage::util::backup_creation_date_time(&backup_id).unwrap(), + completed_at: None, + failure_reason: None, + source_table_details_json: manifest["source_table"].clone(), + owner_generation: 0, + heartbeat_at: 0.0, + }; + let claim = engine.claim_backup(&row).await.expect("claim the row"); + let written = claim + .set_status( + JobStatus::Creating, + JobStatus::Available, + &StatusFields { + item_count: Some(3), + size_bytes: Some(1234), + completed: true, + ..StatusFields::default() + }, + ) + .await + .expect("the AVAILABLE write"); + assert_eq!(written, WriteOutcome::Matched); + drop(claim); + BackupRow { + status: BackupStatus::Available, + ..row + } +} + +fn registration(row: &BackupRow, spool_host: Option<&str>) -> RestoreRegistrationRow { + RestoreRegistrationRow { + backup: row.key(), + source_table_arn: format!( + "arn:aws:dynamodb:{REGION}:{ACCOUNT}:table/{}", + row.table_name + ), + restore_date_time: row.created_at, + store: row.store.clone(), + spool_host: spool_host.map(str::to_owned), + owner_generation: 0, + } +} + +/// The target: a composite-key table with a GSI on `g`, an LSI on `l` and, +/// when asked, a vector index on `emb` (three dimensions). +fn target_input(name: &str, vector: bool) -> CreateTableInput { + let mut input: CreateTableInput = serde_json::from_value(serde_json::json!({ + "TableName": name, + "KeySchema": [ + {"AttributeName": "pk", "KeyType": "HASH"}, + {"AttributeName": "sk", "KeyType": "RANGE"} + ], + "AttributeDefinitions": [ + {"AttributeName": "pk", "AttributeType": "S"}, + {"AttributeName": "sk", "AttributeType": "N"}, + {"AttributeName": "g", "AttributeType": "S"}, + {"AttributeName": "l", "AttributeType": "S"} + ], + "BillingMode": "PAY_PER_REQUEST", + "GlobalSecondaryIndexes": [{ + "IndexName": "gix", + "KeySchema": [{"AttributeName": "g", "KeyType": "HASH"}], + "Projection": {"ProjectionType": "KEYS_ONLY"} + }], + "LocalSecondaryIndexes": [{ + "IndexName": "lix", + "KeySchema": [ + {"AttributeName": "pk", "KeyType": "HASH"}, + {"AttributeName": "l", "KeyType": "RANGE"} + ], + "Projection": {"ProjectionType": "ALL"} + }] + })) + .expect("input"); + if vector { + input.vector_indexes = Some(vec![VectorIndexSpecification { + index_name: "vix".to_owned(), + dimensions: 3, + distance_function: DistanceFunction::Cosine, + vector_attribute: VectorAttribute { + attribute_name: "emb".to_owned(), + }, + search_schema: None, + projection: Some(Projection { + projection_type: ProjectionType::All, + non_key_attributes: None, + }), + }]); + } + input +} + +/// Item `i` of the fixture: every item has `pk` and `sk`, every second one +/// `g` (the GSI's key), every third one `l` (the LSI's), every fifth one +/// `emb` (the vector). +fn item(i: usize) -> Item { + let mut item = + serde_json::json!({"pk": {"S": format!("p{}", i % 7)}, "sk": {"N": i.to_string()}}); + if i.is_multiple_of(2) { + item["g"] = serde_json::json!({"S": format!("g{}", i % 4)}); + } + if i.is_multiple_of(3) { + item["l"] = serde_json::json!({"S": format!("l{i}")}); + } + if i.is_multiple_of(5) { + item["emb"] = serde_json::json!({"L": [{"N": "0.1"}, {"N": "0.2"}, {"N": "0.3"}]}); + } + serde_json::from_value(item).expect("item") +} + +fn items(range: std::ops::Range) -> Vec { + range.map(item).collect() +} + +fn stream(items: Vec) -> BoxStream<'static, Result> { + Box::pin(futures::stream::iter(items.into_iter().map(Ok))) +} + +fn file(index: u32, start_at: u64) -> BulkLoadFile { + BulkLoadFile { + file_index: index, + start_at, + reapplying: false, + } +} + +fn key_of(desc: &TableDescription) -> RestoreTargetKey { + RestoreTargetKey { + account_id: ACCOUNT.to_owned(), + table_name: desc.table_name.clone(), + table_id: desc.table_id.clone(), + } +} + +/// Register a restore of `row` into `target` and return the description, its +/// key and the claim. +async fn register( + engine: &SqliteEngine, + row: &BackupRow, + target: &str, + vector: bool, + spool_host: Option<&str>, +) -> (TableDescription, RestoreTargetKey, Box) { + let (desc, claim) = engine + .claim_restore( + ACCOUNT, + target_input(target, vector), + ®istration(row, spool_host), + ) + .await + .expect("register the restore"); + let key = key_of(&desc); + (desc, key, claim) +} + +/// A claim on the record under `generation`, as a job whose record passed to +/// another owner still holds one. +fn stale_claim(engine: &SqliteEngine, table_id: &str, generation: u64) -> SqliteJobClaim { + SqliteJobClaim::for_restore(engine, table_id, generation, JobPermit::detached()) +} + +// --- reads -------------------------------------------------------------------- + +async fn table_status(engine: &SqliteEngine, name: &str) -> Option { + sqlx::query_scalar("SELECT table_status FROM tables WHERE account_id = ? AND table_name = ?") + .bind(ACCOUNT) + .bind(name) + .fetch_optional(&engine.pool) + .await + .expect("table status") +} + +async fn record(engine: &SqliteEngine, key: &RestoreTargetKey) -> RestoreJobRow { + engine + .read_restore_job(key) + .await + .expect("read the record") + .expect("the record exists") +} + +/// `(status, kind, owner_generation, heartbeat_at, spool_host, file_index, +/// items_applied, completed_at)` of the record, straight from the table. +type RecordColumns = ( + String, + String, + i64, + String, + Option, + Option, + Option, + Option, +); + +async fn record_columns(engine: &SqliteEngine, table_id: &str) -> RecordColumns { + sqlx::query_as( + "SELECT status, kind, owner_generation, heartbeat_at, spool_host, file_index, \ + items_applied, completed_at FROM table_restores WHERE table_id = ?", + ) + .bind(table_id) + .fetch_one(&engine.pool) + .await + .expect("read table_restores") +} + +async fn rows_in(engine: &SqliteEngine, physical: &str) -> i64 { + sqlx::query_scalar(&format!("SELECT COUNT(*) FROM {physical}")) + .fetch_one(&engine.pool) + .await + .expect("count rows") +} + +async fn base_rows(engine: &SqliteEngine, table_id: &str) -> i64 { + rows_in(engine, &crate::data::data_table_name(table_id)).await +} + +/// `(index_name, index_id, index_status)` of the table's secondary indexes. +async fn indexes(engine: &SqliteEngine, table_id: &str) -> Vec<(String, String, String)> { + sqlx::query_as( + "SELECT index_name, index_id, index_status FROM indexes WHERE table_id = ? \ + ORDER BY index_name", + ) + .bind(table_id) + .fetch_all(&engine.pool) + .await + .expect("read indexes") +} + +async fn index_rows(engine: &SqliteEngine, table_id: &str, name: &str) -> i64 { + let id = indexes(engine, table_id) + .await + .into_iter() + .find(|(n, _, _)| n == name) + .map(|(_, id, _)| id) + .expect("the index exists"); + rows_in(engine, &crate::data::index_table_name(&id)).await +} + +/// `(index_id, index_status, backfilling)` of the table's vector indexes. +async fn vector_indexes( + engine: &SqliteEngine, + table_id: &str, +) -> Vec<(String, String, Option)> { + sqlx::query_as( + "SELECT index_id, index_status, backfilling FROM vector_indexes WHERE table_id = ?", + ) + .bind(table_id) + .fetch_all(&engine.pool) + .await + .expect("read vector_indexes") +} + +async fn vector_rows(engine: &SqliteEngine, table_id: &str) -> i64 { + let (id, _, _) = vector_indexes(engine, table_id) + .await + .pop() + .expect("the vector index exists"); + rows_in(engine, &crate::data::vector_table_name(table_id, &id)).await +} + +async fn progress(engine: &SqliteEngine, table_id: &str) -> (Option, Option) { + let (_, _, _, _, _, file_index, items_applied, _) = record_columns(engine, table_id).await; + (file_index, items_applied) +} + +async fn tables_named(engine: &SqliteEngine, name: &str) -> i64 { + sqlx::query_scalar("SELECT COUNT(*) FROM tables WHERE account_id = ? AND table_name = ?") + .bind(ACCOUNT) + .bind(name) + .fetch_one(&engine.pool) + .await + .expect("count tables") +} + +// --- the registration ----------------------------------------------------------- + +/// `claim_restore` takes the name's slot and creates the CREATING target with +/// no scheduled transition and its RUNNING `manifest` record carrying the +/// store, the spool host and the claim's generation in one transaction; the +/// description carries the `RestoreSummary`; the record reads back through +/// `read_restore_job` and `list_restore_jobs`; the heartbeat confirms and +/// refreshes the row; the secondary indexes are ACTIVE from creation and the +/// vector index CREATING with its table; dropping the claim frees the slot. +#[tokio::test] +async fn claim_restore_registers_the_target_with_its_record() { + let engine = engine().await; + let row = available_backup(&engine, "Music").await; + let slot = restore_slot_key(ACCOUNT, "Restored"); + assert!(!engine.job_slots.holds(&slot)); + let (desc, key, claim) = register(&engine, &row, "Restored", true, Some("host-a:/spool")).await; + assert_eq!(desc.table_status, TableStatus::Creating); + let summary = desc.restore_summary.as_ref().expect("a RestoreSummary"); + assert_eq!( + summary.source_backup_arn.as_deref(), + Some(row.backup_arn.as_str()) + ); + assert_eq!( + summary.source_table_arn.as_deref(), + Some(format!("arn:aws:dynamodb:{REGION}:{ACCOUNT}:table/Music").as_str()) + ); + assert_eq!(summary.restore_date_time, row.created_at); + assert!(summary.restore_in_progress); + assert!( + engine.job_slots.holds(&slot), + "the claim holds the name's slot" + ); + + let (status, kind, generation, heartbeat_at, spool_host, file_index, items_applied, done) = + record_columns(&engine, &key.table_id).await; + assert_eq!((status.as_str(), kind.as_str()), ("RUNNING", "manifest")); + assert_eq!(generation, i64::try_from(claim.generation()).unwrap()); + assert!(heartbeat_at.starts_with("20"), "{heartbeat_at}"); + assert_eq!(spool_host.as_deref(), Some("host-a:/spool")); + assert_eq!((file_index, items_applied, done), (None, None, None)); + let transition: Option = + sqlx::query_scalar("SELECT status_transition_at FROM tables WHERE table_id = ?") + .bind(&key.table_id) + .fetch_one(&engine.pool) + .await + .expect("read the transition"); + assert_eq!( + transition, None, + "no scheduled transition: the job flips the table" + ); + + let job = record(&engine, &key).await; + assert_eq!(job.target, key); + assert_eq!(job.source_backup_arn, row.backup_arn); + assert_eq!(job.kind, RestoreKind::Manifest); + assert_eq!(job.status, JobStatus::Running); + assert_eq!(job.owner_generation, claim.generation()); + assert_eq!(job.store, Some(store())); + assert_eq!(job.spool_host.as_deref(), Some("host-a:/spool")); + assert_eq!((job.file_index, job.items_applied), (None, None)); + assert_eq!( + (job.failure_reason.as_deref(), job.completed_at), + (None, None) + ); + let running = engine + .list_restore_jobs(Some(JobStatus::Running)) + .await + .expect("list"); + assert_eq!(running, vec![job.clone()]); + assert_eq!( + engine.list_restore_jobs(None).await.expect("list all"), + vec![job.clone()] + ); + assert!( + engine + .list_restore_jobs(Some(JobStatus::Done)) + .await + .expect("list done") + .is_empty() + ); + assert_eq!( + engine.backup_in_use(&row.key()).await.expect("in use"), + Some("Restored".to_owned()), + "the backup is in use while its target is CREATING" + ); + assert!( + engine + .list_dead_restore_jobs(STALE) + .await + .expect("dead") + .is_empty(), + "a fresh record is not dead" + ); + + tokio::time::sleep(Duration::from_millis(20)).await; + assert_eq!( + claim.heartbeat().await.expect("heartbeat"), + Heartbeat::Confirmed + ); + let after = record(&engine, &key).await; + assert!( + after.heartbeat_at > job.heartbeat_at, + "{} > {}", + after.heartbeat_at, + job.heartbeat_at + ); + assert_eq!( + after.owner_generation, + claim.generation(), + "the heartbeat left the rest of the record" + ); + let read = claim.read().await.expect("read through the claim"); + assert_eq!(read.status, JobStatus::Running); + assert_eq!(read.owner_generation, claim.generation()); + + for (name, _, status) in indexes(&engine, &key.table_id).await { + assert_eq!(status, "ACTIVE", "secondary index {name} is created ACTIVE"); + } + let vectors = vector_indexes(&engine, &key.table_id).await; + assert_eq!(vectors.len(), 1); + assert_eq!( + (vectors[0].1.as_str(), vectors[0].2), + ("CREATING", None), + "the vector index is CREATING with its table and not a backfill" + ); + + drop(claim); + assert!( + !engine.job_slots.holds(&slot), + "dropping the claim frees the slot" + ); + assert_eq!( + table_status(&engine, "Restored").await.as_deref(), + Some("CREATING") + ); +} + +/// DeleteBackup winning: `start_backup_delete` moves the row to DELETING and a +/// registration after it finds the backup gone, creates no target, keeps no +/// slot and writes no record. The two also serialize when they contend on +/// the write lock: started together behind it, the outcomes are one of the +/// two consistent pairs and never a target beside a DELETING row. +#[tokio::test] +async fn delete_backup_winning_before_the_registration_leaves_no_target() { + let engine = engine().await; + let row = available_backup(&engine, "Music").await; + assert_eq!( + engine + .start_backup_delete(&row.key()) + .await + .expect("delete"), + DeleteStart::Deleting + ); + let refused = engine + .claim_restore( + ACCOUNT, + target_input("Restored", false), + ®istration(&row, None), + ) + .await + .err() + .expect("the registration finds the backup gone"); + assert!( + is_backup_not_found(&refused), + "the registration finds the backup gone: {refused:?}" + ); + assert_eq!(tables_named(&engine, "Restored").await, 0, "no target"); + assert!( + !engine + .job_slots + .holds(&restore_slot_key(ACCOUNT, "Restored")) + ); + assert!( + engine + .list_restore_jobs(None) + .await + .expect("list") + .is_empty(), + "no record" + ); + + // Contended: both behind the write lock, released together. + let engine = Arc::new(engine); + for attempt in 0..6 { + let row = available_backup(&engine, &format!("Music{attempt}")).await; + let name = format!("Restored{attempt}"); + let held = engine.write_lock.lock().await; + let registering = tokio::spawn({ + let engine = Arc::clone(&engine); + let row = row.clone(); + let name = name.clone(); + async move { + engine + .claim_restore( + ACCOUNT, + target_input(&name, false), + ®istration(&row, None), + ) + .await + } + }); + let deleting = tokio::spawn({ + let engine = Arc::clone(&engine); + let key = row.key(); + async move { engine.start_backup_delete(&key).await } + }); + tokio::time::sleep(Duration::from_millis(20)).await; + assert!(!registering.is_finished() && !deleting.is_finished()); + drop(held); + let registered = tokio::time::timeout(Duration::from_secs(10), registering) + .await + .expect("the registration ends") + .expect("no panic"); + let deleted = tokio::time::timeout(Duration::from_secs(10), deleting) + .await + .expect("the delete ends") + .expect("no panic") + .expect("start_backup_delete"); + match (registered, deleted) { + (Ok((desc, claim)), DeleteStart::InUse(table)) => { + assert_eq!(table, name, "attempt {attempt}: the registration won"); + assert_eq!( + table_status(&engine, &name).await.as_deref(), + Some("CREATING") + ); + drop(claim); + assert!( + engine + .remove_restore_target(&key_of(&desc)) + .await + .expect("remove") + ); + } + (Err(e), DeleteStart::Deleting) => { + assert!(is_backup_not_found(&e), "attempt {attempt}: {e:?}"); + assert_eq!(tables_named(&engine, &name).await, 0, "attempt {attempt}"); + } + (registered, deleted) => panic!( + "attempt {attempt}: the two outcomes are not one ordering: {:?} / {deleted:?}", + registered.map(|(d, _)| d.table_status) + ), + } + } +} + +/// The registration winning: the backup is in use (`start_backup_delete` +/// answers the target's name), a second registration of the backup into +/// another name is `BackupInUse` naming the target and leaves no slot, and +/// the row stays AVAILABLE. +#[tokio::test] +async fn a_registration_winning_before_delete_backup_keeps_the_backup_in_use() { + let engine = engine().await; + let row = available_backup(&engine, "Music").await; + let (_desc, _key, _claim) = register(&engine, &row, "First", false, None).await; + assert_eq!( + engine + .start_backup_delete(&row.key()) + .await + .expect("delete"), + DeleteStart::InUse("First".to_owned()) + ); + let second = engine + .claim_restore( + ACCOUNT, + target_input("Second", false), + ®istration(&row, None), + ) + .await + .err() + .expect("a second registration of the backup is refused"); + match second { + // The service's text names the backup, not the target. + StorageError::BackupInUse(m) => assert_eq!(m, backup_in_use_message(&row.backup_arn)), + other => panic!("{other:?}"), + } + assert_eq!(tables_named(&engine, "Second").await, 0); + assert!(!engine.job_slots.holds(&restore_slot_key(ACCOUNT, "Second"))); + let arn = extenddb_core::backup_format::BackupArn::parse(&row.backup_arn).unwrap(); + assert_eq!( + engine + .read_backup_row(&arn) + .await + .expect("read") + .expect("the row") + .status, + BackupStatus::Available + ); +} + +/// A taken name: an existing table is `TableAlreadyExists`; a CREATING +/// restore target is `TableBeingRestored` naming its backup, both while its +/// claim holds the slot and after the claim is gone (the committed row). +#[tokio::test] +async fn a_taken_name_is_refused_as_create_table_or_as_a_restore_in_progress() { + let engine = engine().await; + let row = available_backup(&engine, "Music").await; + engine + .create_table_impl(ACCOUNT, target_input("Plain", false), false) + .await + .expect("a table"); + let taken = engine + .claim_restore( + ACCOUNT, + target_input("Plain", false), + ®istration(&row, None), + ) + .await + .err() + .expect("the name is taken"); + assert!( + matches!(&taken, StorageError::TableAlreadyExists(n) if n == "Plain"), + "{taken:?}" + ); + assert!(!engine.job_slots.holds(&restore_slot_key(ACCOUNT, "Plain"))); + + let other = available_backup(&engine, "Other").await; + let (_desc, _key, claim) = register(&engine, &row, "Target", false, None).await; + async fn refuse(engine: &SqliteEngine, other: &BackupRow) -> StorageError { + engine + .claim_restore( + ACCOUNT, + target_input("Target", false), + ®istration(other, None), + ) + .await + .err() + .expect("the name is a restore target") + } + // While the first claim holds the slot: refused on the slot, the + // committed CREATING row naming the backup. + match refuse(&engine, &other).await { + StorageError::TableBeingRestored { + table_name, + backup_arn, + } => { + assert_eq!(table_name, "Target"); + assert_eq!(backup_arn, row.backup_arn, "the slot held"); + } + other => panic!("slot held: {other:?}"), + } + // The claim gone, the target still CREATING: refused in the + // registration transaction on the committed row. + drop(claim); + assert!(!engine.job_slots.holds(&restore_slot_key(ACCOUNT, "Target"))); + match refuse(&engine, &other).await { + StorageError::TableBeingRestored { + table_name, + backup_arn, + } => { + assert_eq!(table_name, "Target"); + assert_eq!( + backup_arn, row.backup_arn, + "the slot free, the row committed" + ); + } + other => panic!("slot free: {other:?}"), + } + assert!( + !engine.job_slots.holds(&restore_slot_key(ACCOUNT, "Target")), + "a refused registration returns the slot" + ); +} + +// --- the takeover --------------------------------------------------------------- + +/// `claim_restore_dead`: refused while the owner's claim holds the slot, +/// whatever the heartbeat says; a record whose slot no task holds is taken at +/// once, fresh heartbeat or not, with a fresh generation, the claimant's +/// `spool_host` and `heartbeat_at` refreshed to the database's clock (an +/// aged record is live to the dead listing at once); a third taker is +/// refused; the old generation's heartbeat is lost and its writes match +/// nothing; the successor's FAILED write matches and leaves the progress; a +/// DONE record, a legacy record and a missing one are `None` and hold no +/// slot. +#[tokio::test] +async fn a_dead_record_is_taken_over_with_the_claimants_spool_and_a_fresh_heartbeat() { + let engine = engine().await; + let row = available_backup(&engine, "Music").await; + let (_desc, key, claim) = register(&engine, &row, "Restored", false, Some("host-a:/a")).await; + let old_generation = claim.generation(); + engine + .bulk_load(&key, claim.as_ref(), file(0, 0), stream(items(0..10))) + .await + .expect("a first file"); + assert_eq!(progress(&engine, &key.table_id).await, (Some(0), Some(10))); + assert!( + engine + .claim_restore_dead(&key, STALE, Some("host-b:/b")) + .await + .expect("claim") + .is_none(), + "a live claim's slot refuses the takeover" + ); + // The dead owner's last beat, an hour ago: the record is in the dead + // listing until the takeover refreshes it. + sqlx::query( + "UPDATE table_restores SET heartbeat_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now', \ + '-1 hour') WHERE table_id = ?", + ) + .bind(&key.table_id) + .execute(&engine.pool) + .await + .expect("age the record"); + let before = record_columns(&engine, &key.table_id).await; + assert_eq!(before.4.as_deref(), Some("host-a:/a")); + + drop(claim); + assert_eq!( + engine + .list_dead_restore_jobs(STALE) + .await + .expect("dead") + .len(), + 1, + "the aged record is listed before the takeover" + ); + let successor = engine + .claim_restore_dead(&key, STALE, Some("host-b:/b")) + .await + .expect("claim") + .expect("a record whose slot no task holds is dead at once"); + assert_ne!(successor.generation(), old_generation); + let after = record_columns(&engine, &key.table_id).await; + assert_eq!(after.0, "RUNNING"); + assert_eq!( + after.2, + i64::try_from(successor.generation()).unwrap(), + "the successor's generation" + ); + assert!( + after.3 > before.3, + "the takeover refreshed heartbeat_at: {} was {}", + after.3, + before.3 + ); + let age_secs: f64 = sqlx::query_scalar( + "SELECT (julianday('now') - julianday(heartbeat_at)) * 86400.0 FROM table_restores \ + WHERE table_id = ?", + ) + .bind(&key.table_id) + .fetch_one(&engine.pool) + .await + .expect("the record's age"); + assert!( + age_secs.abs() < 5.0, + "the refreshed heartbeat is the database's now: {age_secs} s old" + ); + assert!( + engine + .list_dead_restore_jobs(STALE) + .await + .expect("dead") + .is_empty(), + "the taken-over record is live to the listing at once" + ); + assert_eq!( + after.4.as_deref(), + Some("host-b:/b"), + "the claimant's spool" + ); + assert_eq!( + (after.5, after.6), + (Some(0), Some(10)), + "the progress stays" + ); + assert!( + engine + .job_slots + .holds(&restore_slot_key(ACCOUNT, "Restored")) + ); + assert!( + engine + .claim_restore_dead(&key, STALE, None) + .await + .expect("claim") + .is_none(), + "a third taker is refused" + ); + + let old = stale_claim(&engine, &key.table_id, old_generation); + assert_eq!(old.heartbeat().await.expect("beat"), Heartbeat::Lost); + assert_eq!( + old.set_status( + JobStatus::Running, + JobStatus::Failed, + &StatusFields { + failure_reason: Some("old owner".to_owned()), + completed: true, + ..StatusFields::default() + } + ) + .await + .expect("write"), + WriteOutcome::NotMatched + ); + assert_eq!( + engine + .finish_restore(&key, &old, 10, 100) + .await + .expect("finish"), + Finished::NotMatched(format!( + "the job record of Restored is not RUNNING under generation {old_generation}, or \ + the table is not CREATING" + )) + ); + assert_eq!( + table_status(&engine, "Restored").await.as_deref(), + Some("CREATING") + ); + assert_eq!( + successor.heartbeat().await.expect("beat"), + Heartbeat::Confirmed + ); + assert_eq!( + successor + .set_status( + JobStatus::Running, + JobStatus::Failed, + &StatusFields { + item_count: Some(99), + failure_reason: Some("target_not_empty".to_owned()), + completed: true, + ..StatusFields::default() + } + ) + .await + .expect("write"), + WriteOutcome::Matched + ); + let failed = record_columns(&engine, &key.table_id).await; + assert_eq!(failed.0, "FAILED"); + assert!(failed.7.is_some(), "the FAILED write stamps completed_at"); + assert_eq!( + (failed.5, failed.6), + (Some(0), Some(10)), + "a status write leaves the progress; item_count is not items_applied" + ); + let read = successor.read().await.expect("read"); + assert_eq!(read.status, JobStatus::Failed); + assert_eq!(read.failure_reason.as_deref(), Some("target_not_empty")); + assert_eq!( + successor + .set_status( + JobStatus::Running, + JobStatus::Failed, + &StatusFields::default() + ) + .await + .expect("write"), + WriteOutcome::NotMatched, + "a FAILED record is not RUNNING" + ); + assert!( + engine + .list_dead_restore_jobs(STALE) + .await + .expect("dead") + .is_empty(), + "a fresh FAILED record is not listed yet" + ); + drop(successor); + assert!( + !engine + .job_slots + .holds(&restore_slot_key(ACCOUNT, "Restored")), + "dropping the successor frees the slot" + ); + assert!( + engine + .claim_restore_dead(&key, STALE, None) + .await + .expect("claim") + .is_none(), + "a FAILED record is not taken over" + ); + assert!( + !engine + .job_slots + .holds(&restore_slot_key(ACCOUNT, "Restored")), + "a refused takeover holds no slot" + ); + let listed = engine + .list_restore_jobs(Some(JobStatus::Failed)) + .await + .expect("list"); + assert_eq!(listed.len(), 1); + assert_eq!( + listed[0].failure_reason.as_deref(), + Some("target_not_empty") + ); + assert_eq!(listed[0].spool_host.as_deref(), Some("host-b:/b")); + assert!(listed[0].completed_at.is_some()); + + // A legacy record is never taken over: the reconciler fails and removes it. + let legacy = engine + .create_table_impl(ACCOUNT, target_input("Legacy", false), true) + .await + .expect("a CREATING table"); + sqlx::query(&format!( + "INSERT INTO table_restores (table_id, source_backup_arn, source_table_arn, \ + restore_date_time, status, kind, owner_generation, heartbeat_at) \ + VALUES (?, 'arn:legacy', 'arn:aws:dynamodb:us-east-1:123456789012:table/Music', {NOW}, \ + 'RUNNING', 'legacy', 7, '1970-01-01T00:00:00.000Z')", + NOW = crate::backup_catalog::NOW + )) + .bind(&legacy.table_id) + .execute(&engine.pool) + .await + .expect("a legacy record"); + let legacy_key = key_of(&legacy); + assert!( + engine + .claim_restore_dead(&legacy_key, STALE, None) + .await + .expect("claim") + .is_none(), + "a legacy record is not taken over" + ); + assert!(!engine.job_slots.holds(&restore_slot_key(ACCOUNT, "Legacy"))); + let dead = engine.list_dead_restore_jobs(STALE).await.expect("dead"); + assert_eq!(dead.len(), 1, "{dead:?}"); + assert_eq!(dead[0].kind, RestoreKind::Legacy); + assert_eq!(dead[0].target, legacy_key); + assert_eq!( + (dead[0].store.clone(), dead[0].spool_host.clone()), + (None, None) + ); + let missing = RestoreTargetKey { + account_id: ACCOUNT.to_owned(), + table_name: "Nothing".to_owned(), + table_id: "no-such-id".to_owned(), + }; + assert!( + engine + .claim_restore_dead(&missing, STALE, None) + .await + .expect("claim") + .is_none() + ); + assert!( + !engine + .job_slots + .holds(&restore_slot_key(ACCOUNT, "Nothing")) + ); + assert!( + engine + .read_restore_job(&missing) + .await + .expect("read") + .is_none() + ); +} + +// --- the load ------------------------------------------------------------------- + +/// `bulk_load` writes the base rows, every secondary index's rows and the +/// vector rows across the batch boundary, recording `(file_index, +/// items_applied)` on the record; a file resumed at `start_at` records the +/// running total; a key the backup holds twice, within a file or against an +/// earlier file, fails the file with nothing of its batch committed and the +/// progress untouched; an `Err` item fails the file the same way; another +/// account's key is refused before any write. +#[tokio::test] +async fn bulk_load_writes_each_batch_with_its_indexes_and_progress() { + let engine = engine().await; + let row = available_backup(&engine, "Music").await; + let (_desc, key, claim) = register(&engine, &row, "Restored", true, None).await; + let n = RESTORE_BATCH_ITEMS + 500; + let applied = engine + .bulk_load(&key, claim.as_ref(), file(0, 0), stream(items(0..n))) + .await + .expect("load file 0"); + assert_eq!(applied as usize, n); + assert_eq!(base_rows(&engine, &key.table_id).await as usize, n); + assert_eq!( + index_rows(&engine, &key.table_id, "gix").await as usize, + n / 2 + ); + assert_eq!( + index_rows(&engine, &key.table_id, "lix").await as usize, + n.div_ceil(3) + ); + assert_eq!(vector_rows(&engine, &key.table_id).await as usize, n / 5); + assert_eq!( + progress(&engine, &key.table_id).await, + (Some(0), Some(n as i64)) + ); + assert_eq!( + engine.count_restore_target(&key).await.expect("count") as usize, + n + ); + let listed = record(&engine, &key).await; + assert_eq!( + (listed.file_index, listed.items_applied), + (Some(0), Some(n as u64)) + ); + + // A file resumed after five of its items: the record carries the total. + let applied = engine + .bulk_load(&key, claim.as_ref(), file(1, 5), stream(items(n..n + 3))) + .await + .expect("load the rest of file 1"); + assert_eq!(applied, 3); + assert_eq!(progress(&engine, &key.table_id).await, (Some(1), Some(8))); + assert_eq!(base_rows(&engine, &key.table_id).await as usize, n + 3); + + // A key twice within a file. + let mut twice = items(n + 3..n + 6); + twice.push(item(n + 4)); + let e = engine + .bulk_load(&key, claim.as_ref(), file(2, 0), stream(twice)) + .await + .expect_err("a key twice fails the file"); + match &e { + StorageError::Validation(m) => { + assert!( + m.starts_with("holds a primary key the target already has"), + "the backend names no file (the engine prefixes the file's key): {m}" + ); + assert!(m.contains("the backup holds a key twice"), "{m}"); + } + other => panic!("{other:?}"), + } + assert_eq!( + base_rows(&engine, &key.table_id).await as usize, + n + 3, + "nothing of the failed batch committed" + ); + assert_eq!( + index_rows(&engine, &key.table_id, "gix").await as usize, + (n + 3).div_ceil(2), + "no index row of the failed batch" + ); + assert_eq!(progress(&engine, &key.table_id).await, (Some(1), Some(8))); + + // A key of file 0 in file 3. + let mut across = items(n + 6..n + 8); + across.push(item(1)); + let e = engine + .bulk_load(&key, claim.as_ref(), file(3, 0), stream(across)) + .await + .expect_err("a key of an earlier file fails the file"); + assert!( + matches!(&e, StorageError::Validation(m) if m.starts_with("holds a primary key the target already has")), + "{e:?}" + ); + assert_eq!(base_rows(&engine, &key.table_id).await as usize, n + 3); + assert_eq!(progress(&engine, &key.table_id).await, (Some(1), Some(8))); + + // An `Err` item mid-stream: the error, nothing committed. + let failing: BoxStream<'static, Result> = Box::pin( + futures::stream::iter(items(n + 10..n + 12).into_iter().map(Ok)).chain( + futures::stream::once(async { + Err(StorageError::Validation( + "item cannot be restored".to_owned(), + )) + }), + ), + ); + let e = engine + .bulk_load(&key, claim.as_ref(), file(4, 0), failing) + .await + .expect_err("an Err item fails the file"); + assert!( + matches!(&e, StorageError::Validation(m) if m == "item cannot be restored"), + "{e:?}" + ); + assert_eq!(base_rows(&engine, &key.table_id).await as usize, n + 3); + assert_eq!(progress(&engine, &key.table_id).await, (Some(1), Some(8))); + + // Another account's key names no restore target of this account. + let foreign = RestoreTargetKey { + account_id: "111111111111".to_owned(), + ..key.clone() + }; + assert!(matches!( + engine.count_restore_target(&foreign).await, + Err(StorageError::TableNotFound(_)) + )); + assert!(matches!( + engine + .bulk_load(&foreign, claim.as_ref(), file(5, 0), stream(items(0..1))) + .await, + Err(StorageError::TableNotFound(_)) + )); + assert_eq!(base_rows(&engine, &key.table_id).await as usize, n + 3); +} + +/// A resume from a mid-file `items_applied`: the engine hands the rest of the +/// file with `start_at` set, and exactly those items land, the record's total +/// becoming the file's count. +#[tokio::test] +async fn a_resume_from_a_mid_file_progress_applies_exactly_the_rest() { + let engine = engine().await; + let row = available_backup(&engine, "Music").await; + let (_desc, key, claim) = register(&engine, &row, "Restored", false, None).await; + let applied = engine + .bulk_load(&key, claim.as_ref(), file(0, 0), stream(items(0..60))) + .await + .expect("the first run's batch"); + assert_eq!(applied, 60); + assert_eq!(progress(&engine, &key.table_id).await, (Some(0), Some(60))); + // The successor: `start_at` 60, the stream from item 60. + let applied = engine + .bulk_load(&key, claim.as_ref(), file(0, 60), stream(items(60..100))) + .await + .expect("the resumed rest of the file"); + assert_eq!(applied, 40, "exactly the rest"); + assert_eq!(progress(&engine, &key.table_id).await, (Some(0), Some(100))); + assert_eq!(engine.count_restore_target(&key).await.expect("count"), 100); + assert_eq!(index_rows(&engine, &key.table_id, "gix").await, 50); + assert_eq!(index_rows(&engine, &key.table_id, "lix").await, 34); + + // An empty file (a resume whose `start_at` is the file's whole count, + // or a data file with no items) records its progress as complete. + let applied = engine + .bulk_load(&key, claim.as_ref(), file(1, 0), stream(Vec::new())) + .await + .expect("an empty file"); + assert_eq!(applied, 0); + assert_eq!( + progress(&engine, &key.table_id).await, + (Some(1), Some(0)), + "an empty file's progress is recorded" + ); + let applied = engine + .bulk_load(&key, claim.as_ref(), file(2, 7), stream(Vec::new())) + .await + .expect("a file resumed at its end"); + assert_eq!(applied, 0); + assert_eq!( + progress(&engine, &key.table_id).await, + (Some(2), Some(7)), + "`start_at` is the recorded total of an empty rest" + ); +} + +/// A batch closes at the first item whose text takes it past +/// `RESTORE_BATCH_BYTES`, long before the count cap: a file of 100 items of +/// about 350 KB each (35 MB of text against the 32 MiB bound) is written in +/// two transactions, the first committed with its progress before the +/// stream hands over its last item. +#[tokio::test] +async fn a_batch_closes_at_the_byte_bound_before_the_count_cap() { + let engine = engine().await; + let row = available_backup(&engine, "Music").await; + let (_desc, key, claim) = register(&engine, &row, "Restored", false, None).await; + const N: usize = 100; + const TEXT: usize = 350_000; + const { + assert!(N < RESTORE_BATCH_ITEMS); + // 100 items cross the byte bound once. + assert!(N * TEXT > RESTORE_BATCH_BYTES && (N - 2) * TEXT < 2 * RESTORE_BATCH_BYTES); + } + let big = |i: usize| -> Item { + let mut it = item(i); + it.insert("v".to_owned(), AttributeValue::S("x".repeat(TEXT))); + it + }; + // The stream reads the record when asked for the last item: by then the + // byte bound has closed and committed the first batch. + let pool = engine.pool.clone(); + let table_id = key.table_id.clone(); + let seen = Arc::new(std::sync::Mutex::new(None)); + let seen_by_stream = Arc::clone(&seen); + let bounded: BoxStream<'static, Result> = + Box::pin(futures::stream::unfold(0usize, move |i| { + let pool = pool.clone(); + let table_id = table_id.clone(); + let seen = Arc::clone(&seen_by_stream); + async move { + if i >= N { + return None; + } + if i == N - 1 { + let progress: (Option, Option) = sqlx::query_as( + "SELECT file_index, items_applied FROM table_restores WHERE table_id = ?", + ) + .bind(&table_id) + .fetch_one(&pool) + .await + .expect("progress"); + *seen.lock().expect("seen") = Some(progress); + } + Some((Ok(big(i)), i + 1)) + } + })); + let applied = engine + .bulk_load(&key, claim.as_ref(), file(0, 0), bounded) + .await + .expect("load"); + assert_eq!(applied as usize, N); + let (file_index, items_applied) = seen + .lock() + .expect("seen") + .expect("the stream read the record at its last item"); + assert_eq!( + file_index, + Some(0), + "the first batch committed with its progress" + ); + let first_batch = items_applied.expect("the first batch's total") as usize; + assert!( + 0 < first_batch && first_batch < N, + "the first batch closed at the byte bound with {first_batch} of {N} items" + ); + assert!( + first_batch * TEXT < RESTORE_BATCH_BYTES + TEXT + 1_000 * N, + "the batch closed at the first item past the bound: {first_batch} items" + ); + assert_eq!( + progress(&engine, &key.table_id).await, + (Some(0), Some(N as i64)) + ); + assert_eq!(base_rows(&engine, &key.table_id).await as usize, N); +} + +/// A claim lost between two batches of one file: the second batch's progress +/// write matches no record, the batch rolls back whole (base, index and +/// vector rows), and `bulk_load` fails as a lost claim; the first batch and +/// its progress stand. +#[tokio::test] +async fn a_lost_claim_mid_file_rolls_its_batch_back() { + let engine = engine().await; + let row = available_backup(&engine, "Music").await; + let (_desc, key, claim) = register(&engine, &row, "Restored", true, None).await; + let n = RESTORE_BATCH_ITEMS + 200; + // The stream hands the first batch, then takes the record away before + // the first item of the second. + let pool = engine.pool.clone(); + let table_id = key.table_id.clone(); + let taken: BoxStream<'static, Result> = + Box::pin(futures::stream::unfold(0usize, move |i| { + let pool = pool.clone(); + let table_id = table_id.clone(); + async move { + if i >= n { + return None; + } + if i == RESTORE_BATCH_ITEMS { + sqlx::query( + "UPDATE table_restores SET owner_generation = owner_generation + 1 \ + WHERE table_id = ?", + ) + .bind(&table_id) + .execute(&pool) + .await + .expect("another owner takes the record"); + } + Some((Ok(item(i)), i + 1)) + } + })); + let e = engine + .bulk_load(&key, claim.as_ref(), file(0, 0), taken) + .await + .expect_err("the lost claim ends the file"); + assert!( + matches!(&e, StorageError::Internal(m) if m.starts_with("claim lost")), + "{e:?}" + ); + assert_eq!( + base_rows(&engine, &key.table_id).await as usize, + RESTORE_BATCH_ITEMS, + "the second batch's base rows rolled back" + ); + assert_eq!( + index_rows(&engine, &key.table_id, "gix").await as usize, + RESTORE_BATCH_ITEMS / 2, + "the second batch's index rows rolled back" + ); + assert_eq!( + vector_rows(&engine, &key.table_id).await as usize, + RESTORE_BATCH_ITEMS / 5, + "the second batch's vector rows rolled back" + ); + assert_eq!( + progress(&engine, &key.table_id).await, + (Some(0), Some(RESTORE_BATCH_ITEMS as i64)), + "the first batch's progress stands" + ); + assert_eq!(claim.heartbeat().await.expect("beat"), Heartbeat::Lost); +} + +// --- the flip ------------------------------------------------------------------- + +/// `finish_restore`: a stale generation is `NotMatched` and changes nothing; +/// the claim's flips the record DONE with `completed_at` and the progress +/// left, the table ACTIVE with its counts and no transition, and every +/// secondary and vector index ACTIVE in one transaction; the heartbeat is +/// lost after DONE; a second flip is `NotMatched`; the backup is no longer +/// in use; `describe_table` reports ACTIVE with the count. +/// `remove_restore_target` of a CREATING target removes it with its vector +/// data table. +#[tokio::test] +async fn finish_restore_flips_the_table_and_every_index() { + let engine = engine().await; + let row = available_backup(&engine, "Music").await; + let (_desc, key, claim) = register(&engine, &row, "Restored", true, None).await; + engine + .bulk_load(&key, claim.as_ref(), file(0, 0), stream(items(0..30))) + .await + .expect("load"); + // A secondary index left CREATING by hand, beside the vector index the + // creation left CREATING; the vector row's `backfilling` set as a backfill + // in progress leaves it, so the flip's `backfilling = NULL` is observed + // (an ACTIVE row may not carry it: `chk_vector_active_has_no_backfilling`). + sqlx::query( + "UPDATE indexes SET index_status = 'CREATING' WHERE table_id = ? AND index_name = 'gix'", + ) + .bind(&key.table_id) + .execute(&engine.pool) + .await + .expect("a CREATING index"); + sqlx::query("UPDATE vector_indexes SET backfilling = 1 WHERE table_id = ?") + .bind(&key.table_id) + .execute(&engine.pool) + .await + .expect("a backfilling vector index"); + assert_eq!( + vector_indexes(&engine, &key.table_id) + .await + .iter() + .map(|(_, status, backfilling)| (status.as_str(), *backfilling)) + .collect::>(), + vec![("CREATING", Some(true))], + "the fixture's one vector index is CREATING and backfilling before the flip" + ); + + let stale = stale_claim(&engine, &key.table_id, claim.generation() + 1); + assert!(matches!( + engine + .finish_restore(&key, &stale, 30, 3000) + .await + .expect("finish"), + Finished::NotMatched(_) + )); + assert_eq!( + table_status(&engine, "Restored").await.as_deref(), + Some("CREATING") + ); + assert_eq!(record_columns(&engine, &key.table_id).await.0, "RUNNING"); + + assert_eq!( + engine + .finish_restore(&key, claim.as_ref(), 30, 3000) + .await + .expect("finish"), + Finished::Active + ); + let (status, _, _, _, _, file_index, items_applied, completed_at) = + record_columns(&engine, &key.table_id).await; + assert_eq!(status, "DONE"); + assert!(completed_at.is_some(), "DONE stamps completed_at"); + assert_eq!( + (file_index, items_applied), + (Some(0), Some(30)), + "the DONE write leaves the progress" + ); + let table: (String, i64, i64, Option) = sqlx::query_as( + "SELECT table_status, item_count, table_size_bytes, status_transition_at \ + FROM tables WHERE table_id = ?", + ) + .bind(&key.table_id) + .fetch_one(&engine.pool) + .await + .expect("the table row"); + assert_eq!(table, ("ACTIVE".to_owned(), 30, 3000, None)); + for (name, _, status) in indexes(&engine, &key.table_id).await { + assert_eq!(status, "ACTIVE", "every secondary index is ACTIVE ({name})"); + } + for (id, status, backfilling) in vector_indexes(&engine, &key.table_id).await { + assert_eq!( + (status.as_str(), backfilling), + ("ACTIVE", None), + "every vector index is ACTIVE with `backfilling` cleared ({id})" + ); + } + assert_eq!(claim.heartbeat().await.expect("beat"), Heartbeat::Lost); + assert!(matches!( + engine + .finish_restore(&key, claim.as_ref(), 30, 3000) + .await + .expect("finish"), + Finished::NotMatched(_) + )); + assert_eq!( + engine.backup_in_use(&row.key()).await.expect("in use"), + None + ); + let described = engine + .describe_table( + ACCOUNT, + extenddb_core::types::DescribeTableInput { + table_name: "Restored".to_owned(), + }, + ) + .await + .expect("describe"); + assert_eq!(described.table_status, TableStatus::Active); + assert_eq!(described.item_count, 30); + let done = engine + .list_restore_jobs(Some(JobStatus::Done)) + .await + .expect("list"); + assert_eq!(done.len(), 1); + assert!(done[0].completed_at.is_some()); + assert!( + matches!( + engine.count_restore_target(&key).await, + Err(StorageError::TableNotFound(_)) + ), + "an ACTIVE table is no longer a CREATING restore target" + ); + + // Removal of another target: the table, its record, its index and + // vector data tables. + let other = available_backup(&engine, "Other").await; + let (desc, other_key, other_claim) = register(&engine, &other, "Gone", true, None).await; + engine + .bulk_load( + &other_key, + other_claim.as_ref(), + file(0, 0), + stream(items(0..10)), + ) + .await + .expect("load"); + let (vector_id, _, _) = vector_indexes(&engine, &other_key.table_id) + .await + .pop() + .unwrap(); + let data_tables = || async { + sqlx::query_scalar::<_, i64>( + "SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name IN (?, ?)", + ) + .bind(crate::data::data_table_name(&desc.table_id).trim_matches('"')) + .bind(crate::data::vector_table_name(&desc.table_id, &vector_id).trim_matches('"')) + .fetch_one(&engine.pool) + .await + .expect("sqlite_master") + }; + assert_eq!(data_tables().await, 2); + drop(other_claim); + assert!( + engine + .remove_restore_target(&other_key) + .await + .expect("remove") + ); + assert_eq!(tables_named(&engine, "Gone").await, 0); + assert_eq!( + data_tables().await, + 0, + "the data and vector tables are dropped" + ); + assert!( + engine + .read_restore_job(&other_key) + .await + .expect("read") + .is_none() + ); + assert!( + !engine + .remove_restore_target(&other_key) + .await + .expect("remove again") + ); +} diff --git a/crates/storage-sqlite/src/snapshot_engine.rs b/crates/storage-sqlite/src/snapshot_engine.rs index e8f12a53e..4b719b8fb 100644 --- a/crates/storage-sqlite/src/snapshot_engine.rs +++ b/crates/storage-sqlite/src/snapshot_engine.rs @@ -30,10 +30,10 @@ //! pool free for requests counts snapshots too; when no slot is free the //! open returns `LimitExceeded` at once rather than queueing. The slot count //! is stated once, in `SqliteConfig`'s doc (`config.rs`). The other side of -//! sharing: a `CreateBackup` waits 30 s for a slot, so snapshots holding -//! every slot make every `CreateBackup` wait and then fail with the shared -//! refusal (`backup::slots_exhausted`), which names table snapshots for that -//! reason. +//! sharing: a `CreateBackup` tries the same slots without waiting +//! (`backup::acquire_backup_permit`), so snapshots holding every slot make +//! every `CreateBackup` fail at once with the one `LimitExceededException` +//! text, with no backup running. //! //! In-memory databases are pinned to a single connection (a second connection //! would open a different database), so a stream that held its transaction @@ -55,12 +55,23 @@ //! LIMIT n`), which `SQLite` serves from the primary key index. Multi-part //! range keys fall back to rowid order and report `key_ordered = false`. //! -//! Memory: one page is `LIMIT paging::DEFAULT_FETCH_BATCH` (1000) rows, all -//! of them held as item text at once. At the default 400 KB item limit one -//! row's text can reach 27.7 MB (the measured worst case in #394's line-cap -//! table), so a page is bounded by rows, not bytes. A byte budget (keyset -//! paging with a byte cutoff) is required before the stream gets a caller; -//! the backup design assigns it to the backup job PR. +//! Memory: a page ends at `paging::fetch_batch()` rows (1000) or after the +//! row that takes its item text to or past `paging::fetch_bytes()` (64 MiB, +//! the budget `extenddb_storage::snapshot::paging` states), whichever comes +//! first, so a page holds at most the budget plus one row of item text. The +//! cut is applied while the page's rows are pulled from the query one at a +//! time (`fetch_page`): the pull stops at the cut and the statement is +//! reset, so the stream reads no row past it and the next page starts after +//! the last row pulled. The driver's worker thread runs the statement ahead +//! of the pull and buffers the rows it has read: that run-ahead is bounded by +//! the pool's `row_buffer_size` (`store.rs`), not by the budget, at five +//! rows (the four the channel holds and the one the worker holds while its +//! `send` waits), 160 MiB at the 32 MiB line cap, and is the one term of a +//! page's memory the budget does not state. A page cut by the budget is short +//! without being the end of the table; the end is a page under the row bound +//! that the budget did not cut. The in-memory copy is a different bound: it +//! holds the whole table decoded, and the slot count, not the budget, bounds +//! how many copies exist at once. use extenddb_core::types::{Item, ScalarAttributeType, TableDescription}; use extenddb_storage::error::StorageError; @@ -120,9 +131,10 @@ impl SqliteEngine { // would hold every connection and every writer would wait out the // pool's acquire timeout under the engine write lock. Refuse rather // than queue; the caller bounds concurrency. - let permit = std::sync::Arc::clone(&self.backup_server_slots) - .try_acquire_owned() - .map_err(|_| crate::backup::slots_exhausted(self.backup_server_slot_count))?; + let permit = crate::backup::acquire_backup_permit( + &self.backup_server_slots, + self.backup_server_slot_count, + )?; let mut tx: Transaction<'static, Sqlite> = self.pool.begin().await.map_err(|e| db_err(&e))?; @@ -265,9 +277,10 @@ fn keyset_stream( buffer: Vec::new().into_iter(), done: false, }; - // Read the page size once so a change to the test knob during a stream - // cannot end it early or late. + // Read the page size and the byte budget once so a change to either + // test knob during a stream cannot end it early or late. let batch = paging::fetch_batch(); + let budget = paging::fetch_bytes(); let limit = i64::try_from(batch).unwrap_or(i64::MAX); Box::pin(futures::stream::unfold( state, @@ -281,8 +294,16 @@ fn keyset_stream( } let tx = state.tx.as_mut()?; paging::record_fetch(); - let page = fetch_page(tx, &state.table, &state.paging, &state.cursor, limit).await; - let (items, next_cursor) = match page { + let page = fetch_page( + tx, + &state.table, + &state.paging, + &state.cursor, + limit, + budget, + ) + .await; + let Page { items, next, cut } = match page { Ok(page) => page, Err(e) => { state.done = true; @@ -291,10 +312,13 @@ fn keyset_stream( return Some((Err(e), state)); } }; - if let Some(next) = next_cursor { + if let Some(next) = next { state.cursor = next; } - if items.len() < batch { + // A short page shows the end only when the budget did not + // cut it: a cut page is short because the pull stopped, not + // because the table did. + if items.len() < batch && !cut { state.done = true; if let Some(tx) = state.tx.take() { let _ = tx.commit().await; @@ -307,127 +331,118 @@ fn keyset_stream( )) } -/// Query one page of up to `limit` rows for the paging mode and cursor. -async fn query_page( - tx: &mut Transaction<'static, Sqlite>, - table: &str, - paging: &Paging, - cursor: &Cursor, - limit: i64, -) -> Result, StorageError> { - match (paging, cursor) { +/// The SQL of one page for the paging mode and cursor: the key columns and +/// `item_data`, in key order, after the cursor when there is one, `LIMIT ?` +/// last. +fn page_sql(table: &str, paging: &Paging, cursor: &Cursor) -> Result { + Ok(match (paging, cursor) { (Paging::Pk, Cursor::Start) => { - sqlx::query(&format!( - "SELECT pk, item_data FROM {table} ORDER BY pk LIMIT ?" - )) - .bind(limit) - .fetch_all(&mut **tx) - .await + format!("SELECT pk, item_data FROM {table} ORDER BY pk LIMIT ?") } - (Paging::Pk, Cursor::Pk(last)) => { - sqlx::query(&format!( - "SELECT pk, item_data FROM {table} WHERE pk > ? ORDER BY pk LIMIT ?" - )) - .bind(last) - .bind(limit) - .fetch_all(&mut **tx) - .await + (Paging::Pk, Cursor::Pk(_)) => { + format!("SELECT pk, item_data FROM {table} WHERE pk > ? ORDER BY pk LIMIT ?") } (Paging::PkSk { sk_col, .. }, Cursor::Start) => { - sqlx::query(&format!( - "SELECT pk, {sk_col}, item_data FROM {table} ORDER BY pk, {sk_col} LIMIT ?" - )) - .bind(limit) - .fetch_all(&mut **tx) - .await - } - (Paging::PkSk { sk_col, .. }, Cursor::PkSkText(pk, sk)) => { - sqlx::query(&format!( - "SELECT pk, {sk_col}, item_data FROM {table} \ - WHERE (pk, {sk_col}) > (?, ?) ORDER BY pk, {sk_col} LIMIT ?" - )) - .bind(pk) - .bind(sk) - .bind(limit) - .fetch_all(&mut **tx) - .await + format!("SELECT pk, {sk_col}, item_data FROM {table} ORDER BY pk, {sk_col} LIMIT ?") } - (Paging::PkSk { sk_col, .. }, Cursor::PkSkBlob(pk, sk)) => { - sqlx::query(&format!( + (Paging::PkSk { sk_col, .. }, Cursor::PkSkText(..) | Cursor::PkSkBlob(..)) => { + format!( "SELECT pk, {sk_col}, item_data FROM {table} \ WHERE (pk, {sk_col}) > (?, ?) ORDER BY pk, {sk_col} LIMIT ?" - )) - .bind(pk) - .bind(sk) - .bind(limit) - .fetch_all(&mut **tx) - .await + ) } (Paging::Rowid, Cursor::Start) => { - sqlx::query(&format!( - "SELECT rowid, item_data FROM {table} ORDER BY rowid LIMIT ?" - )) - .bind(limit) - .fetch_all(&mut **tx) - .await + format!("SELECT rowid, item_data FROM {table} ORDER BY rowid LIMIT ?") } - (Paging::Rowid, Cursor::Rowid(last)) => { - sqlx::query(&format!( - "SELECT rowid, item_data FROM {table} WHERE rowid > ? ORDER BY rowid LIMIT ?" - )) - .bind(last) - .bind(limit) - .fetch_all(&mut **tx) - .await + (Paging::Rowid, Cursor::Rowid(_)) => { + format!("SELECT rowid, item_data FROM {table} WHERE rowid > ? ORDER BY rowid LIMIT ?") } _ => { return Err(StorageError::Internal( "snapshot cursor does not match its paging mode".to_owned(), )); } - } - .map_err(|e| db_err(&e)) + }) +} + +/// The cursor a page's last row leaves for the next page. +fn cursor_after(row: &sqlx::sqlite::SqliteRow, paging: &Paging) -> Result { + Ok(match paging { + Paging::Pk => Cursor::Pk(row.try_get("pk").map_err(|e| db_err(&e))?), + Paging::PkSk { sk_col, sk_type } => { + let pk: String = row.try_get("pk").map_err(|e| db_err(&e))?; + match sk_type { + ScalarAttributeType::B => Cursor::PkSkBlob( + pk, + row.try_get::, _>(*sk_col).map_err(|e| db_err(&e))?, + ), + ScalarAttributeType::S | ScalarAttributeType::N => Cursor::PkSkText( + pk, + row.try_get::(*sk_col).map_err(|e| db_err(&e))?, + ), + } + } + Paging::Rowid => Cursor::Rowid(row.try_get("rowid").map_err(|e| db_err(&e))?), + }) +} + +/// One page as [`fetch_page`] returns it. +struct Page { + items: Vec, + /// The cursor for the next page; `None` for an empty page. + next: Option, + /// Whether the byte budget ended the page before `limit` rows. + cut: bool, } -/// One page: items and the cursor for the next page. Fewer items than -/// `limit` means the table is exhausted. +/// One page: at most `limit` rows, pulled one at a time from the page query +/// and stopped after the row that takes the page's item text to or past +/// `budget` bytes, so a page holds at most the budget plus one row and at +/// least one row, and the stream pulls no row past the cut (the statement is +/// reset when the row stream is dropped, with the rows the driver's worker +/// read ahead, at most the pool's `row_buffer_size` plus the one the worker +/// holds, five; the next page starts at the cursor the last pulled row +/// leaves). Fewer rows than `limit` +/// without a cut means the table is exhausted. async fn fetch_page( tx: &mut Transaction<'static, Sqlite>, table: &str, paging: &Paging, cursor: &Cursor, limit: i64, -) -> Result<(Vec, Option), StorageError> { - let rows = query_page(tx, table, paging, cursor, limit).await?; - let mut items = Vec::with_capacity(rows.len()); + budget: usize, +) -> Result { + use futures::TryStreamExt as _; + let sql = page_sql(table, paging, cursor)?; + let query = sqlx::query(&sql); + let query = match cursor { + Cursor::Start => query, + Cursor::Pk(pk) => query.bind(pk), + Cursor::PkSkText(pk, sk) => query.bind(pk).bind(sk), + Cursor::PkSkBlob(pk, sk) => query.bind(pk).bind(sk.as_slice()), + Cursor::Rowid(rowid) => query.bind(rowid), + } + .bind(limit); + let mut rows = query.fetch(&mut **tx); + let mut items = Vec::new(); + let mut bytes = 0usize; let mut next = None; - for row in &rows { + let mut cut = false; + while let Some(row) = rows.try_next().await.map_err(|e| db_err(&e))? { let data: String = row.try_get("item_data").map_err(|e| db_err(&e))?; + bytes += data.len(); let item: Item = serde_json::from_str(&data) .map_err(|e| StorageError::Internal(format!("Parse item: {e}")))?; items.push(item); + next = Some(cursor_after(&row, paging)?); + if bytes >= budget { + cut = true; + break; + } } - if let Some(last) = rows.last() { - next = Some(match paging { - Paging::Pk => Cursor::Pk(last.try_get("pk").map_err(|e| db_err(&e))?), - Paging::PkSk { sk_col, sk_type } => { - let pk: String = last.try_get("pk").map_err(|e| db_err(&e))?; - match sk_type { - ScalarAttributeType::B => Cursor::PkSkBlob( - pk, - last.try_get::, _>(*sk_col) - .map_err(|e| db_err(&e))?, - ), - ScalarAttributeType::S | ScalarAttributeType::N => Cursor::PkSkText( - pk, - last.try_get::(*sk_col).map_err(|e| db_err(&e))?, - ), - } - } - Paging::Rowid => Cursor::Rowid(last.try_get("rowid").map_err(|e| db_err(&e))?), - }); - } - Ok((items, next)) + drop(rows); + paging::record_page_bytes(bytes); + Ok(Page { items, next, cut }) } #[cfg(test)] @@ -549,6 +564,33 @@ mod tests { }, ) .await; + // The byte budget: pages are cut after the row that reaches it, every + // row streams once across cut pages, and the rows past a cut are + // never read (the pull stops at the cut), which `rows_past_cut` + // reports as zero here. + let bounds = snapshot_conformance::run_page_bounds( + &hooks, + &*engine, + ACCOUNT, + "snap_bounds", + || async { + engine + .process_control_plane_transitions() + .await + .expect("pump"); + }, + ) + .await; + assert_eq!( + bounds.rows_past_cut, 0, + "SQLite pulls rows one at a time and stops at the cut: {bounds:?}" + ); + assert_eq!( + bounds.pages, + (snapshot_conformance::PAGE_BOUNDS_ITEMS as u64) + .div_ceil(snapshot_conformance::PAGE_BOUNDS_ROWS as u64), + "a page is exactly the budget's rows: {bounds:?}" + ); let pool = engine.pool.clone(); snapshot_conformance::run_refusals( &hooks, @@ -696,19 +738,21 @@ mod tests { .await .expect_err("no slot left"); assert!(matches!(err, StorageError::LimitExceeded(_)), "{err}"); - // CreateBackup's admission on the same slots, with its 30 s wait - // shortened: with snapshots holding every slot and no backup running, - // the refusal must name snapshots. + // CreateBackup's admission on the same slots: with snapshots holding + // every slot and no backup running, it is refused at once with the + // one text. match crate::backup::acquire_backup_permit( - std::sync::Arc::clone(&engine.backup_server_slots), + &engine.backup_server_slots, engine.backup_server_slot_count, - std::time::Duration::from_millis(50), - ) - .await - { - Err(StorageError::LimitExceeded(m)) => assert!( - m.contains("backups or table snapshots") && m.contains(&format!("at most {slots}")), - "{m}" + ) { + Err(StorageError::LimitExceeded(m)) => assert_eq!( + m, + extenddb_storage::backup_catalog::limit_exceeded_message( + extenddb_storage::backup_catalog::JobBound::Counted { + name: crate::backup::BACKUP_SERVER_SLOTS_NAME, + limit: slots, + } + ) ), other => panic!( "CreateBackup is refused while snapshots hold every slot: {:?}", diff --git a/crates/storage-sqlite/src/store.rs b/crates/storage-sqlite/src/store.rs index 4128890fe..a8f7c8de0 100644 --- a/crates/storage-sqlite/src/store.rs +++ b/crates/storage-sqlite/src/store.rs @@ -38,17 +38,24 @@ //! so the invariant stays auditable, is init-time bootstrap (in this file and //! in `bootstrapper.rs`), which runs before the server serves traffic. +use std::str::FromStr; use std::sync::Arc; use std::sync::atomic::AtomicU64; use extenddb_storage::error::StorageError; use sqlx::SqlitePool; -use sqlx::sqlite::SqlitePoolOptions; +use sqlx::sqlite::{SqliteConnectOptions, SqlitePoolOptions}; use tokio::sync::Mutex; use crate::INDEX_PROPAGATION_DELAY_QUERY; use crate::sqlite_util::sqlite_url; +/// Rows the driver's worker buffers ahead of the reading task (see +/// `SqliteEngine::new_for_server`). The rows ahead of the reader are one +/// more than this: the channel holds this many and the worker, blocked on +/// its `send`, holds the next row it has already stepped and copied. +const ROW_BUFFER_ROWS: usize = 4; + /// The SQLite storage engine. Implements `StorageEngine` (all data/metadata/ /// stream/worker/backup traits) over a single SQLite database. #[derive(Clone)] @@ -86,17 +93,55 @@ pub struct SqliteEngine { /// it; see [`crate::backup::backup_server_slot_count`]. pub(crate) backup_server_slot_count: usize, pub(crate) backup_server_slots: Arc, + /// The in-process claims of the backup and restore jobs this process + /// runs; see [`crate::backup_catalog::JobSlots`]. + pub(crate) job_slots: Arc, +} + +/// The connection options of every pool connection. The driver runs each +/// statement on a worker thread that buffers rows ahead of the task reading +/// them (sqlx's `row_buffer_size`, 50 by default) over a bounded channel; +/// the worker steps and copies the next row before its `send` blocks, so +/// the rows ahead of the reader are the buffer plus one. A table snapshot +/// reads rows of up to the 32 MiB line cap and stops a page at the byte +/// budget, so a 50-row buffer is up to 1.6 GB the page never asked for; +/// [`ROW_BUFFER_ROWS`] bounds the run-ahead at five rows (160 MiB at the +/// line cap), which costs ordinary queries a channel hand-off every four +/// rows. The test at the end of this module counts the rows stepped ahead. +pub(crate) fn connect_options(url: &str) -> Result { + Ok(SqliteConnectOptions::from_str(url) + .map_err(|e| StorageError::Connection(e.to_string()))? + .row_buffer_size(ROW_BUFFER_ROWS)) } impl SqliteEngine { /// Open (creating if necessary) the SQLite database and build the engine. /// /// `path_or_url` accepts a filesystem path, `:memory:`, or a `sqlite:` URL. + /// The pool is sized for no backup jobs; a server opens the engine with + /// [`Self::new_for_server`]. pub async fn new( path_or_url: &str, pool_size: u32, region: &str, max_item_size_bytes: usize, + ) -> Result { + Self::new_for_server(path_or_url, pool_size, 0, region, max_item_size_bytes).await + } + + /// [`Self::new`] for a server that runs up to `max_concurrent_jobs` + /// backup and restore jobs at once: a file-backed pool is raised, with a + /// warning naming both numbers, until its server slots admit every job + /// the engine's permit admits (`backup::pool_size_for_jobs`; the rule is + /// stated on `SqliteConfig::pool_size`), so no job the permit admitted + /// is refused by the slot. The in-memory pool has one connection by + /// nature and is not raised. + pub async fn new_for_server( + path_or_url: &str, + pool_size: u32, + max_concurrent_jobs: usize, + region: &str, + max_item_size_bytes: usize, ) -> Result { let url = sqlite_url(path_or_url); // Use the same parsed-location classifier as the serve lock. String @@ -104,6 +149,18 @@ impl SqliteEngine { let in_memory = crate::serve_lock::database_file(path_or_url) .map_err(StorageError::Connection)? .is_none(); + let for_jobs = if in_memory { + pool_size + } else { + crate::backup::pool_size_for_jobs(pool_size, max_concurrent_jobs) + }; + if for_jobs != pool_size { + tracing::warn!( + "{}", + crate::backup::pool_raise_message(pool_size, for_jobs, max_concurrent_jobs) + ); + } + let pool_size = for_jobs; // An in-memory database lives only inside its own connection: a second // connection opens a *separate* empty database. So for an in-memory // location we pin the pool to a single connection that is never @@ -126,13 +183,14 @@ impl SqliteEngine { if in_memory { opts = opts.idle_timeout(None).max_lifetime(None); } + let connect = connect_options(&url)?; let pool = opts .after_connect(|conn, _| { Box::pin( async move { crate::sqlite_util::set_connection_pragmas(conn, true).await }, ) }) - .connect(&url) + .connect_with(connect) .await .map_err(|e| StorageError::Connection(e.to_string()))?; @@ -177,6 +235,7 @@ impl SqliteEngine { serve_lock: None, backup_server_slot_count, backup_server_slots: Arc::new(tokio::sync::Semaphore::new(backup_server_slot_count)), + job_slots: Arc::default(), }) } @@ -560,20 +619,26 @@ mod d1_write_lock_tests { #[tokio::test] async fn delete_backup_waits_for_the_write_lock() { - use extenddb_storage::BackupEngine; + use extenddb_storage::{JobPermit, LegacyBackupReader}; let engine = engine().await; create_table(&engine, "d1-bkp-t").await; // The backup must exist before the lock window: delete_backup resolves // it with a read first and returns early when it is missing, which // would complete without ever reaching the writes under test. - let details = BackupEngine::create_backup(&engine, "000000000000", "d1-bkp-t", "b") - .await - .expect("backup"); + let details = LegacyBackupReader::create( + &engine, + "000000000000", + "d1-bkp-t", + "b", + JobPermit::detached(), + ) + .await + .expect("backup"); let e = engine.clone(); assert_serialized( &engine, async move { - BackupEngine::delete_backup(&e, "000000000000", &details.backup_arn) + LegacyBackupReader::delete(&e, "000000000000", &details.backup_arn) .await .expect("delete backup"); }, @@ -790,3 +855,65 @@ mod d1_write_lock_tests { .await; } } + +#[cfg(test)] +mod row_buffer_tests { + use std::sync::Arc; + use std::sync::atomic::{AtomicUsize, Ordering}; + + use futures::TryStreamExt; + + /// The rows the driver's worker steps ahead of a reader that pulls one + /// row and stops, on a connection with the engine's options: a collation + /// the query's `WHERE` applies to every stepped row counts the steps, a + /// table of large rows (1 MiB each, 40 of them) is read through a plain + /// scan, the reader takes one row and drops the stream, and the steps + /// beyond that row are the buffer plus the one the worker holds, five. + /// With sqlx's default buffer of 50 the count is 51, 1.6 GB of lines at + /// the cap a page never asked for. + #[tokio::test] + async fn the_driver_steps_five_rows_ahead_of_the_pages_reader() { + let steps = Arc::new(AtomicUsize::new(0)); + let counting = Arc::clone(&steps); + let options = super::connect_options("sqlite::memory:") + .expect("options") + .collation("counting", move |a: &str, b: &str| { + counting.fetch_add(1, Ordering::SeqCst); + a.cmp(b) + }); + let mut conn = sqlx::ConnectOptions::connect(&options) + .await + .expect("connect"); + sqlx::query("CREATE TABLE big (pk TEXT COLLATE counting, item_data TEXT)") + .execute(&mut conn) + .await + .expect("create"); + let payload = "x".repeat(1024 * 1024); + for i in 0..40 { + sqlx::query("INSERT INTO big (pk, item_data) VALUES (?, ?)") + .bind(format!("k{i:03}")) + .bind(&payload) + .execute(&mut conn) + .await + .expect("insert"); + } + steps.store(0, Ordering::SeqCst); + // The scan evaluates the comparison on each row it steps, so the + // collation is called once per step; no index, no sort, so no + // comparison runs ahead of the stepping. + let mut rows = sqlx::query("SELECT item_data FROM big WHERE pk <> 'none'").fetch(&mut conn); + let first = rows.try_next().await.expect("fetch").expect("a row"); + drop(first); + // The worker runs on until its channel is full and its next send + // blocks; give it the time to get there. + tokio::time::sleep(std::time::Duration::from_millis(300)).await; + let stepped = steps.load(Ordering::SeqCst); + drop(rows); + assert_eq!( + stepped - 1, + super::ROW_BUFFER_ROWS + 1, + "{stepped} rows stepped for one pulled: the buffer plus the row the worker holds" + ); + assert_eq!(stepped - 1, 5); + } +} diff --git a/crates/storage-sqlite/tests/backup_job_panic.rs b/crates/storage-sqlite/tests/backup_job_panic.rs new file mode 100644 index 000000000..14bd52ed8 --- /dev/null +++ b/crates/storage-sqlite/tests/backup_job_panic.rs @@ -0,0 +1,338 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! A CreateBackup job whose task panics on SQLite leaves a CREATING row that +//! the reconciler's next cycle resolves to FAILED, in the same process, with +//! no restart (design 4(e): at runtime a row whose in-process slot no task +//! holds is dead; PR 8's acceptance). +//! +//! The engine is built the way the server builds it (the backend's server +//! components factory, its workers spawned through the runtime hooks, so a +//! table reaches ACTIVE as it does under `serve`), the store is a filesystem +//! store whose `put` panics, and the request goes through the engine's +//! `dispatch`. The job's task panics at its first put; the task's drop +//! releases the `Arc` and with it the row's `JobSlot`, so the +//! SQLite catalog's `claim_dead_backup` takes the row over at once, whatever +//! its heartbeat says, and the reconciler writes FAILED (no manifest) and +//! then, since the dead owner's heartbeat is within the window, leaves the +//! prefix for a later cycle. + +use std::os::unix::fs::DirBuilderExt; +use std::path::PathBuf; +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::Duration; + +use extenddb_backup_store::{BackupStore, ByteStream, FilesystemStore, ObjectMeta, StoreError}; +use extenddb_core::backup_format::BackupArn; +use extenddb_core::limits::LimitsConfig; +use extenddb_engine::reconciler::{self, OWNER_DEAD_NO_MANIFEST, ReconcilerState}; +use extenddb_engine::{ + BackupRuntime, ConfiguredStore, OperationContext, check_configured_store, dispatch, +}; +use extenddb_storage::backup_catalog::{BackupStatus, StoreKind}; +use extenddb_storage::server_components::ServerComponentsOptions; +use extenddb_storage::{CancellationToken, WorkerContext}; +use extenddb_storage_sqlite::SqliteConfig; +use futures::future::BoxFuture; +use futures::stream::BoxStream; +use serde_json::json; + +const REGION: &str = "us-east-1"; + +/// A filesystem store whose `put` panics: every other call reaches the real +/// store, so the identity check, the listing and the delete behave as on a +/// server; only the job's first object write dies. +struct PanicOnPut { + inner: FilesystemStore, + puts: AtomicUsize, +} + +impl BackupStore for PanicOnPut { + fn put<'a>( + &'a self, + key: &'a str, + _body: ByteStream, + ) -> BoxFuture<'a, Result> { + self.puts.fetch_add(1, Ordering::SeqCst); + Box::pin(async move { panic!("injected panic in put of {key}") }) + } + + fn put_if_absent<'a>( + &'a self, + key: &'a str, + body: ByteStream, + ) -> BoxFuture<'a, Result, StoreError>> { + self.inner.put_if_absent(key, body) + } + + fn get<'a>(&'a self, key: &'a str) -> BoxFuture<'a, Result> { + self.inner.get(key) + } + + fn head<'a>(&'a self, key: &'a str) -> BoxFuture<'a, Result, StoreError>> { + self.inner.head(key) + } + + fn list<'a>(&'a self, prefix: &'a str) -> BoxStream<'a, Result> { + self.inner.list(prefix) + } + + fn delete_prefix<'a>(&'a self, prefix: &'a str) -> BoxFuture<'a, Result> { + self.inner.delete_prefix(prefix) + } + + fn validate(&self) -> BoxFuture<'_, Result<(), StoreError>> { + self.inner.validate() + } +} + +/// A scratch directory under the cargo target directory, removed on drop. +struct Scratch(PathBuf); + +impl Drop for Scratch { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + +async fn call( + ctx: &OperationContext, + operation: &str, + body: serde_json::Value, +) -> serde_json::Value { + dispatch(operation, body, ctx, "127.0.0.1:0") + .await + .unwrap_or_else(|e| panic!("{operation}: {e:?}")) + .body +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_job_task_that_panics_leaves_a_row_the_next_cycle_resolves_without_a_restart() { + let scratch = Scratch( + PathBuf::from(env!("CARGO_TARGET_TMPDIR")) + .join(format!("backup-job-panic-{}", uuid::Uuid::new_v4())), + ); + std::fs::create_dir_all(&scratch.0).expect("scratch directory"); + let db_path = scratch.0.join("catalog.sqlite"); + let root = scratch.0.join("backups"); + std::fs::DirBuilder::new() + .mode(0o700) + .create(&root) + .expect("root"); + + // The engine as the server builds it: a dev bootstrap of a fresh file + // (the migrations, the default account), the workers spawned. + let config = SqliteConfig { + path: db_path.to_str().expect("UTF-8 path").to_owned(), + pool_size: 4, + }; + let components = (extenddb_storage_sqlite::backend().server_components)(&config, REGION, { + let mut options = ServerComponentsOptions::default(); + options.bootstrap_if_uninitialized = true; + options.release = "test"; + options.max_concurrent_jobs = 4; + options + }) + .await + .unwrap_or_else(|e| panic!("server components: {e:?}")); + let storage = components.engine; + let shutdown = CancellationToken::new(); + let (_layer, reload_handle) = + tracing_subscriber::reload::Layer::new(tracing_subscriber::EnvFilter::new("info")); + let worker_ctx = WorkerContext { + metrics: Arc::new(extenddb_core::metrics::MetricsCollector::new()), + catalog_store: components.catalog_store.clone(), + reload_handle, + config_log_level: "info".to_owned(), + shutdown: shutdown.clone(), + }; + let hooks = components + .runtime_hooks + .expect("the SQLite backend has runtime hooks"); + let workers = hooks.spawn_workers(&worker_ctx).await; + let account_id: String = { + let pool = sqlx::SqlitePool::connect(&format!("sqlite://{}", db_path.display())) + .await + .expect("a second connection to the catalog"); + sqlx::query("INSERT OR REPLACE INTO settings (key, value) VALUES ('control_plane_delay_seconds', '0.05')") + .execute(&pool) + .await + .expect("setting"); + let id = sqlx::query_scalar("SELECT value FROM settings WHERE key = 'default_account_id'") + .fetch_one(&pool) + .await + .expect("the bootstrapped account"); + pool.close().await; + id + }; + + // The store: a healthy filesystem store (its identity file created by + // the check the server runs at startup) whose `put` panics. + let store = Arc::new(PanicOnPut { + inner: FilesystemStore::open(&root) + .await + .expect("filesystem store"), + puts: AtomicUsize::new(0), + }); + let configured = ConfiguredStore::checking( + StoreKind::Filesystem, + root.to_str().expect("UTF-8 path").to_owned(), + store.clone(), + None, + ); + let outcome = check_configured_store(&configured, &*storage, "sqlite").await; + assert!(outcome.is_ok(), "the store's first check: {outcome:?}"); + configured.record_check(outcome); + let stale_job_after = Duration::from_secs(300); + let runtime = Arc::new( + BackupRuntime::new(4) + .with_stale_job_after(stale_job_after) + .with_store(configured) + .with_source("sqlite", "test") + .with_shutdown(shutdown.child_token()), + ); + let ctx = OperationContext { + storage: storage.clone(), + backup: runtime.clone(), + limits: Arc::new(LimitsConfig::default()), + region: Arc::from(REGION), + account_id: Arc::from(account_id.as_str()), + import_paths: Arc::from(Vec::new()), + export_paths: Arc::from(Vec::new()), + authorized_key_info: Default::default(), + auth_cache: extenddb_auth::AuthCacheRegistry::empty(), + table_key_info_lookup: None, + }; + + call( + &ctx, + "CreateTable", + json!({ + "TableName": "Music", + "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}], + "AttributeDefinitions": [{"AttributeName": "pk", "AttributeType": "S"}], + "BillingMode": "PAY_PER_REQUEST" + }), + ) + .await; + let deadline = std::time::Instant::now() + Duration::from_secs(30); + loop { + let desc = call(&ctx, "DescribeTable", json!({"TableName": "Music"})).await; + if desc["Table"]["TableStatus"] == "ACTIVE" { + break; + } + assert!( + std::time::Instant::now() < deadline, + "Music did not become ACTIVE: {desc}" + ); + tokio::time::sleep(Duration::from_millis(50)).await; + } + call( + &ctx, + "PutItem", + json!({"TableName": "Music", "Item": {"pk": {"S": "a"}, "n": {"N": "1"}}}), + ) + .await; + + // The request answers CREATING; the job's task then panics at its first + // put and ends. + let details = call( + &ctx, + "CreateBackup", + json!({"TableName": "Music", "BackupName": "nightly"}), + ) + .await; + assert_eq!( + details["BackupDetails"]["BackupStatus"], "CREATING", + "{details}" + ); + let arn_text = details["BackupDetails"]["BackupArn"] + .as_str() + .expect("an ARN") + .to_owned(); + let arn = BackupArn::parse(&arn_text).expect("a backup ARN"); + let deadline = std::time::Instant::now() + Duration::from_secs(30); + while runtime.running_jobs() > 0 { + assert!( + std::time::Instant::now() < deadline, + "the job task did not end" + ); + tokio::time::sleep(Duration::from_millis(20)).await; + } + assert_eq!( + store.puts.load(Ordering::SeqCst), + 1, + "the task died at its first put" + ); + let row = storage + .read_backup_row(&arn) + .await + .expect("read the row") + .expect("the row exists"); + assert_eq!( + row.status, + BackupStatus::Creating, + "the panicking task wrote nothing" + ); + let heartbeat_before = row.heartbeat_at; + assert_eq!( + runtime.free_slots(), + 4, + "the task's drop returned the job permit" + ); + + // One reconciler cycle in the same process: the slot no task holds makes + // the row dead at once, and without a manifest it is FAILED. + let mut state = ReconcilerState::with_shutdown(shutdown.child_token()); + let report = reconciler::cycle(&storage, &runtime, stale_job_after, "sqlite", &mut state) + .await + .expect("a reconciler cycle"); + assert_eq!(report.store_healthy, Some(true), "{report:?}"); + assert_eq!( + report.creating_resolved, + vec![(arn_text.clone(), BackupStatus::Failed)], + "{report:?}" + ); + let row = storage + .read_backup_row(&arn) + .await + .expect("read the row") + .expect("the row exists"); + assert_eq!(row.status, BackupStatus::Failed); + assert_eq!(row.failure_reason.as_deref(), Some(OWNER_DEAD_NO_MANIFEST)); + assert!( + row.completed_at.is_some(), + "the FAILED write stamps completed_at" + ); + assert_eq!( + row.heartbeat_at, heartbeat_before, + "the takeover left the heartbeat" + ); + // The dead owner's heartbeat is within the window, so the prefix (empty + // here: the put never landed) is the FAILED-prefix duty's once it is + // stale, not this cycle's. + assert!(report.failed_prefixes_deleted.is_empty(), "{report:?}"); + let described = dispatch( + "DescribeBackup", + json!({"BackupArn": arn_text}), + &ctx, + "127.0.0.1:0", + ) + .await; + match described { + Err(extenddb_core::error::DynamoDbError::BackupNotFoundException(_)) => {} + Err(other) => panic!("DescribeBackup of the FAILED row: {other:?}"), + Ok(answer) => panic!("DescribeBackup of the FAILED row answered {}", answer.body), + } + // A second cycle has nothing left to resolve. + let report = reconciler::cycle(&storage, &runtime, stale_job_after, "sqlite", &mut state) + .await + .expect("a second cycle"); + assert!(report.creating_resolved.is_empty(), "{report:?}"); + + shutdown.cancel(); + for worker in workers { + let _ = tokio::time::timeout(Duration::from_secs(5), worker).await; + } +} diff --git a/crates/storage-sqlite/tests/fixtures/golden/0006_backup_catalog_index.txt b/crates/storage-sqlite/tests/fixtures/golden/0006_backup_catalog_index.txt new file mode 100644 index 000000000..10e237364 --- /dev/null +++ b/crates/storage-sqlite/tests/fixtures/golden/0006_backup_catalog_index.txt @@ -0,0 +1,47 @@ +-- SELECT t.table_name, t.table_status, r.status, r.kind, r.owner_generation, r.heartbeat_at, r.store_kind, r.completed_at = t.creation_date_time FROM table_restores r JOIN tables t ON t.table_id = r.table_id ORDER BY t.table_name +cu_orders ACTIVE DONE legacy 0 1970-01-01T00:00:00.000Z NULL 1 +interrupted CREATING RUNNING legacy 0 1970-01-01T00:00:00.000Z NULL NULL +-- SELECT name FROM pragma_table_info('table_restores') ORDER BY cid +table_id +source_backup_arn +source_table_arn +restore_date_time +status +kind +store_kind +store_root +store_id +owner_generation +heartbeat_at +spool_host +file_index +items_applied +failure_reason +completed_at +-- SELECT name, type FROM sqlite_master WHERE name LIKE 'backup_%' OR name LIKE 'idx_backup_%' OR name = 'idx_table_restores_status' ORDER BY name +backup_definitions table +backup_index table +backup_items table +backup_maintenance table +backups table +idx_backup_index_account_table index +idx_backup_index_backup_id index +idx_backup_index_store_status index +idx_backup_items_arn index +idx_backups_table index +idx_table_restores_status index +-- SELECT notes FROM extenddb_migrations WHERE id = 6 +added table_restores.status +added table_restores.kind +added table_restores.store_kind +added table_restores.store_root +added table_restores.store_id +added table_restores.owner_generation +added table_restores.heartbeat_at +added table_restores.spool_host +added table_restores.file_index +added table_restores.items_applied +added table_restores.failure_reason +added table_restores.completed_at +backfilled 2 restore registration(s) as legacy restores, 1 of them RUNNING with a dead owner for the reconciler to remove +wrote settings.catalog_id diff --git a/crates/storage-sqlite/tests/restore_job_resume.rs b/crates/storage-sqlite/tests/restore_job_resume.rs new file mode 100644 index 000000000..2b1a5dd81 --- /dev/null +++ b/crates/storage-sqlite/tests/restore_job_resume.rs @@ -0,0 +1,491 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! A store-backed restore on SQLite whose owner died mid-file is resumed by +//! one reconciler cycle in the same process, from the record's +//! `(file_index, items_applied)`, and ends ACTIVE with the backup's item +//! count (design 4(e) and 5.3 step 5 on this backend: a RUNNING `manifest` +//! record whose in-process slot no task holds is dead at once, and the +//! progress on the record is where the resume starts). +//! +//! The engine is built the way the server builds it (the backend's server +//! components factory, its workers spawned through the runtime hooks), the +//! store is a real filesystem store, and the backup is taken through the +//! engine's `dispatch` with data files small enough to make several. The +//! dead owner is played at the storage level: `claim_restore`, `bulk_load` +//! of the first file and of half of the second, the claim dropped (its slot +//! freed) and the record's heartbeat aged as the server's restart ages it. +//! The cycle takes the record over, the resume loads the rest, and the table +//! reaches ACTIVE with every item once; a second cycle finds nothing. + +use std::os::unix::fs::DirBuilderExt; +use std::path::PathBuf; +use std::sync::Arc; +use std::time::Duration; + +use extenddb_backup_store::FilesystemStore; +use extenddb_core::backup_format::{ + BackupArn, EXTENDDB_MANIFEST_FILE, ExtenddbManifest, data_file_key, +}; +use extenddb_core::limits::LimitsConfig; +use extenddb_core::types::Item; +use extenddb_engine::reconciler::{self, ReconcilerState}; +use extenddb_engine::{ + BackupRuntime, ConfiguredStore, JobSettings, OperationContext, check_configured_store, dispatch, +}; +use extenddb_storage::backup_catalog::{ + JobStatus, RestoreKind, RestoreRegistrationRow, RestoreTargetKey, StoreKind, UNOWNED_GENERATION, +}; +use extenddb_storage::restore_engine::BulkLoadFile; +use extenddb_storage::server_components::ServerComponentsOptions; +use extenddb_storage::{ + CancellationToken, RestoreTableOverrides, WorkerContext, + backup_definition::restore_input_from_manifest, +}; +use extenddb_storage_sqlite::SqliteConfig; +use serde_json::json; + +const REGION: &str = "us-east-1"; +const ITEMS: usize = 300; + +/// A scratch directory under the cargo target directory, removed on drop. +struct Scratch(PathBuf); + +impl Drop for Scratch { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + +async fn call( + ctx: &OperationContext, + operation: &str, + body: serde_json::Value, +) -> serde_json::Value { + dispatch(operation, body, ctx, "127.0.0.1:0") + .await + .unwrap_or_else(|e| panic!("{operation}: {e:?}")) + .body +} + +async fn wait_table_status(ctx: &OperationContext, table: &str, status: &str) -> serde_json::Value { + let deadline = std::time::Instant::now() + Duration::from_secs(30); + loop { + let desc = call(ctx, "DescribeTable", json!({"TableName": table})).await; + if desc["Table"]["TableStatus"] == status { + return desc; + } + assert!( + std::time::Instant::now() < deadline, + "{table} did not reach {status}: {desc}" + ); + tokio::time::sleep(Duration::from_millis(50)).await; + } +} + +async fn scan_count(ctx: &OperationContext, table: &str, index: Option<&str>) -> usize { + let mut count = 0; + let mut start: Option = None; + loop { + let mut body = json!({"TableName": table, "Select": "COUNT"}); + if let Some(index) = index { + body["IndexName"] = json!(index); + } + if let Some(key) = &start { + body["ExclusiveStartKey"] = key.clone(); + } + let page = call(ctx, "Scan", body).await; + count += usize::try_from(page["Count"].as_u64().expect("Count")).unwrap(); + match page.get("LastEvaluatedKey") { + Some(key) if !key.is_null() => start = Some(key.clone()), + _ => return count, + } + } +} + +/// The items of data file `index` of the backup under `root`, decoded as the +/// engine's load decodes them. +fn file_items( + root: &std::path::Path, + arn: &BackupArn, + manifest: &ExtenddbManifest, + index: usize, +) -> Vec { + let entry = &manifest.data_files[index]; + // Data file keys count from one; `BulkLoadFile::file_index` from zero. + assert_eq!(entry.key, data_file_key(u32::try_from(index + 1).unwrap())); + let compressed = std::fs::File::open(root.join(arn.store_prefix()).join(&entry.key)) + .expect("the data file in the store"); + let mut reader = manifest + .reader_prehashed(entry, compressed) + .expect("a reader"); + let mut items = Vec::new(); + while let Some(item) = reader.next_item().expect("an item") { + items.push(item); + } + assert_eq!(items.len() as u64, entry.item_count); + items +} + +fn stream( + items: Vec, +) -> futures::stream::BoxStream<'static, Result> { + Box::pin(futures::stream::iter(items.into_iter().map(Ok))) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_dead_restore_is_resumed_by_one_cycle_from_its_recorded_progress() { + let scratch = Scratch( + PathBuf::from(env!("CARGO_TARGET_TMPDIR")) + .join(format!("restore-job-resume-{}", uuid::Uuid::new_v4())), + ); + std::fs::create_dir_all(&scratch.0).expect("scratch directory"); + let db_path = scratch.0.join("catalog.sqlite"); + let root = scratch.0.join("backups"); + std::fs::DirBuilder::new() + .mode(0o700) + .create(&root) + .expect("root"); + + // The engine as the server builds it. + let config = SqliteConfig { + path: db_path.to_str().expect("UTF-8 path").to_owned(), + pool_size: 4, + }; + let components = (extenddb_storage_sqlite::backend().server_components)(&config, REGION, { + let mut options = ServerComponentsOptions::default(); + options.bootstrap_if_uninitialized = true; + options.release = "test"; + options.max_concurrent_jobs = 4; + options + }) + .await + .unwrap_or_else(|e| panic!("server components: {e:?}")); + let storage = components.engine; + let shutdown = CancellationToken::new(); + let (_layer, reload_handle) = + tracing_subscriber::reload::Layer::new(tracing_subscriber::EnvFilter::new("info")); + let worker_ctx = WorkerContext { + metrics: Arc::new(extenddb_core::metrics::MetricsCollector::new()), + catalog_store: components.catalog_store.clone(), + reload_handle, + config_log_level: "info".to_owned(), + shutdown: shutdown.clone(), + }; + let hooks = components + .runtime_hooks + .expect("the SQLite backend has runtime hooks"); + let workers = hooks.spawn_workers(&worker_ctx).await; + let catalog = sqlx::SqlitePool::connect(&format!("sqlite://{}", db_path.display())) + .await + .expect("a second connection to the catalog"); + for (key, value) in [ + ("control_plane_delay_seconds", "0.05"), + ("index_propagation_delay_ms", "0"), + ] { + sqlx::query("INSERT OR REPLACE INTO settings (key, value) VALUES (?, ?)") + .bind(key) + .bind(value) + .execute(&catalog) + .await + .expect("setting"); + } + let account_id: String = + sqlx::query_scalar("SELECT value FROM settings WHERE key = 'default_account_id'") + .fetch_one(&catalog) + .await + .expect("the bootstrapped account"); + + // A healthy filesystem store, its identity file written by the check + // the server runs at startup; data files of about 4 KB so the backup + // has several. + let store = Arc::new( + FilesystemStore::open(&root) + .await + .expect("filesystem store"), + ); + let configured = ConfiguredStore::checking( + StoreKind::Filesystem, + root.to_str().expect("UTF-8 path").to_owned(), + store, + None, + ); + let outcome = check_configured_store(&configured, &*storage, "sqlite").await; + assert!(outcome.is_ok(), "the store's first check: {outcome:?}"); + configured.record_check(outcome); + let stale_job_after = Duration::from_secs(300); + let runtime = Arc::new( + BackupRuntime::new(4) + .with_stale_job_after(stale_job_after) + .with_store(configured) + .with_source("sqlite", "test") + .with_job_settings(JobSettings { + data_file_target_bytes: 4096, + ..JobSettings::default() + }) + .with_limits(Arc::new(LimitsConfig::default())) + .with_shutdown(shutdown.child_token()), + ); + let ctx = OperationContext { + storage: storage.clone(), + backup: runtime.clone(), + limits: Arc::new(LimitsConfig::default()), + region: Arc::from(REGION), + account_id: Arc::from(account_id.as_str()), + import_paths: Arc::from(Vec::new()), + export_paths: Arc::from(Vec::new()), + authorized_key_info: Default::default(), + auth_cache: extenddb_auth::AuthCacheRegistry::empty(), + table_key_info_lookup: None, + }; + + call( + &ctx, + "CreateTable", + json!({ + "TableName": "Music", + "KeySchema": [ + {"AttributeName": "pk", "KeyType": "HASH"}, + {"AttributeName": "sk", "KeyType": "RANGE"} + ], + "AttributeDefinitions": [ + {"AttributeName": "pk", "AttributeType": "S"}, + {"AttributeName": "sk", "AttributeType": "N"}, + {"AttributeName": "g", "AttributeType": "S"} + ], + "BillingMode": "PAY_PER_REQUEST", + "GlobalSecondaryIndexes": [{ + "IndexName": "gix", + "KeySchema": [{"AttributeName": "g", "KeyType": "HASH"}], + "Projection": {"ProjectionType": "KEYS_ONLY"} + }] + }), + ) + .await; + wait_table_status(&ctx, "Music", "ACTIVE").await; + for i in 0..ITEMS { + let mut item = json!({ + "pk": {"S": format!("p{}", i % 7)}, + "sk": {"N": i.to_string()}, + "note": {"S": format!("item {i} of the fixture, padded to make the files small {}", "x".repeat(40))} + }); + if i % 2 == 0 { + item["g"] = json!({"S": format!("g{}", i % 4)}); + } + call(&ctx, "PutItem", json!({"TableName": "Music", "Item": item})).await; + } + + // The backup, to the store, AVAILABLE. + let details = call( + &ctx, + "CreateBackup", + json!({"TableName": "Music", "BackupName": "nightly"}), + ) + .await; + let arn_text = details["BackupDetails"]["BackupArn"] + .as_str() + .expect("an ARN") + .to_owned(); + let deadline = std::time::Instant::now() + Duration::from_secs(60); + loop { + let described = call(&ctx, "DescribeBackup", json!({"BackupArn": arn_text})).await; + if described["BackupDescription"]["BackupDetails"]["BackupStatus"] == "AVAILABLE" { + break; + } + assert!( + std::time::Instant::now() < deadline, + "the backup did not become AVAILABLE: {described}" + ); + tokio::time::sleep(Duration::from_millis(50)).await; + } + while runtime.running_jobs() > 0 { + tokio::time::sleep(Duration::from_millis(20)).await; + } + let arn = BackupArn::parse(&arn_text).expect("a backup ARN"); + let row = storage + .read_backup_row(&arn) + .await + .expect("read the row") + .expect("the row exists"); + let manifest_bytes = std::fs::read(root.join(arn.store_prefix()).join(EXTENDDB_MANIFEST_FILE)) + .expect("manifest"); + let manifest = ExtenddbManifest::from_json_bytes(&manifest_bytes).expect("a manifest"); + assert!( + manifest.data_files.len() >= 3, + "the backup has several data files: {}", + manifest.data_files.len() + ); + let total: u64 = manifest.data_files.iter().map(|f| f.item_count).sum(); + assert_eq!(total as usize, ITEMS); + + // The dead owner: the registration, file 0 whole, half of file 1, then + // gone (the claim dropped frees the slot; the heartbeat aged as the + // server's restart ages every RUNNING record). + let input = restore_input_from_manifest( + &manifest.source_table, + "Restored", + &RestoreTableOverrides::default(), + ); + let registration = RestoreRegistrationRow { + backup: row.key(), + source_table_arn: manifest.source_table.table_arn.clone(), + restore_date_time: row.created_at, + store: row.store.clone(), + spool_host: None, + owner_generation: UNOWNED_GENERATION, + }; + let (desc, claim) = storage + .claim_restore(&account_id, input, ®istration) + .await + .expect("register the restore"); + let key = RestoreTargetKey { + account_id: account_id.clone(), + table_name: desc.table_name.clone(), + table_id: desc.table_id.clone(), + }; + let file0 = file_items(&root, &arn, &manifest, 0); + let file1 = file_items(&root, &arn, &manifest, 1); + let half = file1.len() / 2; + assert!(half > 0); + storage + .bulk_load( + &key, + claim.as_ref(), + BulkLoadFile { + file_index: 0, + start_at: 0, + reapplying: false, + }, + stream(file0.clone()), + ) + .await + .expect("file 0"); + storage + .bulk_load( + &key, + claim.as_ref(), + BulkLoadFile { + file_index: 1, + start_at: 0, + reapplying: false, + }, + stream(file1[..half].to_vec()), + ) + .await + .expect("half of file 1"); + let dead_generation = claim.generation(); + drop(claim); + let before = storage + .read_restore_job(&key) + .await + .expect("read") + .expect("the record"); + assert_eq!(before.status, JobStatus::Running); + assert_eq!(before.kind, RestoreKind::Manifest); + assert_eq!( + (before.file_index, before.items_applied), + (Some(1), Some(half as u64)) + ); + assert_eq!( + storage.count_restore_target(&key).await.expect("count") as usize, + file0.len() + half + ); + sqlx::query( + "UPDATE table_restores SET heartbeat_at = '1970-01-01T00:00:00.000Z' WHERE table_id = ?", + ) + .bind(&key.table_id) + .execute(&catalog) + .await + .expect("age the heartbeat as a restart does"); + let desc_before = call(&ctx, "DescribeTable", json!({"TableName": "Restored"})).await; + assert_eq!(desc_before["Table"]["TableStatus"], "CREATING"); + assert!( + desc_before["Table"]["RestoreSummary"]["RestoreInProgress"] + .as_bool() + .unwrap_or(false), + "{desc_before}" + ); + + // One cycle: the record's slot no task holds makes it dead at once; the + // resume is spawned through the runtime. + let mut state = ReconcilerState::with_shutdown(shutdown.child_token()); + let report = reconciler::cycle(&storage, &runtime, stale_job_after, "sqlite", &mut state) + .await + .expect("a reconciler cycle"); + assert_eq!(report.store_healthy, Some(true), "{report:?}"); + assert_eq!( + report.manifest_restores_resumed, + vec!["Restored".to_owned()], + "{report:?}" + ); + assert!(report.manifest_restores_failed.is_empty(), "{report:?}"); + assert!(report.legacy_restores_removed.is_empty(), "{report:?}"); + let taken = storage + .read_restore_job(&key) + .await + .expect("read") + .expect("the record"); + assert_ne!( + taken.owner_generation, dead_generation, + "the takeover minted a generation" + ); + assert_eq!(taken.spool_host, None, "a filesystem store has no spool"); + let deadline = std::time::Instant::now() + Duration::from_secs(60); + while runtime.running_jobs() > 0 { + assert!( + std::time::Instant::now() < deadline, + "the resumed restore did not end" + ); + tokio::time::sleep(Duration::from_millis(20)).await; + } + + // The table ACTIVE with every item once; the record DONE at the last + // file's count; the GSI answering. + let desc_after = wait_table_status(&ctx, "Restored", "ACTIVE").await; + assert_eq!( + desc_after["Table"]["ItemCount"], + json!(0), + "the restored table's ItemCount is 0 at ACTIVE, as the service reports it" + ); + assert!( + desc_after["Table"].get("RestoreSummary").is_none(), + "{desc_after}" + ); + assert_eq!(scan_count(&ctx, "Restored", None).await, ITEMS); + assert_eq!(scan_count(&ctx, "Restored", Some("gix")).await, ITEMS / 2); + assert_eq!(scan_count(&ctx, "Music", None).await, ITEMS); + let done = storage + .read_restore_job(&key) + .await + .expect("read") + .expect("the record"); + assert_eq!(done.status, JobStatus::Done); + assert!(done.completed_at.is_some()); + let last = manifest.data_files.len() - 1; + assert_eq!( + (done.file_index, done.items_applied), + ( + Some(u32::try_from(last).unwrap()), + Some(manifest.data_files[last].item_count) + ), + "the progress names the last file complete" + ); + assert_eq!(done.owner_generation, taken.owner_generation); + assert_eq!( + storage.backup_in_use(&row.key()).await.expect("in use"), + None + ); + + // A second cycle has nothing left to resume. + let report = reconciler::cycle(&storage, &runtime, stale_job_after, "sqlite", &mut state) + .await + .expect("a second cycle"); + assert!(report.manifest_restores_resumed.is_empty(), "{report:?}"); + assert!(report.manifest_restores_failed.is_empty(), "{report:?}"); + + catalog.close().await; + shutdown.cancel(); + for worker in workers { + let _ = tokio::time::timeout(Duration::from_secs(5), worker).await; + } +} diff --git a/crates/storage/src/backup_catalog.rs b/crates/storage/src/backup_catalog.rs new file mode 100644 index 000000000..52772fc7d --- /dev/null +++ b/crates/storage/src/backup_catalog.rs @@ -0,0 +1,1823 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The catalog index of backups and restore jobs, and the frozen reader of +//! the in-catalog backups earlier releases wrote. +//! +//! The engine (`crates/engine/src/backup.rs`) is backend-agnostic, and every +//! write to a catalog row (the claim, the heartbeat, the conditional status +//! writes, the store-scoped listings, the restore job record) has to be +//! written by the backend that holds the row. [`BackupCatalog`] is how the +//! engine reaches the index; [`JobClaim`] is the handle a job holds on its +//! row; [`LegacyBackupReader`] is the frozen path to backups stored in the +//! catalog itself (`backups` and `backup_items`), which stays until that +//! format is retired. +//! +//! Rows are addressed by [`BackupKey`] (account, backup id, ARN), built from +//! a parsed [`BackupArn`]; no method takes a bare backup id. A restore target +//! is addressed by [`RestoreTargetKey`] (account, name, table id), the table +//! id guarding against a table re-created under the name. +//! +//! Status mapping to the API ([`BackupStatus::reported`]): CREATING and +//! AVAILABLE as themselves; DELETING is reported as DELETED by DescribeBackup +//! and DeleteBackup and is not listed; DELETED and FAILED rows are tombstones +//! that are neither described nor listed. + +use std::sync::Arc; +use std::time::Duration; + +use extenddb_core::backup_format::{BackupArn, SourceTableSchema}; +use extenddb_core::types::{ + BackupDescription, BackupDetails, BackupSummary, BillingMode, CreateTableInput, + GlobalSecondaryIndexInfo, LocalSecondaryIndexInfo, ProvisionedThroughput, SourceTableDetails, + SourceTableFeatureDetails, TableDescription, +}; +use futures::future::BoxFuture; +use futures::stream::BoxStream; + +use crate::backup_definition::{BackupRestoreSource, RestoreRequest}; +use crate::error::StorageError; + +/// `BackupType` of every backup this server creates. +pub const BACKUP_TYPE_USER: &str = "USER"; + +/// The sentinel `owner_generation` the migration writes on a restore +/// registration that existed before the job record had one; no job ever +/// holds it. +pub const UNOWNED_GENERATION: u64 = 0; + +/// A fresh `owner_generation`: random, never [`UNOWNED_GENERATION`], and +/// below `i64::MAX` so every backend's integer column holds it. +#[must_use] +pub fn new_generation() -> u64 { + loop { + let g = rand::random::() >> 1; + if g != UNOWNED_GENERATION { + return g; + } + } +} + +/// Connections a `[storage.postgres] pool_size` must have for +/// `max_concurrent_jobs` jobs: twice the jobs, so a backend that admits at +/// most half its pool as open snapshots (#395) still admits every job the +/// permit allows, plus ten for requests. The caller takes the larger of this +/// and its own minimum pool size. A job's claim connection is outside the +/// pool and not counted here. +#[must_use] +pub fn pool_size_for_jobs(max_concurrent_jobs: usize) -> u32 { + let jobs = u32::try_from(max_concurrent_jobs).unwrap_or(u32::MAX); + jobs.saturating_mul(2).saturating_add(10) +} + +/// The floor of a job's heartbeat period on every backend: the store-backed +/// jobs run at `max([backup] heartbeat_interval_secs, JOB_HEARTBEAT_INTERVAL)` +/// (`BackupConfig::effective_heartbeat_interval_secs`, which the server +/// hands the jobs), the in-catalog paths and the maintenance claim at the +/// constant. `BackupConfig::validate` floors `stale_job_after_secs` on this +/// period, so a configured interval shorter than it cannot make the +/// reconciler declare a live job dead between two heartbeats. +pub const JOB_HEARTBEAT_INTERVAL: Duration = Duration::from_secs(15); + +/// The bound a job refusal names, so the text says which one was met +/// (an operator reading the engine's `max_concurrent_jobs` refusal and +/// SQLite's slot refusal, which differ in number, can tell them apart). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum JobBound<'a> { + /// A counted bound of this instance: `name` (the config key or the + /// backend's slot formula) admits `limit` jobs at once. + Counted { name: &'a str, limit: usize }, + /// The database server refused the job's dedicated connection + /// (PostgreSQL's `too_many_connections`): the bound is the server's own + /// connection limit, which this process does not know. + ServerConnections, +} + +/// The text of the `LimitExceededException` a CreateBackup or +/// RestoreTableFromBackup answers when a bound on running jobs is met. One +/// implementation: the engine's `max_concurrent_jobs` permit, a backend's own +/// admission (SQLite's snapshot slots) and PostgreSQL's claim connection +/// answer the same wire refusal, each naming its bound. +#[must_use] +pub fn limit_exceeded_message(bound: JobBound<'_>) -> String { + let detail = match bound { + JobBound::Counted { name, limit } => format!("at most {limit} at once, the {name} bound"), + JobBound::ServerConnections => { + "the database server has no connection slot left for the job".to_owned() + } + }; + format!("Too many backup and restore jobs are running on this server ({detail}); retry later") +} + +/// Where a backup's bytes live. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum StoreKind { + Filesystem, + S3, +} + +impl StoreKind { + /// The value stored in the `store_kind` column. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Filesystem => "filesystem", + Self::S3 => "s3", + } + } + + /// The kind a `store_kind` column holds. + #[must_use] + pub fn parse(s: &str) -> Option { + match s { + "filesystem" => Some(Self::Filesystem), + "s3" => Some(Self::S3), + _ => None, + } + } +} + +impl std::fmt::Display for StoreKind { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +/// The store a row belongs to: kind, the configured root at creation time, +/// and the id in the root's `extenddb-store.json`. Every duty that reads or +/// writes a backup row acts only on rows of the configured store. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct StoreIdentity { + pub store_kind: StoreKind, + pub store_root: String, + pub store_id: String, +} + +/// The status column of a backup row. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum BackupStatus { + Creating, + Available, + Deleting, + Deleted, + Failed, +} + +impl BackupStatus { + /// The value stored in the `status` column. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Creating => "CREATING", + Self::Available => "AVAILABLE", + Self::Deleting => "DELETING", + Self::Deleted => "DELETED", + Self::Failed => "FAILED", + } + } + + /// The status a column holds. + /// + /// # Errors + /// + /// `Internal` for a value no release writes. + pub fn parse(s: &str) -> Result { + match s { + "CREATING" => Ok(Self::Creating), + "AVAILABLE" => Ok(Self::Available), + "DELETING" => Ok(Self::Deleting), + "DELETED" => Ok(Self::Deleted), + "FAILED" => Ok(Self::Failed), + other => Err(StorageError::Internal(format!( + "backup row carries an unknown status {other:?}" + ))), + } + } + + /// The `BackupStatus` the API reports for a row of this status: DELETING + /// is reported as DELETED, so a client retrying DeleteBackup sees the + /// terminal status. + #[must_use] + pub const fn reported(self) -> &'static str { + match self { + Self::Deleting => "DELETED", + other => other.as_str(), + } + } + + /// Whether DescribeBackup answers for a row of this status (CREATING, + /// AVAILABLE, and DELETING as DELETED). DELETED and FAILED rows are + /// `BackupNotFoundException`. + #[must_use] + pub const fn is_described(self) -> bool { + matches!(self, Self::Creating | Self::Available | Self::Deleting) + } + + /// Whether ListBackups lists a row of this status (CREATING and + /// AVAILABLE only). + #[must_use] + pub const fn is_listed(self) -> bool { + matches!(self, Self::Creating | Self::Available) + } +} + +impl std::fmt::Display for BackupStatus { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +/// One backup row's address: the account, the backup id and the ARN as the +/// row stores it. Built from a parsed [`BackupArn`] or from a row read. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct BackupKey { + pub account_id: String, + pub backup_id: String, + pub backup_arn: String, +} + +impl BackupKey { + /// The key of the row `arn` names. + #[must_use] + pub fn from_arn(arn: &BackupArn) -> Self { + Self { + account_id: arn.account_id().to_owned(), + backup_id: arn.backup_id().to_owned(), + backup_arn: arn.to_arn_string(), + } + } +} + +/// One restore target's address. The table id guards against a table +/// re-created under the same name after the target was removed. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct RestoreTargetKey { + pub account_id: String, + pub table_name: String, + pub table_id: String, +} + +/// The `TimeRangeLowerBound` (inclusive) and `TimeRangeUpperBound` +/// (exclusive) of ListBackups, epoch seconds, applied to `created_at`. +#[derive(Debug, Clone, Copy, Default, PartialEq)] +pub struct TimeBounds { + pub lower: Option, + pub upper: Option, +} + +impl TimeBounds { + /// Whether `created_at` (epoch seconds) is within the bounds. + #[must_use] + pub fn contains(&self, created_at: f64) -> bool { + self.lower.is_none_or(|lower| created_at >= lower) + && self.upper.is_none_or(|upper| created_at < upper) + } +} + +/// A row of `backup_index`. Times are epoch seconds; `created_at` is the +/// id's instant, so `BackupCreationDateTime`, the ARN and the ListBackups +/// cursor agree. +#[derive(Debug, Clone, PartialEq)] +pub struct BackupRow { + pub backup_id: String, + pub account_id: String, + pub table_name: String, + pub backup_arn: String, + pub backup_name: String, + pub status: BackupStatus, + pub store: StoreIdentity, + pub size_bytes: i64, + pub item_count: i64, + pub created_at: f64, + pub completed_at: Option, + pub failure_reason: Option, + /// The manifest's `source_table` object, verbatim + /// ([`SourceTableSchema`]); the API's source-table shapes derive from it. + pub source_table_details_json: serde_json::Value, + pub owner_generation: u64, + /// The last heartbeat by the owning job, on the database's clock. + pub heartbeat_at: f64, +} + +impl BackupRow { + /// The row's address. + #[must_use] + pub fn key(&self) -> BackupKey { + BackupKey { + account_id: self.account_id.clone(), + backup_id: self.backup_id.clone(), + backup_arn: self.backup_arn.clone(), + } + } + + /// The source table the row recorded. + /// + /// # Errors + /// + /// `Internal` when `source_table_details_json` is not a manifest + /// `source_table` object. + pub fn source_table(&self) -> Result { + serde_json::from_value(self.source_table_details_json.clone()).map_err(|e| { + StorageError::Internal(format!( + "backup {} carries a source table this build cannot read: {e}", + self.backup_arn + )) + }) + } + + /// `BackupDetails` as DescribeBackup and DeleteBackup report them, with + /// the status mapped by [`BackupStatus::reported`]. + #[must_use] + pub fn details(&self) -> BackupDetails { + BackupDetails { + backup_arn: self.backup_arn.clone(), + backup_name: self.backup_name.clone(), + backup_status: self.status.reported().to_owned(), + backup_type: BACKUP_TYPE_USER.to_owned(), + backup_size_bytes: self.size_bytes, + backup_creation_date_time: self.created_at, + } + } + + /// The DescribeBackup description: the details, `SourceTableDetails` and + /// `SourceTableFeatureDetails` derived from the recorded source table. + /// `SourceTableDetails.ProvisionedThroughput` is always set (`{0, 0}` for + /// an on-demand table, as the service reports it); a GSI's throughput is + /// `{0, 0}` on an on-demand table too. `SourceTableFeatureDetails` is + /// always set, `{}` for a source that recorded no index, stream, TTL or + /// SSE (as the service reports it, measured on DynamoDB); the inner + /// members the source recorded nothing for are omitted. + /// + /// # Errors + /// + /// As [`Self::source_table`]. + pub fn description(&self) -> Result { + let source = self.source_table()?; + let zero = ProvisionedThroughput { + read_capacity_units: 0, + write_capacity_units: 0, + }; + let on_demand = source.billing_mode == BillingMode::PayPerRequest; + let provisioned_throughput = if on_demand { + Some(zero.clone()) + } else { + source + .provisioned_throughput + .clone() + .or_else(|| Some(zero.clone())) + }; + let features = SourceTableFeatureDetails { + local_secondary_indexes: (!source.local_secondary_indexes.is_empty()).then(|| { + source + .local_secondary_indexes + .iter() + .map(|i| LocalSecondaryIndexInfo { + index_name: i.index_name.clone(), + key_schema: i.key_schema.clone(), + projection: i.projection.clone(), + }) + .collect() + }), + global_secondary_indexes: (!source.global_secondary_indexes.is_empty()).then(|| { + source + .global_secondary_indexes + .iter() + .map(|i| GlobalSecondaryIndexInfo { + index_name: i.index_name.clone(), + key_schema: i.key_schema.clone(), + projection: i.projection.clone(), + provisioned_throughput: if on_demand { + Some(zero.clone()) + } else { + i.provisioned_throughput.clone() + }, + }) + .collect() + }), + stream_description: source.stream_specification.clone(), + time_to_live_description: source.time_to_live.clone(), + sse_description: source.sse_description.clone(), + }; + Ok(BackupDescription { + backup_details: self.details(), + source_table_details: SourceTableDetails { + table_name: source.table_name.clone(), + table_id: source.table_id.clone(), + table_arn: source.table_arn.clone(), + key_schema: source.key_schema.clone(), + item_count: source.item_count, + table_size_bytes: source.table_size_bytes, + billing_mode: Some(billing_mode_name(source.billing_mode).to_owned()), + table_creation_date_time: source.table_creation_date_time, + provisioned_throughput, + on_demand_throughput: source.on_demand_throughput.clone(), + }, + source_table_feature_details: Some(features), + }) + } + + /// The ListBackups summary. It needs only `table_id` and `table_arn` of + /// the recorded source table, and reads those two members directly, so a + /// row whose `source_table_details_json` a later build extended with a + /// member this build requires still lists; DescribeBackup of the same + /// row is the internal error [`Self::description`] reports. + /// + /// # Errors + /// + /// `Internal` when the recorded source carries no `table_id` or + /// `table_arn` string. + pub fn summary(&self) -> Result { + let member = |name: &str| -> Result { + self.source_table_details_json + .get(name) + .and_then(serde_json::Value::as_str) + .map(str::to_owned) + .ok_or_else(|| { + StorageError::Internal(format!( + "backup {} carries a source table without a {name} this build can read", + self.backup_arn + )) + }) + }; + Ok(BackupSummary { + backup_arn: self.backup_arn.clone(), + backup_name: self.backup_name.clone(), + table_name: self.table_name.clone(), + table_id: member("table_id")?, + table_arn: member("table_arn")?, + backup_status: self.status.reported().to_owned(), + backup_type: BACKUP_TYPE_USER.to_owned(), + backup_size_bytes: self.size_bytes, + backup_creation_date_time: self.created_at, + }) + } +} + +/// The wire name of a billing mode. +#[must_use] +pub const fn billing_mode_name(mode: BillingMode) -> &'static str { + match mode { + BillingMode::Provisioned => "PROVISIONED", + BillingMode::PayPerRequest => "PAY_PER_REQUEST", + } +} + +/// What `start_backup_delete` found under the row's lock. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum DeleteStart { + /// The row was AVAILABLE and is now DELETING. + Deleting, + /// A restore target registered from the backup is CREATING; its name. + InUse(String), + /// The row was not AVAILABLE; its status. + Status(BackupStatus), +} + +/// What [`BackupCatalog::forget_available_backup_row`] did. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ForgetRow { + /// The row was AVAILABLE and is gone. + Removed, + /// A restore target registered from the backup is CREATING; its name. + /// The row is left. + InUse(String), + /// The row was not AVAILABLE; its status. The row is left. + Status(BackupStatus), +} + +/// What a heartbeat found. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Heartbeat { + /// The row still carries the claim's generation and the claim is held. + Confirmed, + /// The row is gone, carries another generation, or the claim connection + /// no longer holds the claim. The caller treats an `Err` the same way. + Lost, +} + +/// The status a job record moves through: CREATING, AVAILABLE and FAILED for +/// a backup row; RUNNING, DONE and FAILED for a restore job record. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum JobStatus { + Creating, + Available, + Running, + Done, + Failed, +} + +impl JobStatus { + /// The value stored in the record's status column. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Creating => "CREATING", + Self::Available => "AVAILABLE", + Self::Running => "RUNNING", + Self::Done => "DONE", + Self::Failed => "FAILED", + } + } + + /// The status a column holds. + /// + /// # Errors + /// + /// `Internal` for a value no release writes. + pub fn parse(s: &str) -> Result { + match s { + "CREATING" => Ok(Self::Creating), + "AVAILABLE" => Ok(Self::Available), + "RUNNING" => Ok(Self::Running), + "DONE" => Ok(Self::Done), + "FAILED" => Ok(Self::Failed), + other => Err(StorageError::Internal(format!( + "job record carries an unknown status {other:?}" + ))), + } + } +} + +impl std::fmt::Display for JobStatus { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +/// The fields a status write sets beside the status. A `None` leaves the +/// column as it is; `completed` stamps `completed_at` with the database's +/// `now()`. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct StatusFields { + pub size_bytes: Option, + pub item_count: Option, + pub failure_reason: Option, + /// The manifest's `source_table` object, which the AVAILABLE write of a + /// backup row carries so DescribeBackup reports the counts the manifest + /// has; a restore job record has no such column and ignores it. + pub source_table_details_json: Option, + pub completed: bool, +} + +/// The column an age in [`BackupCatalog::list_backup_rows_older_than`] is +/// measured on, against the database's own `now()`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum StaleClock { + /// `heartbeat_at`: the owner's last sign of life. + Heartbeat, + /// `completed_at`: when the row reached its terminal status. A row with + /// `completed_at` unset is never older than any age on this clock. + Completed, +} + +/// What a conditional write did. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum WriteOutcome { + /// Exactly one row matched the condition and was written. + Matched, + /// No row matched: the record is gone, carries another generation, or + /// is not in the `from` status. + NotMatched, + /// The statement may or may not have been applied (the connection failed + /// after it was sent). The caller re-reads the record ([`JobClaim::read`]) + /// rather than treating this as a failure. + Unknown, +} + +/// A job record as [`JobClaim::read`] returns it. +#[derive(Debug, Clone, PartialEq)] +pub struct JobRow { + pub status: JobStatus, + pub owner_generation: u64, + /// Epoch seconds, on the database's clock. + pub heartbeat_at: f64, + pub failure_reason: Option, +} + +/// How a restore job reads its backup. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum RestoreKind { + /// Through the [`LegacyBackupReader`]: an in-catalog backup. + Legacy, + /// From a manifest in the store. + Manifest, +} + +impl RestoreKind { + /// The value stored in the `kind` column. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Legacy => "legacy", + Self::Manifest => "manifest", + } + } + + /// The kind a column holds. + /// + /// # Errors + /// + /// `Internal` for a value no release writes. + pub fn parse(s: &str) -> Result { + match s { + "legacy" => Ok(Self::Legacy), + "manifest" => Ok(Self::Manifest), + other => Err(StorageError::Internal(format!( + "restore job carries an unknown kind {other:?}" + ))), + } + } +} + +/// A restore job record: the row [`BackupCatalog::claim_restore`] writes +/// beside a CREATING target, as [`BackupCatalog::read_restore_job`] and +/// [`BackupCatalog::list_restore_jobs`] return it, and the one the +/// reconciler acts on (RUNNING with a dead owner, or FAILED with its target +/// still CREATING, [`BackupCatalog::list_dead_restore_jobs`]). +#[derive(Debug, Clone, PartialEq)] +pub struct RestoreJobRow { + pub target: RestoreTargetKey, + pub source_backup_arn: String, + pub kind: RestoreKind, + /// RUNNING, DONE or FAILED; the reconciler reads it to decide whether + /// the FAILED write is still to be made. + pub status: JobStatus, + pub owner_generation: u64, + /// Epoch seconds, on the database's clock. + pub heartbeat_at: f64, + /// The store the job reads its backup from; `None` on a legacy record, + /// which reads the catalog. + pub store: Option, + /// `:` of the instance that downloads the backup + /// into its spool; `None` for a filesystem store, which is read in + /// place, and on a legacy record. A takeover rewrites it to the + /// claimant's. + pub spool_host: Option, + /// The last data file whose progress the job recorded, as the backend + /// records it (SQLite and Cassandra write it per batch, MongoDB per + /// file; PostgreSQL keeps its progress in the data database and leaves + /// this `None`). + pub file_index: Option, + /// Items of `file_index` applied so far, where the backend records + /// mid-file progress (SQLite and Cassandra). + pub items_applied: Option, + pub failure_reason: Option, + /// Epoch seconds, on the database's clock; set by the DONE and FAILED + /// writes. + pub completed_at: Option, +} + +impl RestoreJobRow { + /// The row as the four backends' `list_dead_restore_jobs` built it + /// before the restore job existed: the six members the reconciler's + /// duties on legacy and dead records read, the rest `None`. + #[must_use] + pub fn reconciler_row( + target: RestoreTargetKey, + source_backup_arn: String, + kind: RestoreKind, + status: JobStatus, + owner_generation: u64, + heartbeat_at: f64, + ) -> Self { + Self { + target, + source_backup_arn, + kind, + status, + owner_generation, + heartbeat_at, + store: None, + spool_host: None, + file_index: None, + items_applied: None, + failure_reason: None, + completed_at: None, + } + } +} + +/// What [`BackupCatalog::claim_restore`] returns: the CREATING target's +/// description with its `RestoreSummary`, and the claim on its record. +pub type RestoreRegistered = (TableDescription, Box); + +/// What a restore's registration writes beside the target table it creates +/// (the design's 5.3 step 4): the backup the job reads, the provenance the +/// target's `RestoreSummary` reports, the store, the claimant's spool and +/// the claim's generation. The record is born RUNNING and `kind = manifest`. +#[derive(Debug, Clone, PartialEq)] +pub struct RestoreRegistrationRow { + pub backup: BackupKey, + /// Reported as `RestoreSummary.SourceTableArn`. + pub source_table_arn: String, + /// Reported as `RestoreSummary.RestoreDateTime`, epoch seconds: the + /// backup's creation time, as the service reports it. + pub restore_date_time: f64, + pub store: StoreIdentity, + /// `:` when the job downloads into a spool (an S3 + /// store); `None` for a filesystem store. + pub spool_host: Option, + /// The claim's fencing token, written on the record with the target. + pub owner_generation: u64, +} + +/// The names of the pause points inside the backends' restore writes, one +/// spelling each: the backend passes its constant to +/// [`restore_pause_point`], and the kill-point harness names the same +/// constant in `EXTENDDB_BACKUP_PAUSE`. The engine's own kill points are its +/// `KillPoint` enum; these four live here because the backends cannot see +/// the engine. +/// +/// SQLite's `bulk_load`: between two batches of one file, after the first +/// batch's transaction committed with its progress. +pub const RESTORE_SQLITE_MID_BATCH: &str = "restore_sqlite_mid_batch"; +/// MongoDB's `bulk_load`: after the file's last `insert_many` returned, +/// before the record's `file_index` write. +pub const RESTORE_MONGODB_AFTER_INSERT_MANY: &str = "restore_mongodb_after_insert_many"; +/// Cassandra's `bulk_load`: after a batch's writes, before its progress +/// lightweight transaction. +pub const RESTORE_CASSANDRA_BEFORE_PROGRESS: &str = "restore_cassandra_before_progress"; +/// PostgreSQL's `finish_restore`: after the ACTIVE flip committed, before +/// the `restore_progress` rows are deleted. +pub const RESTORE_POSTGRES_BEFORE_PROGRESS_DELETE: &str = "restore_postgres_before_progress_delete"; + +/// A pause point inside a backend's `bulk_load` or its progress write, for +/// the binary-level kill-point harness: with `test-hooks` it pauses while +/// `EXTENDDB_BACKUP_PAUSE` names `name` (one of the constants above), until +/// the file `EXTENDDB_BACKUP_PAUSE_RELEASE` names exists, logging `restore +/// job reached kill point ` as the engine's own kill points do (one +/// harness drives both); without the feature it is a no-op. +#[cfg(feature = "test-hooks")] +pub async fn restore_pause_point(name: &str) { + if std::env::var("EXTENDDB_BACKUP_PAUSE").ok().as_deref() != Some(name) { + return; + } + let release = std::env::var("EXTENDDB_BACKUP_PAUSE_RELEASE").ok(); + tracing::warn!( + "restore job reached kill point {name}; paused until {} exists", + release + .as_deref() + .unwrap_or("no release file (the pause does not end)") + ); + loop { + if let Some(release) = &release + && std::path::Path::new(release).exists() + { + tracing::warn!("restore job released from kill point {name}"); + return; + } + tokio::time::sleep(Duration::from_millis(100)).await; + } +} + +/// [`restore_pause_point`] without `test-hooks`: nothing. +#[cfg(not(feature = "test-hooks"))] +#[allow(clippy::unused_async)] +pub async fn restore_pause_point(_name: &str) {} + +/// A legacy CREATING backup row whose create died. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AbandonedLegacyBackup { + pub account_id: String, + pub backup_arn: String, + /// Whether the backend removed the row, its items and its definition in + /// the call that reported it: SQLite does, in one write under its engine + /// write lock limited to CREATING rows whose slot no task holds, so a + /// create that commits AVAILABLE and frees its slot between a read and + /// the removal keeps its backup. The reconciler then only logs it. A + /// backend that only reports (Cassandra) leaves `false`, and the + /// reconciler removes the row through [`LegacyBackupReader::delete`]. + pub removed: bool, +} + +/// One of the `max_concurrent_jobs` slots of this instance, held by a backup +/// or restore job for its life. The engine takes it last among a request's +/// checks and hands it to the backend with the job; dropping it frees the +/// slot. +#[derive(Debug)] +pub struct JobPermit(#[allow(dead_code)] tokio::sync::OwnedSemaphorePermit); + +impl JobPermit { + /// Take a free slot of `slots`, or `None` when every slot is held, without + /// waiting. + #[must_use] + pub fn try_acquire(slots: &Arc) -> Option { + Arc::clone(slots).try_acquire_owned().ok().map(Self) + } + + /// Take a slot of `slots`, waiting up to `wait` for one to be freed, or + /// `None` when none was within that time. The request holds nothing while + /// it waits: the engine takes the permit last among a request's checks, + /// before the backend opens a snapshot or a transaction. A free slot is + /// taken at once, whatever `wait` is. + pub async fn acquire_within( + slots: &Arc, + wait: Duration, + ) -> Option { + tokio::time::timeout(wait, Arc::clone(slots).acquire_owned()) + .await + .ok() + .and_then(|permit| permit.ok().map(Self)) + } + + /// A permit of a slot nobody else can take: for a caller that runs a + /// job outside the engine's slots (a tool, a backend's own tests). + #[must_use] + pub fn detached() -> Self { + Self::try_acquire(&Arc::new(tokio::sync::Semaphore::new(1))).expect("a fresh slot") + } +} + +/// The text every backend and the engine answer for a backup a CREATING +/// restore target still reads (`BackupInUseException` on the wire): the +/// service's, which names the backup and not the table (measured on +/// DynamoDB: `Backup is being used to restore another table: `). One +/// implementation: the refusal is wire-visible and was spelled in each +/// backend before. The table's name goes to the log ([`backup_in_use`]). +#[must_use] +pub fn backup_in_use_message(backup_arn: &str) -> String { + format!("Backup is being used to restore another table: {backup_arn}") +} + +/// [`backup_in_use_message`] as the storage error the engine maps, with the +/// name of the CREATING target that holds the backup in use logged, since the +/// wire text does not carry it. +#[must_use] +pub fn backup_in_use(table: &str, backup_arn: &str) -> StorageError { + tracing::info!("backup {backup_arn} is in use: a restore into {table} is still CREATING"); + StorageError::BackupInUse(backup_in_use_message(backup_arn)) +} + +/// The one signal of a missing backup every backend emits inside a +/// `Validation` error, which the engine turns into `BackupNotFoundException` +/// ([`is_backup_not_found`]). +pub const BACKUP_NOT_FOUND: &str = "Backup not found"; + +/// The error for a backup the catalog has no row for, or one of another +/// account, or (on the legacy reader) one that is not AVAILABLE. +#[must_use] +pub fn backup_not_found(backup_arn: &str) -> StorageError { + StorageError::Validation(format!("{BACKUP_NOT_FOUND}: {backup_arn}")) +} + +/// Whether a backend error says the backup is not there. +#[must_use] +pub fn is_backup_not_found(e: &StorageError) -> bool { + matches!(e, StorageError::Validation(m) if m.contains(BACKUP_NOT_FOUND)) +} + +/// The catalog index of backups and restore jobs, implemented by each +/// backend. The engine reaches every row through it; the per-backend +/// mechanics (the lock on a row, the claim, the clock) are the backend's. +pub trait BackupCatalog: Send + Sync { + /// The catalog's random id, written at `init` and by the migration that + /// created the index for an existing catalog. + fn catalog_id(&self) -> BoxFuture<'_, Result>; + + /// Insert `row` with status CREATING under a new claim, and return the + /// claim. The claim precedes the insert or is the insert itself, so no + /// CREATING row ever exists unowned (the AVAILABLE rows + /// [`Self::insert_backup_row`] writes are unowned and have no job). + /// + /// # Errors + /// + /// `AlreadyExists` when a live owner holds the row's claim or the row + /// exists. + fn claim_backup<'a>( + &'a self, + row: &'a BackupRow, + ) -> BoxFuture<'a, Result, StorageError>>; + + /// Insert `row` unowned, if no row with its key exists: `backup sync`'s + /// insert for a prefix it verified. The row is written as given (its + /// status, sync's being AVAILABLE, its store, sizes, `created_at`, + /// `failure_reason` and source table) except for three columns: + /// `owner_generation` is [`UNOWNED_GENERATION`], and `completed_at` and + /// `heartbeat_at` are the database's clock; the row's own values for + /// those are ignored. No claim is taken: an unowned row has no job. + /// Sync passes AVAILABLE; an unowned row of another status has no job to + /// move it (on SQLite an unowned CREATING row has no slot, and the next + /// cycle resolves it as dead). + /// + /// Returns `false`, writing nothing, when a row with the key exists, of + /// whatever status or store, with one exception on Cassandra: a stored + /// row equal to this one in every column the insert writes (unowned, the + /// same store, sizes and source table) counts as inserted, and its list + /// and status entries are written, which completes an insert that + /// stopped between the row and its entries. On PostgreSQL, SQLite and + /// MongoDB, which + /// index `backup_id` alone as unique, a row with the backup id under + /// another account also exists for this rule. Cassandra keys its rows + /// by the account and the ARN and indexes no backup id, so there the + /// rule is per ARN spelling: a row for the same id under another region + /// or partition does not count. `backup sync` closes that gap itself: + /// before the insert it reads the row under each spelling of the + /// manifest's ARN (this deployment's region and the manifest's own) and + /// skips a backup the other spelling has a row for, so sync never gives + /// an account two rows of one backup id under the manifest's account, + /// table and id on any backend. A row under the + /// manifest's own spelling is left to this method, which answers `false` + /// for it or, on Cassandra, completes it under the exception above. + /// + /// # Errors + /// + /// The backend's error for a failed write. On Cassandra an answer lost + /// in transit, and a `NotMatched` answer, is settled by a serial read of + /// the row: this row counts as inserted and another as existing. No row + /// after a lost answer means not written, and the write is sent once + /// more; no row after `NotMatched` answers `false` (a legacy in-catalog + /// backup holds the ARN, which the read does not return). An outcome + /// still unknown after that is an error. A failed batch of the entries + /// is sent once more before the error, and the next insert of the same + /// row writes them. + fn insert_backup_row<'a>( + &'a self, + row: &'a BackupRow, + ) -> BoxFuture<'a, Result>; + + /// Every row of `store` with `status`, or of every status for `None`. + fn list_backup_rows<'a>( + &'a self, + status: Option, + store: &'a StoreIdentity, + ) -> BoxStream<'a, Result>; + + /// The newest AVAILABLE row of `store` (the last in the + /// (`created_at`, `backup_arn`) order [`Self::list_backup_rows`] lists + /// in), or `None` when the store has none. The store's manifest probe + /// reads it each reconciler cycle, so it reads one row and not the + /// whole listing. + fn newest_available_backup_row<'a>( + &'a self, + store: &'a StoreIdentity, + ) -> BoxFuture<'a, Result, StorageError>>; + + /// The account's rows, of `table_name` when given, created within + /// `bounds`, of every status: the caller decides which statuses are + /// listed ([`BackupStatus::is_listed`]). + fn list_account_backup_rows<'a>( + &'a self, + account_id: &'a str, + table_name: Option<&'a str>, + bounds: &'a TimeBounds, + ) -> BoxStream<'a, Result>; + + /// The row `arn` names, read by the ARN's account and backup id and + /// returned only when the row's `account_id`, `table_name` and + /// `backup_arn` equal the ARN's; `None` otherwise, of whatever status. + fn read_backup_row<'a>( + &'a self, + arn: &'a BackupArn, + ) -> BoxFuture<'a, Result, StorageError>>; + + /// DeleteBackup's first step: under the row's lock, the in-use check and + /// the AVAILABLE-to-DELETING write in one transaction. + /// + /// # Errors + /// + /// `Validation("Backup not found: ")` when there is no such row. + fn start_backup_delete<'a>( + &'a self, + key: &'a BackupKey, + ) -> BoxFuture<'a, Result>; + + /// DELETING to DELETED, conditional on the row being DELETING; `false` + /// when it was not. + fn set_backup_deleted<'a>( + &'a self, + key: &'a BackupKey, + ) -> BoxFuture<'a, Result>; + + /// Remove a DELETED or FAILED tombstone; `false` when the row is not one. + fn remove_backup_row<'a>( + &'a self, + key: &'a BackupKey, + ) -> BoxFuture<'a, Result>; + + /// Remove an AVAILABLE row outright, leaving no DELETING or DELETED + /// tombstone, so no deletion duty ever acts on the backup's objects: + /// `backup sync --prune-missing`'s write for a row whose manifest the + /// configured root does not hold, whose bytes may be under another + /// host's root of the same path. Under the row's lock, as + /// [`Self::start_backup_delete`] takes it, the status check and the + /// in-use rule in one step with the removal: a row a CREATING restore + /// target reads is left ([`ForgetRow::InUse`]), and so is a row another + /// writer moved ([`ForgetRow::Status`]). The store is not touched. + /// + /// # Errors + /// + /// `Validation("Backup not found: ")` when there is no such row. + fn forget_available_backup_row<'a>( + &'a self, + key: &'a BackupKey, + ) -> BoxFuture<'a, Result>; + + /// Remove a CREATING row whose owner is dead: `heartbeat_at` older than + /// `stale_after` on the catalog's clock. `false` when the row is not + /// CREATING, is gone, or has a live owner. The maintenance command's + /// write for a CREATING row its wait saw no live owner for, after it has + /// deleted the row's prefix; the row's own job, were it alive, would + /// find its claim lost at its next heartbeat. + fn remove_dead_creating_backup_row<'a>( + &'a self, + key: &'a BackupKey, + stale_after: Duration, + ) -> BoxFuture<'a, Result>; + + /// Take over a CREATING row whose owner is dead, under a fresh + /// generation, and return the claim; the reconciler's way onto a row a + /// job left behind. Dead is `heartbeat_at` older than `stale_after` on + /// the database's clock (on PostgreSQL, once the owner's claim session + /// has ended as well: the advisory lock is tried first, so a stopped + /// owner whose session lives keeps its row), except on SQLite, where a + /// CREATING row whose in-process slot no task holds is dead at once (a + /// restart frees every slot, a job that panics drops its slot) and + /// `stale_after` is not read. + /// `heartbeat_at` is left as the dead owner wrote it, so the stale + /// window of the FAILED-prefix duty counts from the owner's last beat + /// and not from the takeover. `None` when the row is not CREATING, is + /// gone, has a live owner, or is already claimed by another task; the + /// old generation's writes through its [`JobClaim`] find + /// [`WriteOutcome::NotMatched`] from then on. + fn claim_dead_backup<'a>( + &'a self, + key: &'a BackupKey, + stale_after: Duration, + ) -> BoxFuture<'a, Result>, StorageError>>; + + /// The rows of `store` with `status` whose `clock` column is older than + /// `age` on the database's clock: the maintenance command's test of a + /// live owner (a CREATING or FAILED row it does not list under the + /// window on [`StaleClock::Heartbeat`] has one), and the reconciler's + /// listing for the FAILED-prefix delay (`Failed`, + /// [`StaleClock::Heartbeat`]) and the tombstone retention (`Failed` or + /// `Deleted`, [`StaleClock::Completed`]). The comparison is the + /// backend's, in its own query, so the engine never compares a row's + /// timestamp with a clock of its own. + fn list_backup_rows_older_than<'a>( + &'a self, + status: BackupStatus, + store: &'a StoreIdentity, + clock: StaleClock, + age: Duration, + ) -> BoxStream<'a, Result>; + + /// Remove the restore registrations a backend keeps beside its rows that + /// name a target no longer a CREATING restore target of that backup, and + /// return the ARNs of the backups touched. Only Cassandra keeps such a + /// map (`active_restores`, maintained by lightweight transactions whose + /// removal step can be lost with its owner); every other backend derives + /// the in-use check from the target rows themselves and has nothing to + /// sweep, which is what the default body says. + fn sweep_restore_registrations(&self) -> BoxFuture<'_, Result, StorageError>> { + Box::pin(async { Ok(Vec::new()) }) + } + + /// The name of a CREATING restore target registered from the backup, if + /// any: the rule under which DeleteBackup and a second restore of the + /// backup are refused with `BackupInUseException`. Reads the target's + /// status, not the job's. + fn backup_in_use<'a>( + &'a self, + key: &'a BackupKey, + ) -> BoxFuture<'a, Result, StorageError>>; + + /// Remove a CREATING restore target, its job record and its progress + /// rows, conditional on the table still being a CREATING restore target + /// with this table id, so this and the ACTIVE flip exclude each other. + /// `false` when the table is no longer one. Not DeleteTable, which + /// refuses a restore target. Also resumes, from the drop, the removal of + /// a FAILED record's target that an earlier call left DELETING with its + /// registration (the state [`Self::list_dead_restore_jobs`] names). + fn remove_restore_target<'a>( + &'a self, + key: &'a RestoreTargetKey, + ) -> BoxFuture<'a, Result>; + + /// [`Self::remove_restore_target`] for the `extenddb backup + /// remove-target` command, which runs with no claim of its own: a + /// backend whose owner holds a lock that outlives its heartbeat + /// (PostgreSQL's per-session advisory lock) takes that lock here first, + /// so the removal and a reconciler's takeover of the same target exclude + /// each other and a frozen owner whose session still holds the lock + /// keeps its target; `false` when the lock is held. The default is + /// [`Self::remove_restore_target`]: the job's own failure path and the + /// reconciler's resume already hold the claim when they remove, so the + /// lock lives only on this variant, and the other backends fence the + /// takeover on the record's generation and heartbeat alone. + fn remove_restore_target_exclusive<'a>( + &'a self, + key: &'a RestoreTargetKey, + ) -> BoxFuture<'a, Result> { + self.remove_restore_target(key) + } + + /// The restore job records the reconciler acts on, two kinds: RUNNING + /// records whose `heartbeat_at` is older than `stale_after` on the + /// database's clock (the owner is dead by every backend's rule; the + /// reconciler fails or resumes them), and FAILED records whose table is + /// still a CREATING restore target (the FAILED write landed and the + /// target's removal did not; the reconciler removes the target), listed + /// once the record's `completed_at`, or its `heartbeat_at` where that is + /// unset, is older than `stale_after`, so a job still on its own failure + /// path is left to finish it. On a backend whose removal is more than one + /// write (Cassandra: a claim CREATING to DELETING, then the drop), a + /// FAILED record whose target is DELETING with its registration is + /// returned once stale on the same rule, and + /// [`Self::remove_restore_target`] resumes the removal; PostgreSQL and + /// SQLite remove in one transaction and never leave that state, and + /// MongoDB deletes the document first, so neither does it. + fn list_dead_restore_jobs( + &self, + stale_after: Duration, + ) -> BoxFuture<'_, Result, StorageError>>; + + /// Every RUNNING restore job record, by status alone, whatever its + /// heartbeat says: the maintenance command's wait set, which must see a + /// record stamped in the listing's own instant (or ahead of the + /// listing's clock) as well as an old one. Not a liveness test. + fn list_running_restore_jobs(&self) -> BoxFuture<'_, Result, StorageError>>; + + /// RUNNING to FAILED with `reason` on the target's job record, conditional + /// on RUNNING and on the record still carrying the `owner_generation` and + /// `heartbeat_at` the reconciler listed (design 4(e)): a heartbeat that + /// lands after the listing changes `heartbeat_at`, so the write matches + /// nothing and the live owner keeps its record. The write sets a fresh + /// `owner_generation`, fencing the dead job's writes out. `false` when the + /// record was not RUNNING under those values or is gone. The reconciler's + /// write for a dead legacy restore, which is never resumed, and for a + /// manifest restore it cannot resume (a store other than the configured + /// one, a maintenance claim held). + fn fail_restore_job<'a>( + &'a self, + key: &'a RestoreTargetKey, + owner_generation: u64, + heartbeat_at: f64, + reason: &'a str, + ) -> BoxFuture<'a, Result>; + + /// A store-backed restore's registration (the design's 5.3 step 4): + /// create the target table CREATING with no scheduled transition and + /// its RUNNING `kind = manifest` job record, `registration`'s store, + /// `spool_host` and generation on it, under the + /// lock on the backup's `backup_index` row that + /// [`Self::start_backup_delete`] takes, re-checking there that the row + /// is AVAILABLE and that no CREATING target of the backup exists. The + /// catalog rows commit first and the target's data tables (on MongoDB + /// its collections and index metadata) are created after the commit, in + /// the order [`RestoreEngine::restore_target_ready`]'s doc states per + /// backend, so a kill right after the commit leaves a RUNNING record + /// over a target with no data table; the reconciler's resume checks + /// `restore_target_ready` before it resumes a record. A + /// DeleteBackup and a registration of one backup therefore order wholly + /// one before the other. Returns the CREATING description, its + /// `RestoreSummary` filled from `registration`, and the claim the job + /// heartbeats and writes its status through; the record is born owned, + /// as [`Self::claim_backup`]'s row is. The record's `heartbeat_at` is + /// the database's clock as the backend stamps it at the insert: on + /// PostgreSQL `now()` is the registration transaction's start, so a + /// registration that waited on the `backup_index` row's lock, or whose + /// data DDL after the commit took long, hands back a record whose + /// heartbeat is that much older than its return; the engine's first beat, + /// sent before the registration's reply, refreshes it. + /// + /// # Errors + /// + /// `Validation("Backup not found: ")` ([`is_backup_not_found`]) + /// when the row is missing or not AVAILABLE; `BackupInUse` when a target + /// of the backup is CREATING; `TableAlreadyExists` when the name is + /// taken, or `TableBeingRestored` when it is a CREATING restore target; + /// `LimitExceeded` when the backend admits no further job. + fn claim_restore<'a>( + &'a self, + account_id: &'a str, + input: CreateTableInput, + registration: &'a RestoreRegistrationRow, + ) -> BoxFuture<'a, Result>; + + /// Take over a RUNNING manifest record whose owner is dead (the design's + /// 4(e)): a fresh generation, the claimant's `spool_host` and + /// `heartbeat_at` set to the database's clock, in one conditional write, + /// so a second taker finds the record live at once. The refresh is where + /// this takeover differs from [`Self::claim_dead_backup`]: the backup + /// takeover leaves `heartbeat_at` because the FAILED-prefix duty's stale + /// window counts from it, while nothing waits on a restore record's old + /// heartbeat after a takeover (the FAILED-target duty ages on + /// `completed_at`, which every FAILED write of a resume stamps, and the + /// maintenance claim waits on RUNNING, not on the beat); without the + /// refresh a second reconciler's cycle landing before the successor's + /// first beat (one period after the takeover) would take the same record + /// again. Dead is `heartbeat_at` older than `stale_after` on the + /// database's clock; on PostgreSQL the owner's advisory lock on the + /// target name is tried first, so a stopped owner whose session lives + /// keeps its record, as [`Self::claim_dead_backup`] says; on SQLite the + /// name's in-process slot is the test instead and `stale_after` is not + /// read: a record whose slot no task of this process holds is dead at + /// once (a restart frees every slot and ages every RUNNING record), and + /// one whose slot a task holds is alive whatever its heartbeat says. + /// `None` when the record is not RUNNING, is gone, or has a live owner; + /// the old generation's writes find [`WriteOutcome::NotMatched`] from + /// then on. On Cassandra the takeover also re-applies the target's + /// `active_restores` entry on the backup's row when it is absent (a + /// crash between the target's DDL and the registration), so the backup + /// stays in use for the resumed load. + fn claim_restore_dead<'a>( + &'a self, + key: &'a RestoreTargetKey, + stale_after: Duration, + spool_host: Option<&'a str>, + ) -> BoxFuture<'a, Result>, StorageError>>; + + /// The job record of the target `key` names, of whatever status; `None` + /// when the table is not a restore target with this table id. + fn read_restore_job<'a>( + &'a self, + key: &'a RestoreTargetKey, + ) -> BoxFuture<'a, Result, StorageError>>; + + /// Every job record with `status`, or of every status for `None`: the + /// reconciler's listing of RUNNING targets for its sweeps. + fn list_restore_jobs<'a>( + &'a self, + status: Option, + ) -> BoxFuture<'a, Result, StorageError>>; + + /// The distinct `store_id`s the catalog's rows record for a root, of + /// every status: the store identity check reads them before it knows an + /// id. Empty when no row names the root. + fn read_store_ids<'a>( + &'a self, + store_kind: StoreKind, + store_root: &'a str, + ) -> BoxFuture<'a, Result, StorageError>>; + + /// Take the catalog-level maintenance claim for `holder` (a label naming + /// the command and the host it runs on, for the operator who finds it + /// held). The claim is one record, `backup_maintenance`, born owned + /// with a fresh generation and heartbeated through the handle; a holder + /// whose heartbeat is older than `stale_after` on the catalog's clock is + /// dead and its claim is taken over. PostgreSQL is the exception to the + /// liveness rule on the holder's side: there the holder's session holds + /// an advisory lock, a holder whose session ended is dead whatever its + /// heartbeat says, and `stale_after` is read by [`check_maintenance_claim`] + /// alone; on SQLite a server's startup under the serve lock removes the + /// record outright, since no holder can be alive there. + /// + /// [`check_maintenance_claim`]: BackupCatalog::check_maintenance_claim + /// + /// # Errors + /// + /// `AlreadyExists` naming the holder when a live one holds the claim: a + /// second maintenance command fails at once. + fn maintenance_claim<'a>( + &'a self, + holder: &'a str, + stale_after: Duration, + ) -> BoxFuture<'a, Result, StorageError>>; + + /// Whether the catalog-level maintenance claim is held by a live holder: + /// a record whose heartbeat is younger than `stale_after` on the + /// catalog's clock. Jobs check it and refuse while it is held; the + /// reconciler runs only the duties the holder waits on. Where in the + /// job's order the check runs depends on the job: the in-catalog create + /// and restore of this release commit their row inside the backend and + /// detach, so the engine checks after every other refusal and before the + /// row exists (the engine's backup module states it); the store-backed + /// job of a later release owns its row from the engine and checks after + /// its own claim and row exist (the design's 5.1 step 2). + fn check_maintenance_claim( + &self, + stale_after: Duration, + ) -> BoxFuture<'_, Result>; +} + +/// The holder's handle on the catalog-level maintenance claim: the +/// heartbeat that keeps it, and the release that ends it. Dropping the handle +/// without `release` leaves the record for the stale rule to expire. +pub trait MaintenanceClaim: Send + Sync { + /// The fencing token the claim was taken with. + fn generation(&self) -> u64; + + /// Refresh `heartbeat_at` on the catalog's clock and confirm the claim is + /// still this holder's. The caller treats an `Err` as [`Heartbeat::Lost`]. + fn heartbeat(&self) -> BoxFuture<'_, Result>; + + /// Remove the record, conditioned on the generation; a record another + /// holder took over is left to it. Idempotent. + fn release(&self) -> BoxFuture<'_, Result<(), StorageError>>; +} + +/// A job's handle on its row: the claim that excludes a second owner and +/// lets a successor detect a dead one, and the writes the job makes through +/// it, each conditioned on the claim's generation. +pub trait JobClaim: Send + Sync { + /// The fencing token the claim was taken with. + fn generation(&self) -> u64; + + /// Refresh `heartbeat_at` on the database's clock and confirm the claim + /// is still held. The caller treats an `Err` as [`Heartbeat::Lost`]. + fn heartbeat(&self) -> BoxFuture<'_, Result>; + + /// [`Self::heartbeat`] under `bound`: `None` when the beat did not + /// return within it, which the caller treats as a lost claim. The default + /// bounds the whole call. A backend whose beat queues behind a lock of + /// its own process before it reaches the catalog (SQLite's engine write + /// lock, held by a GSI backfill or a large write for as long as it + /// takes) measures the bound from the moment the beat holds that lock, + /// so a writer holding it past a period delays the beat and does not end + /// the job: the queue is this process's, not a sign the claim is lost. + fn heartbeat_within( + &self, + bound: Duration, + ) -> BoxFuture<'_, Option>> { + Box::pin(async move { tokio::time::timeout(bound, self.heartbeat()).await.ok() }) + } + + /// Whether [`Self::heartbeat_within`] can wait outside its bound before + /// the bound starts (SQLite's beat queues behind the engine write lock + /// of its own process, so the wait is this process's and never a sign of + /// a takeover). The engine's pre-put check treats a beat in flight as + /// confirming the claim for as long as it waits where this is `true`, + /// and only within one bound of the beat's start where it is `false` + /// (the default): a process stopped mid-beat and resumed would otherwise + /// pass the check until its timer raised the loss, while another owner + /// may have taken the row over during the stop. + fn heartbeat_parks_before_its_bound(&self) -> bool { + false + } + + /// Move the record from `from` to `to`, conditioned on the claim's + /// generation and on the record being in `from`. + fn set_status<'a>( + &'a self, + from: JobStatus, + to: JobStatus, + fields: &'a StatusFields, + ) -> BoxFuture<'a, Result>; + + /// Re-read the record, after a write whose outcome is unknown. + fn read(&self) -> BoxFuture<'_, Result>; +} + +/// The frozen path to backups stored in the catalog itself: the create, +/// describe, list, delete, restore-source and restore paths of the +/// in-catalog format, which each backend keeps as a copy of its own code for +/// reading and decoding that shape. The engine consults the catalog index +/// first and falls back to this reader; `create` is the CreateBackup path +/// while no `[backup.store]` is configured. +/// +/// The one part that is not frozen is the restore's registration: a restore +/// registers through the restore job fields (`kind = legacy`, the claim), so +/// the reconciler finds a restore whose owner died as it finds any other. +pub trait LegacyBackupReader: Send + Sync { + /// Create an in-catalog backup of a table, snapshotting all items. The + /// permit is the job's slot, released when the create ends. + fn create( + &self, + account_id: &str, + table_name: &str, + backup_name: &str, + permit: JobPermit, + ) -> BoxFuture<'_, Result>; + + /// Describe a backup by ARN, scoped to the owning account. + /// + /// Backups belong to an account, so implementations must match on both + /// `account_id` and `backup_arn` and report a missing backup when the ARN + /// belongs to another account. + fn describe( + &self, + account_id: &str, + backup_arn: &str, + ) -> BoxFuture<'_, Result>; + + /// List the account's in-catalog backups, optionally only those of + /// `table_name`, in any order. Each summary carries the source table's + /// `TableId`. + fn list( + &self, + account_id: &str, + table_name: Option<&str>, + ) -> BoxFuture<'_, Result, StorageError>>; + + /// Delete a backup by ARN, scoped to the owning account; an ARN owned by + /// another account is reported as missing. `BackupInUse` while a restore + /// target of the backup is CREATING. + fn delete( + &self, + account_id: &str, + backup_arn: &str, + ) -> BoxFuture<'_, Result>; + + /// The backup's recorded source, for an AVAILABLE backup owned by + /// `account_id`. + /// + /// The engine resolves the restore target and runs every refusal from + /// this, and fills DescribeBackup's `SourceTableFeatureDetails` and + /// `SourceTableDetails.ProvisionedThroughput` from it. `definition` is the + /// stored [`crate::backup_definition::BackupTableDefinition`] (with + /// `time_to_live` and `stream_specification` as captured at + /// CreateBackup), or `None` for a backup recorded without one. + /// + /// # Errors + /// + /// `StorageError::Validation("Backup not found: ")` if the backup is + /// missing, owned by another account, or not AVAILABLE. + /// `StorageError::Unsupported` for a stored definition this build cannot + /// read. + fn restore_source( + &self, + account_id: &str, + backup_arn: &str, + ) -> BoxFuture<'_, Result>; + + /// Restore `request.input.table_name` from `request.backup_arn`. + /// + /// `request.input` is resolved and validated by the engine; create exactly + /// that table, CREATING. In the same atomic step that re-checks the backup + /// is still AVAILABLE and not in use, register the restore (the job + /// record, `kind = legacy`, with the claim) so DescribeTable can report + /// `RestoreSummary`, DeleteBackup refuses the source, and DeleteTable + /// refuses the target while it is being restored. Then return the CREATING + /// description at once. `permit` is the job's slot, released when the + /// copy ends. + /// + /// The item copy runs in a detached task that owns its own clones, so + /// the request future being dropped does not stop it. The task flips the + /// target ACTIVE when the copy completes, or removes the target on + /// failure. + /// + /// # Errors + /// + /// Returned synchronously, before the copy starts: + /// `StorageError::Validation("Backup not found: ")` if the backup is + /// not AVAILABLE at registration; `StorageError::BackupInUse` if a + /// target of it is CREATING; `StorageError::TableAlreadyExists(name)` if + /// the name is taken; `StorageError::TableBeingRestored` if it is a + /// CREATING restore target. + fn restore( + &self, + account_id: &str, + request: RestoreRequest, + permit: JobPermit, + ) -> BoxFuture<'_, Result>; + + /// In-catalog CREATING backup rows whose create is dead: on a backend + /// whose legacy create commits a CREATING row before the copy, those + /// whose heartbeat is older than `stale_after`, or that no running create + /// holds. The reconciler removes each through [`Self::delete`], unless the + /// backend removed it in this call ([`AbandonedLegacyBackup::removed`]). + /// Empty on a backend whose legacy create commits its row AVAILABLE with + /// the copy. + fn abandoned_creating_backups( + &self, + stale_after: Duration, + ) -> BoxFuture<'_, Result, StorageError>>; +} + +#[cfg(test)] +mod tests { + use extenddb_core::types::{KeyType, ProjectionType}; + + use super::*; + + /// The `source_table` object of the manifest fixture + /// `crates/core/testdata/extenddb-manifest/v1.json`. + fn fixture_source_table() -> serde_json::Value { + let manifest: serde_json::Value = serde_json::from_str(include_str!( + "../../core/testdata/extenddb-manifest/v1.json" + )) + .expect("fixture parses"); + manifest["source_table"].clone() + } + + fn row(status: BackupStatus) -> BackupRow { + BackupRow { + backup_id: "01757500000123-0badcafe".to_owned(), + account_id: "123456789012".to_owned(), + table_name: "Music".to_owned(), + backup_arn: + "arn:aws:dynamodb:us-east-1:123456789012:table/Music/backup/01757500000123-0badcafe" + .to_owned(), + backup_name: "music-nightly".to_owned(), + status, + store: StoreIdentity { + store_kind: StoreKind::Filesystem, + store_root: "/var/lib/extenddb/backups".to_owned(), + store_id: "store-1".to_owned(), + }, + size_bytes: 61828, + item_count: 100_000, + created_at: 1_757_500_000.123, + completed_at: None, + failure_reason: None, + source_table_details_json: fixture_source_table(), + owner_generation: 7, + heartbeat_at: 1_757_500_001.0, + } + } + + #[test] + fn a_description_derives_from_the_manifest_source_table() { + let desc = row(BackupStatus::Available).description().unwrap(); + let d = &desc.backup_details; + assert_eq!(d.backup_status, "AVAILABLE"); + assert_eq!(d.backup_type, "USER"); + assert_eq!(d.backup_size_bytes, 61828); + assert_eq!(d.backup_creation_date_time, 1_757_500_000.123); + let s = &desc.source_table_details; + assert_eq!(s.table_name, "Music"); + assert_eq!(s.table_id, "12345a12-abcd-123a-ab12-1234abc12345"); + assert_eq!( + s.table_arn, + "arn:aws:dynamodb:us-east-1:123456789012:table/Music" + ); + assert_eq!(s.key_schema.len(), 2); + assert_eq!(s.key_schema[1].key_type, KeyType::Range); + assert_eq!(s.item_count, 100_000); + assert_eq!(s.table_size_bytes, 123_456); + assert_eq!(s.billing_mode.as_deref(), Some("PROVISIONED")); + assert_eq!(s.table_creation_date_time, 1_757_400_000.0); + assert_eq!( + s.provisioned_throughput, + Some(ProvisionedThroughput { + read_capacity_units: 5, + write_capacity_units: 5, + }) + ); + assert_eq!( + s.on_demand_throughput + .as_ref() + .and_then(|o| o.max_read_request_units), + Some(100) + ); + let f = desc.source_table_feature_details.expect("features"); + let gsi = &f.global_secondary_indexes.unwrap()[0]; + assert_eq!(gsi.index_name, "GenreIndex"); + assert_eq!(gsi.projection.projection_type, ProjectionType::Include); + assert_eq!( + gsi.provisioned_throughput, + Some(ProvisionedThroughput { + read_capacity_units: 1, + write_capacity_units: 1, + }) + ); + let lsi = &f.local_secondary_indexes.unwrap()[0]; + assert_eq!(lsi.index_name, "TitleIndex"); + assert_eq!(lsi.projection.projection_type, ProjectionType::KeysOnly); + assert!(f.stream_description.unwrap().stream_enabled); + assert_eq!( + f.time_to_live_description + .unwrap() + .attribute_name + .as_deref(), + Some("expires_at") + ); + let sse = f.sse_description.unwrap(); + assert_eq!(sse.status, "ENABLED"); + assert_eq!(sse.sse_type, Some(extenddb_core::types::SseType::KMS)); + } + + #[test] + fn an_on_demand_source_reports_zero_throughput_on_the_table_and_its_indexes() { + let mut row = row(BackupStatus::Available); + row.source_table_details_json["billing_mode"] = "PAY_PER_REQUEST".into(); + let desc = row.description().unwrap(); + let zero = ProvisionedThroughput { + read_capacity_units: 0, + write_capacity_units: 0, + }; + assert_eq!( + desc.source_table_details.billing_mode.as_deref(), + Some("PAY_PER_REQUEST") + ); + assert_eq!( + desc.source_table_details.provisioned_throughput, + Some(zero.clone()) + ); + let f = desc.source_table_feature_details.unwrap(); + assert_eq!( + f.global_secondary_indexes.unwrap()[0].provisioned_throughput, + Some(zero) + ); + } + + /// A source that recorded no index, stream, TTL or SSE still reports + /// `SourceTableFeatureDetails`, as `{}` (the service's answer for such a + /// table), with every inner member omitted. + #[test] + fn a_source_without_features_reports_empty_feature_details() { + let mut row = row(BackupStatus::Available); + let s = &mut row.source_table_details_json; + s["global_secondary_indexes"] = serde_json::json!([]); + s["local_secondary_indexes"] = serde_json::json!([]); + s["stream_specification"] = serde_json::Value::Null; + s["time_to_live"] = serde_json::Value::Null; + s["sse_description"] = serde_json::Value::Null; + let desc = row.description().unwrap(); + let features = desc + .source_table_feature_details + .expect("the member is always present"); + assert!(features.is_empty()); + assert_eq!( + serde_json::to_value(&features).unwrap(), + serde_json::json!({}) + ); + } + + #[test] + fn a_summary_carries_the_source_table_id_and_arn() { + let s = row(BackupStatus::Creating).summary().unwrap(); + assert_eq!(s.table_id, "12345a12-abcd-123a-ab12-1234abc12345"); + assert_eq!( + s.table_arn, + "arn:aws:dynamodb:us-east-1:123456789012:table/Music" + ); + assert_eq!(s.table_name, "Music"); + assert_eq!(s.backup_status, "CREATING"); + assert_eq!(s.backup_creation_date_time, 1_757_500_000.123); + } + + #[test] + fn deleting_is_reported_as_deleted_and_tombstones_are_hidden() { + assert_eq!( + row(BackupStatus::Deleting).details().backup_status, + "DELETED" + ); + assert!(BackupStatus::Deleting.is_described()); + assert!(!BackupStatus::Deleting.is_listed()); + for status in [BackupStatus::Deleted, BackupStatus::Failed] { + assert!(!status.is_described(), "{status}"); + assert!(!status.is_listed(), "{status}"); + } + for status in [BackupStatus::Creating, BackupStatus::Available] { + assert!(status.is_described(), "{status}"); + assert!(status.is_listed(), "{status}"); + assert_eq!(status.reported(), status.as_str()); + } + } + + #[test] + fn a_source_table_this_build_cannot_read_is_internal() { + let mut row = row(BackupStatus::Available); + row.source_table_details_json = serde_json::json!({"table_name": "x"}); + assert!(matches!(row.description(), Err(StorageError::Internal(_)))); + assert!(matches!(row.summary(), Err(StorageError::Internal(_)))); + } + + /// A source table a later build extended with a member this build + /// requires (here: every member but the two the summary reads is gone) + /// still summarizes; only the description needs the whole object. + #[test] + fn a_summary_needs_only_the_source_table_id_and_arn() { + let mut row = row(BackupStatus::Available); + row.source_table_details_json = serde_json::json!({ + "table_id": "12345a12-abcd-123a-ab12-1234abc12345", + "table_arn": "arn:aws:dynamodb:us-east-1:123456789012:table/Music", + "member_this_build_does_not_know": true, + }); + assert!(matches!(row.description(), Err(StorageError::Internal(_)))); + let s = row.summary().unwrap(); + assert_eq!(s.table_id, "12345a12-abcd-123a-ab12-1234abc12345"); + assert_eq!( + s.table_arn, + "arn:aws:dynamodb:us-east-1:123456789012:table/Music" + ); + row.source_table_details_json["table_arn"] = serde_json::json!(7); + let e = row.summary().unwrap_err(); + assert!(e.to_string().contains("table_arn"), "{e}"); + } + + #[test] + fn statuses_round_trip_through_their_column_values() { + for status in [ + BackupStatus::Creating, + BackupStatus::Available, + BackupStatus::Deleting, + BackupStatus::Deleted, + BackupStatus::Failed, + ] { + assert_eq!(BackupStatus::parse(status.as_str()).unwrap(), status); + } + assert!(BackupStatus::parse("ACTIVE").is_err()); + for status in [ + JobStatus::Creating, + JobStatus::Available, + JobStatus::Running, + JobStatus::Done, + JobStatus::Failed, + ] { + assert_eq!(JobStatus::parse(status.as_str()).unwrap(), status); + } + for kind in [RestoreKind::Legacy, RestoreKind::Manifest] { + assert_eq!(RestoreKind::parse(kind.as_str()).unwrap(), kind); + } + for kind in [StoreKind::Filesystem, StoreKind::S3] { + assert_eq!(StoreKind::parse(kind.as_str()), Some(kind)); + } + assert_eq!(StoreKind::parse("gcs"), None); + } + + #[test] + fn a_key_is_built_from_the_parsed_arn() { + let arn = BackupArn::parse( + "arn:aws:dynamodb:us-east-1:123456789012:table/Music/backup/01757500000123-0badcafe", + ) + .unwrap(); + let key = BackupKey::from_arn(&arn); + assert_eq!(key.account_id, "123456789012"); + assert_eq!(key.backup_id, "01757500000123-0badcafe"); + assert_eq!(key.backup_arn, arn.to_arn_string()); + assert_eq!(row(BackupStatus::Available).key(), key); + } + + #[test] + fn time_bounds_are_lower_inclusive_and_upper_exclusive() { + let bounds = TimeBounds { + lower: Some(10.0), + upper: Some(20.0), + }; + assert!(bounds.contains(10.0)); + assert!(bounds.contains(19.999)); + assert!(!bounds.contains(20.0)); + assert!(!bounds.contains(9.999)); + assert!(TimeBounds::default().contains(0.0)); + } + + #[test] + fn the_pool_rule_is_twice_the_jobs_plus_ten() { + assert_eq!(pool_size_for_jobs(4), 18); + assert_eq!(pool_size_for_jobs(0), 10); + assert_eq!(pool_size_for_jobs(16), 42); + } + + #[tokio::test] + async fn a_permit_is_taken_without_waiting_and_freed_on_drop() { + let slots = Arc::new(tokio::sync::Semaphore::new(1)); + let held = JobPermit::try_acquire(&slots).expect("a free slot"); + assert!(JobPermit::try_acquire(&slots).is_none()); + drop(held); + assert!(JobPermit::try_acquire(&slots).is_some()); + } + + /// `acquire_within` takes a free slot at once whatever the wait, gives up + /// once the wait passes with every slot held, and takes a slot freed + /// during the wait. + #[tokio::test] + async fn a_permit_is_waited_for_within_the_bound() { + let slots = Arc::new(tokio::sync::Semaphore::new(1)); + let held = JobPermit::acquire_within(&slots, Duration::ZERO) + .await + .expect("a free slot is taken at once"); + assert!( + JobPermit::acquire_within(&slots, Duration::from_millis(20)) + .await + .is_none(), + "every slot held past the bound" + ); + let slots2 = Arc::clone(&slots); + let waiter = tokio::spawn(async move { + JobPermit::acquire_within(&slots2, Duration::from_secs(5)).await + }); + tokio::time::sleep(Duration::from_millis(20)).await; + drop(held); + assert!( + waiter.await.unwrap().is_some(), + "the freed slot went to the waiter" + ); + } + + /// A generation is never the unowned sentinel and always below 2^63, so + /// every backend's signed integer column holds it unchanged and two + /// claims never share a fencing token through saturation. Deterministic: + /// 10,000 draws, each checked. + #[test] + fn generations_are_non_zero_and_fit_a_signed_integer_column() { + for _ in 0..10_000 { + let g = new_generation(); + assert_ne!(g, UNOWNED_GENERATION); + assert!(g < 1u64 << 63, "{g} does not fit i64"); + assert_eq!(u64::try_from(i64::try_from(g).unwrap()).unwrap(), g); + } + } + + /// The one refusal text, by bound: a counted bound names its count and + /// its name in one parenthesis; the server's connection slots name no + /// count. Both carry the sentence a client matches on. + #[test] + fn limit_exceeded_message_names_the_bound() { + assert_eq!( + limit_exceeded_message(JobBound::Counted { + name: "[backup] max_concurrent_jobs", + limit: 4, + }), + "Too many backup and restore jobs are running on this server (at most 4 at once, \ + the [backup] max_concurrent_jobs bound); retry later" + ); + assert_eq!( + limit_exceeded_message(JobBound::ServerConnections), + "Too many backup and restore jobs are running on this server (the database server \ + has no connection slot left for the job); retry later" + ); + } +} diff --git a/crates/storage/src/backup_definition.rs b/crates/storage/src/backup_definition.rs index 411b22ab6..db7f5247a 100644 --- a/crates/storage/src/backup_definition.rs +++ b/crates/storage/src/backup_definition.rs @@ -24,6 +24,7 @@ //! alter the meaning of backups already on disk. A reader that finds a version //! it does not know refuses the restore rather than guessing. +use extenddb_core::backup_format::SourceTableSchema; use extenddb_core::types::{ AttributeDefinition, BillingMode, CreateTableInput, GsiInput, KeySchemaElement, KeyType, LsiInput, OnDemandThroughput, ProvisionedThroughput, StreamSpecification, @@ -334,7 +335,7 @@ const LEGACY_DEFAULT_THROUGHPUT: i64 = 5; /// A backup as the engine needs it to restore it and to describe it. /// -/// Returned by `BackupEngine::backup_restore_source` for an AVAILABLE backup. +/// Returned by `LegacyBackupReader::restore_source` for an AVAILABLE backup. #[derive(Debug, Clone, PartialEq)] pub struct BackupRestoreSource { pub backup_arn: String, @@ -437,6 +438,75 @@ impl BackupRestoreSource { } } +/// What the engine answers a `RestoreTableFromBackup` with, beside the +/// target's description: the three members of the `RestoreSummary`, from a +/// legacy backup's [`BackupRestoreSource`] or a store-backed backup's +/// manifest. +#[derive(Debug, Clone, PartialEq)] +pub struct RestoreSource { + pub backup_arn: String, + pub source_table_arn: String, + /// Epoch seconds; reported as `RestoreSummary.RestoreDateTime`, as the + /// service does. + pub backup_creation_date_time: f64, +} + +impl From<&BackupRestoreSource> for RestoreSource { + fn from(source: &BackupRestoreSource) -> Self { + Self { + backup_arn: source.backup_arn.clone(), + source_table_arn: source.source_table_arn.clone(), + backup_creation_date_time: source.backup_creation_date_time, + } + } +} + +/// The restore target's `CreateTableInput` for a store-backed backup: the +/// manifest's `source_table` with the request's billing, throughput and SSE +/// overrides applied, by the rule [`BackupTableDefinition::apply_to`] applies +/// to a legacy definition (one rule for both paths: the GSI throughput +/// defaulting, the on-demand normalization, `TableClass` not restored). The +/// engine applies the index and on-demand overrides to the result and +/// validates it with `validate_create_table`. `vector_indexes` is the +/// manifest's list when it is non-empty (a target without vector support has +/// refused it before this is called), else `None`; the recorded SSE +/// description maps back to the specification the backends derive it from +/// (an `ENABLED` description was an enabled `KMS` specification). Streams, +/// TTL and tags are not restored, as the service does not. +#[must_use] +pub fn restore_input_from_manifest( + source: &SourceTableSchema, + target_table_name: &str, + overrides: &RestoreTableOverrides, +) -> CreateTableInput { + let mut input = CreateTableInput { + table_name: target_table_name.to_owned(), + key_schema: source.key_schema.clone(), + attribute_definitions: source.attribute_definitions.clone(), + ..Default::default() + }; + let definition = BackupTableDefinition { + version: BACKUP_DEFINITION_VERSION, + billing_mode: crate::backup_catalog::billing_mode_name(source.billing_mode).to_owned(), + provisioned_throughput: source.provisioned_throughput.clone(), + global_secondary_indexes: source.global_secondary_indexes.clone(), + local_secondary_indexes: source.local_secondary_indexes.clone(), + table_class: source.table_class.clone(), + sse_specification: source + .sse_description + .as_ref() + .filter(|sse| sse.status == "ENABLED") + .map(|_| serde_json::json!({"Enabled": true, "SSEType": "KMS"})), + on_demand_throughput: source.on_demand_throughput.clone(), + vector_index_names: Vec::new(), + time_to_live: None, + stream_specification: None, + }; + definition.apply_to(&mut input, overrides); + input.vector_indexes = source.vector_indexes.clone().filter(|v| !v.is_empty()); + input +} + /// What a backend is asked to do for one `RestoreTableFromBackup`, after the /// engine has resolved and validated it. #[derive(Debug, Clone, PartialEq)] diff --git a/crates/storage/src/config.rs b/crates/storage/src/config.rs index c9be8335c..1136e4d13 100644 --- a/crates/storage/src/config.rs +++ b/crates/storage/src/config.rs @@ -59,6 +59,29 @@ pub trait StorageConfig: Send + Sync + std::fmt::Debug { Vec::new() } + /// The local file the database lives in, for a backend whose database is + /// one file on this host (SQLite with a file path); `None` for a server + /// database and for an in-memory database. The backup configuration + /// derives its per-backend defaults from it: a store root and a restore + /// spool beside the file, and the rule that neither may sit in the + /// file's directory. The default is `None`. + fn database_file(&self) -> Option { + None + } + + /// [`Self::database_file`] with the reason a configured location names + /// no usable file: `Ok(None)` for a server database and for an + /// in-memory one, `Err` for a location the backend cannot parse (which + /// [`Self::database_file`] also answers `None` for). The default is + /// `Ok` of [`Self::database_file`]. + /// + /// # Errors + /// + /// The backend's reason the location is not usable. + fn database_location(&self) -> Result, String> { + Ok(self.database_file()) + } + /// Clone this config into a boxed trait object. fn clone_box(&self) -> Box; @@ -88,38 +111,39 @@ impl Clone for Box { } /// The warning for a `server.region` outside `[a-z0-9-]`, or `None` when the -/// region is in that class. +/// region is in that class (the `BackupArn` parser's predicate, +/// [`extenddb_core::backup_format::is_region`], so the warning and the parser +/// cannot disagree). /// /// The region is baked into every backup ARN the server issues /// (`arn:aws:dynamodb:::table//backup/`), and the /// `BackupArn` parser accepts a region in `[a-z0-9-]` only, as the service -/// does. A deployment configured outside that class therefore issues ARNs -/// it refuses on `DescribeBackup`, `DeleteBackup`, and -/// `RestoreTableFromBackup` with `ValidationException` "Invalid Request: -/// BackupArn is not valid". The server starts anyway (the setting is -/// pre-existing and the data plane does not depend on it); the operator is -/// told at startup. +/// does. The engine checks the region before forming a backup ARN, so on a +/// deployment configured outside that class `CreateBackup` is refused with a +/// `ValidationException` naming `server.region`; backups taken before the +/// upgrade under such a region keep their stored ARN, which `DescribeBackup`, +/// `DeleteBackup` and `RestoreTableFromBackup` refuse as "Invalid Request: +/// BackupArn is not valid" as before. The server starts anyway (the setting +/// is pre-existing and the data plane does not depend on it); the operator +/// is told at startup. #[must_use] pub fn server_region_warning(region: &str) -> Option { - let well_formed = !region.is_empty() - && region - .bytes() - .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-'); - if well_formed { + if extenddb_core::backup_format::is_region(region) { return None; } Some(StartupWarning { title: format!( - "server.region {region:?} is outside [a-z0-9-]; the backup ARNs this server \ - issues will not resolve." + "server.region {region:?} is outside [a-z0-9-]; CreateBackup is refused on this \ + server." ), details: vec![ "Every backup ARN carries the configured region, and BackupArn is accepted \ - only with a region in [a-z0-9-], so DescribeBackup, DeleteBackup, and \ - RestoreTableFromBackup answer ValidationException \"Invalid Request: \ - BackupArn is not valid\" for every backup this server creates, and \ - ListBackups refuses the cursor it hands out, so a listing stops at its \ - first page." + only with a region in [a-z0-9-], so CreateBackup answers ValidationException \ + naming server.region (no backup ARN can be formed), and the backups created \ + under this region before the upgrade keep an ARN that DescribeBackup, \ + DeleteBackup, and RestoreTableFromBackup refuse with ValidationException \ + \"Invalid Request: BackupArn is not valid\"; ListBackups refuses a cursor that is \ + such an ARN (a listing of backups created under a valid region pages)." .to_owned(), "Set server.region to a value in [a-z0-9-] (for example us-east-1). Backups \ already created keep the old region in their stored ARN; see the upgrade \ @@ -160,6 +184,15 @@ mod tests { "{region:?}: {text}" ); assert!(text.contains("DescribeBackup"), "{region:?}: {text}"); + assert!( + text.contains("ListBackups refuses a cursor that is such an ARN"), + "{region:?}: {text}" + ); + assert!(!text.contains("first page"), "{region:?}: {text}"); + assert!( + text.contains("CreateBackup answers ValidationException naming server.region"), + "{region:?}: {text}" + ); // The warning agrees with the parser it speaks for. let arn = crate::util::backup_arn(region, "123456789012", "Music", "01489602797149-73d8d5bc"); diff --git a/crates/storage/src/error.rs b/crates/storage/src/error.rs index c936675a5..848a25f00 100755 --- a/crates/storage/src/error.rs +++ b/crates/storage/src/error.rs @@ -63,6 +63,10 @@ pub enum StorageError { /// `BackupInUseException`. #[error("{0}")] BackupInUse(String), + /// A catalog row that already exists, or whose claim a live owner holds + /// (`BackupCatalog::claim_backup`). + #[error("Already exists: {0}")] + AlreadyExists(String), /// A failure that is expected to succeed on retry: I/O errors, pool /// timeouts, SQLITE_BUSY / SQLITE_LOCKED. Exists so queue workers can tell /// "this row can never be applied" (drop it, or the whole queue stalls) diff --git a/crates/storage/src/lib.rs b/crates/storage/src/lib.rs index f5e744799..1cacda993 100755 --- a/crates/storage/src/lib.rs +++ b/crates/storage/src/lib.rs @@ -9,6 +9,7 @@ pub mod authorization_store; pub mod backend; +pub mod backup_catalog; pub mod backup_definition; pub mod bootstrapper; pub mod config; @@ -22,6 +23,7 @@ pub mod migrations; pub mod operations; pub mod password; pub mod policy_migration; +pub mod restore_engine; pub mod secret_cipher; pub mod server_components; pub mod settings_store; @@ -37,7 +39,13 @@ pub mod vector_catalog; pub mod vector_lifecycle; pub use backend::{Backend, BackendAlreadySet, backend_name, set_backend, try_backend}; -pub use backup_definition::{BackupRestoreSource, RestoreRequest, RestoreTableOverrides}; +pub use backup_catalog::{ + BackupCatalog, JobClaim, JobPermit, LegacyBackupReader, MaintenanceClaim, +}; +pub use backup_definition::{ + BackupRestoreSource, RestoreRequest, RestoreSource, RestoreTableOverrides, +}; +pub use restore_engine::{BulkLoadFile, Finished, RestoreEngine}; pub use snapshot::{SnapshotEngine, SnapshotInfo, TableSnapshot}; pub use transact::{IdempotencyKey, TransactGetOp, TransactWriteOp}; @@ -711,105 +719,6 @@ pub trait WorkerStore: Send + Sync { ) -> BoxFuture<'_, Result, StorageError>>; } -/// Backup operations. -/// -/// Point-in-time recovery is deliberately absent: no backend implements it, so -/// the engine answers `DescribeContinuousBackups`, `UpdateContinuousBackups`, -/// and `RestoreTableToPointInTime` itself without consulting storage. -pub trait BackupEngine: Send + Sync { - /// Create a backup of a table, snapshotting all items. - fn create_backup( - &self, - account_id: &str, - table_name: &str, - backup_name: &str, - ) -> BoxFuture<'_, Result>; - - /// Describe a backup by ARN, scoped to the owning account. - /// - /// Backups belong to an account, so implementations must match on both - /// `account_id` and `backup_arn` and report a missing backup when the ARN - /// belongs to another account — consistent with `list_backups`, which is - /// already account-scoped. - fn describe_backup( - &self, - account_id: &str, - backup_arn: &str, - ) -> BoxFuture<'_, Result>; - - /// List the account's backups, optionally only those of `table_name`. - /// - /// Returns every matching backup in any order; the engine sorts, applies - /// `Limit` and `ExclusiveStartBackupArn`, and sets - /// `LastEvaluatedBackupArn`. Each summary carries the source table's - /// `TableId`. - fn list_backups( - &self, - account_id: &str, - table_name: Option<&str>, - ) -> BoxFuture<'_, Result, StorageError>>; - - /// Delete a backup by ARN, scoped to the owning account. - /// - /// Same scoping requirement as `describe_backup`: an ARN owned by another - /// account must be reported as missing rather than deleted. - fn delete_backup( - &self, - account_id: &str, - backup_arn: &str, - ) -> BoxFuture<'_, Result>; - - /// The backup's recorded source, for an AVAILABLE backup owned by - /// `account_id`. - /// - /// The engine resolves the restore target and runs every refusal from - /// this, and fills DescribeBackup's `SourceTableFeatureDetails` and - /// `SourceTableDetails.ProvisionedThroughput` from it. `definition` is the - /// stored [`backup_definition::BackupTableDefinition`] (with - /// `time_to_live` and `stream_specification` as captured at - /// CreateBackup), or `None` for a backup recorded without one. - /// - /// # Errors - /// - /// `StorageError::Validation("Backup not found: ")` if the backup is - /// missing, owned by another account, or not AVAILABLE. - /// `StorageError::Unsupported` for a stored definition this build cannot - /// read ([`backup_definition::BackupTableDefinition::from_json`]). - fn backup_restore_source( - &self, - account_id: &str, - backup_arn: &str, - ) -> BoxFuture<'_, Result>; - - /// Restore `request.input.table_name` from `request.backup_arn`. - /// - /// `request.input` is resolved and validated by the engine; create exactly - /// that table, CREATING. In the same atomic step that re-checks the backup - /// is still AVAILABLE, register the restore (source backup ARN, source - /// table ARN, `restore_date_time`) so DescribeTable can report - /// `RestoreSummary`, DeleteBackup refuses the source, and DeleteTable - /// refuses the target while it is being restored. Then return the CREATING - /// description at once. - /// - /// The item copy runs in a detached task (`tokio::spawn`) that owns its - /// own clones, so the request future being dropped (a client timeout) does - /// not stop it. The task flips the target ACTIVE when the copy completes, - /// or removes the target on failure. - /// - /// # Errors - /// - /// Returned synchronously, before the copy starts: - /// `StorageError::Validation("Backup not found: ")` if the backup is - /// not AVAILABLE at registration; `StorageError::TableAlreadyExists(name)` - /// if the name is taken; `StorageError::LimitExceeded` if the backend's - /// restore capacity is exhausted. - fn restore_table_from_backup( - &self, - account_id: &str, - request: RestoreRequest, - ) -> BoxFuture<'_, Result>; -} - /// Supertrait combining all `DynamoDB` operation traits. /// /// All storage backends must implement this to provide a complete @@ -820,7 +729,9 @@ pub trait StorageEngine: + DataEngine + MetadataEngine + StreamEngine - + BackupEngine + + BackupCatalog + + LegacyBackupReader + + RestoreEngine + SnapshotEngine + WorkerStore + Send @@ -828,13 +739,15 @@ pub trait StorageEngine: { } -// Blanket implementation: any type implementing all 7 traits is a StorageEngine +// Blanket implementation: any type implementing all 9 traits is a StorageEngine impl StorageEngine for T where T: TableEngine + DataEngine + MetadataEngine + StreamEngine - + BackupEngine + + BackupCatalog + + LegacyBackupReader + + RestoreEngine + SnapshotEngine + WorkerStore + Send diff --git a/crates/storage/src/migrations/registry.rs b/crates/storage/src/migrations/registry.rs index dc426fd18..d6371ebe8 100644 --- a/crates/storage/src/migrations/registry.rs +++ b/crates/storage/src/migrations/registry.rs @@ -67,6 +67,21 @@ migration_ids! { /// it. DropContinuousBackups = 5, "drop_continuous_backups", "drop the continuous_backups table, which no request reads or writes since #335"; + /// Creates the catalog index of backups (`backup_index`), the restore + /// job fields on the restore registrations, the maintenance claim record + /// and the catalog id. + BackupCatalogIndex = 6, "backup_catalog_index", + "create the backup catalog index, the restore job fields, the maintenance claim \ + record and the catalog id"; + /// Creates `restore_progress` in the PostgreSQL data database, where a + /// store-backed restore records each data file it applied; the other + /// backends keep that progress on the job record and have nothing to do. + /// Declared `Kind::Breaking` on PostgreSQL although a new table is + /// additive by CONTRIBUTING's step 4: declaring `Kind::Additive` needs + /// the old-binary job, which does not exist yet. + RestoreProgress = 7, "restore_progress", + "create the restore progress table of store-backed restores in the PostgreSQL data \ + database"; } /// Highest id that has shipped in a release. Bodies at or below it are frozen: diff --git a/crates/storage/src/restore_engine.rs b/crates/storage/src/restore_engine.rs new file mode 100644 index 000000000..650b5b534 --- /dev/null +++ b/crates/storage/src/restore_engine.rs @@ -0,0 +1,169 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The data-plane side of a store-backed restore: how a backend takes the +//! items of one verified data file into a CREATING target, records how far +//! it got, flips the target ACTIVE, and counts what it holds. +//! +//! The engine (`crates/engine/src/backup/restore.rs`) owns the job: it +//! downloads and verifies every file, decodes and validates the items, and +//! hands them to [`RestoreEngine::bulk_load`] one file at a time. The +//! backend owns the writes: the base rows, the index rows and the vector +//! rows in its own batches and transactions, the progress record its +//! resume reads, and the ACTIVE flip conditioned on the claim. Progress is +//! recorded inside `bulk_load`, never by a separate call, because the +//! backends record it at different points (PostgreSQL per file in the data +//! database, SQLite, MongoDB and Cassandra per batch) and only the +//! write that applied the items knows when they are durable. + +use extenddb_core::types::Item; +use futures::future::BoxFuture; +use futures::stream::BoxStream; + +use crate::backup_catalog::{JobClaim, RestoreTargetKey}; +use crate::error::StorageError; + +/// Items per write batch of the SQLite and Cassandra `bulk_load` (SQLite's +/// write transaction, Cassandra's run of upserts before a progress write): +/// the count cap of the batch term of a restore's memory (the design's +/// section 7), and the unit other writers wait for on SQLite. +pub const RESTORE_BATCH_ITEMS: usize = 10_000; + +/// Bytes of serialized item text per write batch of the SQLite and Cassandra +/// `bulk_load`: a batch closes at the first item that takes its `item_data` +/// text past this, so the batch term of a restore's memory is bounded in +/// bytes as well as in items (10,000 items of the 400 KB limit would be +/// 3.9 GiB). Measured on the JSON text each backend writes as the row's +/// `item_data`, not on the service's `item_size_bytes`. +pub const RESTORE_BATCH_BYTES: usize = 32 * 1024 * 1024; + +/// Which data file a [`RestoreEngine::bulk_load`] call carries, and where in +/// it the items start. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct BulkLoadFile { + /// The file's index in the manifest's `data_files`. + pub file_index: u32, + /// Items of this file already applied by an earlier run, which the + /// engine has skipped: the stream holds the rest. The backend records + /// `start_at` plus the items it applied as the file's running total. + pub start_at: u64, + /// The file is the one a resumed job may have applied in part without a + /// record of it (a backend that records progress per file and died + /// after some of the file's writes). A backend whose writes are plain + /// inserts may suppress duplicate-key failures for this file only; + /// elsewhere a duplicate key is the backup's and fails the file. + pub reapplying: bool, +} + +/// What [`RestoreEngine::finish_restore`] did. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Finished { + /// The record is DONE and the table and every index are ACTIVE. + Active, + /// The record is DONE and the table waits for its index backfill; the + /// backend's worker flips it ACTIVE once every index has left CREATING + /// (MongoDB). + Pending, + /// The record was not RUNNING under the claim's generation, or the + /// table was not a CREATING restore target: nothing was written. Carries + /// the backend's account of what it found. + NotMatched(String), +} + +/// The data-plane operations of a store-backed restore, implemented by each +/// backend beside [`crate::backup_catalog::BackupCatalog`]. +pub trait RestoreEngine: Send + Sync { + /// Write one verified data file's items, from `file.start_at` on, into + /// the CREATING target: the base rows and the rows of every secondary + /// and vector index the target has, in the backend's own batches, and + /// the file's progress as the backend records it (the design's 5.3 step + /// 5), under `claim`'s generation where the backend's record carries it. + /// Returns the items it applied from the stream. + /// + /// A duplicate primary key, within the file or against rows an earlier + /// file wrote, fails the file (the backup holds a key twice), except as + /// [`BulkLoadFile::reapplying`] says. An `Err` item of the stream (a + /// decode or validation failure the engine found) fails the file with + /// that error, and nothing of a failed file is recorded as applied where + /// the backend writes the file in one transaction. + fn bulk_load<'a>( + &'a self, + target: &'a RestoreTargetKey, + claim: &'a dyn JobClaim, + file: BulkLoadFile, + items: BoxStream<'a, Result>, + ) -> BoxFuture<'a, Result>; + + /// The end of a restore (the design's 5.3 step 6): the record RUNNING to + /// DONE and the table and every index CREATING to ACTIVE with + /// `item_count` and `size_bytes`, in one write where the backend can, + /// conditional on the claim's generation and on the record RUNNING and + /// the table CREATING ([`Finished::Active`]); or DONE with the ACTIVE + /// flip left to the backend's index backfill ([`Finished::Pending`]). + /// [`Finished::NotMatched`] when the record or the table was not the + /// claim's. A backend that keeps progress outside the record removes it + /// after the flip. + fn finish_restore<'a>( + &'a self, + target: &'a RestoreTargetKey, + claim: &'a dyn JobClaim, + item_count: u64, + size_bytes: u64, + ) -> BoxFuture<'a, Result>; + + /// The items the target's base table holds: the count check every + /// restore ends with (equal to the manifest's total, else the backup + /// holds a key twice), and the emptiness check a resume with no recorded + /// progress makes. + fn count_restore_target<'a>( + &'a self, + key: &'a RestoreTargetKey, + ) -> BoxFuture<'a, Result>; + + /// Whether the target's data tables exist as the registration leaves + /// them: the reconciler's check before it resumes a dead owner's record. + /// Every backend commits the record before the target's data DDL + /// (PostgreSQL and SQLite create the data table after the catalog + /// transaction, MongoDB the collections and the index metadata after the + /// insert's commit, Cassandra the data table after its registration + /// LWT), so an owner killed in between leaves a RUNNING record over a + /// target with no table, or on MongoDB with its index metadata missing; + /// `false` names that record, which the reconciler fails and removes + /// instead of resuming. `Err` when `key` is not a CREATING restore + /// target with this table id, or the read failed. + fn restore_target_ready<'a>( + &'a self, + key: &'a RestoreTargetKey, + ) -> BoxFuture<'a, Result>; + + /// The progress rows a backend keeps outside its job record, as + /// `(file_index, items_applied)` pairs: PostgreSQL's `restore_progress` + /// rows in the data database. Empty by default: the other backends + /// carry the progress on the record ([`crate::backup_catalog::RestoreJobRow`]). + fn read_restore_progress<'a>( + &'a self, + key: &'a RestoreTargetKey, + ) -> BoxFuture<'a, Result, StorageError>> { + let _ = key; + Box::pin(async { Ok(Vec::new()) }) + } + + /// Remove progress rows of targets not among `running` (the table ids + /// of RUNNING restore targets, read from the catalog): the reconciler's + /// sweep of rows a crash left behind where the progress lives apart + /// from the record. The candidates are a positive list: the backend + /// reads the distinct table ids the progress rows name first, then the + /// catalog's RUNNING records, and deletes only the rows of ids in the + /// first set that are in neither the second nor `running`; a target + /// whose first row lands after the first read is never a candidate, and + /// one registered before it is RUNNING at the second read, so a row of + /// a live job is never deleted. Returns the rows removed; `0` by + /// default. + fn sweep_restore_progress<'a>( + &'a self, + running: &'a [String], + ) -> BoxFuture<'a, Result> { + let _ = running; + Box::pin(async { Ok(0) }) + } +} diff --git a/crates/storage/src/server_components.rs b/crates/storage/src/server_components.rs index 900fe84bd..c685e383a 100644 --- a/crates/storage/src/server_components.rs +++ b/crates/storage/src/server_components.rs @@ -100,6 +100,38 @@ pub struct ServerComponentsOptions { /// The ExtendDB release that is starting, recorded in the ledger rows of /// migrations the factory runs (a dev build, an in-memory database). pub release: &'static str, + /// `backup.max_concurrent_jobs`: the backup and restore jobs the + /// instance runs at once. A backend whose pool has to admit every job + /// sizes it from this ([`crate::backup_catalog::pool_size_for_jobs`]); + /// `0` (the default outside a server) asks for no job capacity. + pub max_concurrent_jobs: usize, + /// The caller is a command that reads the catalog beside a running + /// server (`extenddb backup list`), not a server: no serve lock is taken + /// (on SQLite the file is opened as another reader of a served + /// database) and none of the startup recovery a server runs (see + /// `startup_recovery`, which this implies off), since the server that + /// owns the catalog does that. A command that writes (`backup purge`) + /// leaves this `false`: on SQLite it takes the serve lock, as a server + /// does. + pub beside_server: bool, + /// Whether the factory runs the startup recovery a server runs + /// (interrupted control-plane transitions, incomplete indexes, + /// PostgreSQL's GSI queue workers, MongoDB's repair of documents stored + /// without their key, SQLite's expiry of interrupted restores and of a + /// left-behind maintenance claim). `true` for a server; a command that + /// writes (`backup purge`) sets it `false` beside `beside_server = + /// false`, so on PostgreSQL, MongoDB and Cassandra it runs none of that + /// work beside the live servers and exits with none of it half done, + /// and on SQLite it holds the serve lock and resolves what it needs + /// itself. Read only when `beside_server` is `false`. + pub startup_recovery: bool, + /// The command that opens the catalog, when it is not `serve` + /// (`extenddb backup list`, `extenddb backup purge`). A backend whose + /// factory takes a serving registration names it after the command + /// (PostgreSQL's session `application_name`, MongoDB's and Cassandra's + /// `extenddb_servers` entry), which a migration waiting for the + /// registrations prints; `None` is `extenddb serve`. + pub command: Option<&'static str>, } impl Default for ServerComponentsOptions { @@ -108,6 +140,10 @@ impl Default for ServerComponentsOptions { bootstrap_if_uninitialized: false, max_item_size_bytes: extenddb_core::limits::LimitsConfig::default().max_item_size_bytes, release: "", + max_concurrent_jobs: 0, + beside_server: false, + startup_recovery: true, + command: None, } } } @@ -139,7 +175,7 @@ pub type ServerComponentsFactory = /// /// A `region` the `BackupArn` parser would refuse is logged at warn first /// ([`crate::config::server_region_warning`]): the server still starts, but -/// the operator is told its own backup ARNs will not resolve. +/// the operator is told `CreateBackup` is refused under it. pub async fn create_server_components( config: &(dyn StorageConfig + 'static), region: &str, diff --git a/crates/storage/src/snapshot.rs b/crates/storage/src/snapshot.rs index 10f14d790..0f692e5f9 100644 --- a/crates/storage/src/snapshot.rs +++ b/crates/storage/src/snapshot.rs @@ -65,8 +65,12 @@ //! of each backend's `snapshot_engine.rs` are the record of what the //! snapshots do. -use extenddb_core::backup_format::KeyOrder; -use extenddb_core::types::{IndexStatus, Item, TableDescription}; +use extenddb_core::backup_format::{KeyOrder, SourceTableSchema}; +use extenddb_core::types::{ + BillingMode, GsiInput, IndexStatus, Item, LsiInput, ProvisionedThroughput, + ProvisionedThroughputDescription, TableDescription, TimeToLiveDescription, + VectorIndexSpecification, +}; use futures::future::BoxFuture; use futures::stream::BoxStream; @@ -89,26 +93,37 @@ pub use extenddb_core::backup_format::SnapshotDescriptor as SnapshotInfo; /// so the check can set it and observe its effect without a per-backend /// setter that a test could forget to call. pub mod paging { - /// Rows or documents a backend fetches per round trip. Measured on + /// Rows or documents a backend fetches per round trip, at most. Measured on /// localhost `PostgreSQL` 16 with 318-byte items: 1000 and 10000 read a /// million rows in the same time; 1000 holds less memory. Over a 1 ms /// network link the larger fetch would save about a second per million /// rows, which is why the size is a constant and not a configuration knob. - /// - /// The bound is rows, not bytes. `PostgreSQL`'s `FETCH` and `SQLite`'s - /// `LIMIT` each hold up to this many rows of item text in memory at - /// once, and at the default 400 KB item limit one row's text can reach - /// 27.7 MB (the measured worst case in #394's line-cap table), so one - /// fetch is bounded by rows, not bytes. A byte budget (keyset paging with a byte cutoff) is - /// required before the stream gets a caller; the backup design assigns - /// it to the backup job PR. `MongoDB`'s batches are bounded by the - /// server's 16 MiB reply limit as well as by this count. pub const DEFAULT_FETCH_BATCH: usize = 1000; + /// Bytes of item text a page holds, at most, before the page is cut: a + /// page ends at [`DEFAULT_FETCH_BATCH`] rows or after the row that takes + /// its running total of item text to or past this budget, whichever + /// comes first, so a page holds at most the budget plus one row and at + /// least one row. At the default 400 KB item limit one row's text can + /// reach 27.7 MB (the measured worst case in #394's line-cap table), so a + /// row bound alone would let a page of such rows hold gigabytes; the + /// budget is the term of the backup job's memory formula the stream + /// contributes. `PostgreSQL` applies the cut in its page query (a running + /// sum over the key order, so the server never returns the rows past the + /// cut), `SQLite` while it pulls the page's rows, and `MongoDB` is bounded + /// by the server's 16 MiB reply instead. + pub const SNAPSHOT_FETCH_BYTES: usize = 64 * 1024 * 1024; + #[cfg(feature = "test-hooks")] static OVERRIDE: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); #[cfg(feature = "test-hooks")] + static BYTES_OVERRIDE: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); + #[cfg(feature = "test-hooks")] static FETCHES: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + #[cfg(feature = "test-hooks")] + static MAX_PAGE_BYTES: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); + #[cfg(feature = "test-hooks")] + static ROWS_PAST_CUT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); /// The page size a stream is built with. Backends read it once when they /// open a stream, so a change to the knob during a stream cannot end it @@ -128,12 +143,29 @@ pub mod paging { } } + /// The byte budget a stream is built with ([`SNAPSHOT_FETCH_BYTES`], or + /// the `test-hooks` override). Read once per open, like [`fetch_batch`]. + #[must_use] + pub fn fetch_bytes() -> usize { + #[cfg(feature = "test-hooks")] + { + match BYTES_OVERRIDE.load(std::sync::atomic::Ordering::Relaxed) { + 0 => SNAPSHOT_FETCH_BYTES, + n => n, + } + } + #[cfg(not(feature = "test-hooks"))] + { + SNAPSHOT_FETCH_BYTES + } + } + /// Record one round trip to the backend for a page of items. A no-op /// without `test-hooks`. /// /// The counter is trusted: the conformance check reads it to confirm /// the backend paged at the knob's size, so a backend must call this - /// exactly once per round trip (one `FETCH`, one page query, one + /// exactly once per round trip (one page query, one /// `find` or `getMore`), never per item. A backend that ignored the /// knob and called it per item would pass the paging assertions with /// the whole table read in one trip. `MongoDB`'s driver hides `getMore` @@ -145,6 +177,27 @@ pub mod paging { FETCHES.fetch_add(1, std::sync::atomic::Ordering::Relaxed); } + /// Record the bytes of item text one page returned, so the conformance + /// check can read the largest page (`max_page_bytes`). Called once per + /// page by the backends that apply the byte cut; a no-op without + /// `test-hooks`. + pub fn record_page_bytes(bytes: usize) { + #[cfg(feature = "test-hooks")] + MAX_PAGE_BYTES.fetch_max(bytes, std::sync::atomic::Ordering::Relaxed); + #[cfg(not(feature = "test-hooks"))] + let _ = bytes; + } + + /// Record rows the database read for a page and dropped at the byte cut + /// (`PostgreSQL`'s outer `WHERE`; at most the page's row bound less one). + /// A no-op without `test-hooks`. + pub fn record_rows_past_cut(n: u64) { + #[cfg(feature = "test-hooks")] + ROWS_PAST_CUT.fetch_add(n, std::sync::atomic::Ordering::Relaxed); + #[cfg(not(feature = "test-hooks"))] + let _ = n; + } + /// Set the page size for every stream opened afterwards; `0` restores the /// default. #[cfg(feature = "test-hooks")] @@ -152,6 +205,21 @@ pub mod paging { OVERRIDE.store(rows, std::sync::atomic::Ordering::Relaxed); } + /// Set the byte budget for every stream opened afterwards; `0` restores + /// [`SNAPSHOT_FETCH_BYTES`]. + #[cfg(feature = "test-hooks")] + pub fn set_fetch_bytes(bytes: usize) { + BYTES_OVERRIDE.store(bytes, std::sync::atomic::Ordering::Relaxed); + } + + /// Zero [`max_page_bytes`] and [`rows_past_cut`], so a check reads only + /// the pages it drives afterwards. + #[cfg(feature = "test-hooks")] + pub fn reset_page_counters() { + MAX_PAGE_BYTES.store(0, std::sync::atomic::Ordering::Relaxed); + ROWS_PAST_CUT.store(0, std::sync::atomic::Ordering::Relaxed); + } + /// Round trips recorded by every stream in this process so far. #[cfg(feature = "test-hooks")] #[must_use] @@ -159,6 +227,22 @@ pub mod paging { FETCHES.load(std::sync::atomic::Ordering::Relaxed) } + /// The largest page (bytes of item text) recorded since the last + /// [`reset_page_counters`]. + #[cfg(feature = "test-hooks")] + #[must_use] + pub fn max_page_bytes() -> usize { + MAX_PAGE_BYTES.load(std::sync::atomic::Ordering::Relaxed) + } + + /// Rows read past a page's cut, summed since the last + /// [`reset_page_counters`]. + #[cfg(feature = "test-hooks")] + #[must_use] + pub fn rows_past_cut() -> u64 { + ROWS_PAST_CUT.load(std::sync::atomic::Ordering::Relaxed) + } + /// The two points in `open_table_snapshot` at which every backend calls /// [`open_pause`]. Under `test-hooks` the conformance check holds an open /// at one of them while it commits writes and an `UpdateTable`, to check @@ -177,9 +261,8 @@ pub mod paging { /// layout the pause is before the `LOCK TABLE`, in the window where a /// `DeleteTable` can still drop the data table. AfterDefinition, - /// The data snapshot is pinned; the item read (`PostgreSQL`'s `DECLARE - /// ... CURSOR`, `MongoDB`'s `find`, `SQLite`'s first page) has not - /// started. + /// The data snapshot is pinned; the item read (`PostgreSQL`'s and + /// `SQLite`'s first page, `MongoDB`'s `find`) has not started. BeforeItems, } @@ -252,6 +335,116 @@ pub fn key_schema_is_orderable(description: &TableDescription) -> bool { KeyOrder::resolve(&description.key_schema, &description.attribute_definitions).is_ok() } +/// The manifest's `source_table` object for a table, built from the +/// description a snapshot returned and the table's TTL description, which is +/// not part of a [`TableSnapshot`] and is read beside the open +/// (`MetadataEngine::describe_ttl`). One derivation for every backend, so the +/// row a CreateBackup inserts and the manifest its job writes carry the same +/// object; the legacy format's `BackupTableDefinition` is a different shape +/// and stays its own. +/// +/// `billing_mode` is the summary's, PROVISIONED when the description has no +/// summary; `provisioned_throughput` and each GSI's are written only for a +/// PROVISIONED table (an on-demand table's description echoes zeros, which +/// no CreateTable accepts). `vector_indexes` is `None` when the description +/// carries none (a backend without vector support) and the list of +/// specifications otherwise, empty included, so a reader can tell the two +/// apart. `item_count` and `table_size_bytes` are the description's, which +/// are the stats pass's figures: the job replaces `item_count` with the +/// stream's count before it writes the manifest. +/// +/// # Errors +/// +/// [`StorageError::Internal`] when the description's `TableClassSummary` has +/// no `TableClass` string: no in-tree backend writes such a summary, and +/// dropping the class silently would make a backup that restores as a +/// different table than it describes. +pub fn source_table_schema( + description: &TableDescription, + ttl: Option, +) -> Result { + let billing_mode = description + .billing_mode_summary + .as_ref() + .map_or(BillingMode::Provisioned, |s| s.billing_mode); + let provisioned = billing_mode == BillingMode::Provisioned; + let throughput = |d: &ProvisionedThroughputDescription| { + provisioned.then_some(ProvisionedThroughput { + read_capacity_units: d.read_capacity_units, + write_capacity_units: d.write_capacity_units, + }) + }; + let table_class = match &description.table_class_summary { + None => None, + Some(summary) => Some( + summary + .get("TableClass") + .and_then(serde_json::Value::as_str) + .ok_or_else(|| { + StorageError::Internal(format!( + "table {} has a TableClassSummary without a TableClass: {summary}", + description.table_name + )) + })? + .to_owned(), + ), + }; + let global_secondary_indexes = description + .global_secondary_indexes + .iter() + .flatten() + .map(|g| GsiInput { + index_name: g.index_name.clone(), + key_schema: g.key_schema.clone(), + projection: g.projection.clone(), + provisioned_throughput: g.provisioned_throughput.as_ref().and_then(throughput), + }) + .collect(); + let local_secondary_indexes = description + .local_secondary_indexes + .iter() + .flatten() + .map(|l| LsiInput { + index_name: l.index_name.clone(), + key_schema: l.key_schema.clone(), + projection: l.projection.clone(), + }) + .collect(); + let vector_indexes = description.vector_indexes.as_ref().map(|indexes| { + indexes + .iter() + .map(|v| VectorIndexSpecification { + index_name: v.index_name.clone(), + dimensions: v.dimensions, + distance_function: v.distance_function, + vector_attribute: v.vector_attribute.clone(), + search_schema: v.search_schema.clone(), + projection: v.projection.clone(), + }) + .collect() + }); + Ok(SourceTableSchema { + table_name: description.table_name.clone(), + table_id: description.table_id.clone(), + table_arn: description.table_arn.clone(), + table_creation_date_time: description.creation_date_time, + key_schema: description.key_schema.clone(), + attribute_definitions: description.attribute_definitions.clone(), + billing_mode, + provisioned_throughput: throughput(&description.provisioned_throughput), + on_demand_throughput: description.on_demand_throughput.clone(), + table_class, + sse_description: description.sse_description.clone(), + global_secondary_indexes, + local_secondary_indexes, + vector_indexes, + stream_specification: description.stream_specification.clone(), + time_to_live: ttl, + item_count: description.item_count, + table_size_bytes: description.table_size_bytes, + }) +} + /// A consistent view of one table: its definition, how the snapshot was /// taken, and every item as of the same instant. pub struct TableSnapshot { @@ -396,8 +589,199 @@ pub trait SnapshotEngine: Send + Sync { #[cfg(test)] mod tests { use super::*; + use extenddb_core::backup_format::ExtenddbManifest; use extenddb_core::types::GsiDescription; + /// A description as the storage layer reports it: PROVISIONED, one GSI + /// (with throughput), one LSI, one vector index, a stream, a table class + /// and SSE; the table of the manifest fixture, so the schema can be + /// spliced into it. + fn described() -> TableDescription { + serde_json::from_value(serde_json::json!({ + "TableName": "Music", + "TableId": "12345a12-abcd-123a-ab12-1234abc12345", + "TableArn": "arn:aws:dynamodb:us-east-1:123456789012:table/Music", + "TableStatus": "ACTIVE", + "CreationDateTime": 1757400000.0, + "TableSizeBytes": 123456, + "ItemCount": 100000, + "KeySchema": [ + {"AttributeName": "Artist", "KeyType": "HASH"}, + {"AttributeName": "SongTitle", "KeyType": "RANGE"} + ], + "AttributeDefinitions": [ + {"AttributeName": "Artist", "AttributeType": "S"}, + {"AttributeName": "SongTitle", "AttributeType": "S"}, + {"AttributeName": "Genre", "AttributeType": "S"} + ], + "ProvisionedThroughput": { + "ReadCapacityUnits": 5, "WriteCapacityUnits": 5, "NumberOfDecreasesToday": 0 + }, + "BillingModeSummary": {"BillingMode": "PROVISIONED"}, + "GlobalSecondaryIndexes": [{ + "IndexName": "GenreIndex", + "KeySchema": [{"AttributeName": "Genre", "KeyType": "HASH"}], + "Projection": {"ProjectionType": "INCLUDE", "NonKeyAttributes": ["Year"]}, + "IndexStatus": "ACTIVE", + "ProvisionedThroughput": { + "ReadCapacityUnits": 1, "WriteCapacityUnits": 1, "NumberOfDecreasesToday": 0 + }, + "IndexSizeBytes": 10, "ItemCount": 3, + "IndexArn": "arn:aws:dynamodb:us-east-1:123456789012:table/Music/index/GenreIndex" + }], + "LocalSecondaryIndexes": [{ + "IndexName": "TitleIndex", + "KeySchema": [ + {"AttributeName": "Artist", "KeyType": "HASH"}, + {"AttributeName": "SongTitle", "KeyType": "RANGE"} + ], + "Projection": {"ProjectionType": "KEYS_ONLY"}, + "IndexSizeBytes": 10, "ItemCount": 3, + "IndexArn": "arn:aws:dynamodb:us-east-1:123456789012:table/Music/index/TitleIndex" + }], + "VectorIndexes": [{ + "IndexName": "embeddings", + "VectorAttribute": {"AttributeName": "vec"}, + "Dimensions": 1024, + "DistanceFunction": "COSINE", + "IndexStatus": "ACTIVE", + "IndexSizeBytes": 0, "ItemCount": 0, + "IndexArn": "arn:aws:dynamodb:us-east-1:123456789012:table/Music/index/embeddings" + }], + "StreamSpecification": {"StreamEnabled": true, "StreamViewType": "NEW_AND_OLD_IMAGES"}, + "DeletionProtectionEnabled": false, + "SSEDescription": { + "Status": "ENABLED", "SSEType": "KMS", + "KMSMasterKeyArn": "arn:aws:kms:us-east-1:123456789012:key/abcd" + }, + "TableClassSummary": {"TableClass": "STANDARD"}, + "OnDemandThroughput": {"MaxReadRequestUnits": 100, "MaxWriteRequestUnits": 50} + })) + .expect("a table description") + } + + fn ttl() -> TimeToLiveDescription { + serde_json::from_value(serde_json::json!({ + "TimeToLiveStatus": "ENABLED", + "AttributeName": "expires_at" + })) + .expect("a TTL description") + } + + /// The fixture manifest with `source_table` replaced by `schema`, parsed + /// back through the reader's own checks. + fn manifest_with(schema: &SourceTableSchema) -> Result { + let mut manifest: serde_json::Value = serde_json::from_str(include_str!( + "../../core/testdata/extenddb-manifest/v1.json" + )) + .expect("fixture"); + manifest["source_table"] = serde_json::to_value(schema).expect("serialize"); + let bytes = serde_json::to_vec(&manifest).expect("bytes"); + ExtenddbManifest::from_json_bytes(&bytes).map_err(|e| e.to_string()) + } + + /// The schema built from a full description is the fixture's own + /// `source_table`, member for member, and reads back through the + /// manifest's checks. + #[test] + fn a_described_table_becomes_the_fixtures_source_table() { + let schema = source_table_schema(&described(), Some(ttl())).expect("schema"); + let fixture: serde_json::Value = serde_json::from_str(include_str!( + "../../core/testdata/extenddb-manifest/v1.json" + )) + .expect("fixture"); + assert_eq!( + serde_json::to_value(&schema).expect("serialize"), + fixture["source_table"] + ); + let manifest = manifest_with(&schema).expect("the manifest reads back"); + assert_eq!(manifest.source_table, schema); + assert_eq!( + manifest.source_table.vector_indexes.as_ref().map(Vec::len), + Some(1), + "the vector index specification is carried" + ); + } + + /// The members the reader requires even when absent (`vector_indexes`, + /// `stream_specification`, `time_to_live`) are written as explicit + /// nulls, so the document reads back; an on-demand table writes no + /// throughput for itself or its GSIs. + #[test] + fn absent_members_are_written_as_nulls_and_on_demand_drops_throughput() { + let mut description = described(); + description.vector_indexes = None; + description.stream_specification = None; + description.table_class_summary = None; + description.billing_mode_summary = Some(extenddb_core::types::BillingModeSummary { + billing_mode: BillingMode::PayPerRequest, + last_update_to_pay_per_request_date_time: None, + }); + let schema = source_table_schema(&description, None).expect("schema"); + let json = serde_json::to_value(&schema).expect("serialize"); + for member in [ + "vector_indexes", + "stream_specification", + "time_to_live", + "table_class", + ] { + assert_eq!(json[member], serde_json::Value::Null, "{member}"); + assert!( + json.as_object().unwrap().contains_key(member), + "{member} must be present as null, not missing" + ); + } + assert_eq!(schema.billing_mode, BillingMode::PayPerRequest); + assert_eq!(schema.provisioned_throughput, None); + assert_eq!( + schema.global_secondary_indexes[0].provisioned_throughput, + None + ); + assert_eq!( + json["global_secondary_indexes"].as_array().map(Vec::len), + Some(1) + ); + assert_eq!( + json["local_secondary_indexes"].as_array().map(Vec::len), + Some(1) + ); + manifest_with(&schema).expect("the manifest reads back"); + } + + /// A description without a billing-mode summary is PROVISIONED with the + /// description's throughput; a table without vector support writes + /// `vector_indexes: null`, one with support and no indexes `[]`. + #[test] + fn billing_defaults_to_provisioned_and_vector_support_is_told_from_no_indexes() { + let mut description = described(); + description.billing_mode_summary = None; + description.vector_indexes = Some(vec![]); + let schema = source_table_schema(&description, None).expect("schema"); + assert_eq!(schema.billing_mode, BillingMode::Provisioned); + assert_eq!( + schema.provisioned_throughput, + Some(ProvisionedThroughput { + read_capacity_units: 5, + write_capacity_units: 5 + }) + ); + assert_eq!(schema.vector_indexes, Some(vec![])); + description.vector_indexes = None; + let schema = source_table_schema(&description, None).expect("schema"); + assert_eq!(schema.vector_indexes, None); + } + + #[test] + fn a_table_class_summary_without_a_class_is_an_error() { + let mut description = described(); + description.table_class_summary = Some(serde_json::json!({"Other": 1})); + let err = source_table_schema(&description, None).expect_err("refused"); + assert!( + matches!(&err, StorageError::Internal(m) if m.contains("TableClassSummary")), + "{err}" + ); + } + #[test] fn deleting_indexes_are_dropped_and_others_kept() { let gsi = |name: &str, status: &str| -> GsiDescription { diff --git a/crates/storage/src/snapshot_conformance.rs b/crates/storage/src/snapshot_conformance.rs index 04b58c9b2..7429836e7 100644 --- a/crates/storage/src/snapshot_conformance.rs +++ b/crates/storage/src/snapshot_conformance.rs @@ -375,11 +375,39 @@ where /// Drain `stream` into `into`, each item within [`DRAIN_TIMEOUT`] (see /// [`next_within`]); `stage` names the drain in the panic. async fn drain_within(stream: &mut S, stage: &str, into: &mut Vec) +where + S: futures::Stream> + Unpin, +{ + drain_at_most(stream, stage, into, usize::MAX).await; +} + +/// [`drain_within`] with a bound on the count: a stream that has yielded +/// more than `at_most` items is repeating rows (a seek that is inclusive +/// instead of exclusive yields the row it was seeked past on every page, and +/// a cursor that never advances yields the first page forever), and the +/// drain panics at once naming `stage` and the first key yielded twice, so +/// the check fails in seconds instead of running until another test's wait +/// on the hooks lock gives out. +async fn drain_at_most(stream: &mut S, stage: &str, into: &mut Vec, at_most: usize) where S: futures::Stream> + Unpin, { while let Some(next) = next_within(stream, stage, into.len()).await { into.push(next.unwrap_or_else(|e| panic!("{stage}: snapshot stream item: {e}"))); + if into.len() > at_most { + let mut seen = std::collections::HashSet::new(); + let repeated = into + .iter() + .map(key_id) + .find(|id| !seen.insert(id.clone())) + .unwrap_or_else(|| "(no key yielded twice)".to_owned()); + panic!( + "{stage}: the stream yielded {} items of a table of {at_most}; the first key \ + yielded twice is {repeated} (the stream repeats rows: an inclusive seek or a \ + cursor that does not advance)", + into.len() + ); + } } } @@ -566,6 +594,26 @@ impl Drop for PageSize { } } +/// Sets the shared snapshot byte budget ([`paging::set_fetch_bytes`]) and +/// restores the default when dropped, as [`PageSize`] does for the row +/// bound; hold [`hooks_lock`] while one is alive. +pub struct ByteBudget(()); + +impl ByteBudget { + /// Set the byte budget to `bytes` until the returned value is dropped. + #[must_use] + pub fn set(bytes: usize) -> Self { + paging::set_fetch_bytes(bytes); + Self(()) + } +} + +impl Drop for ByteBudget { + fn drop(&mut self) { + paging::set_fetch_bytes(0); + } +} + /// What the check expects of the backend under test. #[derive(Debug, Clone, Copy)] pub struct Expectations { @@ -1465,6 +1513,206 @@ async fn key_shapes( } } +/// Items [`run_page_bounds`] fills its table with: more than one row bound's +/// worth at the budget it sets, and ten past a multiple of the rows a page +/// keeps, so the last page is one the cut shortens (a backend that ends the +/// stream on a short page without asking whether the cut shortened it loses +/// those rows). +pub const PAGE_BOUNDS_ITEMS: usize = 610; + +/// Rows of item text one page of [`run_page_bounds`]'s second pass holds at +/// the budget it sets: the budget is this many rows' text. +pub const PAGE_BOUNDS_ROWS: usize = 8; + +/// What [`run_page_bounds`] measured, for a backend's own assertions beside +/// it. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct PageBounds { + /// Bytes of item text of one row of the table (every row is the same + /// size). + pub row_bytes: usize, + /// The byte budget the second pass ran under: [`PAGE_BOUNDS_ROWS`] rows. + pub budget: usize, + /// Round trips the second pass made. + pub pages: u64, + /// Rows the backend recorded as read past a cut during the second pass + /// ([`paging::rows_past_cut`]; a backend that cuts in its query records + /// them, one that cuts while pulling rows records none). + pub rows_past_cut: u64, +} + +fn page_bounds_item(i: usize) -> Item { + // Fixed-width keys and a fixed padding, so every row's text is the same + // length and the budget can be set to a whole number of rows. + [ + ("pk".to_owned(), s(format!("p{i:05}"))), + ("sk".to_owned(), n(100_000 + i)), + ("v".to_owned(), s("x".repeat(1000))), + ] + .into_iter() + .collect() +} + +/// Check that the stream pages by bytes: no page holds more than the budget +/// plus one row, and the cut does not lose rows. `hooks` is the caller's +/// [`hooks_lock`] guard; `table` must not exist; `pump` runs while the table +/// becomes `ACTIVE`. +/// +/// The table is [`PAGE_BOUNDS_ITEMS`] rows of one size. A first pass at a +/// budget of one byte reads one row per page, which measures the row's text +/// size as the backend counts it (`row_bytes`) and shows the budget took +/// effect (as many round trips as rows). The second pass sets the budget to +/// [`PAGE_BOUNDS_ROWS`] rows and the row bound to its default, and asserts: +/// every item once (completeness across cut pages, the last one included; +/// both passes stop and fail as soon as the stream has yielded more items +/// than the table holds, naming the key yielded twice, so a stream that +/// repeats rows fails in seconds rather than running until the hooks lock's +/// wait gives out); +/// more round trips than the row bound alone would make (the budget, not the +/// row bound, paged the table: a counter beside the knob, since the stream's +/// contents alone cannot show the knob took effect); the largest page at least the +/// budget (a page was cut) and under the budget plus one row (the rule: a +/// page ends after the row that takes it to or past the budget, so a page +/// cut one row late shows as exactly the budget plus one row); and, when the +/// stream is `key_ordered`, ascending order. +/// +/// # Panics +/// +/// On any violation, with the figures. +pub async fn run_page_bounds( + hooks: &HooksGuard, + engine: &E, + account: &str, + table: &str, + mut pump: F, +) -> PageBounds +where + E: SnapshotEngine + TableEngine + DataEngine + Send + Sync, + F: FnMut() -> Fut, + Fut: std::future::Future, +{ + let _ = hooks; + engine + .create_table(account, table_input(table)) + .await + .expect("create the page-bounds table"); + wait_active(engine, account, table, &mut pump).await; + let ki = key_info(engine, account, table).await; + for i in 0..PAGE_BOUNDS_ITEMS { + put(engine, &ki, page_bounds_item(i)).await; + } + let expected: BTreeSet = (0..PAGE_BOUNDS_ITEMS) + .map(|i| key_id(&page_bounds_item(i))) + .collect(); + + // First pass: one row per page. + let row_bytes = { + let _budget = ByteBudget::set(1); + let _rows = PageSize::set(0); + paging::reset_page_counters(); + let fetches_before = paging::fetches(); + let mut snapshot = engine + .open_table_snapshot(account, table) + .await + .expect("open at a one-byte budget"); + let mut seen = Vec::new(); + drain_at_most( + &mut snapshot.items, + "page bounds, one-byte budget", + &mut seen, + PAGE_BOUNDS_ITEMS, + ) + .await; + drop(snapshot); + assert_eq!(seen.len(), PAGE_BOUNDS_ITEMS, "one-byte budget: items read"); + let fetches = paging::fetches() - fetches_before; + assert!( + fetches >= PAGE_BOUNDS_ITEMS as u64, + "one-byte budget: {fetches} round trips for {PAGE_BOUNDS_ITEMS} rows; the budget \ + did not cut the pages to one row" + ); + let row_bytes = paging::max_page_bytes(); + assert!( + row_bytes > 1000, + "one row of item text measured as {row_bytes} bytes" + ); + row_bytes + }; + + // Second pass: a budget of a whole number of rows, the row bound at its + // default. + let budget = row_bytes * PAGE_BOUNDS_ROWS; + let _budget = ByteBudget::set(budget); + let _rows = PageSize::set(0); + paging::reset_page_counters(); + let fetches_before = paging::fetches(); + let mut snapshot = engine + .open_table_snapshot(account, table) + .await + .expect("open at the row-multiple budget"); + let key_ordered = snapshot.key_ordered; + let order = extenddb_core::backup_format::KeyOrder::resolve( + &snapshot.description.key_schema, + &snapshot.description.attribute_definitions, + ) + .expect("the page-bounds table has an orderable key"); + let mut seen = Vec::new(); + drain_at_most( + &mut snapshot.items, + "page bounds, row-multiple budget", + &mut seen, + PAGE_BOUNDS_ITEMS, + ) + .await; + drop(snapshot); + let pages = paging::fetches() - fetches_before; + + let mut got = BTreeSet::new(); + for item in &seen { + let id = key_id(item); + assert!(got.insert(id.clone()), "page bounds: duplicate item {id}"); + } + assert_eq!( + got, expected, + "page bounds: the stream did not yield every row exactly once (a cut page lost or \ + repeated rows)" + ); + let row_bound_pages = PAGE_BOUNDS_ITEMS.div_ceil(paging::fetch_batch()) as u64; + assert!( + pages > row_bound_pages, + "page bounds: {pages} round trips, no more than the row bound alone would make \ + ({row_bound_pages}); the byte budget did not page the table" + ); + let largest = paging::max_page_bytes(); + assert!( + largest >= budget, + "page bounds: the largest page holds {largest} bytes, under the budget of {budget}; no \ + page was cut by bytes" + ); + assert!( + largest < budget + row_bytes, + "page bounds: the largest page holds {largest} bytes, the budget plus a row or more \ + ({budget} + {row_bytes}); a page is cut one row late" + ); + if key_ordered { + for pair in seen.windows(2) { + assert_eq!( + order.compare(&pair[0], &pair[1]).expect("comparable keys"), + std::cmp::Ordering::Less, + "page bounds: {} is not before {}", + key_id(&pair[0]), + key_id(&pair[1]) + ); + } + } + PageBounds { + row_bytes, + budget, + pages, + rows_past_cut: paging::rows_past_cut(), + } +} + /// Checks that a snapshot cannot be opened on a missing or non-`ACTIVE` /// table. `hooks` is the caller's [`hooks_lock`] guard. `table` must not /// exist; `set_updating` flips that table's catalog diff --git a/devtools/backup-kill-points.py b/devtools/backup-kill-points.py new file mode 100755 index 000000000..3480201c8 --- /dev/null +++ b/devtools/backup-kill-points.py @@ -0,0 +1,1151 @@ +#!/usr/bin/env python3 +"""The API half of kill-points.sh and bench-rss.sh: load a table, start a +backup, poll it, read the store. + +Usage (every call reads EXTENDDB_TEST_ENDPOINT and the AWS_* credentials +provision-test-credentials exported): + kill-points.py create-table [--items N] [--pad BYTES] [--gsis N] [--threads N] + kill-points.py create-backup -> prints the ARN + kill-points.py status -> CREATING|AVAILABLE|NOTFOUND + kill-points.py wait-available [--timeout S] -> prints size,count,duration + kill-points.py describe -> JSON + kill-points.py put-worst [--count N] [--sets N] [--small N] [--interleave N] + -> the worst-expansion items after N small ones, or at every Nth key among them + kill-points.py put-random [--count N] [--bytes N] +""" +from __future__ import annotations + +import argparse +import base64 +import json +import os +import sys +import threading +import time +from concurrent.futures import ThreadPoolExecutor + +import boto3 +import urllib3 +from botocore.config import Config +from botocore.exceptions import ClientError + +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) +ENDPOINT = os.environ.get("EXTENDDB_TEST_ENDPOINT", "") + + +def client(): + return boto3.client( + "dynamodb", + endpoint_url=ENDPOINT, + region_name="us-east-1", + verify=False, + config=Config(retries={"max_attempts": 10, "mode": "standard"}, read_timeout=120), + ) + + +def wait_active(c, name: str) -> None: + while c.describe_table(TableName=name)["Table"]["TableStatus"] != "ACTIVE": + time.sleep(0.2) + # Every GSI ACTIVE too. + while any( + g.get("IndexStatus") != "ACTIVE" + for g in c.describe_table(TableName=name)["Table"].get("GlobalSecondaryIndexes", []) + ): + time.sleep(0.2) + + +def create_table(args) -> None: + c = client() + attrs = [ + {"AttributeName": "pk", "AttributeType": "S"}, + {"AttributeName": "sk", "AttributeType": "N"}, + ] + gsis = [] + for i in range(args.gsis): + attrs.append({"AttributeName": f"g{i}", "AttributeType": "S"}) + gsis.append( + { + "IndexName": f"by-g{i}", + "KeySchema": [ + {"AttributeName": f"g{i}", "KeyType": "HASH"}, + {"AttributeName": "sk", "KeyType": "RANGE"}, + ], + "Projection": {"ProjectionType": "ALL"}, + } + ) + kwargs = {"GlobalSecondaryIndexes": gsis} if gsis else {} + c.create_table( + TableName=args.name, + KeySchema=[ + {"AttributeName": "pk", "KeyType": "HASH"}, + {"AttributeName": "sk", "KeyType": "RANGE"}, + ], + AttributeDefinitions=attrs, + BillingMode="PAY_PER_REQUEST", + **kwargs, + ) + wait_active(c, args.name) + n = args.items + if n == 0: + return + local = threading.local() + started = time.monotonic() + + def write(start: int) -> None: + if not hasattr(local, "c"): + local.c = client() + reqs = [] + for i in range(start, min(start + 25, n)): + item = { + "pk": {"S": f"p{i % 1000:05d}"}, + "sk": {"N": str(i)}, + "pad": {"S": "x" * args.pad}, + } + for g in range(args.gsis): + item[f"g{g}"] = {"S": f"v{(i // (g + 1)) % 97}"} + reqs.append({"PutRequest": {"Item": item}}) + pending = {args.name: reqs} + while pending: + pending = local.c.batch_write_item(RequestItems=pending).get("UnprocessedItems") or {} + if pending: + time.sleep(0.05) + + with ThreadPoolExecutor(max_workers=args.threads) as pool: + list(pool.map(write, range(0, n, 25))) + wait_active(c, args.name) + print(f"loaded {n} items in {time.monotonic() - started:.1f} s", file=sys.stderr) + + +def put_worst(args) -> None: + """#394's worst-expansion lines (`data_file.rs`'s line-cap table): an + item of 2,262 `NS` sets of the 90 values `1.0E-130`..`9.9E-130` is + 409,429 bytes as the service sizes items and 27.7 MB as a data-file line + (and as stored item text, the server normalizing each number to its + 133-character decimal form), placed after `--small` small rows (their + `a...` keys sort before `zz...`) or, with `--interleave N`, at every Nth + key among them (the shape of a table whose large items are a minority: + key order does not sort items by size).""" + c = client() + every = args.interleave + big_positions = set([i for i in range(args.small) if every and i % every == every - 1][: args.count]) + small = [ + {"PutRequest": {"Item": {"pk": {"S": f"a{i:04d}"}, "sk": {"N": str(i)}, "pad": {"S": "x" * 50}}}} + for i in range(args.small) if i not in big_positions + ] + for i in range(0, len(small), 25): + pending = {args.name: small[i : i + 25]} + while pending: + pending = c.batch_write_item(RequestItems=pending).get("UnprocessedItems") or {} + values = [f"{a}.{b}E-130" for a in range(1, 10) for b in range(10)] + # MongoDB stores an item as one BSON document, capped at 16 MiB, so + # #394's 2,262-set shape (28.9 MB as BSON) cannot be stored there; --sets + # lowers the count to the largest row the backend takes. + sets = [{"NS": values} for _ in range(args.sets)] + if every: + for i in sorted(big_positions): + c.put_item(TableName=args.name, Item={"pk": {"S": f"a{i:04d}"}, "sk": {"N": str(i)}, "l": {"L": sets}}) + print(f"worst-expansion: {args.small} small items with {len(big_positions)} list-of-{args.sets}-NS items interleaved every {every}", file=sys.stderr) + else: + for i in range(args.count): + c.put_item(TableName=args.name, Item={"pk": {"S": f"zz{i:03d}"}, "sk": {"N": "0"}, "l": {"L": sets}}) + print(f"worst-expansion: {args.small} small items then {args.count} list-of-{args.sets}-NS items", file=sys.stderr) + + +def put_random(args) -> None: + """Incompressible items: `B` attributes of random bytes, `args.bytes` in + all across `args.count` items.""" + c = client() + per = args.bytes // args.count + for i in range(args.count): + c.put_item( + TableName=args.name, + Item={"pk": {"S": f"r{i:05d}"}, "sk": {"N": "0"}, "blob": {"B": os.urandom(per)}}, + ) + print(f"random: {args.count} items of {per} bytes", file=sys.stderr) + + +def create_backup(args) -> None: + c = client() + d = c.create_backup(TableName=args.name, BackupName=args.backup_name)["BackupDetails"] + print(d["BackupArn"]) + print(d["BackupStatus"], file=sys.stderr) + + +def status(args) -> None: + c = client() + try: + print(c.describe_backup(BackupArn=args.arn)["BackupDescription"]["BackupDetails"]["BackupStatus"]) + except ClientError as e: + if e.response["Error"]["Code"] == "BackupNotFoundException": + print("NOTFOUND") + else: + raise + + +def wait_available(args) -> None: + c = client() + started = time.monotonic() + while True: + d = c.describe_backup(BackupArn=args.arn)["BackupDescription"] + s = d["BackupDetails"]["BackupStatus"] + if s == "AVAILABLE": + print( + json.dumps( + { + "size_bytes": d["BackupDetails"]["BackupSizeBytes"], + "item_count": d["SourceTableDetails"]["ItemCount"], + "duration_s": round(time.monotonic() - started, 2), + } + ) + ) + return + if s != "CREATING": + print(f"backup is {s}", file=sys.stderr) + sys.exit(2) + if time.monotonic() - started > args.timeout: + print("timeout", file=sys.stderr) + sys.exit(3) + time.sleep(0.2) + + +def describe(args) -> None: + c = client() + print(json.dumps(c.describe_backup(BackupArn=args.arn)["BackupDescription"], default=str, indent=1)) + + +def main() -> None: + p = argparse.ArgumentParser() + sub = p.add_subparsers(dest="cmd", required=True) + a = sub.add_parser("create-table") + a.add_argument("name") + a.add_argument("--items", type=int, default=100000) + a.add_argument("--pad", type=int, default=200) + a.add_argument("--gsis", type=int, default=2) + a.add_argument("--threads", type=int, default=16) + a.set_defaults(fn=create_table) + a = sub.add_parser("put-worst") + a.add_argument("name") + a.add_argument("--count", type=int, default=20) + a.add_argument("--sets", type=int, default=2262) + a.add_argument("--small", type=int, default=1000) + a.add_argument("--interleave", type=int, default=0, help="place the large items at every Nth key among the small ones instead of after them") + a.set_defaults(fn=put_worst) + a = sub.add_parser("put-random") + a.add_argument("name") + a.add_argument("--count", type=int, default=256) + a.add_argument("--bytes", type=int, default=64 * 1024 * 1024) + a.set_defaults(fn=put_random) + a = sub.add_parser("create-backup") + a.add_argument("name") + a.add_argument("backup_name") + a.set_defaults(fn=create_backup) + a = sub.add_parser("status") + a.add_argument("arn") + a.set_defaults(fn=status) + a = sub.add_parser("wait-available") + a.add_argument("arn") + a.add_argument("--timeout", type=float, default=1800) + a.set_defaults(fn=wait_available) + a = sub.add_parser("describe") + a.add_argument("arn") + a.set_defaults(fn=describe) + args = p.parse_args() + args.fn(args) + + + + + +# =========================================================================== +# PR 9: the restore kill points. +# +# Each run builds one backend's deployment from the hooks binary +# (`--features extenddb-app/backup-test-hooks`, whose feature chain pulls +# `extenddb-storage/test-hooks`, so the engine's restore points and the +# backend's own point are armed), `reconcile_interval_secs = 2`, +# `stale_job_after_secs = 45`; loads a table with one GSI (`by-g`, every +# item carries `g`), takes a backup, then starts a restore paused at the +# named point (`EXTENDDB_BACKUP_PAUSE`), kills the server by PID, restarts +# it without the pause, and asserts the outcome: at every ACTIVE outcome +# the base table's `Scan` count and the GSI's `Scan` count both equal +# `--items` (a resumed Cassandra restore once left the GSI short while the +# base count was exact, so the base count alone is not the assertion). +# Logs under `--work`. +# +# kill-points.py restore \ +# --bin [--items N] [--work DIR] [--second-reach] \ +# [--vector] [--source-bin ] +# +# `pg_terminate_backend` is not a pause point of its own: the server is +# paused at `restore_after_file_commit`, the session holding the claim's +# advisory lock is terminated while the job is paused (psql must answer a +# row), the script waits past one heartbeat period and releases, and asserts +# the job's lost-claim stop, the record RUNNING with its progress row, the +# reconciler's takeover no earlier than `stale_job_after_secs` after the +# owner's last beat (read from the record at the termination; the beat +# precedes the termination by up to one period), and the exact counts. `--vector` gives the source table a +# vector index; with `restore_between_failed_and_removal` on postgres the +# cell then asserts the removed target's data table and `_ddb_vec_` relation +# are gone from the data database. +# +# Cassandra has no `SnapshotEngine`, so its CreateBackup is the in-catalog +# format and no backup taken on a Cassandra server restores through the +# store-backed job. A Cassandra kill point therefore takes its backup on a +# SQLite source server (`--source-bin`, a SQLite build), copies the prefix +# under the Cassandra target's root and seeds the target's `backup_index` +# row as `devtools/backup-restore-matrix` does (and as `backup sync +# --adopt-store` will), then restores on the paused Cassandra server. The +# matrix script is imported for `copy_prefix`, `prefix_of` and `seed_row`. +# +# `--second-reach` kills at the point's second reach instead of its first: +# the first pause is released (the release file written, then removed once +# the job logs its release), and the server is SIGKILLed when the job +# reaches the point again. For `restore_mongodb_after_insert_many` and +# `restore_cassandra_before_progress` that puts the kill after an earlier +# batch's progress is recorded, so the resume continues from that progress +# (MongoDB: each batch commits with its progress, nothing is re-applied; +# Cassandra: at most one batch re-applied by the upsert). The first-reach +# kill of `restore_cassandra_before_progress` leaves the first batch's items +# with no progress recorded; the resume finds the count at or below the +# backup's total and re-applies the first file with upserts (the engine's +# `target_not_empty` rule fails only a count above the total). Every kill +# point's outcome is therefore ACTIVE with the exact count, except the +# tampered-file point, whose target is removed. +# +# The services are local (PostgreSQL 55432, MongoDB 27117, Cassandra 9042). +# Every server is stopped by the PID this script started, never by a +# pattern over the process list; every scratch database, keyspace and the +# fixed MongoDB databases are dropped. +# =========================================================================== + +import os +import re +import shutil +import signal +import subprocess + +import requests +import urllib.parse # noqa: F401 + +PG = {"host": "127.0.0.1", "port": "55432", "user": "postgres", "password": "devpass"} +# The MongoDB replica set a MongoDB kill point uses (its fixed databases are +# dropped per run, so give the script a replica set of its own). +MONGODB_URI = os.environ.get("EXTENDDB_TEST_MONGODB_URI") or "mongodb://127.0.0.1:27117/?directConnection=true" +ACCOUNT = "123456789012" +ADMIN_PASSWORD = "TestPass1!" +REGION = "us-east-1" + + +def _say(msg: str) -> None: + print(f"[{time.strftime('%H:%M:%S')}] {msg}", flush=True) + + +def _repo() -> str: + """The ExtendDB checkout whose `devtools/` scripts this script calls + (`provision-test-credentials`, `backup-restore-matrix`): `EXTENDDB_REPO`, + or the working directory when the script is run from the checkout.""" + return os.path.abspath(os.environ.get("EXTENDDB_REPO") or os.getcwd()) + + +def _free_port() -> int: + import socket + with socket.socket() as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + + +def _psql(stmt: str, database: str = "postgres") -> str: + env = dict(os.environ, PGPASSWORD=PG["password"]) + r = subprocess.run( + ["psql", "-h", PG["host"], "-p", PG["port"], "-U", PG["user"], "-d", database, + "-v", "ON_ERROR_STOP=1", "-tAq", "-c", stmt], + env=env, capture_output=True, text=True, + ) + if r.returncode != 0: + raise SystemExit(f"psql failed: {r.stderr}\n{stmt}") + return r.stdout.strip() + + +def _sqlite_connect(path: str): + try: + import pysqlite3 as sqlite3 # type: ignore + except ImportError: + import sqlite3 # type: ignore + return sqlite3.connect(path, timeout=10) + + +class Deployment: + """One backend's server for a kill-point run, from the hooks binary.""" + + def __init__(self, backend: str, binary: str, work: str): + self.backend = backend + self.bin = binary + self.work = work + os.makedirs(work, exist_ok=True) + self.config = os.path.join(work, "extenddb.toml") + self.root = os.path.join(work, "root") + self.spool = os.path.join(work, "spool") + self.port = _free_port() + self.endpoint = f"https://127.0.0.1:{self.port}" + self.tag = f"kp{int(time.time()) % 1_000_000}" + self.proc = None + self.creds = {} + + def _free_port_again(self) -> None: + self.port = _free_port() + self.endpoint = f"https://127.0.0.1:{self.port}" + + def init(self) -> None: + env = dict(os.environ, EXTENDDB_ADMIN_PASSWORD=ADMIN_PASSWORD) + if self.backend == "sqlite": + cmd = [self.bin, "init", "--backend", "sqlite", "--sqlite-path", + os.path.join(self.work, "db.sqlite"), "--config", self.config] + elif self.backend == "postgres": + cmd = [self.bin, "init", "--pg-host", PG["host"], "--pg-port", PG["port"], + "--pg-user", PG["user"], "--pg-pass", PG["password"], + "--data-db", f"{self.tag}_data", "--catalog-db", f"{self.tag}_catalog", + "--extenddb-user", f"{self.tag}_app", "--config", self.config] + elif self.backend == "mongodb": + self._mongo_drop() + with open(self.config, "w") as f: + f.write(f'[storage]\nbackend = "mongodb"\n\n[storage.mongodb]\nconnection_string = "{MONGODB_URI}"\n') + cmd = [self.bin, "init", "--backend", "mongodb", "--config", self.config, "--overwrite"] + elif self.backend == "cassandra": + cmd = [self.bin, "init", "--backend", "cassandra", + "--cassandra-contact-points", "127.0.0.1:9042", + "--cassandra-user", "cassandra", "--cassandra-pass", "cassandra", + "--config", self.config] + else: + raise SystemExit(f"unknown backend {self.backend}") + with open(os.path.join(self.work, "init.log"), "w") as log: + r = subprocess.run(cmd, cwd=self.work, env=env, stdout=log, stderr=subprocess.STDOUT) + if r.returncode != 0: + print(open(os.path.join(self.work, "init.log")).read()) + raise SystemExit(f"init of the {self.backend} server failed") + self._write_config() + + def _write_config(self) -> None: + home = os.path.expanduser("~") + os.makedirs(self.root, mode=0o700, exist_ok=True) + store = f'[backup.store]\nkind = "filesystem"\n[backup.store.filesystem]\npath = "{self.root}"\n' + backup = "[backup]\nreconcile_interval_secs = 2\nstale_job_after_secs = 45\nmax_concurrent_jobs = 2\ndata_file_target_bytes = 33554432\n" + if self.backend == "sqlite": + text = open(self.config).read() + text = re.sub(r"(?m)^[ \t]*#?[ \t]*port[ \t]*=[ \t]*\d+.*$", f"port = {self.port}", text, count=1) + text = re.sub(r'(?m)^run_dir = .*$', f'run_dir = "{self.work}/run"', text) + text += "\n" + backup + store + open(self.config, "w").write(text) + elif self.backend == "postgres": + text = open(self.config).read() + text = re.sub(r"(?m)^[ \t]*#?[ \t]*port[ \t]*=[ \t]*\d+.*$", f"port = {self.port}", text, count=1) + text = re.sub(r'(?m)^run_dir = .*$', f'run_dir = "{self.work}/run"', text) + text += "\n" + backup + store + open(self.config, "w").write(text) + else: + conn = (MONGODB_URI if self.backend == "mongodb" + else "") + storage = (f'[storage]\nbackend = "mongodb"\n\n[storage.mongodb]\nconnection_string = "{MONGODB_URI}"\n' + if self.backend == "mongodb" else + # `replication_factor = 1` as `init` writes it for a single + # node: the default is 3, and the data keyspace the first + # CreateTable creates under RF 3 on one node answers every + # lightweight transaction and quorum read with "No nodes + # available in query plan" (the catalog keyspace, created by + # `init` under RF 1, keeps working, so the server starts and + # the table reports ACTIVE before the first write fails). + '[storage]\nbackend = "cassandra"\n\n[storage.cassandra]\n' + 'contact_points = ["127.0.0.1:9042"]\nusername = "cassandra"\npassword = "cassandra"\n' + 'keyspace_prefix = "extenddb"\nreplication_factor = 1\ndatacenter = "datacenter1"\n') + open(self.config, "w").write( + f'[server]\nbind_addr = "127.0.0.1"\nport = {self.port}\nregion = "{REGION}"\n' + f'run_dir = "{self.work}/run"\n\n' + f'[server.tls]\ncert_path = "{home}/.extenddb/tls/cert.pem"\n' + f'key_path = "{home}/.extenddb/tls/key.pem"\n\n' + f'{storage}\n[auth]\nprovider = "builtin"\n\n{backup}\n{store}') + + def start(self, log_name: str, pause: str | None = None) -> None: + env = dict(os.environ) + if pause: + env["EXTENDDB_BACKUP_PAUSE"] = pause + env["EXTENDDB_BACKUP_PAUSE_RELEASE"] = os.path.join(self.work, "release") + self.log_path = os.path.join(self.work, log_name) + self.log = open(self.log_path, "a") + self.proc = subprocess.Popen( + [self.bin, "serve", "--foreground", "--config", self.config], + cwd=self.work, env=env, stdout=self.log, stderr=subprocess.STDOUT) + for _ in range(300): + try: + if requests.get(f"{self.endpoint}/health", verify=False, timeout=2).status_code == 200: + break + except requests.RequestException: + pass + if self.proc.poll() is not None: + break + time.sleep(0.2) + if self.proc.poll() is not None: + raise SystemExit(f"the {self.backend} server exited at start; see {self.log_path}") + _say(f"{self.backend} pid {self.proc.pid} on {self.port} ({log_name})") + + def sigkill(self) -> None: + if self.proc and self.proc.poll() is None: + self.proc.send_signal(signal.SIGKILL) + self.proc.wait(timeout=10) + _say(f"SIGKILLed pid {self.proc.pid}") + + def sigterm(self) -> int: + self.proc.send_signal(signal.SIGTERM) + code = self.proc.wait(timeout=60) + _say(f"SIGTERM pid {self.proc.pid}, exit {code}") + return code + + def stop(self) -> None: + if self.proc and self.proc.poll() is None: + self.proc.send_signal(signal.SIGTERM) + try: + self.proc.wait(timeout=60) + except subprocess.TimeoutExpired: + self.proc.send_signal(signal.SIGKILL) + self.proc.wait(timeout=10) + + def provision(self) -> None: + env = dict(os.environ, EXTENDDB_TEST_ENDPOINT=self.endpoint, EXTENDDB_ADMIN_USER="admin", + EXTENDDB_ADMIN_PASSWORD=ADMIN_PASSWORD, EXTENDDB_CA_CERT=os.path.expanduser("~/.extenddb/tls/cert.pem")) + r = subprocess.run([sys.executable, os.path.join(_repo(), "devtools", "provision-test-credentials")], + env=env, capture_output=True, text=True) + if r.returncode != 0: + raise SystemExit(f"provisioning failed: {r.stderr}") + for line in r.stdout.splitlines(): + m = re.match(r"export (\w+)=(.*)", line.strip()) + if m: + self.creds[m.group(1)] = m.group(2) + os.environ[m.group(1)] = m.group(2) + os.environ["EXTENDDB_TEST_ENDPOINT"] = self.endpoint + for key, value in [("control_plane_delay_seconds", "0.05"), ("index_propagation_delay_ms", "0")]: + subprocess.run([self.bin, "settings", "--config", self.config, "set", key, value], + cwd=self.work, capture_output=True) + + def teardown(self) -> None: + self.stop() + if self.backend == "postgres": + _psql(f"DROP DATABASE IF EXISTS {self.tag}_data WITH (FORCE)") + _psql(f"DROP DATABASE IF EXISTS {self.tag}_catalog WITH (FORCE)") + _psql(f"DROP ROLE IF EXISTS {self.tag}_app") + elif self.backend == "mongodb": + self._mongo_drop() + elif self.backend == "cassandra": + try: + from cassandra.auth import PlainTextAuthProvider + from cassandra.cluster import Cluster + cluster = Cluster(["127.0.0.1"], port=9042, + auth_provider=PlainTextAuthProvider("cassandra", "cassandra")) + s = cluster.connect() + for ks in ("extenddb_catalog", f"extenddb_account_{ACCOUNT}"): + s.execute(f"DROP KEYSPACE IF EXISTS {ks}") + cluster.shutdown() + except Exception as e: # noqa: BLE001 + print(f"cassandra teardown: {e}") + + def _mongo_drop(self) -> None: + """Drop the fixed databases and wait until each drop has completed: on + a replica set `dropDatabase` is two-phase and a collection created + while the database is drop-pending fails with code 215 or vanishes + when the drop lands (a server started over a pending drop answered + `ResourceNotFoundException` to `BatchWriteItem` into an ACTIVE + table). The probe creates a collection until the server accepts it, + then drops the collection, which leaves no pending database.""" + from pymongo import MongoClient + from pymongo.errors import OperationFailure + c = MongoClient(MONGODB_URI) + try: + for name in ("extenddb_catalog", "extenddb_data"): + c.drop_database(name) + deadline = time.monotonic() + 120 + for name in ("extenddb_catalog", "extenddb_data"): + while True: + try: + c[name].create_collection("drop_probe") + except OperationFailure as e: + if e.code == 215 and time.monotonic() < deadline: + time.sleep(0.25) + continue + raise + c[name].drop_collection("drop_probe") + break + finally: + c.close() + + +def _ddb(dep: Deployment): + import boto3 + from botocore.config import Config + return boto3.client("dynamodb", endpoint_url=dep.endpoint, region_name=REGION, verify=False, + aws_access_key_id=dep.creds["AWS_ACCESS_KEY_ID"], + aws_secret_access_key=dep.creds["AWS_SECRET_ACCESS_KEY"], + config=Config(retries={"max_attempts": 10, "mode": "standard"}, read_timeout=120)) + + +# The points and which backends they apply to. +RESTORE_POINTS = { + "restore_after_download": ["postgres", "sqlite", "mongodb", "cassandra"], + "restore_after_file_commit": ["postgres", "sqlite", "mongodb", "cassandra"], + "restore_sqlite_mid_batch": ["sqlite"], + "restore_cassandra_before_progress": ["cassandra"], + "restore_mongodb_after_insert_many": ["mongodb"], + "restore_postgres_before_progress_delete": ["postgres"], + "restore_mongodb_before_worker_flip": ["mongodb"], + "restore_between_failed_and_removal": ["postgres", "sqlite", "mongodb", "cassandra"], + "restore_fenced_out": ["postgres"], + "pg_terminate_backend": ["postgres"], + "sigterm": ["postgres", "sqlite", "mongodb", "cassandra"], +} + + +def _wait_table_active(ddb, name: str, timeout: float = 120) -> None: + """Poll DescribeTable until the table and every GSI are ACTIVE, the way + the fidelity suite's `wait_for_active` does, before any load.""" + deadline = time.monotonic() + timeout + while True: + t = ddb.describe_table(TableName=name)["Table"] + gsis = t.get("GlobalSecondaryIndexes") or [] + if t["TableStatus"] == "ACTIVE" and all(g.get("IndexStatus", "ACTIVE") == "ACTIVE" for g in gsis): + return + assert time.monotonic() < deadline, f"{name} did not reach ACTIVE" + time.sleep(0.3) + + +def _batch_write_ready(ddb, name: str, reqs) -> None: + """Write one batch, tolerating the load-readiness window on a freshly + created table: a `ResourceNotFoundException` on MongoDB (a collection not + yet visible) or an `InternalServerError` on Cassandra (schema not settled + on the coordinator) right after the table reports ACTIVE is a harness + race, not a restore defect, so re-poll ACTIVE and retry the batch for up + to 60 s rather than aborting the load.""" + from botocore.exceptions import ClientError + + deadline = time.monotonic() + 240 + pending = {name: reqs} + while pending: + try: + pending = ddb.batch_write_item(RequestItems=pending).get("UnprocessedItems") or {} + except ClientError as e: + code = e.response["Error"]["Code"] + if code in ("ResourceNotFoundException", "InternalServerError") and time.monotonic() < deadline: + _wait_table_active(ddb, name) + time.sleep(2) + continue + raise + + +def _signed_post(dep: Deployment, operation: str, body: dict): + """A hand-signed request for an operation botocore's model lacks + (CreateTable with VectorIndexes), as `devtools/backup-restore-matrix` + sends it.""" + import json + import boto3 + from botocore.auth import SigV4Auth + from botocore.awsrequest import AWSRequest + body_bytes = json.dumps(body).encode() + headers = {"X-Amz-Target": f"DynamoDB_20120810.{operation}", "Content-Type": "application/x-amz-json-1.0"} + frozen = boto3.Session(aws_access_key_id=dep.creds["AWS_ACCESS_KEY_ID"], + aws_secret_access_key=dep.creds["AWS_SECRET_ACCESS_KEY"]).get_credentials().get_frozen_credentials() + req = AWSRequest(method="POST", url=dep.endpoint, data=body_bytes, headers=headers) + SigV4Auth(frozen, "dynamodb", REGION).add_auth(req) + return requests.post(dep.endpoint, data=body_bytes, headers=dict(req.headers), verify=False) + + +def _create_table_and_backup(dep: Deployment, items: int, big_file: bool, vector: bool = False): + ddb = _ddb(dep) + name = f"kp{dep.tag}" + attrs = [{"AttributeName": "pk", "AttributeType": "S"}, {"AttributeName": "sk", "AttributeType": "N"}, + {"AttributeName": "g", "AttributeType": "S"}] + body = dict(TableName=name, KeySchema=[{"AttributeName": "pk", "KeyType": "HASH"}, + {"AttributeName": "sk", "KeyType": "RANGE"}], + AttributeDefinitions=attrs, BillingMode="PAY_PER_REQUEST", + GlobalSecondaryIndexes=[{"IndexName": "by-g", + "KeySchema": [{"AttributeName": "g", "KeyType": "HASH"}, + {"AttributeName": "sk", "KeyType": "RANGE"}], + "Projection": {"ProjectionType": "ALL"}}]) + if vector: + # ExtendDB's VectorIndexes member is not in botocore's model, so the + # CreateTable goes through a signed POST, as the matrix script's does. + body["VectorIndexes"] = [{"IndexName": "by-emb", "Dimensions": 3, "DistanceFunction": "COSINE", + "VectorAttribute": {"AttributeName": "emb"}, + "Projection": {"ProjectionType": "ALL"}}] + r = _signed_post(dep, "CreateTable", body) + if r.status_code != 200: + raise SystemExit(f"CreateTable with a vector index: {r.status_code} {r.text}") + else: + ddb.create_table(**body) + _wait_table_active(ddb, name) + if dep.backend == "cassandra": + # Cassandra's DDL settles on the coordinator a moment after the table + # reports ACTIVE; a load that starts in that gap meets "No nodes + # available in query plan" or an InternalServerError from the LWT. + # `_batch_write_ready` retries through it; this settle wait shortens it. + time.sleep(3) + # big_file forces more than 10,000 items into one data file for the + # SQLite and Cassandra mid-batch points (each sits between two batches of + # one file); otherwise 4,000-byte items give two or three data files at + # the 32 MiB file target. (A Cassandra deployment never takes the backup + # here: its backup is the in-catalog format, module comment.) + pad = 20 if big_file else 4000 + reqs = [] + for i in range(items): + item = {"pk": {"S": f"p{i % 50:04d}"}, "sk": {"N": str(i)}, "g": {"S": f"g{i % 7}"}, "pad": {"S": "x" * pad}} + if vector and i % 5 == 0: + item["emb"] = {"L": [{"N": "0.1"}, {"N": str(0.01 * (i % 50))}, {"N": "0.3"}]} + reqs.append({"PutRequest": {"Item": item}}) + if len(reqs) == 25 or i == items - 1: + _batch_write_ready(ddb, name, reqs) + reqs = [] + arn = ddb.create_backup(TableName=name, BackupName="kpbackup")["BackupDetails"]["BackupArn"] + deadline = time.monotonic() + 600 + while True: + d = ddb.describe_backup(BackupArn=arn)["BackupDescription"]["BackupDetails"]["BackupStatus"] + if d == "AVAILABLE": + break + assert d == "CREATING", d + assert time.monotonic() < deadline, "backup did not become AVAILABLE" + time.sleep(0.2) + return ddb, name, arn + + +def _count(ddb, name: str, index: str | None = None) -> int: + """The `Scan` count of the table, or of its GSI `index` (an eventually + consistent read, the only kind a GSI takes).""" + total, start = 0, None + while True: + kw = {"TableName": name, "Select": "COUNT"} + if index: + kw["IndexName"] = index + else: + kw["ConsistentRead"] = True + if start: + kw["ExclusiveStartKey"] = start + r = ddb.scan(**kw) + total += r["Count"] + start = r.get("LastEvaluatedKey") + if not start: + return total + + +def _assert_exact(ddb, name: str, items: int, what: str) -> None: + """The base table and the GSI `by-g` both hold `items` rows. Every item + of `_create_table_and_backup`'s fixture carries `g`, so a GSI short of + the base count is an index row the restore lost (a re-applied batch on + Cassandra deleted its index rows before #742 was corrected).""" + n = _count(ddb, name) + assert n == items, f"{what}: restored {n}, expected {items}" + g = _count(ddb, name, index="by-g") + assert g == items, f"{what}: the GSI by-g holds {g} rows, expected {items}" + + +def _wait_reached(dep: Deployment, point: str, timeout: float = 120, times: int = 1) -> bool: + """True once the log holds `times` lines of the point being reached.""" + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + with open(dep.log_path, errors="replace") as f: + if sum(1 for line in f if f"reached kill point {point}" in line) >= times: + return True + if dep.proc.poll() is not None: + return False + time.sleep(0.2) + return False + + +def _release_first_reach(dep: Deployment, point: str) -> None: + """Release the paused job from the point's first reach and remove the + release file again, so the next reach pauses. The pause polls the file + every 100 ms and the job reaches the point again as soon as the next + file's or batch's writes are done, so the log is polled every 10 ms for + the release line and the file removed at once; a second reach the file + still released (a next write that took less than that) is a setup + failure, not a kill, and the run says so.""" + release = os.path.join(dep.work, "release") + open(release, "w").close() + deadline = time.monotonic() + 30 + while True: + with open(dep.log_path, errors="replace") as f: + released = sum(1 for line in f if f"released from kill point {point}" in line) + if released >= 1: + break + if time.monotonic() > deadline: + raise SystemExit(f"the job was not released from {point}; see {dep.log_path}") + time.sleep(0.01) + os.remove(release) + if released >= 2: + raise SystemExit(f"the second reach of {point} was released before the file was removed; raise --items") + _say(f"{dep.backend}/{point}: first reach released; waiting for the second reach") + + +def _log_has(dep: Deployment, needle: str, timeout: float = 20) -> str | None: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + with open(dep.log_path, errors="replace") as f: + for line in f: + if needle in line: + return line + time.sleep(0.25) + return None + + +def _tamper_a_data_file(dep: Deployment, arn: str) -> str: + """Corrupt one data file's bytes under the store root so its sha256 no + longer matches the manifest, which fails the restore in its verification + pass before any item is written (5.3 step 3). Returns the file path.""" + import glob + + files = sorted(glob.glob(os.path.join(dep.root, "**", "data", "*.json.gz"), recursive=True)) + if not files: + raise SystemExit(f"no data files under {dep.root} to tamper") + victim = files[-1] + with open(victim, "r+b") as f: + f.seek(0, os.SEEK_END) + size = f.tell() + f.seek(size // 2) + b = f.read(16) + f.seek(size // 2) + f.write(bytes((x ^ 0xFF) for x in b)) + _say(f"tampered {os.path.relpath(victim, dep.root)}") + return victim + + +def _matrix_module(): + """`devtools/backup-restore-matrix` imported as a module, for the prefix + copy and the `backup_index` seed a Cassandra target needs.""" + import importlib.machinery + import importlib.util + + path = os.path.join(_repo(), "devtools", "backup-restore-matrix") + # The script has no `.py` suffix, so the loader is named explicitly. + loader = importlib.machinery.SourceFileLoader("backup_restore_matrix", path) + spec = importlib.util.spec_from_loader("backup_restore_matrix", loader) + module = importlib.util.module_from_spec(spec) + saved = sys.argv + sys.argv = ["backup-restore-matrix"] + try: + spec.loader.exec_module(module) + finally: + sys.argv = saved + return module + + +def _adopt_backup(source: Deployment, target: Deployment, arn: str) -> None: + """Copy the source's backup prefix under the target's root and seed the + target's `backup_index` row, the matrix's stand-in for `backup sync + --adopt-store`.""" + import json + import types + + mx = _matrix_module() + src_prefix = mx.prefix_of(source.root, arn) + dst_prefix = mx.prefix_of(target.root, arn) + mx.copy_prefix(src_prefix, dst_prefix, target.root) + with open(os.path.join(dst_prefix, "extenddb-manifest.json")) as f: + manifest = json.load(f) + + def store_id() -> str: + with open(os.path.join(target.root, "extenddb-store.json")) as f: + return json.load(f)["store_id"] + + adapter = types.SimpleNamespace(backend=target.backend, root=target.root, work=target.work, + tag=target.tag, store_id=store_id) + mx.seed_row(adapter, arn, manifest, manifest["backup_size_bytes"], manifest["source_table"]["item_count"]) + _say(f"backup {arn} adopted by the {target.backend} target (prefix copied, backup_index row seeded)") + + +def _run_restore_point(backend: str, point: str, binary: str, items: int, work: str, + second_reach: bool = False, source_bin: str | None = None, + vector: bool = False) -> int: + """One restore kill point on one backend. Returns 0 on pass, 1 on a + failed assertion, 2 on setup error. With `second_reach` the SIGKILL is + taken at the point's second reach (module comment). On Cassandra the + backup is taken on a SQLite source (`source_bin`) and adopted by the + target (module comment). With `vector` the + source table carries a vector index and the tamper cell on PostgreSQL + asserts the removed target's data table and vector relation gone.""" + import boto3 # noqa: F401 + from botocore.exceptions import ClientError + + shutil.rmtree(work, ignore_errors=True) + os.makedirs(work, exist_ok=True) + dep = Deployment(backend, binary, work) + big_file = point in ("restore_sqlite_mid_batch", "restore_cassandra_before_progress") + if second_reach and point in ("sigterm", "pg_terminate_backend", "restore_fenced_out"): + raise SystemExit(f"--second-reach does not apply to {point}") + if vector and (backend != "postgres" or point != "restore_between_failed_and_removal"): + raise SystemExit("--vector applies to restore_between_failed_and_removal on postgres") + source = None + if backend == "cassandra": + if not source_bin: + raise SystemExit("a Cassandra kill point needs --source-bin (a SQLite build) for its backup") + source = Deployment("sqlite", os.path.abspath(source_bin), os.path.join(work, "source")) + rc = 0 + try: + if source is not None: + source.init() + source.start("source.log") + source.provision() + _, name, arn = _create_table_and_backup(source, items, big_file) + source.stop() + _say(f"source (sqlite) backup {arn} of {items} items taken and the source stopped") + dep.init() + if point == "sigterm": + dep.start("serve.log") + elif point == "pg_terminate_backend": + # Not a pause point of its own: the claim session is terminated + # while the job is paused after its first file's commit. + dep.start("serve-paused.log", pause="restore_after_file_commit") + else: + dep.start("serve-paused.log", pause=point) + dep.provision() + if source is not None: + _adopt_backup(source, dep, arn) + ddb = _ddb(dep) + else: + ddb, name, arn = _create_table_and_backup(dep, items, big_file, vector=vector) + target = f"{name}r" + if point == "restore_between_failed_and_removal": + _tamper_a_data_file(dep, arn) + resp = ddb.restore_table_from_backup(TargetTableName=target, BackupArn=arn)["TableDescription"] + assert resp["TableStatus"] == "CREATING", resp + vector_relation = None + if vector: + # The registration created the target's catalog rows; read the + # table id and the vector index id so the data relations can be + # probed once the failing restore's target is gone. + vids = _psql("SELECT t.table_id, v.index_id FROM tables t JOIN vector_indexes v ON v.table_id = t.table_id " + f"WHERE t.table_name = '{target}' AND t.account_id = '{ACCOUNT}'", database=f"{dep.tag}_catalog") + assert vids and "|" in vids, f"the target's vector index row was not found: {vids!r}" + vector_relation = tuple(vids.splitlines()[0].split("|")) + _say(f"{backend}/{point}: the target's table id {vector_relation[0]}, vector index id {vector_relation[1]}") + + if point == "sigterm": + time.sleep(0.5) + status = ddb.describe_table(TableName=target)["Table"]["TableStatus"] + if status != "CREATING": + raise SystemExit(f"the restore finished ({status}) before SIGTERM; raise --items") + dep.sigterm() + log = open(dep.log_path, errors="replace").read() + assert "stopped by the server's shutdown" in log, log[-2000:] + assert ": FAILED (" not in log, log[-2000:] + dep.start("serve-restart.log") + dep.provision() + ddb = _ddb(dep) + deadline = time.monotonic() + 240 + while ddb.describe_table(TableName=target)["Table"]["TableStatus"] != "ACTIVE": + assert time.monotonic() < deadline, "the restore did not resume" + time.sleep(0.5) + _assert_exact(ddb, target, items, point) + _say(f"{backend}/{point}: RUNNING across restart, resumed ACTIVE with {items} items (base and GSI exact)") + return 0 + + if point == "pg_terminate_backend": + # The job is paused at restore_after_file_commit (file 0 committed, + # one restore_progress row). Terminate the session that holds the + # claim's advisory lock while the job is paused, wait past one + # heartbeat period so the timer's next beat meets the terminated + # session, then release: the job stops at its next check as a + # lost claim with no further catalog or table access, the record + # stays RUNNING with its progress row, and the same server's + # reconciler takes it over once heartbeat_at is stale (45 s after + # the last beat, so at least stale_job_after_secs after the + # termination) and finishes it. + paused_at = "restore_after_file_commit" + if not _wait_reached(dep, paused_at, timeout=300): + raise SystemExit(f"the kill point {paused_at} was not reached; see {dep.log_path}") + out = _psql("SELECT pg_terminate_backend(l.pid) FROM pg_locks l " + "JOIN pg_stat_activity a ON a.pid = l.pid " + "WHERE l.locktype = 'advisory' AND l.objsubid = 1 " + f"AND a.datname = '{dep.tag}_catalog' AND a.pid <> pg_backend_pid()", database="postgres") + terminated_at = time.monotonic() + assert out.strip() == "t", f"no claim session was terminated (psql answered {out!r})" + # The stale window counts from the owner's last confirmed beat on + # the database's clock, which precedes the termination by up to + # one heartbeat period; read that age now (no beat lands on the + # terminated session, so it stays until the takeover). + beat_age = float(_psql("SELECT extract(epoch FROM now() - heartbeat_at) FROM table_restores", + database=f"{dep.tag}_catalog")) + _say(f"{backend}/{point}: terminated the claim session (advisory objsubid=1) while paused at {paused_at}; " + f"the last beat was {beat_age:.1f} s old") + time.sleep(20) + open(os.path.join(dep.work, "release"), "w").close() + stopped = _log_has(dep, "stopped with no further catalog or table access", timeout=120) + assert stopped, "the job did not stop as a lost claim after its claim session was terminated" + _say(f"{backend}/{point}: the job stopped {time.monotonic() - terminated_at:.1f} s after the termination: " + f"{stopped.strip()[-240:]}") + rec = _psql("SELECT status, file_index, items_applied FROM table_restores", database=f"{dep.tag}_catalog") + prog = _psql("SELECT file_index, items_applied FROM restore_progress ORDER BY 1", database=f"{dep.tag}_data") + assert rec.startswith("RUNNING|"), f"the record is not RUNNING after the lost claim: {rec!r}" + assert prog, "no restore_progress row survived the lost claim" + _say(f"{backend}/{point}: record {rec!r}; progress rows {prog.replace(chr(10), ' ')!r}") + took = _log_has(dep, "taken over by this instance and resumed", timeout=180) + assert took, "the reconciler did not take the record over" + took_after = time.monotonic() - terminated_at + assert took_after + beat_age >= 45, (f"the takeover came {took_after:.1f} s after the termination and " + f"{took_after + beat_age:.1f} s after the last beat, inside stale_job_after_secs (45)") + _say(f"{backend}/{point}: takeover {took_after:.1f} s after the termination, {took_after + beat_age:.1f} s " + f"after the last beat: {took.strip()[-200:]}") + # The pause is the server's, so the successor's resume pauses at + # the same point; release it. + time.sleep(0.5) + open(os.path.join(dep.work, "release"), "w").close() + deadline = time.monotonic() + 240 + while True: + st = ddb.describe_table(TableName=target)["Table"]["TableStatus"] + if st == "ACTIVE": + break + assert time.monotonic() < deadline, f"stuck at {st}" + time.sleep(0.5) + _assert_exact(ddb, target, items, point) + _say(f"{backend}/{point}: lost claim, record RUNNING with its progress row, takeover {took_after:.1f} s " + f"after the termination, resumed ACTIVE with {items} items (base and GSI exact)") + return 0 + + if point == "restore_fenced_out": + # The job pauses after loading every file, with its heartbeat + # timer stopped (restore.rs). The claim session is terminated so + # the advisory lock frees (on PostgreSQL the lock, not the stale + # heartbeat alone, is what the reconciler waits on, 4(e)); the + # same server's reconciler then takes the record over on a free + # permit (max_concurrent_jobs = 2) once the heartbeat is stale and + # finishes the restore. Releasing the paused job last, its own + # DONE write matches no record (the successor's generation fenced + # it out) and it makes no table access. + if not _wait_reached(dep, point): + raise SystemExit(f"the kill point {point} was not reached; see {dep.log_path}") + _say(f"{backend}/{point}: reached; terminating the claim session so its lock frees") + _psql("SELECT pg_terminate_backend(l.pid) FROM pg_locks l " + "JOIN pg_stat_activity a ON a.pid = l.pid " + "WHERE l.locktype = 'advisory' AND l.objsubid = 1 " + f"AND a.datname = '{dep.tag}_catalog' AND a.pid <> pg_backend_pid()", database="postgres") + took = _log_has(dep, "taken over by this instance and resumed", timeout=120) + assert took, "the reconciler did not take the fenced-out record over" + # The server's pause is global, so the successor's resume job also + # pauses at this point with its own heartbeat stopped; release now + # so it finishes. The fenced original then proceeds to its own DONE + # write, which matches no record (the successor bumped the + # generation on takeover) and makes no table access. + open(os.path.join(dep.work, "release"), "w").close() + deadline = time.monotonic() + 240 + while ddb.describe_table(TableName=target)["Table"]["TableStatus"] != "ACTIVE": + assert time.monotonic() < deadline, "the successor did not finish the restore" + time.sleep(0.5) + _assert_exact(ddb, target, items, f"{point} (the successor's table)") + line = _log_has(dep, "no table access was made", timeout=30) + assert line, "the released fenced-out job did not log its NotMatched DONE write" + _say(f"{backend}/{point}: successor finished ACTIVE with {items} items; " + f"the released job's DONE write matched no record ({line.strip()[-80:]})") + return 0 + + # The SIGKILL points: wait for the line, SIGKILL, restart, assert. + if not _wait_reached(dep, point, timeout=300): + raise SystemExit(f"the kill point {point} was not reached; see {dep.log_path}") + if second_reach: + _release_first_reach(dep, point) + if not _wait_reached(dep, point, timeout=300, times=2): + raise SystemExit(f"the kill point {point} was not reached a second time; see {dep.log_path}") + with open(dep.log_path, errors="replace") as f: + released = sum(1 for line in f if f"released from kill point {point}" in line) + if released >= 2: + raise SystemExit(f"the second reach of {point} was released before the file was removed; raise --items") + _say(f"{backend}/{point}: reached a second time; SIGKILL") + else: + _say(f"{backend}/{point}: reached; SIGKILL") + dep.sigkill() + dep.start("serve-restart.log") + dep.provision() + ddb = _ddb(dep) + # The FAILED-target duty waits out the stale window (45 s) before it + # removes the target, so give it more than one window. + deadline = time.monotonic() + 240 + + # `restore_cassandra_before_progress` at its first reach pauses + # before the first batch's progress is recorded, so the kill leaves + # the batch's items with no recorded progress. The resume counts the + # target, finds no more items than the backup's total, and re-applies + # the first file with upserts (the engine fails `target_not_empty` + # only above the total), so the outcome below is ACTIVE with the exact + # count, as for every other point. (MongoDB commits each batch with + # its progress, so its first reach leaves nothing unrecorded.) + + want_gone = point == "restore_between_failed_and_removal" + while True: + try: + st = ddb.describe_table(TableName=target)["Table"]["TableStatus"] + except ClientError as e: + assert e.response["Error"]["Code"] == "ResourceNotFoundException", e.response["Error"] + st = "GONE" + if want_gone and st == "GONE": + break + if not want_gone and st == "ACTIVE": + break + assert time.monotonic() < deadline, f"stuck at {st} for {point}" + time.sleep(0.5) + if not want_gone: + _assert_exact(ddb, target, items, point) + _say(f"{backend}/{point}{' (second reach)' if second_reach else ''}: resumed and ACTIVE with {items} items (base and GSI exact)") + if second_reach: + for needle in ("taken over by this instance and resumed", "resumed from data file", ": ACTIVE, "): + line = _log_has(dep, needle, timeout=5) + if line: + _say(f" {line.strip()[-200:]}") + if point == "restore_postgres_before_progress_delete": + line = _log_has(dep, "restore progress row", timeout=20) + _say(f"{backend}/{point}: {line.strip() if line else 'progress rows swept (no line seen)'}") + else: + line = _log_has(dep, "target", timeout=5) + _say(f"{backend}/{point}: FAILED target removed after the stale window (target gone)") + if vector_relation is not None: + table_id, index_id = vector_relation + left = _psql(f"SELECT to_regclass('\"_ddb_{table_id}\"') IS NOT NULL, " + f"to_regclass('\"_ddb_vec_{index_id}\"') IS NOT NULL", database=f"{dep.tag}_data") + assert left == "f|f", f"the removed target left a data relation behind (data table, vector relation exist: {left!r})" + _say(f"{backend}/{point} (vector): the data table _ddb_{table_id} and the vector relation " + f"_ddb_vec_{index_id} are gone with the target") + return 0 + except AssertionError as e: + print(f"KILL-POINT FAILED {backend}/{point}: {e}") + return 1 + except SystemExit as e: + print(f"KILL-POINT SETUP {backend}/{point}: {e}") + return 2 + finally: + try: + dep.teardown() + except Exception as e: # noqa: BLE001 + print(f"teardown {backend}/{point}: {e}") + if source is not None: + source.stop() + _say(f"logs under {work}") + + +def _restore_main(argv) -> None: + p = argparse.ArgumentParser(prog="kill-points.py restore") + p.add_argument("backend", choices=["sqlite", "postgres", "mongodb", "cassandra"]) + p.add_argument("point") + p.add_argument("--bin", required=True) + p.add_argument("--items", type=int, default=12000) + p.add_argument("--work", required=True) + p.add_argument("--second-reach", action="store_true", + help="SIGKILL at the point's second reach (after an earlier progress record)") + p.add_argument("--source-bin", help="a SQLite build that takes the backup a Cassandra target restores") + p.add_argument("--vector", action="store_true", + help="give the source table a vector index; restore_between_failed_and_removal on postgres then asserts the removed target's vector relation gone") + a = p.parse_args(argv) + if a.point not in RESTORE_POINTS: + raise SystemExit(f"unknown point {a.point}; known: {', '.join(RESTORE_POINTS)}") + if a.backend not in RESTORE_POINTS[a.point]: + raise SystemExit(f"{a.point} does not apply to {a.backend}") + sys.exit(_run_restore_point(a.backend, a.point, os.path.abspath(a.bin), a.items, os.path.abspath(a.work), + second_reach=a.second_reach, source_bin=a.source_bin, + vector=a.vector)) + + +# Dispatch `kill-points.py restore ...` to the restore orchestrator, every +# other subcommand to PR 8's API helper. +if __name__ == "__main__": + if len(sys.argv) > 1 and sys.argv[1] == "restore": + _restore_main(sys.argv[2:]) + else: + main() diff --git a/devtools/backup-kill-points.sh b/devtools/backup-kill-points.sh new file mode 100755 index 000000000..1c1f772fc --- /dev/null +++ b/devtools/backup-kill-points.sh @@ -0,0 +1,284 @@ +#!/usr/bin/env bash +# The backup job's kill points, at the binary level (design +# `docs/design/15-backup-restore.md`: the PR 8 acceptance and the testing +# summary). One deployment per backend, a server built with +# `cargo build --release --features extenddb-app/backup-test-hooks` (SQLite +# and S3 points: `--no-default-features --features sqlite,extenddb-app/backup-test-hooks`), +# `reconcile_interval_secs = 2`, `stale_job_after_secs = 45`, a 100k-item +# composite-key table with two GSIs. Run from a checkout; the API half is +# `devtools/backup-kill-points.py` beside this script. +# +# devtools/backup-kill-points.sh [after_data_file|before_manifest|after_manifest|sigterm ...] +# +# Per point: `after_data_file`, `before_manifest` and `after_manifest` set +# EXTENDDB_BACKUP_PAUSE on the server, run CreateBackup, wait for the "reached +# kill point" line, `kill -9` the server by its PID, and restart it without +# the pause; then the row and the prefix are watched through `extenddb backup +# list` and the store. +# after_data_file: no manifest under the prefix after the kill; the row +# CREATING, then FAILED (SQLite: the first cycle, the slot rule; PostgreSQL: +# after the 45 s window), then the prefix deleted (once the row's heartbeat +# is stale, within the window plus a cycle); DescribeBackup BackupNotFound. +# before_manifest: the three documents (manifest-files, manifest-summary, +# the sha256 sidecar) and the data file under the prefix after the kill +# and no manifest; then as after_data_file (FAILED `owner dead`, the +# prefix deleted once the heartbeat is stale): a prefix without +# `extenddb-manifest.json` is never resolved AVAILABLE, whatever else is +# under it. +# after_manifest: the row AVAILABLE after the cycle with the manifest's +# size and count, the prefix intact, DescribeBackup AVAILABLE. +# sigterm: no pause; a larger table (SIGTERM_ITEMS, default 1,000,000); +# SIGTERM while the row is CREATING; the row FAILED `shutdown`, the prefix +# empty, `drained` in the log; on the S3 store no multipart upload left; +# the exit is timed by polling /proc/ at 50 ms. +# Every run is recorded under $LOGS_DIR/// (LOGS_DIR defaults +# to discussions/kill-points under the checkout, which git ignores). +# +# Environment: EXTENDDB_BIN (the server binary built with the hooks; default +# target/release/extenddb under the checkout), ITEMS (default 100000), +# SIGTERM_ITEMS (default 1000000), LOGS_DIR, PG_PORT (default 55432) and +# S3_ENDPOINT (default http://127.0.0.1:9200, a moto server), the services +# the points use. +# `A && say ... || fail ...`: `say` never fails, so the `fail` branch runs only +# when the check before it fails. +# shellcheck disable=SC2015 +set -u +BACKEND=${1:?backend}; shift +POINTS=("$@"); [ ${#POINTS[@]} -eq 0 ] && POINTS=(after_data_file before_manifest after_manifest sigterm) +HERE=$(cd "$(dirname "$0")" && pwd) +REPO=$(cd "$HERE/.." && pwd) +LOGS=${LOGS_DIR:-$REPO/discussions/kill-points}/$BACKEND +ITEMS=${ITEMS:-100000} +SIGTERM_ITEMS=${SIGTERM_ITEMS:-1000000} +PG_PORT=${PG_PORT:-55432} +S3_ENDPOINT=${S3_ENDPOINT:-http://127.0.0.1:9200} +case $BACKEND in + sqlite|s3|postgres) ;; + *) echo "backend must be sqlite, postgres or s3"; exit 2 ;; +esac +BIN=${EXTENDDB_BIN:-$REPO/target/release/extenddb} +[ -x "$BIN" ] || { echo "build $BIN with --features extenddb-app/backup-test-hooks first (or set EXTENDDB_BIN)"; exit 2; } +"$BIN" version 2>/dev/null | grep -q "backup-test-hooks enabled" || { echo "$BIN was not built with --features extenddb-app/backup-test-hooks"; exit 2; } +KP="python3 $HERE/backup-kill-points.py" +export EXTENDDB_REPO=$REPO +export S3_ENDPOINT +export AWS_ACCESS_KEY_ID=testing AWS_SECRET_ACCESS_KEY=testing AWS_REGION=us-east-1 AWS_EC2_METADATA_DISABLED=true +export EXTENDDB_ADMIN_PASSWORD='TestPass1!' + +say() { echo "[$(date +%H:%M:%S)] $*"; } +# The points run in pipelines (subshells), so a failure is a FAIL line in the +# point's log, which the end of the script greps for. +fail() { say "FAIL: $*"; } + +# --- one deployment per point -------------------------------------------------- +# $1 = point; sets WORK, PORT, ROOT (filesystem) or BUCKET (s3), TAG +deploy() { + local point=$1 + WORK=$LOGS/$point/work; rm -rf "$WORK"; mkdir -p "$WORK"; cd "$WORK" || exit 1 + PORT=$((19000 + RANDOM % 900)) + TAG="kp$(date +%s | tail -c 6)" + case $BACKEND in + sqlite|s3) + "$BIN" init --backend sqlite --sqlite-path "$WORK/db.sqlite" --config extenddb.toml > init.log 2>&1 || { cat init.log; exit 1; } ;; + postgres) + "$BIN" init --pg-host 127.0.0.1 --pg-port "$PG_PORT" --pg-user postgres --pg-pass devpass \ + --data-db "${TAG}_data" --catalog-db "${TAG}_catalog" --extenddb-user "${TAG}_app" --config extenddb.toml > init.log 2>&1 || { cat init.log; exit 1; } ;; + esac + python3 -c "import sys; sys.path.insert(0, '$REPO/tests'); import lifecycle_helpers as lh; lh._patch_config_port('extenddb.toml', $PORT)" + printf '\n[backup]\nreconcile_interval_secs = 2\nstale_job_after_secs = 45\n' >> extenddb.toml + case $BACKEND in + sqlite) ROOT=$WORK/backups ;; + postgres) ROOT=$WORK/backups; mkdir -m 0700 "$ROOT" + printf '[backup.store]\nkind = "filesystem"\n[backup.store.filesystem]\npath = "%s"\n' "$ROOT" >> extenddb.toml ;; + s3) BUCKET="kp-$point-$(date +%s | tail -c 6)" + python3 -c "import boto3; boto3.client('s3', endpoint_url='$S3_ENDPOINT', region_name='us-east-1').create_bucket(Bucket='$BUCKET')" + printf '[backup.store]\nkind = "s3"\n[backup.store.s3]\nbucket = "%s"\nendpoint = "%s"\nforce_path_style = true\nregion = "us-east-1"\n' "$BUCKET" "$S3_ENDPOINT" >> extenddb.toml ;; + esac + export EXTENDDB_TEST_ENDPOINT="https://127.0.0.1:$PORT" +} + +# start [env assignments...] +start() { + local log=$1; shift + env "$@" "$BIN" serve --foreground --config extenddb.toml > "$WORK/$log" 2>&1 & + SPID=$! + for _ in $(seq 1 100); do curl -sk "$EXTENDDB_TEST_ENDPOINT/health" > /dev/null 2>&1 && break; sleep 0.2; done + curl -sk "$EXTENDDB_TEST_ENDPOINT/health" > /dev/null || { fail "server did not start"; tail -5 "$WORK/$log"; kill -9 "$SPID" 2>/dev/null; exit 1; } + say "server pid $SPID on $PORT ($log)" +} + +provision() { + eval "$(cd "$REPO" && EXTENDDB_ADMIN_USER=admin EXTENDDB_CA_CERT=$HOME/.extenddb/tls/cert.pem python3 devtools/provision-test-credentials 2> "$WORK/creds.err")" || { cat "$WORK/creds.err"; exit 1; } + "$BIN" settings --config extenddb.toml set control_plane_delay_seconds 0.05 > /dev/null 2>&1 +} + +# The row as `backup list` prints it (status, size, count, reason). +row() { "$BIN" backup list --config extenddb.toml 2>/dev/null | grep -F "$1" | head -1; } +row_failed() { "$BIN" backup list --failed --config extenddb.toml 2>/dev/null | grep -F "$1" | head -1; } +row_status() { local r; r=$(row "$1"); [ -n "$r" ] && echo "${r%% *}" || { r=$(row_failed "$1"); [ -n "$r" ] && echo "${r%% *}" || echo GONE; }; } + +# The objects under the backup's prefix, one key per line. +prefix_keys() { + local arn=$1 table=$2 id=${1##*/} + case $BACKEND in + sqlite|postgres) [ -d "$ROOT/123456789012/$table/$id" ] && (cd "$ROOT/123456789012/$table/$id" && find . -type f | sed 's|^\./||' | sort) ;; + s3) python3 - "$BUCKET" "123456789012/$table/$id/" <<'EOF' +import boto3, os, sys +s3 = boto3.client("s3", endpoint_url=os.environ["S3_ENDPOINT"], region_name="us-east-1") +for o in s3.list_objects_v2(Bucket=sys.argv[1], Prefix=sys.argv[2]).get("Contents", []): + print(o["Key"][len(sys.argv[2]):]) +EOF + ;; + esac +} +manifest_json() { + local arn=$1 table=$2 id=${1##*/} + case $BACKEND in + sqlite|postgres) cat "$ROOT/123456789012/$table/$id/extenddb-manifest.json" ;; + s3) python3 -c "import boto3,sys; print(boto3.client('s3', endpoint_url='$S3_ENDPOINT', region_name='us-east-1').get_object(Bucket='$BUCKET', Key='123456789012/$table/$id/extenddb-manifest.json')['Body'].read().decode())" ;; + esac +} +multipart_uploads() { + python3 -c "import boto3; print(len(boto3.client('s3', endpoint_url='$S3_ENDPOINT', region_name='us-east-1').list_multipart_uploads(Bucket='$BUCKET').get('Uploads', [])))" +} + +wait_status() { # arn expected timeout + local arn=$1 want=$2 t=$3 s + for _ in $(seq 1 "$t"); do s=$(row_status "$arn"); [ "$s" = "$want" ] && { echo "$s"; return 0; }; sleep 1; done + echo "$s"; return 1 +} + +stop_server() { kill -TERM "$SPID" 2>/dev/null; wait "$SPID" 2>/dev/null; } +teardown() { + case $BACKEND in + postgres) PGPASSWORD=devpass psql -h 127.0.0.1 -p "$PG_PORT" -U postgres -q -c "DROP DATABASE IF EXISTS ${TAG}_data" -c "DROP DATABASE IF EXISTS ${TAG}_catalog" -c "DROP ROLE IF EXISTS ${TAG}_app" > /dev/null 2>&1 ;; + s3) python3 - "$BUCKET" <<'EOF' +import boto3, os, sys +s3 = boto3.client("s3", endpoint_url=os.environ["S3_ENDPOINT"], region_name="us-east-1") +b = sys.argv[1] +for o in s3.list_objects_v2(Bucket=b).get("Contents", []): s3.delete_object(Bucket=b, Key=o["Key"]) +for u in s3.list_multipart_uploads(Bucket=b).get("Uploads", []): s3.abort_multipart_upload(Bucket=b, Key=u["Key"], UploadId=u["UploadId"]) +s3.delete_bucket(Bucket=b) +EOF + ;; + esac + rm -rf "$WORK/db.sqlite" "$WORK/db.sqlite-wal" "$WORK/db.sqlite-shm" "$WORK/backups" "$WORK/backup-spool" "$WORK/run" +} + +# --- the points --------------------------------------------------------------- +run_pause_point() { # after_data_file | before_manifest | after_manifest + local point=$1 table=kp_$1 + deploy "$point" + start serve-paused.log EXTENDDB_BACKUP_PAUSE="$point" + provision + $KP create-table "$table" --items "$ITEMS" 2>> "$WORK/load.log" || { fail "load"; stop_server; teardown; return; } + local arn; arn=$($KP create-backup "$table" "$point") || { fail "CreateBackup"; stop_server; teardown; return; } + say "backup $arn" + for _ in $(seq 1 600); do grep -q "reached kill point $point" "$WORK/serve-paused.log" && break; sleep 0.1; done + grep -q "reached kill point $point" "$WORK/serve-paused.log" || { fail "the kill point was not reached"; stop_server; teardown; return; } + say "reached kill point $point; row before the kill: $(row_status "$arn")" + kill -9 "$SPID"; wait "$SPID" 2>/dev/null + say "SIGKILLed $SPID" + prefix_keys "$arn" "$table" > "$WORK/prefix-after-kill.txt" + say "prefix after the kill: $(tr '\n' ' ' < "$WORK/prefix-after-kill.txt")" + case $point in + after_data_file) + grep -q "extenddb-manifest.json" "$WORK/prefix-after-kill.txt" && fail "a manifest exists after the kill at after_data_file" + grep -q "^data/000001.json.gz$" "$WORK/prefix-after-kill.txt" || fail "the data file is not under the prefix" ;; + before_manifest) + grep -q "^extenddb-manifest.json$" "$WORK/prefix-after-kill.txt" && fail "a manifest exists after the kill at before_manifest" + for k in data/000001.json.gz manifest-files.json manifest-summary.json extenddb-manifest.sha256; do + grep -q "^$k$" "$WORK/prefix-after-kill.txt" || fail "$k is not under the prefix after the kill" + done + [ "$(wc -l < "$WORK/prefix-after-kill.txt")" -eq 4 ] || fail "expected the four objects before the manifest put" ;; + after_manifest) + [ "$(wc -l < "$WORK/prefix-after-kill.txt")" -eq 5 ] || fail "expected the five objects after the manifest put" ;; + esac + [ "$(row_status "$arn")" = CREATING ] || fail "the row is $(row_status "$arn") after the kill, not CREATING" + local killed_at; killed_at=$(date +%s) + start serve-restarted.log + case $point in + after_data_file|before_manifest) + local bound=8; [ "$BACKEND" = postgres ] && bound=70 + local s; s=$(wait_status "$arn" FAILED $bound) && say "FAILED after $(( $(date +%s) - killed_at )) s (bound $bound)" || fail "the row is $s after $bound s, not FAILED" + row_failed "$arn" | grep -o 'failure_reason=.*' | tee "$WORK/reason.txt" + grep -q "owner dead" "$WORK/reason.txt" || fail "the reason does not say the owner was dead" + [ "$($KP status "$arn")" = NOTFOUND ] || fail "DescribeBackup is not BackupNotFound" + # The prefix goes once the dead owner's heartbeat is stale (45 s from + # the last beat before the kill) at the following cycle. + for _ in $(seq 1 70); do [ -z "$(prefix_keys "$arn" "$table")" ] && break; sleep 1; done + [ -z "$(prefix_keys "$arn" "$table")" ] && say "prefix deleted after $(( $(date +%s) - killed_at )) s" || fail "the prefix still holds: $(prefix_keys "$arn" "$table" | tr '\n' ' ')" + grep -E "owner (was|is) dead|deleted .* object\(s\) left under the prefix" "$WORK/serve-restarted.log" | sed 's/\x1b\[[0-9;]*m//g' | cut -c1-260 ;; + after_manifest) + local bound=8; [ "$BACKEND" = postgres ] && bound=70 + local s; s=$(wait_status "$arn" AVAILABLE $bound) && say "AVAILABLE after $(( $(date +%s) - killed_at )) s (bound $bound)" || fail "the row is $s after $bound s, not AVAILABLE" + manifest_json "$arn" "$table" > "$WORK/manifest.json" + local msize mcount rsize rcount + msize=$(python3 -c "import json; m=json.load(open('$WORK/manifest.json')); print(m['backup_size_bytes'])") + mcount=$(python3 -c "import json; m=json.load(open('$WORK/manifest.json')); print(sum(f['item_count'] for f in m['data_files']))") + read -r _ _ _ rsize rcount _ <<< "$(row "$arn")" + say "manifest size=$msize count=$mcount; row size=$rsize count=$rcount" + [ "$msize" = "$rsize" ] && [ "$mcount" = "$rcount" ] && [ "$mcount" = "$ITEMS" ] || fail "the row's size and count are not the manifest's" + [ "$(prefix_keys "$arn" "$table" | wc -l)" -eq 5 ] || fail "the prefix did not survive whole" + [ "$($KP status "$arn")" = AVAILABLE ] || fail "DescribeBackup is not AVAILABLE" + $KP describe "$arn" > "$WORK/describe.json" + grep -E "owner was dead" "$WORK/serve-restarted.log" | sed 's/\x1b\[[0-9;]*m//g' | cut -c1-200 ;; + esac + stop_server + teardown +} + +run_sigterm() { + local table=kp_sigterm + deploy sigterm + start serve.log + provision + # No GSIs on this table: the point is the upload's interruption, and a + # SQLite load with two GSIs runs at a third of the speed. + $KP create-table "$table" --items "$SIGTERM_ITEMS" --gsis 0 2>> "$WORK/load.log" || { fail "load"; stop_server; teardown; return; } + local arn; arn=$($KP create-backup "$table" sigterm) || { fail "CreateBackup"; stop_server; teardown; return; } + say "backup $arn" + for _ in $(seq 1 300); do grep -q "backup $arn: started" "$WORK/serve.log" && break; sleep 0.1; done + sleep 1 + local before; before=$($KP status "$arn") + say "row before SIGTERM: $before" + kill -TERM "$SPID"; local sent; sent=$(date +%s%3N) + # The exit is timed by polling /proc/ at 50 ms (a fixed sleep before + # `wait` would report its own length as the exit time). A job that has not + # ended 5 s after SIGTERM is dumped (every thread's stack) for the record + # before the 30 s drain bound ends it. + local waited=0 + while [ -d "/proc/$SPID" ] && [ -f "/proc/$SPID/status" ] && ! grep -q "^State:.*Z" "/proc/$SPID/status" 2>/dev/null; do + sleep 0.05; waited=$(( waited + 50 )) + if [ "$waited" -eq 5000 ]; then + say "the server is still running 5 s after SIGTERM; dumping its threads to $WORK/stacks.txt" + eu-stack -p "$SPID" > "$WORK/stacks.txt" 2>&1 || true + fi + [ "$waited" -ge 40000 ] && break + done + local gone; gone=$(date +%s%3N) + wait "$SPID" 2>/dev/null + say "server exited $(( gone - sent )) ms after SIGTERM (polled at 50 ms)" + [ "$before" = CREATING ] || fail "the row was $before before SIGTERM: the backup had finished; raise SIGTERM_ITEMS" + sed 's/\x1b\[[0-9;]*m//g' "$WORK/serve.log" | grep -E "backup job\(s\)|drained|FAILED \(shutdown\)" | cut -c1-220 + grep -q "drained" "$WORK/serve.log" || fail "no 'drained' line" + grep -q "FAILED (shutdown); its objects were deleted" "$WORK/serve.log" || fail "no FAILED (shutdown) line with the objects deleted" + local s; s=$(row_status "$arn"); [ "$s" = FAILED ] && say "row FAILED" || fail "the row is $s, not FAILED" + row_failed "$arn" | grep -o 'failure_reason=.*' | tee "$WORK/reason.txt" + grep -q 'failure_reason="shutdown"' "$WORK/reason.txt" || fail "the reason is not shutdown" + local left; left=$(prefix_keys "$arn" "$table"); [ -z "$left" ] && say "prefix empty" || fail "the prefix still holds: $left" + [ "$BACKEND" = s3 ] && { local n; n=$(multipart_uploads); [ "$n" = 0 ] && say "no multipart upload left" || fail "$n multipart upload(s) left"; } + teardown +} + +mkdir -p "$LOGS" +for point in "${POINTS[@]}"; do + say "=== $BACKEND $point ===" + case $point in + after_data_file|before_manifest|after_manifest) run_pause_point "$point" 2>&1 | tee "$LOGS/$point.log" ;; + sigterm) run_sigterm 2>&1 | tee "$LOGS/sigterm.log" ;; + *) echo "unknown point $point"; exit 2 ;; + esac +done +grep -l "FAIL:" "$LOGS"/*.log 2>/dev/null && exit 1 +say "all points passed for $BACKEND" diff --git a/devtools/backup-restore-matrix b/devtools/backup-restore-matrix new file mode 100755 index 000000000..c4ff4ef2a --- /dev/null +++ b/devtools/backup-restore-matrix @@ -0,0 +1,922 @@ +#!/usr/bin/env python3 +"""One cell of the cross-backend restore matrix, at the binary level. + +A cell proves that a backup taken on the source backend restores, through the +store, into the target backend: the restored table has the source's items, +key schema, secondary indexes, throughput, SSE and (where the target supports +them) vector indexes, and its class is STANDARD. The target's catalog has no +`backup_index` row for the source's backup until `extenddb backup sync` on +the target adopts the prefix the cell copies under a root the target owns, +so a cell proves the adoption and the restore. With --adopt-store the cell +copies nothing: the target's store path is the source's root, and `backup +sync --adopt-store` makes the target's catalog its owner, after which the +source's server finds the store owned by another catalog. +`matrix-nightly.yml` runs the twelve plain cells, the duplicate-key and +vector cells and two --adopt-store cells through this committed script. + + backup-restore-matrix --source --target + [--items N] [--vector] [--duplicate-key] [--adopt-store] [--work DIR] + [--bin-sqlite PATH] [--bin-postgres PATH] [--bin-mongodb PATH] [--bin-cassandra PATH] + +Starts a source server from the backend's binary with a filesystem store at +its own root, creates a composite-key table with a GSI and an LSI (and, with +--vector, a vector index), writes --items items carrying every attribute +type, takes a backup and waits for AVAILABLE. Starts the target server with +its own empty root (it writes its own extenddb-store.json), copies the +source's backup prefix under the target root and runs `extenddb backup sync` +on the target (beside its server, or on SQLite with the server stopped, since +there the command takes the serve lock), which must adopt the backup, then +restores, waits for ACTIVE, compares the item set and the +description (key schema, indexes, throughput, SSE; the table class STANDARD), +queries each GSI, and tears both servers down by PID and drops their +databases. With --duplicate-key the copied prefix is replaced by a hand-built +unordered backup whose two data files share one key, and the cell passes +when the restore fails (the target gone, no table) instead. --adopt-store +takes neither --duplicate-key nor a MongoDB source with a MongoDB target +(the two catalogs would share the backend's fixed names). + +The two servers connect to PostgreSQL at 127.0.0.1:55432 (postgres/devpass), +MongoDB at 127.0.0.1:27117 and Cassandra at 127.0.0.1:9042, the coordinates +`matrix-nightly.yml` publishes its services on. Each server is an `init` with +its port patched, `[backup]` with short intervals and `serve --foreground`, +the test credentials provisioned through devtools/provision-test-credentials. +The MongoDB and Cassandra deployments use the backends' fixed names (the +`extenddb_catalog` and `extenddb_data` databases; the `extenddb_catalog` and +`extenddb_account_` keyspaces) and drop them in `init` and +`teardown`, so two cells with a MongoDB or Cassandra role, or a cell beside +another process on those names, destroy each other's catalogs: run such cells +one at a time, as the nightly does. The MongoDB and Cassandra roles need +`pymongo` and `cassandra-driver` (requirements-matrix.txt). + +Logs go under --work (default `$TMPDIR/backup-restore-matrix/`). +Exit 0 when the cell passes, 1 on an assertion failure or a setup error. +""" +from __future__ import annotations + +import argparse +import base64 +import gzip +import hashlib +import json +import os +import re +import shutil +import signal +import subprocess +import sys +import tempfile +import time +import urllib.parse + +import boto3 +import requests +import urllib3 +from botocore.auth import SigV4Auth +from botocore.awsrequest import AWSRequest +from botocore.config import Config +from botocore.exceptions import ClientError + +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + +HERE = os.path.dirname(os.path.abspath(__file__)) +# The script lives at /devtools/backup-restore-matrix; the repo root +# is its parent. A caller overrides the binaries with --bin-. +REPO = os.path.abspath(os.path.join(HERE, "..")) +ACCOUNT = "123456789012" +REGION = "us-east-1" +ADMIN_PASSWORD = "TestPass1!" +PG = {"host": "127.0.0.1", "port": "55432", "user": "postgres", "password": "devpass"} +MONGODB_URI = "mongodb://127.0.0.1:27117/?directConnection=true" + +DEFAULT_BINS = { + "sqlite": os.path.join(REPO, "target", "rel-sqlite", "release", "extenddb"), + "postgres": os.path.join(REPO, "target", "rel-postgres", "release", "extenddb"), # or target/release + "mongodb": os.path.join(REPO, "target", "rel-mongodb", "release", "extenddb"), + "cassandra": os.path.join(REPO, "target", "rel-cassandra", "release", "extenddb"), +} + + +def say(msg: str) -> None: + print(f"[{time.strftime('%H:%M:%S')}] {msg}", flush=True) + + +class CellFailed(Exception): + pass + + +def check(cond: bool, msg: str) -> None: + if not cond: + raise CellFailed(msg) + + +def free_port() -> int: + import socket + + with socket.socket() as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + + +# --- deployments ----------------------------------------------------------------- + + +class Deployment: + """One server: its binary, work directory, config, store root, PID and + the credentials provisioned on it.""" + + def __init__(self, backend: str, binary: str, work: str, role: str): + self.backend = backend + self.bin = binary + self.work = os.path.join(work, role) + self.role = role + os.makedirs(self.work, exist_ok=True) + self.config = os.path.join(self.work, "extenddb.toml") + self.root = os.path.join(self.work, "root") + self.port = free_port() + self.endpoint = f"https://127.0.0.1:{self.port}" + self.tag = f"mx{int(time.time()) % 1_000_000}{role[0]}" + self.proc: subprocess.Popen | None = None + self.creds: dict[str, str] = {} + + # -- init and config -- + + def init(self) -> None: + env = dict(os.environ, EXTENDDB_ADMIN_PASSWORD=ADMIN_PASSWORD) + os.makedirs(self.root, mode=0o700, exist_ok=True) + if self.backend == "mongodb": + # The MongoDB backend uses fixed database names + # (extenddb_catalog / extenddb_data), so a cell drops them before + # it inits and after it tears down, and a MongoDB source and a + # MongoDB target never run at once (the source stops before the + # target starts). + self.mongo_drop_databases() + with open(self.config, "w") as f: + f.write( + f'[storage]\nbackend = "mongodb"\n\n[storage.mongodb]\n' + f'connection_string = "{MONGODB_URI}"\n' + ) + cmd = [self.bin, "init", "--backend", "mongodb", "--config", self.config, + "--overwrite"] + with open(os.path.join(self.work, "init.log"), "w") as log: + r = subprocess.run(cmd, cwd=self.work, env=env, stdout=log, stderr=subprocess.STDOUT) + if r.returncode != 0: + print(open(os.path.join(self.work, "init.log")).read()) + raise SystemExit(f"init of the {self.role} failed") + # `init` regenerates the config without the port, run_dir and + # backup sections the cell needs; write the whole config, as + # devtools/run-mongodb-tests does. + home = os.path.expanduser("~") + with open(self.config, "w") as f: + f.write( + f'[server]\nbind_addr = "127.0.0.1"\nport = {self.port}\n' + f'region = "{REGION}"\nrun_dir = "{self.work}/run"\n\n' + f'[server.tls]\ncert_path = "{home}/.extenddb/tls/cert.pem"\n' + f'key_path = "{home}/.extenddb/tls/key.pem"\n\n' + f'[storage]\nbackend = "mongodb"\n\n[storage.mongodb]\n' + f'connection_string = "{MONGODB_URI}"\n\n' + f'[auth]\nprovider = "builtin"\n\n' + f'[backup]\nreconcile_interval_secs = 2\nstale_job_after_secs = 45\n\n' + f'[backup.store]\nkind = "filesystem"\n' + f'[backup.store.filesystem]\npath = "{self.root}"\n' + ) + return + if self.backend == "sqlite": + cmd = [self.bin, "init", "--backend", "sqlite", "--sqlite-path", + os.path.join(self.work, "db.sqlite"), "--config", self.config] + elif self.backend == "postgres": + cmd = [self.bin, "init", "--pg-host", PG["host"], "--pg-port", PG["port"], + "--pg-user", PG["user"], "--pg-pass", PG["password"], + "--data-db", f"{self.tag}_data", "--catalog-db", f"{self.tag}_catalog", + "--extenddb-user", f"{self.tag}_app", "--config", self.config] + elif self.backend == "cassandra": + cmd = [self.bin, "init", "--backend", "cassandra", + "--cassandra-contact-points", "127.0.0.1:9042", + "--cassandra-user", "cassandra", "--cassandra-pass", "cassandra", + "--config", self.config] + else: + raise NotImplementedError(f"deployment of a {self.backend} server") + with open(os.path.join(self.work, "init.log"), "w") as log: + r = subprocess.run(cmd, cwd=self.work, env=env, stdout=log, stderr=subprocess.STDOUT) + if r.returncode != 0: + print(open(os.path.join(self.work, "init.log")).read()) + raise SystemExit(f"init of the {self.role} failed") + text = open(self.config).read() + text, n = re.subn(r"(?m)^[ \t]*#?[ \t]*port[ \t]*=[ \t]*\d+.*$", f"port = {self.port}", text, count=1) + assert n == 1, "no port line in the config" + text = re.sub(r'(?m)^run_dir = .*$', f'run_dir = "{self.work}/run"', text) + os.makedirs(self.root, mode=0o700, exist_ok=True) + text += ( + "\n[backup]\nreconcile_interval_secs = 2\nstale_job_after_secs = 45\n" + f'\n[backup.store]\nkind = "filesystem"\n[backup.store.filesystem]\npath = "{self.root}"\n' + ) + open(self.config, "w").write(text) + + def mongo_drop_databases(self) -> None: + """Drop the MongoDB backend's fixed catalog and data databases and wait + until each drop has completed. On a replica set `dropDatabase` is + two-phase: the database stays drop-pending until the drop is + majority-committed, a collection created meanwhile fails with + `DatabaseDropPending` (code 215) or vanishes with the database when + the drop lands, and a server started over a pending drop then answers + `ResourceNotFoundException` to writes into a table it reported + ACTIVE. The probe below creates a collection in each database until + the server accepts it, then drops the collection (a collection drop + leaves no pending database).""" + from pymongo import MongoClient + from pymongo.errors import OperationFailure + + client = MongoClient(MONGODB_URI) + try: + for name in ("extenddb_catalog", "extenddb_data"): + client.drop_database(name) + deadline = time.monotonic() + 120 + for name in ("extenddb_catalog", "extenddb_data"): + while True: + try: + client[name].create_collection("drop_probe") + except OperationFailure as e: + if e.code == 215 and time.monotonic() < deadline: + time.sleep(0.25) + continue + raise + client[name].drop_collection("drop_probe") + break + finally: + client.close() + + # -- the process -- + + def start(self, log_name: str = "serve.log", extra_env: dict | None = None) -> None: + env = dict(os.environ, **(extra_env or {})) + self.log = open(os.path.join(self.work, log_name), "a") + self.proc = subprocess.Popen( + [self.bin, "serve", "--foreground", "--config", self.config], + cwd=self.work, env=env, stdout=self.log, stderr=subprocess.STDOUT, + ) + for _ in range(300): + try: + if requests.get(f"{self.endpoint}/health", verify=False, timeout=2).status_code == 200: + break + except requests.RequestException: + pass + if self.proc.poll() is not None: + break + time.sleep(0.2) + else: + self.stop() + raise SystemExit(f"the {self.role} server did not start; see {self.log.name}") + if self.proc.poll() is not None: + raise SystemExit(f"the {self.role} server exited at start; see {self.log.name}") + say(f"{self.role} ({self.backend}) pid {self.proc.pid} on {self.port}") + + def stop(self) -> None: + """SIGTERM by PID, then SIGKILL if it outlives the drain.""" + if self.proc is None or self.proc.poll() is not None: + return + self.proc.send_signal(signal.SIGTERM) + try: + self.proc.wait(timeout=60) + except subprocess.TimeoutExpired: + self.proc.kill() + self.proc.wait(timeout=10) + say(f"{self.role} pid {self.proc.pid} stopped (exit {self.proc.returncode})") + + def teardown(self) -> None: + self.stop() + if self.backend == "postgres": + psql(f"DROP DATABASE IF EXISTS {self.tag}_data WITH (FORCE)") + psql(f"DROP DATABASE IF EXISTS {self.tag}_catalog WITH (FORCE)") + psql(f"DROP ROLE IF EXISTS {self.tag}_app") + elif self.backend == "mongodb": + self.mongo_drop_databases() + elif self.backend == "cassandra": + # The Cassandra target server used the default keyspace prefix + # `extenddb`; drop its catalog and this account's data keyspace. + # The names are fixed, so another cell or process on the same + # node loses its catalog here (module docstring). + try: + from cassandra.auth import PlainTextAuthProvider + from cassandra.cluster import Cluster + + cluster = Cluster( + ["127.0.0.1"], port=9042, + auth_provider=PlainTextAuthProvider("cassandra", "cassandra"), + ) + session = cluster.connect() + for ks in ("extenddb_catalog", f"extenddb_account_{ACCOUNT}"): + session.execute(f"DROP KEYSPACE IF EXISTS {ks}") + cluster.shutdown() + except Exception as e: # noqa: BLE001 + print(f"cassandra teardown: {e}") + + # -- credentials and clients -- + + def provision(self) -> None: + env = dict(os.environ, EXTENDDB_TEST_ENDPOINT=self.endpoint, EXTENDDB_ADMIN_USER="admin", + EXTENDDB_ADMIN_PASSWORD=ADMIN_PASSWORD) + r = subprocess.run([sys.executable, os.path.join(REPO, "devtools", "provision-test-credentials")], + env=env, capture_output=True, text=True) + if r.returncode != 0: + raise SystemExit(f"provisioning on the {self.role} failed: {r.stderr}") + for line in r.stdout.splitlines(): + m = re.match(r"export (\w+)=(.*)", line.strip()) + if m: + self.creds[m.group(1)] = m.group(2) + for key, value in [("control_plane_delay_seconds", "0.05"), ("index_propagation_delay_ms", "0")]: + subprocess.run([self.bin, "settings", "--config", self.config, "set", key, value], + cwd=self.work, capture_output=True) + + def client(self): + return boto3.client( + "dynamodb", endpoint_url=self.endpoint, region_name=REGION, verify=False, + aws_access_key_id=self.creds["AWS_ACCESS_KEY_ID"], + aws_secret_access_key=self.creds["AWS_SECRET_ACCESS_KEY"], + config=Config(retries={"max_attempts": 10, "mode": "standard"}, read_timeout=120), + ) + + def signed_post(self, operation: str, body: dict) -> requests.Response: + """A hand-signed request for an operation botocore's model lacks + (CreateTable with VectorIndexes, DescribeTable's VectorIndexes, + SearchVectors).""" + body_bytes = json.dumps(body).encode() + headers = {"X-Amz-Target": f"DynamoDB_20120810.{operation}", + "Content-Type": "application/x-amz-json-1.0"} + frozen = boto3.Session( + aws_access_key_id=self.creds["AWS_ACCESS_KEY_ID"], + aws_secret_access_key=self.creds["AWS_SECRET_ACCESS_KEY"], + ).get_credentials().get_frozen_credentials() + req = AWSRequest(method="POST", url=self.endpoint, data=body_bytes, headers=headers) + SigV4Auth(frozen, "dynamodb", REGION).add_auth(req) + return requests.post(self.endpoint, data=body_bytes, headers=dict(req.headers), verify=False) + + def store_id(self) -> str: + return json.load(open(os.path.join(self.root, "extenddb-store.json")))["store_id"] + + +def psql(statement: str, database: str = "postgres", tuples: bool = False) -> str: + env = dict(os.environ, PGPASSWORD=PG["password"]) + cmd = ["psql", "-h", PG["host"], "-p", PG["port"], "-U", PG["user"], "-d", database, "-v", "ON_ERROR_STOP=1", "-q"] + if tuples: + cmd.append("-tA") + r = subprocess.run(cmd + ["-c", statement], env=env, capture_output=True, text=True) + if r.returncode != 0: + raise SystemExit(f"psql failed: {r.stderr}\n{statement}") + return r.stdout.strip() + + +# --- the table, the items, the backup -------------------------------------------- + +TABLE = "MatrixSource" +TARGET = "MatrixTarget" + + +def table_input(name: str, vector: bool) -> dict: + body = { + "TableName": name, + "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}, {"AttributeName": "sk", "KeyType": "RANGE"}], + "AttributeDefinitions": [ + {"AttributeName": "pk", "AttributeType": "S"}, + {"AttributeName": "sk", "AttributeType": "N"}, + {"AttributeName": "g", "AttributeType": "S"}, + {"AttributeName": "l", "AttributeType": "S"}, + ], + "BillingMode": "PROVISIONED", + "ProvisionedThroughput": {"ReadCapacityUnits": 7, "WriteCapacityUnits": 9}, + "GlobalSecondaryIndexes": [{ + "IndexName": "by-g", + "KeySchema": [{"AttributeName": "g", "KeyType": "HASH"}, {"AttributeName": "sk", "KeyType": "RANGE"}], + "Projection": {"ProjectionType": "ALL"}, + "ProvisionedThroughput": {"ReadCapacityUnits": 3, "WriteCapacityUnits": 4}, + }], + "LocalSecondaryIndexes": [{ + "IndexName": "by-l", + "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}, {"AttributeName": "l", "KeyType": "RANGE"}], + "Projection": {"ProjectionType": "KEYS_ONLY"}, + }], + "SSESpecification": {"Enabled": True, "SSEType": "KMS"}, + } + if vector: + # Vector indexes are accepted on on-demand tables only. + body["BillingMode"] = "PAY_PER_REQUEST" + del body["ProvisionedThroughput"] + del body["GlobalSecondaryIndexes"][0]["ProvisionedThroughput"] + body["VectorIndexes"] = [{ + "IndexName": "by-emb", "Dimensions": 3, "DistanceFunction": "COSINE", + "VectorAttribute": {"AttributeName": "emb"}, "Projection": {"ProjectionType": "ALL"}, + }] + return body + + +def item(i: int, vector: bool) -> dict: + """Item `i` with every attribute type; every second item carries the GSI + key, every third the LSI key, every fifth the vector.""" + it = { + "pk": {"S": f"p{i % 11:03d}"}, + "sk": {"N": str(i)}, + "s": {"S": f"text {i} \u00e9\u2028"}, + "n": {"N": f"{i}.25"}, + "big": {"N": "12345678901234567890123456789012345678"}, + "b": {"B": bytes([i % 256, 0, 255, 7])}, + "ss": {"SS": [f"a{i}", "b", "c"]}, + "ns": {"NS": ["1", "2.5", str(-i)]}, + "bs": {"BS": [b"\x00\x01", bytes([i % 256])]}, + "bool": {"BOOL": i % 2 == 0}, + "null": {"NULL": True}, + "list": {"L": [{"S": "x"}, {"N": "1"}, {"M": {"k": {"BOOL": False}}}]}, + "map": {"M": {"nested": {"M": {"deep": {"L": [{"NULL": True}]}}}, "n": {"N": "3"}}}, + } + if i % 2 == 0: + it["g"] = {"S": f"g{i % 4}"} + if i % 3 == 0: + it["l"] = {"S": f"l{i:05d}"} + if vector and i % 5 == 0: + it["emb"] = {"L": [{"N": "0.1"}, {"N": str(0.01 * (i % 50))}, {"N": "0.3"}]} + return it + + +def create_and_fill(d: Deployment, items: int, vector: bool) -> None: + body = table_input(TABLE, vector) + r = d.signed_post("CreateTable", body) + check(r.status_code == 200, f"CreateTable on the source: {r.status_code} {r.text}") + c = d.client() + wait_active(d, TABLE) + reqs = [] + for i in range(items): + reqs.append({"PutRequest": {"Item": item(i, vector)}}) + if len(reqs) == 25 or i == items - 1: + pending = {TABLE: reqs} + while pending: + pending = c.batch_write_item(RequestItems=pending).get("UnprocessedItems") or {} + reqs = [] + say(f"source: {items} items written") + + +def wait_active(d: Deployment, name: str, timeout: float = 600) -> dict: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + desc = d.signed_post("DescribeTable", {"TableName": name}) + if desc.status_code == 200: + table = desc.json()["Table"] + indexes = (table.get("GlobalSecondaryIndexes") or []) + (table.get("VectorIndexes") or []) + if table["TableStatus"] == "ACTIVE" and all(x.get("IndexStatus", "ACTIVE") == "ACTIVE" for x in indexes): + return table + time.sleep(0.2) + raise CellFailed(f"{name} did not become ACTIVE on the {d.role} within {timeout} s") + + +def take_backup(d: Deployment) -> tuple[str, dict]: + c = d.client() + arn = c.create_backup(TableName=TABLE, BackupName="matrix")["BackupDetails"]["BackupArn"] + deadline = time.monotonic() + 600 + while True: + desc = c.describe_backup(BackupArn=arn)["BackupDescription"] + status = desc["BackupDetails"]["BackupStatus"] + if status == "AVAILABLE": + say(f"source: backup {arn} AVAILABLE ({desc['BackupDetails']['BackupSizeBytes']} bytes)") + return arn, desc + check(status == "CREATING", f"the backup is {status}") + check(time.monotonic() < deadline, "the backup did not become AVAILABLE") + time.sleep(0.2) + + +def copy_prefix(source_prefix: str, target_prefix: str, target_root: str) -> None: + """Copy the backup's prefix under the target root, every directory + 0700 and every file 0600 (the store refuses a root others can read).""" + rel = os.path.relpath(os.path.dirname(target_prefix), target_root) + path = target_root + for part in rel.split(os.sep): + path = os.path.join(path, part) + if not os.path.isdir(path): + os.mkdir(path, 0o700) + os.chmod(path, 0o700) + shutil.copytree(source_prefix, target_prefix) + for dirpath, dirnames, filenames in os.walk(target_prefix): + os.chmod(dirpath, 0o700) + for name in filenames: + os.chmod(os.path.join(dirpath, name), 0o600) + + +def prefix_of(root: str, arn: str) -> str: + backup_id = arn.rsplit("/", 1)[1] + table = arn.split(":table/")[1].split("/")[0] + return os.path.join(root, ACCOUNT, table, backup_id) + + +# --- a hand-built unordered backup with one key in two files ----------------------- + + +def write_duplicate_key_prefix(prefix: str, manifest: dict, vector: bool) -> dict: + """Replace the data files under `prefix` with two unordered files that + share the key of item 0, and rewrite the manifest and its sidecar; every + file written here is 0600 and the data directory 0700, as `copy_prefix` + leaves the rest of the prefix.""" + data_dir = os.path.join(prefix, "data") + shutil.rmtree(data_dir) + os.makedirs(data_dir, mode=0o700) + os.chmod(data_dir, 0o700) + files = [] + total = 0 + for index, rows in enumerate([[item(i, vector) for i in range(0, 6)], [item(i, vector) for i in range(6, 12)] + [item(0, vector)]], start=1): + lines = b"".join(json.dumps({"Item": r}, separators=(",", ":"), default=b64).encode() + b"\n" for r in rows) + import io + + buffer = io.BytesIO() + with gzip.GzipFile(fileobj=buffer, mode="wb", mtime=0) as gz: + gz.write(lines) + compressed = buffer.getvalue() + key = f"data/{index:06d}.json.gz" + with open(os.path.join(prefix, key), "wb") as f: + f.write(compressed) + os.chmod(os.path.join(prefix, key), 0o600) + files.append({ + "key": key, "item_count": len(rows), + "sha256": hashlib.sha256(compressed).hexdigest(), + "md5": base64.b64encode(hashlib.md5(compressed).digest()).decode(), + "bytes": len(compressed), "uncompressed_bytes": len(lines), "key_range": None, + }) + total += len(compressed) + manifest = dict(manifest) + manifest["item_order"] = "unordered" + manifest["data_files"] = files + manifest["backup_size_bytes"] = total + manifest["source_table"] = dict(manifest["source_table"], item_count=13) + text = json.dumps(manifest, separators=(",", ":")).encode() + with open(os.path.join(prefix, "extenddb-manifest.json"), "wb") as f: + f.write(text) + os.chmod(os.path.join(prefix, "extenddb-manifest.json"), 0o600) + with open(os.path.join(prefix, "extenddb-manifest.sha256"), "w") as f: + f.write(f"{hashlib.sha256(text).hexdigest()} extenddb-manifest.json\n") + os.chmod(os.path.join(prefix, "extenddb-manifest.sha256"), 0o600) + write_ddb_manifests(prefix, files, total) + return manifest + + +def write_ddb_manifests(prefix: str, files: list[dict], size_bytes: int) -> None: + """Rewrite the two DynamoDB export manifests for the data files now under + `prefix`, so the prefix is a whole backup in the format's sense (one + `md5Checksum` and one `itemCount` per file, agreeing with the + extenddb manifest by construction): `manifest-files.json` is one line per + file, `manifest-summary.json` keeps the source's document with the new + item count and size.""" + summary_path = os.path.join(prefix, "manifest-summary.json") + with open(summary_path) as f: + summary = json.load(f) + s3_prefix = summary["s3Prefix"] + with open(os.path.join(prefix, "manifest-files.json"), "w") as f: + for entry in files: + line = { + "itemCount": entry["item_count"], + "md5Checksum": entry["md5"], + "dataFileS3Key": f"{s3_prefix}/{entry['key']}" if s3_prefix else entry["key"], + } + f.write(json.dumps(line, separators=(",", ":")) + "\n") + os.chmod(os.path.join(prefix, "manifest-files.json"), 0o600) + summary["itemCount"] = sum(entry["item_count"] for entry in files) + summary["billedSizeBytes"] = size_bytes + with open(summary_path, "w") as f: + f.write(json.dumps(summary, separators=(",", ":"))) + os.chmod(summary_path, 0o600) + + +def b64(value): + if isinstance(value, bytes): + return base64.b64encode(value).decode() + raise TypeError(type(value)) + + +# --- the adoption -------------------------------------------------------------- + + +def backup_sync(target: Deployment, *options: str) -> str: + """`extenddb backup sync [options]` on the target, its output logged to + `sync.log`: beside the server on PostgreSQL, MongoDB and Cassandra; on + SQLite, where the command takes the serve lock, with the server stopped + and started again after it.""" + if target.backend == "sqlite": + target.stop() + r = subprocess.run( + [target.bin, "backup", "sync", *options, "--config", target.config], + cwd=target.work, capture_output=True, text=True, stdin=subprocess.DEVNULL, timeout=600, + ) + with open(os.path.join(target.work, "sync.log"), "a") as log: + log.write(f"$ extenddb backup sync {' '.join(options)}: exit {r.returncode}\n{r.stdout}{r.stderr}\n") + if target.backend == "sqlite": + target.start() + check(r.returncode == 0, + f"backup sync {' '.join(options)} on the target exited {r.returncode}: {(r.stdout + r.stderr)[-3000:]}") + return r.stdout + + +def store_health(d: Deployment) -> dict: + """The `backup_store` component of the server's /health.""" + return requests.get(f"{d.endpoint}/health", verify=False, timeout=5).json().get("components", {}).get("backup_store", {}) + + +def wait_store(d: Deployment, predicate, what: str, timeout: float = 60) -> dict: + deadline = time.monotonic() + timeout + while True: + component = store_health(d) + if predicate(component): + return component + check(time.monotonic() < deadline, f"the {d.role}'s store is not {what} after {timeout} s: {component}") + time.sleep(0.5) + + +def adopt_backup(target: Deployment, arn: str, adopt_store: bool) -> None: + """Sync the target's catalog with its root and require the one backup + adopted: a plain `backup sync` for a prefix copied under a root the target + owns, `--adopt-store` for the source's root itself.""" + out = backup_sync(target, *(["--adopt-store"] if adopt_store else [])) + if adopt_store: + check("Adopted the root: extenddb-store.json named catalog" in out, f"--adopt-store did not adopt the root: {out[-3000:]}") + check(f" {arn}" in out.splitlines() and "1 backup(s) adopted" in out, + f"backup sync did not adopt {arn}: {out[-3000:]}") + wait_store(target, lambda c: c.get("status") == "healthy", "healthy") + say(f"target: backup sync adopted {arn}" + (" with the root (--adopt-store)" if adopt_store else "")) + + +# --- comparison --------------------------------------------------------------- + + +def normalize(it: dict) -> str: + def norm(v): + if isinstance(v, dict): + if len(v) == 1: + (k, inner), = v.items() + if k in ("SS", "NS"): + return {k: sorted(inner)} + if k == "BS": + return {k: sorted(base64.b64encode(x).decode() for x in inner)} + if k == "B": + return {k: base64.b64encode(inner).decode()} + if k == "N": + return {k: str(inner)} + return {k: norm(x) for k, x in v.items()} + if isinstance(v, list): + return [norm(x) for x in v] + return v + return json.dumps(norm(it), sort_keys=True) + + +def scan_all(d: Deployment, name: str, index: str | None = None) -> list[str]: + c = d.client() + out, kwargs = [], {"TableName": name} + if index: + kwargs["IndexName"] = index + while True: + page = c.scan(**kwargs) + out.extend(normalize(i) for i in page["Items"]) + if "LastEvaluatedKey" not in page: + return sorted(out) + kwargs["ExclusiveStartKey"] = page["LastEvaluatedKey"] + + +def compare_descriptions(source: dict, target: dict) -> None: + def keys(x): + return [(k["AttributeName"], k["KeyType"]) for k in x["KeySchema"]] + check(keys(source) == keys(target), "key schema differs") + check(sorted((a["AttributeName"], a["AttributeType"]) for a in source["AttributeDefinitions"]) + == sorted((a["AttributeName"], a["AttributeType"]) for a in target["AttributeDefinitions"]), "attribute definitions differ") + for kind in ("GlobalSecondaryIndexes", "LocalSecondaryIndexes"): + s = {g["IndexName"]: (keys(g), g["Projection"]["ProjectionType"]) for g in source.get(kind) or []} + t = {g["IndexName"]: (keys(g), g["Projection"]["ProjectionType"]) for g in target.get(kind) or []} + check(s == t, f"{kind} differ: {s} vs {t}") + def units(x): + pt = x.get("ProvisionedThroughput") or {} + return (pt.get("ReadCapacityUnits", 0), pt.get("WriteCapacityUnits", 0)) + for g in target.get("GlobalSecondaryIndexes") or []: + sg = next(x for x in source["GlobalSecondaryIndexes"] if x["IndexName"] == g["IndexName"]) + check(units(g) == units(sg), f"GSI {g['IndexName']} throughput differs: {units(g)} vs {units(sg)}") + check(units(source) == units(target), f"table throughput differs: {units(source)} vs {units(target)}") + check((source.get("BillingModeSummary") or {}).get("BillingMode", "PROVISIONED") + == (target.get("BillingModeSummary") or {}).get("BillingMode", "PROVISIONED"), "billing mode differs") + check((source.get("SSEDescription") or {}).get("Status") == (target.get("SSEDescription") or {}).get("Status"), "SSE differs") + check((target.get("TableClassSummary") or {}).get("TableClass", "STANDARD") == "STANDARD", "the target's table class is not STANDARD") + sv = {v["IndexName"]: (v["Dimensions"], v["DistanceFunction"]) for v in source.get("VectorIndexes") or []} + tv = {v["IndexName"]: (v["Dimensions"], v["DistanceFunction"]) for v in target.get("VectorIndexes") or []} + check(sv == tv, f"vector indexes differ: {sv} vs {tv}") + + +# --- the cell ------------------------------------------------------------------ + + +def run(args) -> int: + cell_tag = ( + f"{args.source}-{args.target}" + + ("-vector" if args.vector else "") + + ("-dup" if args.duplicate_key else "") + + ("-adopt" if args.adopt_store else "") + ) + work = args.work or os.path.join(tempfile.gettempdir(), "backup-restore-matrix", cell_tag) + work = os.path.abspath(work) + shutil.rmtree(work, ignore_errors=True) + os.makedirs(work) + bins = {b: os.path.abspath(getattr(args, f"bin_{b}") or DEFAULT_BINS[b]) for b in DEFAULT_BINS} + for role, backend in (("source", args.source), ("target", args.target)): + check(os.path.isfile(bins[backend]), f"no {backend} binary at {bins[backend]} for the {role}") + source = Deployment(args.source, bins[args.source], work, "source") + target = Deployment(args.target, bins[args.target], work, "target") + rc = 0 + try: + source.init() + source.start() + source.provision() + create_and_fill(source, args.items, args.vector) + source_desc = wait_active(source, TABLE) + arn, backup_desc = take_backup(source) + source_items = scan_all(source, TABLE) + source_prefix = prefix_of(source.root, arn) + manifest = json.load(open(os.path.join(source_prefix, "extenddb-manifest.json"))) + source.stop() + + if args.adopt_store: + # No copy: the target's store is the source's root, which names + # the source's catalog until the target adopts it. + target.root = source.root + target.init() + target.start() + target.provision() + if args.adopt_store: + with open(os.path.join(source.root, "extenddb-store.json")) as f: + owner = json.load(f)["catalog_id"] + wait_store(target, lambda c: c.get("reason") == "owned by another catalog", "owned by another catalog") + adopt_backup(target, arn, adopt_store=True) + with open(os.path.join(source.root, "extenddb-store.json")) as f: + check(json.load(f)["catalog_id"] != owner, "the root's file still names the source's catalog") + source.start("serve-after-adoption.log") + component = wait_store(source, lambda c: c.get("reason") == "owned by another catalog", + "owned by another catalog") + check(component.get("status") == "unhealthy", f"the source's store after the adoption: {component}") + say(f"source: after the adoption its store is {component}") + source.stop() + else: + check(os.path.isfile(os.path.join(target.root, "extenddb-store.json")), "the target wrote no identity file") + target_prefix = prefix_of(target.root, arn) + copy_prefix(source_prefix, target_prefix, target.root) + if args.duplicate_key: + manifest = write_duplicate_key_prefix(target_prefix, manifest, args.vector) + say("target: the prefix now holds an unordered backup with one key in two files") + adopt_backup(target, arn, adopt_store=False) + + c = target.client() + described = c.describe_backup(BackupArn=arn)["BackupDescription"] + check(described["BackupDetails"]["BackupStatus"] == "AVAILABLE", "the adopted row does not describe AVAILABLE") + check(described["BackupDetails"]["BackupName"] == "matrix", f"the adopted row's name: {described['BackupDetails']}") + check(described["BackupDetails"]["BackupSizeBytes"] == manifest["backup_size_bytes"], + f"the adopted row's size: {described['BackupDetails']} against the manifest's {manifest['backup_size_bytes']}") + started = time.monotonic() + + if args.vector and args.target == "mongodb": + # A manifest with vector indexes into a backend without vector + # support is refused before any table is created (section 1, + # 5.3 step 2). + try: + c.restore_table_from_backup(TargetTableName=TARGET, BackupArn=arn) + except ClientError as e: + err = e.response["Error"] + check(err.get("Code") == "ValidationException", + f"the refusal is a ValidationException: {err}") + check("vector" in err.get("Message", "").lower(), + f"the refusal names vector indexes: {err}") + probe = target.signed_post("DescribeTable", {"TableName": TARGET}) + check(probe.status_code == 400 and "ResourceNotFound" in probe.text, + f"a table was created despite the refusal: {probe.status_code} {probe.text}") + say(f"target: the vector manifest was refused before any table: {err['Message'][:160]}") + print("CELL PASSED (vector manifest refused)") + return 0 + raise CellFailed("a vector manifest into MongoDB was not refused") + + # A vector manifest is refused into a backend without vector support + # (Cassandra), synchronously and before any table is created. + if args.vector and args.target == "cassandra": + try: + c.restore_table_from_backup(TargetTableName=TARGET, BackupArn=arn) + except ClientError as e: + err = e.response["Error"] + check( + err.get("Code") == "ValidationException" and "vector index" in err.get("Message", ""), + f"the vector manifest was not refused with a vector ValidationException: {err}", + ) + probe = target.signed_post("DescribeTable", {"TableName": TARGET}) + check( + probe.status_code == 400 and "ResourceNotFound" in probe.text, + f"a table was created for a refused vector restore: {probe.status_code} {probe.text}", + ) + say("target: the vector manifest was refused before any table was created") + print("CELL PASSED (vector manifest refused)") + return 0 + raise CellFailed("the vector manifest into Cassandra was not refused") + + try: + resp = c.restore_table_from_backup(TargetTableName=TARGET, BackupArn=arn) + except ClientError as e: + raise CellFailed(f"RestoreTableFromBackup refused: {e.response['Error']}") from e + desc = resp["TableDescription"] + check(desc["TableStatus"] == "CREATING", f"the restore answered {desc['TableStatus']}") + summary = desc.get("RestoreSummary") or {} + check(summary.get("SourceBackupArn") == arn and summary.get("RestoreInProgress") is True, + f"the response carries no RestoreSummary: {summary}") + say(f"target: restore of {arn} into {TARGET} registered CREATING") + + if args.duplicate_key: + deadline = time.monotonic() + 600 + while time.monotonic() < deadline: + r = target.signed_post("DescribeTable", {"TableName": TARGET}) + if r.status_code == 400 and "ResourceNotFound" in r.text: + break + # Cassandra removes a failed restore target in two writes + # (CREATING to DELETING, then the drop), so DELETING is a + # transient state on the way out, not a reached ACTIVE. + check( + r.status_code != 200 + or r.json()["Table"]["TableStatus"] in ("CREATING", "DELETING"), + f"a backup with a duplicated key reached {r.json()['Table']['TableStatus']}", + ) + time.sleep(0.5) + else: + raise CellFailed("the target of the duplicated-key backup was not removed") + # The job logs its FAILED line just after it removes the target, + # so poll the log for a short window rather than reading it the + # instant the table disappears. + log_deadline = time.monotonic() + 15 + failed = [] + while time.monotonic() < log_deadline: + log = open(os.path.join(target.work, "serve.log")).read() + failed = [l for l in log.splitlines() if "FAILED" in l and TARGET in l] + if failed: + break + time.sleep(0.25) + check(bool(failed), "the server log does not record the FAILED restore") + check(any("holds a key twice" in l or "the manifest lists" in l for l in failed), + f"the failure names neither the duplicate key nor the count: {failed[-1][:300]}") + say(f"target: {failed[-1].split(': FAILED', 1)[-1][:200]}") + say(f"target: the duplicated-key restore failed and the target is gone after {time.monotonic() - started:.1f} s") + print("CELL PASSED (duplicate key refused)") + return 0 + + target_desc = wait_active(target, TARGET) + say(f"target: {TARGET} ACTIVE after {time.monotonic() - started:.1f} s") + check(target_desc.get("RestoreSummary") is None, "an ACTIVE table still reports a RestoreSummary") + target_items = scan_all(target, TARGET) + check(len(target_items) == args.items, f"the target holds {len(target_items)} items, the source wrote {args.items}") + check(target_items == source_items, "the item sets differ") + compare_descriptions(source_desc, target_desc) + for gsi in target_desc.get("GlobalSecondaryIndexes") or []: + name = gsi["IndexName"] + rows = c.query(TableName=TARGET, IndexName=name, + KeyConditionExpression="g = :g", ExpressionAttributeValues={":g": {"S": "g0"}})["Items"] + expected = sum(1 for i in range(args.items) if i % 2 == 0 and i % 4 == 0) + check(len(rows) == expected, f"Query on {name} returned {len(rows)} items, expected {expected}") + lsi_rows = c.query(TableName=TARGET, IndexName="by-l", + KeyConditionExpression="pk = :p", ExpressionAttributeValues={":p": {"S": "p000"}})["Items"] + expected_l = sum(1 for i in range(args.items) if i % 11 == 0 and i % 3 == 0) + check(len(lsi_rows) == expected_l, f"Query on by-l returned {len(lsi_rows)} items, expected {expected_l}") + if args.vector: + r = target.signed_post("SearchVectors", {"TableName": TARGET, "IndexName": "by-emb", + "SearchVector": [{"N": "0.1"}, {"N": "0.0"}, {"N": "0.3"}], + "TopK": 3}) + check(r.status_code == 200, f"SearchVectors on the target: {r.status_code} {r.text}") + results = r.json().get("SearchResults") or [] + check(len(results) == 3, f"SearchVectors returned {len(results)} results, expected 3") + # The index strips the vector from the projected item; the items it + # answers are the ones that carried one (every fifth). + keys = [int(x["Item"]["sk"]["N"]) for x in results] + check(all(k % 5 == 0 for k in keys), f"SearchVectors answered items that carried no vector: {keys}") + check(described["SourceTableDetails"]["ItemCount"] == args.items, "DescribeBackup's item count differs from the items written") + say("target: item set, description, GSI and LSI queries match the source") + print("CELL PASSED") + return 0 + except CellFailed as e: + print(f"CELL FAILED: {e}") + return 1 + finally: + for d in (target, source): + try: + d.teardown() + except Exception as e: # noqa: BLE001 + print(f"teardown of the {d.role}: {e}") + print(f"logs under {work}") + + +def main() -> None: + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("--source", required=True, choices=["sqlite", "postgres", "mongodb"]) + p.add_argument("--target", required=True, choices=["sqlite", "postgres", "mongodb", "cassandra"]) + p.add_argument("--items", type=int, default=1200) + p.add_argument("--vector", action="store_true") + p.add_argument("--duplicate-key", action="store_true") + p.add_argument("--adopt-store", action="store_true", + help="point the target's store at the source's root and adopt it with `backup sync --adopt-store`") + p.add_argument("--work") + for b in DEFAULT_BINS: + p.add_argument(f"--bin-{b}") + args = p.parse_args() + if args.adopt_store and args.duplicate_key: + p.error("--adopt-store copies no prefix, so it takes no --duplicate-key") + if args.adopt_store and args.source == "mongodb" and args.target == "mongodb": + p.error("--adopt-store keeps the source's catalog for its health check; two MongoDB catalogs share fixed names") + sys.exit(run(args)) + + +if __name__ == "__main__": + main() diff --git a/devtools/catalog_checks.py b/devtools/catalog_checks.py index b687def35..3dc664bf8 100644 --- a/devtools/catalog_checks.py +++ b/devtools/catalog_checks.py @@ -13,6 +13,7 @@ command's output indented under a FAIL; `finish` prints the count of failed checks or `all checks passed` and returns the script's exit code. """ +import os import signal import subprocess import time @@ -125,6 +126,20 @@ def stop(proc: subprocess.Popen) -> None: proc.wait() +def with_filesystem_store(config: str, work: str) -> tuple[str, str]: + """A copy of `config` with a filesystem backup store at `/backups` + (created `0700`), for the backup commands' store-configured runs; the + copy's path and the root's canonical path, as the commands print it.""" + root = os.path.realpath(os.path.join(work, "backups")) + os.mkdir(root, 0o700) + store_config = os.path.join(work, "extenddb-store.toml") + with open(config, encoding="utf-8") as f: + base = f.read() + with open(store_config, "w", encoding="utf-8") as f: + f.write(f'{base}\n[backup.store]\nkind = "filesystem"\n\n[backup.store.filesystem]\npath = "{root}"\n') + return store_config, root + + def finish() -> int: """Print the outcome of every check reported, and return the exit code.""" if failures: @@ -162,7 +177,9 @@ def start_mongo(container: str, port: int, mongod_args: tuple[str, ...] = (), mo if started.returncode != 0: return report(False, f"start {container}", started.stderr) for _ in range(60): - if mongosh(container, port, "db.runCommand({ping: 1}).ok")[1].strip().endswith("1"): + # The whole output must be the `1`: a refused connection's error + # names the port, and a port ending in 1 would pass an `endswith`. + if mongosh(container, port, "db.runCommand({ping: 1}).ok")[1].strip() == "1": break time.sleep(1) mongosh(container, port, f"rs.initiate({{_id: 'rs0', members: [{{_id: 0, host: 'localhost:{port}'}}]}})") diff --git a/devtools/check-cassandra-catalog-lock b/devtools/check-cassandra-catalog-lock index d27061206..a0d578823 100755 --- a/devtools/check-cassandra-catalog-lock +++ b/devtools/check-cassandra-catalog-lock @@ -49,7 +49,17 @@ import tempfile import time import tomllib -from catalog_checks import expect_exit, finish, healthy, report, run, start, stop, watch_lost_registration +from catalog_checks import ( + expect_exit, + finish, + healthy, + report, + run, + start, + stop, + watch_lost_registration, + with_filesystem_store, +) ADMIN_USER = os.environ.get("CASSANDRA_ADMIN_USER", "cassandra") ADMIN_PASSWORD = os.environ.get("CASSANDRA_ADMIN_PASSWORD", "cassandra") @@ -230,6 +240,47 @@ def main(binary: str, container: str, contact_point: str, port: int) -> int: ) report(time.monotonic() - started < 30, "the stopped server's registration was removed") + # The backup commands register as a server does and release the + # registration when they exit, whatever the outcome, so a migration + # right after them does not wait for the row's TTL. With no store + # configured the command releases on its no-store path before it + # prints; with a filesystem store (a root under the work directory) + # it opens the store, runs, and releases when it closes the catalog, + # the path a store-backed deployment takes. + backup = [binary, "backup", "--config", config] + expect_exit(0, "backup list with no server running", backup + ["list"], env, contains="none (") + expect_exit( + 0, + "backup purge's dry run with no server running", + backup + ["purge"], + env, + contains="Dry run: nothing was deleted", + ) + store_config, root = with_filesystem_store(config, work) + backup = [binary, "backup", "--config", store_config] + expect_exit( + 0, + "backup list with a filesystem store configured", + backup + ["list"], + env, + contains=f"=== backups in the filesystem store {root}", + ) + expect_exit( + 0, + "backup purge's dry run with a filesystem store configured", + backup + ["purge"], + env, + contains="Dry run: nothing was deleted. Run again with --confirm", + ) + started = time.monotonic() + expect_exit( + 0, + "migrate --yes right after the backup commands, without waiting for a registration's TTL", + migrate + ["--yes", "--lock-timeout", "5"], + env, + ) + report(time.monotonic() - started < 30, "the backup commands' registration was removed") + keyspace = f"{stem}_catalog" code, output = cqlsh( container, diff --git a/devtools/check-mongodb-catalog-lock b/devtools/check-mongodb-catalog-lock index f13f764e6..762b23fdc 100755 --- a/devtools/check-mongodb-catalog-lock +++ b/devtools/check-mongodb-catalog-lock @@ -14,7 +14,9 @@ extenddb-check-lock-, published on 127.0.0.1 only: - the migrate preview and verify run beside a server and exit 0, and migrate --yes exits 6 naming the server's host and pid; - a server stopped with SIGTERM deletes its registration, so migrate --yes - then runs at once; + then runs at once; `backup list` and `backup purge`'s dry run, with and + without a store configured, release the registration they take, so + migrate --yes right after them runs at once too; - while a migration holds the lock, a server exits 6 and a second migrate --yes exits 6, and the preview still runs; - a server refuses a catalog behind this binary with 3, and the command it @@ -51,6 +53,7 @@ from catalog_checks import ( start_mongo, stop, watch_lost_registration, + with_filesystem_store, ) @@ -115,6 +118,30 @@ def main(binary: str, mongo_port: int, api_port: int) -> int: report(live_servers(container, mongo_port) == 0, "the stopped server deleted its registration") expect_exit(0, "migrate --yes once the server stopped", migrate + ["--yes", "--lock-timeout", "2"], env) + # The backup commands register as a server does and release the + # registration when they exit, whatever the outcome, so a migration + # right after them does not wait for the document to expire 45 s + # after its last renewal. With no store configured the command + # releases on its no-store path before it prints; with a filesystem + # store (a root under the work directory) it opens the store, runs, + # and releases when it closes the catalog, the path a store-backed + # deployment takes. + backup = [binary, "backup", "--config", "extenddb.toml"] + expect_exit(0, "backup list with no server running", backup + ["list"], env, contains="none (") + expect_exit(0, "backup purge's dry run with no server running", backup + ["purge"], env, + contains="Dry run: nothing was deleted") + store_config, root = with_filesystem_store(os.path.join(work, "extenddb.toml"), work) + backup = [binary, "backup", "--config", store_config] + expect_exit(0, "backup list with a filesystem store configured", backup + ["list"], env, + contains=f"=== backups in the filesystem store {root}") + expect_exit(0, "backup purge's dry run with a filesystem store configured", backup + ["purge"], env, + contains="Dry run: nothing was deleted. Run again with --confirm") + report(live_servers(container, mongo_port) == 0, "the backup commands released their registration") + started = time.monotonic() + expect_exit(0, "migrate --yes right after the backup commands", migrate + ["--yes", "--lock-timeout", "2"], + env) + report(time.monotonic() - started < 20, "migrate --yes did not wait for a backup command's registration") + # Another migration's lock, as its claim writes it. mongosh(container, mongo_port, "db.getSiblingDB('extenddb_catalog').settings.insertOne(" "{_id: 'migration_lock', owner: 'check', expires_at: new Date(Date.now() + 120000)})") diff --git a/devtools/check-postgres-catalog-lock b/devtools/check-postgres-catalog-lock index 81a76f794..6fbb5693e 100755 --- a/devtools/check-postgres-catalog-lock +++ b/devtools/check-postgres-catalog-lock @@ -45,7 +45,7 @@ import urllib.parse import psycopg2 -from catalog_checks import expect_exit, finish, healthy, report, run, start, stop +from catalog_checks import expect_exit, finish, healthy, report, run, start, stop, with_filesystem_store NAMESPACE = 0x0045_4442 # the advisory key namespace of crates/storage-postgres MIGRATION_KEY = 1 @@ -174,6 +174,46 @@ def main(binary: str, admin_url: str, port: int) -> int: ) stop(server) + # The backup commands register as a server does (the session's + # application_name) and release the registration when they exit, + # whatever the outcome, so a migration right after them finds no + # session to wait for. With no store configured the command releases + # on its no-store path before it prints; with a filesystem store (a + # root under the work directory) it opens the store, runs, and + # releases when it closes the catalog, the path a store-backed + # deployment takes. + backup = [binary, "backup", "--config", config] + expect_exit(0, "backup list with no server running", backup + ["list"], env, contains="none (") + expect_exit( + 0, + "backup purge's dry run with no server running", + backup + ["purge"], + env, + contains="Dry run: nothing was deleted", + ) + store_config, root = with_filesystem_store(config, work) + backup = [binary, "backup", "--config", store_config] + expect_exit( + 0, + "backup list with a filesystem store configured", + backup + ["list"], + env, + contains=f"=== backups in the filesystem store {root}", + ) + expect_exit( + 0, + "backup purge's dry run with a filesystem store configured", + backup + ["purge"], + env, + contains="Dry run: nothing was deleted. Run again with --confirm", + ) + expect_exit( + 0, + "migrate --yes right after the backup commands", + migrate + ["--yes", "--lock-timeout", "1"], + env, + ) + holder = psycopg2.connect(f"{admin_url.rstrip('/')}/{stem}_catalog") with holder.cursor() as cur: cur.execute("SELECT pg_advisory_xact_lock(%s, %s)", (NAMESPACE, MIGRATION_KEY)) diff --git a/devtools/run-mongodb-tests b/devtools/run-mongodb-tests index a41cc78f0..57eaba215 100755 --- a/devtools/run-mongodb-tests +++ b/devtools/run-mongodb-tests @@ -16,6 +16,13 @@ # --mongo-port PORT Host port to publish mongo on (default: 27021) # --output DIR Directory for logs and generated config # (default: /tmp/run-mongodb-tests-) +# --backup-store KIND filesystem (default): the server gets a filesystem +# [backup.store] under the output directory and the +# suites read EXTENDDB_TEST_BACKUP_STORE=filesystem; +# none: no [backup.store], the in-catalog backup path, +# EXTENDDB_TEST_BACKUP_STORE=none. The legacy LSI +# fallback cell of tests/test_mongodb_backup_restore.py +# runs only under none. # --keep Do not tear down the mongo container / server # on exit (useful for post-run inspection) # --mongo-only Start the replica set, print its URI, and exit without @@ -23,6 +30,13 @@ # For tests that talk to MongoDB directly (the storage # crate's snapshot tests). Tear down with # docker rm -f extenddb-runtests-mongo-. +# --backup-store-root DIR +# The filesystem store's root (default: backups/ under +# the output directory), created 0700 if missing; a +# fresh directory per run, since the store's identity +# file names the catalog that owns it and this runner +# initializes a new catalog each time. Implies +# --backup-store filesystem. # --rust-live Also run the Rust live tests of the storage-mongodb # crate against this runner's replica set. They are # `#[ignore]`d, because they need a replica set, and @@ -37,6 +51,7 @@ # devtools/run-mongodb-tests -- --pytest --filter test_put_item # devtools/run-mongodb-tests --keep -- --pytest --filter test_rfc0003 # devtools/run-mongodb-tests --rust-live -- --rust --rust-integration +# devtools/run-mongodb-tests --backup-store none -- --pytest --filter test_mongodb_backup_restore # # Prerequisites: # - Docker running @@ -60,6 +75,8 @@ OUTPUT_DIR="" KEEP=false MONGO_ONLY=false RUST_LIVE=false +BACKUP_STORE_ROOT="" +BACKUP_STORE=filesystem RUN_TESTS_ARGS=() while [[ $# -gt 0 ]]; do @@ -67,11 +84,18 @@ while [[ $# -gt 0 ]]; do --port) BIND_PORT="$2"; shift 2 ;; --mongo-port) MONGO_PORT="$2"; shift 2 ;; --output) OUTPUT_DIR="$2"; shift 2 ;; + --backup-store) + case "$2" in + filesystem|none) BACKUP_STORE="$2" ;; + *) echo "error: --backup-store takes filesystem or none, not: $2" >&2; exit 1 ;; + esac + shift 2 ;; --keep) KEEP=true; shift ;; --mongo-only) MONGO_ONLY=true; KEEP=true; shift ;; --rust-live) RUST_LIVE=true; shift ;; + --backup-store-root) BACKUP_STORE_ROOT="$2"; BACKUP_STORE=filesystem; shift 2 ;; --) shift; RUN_TESTS_ARGS=("$@"); break ;; - -h|--help) sed -n '5,45p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;; + -h|--help) sed -n '5,60p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;; *) echo "error: unknown option: $1" >&2; exit 1 ;; esac done @@ -134,7 +158,7 @@ cleanup() { if $KEEP; then echo "" echo "=== --keep set; leaving mongo + extenddb running ===" - echo " server log (startup): $SERVER_LOG" + echo " server log: $SERVER_LOG" echo " mongo container: $CONTAINER_NAME (port $MONGO_PORT)" echo " tear down manually: $BINARY stop --config $CONFIG; docker rm -f $CONTAINER_NAME" return @@ -221,7 +245,38 @@ EXTENDDB_ADMIN_PASSWORD="$ADMIN_PASSWORD" "$BINARY" init \ # `init` regenerates the config with default port 18443 and no # `[import]` / `[export]` sections. Rewrite with the port + defaults the # integration suite needs. This overwrite falls away once `extenddb init` -# grows flags for these knobs. +# grows flags for these knobs. With `--backup-store filesystem` (the +# default) the `[backup.store]` is a filesystem root under the output +# directory: CreateBackup is the store-backed job and RestoreTableFromBackup +# the store-backed restore, which the MongoDB restore cells +# (tests/test_mongodb_backup_restore.py) reach through the root +# (EXTENDDB_TEST_BACKUP_ROOT) and the server's log (EXTENDDB_TEST_SERVER_LOG); +# the suites read EXTENDDB_TEST_BACKUP_STORE=filesystem to expect that path. +# With `--backup-store none` the section is absent, CreateBackup writes the +# in-catalog format and the suites read EXTENDDB_TEST_BACKUP_STORE=none. +BACKUP_ROOT="${BACKUP_STORE_ROOT:-$OUTPUT_DIR/backups}" +STORE_SECTION="" +if [[ "$BACKUP_STORE" == "filesystem" ]]; then + if [[ -z "$BACKUP_STORE_ROOT" ]]; then + # A root left by an earlier run of a reused --output directory + # carries that run's `extenddb-store.json`, which names that run's + # catalog; the container is recreated every run, so the new catalog's + # id differs and the server would find the store "owned by another + # catalog". Start from an empty root. A root the caller named is + # theirs to clear: it is created and made owner-only, nothing more. + rm -rf "$BACKUP_ROOT" + fi + mkdir -p "$BACKUP_ROOT" + chmod 0700 "$BACKUP_ROOT" + echo " backup store: filesystem at $BACKUP_ROOT" + STORE_SECTION=" +[backup.store] +kind = \"filesystem\" + +[backup.store.filesystem] +path = \"$BACKUP_ROOT\" +" +fi cat > "$CONFIG" <"$SERVER_LOG" 2>&1 +# The server runs in the foreground in the background of this shell, with its +# PID file written so `extenddb stop` (the cleanup) can signal it, and its +# whole log in SERVER_LOG: the restore cells read the job's log lines there, +# where a daemonized server's would go to syslog. +"$BINARY" serve --foreground --write-pid-file --config "$CONFIG" >"$SERVER_LOG" 2>&1 & for i in $(seq 1 30); do if curl -sk "https://127.0.0.1:$BIND_PORT/health" >/dev/null 2>&1; then @@ -320,6 +379,18 @@ export EXTENDDB_TEST_MONGODB_CONTAINER="$CONTAINER_NAME" # physical index lifecycle after exercising the DynamoDB API. export EXTENDDB_TEST_MONGODB_CONTAINER="$CONTAINER_NAME" +# Where the server writes its backups, which run-tests requires for --pytest +# and the backup suites read to pick their branch (filesystem: the +# store-backed job and restore; none: the in-catalog format). The MongoDB +# restore cells rewrite a backup's prefix under the store's root and read the +# restore job's log lines from the server's log; without a store there is no +# root and those cells skip, while the legacy LSI fallback cell runs. +export EXTENDDB_TEST_BACKUP_STORE="$BACKUP_STORE" +if [[ "$BACKUP_STORE" == "filesystem" ]]; then + export EXTENDDB_TEST_BACKUP_ROOT="$BACKUP_ROOT" +fi +export EXTENDDB_TEST_SERVER_LOG="$SERVER_LOG" + # Some tests read EXTENDDB_CONFIG to invoke `extenddb settings` against # the running instance. Without this, they fall back to `./extenddb.toml` # in the repo root, which is a stale dev config that points at a @@ -357,6 +428,39 @@ if $RUST_LIVE; then echo " ✗ MongoDB live Rust tests failed" RC=1 fi + # The backup engine's live tests: the restore claim, the heartbeat, the + # reconciler's listing and FAILED write, the delete and restore orderings + # against a replica set. They are the only tests of several of those + # conditions, so they run here with the replica set required (a missing + # server is a failure, not a skip), one at a time as the tests above. + echo "" + echo "=== MongoDB backup engine live Rust tests (storage-mongodb, --ignored) ===" + if EXTENDDB_TEST_MONGODB_REQUIRED=1 \ + cargo test -p extenddb-storage-mongodb --lib -- \ + --ignored --test-threads=1 backup_engine::live_tests::; then + echo " ✓ MongoDB backup engine live Rust tests passed" + else + echo " ✗ MongoDB backup engine live Rust tests failed" + RC=1 + fi + + + # The restore fence cells need the `test-hooks` feature (the GSI backfill + # gate `bulk_load` honours is compiled only under it), and no other job + # builds that feature, so the takeover fence has no CI coverage without + # this step. Run them one at a time against the same replica set. + echo "" + echo "=== MongoDB restore fence cells (storage-mongodb, test-hooks) ===" + if cargo test -p extenddb-storage-mongodb --lib --features test-hooks -- \ + --ignored --test-threads=1 \ + backup_engine::live_tests::a_takeover_during_bulk_load_fences_the_file_index_write \ + backup_engine::live_tests::live_bulk_load_stops_at_a_lost_claim_between_two_insert_many_calls \ + backup_engine::live_tests::live_bulk_load_records_each_batch_and_a_resume_continues_from_items_applied; then + echo " ✓ MongoDB restore fence cells passed" + else + echo " ✗ MongoDB restore fence cells failed" + RC=1 + fi fi exit $RC diff --git a/devtools/run-tests b/devtools/run-tests index ae7984896..4591846ba 100755 --- a/devtools/run-tests +++ b/devtools/run-tests @@ -38,6 +38,13 @@ # --parallel[=N] run pytest in parallel (default: 1/3 of CPU cores) # --config PATH config file path (default: extenddb.toml in repo root) # +# Environment (--pytest against extenddb): +# EXTENDDB_TEST_BACKUP_STORE where the server writes its backups: filesystem, +# s3 or none; the backup tests derive their +# expected outcome from it, so it is required +# (the Rust integration suite's backup tests read +# it too and fail when it is unset). +# # REQ-TEST-001, D-1, D-2, D-3, D-4, D-5 set -euo pipefail @@ -98,6 +105,15 @@ Options: --filter PATTERN filter for cargo test / pytest -k --parallel[=N] run pytest in parallel (default: 1/3 of CPU cores) --config PATH config file path + +Environment (--extenddb with an integration suite): + EXTENDDB_TEST_ENDPOINT the server, https://localhost: (required) + EXTENDDB_ADMIN_PASSWORD the password from 'extenddb init' (required) + EXTENDDB_TEST_BACKUP_STORE where the server writes its backups: filesystem, + s3 or none (required for --pytest; the Rust + integration suite's backup tests read it too) +Environment (--real-dynamodb): EXTENDDB_TEST_ENDPOINT and EXTENDDB_ADMIN_PASSWORD + unset; AWS credentials from the SDK's chain or AWS_ACCESS_KEY_ID. EOF exit 1 } @@ -187,6 +203,25 @@ validate_local_target() { echo " This is the password from '${prefix_lower} init'." exit 1 fi + + # EXTENDDB_TEST_BACKUP_STORE must say where the server writes its backups. + # The backup tests derive their expected outcome from it (the store-backed + # create backs up vector tables and is asynchronous; the in-catalog create + # refuses them), so a run cannot leave it to be guessed. + if $RUN_PYTEST; then + case "${EXTENDDB_TEST_BACKUP_STORE:-}" in + filesystem|s3|none) ;; + *) + echo "error: --${prefix_lower} --pytest requires EXTENDDB_TEST_BACKUP_STORE to be set." + echo " Set it to the server's [backup.store]: 'filesystem' or 's3' for a configured" + echo " store (a file-backed SQLite server without the section has the defaulted" + echo " filesystem root), 'none' for a server without one (kind = \"disabled\", or" + echo " PostgreSQL, MongoDB and Cassandra without the section)." + echo " Got: '${EXTENDDB_TEST_BACKUP_STORE:-}'" + exit 1 + ;; + esac + fi } validate_real_dynamodb() { @@ -244,22 +279,42 @@ fi echo "" # --- Health check (local targets only) --- +# With EXTENDDB_TEST_BACKUP_STORE at filesystem or s3 the check also waits for +# the backup store component to report healthy: /health answers 200 while the +# component is still "checking", and the backup suites assert the component +# is healthy at their start, so what is waited for here is what they assert. health_check() { local endpoint="$1" local timeout=30 local interval=1 local elapsed=0 + local want_store=false + case "${EXTENDDB_TEST_BACKUP_STORE:-}" in + filesystem|s3) want_store=true ;; + esac echo "=== Health check: ${endpoint}/health ===" while [[ $elapsed -lt $timeout ]]; do - if curl -skf -o /dev/null "${endpoint}/health" 2>/dev/null; then - echo " ✓ Server is healthy (${elapsed}s)" - return 0 + local body + if body=$(curl -skf "${endpoint}/health" 2>/dev/null) && [[ -n "$body" ]]; then + if ! $want_store; then + echo " ✓ Server is healthy (${elapsed}s)" + return 0 + fi + if printf '%s' "$body" | python3 -c 'import json, sys +h = json.load(sys.stdin) +sys.exit(0 if h.get("components", {}).get("backup_store", {}).get("status") == "healthy" else 1)' 2>/dev/null; then + echo " ✓ Server and its backup store are healthy (${elapsed}s)" + return 0 + fi fi sleep "$interval" elapsed=$((elapsed + interval)) done echo " ✗ Server health check failed after ${timeout}s" + if $want_store; then + echo " The backup store component did not report healthy; last: $(curl -sk "${endpoint}/health" 2>/dev/null)" + fi echo " Is the server running? Check: curl -sk ${endpoint}/health" return 1 } @@ -467,8 +522,15 @@ if $RUN_PYTEST; then echo " endpoint: $EXTENDDB_TEST_ENDPOINT" fi echo " AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID:-}" - # CLI lifecycle tests manage their own server; exclude from main suite - PYTEST_ARGS=(python3 -m pytest tests/ -v --ignore=tests/python --ignore=tests/test_cli_lifecycle.py --ignore=tests/test_cli_container_readiness.py --ignore=tests/test_cli_migrate_concurrency.py --ignore=tests/test_cli_vector_catalog_migration.py --ignore=tests/test_gsi_async_queue.py) + # CLI lifecycle tests manage their own server; exclude from main suite. + # tests/test_backup_store_config.py does too: it runs below, in the CLI + # lifecycle list on PostgreSQL and in its own step on a SQLite build. + # tests/test_backup_job.py starts its own servers as well; CI runs it in + # the integration workflows' backup steps, this script does not run it. + # -rP: the summary repeats the passed tests that printed something (the + # backup cells print which branch ran and which store the server reports), + # which pytest's capture would otherwise keep out of the log. + PYTEST_ARGS=(python3 -m pytest tests/ -v -rP --ignore=tests/python --ignore=tests/test_cli_lifecycle.py --ignore=tests/test_cli_container_readiness.py --ignore=tests/test_cli_migrate_concurrency.py --ignore=tests/test_cli_vector_catalog_migration.py --ignore=tests/test_gsi_async_queue.py --ignore=tests/test_backup_store_config.py --ignore=tests/test_backup_job.py) # Parallel execution (--parallel): distribute by file so module/class-scoped # fixtures stay within a single worker. if [[ -n "$PARALLEL" ]] && python3 -c "import xdist" 2>/dev/null; then @@ -560,7 +622,7 @@ if $RUN_PYTEST \ && -n "${EXTENDDB_TEST_PG_CONNECTION_STRING:-}" ]]; then CLI_OUTFILE="discussions/test-cli-${HASH}.txt" echo "=== CLI lifecycle tests → ${CLI_OUTFILE} ===" - CLI_ARGS=(python3 -m pytest tests/test_cli_lifecycle.py tests/test_cli_container_readiness.py tests/test_cli_migrate_concurrency.py tests/test_cli_vector_catalog_migration.py -v) + CLI_ARGS=(python3 -m pytest tests/test_cli_lifecycle.py tests/test_cli_container_readiness.py tests/test_cli_migrate_concurrency.py tests/test_cli_vector_catalog_migration.py tests/test_backup_store_config.py -v) if [[ -n "$FILTER" ]]; then CLI_ARGS+=(-k "$FILTER") fi @@ -573,6 +635,37 @@ if $RUN_PYTEST \ echo "" fi +# --- Backup store wire tests on a SQLite build (run last — they start/stop their own servers) --- +# tests/test_backup_store_config.py's SQLite cells: the default root, the +# S3 store's health against moto (skipped with a reason unless +# EXTENDDB_TEST_S3_ENDPOINT names a moto endpoint), `backup list` and +# `backup purge` with their signals and exit codes. Its PostgreSQL cells run +# in the CLI lifecycle list above. The build decides: the step runs when the +# binary reports itself a SQLite build. +if $RUN_PYTEST && [[ "$TARGET" != "real-dynamodb" ]]; then + WIRE_BINARY="${EXTENDDB_BINARY:-target/release/$TARGET}" + if [[ ! -x "$WIRE_BINARY" ]]; then + WIRE_BINARY="target/debug/$TARGET" + fi + if [[ -x "$WIRE_BINARY" ]] && "$WIRE_BINARY" version 2>/dev/null | grep -q '(sqlite)'; then + WIRE_OUTFILE="discussions/test-backup-store-wire-${HASH}.txt" + echo "=== Backup store wire tests (SQLite build) → ${WIRE_OUTFILE} ===" + echo " binary: $WIRE_BINARY" + echo " S3 endpoint: ${EXTENDDB_TEST_S3_ENDPOINT:-none (the S3 cells skip)}" + WIRE_ARGS=(python3 -m pytest tests/test_backup_store_config.py -v) + if [[ -n "$FILTER" ]]; then + WIRE_ARGS+=(-k "$FILTER") + fi + if EXTENDDB_BINARY="$WIRE_BINARY" "${WIRE_ARGS[@]}" 2>&1 | tee "$WIRE_OUTFILE"; then + echo " ✓ Backup store wire tests passed" + else + echo " ✗ Backup store wire tests failed" + PASS=false + fi + echo "" + fi +fi + # --- Catalog check --- if $RUN_CATALOG_CHECK; then if [[ "$TARGET" == "real-dynamodb" ]]; then diff --git a/docker-compose.cassandra.yml b/docker-compose.cassandra.yml index 1ce90a03c..d71128933 100644 --- a/docker-compose.cassandra.yml +++ b/docker-compose.cassandra.yml @@ -253,7 +253,7 @@ services: - "127.0.0.1:${EXTENDDB_PORT:-18443}:18443" volumes: - extenddb-state:/var/lib/extenddb - stop_grace_period: 15s + stop_grace_period: 80s restart: unless-stopped volumes: diff --git a/docker-compose.mongodb.yml b/docker-compose.mongodb.yml index d9514a6bb..baff12ba0 100644 --- a/docker-compose.mongodb.yml +++ b/docker-compose.mongodb.yml @@ -133,7 +133,7 @@ services: - "127.0.0.1:${EXTENDDB_PORT:-18443}:18443" volumes: - extenddb-state:/var/lib/extenddb - stop_grace_period: 15s + stop_grace_period: 80s restart: unless-stopped volumes: diff --git a/docker-compose.yml b/docker-compose.yml index b4d5a2060..6b00a1f6a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -99,7 +99,7 @@ services: - "127.0.0.1:${EXTENDDB_PORT:-18443}:18443" volumes: - extenddb-state:/var/lib/extenddb - stop_grace_period: 15s + stop_grace_period: 80s restart: unless-stopped volumes: diff --git a/docs/design/02-high-level-design.md b/docs/design/02-high-level-design.md index af78ee6bf..9ee5e0a68 100755 --- a/docs/design/02-high-level-design.md +++ b/docs/design/02-high-level-design.md @@ -376,12 +376,15 @@ Total connection footprint on the PostgreSQL server: pool because its ownership lock is session-scoped - one connection while a schema migration runs, pinned outside the pools so its transaction-scoped lock cannot move between backends - -So the total is `2 * pool_size + catalog_pool_size`, plus builds and migrations. -There is no separate poller pool: the log-level poller borrows from the management -pool like every other background poller. `max_connections` is per cluster rather -than per database, so both `pool_size` pools draw on the same budget: the defaults -need 60 of PostgreSQL's default 100. +- one connection per running backup or restore job, at most `[backup] + max_concurrent_jobs` (default 4), opened outside every pool because the job's + claim is a session-level advisory lock + +So the total is `2 * pool_size + catalog_pool_size + max_concurrent_jobs`, plus +builds and migrations. There is no separate poller pool: the log-level poller +borrows from the management pool like every other background poller. +`max_connections` is per cluster rather than per database, so both `pool_size` +pools draw on the same budget: the defaults need 64 of PostgreSQL's default 100. ### 6.3 Row-Level Locking diff --git a/docs/design/13-storage-mongodb.md b/docs/design/13-storage-mongodb.md index b1219ccbc..cc4f74af3 100644 --- a/docs/design/13-storage-mongodb.md +++ b/docs/design/13-storage-mongodb.md @@ -837,6 +837,13 @@ opens the engine, and refuses a catalog with pending migrations or with a breaking migration this binary does not know, as on every backend. [ADR-0023](../adr/0023-catalog-migrations.md) records the design. +The factory registers and reads the status before it builds the engine +or starts the startup key repair, so a server that refuses has written +nothing but its registration, which it deletes. A command that opens +the catalog beside a server (`ServerComponentsOptions::beside_server`: +`extenddb backup list`, the purge's dry run) starts no repair; the +server does. + ## 6. Crate Structure ``` diff --git a/docs/design/15-backup-restore-bench-pr8.md b/docs/design/15-backup-restore-bench-pr8.md new file mode 100644 index 000000000..51a3ccaa0 --- /dev/null +++ b/docs/design/15-backup-restore-bench-pr8.md @@ -0,0 +1,481 @@ +# PR 8 memory and duration record (`bench-rss.sh`, 2026-10-31 and 2026-11-01) + +This is the record the design cites (section 7, revision item 31 of +`15-backup-restore.md`). `kill-points.py`'s API half is committed as +`devtools/backup-kill-points.py`, with the kill-point driver +`devtools/backup-kill-points.sh`; `bench-rss.sh` and the raw samples +(`/proc//status` every 200 ms per run, the servers' logs) are kept +beside the PR outside the repository, on the measuring host; +the figures below are what they produced. The runs of 2026-11-01, after the +PostgreSQL page's row bound changed, are in the section at the end and in the +sentences they correct; the runs of the same day after the row bound gained +its memory are in the last section. + +What was measured: the release binary (`cargo build --release`, the default +PostgreSQL features; `--no-default-features --features sqlite`; `--features +mongodb-test-hooks`) running `serve --foreground` with a filesystem backup +store (SQLite: the defaulted root beside the database file), sampled from +`/proc//status` every 200 ms (`VmRSS` and `VmHWM`). Each table is loaded +through the wire (`kill-points.py create-table`: BatchWriteItem from 16 +threads, items of a 200-byte `S` pad under a composite key with two GSIs), +then the server is restarted so the process's high-water mark covers the +backup alone; the idle figure is read right before `CreateBackup`, the peak +is `VmHWM` once `DescribeBackup` reports `AVAILABLE`. Samples: +`rss//{host.txt,results.txt,.json,.samples}` with `` the +backend for the record's runs and `sqlite-itemchannel1`, +`sqlite-itemchannel-swapped`, `sqlite-arena1`, `sqlite-worst24`, +`sqlite-rowbuffer4` for the comparison runs below; the series of 2026-11-01 +under `sqlite-worst-{default,arena1}/` and `postgres/`. + +Host (`host.txt`): 16 cores, 30.3 GiB, Linux 5.10.270 (amzn2int); PostgreSQL +16.15 in a container on loopback (55432); the SQLite the binary bundles +(libsqlite3-sys 0.30.1, SQLite 3.46; the host's 3.7.17 is unused); MongoDB +7.0.43 single-node replica set in a container (27117). What was not held +constant, stated before any figure here is read across runs: the host ran +other work during every run (the PostgreSQL runs +beside two 1M-item loads and a wire suite; the SQLite and MongoDB runs beside +the Cassandra integration suite and the kill-point runs), so the durations +are upper bounds and comparable only within one back-to-back pair; the peaks +are the process's own and do not depend on the host's load. Everything is on +loopback: nothing here says what a network database or a remote S3 store does +to the duration; the upload window of an S3 store (`upload_part_size_bytes * +max_concurrent_uploads`) is not in these figures, which use the filesystem +store's one write buffer. The MongoDB binary was built before the pipeline's +last change (the sink's `BrokenPipe` answer after a stop, which touches no +allocation); the PostgreSQL and SQLite figures of the record are from binaries +of the PR's first build (`ITEM_CHANNEL = 4`); the PostgreSQL page changed +after these runs (the row bound from the largest row, the rows parsed as they +arrive), so the PostgreSQL worst-shape row below is the first build's and the +section of 2026-11-01 carries what changed. + +## The bound the acceptance asserts: peak(1M) − peak(100k) < 112 MiB + +The bound the script asserts is 64 + 32 + 16 MiB (the page budget, the line +cap, the channels). MongoDB's "1M" run is 500k: its 100k load took 361 s +and the 500k load 1,604 s through the wire on this host, so a 1M load would +have run for close to an hour; the 500k run is recorded in its place and the +difference is against it. + +| backend | run | items | idle RSS | peak (VmHWM) | peak − idle | AVAILABLE after | backup size | +|---|---|---|---|---|---|---|---| +| PostgreSQL | 100k | 100,000 | 24.6 MiB | 35.9 MiB | 11.3 MiB | 1.85 s | 847,689 B | +| PostgreSQL | 1M | 1,000,000 | 24.8 MiB | 45.4 MiB | 20.6 MiB | 36.1 s | 6,452,803 B | +| PostgreSQL | 1M − 100k | | | **+9.5 MiB** | | | PASS (bound 112) | +| SQLite | 100k | 100,000 | 20.4 MiB | 65.4 MiB | 45.0 MiB | 1.44 s | 847,689 B | +| SQLite | 1M | 1,000,000 | 22.8 MiB | 73.3 MiB | 50.5 MiB | 12.45 s | 6,452,803 B | +| SQLite | 1M − 100k | | | **+7.9 MiB** | | | PASS (bound 112) | +| MongoDB | 100k | 100,000 | 26.8 MiB | 36.4 MiB | 9.6 MiB | 1.85 s | 962,185 B | +| MongoDB | 500k (in the 1M row's place) | 500,000 | 25.7 MiB | 41.2 MiB | 15.5 MiB | 8.81 s | 4,869,108 B | +| MongoDB | 500k − 100k | | | **+4.8 MiB** | | | PASS (bound 112) | + +The page budget is never reached by 200-byte rows (a page is `n` rows, at +most `DEFAULT_FETCH_BATCH` = 1000, about 250 KB of text), and the data files +roll at 64 MiB of uncompressed text without the file being held (the encoder +streams 1 MiB chunks into the put), so the peak over idle is a few tens of +MiB on every backend and moves by under 10 MiB from 100k to 1M rows. SQLite's +45 MiB over idle at 100k is the backend's own: the pool's connections each +hold SQLite's page cache, which a backup's reads fill; it is bounded by the +pool and does not grow with the table (the 1M run adds 7.9 MiB to it); the +constant is `PRAGMA cache_size = -32000` +(`crates/storage-sqlite/src/sqlite_util.rs`), 31 MiB per connection, so the +pool's bound is 10 × 31 MiB at the default pool and 34 × 31 MiB at the raised +one, and the 45 MiB is about one and a half connections' caches filled, not +the pool's. The duration scales +with the items: 36 s for 1M on PostgreSQL under a loaded host (the server's +log line says 36.1 s of job time; 1.9 s for 100k), 12.5 s on SQLite, 8.8 s +for 500k on MongoDB. + +Rows read past the cut on PostgreSQL (the `test-hooks` counter +`paging::rows_past_cut`, `crates/storage-postgres/tests/table_snapshot.rs::pages_are_cut_at_the_byte_budget`, +run with `--nocapture`): on the first build **602 rows over 77 pages of 8 +rows** with `n` = 16 for every page (the row bound from the average row, whose +floor was 16; 8 per page where the budget cut each page in half), on both +layouts; after the paging change **8 rows over 77 pages**, all on the first page (the +row bound is then the budget divided by the largest row, 8, so every later +page reads exactly the rows it keeps), asserted exactly. The SQLite stream +pulls no row past its cut (its `rows_past_cut` is 0 and asserted so by +`snapshot_engine.rs`'s test); the driver's worker reads ahead of it, which +the worst-expansion runs below measure. + +## The worst-expansion run + +`put-worst`: 1,000 small rows (`a0000`.., a 50-byte pad) and then 12 items +of #394's worst-expansion shape (`zz000`..: a list of 2,262 `NS` sets of the +90 values `1.0E-130`..`9.9E-130`; 409,429 bytes as the service sizes an +item; 27.7 MB as a data-file line and as stored item text, every number +normalized to its 133-character decimal form; about 34 MB decoded, 203,580 +`String`s of 133 bytes with their headers). The page the cut gives on such +rows is three rows (the rows whose predecessors' bytes are under 64 MiB: 0, +27.7, 55.4 MB), 83 MB of text, so 12 rows are four pages. MongoDB stores an +item as one BSON document and refuses the shape at 28.9 MB (`BSONObjectTooLarge`, +the 16 MiB cap; `rss-mongodb-worst.out`'s first attempt), so its run holds +12 items of 1,100 sets (13.5 MB lines, `--sets 1100`): there every row is a +page of its own (the server's 16 MiB reply bound) and the shape is half the +size of the other two backends'. + +| backend | build | worst rows | idle RSS | peak (VmHWM) | peak − idle | AVAILABLE after | backup size | +|---|---|---|---|---|---|---|---| +| PostgreSQL | `ITEM_CHANNEL = 4` | 12 | 24.5 MiB | **904.5 MiB** | 880 MiB | 18.2 s | 3,793,555 B | +| SQLite | `ITEM_CHANNEL = 4`, the 15:34 build (before the sink's `BrokenPipe` answer) | 12 | 21.1 MiB | **782.7 MiB** | 762 MiB | 8.16 s | 3,793,555 B | +| SQLite | `ITEM_CHANNEL = 4`, the first build (with the `BrokenPipe` answer) | 12 | 20.7 MiB | 602.6 MiB | 582 MiB | 8.97 s | 3,793,555 B | +| SQLite | `ITEM_CHANNEL = 1` | 12 | 20.9 MiB | 555.4 MiB | 535 MiB | 8.13 s | 3,793,555 B | +| SQLite | `ITEM_CHANNEL = 1`, back to back with the row above it | 12 | 20.4 MiB | 552.2 MiB | 532 MiB | 7.95 s | 3,793,555 B | +| SQLite | `ITEM_CHANNEL = 4`, `MALLOC_ARENA_MAX=1` | 12 | 22.0 MiB | 529.5 MiB | 507 MiB | 8.76 s | 3,793,555 B | +| SQLite | `ITEM_CHANNEL = 4` | **24** | 20.3 MiB | **919.4 MiB** | 899 MiB | 15.7 s | 7,567,821 B | +| MongoDB | `ITEM_CHANNEL = 4`, 1,100 sets | 12 | 25.0 MiB | 498.7 MiB | 474 MiB | 4.7 s | 1,851,153 B | + +The driver's run-ahead on SQLite, measured back to back on one 24-row table +(`rss/sqlite-rowbuffer4/results.txt`; the final tree sets `row_buffer_size` +to four on the pool, `crates/storage-sqlite/src/store.rs`): + +| SQLite build | worst rows | idle RSS | peak (VmHWM) | peak − idle | AVAILABLE after | +|---|---|---|---|---|---| +| `row_buffer_size` 4 (final tree) | 24 | 20.2 MiB | 716.7 MiB | 696 MiB | 15.9 s | +| `row_buffer_size` 50 (sqlx's default) | 24 | 22.5 MiB | 866.2 MiB | 844 MiB | 15.7 s | +| `row_buffer_size` 4, the 1M-row table | 1,000,000 small | 22.3 MiB | 67.3 MiB | 45.0 MiB | 11.43 s | +| `row_buffer_size` 50, the 1M-row table | 1,000,000 small | 21.1 MiB | 67.6 MiB | 46.5 MiB | 11.42 s | + +The samples show the shape of it: with 50 the RSS reaches 850 MiB within a +second of the start (the worker has read every one of the 24 rows while the +stream parses the first page's three), falls to 280 MiB when the first page's +cut drops the statement, and climbs again on the second; with four it climbs +with the pipeline (236, 292, 339, 424 MiB and on) to its peak. A 1M-row +backup takes the same 11.4 s with either, so the pool keeps four. + +The terms, by the pipeline's parts, for a row of `r` = 27.7 MB of text whose +decoded `Item` is `d` ≈ 34 MB: + +- the page's text: on the first build PostgreSQL fetched the page whole + (`fetch_all`, 83 MB as the driver's rows) and copied each row's text out + as it extracted it, about one page plus one row; after the paging change it pulls + the rows from the connection one at a time and parses each as it arrives, + so one row, as SQLite, which pulls the rows one at a time; +- the page's decoded rows (up to 3 × `d` = 102 MB) in the stream's buffer + while the pipeline takes them one by one; +- one row's parse: the text, the deserializer's intermediate value and the + item, `r + 2d` ≈ 96 MB, for the row being parsed; +- the pipeline's items: the feeder's, the item channel's (`ITEM_CHANNEL`) + and the encoder's, (2 + `ITEM_CHANNEL`) × `d`: 204 MB at four, 102 MB at + one; +- the encoder's line buffer (one line, `r`), its gzip window, the chunk + sink's 1 MiB and the chunk channel's four 1 MiB chunks; +- on SQLite, the driver's run-ahead (below). + +At the instant the stream fetches a new page while the pipeline is full the +terms sum to about 400 MB with a four-item channel and about 300 MB with a +one-item channel. The high-water marks are 1.4 to 2.2 times that, and the +difference is the allocator's: the row's 203,580 short strings are allocated +on the tokio worker that parses the row and freed on the blocking thread +that encodes it, and glibc's malloc keeps that memory in its arenas between +pages. Evidence: two builds of the same pipeline code (the table's two 12-row +rows, the 15:34 build and the first build) peaked at 762 and 582 MiB over +idle on one shape, two runs of one first-build binary at 12 rows peaked at +574 and 805 MiB (the terms do not vary; the allocator's state does), and +`MALLOC_ARENA_MAX=1` took the same run to 507 MiB. The 24-row run on SQLite +(899 MiB over idle against 582 to 762 for 12 rows) adds one term the 12-row +runs hid: sqlx's SQLite driver runs each statement on a worker thread that +buffers rows ahead of the reading task (`row_buffer_size`, 50 by default), +so while the stream parses its page's three rows the worker has already read +the rows after them, up to 50 or the table's end, whichever comes first (12 +rows in the first runs, 24 in this one, 50 × the line cap, 1.6 GB, on a +large table of such rows); the page's cut drops the statement and frees them. +`rows_past_cut` is 0 because it counts the rows the stream pulls, not the +rows the driver read. The final tree sets the +pool's `row_buffer_size` to four (the table above): the run-ahead is then at +most five rows (the four the channel holds and the one the worker has stepped +and copied while its `send` waits; 160 MiB at the line cap) and the SQLite +peak is the pipeline's terms and the allocator's share. + +The item channel. The proposal was a one-item channel to save the +three items the four-item channel holds. Measured back to back on SQLite on +one loaded 1M-row table, each build once in each order +(`rss/sqlite-itemchannel1/results.txt`, `rss/sqlite-itemchannel-swapped/results.txt`): + +| order | `ITEM_CHANNEL = 4` | `ITEM_CHANNEL = 1` | +|---|---|---| +| one-item build first (18:22) | 12.45 s | 21.85 s | +| four-item build first (18:33) | 12.24 s | 21.41 s | + +A one-item channel costs a 1M-row backup 75% of its duration (every item is +a hand-off between the feeder task and the encoder's blocking thread) and +saves about 50 MiB of the worst-shape peak (582 against 532 MiB over idle, +back to back), so `ITEM_CHANNEL` stays at four and `writer.rs` says why. + +The terms do not grow with the number of small rows (the 1M run) and grow +with the row (every term but the budget and the chunks is a multiple of `r` +or `d`), which the line cap (32 MiB) and `max_item_size_bytes` bound; the +process's peak, with glibc's default arenas, grows with the number of large +rows a job reads, and does not under `MALLOC_ARENA_MAX=1` (the series of +2026-11-01 below). Section 7 of the design states these terms with the figures; +revision item 31 carries the correction (the earlier text, "the budget plus +one row of text, plus its decoded form", named the first term alone). + +## The incompressible run + +`put-random`: 256 items each holding a 256 KiB `B` attribute of +`os.urandom` bytes (64 MiB in all; 85 MB as base64 text). + +| backend | idle RSS | peak (VmHWM) | peak − idle | AVAILABLE after | backup size | +|---|---|---|---|---|---| +| PostgreSQL | 24.6 MiB | 145.1 MiB | 120.5 MiB | 5.9 s | 68,016,482 B | +| SQLite | 22.7 MiB | 97.5 MiB | 74.8 MiB | 4.3 s | 68,016,702 B | +| MongoDB | 25.6 MiB | 115.8 MiB | 90.2 MiB | 4.9 s | 68,016,522 B | + +The backup's size (68 MB) is the data's: gzip shrank nothing. The peak over +idle is one page of text (the 85 MB fits one page: 256 rows of 342 KB, which +PostgreSQL holds whole and SQLite a row at a time) plus the decoded items +(the 64 MiB of bytes) plus the channels; the encoder did not hold the file +(a 68 MB file through a 4 MiB chunk window), which is what this run proves. + +## Durations for the record (the design's RTO baseline, PR 11) + +CreateBackup to AVAILABLE, the `DescribeBackup` poll's figure, a composite +key with two GSIs, 200-byte items, a filesystem store on the same disk, the +host loaded as said above: + +| backend | 100k items | 1M items | 500k items | +|---|---|---|---| +| PostgreSQL 16.15 (loopback, separate data and catalog databases) | 1.85 s | 36.1 s | | +| SQLite 3.46 (file) | 1.44 s | 12.45 s (12.24 and 12.45 s in the comparison runs, no GSIs) | | +| MongoDB 7.0 (loopback replica set) | 1.85 s | not run | 8.81 s | + +The worst shape (12 rows of 27.7 MB, 332 MB of text): 18.2 s on PostgreSQL +on the first build and 16.5 s after the paging change (the section of 2026-11-01), +8.0 to 9.0 s on SQLite; the incompressible 64 MiB: 5.9, 4.3 and 4.9 s. The +loads themselves (the wire, 16 threads of BatchWriteItem): 100k in 217 s on +SQLite with two GSIs and 1M in 6,531 s; 1M in 463 s without GSIs; MongoDB +100k in 361 s and 500k in 1,604 s. + +## 2026-11-01: the allocator series and the PostgreSQL page's cost + +The worst shape (1,000 small rows then N rows of 27.7 MB) in two series of +six runs, each series on one SQLite release build, each run a fresh +deployment, `bench-rss.sh sqlite worst` with `WORST_COUNT` 12, 24, 48, first +with glibc's default arenas and then with `MALLOC_ARENA_MAX=1` in the +server's environment. Series 1: the build before the feeder's line +measurement was added (it measures no line at the default item limit, so the +pipeline's allocation is the same), a quiet host, samples under +`rss-series1/rss/`. Series 2: the final build, with the kill-point runs of the +same day running beside it on the host, samples under +`rss/sqlite-worst-{default,arena1}/`; each binary's md5 is recorded beside +its samples: + +| allocator | worst rows | series 1, quiet host: idle / peak / over idle, AVAILABLE after | series 2, final build, loaded host: idle / peak / over idle, AVAILABLE after | +|---|---|---|---| +| default arenas | 12 | 24.0 / 489.6 / **466 MiB**, 8.74 s | 22.7 / 729.3 / **707 MiB**, 9.15 s | +| default arenas | 24 | 23.7 / 611.4 / **588 MiB**, 18.07 s | 20.6 / 720.4 / **700 MiB**, 17.89 s | +| default arenas | 48 | 22.4 / 782.1 / **760 MiB**, 36.56 s | 23.2 / 1304.5 / **1,281 MiB**, 35.96 s | +| `MALLOC_ARENA_MAX=1` | 12 | 22.1 / 503.3 / **481 MiB**, 9.57 s | 24.0 / 476.2 / **452 MiB**, 9.99 s | +| `MALLOC_ARENA_MAX=1` | 24 | 23.5 / 475.6 / **452 MiB**, 19.55 s | 20.6 / 500.8 / **480 MiB**, 19.32 s | +| `MALLOC_ARENA_MAX=1` | 48 | 22.0 / 501.5 / **480 MiB**, 39.45 s | 22.2 / 477.6 / **455 MiB**, 39.67 s | + +The terms of the section above at this row (`r` = 26.4 MiB, `d` ≈ 32.4 MiB: +the page's three decoded rows 97 MiB, one row's parse 91 MiB, the pipeline's +six items 195 MiB, the line buffer 26 MiB, the chunks 5 MiB, the five-row +run-ahead 132 MiB) sum to 545 MiB on SQLite. With the default arenas the peak +over idle is 0.86 to 1.39 of that sum in the first series and 1.28 to 2.35 in the +second, grows with the rows in both, and differs by 240 to 520 MiB between +the two series at the same count (runs of the first build on a loaded host +reached 1,096 MiB at 48 rows, and its 12-row runs of one binary spread over +574 to 805 MiB); under one arena it is 0.83 to 0.88 of the sum in both series and +flat from 12 to 48 rows. So the pipeline's terms do not grow with the table, +the growth and the spread under the default arenas are the allocator's +retention of the strings freed on the encoder's thread (how much it keeps +depends on the host's state at the time, which is why the host's load moves +it), and the bound the design states (section 7) is the terms at the line +cap times 1.2 per job under `MALLOC_ARENA_MAX=1`, which the admin guide and +the deployment guide's service unit now carry; with the default arenas the +design records the growth and names 2.35 of the terms at 48 rows as the +largest measured. The duration grows with the rows as it should (the 48-row +table is 1.3 GB of text) and is the same under either setting. + +The PostgreSQL page (the probe `pg_adapt_probe.py`, kept with the scripts, +run twice with the first build's and the changed row-bound rule on a +1,000-small-then-60-worst table, `EXPLAIN (ANALYZE, BUFFERS)` of page 3): + +| row bound rule | page 2 (`n` = 1000, reads 984 small + 16 worst, keeps 987) | page 3: `n` asked | rows read / past the cut | temp written (8 KiB blocks) | page 3 time | pages 4, 5 (`n`, time) | +|---|---|---|---|---|---|---| +| the first build: budget / average row, floor 16 | 4.6 s | 810 | 57 / 54 | 757,844 (5.9 GB) | 18.2 s (43.7 s on a loaded host) | 16, 4.4 s each (13 rows past the cut) | +| the change: budget / largest row, floor 4 | 14.0 s | 4 | 4 / 1 | 26,564 (208 MiB) | 1.13 s | 4, 1.2 s each (1 row past the cut) | + +Page 2 is the over-read of this shape (it asks for the rows the small rows +justified and meets the large ones: up to `n` rows of up to the line cap); +its two timings differ by the host, not the rule, since both runs issue the +same query. On a table whose large rows are scattered among small ones that +page is not the only one: with the bound sized from the previous page alone +it recurs every other page, which the last section measures and the row +bound's memory corrects. The PostgreSQL release build with the changed page under +`bench-rss.sh postgres 100k 1m worst` with `GSIS=0` (one deployment, the +runs in that order, samples under `postgres/`): 100k, idle +23.7 MiB, peak 30.4 MiB, 1.87 s; 1M, idle 22.6 MiB, peak 34.4 MiB, 13.27 s +(peak(1M) − peak(100k) = +4.0 MiB, bound 112, PASS; the record's 36.1 s was +under two 1M loads and a wire suite); the worst shape at 12 rows, peak +790.2 MiB (`VmHWM`) after 16.5 s, against the first build's 904.5 MiB and 18.2 s, +with the page-text term gone and each large-row page reading one row past +its cut rather than thirteen (the idle before this run, 122.7 MiB, is the +restarted process's startup work on a table of 27.7 MB rows, not the 1M +load: on a fresh deployment holding only the worst table the restarted +process idled at 138 and 164 MiB in two later runs, climbing from 79 to +168 MiB within two seconds of the listener answering and before +`CreateBackup`, so the over-idle figure, 667 MiB, is not comparable with +the record's 880; the `VmHWM` is). The backup sizes of the `GSIS=0` tables (366,631 and 3,017,864 +bytes) are smaller than the record's because the items carry no GSI +attributes; the backup reads the base table either way. The plan near the end of a table is the planner's: on +the 100k-row test table the index scan holds ten rows from the end, and on a +1,060-row table a page after the third of the last 60 at `n` = 810 is a +bitmap scan of the primary key index and a `Sort` of the 57 rows after the +seek (PostgreSQL 16; `the_last_page_sorts_the_rows_after_the_seek` pins +both). + +## 2026-11-01, later: the row bound's memory, the server-side walk that was measured and not taken, the allocator on PostgreSQL + +The shape the sections above do not cover is a table whose large rows are +scattered among small ones, which is what a table whose large items are a +minority looks like (key order does not sort items by size; `put-worst` +groups its `zz` keys after the `a` keys). With the row bound sized from the +previous page alone, a page that holds a 27 MB row sets the next bound to 4, +four small rows set the one after back to 1000, and that page meets large +rows again: the cycle repeats every other page for the whole table. + +The SQL probe (`pg_memory.py`, kept with the scripts; the window-form page +query as `page_sql` writes it, a complete walk of a 1,000-row table with 30 +rows of 27 MB text at every 33rd key, PostgreSQL 16.15, `work_mem` 4 MB, +one plain run per rule, the server's `temp_bytes` for that run): + +| row bound rule | pages | rows the inner `LIMIT` read for 1,000 kept | `temp_bytes` | wall | +|---|---|---|---|---| +| the largest row of the previous page | 22 (`n` 16, 1000, 4, 1000, 4, …) | 5,555 | 13.38 GB | 71.5 s | +| the largest row of the last four pages | 50 (16, 1000, 4, 4, 4, 4, 1000, …) | 5,555 | 13.38 GB | 64.6 s | +| the largest row of the last pages holding at least 1,000 rows | 228 (16, 1000, then 4 to the end) | 1,901 | 3.89 GB | 22.4 s | + +Four pages of memory read the same rows as one: the four pages after a +large row hold 16 small rows, so the bound returns to 1000 as before and +the 1000-row page recurs every fifth page instead of every other. A memory +in rows holds the bound at 4 for a row bound's worth of rows after the last +large one, so on this table it never returns to 1000, and the one over-read +left is the first page that meets the large rows (984 rows read for 83 +kept). That is the rule the tree carries (`RowBoundMemory`, +`snapshot_engine.rs`): the bound that remains is the second page, and then +the page that follows a row bound's worth of small rows, each reading up to +`n` rows of up to the line cap, and such a page recurs at most once per row +bound's worth of small rows (on a table whose large rows are rarer than one +per thousand keys, at every large row). `the_row_bound_remembers_large_rows_among_small_ones` +pins the figure on a 300-row table with nine such rows: 201 rows read past +the cut, all on the second page, over 53 round trips (306 over 7 with one +page of memory; 103 round trips with a floor of 2). + +The release binary (`pg-interleave.sh`, kept with the scripts; one +deployment with a filesystem store, two tables of 20,000 small rows loaded +through the wire with 30 worst-expansion items, at keys `a0032`, `a0065`, +… `a0989` in one and `zz000`.. in the other, the server restarted before +each backup, the data database's `temp_bytes` during it, both binaries on +the same tables back to back; host load 3.6 to 8.6): + +| table | row bound rule (binary) | AVAILABLE after | data database `temp_bytes` | server `VmHWM` | +|---|---|---|---|---| +| interleaved | the previous page (`80f2d287`) | 72.0 s | 9.21 GB | 905 MiB | +| interleaved | the memory (`e020f82e`, the tree) | 54.6 s | 3.18 GB | 900 MiB | +| grouped | the previous page | 39.8 s | 2.85 GB | 1,143 MiB | +| grouped | the memory | 38.9 s | 2.85 GB | 1,066 MiB | + +The interleaved table's remaining 16 s and 3.2 GB over the grouped one are +the second page, whose `LIMIT 1000` reads keys 17 to 1,016, which hold all +30 large rows (3 kept, 27 past the cut), for the 83 it keeps; the grouped +table is unchanged (its one 1000-row page meets small rows). A second pair on the same host +earlier that hour: 77.4 s and 9.21 GB for the previous-page rule on the +interleaved table, 38.9 s and 2.85 GB grouped. The `VmHWM` figures are +under glibc's default arenas and vary by about 200 MiB between runs; they +are reported, not rested on. + +Measured and not taken: a page query that stops at the budget on the server, +a recursive walk of the primary key one index probe per row +(`WITH RECURSIVE … LATERAL … ORDER BY LIMIT 1`, the running total +carried from step to step, the walk ending after the row that reaches the +budget or at `n` steps), so the server serializes exactly the rows the page +returns and reads none past the cut. On the same two tables and binaries it +took the interleaved table in 38.8 s with 2.51 GB of temp and the grouped +one in 38.0 s with 2.51 GB (the three rows of each large-row page spilled +three times: the work table, the CTE's materialization, the sort), and the +first-page and middle-page plans on the six key shapes were the two index +probes with no other scan of the relation. Its cost is one index probe per +row, which a table of small rows pays on every row: the 1M-row backup +(`bench-rss.sh postgres 100k 1m worst`, `GSIS=0`, the two binaries back to +back on one loaded table) took 23.07 s against 11.85 s for the window form +(+95%; the 100k run 1.64 s; the worst shape at 12 rows 13.4 s and 658 MiB +`VmHWM` against 17.2 s and 553 MiB). The rule for the choice was a 1M-row +duration under 21 s (within 60% of 13 s), so the window form with the +memory stays; the walk's query, tests and figures are kept with the scripts +(`option1/`). A walk in chunks of a few rows per step would bound the +over-read at the chunk less one and cost a probe per chunk; it was not +measured. + +The 1M-row backup with the memory rule, the same back-to-back method: 13.69 s +against 12.06 s for the previous-page rule (`VmHWM` 36.7 against 36.3 MiB; +peak(1M) − peak(100k) = +4.1 MiB, bound 112, PASS); the rule changes no +page of a table with no large row (every page is the row bound either way), +and the tree's binary carries that day's other changes to the job's +validator and feeder as well, so the difference is the host's and theirs, +not the page's. + +What the memory costs where the previous-page rule had no over-read: after +a page holding a row at the line cap the bound stays at the floor for a row +bound's worth of rows, 250 pages of four, whatever those rows hold, so a +table whose large rows are rarer than one per row bound pays 250 round +trips per large row where one page of memory pages it at the row bound (a +1000-row page holding one 27 MB row and 999 small ones is under the budget +and kept whole). Measured on a 20,000-row table with a 27 MB row at every +1,100th key (18 large rows, loaded through the wire with `kill-points.py +put-worst --small 20000 --count 18 --interleave 1100`; the PostgreSQL +release build against a one-line build with one page of memory, the data +table's `idx_scan` counting the page queries; `pg-rare.sh`, kept with the +scripts): 2,585 page queries against 38, 25.6 and 24.9 s in two runs +against 25.1 s, 1.06 against 1.09 GB of temp, the same 5,821,368-byte +backup; the SQL mirror (`pg_probe.py`) walks the same shape in 2,508 pages +against 38 with 0 rows past the cut under either rule. On loopback the +extra round trips cost about 0.3 ms each and disappear under the large rows' +own serialization (about 1.3 s per 27 MB row); on a 1M-row table with 900 +such rows they are 125,000 round trips, about 40 s on loopback and about two +minutes on a 1 ms link, against the 20 minutes the large rows themselves +take. A bound that doubles after each page without a large row (4, 8, 16, +…) would reach the row bound after eight small pages instead of 250 with +the same rows read past the cut on the three shapes the mirror walked; it +was not taken, the cost above being bounded and the rule stated. +The worst shape at 12 rows under the default arenas: 787.8 MiB `VmHWM` in +17.7 s against 750.1 MiB in 17.5 s (the idle before the first of the two, +241.6 MiB, is the restarted process's startup work on the table of 27.7 MB +rows, the caveat of the previous section). + +The allocator on PostgreSQL (`MALLOC_ARENA_MAX=1 bench-rss.sh postgres +worst`, `GSIS=0`, the tree's binary, a fresh deployment): the worst shape +at 12 rows peaked at **339.9 MiB `VmHWM`** in 18.7 s (idle 144.2 MiB, the +startup work above; another measurement of the same shape on a build of the +same day: 346.3 MiB, 208 MiB over an idle of 138.0 MiB, 22.4 s), against +750 to 788 MiB under the default arenas. The terms of the section above on +PostgreSQL at this row are 413 MiB (545 MiB less the five-row run-ahead, +which PostgreSQL has not), so the one-arena peak is 0.82 of the terms here, +0.84 in the other measurement, under the 1.2 the design states, which was +measured on SQLite alone until now. + +What one arena costs the request path. On the SQLite release build with a +50k-item composite-key table, one boto3 client per thread, three +alternating repeats, 20 s per operation (`sqlite-ab.sh`, +`workload.py`, kept with the scripts; host load 2.4 to 5.2): GetItem at +8 threads 624, 621, 616 req/s against 605, 588, 598 under one arena +(−3.8%); Query of a 50-item partition at 8 threads 414, 407, 406 against +398, 391, 395 (−3.5%); Scan without Limit at 4 threads 47.7k, 46.7k, 46.0k +items/s against 44.2k, 42.7k, 43.9k (−6.9%); Scan with Limit 1000 over the +table 46.1k, 42.9k, 43.6k against 40.7k, 40.4k, 39.3k (−9.2%). On the +PostgreSQL release build (`pg-ab.sh`, the same workload and method, the +tree's binary, host load 2.6 to 3.8): GetItem 600, 631, 606 against 617, +623, 617 (+1.1%); Query 398, 413, 407 against 402, 401, 408 (−0.6%); Scan +46.5k, 46.2k, 47.3k against 46.1k, 45.8k, 45.0k (−2.1%); paged Scan 45.0k, +44.3k, 45.9k against 44.3k, 44.1k, 44.1k (−2.1%). glibc with one arena +serializes every allocation of the worker threads on one lock; SQLite's +request path runs in the process (the JSON parse and render per item and +the database's own work), where PostgreSQL's spends its time on the +database's round trip, which is why the setting shows on SQLite and is +within the spread between repeats on PostgreSQL but for the scans. A Python +client bounds these rates; the server's own share of the loss may be larger +under a faster client. The deployment guide's service unit carries the line +commented out, with the condition (a host that backs up tables of large +items) and this cost beside it. diff --git a/docs/design/15-backup-restore-bench-pr9.md b/docs/design/15-backup-restore-bench-pr9.md new file mode 100644 index 000000000..24e190dd1 --- /dev/null +++ b/docs/design/15-backup-restore-bench-pr9.md @@ -0,0 +1,310 @@ +# PR 9 restore memory and duration record (2026-11-02; the index, 60,000-item, input-order and 3M-row runs added the same day) + +This is the record the design cites for the restore (section 7, revision item +30 of `15-backup-restore.md`) and the admin guide's `max_concurrent_jobs` row +quotes. The scripts (`bench-restore-rss.sh`, `kill-points.py`) and the raw +samples (`/proc//status` every 200 ms per run, the servers' logs) are +kept beside the PR outside the repository, on the measuring host; the figures +below are what they produced. The backup side's record is +`15-backup-restore-bench-pr8.md`. + +## Method + +The release binary (`cargo build --release -p extenddb --no-default-features +--features sqlite` and `--features postgres`, built from the PR's final tree) +running `serve --foreground` with a filesystem backup store on local disk. +Each run: a fresh deployment (`init`, the port patched, `[backup] +max_concurrent_jobs = 2`), the table loaded through the wire +(`kill-points.py create-table`: BatchWriteItem from 16 threads, composite +key, no GSI; or `put-worst` for the worst shape), `CreateBackup` and the wait +for `AVAILABLE`, then the server stopped and started again so the process's +high-water mark covers the restore alone; `RestoreTableFromBackup` into a new +table; `VmHWM` read right before the request (the idle figure) and once +`DescribeTable` reports `ACTIVE` (the peak), sampled every 200 ms in between; +"over idle" is the difference. The duration is from the request to `ACTIVE` +as the client saw it; the job's own line gives the same figure within 0.2 s. + +The worst shape is #394's worst-expansion item (`data_file.rs`'s line-cap +table): a list of 2,262 number sets whose members are `1.0E-130`, 409 KB as +the service sizes an item, 27.7 MB as a data-file line, about 34 MB as a +decoded `Item`; each `worst` table holds 1,000 small items and then `N` +such items. `big1200` is 1,200 items of a 350,000-byte string under one +hash key, written into one data file (`data_file_target_bytes` raised past +the table), the shape on which the SQLite batch was measured on the PR's first +build, before it had a byte bound. The 100k and 1M tables are 200-byte `S` pads under a +composite key and have no secondary index, and their data files are in key +order (the server's own backup of a table loaded through the wire is +`key_ordered`); every figure in the first table below is for that shape, a +table with secondary indexes restores in more memory on SQLite (the second +table), and an unordered backup of the same items restores in more memory and +time on SQLite (the input-order table). The 60,000-item runs are the same items +and key, no index: the shape the differences document's in-catalog restore +timings were measured on. + +Host: 16 cores (Xeon Platinum 8124M), 30 GiB, Linux 5.10 (amzn2int), glibc +2.26; PostgreSQL 16.15 in a container on loopback (55432); the SQLite the +binary bundles. What was not held constant: the host ran other work during +the runs (the four hooks builds during the first six runs, with the load +average between 8 and 13 on 16 cores; a MongoDB kill-point run and the +matrix cells beside the later ones, load 2 to 7), so the durations are upper +bounds and comparable only within one back-to-back pair; the peaks are the +process's own and do not depend on the host's load. Everything is on +loopback and on a filesystem store: nothing here says what a network database +or an S3 store's download does to the duration, and the spool's disk is not +in these figures. + +## The figures + +| backend | shape | data files | idle `VmHWM` | peak `VmHWM` | peak − idle | `ACTIVE` after | downloaded | +|---|---|---|---|---|---|---|---| +| SQLite | worst 12 | 4 | 22.1 MiB | 870.5 MiB | **848 MiB** | 17.7 s | 3,793,555 B | +| SQLite | worst 24 | 8 | 20.4 MiB | 869.2 MiB | **848 MiB** | 26.8 s | 7,567,821 B | +| SQLite | worst 48 | 16 | 22.1 MiB | 1,035.1 MiB | **1,013 MiB** | 57.0 s | 15,116,357 B | +| SQLite | worst 24, `MALLOC_ARENA_MAX=1` | 8 | 20.2 MiB | 412.9 MiB | **392 MiB** | 20.5 s | 7,567,821 B | +| PostgreSQL | worst 24 | 8 | 21.4 MiB | 799.8 MiB | **778 MiB** | 22.5 s | 7,567,821 B | +| PostgreSQL | worst 24, `MALLOC_ARENA_MAX=1` | 8 | 21.4 MiB | 448.8 MiB | **427 MiB** | 22.7 s | 7,567,821 B | +| SQLite | big1200 (1,200 × 350 KB, one file) | 1 | 20.8 MiB | 291.5 MiB | **270 MiB** | 3.1 s | 450,922 B | +| PostgreSQL | big1200 | 1 | 21.4 MiB | 177.8 MiB | **156 MiB** | 3.3 s | 450,922 B | +| SQLite | 100k × 200 B | 1 | 21.9 MiB | 60.2 MiB | **38 MiB** | 2.9 s | 366,631 B | +| SQLite | 1M × 200 B, key-ordered lines | 4 | 21.2 MiB | 110.4 MiB | **89 MiB** | 26.8 s | 3,017,864 B | +| PostgreSQL | 100k × 200 B | 1 | 21.4 MiB | 32.0 MiB | **10.7 MiB** | 1.1 s | 366,631 B | +| PostgreSQL | 1M × 200 B, key-ordered lines | 4 | 21.5 MiB | 37.8 MiB | **16.2 MiB** | 9.5 s | 3,017,864 B | + +The SQLite 1M run's high-water mark over time: 60 MiB at 2.7 s (the 100k +run's peak), rising to 87 MiB by 20.8 s as the load went on, then 110 MiB in +the last second, at the count and the flip. The growth during the load is the +write connection's page cache filling (`PRAGMA cache_size = -32000`, +`crates/storage-sqlite/src/sqlite_util.rs`: 31 MiB per connection, bounded by +the pool, as the backup record says of its 45 MiB); the final 23 MiB is the +`COUNT(*)` over the base table: the counting connection's page cache filling +with the table's b-tree pages, bounded by the same `cache_size` (32 MiB) and +measured at the same size at 3M rows (22 to 28 MiB, the input-order table +below), so it does not grow with the table past the cache. On PostgreSQL the 1M run's high-water mark reached 90% of its final +value within 3 s and the count added nothing visible. 1M − 100k: +51 MiB on +SQLite (+27 of them during the load), +5.5 MiB on PostgreSQL. Neither +materializes the table (1M items of 200 bytes are over 200 MB of items). + +## The terms the code bounds + +Per running restore, times `max_concurrent_jobs`: + +- the parsed manifest, for the whole job (one entry per data file); +- the decoder's line, at most `MAX_LINE_BYTES` (32 MiB, `data_file.rs`), and + the deserializer's intermediate value for it (the backup record's "one + row's parse", `r + 2d`: 109 MiB at the line cap); +- the decoded items of up to `LOAD_CHUNK_BYTES` (32 MiB, + `crates/engine/src/backup/restore/load.rs`) of data-file line text per + chunk, the chunk closing at the first item that takes it to the bound (so at + most the bound plus one line, 64 MiB of text at the cap) or at + `LOAD_CHUNK_ITEMS` (1,000) items, at four chunk positions: two in the + channel to `bulk_load`, the one being filled and the one being drained. The + bound counts the bytes the decoder read for the chunk's items, the unit the + backup side budgets in, not the service's `item_size_bytes` (which is 68 + times smaller for the worst shape and bounded nothing). `bulk_load` is one + data file at a time, so the chunks in flight never hold more than one file's + items: at most the source's `data_file_target_bytes` plus one line of text + (64 + 32 MiB at the defaults), a knob the restoring host does not set; +- the backend's write batch: on SQLite and Cassandra at most + `RESTORE_BATCH_BYTES` (32 MiB, `crates/storage/src/restore_engine.rs`) of + serialized item text plus one item, or `RESTORE_BATCH_ITEMS` (10,000) + items when they are smaller, held beside the decoded items of that text + (SQLite collects `(Item, item_json)`; Cassandra serializes once to measure + and again to write); on PostgreSQL at most `COPY_BATCH_BYTES` (4 MiB, + `crates/storage/src/backup_definition.rs`) of item text plus one item, or + `COPY_BATCH_ITEMS` (500) rows, as `Vec<(Item, String)>` (the item and its + JSON text, two copies), one transaction per file; on MongoDB 1,000 + documents per `insert_many`, bounded in bytes only by the data file + (`bulk_load` is per file), since the driver splits a call by the server's + message bound on its own; +- on SQLite, the `COUNT(*)` after the last file: the counting connection's + page cache (`PRAGMA cache_size = -32000`, 32 MiB), which the count fills + with the base table's b-tree pages and which stops at that setting whatever + the table holds (22 to 28 MiB measured at 1M and at 3M rows, the input-order + table below). + +None of the terms grows with the table: the chunk and batch terms grow with +the item, which `max_item_size_bytes` bounds, the line cap bounds the line, +and `cache_size` bounds the count's pages. These are the restore's own terms; a table with +secondary or vector indexes adds, on SQLite, the backend's own per-connection +page cache for the index b-trees (below), which is not a buffer of the +restore and which the batch bound does not cover. The decoded `Item` of the worst shape is about 1.2 times +its line, so at the line cap the chunk positions hold up to 4 × 64 MiB × 1.2 ≈ +307 MiB of decoded items when a file holds that much, the SQLite or Cassandra +batch up to 32 MiB of text plus about 77 MiB of decoded items, and the +PostgreSQL batch 4 MiB plus one 32 MiB item twice. The measured worst-shape +peaks are above those terms under glibc's default allocator for the reason +the backup record gives: the items' strings are allocated on one thread (the +decoder's blocking thread) and freed on another (the backend's write), and +the per-thread arenas keep that memory, so the high-water mark grows with the +number of large items a restore reads (848 MiB at 12 and 24 items, 1,013 MiB +at 48 on SQLite, with no plateau measured) and varies between runs. Under +`MALLOC_ARENA_MAX=1` the peak is the terms' and did not grow from 12 to 24 +items on the backup side; here it is 392 MiB (SQLite) and 427 MiB +(PostgreSQL) at 24 items. + +Before the bounds (the figures measured the same way on the PR's first +build, kept beside this record): SQLite worst 12, +24 and 48 at 616, 891 and 925 MiB, worst 24 under one arena 370 MiB; +PostgreSQL worst 24 at 861 MiB, under one arena 428 MiB; the 1,200 × 350 KB +table on SQLite 435 MiB over idle (`VmRSS`, sampled every 100 ms in the +process that had taken the backup) against 127 MiB on PostgreSQL, the whole +file's items in one batch. With the batch bounded at 32 MiB of text the same +table holds 270 MiB on SQLite (`VmHWM` from a fresh start, so not the same +figure's method; the one-file table of 420 MB of items now loads in batches +of about 95 items) and 156 MiB on PostgreSQL. The worst-shape SQLite figures +with the default arenas moved with the allocator between the two series (616 +to 848 at 12 items, 925 to 1,013 at 48), as the backup record's series did; +the one-arena figures, which measure the terms, are 370 to 392 MiB on SQLite +and 428 to 427 MiB on PostgreSQL. + +## Tables with secondary indexes + +The same 200-byte items under the same composite key with two GSIs +(`by-g0`, `by-g1`, hash key `g` with 97 values, range key `sk`, `ALL` +projection), measured the same way (a fresh start, `VmHWM`), after every GSI +reported `ACTIVE` and with each GSI's `Scan` count checked against the item +count (equal in every run). "head" is the build of the tree before the +spool-sweep order, the `finish_restore` re-read and the `remove-target` +command were added, "final" the build with them; nothing in the restore's +load changed between the two, and the runs differ as the allocator's runs do. + +| backend | shape | build | idle `VmHWM` | peak `VmHWM` | peak − idle | `ACTIVE` after | load average at start | +|---|---|---|---|---|---|---|---| +| SQLite | 100k × 200 B, two GSIs | head | 21.4 MiB | 164.8 MiB | **143 MiB** | 10.2 s | 2.9 | +| SQLite | 100k × 200 B, two GSIs | final | 21.3 MiB | 246.4 MiB | **225 MiB** | 12.3 s | 12.1 (release builds running) | +| SQLite | 100k × 200 B, two GSIs | final, repeated | 21.1 MiB | 164.1 MiB | **143 MiB** | 13.9 s | 1.8 | +| SQLite | 300k × 200 B, two GSIs | head | 22.0 MiB | 187.1 MiB | **165 MiB** | 51.7 s | 8.1 | +| SQLite | 300k × 200 B, two GSIs | final | 21.7 MiB | 193.5 MiB | **172 MiB** | 51.9 s | 6.1 | +| PostgreSQL | 100k × 200 B, two GSIs | head | 21.4 MiB | 50.8 MiB | **29.3 MiB** | 9.9 s | 8.4 | +| PostgreSQL | 100k × 200 B, two GSIs | final | 23.3 MiB | 46.2 MiB | **22.9 MiB** | 6.2 s | 4.6 | +| PostgreSQL | 300k × 200 B, two GSIs | head | 21.3 MiB | 52.1 MiB | **30.8 MiB** | 19.7 s | 4.8 | +| PostgreSQL | 300k × 200 B, two GSIs | final | 24.0 MiB | 57.6 MiB | **33.5 MiB** | 21.4 s | 9.7 | + +With two GSIs the SQLite restore holds three to six times the no-index +figure at 100k (143 to 225 MiB in three runs against 38 and 43 MiB) and +grows from 100k to 300k (143 to 165 and 172 MiB); PostgreSQL holds 23 to +34 MiB against 10.7 MiB and grows little. In the SQLite runs the load itself +climbs to about 105 to 125 MiB of `VmRSS` before the last batch and the final +`COUNT(*)` and flip add the rest (50 MiB in the 143 MiB runs, 120 MiB in the +225 MiB run). The term is SQLite's, not the restore's: the restore's own +buffer is the one batch of `(Item, item_json)` bounded by `RESTORE_BATCH_BYTES` +whatever the indexes are, and the extra resident memory is the write +connection's page cache (`PRAGMA cache_size = -32000`, 32 MiB per connection, +`crates/storage-sqlite/src/sqlite_util.rs`) holding the index tables' b-tree +pages, which a load of scattered index keys dirties across the whole tree, +plus glibc's per-arena free lists (the figure falls by about a third under +`MALLOC_ARENA_MAX=1`: 72 MiB against 119 MiB at 100k in another run on the +same shape). It is bounded by `cache_size` per connection and grows +sub-linearly with the table, not with the restore's buffers; a smaller batch +for an indexed table was built and measured (91.7 MiB against 92.7 MiB over +idle at 100k in that pair of runs: the cache persists across batches, so +smaller transactions do not lower the peak) and not kept. The published +no-index figures stand for the no-index shape; size a SQLite host that +restores indexed tables for the no-index figure plus about 32 MiB of page +cache per running restore, and read the run-to-run spread above as the +allocator's. + +## Input order and the count term + +The 1M figures above are for the server's own backup, whose data files are +in key order. The same items (200-byte `S` pads, composite key, no index) +as an unordered backup, the prefix built by hand with the hash keys cycling +(`pk = i % 1000`, the shape of every MongoDB-sourced backup and every +multi-part-key table), and once more in key order (`pk = i // 1000`, 1,000 +items per hash key), at 1M and at 3M rows, on two builds of this PR: +"committed", the pushed head's, and "final", the build after the last +corrections (the PostgreSQL vector-relation drop on a removed target, the +Cassandra progress re-read, the job's first beat before the reply; nothing in +the SQLite load changed between the two). Sampled every 50 ms. +"count + flip" is the resident set's growth in the restore's last stretch +(the peak less the `VmRSS` one second before `ACTIVE`), which is the final +`COUNT(*)` and the flip; the host's load average at the start is beside each +run (the hooks builds and other suites ran beside the SQLite runs). + +| backend | shape | build | peak − idle | count + flip | `ACTIVE` after | load | +|---|---|---|---|---|---|---| +| SQLite | 1M, key-ordered lines | final | **98.1 MiB** | 21.9 MiB | 56.3 s | 16.2 | +| SQLite | 1M, unordered lines | final | **163.5 MiB** | 13.6 MiB | 90.3 s | 18.0 | +| SQLite | 1M, unordered lines | committed | **144.4 MiB** | 13.4 MiB | 66.2 s | 3.1 | +| SQLite | 3M, key-ordered lines | final | **109.3 MiB** | 27.6 MiB | 102.7 s | 19.1 | +| SQLite | 3M, key-ordered lines | committed | **116.4 MiB** | 22.9 MiB | 84.9 s | 4.5 | +| SQLite | 3M, unordered lines | final | **173.5 MiB** | 12.8 MiB | 264.9 s | 10.9 | +| SQLite | 3M, unordered lines | committed | **170.8 MiB** | 14.4 MiB | 274.7 s | 6.0 | +| PostgreSQL | 1M, unordered lines | final | **16.4 MiB** | 0.1 MiB | 17.5 s | 5.6 | + +Two things the table shows. The input order is a term of SQLite's, not of +the restore's: an unordered backup of 1M items holds 1.5 to 1.8 times the +key-ordered figure (144 to 164 MiB against 87 to 98) and takes two to three +times as long, because scattered inserts dirty pages across the whole base +b-tree where ordered inserts append to its right edge, and the write +connection's page cache (`cache_size`, 32 MiB) plus the allocator's arenas +hold the difference; the restore's own buffers (the chunk and the batch) are +the same in both runs, and PostgreSQL does not depend on the order (16.2 MiB +ordered, 16.4 unordered). The ordered 1M run on this binary reproduced the +first table's figure (98 against 89 MiB over idle; 86.9 MiB in another run +of the same shape on the committed build). And the count's term does not grow +with the table: 22 to 28 MiB at 1M and at 3M rows in key order, 13 to 14 MiB +unordered at both sizes, every figure under the counting connection's +`cache_size` (32 MiB), which is what bounds it; from 1M to 3M rows the whole +peak moved by +11 MiB key-ordered and +7 to +10 MiB unordered while the +table tripled. The durations are upper bounds: the SQLite runs on the final +build ran under a load average of 11 to 19 (the ordered 1M run took 56 s +against 26.8 s on the quieter host of the first table). + +## The 60,000-item shape, store-backed + +The differences document's "How `RestoreTableFromBackup` runs" row quotes +response and `ACTIVE` times measured on an earlier build's in-catalog restore +of a 60,000-item table with no secondary index. The same shape (60,000 items +of 200-byte `S` pads under a composite key, no index) restored from a +filesystem store on the final binaries, the host's load average 5.4 to 7.6 +with other runs beside it: + +| backend | response | `ACTIVE` after | peak − idle | data files / bytes | +|---|---|---|---|---| +| SQLite | 0.016 s | 1.85 s | 38.3 MiB | 1 / 360,532 | +| PostgreSQL | 0.051 s | 0.67 s | 9.3 MiB | 1 / 360,532 | +| MongoDB | 0.055 s | 3.74 s (the job's DONE at 1.8 s; the backfill worker's flip after) | 30.1 MiB | 1 / 360,532 | + +Cassandra was not measured on this shape (its own CreateBackup is in-catalog, +so a store-backed Cassandra restore needs a backup taken elsewhere and +adopted, which the kill-point and matrix runs do at 12,000 items). The +in-catalog figures the differences document quotes (response 0.032, 0.012, +0.053, 0.151 s; `ACTIVE` after 1.5, 2.1, 3.6, 11.2 s on PostgreSQL, SQLite, +MongoDB, Cassandra) were measured on another build and host and are not +comparable run for run; the store-backed response includes the sidecar and +manifest reads, and its `ACTIVE` time the download and the per-file +verification. + +## The sizing sentence + +Size a host for about 450 MiB per running restore of a table of large items +under `MALLOC_ARENA_MAX=1` on SQLite or PostgreSQL (392 and 427 MiB measured +at 24 items of 27.7 MB), times `max_concurrent_jobs`; under the default +arenas the peak grows with the number of large items a restore reads and no +bound was measured (1,013 MiB over idle at 48 such items on SQLite is the +largest seen; 778 MiB at 24 on PostgreSQL). A restore of 200-byte items holds +under 100 MiB on SQLite and under 20 MiB on PostgreSQL at 1M items when the +table has no secondary index and the backup's lines are in key order (the +server's own backups of hash-only and single-range-key tables); an unordered +backup of the same items (every MongoDB-sourced backup and every +multi-part-key table) held 144 to 164 MiB over idle on SQLite at 1M and 171 to +174 MiB at 3M (the input-order section), PostgreSQL 16 MiB either way; with two GSIs SQLite held 143 to 225 MiB at +100k and 165 to 172 MiB at 300k (the section above), PostgreSQL 23 to 34 MiB. A +restore is not "much less" than a backup of the same table: the backup +record's worst-shape figures are 466 to 1,281 MiB under the default arenas +and 452 to 481 MiB under one arena. + +## Durations (the RTO baseline the testing summary names) + +On this host, loopback, filesystem store: 1M items of 200 bytes in key order +restore in 9.5 s on PostgreSQL and 26.8 s on SQLite (100k: 1.1 s and 2.9 s; +60k: 0.67 s and 1.85 s, and 3.74 s on MongoDB); the same 1M items as an +unordered backup in 17.5 s on PostgreSQL and 66 to 90 s on SQLite, and 3M +items on SQLite in 85 to 103 s key-ordered and 265 to 275 s unordered (the +input-order section, on a host whose load average ran between 4 and 19); with two GSIs 100k in 6.2 to 9.9 s on +PostgreSQL and 10.2 to 13.9 s on SQLite and 300k in 19.7 to 21.4 s and 51.7 +to 51.9 s, on a host whose load average ran between 1.8 and 12; 24 items of +27.7 MB in 22.5 s on PostgreSQL and 26.8 s on SQLite; 1,200 items of 350 KB in +3.3 s and 3.1 s. The backup side's 1M figures are in the PR 8 record (36 s on +PostgreSQL under a loaded host, 12.5 s on SQLite). diff --git a/docs/design/15-backup-restore.md b/docs/design/15-backup-restore.md index ebaf3318c..7d6b12ed3 100644 --- a/docs/design/15-backup-restore.md +++ b/docs/design/15-backup-restore.md @@ -61,9 +61,9 @@ A data file line (one `{"Item": ...}` object and its newline) is at most 32 MiB `backup_creation_date_time` is the instant `open_table_snapshot` returned, read just after the snapshot pinned, and it is the instant encoded in the backup id: CreateBackup opens the snapshot first (after any wait behind an in-flight UpdateTable), then mints the id from that millisecond (5.1 step 2), so the ARN, the manifest and the catalog's `created_at` carry one time. -A restore rejects a backup whose data files contain the same primary key twice, across files as well as within one. For key-ordered backups the manifest's `key_range` check rejects overlap before any file is downloaded and the per-file key-range check proves the rest, on every target. For unordered backups a plain `INSERT` on PostgreSQL and SQLite fails on a duplicate and so does MongoDB's `_id` unique index (E11000), but a Cassandra write is an upsert and detects nothing, so the general rule is a count: after the last file the restore counts the target's items and requires the count to equal the sum of the manifest's `item_count`; a shortfall is a duplicate and fails the restore. +A restore rejects a backup whose data files contain the same primary key twice, across files as well as within one. For key-ordered backups the manifest's `key_range` check rejects overlap before any file is downloaded and the per-file key-range check proves the rest, on every target. For unordered backups a plain `INSERT` on PostgreSQL and SQLite fails on a duplicate and so does MongoDB's `_id` unique index (E11000, suppressed only on the one unrecorded file a resume re-applies, 5.3 step 5), but a Cassandra write is an upsert and detects nothing, so the general rule is a count: after the last file of every backup, key-ordered or not, the restore counts the target's items (`count_restore_target`, section 3) and requires the count to equal the sum of the manifest's `item_count`; a shortfall is a duplicate and fails the restore. -- Numbers are DynamoDB JSON strings, so the 38-digit precision survives the file. A restore into MongoDB of an `N` sort-key value beyond Decimal128's 34 significant digits fails with ValidationException and the restore reports FAILED (see 5.3); it never rounds. Other `N` values are stored as strings there and keep their 38 digits. +- Numbers are DynamoDB JSON strings, so the 38-digit precision survives the file. A restore into MongoDB of an `N` sort-key value beyond Decimal128's 34 significant digits fails in `bulk_load` with the write path's ValidationException text, which becomes the restore's FAILED reason (5.3 step 7: no wire exception, the client sees the target disappear); it never rounds. Other `N` values are stored as strings there and keep their 38 digits, so a 38-digit non-key number restores intact. ### 2. Backup store abstraction @@ -77,12 +77,12 @@ Three store facts that rules of sections 4 and 5 rest on, and the one method a l ### 3. Storage trait changes -`SnapshotEngine` and `TableSnapshot` are `crates/storage/src/snapshot.rs` and the four `snapshot_engine.rs` files (PR 5, #395); the module docs there carry the per-backend mechanism, `snapshot.rs`'s trait docs state what `description` is and which errors end a stream, and `docs/troubleshooting.md` carries the operator side of the PostgreSQL DeleteTable change. The fetch size of every backend's stream is one shared constant, `extenddb_storage::snapshot::paging::DEFAULT_FETCH_BATCH` (1000), with a `test-hooks` override and a fetch counter the conformance check reads. +`SnapshotEngine` and `TableSnapshot` are `crates/storage/src/snapshot.rs` and the four `snapshot_engine.rs` files (PR 5, #395); the module docs there carry the per-backend mechanism, `snapshot.rs`'s trait docs state what `description` is and which errors end a stream, and `docs/troubleshooting.md` carries the operator side of the PostgreSQL DeleteTable change. The fetch size of every backend's stream is one shared constant, `extenddb_storage::snapshot::paging::DEFAULT_FETCH_BATCH` (1000), with a `test-hooks` override and a fetch counter the conformance check reads; beside it PR 8 adds the byte budget `SNAPSHOT_FETCH_BYTES` (64 MiB): on PostgreSQL and SQLite a page ends at the row bound or after the row that takes its item text to or past the budget, whichever comes first, so a page holds at least one row and at most the budget plus one row, with the cut applied in the page query on PostgreSQL (a keyset page in the snapshot's transaction, three layers with the cut in the outer one; the server reads up to `n - 1` rows past the cut, PR 8's section states the bound) and while the rows are pulled on SQLite (PR 8's section states the page and what it costs the server); a MongoDB batch is under the budget by the server's 16 MiB reply bound and applies no cut. - Cassandra: `open_table_snapshot` returns `Unsupported`. Cassandra has no `SnapshotEngine`, so the engine routes CreateBackup there to the legacy create (`LegacyBackupReader::create`, the #384 in-catalog path) whether or not `[backup.store]` is configured, the path every backend takes while no store is configured (section 7); the off-host path is never taken there. That stays until PR 14, by which time either a token-range `SnapshotEngine` with a documented non-point-in-time `SnapshotInfo.kind` (`cassandra_token_scan`) has been accepted as a design change to this section, or Cassandra backups are documented as unavailable. RFC-0003 (backend acceptance criteria) has no backup criterion today; the criterion this trait implies ("a backend that cannot fix an instant across a table returns `Unsupported` rather than a best-effort stream") is to be proposed there in a separate docs change, not in PR 5. The interfaces later PRs add to `extenddb-storage`, each with its contract; the signatures and the row keys are in the section of the PR that adds them (Phasing): -- `bulk_load(target_table_id, claim, file_index, start_at, items) -> Result`, a new `StorageEngine` method each backend owns (PR 9a), writes the items of one verified data file from item `start_at` on, and their index rows, into a CREATING restore target, records progress as 5.3 step 5 specifies for the backend under the job's `claim` (so a progress write conditioned on the generation has it) and returns how many items it applied. There is one spelling of that signature; 5.3 step 5 calls it. +- `RestoreEngine` (PR 9a), a trait in `StorageEngine`'s bound (`StorageEngine` has no methods of its own) that each backend implements: `bulk_load(target, claim, BulkLoadFile { file_index, start_at, reapplying }, items) -> Result` writes the items of one verified data file from item `start_at` on, and their index rows, into a CREATING restore target, records progress in the write that applied the items as 5.3 step 5 specifies for the backend (the job's `claim` carries the generation the record's writes are conditioned on) and returns how many items it applied; `finish_restore(target, claim, item_count, size_bytes) -> Finished { Active, Pending, NotMatched }` is the ACTIVE flip of 5.3 step 6; `count_restore_target(target)` is the count of section 1; `read_restore_progress` and `sweep_restore_progress(running)` read and sweep PostgreSQL's `restore_progress` rows (defaults elsewhere). There is one spelling of each signature, in PR 9a's section; 5.3 step 5 calls `bulk_load`. - The engine is backend-agnostic (main's `crates/engine` reaches every backend through the `extenddb_storage` traits), and nothing in sections 4 and 5 that touches a catalog row (the claim, the heartbeat, the conditional status writes, the store-scoped listings, the restore job record) can be written in it. `BackupCatalog`, a trait in `extenddb-storage` that PR 6 adds and each backend implements, is how the engine reaches the catalog index of section 4; PRs 8, 9a and 10 add methods to the same trait as their sections need them. Section 4's "Authority" line names this trait beside the engine module that binds it. - `LegacyBackupReader` is a trait in `extenddb-storage` with `describe`, `list`, `restore` and `delete`, plus `create`, which every backend keeps until PR 14: the engine calls it for CreateBackup when no `[backup.store]` is configured (section 7), logging a deprecation warning that names `[backup.store]` on each call, and on Cassandra always, since it has no `SnapshotEngine` (the decision before PR 14 settles whether that stays), and a legacy create takes a `max_concurrent_jobs` permit like any job, returning `LimitExceededException` when none is free (5.1 step 1), since PR 6 removes the slot semaphores #384's creates wait on; each backend implements it with a frozen copy of its #384 code for reading and decoding the legacy shape (not calls through trait methods later PRs remove), and the engine holds `Option>`. PR 14 removes the trait. @@ -92,23 +92,21 @@ The invariants and state machines every later PR honours, by lettered part; the (a) The backup row. The engine keeps a catalog row per backup, in `backup_index` (PR 6's schema), as the fast index for Describe and List, and as the state machine record; `status` is CREATING, AVAILABLE, DELETING, DELETED or FAILED. Status mapping to the API: CREATING and AVAILABLE as themselves; DELETING is reported as DELETED by DescribeBackup and DeleteBackup (a client retrying DeleteBackup sees the terminal status) and is not listed; DELETED and FAILED rows are not described (BackupNotFoundException, which is what the live service returns for a deleted backup) and not listed. FAILED and DELETED rows are tombstones the reconciler owns. Transitions and who performs them: CreateBackup inserts the row CREATING under the job's claim (5.1 step 2); the job writes AVAILABLE after the manifest, or FAILED, each conditional on its generation and on `status = CREATING` (5.1 steps 4, 5); the reconciler resolves a CREATING row whose owner is dead to AVAILABLE if a complete, verified manifest exists in the configured store, else to FAILED, under the same condition (5.1 step 6); DeleteBackup moves an AVAILABLE row to DELETING under the row's lock and to DELETED once the prefix lists empty (5.2); the reconciler finishes a DELETING row, deletes the prefix of a FAILED row once its `heartbeat_at` is older than `stale_job_after_secs` and of a DELETED row that still lists objects, and removes a FAILED or DELETED row once its prefix is empty and its `completed_at` is older than `failed_row_retention_secs`. -(b) The restore job record. The engine keeps one restore job record per restore target, called `restore_jobs` in this document. It is #384's restore registration extended with `status` (RUNNING, FAILED, DONE), `kind` (legacy or manifest), the store it reads, the claim (`owner_generation`, `heartbeat_at`), `spool_host`, per-batch progress, `failure_reason` and `completed_at` (PR 6's schema). Transitions: the request writes the row RUNNING with the target table in one transaction (5.3 step 4); the job marks it DONE with the ACTIVE flip, or FAILED, each conditional on its generation and on RUNNING (5.3 steps 6, 7); the reconciler claims a RUNNING row whose owner is dead and resumes or fails it (5.3 step 7); on MongoDB the job's DONE write precedes the worker's ACTIVE flip and the reconciler leaves such a row alone (5.3 step 6). The record is the table's child (`table_restores` cascades with the table on PostgreSQL and SQLite; it is fields of the table on MongoDB and Cassandra), so it lives exactly as long as the table: a DONE record stays while the table exists, `RestoreSummary` is reported only while the table is CREATING (as main does and `restore_summary_is_reported_only_while_creating` pins), and a FAILED record goes with its table when step 7 deletes it, the failure reported by the job's log line and `restore_jobs_total{outcome="failed"}`. +(b) The restore job record. The engine keeps one restore job record per restore target, called `restore_jobs` in this document. It is #384's restore registration extended with `status` (RUNNING, FAILED, DONE), `kind` (legacy or manifest), the store it reads, the claim (`owner_generation`, `heartbeat_at`), `spool_host`, its progress (`file_index` and `items_applied`: per batch on SQLite and Cassandra, `file_index` per file on MongoDB; PostgreSQL records progress in `restore_progress` rows of the data database instead, 5.3 step 5), `failure_reason` and `completed_at` (PR 6's schema; `RestoreJobRow` carries every member). Transitions: the request writes the row RUNNING with the target table in one transaction (5.3 step 4); the job marks it DONE with the ACTIVE flip, or FAILED, each conditional on its generation and on RUNNING (5.3 steps 6, 7); the reconciler claims a RUNNING row whose owner is dead and resumes or fails it (5.3 step 7); on MongoDB the job's DONE write precedes the worker's ACTIVE flip and the reconciler leaves such a row alone (5.3 step 6). The record is the table's child (`table_restores` cascades with the table on PostgreSQL and SQLite; it is fields of the table on MongoDB and Cassandra), so it lives exactly as long as the table: a DONE record stays while the table exists, `RestoreSummary` is reported only while the table is CREATING (as main does and `restore_summary_is_reported_only_while_creating` pins), and a FAILED record goes with its table when step 7 deletes it, the failure reported by the job's log line and `RestoreJobs{failed}` (section 9). -(c) Store identity and health. Each catalog has a random `catalog_id` (`BackupCatalog::catalog_id`); the store's root carries `extenddb-store.json`, holding a random store id and the `catalog_id` of the catalog that owns the root, and every backup row records the store id in `store_id`; server startup, every sync and every reconciler cycle read the file and check it against the catalog (PR 7's section: how the file is created). The store is unhealthy when the file names another catalog, when its store id differs from the `store_id` of the catalog's rows for the configured `store_kind` and `store_root`, when the catalog has rows for the root and the root has no file, or when those rows carry more than one `store_id` (`BackupCatalog::read_store_ids`, section 3). An unhealthy store is not a startup refusal (section 7): the reason is logged, `/health` reports the store component unhealthy, the operations that touch the store are refused, the reconciler acts on no row of the store, and the check runs again each reconciler cycle; a root whose file names another catalog stays unhealthy until an operator adopts it with `backup sync --adopt-store` (PR 10), and the previous owner then finds the store unhealthy and stops acting on it. Instances that share a catalog must share one store root (an S3 bucket and prefix or a shared mount): host-local roots at one path would give each instance different bytes under one `store_root`, so startup and each reconciler cycle also probe the root independently of the host, `head`ing the manifest of the most recent AVAILABLE row and marking the store unhealthy if it is absent. Every duty that reads or writes a backup row acts only on rows of the configured store (`store_kind`, `store_root` and `store_id`), as sync does, the legacy duty excepted: a CREATING or DELETING row of another store is reported and left alone (`backup sync --move-store`, PR 10, resolves it), a FAILED or DELETED row of another store is never removed and its prefix never deleted, and the emptiness check before a row is removed looks in the configured store, which is also what finds an object left by a put that overlapped the prefix's deletion (section 2); so pointing the configuration at a copy of a store never deletes the copy, and no row is resolved against a store that does not hold its bytes. +(c) Store identity and health. Each catalog has a random `catalog_id` (`BackupCatalog::catalog_id`); the store's root carries `extenddb-store.json`, holding a random store id and the `catalog_id` of the catalog that owns the root, and every backup row records the store id in `store_id`; server startup, every sync and every reconciler cycle read the file and check it against the catalog (PR 7's section: how the file is created). The store is unhealthy when the file names another catalog, when its store id differs from the `store_id` of the catalog's rows for the configured `store_kind` and `store_root`, when the catalog has rows for the root and the root has no file, or when those rows carry more than one `store_id` (`BackupCatalog::read_store_ids`, section 3). An unhealthy store is not a startup refusal (section 7): the reason is logged, `/health` reports the store component unhealthy, the operations that touch the store are refused, the reconciler acts on no row of the store, and the check runs again each reconciler cycle; a root whose file names another catalog stays unhealthy until an operator adopts it with `backup sync --adopt-store` (PR 10; a server on an in-memory database adopts, at startup, a root whose file another in-memory catalog wrote, revision item 34). The previous owner keeps acting on the root until a later check that can read its catalog finds the store unhealthy, at the earliest one reconciler interval after the cycle in progress ends, unless its deletion fence stops it first: each deletion from the store, DeleteBackup's and purge's row change before it, and each CreateBackup before its first put, re-reads the file and is refused, the store marked unhealthy at once, when the file names another catalog (revision item 34). A write already past the fence's read still lands, so every server of the previous catalog is stopped before an adoption. Instances that share a catalog must share one store root (an S3 bucket and prefix or a shared mount): host-local roots at one path would give each instance different bytes under one `store_root`, so startup and each reconciler cycle also probe the root independently of the host, `head`ing the manifest of the most recent AVAILABLE row and marking the store unhealthy if it is absent. The probe needs an AVAILABLE row, and until one exists the guard has a gap: two instances with host-local roots at one path and a copied `extenddb-store.json` (one id in both files, so the identity check passes on each) are not told apart, the dead-owner duty of one fails a killed instance's complete backup under the other directory (no manifest in its own) and the killed instance's restart deletes that backup's prefix, the only copy (measured on PostgreSQL, PR 8's review; revision item 32). The rule is stated in the admin guide: one shared root from the first backup on, and the identity file is never copied between roots. Every duty that reads or writes a backup row acts only on rows of the configured store (`store_kind`, `store_root` and `store_id`), as sync does, the legacy duty excepted: a CREATING or DELETING row of another store is reported and left alone (`backup sync --move-store`, PR 10, resolves it), a FAILED or DELETED row of another store is never removed and its prefix never deleted, and the emptiness check before a row is removed looks in the configured store, which is also what finds an object left by a put that overlapped the prefix's deletion (section 2); so pointing the configuration at a copy of a store never deletes the copy, and no row is resolved against a store that does not hold its bytes. (d) Identities. The ARN encodes partition, region, account, table, and backup id, and the key prefix is derived from account, table and id deterministically by `BackupArn::store_prefix`, under the configured store's prefix, and no row stores it, so restore, sync, purge and the reconciler derive it the same way and `describe_backup(arn)` works from the store alone after a sync. The engine, the writer and sync all parse and format ARNs through one implementation, `BackupArn` in `extenddb_core::backup_format` (#394); the engine keeps #384's region handling: it resolves a backup under the ARN as written and under the ARN re-spelled in its own region, and refuses a partition other than its own. `backup_creation_date_time` and `created_at` are the id's instant, so the ARN, the manifest, the API's reported time and the ListBackups cursor agree by construction. One row is addressed by `BackupKey` (`account_id`, `backup_id`, `backup_arn`): PostgreSQL, SQLite and MongoDB key `backup_index` by (`account_id`, `backup_id`), Cassandra keys `backups_by_arn` by (`account_id`, `backup_arn`); no method takes a bare backup id, and the engine treats a row whose `account_id`, `table_name` or `backup_arn` (after that re-spelling) differs from the request's as absent. One target is addressed by `RestoreTargetKey` (`account_id`, `table_name`, `table_id`), the table id guarding against a table re-created under the name. -(e) The claim. There is no single-instance lease on PostgreSQL or MongoDB, so each job holds a claim that both excludes a second owner and lets a successor detect a dead one. Every backend sets `owner_generation` (a fresh random value) at claim time, and every catalog write by the job is `WHERE owner_generation = $gen`, so a job that lost its claim cannot publish over a successor's work. No row ever exists unowned: on every backend the claim precedes the row's insert or is the insert itself (5.1 step 2); what the claim is per backend is PR 6's section. Who is dead: on SQLite every CREATING row at startup, and at runtime a row whose in-process slot no task holds; on PostgreSQL a row whose advisory lock is free; on MongoDB and Cassandra a row whose `heartbeat_at` is older than `stale_job_after_secs` (default 300). Staleness is always measured on the database's clock, never on an application host's. The reconciler takes the same claim as a worker, with a fresh `owner_generation` that fences the dead job's generation out, leaves `heartbeat_at` untouched (a refreshed heartbeat would restart the stale window the prefix deletion waits for), writes AVAILABLE or FAILED at once, both conditioned on `status = CREATING`, and never continues the upload. A job whose conditional write matches no row, or whose claim connection errors, performs no further store or table operation, deletes of a prefix or of a restore target included (5.1 step 5, 5.3 step 7); on PostgreSQL a restore target is resumed or deleted only once the row's `heartbeat_at` is older than `stale_job_after_secs`, since a free advisory lock shows that the owner's catalog session ended, not that its process stopped (5.3 step 7). +(e) The claim. There is no single-instance lease on PostgreSQL or MongoDB, so each job holds a claim that both excludes a second owner and lets a successor detect a dead one. Every backend sets `owner_generation` (a fresh random value) at claim time, and every catalog write by the job is `WHERE owner_generation = $gen`, so a job that lost its claim cannot publish over a successor's work. No CREATING row exists unowned: on every backend the claim precedes the row's insert or is the insert itself (5.1 step 2); the AVAILABLE rows `backup sync` inserts are unowned (`owner_generation` 0) and have no job, and no duty reads an AVAILABLE row's generation (revision item 34); what the claim is per backend is PR 6's section. Who is dead: on SQLite every CREATING backup row and every RUNNING store-backed restore record at startup (the restart ages every RUNNING restore's heartbeat to the epoch), and at runtime a row or record whose in-process slot no task holds (on SQLite the store-backed restore's claim holds the `restore:/
    ` slot from before the target exists to the task's end; the legacy restore holds no slot, and a legacy and a store-backed restore into one name exclude each other through the name's `tables` row under the write transaction (`BEGIN IMMEDIATE`: the CREATING-target check, then the unique name)); on PostgreSQL, MongoDB and Cassandra a row whose `heartbeat_at` is older than `stale_job_after_secs` (default 300), the rule MongoDB and Cassandra share; PostgreSQL adds the lock. On PostgreSQL the advisory lock is the claim's proof against a second owner and not a liveness test on its own: a free lock shows that the owner's catalog session ended, not that its process stopped (a session a managed server dropped leaves a process whose data writes carry no generation), and the owner itself aborts at its next heartbeat inside the stale window, so the reconciler takes over only a row whose heartbeat is stale **and** whose lock is free: a held lock with a stale heartbeat is a process that is stopped, not dead (SIGSTOP, a paused VM), and its row stays with it until its session ends or it resumes and finishes; a host that vanished releases the lock when the server's keepalives give up (about a minute with `CLAIM_KEEPALIVES`), so the takeover waits for the later of the two. Staleness is always measured on the database's clock, never on an application host's: the engine holds no clock, and every age test is a predicate of the backend's own (`BackupCatalog::claim_dead_backup(key, stale_after)`, `list_backup_rows_older_than(status, store, clock, age)` on the heartbeat or the completion column, PR 8's section). The reconciler takes over a dead row through `claim_dead_backup`, the same claim as a worker's with a fresh `owner_generation` that fences the dead job's generation out (PostgreSQL's lock and SQLite's slot refuse a second taker outright; on MongoDB and Cassandra the generation fence orders two takers, since the successor is a live owner to a later taker only once it heartbeats, and the reconciler writes its one status before any beat, so two instances' reconcilers can both take over one dead row within that gap and exactly one resolution lands, the other's write finding no row; Cassandra's conditional write also carries the heartbeat it read, so a beat landing between the read and the write refuses the takeover), leaves `heartbeat_at` untouched (a refreshed heartbeat would restart the stale window the prefix deletion waits for), writes AVAILABLE or FAILED at once, both conditioned on `status = CREATING`, and never continues the upload. A dead RUNNING store-backed restore record is taken over through `claim_restore_dead(key, stale_after, spool_host)` under the same rules (PostgreSQL's name lock and SQLite's slot refuse a taker outright; MongoDB and Cassandra condition on the stale heartbeat, Cassandra's lightweight transaction also on the heartbeat it read and on `kind = manifest`), which writes the claimant's `spool_host` with the fresh generation and never takes over a legacy record; the successor resumes the restore from the recorded progress (5.3 step 7). A restore record's takeover differs from a backup row's in one write: `claim_restore_dead` sets `heartbeat_at` to the database's clock in the same conditional write as the generation and `spool_host` (PostgreSQL `now()`, SQLite `strftime('%Y-%m-%dT%H:%M:%fZ', 'now')`, MongoDB `$$NOW`, Cassandra `toTimestamp(now())`), so a second instance's reconciler, whose cycle lands before the successor's first beat (one `heartbeat_interval_secs` after the takeover), finds the record live and is refused. The backup takeover leaves `heartbeat_at` because the FAILED-prefix duty's stale window counts from the owner's last beat; nothing waits on a restore record's old heartbeat after a takeover (the FAILED-target duty ages on `COALESCE(completed_at, heartbeat_at)` and every FAILED write of a resume stamps `completed_at`; the maintenance claim waits on RUNNING, not on the beat). Per backend, the second taker is refused by: PostgreSQL the successor's advisory lock on the target name, then the fresh heartbeat; SQLite the successor's in-process slot (one process serves the file), and the record the startup step had aged is no longer listed as dead once the takeover refreshed its heartbeat; MongoDB the `$expr` staleness term of the takeover's own filter against the refreshed `heartbeat_at`; Cassandra the takeover's serial read, which sees the refreshed `restore_heartbeat_at` and stops before its lightweight transaction. So the two-taker argument of the backup takeover is, for a restore, "a second taker finds the record live"; the belt under it is that every progress write is fenced by the generation (a first taker's next write would match nothing and its job end with no table access) and that a resume's first file is `reapplying`, which covers a partial file on MongoDB and which Cassandra's upserts collapse, so a later change to either mechanism (a progress write without the generation, `reapplying` on a later file) is seen to matter here. The takeover on Cassandra also re-applies the target's `active_restores[table_id] = name` entry on the backup's row when it is absent (one serial read, then `UPDATE ... IF backup_status = 'AVAILABLE'`): the registration writes the target's row and its DDL before the entry, so an owner that died between the two left a resumable record whose backup nothing held in use, and `DeleteBackup` would have succeeded under the resumed load. A backup no longer AVAILABLE keeps no entry; the resumed load fails on the missing prefix and the target is removed. The engine's first confirmed beat of a fresh restore is the return of the one heartbeat the task sends right after `claim_restore` returns and before the reply (5.3 step 4), and of a resumed restore the instant `claim_restore_dead` returned, taken by the reconciler beside the takeover and handed to the job; a takeover's `heartbeat_at` was stamped inside that write, so the instant is at or after the beat it stands for by the write's own duration, and the catalog reads between a takeover and the resumed job's start count against the two-period window rather than widen it. The registration's own `heartbeat_at` stamp is the catalog's clock at some point of the registration, not its end: on PostgreSQL `now()` is the transaction's start, so a registration stalled on the `backup_index` lock or the DDL commits a stale `heartbeat_at`; the job's first beat, sent before the reply, refreshes it. Two readers judge a RUNNING record's owner by the heartbeat alone and need that beat: the reconciler under a maintenance claim (a dead record is failed and its target removed, or failed `target_not_empty`) and `extenddb backup remove-target`; the lock fences only `claim_restore_dead`. A first beat that finds the claim lost, or fails, ends the task before the reply: the record is failed with reason "the claim was lost before the job started" (a write that matches nothing when the claim is another's) and the request is answered `InternalServerError`. What bounds a registration's duration per backend: PostgreSQL holds the `backup_index` row's `FOR UPDATE` lock and the target name's advisory lock through the registration, so `claim_restore_dead` is refused however long it takes; SQLite holds the in-process `restore:/
    ` slot from before the target exists; MongoDB runs the registration in a transaction bounded by `transactionLifetimeLimitSeconds` (default 60 s); Cassandra's registration writes are lightweight transactions bounded by the coordinator's `write_request_timeout_in_ms` and `cas_contention_timeout_in_ms` (Cassandra defaults 2 s and 1 s), and the manifest registration runs no heartbeat during the target's DDL; a registration whose DDL outlasts the stale window can lose its record to a takeover, and its refusal path then conditions the target's removal on its own generation and removes nothing (the registration answers an error naming the lost claim, and the record's owner finishes the restore). A registration that took two periods does not end the job at its first check, since the two-period rule counts from the first beat's return. A job whose conditional write matches no row, or whose conditional write's outcome stays unknown after the re-read, performs no further store or table operation, deletes of a prefix or of a restore target included (5.1 step 5, 5.3 step 7); each unknown outcome is followed by a re-read; the write is sent once more at most; after the second write a re-read that shows the row in the target status counts as matched and anything else as matching no row. The second write is for a backend whose claim connection survives the unknown outcome (MongoDB and Cassandra, whose claim writes go through the shared client); on PostgreSQL an unknown outcome is an error after which the claim session's one connection is dead, so the second write fails with the session, its re-read settles it as matching no row, and the row is the reconciler's once the session's end has freed the advisory lock. PostgreSQL's lock rule applies to the resume's takeover alone: the two writes the reconciler makes to a dead RUNNING manifest record without resuming it, FAILED `OWNER_DEAD` (the target removed) or `target_not_empty` (the target left, 5.3 step 7) under a maintenance claim and FAILED `OTHER_STORE`, test the heartbeat alone on every backend, because the operator's command waits on the first and the second is a configuration this design forbids (4(c): instances of one catalog share one root), so neither waits for a stopped owner's session to end: both go through `fail_restore_job`, which conditions on RUNNING alone after `list_dead_restore_jobs` selected the record by heartbeat age, and on PostgreSQL they do not try the owner's advisory lock, so a stopped owner whose session lives keeps its record on the resume path and loses it on these two. A FAILED `restore_jobs` row whose target table is still CREATING (the FAILED write landed and the target's removal did not, whether the job's own or the reconciler's) is the FAILED-target duty: the reconciler lists it once the row's `completed_at` (or, where none was stamped, its `heartbeat_at`) is older than `stale_job_after_secs`, so a writer still on its own failure path finishes it, and removes the target through `remove_restore_target` with no second FAILED write; PR 6's reconciler runs it for every kind of record, since on this rule the kind does not matter (PR 8 adds the `target_not_empty` exception). On Cassandra the removal is itself two writes, a lightweight transaction claiming the target CREATING to DELETING and then the drop, so the duty also lists a FAILED row whose target is DELETING with its registration still present, on the same staleness rule, and `remove_restore_target` resumes the removal from the drop; PostgreSQL and SQLite remove in one transaction and MongoDB deletes the document first, so neither leaves that state. While the server holds no serving registration (PostgreSQL, MongoDB and Cassandra) it does what the module doc of `crates/server/src/registration.rs` says: it refuses new requests, stops its background workers, the reconciler among them, and starts a new set once it registers again, and the work requests started goes on. Two points that doc leaves to this design: a reconciler cycle running at the stop spawns no resume once its own token has fired (the record is left for the next set's reconciler, which starts from a fresh state), and the backup and restore jobs, resumes included, run on the jobs' token, a child of the server's shutdown token and not of the workers', so a job running when the registration is lost goes on writing the catalog and the store until it ends or the server stops, beside a migration that may already have taken the catalog. Gating the running jobs on the registration (a job stopping while the server is unregistered) is GitHub issue #755. -(f) The maintenance claim. `backup purge`, `backup remove-target` and `backup sync` with `--move-store`, `--forget-store`, `--forget-backup`, `--prune-missing` or `--adopt-store` change rows or bytes that jobs on any instance may be using, and a job permit (5.1 step 1) is per instance, so these commands take a catalog-level maintenance claim first (PR 10's section: the claim per backend and the SQLite serve lock). Every backup and restore job checks the claim after its own claim and row exist (5.1 step 2, 5.3 step 4) and refuses while it is held; while a claim is held the reconciler runs only the duties that resolve the states the command waits on (DELETING rows, rows whose owner is dead, the FAILED-target duty) and skips every other duty; the command waits until no CREATING backup row with a live owner, no DELETING row of the configured store and no RUNNING `restore_jobs` row remains, and refuses and releases the claim if they have not gone within `2 * stale_job_after_secs`, which it prints; a second maintenance command while one holds the claim fails at once. Because the job checks after its row exists and the command checks rows after its claim exists, one of the two always sees the other. +(f) The maintenance claim. `backup purge`, `backup remove-target` and `backup sync` with `--move-store`, `--forget-store`, `--forget-backup`, `--prune-missing` or `--adopt-store` change rows or bytes that jobs on any instance may be using, and a job permit (5.1 step 1) is per instance, so these commands take a catalog-level maintenance claim first (PR 10's section: the claim per backend and the SQLite serve lock). Plain `backup sync` takes it too where it runs beside servers (PostgreSQL, MongoDB, Cassandra; on SQLite the serve lock already excludes purge), without the wait for running jobs: its insert-if-absent orders it against a CreateBackup, and the claim keeps it from adopting a prefix purge has un-rowed and not yet deleted (revision item 34). Every backup job checks the claim before its permit and so before its row exists (5.1 step 2; the in-catalog create and restore the same, revision items 29 and 32), the restore job as 5.3 step 4 states, and each refuses while it is held; while a claim is held the reconciler runs only the duties that resolve the states the command waits on (DELETING rows, rows whose owner is dead, the FAILED-target duty) and skips every other duty; the command waits until no CREATING or FAILED backup row of the configured store has a live owner (a heartbeat within `stale_job_after_secs` on the catalog's clock, read through `list_backup_rows_older_than`; a FAILED row's heartbeat may still be refreshed by a job that has not seen its claim lost) and no RUNNING `restore_jobs` row remains, a DELETING row not being waited on (nothing heartbeats one; the command finishes it as it finishes every row), and refuses and releases the claim if they have not gone within `2 * stale_job_after_secs`, which it prints; a second maintenance command while one holds the claim fails at once. Because the command's wait lists the rows again each time it waits, a job whose row lands after the command's claim is seen by the wait, which waits for it; a job that checked before the claim was taken runs to its end, as a job started before the command would. -(g) The store, the rows and sync. The store is the source of truth; the row is a cache. Sync considers only rows of the configured store (`store_kind`, `store_root`, `store_id`); rows for another store are reported, not touched, and `backup sync --forget-store ` is the explicit opt-in to drop them: it deletes that root's catalog rows outright, leaving no DELETED tombstone, so no deletion duty ever acts on that store's bytes. Sync reports a row whose manifest is gone and marks it DELETING only with `--prune-missing`; without a resolving option (PR 10) it never modifies CREATING, DELETING, DELETED or FAILED rows nor adopts their prefixes, and it never adopts a prefix without a manifest, one whose sidecar does not match, or one whose listing lacks a data file the manifest lists or holds it at another size. Neither Describe nor Restore reads the store for an ARN that has no row: a backup that exists only in the store is BackupNotFound until `backup sync` adopts it, so these rules are the only path from a prefix to an AVAILABLE row. ListBackups lists rows of another store like any other (the response shape has no marker); Describe of such a row returns it and logs a warning; DeleteBackup and Restore of such a row return a ValidationException saying the backup is in a store other than the configured one, without naming either root. No API error names a store root. +(g) The store, the rows and sync. The store is the source of truth; the row is a cache. Sync considers only rows of the configured store (`store_kind`, `store_root`, `store_id`); rows for another store are reported, not touched, and `backup sync --forget-store ` is the explicit opt-in to drop them: it deletes that root's catalog rows outright, leaving no DELETED tombstone, so no deletion duty ever acts on that store's bytes. Sync reports a row whose manifest is gone and removes it only with `--prune-missing`, outright, as `--forget-store` removes rows: no DELETING or DELETED tombstone is left and no object is deleted, so on host-local roots a prune on the instance that lacks the bytes leaves them on the other, where a later sync adopts them again (revision item 34); without a resolving option (PR 10) it never modifies CREATING, DELETING, DELETED or FAILED rows nor adopts their prefixes, and it never adopts a prefix without a manifest, one whose sidecar does not match, or one whose listing lacks a data file the manifest lists or holds it at another size. Neither Describe nor Restore reads the store for an ARN that has no row: a backup that exists only in the store is BackupNotFound until `backup sync` adopts it, so these rules are the only path from a prefix to an AVAILABLE row. ListBackups lists rows of another store like any other (the response shape has no marker), and Describe answers such a row by its lifecycle state as it answers any row (AVAILABLE described, DELETING and DELETED as DELETED, CREATING as CREATING, FAILED not found), with no warning: the store a row belongs to is the operator's concern (`backup list` prints every row's store), not the client's. DeleteBackup and Restore check the row's own state first, as for a row of the configured store (a CREATING row is BackupInUseException, a DELETED or FAILED row BackupNotFoundException, and for Restore a DELETING row too), and then refuse what is left, an AVAILABLE row and for DeleteBackup a DELETING one (whose deletion only its own store's server can finish), with a ValidationException saying the backup is in a store other than the configured one, without naming either root. No API error names a store root. (h) Trust boundary of sync: a principal that can write under an account's prefix can forge a backup for that account. Checksums do not help, they are forgeable by the same principal. The deployment guide states that write access to an account's prefix is backup-administrator authority for that account, and gives per-account IAM prefix policies. A server-side MAC over manifests is not in v1; if hostile writers with bucket access are in scope for a deployment, the operator restricts write access rather than relying on ExtendDB to detect forgeries. -(i) Authority: the catalog migration PR 6 adds (PostgreSQL 005, SQLite schema, the MongoDB `backup_index` collection, `tables` fields and version step, and Cassandra V007, each writing -catalog 0.0.6) -the `BackupCatalog` trait of section 3 that each backend implements, and the engine's `backup` module (`crates/engine/src/backup/`), which binds it; this section is the specification they implement. The MongoDB version step and the mixed-version rule are PR 6's section. +(i) Authority: the catalog migration PR 6 adds (migration id 6, `backup_catalog_index`, on every backend, through the numbered runner that PostgreSQL and SQLite use and #681 put Cassandra and MongoDB on; PR 9a's `restore_progress` is id 7), the `BackupCatalog` trait of section 3 that each backend implements, and the engine's `backup` module (`crates/engine/src/backup/`), which binds it; this section is the specification they implement. The mixed-version rule is PR 6's section. PR 9 adds migration id 7, `restore_progress` (`MigrationId::RestoreProgress`), a `StoreKind::Data` body on PostgreSQL only (`data_migrations/0007_restore_progress.sql`), declared `NotApplicable` on SQLite, MongoDB and Cassandra, which keep a store-backed restore's progress on its job record (SQLite's `table_restores` row, MongoDB's `restore_job` field, Cassandra's catalog `tables` row) (PR 9a's section). ### 5. Operations @@ -116,29 +114,29 @@ Per operation, the contract: the request checks in their order, the error each r #### 5.1 CreateBackup -1. Validate the request as #384 does and in its order, every check before the permit: `TableName` resolved (a table ARN is accepted), the table exists and is ACTIVE for the account (`TableNotFoundException`, "Table not found: ", otherwise, whether or not a permit is free), the table name fits the store prefix (`BackupArn`: a name over 255 bytes is refused with a ValidationException naming the limit). #384's `refuse_backup_of_vector_table` is not among the off-host path's checks (the manifest records vector indexes, section 1); the engine keeps the refusal in front of `LegacyBackupReader::create`, the path of every backend without `[backup.store]` and of Cassandra always (section 7), and the tests that change are PR 8's and PR 9a's sections. A second CreateBackup on a table that already has a CREATING backup is allowed and takes its own snapshot; the `BackupInUseException` the service models on CreateBackup is not raised. Then reserve a job slot: `max_concurrent_jobs` is a per-instance semaphore the request `try_acquire`s (the maintenance claim of section 4 is the cross-instance exclusion), and the permit lives with the job; if none is free return `LimitExceededException` before opening anything, because a queued snapshot would hold a PostgreSQL transaction or a MongoDB snapshot session while it waited; RestoreTableFromBackup and the legacy create take a permit from the same semaphore and return the same error (5.3 step 1, section 3). The default is 4 (section 7), so one running restore does not turn every other backup operation on the instance into `LimitExceededException`. -2. Open the table snapshot (`SnapshotEngine::open_table_snapshot`) in the request. The pin happens during the request, so the backup holds every write acknowledged before the request was received, which is stricter than the service's one-minute window. Mint the backup id from the millisecond the open returned, just after the pin (`format_backup_id`), and the owner generation, then, through `BackupCatalog::claim_backup` (section 3), insert the catalog row with status CREATING, `created_at` = the id's instant, `owner_generation`, `heartbeat_at` = the database's `now()`, and `source_table_details_json` taken from the snapshot's description; then check the maintenance claim (section 4): if it is held, set the row FAILED with reason `maintenance`, release the snapshot, the lock and the permit, and return `LimitExceededException`; otherwise return `BackupDetails` with BackupStatus CREATING. Everything from the open to the hand-off in step 3 runs in a detached task whose result the request awaits, so a client that disconnects mid-request cannot leave a row whose owner was never spawned; if the row insert fails the snapshot is dropped, the lock released, the permit returned, and nothing was written. +1. Validate the request as #384 does and in its order, every check before the permit: `TableName` resolved (a table ARN is accepted), the table exists and is ACTIVE for the account (`TableNotFoundException`, "Table not found: ", otherwise, whether or not a permit is free), the server's region fits a backup ARN (`check_backup_arn_fits`: a `server.region` outside `[a-z0-9-]` is a ValidationException naming the setting, since no backup ARN can be formed for any table under it), and on the store-backed path the table name fits a store prefix (`check_store_prefix_fits`: a name over 255 bytes or made only of dots, the store-prefix rules, is a ValidationException naming the rule, applied before the ARN is formed so the limit is named whatever the name's length; the in-catalog create writes no prefix and applies no name rule, as the service backs up any table it can create: a table named `...` backs up, measured). #384's `refuse_backup_of_vector_table` is not among the off-host path's checks (the manifest records vector indexes, section 1); the engine keeps the refusal in front of `LegacyBackupReader::create`, the path of every backend without `[backup.store]` and of Cassandra always (section 7), and the tests that change are PR 8's and PR 9a's sections. A second CreateBackup on a table that already has a CREATING backup is allowed and takes its own snapshot; the `BackupInUseException` the service models on CreateBackup is not raised. Then, on the store-backed path, the store must be healthy (section 7: a store that is unhealthy or whose first check is pending refuses with the ValidationException naming its kind and the category, before the permit); then reserve a job slot: `max_concurrent_jobs` is a per-instance semaphore the request acquires a permit from, waiting up to 30 s for one (`JobPermit::acquire_within`, `PERMIT_WAIT`; the bound the PostgreSQL and MongoDB restore slots of earlier releases waited, and the service accepts bursts over the count: five restores and six CreateBackup calls sent together were all accepted; the maintenance claim of section 4 being the cross-instance exclusion), and the permit lives with the job; if none is freed within the bound return `LimitExceededException` before opening anything. The request holds nothing while it waits, since the permit is the last of its checks and the snapshot is opened after it; RestoreTableFromBackup and the legacy create take a permit from the same semaphore and return the same error (5.3 step 1, section 3). The default is 4 (section 7), so one running restore does not turn every other backup operation on the instance into `LimitExceededException`. +2. Open the table snapshot (`SnapshotEngine::open_table_snapshot`) in the request. The pin happens during the request, after the request's checks and after any wait for a permit in step 1 (up to 30 s), so the backup is the table at the open: it holds every write acknowledged before the open and none acknowledged after it, a write acknowledged while the request waited for a permit included (measured in PR 8's second review round: a `PutItem` acknowledged 1.01 s after a `CreateBackup` that waited for a permit was in the backup, 2 of 2 runs; the service, measured in PR 8's review, held every write acknowledged before `CreateBackup` was sent and the 18 acknowledged in the 0.1 s up to `BackupCreationDateTime`, and none of the 11,754 acknowledged in the 75 s that followed: an instant inside the request there too). Mint the backup id from the millisecond the open returned, just after the pin (`format_backup_id`), and the owner generation, then, through `BackupCatalog::claim_backup` (section 3), insert the catalog row with status CREATING, `created_at` = the id's instant, `owner_generation`, `heartbeat_at` = the database's `now()`, and `source_table_details_json` the manifest's `source_table` object built from the snapshot's description and the table's TTL, which is read beside the open and not under the snapshot (`extenddb_storage::snapshot::source_table_schema`; its `item_count` and `table_size_bytes` are the description's figures, the stats pass's, until the AVAILABLE write replaces the object with the manifest's, whose count is the stream's and whose size is the DynamoDB sizes of the items written, summed, so DescribeBackup's `ItemCount` and `TableSizeBytes` are the table's figures as last counted while CREATING and the backup's exact figures once AVAILABLE); then return `BackupDetails` with BackupStatus CREATING. The maintenance claim (section 4) is checked by the request before the permit, as the in-catalog create checks it (revision item 29), so a held claim refuses the request with `LimitExceededException` and inserts no row: a row written FAILED with reason `maintenance` for the refusal carried the insert's heartbeat, and `backup purge`'s wait counted it as a live job for the stale window (revision item 32). The open and the insert are selected against the shutdown token, so a server that is stopping neither waits on an open behind a table's lock nor inserts a row; the request's `ServiceUnavailable` for that case reaches a client only when the token is cancelled while the listener still serves, which is a registration loss (the restartable worker set stopping, `BackupRuntime::cancel_jobs`), since on SIGTERM the listener's 5 s connection drain closes the connection before `drain_then_release` cancels the jobs' token (measured: a `CreateBackup` waiting in the open at SIGTERM saw its connection closed 8.2 s after sending). Everything from the open to the hand-off in step 3 runs in a detached task whose result the request awaits, so a client that disconnects mid-request cannot leave a row whose owner was never spawned; if the row insert fails the snapshot is dropped, the lock released, the permit returned, and nothing was written. 3. The claim: section 4(e) states the rules; what the claim is on each backend, the claim connection's settings and the hand-off to the worker are PR 6's and PR 8's sections. -4. The job streams items from the snapshot into gzip data files through `DataFileEncoder`; a file closes once its `uncompressed_bytes` reaches `data_file_target_bytes` (default 64 MiB), so a file exceeds the knob by less than one line, at most `MAX_LINE_BYTES` (32 MiB, section 1), which is also the knob's minimum. Every item is checked, before it is encoded, with the item validator PR 8 adds to `extenddb-core` (`extenddb_core::validation::validate_restorable_item`, with this instance's `max_item_size_bytes`; the restore binds the same function, 5.3 step 5), so no backup that no restore can load is reported AVAILABLE; that is a regression from main for a table that already holds a value #618 let in (Drawbacks), and the reason reaches the operator only (`backup list --failed` and the log); the client sees CREATING and then BackupNotFound. The heartbeat is written on a timer (`heartbeat_interval_secs`, default 15) for as long as the job task is alive, independent of upload progress; before each put the job checks that its latest heartbeat confirmed the claim within `heartbeat_interval_secs` and starts no put otherwise; a heartbeat that does not confirm the claim, or that has not returned within `heartbeat_interval_secs`, aborts the job without further store writes, deletes included. `stale_job_after_secs` must exceed several heartbeat intervals; startup refuses a value under `3 * heartbeat_interval_secs`, which is what makes that detection precede a reconciler's deletion. After the last file: write `manifest-files.json`, `manifest-summary.json`, `extenddb-manifest.sha256` (the order #394's `BackupDocuments::build` doc gives); re-check the claim; then write `extenddb-manifest.json` and update the row to AVAILABLE with size and count (`JobClaim::set_status(CREATING, AVAILABLE)`, conditional on the generation and on `status = CREATING`). An update whose outcome is unknown is followed by a re-read of the row, not treated as a failure; one that matches zero rows ends the job with no store access. Store writes are not fenced by the generation; the re-check narrows the window in which a fenced-out job can still write a manifest, and the FAILED tombstone (section 4) closes what remains: sync never adopts a prefix whose row is FAILED, and the reconciler never deletes a prefix while the row's `heartbeat_at` is younger than `stale_job_after_secs` on the catalog database's clock, so a late manifest from a fenced-out job lands under a tombstone that sync will not adopt. -5. On any failure, including `SnapshotExpired`: set the row FAILED with the reason, conditional on the generation and on `status = CREATING`, so a job whose AVAILABLE update committed with its response lost cannot turn a backup a client has seen AVAILABLE into a deleted one. Only if that write reports exactly one row matched does the job stop its uploads, await its in-flight puts and `delete_prefix` the partial backup (which on S3 also aborts any upload still open under it); a write that matches zero rows, or any error on the claim connection, ends the job with no further store access of any kind, because the holder of the claim owns the prefix. If the delete fails the reconciler retries it (section 4). A backup never resumes from a cursor position, so a lost snapshot means a new CreateBackup. DescribeBackup returns BackupNotFound; ListBackups omits it. -6. Shutdown: the snapshot lives in the process and dies with it, so a cancelled job can never complete. On cancellation the worker runs step 5 with reason `shutdown`, and the server's shutdown awaits each job's failure path within the drain window before the runtime stops: runtime shutdown sends no abort for a put still in flight (section 2), so step 5's `delete_prefix` is what sweeps its upload. If the catalog is unreachable it leaves the row CREATING for the reconciler; if only the store is unreachable the row is FAILED and the reconciler deletes the prefix later. Crash: on startup and every reconciler cycle, a CREATING row is resolved when its owner is dead, by the test and the claim of section 4(e); the prefix of a FAILED row is deleted only once `heartbeat_at` is older than `stale_job_after_secs`, the delay step 4 relies on. A restarted process cannot finish a CREATING backup; AVAILABLE from an existing manifest or FAILED are the only two outcomes. +4. The job streams items from the snapshot into gzip data files through `DataFileEncoder`; a file closes once its `uncompressed_bytes` reaches `data_file_target_bytes` (default 64 MiB), so a file exceeds the knob by less than one line, at most `MAX_LINE_BYTES` (32 MiB, section 1), which is also the knob's minimum. Every item is checked, before it is encoded, with the item validator PR 8 adds to `extenddb-core` (`extenddb_core::validation::validate_restorable_item`, with this instance's `max_item_size_bytes`, hash and range key sizes and attribute name size (`RestorableLimits`); the restore binds the same function, 5.3 step 5), so no backup that no restore can load is reported AVAILABLE. An item over the default item limit has its data file line measured against the line cap through a counting writer that holds no line, and one over the cap (only possible above the default item limit) is refused through the same count-and-log path as a #618 item (`RestorableItemReason::LineTooLong`, whose reason names the cap, the line's own length being unknown once the count stops); that is a regression from main for a table that already holds a value #618 let in (Drawbacks), and the reason reaches the operator only (`backup list --failed` and the log); the client sees CREATING and then BackupNotFound. The heartbeat is written on a timer (`heartbeat_interval_secs`, default 15) for as long as the job task is alive, independent of upload progress; before each put the job checks that no loss has been raised and that its latest heartbeat confirmed the claim within two periods (one for the timer, one for the beat to return: a beat sent at the tick returns up to one period later) or that a beat is in flight and has not failed, and starts no put otherwise (the in-flight clause is for a beat parked outside its bound on SQLite's engine write lock, behind a writer holding it past two periods: there the beat waits on this process's own queue, one process serves the file and the reconciler's liveness test is the in-process slot, so a parked beat cannot coincide with a takeover; on the other backends the bound covers the whole call and the clause holds only within one period of the beat's start, the backend saying which through `JobClaim::heartbeat_parks_before_its_bound`, since a process stopped mid-beat and resumed keeps the beat in flight until its timer runs again, a window of tens of microseconds in which another owner may hold the row (the maintainer measured 64 and 85 µs of passing checks with the clause unbounded); a beat that returns Lost, fails or outlives its bound raises the loss, which the check reads first, revision item 32); the claim's insert stands for the first confirmed beat (the job takes the instant before `claim_backup` and the heartbeat's watch starts from it), so the maintenance check between the insert and the job's start counts against the two periods and a first put that comes later than two periods after the insert, with no beat in flight, is refused as a lost claim, the row left to the reconciler; a heartbeat that does not confirm the claim, or that has not returned within one period, aborts the job without further store writes, deletes included, and without a catalog write either, the claim connection being the one that failed (the row is left to the reconciler). `stale_job_after_secs` must exceed several heartbeat periods; startup refuses a value under three times the period the jobs run at, the larger of `heartbeat_interval_secs` and the 15 s constant `extenddb_storage::backup_catalog::JOB_HEARTBEAT_INTERVAL` (so 45 s at the least): the store-backed job heartbeats at that larger value, the in-catalog paths at the constant until PR 14 removes them; two periods stay inside three, so a job whose claim connection has died stops its puts before a reconciler's deletion; on MongoDB and Cassandra a process stopped across the window (SIGSTOP) resumes the put it had in flight after the takeover, which the FAILED tombstone and the FAILED-prefix retry of step 5 and section 4 cover; on PostgreSQL its claim session holds the advisory lock while it is stopped, so no takeover happens while the session lives; on resume the job either completes under its own generation or, when the stop outlasted two heartbeat periods (one, with a beat in flight at the stop), stops as a lost claim with no catalog write (the beat in flight is reported late, or the next put finds the last confirmed beat over two periods old), its session closes and the reconciler resolves the row (4(e)). After the last file: write `manifest-files.json`, `manifest-summary.json`, `extenddb-manifest.sha256` (the order #394's `BackupDocuments::build` doc gives); re-check the claim; then write `extenddb-manifest.json` and update the row to AVAILABLE with size and count (`JobClaim::set_status(CREATING, AVAILABLE)`, conditional on the generation and on `status = CREATING`). An update whose outcome is unknown is followed by a re-read of the row, not treated as a failure (4(e): one more write when the row is still CREATING under the job's generation); one that matches zero rows ends the job with no store access. Store writes are not fenced by the generation; the re-check narrows the window in which a fenced-out job can still write a manifest, and the FAILED tombstone (section 4) closes what remains: sync never adopts a prefix whose row is FAILED, and the reconciler never deletes a prefix while the row's `heartbeat_at` is younger than `stale_job_after_secs` on the catalog database's clock, so a late manifest from a fenced-out job lands under a tombstone that sync will not adopt. +5. On any failure, including `SnapshotExpired`: set the row FAILED with the reason, conditional on the generation and on `status = CREATING`, so a job whose AVAILABLE update committed with its response lost cannot turn a backup a client has seen AVAILABLE into a deleted one. The pipeline ends its puts before the FAILED write (the failing put has ended, and a stopped encoder ends the body of the one running, which the store then fails or completes; nothing is started after the failure) and the feeder stops reading at once when the uploader fails, so the write does not wait for the snapshot stream's next page, and there is no put in flight at the write; only if that write reports exactly one row matched does the job `delete_prefix` the partial backup (which on S3 also aborts any upload still open under it); a write that matches zero rows, or any error on the claim connection, ends the job with no further store access of any kind, because the holder of the claim owns the prefix. If the delete fails the reconciler retries it (section 4). A backup never resumes from a cursor position, so a lost snapshot means a new CreateBackup. DescribeBackup returns BackupNotFound; ListBackups omits it. +6. Shutdown: the snapshot lives in the process and dies with it; a job cancelled before its manifest put never completes; one cancelled after it completes AVAILABLE, since the manifest is the commit record. On cancellation the worker runs step 5 with reason `shutdown`, and the server's shutdown awaits each job's failure path within the drain window before the runtime stops: runtime shutdown sends no abort for a put still in flight (section 2), so step 5's `delete_prefix` is what sweeps its upload. If the catalog is unreachable it leaves the row CREATING for the reconciler; if only the store is unreachable the row is FAILED and the reconciler deletes the prefix later. Crash: on startup and every reconciler cycle, a CREATING row is resolved when its owner is dead, by the test and the claim of section 4(e); the prefix of a FAILED row is deleted only once `heartbeat_at` is older than `stale_job_after_secs`, the delay step 4 relies on; on a row the reconciler itself just failed that window is counted from the dead owner's last beat, so the FAILED-prefix duty of the same cycle, which runs after the dead-owner duty, deletes it as soon as that beat is stale. A restarted process cannot finish a CREATING backup; AVAILABLE from an existing manifest or FAILED are the only two outcomes. #### 5.2 DescribeBackup, ListBackups, DeleteBackup Served from the catalog index with the legacy fallback. DescribeBackup reads the row by the ARN's account and backup id (`BackupCatalog::read_backup_row`) and returns BackupNotFoundException unless the row's `account_id`, `table_name` and `backup_arn` (after the region re-spelling of section 4) equal the request's, as 5.3 step 1 does. ListBackups implements the full request as #384 implements it (the members, the (`created_at`, `backup_arn`) order and the cursor rules with #384's two exceptions), except that `TableName` may be a table name or a table ARN, as the service accepts it (measured on DynamoDB) and as main's CreateBackup already does; the differences doc's `ListBackups` paging row changes with PR 6 (section 10). -DeleteBackup on a CREATING row returns `BackupInUseException`, as on the service; while the backup is in use likewise; on a DELETING row it resumes the deletion inline; on a DELETED or FAILED row it returns `BackupNotFoundException`, as Restore does; on a row of another store it returns the ValidationException of section 4, and while the configured store is unhealthy, on a row of that store the ValidationException of section 7. A backup is in use while a restore target registered from it is CREATING (`BackupCatalog::backup_in_use`), main's rule on every backend, which 5.3 step 1 applies to a second restore of the backup; the rule reads the target's status, not the job's, so on MongoDB it also covers the window between the job's DONE write and the worker's ACTIVE flip (5.3 step 6). DeleteBackup runs the in-use check and its DELETING write in one transaction under the backup row's lock (`BackupCatalog::start_backup_delete`), and a restore's registration (5.3 step 4) re-checks the in-use rule and `status = AVAILABLE` under the same lock, so the two order wholly one before the other, as on main: a DeleteBackup that wins leaves the registration a row that is not AVAILABLE (`BackupNotFoundException`, no target left), and a registration that wins makes the DeleteBackup return `BackupInUseException`; so no restore loads from a backup a DeleteBackup has started to delete, on any backend. Otherwise: set the row DELETING (a tombstone sync honours), delete `extenddb-manifest.json` first so the prefix is no longer a complete backup to any reader (`delete_prefix` on the manifest's exact key, section 2), then `delete_prefix` for the rest, then set the row DELETED (`set_backup_deleted`) once a `list` of the prefix returns nothing; if any step fails the row stays DELETING and the API returns InternalServerError, and the reconciler finishes the deletion later (section 4). The request returns the description with BackupStatus DELETED after the row reaches DELETED. +DeleteBackup on a CREATING row returns `BackupInUseException`, as on the service; while the backup is in use likewise; on a DELETING row it resumes the deletion inline; on a DELETED or FAILED row it returns `BackupNotFoundException`, as Restore does; on a row of another store it returns the ValidationException of section 4, and while the configured store is unhealthy, on a row of that store the ValidationException of section 7. A backup is in use while a restore target registered from it is CREATING (`BackupCatalog::backup_in_use`), main's rule on every backend, which 5.3 step 1 applies to a second restore of the backup; the rule reads the target's status, not the job's, so on MongoDB it also covers the window between the job's DONE write and the worker's ACTIVE flip (5.3 step 6). DeleteBackup runs the in-use check and its DELETING write in one transaction under the backup row's lock (`BackupCatalog::start_backup_delete`), and a store-backed restore's registration (5.3 step 4) re-checks the in-use rule and `status = AVAILABLE` under the same lock (the legacy registration keeps the `backups` row's), so the two order wholly one before the other, as on main: a DeleteBackup that wins leaves the registration a row that is not AVAILABLE (`BackupNotFoundException`, no target left), and a registration that wins makes the DeleteBackup return `BackupInUseException`; so no restore loads from a backup a DeleteBackup has started to delete, on any backend. Otherwise: set the row DELETING (a tombstone sync honours), delete `extenddb-manifest.json` first so the prefix is no longer a complete backup to any reader (`delete_prefix` on the manifest's exact key, section 2), then `delete_prefix` for the rest, then set the row DELETED (`set_backup_deleted`) once a `list` of the prefix returns nothing; the store calls run under the store's check bound and in-flight rule (section 7); the request answers the description with BackupStatus DELETED once the DELETING write has committed, whatever the deletion does (every reader reports the backup deleted from that write on, and the service has no counterpart for a failed object deletion); if a step fails the row stays DELETING, the failure is logged, and the reconciler finishes the deletion later (section 4) (revision item 30). #### 5.3 RestoreTableFromBackup -The request runs steps 1, 2 and 4 and returns CREATING in milliseconds plus the manifest parse, as DynamoDB does; the job runs steps 3, 5 and 6. The request's own awaits are the checks of step 1 in their order, the permit, the manifest read and the validation of step 2; the step 4 transaction and the spawn of the job run together in a detached task whose result the request awaits, so a client disconnect cannot commit the table and the RUNNING row without an owner. The table exists from the response onwards, so no second restore or CreateTable can take the name while files download, but it accepts no reads or writes until every file has verified and loaded. +The request runs steps 1, 2 and 4 and returns CREATING in milliseconds plus the manifest parse, as DynamoDB does; the job runs steps 3, 5 and 6. On every backend the request's wait for step 4 is bounded (`RESTORE_REGISTRATION_TIMEOUT`, 60 s, the engine's constant; a connection pool that has stopped handing out connections is the case that meets it): past it the request answers `InternalServerError` while the detached task goes on, so the target can appear after the error, holding the backup in use and the name until the load ends, and DescribeTable is where the client finds it (the differences doc's row on how `RestoreTableFromBackup` runs states it for operators). There are two detaches. Once the request is authenticated and authorized, its whole body (the checks of step 1 in their order, the permit, the manifest read and the validation of step 2, then step 4) runs in a task of its own, so a client disconnect after authorization cancels nothing (#384's rule); inside it, the step 4 transaction and the spawn of the job run together in a second detached task whose result the request awaits under the 60 s bound, so the bound can return while the registration goes on and a client disconnect cannot commit the table and the RUNNING row without an owner. The table exists from the response onwards, so no second restore or CreateTable can take the name while files download, but it accepts no reads or writes until every file has verified and loaded. -1. Check the request in the order main uses and the differences doc records, every refusal that needs no store access before the permit: the overrides' shape (below); the ARN with `BackupArn` (the account must equal the caller's before any store access); the target name (`TableAlreadyExistsException` when a table of that name exists, `TableInUseException` when it is a CREATING restore target); the backup row, read by the ARN's account and backup id and accepted only when its `account_id`, `table_name` and `backup_arn` (after the region re-spelling of section 4) equal the request's, or, if there is no such row, resolved through the `LegacyBackupReader` (section 3): an ARN with neither is `BackupNotFoundException`, so an ARN of account B that carries the id of account A's backup is BackupNotFound and never A's row, a CREATING row returns `BackupInUseException`, a backup in use (the rule of 5.2) returns `BackupInUseException` naming that table, a DELETING, DELETED or FAILED row returns `BackupNotFoundException`, a row of another store returns the ValidationException of section 4 and a row of an unhealthy configured store the one of section 7, none of them reading the store; then, last, the job permit (`LimitExceededException` when none is free). Only then read `extenddb-manifest.sha256` and `extenddb-manifest.json`, verify the sidecar against the manifest bytes, then `format_version`, the ARN and its agreement with `source_table`, and the data-file list rules (`ExtenddbManifest::from_json_bytes`). A manifest missing behind an AVAILABLE row returns `BackupNotFoundException` and logs the row for `backup sync --prune-missing`. +1. Check the request in the order main uses and the differences doc records, every refusal that needs no store access before the permit: the overrides' shape (below); the ARN with `BackupArn` (the account must equal the caller's before any store access); the target name's syntax; the backup row, read by the ARN's account and backup id and accepted only when its `account_id`, `table_name` and `backup_arn` (after the region re-spelling of section 4) equal the request's, or, if there is no such row, resolved through the `LegacyBackupReader` (section 3): an ARN with neither is `BackupNotFoundException`, so an ARN of account B that carries the id of account A's backup is BackupNotFound and never A's row, a CREATING row returns `BackupInUseException`, a DELETING, DELETED or FAILED row returns `BackupNotFoundException`; then the target name under the account (`TableAlreadyExistsException` when a table of that name exists, `TableInUseException` when it is a CREATING restore target), after the backup is known to exist and before its in-use rule and the overrides are checked, the order the service uses (a restore of a missing backup into an existing table, into a table CreateTable is still creating or into a CREATING restore target is `BackupNotFoundException`, an in-use backup into an existing table `TableAlreadyExistsException` and a backup into its own CREATING target `TableInUseException`, all measured); then, for a row, a backup in use (the rule of 5.2) returns `BackupInUseException` naming the backup, a row of another store returns the ValidationException of section 4 and a row of an unhealthy configured store the one of section 7, none of them reading the store; then, last, the job permit (`LimitExceededException` when none is freed within the 30 s wait of 5.1). Only then read `extenddb-manifest.sha256` and `extenddb-manifest.json`, verify the sidecar against the manifest bytes, then `format_version`, the ARN and its agreement with `source_table`, and the data-file list rules (`ExtenddbManifest::from_json_bytes`). The request's sidecar and manifest reads collect at most 4 KiB and 16 MiB; an object over its bound behind an AVAILABLE row is a `ValidationException` naming a corrupt prefix, as a bad sidecar is, and a resume meeting one is FAILED with the same reason. A manifest missing behind an AVAILABLE row returns `BackupNotFoundException` and logs the row for `backup sync --prune-missing`. 2. Validate that the target backend supports every feature in `source_table` that a restore reproduces (a non-empty `vector_indexes` into a backend without vector support is the one case, section 1); refuse before creating anything otherwise. Unlike #384 (`ensure_single_part_base_key`), this path restores multi-part-key tables (as `unordered` backups, section 1): `bulk_load` keys each row and derives its index rows with the functions the backend's item write path uses (section 3), so a restored multi-part-key table is laid out as a natively created one. -3. (Job.) Write the verified manifest and sidecar to the spool, then download every data file to the spool (PR 7's section), hashing each as it arrives and comparing with the manifest's `sha256`, `md5` and `bytes`; on a filesystem store the files are verified in place and no copy is made. A transport error during a download is retried with backoff for up to 15 minutes (a constant, not a knob in v1), the row kept RUNNING and heartbeated, before the restore fails (step 7); a checksum or size mismatch is not retried and fails the restore, naming the file, before any item has been written. Spooling everything before loading is what makes "no item is written before every checksum verifies" true for the second file as well as the first. The spool's `.verified` markers, `spool_host` and the resume rules are PR 9a's section. -4. Create the target table and the `restore_jobs` row (RUNNING, with the job's claim, `kind = manifest` and the store the backup row names; section 4) in one catalog transaction (PostgreSQL and SQLite), one session transaction (MongoDB) or, on Cassandra, one `INSERT ... IF NOT EXISTS` of the `tables` row, whose columns are the job row, so no crash leaves a CREATING table without a job row. If another CreateTable or restore took the name between steps 1 and 4, the request returns `TableAlreadyExistsException` (or `TableInUseException` when the winner is itself a CREATING restore target) and the job row insert rolls back with it; CreateTable keeps `ResourceInUseException`. The registration (`BackupCatalog::claim_restore`) re-checks inside its transaction, under the backup row's lock that DeleteBackup's `start_backup_delete` takes (5.2), that the backup row is still AVAILABLE and that no restore target registered from it is CREATING: a row that is no longer AVAILABLE returns `BackupNotFoundException`, a backup in use returns `BackupInUseException`, and no target is left. After the transaction commits the job checks the maintenance claim (section 4); if it is held, the job runs step 7 with reason `maintenance` and the request returns `LimitExceededException`. Otherwise return `TableDescription` with TableStatus CREATING. Overrides are applied here (below). A restore the reconciler resumes is a job and takes a `max_concurrent_jobs` permit like a new one; when none is free the row is skipped until the next cycle. -5. (Job, after step 3.) Per data file: read the verified file through #394's `ExtenddbManifest::reader_prehashed`, apply the per-file key-range check of section 1 when the backup is key-ordered, validate every item with PR 8's item validator with the target's limits and fail the restore on the first invalid item, then write the file through `bulk_load(target_table_id, claim, file_index, start_at, items)` (section 3), with `start_at` the recorded `items_applied` of a file a resume stopped in and 0 otherwise. Each backend writes base rows and index rows together and records progress so that a crash cannot leave an item applied twice: in one transaction with the data on PostgreSQL and SQLite, and by idempotent re-application of the one unrecorded file or batch on MongoDB and Cassandra (PR 9a's section: the batch sizes and progress records). The restored table has no streams and no TTL, so there is nothing to bypass. -6. (Job.) After all files, and for an unordered backup after the count of section 1 matched, the job flips the target to ACTIVE conditional on the table still being CREATING as a restore target and on the job's `owner_generation`; on PostgreSQL and SQLite the table and every index of the target flip in one catalog transaction, which also marks the `restore_jobs` row DONE; on Cassandra the index rows are created ACTIVE with the table and the lightweight transaction on the `tables` row flips the table and marks the job row DONE. A Query on each GSI, and SearchVectors where the target supports it, therefore sees every restored item as soon as the target reports ACTIVE. MongoDB's sequence differs, because the existing backfill worker, not the job, promotes the indexes: the job marks the `restore_jobs` row DONE (conditioned on its generation and on RUNNING) and sets `restore_backfill_pending` in one session transaction, and the job task ends there; the worker promotes each index and then flips the table to ACTIVE, conditioned on `restore_backfill_pending` and on CREATING, not on the job's generation; and the reconciler never treats a row with `restore_backfill_pending` as a dead RUNNING job (the row is DONE). DeleteTable on a restore target is refused with `ResourceInUseException`, as #384 does and as the service does, which is why the job and the reconciler remove a target through `remove_restore_target` (step 7); there is no user-facing way to cancel a running restore. UpdateTable on a CREATING table is refused as today. -7. Failure (verification or load): the job marks the `restore_jobs` row FAILED with the reason, conditional on its generation. Only if that write matched exactly one row does the job remove the partially created table, through `BackupCatalog::remove_restore_target` (section 3), keyed by the target's account, name and table id and conditional on the table still being a CREATING restore target, which also deletes its progress rows, never through the DeleteTable API; a write that matches zero rows ends the job with only its spool removed, because the claim holder owns the table. DescribeTable then returns ResourceNotFound. Crash mid-restore: the reconciler claims RUNNING `restore_jobs` rows whose owner is dead (section 4(e)) and resumes from the recorded progress, reading the store the job row records if it is the configured store, failing the restore and naming `backup sync --move-store` if it is not, and resuming nothing while the configured store is unhealthy; on PostgreSQL only once the row's `heartbeat_at` is older than `stale_job_after_secs`, the delay the prefix deletion already waits, since a free advisory lock shows that the owner's catalog session ended, not that its process stopped, and the restore's data writes carry no generation; the owner aborts at its next heartbeat, inside that window. A legacy RUNNING restore whose owner is dead is marked FAILED and its target removed, never resumed (section 3). Shutdown: a restore job stops at its next batch without writing a status, so step 7 does not run; its row stays RUNNING with its spool and its progress, for a successor once the owner is dead by its backend's rule. +3. (Job.) Write the verified manifest and sidecar to the spool, then download every data file to the spool (PR 7's section), hashing each as it arrives and comparing with the manifest's `sha256`, `md5` and `bytes`; on a filesystem store the files are verified in place, with no spool: a hashing `get` pass over every file before any load, then a second `get` of each file as it is loaded, hashed again through `ExtenddbManifest::reader`, so a file replaced between the passes fails in the load; on a filesystem store the load's second `get` hands items to `bulk_load` as they decode and checks the digest at the end of the stream, so the items of a file replaced between the two passes may be written to the CREATING target (committed, on a backend that commits per batch) before the end-of-stream check fails the file and the failure path removes the target; the store root's owner-only mode (section 2) is the guard. A `StoreError::Transport` during a download (at the `get` or mid-body) is retried for up to `DOWNLOAD_RETRY_BOUND`, 15 minutes (a constant, not a knob in v1), after `min(2^n s, 60 s)` plus up to a second of jitter, the row kept RUNNING and heartbeated, before the restore fails (step 7); the bound counts from the job's first download attempt, one clock for every file (`download_all` takes it once and every file's retries count against it), so a store that stays down costs a restore the bound once, not the bound per file; the failure names the file whose attempt crossed it; a filesystem store's `get` errors are `Io` and fail at once, so the retry covers S3 stores; `NotFound`, a denial, and a checksum or size mismatch are not retried and fail the restore, naming the file, before any item has been written. Spooling everything before loading is what makes "no item is written before every checksum verifies" true for the second file as well as the first. On an S3 store the manifest and its sidecar are written to the spool entry before the first download, each as a data file is (`.partial`, synced, renamed), so a crash during the write leaves the earlier document or none and never a truncated one; a resume that read the pair from the spool does not write it again, and a spooled pair that fails verification (the sidecar, the parse, the record's ARN) is replaced by the store's, with one WARN line. The spool's `.verified` markers, `spool_host` and the resume rules are PR 9a's section. +4. Create the target table and the `restore_jobs` row (RUNNING, with the job's claim, `kind = manifest` and the store the backup row names; section 4) in one catalog transaction (PostgreSQL and SQLite), one session transaction (MongoDB) or, on Cassandra, one `INSERT ... IF NOT EXISTS` of the `tables` row, whose columns are the job row, so no crash leaves a CREATING table without a job row. If another CreateTable or restore took the name between steps 1 and 4, the request returns `TableAlreadyExistsException` (or `TableInUseException` when the winner is itself a CREATING restore target) and the job row insert rolls back with it; CreateTable keeps `ResourceInUseException`. The registration (`BackupCatalog::claim_restore`) re-checks inside its transaction, under the `backup_index` row's lock that DeleteBackup's `start_backup_delete` takes (5.2: PostgreSQL `SELECT ... FOR UPDATE`, SQLite the `BEGIN IMMEDIATE` write transaction, MongoDB `$inc lock_seq` on the document, Cassandra the `active_restores` lightweight transaction on the backup's partition), that the backup row is still AVAILABLE and that no restore target registered from it is CREATING: a row that is no longer AVAILABLE returns `BackupNotFoundException`, a backup in use returns `BackupInUseException`, and no target is left. The task then sends one heartbeat through the claim and takes its return as the instant the heartbeat's two-period rule counts from (`heartbeat.rs`), checks the maintenance claim, and replies: a first beat that finds the claim lost, or fails, runs step 7 with reason "the claim was lost before the job started" and the request returns `InternalServerError` (4(e)); a held maintenance claim (section 4; this release's in-catalog restore checks it before the registration, revision item 29) runs step 7 with reason `maintenance` and the request returns `LimitExceededException`. Otherwise return the `TableDescription` the registration produced, with TableStatus CREATING and its `RestoreSummary`. Overrides are applied here (below). A restore the reconciler resumes is a job and takes a `max_concurrent_jobs` permit like a new one; when none is free the row is skipped until the next cycle. +5. (Job, after step 3.) Per data file: read the verified file through #394's `ExtenddbManifest::reader_prehashed` from the spool and `ExtenddbManifest::reader` in place (a filesystem store's file, hashed again as it is read), apply the per-file key-range check of section 1 when the backup is key-ordered, validate every item with PR 8's item validator with the target's limits and fail the restore on the first invalid item, then write the file through one `bulk_load(target, claim, BulkLoadFile { file_index, start_at, reapplying }, items)` (section 3) over a stream fed in chunks of at most `LOAD_CHUNK_ITEMS` (1000) items or `LOAD_CHUNK_BYTES` (32 MiB) of data-file line text (the unit the backup side budgets in; a chunk closes at the first item that takes it to the bound, so it holds at most the bound plus one line) through a channel of two chunks, with `start_at` the recorded `items_applied` of a file a resume stopped in and 0 otherwise, and `reapplying` true for the first file a resume meets and false after. Each backend writes base rows and index rows together and records progress in the write that applied the items, so that a crash cannot leave an item applied twice: in one transaction with the data on PostgreSQL (a `restore_progress` row per file in the data database, fenced by the base table's primary key and the progress row's own key rather than by the generation, which fences only the record's writes in the catalog) and SQLite (the record's `(file_index, items_applied)` per batch), and by idempotent re-application of the one unrecorded file or batch on MongoDB (the record's `file_index` per file; E11000 suppressed only on the `reapplying` file) and Cassandra (the record's progress per batch under a lightweight transaction; `put_restored_item` is an upsert). The batches: PostgreSQL one data transaction per file, the base rows in multi-row INSERTs closed at 500 rows or 4 MiB of item text (`COPY_BATCH_ITEMS`, `COPY_BATCH_BYTES`), the progress row last in the transaction; SQLite one write transaction per batch, closed at 10,000 items or at the first item that takes the batch's serialized `item_data` text past 32 MiB (`RESTORE_BATCH_ITEMS`, `RESTORE_BATCH_BYTES` in `extenddb_storage::restore_engine`), the record's progress last in the transaction; MongoDB `insert_many(ordered = false)` calls of 1,000 documents (the driver splits a call by the server's message and write-batch bounds; 1,000 is the unit a half-applied file is re-applied in), the record's `file_index` written after the file; Cassandra upserts in batches closed at 10,000 items or 32 MiB of serialized item text, the same two constants, each batch followed by the progress lightweight transaction; an empty file records its progress on every backend: `items_applied = start_at` on PostgreSQL, SQLite and Cassandra, the `file_index` alone on MongoDB, whose record carries no `items_applied` (PR 9a's section). Liveness inside a file: the feeder checks at every chunk boundary that the heartbeat confirmed the claim within two periods and ends the stream as a lost claim otherwise; on MongoDB and Cassandra `bulk_load` reads the record before each write batch (MongoDB one `find_one` on the target's `_id`, `table_id`, `restore_job.status` and `restore_job.owner_generation` before each `insert_many`; Cassandra a lightweight transaction on the record, the heartbeat refreshed `IF` RUNNING under the generation, before each batch of writes, after reading the coordinator's clock as the batch's write timestamp) and ends the file as a lost claim when the record is not RUNNING under its generation, so a writer whose record another instance took over while it stalled lands at most the one batch it was in, and on Cassandra that batch lands `USING TIMESTAMP` the reading taken before the check, below every write and delete the successor made since, so it changes nothing (the PR 9a load paragraph); on Cassandra an unknown outcome of the progress lightweight transaction (a write timeout or failure, a lost connection) is decided by a `LOCAL_SERIAL` re-read of the record: RUNNING under the generation and carrying the values written is applied; RUNNING under the generation with the earlier values is one retry, whose own unknown outcome is re-read the same way, and a second unknown outcome the record does not carry fails the restore; anything else is the lost claim. A stalled owner whose record a successor took over, flipped ACTIVE and whose clients then wrote, lands nothing that outlives the successor's state: on MongoDB the batch's `insert_many` and its progress commit in one transaction whose first write is the claim's fence, so a batch that reaches the server after the takeover is refused whole; on Cassandra the batch's writes carry the timestamp read before its claim check, below every write and delete the successor and its clients made since, so the cells land and lose to the later ones and a key a client deleted stays deleted (the stalled writer's progress write then finds the record not its own and the load stops). The rule on Cassandra needs the ExtendDB hosts' clocks to agree, as the version scheme already does. PostgreSQL's window stays as stated: nothing in its `bulk_load` conditions on the generation; the base table's primary key and the claim's liveness, confirmed before each file, are its fence, the advisory lock holds the record through a stall while the owner's session lives, and the window that remains is a session the database server dropped during a stall longer than the stale window (an `idle_session_timeout`, not a default): the stalled file's transaction can commit after a takeover, and the successor's own load of that file then fails on the duplicate key and the restore is FAILED and its target removed, a lost restore and not wrong data. The restored table has no streams and no TTL, so there is nothing to bypass. +6. (Job.) After all files, and after the count of section 1 matched, the job flips the target to ACTIVE through `finish_restore` (section 3), conditional on the table still being CREATING as a restore target and on the job's `owner_generation` (`NotMatched` otherwise: an error line and no table access); on PostgreSQL and SQLite the table and every index of the target flip in one catalog transaction, which also marks the `restore_jobs` row DONE; on Cassandra the index rows are created ACTIVE with the table and the lightweight transaction on the `tables` row flips the table and marks the job row DONE (`Active`, never `Pending`, there). A `finish_restore` that returns an error is a conditional write whose outcome is unknown and is handled as the status writes are: the record is re-read through the claim; DONE under the job's generation is the write landed and the restore complete (one WARN naming the error); RUNNING under it is the write not taken, sent once more at most and its own error re-read the same way; anything else, a second error included, is the DONE write that matched no record, after which the job makes no table access and the reconciler resumes the record once stale. `finish_restore` writes the manifest's `source_table.table_size_bytes` as the restored table's size until the statistics pass rewrites it; `DescribeTable` of a restored table reports the source table's size, not the compressed size of the backup's files. A Query on each GSI, and SearchVectors where the target supports it, therefore sees every restored item as soon as the target reports ACTIVE (on Cassandra this holds for a resumed restore's re-applied batch because the write path rewrites an unchanged item's index rows instead of deleting them, #424, the PR 9a load paragraph). MongoDB's sequence differs, because the existing backfill worker, not the job, promotes the indexes: the job marks the `restore_jobs` row DONE (conditioned on its generation and on RUNNING) and sets `restore_backfill_pending` in one session transaction (`finish_restore` returns `Pending`), and the job task ends there; the worker promotes each index and then flips the table to ACTIVE, conditioned on `restore_backfill_pending` and on CREATING, not on the job's generation; and the reconciler never treats a row with `restore_backfill_pending` as a dead RUNNING job (the row is DONE). DeleteTable on a restore target is refused with `ResourceInUseException`, as #384 does and as the service does, which is why the job and the reconciler remove a target through `remove_restore_target` (step 7); there is no user-facing way to cancel a running restore. UpdateTable on a CREATING table is refused as today. +7. Failure (verification or load): the job marks the `restore_jobs` row FAILED with the reason, conditional on its generation. Only if that write matched exactly one row does the job remove the partially created table, through `BackupCatalog::remove_restore_target` (section 3), keyed by the target's account, name and table id and conditional on the table still being a CREATING restore target, which also deletes its progress rows, never through the DeleteTable API; a write that matches zero rows ends the job with no further table access and its spool entry left to the sweep, because the claim holder owns the table and may be reading that entry (the entry is keyed by table id, so a same-host successor shares it). DescribeTable then returns ResourceNotFound. Crash mid-restore: once the server's shutdown has begun (the jobs' token is cancelled) the reconciler takes no dead record over and writes no FAILED on the maintenance or other-store arm: the record is left as it is for the next instance's reconciler, since a resume spawned on the way out would stop at its first check and leave the record with a fresh heartbeat, a second stale window for the next instance to wait. Otherwise the reconciler claims RUNNING `restore_jobs` rows whose owner is dead (`claim_restore_dead`, section 4(e)) and resumes from the recorded progress (a dead record met under a maintenance claim is failed through the unclaimed RUNNING-only write before its store is looked at: `target_not_empty` with its target left for `remove-target` (which runs under the same claim) when it recorded no progress and its target holds more items than the backup's row counts, else `OWNER_DEAD` with its target removed; the command's wait is satisfied either way, the record being no longer RUNNING; a record of another store is failed `OTHER_STORE` below, both arms on the heartbeat alone on every backend, 4(e); a resume takes a job permit and is skipped to the next cycle without one; a resume with no recorded progress into a target holding more items than the backup's total (the backup row's `item_count`) is FAILED `target_not_empty` and its target left for `extenddb backup remove-target --account
    ` (section 4's duty list), which ships with PR 9, the rule for a catalog restored from a dump that names a restore which finished and was written to since; a target holding no more than the total with no recorded progress is a first data file applied in part (on Cassandra, which records a batch's progress after its writes, a crash before the first progress record, the first 10,000 items or 32 MiB, leaves this; MongoDB commits each batch with its progress and leaves nothing unrecorded) and is resumed with the first file re-applied under `reapplying`, every backend's load being idempotent for such a file; the takeover after one instance's fifth of a record (`RESTORE_TAKEOVERS`, counted in the reconciler's state, so per instance and reset by a restart) fails it `owner dead: taken over 5 times by one instance without the restore ending` and removes its target as `OWNER_DEAD` does, so a record whose every resume ends RUNNING again (a progress read or the final count that fails every time, a load that stops with the record RUNNING) is not retaken every stale window for as long as the server runs), reading the store the job row records if it is the configured store, failing the restore and naming `backup sync --move-store` if it is not (`OTHER_STORE`, the target left to the FAILED-target duty), and resuming nothing while the configured store is unhealthy; on SQLite at once after a restart, by the slot rule; on PostgreSQL, as on MongoDB and Cassandra, once the row's `heartbeat_at` is older than `stale_job_after_secs`, the delay the prefix deletion already waits (4(e): the advisory lock is the claim's exclusion, not the liveness rule; the owner aborts at its next heartbeat, inside that window). A legacy RUNNING restore whose owner is dead is marked FAILED and its target removed, never resumed (section 3). The reconciler's sweep of `restore_progress` rows (PostgreSQL) is a positive list: the backend reads the distinct table ids the progress rows name, then the catalog's RUNNING records, and deletes only the rows of ids in the first set that are in neither the second nor the reconciler's own RUNNING list; a target whose first row lands after the first read is never a candidate, and one registered before it is RUNNING at the second read, so a live job's row is never deleted. Shutdown: a restore job stops at its next chunk without writing a status, so step 7 does not run; its row stays RUNNING with its spool and its progress, for a successor once the owner is dead by its backend's rule; it records `RestoreJobs{cancelled}` and no worker metric (section 9), where a lost claim or a DONE write that matched no record records `RestoreJobs{failed}` and `WorkerErrorCount{RestoreJob}` and leaves the spool entry to the sweep: the reconciler's spool sweep lists the spool directory first and the RUNNING records second, and removes only the entries it listed that no RUNNING record of this host's `spool_host` names, so a restore whose registration commits and whose entry opens between the two reads is never a candidate; an entry its own job removed between the listing and the removal is done, not an error. Restore overrides. #384 applies every override the operation models as the service applies it (`crates/engine/src/backup/overrides.rs`) and refuses only `VectorIndexOverride`, which the botocore model this project targets does not carry, with a typed ValidationException. The v1 contract keeps exactly that: the new path applies the overrides at step 4 to the manifest's `source_table` before the target is created, through the same `overrides` module, and `VectorIndexOverride` keeps the refusal. The differences and limits docs' override rows stay as they are. @@ -159,21 +157,21 @@ Export gains the store as a target and uses `open_table_snapshot`, which fixes t ```toml [backup] data_file_target_bytes = 67108864 # uncompressed bytes per data file; a file closes once it reaches this, so it exceeds it by less than one line (5.1 step 4); minimum 32 MiB, the line cap -heartbeat_interval_secs = 15 # job heartbeat timer (5.1 step 4); #384's restore heartbeat interval -stale_job_after_secs = 300 # heartbeat age after which a MongoDB or Cassandra job is dead, and the delay before a reconciler deletes a dead job's prefix on every backend; a new value (#384's restore grace is 60 s) +heartbeat_interval_secs = 15 # the store-backed job's heartbeat timer, floored at JOB_HEARTBEAT_INTERVAL (15 s), which the in-catalog paths keep (5.1 step 4); #384's restore heartbeat interval +stale_job_after_secs = 300 # heartbeat age after which a PostgreSQL, MongoDB or Cassandra job is dead (4(e)), and the delay before a reconciler deletes a dead job's prefix on every backend; at least three times the period the jobs run at, max(heartbeat_interval_secs, JOB_HEARTBEAT_INTERVAL), so 45 at the least (5.1 step 4); a new value (#384's restore grace is 60 s) failed_row_retention_secs = 604800 # FAILED tombstones kept for the operator, 7 days reconcile_interval_secs = 60 # reconciler cycle (section 4) -max_concurrent_jobs = 4 # backup and restore jobs running at once on this instance, legacy creates and restores included; a further CreateBackup or RestoreTableFromBackup is LimitExceeded +max_concurrent_jobs = 4 # backup and restore jobs running at once on this instance, legacy creates and restores included; a further CreateBackup or RestoreTableFromBackup waits up to 30 s for a slot and is then LimitExceeded, the text naming the bound it met; at most 1024; on PostgreSQL, pool_size and catalog_pool_size are raised to at least 2 * max_concurrent_jobs + 10 (below) spool_path = "" # restore download area for an S3 store; defaults beside the database file on SQLite, required on the other backends ``` `[backup.store]` and its `filesystem` and `s3` tables are the crate's `BackupStoreConfig` (`crates/backup-store/src/config.rs` module doc, #393: the TOML block with every key, its default and `open`'s refusals); `kind = "disabled"`, the per-backend default root, the spool and `backup purge` are PR 7's section. -On SQLite (file-backed) a missing `[backup]` section is equivalent to `kind = "filesystem"` with `path` = `backups/` beside the database file; a `validate` failure of that defaulted root is a logged warning that leaves the store unconfigured, not a startup error, so an upgrade never stops a server that never configured backups, while an explicitly configured root that fails `validate` is still a startup error. On PostgreSQL and MongoDB a missing `[backup.store]` leaves the off-host path unconfigured and startup logs that; `kind = "disabled"` leaves it unconfigured on purpose on any backend. Without a configured store, CreateBackup on every backend uses the legacy in-catalog create (`LegacyBackupReader::create`, section 3) and logs a deprecation warning naming `[backup.store]` on each call, until PR 14 removes that path; after it CreateBackup without a configured store returns a ValidationException naming `[backup.store]`. Legacy backups stay readable, listable, restorable and deletable either way. `sqlite-memory` (any `:memory:` database path) has no database file and so no default root: CreateBackup there takes the off-host path only with `[backup.store]` set and the legacy create without it. Cassandra has no `SnapshotEngine`, so it uses the legacy create even with a store configured (section 3); a configured store serves restores of off-host backups into Cassandra and nothing else there. +On SQLite (file-backed) a missing `[backup]` section is equivalent to `kind = "filesystem"` with `path` = `backups/` beside the database file; a `validate` failure of that defaulted root is a logged warning that leaves the store unconfigured, not a startup error, so an upgrade never stops a server that never configured backups, while an explicitly configured root that fails `validate` is still a startup error. The warning holds only while the catalog has no backup rows for the defaulted root (`read_store_ids`); once it has, the root has been used and is handled as an explicitly configured one (revision item 29). On PostgreSQL and MongoDB a missing `[backup.store]` leaves the off-host path unconfigured and startup logs that; `kind = "disabled"` leaves it unconfigured on purpose on any backend. Without a configured store, CreateBackup on every backend uses the legacy in-catalog create (`LegacyBackupReader::create`, section 3) and logs a deprecation warning naming `[backup.store]` on each call, until PR 14 removes that path; after it CreateBackup without a configured store returns a ValidationException naming `[backup.store]`. Legacy backups stay readable, listable, restorable and deletable either way. `sqlite-memory` (any `:memory:` database path) has no database file and so no default root: CreateBackup there takes the off-host path only with `[backup.store]` set and the legacy create without it. Cassandra has no `SnapshotEngine`, so it uses the legacy create even with a store configured (section 3); a configured store serves restores of off-host backups into Cassandra and nothing else there. -Memory per running job, which operators size hosts from: a backup holds the upload window (`upload_part_size_bytes * max_concurrent_uploads` on an S3 store, one write buffer on a filesystem store), the read path, the gzip window and the manifest's data-file entries until the manifest is written; with the byte budget PR 8 adds to the PostgreSQL and SQLite streams (`SNAPSHOT_FETCH_BYTES`, a constant beside `DEFAULT_FETCH_BATCH`, not a `[backup]` key in v1) the read path holds at most one page, the budget plus one row of text, plus its decoded form, and neither term grows with `data_file_target_bytes` or with the table, except on `sqlite-memory`, whose open copies the whole table (section 3). A restore holds one line's decode plus one insert batch plus the parsed manifest for the whole job; neither grows with the table. Each running backup also holds one connection from the PostgreSQL data pool for its duration, and on the separate layout one catalog-pool connection while its snapshot opens (section 3); the backend admits at most half the pool as open snapshots and refuses more with `LimitExceeded`; a running restore holds one data connection and has no slot of its own after PR 6; and each PostgreSQL job holds one dedicated claim connection outside the pool. `pool_size` sizes both pools, and the rule is `pool_size >= max(MIN_POOL_SIZE, 2 * max_concurrent_jobs + 10)`: twice the jobs, so the half-pool bound admits every job `max_concurrent_jobs` allows, plus ten for requests; 18 at the defaults, under the default `pool_size` of 20. It applies whether or not `[backup.store]` is configured, because legacy restores are jobs too. Startup raises a smaller `pool_size` to that value with a warning naming both numbers, as it already raises a `pool_size` under `MIN_POOL_SIZE` (10) today. +Memory per running job, which operators size hosts from. A backup's terms, for a row of `r` bytes of item text whose decoded `Item` is `d` bytes (`d` is about 1.2 × `r` for a row of many short strings, #394's worst shape, where every `N` is a 133-character string): the read path holds the page's decoded rows, `d` each, while they wait for the pipeline, at most the rows of the budget PR 8 adds to the PostgreSQL and SQLite streams (`SNAPSHOT_FETCH_BYTES`, 64 MiB, a constant beside `DEFAULT_FETCH_BATCH`, not a `[backup]` key in v1) plus one row, and one row's parse, `r + 2d`, since the deserializer builds an intermediate value before the item; PostgreSQL pulls the page's rows from the connection one at a time and parses each as it arrives, so it holds no page text beyond the row being parsed, and SQLite pulls a row at a time with the driver's worker at most five rows ahead of the pull, `r` each (`row_buffer_size` on the pool, four, plus the row the worker holds while its `send` waits; 160 MiB at the line cap; the default of 50 made the run-ahead, not the page, the largest term on a table of large rows). The pipeline holds the feeder's item, the item channel's items (`ITEM_CHANNEL`, four) and the encoder's item, `d` each; the encoder's line buffer (one line, at most 32 MiB, the line cap) and gzip window; the feeder's measurement of an item over the default `max_item_size_bytes` holds no line (it counts the line's bytes through a writer that stops at the cap), so the terms are the same on an instance that raises the limit, at the largest stored row; the chunk sink's 1 MiB buffer and the chunk channel (four 1 MiB chunks); the upload window (`upload_part_size_bytes * max_concurrent_uploads` on an S3 store, one write buffer on a filesystem store); and the manifest's data-file entries until the manifest is written. No term grows with `data_file_target_bytes` or with the table, except on `sqlite-memory`, whose open copies the whole table (section 3); every term but the budget and the chunks grows with the row, which `max_item_size_bytes` bounds (the line cap bounds the line the job writes and names the item it refuses; the row is read and decoded before the feeder measures it). The sizing rule, per running job, is the terms at the line cap times the allocator's factor, times `max_concurrent_jobs`; the rule is stated at the default `max_item_size_bytes`, where no stored item's line exceeds the cap (27.7 MB is the largest line a 400 KB item expands to, `MAX_LINE_BYTES`'s doc); on an instance that raises it the page's decoded rows, one row's parse and the pipeline's items are at the largest stored row, which only `max_item_size_bytes` bounds (about 68 times the item's size as a line, for #394's worst shape). The terms at the line cap (`r` = 32 MiB, `d` = 1.2 `r`): the page's rows, 3 `d` = 115 MiB; one row's parse, `r` + 2 `d` = 109 MiB; the pipeline's six items, 6 `d` = 230 MiB; the line buffer, 32 MiB; the chunks, 5 MiB; on SQLite the run-ahead, 5 `r` = 160 MiB: about 490 MiB on PostgreSQL and 650 MiB on SQLite, an upper bound, since the run-ahead and a full pipeline are not both at their largest at one instant. The allocator's factor is glibc's: the row's 200,000 short strings are allocated on the tokio worker that parses the row and freed on the blocking thread that encodes it, and with glibc's default per-thread arenas that memory stays in the arenas between pages, so the process's high-water mark grows with the number of large rows the job reads and varies by about 200 MiB between identical runs; under `MALLOC_ARENA_MAX=1` (one arena, set in the server's environment on a host that backs up tables of large items; the admin guide's `max_concurrent_jobs` row carries it and the deployment guide's service unit carries the line commented out with that condition, since one arena serializes the worker threads' allocations on one lock and costs the request path 3.5 to 9.2% on SQLite and about 2% on PostgreSQL (the scans, 2.1%), measured on GetItem, Query and Scan with a Python client, the record) the peak is the terms' and does not grow with the table. Measured on PR 8's 16-core host at 27.7 MB rows (`r` = 26.4 MiB, three rows to a page; the terms' sum 545 MiB on SQLite), 1,000 small rows then 12, 24 and 48 such rows, each series six runs back to back on one SQLite binary (`docs/design/15-backup-restore-bench-pr8.md`; revision item 31): with the default arenas the peak over idle was 466, 588 and 760 MiB on a quiet host and 707, 700 and 1,281 MiB on the final build with the kill-point runs beside it (0.86 to 2.35 of the terms, growing with the rows and varying by hundreds of MiB between series; runs of the PR's first build on a loaded host reached 1,096 MiB at 48 rows; no plateau was measured), and under `MALLOC_ARENA_MAX=1` 481, 452 and 480 MiB and then 452, 480 and 455 MiB (0.83 to 0.88 of the terms, flat from 12 to 48 rows in both series; the SQLite ratios are of the peak over idle), and on PostgreSQL at 12 rows 340 MiB `VmHWM` under one arena against 750 to 788 MiB under the default arenas (0.82 of the terms at that row on PostgreSQL, 413 MiB, which has no run-ahead; that ratio is of the `VmHWM`, the process's whole high-water mark, and about 0.47 over its idle of 144 MiB); so the bound an operator can hold the process to is the terms at the line cap times 1.2 per job under `MALLOC_ARENA_MAX=1`, and under the default arenas a figure that grows with the large rows a job reads, 2.35 of the terms at 48 rows being the largest measured. A one-item channel saved about 50 MiB of the peak and cost a 1M-row backup 75% of its duration, so the channel stays at four. With 200-byte rows every term that grows with the row is under 1 MiB, the chunk channel and the sink hold their 5 MiB whatever the row, and the high-water mark of a 1M-row backup is within 10 MiB of a 100k-row backup's on PostgreSQL and SQLite (the bound the acceptance asserts is 112 MiB: the budget, the line cap and the channels) and the 500k-row backup's within 5 MiB on MongoDB, where the 500k run stands in for the 1M run (its load alone took 27 minutes through the wire). A running restore holds the parsed manifest, the decoder's line (at most `MAX_LINE_BYTES`) and the deserializer's intermediate value for it, the decoded items of `LOAD_CHUNK_BYTES` (32 MiB) of data-file text plus one line per chunk (the chunk closes with the item that reaches the bound) at four chunk positions (two in the channel to `bulk_load`, the one being filled, the one being drained), and the backend's write batch, times `max_concurrent_jobs`; the chunk bound counts the bytes the decoder read for the chunk's items, the unit the backup side budgets in, not the service's item sizing, and a chunk also closes at `LOAD_CHUNK_ITEMS` (1,000) items. The backend's write batch: at most 32 MiB of item text on SQLite and Cassandra (10,000 items when they are smaller), held beside the decoded items of that text, 4 MiB per INSERT statement on PostgreSQL (500 rows when they are smaller; one transaction per file; the item and its text, two copies), 1,000 documents per `insert_many` on MongoDB (the driver splits the call; the documents are held until it returns, bounded in bytes only by the data file, since `bulk_load` is per file). A `bulk_load` is one data file at a time, so the chunks in flight never hold more than one file's items, at most the source's `data_file_target_bytes` plus one line of text. None of these grows with the table: they grow with the item, which `max_item_size_bytes` bounds, and SQLite's `COUNT(*)` after the last file costs the counting connection's page cache, bounded by `cache_size` (32 MiB; 22 to 28 MiB measured at 1M and at 3M rows, the record). The input order of the backup's lines is a term of SQLite's beside these: a key-ordered backup (the server's own, of a hash-only or single-range-key table) appends to the base b-tree's right edge, an unordered one (every MongoDB-sourced backup and every multi-part-key table) dirties pages across the whole tree, and the write connection's page cache plus the allocator's arenas hold the difference (the record's input-order table: 1M items of 200 bytes 87 to 98 MiB key-ordered against 144 to 164 MiB unordered on SQLite, in 27 to 56 s against 66 to 90 s; PostgreSQL 16 MiB either way). A table with secondary or vector indexes adds no term of the restore's: on SQLite it adds the backend's own per-connection page cache for the index b-trees (`cache_size`, 32 MiB), bounded by that setting and growing sub-linearly with the table (the record's index table), on PostgreSQL a few MiB. Measured (`docs/design/15-backup-restore-bench-pr9.md`; revision item 33), `VmHWM` over idle from a fresh start, 24 worst-shape items of 27.7 MB: 848 MiB on SQLite and 778 MiB on PostgreSQL under the default arenas (848 and 1,013 MiB at 12 and 48 items on SQLite: the peak grows with the large items a restore reads, as the backup's does, and no bound was measured), 392 MiB on SQLite and 427 MiB on PostgreSQL under `MALLOC_ARENA_MAX=1`; 1,200 items of 350 KB in one file 270 MiB on SQLite and 156 MiB on PostgreSQL; 1M items of 200 bytes under a composite key with no secondary index, the backup's lines in key order, 89 MiB on SQLite and 16 MiB on PostgreSQL (100k: 38 and 10.7 MiB), restored in 26.8 s and 9.5 s; the same items under the same key with two full-projection GSIs 143 to 225 MiB on SQLite at 100,000 items in three runs and 165 to 172 MiB at 300,000, and 23 to 34 MiB on PostgreSQL at both: the excess on SQLite is the backend's own per-connection page cache (`cache_size`, 32 MiB) holding the index tables' b-tree pages plus the allocator's arenas (about a third less under `MALLOC_ARENA_MAX=1`), not the restore's buffers, which the 32 MiB batch bound holds whatever the indexes; it is bounded by `cache_size` and grows sub-linearly with the table. A restore of a table of large items is of the same order as its backup, not less; the sizing rule is the record's: about 450 MiB per running restore of large items under one arena, times `max_concurrent_jobs`. Each running backup also holds one connection from the PostgreSQL data pool for its duration, and on the separate layout one catalog-pool connection while its snapshot opens (section 3); the backend admits at most half the pool (after the raise below) as open snapshots and refuses more with `LimitExceeded`; a running restore holds one data connection and has no slot of its own after PR 6; and each PostgreSQL job holds one dedicated claim connection outside the pool. `pool_size` sizes both pools, and the rule is `pool_size >= max(MIN_POOL_SIZE, 2 * max_concurrent_jobs + 10)`: twice the jobs, so the half-pool bound admits every job `max_concurrent_jobs` allows, plus ten for requests; 18 at the defaults, under the default `pool_size` of 20. It applies whether or not `[backup.store]` is configured, because legacy restores are jobs too. Startup raises a smaller `pool_size` to that value with a warning naming both numbers, as it already raises a `pool_size` under `MIN_POOL_SIZE` (10) today. On SQLite the snapshot slot (`backup_server_slot_count`, `pool_size / 2 - 1` capped at 16, shared by the in-catalog create and the snapshot open) stays the backend's admission bound, as PostgreSQL's half-pool slot does, and a server raises `[storage.sqlite] pool_size` to `2 * (max_concurrent_jobs + 1)` (at most 34) with a warning naming both numbers so the slot refuses no job the permit admitted; 10 at the defaults, the default pool; above 16 jobs the slot refuses the seventeenth. `backup sync`, which is not a job, keeps of the configured root's rows (every status) each row's status, store id and table and the key of each AVAILABLE row of the configured store id, dropping the rest of each row as it is read, and holds the objects of the one prefix the listing is in with their sizes and the prefix of each manifest it listed; every backend's `list_backup_rows` collects its whole result before it yields the first row, so the read of the rows is sync's largest term on a catalog with many rows of the root (tombstones within their retention included): about 22.5 KiB of peak RSS per row of a table with five GSIs (1,452 bytes of `source_table_details_json`; 610 MiB at 25,000 rows and 1,144 MiB at 50,000, against 47 MiB with none, and 148 MiB at 50,000 rows whose JSON is `{}`, so nearly all of it is the parsed description), measured on a SQLite debug build on a 16-core x86_64 Linux host under glibc's default arenas, the same peak `backup list` reaches over those rows; the filesystem store builds its listing of the whole root in one `Vec` before it yields the first key (about 72 bytes plus the key per object, the store trait's doc), and the S3 store lists a page of 1,000 keys at a time. Measured on a filesystem store with 100 objects per prefix and 60-byte keys: 45 MiB of peak RSS for an empty root, 66 MiB for 100,000 objects and 98 MiB for 300,000, about 190 bytes per object, nearly all of it the store's listing (revision item 34). The report keeps an entry per prefix for the whole run (it lists every prefix the catalog already has a row for among the skipped ones), so at one object per prefix the per-prefix terms double that: about 350 bytes per prefix, 62 MiB of peak RSS for 50,000 such prefixes and 112 MiB for 200,000 against 45 MiB for an empty root (and about 165 bytes per object at 100 objects per prefix), measured on a SQLite debug build of PR 10 with a filesystem root and no rows. On an in-memory SQLite server that cost is paid in the serving process at every start. -A configuration error (a missing or denied bucket, a part size, prefix or key-length problem, an insecure endpoint, a root that fails the owner-only rule) is a startup error naming the path or bucket, except for the defaulted SQLite root above, where it is a warning. A transport failure (no region, no credentials, an unreachable endpoint), and a failure of section 4's identity check or manifest probe, do not stop startup, because a store problem must not take the data plane out of rotation (section 9): the failure is logged with its cause, the store component is reported unhealthy, the operations that touch the store refuse with a ValidationException that names the store's kind and not its root (CreateBackup on the off-host path, and RestoreTableFromBackup and DeleteBackup of an off-host backup; Describe and List, legacy backups and Cassandra's legacy create are unaffected), the reconciler acts on no row of the store (its legacy duty, which touches no store, still runs) and resumes no restore, and the check runs again each reconciler cycle until it passes. +A configuration error that needs no request to the store (a part size, prefix or key-length problem, an insecure endpoint, a root that fails the owner-only rule or whose ancestors others can write to, a filesystem root that is or contains the database directory, a spool rule) is a startup error naming the path or bucket, except for the defaulted SQLite root above while the catalog has no rows for it, where it and a failure of the root's identity check are a warning that leaves the store unconfigured. A filesystem root's `validate` also writes a probe file, a request to the store: an I/O failure there or in resolving the root is a store condition that degrades the store (`failed validation`), as the same condition one reconciler cycle later does, not a startup error. A filesystem store's `validate` and identity check run before the listener, bounded by a timeout, so a mount that stops answering after the root is opened leaves the server starting with the store `checking` (a root that does not answer at startup holds the server before the listener, as the database directory would); an S3 store's are requests to the endpoint and run after the listener is up, in the reconciler's first cycle, with the store `checking` until the first result reports, so a stalled endpoint never holds the data plane (revision item 29). Every later check runs under the same bound (5 s for a filesystem root; 30 s for a bucket, under the SDK's per-attempt timeout so no retry outlives the check) and against the shutdown token: a store that has stopped answering costs a reconciler cycle its bound and nothing more, the cycle's other duties run, the check is recorded as a transport failure, and neither the drain at shutdown nor the process's exit waits on it (a filesystem root's abandoned call stays parked on a thread until the mount answers, no later check starts while it is, the root's calls run one at a time so that calls started before the first reached its bound wait for it under the bound and then have the bound for their own call, rather than parking a thread each, and the binary ends its runtime with a 5 s bound). A transport failure (no region, no credentials, an unreachable endpoint, a check that did not finish within its bound), a missing or denied bucket found after the listener, and a failure of section 4's identity check or manifest probe, do not stop startup, because a store problem must not take the data plane out of rotation (section 9): the failure is logged with its cause, the store component is reported unhealthy with the problem's category as its reason, the operations that touch the store refuse with a ValidationException that names the store's kind and the category and not its root (CreateBackup on PostgreSQL, SQLite and MongoDB, and RestoreTableFromBackup and DeleteBackup of an off-host backup; Describe and List, legacy backups and the in-catalog CreateBackup are unaffected), the reconciler acts on no row of the store (its legacy duty, which touches no store, still runs) and resumes no restore, and the check runs again each reconciler cycle until it passes. Authority: the `[backup]` struct in `crates/config` (PR 6 adds it with the job keys; PR 7 adds `store` and its validation) and `BackupStoreConfig` in `crates/backup-store`; defaults above are the ones they encode. @@ -187,7 +185,7 @@ Authority: the `[backup]` struct in `crates/config` (PR 6 adds it with the job k ### 9. Observability -Metrics: `backup_jobs_total{outcome=succeeded|failed|cancelled}`, `backup_job_duration_seconds`, `backup_bytes_uploaded_total`, `restore_jobs_total{outcome}`, `restore_job_duration_seconds`, `backup_snapshot_age_seconds` (gauge while a job runs), `backup_store_errors_total{op}`. Logs: one info line at job start and completion with ARN, item count, bytes, duration; one error line per failure with the reason. The `/health` readiness work from the operations report should include store reachability as a component, not as a hard failure (a backup store outage should not take the data plane out of rotation). `/health` is a static 200 today; the component model is new work in the configuration PR, and a transport failure of the store, or a failure of its identity check or manifest probe (section 4), marks the component unhealthy rather than stopping the server (section 7). Metric names are variants of the closed `MetricName` enum in `crates/core/src/metrics`; the existing `WorkerLastSuccess`, `WorkerCycleLatency` and `WorkerErrorCount` are emitted by the backup worker and the reconciler like the other workers. +Metrics: `backup_jobs_total{outcome=succeeded|failed|cancelled}`, `backup_job_duration_seconds`, `backup_bytes_uploaded_total`, `restore_jobs_total{outcome}`, `restore_job_duration_seconds`, `restore_bytes_downloaded_total`, `backup_snapshot_age_seconds` (gauge while a job runs), `backup_store_errors_total{op}`. Logs: one info line at job start and completion with ARN, item count, bytes, duration; one error line per failure with the reason. The `/health` readiness work from the operations report should include store reachability as a component, not as a hard failure (a backup store outage should not take the data plane out of rotation). `/health` is a static 200 today; the component model is new work in the configuration PR, and a transport failure of the store, or a failure of its identity check or manifest probe (section 4), marks the component unhealthy rather than stopping the server (section 7). Metric names are variants of the closed `MetricName` enum in `crates/core/src/metrics`, and `/metrics` emits them under those names as JSON (`render_metrics`, `docs/design/06-component-server.md`: the endpoint is not Prometheus exposition format; the deployment guide tells operators to convert); the snake-case spellings above are the names a converter would give them, not a rendering the server does. PR 8 adds `BackupJobs` (one count per job end, `operation` = the outcome, `succeeded`, `failed` or `cancelled`), `BackupJobDuration` (the job's duration in microseconds, like every duration the collector records, with the same `operation`), `BackupBytesUploaded` (bytes put to the store), `BackupSnapshotAge` (seconds since the snapshot instant, set at each confirmed heartbeat) and `BackupStoreErrors` (`operation` = the store call that failed: the job's `put` and `delete_prefix`, from PR 9a a restore's `get`, and the reconciler's `get` and `list` of the manifest probe and the deleting duties and its `delete_prefix`, an absent object not counted), with `QuerySource::BackupJob`; `BackupBytesUploaded` is recorded for a job that ended AVAILABLE, and `WorkerErrorCount` under `BackupJob` for a FAILED job and not for one the shutdown cancelled (which records `BackupJobs` with `cancelled` and neither worker metric); the restore variants are PR 9a's: `RestoreJobs` (`operation` = the outcome), `RestoreJobDuration` and `RestoreBytesDownloaded`, with `QuerySource::RestoreJob` for the worker metrics, and `BackupStoreErrors{get}` on every failed `get` of a restore job: the request's and a resume's two document reads and each download attempt the store failed, retried (a transport error) or not (an absent file, a refusal, an I/O error); a digest mismatch and the spool's own errors are not the store's. `RestoreBytesDownloaded` counts the bytes of each data-file download attempt as they arrived, verified or not; the manifest and sidecar reads and a filesystem store's second read of each file are not counted. `RestoreJobs{failed}` counts the job's FAILED writes and the ends at which it learnt the record is another's; the reconciler's own FAILED writes (`target_not_empty`, `OTHER_STORE`, `OWNER_DEAD` under a maintenance claim) are counted in its cycle summary line and `CycleReport`, not in `RestoreJobs`. `WorkerErrorCount{RestoreJob}` accompanies each of those job ends (the maintenance refusal before the job ran, a FAILED write, a lost claim, a DONE write that matched no record); a job the shutdown stopped records `RestoreJobs{cancelled}` and no worker metric; `WorkerLastSuccess{RestoreJob}` is recorded on a restore that reached DONE. The existing `WorkerLastSuccess`, `WorkerCycleLatency` and `WorkerErrorCount` are emitted by the reconciler like the other workers and by the backup job at its end under `BackupJob`. ### 10. Documentation @@ -202,18 +200,31 @@ Which document and which row each PR changes; what each row says is in the ownin | differences doc: vector-backup row | PR 8 (CreateBackup) and PR 9a (the restore's refusal) | | differences doc: `ListBackups` paging row (table ARN in `TableName`) | PR 6 | | differences doc: multi-part-key row | PR 9a | -| differences doc: restore-interruption row | PR 8 | +| differences doc: restore-interruption row | PR 8 (the reconciler) and PR 9a (the store-backed resume per backend, SIGTERM, the kill-point outcomes) | +| differences doc: how `RestoreTableFromBackup` runs (the 60 s registration bound on every backend; the measured durations in that row are the base's in-catalog figures), the DeleteBackup and concurrent-restore rows (the `backup_index` lock per backend) | PR 9a | +| differences doc: new row for the MongoDB Decimal128 sort-key rule | PR 9a | | differences doc: Import and Export rows | PR 13 | | differences doc: new row for CreateBackup of a table holding an `N` value #618 let in | PR 8 | -| limits doc: backup rows | PR 7 (the knobs) and PR 8 (the line cap and file size) | +| differences doc: where-backups-live and backup-creation rows (the store-backed, asynchronous create) | PR 8 | +| limits doc: backup rows | PR 7 (the knobs) and PR 8 (the line cap and file size, the SQLite pool raise, the backup-contents row, the "Backup store" row rewritten: CreateBackup writes to the store and is refused while the store is unhealthy) | | differences and limits docs: the override rows stay | PR 9a confirms; no edit | | admin guide: database names | PR 11 | +| admin guide: the `[backup]` rows (`data_file_target_bytes`, `heartbeat_interval_secs`, `stale_job_after_secs`, `failed_row_retention_secs`, `reconcile_interval_secs`, `max_concurrent_jobs` with the memory figures) and the `[backup.store]` section (the store-backed create, the job's two log lines, the unhealthy-store refusal) | PR 8 | +| deployment guide: the service unit's `MALLOC_ARENA_MAX=1` line | PR 8 | +| `docs/design/README.md` row 15b (the measurement record) | PR 8 | +| `docs/design/15-backup-restore-bench-pr9.md` (the restore memory and duration record) and `docs/design/README.md` row 15c | PR 9a | | usage guide: "backups not supported" line | PR 8 | -| upgrade manual: stop pre-PR-6 servers before `extenddb migrate`; a restore in flight at the stop; the rollback paragraph | PR 6 | +| usage guide: the backups row's RestoreTableFromBackup ✓ (from the store into every backend) | PR 9a | +| upgrade manual: the restore of a store-backed backup lands, `kind = "disabled"` no longer needed for it, `spool_path` for an S3 store, migration 7 and `extenddb migrate`, the rollback paragraph's progress-row sentence | PR 9a | +| admin guide: the `spool_path` row (what the spool holds, its disk cost, the sweep), the `max_concurrent_jobs` row's restore memory terms, `backup list` lists no restore | PR 9a | +| limits doc: the concurrent-restores row (the registration bound, the spool term, Cassandra's `COUNT(*)`) and the backup-contents row (restores into any backend, vector where supported, multi-part keys) | PR 9a | +| `docs/schema/postgres.txt`: the `restore_progress` table | PR 9a | +| `.github/workflows/matrix-nightly.yml` and `devtools/backup-restore-matrix` (the nightly's script) | PR 9b | +| upgrade manual: stop pre-PR-6 servers before `extenddb migrate`; a restore in flight at the stop; the rollback paragraph | PR 6 (and PR 8 for the store-backed backups a restored catalog no longer lists) | | upgrade manual: CreateBackup without `[backup.store]`, `kind = "disabled"`, the SQLite default root, shared store roots and `--adopt-store` | PR 7 (and PR 10 for `--adopt-store`) | -| upgrade manual: the SQLite serve lock; `migrate-legacy` and its refusals | PR 10 | +| upgrade manual: the SQLite serve lock; `migrate-legacy` and its refusals | PR 10 (`migrate-legacy`: #821, item 34) | | upgrade manual: PR 14's migration refusal, `--force-drop-legacy-backups`, CreateBackup without a store after PR 14 | PR 14 | -| upgrade manual: the #618 regression on the off-host path and the way out | PR 8 | +| upgrade manual: the #618 regression on the off-host path and the way out; the `[backup.store]` paragraph's asynchronous create and the restore of a store-backed backup deferred to PR 9a | PR 8 | | `docs/troubleshooting.md` DeleteTable entry and admin guide `control_plane_delay_seconds` row | #395 (in its diff) | | ADR-0019 and the Cassandra RFC's "Backup and restore" section marked as superseded; the admin and upgrade manuals' backup sections | PR 11 | | `docs/design/README.md` line 22 | PR 2 | @@ -230,7 +241,7 @@ Phase 1, portable format and store (the v1 blocker). Numbering is the PR series; Phase 1's PRs, one line each; the PRs still to be written have a section each below with the same five parts: goal, the invariants of sections 4 and 5 it honours, the interfaces it adds, the state transitions it owns, and its acceptance (its tests, its kill points from the testing summary, the main tests that must pass unchanged, and the documentation rows of section 10 it corrects). What a section does not state is the PR's design, written and reviewed with its code. - PR 0 (#384, merged): the in-catalog backup path in its final form, which this series replaces; the admin and upgrade manuals' backup sections describe it. -- PR 1 (#335, rebased): the PITR surface of section 6; its section is below. +- PR 1 (#335, merged): the PITR surface of section 6; its section is below. - PR 2 (#396): this document, revised. - PR 3 (#393, supersedes #336): `crates/backup-store`, the store of section 2; its README and the module docs of `lib.rs`, `config.rs`, `key.rs`, `fs.rs` and `s3.rs` are its documentation. - PR 4 (#394, supersedes #337): `crates/core/src/backup_format`, the format of section 1; its module docs and `crates/core/testdata/extenddb-manifest/README.md` are its documentation. @@ -239,17 +250,13 @@ Phase 1's PRs, one line each; the PRs still to be written have a section each be ### PR 1: PITR surface Goal, interfaces and acceptance, as its line states; section 6 is its contract: -- PR 1 (#335, open and last updated 2026-09-14; it needs a rebase onto main now that #384 has merged): PITR surface per section 6, smaller than #335 because #384 already removed the trait's `restore_table_to_point_in_time`. PostgreSQL migration 004 writing - catalog 0.0.5 - (#384 holds migration 003 and the current catalog version), SQLite schema bump, Cassandra V006 writing that version (#384 holds V005; #338 ended at V004), MongoDB drops the stale `continuous_backups` collection on migrated catalogs and keeps its own expected version (`0.0.2`; it has no versioned migration step until PR 6, section 4). `TableNotFoundException` for a missing table on all three operations (section 6). Tests: three operations on all four backends. +- PR 1 (#335, merged): PITR surface per section 6, smaller than the draft of #335 because #384 had already removed the trait's `restore_table_to_point_in_time`; the engine answers the three operations without storage (`crates/engine/src/backup/pitr.rs`), and migration 5, `drop_continuous_backups`, drops the `continuous_backups` table on PostgreSQL, SQLite and Cassandra and the collection on MongoDB. `TableNotFoundException` for a missing table on all three operations (section 6). Tests: three operations on all four backends. Documentation: the differences doc's PITR row, rewritten as section 6 says. ### PR 6: catalog index, `BackupCatalog`, `JobClaim`, the legacy reader Goal: the catalog index of section 4 on every backend and the engine-side Describe, List and Delete over it, with the legacy reader beside it. Its line names the interfaces and carries the acceptance: -- PR 6: catalog index per section 4, the `BackupCatalog` trait and `JobClaim` of section 3 implemented by every backend, the `backup_index` row (a new table or collection on PostgreSQL, SQLite and MongoDB; the reused legacy tables with `store_kind` set, plus `backup_index_by_status`, on Cassandra) and the restore job fields on #384's registration record, with the migration's backfill of `status`, `heartbeat_at` (the epoch, not NULL) and a sentinel `owner_generation` on the registrations that exist (PostgreSQL migration 005 and - catalog 0.0.6 - reserved now; SQLite schema; Cassandra V007 writing that version; MongoDB `backup_index` collection and `tables` fields, the versioned migration step MongoDB lacks today and its bump from `0.0.2` (section 4; the expected version is copied three times, twice in `bootstrapper.rs` and once in `operations.rs`, and this PR collapses it to one constant); `RESTORE_LOCK_SEED` made `pub` with `#[doc(hidden)]` so its copy in `tests/backup_restore.rs` binds to it; the `store_id` column, the `backup_maintenance` record and Cassandra's write order across its three backup tables), the `[backup]` struct in `crates/config` with the job keys of section 7 (everything but `store`, which PR 7 adds) and the `stale_job_after_secs >= 3 * heartbeat_interval_secs` check, since this PR's reconciler and permit read them, `max_concurrent_jobs` (default 4) replacing the per-backend slot semaphores (`restore_server_slot_count` on PostgreSQL and MongoDB, `backup_server_slot_count` on SQLite removed) with the request order of 5.1 step 1 and 5.3 step 1, the permit last, and the reconciler period and maintenance-claim check of section 4, engine-side Describe, List (the handler's existing paging, filtering and cursor exceptions kept, over the catalog index merged with the legacy reader through `list_account_backup_rows`, and a table ARN accepted in `TableName` as main's CreateBackup accepts it), Delete (BackupInUse on a CREATING row and while a restore target registered from the backup is CREATING; DELETING ordering; a ValidationException naming no root for a row of another store) over the new rows, `LegacyBackupReader::create` kept on every backend as the CreateBackup path while no store is configured, with its deprecation warning, and on Cassandra whatever the store configuration, with the `LegacyBackupReader` trait and fallback, the legacy restore registering through the new job fields and, with a dead owner, marked FAILED and its target removed through `remove_restore_target` (section 3); the engine adopts `BackupArn`, addresses every backup row by account and backup id (`BackupKey`; `backup_index` keyed by (`account_id`, `backup_id`), no `store_key_prefix` column) and every restore target by account, name and table id, and derives the API's source-table shapes from `source_table_details_json`; the `catalog_id`, written at init and by the migration for an existing catalog; DeleteBackup's in-use check and DELETING write in one transaction under the row's lock (`start_backup_delete`), and on Cassandra `active_restores` serving new rows; the legacy create taking a `max_concurrent_jobs` permit, and Cassandra's legacy create heartbeated so the reconciler removes a dead one's CREATING row; the job row's `kind` and store columns; #384's MongoDB claim markers, Cassandra `restore_owner`, PostgreSQL `RestoreOwner` and their startup sweeps are replaced by the generation column and the reconciler; `RESTORE_LOCK_SEED` becomes the shared seed. Acceptance: all existing backup tests pass unchanged against backups created by the PR 0 binary (run the binary, not the unit), the existing paging tests and `test_restore_into_a_creating_target_and_a_second_restore_of_its_backup_are_refused` included (the vector tests change in PR 8 and 9a, not here); a catalog at #384's version holding a CREATING restore target left by SIGKILL migrates with that registration's `heartbeat_at` at the epoch, and the first reconciler pass removes the target, run on PostgreSQL and on MongoDB; ListBackups with a table ARN in `TableName` returns what the table name returns; DescribeBackup of an ARN of account B that carries the id of account A's backup returns BackupNotFound; a DeleteBackup and a legacy restore of one backup serialize, main's `delete_backup_winning_before_restore_leaves_no_target` and `restore_registration_winning_before_delete_returns_backup_in_use` pass unchanged for the legacy path, and the same two orderings run against the engine's DeleteBackup and RestoreTableFromBackup over an off-host backup (a `backup_index` row), gated on the `backup_index` row lock, a DeleteBackup that wins leaving the registration BackupNotFound and no target and a registration that wins leaving DeleteBackup BackupInUse; a server built before this PR, started against the migrated catalog, is the mixed-version case section 10 forbids and is not tested. Depends on PR 4 (`BackupArn`) and PR 5 (`SnapshotInfo`). +- PR 6: catalog index per section 4, the `BackupCatalog` trait and `JobClaim` of section 3 implemented by every backend, the `backup_index` row (a new table or collection on PostgreSQL, SQLite and MongoDB; the reused legacy tables with `store_kind` set, plus `backup_index_by_status`, on Cassandra) and the restore job fields on #384's registration record, with the migration's backfill of `status`, `heartbeat_at` (the epoch, not NULL) and a sentinel `owner_generation` on the registrations that exist (migration id 6 on every backend: a file body on PostgreSQL, code bodies on SQLite, MongoDB and Cassandra, Cassandra's executing `0006_backup_catalog_index.cql` before its backfill; the MongoDB `backup_index` collection and `tables` fields, section 4; `RESTORE_LOCK_SEED` made `pub` with `#[doc(hidden)]` so its copy in `tests/backup_restore.rs` binds to it; the `store_id` column, the `backup_maintenance` record and Cassandra's write order across its three backup tables), the `[backup]` struct in `crates/config` with the job keys of section 7 (everything but `store`, which PR 7 adds) and the check that `stale_job_after_secs` is at least three times the effective heartbeat period, `max(heartbeat_interval_secs, JOB_HEARTBEAT_INTERVAL)` (section 7), since this PR's reconciler and permit read them, `max_concurrent_jobs` (default 4, at most 1024) replacing the per-backend restore slot semaphores (`restore_server_slot_count` on PostgreSQL and MongoDB removed; SQLite's `backup_server_slot_count` kept for snapshot opens, which PR 8 keeps as the backend's admission bound and raises the pool for (section 7); the legacy create tries it without waiting and answers the one `LimitExceededException` text, `limit_exceeded_message`) with the request order of 5.1 step 1 and 5.3 step 1, the permit last, and the reconciler period and maintenance-claim check of section 4, engine-side Describe, List (the handler's existing paging, filtering and cursor exceptions kept, over the catalog index merged with the legacy reader through `list_account_backup_rows`, and a table ARN accepted in `TableName` as main's CreateBackup accepts it), Delete (BackupInUse on a CREATING row and while a restore target registered from the backup is CREATING; DELETING ordering; a ValidationException naming no root for a row of another store) over the new rows, `LegacyBackupReader::create` kept on every backend as the CreateBackup path while no store is configured, with its deprecation warning, and on Cassandra whatever the store configuration, with the `LegacyBackupReader` trait and fallback, the legacy restore registering through the new job fields and, with a dead owner, marked FAILED and its target removed through `remove_restore_target` (section 3); the engine adopts `BackupArn`, addresses every backup row by account and backup id (`BackupKey`; `backup_index` keyed by (`account_id`, `backup_id`), no `store_key_prefix` column) and every restore target by account, name and table id, and derives the API's source-table shapes from `source_table_details_json`; the `catalog_id`, written at init and by the migration for an existing catalog; DeleteBackup's in-use check and DELETING write in one transaction under the row's lock (`start_backup_delete`), and on Cassandra `active_restores` serving new rows; the legacy create taking a `max_concurrent_jobs` permit, and Cassandra's legacy create heartbeated so the reconciler removes a dead one's CREATING row; the job row's `kind` and store columns; #384's MongoDB claim markers, Cassandra `restore_owner`, PostgreSQL `RestoreOwner` and their startup sweeps are replaced by the generation column and the reconciler; `RESTORE_LOCK_SEED` becomes the shared seed. Acceptance: all existing backup tests pass unchanged against backups created by the PR 0 binary (run the binary, not the unit), the existing paging tests and `test_restore_into_a_creating_target_and_a_second_restore_of_its_backup_are_refused` included (the vector tests change in PR 8 and 9a, not here); a catalog at #384's version holding a CREATING restore target left by SIGKILL migrates with that registration's `heartbeat_at` at the epoch, and the first reconciler pass removes the target, run on PostgreSQL and on MongoDB; ListBackups with a table ARN in `TableName` returns what the table name returns; DescribeBackup of an ARN of account B that carries the id of account A's backup returns BackupNotFound; a DeleteBackup and a legacy restore of one backup serialize, main's `delete_backup_winning_before_restore_leaves_no_target` and `restore_registration_winning_before_delete_returns_backup_in_use` pass unchanged for the legacy path, the `backup_index` row lock observed on PostgreSQL by a two-transaction test of `start_backup_delete` (one holding the row `FOR UPDATE` while it registers a CREATING target, the delete waiting and then answering BackupInUse); the same two orderings run against the engine's DeleteBackup and RestoreTableFromBackup over an off-host backup (a `backup_index` row), gated on that lock, a DeleteBackup that wins leaving the registration BackupNotFound and no target and a registration that wins leaving DeleteBackup BackupInUse, deferred to PR 9a, which adds the first restore of a `backup_index` row (on this PR no restore of one exists); the reconciler's FAILED-target duty of 4(e) on a FAILED record whose target is still CREATING, listed once stale and removed; a server built before this PR, started against the migrated catalog, is the mixed-version case section 10 forbids and is not tested. Depends on PR 4 (`BackupArn`) and PR 5 (`SnapshotInfo`). Invariants honoured: 4(a) and (b) (the two records and their transitions), 4(d) (every row by `BackupKey`, every target by `RestoreTargetKey`), 4(e) (the claim), 4(f) (the maintenance claim its reconciler period checks), 5.2 (DeleteBackup's lock and ordering against a registration), and the Goals' unchanged request and response shapes. @@ -259,7 +266,7 @@ backup_id 14-digit zero-padded epoch millis of the snapshot instant, `-` account_id, table_name, backup_arn, backup_name, status CREATING | AVAILABLE | DELETING | DELETED | FAILED store_kind filesystem | s3 -store_root the configured root at creation time: the canonical directory for a filesystem store, `s3:///` for S3, with `?endpoint=` appended when an endpoint is configured (the URL without userinfo, query or fragment) +store_root the configured root at creation time: the canonical directory for a filesystem store, `s3:///` for S3, with `?endpoint=` appended when the client uses an endpoint other than the SDK's default (configured, or from the SDK's environment and shared config), the URL in one spelling: lowercase scheme and host, no default port, no `/` for an empty path, no userinfo, query or fragment store_id the id in the store's `extenddb-store.json` (store identity, below) size_bytes, item_count, created_at (= backup_creation_date_time, the id's instant), completed_at, failure_reason, source_table_details_json the manifest's source_table object, verbatim @@ -272,15 +279,15 @@ The `restore_jobs` record, #384's registration extended: on PostgreSQL and SQLit status RUNNING | FAILED | DONE kind legacy | manifest: a restore through the `LegacyBackupReader` or from a manifest (set at 5.3 step 4; PR 6's backfill writes legacy) store_kind, store_root, store_id the store the restore reads, recorded at 5.3 step 4 (null for a legacy restore) -owner_generation the same claim as a backup job (5.1 step 3); replaces #384's restore_operation_id (MongoDB) and restore_owner (Cassandra, where a non-key column cannot be renamed: V007 adds owner_generation beside restore_owner, which PR 14 drops) +owner_generation the same claim as a backup job (5.1 step 3); replaces #384's restore_operation_id (MongoDB) and restore_owner (Cassandra, where a non-key column cannot be renamed: migration 6 adds restore_owner_generation beside restore_owner, which PR 14 drops) heartbeat_at replaces #384's restore_heartbeat / restore_heartbeat_at spool_host the host whose spool holds the downloaded files (5.3 step 3) -file_index, items_applied per-batch progress on SQLite and Cassandra (5.3 step 5); unused on PostgreSQL (see `restore_progress`) and MongoDB (per-file record) +file_index, items_applied the restore's progress: per batch on SQLite and Cassandra, `file_index` per file on MongoDB (`items_applied` unused there); unused on PostgreSQL, whose progress is `restore_progress` rows in the data database (migration 7, 5.3 step 5) failure_reason, completed_at ``` -PR 6's migration backfills the new fields on the registrations that exist at migration time, on all four backends: `kind = legacy`, `status = DONE` where the table is not CREATING, and where it is (a restore the upgrade's stop interrupted), `status = RUNNING` with `heartbeat_at` set to the epoch (1970-01-01T00:00:00Z), not NULL, on every backend, so every backend's staleness test selects them (PostgreSQL's test also has an `IS NULL` arm as a second guard, 5.1 step 3), and a sentinel `owner_generation` (0) that no job holds; the first reconciler pass fails those rows as legacy restores (section 3) and removes their targets (`remove_restore_target`), which is what #384's startup sweeps did for such a target. -On Cassandra: On Cassandra the backup row is denormalized into `backups_by_arn`, `backups_by_account` and `backups_by_table`, the three legacy tables reused, with `store_kind IS NOT NULL` marking a new row and `store_kind IS NULL` a legacy one. The two list tables carry no status, so the rule between the three is presence, not status: a `backups_by_account` or `backups_by_table` row exists iff the authoritative `backups_by_arn` row is AVAILABLE. `backups_by_arn` is authoritative and every writer writes it first, so a writer that dies between the writes leaves a state the next sync or reconciler cycle completes; `backups_by_arn` is partitioned by `(account_id, backup_arn)`, so a status-keyed read has no access path in it, and V007 adds `backup_index_by_status`, primary key `((store_root, backup_status), backup_arn)`, one partition per status for a root, which every status write maintains beside the authoritative row. -Versions and the mixed-version rule: MongoDB's expected version is `0.0.2` on main and #384 added its restore fields without a bump; PR 1 leaves MongoDB at `0.0.2`, and PR 6 adds the versioned migration step MongoDB lacks (a `$set` of the version after the collection and field changes, recorded in `schema_history`) and moves it from `0.0.2` to the version the other backends write in PR 6. On PostgreSQL, SQLite and MongoDB it is a new table or collection, `backup_index`, with primary key (`account_id`, `backup_id`) and `backup_id` unique; the legacy `backups` table or collection stays for the `LegacyBackupReader` (section 3), whose queries are scoped to it, so neither reader sees the other's rows. What this leaves is mixed versions on one catalog. A pre-PR-6 server does not read `backup_index`, so it would not list a backup a PR 6 server created, which is the benign direction; the hazard is the reverse: a PR 6 server sees legacy rows only through the frozen reader, so a backup or restore an old server runs during the mixed period is a legacy job whose registration the new server's reconciler does not own (PR 6 replaced #384's sweeps), and a legacy restore the old server abandons is swept by nobody until that server restarts. Section 10 tells operators to stop every pre-PR-6 server before `extenddb migrate`. +PR 6's migration backfills the new fields on the registrations that exist at migration time, on all four backends: `kind = legacy`, `status = DONE` where the table is not CREATING, and where it is (a restore the upgrade's stop interrupted), `status = RUNNING` with `heartbeat_at` set to the epoch (1970-01-01T00:00:00Z), not NULL, on every backend, so every backend's staleness test selects them (the column is NOT NULL after the migration on PostgreSQL, so its test has no `IS NULL` arm), and a sentinel `owner_generation` (0) that no job holds; the first reconciler pass fails those rows as legacy restores (section 3) and removes their targets (`remove_restore_target`), which is what #384's startup sweeps did for such a target. +On Cassandra: On Cassandra the backup row is denormalized into `backups_by_arn`, `backups_by_account` and `backups_by_table`, the three legacy tables reused, with `store_kind IS NOT NULL` marking a new row and `store_kind IS NULL` a legacy one. The two list tables carry no status, so the rule between the three is presence, and it differs by writer: for an index row (`store_kind` set) the list rows exist for as long as the authoritative row does, of whatever status, and every reader re-reads the authoritative row and applies its status; the legacy create keeps its publish-time rule, writing its list rows only when its row becomes AVAILABLE. `backups_by_arn` is authoritative and every writer writes it first, so a writer that dies between the writes leaves a state the next sync or reconciler cycle completes; `backups_by_arn` is partitioned by `(account_id, backup_arn)`, so a status-keyed read has no access path in it, and migration 6 adds `backup_index_by_status`, primary key `((store_root, backup_status), backup_arn)`, one partition per status for a root, which every status write maintains beside the authoritative row. +Versions and the mixed-version rule: PR 6's change is migration 6 on every backend, recorded in each store's ledger like every other migration (#681 put Cassandra and MongoDB on the runner PostgreSQL and SQLite use, so no backend keeps a catalog version string of its own), and `serve` refuses a catalog without it as it refuses any catalog with a pending migration. On PostgreSQL, SQLite and MongoDB it is a new table or collection, `backup_index`, with primary key (`account_id`, `backup_id`) and `backup_id` unique; the legacy `backups` table or collection stays for the `LegacyBackupReader` (section 3), whose queries are scoped to it, so neither reader sees the other's rows. What this leaves is mixed versions on one catalog. A pre-PR-6 server does not read `backup_index`, so it would not list a backup a PR 6 server created, which is the benign direction; the hazard is the reverse: a PR 6 server sees legacy rows only through the frozen reader, so a backup or restore an old server runs during the mixed period is a legacy job whose registration the new server's reconciler does not own (PR 6 replaced #384's sweeps), and a legacy restore the old server abandons is swept by nobody until that server restarts. Section 10 tells operators to stop every pre-PR-6 server before `extenddb migrate`. Interfaces it adds. The trait methods below are PR 6's, with the key each addresses rows by; the methods PR 7, PR 9a and PR 10 add to the same trait are in their sections, and the per-backend mechanics behind each method are this PR's design. Lifetimes elided. ```rust @@ -308,7 +315,22 @@ pub trait BackupCatalog: Send + Sync { // the backend-internal removal of a CREATING restore target (5.3 step 7), not DeleteTable, which refuses one; // conditional on the table still being a CREATING restore target with this table id, so it and the ACTIVE flip // exclude each other; it also deletes the target's `restore_progress` rows; `false` when the table is no longer one. - fn check_maintenance_claim(&self) -> BoxFuture>; // true while one is held; jobs call it + fn list_dead_restore_jobs(&self, stale_after: Duration) -> BoxFuture, StorageError>>; // PR 6 + // RUNNING records whose heartbeat is older than `stale_after` on the database's clock, and FAILED records whose + // table is still a CREATING restore target (on Cassandra a DELETING one with its registration) once the record is + // that old; the reconciler's listing (4(e)) + fn fail_restore_job(&self, key: &RestoreTargetKey, owner_generation: u64, heartbeat_at: f64, reason: &str) -> BoxFuture>; // PR 6 + // RUNNING to FAILED with the reason, conditional on RUNNING and on the record still carrying the `owner_generation` + // and `heartbeat_at` the reconciler listed (4(e): a heartbeat that lands after the listing makes it match nothing), + // written with a fresh generation; `false` when not RUNNING under those values or gone; for a dead legacy restore + fn check_maintenance_claim(&self, stale_after: Duration) -> BoxFuture>; // true while a live holder has it; jobs call it with stale_job_after_secs +} + +pub trait LegacyBackupReader: Send + Sync { + // ... the frozen create, describe, list, delete, restore-source and restore paths, and: + fn abandoned_creating_backups(&self, stale_after: Duration) -> BoxFuture, StorageError>>; // PR 6 + // in-catalog CREATING rows whose create is dead (heartbeat older than `stale_after`, or no running create holds it); + // the reconciler removes each through `delete` } pub trait JobClaim: Send + Sync { @@ -320,15 +342,15 @@ pub trait JobClaim: Send + Sync { } ``` `BackupEngine` on the storage trait is reduced to what only a backend can do. Removed: `create_backup`, `describe_backup`, `list_backups`, `delete_backup`, `backup_restore_source`, `restore_table_from_backup` (#384 already removed `restore_table_to_point_in_time`). The engine owns those now. Added: `LegacyBackupReader` (section 3 states its contract): the one part that is not frozen is the restore's registration: a legacy restore registers through the `restore_jobs` fields of section 4, with `kind = legacy`, and holds the same claim as a new restore (on Cassandra it also registers in the backup row's `active_restores` map, as every restore there does, section 4, so the frozen DeleteBackup's guard sees it), so the reconciler finds a legacy restore whose owner died as it finds a new one (PR 6 removes #384's per-backend sweeps, and a frozen copy of the registration would be swept by nobody). A legacy RUNNING restore whose owner is dead is marked FAILED and its target removed (`remove_restore_target`, above), as #384's sweep does, and is never resumed: a legacy copy has no data files, spool or progress record to resume from. Cassandra's and SQLite's legacy creates commit a CREATING row before the copy; Cassandra's is the one that needs heartbeating, so it is not frozen either: its insert also sets `owner_generation` and `heartbeat_at`, the engine heartbeats it on the job timer, and the reconciler removes a legacy CREATING row and its `backup_items` through the reader's `delete` once that heartbeat is older than `stale_job_after_secs`; on SQLite, one process, a legacy CREATING row that no running create holds is removed the same way at startup and at runtime; the PostgreSQL and MongoDB legacy creates commit their row AVAILABLE with the copy and leave none. Describe, List, Delete and Restore on the engine consult the new catalog index first and fall back to the reader; List merges the two sources into one (`created_at`, `backup_arn`) order, which works with a bare-ARN cursor because `created_at` is the id's instant on both sides for every id that carries a 14-digit instant (section 4); the two cursor exceptions #384's handler makes are kept (5.2). The frozen reader keeps #384's behaviour for a legacy backup of a table with vector indexes (refused at restore on PostgreSQL, restored without its vector indexes on SQLite, as `docs/differences-from-dynamodb.md` records), and the legacy create keeps main's refusal of such a table, which the engine runs in front of `LegacyBackupReader::create` as main runs it in front of the backend's create (5.1 step 1); only the off-host path backs such tables up and restores them. -What the claim is per backend (4(e) states the rules): PostgreSQL: `pg_try_advisory_lock(hashtextextended('backup:' || backup_id, RESTORE_LOCK_SEED))` on a dedicated, non-pooled connection to the catalog database (outside `pool_size`), taken before the row exists and held for the job's life; the seed is the one constant #384's restore lock already uses, which PR 6 makes `pub` with `#[doc(hidden)]`, so there is one lock key space; the job's heartbeats go through that same connection, so a managed server dropping the idle session surfaces as a failed heartbeat and the job aborts instead of continuing unowned. MongoDB: the row is the claim, born owned by an `update_one` with `upsert: true`, a filter on `owner_generation: {$exists: false}` and an update pipeline that sets the generation and `heartbeat_at: "$$NOW"`, so no row ever exists unowned; a heartbeat that does not return exactly one matched document means the claim may be lost and the job aborts. SQLite: one process, so the claim is an in-process slot; a restart is proof of death, and so is a slot no task holds. Cassandra (a restore target only): the row is the claim, taken with a lightweight transaction `IF owner_generation = ` (or `IF NOT EXISTS` at creation), and each heartbeat is a conditional `UPDATE` on the coordinator's clock; a heartbeat whose `[applied]` is false or that errors aborts the job. The lock is PostgreSQL's `FOR UPDATE` on the `backup_index` row, SQLite's write transaction, a write to the backup's `backup_index` document in a MongoDB session transaction (an `$inc` of a counter, so the write always modifies the document and two such transactions conflict), and on Cassandra the lightweight transactions on the `backups_by_arn` row (section 4). +What the claim is per backend (4(e) states the rules): PostgreSQL: `pg_try_advisory_lock(hashtextextended('backup:' || backup_id, RESTORE_LOCK_SEED))` on a dedicated, non-pooled connection to the catalog database (outside `pool_size`), taken before the row exists and held for the job's life; the seed is the one constant #384's restore lock already uses, which PR 6 makes `pub` with `#[doc(hidden)]`, so there is one lock key space; the job's heartbeats go through that same connection, so a managed server dropping the idle session surfaces as a failed heartbeat and the job aborts instead of continuing unowned. MongoDB: the row is the claim, born owned by an `update_one` with `upsert: true`, a filter on `owner_generation: {$exists: false}` and an update pipeline that sets the generation and `heartbeat_at: "$$NOW"`, so no row ever exists unowned; a heartbeat that does not return exactly one matched document means the claim may be lost and the job aborts. SQLite: one process, so the claim is an in-process slot; a restart is proof of death, and so is a slot no task holds. Cassandra (a restore target only): the row is the claim, taken with a lightweight transaction `IF owner_generation = ` (or `IF NOT EXISTS` at creation), and each heartbeat is a conditional `UPDATE` on the coordinator's clock; a heartbeat whose `[applied]` is false, or that errors three times in a row (one error under a loaded node does not stop a job; a beat that hangs is not counted, and the per-bucket poll of the claim bounds that copy to one more bucket), aborts the job, which then touches neither the row nor its data. The lock is PostgreSQL's `FOR UPDATE` on the `backup_index` row, SQLite's write transaction, a write to the backup's `backup_index` document in a MongoDB session transaction (an `$inc` of a counter, so the write always modifies the document and two such transactions conflict), and on Cassandra the lightweight transactions on the `backups_by_arn` row (section 4). -State transitions it owns: AVAILABLE to DELETING and DELETING to DELETED (5.2); a dead legacy restore to FAILED with its target removed, and a dead legacy CREATING row removed (above); the migration's backfill writes DONE or RUNNING with the epoch heartbeat (above). +State transitions it owns: AVAILABLE to DELETING and DELETING to DELETED (5.2); a dead legacy restore to FAILED with its target removed, the target of a FAILED record removed (the FAILED-target duty, 4(e)), and a dead legacy CREATING row removed (above); the migration's backfill writes DONE or RUNNING with the epoch heartbeat (above). Documentation rows it corrects: the differences doc's `ListBackups` paging row's statement that the ARN form of `TableName` is refused (accepted from PR 6, as the service accepts it; 5.2); no row changes for DeleteBackup racing a restore of the same backup: the two serialize as on main (5.2). Upgrade manual: stop every pre-PR-6 server before `extenddb migrate`, because an old server neither reads `backup_index` nor marks the rows it writes (section 4); a restore in flight at that stop is failed and its target removed on the first reconciler pass (section 4); and the rollback paragraph inverts: today's manual says a rollback to a pre-upgrade catalog removes every backup created after the upgrade because they lived in the catalog, whereas after PR 6 a catalog restored from a pre-upgrade dump loses the `backup_index` rows while the store keeps the bytes, which the old binary cannot read and `backup sync` re-adopts after the re-upgrade. ### PR 7: configuration, the store identity file, `backup purge` Goal: the `[backup.store]` configuration with its validation, the store identity file and the health component, and `backup purge`. Its line names the interfaces and carries the acceptance: -- PR 7: configuration: `store` (`[backup.store]`) added to the `[backup]` struct PR 6 adds, with its validation (section 7, including the absent-section default and its warning-only validation, `kind = "disabled"`, the legacy create as the fallback without a store, the spool path rule and the spool `.lock`, the pool-size check), init template, both banners, store `validate()` at startup with the store-identity check around it (`extenddb-store.json` created with `put_if_absent`, which this PR adds to the `BackupStore` trait and both stores, then read and compared with the rows through `read_store_ids` and with the catalog's `catalog_id`; the manifest probe; both run again each reconciler cycle, and a failure marks the store unhealthy rather than stopping startup), the configuration-error versus transport-failure split, `/health` component, destroy semantics, `backup purge` deleting each configured-store row's prefix and the row. Acceptance: a bad bucket or an explicitly configured unwritable root fails startup with the name in the message; a config without `[backup]` starts and creates the default root `0700`; a config without `[backup]` on SQLite under a group-writable parent starts, logs the warning and leaves the store unconfigured, so CreateBackup there uses the legacy create with its warning, while the same root configured explicitly fails startup; an S3 store whose endpoint is unreachable at startup starts with the store component unhealthy and CreateBackup refusing, and recovers when the endpoint answers; a spool path inside or containing the root is refused, a spool path another live instance holds is refused, and on PostgreSQL an S3 store without `spool_path` fails startup; a backup created on `sqlite-memory` with `[backup.store.filesystem] path` set restores, and without `[backup.store]` CreateBackup there creates a legacy backup with the warning; a root whose `extenddb-store.json` names another catalog, carries another id than the rows, or is missing while rows exist starts with the store unhealthy (logged, `/health` unhealthy, CreateBackup, an off-host restore and DeleteBackup of an off-host backup refused naming the store's kind and not its root, the reconciler acting on no row of the store) and recovers once the file is restored; two instances of one catalog started against one empty root end with one id in the file and on the rows; two instances of one catalog with host-local filesystem roots at one path: the second, with no file, is unhealthy, and with the first's file copied to it the manifest probe marks it unhealthy; on either store a `put_if_absent` of a key that exists returns `None` and leaves the first object; purge takes the maintenance claim, waits for a CreateBackup running on a second instance, no job starts while it holds the claim, and afterwards no row of the configured store remains while `extenddb-store.json` and a prefix with no row do; PostgreSQL without `[backup.store]` creates an in-catalog backup, logs the deprecation warning naming `[backup.store]` and keeps serving legacy backups. Also `extenddb backup list` and `--failed`, printing `store_root`. Depends on PR 3. +- PR 7: configuration: `store` (`[backup.store]`) added to the `[backup]` struct PR 6 adds, with its validation (section 7, including the absent-section default and its warning-only validation, `kind = "disabled"`, the legacy create as the fallback without a store, the spool path rule and the spool `.lock`, the pool-size check), init template, both banners, store `validate()` with the store-identity check around it, before the listener for a filesystem store and in the reconciler's first cycle for an S3 store, the store `checking` until then (`extenddb-store.json` created with `put_if_absent`, which this PR adds to the `BackupStore` trait and both stores, then read and compared with the rows through `read_store_ids` and with the catalog's `catalog_id`; the manifest probe; both run again each reconciler cycle, and a failure marks the store unhealthy rather than stopping startup), the configuration-error versus transport-failure split, `/health` component, destroy semantics, `backup purge` deleting each configured-store row's prefix and the row. Acceptance: a bucket name `open` refuses or an explicitly configured unwritable root fails startup with the name in the message, and a missing or denied bucket makes the store unhealthy after the listener; a config without `[backup]` starts and creates the default root `0700`; a config without `[backup]` on SQLite under a group-writable parent starts, logs the warning and leaves the store unconfigured, so CreateBackup there uses the legacy create with its warning, while the same root configured explicitly fails startup; a row seeded under the defaulted SQLite root with its manifest missing starts the server with the store configured and unhealthy (`/health` degraded, `manifest probe failed`), the row's restore and delete refused naming the kind and the category, `backup purge --confirm` and `backup list` reaching the store, and the store healthy again once the manifest is back; a relative `[backup.store.filesystem] path` under daemon-mode `serve` resolves against the shell's directory; an S3 store whose endpoint is unreachable at startup serves with the store component `checking` and then unhealthy, the in-catalog CreateBackup still answering, a stalled endpoint holding the listener for no longer than its own startup and each reconciler cycle for no longer than the check's 30 s bound (the restore of a seeded row refused naming the pending check, then `unreachable`; the next cycle's check reaching the endpoint within seconds of the abandoned one), and recovers when the endpoint answers; a spool path inside or containing the root is refused, a spool path another live instance holds is refused, and on PostgreSQL an S3 store without `spool_path` fails startup; a backup created on `sqlite-memory` with `[backup.store.filesystem] path` set restores, and without `[backup.store]` CreateBackup there creates a legacy backup with the warning; a root whose `extenddb-store.json` names another catalog, carries another id than the rows, or is missing while rows exist starts with the store unhealthy (logged, `/health` unhealthy with the category as its reason, an off-host restore and DeleteBackup of an off-host backup refused naming the store's kind and the category and not its root, the in-catalog CreateBackup unaffected, the reconciler acting on no row of the store) and recovers once the file is restored, on an explicitly configured root and on the defaulted SQLite root once the catalog has rows for it; two instances of one catalog started against one empty root end with one id in the file and on the rows; two instances of one catalog with host-local filesystem roots at one path: the second, with no file, is unhealthy, and with the first's file copied to it the manifest probe marks it unhealthy; on either store a `put_if_absent` of a key that exists returns `None` and leaves the first object; purge takes the maintenance claim and heartbeats it for its whole run, waits for a CreateBackup running on a second instance (a CREATING or FAILED row with a live heartbeat), no job starts while it holds the claim (the in-catalog create and restore answer LimitExceededException), a CREATING row with a dead owner is purged with the rest, and afterwards no row of the configured store remains while `extenddb-store.json` and a prefix with no row do; PostgreSQL without `[backup.store]` creates an in-catalog backup, logs the deprecation warning naming `[backup.store]` and keeps serving legacy backups. Also `extenddb backup list` and `--failed`, printing `store_root` and `store_id`, a read beside a server (no SQLite serve lock, nothing created, an unhealthy store listed with its problem), as the purge's dry run is. Depends on PR 3. Invariants honoured: 4(c) (store identity and health, the shared-root rule), 4(f) (purge takes the maintenance claim), section 7's rules (an upgrade never stops a server that never configured backups; a configuration error stops startup, a transport or identity failure degrades the store). Interfaces it adds. `put_if_absent`, which PR 7 adds to the trait and both stores for the store-identity file (section 4), creates the object only if no object is under the key and returns `None`, leaving the existing object untouched, if one is: S3 sends `PutObject` with `If-None-Match: *` (a 412 is the key present; a 409 `ConditionalRequestConflict` from a concurrent conditional write is retried), and the filesystem store writes and syncs its temporary as `put` does, the temporary created with `O_CREAT | O_EXCL`, then `link`s it to the key, which fails if the key exists, so the object appears complete or not at all. `deny_unknown_fields` applies to every section; the server's `[backup]` struct holds the crate's `BackupStoreConfig` under its own key, `store`, which keeps one definition of the store shape, in the crate. @@ -336,11 +358,18 @@ Interfaces it adds. `put_if_absent`, which PR 7 adds to the trait and both store fn read_store_ids(&self, store_kind: StoreKind, store_root: &str) -> BoxFuture, StorageError>>; // the distinct `store_id`s recorded for a root: the identity check (section 4), before it knows an id; PR 7 // Maintenance claim (section 4). PR 7 for purge, PR 10 for sync. - fn maintenance_claim(&self) -> BoxFuture, StorageError>>; + fn maintenance_claim(&self, holder: &str, stale_after: Duration) -> BoxFuture, StorageError>>; + // `holder` labels the command and host for the operator who finds the claim held; `MaintenanceClaim` is + // the handle's generation, heartbeat and release (the claim has no status machine, so it is not a `JobClaim`) + fn list_backup_rows_older_than(&self, status: BackupStatus, store: &StoreIdentity, clock: StaleClock, age: Duration) + -> BoxStream>; // PR 7 + // the rows of a store with a status whose `heartbeat_at` (`StaleClock::Heartbeat`) or `completed_at` + // (`StaleClock::Completed`, never older when unset) is older than `age`, compared in the backend's own query + // on the database's clock (4(e)); purge's test of a live owner (4(f)), and PR 8's reconciler listings ``` The identity file (4(c) states what makes a store unhealthy): On a root with no `extenddb-store.json` and a catalog with no rows for it, the server creates the file with `BackupStore::put_if_absent` (section 2: `If-None-Match: *` on S3, a temporary created `O_CREAT | O_EXCL` and `link`ed to the key on the filesystem store), then reads the file and uses the id it holds. Of two instances of one catalog starting against one empty root, one creates the file and the other's create finds it present, and both use the winner's id. An S3-compatible server that ignores `If-None-Match` on `PutObject` makes the create a plain put, last writer wins: startup's re-read then catches a race lost before it and marks the store unhealthy, naming both ids, and a race lost after it is caught by the next check (a sync, a startup or a reconciler cycle), which finds rows carrying an id the file does not. Startup runs the store's `validate` (its checks are #393's README and `validate` doc) and, on both store kinds, the identity check and the manifest probe of section 4, which run again each reconciler cycle; a root on a filesystem that reports a permissive mode (`0777` from a mount without Unix permission semantics) fails `validate` with the path, and one that reports `0700` while enforcing nothing passes, the case the store cannot observe and the deployment guide names. The spool: The spool: restores from an S3 store download to `spool_path`, created `0700` with files `0600`. The default is per backend: on file-backed SQLite it is `backup-spool/` beside the database file; on PostgreSQL, MongoDB, Cassandra and `sqlite-memory` with an S3 store `spool_path` is required and startup refuses without it (a filesystem store needs no spool). Startup refuses a `spool_path` that contains or is contained in the filesystem store root (the store would list and delete it, and sync would read its first component as an account id) and one that is or contains the database file's directory, and the reconciler's spool sweep removes only entries whose names match the job-entry pattern (`restore-
    `), never other files under `spool_path`. `spool_host` is `:`, and startup refuses a `spool_path` whose `.lock` another live instance holds (5.3 step 3). Disk cost of a restore: S3 store, the compressed backup size at `spool_path` while the restore runs; filesystem store, none for the restore, because files are verified in place (section 5.3 step 3). The filesystem store itself holds every backup's compressed size under its root, beside the database on the SQLite default, kept through `extenddb destroy` (below); that is the disk statement a SQLite operator sizes from. -Destroy and purge: `extenddb destroy` removes the catalog and data databases and leaves the backup store untouched: it is the copy that survives destroy. A separate `extenddb backup purge --confirm` deletes the prefix of each of the configured store's rows (`BackupArn::store_prefix`), never a whole account prefix, so a prefix with no row (another catalog's, or a copy) survives, and deletes those catalog rows outright, leaving no tombstones (a row over no bytes would be a backup Describe reports and Restore cannot read); it leaves `extenddb-store.json`, which is under no `/` prefix, so the root keeps its identity, and the `exports/` component (5.4). It takes the maintenance claim of section 4 and refuses if a backup row is still CREATING or DELETING or a `restore_jobs` row RUNNING when the claim's wait ends, so no job on any instance starts between the check and the deletion (the job permits are per instance and would exclude nothing elsewhere), prints what it will delete before deleting, and is audit-logged (section 8). It lands in PR 7. +Destroy and purge: `extenddb destroy` removes the catalog and data databases and leaves the backup store untouched: it is the copy that survives destroy. A separate `extenddb backup purge --confirm` deletes the prefix of each of the configured store's rows (`BackupArn::store_prefix`), never a whole account prefix, so a prefix with no row (another catalog's, or a copy) survives, and deletes those catalog rows outright, leaving no tombstones (a row over no bytes would be a backup Describe reports and Restore cannot read); it leaves `extenddb-store.json`, which is under no `/` prefix, so the root keeps its identity, and the `exports/` component (5.4). It takes the maintenance claim of section 4 and refuses if a CREATING or FAILED backup row still has a live owner or a `restore_jobs` row is still RUNNING when the claim's wait ends (4(f)), so no job on any instance starts between the check and the deletion (the job permits are per instance and would exclude nothing elsewhere); a CREATING row whose owner is dead is deleted with the rest, a DELETING row is finished, and a row a DeleteBackup moves or removes between the command's read and its turn is finished and counted as deleted; it prints what it will delete before deleting, and is audit-logged (section 8). It lands in PR 7. State transitions it owns: the store's health (healthy, unhealthy for each of 4(c)'s conditions, and back once the check passes); purge deletes rows outright, leaving no tombstone. Documentation rows it corrects: limits doc backup rows (the knobs); upgrade manual: CreateBackup without `[backup.store]` keeps creating in-catalog backups with a deprecation warning until PR 14, so a store is to be configured before that release, and `kind = "disabled"` is the opt-out (section 7); the rule the SQLite default root must meet and what the warning means when it does not (section 7); instances that share a catalog must share one store root (section 4). @@ -348,79 +377,103 @@ Documentation rows it corrects: limits doc backup rows (the knobs); upgrade manu ### PR 8: CreateBackup job, worker and reconciler Goal: the CreateBackup job of 5.1 on the filesystem store, with the reconciler and its full duty list, the byte budget on the snapshot streams and the item validator. Its line names the interfaces and carries the acceptance: -- PR 8: CreateBackup job and worker per 5.1 on the filesystem store: snapshot in the request with the per-backend claim, generation-fenced catalog writes, pipelined read/compress/upload, timer heartbeat through the claim connection with abort on any unconfirmed heartbeat, the failure path's delete gated on a matched FAILED write, every job status write conditioned on `status = CREATING` and an ambiguous AVAILABLE update re-read, a table name over 255 bytes refused, cancellation runs the failure path, id minted from the snapshot instant, `LimitExceeded` over `max_concurrent_jobs` and while the maintenance claim is held, the byte budget `SNAPSHOT_FETCH_BYTES` on the PostgreSQL and SQLite snapshot streams, enforced in each page's query, PostgreSQL's cursor replaced by the keyset page under the same snapshot, with the key columns returned under qualified aliases, an outer `ORDER BY` and `n` starting at 16 and sized from the previous page's average row (section 3, 7), the item validator `validate_restorable_item` in `extenddb-core` (one walk, each `N` parsed once, an error naming the item's key and the attribute path), which PR 9a's restore binds, run on every item, with the backup failed after the rest of the snapshot is validated, every refused item logged up to the cap and the count and the first item in the FAILED reason (5.1 step 4, #618), #394's `KeyRangeTracker` and `BackupDocuments::build` in the writer, the server's shutdown awaiting each job's failure path, the PostgreSQL claim connection's keepalive settings, the data-file close rule and the knob's minimum, the SQLite runtime dead-owner test (a slot no task holds), `refuse_backup_of_vector_table` taken out of the off-host path and kept in front of `LegacyBackupReader::create`, `create_backup_refuses_a_table_with_vector_indexes` kept for that path, an off-host round trip of a vector table added and `test_create_backup_of_a_vector_table_is_refused` expecting the outcome of the configured path, read from `EXTENDDB_TEST_BACKUP_STORE`, which the harness exports and CI sets (5.1 step 1), FAILED tombstones, metrics. Acceptance: backup of a 100k-item composite-key table with GSIs on each backend; SIGKILL mid-upload leaves no prefix with a manifest and the row resolves after restart (binary-level); SIGKILL between the manifest put and the AVAILABLE update resolves to AVAILABLE after restart and the prefix survives; SIGTERM mid-upload leaves a FAILED row, an empty prefix and no multipart upload; failure deletes the prefix; a second CreateBackup on the same instance while one runs returns `LimitExceededException`; a 1M-item run records duration and peak RSS, and RSS must not grow from the 100k run to the 1M run (on every backend but `sqlite-memory`, whose open copies the table); the memory test with #394's worst-expansion items stays within section 7's formula, with the worst lines placed after a page of small rows, and no page holds more than the budget plus one row, counted in the stream by a check in `extenddb_storage::snapshot_conformance` that the PostgreSQL and SQLite tests both run; the rows read past each PostgreSQL page's cut stay under its `n` and are recorded beside the RSS figure; `EXPLAIN` of the test table's page query shows an index scan; one cell for each write path #394's `key_order.rs` doc names (BatchWriteItem, TransactWriteItems, UpdateItem on an absent item, ImportTable): a table holding `1E126` or `abc` written through that path (in a non-key attribute and, on a key-ordered table, in the range key; for UpdateItem in the key it creates) gives a FAILED row and no manifest on each backend; with three such items the job validates past the first, logs all three with key, attribute path and reason, and the FAILED reason gives the count and the first and names #618; an item refused for another rule (over `max_item_size_bytes`) gives a reason that does not name #618; the vector wire test fails when `EXTENDDB_TEST_BACKUP_STORE` is unset; without `[backup.store]` CreateBackup of a vector table is refused as on main; a job task that panics on SQLite leaves a row the reconciler resolves without a restart. The reconciler ships in the same PR, with the full duty list of section 4, the S3 leg in CI, and memory tests with incompressible items (64 MiB of random binary attributes, which gzip cannot shrink) and with the worst-expansion shapes of section 7; RSS must match the formula in section 7 in each, proving the encoder streams into the upload and the byte budget and channel bound hold at the largest line. Depends on PR 4, 5, 6, 7. +- PR 8: CreateBackup job and worker per 5.1 on the filesystem store: snapshot in the request with the per-backend claim, generation-fenced catalog writes, pipelined read/compress/upload, timer heartbeat through the claim connection with abort on any unconfirmed heartbeat, the failure path's delete gated on a matched FAILED write, every job status write conditioned on `status = CREATING` and an ambiguous AVAILABLE update re-read, a table name over 255 bytes refused, cancellation runs the failure path, id minted from the snapshot instant, `LimitExceeded` over `max_concurrent_jobs` and while the maintenance claim is held, the byte budget `SNAPSHOT_FETCH_BYTES` on the PostgreSQL and SQLite snapshot streams, in the page query on PostgreSQL and while the rows are pulled on SQLite, PostgreSQL's cursor replaced by the three-layer keyset page under the same snapshot (section 3, 7), the item validator `validate_restorable_item` in `extenddb-core` (one walk, each `N` parsed once, an error naming the item's key and the attribute path), which PR 9a's restore binds, run on every item, with the backup failed after the rest of the snapshot is validated, every refused item logged up to the cap and the count and the first item in the FAILED reason (5.1 step 4, #618), #394's `KeyRangeTracker` and `BackupDocuments::build` in the writer, the server's shutdown awaiting each job's failure path, the PostgreSQL claim connection's keepalive settings, the data-file close rule and the knob's minimum, the SQLite runtime dead-owner test (a slot no task holds), `refuse_backup_of_vector_table` taken out of the off-host path and kept in front of `LegacyBackupReader::create`, `create_backup_refuses_a_table_with_vector_indexes` kept for that path, the create half of an off-host round trip of a vector table added (CreateBackup succeeds, DescribeBackup reaches AVAILABLE with the table's features, ListBackups lists it, and the restore answers the ValidationException naming the store until PR 9a adds the restore half) and `test_vector_table_backup_refused_in_catalog_and_restored_from_store` (named `test_create_backup_of_a_vector_table_is_refused` until PR 9 added the restore half) expecting the outcome of the configured path, read from `EXTENDDB_TEST_BACKUP_STORE`, which every CI pytest job that runs the suite sets (`filesystem` on the PostgreSQL job, which appends a filesystem store after `init`, and on the SQLite job, the defaulted root; `s3` on the SQLite job against moto; `filesystem` on the MongoDB jobs, whose `devtools/run-mongodb-tests` serves a filesystem root under its output directory from PR 9 on, where it was `none` before; `none` on the Cassandra job) and `devtools/run-tests --pytest` refuses to run without (5.1 step 1); the restore cells of `tests/test_backup_restore_fidelity.py` skip on a server with a store, naming PR 9a, until its restore job lands; FAILED tombstones, metrics. Acceptance: backup of a 100k-item composite-key table with GSIs on each backend with a `SnapshotEngine` (PostgreSQL, SQLite, MongoDB; Cassandra keeps the legacy create and its existing cells); SIGKILL mid-upload leaves no prefix with a manifest and the row resolves after restart (binary-level); SIGKILL between the manifest put and the AVAILABLE update resolves to AVAILABLE after restart and the prefix survives; SIGTERM mid-upload leaves a FAILED row, an empty prefix and no multipart upload; failure deletes the prefix; a second CreateBackup on the same instance while one runs waits for the slot (up to 30 s, item 28) and is answered once the first job ended, `LimitExceededException` when the wait outlives the bound; a 1M-item run records duration and peak RSS, and the 1M run's peak must be within 112 MiB of the 100k run's (the budget, the line cap and the channels; on every backend but `sqlite-memory`, whose open copies the table; on MongoDB a 500k run stands in for the 1M run, its load alone taking 27 minutes through the wire); the memory test with #394's worst-expansion items, with the worst lines placed after a page of small rows, has its peak over idle under 1.2 times the sum of section 7's terms at the measured row (the decoded rows in the page and the pipeline, one row's parse, the line buffer, the chunks and SQLite's run-ahead) with `MALLOC_ARENA_MAX=1`, at 12, 24 and 48 worst rows on one SQLite binary and at 12 rows on PostgreSQL, and its peak with glibc's default arenas recorded beside it at the same counts, and no page holds more than the budget plus one row, counted in the stream by a check in `extenddb_storage::snapshot_conformance` that the PostgreSQL and SQLite tests both run; the rows a PostgreSQL page reads past its cut are at most `n - 1`, measured by `EXPLAIN (ANALYZE, BUFFERS)` on a 1,000-small-then-60-worst table (4 read, 1 past the cut at the floor) and pinned by `the_row_bound_remembers_large_rows_among_small_ones` (201 over 53 round trips on a 300-row table with nine 27 MB rows); `EXPLAIN` of the page query on the six key shapes a table can have (hash only, `S`, `N` and `B` range keys, a multi-part range key, a legacy table on the database collation) shows `Limit` over one `Index Scan` on the primary key and no other scan of the relation where the planner's estimate is above `n`, and the bitmap scan with a `Sort` near the end of a small table (PostgreSQL 16), both pinned by a test and stated in the module doc; one cell for each write path #394's `key_order.rs` doc names (BatchWriteItem, TransactWriteItems, UpdateItem on an absent item, ImportTable): a table holding `1E126` or `abc` written through that path (in a non-key attribute and, on a key-ordered table, in the range key; for UpdateItem in the key it creates) gives a FAILED row and no manifest on PostgreSQL and SQLite (the two backends `tests/test_backup_job.py`'s deployment fixture started first; the fixture's MongoDB deployment destroys and initializes the catalog of a replica set of the suite's own, since the database names are fixed, and the `mongodb-backup-store` job runs the suite and the backup selection against a filesystem store) (an `abc` range key is refused by the number column on PostgreSQL and SQLite, so the range-key cell carries `1E126`; UpdateItem validates the values its expression sets, so its cell carries the bad value in the key alone); with three such items the job validates past the first, logs all three with key, attribute path and reason, and the FAILED reason gives the count and the first and names #618; an item refused for another rule (over `max_item_size_bytes`) gives a reason that does not name #618; the vector wire test fails when `EXTENDDB_TEST_BACKUP_STORE` is unset; without `[backup.store]` CreateBackup of a vector table is refused as on main; a job task that panics on SQLite leaves a row the reconciler resolves without a restart. The reconciler ships in the same PR, with the full duty list of section 4, the S3 leg in CI, and memory tests with incompressible items (64 MiB of random binary attributes, which gzip cannot shrink) and with the worst-expansion shapes of section 7; RSS is recorded against the terms of section 7 in each, proving the encoder streams into the upload (the incompressible run's 68 MB file passes through a 4 MiB chunk window) and that the byte budget and the channel bound hold at the largest line (under `MALLOC_ARENA_MAX=1` the worst-expansion run's peak does not grow with the number of worst rows, 452, 480 and 455 MiB over idle at 12, 24 and 48 rows on the final SQLite binary; with glibc's default arenas it does, 707, 700 and 1,281 MiB in the same series, the allocator's retention and not the pipeline's terms, which the one-arena series shows do not grow). Depends on PR 4, 5, 6, 7. Invariants honoured: 4(a) (every transition of the backup row a job or the reconciler performs), 4(c) (the reconciler acts only on rows of the configured store and on none while it is unhealthy), 4(e) (the claim, the fencing of every status write, the FAILED-write gate before any delete), 4(f) (the duties that run under a maintenance claim), 5.1 as written, section 2's three store facts, section 7's memory bound and pool rule, and the Drawbacks' #618 regression. -Interfaces it adds: `extenddb_core::validation::validate_restorable_item` (5.1 step 4), `extenddb_storage::snapshot::paging::SNAPSHOT_FETCH_BYTES`, the worker and the reconciler in `crates/server/src/workers.rs`, the metrics of section 9; the reconciler claims through PR 6's `BackupCatalog::claim_backup`. The snapshot and the claim connection are handed to the backup worker over an in-process channel; the worker is a task in `crates/server/src/workers.rs`, spawned with the server's cancellation token and retained handle like the existing workers. It is needed because four write paths store an `N` value such as `1E126` or `abc` unvalidated on main (#618): BatchWriteItem, TransactWriteItems, UpdateItem on an absent item (which creates the item from its `Key`) and ImportTable, the list #394's `key_order.rs` doc gives. So that one run finds every offending item, the first refusal stops encoding and uploading but not validation: the job reads and checks the rest of the snapshot, logs each refused item (its key, attribute path and reason, with the backup ARN) up to 1000 items and counts the rest, and then fails the backup (step 5) with a FAILED reason that gives the number of refused items and the first one, naming #618 only when that item's refusal is an `N` value; so no backup that no restore can load is reported AVAILABLE. The byte budget (section 7 states the bound): PR 8 replaces #395's cursor on PostgreSQL with pages bounded by bytes in the query, each page a keyset `SELECT` in the same `REPEATABLE READ` transaction, ordered by the data table's primary key and starting after the last key of the page before, with a running byte sum that cuts the page at `SNAPSHOT_FETCH_BYTES` (64 MiB, a constant beside `DEFAULT_FETCH_BATCH`, not a configuration key in v1) and always returns at least one row; the snapshot, the lock and which tables stream `key_ordered` are unchanged. On SQLite a page stops at `DEFAULT_FETCH_BATCH` rows or after the row that reaches `SNAPSHOT_FETCH_BYTES` of item text, whichever comes first, and holds at least one row. The cut rule exists in SQL on PostgreSQL and in Rust on SQLite, so the check that no page holds more than the budget plus one row lives in `extenddb_storage::snapshot_conformance`, which both run; a MongoDB batch is already bounded by the server's 16 MiB reply. -Tests that change (5.1 step 1): `create_backup_refuses_a_table_with_vector_indexes` (`crates/engine/src/backup/tests.rs`) stays for that path, PR 8 adds an off-host round trip of a vector table beside it, and `test_create_backup_of_a_vector_table_is_refused` (`tests/test_backup_restore_fidelity.py`) expects the refusal or the round trip according to the server's configured path, which it reads from `EXTENDDB_TEST_BACKUP_STORE=`, a variable the test harness exports and every CI job that runs the suite sets: it derives its branch from the variable, prints which branch ran, and fails if the variable is unset, rather than accepting either outcome; the restore's `ensure_restorable` refusal of every vector backup narrows to "the target has no vector support" only in PR 9a, which rewrites `vector_index_backups_are_refused_in_the_engine` to that case; the frozen legacy reader keeps #384's behaviour for a legacy vector backup (section 3), and section 10 corrects the differences doc's vector row. +Interfaces it adds: `extenddb_core::validation::validate_restorable_item(item, &RestorableKeySchema, &RestorableLimits)` (5.1 step 4; the key argument is a schema every table has, multi-part keys included, each key attribute checked with the format's own `check_key_value`; `RestorableLimits` is built from `&LimitsConfig`: the item size, the hash and range key sizes and the attribute name size), `RestorableItemReason::{AttributeNameTooLong, LineTooLong}` (the latter carrying `{ cap }`), `SHOWN_VALUE_BYTES` (4096, the bound on each text an error carries other than a key value) and `SHOWN_KEY_VALUE_BYTES` (12,288, the bound on a key value), `extenddb_core::limits::DEFAULT_MAX_ITEM_SIZE_BYTES` (the default item limit, the feeder's threshold), `extenddb_core::backup_format::encode_item_line_len_bounded(item, cap)` (the line's length up to `cap`, measured without a buffer), `extenddb_storage::snapshot::paging::SNAPSHOT_FETCH_BYTES` with `fetch_bytes()`, `record_page_bytes` and `record_rows_past_cut` (counters under `test-hooks`), `extenddb_storage::snapshot::source_table_schema` (5.1 step 2), the job (`crates/engine/src/backup/job.rs` with `job/{writer,heartbeat,failure}.rs`) and the reconciler's new duties (`crates/engine/src/backup/reconciler.rs`) in the engine's backup module, the server handing the engine the shutdown token, the metrics collector, the job settings (`JobSettings`) and the manifest's bucket and key prefix through `BackupRuntime` and `ConfiguredStore`, the metrics of section 9, and on `BackupCatalog`: `claim_dead_backup(key, stale_after)` (the takeover of 4(e); `claim_backup` inserts and takes over nothing), `list_backup_rows_older_than(status, store, clock, age)` (`StaleClock::{Heartbeat, Completed}`, the backend's clock; a row with `completed_at` unset is never older on the completed clock), `sweep_restore_registrations` (Cassandra's `active_restores` sweep, a default empty body elsewhere) and `StatusFields.source_table_details_json` (the AVAILABLE write carries the manifest's `source_table`). The request's detached task opens the snapshot, claims the row, checks the maintenance claim and returns the response through a one-shot channel, then continues as the job with the same owned values, so nothing but the response crosses a channel between the request and the job (the pipeline's item and chunk channels and its blocking encoder are the job's own); the runtime's task tracker holds every job, and the server's shutdown cancels and awaits the jobs' failure paths (`BackupRuntime::drain_jobs`, a 30 s bound) before it drains its workers. It is needed because four write paths store an `N` value such as `1E126` or `abc` unvalidated on main (#618): BatchWriteItem, TransactWriteItems, UpdateItem on an absent item (which creates the item from its `Key`) and ImportTable, the list #394's `key_order.rs` doc gives. So that one run finds every offending item, the first refusal stops encoding and uploading but not validation: the job reads and checks the rest of the snapshot, logs each refused item (its key, attribute path and reason, with the backup ARN) up to 1000 items and counts the rest, and then fails the backup (step 5) with a FAILED reason that gives the number of refused items and the first one, naming #618 only when that item's refusal is an `N` value; so no backup that no restore can load is reported AVAILABLE. The byte budget (section 7 states the bound): PR 8 replaces #395's cursor on PostgreSQL with pages bounded by bytes in the query, each page a keyset `SELECT` in the same `REPEATABLE READ` transaction, ordered by the data table's primary key and starting after the last key of the page before, in three layers (the innermost takes `n` rows after the key under `LIMIT`, since a window function runs before `LIMIT`; the middle adds the running byte sum and the rows' count; the outer keeps the rows whose predecessors' bytes are under `SNAPSHOT_FETCH_BYTES` (64 MiB, a constant beside `DEFAULT_FETCH_BATCH`, not a configuration key in v1) and orders them), so a page holds at least one row and at most the budget plus one, and the stream ends when the inner query returned fewer rows than asked and the cut dropped none of them; `n` starts at 16 and is then the budget divided by the largest row of the last pages holding at least a row bound's worth of rows (`RowBoundMemory`), never under 4 and never over `DEFAULT_FETCH_BATCH`; the rows read past the cut (at most `n - 1`) are read by the server and dropped by the outer layer, and each costs the server one serialization of its item text and two spills of it (the window functions buffer the `n` rows before the outer layer drops them), so a page costs the server `n` rows of text, not the rows it returns: sized from the average, a page of 984 small rows and 3 large ones asked the next page for 810 rows of a table that was then large rows only, 5.9 GB of server temp files and 18 to 44 s for the three rows it returned; sized from the largest row it asks for 4 (208 MiB, 1.1 s); the over-read that remains is the second page, and then the page that follows a row bound's worth of small rows, up to `n` rows of up to the line cap, at most once per row bound's worth of small rows; the key columns come back under qualified aliases so the next cursor is the row's own typed key, and the outer `ORDER BY` states a guarantee the executor gives anyway (a mutant without it passes every behavioural test, and the query text pins it); the snapshot, the lock and which tables stream `key_ordered` are unchanged. On SQLite a page stops at `DEFAULT_FETCH_BATCH` rows or after the row that reaches `SNAPSHOT_FETCH_BYTES` of item text, whichever comes first, and holds at least one row; the rows are pulled one at a time, so the stream pulls none past the cut, while the driver's worker thread reads at most five rows ahead of the pull (sqlx's `row_buffer_size`, set on the pool to four from its default of 50, which read every row of a 24-row worst-shape table ahead of the first page, plus the row the worker holds while its `send` waits; 160 MiB at the line cap), and the stream ends only when a page is short of the row bound and no cut happened. The cut rule exists in SQL on PostgreSQL and in Rust on SQLite, so the check that no page holds more than the budget plus one row lives in `extenddb_storage::snapshot_conformance`, which both run; a MongoDB batch is already bounded by the server's 16 MiB reply. +Tests that change (5.1 step 1): `create_backup_refuses_a_table_with_vector_indexes` (`crates/engine/src/backup/tests.rs`) stays for that path, PR 8 adds the create half of an off-host round trip of a vector table beside it (PR 9a adds the restore half), and `test_vector_table_backup_refused_in_catalog_and_restored_from_store` (`tests/test_backup_restore_fidelity.py`; `test_create_backup_of_a_vector_table_is_refused` until PR 9) expects the refusal or the round trip according to the server's configured path, which it reads from `EXTENDDB_TEST_BACKUP_STORE=`, a variable the test harness exports and every CI job that runs a suite creating backups sets it (the Rust integration suite reads it the same way, `tests/rust/src/backup_restore.rs`): it derives its branch from the variable, prints which branch ran, and fails if the variable is unset, rather than accepting either outcome; the restore's `ensure_restorable` refusal of every vector backup narrows to "the target has no vector support" only in PR 9a, which rewrites `vector_index_backups_are_refused_in_the_engine` to that case; the frozen legacy reader keeps #384's behaviour for a legacy vector backup (section 3), and section 10 corrects the differences doc's vector row. -State transitions it owns: CREATING to AVAILABLE and to FAILED by the job (5.1 steps 4, 5) and by the reconciler (5.1 step 6); every deleting duty of 4(a). The reconciler, in one place: The reconciler runs at startup and then every `reconcile_interval_secs` (default 60, section 7); it acts on no row of the store while the configured store is unhealthy (the legacy duty touches no store and still runs), and it checks the maintenance claim first, and while one is held it runs only the duties that resolve the states a maintenance command waits on: it finishes DELETING rows of the configured store, resolves CREATING backup rows whose owner is dead and fails RUNNING `restore_jobs` rows whose owner is dead rather than resuming them (a resume is a job, which the claim excludes), removing their targets, and runs the FAILED-target duty (such a target keeps its backup in use, so `--forget-backup` refuses until the reconciler has removed it); it skips every other duty, so it never deletes the prefix of a FAILED or DELETED row a `--move-store` is rewriting. Its duties: resolve a CREATING backup row whose owner is dead to AVAILABLE or FAILED; delete the prefix of a FAILED row once its `heartbeat_at` is older than `stale_job_after_secs`, retrying while the prefix lists non-empty; finish a DELETING row (delete the manifest, then the prefix, then mark DELETED once the prefix lists empty); delete the prefix of a DELETED row that still lists objects; remove a FAILED or DELETED backup row once its prefix is empty and its `completed_at` is older than `failed_row_retention_secs` (default 7 days); claim and resume a RUNNING `restore_jobs` row whose owner is dead (on PostgreSQL only once its `heartbeat_at` is older than `stale_job_after_secs`, 5.3 step 7, and a row with no `restore_progress` row is resumed only if the target's data table is empty, otherwise marked FAILED with reason `target_not_empty`, its target left in place and reported for the operator, since a catalog restored from a dump can name a restore that finished and was written to since; the operator inspects that target at the storage level (a CREATING table accepts no reads) and removes it with `extenddb backup remove-target
    ` (PR 10), the one command that removes a FAILED restore's CREATING target and its record, through `remove_restore_target`, under the maintenance claim like the other writing commands, since DeleteTable refuses a restore target (5.3 step 6); a legacy restore (`kind = legacy`) is marked FAILED and its target removed the same way instead, section 3; on MongoDB a row with `restore_backfill_pending` is DONE, not RUNNING, and is never resumed while the backfill worker finishes it, 5.3 step 6), skipping it to the next cycle when no job permit is free; finish a FAILED `restore_jobs` row whose target table is still CREATING, the FAILED-target duty (`BackupCatalog::remove_restore_target`, section 3, not DeleteTable, which refuses a restore target; on PostgreSQL after the same delay; never for reason `target_not_empty`, whose target is the operator's, above); remove a CREATING restore target that has no job row (`remove_restore_target`); remove `restore_progress` rows whose table is not a CREATING restore target; remove spool entries under this instance's `spool_path` whose job is not RUNNING with this instance's `spool_host` (5.3 step 3); on Cassandra, remove an `active_restores` entry of a backup row whose target is not a CREATING restore target of that backup; remove a legacy CREATING backup row whose create is dead, with its payload (section 3). The reconciler's writes to a backup row carry what the job's would (`size_bytes`, `item_count`, `source_table_details_json` from the manifest for AVAILABLE; `completed_at` for both outcomes, the retention clock). On PostgreSQL its claim is an advisory lock on a dedicated connection that it closes when the row is resolved. `extenddb backup list --failed` prints FAILED rows with their `failure_reason`. +State transitions it owns: CREATING to AVAILABLE and to FAILED by the job (5.1 steps 4, 5) and by the reconciler (5.1 step 6); every deleting duty of 4(a). The reconciler, in one place: The reconciler runs at startup and then every `reconcile_interval_secs` (default 60, section 7); it acts on no row of the store while the configured store is unhealthy (the legacy duty touches no store and still runs), and it checks the maintenance claim first, and while one is held it runs only the duties that resolve the states a maintenance command waits on: it finishes DELETING rows of the configured store, resolves CREATING backup rows whose owner is dead and fails RUNNING `restore_jobs` rows whose owner is dead rather than resuming them (a resume is a job, which the claim excludes), removing their targets, and runs the FAILED-target duty (such a target keeps its backup in use, so `--forget-backup` refuses until the reconciler has removed it); it skips every other duty, so it never deletes the prefix of a FAILED or DELETED row a `--move-store` is rewriting. Its duties: resolve a CREATING backup row whose owner is dead to AVAILABLE or FAILED; delete the prefix of a FAILED row once its `heartbeat_at` is older than `stale_job_after_secs`, retrying while the prefix lists non-empty; finish a DELETING row (delete the manifest, then the prefix, then mark DELETED once the prefix lists empty); delete the prefix of a DELETED row that still lists objects; remove a FAILED or DELETED backup row once its prefix is empty and its `completed_at` is older than `failed_row_retention_secs` (default 7 days); claim and resume a RUNNING `restore_jobs` row whose owner is dead (the resume is PR 9a's, with the restore job, through `claim_restore_dead`, 4(e); once its `heartbeat_at` is older than `stale_job_after_secs` on every backend, 5.3 step 7, and a row with no `restore_progress` row is resumed only if the target's data table is empty, otherwise marked FAILED with reason `target_not_empty`, its target left in place and reported for the operator, since a catalog restored from a dump can name a restore that finished and was written to since; the operator inspects that target at the storage level (a CREATING table accepts no reads) and removes it with `extenddb backup remove-target --account
    ` (PR 10's command, shipped with PR 9: on MongoDB and Cassandra a crash before the first progress record, the whole first data file on MongoDB and the first 10,000 items or 32 MiB on Cassandra, leaves written items and no progress, so the ordinary crash there ends in this state and the release that ships the restore ships its remedy, item 33; until it runs, DeleteTable refuses the target, 5.3 step 6, and DeleteBackup and a second restore of the backup are `BackupInUseException`, 5.2). `extenddb backup remove-target --account
    ` removes a restore target DeleteTable refuses (a CREATING table) and its restore record, under the maintenance claim like the other writing commands: a FAILED record's target whatever the reason (`target_not_empty` included), a RUNNING record's target whose owner is dead (the backend's `list_dead_restore_jobs` names it under `stale_job_after_secs` on the database's clock; on SQLite under the serve lock the window is zero and every RUNNING record is listed), and a legacy record's target the same way; it refuses a RUNNING record whose owner is live ("stop it or wait for it"), a DONE record and a table no record names (an ACTIVE table is one or the other), the latter two as "not a restore target". It needs no store, prints the claim it took and one line naming what it removed and the record's state, exits 1 with the refusal otherwise, and logs the removal under `extenddb::audit` with the account, the table and the backup's ARN. The removal is `remove_restore_target`, section 3; a legacy restore (`kind = legacy`) is marked FAILED and its target removed the same way instead, section 3; on MongoDB a row with `restore_backfill_pending` is DONE, not RUNNING, and is never resumed while the backfill worker finishes it, 5.3 step 6), skipping it to the next cycle when no job permit is free; finish a FAILED `restore_jobs` row whose target table is still CREATING, the FAILED-target duty (4(e); landed in PR 6, whose `list_dead_restore_jobs` returns such a row once it is stale; `BackupCatalog::remove_restore_target`, section 3, not DeleteTable, which refuses a restore target; never for reason `target_not_empty`, whose target is the operator's, above); remove `restore_progress` rows whose table is not a CREATING restore target and spool entries under this instance's `spool_path` whose job is not RUNNING with this instance's `spool_host` (5.3 step 3), both from PR 9a, which adds the records they read (the duty that removed a CREATING restore target with no job row is withdrawn: on every backend the job record is the registration itself, so no such state exists after migration 6); on Cassandra, remove an `active_restores` entry of a backup row whose target is not a CREATING restore target of that backup; remove a legacy CREATING backup row whose create is dead, with its payload (section 3). The reconciler's writes to a backup row carry what the job's would (`size_bytes`, `item_count`, `source_table_details_json` from the manifest for AVAILABLE; `completed_at` for both outcomes, the retention clock). On PostgreSQL its claim is an advisory lock on a dedicated connection that it closes when the row is resolved. `extenddb backup list --failed` prints FAILED rows with their `failure_reason`. -Kill points (testing summary): between a data file upload and the manifest; between the manifest put and the AVAILABLE update; SIGTERM during a backup. Main tests that must pass unchanged: #384's backup tests other than the vector tests named above. Documentation rows it corrects: the differences doc's backup consistency row (its MongoDB half, "not point-in-time" today, which the snapshot read concern of section 3 makes one; its Cassandra half stays true), its vector-backup row ("Backing up and restoring a table that has vector indexes": supported on the off-host path from PR 8 and 9a, refused only into a target without vector support, CreateBackup on the legacy create, without `[backup.store]` or on Cassandra, still refused as on main, the legacy reader's behaviour kept for legacy backups; 5.1 step 1), its restore-interruption row (one reconciler with the duties above in place of the per-backend sweeps), a new row for CreateBackup of a table holding an `N` value #618 let in (the backup fails on the off-host path, where main backs it up and restores it; 5.1 step 4), the limits doc's line cap and file size rows, the usage guide's "backups not supported" line; upgrade manual: on the off-host path a backup of a table holding an `N` value the wire refuses (#618) fails where main backs it up, which meets file-backed SQLite without a `[backup]` section and any deployment once a store is configured (5.1 step 4), the reason reaches only the operator (`backup list --failed`, and the log, which names each offending item's key and attribute path with the backup ARN a tenant's support case can cite), and the way out is to rewrite each named item. +Kill points (testing summary): between a data file upload and the manifest; between the three documents and the manifest (`before_manifest`); between the manifest put and the AVAILABLE update; SIGTERM during a backup. Main tests that must pass unchanged: #384's backup tests other than the vector tests named above. Documentation rows it corrects: the differences doc's backup consistency row (its MongoDB half, "not point-in-time" today, which the snapshot read concern of section 3 makes one; its Cassandra half stays true), its vector-backup row ("Backing up and restoring a table that has vector indexes": supported on the off-host path from PR 8 and 9a, refused only into a target without vector support, CreateBackup on the legacy create, without `[backup.store]` or on Cassandra, still refused as on main, the legacy reader's behaviour kept for legacy backups; 5.1 step 1), its restore-interruption row (one reconciler with the duties above in place of the per-backend sweeps), a new row for CreateBackup of a table holding an `N` value #618 let in (the backup fails on the off-host path, where main backs it up and restores it; 5.1 step 4), the limits doc's line cap and file size rows, the usage guide's "backups not supported" line; upgrade manual: on the off-host path a backup of a table holding an `N` value the wire refuses (#618) fails where main backs it up, which meets file-backed SQLite without a `[backup]` section and any deployment once a store is configured (5.1 step 4), the reason reaches only the operator (`backup list --failed`, and the log, which names each offending item's key and attribute path with the backup ARN a tenant's support case can cite), and the way out is to rewrite each named item. ### PR 9a: restore job Goal: the restore job of 5.3 on the same backend over a filesystem store. Its line names the interfaces and carries the acceptance: -- PR 9a: restore job per 5.3, same backend, filesystem store: request creates table and `restore_jobs` row atomically in a detached task and returns CREATING (`TableAlreadyExistsException` or `TableInUseException` on a name race, as #384 returns them; `LimitExceededException` over the job limit); Cassandra claim and heartbeat by lightweight transaction; job verifies every file before loading any, with `.verified` spool markers; `bulk_load(target_table_id, claim, file_index, start_at, items)` per backend with the progress record as 5.3 step 5 specifies for each (PostgreSQL data migration 005 for `restore_progress`, reserved here) and the `BackupCatalog` restore methods of section 3; the MongoDB terminal sequence of 5.3 step 6 (DONE and `restore_backfill_pending` in one transaction, the worker's flip conditioned on the flag); `spool_host` as host and spool path; the item validator PR 8 adds, bound with the target's `max_item_size_bytes` (`N` normalization included); items read through `ExtenddbManifest::reader_prehashed`; the restore's vector refusal narrowed to a target without vector support, with `vector_index_backups_are_refused_in_the_engine` rewritten to that case; duplicate rejection across and within files (#394's per-file key-range check on key-ordered backups, the count after the last file on unordered ones, the MongoDB post-resume count check); the manifest and sidecar spooled for a resume; a PostgreSQL resume or target deletion only after the stale window; conditional ACTIVE flip before the progress rows are deleted, the job's target removal (`remove_restore_target`, not DeleteTable) gated on a matched FAILED write, DeleteTable on a restore target refused as #384 refuses it, the in-use rule of 5.2 checked in step 1 before the permit and again in step 4's registration together with `status = AVAILABLE` under the backup row's lock (MongoDB's `$inc`, Cassandra's `active_restores` lightweight transaction), multi-part-key tables restored; `bulk_load` writing index rows in the batch's transaction, with the base rows, on PostgreSQL and SQLite and the ACTIVE flip setting every index ACTIVE there; download transport errors retried for 15 minutes; a restore stopped by shutdown left RUNNING; the store and `kind` recorded on the job row and a takeover writing its `spool_host`; a PostgreSQL resume with no progress row only into an empty target. Acceptance: round trip preserves items, keys, GSIs, LSIs, throughput, SSE, vector where supported, and restores the table `STANDARD` whatever its class (#384's `table_class_is_not_restored` passes unchanged); tampering file 2's sha256 leaves a table that was never writable and is gone after the job fails, with no item ever written; SIGKILL after file 1's commit then restart writes exactly the backup's item count on every backend (MongoDB via duplicate-key suppression); SIGKILL at each kill point of the testing summary leaves a table that either finishes with the right count or is gone (on MongoDB, killed between the DONE write and the worker's flip, the restarted worker finishes it and the reconciler leaves it alone); an unordered backup carrying one key in two files fails into every target, Cassandra included, and a key-ordered backup whose file leaves its `key_range` fails; a second restore of a backup while a target restored from it is CREATING, and a DeleteBackup of that backup, return `BackupInUseException` on every backend, on MongoDB also between the job's DONE write and the worker's ACTIVE flip, and #384's `test_restore_into_a_creating_target_and_a_second_restore_of_its_backup_are_refused` passes unchanged against the new path; PR 6's two orderings over an off-host backup pass unchanged against this PR's restore; on MongoDB two concurrent registrations of one backup leave one target and return `BackupInUseException` to the other; a restore of an ARN of account B that carries the id of account A's backup returns BackupNotFound; on SQLite and PostgreSQL a Query on each GSI and SearchVectors return the restored items as soon as the target reports ACTIVE; a SIGTERM during a restore leaves its row RUNNING and the restore finishes after the restart; a store outage shorter than the retry bound mid-download leaves the restore RUNNING, and it completes once the store answers. Depends on PR 8. +- PR 9a: restore job per 5.3, same backend, filesystem store (shipped in PR 9; the engine's `check_catalog_row_for_restore` no longer refuses a `backup_index` row with "Backup is in a backup store; this release restores in-catalog backups only", the 24 restore cells of `tests/test_backup_restore_fidelity.py` and `tests/test_auth_permissions.py`'s restore-authorization cell run on a server with a store, and the vector wire test's restore asserts the round trip into a vector-capable target and the refusal into one without): request creates table and `restore_jobs` row atomically in a detached task and returns CREATING (`TableAlreadyExistsException` or `TableInUseException` on a name race, as #384 returns them; `LimitExceededException` over the job limit); Cassandra claim and heartbeat by lightweight transaction; job verifies every file before loading any, with `.verified` spool markers; `bulk_load(target, claim, BulkLoadFile { file_index, start_at, reapplying }, items)` per backend with the progress record as 5.3 step 5 specifies for each (PostgreSQL migration 7 for `restore_progress`) and the `BackupCatalog` restore methods of section 3; the MongoDB terminal sequence of 5.3 step 6 (DONE and `restore_backfill_pending` in one transaction, the worker's flip conditioned on the flag); `spool_host` as host and spool path; the item validator PR 8 adds, bound with the target's `RestorableLimits` (item, hash key, range key and attribute name sizes; `N` normalization included); items read through `ExtenddbManifest::reader_prehashed` from the spool and `ExtenddbManifest::reader` in place; the restore's vector refusal narrowed to a target without vector support, with `vector_index_backups_are_refused_in_the_engine` rewritten to that case; duplicate rejection across and within files (#394's per-file key-range check on key-ordered backups, the count after the last file on unordered ones, the MongoDB post-resume count check); the manifest and sidecar spooled for a resume; a PostgreSQL resume or target deletion only after the stale window; conditional ACTIVE flip before the progress rows are deleted, the job's target removal (`remove_restore_target`, not DeleteTable) gated on a matched FAILED write (`remove_restore_target` on PostgreSQL drops the relations `DeleteTable` drops (`table_storage_relations`): the data table, every secondary index's table and every vector index's `_ddb_vec_` table, their ids read in the claiming transaction), DeleteTable on a restore target refused as #384 refuses it, the in-use rule of 5.2 checked in step 1 before the permit and again in step 4's registration together with `status = AVAILABLE` under the backup row's lock (MongoDB's `$inc`, Cassandra's `active_restores` lightweight transaction), multi-part-key tables restored; `bulk_load` writing index rows in the batch's transaction, with the base rows, on PostgreSQL and SQLite and the ACTIVE flip setting every index ACTIVE there; download transport errors retried for 15 minutes; a restore stopped by shutdown left RUNNING; the store and `kind` recorded on the job row and a takeover writing its `spool_host`; a PostgreSQL resume with no progress row only into an empty target. Acceptance: round trip preserves items, keys, GSIs, LSIs, throughput, SSE, vector where supported, and restores the table `STANDARD` whatever its class (#384's `table_class_is_not_restored` passes unchanged); tampering file 2's sha256 leaves a table that was never writable and is gone after the job fails, with no item ever written; SIGKILL after file 1's commit then restart writes exactly the backup's item count on every backend (MongoDB's duplicate-key suppression is proved by `restore_mongodb_after_insert_many` at its second reach, since a kill after a file's commit resumes MongoDB at the next file with nothing of it written; the Cassandra points run with a SQLite source, since a Cassandra server's own backup is in-catalog, item 33); SIGKILL at each kill point of the testing summary leaves a table that finishes with the right count (the base table's and every GSI's), is gone, or, at the first reach of the two points that pause before a first progress record (`restore_mongodb_after_insert_many`, `restore_cassandra_before_progress`), is FAILED `target_not_empty` for `extenddb backup remove-target`, which removes it (on MongoDB, killed between the DONE write and the worker's flip, the restarted worker finishes it and the reconciler leaves it alone); an unordered backup carrying one key in two files fails into every target, Cassandra included, and a key-ordered backup whose file leaves its `key_range` fails; a second restore of a backup while a target restored from it is CREATING, and a DeleteBackup of that backup, return `BackupInUseException` on every backend, on MongoDB also between the job's DONE write and the worker's ACTIVE flip, and #384's `test_restore_into_a_creating_target_and_a_second_restore_of_its_backup_are_refused` passes unchanged against the new path; the two orderings over an off-host backup (a `backup_index` row), gated on the `backup_index` row lock, run here for the first time, deferred from PR 6 (which has no restore of such a row): a DeleteBackup that wins leaves the registration BackupNotFound and no target, a registration that wins leaves DeleteBackup BackupInUse; on MongoDB and on Cassandra two concurrent registrations of one backup leave one target and return `BackupInUseException` to the other (on Cassandra the registration's lightweight transaction, `IF backup_status = 'AVAILABLE' AND active_restores = null`, is linearized on the backup's partition, so the second finds `active_restores` non-null; the in-use pre-read is advisory); a restore of an ARN of account B that carries the id of account A's backup returns BackupNotFound; on SQLite and PostgreSQL a Query on each GSI and SearchVectors return the restored items as soon as the target reports ACTIVE; a SIGTERM during a restore leaves its row RUNNING and the restore finishes after the restart; a store outage shorter than the retry bound mid-download leaves the restore RUNNING, and it completes once the store answers; a 1M-item restore on PostgreSQL and SQLite records its duration and peak memory (the record above: 9.5 s and 26.8 s, 16 and 89 MiB over idle), the MongoDB and Cassandra figures being PR 11's. Depends on PR 8. Invariants honoured: 4(b) (the restore record and its transitions), 4(d) (every target by `RestoreTargetKey`), 4(e) (the claim, the FAILED-write gate before the target is removed, the PostgreSQL stale window), 4(f) (the job checks the maintenance claim after its row exists), 5.2 (the registration under the backup row's lock), 5.3 as written, section 1's duplicate rule and vector refusal, and the Goals' fidelity list. -Interfaces it adds: `bulk_load` (section 3) and the restore methods of `BackupCatalog`: +Interfaces it adds: `RestoreEngine` (section 3) and the restore methods of `BackupCatalog`: ```rust - // Restore job record (section 4). PR 9a. One target is addressed by `RestoreTargetKey` (`account_id`, - // `table_name`, `table_id`): MongoDB's `tables` `_id` and Cassandra's `tables` primary key are - // (`account_id`, `table_name`), and the table id guards against a table re-created under the name. - fn claim_restore(&self, row: &RestoreJobRow) -> BoxFuture, StorageError>>; // same contract as claim_backup - // at creation it is 5.3 step 4's registration: the target, the job row, and the re-check under the backup row's - // lock that the row is AVAILABLE and not in use (BackupNotFound or BackupInUse otherwise, no target left); - // a takeover writes the claimant's `spool_host` in the same conditional write that changes the generation - fn read_restore_job(&self, key: &RestoreTargetKey) -> BoxFuture, StorageError>>; - fn list_restore_jobs(&self, status: Option) -> BoxStream>; - fn record_progress(&self, claim: &dyn JobClaim, file_index: u32, items_applied: u64) -> BoxFuture>; - // the per-file (or per-batch) progress write of 5.3 step 5, conditioned on the claim's generation - fn count_restore_target(&self, key: &RestoreTargetKey) -> BoxFuture>; - // a storage-level count of a CREATING table: Scan is refused on one, so the count is a backend method + // RestoreEngine, a trait in StorageEngine's bound (PR 9). StorageEngine has no + // methods of its own; bulk_load, finish_restore and the counters live here. + fn bulk_load<'a>(&'a self, target: &'a RestoreTargetKey, claim: &'a dyn JobClaim, + file: BulkLoadFile, items: BoxStream<'a, Result>) + -> BoxFuture<'a, Result>; // BulkLoadFile { file_index, start_at, reapplying } + // applies one data file with its secondary- and vector-index rows, recording progress in the + // same write; `reapplying` lets MongoDB suppress E11000 on the one unrecorded file a resume meets + fn finish_restore<'a>(&'a self, target: &'a RestoreTargetKey, claim: &'a dyn JobClaim, + item_count: u64, size_bytes: u64) -> BoxFuture<'a, Result>; + // Finished { Active, Pending (MongoDB: the worker flips the table), NotMatched(reason) } + fn count_restore_target<'a>(&'a self, key: &'a RestoreTargetKey) -> BoxFuture<'a, Result>; + fn read_restore_progress<'a>(&'a self, key: &'a RestoreTargetKey) // default Ok(vec![]) + -> BoxFuture<'a, Result, StorageError>>; + fn sweep_restore_progress<'a>(&'a self, running: &'a [String]) // default Ok(0) + -> BoxFuture<'a, Result>; + + // Restore job record on BackupCatalog (section 4). PR 9. A target is addressed by RestoreTargetKey + // (account_id, table_name, table_id); the table id guards a name re-created under it. + fn claim_restore<'a>(&'a self, account_id: &'a str, input: CreateTableInput, + registration: &'a RestoreRegistrationRow) + -> BoxFuture<'a, Result<(TableDescription, Box), StorageError>>; + // 5.3 step 4's registration: create the CREATING target and the RUNNING record under the backend's + // own generation (registration.owner_generation is UNOWNED_GENERATION), the re-check under the + // backup_index row's lock that it is AVAILABLE and not in use (BackupNotFound / BackupInUse else); + // returns the description the request answers + fn claim_restore_dead<'a>(&'a self, key: &'a RestoreTargetKey, stale_after: Duration, + spool_host: Option<&'a str>) + -> BoxFuture<'a, Result>, StorageError>>; + // the restore analogue of claim_dead_backup: take over a RUNNING manifest record whose heartbeat is + // stale, writing the claimant's spool_host, a fresh generation and heartbeat_at = now() in one write (4(e)) + fn read_restore_job<'a>(&'a self, key: &'a RestoreTargetKey) + -> BoxFuture<'a, Result, StorageError>>; + fn list_restore_jobs<'a>(&'a self, status: Option) + -> BoxFuture<'a, Result, StorageError>>; + // remove_restore_target and fail_restore_job are PR 6's, used by the job's and the reconciler's failure + // paths; RestoreJobRow carries store, spool_host, file_index, items_applied, failure_reason, completed_at. ``` -`count_restore_target` exists because a CREATING table accepts no reads (5.3), so the count of section 1 cannot be a Scan. The job holds the same claim as a backup job (5.1 step 3), keyed `/
    ` on PostgreSQL (the key and seed #384's `try_restore_lock` already uses, so a legacy restore and a new restore into one name exclude each other through the lock as well as the name), the row itself on MongoDB and Cassandra, and heartbeats on the same timer. -The spool and the resume (5.3 step 3): A file that verifies gets a `.verified` marker in the spool, written after the file's `fsync` and renamed into place, so a spooled file without its marker (a crash mid-download) is re-downloaded on resume. The spool is per instance: the `restore_jobs` row records `spool_host` as `:`, a successor whose `spool_host` differs has an empty spool and downloads everything again (its claim writes its own `spool_host` in the conditional write that takes the row, so the sweep of section 4 never removes the spool of the job that owns the row), and two instances must not share a `spool_path` (startup holds a `.lock` file in the spool with an exclusive `flock`, the serve-lock mechanism SQLite already uses, and refuses a spool whose lock another live process holds). A resume with the same `spool_host` reads the manifest and sidecar from the spool; the sweep of section 4 removes only entries of this `spool_host`. - The restore spool holds plaintext items while a restore runs; it is created `0700` with `0600` files, lives outside the store root, and is removed when the job ends. -The load (5.3 step 5): PostgreSQL: multi-row INSERT in 500-row batches with the GSI, LSI and vector-index rows in the same transaction, not through the propagation queue (section 3), one data transaction per file. The catalog and the data live in different databases, so the per-file progress record is a row in the data database (`restore_progress(table_id, file_index)`, data migration 005, reserved for PR 9a next to the catalog numbers in PR 1 and PR 6), written in the file's own transaction; the catalog job row holds only the terminal status. A file is therefore applied exactly once. SQLite: 10k-item batches, each its own transaction holding the batch's index rows, with the progress record `(file_index, items_applied)` committed in each batch's transaction; MongoDB: `insert_many(ordered=false)` per file, the progress record written after the insert, a resumed restore re-applying the one file whose record is missing with E11000 duplicate-key errors ignored on that file only (any other write error fails the restore), then `countDocuments` checked against the cumulative `item_count` (section 1); Cassandra: #384's restore write path (`put_restored_item`, eight in flight) in 10k-item batches, the progress record written after each batch with a lightweight transaction conditioned on the job's `owner_generation`, so a crash re-applies at most one batch, which is safe because the target accepts no other writes and a put of an identical item changes nothing. The table's `restore_progress` rows live in the data database on PostgreSQL (step 5), so they cannot be in that transaction; they are deleted after it commits, and a crash between the two leaves orphan rows for an ACTIVE table, which are harmless because the reconciler reads progress only for a CREATING restore target and removes progress rows whose table is not one (section 4). The other order, progress rows first, would make a crash between the two resume the restore from file 0 and fail a finished restore on the first duplicate key. +There is no `record_progress`: each backend records progress inside `bulk_load`, in the write that applied the items (item 33). `count_restore_target` exists because a CREATING table accepts no reads (5.3), so the count of section 1 cannot be a Scan. The job holds the same claim as a backup job (5.1 step 3), keyed `/
    ` on PostgreSQL (the key and seed #384's `try_restore_lock` already uses, so a legacy restore and a new restore into one name exclude each other through the lock as well as the name), the in-process slot `restore:/
    ` on SQLite (4(e)), the row itself on MongoDB and Cassandra, and heartbeats on the same timer. +The spool and the resume (5.3 step 3): A file is downloaded to `restore-
    /.json.gz.partial` (`` the file's index zero-padded to six digits, `000003.json.gz`), synced, renamed to `.json.gz` and then given its `.verified` marker, created and synced after the rename, so a spooled file without its marker (a crash mid-download) is re-downloaded on resume and one with it is read through `reader_prehashed`. `.verified` holds the file's sha256; a file is verified for a run only while its marker names the digest the manifest in hand records, so a replaced manifest has no old file accepted. A spool entry is a job's to reuse only while the record named this host before the job's claim: a resume whose record moved to another host and back removes its own old entry before use, and a job that learnt the record is another's (its claim lost, a DONE or FAILED write that matched no record) leaves its entry to the sweep rather than removing it from under the record's owner. The spool is per instance: the `restore_jobs` row records `spool_host` as `:`, a successor whose `spool_host` differs has an empty spool and downloads everything again (its claim writes its own `spool_host` in the conditional write that takes the row, and the sweep of section 4 lists the spool before the records, 5.3 step 7, so it never removes the spool of the job that owns the row), and two instances must not share a `spool_path` (startup holds a `.lock` file in the spool with an exclusive `flock`, the serve-lock mechanism SQLite already uses, and refuses a spool whose lock another live process holds). A resume with the same `spool_host` reads the manifest and sidecar from the spool; the sweep of section 4 removes only entries of this `spool_host`. The restore spool holds plaintext items while a restore runs; it is created `0700` with `0600` files, lives outside the store root, and its entry is removed at every end of the job but the shutdown's, which keeps it for the resume, and the ends at which the job learnt the record is another's (a lost claim, a DONE or FAILED write that matched no record), which leave it to the sweep because a same-host successor may be reading it (5.3 step 7). +The load (5.3 step 5): PostgreSQL: multi-row INSERT in batches of 500 rows or 4 MiB of item text, whichever is reached first, with the GSI, LSI and vector-index rows in the same transaction, not through the propagation queue (section 3), one data transaction per file. The catalog and the data live in different databases, so the per-file progress record is a row in the data database (`restore_progress(table_id, file_index, items_applied)`, migration 7 (a StoreKind::Data body on PostgreSQL only; NotApplicable on SQLite, MongoDB and Cassandra)), written in the file's own transaction; the catalog job row holds only the terminal status. A file is therefore applied exactly once. On PostgreSQL a file's `restore_progress` row is a plain INSERT when the file is applied from its start and an upsert on `(table_id, file_index)` when it is resumed past its start, so a row a decoder panic left short of the file's item count (its progress committed, the job's FAILED write lost to a second fault) is completed by the resume rather than failing the restore on the row's primary key. SQLite: batches of at most 10,000 items or `RESTORE_BATCH_BYTES` (32 MiB) of serialized item text (the batch closes at the first item that takes it past the bound), each its own `BEGIN IMMEDIATE` transaction under the engine write lock holding the batch's base, secondary-index and vector-index rows, with the progress record `(file_index, items_applied)` committed in each batch's transaction conditioned on the generation and on RUNNING (a record another owner took rolls the batch back); MongoDB: `insert_many(ordered=false)` in 1000-document calls at `w: majority` per file, base documents only (the backfill worker fills the index collections), the record's `file_index` written after the insert conditioned on the generation and on RUNNING, a resumed restore re-applying the one file whose record is missing (`reapplying`) with E11000 duplicate-key errors ignored on that file only (any other write error fails the restore), then `count_documents` checked against the cumulative `item_count` (section 1); the re-application needs an earlier file's record, so a crash before the first file's record (anywhere in the first file) is not re-applied: the resume finds written items and no progress and answers `target_not_empty` (5.3 step 7, `extenddb backup remove-target`); Cassandra: each item's base row and every GSI and LSI row in one logged batch at a write timestamp read before the batch's claim check (`put_restored_item_at`, no lightweight transaction, eight in flight; the timestamp is the fence: a batch a stalled writer lands after a takeover carries a timestamp below every write and delete its successor made, so Cassandra keeps the successor's cells and tombstones; the base row's `version` is the timestamp, so a later ordinary write's version is above it, ADR-0027), in batches of at most 10,000 items or `RESTORE_BATCH_BYTES` (32 MiB) of serialized item text, the progress record written after each batch with a lightweight transaction conditioned on the job's `owner_generation` and on RUNNING, so a crash after the first batch's record re-applies at most one batch, and a crash before it leaves the first batch's items with no progress, resumed with the first file re-applied (step 7), which is safe because the target accepts no other writes while CREATING and a put of an identical item rewrites the same base and index rows: a PutItem or UpdateItem of an item whose secondary-index keys do not change rewrites its GSI and LSI rows rather than deleting them (#424: the write path's per-index delete-and-insert in one LOGGED BATCH shared the batch's timestamp, and a tombstone wins a tie, so the two statements left the row deleted, GitHub issue #742; the write path skips the DELETE when the old and new items produce the same index row and lets the INSERT rewrite it, and stamps every index row with its base row's `version`, ADR-0027, so a re-applied restore batch, a resume re-applying at most the one batch it was in, rewrites the same base and index rows under a later stamp and a Query on each GSI sees every restored item as soon as the target reports ACTIVE); a crash before the first batch's record (in the first 10,000 items or 32 MiB) leaves the batch's items with no progress, and the resume, finding no more items than the backup's total, re-applies the first file with the same upserts (5.3 step 7; a count above the total is `target_not_empty`). The table's `restore_progress` rows live in the data database on PostgreSQL (step 5), so they cannot be in that transaction; they are deleted after it commits, and a crash between the two leaves orphan rows for an ACTIVE table, which are harmless because the reconciler reads progress only for a CREATING restore target and removes progress rows whose table is not one (section 4). On PostgreSQL the `restore_progress` rows are deleted on the data pool after the ACTIVE flip has committed; that delete cannot fail the restore (its error is logged and `RestoreTableFromBackup` still finishes ACTIVE), because the table is ACTIVE and the record DONE once the flip commits, and the reconciler's `sweep_restore_progress` removes any rows the delete left, exactly as it does after a crash between the flip and the delete. The other order, progress rows first, would make a crash between the two resume the restore from file 0 and fail a finished restore on the first duplicate key. State transitions it owns: RUNNING at registration (5.3 step 4); RUNNING to DONE with the ACTIVE flip, on MongoDB DONE then the worker's flip (5.3 step 6); RUNNING to FAILED with the target removed through `remove_restore_target` only after the FAILED write matched (5.3 step 7); the resume of a dead RUNNING row, on PostgreSQL only after the stale window (5.3 step 7). -Kill points (testing summary): every restore kill point, the fenced-out restore job and the `pg_terminate_backend` case; SIGTERM during a restore. Main tests that must pass unchanged: #384's override tests (`crates/engine/src/backup/tests.rs`), `table_class_is_not_restored`, `test_restore_into_a_creating_target_and_a_second_restore_of_its_backup_are_refused`, and PR 6's two orderings. Documentation rows it corrects: the differences doc's vector-backup row (the restore's refusal narrowed to a target without vector support) and its multi-part-key row (restored, 5.3 step 2); the differences and limits docs' override rows stay (5.3). +Kill points (testing summary): every restore kill point, the fenced-out restore job and the `pg_terminate_backend` case; SIGTERM during a restore. Main tests that must pass unchanged: #384's override tests (`crates/engine/src/backup/tests.rs`), `table_class_is_not_restored` and `test_restore_into_a_creating_target_and_a_second_restore_of_its_backup_are_refused`; the two orderings over a `backup_index` row are PR 9a's own new tests (deferred from PR 6). Documentation rows it corrects: the differences doc's vector-backup row (the restore's refusal narrowed to a target without vector support) and its multi-part-key row (restored, 5.3 step 2); the differences and limits docs' override rows stay (5.3). ### PR 9b: cross-backend matrix -- PR 9b: cross-backend matrix (twelve cells, nightly: PostgreSQL, SQLite and MongoDB sources into those three and Cassandra), #384's override tests run against the new path per 5.3, MongoDB target refuses a vector manifest, an `N` sort-key value beyond 34 significant digits into MongoDB fails with ValidationException, and a 38-digit non-key number into MongoDB restores intact; the Cassandra-target cells include an unordered backup with a duplicated key. Depends on PR 9a. +- PR 9b: cross-backend matrix (shipped in PR 9): the twelve plain cells (PostgreSQL, SQLite and MongoDB sources into PostgreSQL, SQLite, MongoDB and Cassandra targets), the `--duplicate-key` cells into every target (an unordered backup carrying one key in two files fails into every target, into Cassandra by the count alone) and the `--vector` cells (PostgreSQL and SQLite sources restored into PostgreSQL and SQLite with `SearchVectors` answering, refused before any table into MongoDB and Cassandra), run by `devtools/backup-restore-matrix`: a cell takes a backup on the source server, copies its prefix under a root the target catalog owns and gives the target's catalog its `backup_index` row with `extenddb backup sync` (PR 10; until then the cell seeded the row by hand and proved the restore alone, item 34), then restores and compares the item sets, the descriptions and the index queries; `.github/workflows/matrix-nightly.yml` (a `schedule` cron and `workflow_dispatch`) runs every cell nightly, building the four backend binaries and starting PostgreSQL, a MongoDB replica set and Cassandra as the integration workflows do. #384's override tests run against the new path per 5.3; the MongoDB target refuses a vector manifest before any table, an `N` sort-key value beyond 34 significant digits into MongoDB fails the restore with the write path's ValidationException text as its reason and the target gone, and a 38-digit non-key number into MongoDB restores intact (`tests/test_mongodb_backup_restore.py`). Depends on PR 9a. ### PR 10: `backup sync`, its options, `remove-target`, `migrate-legacy` -Goal: the catalog-loss recovery and the maintenance commands of section 4. Its line names the interfaces and carries the acceptance: -- PR 10: `extenddb backup sync` per section 4 with store-root scoping, the store id, the maintenance claim with its printed wait bound, tombstone and identity checks, the row check before the manifest read and the insert-if-absent, `--prune-missing`, `--forget-store`, `--forget-backup`, `--move-store` (with `list_backup_rows_by_root` and `rewrite_store`), `--adopt-store`, `extenddb backup remove-target
    ` (section 4), the SQLite serve lock on every command that writes and the dead-owner resolutions such a command runs first, and `extenddb backup migrate-legacy |--all` with its re-read, validation and count checks and its refusals (section 3). Acceptance: drop the catalog rows, run sync, Describe and List and Restore work; a prefix without a manifest, with a bad sidecar, with a mismatched ARN, under `changelog/`, or with a CREATING, DELETING or FAILED row is not adopted; rows for another store root are reported and untouched, and the reconciler deletes none of their prefixes; after a store tree is copied to a new root, `--move-store` re-points its rows and Describe and Restore work against the new root; `--move-store` resolves the old root's CREATING and DELETING rows, stops on a row whose data files were not all copied, and refuses a DELETING, DELETED or FAILED row whose copied prefix holds a verified manifest; `--forget-backup` refuses a CREATING row with a live owner and a backup from which a restore target is CREATING (a RUNNING restore's target, a FAILED restore's target not yet removed, and on MongoDB a DONE restore's target awaiting the backfill), and says so and does nothing for a legacy ARN; a second maintenance command while one holds the claim fails at once; on SQLite every writing command refuses to run beside a server; `migrate-legacy` on a backup the PR 0 binary created produces a prefix Describe, List and Restore serve under the original ARN, and the legacy rows are gone; with the volume unmounted (an empty root, no id or another id) the store is unhealthy, sync without the option that resolves it refuses, and no row changes; `--adopt-store` of a root another catalog owns makes this catalog the owner, and the other catalog finds the store unhealthy at its next check; with host-local roots on two instances of one catalog no reconciler or sync on either resolves or deletes a row because its own root lacks the bytes; `migrate-legacy` refuses a backup with a 13-digit id, one holding a value #618 let in and a pre-0.0.4 SQLite backup, keeps their legacy rows and names the reason; without `--prune-missing` a row whose manifest is gone is reported and stays AVAILABLE; after `--forget-store` and the store's return, sync adopts its prefixes; a CreateBackup started on another instance during `--move-store` either finishes before the rewrite or is refused; after `--move-store A` stops on a row with one data file missing under the configured root and `--forget-backup` lets the move finish, sync reports that prefix as incomplete and does not adopt it, and adopts it once the file is copied; a maintenance command started while a DELETING row and a RUNNING restore with a dead owner exist proceeds once the reconciler, which runs those two duties under the claim, has finished the one and failed the other. Depends on PR 6, 8. +Goal: the catalog-loss recovery and the maintenance commands of section 4. As implemented (revision item 34) the section ships as three PRs: #746 plain `backup sync`, `--prune-missing`, `--adopt-store`, the `sqlite-memory` startup sync and `insert_backup_row`, with the acceptance cells those carry; #820 `--move-store`, `--forget-store` and `--forget-backup` with `list_backup_rows_by_root` and `rewrite_store`, and the report of the rows of other roots, with their cells; #821 `migrate-legacy` with its cells; `remove-target` shipped with PR 9 (item 33). Its line names the interfaces and carries the acceptance: +- PR 10: `extenddb backup sync` per section 4 with store-root scoping, the store id, the maintenance claim with its printed wait bound, tombstone and identity checks, the row check before the manifest read and the insert-if-absent, `--prune-missing`, `--forget-store`, `--forget-backup`, `--move-store` (with `list_backup_rows_by_root` and `rewrite_store`), `--adopt-store`, the SQLite serve lock on every command that writes (`extenddb backup remove-target --account
    `, section 4, shipped with PR 9 rather than here, item 33: a crash before the first progress record on MongoDB or Cassandra leaves a `target_not_empty` target that no other command removes) and the dead-owner resolutions such a command runs first, and `extenddb backup migrate-legacy |--all` with its re-read, validation and count checks and its refusals (section 3). Acceptance: drop the catalog rows, run sync, Describe and List and Restore work; a prefix without a manifest, with a bad sidecar, with a mismatched ARN, under `changelog/`, or with a CREATING, DELETING or FAILED row is not adopted; rows for another store root are reported and untouched, and the reconciler deletes none of their prefixes; after a store tree is copied to a new root, `--move-store` re-points its rows and Describe and Restore work against the new root; `--move-store` resolves the old root's CREATING and DELETING rows, stops on a row whose data files were not all copied, and refuses a DELETING, DELETED or FAILED row whose copied prefix holds a verified manifest; `--forget-backup` refuses a CREATING row with a live owner and a backup from which a restore target is CREATING (a RUNNING restore's target, a FAILED restore's target not yet removed, and on MongoDB a DONE restore's target awaiting the backfill), and says so and does nothing for a legacy ARN; a second maintenance command while one holds the claim fails at once; on SQLite every writing command refuses to run beside a server; `migrate-legacy` on a backup the PR 0 binary created produces a prefix Describe, List and Restore serve under the original ARN, and the legacy rows are gone; with the volume unmounted (an empty root, no id or another id) the store is unhealthy, sync without the option that resolves it refuses, and no row changes; `--adopt-store` of a root another catalog owns makes this catalog the owner, and the other catalog finds the store unhealthy at its next check; with host-local roots on two instances of one catalog no reconciler on either deletes a prefix because its own root lacks the bytes, and `--prune-missing` on the instance that lacks them removes the row and deletes nothing: the backup is BackupNotFound on every instance until a sync on the instance that holds the bytes adopts it again (revision item 34); `migrate-legacy` refuses a backup with a 13-digit id, one holding a value #618 let in and a pre-0.0.4 SQLite backup, keeps their legacy rows and names the reason; without `--prune-missing` a row whose manifest is gone is reported and stays AVAILABLE; after `--forget-store` and the store's return, sync adopts its prefixes; a CreateBackup started on another instance during `--move-store` either finishes before the rewrite or is refused; after `--move-store A` stops on a row with one data file missing under the configured root and `--forget-backup` lets the move finish, sync reports that prefix as incomplete and does not adopt it, and adopts it once the file is copied; a maintenance command started while a DELETING row and a RUNNING restore with a dead owner exist proceeds once the reconciler, which runs those two duties under the claim, has finished the one and failed the other. Depends on PR 6, 8. -Invariants honoured: 4(c) (store identity, scoping, adoption only through `--adopt-store`), 4(f) (every writing command takes the maintenance claim), 4(g) (what sync adopts and never adopts; the opt-ins), 4(d) (the ARN checked against the path and `source_table`), and 4(a)'s rule that no deletion duty acts on the bytes of a row the operator forgot. +Invariants honoured: 4(c) (store identity, scoping, adoption only through `--adopt-store` for a durable catalog and, for an in-memory one, at its startup from another in-memory catalog), 4(f) (every writing command takes the maintenance claim), 4(g) (what sync adopts and never adopts; the opt-ins), 4(d) (the ARN checked against the path and `source_table`), and 4(a)'s rule that no deletion duty acts on the bytes of a row the operator forgot. Interfaces it adds: ```rust fn insert_backup_row(&self, row: &BackupRow) -> BoxFuture>; // sync's unowned insert-if-absent fn list_backup_rows_by_root(&self, store_kind: StoreKind, store_root: &str, status: Option) -> BoxStream>; // a root's rows whatever their `store_id`: `--forget-store` and `--move-store` (section 4), which get a root and no id; PR 10 - // on Cassandra this and `read_store_ids` read `backup_index_by_status` one partition per status for the root, - // then each row's `backups_by_arn` row, since the index carries neither `store_kind` nor `store_id` + // on Cassandra this and `read_store_ids` read `backup_index_by_status` one partition per status for the root; + // migration 6's entry carries `store_kind` and `store_id`, so `read_store_ids` needs no second read, and this + // listing reads each row's `backups_by_arn` row for the rest of the row fn rewrite_store(&self, keys: &[BackupKey], store: &StoreIdentity) -> BoxFuture>; // `--move-store`, PR 10 ``` -`backup sync` and its options, each with its contract: Sync lists the root (every object, since the trait has no delimiter listing; only the distinct `/
    //` prefixes are kept, the reserved `changelog` and `exports/` components skipped), checks each prefix against the catalog as the listing reaches it, reads the manifest and sidecar of the prefixes that have no row, and inserts an AVAILABLE row for each verified manifest whose data files the listing contains, every `data_files` key present with its `bytes`, with an insert-if-absent (`BackupCatalog::insert_backup_row`): a CreateBackup whose row landed between the check and the insert keeps its row and sync skips the prefix. Before adopting, sync parses the manifest's `backup_arn` with the one `BackupArn` type in `extenddb-core` and requires the parsed account, table and backup id to equal the listed path components and the `source_table` identity; a mismatch is logged and skipped. Foreign region or partition in an adopted ARN is accepted and logged; requests check only the account. `--move-store ` re-points the rows of that root to the configured store. It first resolves that root's CREATING and DELETING rows itself, against the configured store: a CREATING row becomes AVAILABLE if a verified manifest is there and FAILED otherwise, and a DELETING row whose prefix there holds no manifest has its deletion finished in the configured store. For each AVAILABLE row it compares the configured store's listing of the row's prefix with the manifest's `data_files` (every key present, with its `bytes`) as well as checking the verified manifest, and stops on a row whose copy is incomplete, naming it; `--forget-backup ` deletes one row's catalog entry so the move can go ahead for the rest; it runs under the maintenance claim like the other commands, refuses a CREATING row with a live owner (the job's next conditional write would match nothing and its manifest would be adopted by the next sync) and a backup in use, one from which a restore target is CREATING, and for a legacy ARN, which has no new-format row, says so and does nothing. One rule covers the three tombstone states: the move refuses to carry a DELETING, DELETED or FAILED row whose prefix in the configured store holds a verified manifest, and names it (that prefix is a backup the operator copied before or during its deletion, not a leftover; `--forget-backup` on the row is the way out). Then it rewrites `store_kind`, `store_root` and `store_id`; the reconciler then cleans the leftovers of the moved tombstones in the configured store. `--move-store` and `--forget-store` match `store_root` exactly as stored, the `?endpoint=` suffix of an S3 root included. A root whose file names another catalog stays unhealthy until an operator adopts it with `backup sync --adopt-store`, which takes the maintenance claim, rewrites the file's owner to this catalog and then syncs, recording the file's store id on the rows it adopts; the previous owner finds the store unhealthy at its next check and stops acting on it. On `sqlite-memory` the catalog, and with it the `catalog_id`, is new at every start, so the server runs a plain `backup sync` itself at startup when a store is configured, which adopts a root whose file names a `sqlite-memory` catalog as `--adopt-store` does, rewriting the file's owner to the new `catalog_id` before it syncs, so the root has one owner at a time there too; a root owned by any other catalog is unhealthy there as anywhere, and the backup CLI commands do not exist there (no file to open), so that startup sync is the only catalog-loss recovery there. +`backup sync` and its options, each with its contract: Sync lists the root (every object, since the trait has no delimiter listing; the store lists in key order, so sync holds one prefix's objects at a time and the prefix of each manifest it listed, beside what it keeps of the root's rows, whose read is its larger cost (section 7), the reserved `changelog` and `exports/` components skipped; a filesystem store builds its own listing of the root first, section 7), checks each prefix against the catalog as the listing reaches it, reads the manifest and sidecar of the prefixes that have no row, and inserts an AVAILABLE row for each verified manifest whose data files the listing contains, every `data_files` key present with its `bytes`, with an insert-if-absent (`BackupCatalog::insert_backup_row`): a CreateBackup whose row landed between the check and the insert keeps its row and sync skips the prefix. Before adopting, sync parses the manifest's `backup_arn` with the one `BackupArn` type in `extenddb-core` and requires the parsed account, table and backup id to equal the listed path components and the `source_table` identity; a mismatch is logged and skipped. An ARN of another region is adopted and logged (requests resolve it by the ARN as written); one of another partition than `aws` is skipped, since every request refuses such an ARN before it reads a row (revision item 34). `--move-store ` re-points the rows of that root to the configured store. It first resolves that root's CREATING and DELETING rows itself, against the configured store: a CREATING row becomes AVAILABLE if a verified manifest is there and FAILED otherwise, and a DELETING row whose prefix there holds no manifest has its deletion finished in the configured store. For each AVAILABLE row it compares the configured store's listing of the row's prefix with the manifest's `data_files` (every key present, with its `bytes`) as well as checking the verified manifest, and stops on a row whose copy is incomplete, naming it; `--forget-backup ` deletes one row's catalog entry so the move can go ahead for the rest; it runs under the maintenance claim like the other commands, refuses a CREATING row with a live owner (the job's next conditional write would match nothing and its manifest would be adopted by the next sync) and a backup in use, one from which a restore target is CREATING, and for a legacy ARN, which has no new-format row, says so and does nothing. One rule covers the three tombstone states: the move refuses to carry a DELETING, DELETED or FAILED row whose prefix in the configured store holds a verified manifest, and names it (that prefix is a backup the operator copied before or during its deletion, not a leftover; `--forget-backup` on the row is the way out). Then it rewrites `store_kind`, `store_root` and `store_id`; the reconciler then cleans the leftovers of the moved tombstones in the configured store. `--move-store` and `--forget-store` match `store_root` exactly as stored, the `?endpoint=` suffix of an S3 root included. A root whose file names another catalog stays unhealthy until an operator adopts it with `backup sync --adopt-store`, which takes the maintenance claim, rewrites the file's owner to this catalog and then syncs, recording the file's store id on the rows it adopts (the `backup_index` row PR 9b's matrix seeds by hand is the row this adoption writes); the previous owner acts on the root until a later check that can read its catalog finds the store unhealthy, at the earliest one reconciler interval after the cycle in progress ends, unless its deletion fence (section 4(c)) refuses a deletion or a CreateBackup first, so every server of the previous catalog is stopped before the adoption. On `sqlite-memory` the catalog, and with it the `catalog_id`, is new at every start, so the server runs a plain `backup sync` itself at startup when a store is configured, which adopts a root whose file names a `sqlite-memory` catalog as `--adopt-store` does, rewriting the file's owner to the new `catalog_id` before it syncs, so the root has one owner at a time there too once the previous process has exited (a new in-memory server is started only after the old one has exited; one started beside it adopts the root while the old one still acts on it until a later check or its fence); a root owned by any other catalog is unhealthy there as anywhere, and the backup CLI commands do not exist there (no file to open), so that startup sync is the only catalog-loss recovery there. `migrate-legacy`: PR 10 adds `extenddb backup migrate-legacy |--all`, which streams a legacy backup through the frozen reader's item stream, checking each item with the backup job's item validator (5.1 step 4), into `DataFileEncoder` and the configured store under the legacy backup id, so the ARN survives, with a manifest whose `snapshot.kind` says it is a copy of an in-catalog backup (`legacy_in_catalog`, no marker). Before it inserts the AVAILABLE row or deletes anything it re-reads the manifest it wrote through `ExtenddbManifest::from_json_bytes` and checks that the manifest's item count equals the legacy backup's; only then does it insert the row and delete the legacy rows through the reader's `delete`. It refuses a backup, keeping its legacy rows, deleting the prefix it wrote and naming the reason, when the id carries no instant (the 13-digit and bare shapes, which the manifest parser refuses, section 4), when the validator refuses an item (a value #618 let in, which main's legacy restore copies as stored), and when it cannot state the definition (a pre-0.0.4 SQLite backup recorded no vector state, so `vector_indexes` would be a guess); the way out for such a backup is to restore it through the legacy reader and take a new backup of the restored table, which has a new ARN (section 10). -The maintenance claim per backend and the SQLite serve lock (4(f) states the rules): On PostgreSQL the claim is an advisory lock keyed `hashtextextended('backup-maintenance', RESTORE_LOCK_SEED)`, held exclusively on the command's dedicated connection; a job tests it with `pg_try_advisory_lock_shared` and releases the shared lock at once. On MongoDB it is a document in a `backup_maintenance` collection, claimed and heartbeated like a job row (5.1 step 3) and held while its heartbeat is younger than `stale_job_after_secs`; on Cassandra a `backup_maintenance` row claimed by lightweight transaction with a coordinator-clock heartbeat, read by jobs at `SERIAL`. On SQLite every backup CLI command that writes the catalog takes the database's serve lock, as `destroy` and `migrate` do, so the server must be stopped first and no other claim is needed; with the serve lock held every job owner is dead, so before the wait such a command runs the reconciler's dead-owner resolutions itself (a CREATING backup row to AVAILABLE or FAILED, a RUNNING restore to FAILED with its target removed, a DELETING row finished), and a job the stop interrupted never makes it wait or refuse; `backup list` is a read and runs beside a server. +The maintenance claim per backend and the SQLite serve lock (4(f) states the rules): On PostgreSQL the claim is an advisory lock keyed `hashtextextended('backup_maintenance', RESTORE_LOCK_SEED)` (`MAINTENANCE_LOCK_KEY`), held exclusively on the command's dedicated connection, with the `backup_maintenance` row carrying the holder, the generation and the heartbeat; a job tests the lock with `pg_try_advisory_xact_lock_shared` in one autocommit statement, so the shared lock ends with the statement, and then reads the row's heartbeat age, so a holder whose session ended is still seen held until its heartbeat is older than `stale_job_after_secs`; a command's take tries the exclusive lock a few times over about 10 ms on its connection, so a job's one-statement shared lock does not refuse it. On MongoDB it is a document in a `backup_maintenance` collection, claimed and heartbeated like a job row (5.1 step 3) and held while its heartbeat is younger than `stale_job_after_secs`; on Cassandra a `backup_maintenance` row claimed by lightweight transaction with a coordinator-clock heartbeat, read by jobs at `LOCAL_SERIAL`. On SQLite every backup CLI command that writes the catalog takes the database's serve lock, as `destroy` and `migrate` do, so the server must be stopped first and no other claim is needed; with the serve lock held every job owner is dead, and so is the holder of any maintenance claim row, which such a command takes over whatever its heartbeat says; before the wait it runs the reconciler's dead-owner resolutions itself, as the claim's holder (the cycle takes the held branch without reading the claim back through the zero window) (a CREATING backup row to AVAILABLE or FAILED, a RUNNING restore to FAILED with its target removed, a DELETING row finished) and then waits and removes CREATING rows with a zero stale window, so a job the stop interrupted never makes it wait or refuse whatever its heartbeat says (a RUNNING `manifest` restore, which PR 9a introduces, is the exception revision item 29 states), and a server starting under the serve lock removes a maintenance claim a stopped command left; `backup list` is a read and runs beside a server. -State transitions it owns: a prefix to an AVAILABLE row (adoption); CREATING or DELETING rows of a moved root resolved by `--move-store`; AVAILABLE to DELETING by `--prune-missing`; rows deleted outright by `--forget-store` and `--forget-backup`; a FAILED restore's CREATING target removed by `remove-target`; legacy rows replaced by an AVAILABLE row by `migrate-legacy`. +State transitions it owns: a prefix to an AVAILABLE row (adoption); CREATING or DELETING rows of a moved root resolved by `--move-store`; rows deleted outright by `--prune-missing` (AVAILABLE rows whose manifest is gone), `--forget-store` and `--forget-backup`; a FAILED restore's CREATING target removed by `remove-target`; legacy rows replaced by an AVAILABLE row by `migrate-legacy`. Documentation rows it corrects: upgrade manual: on SQLite the backup CLI commands that write the catalog take the serve lock, so the server is stopped for `backup sync` and its options and for `purge` (section 4); legacy backups are carried forward with `backup migrate-legacy` (above), and one it refuses (an id with no instant, an item the validator refuses, a pre-0.0.4 SQLite definition) is carried forward by restoring it and taking a new backup, which has a new ARN; a store owned by another catalog is adopted only with `backup sync --adopt-store`. ### PR 11: CI and documentation -- PR 11: CI and documentation: drop the `not import_export` filters, the nightly schedule for the matrix, and the documentation per section 10 with RPO and RTO numbers from the PR 8 and 9a runs on a stated host class. Depends on PR 3, 9a and 9b. +- PR 11: CI and documentation: drop the `not import_export` filters; measure the 1M-item restore on MongoDB and Cassandra and the 1M-item vector cell on PostgreSQL and SQLite (the testing summary's two open cells; the nightly schedule for the matrix shipped with PR 9); the documentation per section 10 with RPO and RTO numbers from the PR 8, PR 9 and these runs on a stated host class. Depends on PR 3, 9a and 9b. Documentation it writes (section 10's table): New `docs/manuals/14-backup-and-restore.md`: strategy, RPO and RTO statements per store kind (RPO equals backup interval, no PITR; RTO measured in the acceptance tests and published), configuration, IAM policy example, restore drill procedure, catalog loss recovery with `backup sync`, migrating legacy backups, what a restore preserves and what it does not (the table from the readiness discussion), moving a backup to or from Amazon DynamoDB with ImportTable and export. The IAM and S3-compatible-server material points at `crates/backup-store/README.md` rather than copying it; the admin guide's database names; ADR-0019 and the Cassandra RFC's "Backup and restore" section marked as superseded; the admin and upgrade manuals' backup sections rewritten. Acceptance: the statements the series defers to the deployment guide, each stated in the manual: a filesystem store keeps the same-host failure domain, said in the first sentence of the backup section, with S3 as the production recommendation; encryption at rest is the store's job; a pickup agent runs as the server's user or as root, and a root under a storage administrator's directory, an NFS mount owned by `nobody` or a group-writable parent without the sticky bit is refused; a bind mount or a filesystem without Unix permission semantics is outside what the store can observe; an `AbortIncompleteMultipartUpload` lifecycle rule, and for a versioned bucket the `NoncurrentVersionExpiration` and `ExpiredObjectDeleteMarker` rules; the least-privilege IAM policy example; a PostgreSQL snapshot retains every dead tuple written during the backup and the SQLite WAL grows by the write volume during it, with the sizing figures the manual measures; the claim connection's keepalive bound (about a minute after the loss of the owner's host) and `pg_terminate_backend` on the claim's session as the way to end it sooner; a transaction-mode pooler is unsupported for the claim; the SDK's `trace` logs carry the session token and are not enabled in production; write access to an account's prefix is backup-administrator authority for that account, with per-account IAM prefix policies; MongoDB's `minSnapshotHistoryWindowInSeconds` as the operator setting a slow backup needs; a root that reports `0700` while enforcing nothing. -Parallel today: PR 1, 2, 3, 4 (PR 0 has merged); PR 5 follows PR 4 and PR 6 follows both. Critical path: PR 4, 5, 6, 8, 9a. PR 1 is not on it: migration numbers are reserved up front so a PITR PR and a catalog PR do not serialize on a shared number. +Parallel today: PR 1, 2, 3, 4 (PR 0 has merged); PR 5 follows PR 4 and PR 6 follows both. Critical path: PR 4, 5, 6, 8, 9a. PR 1 was not on it and merged first, as #335 with migration 5; PR 6 took id 6 and PR 9a's `restore_progress` takes id 7 (4(i)). Phase 2, unify export and import (v1 if time allows, otherwise v1.1): ### PR 13: export and import through the store @@ -445,28 +498,28 @@ Phase 3, later (v1.x): - Unit: manifest serde and golden files, key mapping and ARN parsing, store conformance. - Integration, every backend, every PR: round trip with full configuration preservation, composite keys, GSIs and LSIs, vector indexes where supported, legacy fallback, PITR surface. - Consistency: concurrent-writer snapshot test on every backend. -- Crash: SIGKILL during upload and during restore, verified by the reconciler tests. These extend the SIGKILL harness the migration tests already use. Backup kill points: between a data file upload and the manifest; between the manifest put and the AVAILABLE update (the one path to the reconciler's AVAILABLE outcome). Restore kill points: after a file's transaction commits and before the next file starts; MongoDB between `insert_many` and the progress write (the path that exercises E11000 suppression and the count check); Cassandra between a batch's last write and its progress write; SQLite mid-batch; PostgreSQL between the ACTIVE flip and the progress-row deletion; MongoDB between the DONE write (with `restore_backfill_pending`) and the worker's ACTIVE flip, where the reconciler must leave the row alone and the restarted worker must finish; between the FAILED write and the target removal of step 7; a fenced-out restore job that resumes after a successor claimed the row (its FAILED write matches zero rows and the successor's table survives); PostgreSQL, a restore job whose catalog session ended (`pg_terminate_backend` on its claim connection) while its data writes continue: no successor loads into or deletes the target until `heartbeat_at` is stale, and the job aborts at its next heartbeat. DeleteBackup kill points: after the DELETING write; after the manifest delete; after `delete_prefix` and before the DELETED write. Shutdown: SIGTERM during a backup (FAILED, empty prefix, no multipart upload left) and during a restore (row left RUNNING, finished after the restart). +- Crash: SIGKILL during upload and during restore, verified by the reconciler tests. The backup kill points are run by hand against the built binary with `devtools/backup-kill-points.sh` and its API half `devtools/backup-kill-points.py` (a server built with `extenddb-app/backup-test-hooks`, paused by `EXTENDDB_BACKUP_PAUSE` at the named point, killed by PID and restarted, with the row and the prefix watched through `extenddb backup list` and the store; the clippy job lints the feature so the pause code compiles on every push), not by a pytest cell; the restore kill points are run the same way with `devtools/backup-kill-points.py` (every point of every backend runs in the nightly matrix workflow's four `restore-kill-points-` jobs, one per backend, each built with `extenddb-app/backup-test-hooks` and starting its backend as the matrix job does, the Cassandra job building a plain SQLite binary as the source beside its hooks binary: a server built with `extenddb-app/backup-test-hooks`, whose feature chain pulls `extenddb-storage/test-hooks` beside `extenddb-engine/backup-test-hooks` so each backend's own pause point compiles in without the server naming a backend crate, a backup of enough items for two data files, a restore paused at the named point by `EXTENDDB_BACKUP_PAUSE`, `kill -9` by PID, a restart, and the outcome asserted: the exact count after the resume, of the base table and of the GSI `by-g` every item of the fixture carries, or the target gone for a failing restore), on every backend the point applies to (item 33 lists the cells as run on the build before this round, when the first reach of the Cassandra point below ended `target_not_empty`); a Cassandra kill point takes its backup on a SQLite source and seeds the target's row as the matrix does, Cassandra's own CreateBackup being in-catalog (section 3); the Cassandra point that pauses before a batch's progress record (`restore_cassandra_before_progress`) is run at its first reach (a crash before the first progress record leaves the first batch's items with no progress; the resume finds no more items than the backup's total and re-applies the first file with upserts, step 7) and at its second (`--second-reach`, the re-application of at most one batch after a recorded one). MongoDB records each batch's `insert_many` and its progress in one transaction, so a mid-file crash leaves no items the record does not account for and resumes to ACTIVE rather than `target_not_empty`; only a crash before the first batch commits leaves an empty target, which resumes from the start. Backup kill points: between a data file upload and the manifest; between the three documents and the manifest; between the manifest put and the AVAILABLE update (the one path to the reconciler's AVAILABLE outcome). Restore kill points: after every file has verified and before the first load (`restore_after_download`; on an S3 store the spool holds every `.verified` marker, the manifest and the sidecar, and no item was written); after a file's transaction commits and before the next file starts (`restore_after_file_commit`); MongoDB after a batch's `insert_many` and its progress commit in one transaction (`restore_mongodb_after_insert_many`, the path a resume continues from the recorded `items_applied`); Cassandra between a batch's last write and its progress write (`restore_cassandra_before_progress`); SQLite mid-batch (`restore_sqlite_mid_batch`); PostgreSQL between the ACTIVE flip and the progress-row deletion (`restore_postgres_before_progress_delete`); MongoDB between the DONE write (with `restore_backfill_pending`) and the worker's ACTIVE flip (`restore_mongodb_before_worker_flip`), where the reconciler must leave the row alone and the restarted worker must finish; between the FAILED write and the target removal of step 7 (`restore_between_failed_and_removal`); a fenced-out restore job that resumes after a successor claimed the row (`restore_fenced_out`: its DONE write matches zero rows, it makes no table access and the successor's table survives with the exact count); PostgreSQL, a restore job whose catalog session ended (`pg_terminate_backend` on its claim connection) while its data writes continue: no successor loads into or deletes the target until `heartbeat_at` is stale, and the job aborts at its next heartbeat. DeleteBackup kill points: after the DELETING write; after the manifest delete; after `delete_prefix` and before the DELETED write. Shutdown: SIGTERM during a backup (FAILED, empty prefix, no multipart upload left) and during a restore (row left RUNNING, finished after the restart). - Distribution matrix for the filesystem default path: unprivileged local launch, the npm launcher, the container image with and without a volume, `sqlite-memory`. - Cross-backend matrix: nightly. - Live DynamoDB: Phase 2, measured. -- Performance: backup and restore of a 1M-item table per backend, and of a 1M-item table carrying a vector index on the backends that support one, duration and peak memory recorded as the published RTO baseline; memory must stay flat with table size (streaming, never materializing the table). Index contents are not stored in the backup; every index is rebuilt from the items on restore, so the vector cell measures the dominant restore cost. +- Performance: backup and restore of a 1M-item table per backend, and of a 1M-item table carrying a vector index on the backends that support one, duration and peak memory recorded as the published RTO baseline; memory must stay flat with table size (streaming, never materializing the table) in the restore's own terms: on SQLite a table with secondary indexes costs SQLite's page cache for the index b-trees above the no-index figure, an unordered backup costs its page cache across the base b-tree above the key-ordered figure, and the final `COUNT(*)` costs the counting connection's page cache, each bounded by `cache_size` (section 7, the record). The backup side's 1M figures are PR 8's record; the restore's are PR 9a's (`docs/design/15-backup-restore-bench-pr9.md`): 1M items of 200 bytes in key order in 9.5 s on PostgreSQL and 26.8 s on SQLite, the peak over idle 16 and 89 MiB, the growth from 100k to 1M 5.5 MiB on PostgreSQL and 51 MiB on SQLite (the write connection's page cache and the final `COUNT(*)`, each bounded by `cache_size`, not the items; from 1M to 3M rows the SQLite peak moved 11 MiB), and the same 1M items as an unordered backup 16 MiB on PostgreSQL and 144 to 164 MiB on SQLite in 66 to 90 s (SQLite's base b-tree under scattered inserts, not the restore's buffers). The MongoDB and Cassandra 1M restore and the vector cell are still to be measured (PR 11). Index contents are not stored in the backup; every index is rebuilt from the items on restore, so the vector cell measures the dominant restore cost. ## Review checklist for implementation PRs - No data file is uploaded outside a snapshot; the snapshot is opened before the first read and closed after the last. - `extenddb-manifest.json` is written last and only after every other object succeeded. - Every failure path deletes the prefix or leaves it unreachable from List, and the row never says AVAILABLE for an incomplete prefix. -- Restore writes no item before every data file's checksum has verified; a verification failure deletes the CREATING target table, which never accepted a read or a write; the request refuses before creating the table when the target cannot represent the manifest or the key ranges overlap. -- A file's data and its progress record commit in one transaction on PostgreSQL and SQLite (per 10k-item batch on SQLite, with the items-applied count), so no item is applied twice; on MongoDB the progress record follows the insert, re-application of the one unrecorded file is idempotent by duplicate-key suppression (code 11000 only; any other write error fails the restore), and the post-resume document count is checked against the manifest; on Cassandra the progress record follows each batch under a lightweight transaction and at most one batch is re-applied. A key-ordered restore runs the per-file key-range check on every file, and an unordered one ends with a count of the target equal to the manifest's item count. On PostgreSQL the ACTIVE flip precedes the deletion of progress rows; on MongoDB the job's DONE write and `restore_backfill_pending` are one transaction, and the worker's ACTIVE flip is conditioned on that flag and CREATING, not on the job's generation. -- Every status write by a job is conditional on its owner generation and on the row still being in the job's state (`CREATING` for a backup, `RUNNING` for a restore); a job whose conditional write matches no row, or whose claim connection errors, performs no further store or table operation, deletes (of a prefix or of a restore target) included; the reconciler takes the same claim as a worker and leaves `heartbeat_at` untouched. +- Restore writes no item before every data file's checksum has verified once; on a filesystem store the load's second read is hashed again and a file replaced between the passes fails the file, its items removed with the target (5.3 step 3); a verification failure deletes the CREATING target table, which never accepted a read or a write; the request refuses before creating the table when the target cannot represent the manifest or the key ranges overlap. +- A file's data and its progress record commit in one transaction on PostgreSQL (the `restore_progress` row, per file) and SQLite (per batch of at most 10,000 items or 32 MiB of item text, with the items-applied count), so no item is applied twice; on MongoDB the `file_index` record follows the insert, re-application of the one unrecorded file (`BulkLoadFile::reapplying`) is idempotent by duplicate-key suppression (code 11000 only, on that file only; any other write error fails the restore); on Cassandra the progress record follows each batch under a lightweight transaction and at most one batch is re-applied. A key-ordered restore runs the per-file key-range check on every file, and every restore ends with a count of the target equal to the manifest's item count. On PostgreSQL the ACTIVE flip precedes the deletion of progress rows; on MongoDB the job's DONE write and `restore_backfill_pending` are one transaction, and the worker's ACTIVE flip is conditioned on that flag and CREATING, not on the job's generation. +- Every status write by a job is conditional on its owner generation and on the row still being in the job's state (`CREATING` for a backup, `RUNNING` for a restore); a job whose conditional write matches no row, or whose conditional write's outcome stays unknown after the re-read, performs no further store or table operation, deletes (of a prefix or of a restore target) included; the reconciler takes the same claim as a worker and leaves `heartbeat_at` untouched on a backup row and refreshes it on a restore record (4(e)). - The ARN is parsed by `BackupArn` once, in the engine, before any catalog or store access; no other code splits ARN strings. - Account scoping is checked on the ARN before any store access, every backup row is read by account and backup id and accepted only when its account, table and ARN equal the request's, and every restore-target method takes the target's account, name and table id. - The backup job validates every item with `validate_restorable_item` before encoding it, keeps validating after the first refusal, and the restore binds the same function. -- A restore's registration re-checks, under the backup row's lock, that the row is AVAILABLE and not in use, and DeleteBackup's in-use check and DELETING write run in one transaction under the same lock. -- A failed restore's target is removed through `remove_restore_target`, never DeleteTable; on PostgreSQL and SQLite `bulk_load` writes index rows in the batch's transaction, with the base rows, and the ACTIVE flip sets every index of the target ACTIVE. +- A restore's registration re-checks, under the `backup_index` row's lock (5.3 step 4 names it per backend), that the row is AVAILABLE and not in use, and DeleteBackup's in-use check and DELETING write run in one transaction under the same lock. +- A failed restore's target is removed through `remove_restore_target`, never DeleteTable, and only after the FAILED write matched one record (`fail_restore`); on PostgreSQL and SQLite `bulk_load` writes the secondary- and vector-index rows in the batch's transaction, with the base rows, and the ACTIVE flip sets every index of the target ACTIVE. - No API operation or job touches the store while it is unhealthy, the reconciler acts on no row of the store then (its legacy duty, which touches no store, still runs), and only sync with a resolving option and the health check itself read it. - No credential, bucket name with credentials, or presigned URL appears in a log line, and no API error names a store root. -- New workers stop on the shutdown signal within the drain window: a backup runs its failure path, which the server's shutdown awaits, and a restore stops with its row left RUNNING. +- New workers stop on the shutdown signal within the drain window: a backup runs its failure path, which the server's shutdown awaits, and a restore stops with its row left RUNNING and its spool kept. - Tests create composite-key tables, not only hash-only tables. ## Drawbacks @@ -477,7 +530,7 @@ Phase 3, later (v1.x): - A restore from an S3 store spools the whole compressed backup to local disk before loading any of it. - One core at full utilization for the duration of a backup on compression; `max_concurrent_jobs` and `spawn_blocking` keep it off the request threads but do not make it free on a 2-vCPU host. - Two code paths for one release while legacy backups remain readable. -- A backup of a table holding an `N` value #618 let in fails on the off-host path, where main backs it up (5.1 step 4). +- A backup of a table holding an `N` value #618 let in, or an attribute name over the limit that `UpdateItem` and a `TransactWriteItems` `Update` stored before #433, fails on the off-host path, where main backs it up (5.1 step 4). ## Alternatives considered @@ -507,7 +560,7 @@ Changed after a review of the draft against main `9f43d03` and PRs #384, #335, # 3. DeleteBackup deletes the manifest first and tombstones the row as DELETING; FAILED is a tombstone too; sync never adopts a CREATING, DELETING, DELETED or FAILED prefix and only touches rows of the configured store (5.2, section 4). The draft's order could resurrect a half-deleted backup, and a fenced-out job's late manifest could have been adopted over a FAILED row. 4. Restore returns CREATING from the request (table and job row created atomically) and the job verifies every file before loading any; progress is recorded per file in a way a crash cannot split from the data (a data-database row on PostgreSQL, the same transaction on SQLite, idempotent re-application on MongoDB) (5.3). The draft's per-file resume re-applied a file after a crash between commit and checkpoint, which fails on PostgreSQL's plain INSERT and silently picks a winner under UPSERT; an intermediate revision made the request synchronous for the whole download, which contradicted the asynchronous goal. 5. The manifest's self-checksum is replaced by a sidecar over stored bytes (section 1). -6. Per-job claims with fencing replace the assumed single-instance lease, which does not exist on PostgreSQL or MongoDB; no row ever exists unowned (the PostgreSQL lock precedes the row, the MongoDB row is born with its generation), every catalog write is conditional on the generation on every backend, PostgreSQL heartbeats go through the lock connection, and a cancelled job fails rather than waiting an hour in CREATING (5.1, Non-goals). +6. Per-job claims with fencing replace the assumed single-instance lease, which does not exist on PostgreSQL or MongoDB; no CREATING row exists unowned (the PostgreSQL lock precedes the row, the MongoDB row is born with its generation; sync's AVAILABLE rows are unowned, revision item 34), every catalog write is conditional on the generation on every backend, PostgreSQL heartbeats go through the lock connection, and a cancelled job fails rather than waiting an hour in CREATING (5.1, Non-goals). 7. Index population on restore stays per backend (`bulk_load`); the draft's shared backfill worker exists only on MongoDB (5.3 step 6). 8. Failed creations are BackupNotFound, not DELETED (section 4). 9. `item_order`, `key_range`, `md5`, `uncompressed_bytes`, explicit nulls, three-state `vector_indexes`, optional `etag`, `dataFileS3Key` as a full key, partition-aware `BackupArn` (section 1, 4). @@ -527,3 +580,16 @@ Changed after a review of the draft against main `9f43d03` and PRs #384, #335, # 23. (2026-10-29, after the eighth review round, a verification of item 22's text against the companions' code and main.) Main's two PostgreSQL race tests exercise the legacy path and stay there; PR 6 adds the same two orderings against the engine's DeleteBackup and RestoreTableFromBackup over an off-host backup, gated on the `backup_index` row lock, which corrects item 22's "run unchanged" (5.2, section 10, PR 6, PR 9a). Cassandra's restore methods take the full `tables` key because a secondary-index read scatters and cannot carry a lightweight transaction, not because of `ALLOW FILTERING` (section 3). #384's Cassandra `remove_restore_target` is conditional on the table id only, and never running it without `abandon_restore_target`'s conditional step is this design's rule for the trait method (section 3). The restore's ACTIVE flip sets the table and every index in one catalog transaction on PostgreSQL and SQLite; on Cassandra the index rows are created ACTIVE and the lightweight transaction on the `tables` row flips the table alone, and Cassandra has no vector indexes (5.3 step 6). The `target_not_empty` exception is stated once, in section 4's duty list, which names the duty; the claim-held list, the maintenance-claim paragraph and 5.3 step 7 refer to it; a FAILED target keeps `--forget-backup` refusing, not waiting; and the operator path for a `target_not_empty` target is `extenddb backup remove-target
    `, a PR 10 command under the maintenance claim that is audit-logged (section 4, 5.3, 8, PR 10). `sqlite-memory`'s startup sync takes ownership of a root whose file names a `sqlite-memory` catalog as `--adopt-store` does, so one catalog owns the root at a time there too (section 4). Cassandra's and SQLite's legacy creates both commit a CREATING row before the copy, Cassandra's being the one that needs heartbeating; a DeleteBackup that wins against a registration leaves no target rather than creating none; `backup_index_by_status` is partitioned `((store_root, backup_status), backup_arn)`; PR 14's Cassandra step reads the three tables and deletes legacy rows by key, since CQL has no `IS NULL` predicate; section 3's PR 14 sentence names the `--force-drop-legacy-backups` opt-out; the functions `BackupDocuments::build` calls are private to `backup_format`; `bulk_load` writes index rows in the batch's transaction with the base rows wherever the design restates it; the unhealthy-store rule exempts the reconciler's legacy duty, sync with a resolving option and the health check, and PR 7's acceptance names DeleteBackup of an off-host backup among the refused operations; section 8's virtual-hosting example names the two cases the SDK's rules give (a virtual-hostable name on `localhost`, an access point ARN on any custom endpoint); the sentence join in section 3's PostgreSQL bullet, "sees" in 5.3 step 6 and the trait comment on `remove_restore_target`'s first use are corrected; the status line and item 22 carry the commit's date (section 1, 3, 4, 5.2, 5.3, 7, 8, PR 4, PR 7, PR 9a, PR 14, checklist). The companion facts of this round, as their round-8 fixes state them: `--x-s3` and `--xa-s3` names refused, the credential chain's clients given the proxy-less HTTP client when any credential endpoint setting, the IMDS endpoint setting or the container credentials URI is `http://`, so such a credential request never reaches the proxy (without such a setting the chain uses the SDK's default client and `NO_PROXY` must list `169.254.169.254` and `169.254.170.2`), the settings read from a load that sends nothing, the IMDS endpoint setting checked with `http://` allowed only to loopback, `169.254.169.254` and `[fd00:ec2::254]`, `open` without `region` querying IMDS, `SSL_CERT_FILE` and `SSL_CERT_DIR` replacing the system trust store, and SDK-sourced endpoints checked after the shared config is loaded (#393; section 2, 8, PR 3); the store key prefix trimmed of leading and trailing `/` by `BackupDocuments::build` in agreement with the store, `KeyRangeTracker::next_file` carrying ascent across files so an out-of-order boundary fails at the next file's first item, `build` taking each file paired with its ETag, and the sidecar check taking bytes (#394; section 1, 5.1, PR 4); on the separate layout a data commit that fails after the catalog commit retried on a fresh connection for a bounded time before the request returns, the orphan window being a process death or a data database down past that bound, with the troubleshooting page naming how to find and drop such relations (#395; section 3, PR 5). 24. (2026-10-29.) The series is condensed from twelve remaining PRs to ten: the reconciler (formerly PR 8b) ships with the CreateBackup job as PR 8, since a job without its reconciler leaves rows no one resolves, and the CI changes and the documentation (formerly PR 11 and 12) ship as PR 11, since both finalize after PR 9b. Body references to PR 8a, 8b and 12 are renamed; the revision log keeps the old labels as written (section 3, 4, 7, 10, Phasing). 25. (2026-10-29, the split.) The document is cut from 279,584 bytes to 187,722 bytes: sections 1, 2 and 3 now point at the companions' documentation and keep only the rules later PRs build on (the format: `crates/core/src/backup_format` module docs and `crates/core/testdata/extenddb-manifest/README.md`, #394; the store: `crates/backup-store/README.md` and its `config.rs`, `key.rs` and `lib.rs` docs, #393; the snapshot: `crates/storage/src/snapshot.rs`, the four `snapshot_engine.rs` module docs and `docs/troubleshooting.md`, #395); the companions gain the facts only this document stated (the filesystem `list`'s whole-root `Vec` and the empty-prefix IAM policy in #393's README and `lib.rs`; the pre-#361 sort cost and the reason the fetch size is a constant in #395's `snapshot_engine.rs` and `snapshot.rs`, whose module docs already carried the half-pool measurement; a direction sentence in each companion's module doc saying that this overview states the rules the code's callers honour and the crate docs are the record of what the code does); the dependency-change facts of section 2 (lock entries, licences, pins) and the vacuum and WAL growth figures are dropped, the first being the PR description's record and the second the backup manual's. Sections 4 and 5 keep the invariants, the state machines and the per-operation contracts, by lettered part and numbered step; the schema, the per-backend mechanics, the reconciler's duty list, the sync procedure, the claim per backend, the page query and the memory arithmetic moved into a section per later PR under Phasing (PR 1, 6, 7, 8, 9a, 9b, 10, 11, 13, 14), each with its goal, the invariants it honours by reference, the interfaces it adds with their row keys, the transitions it owns and its acceptance list kept whole; section 10 is a table of which document and row each PR changes, the rows themselves in the owning PR's section; the PR 3, 4 and 5 lines go, their PR descriptions and docs being the record of their deliverables. Heading numbers 1 to 10 and the step numbers of 5.1 and 5.3 are unchanged, so items 1 to 24 still resolve; the three `version-literal-ok` markers stay in section 4(i), PR 1's and PR 6's sections. The rewordings the split needed, each keeping its sentence's meaning, are listed in the split's report (`.review/reviews/r8/DONE-SPLIT.md`). +26. (2026-10-30, PR 6 as implemented on #666's registry.) PR 6's migration is id 5 on PostgreSQL and SQLite (one `MigrationId` list for both, ids 1 to 4 taken), and the Cassandra file is V006 and the MongoDB step `002_backup_catalog_index`, both writing `0.0.5` so that one number names one change on every backend; the literals in 4(i) and PR 6's section are updated, PR 1's line keeps its own numbers and renumbers if it lands after this. +27. (2026-10-30, PR 6.) The FAILED-target duty lands in PR 6: `list_dead_restore_jobs` also returns a FAILED `restore_jobs` row whose target is still CREATING once the row is stale, and the reconciler removes the target without a second FAILED write, which closes the state the job's or the reconciler's failure path leaves when the FAILED write lands and the removal does not (4(e), PR 6, PR 8). PostgreSQL's dead rule is the heartbeat alone, as on MongoDB and Cassandra: the advisory lock is the claim's exclusion, not the liveness test, and the `IS NULL` arm item 21 named does not exist, the column being NOT NULL (4(e), 5.3 step 7, PR 6). The stale floor is computed from the period the jobs run at, `max(heartbeat_interval_secs, JOB_HEARTBEAT_INTERVAL)`, the one constant every backend's heartbeat loop reads until the knob reaches the jobs, and `max_concurrent_jobs` has an upper bound (5.1 step 4, section 7, PR 6). SQLite's `backup_server_slot_count` is kept for snapshot opens until PR 8, the legacy create trying it without waiting and answering the one `LimitExceededException` text (PR 6). The engine's `BackupArnSpellings` carries an optional parsed ARN per spelling, so a legacy backup of a name the store-prefix rules refuse (dot-only, over 255 bytes) still describes, restores and deletes through the legacy reader (4(d), PR 6). DeleteBackup over a row runs the in-use check before the other-store check, as a restore does, builds the description before the DELETING write, and a row whose source table the build cannot read is left out of ListBackups rather than failing the listing (5.2, PR 6). The acceptance cell running the two orderings over a `backup_index` row is deferred to PR 9a, with the row lock observed by a two-transaction test of `start_backup_delete` in PR 6 (PR 6). On Cassandra an index row's list rows exist for as long as the row does, whatever its status, readers re-reading the authoritative row; the legacy create keeps its publish-time rule (PR 6). The reconciler records the worker metrics of section 9 (PR 6). On Cassandra a migration 6 that fails in its backfill leaves the migration unrecorded, so `serve` keeps refusing the catalog and `migrate --yes` runs the body again; a Cassandra job stops after three heartbeat errors in a row, as after a heartbeat that does not apply, and the fence the reconciler puts on a dead create's row takes a fresh `owner_generation` (4(e), PR 6). MongoDB's RestoreTableFromBackup waits a bounded time for the registration and answers `InternalServerError` past it while the registration goes on; what the request leaves is stated in 5.3 and the differences document (5.3, PR 6). CreateBackup checks `server.region` before forming a backup ARN and refuses, naming the setting, under a region outside `[a-z0-9-]`, where main created a backup no operation could then address; the store-prefix rules on the name are applied before the ARN is formed so a name over 255 bytes is refused naming the limit whatever its length (5.1 step 1, PR 6). The `LimitExceededException` text names the bound that was met beside its number (`max_concurrent_jobs`, SQLite's snapshot slots, or PostgreSQL's connection slots for the claim connection), one implementation (`limit_exceeded_message`) every refusal binds (5.1 step 1, section 7, PR 6). Section 7 states, where it bounds snapshots by half the pool, that the pool is the raised one (section 7, PR 6). Item 21's `IS NULL` arm is withdrawn by this item alone; item 21 keeps its wording, as item 22 requires. On Cassandra, where the removal is a claim and then a drop, the FAILED-target duty also lists a FAILED record whose target is DELETING with its registration, and `remove_restore_target` resumes the removal from the drop (4(e), PR 6). On MongoDB the in-use check of a restore's registration and of DeleteBackup reads at the transaction read concern, majority-committed data, while DescribeTable reads at `local`, so a request issued within the majority-commit lag after DescribeTable first shows the target ACTIVE can still answer `BackupInUseException` and succeeds when retried; the differences document's concurrent-restores row states it with the measured lag (5.3, PR 6). The rule that a job performs no further operation after a write it cannot confirm is scoped to the conditional write itself: a legacy restore whose claim connection (the heartbeat's) errors runs 5.3 step 7, the conditional FAILED write on a pooled connection and the removal only when that matched, so on PostgreSQL the target is removed at the next heartbeat rather than by the reconciler once the record is stale (4(e), 5.3 step 7, PR 6). On MongoDB `remove_restore_target` deletes the target's catalog document first and returns the target removed once that is gone: a collection drop that fails after it is logged naming the collections and leaves them for `backup sync` (PR 10), there being no record for a later reconciler cycle to retry from (4(e), PR 6). PR 6's interface block lists `list_dead_restore_jobs`, `fail_restore_job` and `LegacyBackupReader::abandoned_creating_backups` with their contracts, as the trait ships them (PR 6). On MongoDB `serve` registers and reads the catalog's status (`register_and_check`, #681) before it builds the data engine or starts the startup key repair of documents stored without their key (#473), so a server that refuses the catalog has written nothing, as on PostgreSQL (PR 6). After #335 and #681 merged, PR 6 is rebased onto main: PR 1 merged as #335 with migration 5, `drop_continuous_backups`, on every backend, so PR 6's migration is id 6 and PR 9a's `restore_progress` id 7; Cassandra and MongoDB migrate through the numbered runner, where PR 6's change is a code body (Cassandra's executing `0006_backup_catalog_index.cql` before the backfill and the catalog id), and no backend keeps a catalog version string, so the version-last rule of Cassandra's former runner, MongoDB's versioned step and their shared version check are withdrawn, the runner's status check being `serve`'s refusal of a catalog without migration 6 on every backend; item 26's numbers are superseded by this sentence, and item 26 keeps its wording, as item 22 requires (4(i), PR 1, PR 6, PR 9a). +28. (2026-11-04, PR 6 after the eighth review round, measured against DynamoDB.) The engine checks a restore's target name after the backup resolves and before the overrides are checked against it, the service's order, so a restore of a missing backup is `BackupNotFoundException` whatever the target name's state (5.3 step 1). `check_backup_arn_fits` keeps the region rule only; the store-prefix rules on the name are the store-backed writer's (PR 8), and the in-catalog create backs up a dot-only or over-255-byte name as main did, as the service backs up any table it can create (5.1 step 1). A table ARN in CreateBackup's or ListBackups's `TableName` whose region is not `server.region` is `ValidationException` "1 validation error detected: Invalid AWS region in ''", the service's text. The job permit is waited for up to 30 s (`JobPermit::acquire_within`, `PERMIT_WAIT`) rather than refused at once, the request holding nothing meanwhile (5.1 step 1, 5.3 step 1, section 7). `fail_restore_job` takes the listed `owner_generation` and `heartbeat_at` and writes FAILED with a fresh generation, so the reconciler's write cannot land over a heartbeat that followed its listing (4(e), the PR 6 interface block). The SQLite restore claim holds a `JobSlots` slot for the copy's life, and `list_dead_restore_jobs` skips a RUNNING record whose slot is held, the SQLite rule of 4(e): the heartbeat queues behind the write lock and is not that process's liveness test. SQLite's `abandoned_creating_backups` removes the dead CREATING rows itself, in one write under the engine write lock limited to rows whose slot no task holds (`AbandonedLegacyBackup::removed`), so a create that commits AVAILABLE between a read and the removal keeps its backup. The Cassandra restore heartbeat requires `restore_status = 'RUNNING'`, so a FAILED write stops the copy as on the other backends. `BackupInUseException` for a restore of an in-use backup carries the service's text, `Backup is being used to restore another table: `, the target's name going to the log. `SourceTableFeatureDetails` is always present, `{}` when the source recorded no feature. The MongoDB registration stamps its first heartbeat with `$$NOW` inside the registration's transaction. A restore job record this build cannot read is logged and skipped by `list_dead_restore_jobs` on every backend, and the reconciler's once-per-record log set lives in `serve`, outside the workers it restarts (4(e), PR 6). +29. (2026-10-30, PR 7 as implemented.) A store problem never gates the in-catalog create: with a configured store that is unhealthy, CreateBackup writes the in-catalog format as it does with no store, and only the store-backed create of PR 8 is refused by `require_healthy_store`; a defaulted SQLite root whose identity check fails (a catalog re-created by `destroy` and `init` under the old root's `extenddb-store.json`) is the same warning as a failed validation, leaving the store unconfigured, and `destroy` and the admin guide say the file must go with the catalog (section 7, PR 7). An S3 store's `validate` and identity check run after the listener is up, in the reconciler's first cycle, with the store `checking` until the first result reports; a filesystem store's run before the listener; the configuration errors that need no request still stop startup; a `validate` refusal of an S3 bucket after the listener makes the store unhealthy (`failed validation`) rather than stopping startup (section 7, 9, PR 7). `/health`'s `reason` is the problem's category alone; the ids and ARNs go to the operator's log line (section 9, PR 7). The identity file is read up to 64 KiB. The S3 root label is built from the endpoint the client uses, in one spelling. A check that cannot read the catalog leaves the store's health unchanged. An existing spool must be owner-only. The maintenance claim is heartbeated on a timer from the take to the release, and PostgreSQL's `check_maintenance_claim` tests the advisory lock before the heartbeat's age (4(f), PR 7). The jobs' side of the claim is implemented in the engine for the in-catalog create and restore, before the permit and, since those paths commit their row inside the backend, before the row exists; the ordering 5.1 step 2 states applies to the store-backed job PR 8 adds (5.1, 5.3, PR 7). `backup list` opens the SQLite catalog without the serve lock, reads the identity file without creating it and runs on an unhealthy store; on SQLite `backup purge` runs the reconciler's duties with a zero window before its wait; a CREATING row with a dead owner is purged as any row, through `remove_dead_creating_backup_row`, a `BackupCatalog` method of PR 7; a FAILED row with a young heartbeat is waited on; the dry run exits 0 (PR 7, PR 10). Cassandra's claim reads run at `LOCAL_SERIAL` and an insert whose answer was lost is settled by a re-read of the generation (PR 7). The PR 10 comment on `backup_index_by_status` is corrected: migration 6's entry carries `store_kind` and `store_id`. The body sentences that paraphrased the earlier rules are edited with this item (section 7's configuration-error paragraph, 4(f) and 5.1 step 2 and 5.3 step 4's claim check, PR 6's `store_root` row, PR 7's interfaces and acceptance), and the filesystem store's startup checks are bounded by a timeout that leaves the store `checking` when a mount stops answering after the root is opened (a root that does not answer at startup holds the server before the listener, as the database directory would). Also: the PostgreSQL check takes its shared lock at transaction scope so a cancelled request leaves no lock on the pool; a dropped `HeldClaim` stops its timer; the purge's dry run is a read like `list`; a CREATING row is removed before its prefix; an empty `filesystem.path` is left for `open` to refuse; the spool meets the root's whole rule, parents included; `backup list` reads the catalog's status without the serve lock. The defaulted SQLite root's warning holds only while the catalog has no backup rows for the root (`read_store_ids(Filesystem, root)`: the upgrade, fresh and re-init cases); with rows for it the root has been used and is handled as an explicitly configured one, so its backups never vanish behind a warning: a failure of its identity check or manifest probe is an unhealthy store (reported, its rows' restores and deletes refused, rechecked each cycle), and a refusal of its `open` or `validate` a startup error naming the root (section 7, PR 7's acceptance). The server's `configure` carries the reason for an absent store out (`NotConfigured`), and `backup list` and `backup purge` print it, so a defaulted root that failed its check is no longer reported as "[backup.store] is absent or disabled". The store's check runs under a per-kind bound in every reconciler cycle (`check_configured_store`: 5 s for a filesystem root, the startup bound; 30 s for a bucket, under the SDK's 60 s per-attempt timeout so a timed-out check is dropped before the SDK retries its request), recording a transport failure naming the bound, and the cycle selects the check against the shutdown token; an unbounded check let a stalled S3 endpoint hold each cycle for the SDK's whole retry budget and a stalled mount hold it for good, with shutdown hitting the drain timeout (section 7). A filesystem root's `validate` reports no transport class; an I/O failure of its probe write or of resolving the root at startup degrades the store (`failed validation`) as the same condition one cycle later does, while the owner-only and existence refusals stay configuration errors (section 7). A transport failure's operator line carries the store error's whole cause chain (the SDK's top link is a bare class). Every store error of a check that is neither a transport failure nor a denial is `failed validation`, the category `validate` gives it, so a bucket deleted under a running server reads as a bucket missing at startup. A filesystem root that is or contains the database directory is refused in `resolve_store`, beside the spool rule (section 7). A filesystem store's synchronous root checks run on the blocking pool so the startup bound fires whichever call stalls. The checking store's refusal says its first check is pending; `/health`'s reason list carries `catalog unreadable`; `record_check` reports whether the category changed. Purge's wait set is CREATING and FAILED rows with a live heartbeat and RUNNING restores, DELETING rows being finished rather than waited on, and its refusal rule is that set at the wait's end; 4(f), the PR 7 "Destroy and purge" paragraph and the PR 10 claim paragraph (`MAINTENANCE_LOCK_KEY`, `pg_try_advisory_xact_lock_shared` then the heartbeat age, `LOCAL_SERIAL`) and the PR 6 and PR 7 signatures (`check_maintenance_claim(stale_after)`, `maintenance_claim(holder, stale_after) -> Box`) are edited to the code with this item. A row a DeleteBackup moves to DELETING or DELETED, or removes, between purge's read and its turn is finished and counted as deleted, not left (the deletion is one idempotent sequence on both sides). On SQLite purge waits and removes CREATING rows with a zero stale window (under the serve lock every owner is dead, so a young heartbeat proves nothing), and the server's startup under the serve lock removes a maintenance claim a stopped command left, so a killed purge costs no stale window; on every backend purge releases the claim on `SIGINT` or `SIGTERM`, at once during the wait and before the next row during the deletion. The wait's three failures (jobs still running at the limit, the claim lost, the catalog unreadable) and the signal are reported each as itself. PostgreSQL's factory starts the GSI queue workers only for a server, not for a command reading beside one. Purge's restore part of the wait set is the RUNNING `restore_jobs` records alone: the dead-job listing it reads with a zero window also returns the FAILED records whose target is still CREATING (the FAILED-target duty's, 4(e)), which are the reconciler's to finish and no job runs under, so they are not waited on; the wait finishes at once on a stopped deployment that holds one. On SQLite the zero-window pass resolves the kinds this release writes; a RUNNING `restore_jobs` record of kind `manifest`, which PR 9a introduces, is left for the reconciler that resumes it and purge waits on it and refuses naming the table, the one state the PR 10 paragraph's "never makes it wait or refuse" does not cover. The reconciler finishes the configured store's DELETING rows while the maintenance claim is held, as 4(f) lists. The binary ends its tokio runtime with a bounded shutdown (5 s) after `serve` and after every command: a filesystem store's check abandoned at its bound leaves its call parked on a blocking thread until the mount answers, and a plain drop of the runtime joined that thread, so a server whose root had stopped answering drained its workers at SIGTERM and then stayed alive until SIGKILL. Each call of a store check runs as a task of its own that an abandoned check does not take with it, and the store records that a call is in flight; no check starts while one is (the store is recorded `unreachable` again, with the reason), so a mount that stays stalled parks one thread rather than one per cycle. A command's identity read (`backup list`, the dry run) runs under the store's check bound as the server's check does, where the unbounded read waited out the SDK's three attempts of 60 s on an endpoint that accepts and never answers. An empty `filesystem.path` is compared with nothing in `resolve_store` (compared as written it was the working directory, refused by name as the database directory when the server starts there) and left for `open` to refuse. A root that refuses `link(2)` with `EPERM` is reported `failed validation` (the README and the deployment guide said `unreachable`). `tests/test_backup_store_config.py` is excluded from `devtools/run-tests`'s pytest run, as its docstring says; a catalog whose rows cannot be read keeps a defaulted root configured, and that arm is tested. On SQLite the purge's own reconciler cycle is told that its caller holds the maintenance claim (`ReconcilerState::for_maintenance_holder`) and takes the held branch without reading the claim; read through the zero dead-owner window, the command's own claim was seen held only when its heartbeat and the check fell in one millisecond, so the branch was decided by scheduler timing (PR 7, PR 10). Purge's test of a live owner is the backend's `list_backup_rows_older_than` on the heartbeat clock, a `BackupCatalog` method of PR 7 that PR 8's reconciler also lists by, where it compared `heartbeat_at` with this host's clock, which a host running ahead of the database read as a shorter wait, not a longer one (4(e), 4(f), PR 7). On SQLite a command under the serve lock takes over a maintenance claim row whatever its heartbeat says, and PostgreSQL's take tries the exclusive lock again over about 10 ms, so a job's one-statement shared lock does not refuse it (PR 10's claim paragraph). On MongoDB a command that opens the catalog beside a server (`backup list`, the dry run) starts no key repair of documents stored without their key (#473); the server does. On PostgreSQL every `backup` command registers as a server does and its registration's session carries the command's name as `application_name`, so a migration that waits for it names it, and a catalog the factory refuses ends the command with the message and exit code `serve` gives (3 behind, 6 while a migration runs). The jobs' refusal while the claim is held names a backup maintenance command, not `purge`, since every maintenance command holds the one claim. The console banner shows a filesystem root in the canonical spelling the rows and the log line use, with the configured spelling only where it differs (PR 7). After #335 and #681 merged, PR 7 is rebased onto PR 6's rebased head: MongoDB's and Cassandra's `serve` register through the numbered runner (#681), so the backup commands register there too, as on PostgreSQL, named after the command (MongoDB's `extenddb_servers` document carries a `command` field; Cassandra's `extenddb_servers` row, which has no column for it, carries the name in `release` after the release and ` as `, a column being a catalog migration), refuse with the same exit codes, and release the registration when they exit, where a registration left to the runtime hooks the command drops would be deleted only by a task the command's exit may never run, and `migrate --yes` would then wait out the row's or document's 45 s; MongoDB's factory starts the key repair in its `build_components`, after `register_and_check`, only for a server (`start_key_repair`); the reconciler, which takes the backup runtime, runs in the worker set the registration supervisor stops and starts (#666) (PR 7). + +30. (2026-11-04, PR 7, second review round.) DeleteBackup of a store-backed row answers the DELETED description once the DELETING write has committed and leaves a failed object deletion to the reconciler, where it answered InternalServerError (5.2). The store's bound covers the store's calls alone, each through one adapter (`ConfiguredStore::calls`): the catalog's reads run beside them, so a slow catalog costs the check time and never marks the store unhealthy, and the deletions DeleteBackup and the reconciler run go through the same adapter, under the same bound and the in-flight rule (section 7; the in-flight mark now means a call abandoned at its bound has not returned). After the second round's review: a filesystem root's calls run one at a time (`ConfiguredStore::call_slot`), a call that finds another running waiting for it under the bound and then having the bound for its own call (third round: one bound over the wait and the call abandoned a 3 s call behind a 3 s call on a healthy root), so two DeleteBackup requests on a stalled mount park one thread between them where each parked its own (the in-flight mark alone was set only once the first call reached its bound); the mark is set under the call task's own lock, only while the task still runs, so a task that ended between the bound's last poll and the mark leaves no mark with nothing to clear it. A store whose startup `validate` failed is built unvalidated, so the next check runs `validate` again rather than reporting the store healthy on a re-read of the identity file. A defaulted SQLite root whose startup checks did not finish keeps its rule (`ConfiguredStore::defaulted`): the reconciler applies it to the first check's outcome and sets a failing root with no rows aside (`set_aside`), as the startup check would have (section 7). The manifest probe reads the newest AVAILABLE row through `BackupCatalog::newest_available_backup_row` rather than listing every one. Purge's wait reads RUNNING restores by status (`BackupCatalog::list_running_restore_jobs`), with no age, so a record stamped in the listing's instant or ahead of the clock holds the wait (4(f)); its deletion loop lives in the engine (`maintenance::purge_rows`) and stops before its next row at a lost claim or a signal; `HeldClaim::release` is bounded (10 s). On SQLite a zero window selects the dead-owner listings and removals by status alone, so a heartbeat stamped ahead of the clock is not a live owner under the serve lock (PR 10's claim paragraph). The backup commands run none of the startup recovery a server runs (`ServerComponentsOptions::startup_recovery`), `purge --confirm` included, which still takes SQLite's serve lock; the signal listener is up before the open in every subcommand, so a signal during the open ends the command with its registration released. PostgreSQL's `release` unlocks the advisory lock before closing the session, so a check right after the release reads the claim free; Cassandra's takeover re-reads on a lost answer as the insert does; MongoDB's claim upsert that matched a document with no `owner_generation` is taken. `tests/test_backup_store_config.py` runs in CI: on the SQLite job with moto, and in the PostgreSQL CLI lifecycle list. Two rows join `docs/differences-from-dynamodb.md`: the maintenance claim's LimitExceededException and the unhealthy store's ValidationException. + +31. (2026-10-31, PR 8 as implemented.) The job and the reconciler live in the engine's backup module (`crates/engine/src/backup/{job,reconciler}.rs`), not in `crates/server/src/workers.rs`, and nothing crosses an in-process channel between the request and the job but the response (the pipeline's item and chunk channels are the job's own): the request's detached task opens the snapshot, claims the row, checks the maintenance claim and answers through a one-shot channel, then continues as the job; the server hands the engine the shutdown token, the metrics collector, the job settings and the manifest's bucket and key prefix through `BackupRuntime` and `ConfiguredStore`, and `drain_jobs` (30 s) runs before the workers drain (PR 8); a stopped pipeline's sink answers the encoder's write with `BrokenPipe`, which nothing retries, where an `Interrupted` answer (std's "try again") kept a job cancelled in the middle of a 300k-row table spinning in the compressor's trailer write through the whole drain with its row left CREATING; the job ends within milliseconds of the signal on every store (5.1 step 5, PR 8). The reconciler's takeover is `BackupCatalog::claim_dead_backup(key, stale_after)`, not `claim_backup`, which inserts; `list_backup_rows_older_than(status, store, clock, age)` with `StaleClock::{Heartbeat, Completed}` carries the database's clock as a backend predicate for the FAILED-prefix delay and the tombstone retention, so the engine never holds a clock; `sweep_restore_registrations` is the Cassandra `active_restores` sweep with a default empty body; `StatusFields` gains `source_table_details_json`, which the job's and the reconciler's AVAILABLE writes fill with the manifest's `source_table` (4(e), PR 8). The PostgreSQL page is three layers, since a window function runs before `LIMIT`, and the stream ends when the inner query returned fewer rows than asked and none was cut (a page cut short of the row bound is not the end); the outer `ORDER BY` is a stated guarantee the executor gives anyway; SQLite's stream pulls no row past its cut (the driver's worker reads at most five rows ahead of it: the four `row_buffer_size` on the pool buffers and the one it holds while its `send` waits), so the rows-past-cut figure is PostgreSQL's alone (section 3, PR 8). `heartbeat_interval_secs` reaches the store-backed job as `max(heartbeat_interval_secs, JOB_HEARTBEAT_INTERVAL)`; the in-catalog paths keep the constant until PR 14, which corrects item 27's "until the knob reaches the jobs" (5.1 step 4, section 7). The job's pre-put check is two periods, not one (a beat sent at the tick returns up to a period later), which stays inside the three-period floor; the check measures from the instant taken before the claim's insert, not from the heartbeat handle's start, so an unbounded maintenance check (a catalog query held behind the pool) cannot admit a first put on a row already past `stale_job_after`; when the pipeline returns no put is in flight, so the failure path's `delete_prefix` follows the FAILED write with nothing to await; a lost or stalled heartbeat ends the job with no catalog write, the claim connection being the one that failed; each unknown outcome of a status write is followed by a re-read; the write is sent once more at most; after the second write a re-read that shows the row in the target status counts as matched and anything else as matching no row; within one cycle the duties run dead-owner first and FAILED-prefix after it, so a row the cycle just failed loses its prefix in that cycle once the dead owner's beat is stale (5.1 steps 4 to 6, 4(e)). SQLite's snapshot slot is kept as the backend's admission bound and the pool raised to `2 * (max_concurrent_jobs + 1)`, at most 34, which is what item 27's "folds it into `max_concurrent_jobs`" became (section 7, PR 6). On SQLite `claim_dead_backup` reads no window: a row whose slot no task holds is dead at once, and the engine calls it on every CREATING row each cycle (one set probe per live row); the dead owner's handle cannot observe `NotMatched` after a takeover on SQLite or PostgreSQL, whose slot and lock are the owner's life, and the engine reaches a taken-over row through `Unknown` and the re-read (4(e)). On MongoDB and Cassandra a successor is a live owner to a later taker only once it heartbeats, and the reconciler writes its one status before any beat, so two instances' reconcilers can both take over one dead row in that gap; the generation fence makes the loser's write `NotMatched` and one resolution lands; Cassandra's takeover condition carries the heartbeat it read (`IF owner_generation = AND backup_status = 'CREATING' AND heartbeat_at = `), which puts the liveness test inside the write; Cassandra's `list_backup_rows_older_than` reads the coordinator clock before its scan where `list_dead_restore_jobs` reads it after, both conservative; MongoDB's `Completed` listing requires the column present, since `age_is_stale` alone reads an absent column as the epoch (4(e), PR 8). MongoDB's stream applies no cut of its own (the server's 16 MiB reply bound is under the budget) and feeds no page-bytes counter, so `run_page_bounds` runs on PostgreSQL and SQLite only (section 3, PR 8); Cassandra's registrations sweep is one range read of `backups_by_arn` per cycle, enough while a catalog holds thousands of backups and not more, and the `backup_size_bytes` of `backups_by_account` and `backups_by_table` is written at the claim and not refreshed by `set_status`, which the index listings do not read (they re-read the row), as before PR 8 (PR 6). The spool sweep and the `restore_progress` sweep land with PR 9a, which adds the records they read, and the duty that removed a CREATING restore target with no job row is withdrawn: the job record is the registration on every backend (section 4, PR 8). The vector "round trip" PR 8 adds is its create half, the restore half being PR 9a's; the vector wire test asserts the store's ValidationException on the restore until then, and the fidelity suite's 24 restore cells skip on a server with a store (PR 8, PR 9a). `source_table.time_to_live` is read beside the open, outside the snapshot, and may trail it by the open's duration; DescribeBackup's `ItemCount` is the stats pass's figure while CREATING and the stream's count once AVAILABLE (5.1 step 2). Section 9 lists the metrics as `/metrics` emits them, the `MetricName` variants PR 8 adds (`BackupJobs`, `BackupJobDuration` in microseconds, `BackupBytesUploaded`, `BackupSnapshotAge`, `BackupStoreErrors`, the outcome and the store call in the operation dimension) as JSON under those names, and says the snake-case spellings are a converter's names and not a rendering the server does (section 9). `EXTENDDB_TEST_BACKUP_STORE` is set by every CI job that runs a suite creating backups, the pytest jobs and the Rust integration jobs (`filesystem` on PostgreSQL, which appends a store after `init`, and on SQLite, the defaulted root; `s3` on the SQLite job against moto; `none` on MongoDB and Cassandra and on the PostgreSQL Rust integration jobs; `run-integration-dev-mode` runs `tests/test_dev_mode_authorization.py` alone, which creates no backup, without it) and `devtools/run-tests --pytest` refuses without it (section 10, PR 8). PostgreSQL's snapshot-slot refusal now goes through `limit_exceeded_message` naming `[storage.postgres] pool_size snapshot-slot` (item 26's one-implementation rule). The acceptance's "on each backend" for the 100k backup reads "on each backend with a `SnapshotEngine`", Cassandra keeping the legacy create, and for the #618 cells "on PostgreSQL and SQLite", the two deployments `tests/test_backup_job.py`'s fixture starts (the MongoDB deployment and its CI job came with the review, revision item 32); an `abc` range key is refused by the number column on PostgreSQL and SQLite, so the range-key cell carries `1E126`, and UpdateItem validates the values its expression sets, so its cell carries the bad value in the key alone (PR 8). The 1M-item RSS runs and the kill points were measured on a 16-core, 30 GiB host (PostgreSQL 16.15 and MongoDB 7.0 in containers on loopback, the bundled SQLite 3.46) and the figures recorded with the PR in `docs/design/15-backup-restore-bench-pr8.md`; the memory bound holds (the 1M run's peak within 10 MiB of the 100k run's on PostgreSQL and SQLite, and the 500k run's within 5 MiB on MongoDB, where the 500k load alone took 27 minutes through the wire on that host; the bound is 112 MiB). Section 7's memory paragraph is rewritten to the terms the worst-expansion run measured: the earlier "the budget plus one row of text, plus its decoded form" left out the page's decoded rows waiting for the pipeline, the items in the feeder, the item channel and the encoder, one row's parse through the deserializer's intermediate value, and the line buffer, and it said nothing of the allocator; with 27.7 MB rows (three to a page) the terms sum to about 400 MB and the high-water mark over idle was 880 MiB on PostgreSQL and 580 to 760 MiB on SQLite across runs (507 MiB under `MALLOC_ARENA_MAX=1`), with MongoDB measured at 474 MiB on 13.5 MB rows, its 16 MiB document cap refusing the full shape; SQLite's driver run-ahead was bounded (sqlx's `row_buffer_size`, 50 rows by default, set to four on the pool: the 24-row run peaked at 844 MiB over idle with 50 and 696 MiB with four, back to back, and a 1M-row backup took 11.4 s with either), and whether the peak then grows with the number of worst rows is settled by the 12, 24 and 48-row series of the 2026-11-01 sentences below (it does with glibc's default arenas and does not under `MALLOC_ARENA_MAX=1`), and a one-item channel saved about 50 MiB of it for a 1M-row backup 75% slower (12.4 s to 21.6 s, both orders), so `ITEM_CHANNEL` stays at four. The PR 8 acceptance's "stays within section 7's formula" and "RSS must match the formula" read "under 1.2 times the sum of section 7's terms at the measured row with `MALLOC_ARENA_MAX=1`, at 12, 24 and 48 worst rows on one binary, the default-arena peak recorded beside it" (section 7, PR 8). The body sentences each of these corrects are edited with this item (4(e), 5.1 steps 2, 4, 5 and 6, section 7's knob comments, memory paragraph and pool paragraph, section 9, section 10's table, PR 6's slot sentence, PR 8's acceptance, interfaces, page paragraph and duty list, PR 9a's lead-in); earlier items keep their wording, as item 22 requires. (2026-11-01, PR 8 as measured and corrected.) The PostgreSQL page's row bound `n` is sized from the previous page's largest row, not its average, and the floor after the first page is 4 (`MIN_NEXT_PAGE_ROWS`; `MIN_PAGE_ROWS`, 16, is the first page's alone), because the server serializes and spills every one of the `n` rows the inner `LIMIT` reads, so a page costs the server `n` rows of text and not the rows it returns: after a page of 984 small rows and 3 large ones the average sized the next page at 810 rows of a table that was then large rows only, 5.9 GB of server temp files and 18 to 44 s for the three rows it returned, where the largest row sizes it at 4 (208 MiB of temp, 1.1 s), and a page of 27 MB rows now reads one row past its cut rather than thirteen (its steady state 1.2 s a page, 4.4 s before); the over-read that remains is the page that follows a row bound's worth of small rows, up to `n` rows of up to the line cap, which the sentence of the same day below bounds (section 3, 7, PR 8). The `EXPLAIN` acceptance holds for a page with the planner's estimate above `n` and not everywhere: on a table of 1,060 rows a page near the end whose `n` is above the rows left gets a bitmap scan of the primary key index and a `Sort` of the rows after the seek under the `Limit` (PostgreSQL 16), while on the 100k-row table the index scan holds ten rows from the end; both are pinned by a test and stated in the module doc (PR 8). PostgreSQL's `fetch_page` pulls the page's rows from the connection one at a time and parses each as it arrives, so the page's text is no longer a term of section 7 (the record's 83 MB at the worst shape); the item channel and the decoded page are unchanged; with that change the PostgreSQL worst-shape run peaked at 790 MiB (`VmHWM`) in 16.5 s against the head's 905 MiB in 18.2 s, and its 1M run's peak was 4.0 MiB over the 100k run's (section 7, PR 8). `run_page_bounds` fails at once, naming the key yielded twice, when a stream has yielded more items than the table holds, so an inclusive seek fails in 3 s where it ran until another test's lock wait gave out (PR 8). SQLite's run-ahead is five rows, not four: sqlx's worker steps and copies the next row before its `send` on the four-row channel blocks, 160 MiB at the line cap (section 7, PR 8). The allocator: the peak over idle of the worst-expansion run grows with the number of large rows under glibc's default arenas and is flat under `MALLOC_ARENA_MAX=1`, which is documented for operators (the admin guide's `[backup]` rows, the deployment guide's service unit, section 7) rather than set from the binary; section 7's memory paragraph is the sizing rule (the terms at the line cap, about 490 MiB per job on PostgreSQL and 650 MiB on SQLite, times the allocator's factor, times `max_concurrent_jobs`), measured at 27.7 MB rows, 1,000 small rows first, 12, 24 and 48 such rows back to back, in two series of six runs each on one SQLite binary: 466, 588 and 760 MiB over idle with the default arenas and 481, 452 and 480 MiB under `MALLOC_ARENA_MAX=1` on a quiet host, then 707, 700 and 1,281 MiB and 452, 480 and 455 MiB on the final build with the kill-point runs beside it (the terms' sum at that row 545 MiB; runs of the PR's first build on a loaded host reached 1,096 MiB at 48 rows), which replaces this item's earlier "the peak did not move between 12 and 24 worst rows" (section 7, PR 8). The #618 cells run on PostgreSQL and SQLite; the RSS acceptance states its 112 MiB bound and MongoDB's 500k run; the testing summary says the backup kill points are run by hand with the posted script, a `before_manifest` point added (the three documents and the data file under the prefix, no manifest, FAILED `owner dead` and the prefix deleted as at `after_data_file`) and the SIGTERM exit timed by polling (55 ms on SQLite, not the script's 5 s sleep); 5.1 step 1 names the store-health check in its order, before the permit; the record is committed as `docs/design/15-backup-restore-bench-pr8.md` and cited by that path, with its SQLite cache explanation corrected to `cache_size = -32000` (31 MiB per connection, `sqlite_util.rs`) and its two 12-row rows labelled by build; the differences document's backup-creation figures are the record's and its restore-interruption row says the prefix of a SQLite row goes once the dead owner's heartbeat is stale, not in the first cycle; the usage guide's backups row says RestoreTableFromBackup restores in-catalog backups only this release and the upgrade manual says a store-backed backup is listed, described and deleted but its restore answers the ValidationException until the restore job lands (section 5.1, 7, 10, testing summary, PR 8). The harness: `tests/test_abac.py`'s backup cell waits for AVAILABLE before DeleteBackup; the job-bound cell's two diagnostics say what each case is; the fidelity suite reads `/health` once and checks that the server's `backup_store` component is the kind `EXTENDDB_TEST_BACKUP_STORE` names and healthy (absent for `none`), the `none` branch of the vector cell captures the ARN of a create accepted against expectation, `_failed_rows` decodes Rust's `Debug` escapes, a cell pins the whole key of a refused item with a UUID hash key and a ten-digit `N` range key in the log line and in `backup list --failed`; the CI start steps of the PostgreSQL, SQLite and S3 jobs wait for the `backup_store` component to be healthy, the pytest lines carry `-rP` so a passed cell's printed branch reaches the log, and the S3 job dumps moto's log on failure and names what its by-name step covers (PR 8). Also: `validate_restorable_item` takes `&RestorableLimits` (the item size, hash key size, range key size and attribute name size of `LimitsConfig`), not `max_item_size_bytes` alone, so the restore's "every item against the target's limits" (5.3 step 5) is the same call; every untrusted text its error carries (a key value, an attribute name, a path segment, the number the wire's reason echoes) is bounded on its own at 4096 bytes of the text the message writes (a key value at 12,288, six times the hash key limit, so a key of control characters is whole too; 4096 is a round figure that keeps a path segment to four times the longest key attribute name), with a 1024-byte prefix and the length for a longer one, so a key of any legal size appears whole in the log line and the FAILED reason; every such text, the attribute path and the echoed number included, is written as a JSON string's content, with U+0085, U+2028 and U+2029 escaped as `\u0085`, `\u2028`, `\u2029` beside the control characters, so neither a control character nor a Unicode line separator in a tenant's item can add a line to the server's log, for a byte-oriented reader and for one that breaks lines the Unicode way (Python's `str.splitlines`) alike; an item whose data file line would exceed `MAX_LINE_BYTES` under a raised `max_item_size_bytes` is refused by the job's feeder through the same count-and-log path (`RestorableItemReason::LineTooLong`, whose reason names the cap, the line's own length being unknown once the count stops), the line measured on the blocking pool for items over the default item limit only, through a counting writer that holds no line, and such an item's validation (the number rule over every member, CPU work proportional to the item's size) run on the blocking pool with it, an item within the default validated on the feeder's own task, so the encoder's own refusal is never the first to see it; a put that fails while the snapshot stream is between pages ends the pipeline at once (the uploader cancels a token the feeder selects on), so the FAILED write does not wait for the stream; the completion log line carries the stream's item count; a job the shutdown cancelled records `BackupJobs{cancelled}` and neither `WorkerLastSuccess` nor `WorkerErrorCount`; `BackupJobDuration` is microseconds from the snapshot's open to the end of the job's last catalog or store call (the AVAILABLE write, or the FAILED write and the prefix's deletion); the reconciler's `CycleReport` counts a dead-CREATING resolution whose write matched no row as left to its owner (`creating_left_to_owner`), and its cycle line counts the dead legacy restores and the FAILED-record targets apart; `reconciler.rs`'s module doc states the FAILED-prefix duty's timing as the code has it, in this cycle on PostgreSQL, MongoDB and Cassandra and once the dead owner's last beat is stale on SQLite (5.1 steps 4 and 5, section 9, PR 8). (2026-11-01, PR 8 as measured and corrected, the row bound's memory.) The sentence above that the over-read happens "once" was false on a table whose large rows are scattered among small ones: with the bound sized from the previous page alone, a large row set the next bound to 4, four small rows set the one after back to 1000, and that page met large rows again, so the 1000-row page recurred every other page for the whole table (5,555 rows read for 1,000 kept, 13.4 GB of server temp files, on a 1,000-row table with 30 such rows; on the release binary the same bytes took 1.9 times as long and 3.2 times the temp files by key layout alone). The bound is now sized from the largest row of the last pages that together hold at least a row bound's worth of rows (`RowBoundMemory`), so it stays at the floor while large rows keep arriving within that span and returns to the row bound a row bound's worth of small rows after the last large one: 1,901 rows read for 1,000 kept and 3.9 GB on the probe (a memory of four pages read the same 5,555 as one page, since four pages of four rows are sixteen rows), and the interleaved table's backup 54.6 s and 3.2 GB of temp against 72.0 s and 9.2 GB, the grouped table unchanged. The bound that remains, stated as it is: the page that follows a row bound's worth of small rows reads up to `n` rows of up to the line cap, at most once per row bound's worth of small rows, and nothing in the window form stops the server at the budget. A page query that does stop there, a recursive walk of the primary key one index probe per row, was measured and not taken: it read exactly the rows it returned on every shape (the interleaved table in 38.8 s with 2.5 GB of temp) and cost one probe per row, 23.1 s against 11.9 s for the 1M-row backup; the record carries both. `the_row_bound_remembers_large_rows_among_small_ones` pins the memory's figures on a 300-row table with nine such rows (201 rows read past the cut over 53 round trips), `the_page_query_is_an_index_scan_on_every_key_shape` pins the index scan on the six key shapes (hash only, `S`, `N` and `B` range keys, a multi-part range key, a legacy table on the database collation, each at `n` 16 and 1000, first page and a page after a key, and the stream over 20,000 rows), and `page_sql_for` takes the collation flag `open_stream` decides, so the legacy table's query is tested as the engine writes it (section 3, 7, PR 8). Section 7's rule is scoped: the terms are at the default `max_item_size_bytes`, where no stored item's line exceeds the cap, and on an instance that raises it they are at the largest line a stored item expands to; the feeder's measurement of an item over the default limit holds no line (a counting writer that stops at the cap), where the earlier text said its buffer was bounded by the cap and the code held the whole line (section 7, PR 8). The one-arena factor is measured on both backends: the PostgreSQL worst shape at 12 rows peaked at 340 MiB `VmHWM` under `MALLOC_ARENA_MAX=1` against 750 to 788 MiB under the default arenas, 0.82 of the terms at that row on PostgreSQL (413 MiB, no run-ahead), and the setting's cost on the request path is measured: 3.5 to 9.2% on SQLite (GetItem, Query, Scan, paged Scan) and about 2% on PostgreSQL (the scans, 2.1%), so the deployment guide's service unit carries the line commented out with the condition and the cost, and section 7 states both (section 7, PR 8). 4(e)'s unknown-outcome rule reads as `write_status` has it: each unknown outcome is followed by a re-read, the write is sent once more at most, and after the second write a re-read that shows the row in the target status counts as matched, anything else as matching no row; the review checklist's bullet says the same (4(e), PR 8). The PR 8 duty list marks the resume of a dead RUNNING restore as PR 9a's, this release's reconciler logging such a record once; section 10's table gains PR 8's rows (the admin guide's `[backup]` rows and `[backup.store]` section, the deployment guide's service unit, the upgrade manual's asynchronous-create and rollback paragraphs, the design README's row 15b); the PR 9a lead-in names `RestorableLimits`; the ratios are given to two places (0.86 to 2.35; 2.35 at 48 rows); the SIGTERM exit is 55 ms in every statement; the record attributes the PostgreSQL worst run's idle to the restarted process's startup work on a table of 27.7 MB rows, not to the 1M load (sections 4, 7, 10, PR 8, PR 9a). (2026-11-01, PR 8 as corrected.) Section 3's parenthesis on the PostgreSQL cut, the PR 8 bullet, its two paging acceptance cells and the Interfaces paragraph's `n` sentence described the recursive walk that was measured and not taken (one row per step, no row read past the cut, two index probes) and the rule before the memory (`n` from the previous page's largest row, the over-read "once"); each is corrected to the shipped form: the three-layer window page with up to `n - 1` rows read past the cut, `n` sized from the memory, the first paging cell naming the record's `EXPLAIN (ANALYZE, BUFFERS)` figure (4 read, 1 past the cut at the floor) and `the_row_bound_remembers_large_rows_among_small_ones` (201 over 53 round trips), the `EXPLAIN` cell the plans the tests pin (`Limit` over one `Index Scan`; the bitmap scan with a `Sort` near the end of a small table) (section 3, PR 8). The over-read's statement gains the second page, which follows the first page's 16 rows and asks for the row bound: the over-read that remains is the second page, and then the page that follows a row bound's worth of small rows, each up to `n` rows of up to the line cap, said the same way in the PR 8 page paragraph, the module doc, the test's doc and the record (the earlier sentence of this item said "the page that follows a row bound's worth of small rows" alone); the floor's cost is stated beside it: after a page holding a row at the line cap the bound stays at the floor for a row bound's worth of rows, 250 pages of four, whatever those rows hold, so a table whose large rows are rarer than one per row bound pays 250 round trips per large row where one page of memory pages it at the row bound (2,585 page queries against 38 on a 20,000-row table with a 27 MB row at every 1,100th key, 25.6 against 25.1 s, 1.06 against 1.09 GB of temp; the record), and the record's "changes no page of a small-row table" reads "of a table with no large row"; the one-page-memory figure the test's doc, the record and the PR text cite is 306 rows past the cut over 7 pages (section 7, PR 8). 4(e) said the PostgreSQL reconciler "reads the heartbeat alone and never probes the lock" and named "the one liveness rule the three share": PostgreSQL's `claim_dead_backup` tries the advisory lock before it tests the heartbeat, so it takes over only a row whose heartbeat is stale and whose lock is free; a stopped owner (SIGSTOP, a paused VM) whose session lives keeps its row until the session ends or it finishes, and a vanished host's lock is released when the server's keepalives give up (about a minute with `CLAIM_KEEPALIVES`), so the takeover waits for the later of the two; 4(e) and 5.1 step 4 now say so (the SIGSTOP sentence scoped to MongoDB and Cassandra, with the PostgreSQL outcome: no takeover while the session lives; on resume the job completes under its own generation or, when the stop outlasted two heartbeat periods, stops as a lost claim with no catalog write and the reconciler resolves the row), the trait doc of `claim_dead_backup`, the admin guide's `stale_job_after_secs` row and the differences document's dead-owner clause gain "on PostgreSQL, once the owner's claim session has ended as well", and item 27's "the heartbeat alone" stands for PR 6's `remove_dead_creating_backup_row`, which `backup purge` still uses (4(e), 5.1 step 4, PR 8). The one-arena cost on PostgreSQL reads "about 2% (the scans, 2.1%)" in section 7, this item, the admin guide, the deployment guide and the PR text, the record's scans being −2.1%; section 7 labels the base of each ratio (the peak over idle on SQLite, the `VmHWM` on PostgreSQL, about 0.47 over idle); the admin guide's `max_concurrent_jobs` row restates section 7's sizing rule (1.2 times the terms per job under `MALLOC_ARENA_MAX=1`, no bound measured under the default arenas, the PostgreSQL figure as `VmHWM`, 2.35) and its `stale_job_after_secs` row the dead-row outcomes (AVAILABLE with a complete manifest, FAILED without, a restore job's target removed); the upgrade manual's rollback paragraph names `backup sync --adopt-store`, as 4(c) requires (section 7, 10, PR 8). `EXTENDDB_TEST_BACKUP_STORE` is set by every CI job that runs a suite creating backups, the Rust integration jobs included: the SQLite Rust integration job's file-backed server has the defaulted root, so its CreateBackup is the asynchronous one, and `tests/rust`'s backup tests assumed the synchronous create and left their backups; the suite now waits for AVAILABLE, deletes what it makes and reads the variable as the Python suite does, and `run-tests`' usage, `CONTRIBUTING.md` and the developer test guide name the variable and the refusal (section 10, PR 8). The PR 8 kill-point list names `before_manifest`, and section 10's limits-doc row the rewritten "Backup store" row (PR 8). 5.1 step 6 says a job cancelled after its manifest put completes AVAILABLE, the manifest being the commit record, where it said a cancelled job can never complete (5.1 step 6, PR 8). On PostgreSQL a job stopped and resumed after more than two heartbeat periods stops as a lost claim with no catalog write, its session's end frees the advisory lock and the reconciler resolves the row, where 5.1 step 4 said it resumes and completes; the unknown-outcome rule's second write is stated as the rule of a backend whose claim connection survives the outcome (MongoDB, Cassandra), PostgreSQL's failing with the session; the reconciler's failed `get`, `list` and `delete_prefix` calls are `BackupStoreErrors` as the job's are; and `extenddb stop` waits for the sum of the server's shutdown bounds, the 30 s job drain included (4(e), 5.1 step 4, section 9, PR 8). After #335 and #681 merged, PR 8 is rebased onto PR 7's rebased head: the server's shutdown drains the jobs before the registration supervisor's return and the worker set's stop (#666), a job still running at the drain's bound keeps the serving registration as a worker that could not be stopped does, so a migration cannot start beside it, `extenddb stop` waits for the sum of the bounds that path has (76 s: 5 s of connections, 30 s of jobs, 10 s for the supervisor, 16 s for the workers' stop, 10 s for the release, 5 s for the runtime), and the reconciler's withdrawn duty is stated after migration 6 (PR 8). After #433 (attribute-name size on every write path) merged, the Drawbacks line names an attribute name over the limit that `UpdateItem` and a `TransactWriteItems` `Update` stored before it beside the #618 values (Drawbacks, PR 8). +32. (2026-11-04, after the maintainer's review of PR 8.) The heartbeat's one-period bound is placed by the backend (`JobClaim::heartbeat_within`, a provided method that bounds the whole call; SQLite's override takes the engine write lock first and bounds only the `UPDATE`, so a writer holding that lock past a period, a GSI backfill say, delays the beat and the claim re-check and ends no job; the re-check's confirmation counts as a beat for the pre-put check; after the maintainer's second round, the pre-put check also passes while a beat is in flight and has not failed, since a backup large enough to reach a put while the beat was parked on the lock past two periods stopped at that put, the maintainer's 180,000-item backup beside a GSI build twice at the third or fourth data file, and the reconciler then failed the row and deleted its objects; the safety argument is in 5.1 step 4 and `heartbeat.rs`'s module doc, and the tests are a SQLite cell whose second data file is put under a hold of the lock, `a_put_during_a_writers_hold_passes_on_the_beat_parked_behind_it`, and an engine cell with a claim whose beat parks, `a_beat_parked_on_the_write_lock_confirms_the_claim_until_it_returns_lost`), and `heartbeat.rs`'s SIGSTOP text says that a job with no beat in flight resumes under its own generation once its next beat succeeds. The store-backed create checks the maintenance claim before the permit, as the in-catalog paths do, and inserts no row for a refusal (5.1 step 2 and 4(f) are edited: a FAILED `maintenance` row carried the insert's heartbeat and `backup purge`'s wait counted it as a live job for the stale window). Before every put the job checks the store as the runtime holds it (healthy, and its identity the row's) and on a failure fails the row with the store's reason and makes no store call, the prefix's deletion included (4(c)'s swapped-root case). The open and the insert are selected against the shutdown token. The uploader cancels the feeder's token before it awaits an encoder, so a put that fails mid-file ends the job while the stream is held. A data file whose stored size differs from the encoder's compressed count fails the job (the dead-owner probe refuses that difference). The AVAILABLE write sets `TableSizeBytes` from the stream as it sets `ItemCount` (the differences document names both). `failed_row_retention_secs` is at least `stale_job_after_secs` (a FAILED row removed sooner left a stopped owner's late object with no row to delete it under). A registration lost while serving cancels the jobs' token, which the runtime replaces for the jobs that start under the new registration; the shutdown sequence after the listener is one function the server's tests drive with a job that outlives a short drain. The plain-HTTP listener's drain is bounded as the TLS listener's is. CreateBackup is bound to the in-catalog path on SQLite where the serve lock is not taken (not Unix), since the slot claim is sound only with one process per file. 4(c) states the gap of the different-roots guard before the first AVAILABLE row. The compose files' `stop_grace_period` is 80 s. The kill-point scripts are committed under `devtools/` and the clippy job lints `backup-test-hooks`; a MongoDB job runs the backup suites against a filesystem store. The tests added are the review's: the lock held past two periods on SQLite, the mid-file put failure with the stream held, the unhealthy and renamed store mid-job, the key-ordered stream out of order, `BackupStoreErrors` for the job's `delete_prefix` and the probe's `list`, the re-read under another generation, the two-period sample, the nesting rule through lists and mixed levels, the keepalive literals, the driver's five-row run-ahead, the beat between the takeover's read and its LWT on Cassandra, `pool_size_for_jobs(11, 4)`, and the retention floor. After the third round the clause is bounded where the beat does not park before its bound (`JobClaim::heartbeat_parks_before_its_bound`, `true` on SQLite only): a beat in flight for more than a period on PostgreSQL, MongoDB or Cassandra confirms nothing, since a process stopped mid-beat and resumed would otherwise pass the check until its timer raised the loss (`a_beat_in_flight_past_its_bound_confirms_the_claim_only_where_the_beat_parks`). + +33. (2026-11-01, PR 9 as implemented — the design's PR 9a and PR 9b as one PR, the maintainer's decision.) The restore job runs from the store on PostgreSQL, SQLite, MongoDB and Cassandra, same-backend and cross-backend, from a filesystem or an S3 store; the matrix and the nightly are PR 9b's, shipped here. `RestoreTableFromBackup` of a `backup_index` row no longer answers the "in a backup store; this release restores in-catalog backups only" refusal `check_catalog_row_for_restore` carried; a healthy configured store's AVAILABLE row is restored, and the refusal is gone from the engine and the differences, usage and upgrade docs (5.3, section 10, PR 9a). The interface: `RestoreEngine`, a new trait in `StorageEngine`'s bound with `bulk_load(target, claim, BulkLoadFile { file_index, start_at, reapplying }, items) -> applied`, `finish_restore(target, claim, item_count, size_bytes) -> Finished { Active, Pending, NotMatched }`, `count_restore_target(target) -> u64`, `read_restore_progress` (default `[]`) and `sweep_restore_progress(running) -> u64` (default 0); `StorageEngine` has no methods of its own, so `bulk_load` is not "a new `StorageEngine` method" (section 3's PR 9a interface list). `BackupCatalog` gains `claim_restore(account_id, input: CreateTableInput, &RestoreRegistrationRow) -> (TableDescription, Box)` (the registration creates the CREATING target and returns the description the request answers; the record is born owned under the backend's generation, `RestoreRegistrationRow.owner_generation` being `UNOWNED_GENERATION` and the backend minting its own), `claim_restore_dead(key, stale_after, spool_host) -> Option>` (the restore analogue of `claim_dead_backup`, writing the claimant's `spool_host`, a fresh generation and a fresh `heartbeat_at` on the database's clock in one conditional write, where the backup takeover leaves the heartbeat: nothing waits on a restore record's old heartbeat after a takeover, and a second taker must find the record live at once, 4(e)), `read_restore_job`, `list_restore_jobs(Option)` and `remove_restore_target`/`fail_restore_job` as the reconciler uses them; `RestoreJobRow` carries `store`, `spool_host`, `file_index`, `items_applied`, `failure_reason`, `completed_at`, so the FAILED-target duty reads a row's reason and the sweeps read its store and `spool_host` (section 3, 4, 5.3, PR 9a, PR 9b). The design's `claim_restore(&RestoreJobRow) -> Box` and `record_progress(&dyn JobClaim, file_index, items_applied)` are gone: there is no `record_progress`; every backend records progress inside `bulk_load`, in the write that applied the items (PostgreSQL a `restore_progress` row in the file's data transaction, SQLite and Cassandra a `tables`-row update per batch, MongoDB a `restore_job.file_index` update per file), and the design's "conditioned on the generation" holds on SQLite, MongoDB and Cassandra but on PostgreSQL the progress row lives in the data database, which holds no job record, so the fence there is the base table's primary key (a file applied twice fails on its first key, a recorded file on the progress row's own key) and the claim's liveness, which the engine confirms before each file and whose loss, the heartbeat's, aborts a file in flight; the generation fences only the record's own writes in the catalog (DONE, FAILED, the takeover) (5.3 step 5, PR 9a). `bulk_load` carries `reapplying` so MongoDB's duplicate suppression is explicit: on a resume the first file the engine meets is `reapplying = true` and every later file false; MongoDB suppresses an all-E11000 `insert_many` only when `reapplying`, else "the backup holds a key twice"; PostgreSQL and SQLite never suppress (their base rows are plain `INSERT`s, a duplicate fails the file whatever the flag, and SQLite has no unrecorded partial write, its progress being in the batch's transaction); Cassandra's `put_restored_item` is an upsert, so a duplicate collapses and `count_restore_target` one short of the manifest's total is the only duplicate check there (5.3 step 6, PR 9a). The restore ends by counting the target against the manifest's item total on every backend, so an unordered backup holding one key in two files and a key-range check that missed are both caught; a key-ordered file runs #394's per-file `KeyRange` check, which ends the file's stream at an item outside its range. The migration for PostgreSQL's `restore_progress` is id 6 (`MigrationId::RestoreProgress`), a `StoreKind::Data` body on PostgreSQL (`data_migrations/0006_restore_progress.sql`), `NotApplicable` on SQLite, and nothing on MongoDB and Cassandra, whose bootstrappers have no `MigrationId` table; this corrects "data migration 005 for `restore_progress`" in the PR 9a section (its acceptance line and its load paragraph) and the PR 6 line on reserving numbers; item 14's "data migration 005 reserved" stands as the history it is (4(i), PR 6, PR 9a). The manifest registration locks the `backup_index` row (the lock `start_backup_delete` takes: PostgreSQL `SELECT ... FOR UPDATE`, SQLite the `BEGIN IMMEDIATE` write transaction, MongoDB `$inc lock_seq` on the document, Cassandra the `active_restores` lightweight transaction on the backup's partition), while the legacy registration keeps the `backups` row; a legacy and a store-backed restore into one name exclude each other (PostgreSQL the advisory `restore_lock_key`, which both paths take; on SQLite the store-backed path's `restore:/
    ` slot and, for a legacy restore, the `tables` row under the write transaction), so a `DeleteBackup` of a `backup_index` row meets only store-backed registrations and the two orderings resolve one before the other (DeleteBackup wins → the registration is `BackupNotFound` and no target; the registration wins → DeleteBackup is `BackupInUse` and the row stays AVAILABLE) (5.2, 5.3, PR 9a). The restore's ACTIVE flip sets the table and every secondary and vector index of the target in one catalog transaction on PostgreSQL and SQLite (an index of a restore target is created ACTIVE, and `activate_restore_target` sets ACTIVE any `indexes` or `vector_indexes` row that is not), on MongoDB the job writes DONE and `restore_backfill_pending` and the backfill worker flips the table ACTIVE once every index has left CREATING (`finish_restore` returns `Pending` there), and on Cassandra the one lightweight transaction flips the table alone, its index rows created ACTIVE (`Active`, never `Pending`), so a Query on each GSI and `SearchVectors` answer as soon as the target reports ACTIVE (5.3 step 6, PR 9a). The request bounds its wait for the detached registration at `RESTORE_REGISTRATION_TIMEOUT` (60 s) on every backend; past it it answers `InternalServerError` while the task goes on and the target appears through `DescribeTable`, so the differences document's MongoDB-only sentence becomes every backend's (5.3 step 4, PR 9a). The maintenance check runs after the registration on the manifest path (5.3 step 4 as written) and before the permit on the legacy path (item 29), both orders in `restore_table_from_backup` and `restore::start`; the restore's first confirmed beat stands at the instant `claim_restore` returned (the record's `heartbeat_at` was stamped inside the registration, before the return, so the instant is at or after the beat it stands for by the registration's remaining duration, which each backend bounds, 4(e)), where the backup job takes the instant before its claim's insert (5.1 step 4), so a registration that took longer than two heartbeat periods does not end the job at its first check; a held claim on the manifest path leaves a FAILED `maintenance` record whose target the job removes before it answers `LimitExceededException`. Only `StoreError::Transport` is retried on a data-file download, for up to `DOWNLOAD_RETRY_BOUND` (15 min, counted from the job's first download attempt, every file's retries against it) with `min(2^n s, 60 s)` plus up to a second of jitter, so the retry covers S3 stores (a filesystem store's `get` errors are `Io` and fail at once); `NotFound`, a denial, and a digest or size mismatch fail at once naming the file. On a filesystem store "verified in place" is two `get` passes, a hashing pass before any load and a decoding pass through `ExtenddbManifest::reader` that hashes again (so a file replaced between the passes fails in the load) with no spool; on an S3 store the files are spooled as `restore-
    /.json.gz` with `.partial` → `sync_all` → rename → `.verified` (the marker holding the file's sha256, compared with the manifest's before the file is trusted; an entry left by a run of this host for a record another host then owned is removed before use) and read through `reader_prehashed`, the spool at `spool_path` (the defaulted `backup-spool/` on file-backed SQLite). A restore the shutdown stops records `RestoreJobs{cancelled}` and no worker metric, keeps its spool and its progress and writes no status (the `SHUTDOWN` reason is a backup's alone); a lost claim or a DONE write that matched no record records `RestoreJobs{failed}` and `WorkerErrorCount{RestoreJob}` with no catalog or table access; `fail_restore` writes FAILED then runs `remove_restore_target` only when that write matched. A job that learnt the record is another's (the lost claim, the DONE write's `NotMatched`, `fail_restore`'s no-table-access arm) leaves its spool entry to the sweep, since a same-host successor may be reading it (the entry is keyed by table id); every other end but the shutdown's removes it (4(e), 5.3 step 7, PR 9a). The load hands the backend one `bulk_load` per file over a stream fed in chunks of at most `LOAD_CHUNK_ITEMS` (1000) items or `LOAD_CHUNK_BYTES` (32 MiB of data-file line text) through a channel of two chunks; a restore holds the parsed manifest, one decoded line and its parse, the decoded items of four chunk positions and the backend's batch (32 MiB of item text on SQLite and Cassandra, 4 MiB per INSERT on PostgreSQL, 1,000 documents on MongoDB), which is section 7's restore sentence, with the figures measured on this PR's final tree (`docs/design/15-backup-restore-bench-pr9.md`: 848 MiB over idle on SQLite and 778 MiB on PostgreSQL at 24 worst-shape items under the default arenas, 392 and 427 MiB under one arena; 89 and 16 MiB at 1M key-ordered items of 200 bytes, in 26.8 s and 9.5 s, 144 to 164 and 16 MiB for the same items unordered), which replace the earlier sentence, "A restore holds one line's decode plus one insert batch plus the parsed manifest for the whole job; neither grows with the table": the terms stand, the figures are measured, and a restore of large items is of the backup's order. The metrics are `MetricName::{RestoreJobs, RestoreJobDuration, RestoreBytesDownloaded}` with the outcome in the operation dimension, `QuerySource::RestoreJob` for the worker metrics, and `BackupStoreErrors{get}` on every failed `get` of a restore job (the request's and a resume's two document reads and each download attempt the store failed, retried or not; a digest mismatch and the spool's own errors are not the store's); section 9's `restore_jobs_total{outcome}` and `restore_job_duration_seconds` are a converter's spellings, as item 31 said for the backup names (section 9, PR 9a). A backend's refusal of an item in `bulk_load` is not a wire exception: on MongoDB an `N` sort-key value past Decimal128's 34 significant digits is the write path's `ValidationException`, which becomes the restore's FAILED reason, and the client sees the target disappear; a 38-digit non-key number is stored as text and restored intact on every backend. The reconciler resumes a dead store-backed restore from its recorded progress (per file on PostgreSQL and MongoDB, per batch on SQLite and Cassandra) under `claim_restore_dead` once the heartbeat is stale (on SQLite at once after a restart, by the record's slot; on PostgreSQL once the owner's claim session has ended as well), re-downloading to its own spool when the dead owner's `spool_host` is not this host's; it fails a dead manifest record under a maintenance claim before it looks at the record's store (`OWNER_DEAD`, the target removed; `target_not_empty`, the target left for `remove-target`, when it recorded no progress and the target holds items), fails a record of a store other than the configured one or of no store with `OTHER_STORE` (naming `backup sync --move-store`, the target left to the FAILED-target duty), skips a resume without a permit, and FAILs a resume with no recorded progress into a non-empty target as `target_not_empty`, left for `extenddb backup remove-target --account
    ` (PR 10's command, shipped with PR 9, below); its two sweeps (`restore_progress` rows of tables that are not RUNNING restore targets; spool entries of restores not RUNNING with this host's `spool_host`) run only without the maintenance claim and read `list_restore_jobs(Some(RUNNING))`, a backend that cannot list them logging once and skipping both (section 4, 5.3 step 7, PR 9a). A store-backed backup of a multi-part-key table is written `unordered` and restores on every backend, the target laid out as a natively created table (`bulk_load` keys rows with the backend's own key functions), while a legacy backup of one keeps the reader's refusal; the #384 override tests run through both the legacy and the manifest paths for every case with a definition and assert the same input or the same refusal, the manifest path deriving the SSE specification from the manifest's `sse_description` where the legacy path keeps the specification as written. The cross-backend matrix (PR 9b): a cell copies the source's backup prefix into a root the target catalog owns and inserts the target's `backup_index` row as `backup sync --adopt-store` will (PR 10), so the cell proves the restore and not the adoption; it runs from the committed `devtools/backup-restore-matrix`, and `.github/workflows/matrix-nightly.yml` (a `schedule` cron and `workflow_dispatch`) runs the twelve plain cells and the duplicate-key and vector cells nightly, building the four backend binaries and starting PostgreSQL, a MongoDB replica set and Cassandra as the integration workflows do; the seed stands in for `backup sync --adopt-store` until PR 10 replaces it. The restore kill points are run by hand with the `kill-points.py` posted with PR 9 (outside the repository, as PR 8's was), one mode per backend where it applies: `restore_after_download`, `restore_after_file_commit`, `restore_sqlite_mid_batch`, `restore_cassandra_before_progress`, `restore_mongodb_after_insert_many`, `restore_postgres_before_progress_delete`, `restore_mongodb_before_worker_flip`, `restore_between_failed_and_removal`, `restore_fenced_out`, `pg_terminate_backend` and `sigterm`, each a SIGKILL or a signal at the named point with a restart and the outcome asserted (the exact count after a resume, or the target gone for a failing restore). Two points pause before the first progress record of a file or batch (`restore_mongodb_after_insert_many`, `restore_cassandra_before_progress`), so a kill at their first reach leaves written items and no progress and the resume answers `target_not_empty` (the rule of 5.3 step 7, asserted); the re-application path (MongoDB's E11000 suppression of the one unrecorded file, Cassandra's re-applied batch) is run by killing at the point's second reach, after an earlier file's or batch's progress is recorded (the script's `--second-reach`): on MongoDB a 24,000-item backup in three files resumed from file 1 with file 0 recorded and reached ACTIVE with the exact count (file 1's 8,227 items re-applied as duplicates, 7,544 applied by the resume); on Cassandra a one-file backup of 12,000 items killed before the second batch's progress write resumed from item 10,000 and reached ACTIVE with 12,000 (2,000 applied by the resume), and `restore_after_file_commit` resumed from file 1 to the exact count. A Cassandra server's CreateBackup is the in-catalog format (section 3), so a Cassandra kill point takes its backup on a SQLite source, copies the prefix under the Cassandra target's root and seeds the target's `backup_index` row as the matrix does; the single-node Cassandra of the harness needs `replication_factor = 1` in the server's configuration, as `init` writes it, since under the default of 3 every lightweight transaction and quorum read on a keyspace the server creates answers "No nodes available in query plan", which is what had blocked these points. The feature chain for the backends' pause points: `extenddb-app/backup-test-hooks` → `extenddb-server/backup-test-hooks` now pulls `extenddb-storage/test-hooks` beside `extenddb-engine/backup-test-hooks`, so a hooks build compiles `extenddb_storage::backup_catalog::restore_pause_point` and each backend's own point (`restore_sqlite_mid_batch`, `restore_cassandra_before_progress`, `restore_mongodb_after_insert_many`, `restore_postgres_before_progress_delete`, each a `pub const` beside `restore_pause_point` in `extenddb_storage::backup_catalog`, the one spelling the backends and the kill-point script use) in without naming a backend crate (the server depends on none directly); the engine's five restore points are its `KillPoint` variants (`RestoreAfterDownload`, `RestoreAfterFileCommit`, `RestoreMongodbBeforeWorkerFlip`, `RestoreBetweenFailedAndRemoval`, `RestoreFencedOut`), and that enum is the engine's list, not the whole list. Corrections made before PR 9 merged: (1) the restore takeover writes `heartbeat_at = now()` in the one conditional write with the generation and `spool_host`, on every backend, so a second taker finds the record live at once; the backup takeover keeps leaving it (4(e) states why the rules differ); (2) `LOAD_CHUNK_BYTES` is summed from the decoded line's length (the decoder's `bytes_decoded` across one item), not `item_size_bytes`; section 7's terms are restated in that unit; (3) the download retry bound is per job: `download_all` takes the job's download start once and every file's retries count against it; the module doc, 5.3 step 3 and this item say so; (4) a job that learnt the record is another's (a lost claim, a DONE or FAILED write that matched no record) leaves its spool entry to the reconciler's sweep; (5) the `.verified` marker holds the file's sha256 and `is_verified` compares it with the manifest's; `Entry::open` removes an entry whose record's `spool_host` was another host's before use; the resume reads the manifest from the spool only when the record named this host before the takeover (it read the record after the takeover, which had just written this host's name); (6) the decoder checks the heartbeat's pre-write rule (`confirms_the_claim`) at every chunk boundary beside the shutdown token; MongoDB and Cassandra re-read the record per batch; (7) a restore's `claimed_at` is the instant `claim_restore` returned, the record's `heartbeat_at` having been stamped inside the registration. Cassandra's takeover re-applies the `active_restores` entry when absent; the SQLite and Cassandra batch is bounded by `RESTORE_BATCH_BYTES` (32 MiB of item text) beside `RESTORE_BATCH_ITEMS` (10,000), both in `extenddb_storage::restore_engine`; MongoDB and Cassandra read the record before each write batch; `sweep_restore_progress` deletes from a positive list; the backends' pause-point names are `pub const`s beside `restore_pause_point` in `extenddb_storage::backup_catalog`, one spelling each (`RESTORE_SQLITE_MID_BATCH`, `RESTORE_MONGODB_AFTER_INSERT_MANY`, `RESTORE_CASSANDRA_BEFORE_PROGRESS`, `RESTORE_POSTGRES_BEFORE_PROGRESS_DELETE`), and the engine's `KillPoint` is its own list; Cassandra's `count_restore_target` reads the catalog first and answers `TableNotFound` for a key that is not a CREATING restore target with that id, as the other three do; the MongoDB registration stamps the record's heartbeat from the server's clock inside its transaction (the registering process's clock is replaced; the test registers from a process an hour behind); a duplicate primary key fails the file with `data file : holds a primary key the target already has (the backup holds a key twice): `, the engine naming the file by its key once and the backends naming no file; SQLite's startup line for an aged RUNNING record names the reconciler's duty per kind. The reconciler's store check runs first in the cycle, so the manifest arm reads this cycle's health, and the skip line names `not yet checked` for a store whose first check has not recorded; a maintenance-held dead manifest restore is counted once, in `manifest_restores_failed_and_removed`; `RestorableKeySchema::resolve` runs in the request with step 2's refusals; the request's second table-name check and second `check_supported` are gone: the parser's identity rule and version gate run first; the engine's kill points log `job reached kill point `; each backend's own point is named beside `restore_pause_point`; `finish_restore`'s `Pending`, the DONE write's `NotMatched`, the pre-write claim checks of `run`, the download's pre-attempt check and `WorkerErrorCount{RestoreJob}` on a FAILED job have engine tests; the #384 override tests run through both paths for every case with a definition. The wire test `test_create_backup_of_a_vector_table_is_refused` is `test_vector_table_backup_refused_in_catalog_and_restored_from_store`, since it asserts the round trip on a server with a store; the multi-part-key wire cell lives in `tests/test_backup_job.py` on a deployment that enables `allow_multipart_table_keys`, compares the restored item set with the source's `Scan` and makes no `GetItem` by the whole key (the preview's item paths address such a table by its first HASH attribute on every backend: a `GetItem` by the whole key finds nothing on PostgreSQL and the first HASH key's one item on SQLite, restored or not). Corrections made before PR 9 merged, second set: `extenddb backup remove-target --account
    ` ships with PR 9 rather than PR 10 (the PR 10 section's clause says so): the `target_not_empty` rule was written for a catalog restored from a dump that names a restore which finished and was written to since, but its common trigger on MongoDB and Cassandra, which record progress after the write, is an ordinary crash before the first progress record (the whole first data file on MongoDB, the first 10,000 items or 32 MiB on Cassandra: the kill leaves written items and no progress), and without the command this release had no way to remove such a target (DeleteTable refuses a restore target, the FAILED-target duty skips the reason, DeleteBackup and a second restore of the backup are `BackupInUseException`); the first-reach runs of `restore_mongodb_after_insert_many` and `restore_cassandra_before_progress` are that state, so the PR 9a acceptance line, the PR 9a load paragraph, the testing summary and the differences doc's restore-interruption row name it as the third outcome beside "finishes with the right count" and "gone", where they had named two; the command is the one section 4's duty list describes (the maintenance claim, a FAILED record's target whatever the reason, a RUNNING record's dead target, a legacy record's target, the three refusals, no store needed, the audit line), its tests are the engine's maintenance tests and one wire cell on SQLite seeded into the state at the catalog, and the two first-reach kill points run it and assert the target gone and DeleteBackup allowed. The spool sweep lists the spool before the records (the design's rule, which the shipped order broke); the spooled manifest and sidecar are written as the data files are and a spooled pair that fails verification yields to the store's; the resume's first confirmed beat is the takeover's return, taken beside `claim_restore_dead`; a `finish_restore` error is an unknown outcome re-read through the claim; `BackupStoreErrors{get}` counts every `get` the store failed; the request's and the resume's document reads are bounded (4 KiB, 16 MiB); the restored table's size is the source table's; `RestorePostgresBeforeProgressDelete` is gone from the engine's `KillPoint` (one spelling each holds: the four backend names are the storage crate's constants and nothing else); a registration that fails after the request stopped waiting is logged. The restore's re-applied batch on Cassandra relies on the write path rewriting an unchanged item's GSI/LSI row instead of deleting it (#742, fixed on main by #424), and the `restore_cassandra_before_progress` second-reach run reaches ACTIVE with the GSI count exact where before #742 was corrected it was one batch short. PostgreSQL's `finish_restore` deletes the progress rows after the flip's commit and does not fail the restore if that delete errors (logged and swept). A file's PostgreSQL progress row is a plain INSERT from `start_at == 0` and an upsert from `start_at > 0`. Two concurrent registrations of one backup leave one target on Cassandra (the registration LWT's `active_restores = null` is the fence). What bounds a registration's duration per backend (the advisory/`FOR UPDATE` lock, the slot, `transactionLifetimeLimitSeconds`, the LWT timeouts), so the `claimed_at`-minus-`heartbeat_at` gap can never reach the stale window, replaces the earlier "conservative" in 4(e) and in this item, which had the direction wrong: an instant later than the stamp makes the pre-write check admit a beat older than two periods by the gap, so the gap needs a bound, not a name. The SQLite restore's published memory figures are the no-index shape; a table with indexes costs SQLite's page cache (`cache_size`) plus arena fragmentation, bounded, not the restore's buffers (the record's index table: 143 to 225 MiB at 100k and 165 to 172 MiB at 300k with two GSIs on SQLite, 23 to 34 MiB on PostgreSQL), and the 60,000-item shape of the differences doc's in-catalog timings was measured store-backed on PostgreSQL, SQLite and MongoDB (0.67, 1.85 and 3.74 s to ACTIVE), the row now labelling which path each figure is. The MongoDB CI jobs' `EXTENDDB_TEST_BACKUP_STORE` is `filesystem` from this PR (`devtools/run-mongodb-tests` serves a filesystem root), where it was `none`; the Cassandra job's stays `none`; the runner's `--backup-store none` is the one way to run the legacy LSI fallback cell, the in-catalog restore path's one wire cell on MongoDB, which no CI job runs. The PR 11 line owns the two unmeasured cells (the MongoDB and Cassandra 1M restores, the vector cell) and no longer the nightly schedule, shipped here. The kill-point cells as run on the pushed head's hooks binaries (4c353191), every ACTIVE outcome asserting the base table's and the GSI's `Scan` counts: `restore_after_download`, `restore_after_file_commit`, `restore_between_failed_and_removal` and `sigterm` on all four backends; `restore_sqlite_mid_batch`; `restore_postgres_before_progress_delete`, `restore_fenced_out` and `pg_terminate_backend` on PostgreSQL; `restore_mongodb_after_insert_many` at its first reach (`target_not_empty`, then `remove-target`) and its second (the re-application, 24,000 items in three files); `restore_mongodb_before_worker_flip`; `restore_cassandra_before_progress` at its first reach (`target_not_empty`, then `remove-target`) and its second (12,000 items, the GSI exact); the Cassandra points with a SQLite source. Rerun on the build after the last corrections (the first beat before the reply, the Cassandra re-read and fence, the PostgreSQL vector drop): `pg_terminate_backend` as reworked (the lost claim at the next check 20 s after the termination, the record RUNNING with its `(0, 8228)` progress row, the takeover 45.6 s after the termination and 46.7 s after the last beat, ACTIVE 24,000 on base and GSI), `restore_after_file_commit` and `restore_mongodb_before_worker_flip` on MongoDB (resumed from data file 1 after 0 of its items, 3,771 applied by the resume, DONE then the worker's flip, 12,000 exact; DONE before the kill and the restarted worker's flip after it), `restore_cassandra_before_progress` at its second reach (resumed after 10,000, 2,000 applied, 12,000 on base and GSI) and `restore_between_failed_and_removal` on PostgreSQL with a vector backup (the target gone after the stale window, its data table and `_ddb_vec_` relation gone with it, where the pushed head's build left the relation). The matrix's four `--duplicate-key` and eight `--vector` cells ran on the final binaries. Corrections made before PR 9 merged, third set. The job's first heartbeat is sent right after `claim_restore` returns and before the reply, and the pre-write check counts its two periods from that beat's return; a `Lost` or failed first beat fails the record ("the claim was lost before the job started") and answers `InternalServerError`. `extenddb backup remove-target` decides a RUNNING owner dead through `list_dead_restore_jobs(owner_window)` on the database's clock, and the purge's wait decides a backup row's owner live through `list_backup_rows_older_than` on the heartbeat clock; neither reads a host clock. The reconciler takes nothing over and writes no FAILED on its manifest arm once the server's shutdown has begun. Under a maintenance claim a dead record with no progress over a target that holds items is failed `target_not_empty` through `fail_restore_job`, not `OWNER_DEAD`. The reconciler's `target_not_empty` lines print `extenddb backup remove-target --account
    `. The restore job's stop line names the heartbeat timer's reason when the timer raised the loss, and the timer logs the reason at WARN. A DONE write whose answer was lost and whose record re-reads DONE under the job's generation ends with "complete (DONE)", not "ACTIVE". A spool document write that fails and a resume that fails before it runs remove this host's entry. The engine's tests of the 60 s bound, the slot held to the job's end and the re-reads' generation checks are cells of their own. Cassandra's progress lightweight transaction re-reads an unknown outcome (applied, one retry, or the lost claim; a second unknown outcome fails the restore), where the earlier text said the restore fails and the reconciler resumes it, which the FAILED write's removal made false. The Cassandra registration's refusal path conditions the target's removal on its own generation; a record a reconciler took over during the DDL keeps its target. PostgreSQL's `abort_restore` drops a vector index's relation with the data table. The PostgreSQL record's `heartbeat_at` is the registration transaction's start. Tests: `test_an_unknown_progress_write_outcome_is_decided_by_a_re_read`, `test_a_registration_taken_over_during_its_ddl_leaves_the_successors_target`, `test_a_mid_file_resume_records_start_at_in_every_batch_progress`, `remove_restore_target_drops_the_vector_relation_with_the_data_table`. The engine's `KillPoint` has five restore variants, named above. SQLite's takeover clock is `strftime('%Y-%m-%dT%H:%M:%fZ', 'now')`, the `NOW` constant of its catalog (4(e) names it). `extenddb backup remove-target` refuses a RUNNING record with a live owner by its own text ("stop it or wait for it"); the DONE and no-record refusals are the two "not a restore target" ones. The published 1M figures are the key-ordered shape's and the record says so beside them, with the same items measured as an unordered backup (SQLite 144 to 164 MiB over idle in 66 to 90 s against 87 to 98 MiB in 27 to 56 s key-ordered; PostgreSQL 16 MiB either way), and SQLite's final `COUNT(*)` is the counting connection's page cache, bounded by `cache_size` (32 MiB) and the same size at 1M and at 3M rows (22 to 28 MiB), where the record had called it the one term that grows with the table from a single measurement. The `pg_terminate_backend` cell of `kill-points.py` pauses the job at `restore_after_file_commit`, terminates the claim session while it is paused (psql must answer a row), waits past one heartbeat period, releases, and asserts the lost-claim stop line, the record RUNNING with its progress row, the takeover no earlier than `stale_job_after_secs` after the owner's last beat and the exact counts; the shipped cell had terminated the session after a fixed sleep and asserted the final count alone, which a 2.6 s restore satisfied without the case ever running. The script's `--vector` switch gives the source table a vector index, and the tamper cell on PostgreSQL then asserts the removed target's data table and `_ddb_vec_` relation gone. `.github/workflows/matrix-nightly.yml` installs `postgresql-client` and checks `psql --version` and the two drivers' import before the cells; it has had no run, since `workflow_dispatch` needs the file on the default branch, so its first run after the merge is the proof of the runner image. The upgrade manual counts migrations 1 to 6 as breaking and migration 7 as declared breaking on PostgreSQL for want of the additive-declaration job (on SQLite, MongoDB and Cassandra it does not apply). The body sentences each of these corrects are edited with this item (section 1's duplicate rule and Decimal128 bullets, section 3's PR 9a interface list, 4(b)'s progress members and metric name, 4(e) and 4(i), 5.2, 5.3's preamble and steps 1 and 3 to 7, section 7's restore memory sentence, section 9, section 10's table, the PR 6, PR 8, PR 9a and PR 9b sections, the PR 10 clause naming what the seed stands in for and the command shipped here, the PR 11 line, the testing summary and the review checklist); earlier items keep their wording, as item 22 requires. With the rebase onto #666's head (2026-11-03): the jobs and the reconciler are not gated by the PostgreSQL serving registration, and 4(e) says what they do in the registration's retry window, GitHub issue #755 being the gate; `backup purge`'s own cycle under the SQLite serve lock fails a dead store-backed restore and removes its target rather than resuming it (4(f)), since the command tells the cycle it holds the claim instead of the cycle reading it through a zero window; the PR 9a load paragraph gives PostgreSQL's batch as 500 rows or 4 MiB, as 5.3 step 5 does; and MongoDB's snapshot open waits, at most 2 s, for the majority-committed snapshot to show a table a `local` read shows ACTIVE, so a store-backed CreateBackup sent as soon as DescribeTable says ACTIVE is not refused as a table that does not exist. With the rebase onto `main` (2026-11-04), after #335, #666 and #681 merged: `restore_progress` is migration 7 (4(i)), declared not applicable in the MongoDB and Cassandra registries #681 added, as on SQLite; 4(e) cites the module doc of `crates/server/src/registration.rs` for what a lost serving registration does, which replaces this item's 2026-11-03 clause on it: the reconciler stops with the server's workers and spawns no resume once its own token has fired, and the running jobs go on, #755 being the gate for them; a restore job joins PR 8's job drain as a backup job does; and `remove-target` releases its serving registration when it exits and is named on MongoDB and Cassandra as on PostgreSQL. With the move onto `main` at e8ca7df3 (2026-11-04), after #424 merged: the Cassandra write path's rule for an unchanged index row (rewritten by its INSERT alone, #742) and its version stamps on every index row are #424's, so this PR no longer changes `sync_indexes` or the GSI worker; the PR 9a load paragraph, 5.3 steps 5 and 6 and this item cite #424 for what a re-applied batch relies on, and a restored item's index rows carry its base row's `version` as #424's write path gives them (`put_restored_item` is that path). After the second review round (2026-11-05): the Cassandra load no longer goes through `put_restored_item`'s lightweight transaction but writes each item's base and index rows in one logged batch `USING TIMESTAMP` a reading taken before the batch's claim check (`put_restored_item_at`), the fence against a batch a stalled writer lands after a takeover (the PR 9a load paragraph); the index rows still carry the base row's `version`, which is now that timestamp. A resume with no recorded progress fails `target_not_empty` only when the target holds more items than the backup's row counts; at or below the total it re-applies the first file with upserts (step 7), so a Cassandra crash before the first progress record resumes to ACTIVE. One instance's sixth takeover of a record fails it (`RESTORE_TAKEOVERS`, step 7). `restore_target_ready` on Cassandra checks each GSI's and LSI's table as well as the data table. The nightly workflow runs every restore kill point of every backend in four jobs (the testing summary); the kill-point script's first-reach cells assert ACTIVE with the exact count and its `--remove-target` flag is gone (the `target_not_empty` state is seeded by `tests/test_backup_job.py`'s `remove-target` cell). + +34. (2026-11-04, PR 10 as implemented, #746.) PR 10 is split in three: #746 ships plain `backup sync`, `--prune-missing`, `--adopt-store`, the `sqlite-memory` startup sync and `BackupCatalog::insert_backup_row`; #820 carries `--move-store`, `--forget-store` and `--forget-backup` with `list_backup_rows_by_root` and `rewrite_store`, which rewrite or delete rows of other stores, and the full report of the rows of other roots; #821 carries `migrate-legacy`, whose data path (the frozen reader, `DataFileEncoder`, the item validator) shares nothing with sync but the insert; `remove-target` shipped with PR 9 (item 33). The PR 10 section's line, the PR 9b paragraph (the matrix now adopts with `backup sync`) and the doc-row table's `migrate-legacy` row say so (PR 9b, PR 10, section 10). Neither the Cassandra sweep of orphan `backup_index_by_status` entries nor the MongoDB sweep of the data collections a failed restore-target removal leaves is in any of the three: the code comments that gave them to `backup sync` now say no sweep removes them yet, and the MongoDB error line names the collections for an operator to drop. `insert_backup_row(row) -> bool` inserts the row as given, unowned (`owner_generation` 0), with `completed_at` and `heartbeat_at` on the database's clock, and answers `false` when a row with its key exists: PostgreSQL `INSERT ... ON CONFLICT DO NOTHING` with no conflict target, so both unique indexes (the key and `backup_id` alone) refuse it, on the pool and with no advisory lock; SQLite `INSERT ... ON CONFLICT DO NOTHING` under the write lock and `BEGIN IMMEDIATE`, without a job slot; MongoDB the claim's pipeline upsert filtered on `owner_generation: {$exists: false}` with the row's status, `owner_generation: 0`, and E11000 (the unique `backup_id`) meaning a row exists; Cassandra the `backups_by_arn` lightweight transaction `IF NOT EXISTS`, then the LOCAL_QUORUM batch of the two list rows and the `backup_index_by_status` entry under the row's status (the batch `claim_backup` wrote inline, now shared), an unknown outcome or a `NotMatched` answer re-read at `LOCAL_SERIAL` and settled as inserted when the row read is this one (its entries then written, which completes an insert that stopped between its two writes), as existing when it is another; no row after a lost answer is sent once more, and no row after `NotMatched` answers `false` (a legacy in-catalog backup holds the ARN); a failed batch of the entries is sent once more before the error. The row is AVAILABLE from the transaction on, so a DeleteBackup between the transaction and the batch moves a status entry the batch has not written yet, and the batch then writes an AVAILABLE entry no later write removes; every reader of the status table re-reads the row and skips it, one wasted read each, until a sweep of orphan entries exists. Cassandra has no `backup_id` uniqueness, so its insert-if-absent is per ARN spelling: the same id under another region is another row there, as the trait doc says, and sync closes the gap itself by reading the row under both spellings of the manifest's ARN (this server's region and the manifest's) before the insert and skipping a backup the other spelling has a row for, while a row under the manifest's own spelling is left to the insert (which answers `false` for it or, on Cassandra, completes its own torn insert), so through sync no account holds two rows of one backup id under the manifest's account, table and id on any backend (section 3, 4); on Cassandra a row of that id under another table that the rows read did not return (a row of another root, or one whose list entries are missing) is not seen by that read, and within one run the row check keeps a second prefix of an id the run adopted out. Sync reads the configured root's rows first, every status and every store id the catalog records for the root (`read_store_ids`, then `list_backup_rows` per id), keeping of each only the status, store id and table by account and backup id and the key of each AVAILABLE row of the configured store id (the backends collect the rows whole first, so the read peaks at about 22.5 KiB per row of a five-GSI table, section 7), and checks each listed prefix against that set before it reads anything under the prefix; the sync contract's "checks each prefix against the catalog as the listing reaches it" holds with the rows read once up front, and the insert-if-absent then decides against a CreateBackup that landed after the read and against rows of other roots. The listing is consumed as it streams: the store lists in key order, so a prefix is complete once the listing passes it, and sync holds that prefix's objects and the prefix of each manifest it listed; the filesystem store's own listing is the larger cost, measured in section 7. A listing that fails part-way keeps the rows inserted before it, does not read the prefix it was in, and makes no missing-manifest report. An insert that answers `false` is re-read to say which row it met: one that landed after the read, one of another store, account or table holding the backup id, or on Cassandra a row whose list entries are missing (a CreateBackup's claim that stopped between its two writes; sync's own insert that stopped there is completed by the next sync of the prefix). The prefix check is the dead-owner duty's: `reconciler::verify_prefix` reads the manifest and sidecar within their bounds, checks the digest and the reader's acceptance and every listed data file at its size from the listing sync already holds; the dead-owner probe keeps its comparison of the whole ARN, and sync compares the manifest's account, table and backup id with the path, adopts an ARN of another region, logging it (requests resolve it by the ARN as written), and skips one of another partition than `aws`, since every request refuses such an ARN before it reads a row. `layout.rs` gains `EXPORTS_COMPONENT`, `CHANGELOG_COMPONENT`, `classify_store_key` and `parse_store_prefix`: keys outside every backup prefix are passed over, the reserved components and a third component of another id shape (only the 14-digit shape carries a manifest) are reported as skipped and never read. The adopted row carries the manifest's name, ARN, size and item total and `source_table`, the id's instant as `created_at`, and the configured store's identity (section 4, PR 10). Rows of other roots are reported in part: sync prints the prefixes whose backup id a row of another root holds (the read under the ARN's spellings, or the insert's re-read). The rows of the configured root under another store id are not a case sync meets: a store whose rows carry another id than its file, or more than one id, is unhealthy, and sync refuses an unhealthy store. A listing of every root's rows needs `list_backup_rows_by_root`, which is #820's, so until then 4(g)'s "rows for another store are reported" holds for that one case (4(g)). Every sync takes the maintenance claim where it runs beside servers (PostgreSQL, MongoDB, Cassandra), plain sync with no wait for running jobs: its insert-if-absent orders it against a CreateBackup, and the claim orders it against `purge`, whose `purge_row` removes a dead CREATING row before it deletes the prefix, which can hold a complete manifest (a job that died after its manifest); a sync between the two steps would adopt bytes purge then deletes. While the claim is held the jobs refuse, as during purge (4(f)), and a sync killed while it holds the claim keeps the jobs of every instance refused until its heartbeat is `stale_job_after_secs` old. The restore's line for a row whose prefix lacks its sidecar says to put the prefix back; only a missing manifest names `--prune-missing`, the one case it removes. On SQLite it opens the catalog as `serve` does, under the serve lock (exit 6 beside a server, the refusal purge gives), after the dead-owner resolutions purge runs, so a DELETING row a stopped server left is finished and a dead CREATING row resolved before sync reads the rows. `--prune-missing` and `--adopt-store` also wait for running jobs as purge does; every sync installs purge's signal handling, checks it before each prefix and exits 1 "stopped" on a signal, releasing the claim it holds and its serving registration. `--prune-missing` removes each AVAILABLE row whose manifest is gone outright, through `BackupCatalog::forget_available_backup_row` (added here on the four backends: under the row's lock as `start_backup_delete` takes it, the status check and the in-use rule with a delete of the row, and on Cassandra its index entries after it), and deletes nothing in the store. The manifest probe sees only the newest AVAILABLE row's manifest, so a host-local root that holds the newest backup passes it while it lacks older ones; with no tombstone left, no deletion duty on the host that holds those bytes acts on them, but the row is gone for every instance: the backup is BackupNotFound on every instance until a sync on the host that holds the bytes adopts it again, which is the host-local acceptance line as amended here. The objects a removed row's prefix still holds stay until an operator deletes the prefix by hand: `purge` deletes only the prefixes of rows, and every later sync reports the prefix as not a complete backup. `--prune-missing` exits 1 when it leaves a row a later run may remove (a store or catalog call failed, the manifest's `head` included, which keeps the row; or a restore is reading the backup, "run the command again once the restore ends") and 0 when the rows it leaves have their manifest back or were moved by another writer. An unhealthy or `checking` store refuses plain sync and `--prune-missing`, exit 1, naming the problem and changing no row. Sync exits 1 when a prefix was skipped for a store read or catalog call that failed (the run is incomplete; the next run adopts it), and a startup sync that met one stays pending, its skips logged at WARN. So when the newest AVAILABLE row's own manifest is gone the store is unhealthy and neither command runs; the way out in this release is to put the prefix back (#820's `--forget-backup` is the other). The serving registration is named `extenddb backup sync`, and the catalog refusals exit 6 and 3 as the other commands' do (4(f), PR 10). `--adopt-store` is `store_identity::adopt`: with the claim held it rewrites `extenddb-store.json` with a plain `put`, the file's `catalog_id` and `backend` changed and its `store_id` and `created_at_ms` kept, reads it back, and syncs with the adopted identity; it logs an `extenddb::audit` intent line before the put and the audit line after the re-read with its outcome (adopted, or race lost), so a put whose caller stopped waiting is on record too. The re-read detects another adopter's put that landed between this put and the re-read (the store trait has no conditional replace); the command reads the file once more after the sync and fails naming the owner when it changed, which detects one that landed during the sync; a put after both reads is found by this catalog's next check. It refuses when this catalog's rows for the root carry another store id, and on a root this catalog owns it says so and syncs. A later check of the previous owner that can read its catalog finds the file naming another catalog, at the earliest one reconciler interval after its cycle in progress ends; until then it acts on the root, except where its deletion fence stops it. `StoreFence` re-reads the file (one `get` through the store's bounded calls, under its check bound and its one call slot on a filesystem root) right before each `delete_prefix` of a backup's objects (`finish_backup_delete`, which DeleteBackup and the reconciler's DELETING duty run, the backup job's failure path, the FAILED-prefix and DELETED-prefix duties, purge), before DeleteBackup's DELETING write and before purge's removal of a dead CREATING row or its DELETING write, before the reconciler removes a tombstone, and before a CreateBackup's row and first put; when the file names another catalog it refuses, records `ForeignCatalog` so the store is unhealthy at once (and a file naming this catalog under another store id than the store's check settled, a root replaced by another root of the catalog, is refused and recorded as `IdMismatch`; DeleteBackup and CreateBackup answer the unhealthy store's `ValidationException`, and the cycle re-tests the store's health after each store duty and runs none after the refusal, the dead-owner duty included), and a file it cannot read, or a root with no file, refuses the call and leaves the health to the next check, except at DeleteBackup's read before the DELETING write, where it is a store failure: the request writes DELETING and answers DELETED, as for a deletion the store does not answer (revision item 30), and the read before the deletion refuses it. A refusal before the row is changed leaves it as read, so DescribeBackup does not answer a refused DeleteBackup with DELETED; one at the read before the deletion (an adoption between the two reads) leaves the row DELETING with nothing deleted and DeleteBackup answering DELETED (revision item 30), which this catalog's reconciler finishes only if this catalog owns the root again. The restore deletes nothing in the store, so it has no fence, and neither has `--prune-missing`'s removal of a row, which deletes no object. A write already past the fence's read lands, so every server of the catalog the file names is stopped before an adoption, and a new in-memory server is started only after the old one has exited; the admin guide, the refusal and the command's output say so (4(c)). For `backup sync` a defaulted SQLite root whose file names another catalog while this catalog has no rows for it is configured and unhealthy (`Caller::SyncCommand`) instead of the warning that leaves it unconfigured (item 29), so `--adopt-store` can adopt the root a catalog re-created by `destroy` and `init` finds; the server and purge keep the warning (4(c), section 7, PR 7). The identity file gains `ephemeral_catalog` (`#[serde(default)]`, so the format stays version 1 both ways), written `true` by a server whose SQLite database is in memory (`database_file()` names no file: the state, not the `sqlite-memory` cargo feature) and `false` by every other writer, an adoption included. The adoption of a root an in-memory catalog owned is part of `store_identity::establish` when this catalog is itself in memory and the file carries the marker, and never of `recheck`, so two in-memory servers on one root do not take it from each other each cycle; a root whose file names a durable catalog stays unhealthy for an in-memory server, as 4(c) requires. The startup sync runs on every start of an in-memory catalog with a store configured, not only after an adoption (on a fresh root it is one listing): for a filesystem store in `serve` before the routes serve; for an S3 store, and for a startup sync not finished, in the reconciler's cycle after a healthy check. Every run is bounded at 60 s (`sync::STARTUP_SYNC_TIMEOUT`), in `serve` and in the cycle, where the duties after it wait at most that bound; a run that stops before it listed the whole root, reaches the bound (the rows it inserted stay, but every run lists the root from its first key, so the sync completes only while listing the whole root fits in the bound; past it the sync stays pending and is retried every cycle), or skipped a prefix for a store or catalog error stays pending. On a filesystem store a run abandoned at the bound does not stop its listing (the walk has no cancel flag): it runs to its end on the blocking pool, holding the listing it builds, and the next cycle starts another walk; and `serve` answers nothing, `/health` included, until the startup sync before the routes ends, while a signal in that time ends the process at once. Until a run finishes, requests for the earlier backups answer `BackupNotFoundException`, `/health` possibly already healthy; the log line that reports the adopted count ends that window, and for a prefix that line reports as having met a store or catalog error, a later cycle's line does. It takes no claim: the process is the catalog's only user (4(c), PR 7, PR 10). Every `extenddb backup` command refuses an in-memory SQLite database before it opens a catalog, exit 1, with a sentence saying why (a location that does not parse is reported with its parse error instead, as `serve` reports it): it would open a new, empty catalog that ends with the command, and a writing command would hand an adopted root to a catalog nothing can open again; this is the sync contract's "the backup CLI commands do not exist there" (PR 10). The matrix (`devtools/backup-restore-matrix`) copies the source's prefix under the target's root and runs `extenddb backup sync` on the target (on SQLite with the target's server stopped) in place of the hand-written row, so a cell proves the adoption and the restore; `--adopt-store` points the target's store at the source's root, adopts it and checks that the source's server then reports the store owned by another catalog, and the nightly runs that cell from SQLite to PostgreSQL and back (PR 9b). 4(c), 4(e), 4(f), 4(g), decision 6, section 7's memory paragraph and the PR 10 contract are edited with this item. diff --git a/docs/design/README.md b/docs/design/README.md index f9806b336..f0ee934ed 100755 --- a/docs/design/README.md +++ b/docs/design/README.md @@ -20,6 +20,8 @@ This directory contains the authoritative design documents for ExtendDB (extendd | 12 | [Auth / Authz Cache](12-auth-authz-cache.md) | In-process cache for IAM data (credentials, policies, tags, table key info); companion to 05 and 06 | | 13 | [MongoDB Storage](13-storage-mongodb.md) | MongoDB storage backend: engine and catalog traits over the official driver, replica-set transactions | | 15 | [Backup and Restore](15-backup-restore.md) | Overview of the off-host backup series: the invariants, the per-operation contracts and a short section per later PR; the format, store and snapshot documentation lives with their crates (`crates/core/src/backup_format`, `crates/backup-store`, `crates/storage/src/snapshot.rs`) | +| 15b | [Backup memory and duration record, PR 8](15-backup-restore-bench-pr8.md) | The measurements section 7 and revision item 31 of the design cite: peak RSS and duration of the 100k, 1M, worst-expansion and incompressible backups per backend, the allocator series, the PostgreSQL page's cost | +| 15c | [Restore memory and duration record, PR 9](15-backup-restore-bench-pr9.md) | The measurements section 7 and revision item 33 of the design cite for the restore: peak RSS and duration of the 100k, 1M, worst-expansion and large-item restores on SQLite and PostgreSQL, with and without one allocator arena, the terms the code bounds and the sizing sentence | ## How to Use These Docs diff --git a/docs/differences-from-dynamodb.md b/docs/differences-from-dynamodb.md index 0686d8445..077062530 100755 --- a/docs/differences-from-dynamodb.md +++ b/docs/differences-from-dynamodb.md @@ -91,11 +91,11 @@ from a request sent to that service or backend. | Vector index update propagation | Eventually consistent, the same model as a GSI | Matches DynamoDB. Maintenance is queued on the same propagation queue as async GSIs, so a search immediately after a write may not see it. Governed by the same `index_propagation_delay_ms` setting; unlike a GSI there is no per-index override. A value of 0 applies maintenance inline in the write's own transaction, which is stricter than the service and exists so a test can assert steady state without waiting. Inline additionally requires the table's propagation queue to be empty: while rows queued during an index build are still draining, a new write queues behind them instead of overtaking them, so a brief search-after-write window exists even at 0 until the queue drains. That window is ordering, not data loss; the write is applied, in order, by the queue worker. That zero behaves differently while an index is still building, and the two backends differ: **PostgreSQL** applies inline only to an index that is already `ACTIVE`, and defers a write to a building index whatever the delay says, because the write must not reach the index ahead of the backfill's older snapshot of the same item; **SQLite** does not check the status on this path, so with a zero delay a write during a backfill is applied inline and bypasses the hold that keeps the two ordered. The SQLite behaviour is tracked as a defect (F-20), not intended, and it is reachable only with the zero delay, which is a test setting. | | `SearchVectors` score at extreme magnitudes (PostgreSQL backend only) | Returns the true distance as a number for any vector of finite components | The score is bounded to a finite value instead, and the bound is not a measured service answer. pgvector accumulates distances in single precision, so magnitudes far below `f32::MAX` overflow inside the extension: Euclidean above about 9.2e18, dot product above about 1.8e19, and cosine at both ends, above about 1.8e19 and below about 3.7e-23, which is where a component's single-precision square rounds to zero. A non-finite score cannot be serialised as JSON at all, so the result is bounded in SQL: `1e308` for an overflowed distance, `-1e308` for an overflowed negated inner product, which the score contract negates so a client sees `1e308` in `Score`, and 1.0 for a cosine that comes back NaN. Ranking is unaffected at the overflow end, because each bound sits at the end its metric overflows towards, so the farthest row stays farthest and the most similar stays most similar; two rows that both overflow tie, and the tie breaks on the base key. At the underflow end cosine loses resolution rather than being bounded, and which side is tiny decides how much. For a tiny **query** vector, its norm underflows while the inner product usually does not, so the quotient is an infinity that pgvector clamps and the reported distance collapses to one of 0, 1 or 2 following the sign of the inner product, with the 1.0 substitute firing only when the vectors are exactly orthogonal and the quotient is therefore 0/0 (measured). For a tiny **stored** vector it is worse: the stored norm is computed in single precision and reaches zero, so the zero-vector guard fires and that row reports 1.0 at every angle, parallel included. A corpus of tiny embeddings therefore loses ranking altogether rather than losing resolution. For a tiny query vector, by contrast, ranking still separates nearer-than-orthogonal from farther and loses resolution only within each half. The SQLite backend owns its own arithmetic, computes in double precision, and reports the true value, which is why this row is scoped to PostgreSQL. | | Vector index deletion window | `UpdateTable` Delete leaves the index in `DELETING` long enough to observe, then removes it | No observable `DELETING` window on either backend: the catalog row is removed inside the `UpdateTable` transaction, so a `DescribeTable` immediately afterwards already omits the index. The index's data table is dropped after that commit, in a separate transaction, and the two backends handle a failure there differently. **PostgreSQL** treats it as best effort, because the data table lives in a different database entirely: a failure is logged and skipped rather than failing the request. **SQLite** propagates it, so a failed drop returns an error from an `UpdateTable` whose catalog change has already committed, which is the more surprising outcome of the two: the index is gone from the catalog and the caller saw a failure. So an operator debugging a leftover `_ddb_vec_*` table should look for that warning rather than assume the delete was incomplete. | -| Backing up and restoring a table that has vector indexes | Restores the table with its vector indexes intact: the configuration survives, items keep their vector attributes, and `SearchVectors` works as soon as the table is `ACTIVE` (measured) | Not supported. On PostgreSQL and SQLite, the two backends with vector indexes, `CreateBackup` of such a table is refused with `ValidationException` "table has vector index(es); restoring a table with vector indexes is not supported by this storage backend", and no backup is created, because a restore cannot carry the index data across. A backup of a vector table taken before catalog 0.0.4 is refused at restore on PostgreSQL with the same message naming the backup; on SQLite such a backup restores without its vector indexes. A backup of a table with no vector indexes restores normally. | +| Backing up and restoring a table that has vector indexes | Restores the table with its vector indexes intact: the configuration survives, items keep their vector attributes, and `SearchVectors` works as soon as the table is `ACTIVE` (measured) | Partial. With a configured `[backup.store]`, `CreateBackup` of such a table succeeds on PostgreSQL and SQLite, the two backends with vector indexes: the backup is written to the store and its manifest records the index specifications, and `DescribeBackup` and `ListBackups` report it as any other backup (wire test on both backends). Without a store (the section absent on PostgreSQL, or `kind = "disabled"`), where `CreateBackup` writes the in-catalog format, it is refused with `ValidationException` "table has vector index(es); restoring a table with vector indexes is not supported by this storage backend" and no backup is created, because that format cannot carry the index data across. `RestoreTableFromBackup` of a store-backed vector backup restores the table with its vector indexes into PostgreSQL and SQLite, the two backends with vector support (`SearchVectors` answers as soon as the table is `ACTIVE`; matrix cells), and is refused into MongoDB or Cassandra with `ValidationException` "backup has vector index(es); restoring a table with vector indexes is not supported by this storage backend" before any target table is created. A backup of a vector table taken before catalog 0.0.4 is refused at restore on PostgreSQL with the same message naming the backup; on SQLite such a backup restores without its vector indexes. A backup of a table with no vector indexes restores normally. | | `SearchVectors` endpoint | Served only on `search-dynamodb..amazonaws.com`. The standard `dynamodb..amazonaws.com` endpoint answers the same request with HTTP 400 `UnknownOperationException` ("This operation is not supported by this endpoint"); every control-plane and item operation stays on the standard endpoint. Signing is unchanged either way (service name `dynamodb`, target prefix `DynamoDB_20120810`) | Served on the same endpoint as every other operation, so a client is pointed at one ExtendDB endpoint for all of them. Two consequences worth knowing: an SDK that resolves a separate search hostname from its endpoint ruleset needs its endpoint overridden to reach ExtendDB, and ExtendDB does **not** reproduce the service's refusal, so a test asserting `UnknownOperationException` for vector search on the base endpoint passes against Amazon DynamoDB and fails here. | | `SearchVectors` result order for equal distances | Measured unstable: three identical searches returned tied rows in three different orders, and a top-k that truncates a tie group keeps an arbitrary subset of it | Deterministic on both backends, by different means. **PostgreSQL** sorts explicitly on the base table's full primary key after the score, so the order is a property of the query. **SQLite** issues no `ORDER BY` and resolves ties by scan order through a stable top-k, so its order is a property of the plan rather than something the query guarantees. Either way a client re-issuing an identical search sees the same order, which is stricter than the service rather than divergent in outcome. Do not rely on the two backends agreeing on which subset of a truncated tie group they keep. | | Vector indexes on the MongoDB backend | Vector indexes and `SearchVectors` are available | Not supported at all. That backend provides no vector search implementation, so the engine's capability gate refuses `CreateTable` and `UpdateTable` carrying `VectorIndexes` and every `SearchVectors` request, with the same capability message a PostgreSQL deployment without pgvector returns. Every other vector row in this document describes the PostgreSQL and SQLite backends. | -| Multi-part base table keys | Not supported | Preview extension (opt-in via `enable_multipart_keys` setting). Standard single/composite keys work identically. | +| Multi-part base table keys | Not supported | Preview extension (opt-in via the `limits.allow_multipart_table_keys` setting). Standard single/composite keys work identically. | | GSI throughput on a `PROVISIONED` table | Every GSI needs `ProvisionedThroughput` of at least 1 read and 1 write unit | A GSI `ProvisionedThroughput` below 1 (including 0/0) is `ValidationException` on `CreateTable` of a `PROVISIONED` table and on `UpdateTable`'s GSI `Create` ("ReadCapacityUnits and WriteCapacityUnits must both be greater than or equal to 1 for index: "; the service's text was not captured). A GSI with no `ProvisionedThroughput` is still accepted on a `PROVISIONED` table, and `UpdateTable` to `PROVISIONED` leaves an on-demand table's GSIs without one; see the `BillingModeOverride` row under Backup and Restore for how a backup of such a table restores | | Projected attributes across all indexes | At most 100 | Enforced on `CreateTable` (and so on `RestoreTableFromBackup`, which validates its target the same way) and on `UpdateTable`'s GSI `Create`: every `NonKeyAttributes` entry of every GSI and LSI is counted, so an attribute projected into two indexes counts twice. Over 100 is `ValidationException` ("Number of projected attributes in all indexes exceeds limit of 100, projected: "; the service's text was not captured) | | Index key validation on `UpdateItem` and transaction writes | A secondary-index key attribute must carry its declared scalar type and may not be empty. On `UpdateItem` a literal operand of the wrong type (`SET k = :v`, `ADD`, `DELETE`, `list_append` of literals) is refused on the statement, ahead of the `ConditionExpression`, naming the attribute and the index; a value computed from the item (an attribute reference, the fallback of `if_not_exists`) is refused after the condition passes with the message "The update expression attempted to update the secondary index key to unsupported type"; a key left empty is refused after the condition passes with the update-expression wording. In `TransactWriteItems` a wrong-typed key (table or index) on any member cancels the transaction with the first such member's `ValidationError` in position and `None` elsewhere; otherwise an empty index key on any member is a top-level `ValidationException`; otherwise members report their own conditions. On a `Put` member both checks come ahead of the member's condition. (Measured on the service.) | Matches on every backend. Two known shared differences: an empty index key supplied through the legacy `AttributeUpdates` parameter is reported with the update-expression wording where the service uses the `PutItem` wording naming the index; and a request whose own value exceeds 400 KB is refused for size by the service before any index-key check, where ExtendDB reports the index-key fault first. | @@ -107,31 +107,35 @@ Behaviour stated as measured, and every number below, was observed with release | Area | DynamoDB | ExtendDB | |------|----------|------| -| What a restore carries across | Key schema, attribute definitions, items, secondary indexes, billing mode and provisioned throughput, on-demand throughput (table and GSI level), SSE settings. Table class is not carried: a restore is `STANDARD` whatever the source's class (measured). Stream settings, TTL, tags, auto scaling, IAM policies, and CloudWatch settings must be reapplied by hand | The same set on all four backends, with one gap: the table's on-demand throughput is carried, a GSI's is not (ExtendDB records no per-index on-demand maximum, and `OnDemandThroughput` is not a member of its GSI definition). Table class is not restored: a `STANDARD_INFREQUENT_ACCESS` source restores as `STANDARD`, as on the service. Streams, TTL, tags, and deletion protection are not restored. A backup written before catalog 0.0.4 (PostgreSQL, SQLite, Cassandra) has no recorded table definition and restores with its keys, items, and billing mode but no secondary indexes. A provisioned one restores with 5/5 throughput on PostgreSQL and SQLite and with its recorded throughput on Cassandra | -| How `RestoreTableFromBackup` runs | Returns at once with the target `CREATING`; the service copies in the background | The same on all four backends: the response carries the target `CREATING`, and the items are copied by a background task in the server, so a client that times out or disconnects does not stop it. Once the request has been authenticated and authorized, the rest of it, from reading the backup to registering the restore, runs in a task of its own, so a client that disconnects after that does not cancel it either: the restore runs and the client finds the target with `DescribeTable`. A client that disconnects while the request is still being authenticated or authorized cancels it, and nothing is created *(from the code)*. Measured with a client whose 1 ms read timeout closed its connection before any response arrived: on all four backends the restore was registered and finished. Measured with a 60,000-item backup of a table with no secondary indexes: response in 0.032 s (PostgreSQL), 0.012 s (SQLite), 0.053 s (MongoDB), 0.151 s (Cassandra); `ACTIVE` after 1.5 s, 2.1 s, 3.6 s, and 11.2 s. On Cassandra the response waits for the target table's schema to be created: 0.75 to 1.3 s with five restores running at once *(earlier build)*, which is longer than a client read timeout of 1 s. A restore runs inside the server process; if the process stops, see the abandoned-restore row | +| What a restore carries across | Key schema, attribute definitions, items, secondary indexes, billing mode and provisioned throughput, on-demand throughput (table and GSI level), SSE settings. Table class is not carried: a restore is `STANDARD` whatever the source's class (measured). Stream settings, TTL, tags, auto scaling, IAM policies, and CloudWatch settings must be reapplied by hand. `DescribeTable` on a freshly restored table reports `ItemCount` 0 and `TableSizeBytes` 0 (measured on 15 restored tables, one of them ten minutes after `ACTIVE`) | The same set on all four backends, with one gap: the table's on-demand throughput is carried, a GSI's is not (ExtendDB records no per-index on-demand maximum, and `OnDemandThroughput` is not a member of its GSI definition). Table class is not restored: a `STANDARD_INFREQUENT_ACCESS` source restores as `STANDARD`, as on the service. Streams, TTL, tags, and deletion protection are not restored. `DescribeTable` on a restored table reports `ItemCount` 0 and `TableSizeBytes` 0 when the table goes `ACTIVE`, as the service does (measured: 0 and 0 on 15 restored tables, one of them ten minutes after it went `ACTIVE`), until the statistics pass next rewrites both from the restored rows *(from the code)*. A backup written before catalog 0.0.4 (PostgreSQL, SQLite, Cassandra) has no recorded table definition and restores with its keys, items, and billing mode but no secondary indexes. A provisioned one restores with 5/5 throughput on PostgreSQL and SQLite and with its recorded throughput on Cassandra | +| How `RestoreTableFromBackup` runs | Returns at once with the target `CREATING`; the service copies in the background | The same on all four backends: the response carries the target `CREATING`, and the items are copied by a background task in the server, so a client that times out or disconnects does not stop it. Once the request has been authenticated and authorized, the rest of it, from reading the backup to registering the restore, runs in a task of its own, so a client that disconnects after that does not cancel it either: the restore runs and the client finds the target with `DescribeTable`. A client that disconnects while the request is still being authenticated or authorized cancels it, and nothing is created *(from the code)*. Measured with a client whose 1 ms read timeout closed its connection before any response arrived: on all four backends the restore was registered and finished. Measured *(in-catalog restore, an earlier build)* with a 60,000-item backup of a table with no secondary indexes: response in 0.032 s (PostgreSQL), 0.012 s (SQLite), 0.053 s (MongoDB), 0.151 s (Cassandra); `ACTIVE` after 1.5 s, 2.1 s, 3.6 s, and 11.2 s. Measured *(store-backed restore, this release)* with the same shape (60,000 items of 200 bytes, composite key, no secondary index) from a filesystem store on loopback: response in 0.051 s (PostgreSQL), 0.016 s (SQLite), 0.055 s (MongoDB); `ACTIVE` after 0.67 s, 1.85 s and 3.74 s (on MongoDB the job's DONE at 1.8 s and the index backfill worker's flip after); 1M items of 200 bytes in key order (the server's own backups) `ACTIVE` after 9.5 s on PostgreSQL and 26.8 s on SQLite, the same items as an unordered backup (a MongoDB source, a multi-part-key table) after 17.5 s and 66 to 90 s (`docs/design/15-backup-restore-bench-pr9.md`, which has the host and its load); the store-backed response includes the sidecar and manifest reads and its `ACTIVE` time the download and the per-file verification, and Cassandra as a store-backed target was measured at 12,000 items only (the kill points: a resumed restore of 12,000 items of 4,000 bytes in two data files reached `ACTIVE` 8.3 s after its takeover on a single-node container). On Cassandra the response waits for the target table's schema to be created: 0.75 to 1.3 s with five restores running at once *(earlier build)*, which is longer than a client read timeout of 1 s. On every backend the request waits at most 60 s (`RESTORE_REGISTRATION_TIMEOUT`) for its background task to register the `CREATING` target of a store-backed restore (a bound against a connection pool that has stopped answering; the manifest and sidecar reads that precede the registration are not under it); past that it answers `InternalServerError` while the task goes on, so the target can appear after the error, holding the backup in use (`BackupInUseException` to `DeleteBackup`) and the name until the restore ends; `DescribeTable` is where a client finds it *(from the code)*. Because that 60 s bound equals botocore's default read timeout, a client whose first attempt times out while the registration is still in flight (a backup's `backup_index` row held under a slow lock) retries: a retry that names the same backup and target waits for the in-flight registration and is answered its `CREATING` description rather than registering a second target, and a restore of a *different* backup into the same name while the first is in flight is `TableInUseException` *(from the code)*. The wait is per server process: each server keeps its own list of in-flight registrations, so a retry waits for the first attempt's registration only when it reaches the server that holds it; a retry that reaches another server behind a load balancer finds the name free until the registration commits and then `TableInUseException` (the registration took the name) or, once the target exists, the same `CREATING` description from the catalog *(from the code)*. The job permit is the last of the request's checks: a restore that finds every `[backup] max_concurrent_jobs` slot held (default 4; backups and restores together, per server process) waits up to 30 s for one and is then `LimitExceededException` ("Too many backup and restore jobs are running on this server (at most N at once, the [backup] max_concurrent_jobs bound); retry later"), creating nothing, where the service accepted five restores sent together (measured on DynamoDB); on the store-backed path the permit is taken before the manifest is read, after the catalog row's state, the target name and the backup's in-use rule are checked, in that order (a restore of a backup into its own CREATING target is `TableInUseException`, measured on DynamoDB). A restore runs inside the server process; if the process stops, see the abandoned-restore row | | `RestoreSummary` | The restore response and `DescribeTable` while `CREATING` report `SourceBackupArn`, `SourceTableArn`, `RestoreDateTime` (the backup's `BackupCreationDateTime`, not the time the restore ran), and `RestoreInProgress=true`. Once the table is `ACTIVE`, `DescribeTable` omits `RestoreSummary` | The same on all four backends. No response that describes a table (`CreateTable`, `UpdateTable`, `DeleteTable`, `DescribeTable`, `RestoreTableFromBackup`) carries `RestoreSummary` unless the table is `CREATING`; what the service's `UpdateTable` returns for a restored table was not measured | | Reads on a restore target | A `GetItem` or `Scan` on the target while `CREATING` is `ResourceNotFoundException`; the first read after `DescribeTable` reports `ACTIVE` succeeds (measured) | The same on all four backends. The server's key-info cache stores a negative entry (5 s TTL by default) for a name the catalog does not have, and does not store one for a name the catalog has in a state other than `ACTIVE`: on a not-found answer it asks the catalog once more with `DescribeTable`, and authorization records the name as missing in the key-info snapshot it hands the engine, so the engine does not look it up again. A `GetItem`, `PutItem`, `UpdateItem`, `DeleteItem`, `Query`, or `Scan` on the target while it is `CREATING` therefore costs exactly one `table_key_info` and one `describe_table` catalog load, uncached, per request (on SQLite five SQL statements: one for the key info, four for `DescribeTable`; on PostgreSQL `describe_table` is two queries), plus one `describe_table` on the first such request for the throttle registry; the first read after `ACTIVE` loads once more and is then cached. Measured on SQLite with a `GetItem` loop on a `CREATING` restore target of 100,000 items: 40 requests refused, 40 key-info and 41 `DescribeTable` loads in the server's statement log; with the hand-off to the engine removed, 80 and 81. In an earlier run of the fidelity test the first `GetItem` after `ACTIVE` answered in 3 ms after 72 refused reads. | | `RestoreTableFromBackup` overrides | Current request members are `BillingModeOverride`, `ProvisionedThroughputOverride`, `GlobalSecondaryIndexOverride`, `LocalSecondaryIndexOverride`, `SSESpecificationOverride`, `OnDemandThroughputOverride`, and `VectorIndexOverride`; DynamoDB applies them to the restored table | See the overrides table below. A refused override is refused before the target table is created | | Members of the `RestoreTableFromBackup` response | The `CREATING` description omits `DeletionProtectionEnabled`, the table's `ProvisionedThroughput.NumberOfDecreasesToday`, and each GSI's `IndexStatus` | The same on all four backends; `DescribeTable` reports the first two. Each GSI's `ProvisionedThroughput` keeps `NumberOfDecreasesToday`; what the service sends there was not measured | -| Restoring into a name that exists | `TableAlreadyExistsException`; into a name that is itself a `CREATING` restore target, `TableInUseException`: "Table: is already being restored from backup: " for the same backup, and "Attempt to change a resource which is still in use: Table: is being used." for a different one (both measured) | `TableAlreadyExistsException` on all four backends for an existing table; `TableInUseException` for a name that is a `CREATING` restore target, whichever backup is restored, always with the same-backup text naming the backup the existing target is being restored from (the service's text for a different backup is not reproduced). Each backend checks the name before the backup and before a restore slot is taken, so the same answer comes back whatever state the backup is in (a missing backup reaches the backend only when it is deleted after the engine has read it) and a client retrying its own restore is answered at once; the target insert is the fence that tells an existing table from a `CREATING` target, so two simultaneous requests cannot both pass. The existing table or target is unchanged | +| Restoring into a name that exists | `TableAlreadyExistsException`; into a name that is itself a `CREATING` restore target, `TableInUseException`: "Table: is already being restored from backup: " for the same backup, and "Attempt to change a resource which is still in use: Table: is being used." for a different one (both measured) | `TableAlreadyExistsException` on all four backends for an existing table; `TableInUseException` for a name that is a `CREATING` restore target, whichever backup is restored, always with the same-backup text naming the backup the existing target is being restored from (the service's text for a different backup is not reproduced). The engine checks the name after the backup is read and before the overrides are checked against it, the order the service uses: a restore of a missing backup into an existing table, into a table `CreateTable` is still creating or into a `CREATING` restore target is `BackupNotFoundException` "Backup not found: " (measured on DynamoDB, twice each, and the same on `main`); an earlier build of this release checked the name first and answered `TableAlreadyExistsException` or `TableInUseException` for those three. The name is still checked before the job permit is taken, so the answer does not depend on whether a permit is free; each backend checks the name again under its registration (a missing backup reaches the backend only when it is deleted after the engine has read it), whose target insert is the fence that tells an existing table from a `CREATING` target, so two simultaneous requests cannot both pass. The existing table or target is unchanged | | `UpdateTable` on a `CREATING` restore target | `ResourceNotFoundException` (measured on DynamoDB) | `ResourceInUseException` on all four backends, the engine's mapping of a table that is not `ACTIVE`. A plain `CreateTable` into the target's name is `ResourceInUseException` "Table already exists: ", the service's text for a taken name (measured on DynamoDB), without the backup ARN. The target is unchanged | -| `BackupArn` that is malformed or belongs to another account | Not an ARN of the right shape, too long (over 1024), a control character, a slash or `..` in the region: `ValidationException` "Invalid Request: BackupArn is not valid". The right shape with an id that is not a backup id (not hex, a wildcard): `ValidationException` "Invalid Backup ARN" (both measured; which input gets which text was captured for one input only: a bare 13-digit id is "Invalid Request: BackupArn is not valid"). Surrounding whitespace, a trailing newline for one, is trimmed, and an ARN naming another region of the caller's account resolves (measured). One in another partition is `ValidationException` (measured; text not captured) | The same two texts on all four backends, split by ExtendDB's parser: a string that is not `arn:aws:dynamodb:::table//backup/` with a region in `[a-z0-9-]`, a digit-only account, a name in `[a-zA-Z0-9_.-]`, only printable ASCII, and at most 1024 bytes after trimming is "Invalid Request: BackupArn is not valid" (so is one in a partition other than `aws`, the only one ExtendDB issues); one of that shape whose id is neither `[0-9]{14}-[0-9a-f]{8}` nor a shape an earlier build issued (13 digits, `-`, 8 hex digits, which every backend issued before the shared id; 13 bare digits, which PostgreSQL v0.1.0 to v0.1.2 issued) is "Invalid Backup ARN"; no build has put a UUID in a backup ARN, so none resolves. Surrounding ASCII whitespace is trimmed (only a trailing newline was measured on the service; trimming all surrounding whitespace is ExtendDB's generalisation), and the region is checked for shape only: the backup is looked up under the ARN rebuilt in this deployment's `server.region` first and, when that is not found and the caller wrote a different ARN, under the ARN as written, so another region's resolves and a backup created under an earlier `server.region` stays reachable by the ARN `ListBackups` reports for it, though not by that ARN rewritten with the new region (see the upgrade manual on `server.region`). A bare 13-digit id (the shape PostgreSQL v0.1.0 to v0.1.2 issued) passes the id check, so such an ARN that names no backup is `BackupNotFoundException`, where the service answers `ValidationException` "Invalid Request: BackupArn is not valid" (measured on DynamoDB). A `server.region` outside `[a-z0-9-]` makes the server's own ARNs "not valid"; the server starts and logs a warning saying so. A well-formed ARN naming another account, of any number of digits, is refused on `DescribeBackup`, `DeleteBackup`, and `RestoreTableFromBackup` before any policy is evaluated, with the service's answers (see "A resource ARN in another account"); in dev mode the engine answers `AccessDeniedException`. The same checks apply to `ListBackups`'s `ExclusiveStartBackupArn`, where another account's or region's cursor is instead a `ValidationException` (see `ListBackups` paging). With authorization on (not dev mode), `DescribeBackup` and `DeleteBackup` first refuse a `BackupArn` that does not parse as a backup ARN at all (a table ARN, an `xrn:` prefix, an empty field) with "Invalid Backup ARN" before any policy is evaluated, as the service does (measured on DynamoDB), so those shapes get that text instead of the engine's. Every other `BackupArn` is authorized as the backup it names in this deployment (the table and id, in the server's region and the caller's account, surrounding whitespace trimmed as the engine trims it) and then checked by the engine as above. Same on all four backends | +| `BackupArn` that is malformed or belongs to another account | Not an ARN of the right shape, too long (over 1024), a control character, a slash or `..` in the region: `ValidationException` "Invalid Request: BackupArn is not valid". The right shape with an id that is not a backup id (not hex, a wildcard): `ValidationException` "Invalid Backup ARN" (both measured; which input gets which text was captured for one input only: a bare 13-digit id is "Invalid Request: BackupArn is not valid"). Surrounding whitespace, a trailing newline for one, is trimmed, and an ARN naming another region of the caller's account resolves (measured). One in another partition is `ValidationException` (measured; text not captured) | The same two texts on all four backends, split by ExtendDB's parser: a string that is not `arn:aws:dynamodb:::table//backup/` with a region in `[a-z0-9-]`, a digit-only account, a name in `[a-zA-Z0-9_.-]`, only printable ASCII, and at most 1024 bytes after trimming is "Invalid Request: BackupArn is not valid" (so is one in a partition other than `aws`, the only one ExtendDB issues); one of that shape whose id is neither `[0-9]{14}-[0-9a-f]{8}` nor a shape an earlier build issued (13 digits, `-`, 8 hex digits, which every backend issued before the shared id; 13 bare digits, which PostgreSQL v0.1.0 to v0.1.2 issued) is "Invalid Backup ARN"; no build has put a UUID in a backup ARN, so none resolves. Surrounding ASCII whitespace is trimmed (only a trailing newline was measured on the service; trimming all surrounding whitespace is ExtendDB's generalisation), and the region is checked for shape only: the backup is looked up under the ARN rebuilt in this deployment's `server.region` first and, when that is not found and the caller wrote a different ARN, under the ARN as written, so another region's resolves and a backup created under an earlier `server.region` stays reachable by the ARN `ListBackups` reports for it, though not by that ARN rewritten with the new region (see the upgrade manual on `server.region`). A bare 13-digit id (the shape PostgreSQL v0.1.0 to v0.1.2 issued) passes the id check, so such an ARN that names no backup is `BackupNotFoundException`, where the service answers `ValidationException` "Invalid Request: BackupArn is not valid" (measured on DynamoDB). Under a `server.region` outside `[a-z0-9-]` no backup ARN can be formed, so `CreateBackup` is refused with a `ValidationException` naming `server.region`, and the backups taken before the upgrade under such a region keep an ARN the three operations refuse as "not valid"; the server starts and logs a warning saying so. A well-formed ARN naming another account, of any number of digits, is refused on `DescribeBackup`, `DeleteBackup`, and `RestoreTableFromBackup` before any policy is evaluated, with the service's answers (see "A resource ARN in another account"); in dev mode the engine answers `AccessDeniedException`. The same checks apply to `ListBackups`'s `ExclusiveStartBackupArn`, where another account's or region's cursor is instead a `ValidationException` (see `ListBackups` paging). With authorization on (not dev mode), `DescribeBackup` and `DeleteBackup` first refuse a `BackupArn` that does not parse as a backup ARN at all (a table ARN, an `xrn:` prefix, an empty field) with "Invalid Backup ARN" before any policy is evaluated, as the service does (measured on DynamoDB), so those shapes get that text instead of the engine's. Every other `BackupArn` is authorized as the backup it names in this deployment (the table and id, in the server's region and the caller's account, surrounding whitespace trimmed as the engine trims it) and then checked by the engine as above. Same on all four backends | | Tag conditions on backups | The backup resource has no `aws:ResourceTag` condition key: a backup has no tags to match | `DescribeBackup`, `DeleteBackup`, and `RestoreTableFromBackup` are authorized on the backup ARN as a resource with no tags. An `aws:ResourceTag/` condition finds the key absent for a backup (and `dynamodb:ResourceTag/` is absent for every resource), so, for example, an Allow conditioned on `StringEquals` never grants on a backup and a Deny conditioned on `StringEquals` never denies one. This holds whatever an earlier build stored under the backup ARN (it accepted `TagResource` on one; see Tagging), and a backup does not take its table's tags. For `RestoreTableFromBackup`, the seven item actions evaluated on `table/` (see the authorization rows below) read that ARN's resource tags. A target that does not exist has no tags, so an `aws:ResourceTag/` condition on those actions finds none and does not grant the restore; a target that exists is evaluated with its tags, and a request that is otherwise valid then passes authorization and fails `TableAlreadyExistsException` (or `TableInUseException` for a `CREATING` restore target) when the condition is met, and fails `AccessDeniedException` naming the first item action when it is not (measured on SQLite). Whether the service reads a target's tags in these evaluations was not measured. Scope backup permissions by ARN (`table//backup/*`). Measured on PostgreSQL and SQLite for the Allow case, including tags written under a backup ARN by the earlier build and then served by this one; the Deny case and role sessions rest on unit tests of the authorization path, which is the same for every backend | | Authorization of `ListBackups` | `dynamodb:ListBackups` is evaluated on `arn:aws:dynamodb:::table/*/backup/*`, or on `table//backup/*` when `TableName` filters the list (measured on DynamoDB) | The same on all four backends (the check runs in the server before the backend is reached). A statement on `*` or `table/*` covers every list; one on `table//backup/*` (or `table//*`) covers only the list filtered to that table; one on the table's own ARN, `table/`, covers none. The unfiltered list is one resource, so a Deny on `table//backup/*` denies the list filtered to that table and not the unfiltered list that would include its backups, which follows from the resources the service evaluates. A `TableName` that is not a string is evaluated as no filter; the engine then refuses the request. These resources have been evaluated since v0.1.14; the admin guide's compatibility notes describe what earlier builds evaluated | | Authorization of `RestoreTableFromBackup` | `dynamodb:RestoreTableFromBackup` on the backup ARN, and `dynamodb:Scan`, `dynamodb:Query`, `dynamodb:GetItem`, `dynamodb:PutItem`, `dynamodb:UpdateItem`, `dynamodb:DeleteItem`, and `dynamodb:BatchWriteItem` on `table/`, each of which must be allowed; `dynamodb:RestoreTableFromBackup` is not evaluated on the target (measured on DynamoDB) | The same eight evaluations on all four backends, in that order: the backup first, then the target under each item action in the order listed, stopping at the first denial, which the `AccessDeniedException` names (`User: ... is not authorized to perform: dynamodb:Scan on resource: arn:aws:dynamodb:::table/`); the service's order was not measured. A missing `TargetTableName` is evaluated as `table/` (an empty name, which no statement but `*`-style wildcards covers); the engine then refuses the request. The seven item-action evaluations read the target ARN's resource tags: a target that does not exist has none, one that exists is evaluated with its tags and the restore then fails `TableAlreadyExistsException` (see Tag conditions on backups). Every DynamoDB condition key (`dynamodb:LeadingKeys`, `dynamodb:Attributes`, `dynamodb:Select`, `dynamodb:ReturnValues`, `dynamodb:ReturnConsumedCapacity`, `dynamodb:FullTableScan`, `dynamodb:EnclosingOperation`) is absent in these seven evaluations, so `ForAllValues:*` and `Null: true` conditions pass and bare `StringEquals` and `Bool` conditions fail: an Allow of the item actions conditioned on `ForAllValues:StringEquals dynamodb:LeadingKeys` grants the restore of every item in the backup, an Allow of `dynamodb:Scan` conditioned on `Bool dynamodb:FullTableScan: false` blocks every restore, and a Deny of `dynamodb:Scan` conditioned on `Bool dynamodb:FullTableScan: true` blocks none; the keys the service sets in these evaluations were not measured. The backup ARN is evaluated as the backup it names in this deployment: the table and backup id from the request, in the server's region and the caller's account, whatever partition, region or account the caller wrote, with surrounding whitespace trimmed as the engine trims it. The engine resolves the same table and id (under the server's region first, then as written for a backup created under an earlier `server.region`), so it never acts on a backup other than the one evaluated. `DescribeBackup` and `DeleteBackup` are evaluated the same way. These resources have been evaluated since v0.1.14 (see the admin guide's compatibility notes). A Deny of `dynamodb:RestoreTableFromBackup` on a table does not block restores into it; deny one of the seven item actions on it instead | -| Multi-part base table keys (preview) | Not supported | A backup of a table created with `enable_multipart_keys` is refused at restore with `ValidationException`; the item paths address such tables by their first HASH and RANGE attribute only *(from the code and storage-level tests; not checked on a running server)* | -| `DeleteBackup` and `DeleteTable` during a restore | `DeleteTable` on the `CREATING` target is `ResourceInUseException`. `DeleteBackup` during a restore was not measured | While the target is `CREATING`, `DeleteTable` on it is `ResourceInUseException` on all four backends, as on the service. `DeleteBackup` on its source backup is `BackupInUseException` on all four: that refusal is ExtendDB's contract, not measured against the service. Both succeed once the target is `ACTIVE`. How the restore and `DeleteBackup` are ordered differs *(from the code)*. PostgreSQL and SQLite register the restore in the transaction that creates the target, holding the backup row, so a concurrent `DeleteBackup` is ordered strictly before (the restore then finds no backup) or after (it finds the restore). MongoDB has no cross-collection transaction: a restore claims the backup document, inserts the target's catalog document, and releases the claim; `DeleteBackup` claims the backup the same way. Cassandra uses two lightweight transactions on different partitions: the target's row is inserted first (`IF NOT EXISTS`, which reserves the name), then the restore is registered on the backup's row (`IF backup_status = 'AVAILABLE'`), which `DeleteBackup` claims on the same partition; a concurrent `DescribeTable` can therefore see a `CREATING` target for a restore that then fails with `BackupNotFoundException`, after which the target is removed. On MongoDB, if `DeleteBackup` returns an internal error or its request is cancelled after the backup's data drop started, the backup stays `AVAILABLE` and in use (`DeleteBackup` and `RestoreTableFromBackup` return `BackupInUseException`) until the heartbeat sweep finishes the delete, about 45 to 80 s later; it is not left restorable without its data. A `DeleteBackup` that stopped while a restore from the same backup is `CREATING` is finished by the sweep after that restore becomes `ACTIVE`. So a `DeleteBackup` that returned an error can still take effect later: besides the case above, one that failed after claiming the backup (for example because it found a restore from it `CREATING`) and whose claim could not then be released, neither in the request nor by the claim guard's release on drop, is finished by the sweep (once no restore from the backup is `CREATING`), although the client was told it failed (not reproduced) *(all from the code and storage-level tests)*. MongoDB reads the catalog from the primary with read concern `local` regardless of the connection string, which the claims rely on; the connection string must use `readPreference=primary` (any other read preference is refused at startup). The write concern follows the connection string and must be `majority` (the server default): a connection string with `w=1` is refused at startup, because a claim acknowledged under `w=1` just before a primary failover can be rolled back | -| Concurrent restores of one backup | A second `RestoreTableFromBackup` of a backup while a restore from it is still `CREATING` is `BackupInUseException` (measured) | The same on all four backends: the second restore is refused with `BackupInUseException` and creates nothing; the check runs inside the transaction that registers the restore (PostgreSQL, with the backup row locked `FOR UPDATE`; SQLite), inside the claim on the backup document followed by a check for a `CREATING` target of the backup (MongoDB), or inside the lightweight transaction on the backup row (Cassandra), so two simultaneous requests cannot both pass. Once the first target is `ACTIVE` the backup restores again. A restore of a backup whose `DeleteBackup` is in progress answers differently by backend (the service's answer was not measured): `BackupInUseException` "Backup is being deleted" on MongoDB, where the backup stays `AVAILABLE` under a delete claim until its data is dropped; `BackupNotFoundException` on Cassandra, where the delete marks it `DELETING` first; and on PostgreSQL and SQLite the restore waits for the delete's transaction and then finds no backup (`BackupNotFoundException`). Earlier builds accepted concurrent restores of one backup on PostgreSQL, SQLite, and Cassandra, and on MongoDB after a 2 s retry | -| Restore interrupted by the server stopping | Managed by the service | The copy runs in the server process, so a stopped process leaves its target `CREATING`. Each backend removes such a target without operator action, logs the removal with the table name (at warn; at info on Cassandra), and then lets `DeleteBackup` and a new restore into the name succeed. Measured after `SIGKILL` mid-copy and an immediate restart *(earlier build)*: **PostgreSQL** removes it on a control-plane pass once the target is older than 60 s and its restore lock is free (removed 60 s after the kill). **SQLite** removes it when the next server starts (0.1 s after the restart). **MongoDB** and **Cassandra** remove it once its owner's heartbeat is more than 60 s old, by a sweep that runs at startup and every 15 s, so about 45 to 80 s after the owner stopped (the heartbeat's last refresh can be up to one 15 s interval before the stop; measured: 60 s on MongoDB, 62 s on Cassandra). Until then `DeleteTable` and `DeleteBackup` are refused as during a running restore. Not covered *(from the code)*: on MongoDB, a target whose base copy had finished is not swept but is finished by the index backfill worker after a restart; if that backfill fails on every attempt, the worker retries it every 5 s with a warning and never gives up, so the target stays `CREATING` (and `DeleteTable` and `DeleteBackup` stay refused) until the cause is fixed or the target's catalog document is removed by hand (no failing case was found to reproduce this). On Cassandra, a target left `CREATING` by a build before catalog 0.0.4 carries no heartbeat and is not swept. The MongoDB and Cassandra heartbeats compare wall clocks across servers (MongoDB uses the database server's clock for refreshes), so a server whose clock is far behind can have a live restore swept | +| Multi-part base table keys (preview) | Not supported | A store-backed backup of a table created under `limits.allow_multipart_table_keys` is written `unordered`, and its restore preserves the item set: the restored table holds what the source's `Scan` held, with the source's key schema and attribute definitions, the target laid out as a natively created table (`bulk_load` keys rows with the backend's own key functions) *(measured on PostgreSQL and SQLite with the preview enabled, `tests/test_backup_job.py`; the engine test `a_multi_part_key_manifest_is_registered`)*. The preview's item paths address such a table by its first HASH attribute on every backend, restored or not: on PostgreSQL a `GetItem` by the whole key finds nothing on the source or on the restored table (a `Query` naming both key attributes does find the item), on SQLite the source itself keeps one item per first HASH key value and a `GetItem` returns that item whatever second key it was given, and a `Query` on the first HASH key alone is refused on both ("Query condition missed key schema element") *(measured on PostgreSQL and SQLite with the preview enabled)*. An in-catalog backup of such a table (no store, or Cassandra) is refused at restore with `ValidationException` *(from the code and storage-level tests)* | +| `DeleteBackup` and `DeleteTable` during a restore | `DeleteTable` on the `CREATING` target is `ResourceInUseException`. `DeleteBackup` during a restore was not measured | While the target is `CREATING`, `DeleteTable` on it is `ResourceInUseException` on all four backends, as on the service. `DeleteBackup` on its source backup is `BackupInUseException` on all four: that refusal is ExtendDB's contract, not measured against the service. Both succeed once the target is `ACTIVE`. How the restore and `DeleteBackup` are ordered differs *(from the code)*. PostgreSQL and SQLite register the restore in the transaction that creates the target, holding the backup row, so a concurrent `DeleteBackup` is ordered strictly before (the restore then finds no backup) or after (it finds the restore). MongoDB runs the restore's registration and `DeleteBackup`'s claim each as one session transaction that first increments a lock field on the backup's catalog document while it is `AVAILABLE`, so the two conflict and one orders wholly before the other: a `DeleteBackup` that wins leaves the registration no `AVAILABLE` backup (`BackupNotFoundException`), a registration that wins makes the `DeleteBackup` `BackupInUseException`. Cassandra uses two lightweight transactions on different partitions: the target's row is inserted first (`IF NOT EXISTS`, which reserves the name), then the restore is registered on the backup's row (`IF backup_status = 'AVAILABLE'`), which `DeleteBackup` claims on the same partition; a concurrent `DescribeTable` can therefore see a `CREATING` target for a restore that then fails with `BackupNotFoundException`, after which the target is removed. On MongoDB `DeleteBackup` writes the backup `DELETING` before it drops the data and `DELETED` after; a `DeleteBackup` whose request fails or is cancelled after the drop started leaves the backup `DELETING`, which `DescribeBackup`, `ListBackups` and a restore treat as gone (`BackupNotFoundException`), and a retry of `DeleteBackup` finishes the drop and answers `DELETED` *(from the code and storage-level tests)*. MongoDB reads the catalog from the primary with read concern `local` regardless of the connection string, which the lock and the job records rely on; the connection string must use `readPreference=primary` (any other read preference is refused at startup). The write concern follows the connection string and must be `majority` (the server default): a connection string with `w=1` is refused at startup, because a `DELETING` write or a job record acknowledged under `w=1` just before a primary failover can be rolled back | +| Concurrent restores of one backup | A second `RestoreTableFromBackup` of a backup while a restore from it is still `CREATING` is `BackupInUseException` (measured) | The same on all four backends: the second restore is refused with `BackupInUseException` and creates nothing; the check runs inside the transaction that registers the restore (PostgreSQL, with the backup row locked `FOR UPDATE`; SQLite), inside the claim on the backup document followed by a check for a `CREATING` target of the backup (MongoDB), or inside the lightweight transaction on the backup row (Cassandra), so two simultaneous requests cannot both pass. For a store-backed restore the registration takes the same lock on the `backup_index` row instead of the legacy `backups` row (PostgreSQL `SELECT ... FOR UPDATE`, SQLite the `BEGIN IMMEDIATE` write transaction, MongoDB a `$inc lock_seq` on the `backup_index` document, Cassandra the `active_restores` lightweight transaction on the backup's partition), the same ordering guarantee, so a legacy restore and a store-backed restore of two different backups never contend and a `DeleteBackup` of a store-backed backup meets only store-backed registrations. Two concurrent registrations of one backup leave one restore target on Cassandra as on MongoDB: the registration's lightweight transaction (`IF backup_status = 'AVAILABLE' AND active_restores = null`) is linearized on the backup's partition, so the second finds `active_restores` non-null and is `BackupInUseException`; the in-use pre-read is advisory *(storage-level tests on both)*. Once the first target is `ACTIVE` the backup restores again; on MongoDB the check reads majority-committed data (at the default transaction read concern) while `DescribeTable` reads at `local`, so a `RestoreTableFromBackup` or `DeleteBackup` issued within the majority-commit lag after `DescribeTable` first shows the target `ACTIVE` can still be `BackupInUseException` and succeeds when retried (the lag is the journal flush plus any concurrent batch write still holding an earlier oplog slot; measured at 7 to 12 ms, once in about 2,000 observations, with a 3,000-item insert running beside the write). A restore of a backup whose `DeleteBackup` is in progress answers `BackupNotFoundException` on every backend (the service's answer was not measured), by two mechanisms: on MongoDB and Cassandra the delete marks the backup `DELETING` before it drops the data, and the restore finds no `AVAILABLE` backup; on PostgreSQL and SQLite the restore waits for the delete's transaction and then finds no backup. Earlier builds accepted concurrent restores of one backup on PostgreSQL, SQLite, and Cassandra, and on MongoDB after a 2 s retry | +| Restore interrupted by the server stopping | Managed by the service | The copy runs in the server process, so a stopped process leaves its target `CREATING` with a RUNNING restore job record whose heartbeat stops. The engine's reconciler, once at startup and then every `backup.reconcile_interval_secs` (default 60 s), marks the record FAILED and removes the target once the heartbeat is older than `backup.stale_job_after_secs` (default 300 s) on the database's clock, on every backend alike, logs the removal with the table name, and then lets `DeleteBackup` and a new restore into the name succeed *(from the code)*. On SQLite a restart ages every record of the stopped process at once (one process serves the file), so the first cycle removes it without the stale window. Until then `DeleteTable` and `DeleteBackup` are refused as during a running restore. A store-backed restore (one run from a `[backup.store]`) is instead resumed, not discarded: its owner's death leaves a RUNNING restore job record with a stale heartbeat and the target `CREATING`, and the reconciler of any instance with the configured store takes the record over under a fresh owner generation once the heartbeat is stale (on SQLite at once after a restart, by the record's slot; on PostgreSQL once the owner's claim session has ended as well), stamping a fresh heartbeat in the same write so no second instance takes it too, and resumes the load from its recorded progress (per data file on PostgreSQL, per batch on SQLite, MongoDB and Cassandra), re-downloading to its own spool when the dead owner's `spool_host` is not this host's; the target reaches `ACTIVE` with the backup's full item count, its secondary indexes complete: a resumed Cassandra restore re-applies its last batch as idempotent upserts that rewrite the same base and index rows (the write path rewrites an unchanged item's GSI and LSI rows rather than deleting them, #424). A resume that finds no recorded progress and a target holding more items than the backup's total is `FAILED` with reason `target_not_empty` and the target is left in place: the rule protects a catalog restored from a dump that names a restore which finished and was written to since. A target holding no more than the total with no recorded progress is a first data file applied in part: on Cassandra, which records a batch's progress after its writes, an ordinary crash before the first progress record (the first 10,000 items or 32 MiB) leaves written items and no progress, and the resume re-applies the first file with idempotent upserts and reaches `ACTIVE` with the exact count *(from the code; the storage and engine cells)*. MongoDB records each batch's documents and its progress in one transaction, so a mid-file crash leaves no items the record does not account for and resumes to `ACTIVE`; only a crash before the first batch commits leaves an empty target, which resumes from the start. A record that one instance's reconciler has taken over five times without the restore ending (a resume whose final count or progress read fails every time, or that stops with the record RUNNING) is failed on the next takeover with reason `owner dead: taken over 5 times by one instance without the restore ending` and its target removed, so a record that can never finish does not hold its name and its backup for as long as the server runs; the count is per instance and a restart starts it over *(from the code)*. The target stays `CREATING` with a `FAILED` record: `DeleteTable` on it is `ResourceInUseException`, `DeleteBackup` of its backup and a second restore of that backup are `BackupInUseException`, and the reconciler does not remove it; `extenddb backup remove-target --account
    ` (the admin guide's `backup` section) removes the target and its record, after which `DeleteBackup` and a new restore into the name succeed. A legacy (in-catalog) restore is still failed and its target removed. Measured *(binary-level, this release)*: SIGTERM during a store-backed restore left the record RUNNING and the restarted server resumed it to `ACTIVE` with every item (on PostgreSQL after the 45 s stale window, on SQLite at once); a SIGKILL at each restore kill point left the target to finish with the exact count (the base table's and the GSI's) or, for a failing restore, gone, except at the first reach of the Cassandra point that pauses before a batch's first progress record: on Cassandra (killed after the first batch's 10,000 upserts, before their record) the resume of that build answered `target_not_empty` and left the target `CREATING`, and `extenddb backup remove-target` then removed it, after which `DescribeTable` was `ResourceNotFoundException` and `DeleteBackup` succeeded; this release resumes that case with upserts to `ACTIVE` *(from the code; the nightly kill-point job runs it)*. On MongoDB a SIGKILL after a file's `insert_many` leaves the batch's items and its progress committed together, so the resume continues to `ACTIVE` rather than `target_not_empty`. Not covered *(from the code)*: on MongoDB, a target whose base copy had finished has its record DONE and is left alone by the reconciler; the index backfill worker finishes it after a restart, and if that backfill fails on every attempt the worker retries it every 5 s with a warning and never gives up, so the target stays `CREATING` (and `DeleteTable` and `DeleteBackup` stay refused) until the cause is fixed or the target's catalog document is removed by hand (no failing case was found to reproduce this). A target left `CREATING` by a build before this release is backfilled by the catalog migration as a RUNNING record with a heartbeat at the epoch, so the first cycle removes it. Heartbeats and the stale window are measured on the database server's clock, so the application hosts' clocks play no part. The same reconciler resolves a store-backed `CreateBackup` the server stopped (one reconciler with one duty list, in place of per-backend sweeps): a `CREATING` backup row whose owner is dead (on PostgreSQL and MongoDB a heartbeat older than `stale_job_after_secs`, on PostgreSQL once the owner's claim session has ended as well; on SQLite, where one process serves the file, every `CREATING` row after a restart and, at runtime, a row whose in-process slot no task holds, so a job task that died is resolved by the next cycle without a restart) is taken over under a fresh owner generation and becomes `AVAILABLE` when a complete, verified manifest is under its prefix in the configured store (the process died between the manifest write and the status write) and `FAILED` otherwise, never continuing the upload; the objects of a `FAILED` backup are deleted once its heartbeat is older than `stale_job_after_secs`; a `DELETING` backup's deletion is finished; a `DELETED` backup's prefix that still lists objects is swept; and `FAILED` and `DELETED` rows are removed once their prefix is empty and their `completed_at` is older than `failed_row_retention_secs` (7 days). Measured *(binary-level, this release)*: SIGKILL between the data file and the manifest left the row `CREATING` and the data file under the prefix, and after the restart the row was `FAILED` in the first cycle on SQLite and 46 s after the kill on PostgreSQL (`stale_job_after_secs = 45`), with the prefix deleted in the same cycle on PostgreSQL (the stale heartbeat is what made the owner dead) and once the dead owner's heartbeat was stale on SQLite, 45 s after the kill (the slot made the owner dead at once; the prefix waits for the heartbeat clock on every backend); the same at a kill between the three documents and the manifest, which leaves the documents under the prefix and no manifest; SIGKILL between the manifest write and the status write left the five objects, and the row was `AVAILABLE` with the manifest's size and count after the restart (SQLite at once, PostgreSQL after 48 s); SIGTERM during an upload left the row `FAILED` with reason `shutdown` and an empty prefix. These duties run only while the configured store is healthy, and under a maintenance claim only the dead-owner and `DELETING` duties run. A `FAILED` backup is `BackupNotFound` to clients; `extenddb backup list --failed` prints it with its reason | | Point-in-time recovery | 35-day window: `UpdateContinuousBackups` enables continuous backups, and `RestoreTableToPointInTime` restores to any second in the window | Not supported; use on-demand backups (`CreateBackup` and `RestoreTableFromBackup`) instead. `DescribeContinuousBackups` reports `ContinuousBackupsStatus: ENABLED` (the service reports this unconditionally) with `PointInTimeRecoveryStatus: DISABLED` and no restorable-time window. `UpdateContinuousBackups` with `PointInTimeRecoveryEnabled: true` returns `ContinuousBackupsUnavailableException`; with `false` it returns the same description. `RestoreTableToPointInTime` checks the source table, then returns `PointInTimeRecoveryUnavailableException` ("Point in time recovery is not enabled for table ''"), the exception the service models on that operation. A table that does not exist is `TableNotFoundException` ("Table not found: ") on all three operations, as on the service. Same on all four backends | -| Where backups live | Managed storage, independent of the table | Inside the database the server already uses, so a lost database loses its backups too: on PostgreSQL in the catalog database (`backups`, `backup_items`, `backup_definitions`, `table_restores`), on SQLite in the one database file, on MongoDB as a `backups` metadata collection in the catalog database and one data collection per backup in the data database, on Cassandra as backup metadata in the catalog keyspace and item payloads in the account's keyspace. Take database-level backups for off-host copies until the backup store program ships | -| Backup creation | `CreateBackup` returns while the backup is `CREATING`; `ListBackups` and `DescribeBackup` expose that state until it becomes `AVAILABLE`. `BackupName` and `TableName` are checked as the model says: length 3 to 255 and `[a-zA-Z0-9_.-]+`, with "1 validation error detected: Value '' at 'backupName' failed to satisfy constraint: " texts, joined under "2 validation errors detected:" when both fail (measured). `TableName` may be the table's ARN (measured). A table that does not exist or is still `CREATING` is `TableNotFoundException` "Table not found: " (measured) | `CreateBackup` copies the items before it returns and returns the backup `AVAILABLE` on all four backends. On PostgreSQL, MongoDB, and Cassandra a concurrent `ListBackups` never saw it `CREATING` *(earlier build)*; on SQLite the backup row is visible as `CREATING` to other requests while the copy runs. Measured *(earlier build)*: 1.3 s for 60,000 items on PostgreSQL, 1.9 s for 60,000 on SQLite, 0.2 s for 20,000 on MongoDB, 5.1 s for 20,000 on Cassandra. MongoDB reports `BackupSizeBytes` 0. The name checks are the service's, with its texts (the length clause before the pattern clause when both fail; the service's order was not captured), counting characters, with `TableName` taking the deployment's table-name limits; before them a 200 KB name was stored and returned in every `ListBackups` page (a 500 on Cassandra) and a NUL byte was a 500 on PostgreSQL. `TableName` as an ARN (`arn:aws:dynamodb:::table/`) is accepted on all four backends: another region's resolves (not measured on the service), another account's is refused with the service's answers (see "A resource ARN in another account"); an SDK built from the model refuses the ARN form client-side, so only a hand-built request sends it. `TableNotFoundException` "Table not found: " for a missing or `CREATING` table on all four backends, built from the request's name (the backends' own payloads differ) | +| Where backups live | Managed storage, independent of the table | With a configured `[backup.store]` (a filesystem root, which is the default beside a SQLite database file, or an S3 bucket and prefix) on PostgreSQL, SQLite and MongoDB: under the store's root, at `/
    //`, as gzip data files of DynamoDB JSON lines (`data/000001.json.gz`, ...) and the manifest documents (`extenddb-manifest.json` last, the commit record), with a `backup_index` row in the catalog as the index; a lost database loses the rows, not the bytes (`extenddb backup sync` gives a new catalog a row for each backup under the root, and `--adopt-store` first makes it the owner of a root another catalog owned). Without a store, and on Cassandra always, inside the database the server already uses, so a lost database loses its backups too: on PostgreSQL in the catalog database (`backups`, `backup_items`, `backup_definitions`, `table_restores`), on SQLite in the one database file, on MongoDB as a `backups` metadata collection in the catalog database and one data collection per backup in the data database, on Cassandra as backup metadata in the catalog keyspace and item payloads in the account's keyspace; take database-level backups for off-host copies there | +| Backup creation | `CreateBackup` returns while the backup is `CREATING`; `ListBackups` and `DescribeBackup` expose that state until it becomes `AVAILABLE`. `BackupName` and `TableName` are checked as the model says: length 3 to 255 and `[a-zA-Z0-9_.-]+`, with "1 validation error detected: Value '' at 'backupName' failed to satisfy constraint: " texts, joined under "2 validation errors detected:" when both fail (measured). `TableName` may be the table's ARN (measured). A table that does not exist or is still `CREATING` is `TableNotFoundException` "Table not found: " (measured) | With a configured `[backup.store]` on PostgreSQL, SQLite and MongoDB, `CreateBackup` takes the table's snapshot and writes the catalog row in the request and returns the backup `CREATING`; a job then uploads it and the backup becomes `AVAILABLE`, which `DescribeBackup` and `ListBackups` show as on the service (`BackupSizeBytes` is 0 while `CREATING` and the compressed size once `AVAILABLE`; `ItemCount` and `TableSizeBytes` in `SourceTableDetails` are the table's figures as last counted by the stats pass while `CREATING`, which is 0 and 0 for a table the stats pass has not reached yet, such as one loaded seconds before CreateBackup, and the backup's exact figures once `AVAILABLE`: the items written and the sum of their DynamoDB sizes; the service reported 0 and 0 in every fresh-table backup measured). `CREATING` lasts the whole upload (the durations below), and while it lasts `DeleteBackup` and `RestoreTableFromBackup` of the backup answer `BackupInUseException` "Backup is being created: " on all three backends; the 16 service backups measured were `AVAILABLE` 12 to 90 ms after `CreateBackup` answered, so a client that deletes or restores a backup right after creating it polls `DescribeBackup` first here. The backup is the table as it stood when its snapshot opened, which is inside the request but after the request's checks (the table's state, the store's health, the maintenance claim) and after any wait for a job permit, up to 30 s (below): every write acknowledged before the open is in the backup and none acknowledged after it, so a `PutItem` acknowledged while the request waited for a permit is in the backup (measured: a write acknowledged 1.01 s after a `CreateBackup` that waited for a permit was in it, 2 of 2 runs), and `BackupCreationDateTime` is the open's millisecond. Measured on the service: a restored backup contained every write acknowledged before `CreateBackup` was sent and the 18 writes acknowledged in the 0.1 s up to `BackupCreationDateTime`, and none of the 11,754 acknowledged in the 75 s that followed, so the service takes the table at an instant inside the request as well. A backup the job cannot finish (a store failure, an item no restore can load, a shutdown) is `FAILED`, which is `BackupNotFound` to clients, and its objects are deleted (the restore-interruption row). A `CreateBackup` still waiting in its snapshot open or row insert when the server stops writes no row; it is answered `ServiceUnavailable` "The server is shutting down. Retry the request." only after a registration loss, where the jobs' token is cancelled while the listener still serves, since on SIGTERM the listener's 5 s connection drain closes the connection before that token is cancelled (measured: the connection closed 8.2 s after the request was sent). Measured *(this release, 16-core host, filesystem store, the record in `docs/design/15-backup-restore-bench-pr8.md`)*: 100,000 items of a composite-key table with two GSIs `AVAILABLE` in 1.85 s on PostgreSQL and 1.44 s on SQLite, 1,000,000 in 36.1 s and 12.45 s. Without a store, and on Cassandra always, `CreateBackup` copies the items before it returns and returns the backup `AVAILABLE`. On PostgreSQL, MongoDB, and Cassandra a concurrent `ListBackups` never saw an in-catalog backup `CREATING` *(earlier build)*; on SQLite the backup row is visible as `CREATING` to other requests while the copy runs. Measured *(earlier build, in-catalog)*: 1.3 s for 60,000 items on PostgreSQL, 1.9 s for 60,000 on SQLite, 0.2 s for 20,000 on MongoDB, 5.1 s for 20,000 on Cassandra. MongoDB's in-catalog backups report `BackupSizeBytes` 0. The name checks are the service's, with its texts (the length clause before the pattern clause when both fail; the service's order was not captured), counting characters, with `TableName` taking the deployment's table-name limits; before them a 200 KB name was stored and returned in every `ListBackups` page (a 500 on Cassandra) and a NUL byte was a 500 on PostgreSQL. `TableName` as an ARN (`arn:aws:dynamodb:::table/`) is accepted on all four backends: an ARN whose region is not the server's (`server.region`, compared as written, so `US-EAST-1` differs from `us-east-1`) is `ValidationException` "1 validation error detected: Invalid AWS region in ''" (the service's answer, measured on DynamoDB twice; an earlier build resolved it to the server region's table), another account's is refused with the service's answers (see "A resource ARN in another account"), the account checked before the region, so another account's ARN in another region is `AccessDeniedException` (measured on DynamoDB for CreateBackup and ListBackups, twice each); an SDK built from the model refuses the ARN form client-side, so only a hand-built request sends it. `TableNotFoundException` "Table not found: " for a missing or `CREATING` table on all four backends, built from the request's name (the backends' own payloads differ). Without a store, and on Cassandra, a table the deployment can create is backed up whatever its name, as the service does (measured on DynamoDB: a table named `...` answered `ContinuousBackupsUnavailableException` "Backups are being enabled for the table: .... Please retry later" 13.2 s after `CreateTable` and was backed up at 33.7 s): the in-catalog format writes no store prefix. With a store, a name over 255 bytes (reachable only with `[limits] max_table_name_length` raised) or made only of dots fits no store prefix, so the store-backed create refuses it with `ValidationException` before it opens a snapshot or writes a row ("TableName cannot be backed up: a backup ARN cannot be formed for it: invalid backup ARN: table name of N bytes is longer than 255" / "... invalid backup ARN: table name \"...\" is only dots"). Every table under a `server.region` outside `[a-z0-9-]` is refused (the `BackupArn` row); the checks run after the table's ACTIVE probe and before the job permit, so the refusal takes no slot. The permit is the last check: a request that finds every `[backup] max_concurrent_jobs` slot held (default 4) waits up to 30 s for one and is then `LimitExceededException` ("Too many backup and restore jobs are running on this server (at most N at once, the [backup] max_concurrent_jobs bound); retry later"), where the service accepted six `CreateBackup` calls sent together (measured on DynamoDB, two runs) | | `DeleteBackup` response | The full `DescribeBackup` description with `BackupStatus` `DELETED`, including `SourceTableDetails.ProvisionedThroughput` (a required member) and `SourceTableFeatureDetails` (measured) | The same on all four backends: the engine reads the backup's recorded definition before the delete and fills the description as `DescribeBackup` does, with `BackupStatus` `DELETED`. A backup whose definition this build cannot read or that does not parse is still deleted and described without feature details (logged at warn), which means a damaged `PROVISIONED` backup is described with on-demand `ProvisionedThroughput` `{0, 0}`: ExtendDB's choice, deletability over the throughput member of a row that is being removed, where `DescribeBackup` on the same row is an `InternalServerError`; a failure to reach the database fails the request and deletes nothing. | +| A backup maintenance command holding the catalog's claim | Not applicable: the service has no maintenance command | While `extenddb backup purge` holds the catalog-level maintenance claim, `CreateBackup` and `RestoreTableFromBackup` on every instance of the catalog answer HTTP 400 `LimitExceededException` "A backup maintenance command holds this catalog's maintenance claim; backup and restore jobs are refused until it finishes", after every other check of the request (a request refused on its own terms, a bad table name or override, gets its own error) and before the job permit. The refusal ends when the command releases the claim, or when a command that died goes stale after `[backup] stale_job_after_secs`. | +| A store-backed backup while the configured store is unhealthy | Not applicable: managed storage | `CreateBackup` on PostgreSQL, SQLite and MongoDB with a configured `[backup.store]`, and `DeleteBackup` and `RestoreTableFromBackup` of a backup in that store, while the store is unhealthy or its first check is pending, answer HTTP 400 `ValidationException` "The filesystem backup store configured on this server is unhealthy (owned by another catalog); the operation is refused until the store's check passes", the kind (`filesystem`, `s3`) and the category in parentheses being the store's (`/health` reports the same category); before the first check of a store checked after the listener: "The s3 backup store configured on this server has not passed a check yet (its first check is pending); the operation is refused until it does". `DescribeBackup` and `ListBackups` still describe such a backup. Once `DeleteBackup`'s DELETING write has committed the request answers the description with `BackupStatus` `DELETED` whatever the object deletion does; a deletion the store does not answer within its bound is left DELETING for the reconciler, logged, and the request still answers DELETED. `CreateBackup`'s refusal comes after the table's ACTIVE probe and the name checks and before the maintenance check and the job permit, so it takes no slot and writes no row; the in-catalog create (no store, Cassandra) touches no store and is not refused. | | `DescribeBackup` when the definition cannot be read | Not applicable | A definition row that does not parse, a renamed column, or a revoked privilege (`Internal` from the backend) is an `InternalServerError`, as it is for `RestoreTableFromBackup` on the same read; only a definition version this build does not know is described without `SourceTableFeatureDetails`. | -| Consistency of a backup | Point-in-time snapshot of the table | **PostgreSQL** and **SQLite** take the items from one snapshot. How *(from the code)*: PostgreSQL holds the table's catalog row shared from the first definition read until the data snapshot (`REPEATABLE READ`, pinned by a read of the table before the row is released) is taken, so the definition recorded is the one in force for the items read; the catalog and the data are separate databases, so this is a barrier, not one cross-database snapshot. SQLite reads all items in one deferred read transaction opened while the engine's write lock was held for the definition read, so writers continue during the copy; that long reader pins WAL checkpointing, so the WAL grows for the duration of the backup by the size of the backup rows written plus any concurrent writes. **MongoDB** (`$out` from the live collection) and **Cassandra** (paging the live table) are not point-in-time: items copied early reflect the table as it was when the copy started, items copied late reflect writes made during the copy. A backup can therefore hold a write without an earlier write it followed, a state the table was never in. Measured *(earlier build)* with a writer that updated an early item and then a late item, pair after pair, during `CreateBackup`: 16 of 66 pairs on MongoDB and 123 of 800 on Cassandra had the late write without the early one; 0 on PostgreSQL (of 228) and SQLite (of 230). Back up a MongoDB or Cassandra table while it is not being written if the backup must be consistent | -| `ListBackups` paging | At most 100 summaries per page, with `LastEvaluatedBackupArn`; `Limit` 1 to 100. `TimeRangeLowerBound`, `TimeRangeUpperBound`, and `BackupType` filter the list. `TableName` is checked as the model says (3 to 255, `[a-zA-Z0-9_.-]+`, the "N validation error(s) detected" texts). An `ExclusiveStartBackupArn` of another account is `ValidationException` "Invalid Request: The supplied ExclusiveStartBackupArn does not match the account"; of another region, or of a table other than `TableName`'s, `ValidationException` ("does not match the region", "does not match the table name predicate"); a time bound before the epoch is `ValidationException`; a lower bound above the upper bound is `ValidationException` "Time range lower bound must be less than or equal to upper bound" (all measured) | The same paging on all four backends: `Limit` outside 1 to 100 is `ValidationException`, `ExclusiveStartBackupArn` resumes after the given backup, and summaries are ordered by creation time then ARN. Summaries carry `TableId`. `TimeRangeLowerBound` (inclusive) and `TimeRangeUpperBound` (exclusive) keep backups created in that range, and `BackupType` keeps backups of that type (`USER` when absent, every type for `ALL`); every ExtendDB backup is `USER`, so `SYSTEM` and `AWS_BACKUP` list nothing. The filters apply before paging, so `Limit` and the cursor count only backups that pass. Measured on SQLite and PostgreSQL with whole-second bounds between two backups taken 2 s apart; the millisecond edges are from the engine's unit tests. The request checks above are the service's, with its texts where captured: the account text in full; the region and table-name texts complete the captured fragments on the account text's pattern ("Invalid Request: The supplied ExclusiveStartBackupArn does not match the region" / "... the table name predicate"), checked in that order after the ARN's shape; a bound before the epoch is refused with ExtendDB's own text naming the member; equal bounds list nothing. An `ExclusiveStartBackupArn` that is not in the listing (excluded by the filters, or deleted since the previous page) is placed by the creation time in its id; one whose id carries no time (the 13-digit ids of earlier builds, with or without the hex suffix: only the 14-digit form carries one) is then a `ValidationException`. A cursor in another region is accepted only when it is the stored ARN of a backup of the account (or of `TableName`), which is what `LastEvaluatedBackupArn` hands out for a backup created under an earlier `server.region`, so paging passes such a backup; any other cursor in another region is refused ("does not match the region"). If that backup is deleted between two pages, its ARN is no longer a stored one and the next page is refused the same way (a same-region cursor would be placed by its time instead); start the listing again. A `server.region` outside `[a-z0-9-]` gives every cursor the server hands out a region the ARN parser refuses, so on such a deployment the second page of any listing is "Invalid Request: BackupArn is not valid". `TableName` here is a table name only: the ARN form `CreateBackup` accepts is refused with the pattern clause (the service's behaviour for `ListBackups` with an ARN was not measured). A numeric `TableName` is refused with the text the service uses for a map ("Start of structure or map found where not expected"), not measured for a number. Every `ListBackups` request reads all of the account's backups (or all of the table's, with `TableName`) from storage and pages them in memory, so walking N backups 100 at a time reads about N²/100 rows in total *(from the code)* | +| Consistency of a backup | Point-in-time snapshot of the table | **PostgreSQL** and **SQLite** take the items from one snapshot, and so does **MongoDB** with a configured `[backup.store]`. How *(from the code)*: with a store, the request opens the table snapshot (`SnapshotEngine`, the module docs of `crates/storage/src/snapshot.rs` and the backends' `snapshot_engine.rs` are the record): on PostgreSQL one `REPEATABLE READ` transaction holds the definition and the items on the shared layout, and on the separate layout the catalog row is held shared until the data transaction is pinned, the items then paged by the data table's primary key under that transaction, each page cut at 64 MiB of item text; on SQLite one deferred read transaction opened without the engine's write lock, paged by key with the same cut; on MongoDB a session with `readConcern: snapshot`, so items copied late reflect the same instant as items copied early. That snapshot is the majority-committed one, which can trail `DescribeTable`'s `local` read by the majority-commit lag, so a CreateBackup sent as soon as `DescribeTable` shows a new table `ACTIVE` could find it missing; the open then waits, at most 2 s, until the snapshot shows the table, and answers `TableNotFoundException` only for a table a `local` read does not show `ACTIVE` either. Without a store, PostgreSQL holds the table's catalog row shared from the first definition read until the data snapshot (`REPEATABLE READ`, pinned by a read of the table before the row is released) is taken, so the definition recorded is the one in force for the items read; the catalog and the data are separate databases, so this is a barrier, not one cross-database snapshot. SQLite reads all items in one deferred read transaction opened while the engine's write lock was held for the definition read, so writers continue during the copy; that long reader pins WAL checkpointing, so the WAL grows for the duration of the backup by the size of the backup rows written plus any concurrent writes. **MongoDB without a store** (`$out` from the live collection) and **Cassandra** (paging the live table) are not point-in-time: items copied early reflect the table as it was when the copy started, items copied late reflect writes made during the copy. A backup can therefore hold a write without an earlier write it followed, a state the table was never in. Measured *(earlier build)* with a writer that updated an early item and then a late item, pair after pair, during `CreateBackup`: 16 of 66 pairs on MongoDB and 123 of 800 on Cassandra had the late write without the early one; 0 on PostgreSQL (of 228) and SQLite (of 230). Back up a Cassandra table, or a MongoDB table on a server without a store, while it is not being written if the backup must be consistent | +| `CreateBackup` of a table holding an `N` value the wire did not validate | Every stored value is a valid number, and a backup holds the table as it is | With a configured `[backup.store]` the backup fails. Four write paths of earlier builds stored an `N` value such as `1E126` or `abc` without validating it (#618: `BatchWriteItem`, `TransactWriteItems`, `UpdateItem` on an absent item in the key it creates, `ImportTable`), and `UpdateItem` and a `TransactWriteItems` `Update` of builds before #433 stored an attribute name over 65,535 bytes without checking it, and the store-backed job validates every item before encoding it with the rule the restore applies (`validate_restorable_item`: every `N` and `NS` member a valid number, the nesting depth, the key shape (each key attribute present, of its declared type, not empty, an `N` key normalized), the item size against the instance's `max_item_size_bytes`, the hash and range key sizes and the attribute name sizes against the instance's limits, and for an item over the default item limit the data file line against the format's 32 MiB cap), so no backup that no restore can load is reported `AVAILABLE`. After the first refusal the job reads and validates the rest of the snapshot, logs each refused item with its key, attribute path and reason beside the backup ARN (up to 1,000, then a count), and fails the backup with a reason giving the count and the first item, naming #618 only when that item's refusal is an `N` value ("3 item(s) cannot be restored; first: key {...}, attribute amount: Number overflow... (#618)"); an item over `max_item_size_bytes` (stored under a higher limit) fails it the same way with a reason naming the size and the limit and not #618. The client sees `CREATING` and then `BackupNotFound`; the reason reaches the operator (`extenddb backup list --failed`, and the log, which a tenant's support case can cite; every tenant-supplied text in the line is written as a JSON string's content, control characters and the Unicode line separators U+0085, U+2028 and U+2029 escaped, so a newline or a line separator in an attribute name or a number value cannot forge a second log record); the way out is to rewrite or delete each named item through the wire (a bad range key cannot be rewritten, since a PutItem with that key is refused; `DeleteItem` removes it). The in-catalog create (no store, Cassandra) backs such a table up and restores it as before (wire tests on PostgreSQL and SQLite for each path) | +| Restoring into MongoDB an `N` key value past Decimal128 | Not applicable | A store-backed restore into the MongoDB backend of a backup holding an `N` sort-key value of more than 34 significant digits (the sort-key column is BSON Decimal128) fails: the restore job applies the file through the write path, whose `ValidationException` ("Numeric sort key value '' exceeds supported precision (Decimal128, 34 significant digits)") becomes the restore's `FAILED` reason, so the client sees the target `CREATING` and then gone and the server log carries the text. PostgreSQL, SQLite and Cassandra restore the same value (their sort-key columns are numeric or decimal). A non-key `N` of up to 38 digits is stored as the item's text and restores intact on every backend, MongoDB included *(wire tests against the MongoDB runner)* | +| `ListBackups` paging | At most 100 summaries per page, with `LastEvaluatedBackupArn`; `Limit` 1 to 100. `TimeRangeLowerBound`, `TimeRangeUpperBound`, and `BackupType` filter the list. `TableName` is checked as the model says (3 to 255, `[a-zA-Z0-9_.-]+`, the "N validation error(s) detected" texts). An `ExclusiveStartBackupArn` of another account is `ValidationException` "Invalid Request: The supplied ExclusiveStartBackupArn does not match the account"; of another region, or of a table other than `TableName`'s, `ValidationException` ("does not match the region", "does not match the table name predicate"); a time bound before the epoch is `ValidationException`; a lower bound above the upper bound is `ValidationException` "Time range lower bound must be less than or equal to upper bound" (all measured) | The same paging on all four backends: `Limit` outside 1 to 100 is `ValidationException`, `ExclusiveStartBackupArn` resumes after the given backup, and summaries are ordered by creation time then ARN. Summaries carry `TableId`. `TimeRangeLowerBound` (inclusive) and `TimeRangeUpperBound` (exclusive) keep backups created in that range, and `BackupType` keeps backups of that type (`USER` when absent, every type for `ALL`); every ExtendDB backup is `USER`, so `SYSTEM` and `AWS_BACKUP` list nothing. The filters apply before paging, so `Limit` and the cursor count only backups that pass. Measured on SQLite and PostgreSQL with whole-second bounds between two backups taken 2 s apart; the millisecond edges are from the engine's unit tests. The request checks above are the service's, with its texts where captured: the account text in full; the region and table-name texts complete the captured fragments on the account text's pattern ("Invalid Request: The supplied ExclusiveStartBackupArn does not match the region" / "... the table name predicate"), checked in that order after the ARN's shape; a bound before the epoch is refused with ExtendDB's own text naming the member; equal bounds list nothing. An `ExclusiveStartBackupArn` that is not in the listing (excluded by the filters, or deleted since the previous page) is placed by the creation time in its id; one whose id carries no time (the 13-digit ids of earlier builds, with or without the hex suffix: only the 14-digit form carries one) is then a `ValidationException`. A cursor in another region is accepted only when it is the stored ARN of a backup of the account (or of `TableName`), which is what `LastEvaluatedBackupArn` hands out for a backup created under an earlier `server.region`, so paging passes such a backup; any other cursor in another region is refused ("does not match the region"). If that backup is deleted between two pages, its ARN is no longer a stored one and the next page is refused the same way (a same-region cursor would be placed by its time instead); start the listing again. Under a `server.region` outside `[a-z0-9-]` the cursor is still the stored ARN: a listing of backups created under a valid region pages, and a cursor that is the ARN of a backup an earlier release created under the invalid region is refused with "Invalid Request: BackupArn is not valid". `TableName` is a table name or a table ARN, as `CreateBackup` accepts it (the service accepts both, measured on DynamoDB): the ARN form lists what the name lists, an ARN whose region is not the server's (compared as written) is `ValidationException` "1 validation error detected: Invalid AWS region in ''" (the service's answer, measured on DynamoDB twice), another account's ARN is `AccessDeniedException`, and the cursor's table check runs against the resolved name. A numeric `TableName` is refused with the text the service uses for a map ("Start of structure or map found where not expected"), not measured for a number. Every `ListBackups` request reads all of the account's backups (or all of the table's, with `TableName`) from storage and pages them in memory, so walking N backups 100 at a time reads about N²/100 rows in total *(from the code)* | | Backup ARN and creation time | Backup id is 14-digit zero-padded epoch milliseconds, `-`, 8 hex digits; `BackupCreationDateTime` has sub-second precision | The same shape on all four backends; `BackupCreationDateTime` is the millisecond in the ARN | -| Running two servers against one MongoDB catalog | Not applicable | MongoDB uses the fixed database names `extenddb_catalog` and `extenddb_data` whatever `--catalog-db` and `--data-db` say, and takes no serve lock: two servers on one replica set share one catalog and both start. The backup and restore sweeps act only on work whose heartbeat is more than 60 s old, so one server does not remove another live server's restore *(from the code and storage-level tests; not checked with two servers)*. Running more than one server per catalog is otherwise unsupported (see the deployment guide) | +| Running two servers against one MongoDB catalog | Not applicable | MongoDB uses the fixed database names `extenddb_catalog` and `extenddb_data` whatever `--catalog-db` and `--data-db` say, and takes no serve lock: two servers on one replica set share one catalog and both start. Each server's reconciler acts only on a restore whose job record's heartbeat is older than `backup.stale_job_after_secs` (default 300 s) on the database's clock, so one server does not remove another live server's restore; every heartbeat a restore writes, the registration's first one included, is stamped with the database's `$$NOW` inside the registration's transaction, so a host whose clock is behind the database's plays no part in that judgement *(from the code and storage-level tests)*. Running more than one server per catalog is otherwise unsupported (see the deployment guide) | ### `RestoreTableFromBackup` overrides @@ -206,7 +210,7 @@ ExtendDB exposes runtime settings that have no DynamoDB equivalent: | `control_plane_delay_seconds` | 5 | Simulated delay for table state transitions (CREATING → ACTIVE, DELETING → removed) | | `index_propagation_delay_ms` | 10 | System-wide default propagation delay for asynchronous secondary-index maintenance (milliseconds), covering GSIs and vector indexes alike. Per-GSI overrides stored in catalog; vector indexes have no per-index override. 0 = synchronous. Accepts the pre-rename name `gsi_propagation_delay_ms` as a deprecated alias, and a catalog created before the rename keeps honouring a value stored under it. | | `throttling_enabled` | `true` | Enable provisioned capacity throttling (token bucket per table/partition) | -| `enable_multipart_keys` | `false` | Enable multi-part base table key extension | +| `limits.allow_multipart_table_keys` | `false` | Enable multi-part base table key extension | | `log_level` | `info` | Runtime log level (trace, debug, info, warn, error) | | `sqlx_log_level` | `warn` | Separate log level for sqlx query traces | | `allow_credential_import` | `true` | Allow importing credentials via the management API | diff --git a/docs/dynamodb-limits.md b/docs/dynamodb-limits.md index 50151741d..39167fb48 100755 --- a/docs/dynamodb-limits.md +++ b/docs/dynamodb-limits.md @@ -128,11 +128,13 @@ Source: [AWS DynamoDB Service Quotas](https://docs.aws.amazon.com/amazondynamodb | Limit | DynamoDB Value | Status | Notes | |-------|---------------|--------|-------| -| Concurrent restores | 50 | Partial | PostgreSQL: one server process runs at most 4 restores per account and at most half its `pool_size` (after the minimum of 10 is applied; 10 with the default `pool_size` of 20), capped at 16, in total. A restore that cannot get both an account slot and a server slot within 30 seconds is refused with `LimitExceededException`. Measured on an earlier build of this branch, with four restores running in one account: a fifth in the same account waited for a slot and answered after 3.1 s, when one of the four finished, while another account's restore answered in 0.05 s. The 30-second refusal is from the code and storage-level tests, not from a running server. MongoDB: the same bound, 4 per account and `min(16, pool/2)` per server where the pool is the engine client's `maxPoolSize` (`max_connections`, or the driver default of 10, so 5 restores), refused with `LimitExceededException` after 30 seconds; a restore request that is not registered within 60 seconds answers an internal error while the restore continues (storage-level tests; measured with 20 concurrent restores on a served default build: all accepted, server responsive). SQLite and Cassandra have no restore limit (five started at once were all answered without waiting for each other, on an earlier build); SQLite bounds `CreateBackup` instead: at most `pool_size / 2 - 1` (4 by default) at once, `LimitExceededException` after 30 seconds | -| `BackupName` and `TableName` on `CreateBackup`, `TableName` on `ListBackups` | Length 3 to 255, `[a-zA-Z0-9_.-]+` | Enforced | The service model's constraints, with its texts ("1 validation error detected: Value '' at 'backupName' failed to satisfy constraint: ...", "2 validation errors detected: ...; ..." when length and pattern both fail). Length is counted in characters. `BackupName` takes the model's 3 to 255; a `TableName` member takes the deployment's `[limits]` `min_table_name_length` and `max_table_name_length` (defaults 3 and 255), as `CreateTable` does, so a table the deployment can create can be backed up and listed. `CreateBackup`'s `TableName` may also be the table's ARN; `ListBackups`'s may not. The value is echoed in the message as sent | +| Concurrent restores | 50 | Partial | Every backend: one server process runs at most `[backup] max_concurrent_jobs` (default 4, at most 1024) backup and restore jobs at once, `CreateBackup` and `RestoreTableFromBackup` together and not per account; a request that finds every slot held waits up to 30 s for one (holding nothing while it waits: the permit is the last of its checks) and is then refused with `LimitExceededException` ("Too many backup and restore jobs are running on this server (at most N at once, the [backup] max_concurrent_jobs bound); retry later") and creates nothing (storage-level and engine tests; measured on PostgreSQL with `max_concurrent_jobs = 1` on an earlier build that did not wait: a running restore made a CreateBackup and a second restore refuse in under 60 ms). On PostgreSQL `pool_size` and `catalog_pool_size` are raised to at least `2 * max_concurrent_jobs + 10` so every admitted job has a connection. SQLite also bounds the snapshot slots `CreateBackup` and table snapshots (#395) share, a count derived from `[storage.sqlite] pool_size` (the formula is stated once, on that setting's documentation in `SqliteConfig`; 4 for the default pool) so half the pool stays free for ordinary requests; a server raises a smaller `pool_size` to `2 * (max_concurrent_jobs + 1)`, at most 34 (the slots are capped at 16), with a warning naming both numbers, so the slot bound refuses no job the `max_concurrent_jobs` permit admitted (no change at the defaults: a pool of 10 has 4 slots for 4 jobs); above 16 jobs a 17th concurrent job is refused by the slot. A `CreateBackup` that finds no slot free is refused at once with the same `LimitExceededException`, its parenthesis naming that bound and its count in place of `max_concurrent_jobs` ("at most N at once, the [storage.sqlite] pool_size snapshot-slot bound"); PostgreSQL bounds open table snapshots at half its (raised) pool the same way, naming `[storage.postgres] pool_size snapshot-slot`; a PostgreSQL job whose dedicated claim connection the database server refuses is refused with the same exception naming the server's connection slots ("the database server has no connection slot left for the job"), with no count; the same error while `extenddb backup purge` holds the catalog's maintenance claim names a backup maintenance command and the claim, not a bound. A restore request on any backend that is not registered within 60 seconds (`RESTORE_REGISTRATION_TIMEOUT`) answers an internal error while the restore continues in the background. A restore from an S3 store holds the compressed backup size at `spool_path` while it runs; a restore's write batch on SQLite and Cassandra is bounded at 32 MiB of item text, measured on the serialized item (not on the service's `item_size_bytes`), and on Cassandra the item is serialized once more to measure it, a CPU cost per item the bound buys; a restore ends by counting the target against the manifest's item total, which on Cassandra is a full `COUNT(*)` of the restored table (one scan of a CREATING table, run once per restore and once more on a resume without recorded progress; no run has measured its cost past 12,000 rows, the largest Cassandra restore measured; a transient failure of this count is retried a few times, and a count that keeps failing leaves the record RUNNING for the reconciler to resume rather than failing the restore and removing its loaded target) | +| `BackupName` and `TableName` on `CreateBackup`, `TableName` on `ListBackups` | Length 3 to 255, `[a-zA-Z0-9_.-]+` | Enforced | The service model's constraints, with its texts ("1 validation error detected: Value '' at 'backupName' failed to satisfy constraint: ...", "2 validation errors detected: ...; ..." when length and pattern both fail). Length is counted in characters. `BackupName` takes the model's 3 to 255; a `TableName` member takes the deployment's `[limits]` `min_table_name_length` and `max_table_name_length` (defaults 3 and 255), as `CreateTable` does, so a table the deployment can create can be listed and backed up whatever its name, as on the service: a name over 255 bytes (with `max_table_name_length` raised) or made only of dots fits no store prefix, so the store-backed create refuses it before it opens a snapshot or writes a row, and the in-catalog create (no store, Cassandra) carries it (see the differences document's backup creation row). Both members may also be the table's ARN; an ARN whose region is not the server's is `ValidationException` "1 validation error detected: Invalid AWS region in ''", the service's text. The value is echoed in the message as sent | | `BackupArn` | 37 to 1024 characters | Maximum enforced; minimum implied | Surrounding ASCII whitespace is trimmed first, then over 1024 bytes, not of the backup ARN shape, a control or non-ASCII character, or a region outside `[a-z0-9-]`, an account that is not digits, or a table name outside `[a-zA-Z0-9_.-]` is `ValidationException` "Invalid Request: BackupArn is not valid"; the right shape with an id that is not a backup id is "Invalid Backup ARN"; a digit-only account that is not the caller's is `AccessDeniedException`. No minimum is checked: the shortest string of the right shape is longer than 37 characters. Another region of the caller's account resolves (the server's region is tried first, then the ARN as written). `ListBackups`'s `ExclusiveStartBackupArn` must also match the account, the region, and `TableName` when given | | `ListBackups` page size | 100 | Enforced | `Limit` 1 to 100, default 100; more rows come back through `LastEvaluatedBackupArn` and `ExclusiveStartBackupArn`. Each request reads every backup of the account (or of the table, with `TableName`) and pages in memory, so walking N backups reads about N²/100 rows. `TimeRangeLowerBound`, `TimeRangeUpperBound`, and `BackupType` filter the list before paging; a bound before the epoch, or a lower bound above the upper bound ("Time range lower bound must be less than or equal to upper bound"), is `ValidationException` | -| Backup contents | Key schema, attribute definitions, items, indexes, billing mode, provisioned throughput, on-demand throughput (table and GSI level), SSE. Table class is not carried (a restore is `STANDARD`) | Partial | ExtendDB restores the same set, except a GSI's on-demand throughput (it records none per index); table class is not restored, as on the service. Streams, TTL, tags, and deletion protection are not restored, as on the service. Backups are stored inside the database the server uses; take database-level backups for off-host copies | +| Backup store | A backup lives in the service's own storage | N/A | `[backup.store]` selects where CreateBackup writes its backups on PostgreSQL, SQLite and MongoDB (Cassandra, any backend without a store, and SQLite on a platform where the serve lock is not taken (not Unix), write the in-catalog format) and where restores and deletes of store-backed backups read: a filesystem root (`backups/` beside a SQLite database file when the section is absent), an S3 bucket and prefix, or `disabled`. The knobs are `[backup]`: `data_file_target_bytes` 64 MiB (minimum 32 MiB), `heartbeat_interval_secs` 15, `stale_job_after_secs` 300 (at least three heartbeats), `failed_row_retention_secs` 7 days, `reconcile_interval_secs` 60, `max_concurrent_jobs` 4 per instance, `spool_path` (required with an S3 store except beside a SQLite database file). A filesystem store holds every backup's compressed size under its root, beside the database on the SQLite default, and `extenddb destroy` leaves it; a restore from an S3 store holds the compressed backup size at `spool_path` while it runs. Instances that share a catalog must share one store root; an unhealthy store refuses CreateBackup where it writes to the store and the restore and delete of store-backed backups with `ValidationException` naming the store's kind, and the server keeps serving (`/health` reports `degraded`). See the admin guide's `[backup.store]` | +| Backup data file | Not specified | N/A | A data file holds one item per line as DynamoDB JSON, gzip-compressed, in key order on a table whose key schema is orderable (`item_order` in the manifest); a line is at most 32 MiB (`MAX_LINE_BYTES`, which holds the largest line a 400 KB item can expand to, 27.7 MB); a file closes at the first item that takes its uncompressed bytes to or past `[backup] data_file_target_bytes` (default 64 MiB, minimum 32 MiB, the line cap), so it exceeds the knob by less than one line. The snapshot streams on PostgreSQL and SQLite page the table by key and cut each page at 64 MiB of item text (`SNAPSHOT_FETCH_BYTES`, a constant), so a page holds at most that plus one row; MongoDB's batches are bounded by the server's 16 MiB reply | +| Backup contents | Key schema, attribute definitions, items, indexes, billing mode, provisioned throughput, on-demand throughput (table and GSI level), SSE. Table class is not carried (a restore is `STANDARD`) | Partial | ExtendDB restores the same set, except a GSI's on-demand throughput (it records none per index); table class is not restored, as on the service. Streams, TTL, tags, and deletion protection are not restored, as on the service (a store-backed backup's manifest records the stream specification and TTL for `DescribeBackup`). With a configured store on PostgreSQL, SQLite and MongoDB the backup is written under the store's root and restores into the backend that took it, and into another backend once `extenddb backup sync` has given the backup a row in the target's catalog (the differences document's "Where backups live" row). Vector indexes are restored where the target supports them, and a backup of a multi-part-key table (the `limits.allow_multipart_table_keys` preview) restores with its item set intact (the engine's `a_multi_part_key_manifest_is_registered` and `tests/test_backup_job.py`'s multi-part cell, which restores such a backup on a server with the preview enabled and compares the restored item set with the source's); without one, and on Cassandra, it is stored inside the database the server uses, so take database-level backups for off-host copies there | | Point-in-time recovery | 35-day window | N/A | Not supported: `UpdateContinuousBackups` refuses to enable it (`ContinuousBackupsUnavailableException`) and `RestoreTableToPointInTime` refuses every restore (`PointInTimeRecoveryUnavailableException`); use on-demand backups (see differences-from-dynamodb.md) | | `RestoreTableFromBackup` overrides | `BillingModeOverride`, `ProvisionedThroughputOverride`, `GlobalSecondaryIndexOverride`, `LocalSecondaryIndexOverride`, `SSESpecificationOverride`, `OnDemandThroughputOverride`, `VectorIndexOverride` | Partial | All but `VectorIndexOverride` are applied as the service applies them: an index override list replaces the backup's indexes of that kind (empty restores none, a subset restores those, a GSI entry's throughput is the restored index's), `OnDemandThroughputOverride` is the restored table's on-demand throughput on an on-demand restore. Refused with `ValidationException` before the target is created: an index the backup does not have or a changed index definition, a throughput member contradicting the billing mode, `{0, 0}`, a `PROVISIONED` restore of a GSI without throughput ("Invalid Request: Must override ProvisionedThroughput if BillingMode is overridden to PROVISIONED for index "), a GSI entry's `OnDemandThroughput` on a `PROVISIONED` restore (on an on-demand restore it is accepted and not recorded), and `VectorIndexOverride`. See differences-from-dynamodb.md | diff --git a/docs/getting-started.md b/docs/getting-started.md index 712f34cfe..cd52dd17f 100755 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -269,7 +269,7 @@ Config: extenddb.toml Catalog database: extenddb_catalog --- Checking catalog migrations... - OK: Current: the catalog is at migration 5 + OK: Current: the catalog is at migration 6 --- Checking catalog connection... OK: Connected to catalog. --- Checking data database... @@ -287,7 +287,7 @@ extenddb runs as a daemon (background process) and logs to syslog. On startup it ```bash ./target/release/extenddb serve --config extenddb.toml -# extenddb 0.1.14 (catalog migration 5) starting on 127.0.0.1:18443 +# extenddb 0.1.14 (catalog migration 6) starting on 127.0.0.1:18443 # storage: postgres (postgresql://extenddb:***@localhost:5432/extenddb_catalog) ``` @@ -1366,7 +1366,7 @@ Each runner requires its tools to be installed. The runner checks prerequisites ```bash ./target/release/extenddb version # extenddb 0.1.14 -# catalog migration 5 (postgres) +# catalog migration 6 (postgres) # commit abc1234 # built 2026-04-17T12:00:00Z ``` @@ -1397,9 +1397,9 @@ The `storage.postgres.catalog_pool_size` setting controls the maximum number of **When to increase:** If you see elevated latency under concurrent load, the pool may be saturated. Requests queue at the pool level when all connections are in use. Increase `pool_size` (and `catalog_pool_size` if auth is enabled) to allow more concurrent transactions. -**Relationship to PostgreSQL `max_connections`:** The total connection footprint is `2 * pool_size + catalog_pool_size`, plus one connection per vector index build running on the server and one while a schema migration runs. The doubled term is the catalog pool and the data pool, both sized by `pool_size`; the third pool is the management pool sized by `catalog_pool_size`. A deployment whose catalog and data share one database is the only shape where the two `pool_size` pools collapse into one, and `extenddb init` does not produce it. Those extra sessions are opened outside both pools on purpose, for two different reasons. A build's ownership lock is session-scoped, so it needs a session that ends when the build does: a pooled connection would return to the pool still holding the lock. The migration lock is transaction-scoped, and needs a pinned connection because its explicit transaction has to retain one PostgreSQL backend until COMMIT, which is what stops a transaction-pooling proxy from letting the lock move between backends. PostgreSQL's default `max_connections` is 100, and the default pool sizes already need 60 of it. Ensure `2 * pool_size + catalog_pool_size`, plus the number of vector index builds you expect to overlap, does not exceed your PostgreSQL `max_connections` setting. +**Relationship to PostgreSQL `max_connections`:** The total connection footprint is `2 * pool_size + catalog_pool_size + [backup] max_concurrent_jobs`, plus one connection per vector index build running on the server and one while a schema migration runs. The `max_concurrent_jobs` term (default 4) is one dedicated connection per running backup or restore job, outside every pool, which holds the job's session-level advisory lock (its claim). The doubled term is the catalog pool and the data pool, both sized by `pool_size`; the third pool is the management pool sized by `catalog_pool_size`. A deployment whose catalog and data share one database is the only shape where the two `pool_size` pools collapse into one, and `extenddb init` does not produce it. Those extra sessions are opened outside both pools on purpose, for two different reasons. A build's ownership lock is session-scoped, so it needs a session that ends when the build does: a pooled connection would return to the pool still holding the lock. The migration lock is transaction-scoped, and needs a pinned connection because its explicit transaction has to retain one PostgreSQL backend until COMMIT, which is what stops a transaction-pooling proxy from letting the lock move between backends. PostgreSQL's default `max_connections` is 100, and the defaults already need 64 of it. Ensure `2 * pool_size + catalog_pool_size + max_concurrent_jobs`, plus the number of vector index builds you expect to overlap, does not exceed your PostgreSQL `max_connections` setting. -**Example:** To support 50 concurrent data operations with auth enabled, set both pools to 50 in `extenddb.toml`. That configuration can open 150 connections, so PostgreSQL needs `max_connections` raised to at least 150 plus headroom for overlapping vector index builds. The default of 100 is not enough for it: an operator who sizes for 100 here meets `FATAL: sorry, too many clients already` at exactly the concurrency the setting was chosen to support. +**Example:** To support 50 concurrent data operations with auth enabled, set both pools to 50 in `extenddb.toml`. That configuration can open 154 connections (the default `max_concurrent_jobs` of 4 included), so PostgreSQL needs `max_connections` raised to at least 154 plus headroom for overlapping vector index builds. The default of 100 is not enough for it: an operator who sizes for 100 here meets `FATAL: sorry, too many clients already` at exactly the concurrency the setting was chosen to support. ```toml [storage.postgres] diff --git a/docs/manuals/01-architecture-guide.md b/docs/manuals/01-architecture-guide.md index 412e37d1a..93ccf3ba9 100755 --- a/docs/manuals/01-architecture-guide.md +++ b/docs/manuals/01-architecture-guide.md @@ -196,7 +196,7 @@ still reports it as running, with the PID shown as unknown. Pass `serve --foreground --write-pid-file` to write one anyway, which restores both commands at the cost of needing a writable `run_dir`. 8. Spawn background tasks (log level polling, throttling polling, GSI delay polling, stream cleanup, TTL expiry, metrics persistence) -9. On SIGTERM/SIGINT: drain connections (5s timeout), exit +9. On SIGTERM/SIGINT: drain connections (5s timeout), end the running backup jobs through their failure path (30s), stop the background tasks (5s, then abort) and give up the serving registration, exit ## Catalog Model diff --git a/docs/manuals/03-usage-guide.md b/docs/manuals/03-usage-guide.md index c165b0dc5..7005f58ed 100755 --- a/docs/manuals/03-usage-guide.md +++ b/docs/manuals/03-usage-guide.md @@ -571,7 +571,7 @@ extenddb reproduces DynamoDB error responses exactly. Common errors: | PartiQL | ✗ (not planned) | | DAX | ✗ (not applicable) | | Global Tables | ✗ (future) | -| On-demand backups (CreateBackup / RestoreTableFromBackup) | ✓ | +| On-demand backups (CreateBackup / RestoreTableFromBackup) | ✓: CreateBackup to the configured filesystem or S3 backup store on PostgreSQL, SQLite and MongoDB (in-catalog without one and on Cassandra); RestoreTableFromBackup from the store into every backend (a filesystem or S3 store; a backup taken on one backend restores into another once the target's catalog has a row for it: copy its prefix under the target's root and run `extenddb backup sync`, or adopt the source's root with `extenddb backup sync --adopt-store`, which moves the root to the target's catalog: stop the source's servers first, and they then report the store owned by another catalog), and of in-catalog backups | | Point-in-time recovery | ✗ (`UpdateContinuousBackups` refuses to enable it; see differences-from-dynamodb.md) | | Import/Export (local filesystem) | ✓ (FileSource instead of S3) | diff --git a/docs/manuals/04-quickstart-setup-guide.md b/docs/manuals/04-quickstart-setup-guide.md index 6e8339c6b..08333d9dd 100755 --- a/docs/manuals/04-quickstart-setup-guide.md +++ b/docs/manuals/04-quickstart-setup-guide.md @@ -130,7 +130,7 @@ Check the version: ```bash ./target/release/extenddb version # extenddb 0.1.14 -# catalog migration 5 (postgres) +# catalog migration 6 (postgres) # commit abc1234 # built 2026-04-17T12:00:00Z ``` @@ -172,7 +172,7 @@ Config: extenddb.toml Catalog database: extenddb_catalog --- Checking catalog migrations... - OK: Current: the catalog is at migration 5 + OK: Current: the catalog is at migration 6 --- Checking catalog connection... OK: Connected to catalog. --- Checking data database... diff --git a/docs/manuals/05-admin-guide.md b/docs/manuals/05-admin-guide.md index e51da206c..5d283c7dc 100755 --- a/docs/manuals/05-admin-guide.md +++ b/docs/manuals/05-admin-guide.md @@ -40,7 +40,7 @@ Or manually: kill ``` -extenddb handles SIGTERM and SIGINT gracefully — it drains active connections for up to 5 seconds before exiting. +extenddb handles SIGTERM and SIGINT gracefully: it drains active connections for up to 5 seconds (the TLS listener and the plain-HTTP listener of a dev-mode build alike), then waits up to 30 seconds for its running backup jobs to end (a job the stop cancels runs its failure path, which marks its row FAILED and deletes what it wrote to the store; on a large S3 prefix that takes longer than the other bounds), then up to 10 seconds for its registration supervisor to return, up to 16 seconds to stop its background workers (5 seconds for them to return, 1 second for the ones then aborted, and 10 seconds more behind a stop the supervisor was still running), up to 10 seconds to give up its serving registration (kept, to expire by itself, while a background worker or a backup job may still run), and 5 seconds for its runtime's blocking threads, and exits. With no backup job running the job wait ends at once, and in an ordinary stop the supervisor, the workers and the release each take moments. A `CreateBackup` that is still waiting in its snapshot open or row insert when the stop begins writes no row, and its client sees the connection closed by the drain (measured: 8.2 s after the request was sent): the jobs' token, which turns such a request into `ServiceUnavailable` "The server is shutting down. Retry the request.", is cancelled after the drain, so that answer reaches a client only after a registration loss, when the token is cancelled while the listener still serves. `extenddb stop` waits up to 76 seconds, the sum of these bounds, before it reports that the server did not exit. Note that `extenddb stop` reads the PID file, which a server started with `serve --foreground` does not write by default. Stop a foreground server through @@ -53,9 +53,36 @@ SIGTERM directly. Alternatively start it with ```bash curl --cacert ~/.extenddb/tls/cert.pem https://127.0.0.1:18443/health -# {"status":"healthy"} +# {"components":{},"status":"healthy"} +# (the order of the keys depends on the build: the default PostgreSQL build +# writes "status" first, the SQLite build "components" first; read the keys) +# with a backup store configured: +# {"components":{"backup_store":{"kind":"filesystem","status":"healthy"}},"status":"healthy"} +# while it cannot be reached: +# {"components":{"backup_store":{"kind":"s3","reason":"unreachable","status":"unhealthy"}},"status":"degraded"} +# right after startup, before an S3 store's first check has returned: +# {"components":{"backup_store":{"kind":"s3","status":"checking"}},"status":"healthy"} ``` +`status` is `healthy`, or `degraded` when a component is unhealthy; the HTTP +status stays 200 either way, because a component problem never takes the data +plane out of rotation. The one component is `backup_store`, present whenever +the server has a store: one `[backup.store]` configures, or the default root +beside a file-backed SQLite database when the section is absent (absent when +no store is configured, when `kind = "disabled"`, and when a default root +failed its check with no rows for it); its `reason` is the problem's category in +one or two words (`unreachable`, `permission denied`, `failed validation`, +`identity file unreadable`, `identity file missing`, `identity mismatch`, +`owned by another catalog`, `manifest probe failed`, `catalog unreadable`) and +nothing else: the endpoint is unauthenticated, so the ids, ARNs and table +names a problem involves are in the server's log line only. An S3 store's +first check runs after the listener is up (the next section); until it +returns the component is `checking`, with the server `healthy`. The +reconciler re-runs the store's checks every `reconcile_interval_secs`, each +run bounded (5 s for a filesystem root, 30 s for a bucket, after which the +check is abandoned and the store is `unreachable` until a later check +answers), and the component recovers on its own. + Or without `curl` — useful inside a minimal container image, and what a Docker `HEALTHCHECK` should call: @@ -67,9 +94,8 @@ echo $? # 0 = healthy, 1 = not `healthcheck` reads the port from the config file, or takes an explicit `--endpoint https://host:port`. It accepts the server's self-signed certificate. -This checks liveness, not readiness. `/health` is a static handler that does not -query PostgreSQL, so it reports healthy even when the database has become -unreachable since startup. That is intentional for a liveness probe: one that +This checks liveness, not readiness. `/health` does not query the database, so +it reports healthy even when the database has become unreachable since startup. That is intentional for a liveness probe: one that failed on a database outage would restart every replica at once and prolong the outage. A database that is unreachable at startup stops the server from listening at all, so that case is still caught. @@ -90,7 +116,7 @@ These settings require a server restart to take effect. |-----|---------|-------------| | `bind_addr` | `127.0.0.1` | Network interface to bind | | `port` | `18443` | HTTP port | -| `region` | `us-east-1` | AWS region for ARN generation. Baked into every backup ARN the server issues, so it must stay within `[a-z0-9-]` (a value outside that class makes the server's own backup ARNs "not valid"; the server logs a warning at startup). A backup ARN of the caller's account in another region is resolved in the server's region first, then as written; see the upgrade manual before changing this on a deployment with backups | +| `region` | `us-east-1` | AWS region for ARN generation. Baked into every backup ARN the server issues, so it must stay within `[a-z0-9-]` (under a value outside that class `CreateBackup` is refused with a `ValidationException` naming `server.region`, since no backup ARN can be formed; backups taken before the upgrade under such a region keep an ARN `DescribeBackup`, `DeleteBackup`, and `RestoreTableFromBackup` refuse as "not valid"; the server logs a warning at startup). A backup ARN of the caller's account in another region is resolved in the server's region first, then as written; see the upgrade manual before changing this on a deployment with backups | #### [storage] @@ -106,17 +132,82 @@ These settings require a server restart to take effect. | `pool_size` | `20` | Maximum concurrent database connections (minimum: 10) | | `catalog_pool_size` | (= `pool_size`) | Maximum connections for management/authz pool (minimum: 10) | -Restores on PostgreSQL are bounded per server process: at most 4 per account and at most half of `pool_size` in total (after its minimum of 10 is applied, capped at 16; 10 with the default `pool_size` of 20). A restore that cannot get both within 30 seconds is refused with `LimitExceededException`. Each running restore holds one connection from this pool for its whole copy, plus one dedicated connection outside the pool for its lock. +Backup and restore jobs are bounded per server process by `[backup] max_concurrent_jobs` (default 4, below), not per account; a request that finds every slot held waits up to 30 s for one and is then refused with `LimitExceededException`. The server raises `pool_size` and `catalog_pool_size` to at least `2 * max_concurrent_jobs + 10` and logs the raise with both numbers, so every job the bound admits has a connection. Each running restore holds one connection from this pool for its whole copy, plus one dedicated connection outside the pool for its claim. -`RestoreTableFromBackup` on PostgreSQL holds a session-level advisory lock for the whole copy, so ExtendDB must reach the catalog database directly or through a session-mode pooler. Behind a transaction-mode pooler (for example PgBouncer with `pool_mode = transaction`) restore locks are not released and the abandoned-restore sweep can remove a restore that is still running; do not run restores through one. +`RestoreTableFromBackup` on PostgreSQL holds a session-level advisory lock (the job's claim) for the whole copy, so ExtendDB must reach the catalog database directly or through a session-mode pooler. Behind a transaction-mode pooler (for example PgBouncer with `pool_mode = transaction`) the lock lands on a pooled server connection and is not released with the job, so later jobs can be refused as already claimed; do not run backups or restores through one. #### [storage.sqlite] | Key | Default | Description | |-----|---------|-------------| -| `path` | `extenddb.sqlite` | Database file (`init --sqlite-path` writes it); `:memory:` for an ephemeral database | +| `path` | `extenddb.sqlite` | Database file (`init --sqlite-path` writes it); `:memory:` for an ephemeral database, whose catalog is new at every start: with `[backup.store]` configured the server adopts the root an earlier in-memory catalog owned and syncs its backups into the new catalog before it serves, within 60 s (an S3 store in the reconciler's first cycle; a sync not finished in 60 s, or one that met a store or catalog error on a prefix, runs again in the next cycle). Every run lists the root from its first key, so the startup sync completes only while listing the whole root fits in 60 s; past that it stays pending, is retried every cycle, and the backups beyond what a run reaches are not served. On a filesystem store a run abandoned at the 60 s bound does not stop its listing: the walk of the root runs to its end on the blocking pool, holding the listing it builds, and the next cycle starts another walk. On a filesystem store the server does not answer, `/health` included, until the startup sync ends (a liveness probe with a short timeout restarts such a server on a large root, each start syncing from the first key), and a signal then ends the process at once. Until the log line `startup sync of the in-memory catalog: N backup(s) adopted` the earlier backups answer `BackupNotFoundException`, although `/health` may already report the store healthy; a prefix that line reports as having met a store or catalog error stays so until a later cycle's line. A server killed inside a DeleteBackup (between the DELETING write and the manifest's deletion), or restarted after a DeleteBackup whose manifest deletion failed, serves that backup again at its next start, since the startup sync adopts the complete prefix; a server killed during a CreateBackup leaves a partial prefix that no command can remove, since every `extenddb backup` command refuses an in-memory database. Start a new in-memory server only after the old one has exited: one started beside it adopts the root while the old one still acts on it until a later check finds it adopted, its deletion fence (the identity paragraph) excepted. Every `extenddb backup` command refuses it | | `pool_size` | `10` | Maximum connections. Writes are serialized by the engine whatever the size. A file database raises `pool_size = 1` to 2 and logs it at info (`SQLite pool_size 1 raised to 2 for a file-backed database: ...`), because a backup holds one reader connection for its whole duration while its batches write through another. Use 3 or more so a running backup does not contend with ordinary requests for connections | +#### [backup] + +The knobs backup and restore jobs and the reconciler read; every key has a default, and the section may be left out. + +| Key | Default | Description | +|-----|---------|-------------| +| `data_file_target_bytes` | `67108864` | Uncompressed bytes per backup data file (minimum 32 MiB, the cap on one line): a data file closes at the first item that takes it to or past this, so it exceeds it by less than one line | +| `heartbeat_interval_secs` | `15` | The job heartbeat timer. The store-backed backup job heartbeats every `max(heartbeat_interval_secs, 15)` s and starts no store write unless a beat confirmed its claim within two periods or a beat is still in flight (on SQLite a beat waits behind any writer holding the engine write lock, a GSI backfill say, for as long as it holds it, and a backup goes on writing meanwhile; on the other backends a beat in flight counts for one period from its start), the claim's insert counting as the first beat; the in-catalog backup and restore paths heartbeat every 15 s whatever this says | +| `stale_job_after_secs` | `300` | Heartbeat age after which a job's owner is dead (on PostgreSQL, once the owner's claim session has ended as well) and the reconciler resolves the job: a dead backup job with a complete manifest under its prefix is AVAILABLE, one without is FAILED and its prefix deleted; a dead in-catalog restore job is FAILED and its target removed; a dead store-backed restore is resumed from its recorded progress (the differences document's restore-interruption row). Must be at least three heartbeat periods, where the period is the larger of `heartbeat_interval_secs` and the 15 s the in-catalog jobs heartbeat at (so 45 at the least, whatever `heartbeat_interval_secs` says); on SQLite a long write transaction delays a heartbeat, so keep a margin above that floor | +| `failed_row_retention_secs` | `604800` | How long FAILED and DELETED backup rows of the configured store are kept once their objects are gone (7 days); `extenddb backup list --failed` prints the FAILED ones with their reason. At least `stale_job_after_secs`: the reconciler deletes a stopped job's late object only under its row, so the row outlives the window in which that job can still write; a smaller value is refused at startup | +| `reconcile_interval_secs` | `60` | The reconciler's cycle; it also runs once at startup. Each cycle checks the store, resolves backup and restore jobs whose owner is dead, finishes deletions, deletes the objects of FAILED backups once their heartbeat is older than `stale_job_after_secs`, and removes tombstones past `failed_row_retention_secs` (the differences document's restore-interruption row lists the duties) | +| `max_concurrent_jobs` | `4` | Backup and restore jobs this instance runs at once (at most 1024); a request over it waits up to 30 s for a slot, then `LimitExceededException`. On PostgreSQL the pool sizes are raised to `2 * max_concurrent_jobs + 10` when smaller; on SQLite `[storage.sqlite] pool_size` is raised to `2 * (max_concurrent_jobs + 1)`, at most 34, when its snapshot slots (`pool_size / 2 - 1`, between 1 and 16) would refuse a job this knob admits. Memory: a running backup holds about 490 MiB (PostgreSQL) or 650 MiB (SQLite) at most in the pipeline's own terms for a table of items at the 32 MiB line cap, a few tens of MiB for 200-byte items (the figures are for the default `max_item_size_bytes`, 400 KB, where no stored item's line exceeds the cap; on an instance that raises it the terms are at the largest line a stored item expands to, about 68 times the item's size for the worst shape); with glibc's default allocator the process's high-water mark grows with the number of large items a job reads (measured 760 MiB over idle at 48 items of 27.7 MB on a quiet host and 1,281 MiB on a loaded one, 1.39 and 2.35 times the terms at that item size) because the items' strings are allocated on one thread and freed on another and the per-thread arenas keep that memory. Set `MALLOC_ARENA_MAX=1` in the server's environment (the deployment guide's service unit carries the line, commented out) on a host that backs up large items: the peak is then the terms' and does not grow with the table (measured 452 to 481 MiB over idle at 12, 24 and 48 items of 27.7 MB on SQLite, in two series on two hosts' loads, and 340 MiB `VmHWM` (about 200 MiB over idle) at 12 items on PostgreSQL); one arena serializes the worker threads' allocations on one lock and costs the request path a few percent (measured 3.5 to 9.2% on GetItem, Query and Scan on SQLite and about 2% on PostgreSQL (the scans, 2.1%), where the request path waits on the database on PostgreSQL). Size the host for 1.2 times the terms (about 590 MiB on PostgreSQL, 780 MiB on SQLite) times this knob under `MALLOC_ARENA_MAX=1`; without it the peak grows with the large rows a job reads and no bound was measured (1,281 MiB over idle at 48 items of 27.7 MB is the largest seen). A running restore holds the parsed manifest, one decoded data-file line and its parse, the decoded items of 32 MiB of data-file text plus one line per chunk (the chunk closes with the item that reaches the bound) at four chunk positions between the decoder and the load (two in the channel, the one being filled, the one being drained; a chunk also closes at 1,000 items, and the chunks never hold more than one data file's items), and the backend's write batch: at most 32 MiB of item text on SQLite and Cassandra (10,000 items when they are smaller), held beside the decoded items of that text; 500 items (or 4 MiB) per INSERT statement inside one transaction per data file on PostgreSQL; 1,000 documents per `insert_many` on MongoDB (the driver splits the call); times this knob. For a table of large items that is the same order as the backup, not less: measured `VmHWM` over idle from a fresh start (`docs/design/15-backup-restore-bench-pr9.md`), 24 items of 27.7 MB restore with a peak of 848 MiB on SQLite and 778 MiB on PostgreSQL under the default allocator (848 and 1,013 MiB at 12 and 48 such items on SQLite: the peak grows with the large items a restore reads, as the backup's does, and no bound was measured), and 392 MiB on SQLite and 427 MiB on PostgreSQL under `MALLOC_ARENA_MAX=1`, the same setting as above; 1,200 items of 350 KB in one data file 270 MiB on SQLite and 156 MiB on PostgreSQL; 1M items of 200 bytes under a composite key with no secondary index, the backup's lines in key order as the server's own backups are, 89 MiB on SQLite (the write connection's page cache and the final `COUNT(*)`, each bounded by `cache_size`; the count's share was 22 to 28 MiB at 1M and at 3M rows) and 16 MiB on PostgreSQL, and the same 1M items as an unordered backup (every MongoDB-sourced backup and every multi-part-key table) 144 to 164 MiB on SQLite in 66 to 90 s against 27 to 56 s key-ordered (scattered inserts dirty the whole base b-tree; the restore's buffers are the same) and 16 MiB on PostgreSQL; the same items under the same key with two full-projection GSIs 143 to 225 MiB on SQLite at 100,000 items in three runs and 165 to 172 MiB at 300,000, and 23 to 34 MiB on PostgreSQL at both: the excess on SQLite is the backend's own per-connection page cache (`cache_size`, 32 MiB) holding the index tables' b-tree pages plus the allocator's arenas (about a third less under `MALLOC_ARENA_MAX=1`), not the restore's buffers, which the 32 MiB batch bound holds whatever the indexes; it is bounded by `cache_size` and grows sub-linearly with the table. Size the host for about 450 MiB per running restore of large items under `MALLOC_ARENA_MAX=1`, times this knob, plus about 32 MiB of page cache per running restore of an indexed table on SQLite, and read the 1M figure for 200-byte items as the key-ordered shape's: an unordered backup of them restores in about 1.5 to 1.8 times that on SQLite | +| `spool_path` | (unset) | Restore download area for an S3 store, created `0700` with files `0600`, holding a `.lock` another instance on the host refuses. A store-backed restore from an S3 store downloads each data file here under `restore-
    /` (`.json.gz`, `` zero-padded to six digits, with a `.verified` marker holding the file's sha256, written only after the file's digests matched and trusted only while it names the digest the manifest records, plus the manifest and sidecar), reads it from there, and removes the entry when the restore ends, except when it ended because the record had become another instance's (its claim lost, a status write that matched no record), where the sweep removes it; an entry left by an earlier run of this host for a record another host then owned is removed before a new run of the same restore uses it; the reconciler sweeps the `restore-` entry of a restore no longer RUNNING with this host's `spool_host`. Its disk cost is the compressed backup of the restore in flight. A filesystem store needs no spool (files are hashed in place and read again). Required with an S3 store, except on file-backed SQLite where it defaults to `backup-spool/` beside the database file. Refused inside or containing a filesystem store root, and when it is or contains the database file's directory | + +#### [backup.store] + +The store CreateBackup writes to on PostgreSQL, SQLite and MongoDB, and the one restores and deletes of store-backed backups read. The section's keys, defaults and the refusals of a bad value are the backup store crate's (`crates/backup-store/src/config.rs`); the table below is what the server adds. One definition of the shape exists, so an unknown key anywhere under `[backup]` is refused at load. + +| Key | Default | Description | +|-----|---------|-------------| +| `kind` | see below | `filesystem`, `s3`, or `disabled` | +| `filesystem.path` | `backups/` beside a SQLite database file | The root directory of a filesystem store. It must exist, be owned by the extenddb user with mode `0700`, and sit under directories owned by root or that user which others cannot write to; `validate` refuses anything else at startup. A relative path, like a relative `spool_path`, is resolved against the working directory the server was started from (the daemon's own working directory is `/`) | +| `s3.bucket`, `s3.prefix`, `s3.region`, `s3.endpoint`, `s3.force_path_style`, `s3.allow_insecure_endpoint`, `s3.upload_part_size_bytes`, `s3.max_concurrent_uploads` | | The S3 store; see the crate's `config` module doc. Credentials come from the AWS SDK's chain and are never in the file | + +When the section is absent, the backend decides: on file-backed SQLite it is a filesystem store at `backups/` beside the database file, created `0700` if missing, whose validation or identity-check failure is a logged warning that leaves the store unconfigured rather than a startup error or an unhealthy store while the catalog has no backup rows for that root (so an upgrade never stops a server that never configured backups, and CreateBackup keeps working on one); once the catalog has rows for the default root, the root has been used and is handled as a configured one: a failure of its identity check or manifest probe is an unhealthy store (reported on `/health`, its rows' restores and deletes refused, `backup purge` refusing on it rather than reporting no store), and a refusal of its `open` or validation a startup error naming the root; on PostgreSQL, MongoDB, Cassandra and an in-memory SQLite database there is no default and the store is unconfigured, logged at startup. `kind = "disabled"` leaves it unconfigured on purpose on any backend. Without a configured store, CreateBackup writes the in-catalog backup format and logs a deprecation warning naming `[backup.store]` on each call. With one configured, CreateBackup on PostgreSQL, SQLite and MongoDB writes the backup to the store: the request takes the table's snapshot (after its checks and after any wait for a job permit, up to 30 s, so the backup is the table at that open and a write acknowledged during the wait is in it), writes the catalog row and answers `CREATING`, and a job of the server uploads gzip data files and the manifest under `/
    //` below the root and marks the row `AVAILABLE` (the job's log lines carry the ARN: `backup : started, items in the table as last counted by the stats pass, key-ordered` (or `unordered`, for a table whose key shape or column collation has no format order) at the start, the stats pass's count, which is 0 on a table the pass has not reached, and `backup : AVAILABLE, items, bytes written in s` at the end, with the backup's own count); Cassandra, which has no table snapshot, keeps the in-catalog create with a store configured. Backups of either format stay listable, describable, deletable and restorable: a store-backed backup is restored by the restore job (the differences document's "How `RestoreTableFromBackup` runs" row; the `spool_path` row above and the `remove-target` command below are its operator surface), an in-catalog one by the previous path. + +Startup opens the configured store and, for a filesystem store, validates it and checks its identity before the listener is up (bounded by 5 s: a mount that stops answering after the root is opened leaves the server starting with the store `checking`; a root that does not answer at startup holds the server before the listener, as the database directory would, with the port bound and silent); an S3 store's validation and identity check are requests to the endpoint, and run after the listener is up, in the reconciler's first cycle, so an endpoint that stalls never holds the data plane down (the component is `checking` until the first result, and the operations that touch the store refuse meanwhile as they do while it is unhealthy). A configuration error, every one decided without a request to the store, stops startup with the key, path or bucket in the message: a root that does not exist or fails the owner-only rule, a filesystem root another user can reach, a filesystem root that is or contains the database directory, a bucket name the SDK routes to S3 Express, a bad part size or prefix, an insecure endpoint, an S3 store without `spool_path` on a backend with no default, a `spool_path` that breaks its placement rule, that another live instance holds or that exists with a mode other than `0700`. A transport failure (no region, no credentials, an endpoint that refuses connections or does not answer within the check's bound), a missing or denied bucket, an I/O failure of a filesystem root's probe write or resolution (a full or read-only disk, a stale network handle), or a failed identity check leaves the server running with the store unhealthy: the problem is logged once at error, `/health` reports the `backup_store` component unhealthy (status `degraded`), CreateBackup (where it writes to the store) and the restore and delete of a store-backed backup are refused with a `ValidationException` that names the store's kind and the problem's category and not its root, the reconciler acts on no row of the store, and the check runs again each `reconcile_interval_secs` until it passes, at which point the server logs `is healthy again` and resumes without a restart. Each check's store calls are bounded (5 s for a filesystem root, 30 s for a bucket); the catalog's own reads (the catalog id, the rows' store ids, the newest AVAILABLE row) run beside the calls and not under the bound, so a slow catalog costs the check time and never marks the store unhealthy. A store that has stopped answering costs a reconciler cycle that bound and nothing more, the cycle's other duties run as before, and the abandoned check is logged; on a filesystem root the abandoned check's call stays parked on one thread until the mount answers, no later cycle starts a check while it is (logged each cycle, the store staying `unreachable`), a root's calls run one at a time so that calls started before the first reached its bound (two `DeleteBackup` requests together, a `DeleteBackup` beside a check) wait for it under the bound, then have the bound for their own call (a slow call behind another finishes, where one bound over both would abandon it), and park no thread of their own, and `stop` or SIGTERM still ends the server, the runtime giving that thread 5 s before the process exits without it (after the job wait of up to 30 s when a backup job is running; see Stopping). The deletions `DeleteBackup` and the reconciler run against the store go through the same bound and rule: `DeleteBackup` answers the description with `BackupStatus` `DELETED` once its DELETING write has committed, and a deletion the store does not answer within the bound, or that leaves an object under the prefix, leaves the row DELETING for the reconciler and is logged, never an error to the client. A default SQLite root whose startup checks did not finish starts `checking` with its rule kept: a first check that fails while the catalog has no rows for it is the same warning and leaves the store unconfigured. CreateBackup on Cassandra is not refused by an unhealthy store: its in-catalog create touches no store. + +The store identity. The root carries `extenddb-store.json`: a random store id and the id of the catalog that owns the root. Startup creates it when the root has no file and the catalog has no rows for the root, and otherwise reads it and compares: a file naming another catalog, a store id that differs from the one the catalog's rows for the root carry, a root with no file while rows exist, or rows carrying more than one id each mark the store unhealthy. Instances that share a catalog must share one store root (one S3 bucket and prefix, or one shared mount): host-local roots configured at the same path give each instance different bytes under one root label, so each check also `head`s the manifest of the newest AVAILABLE backup and marks the store unhealthy when it is absent, which is what a second instance with a local copy of the directory sees. The file is left by `backup purge` and by every deletion; remove it only when retiring the root, or after `extenddb destroy`: a catalog re-created with `init` has a new id, and the old root's file names the old one, so the re-initialised server reports the store owned by another catalog until `extenddb backup sync --adopt-store` makes the new catalog its owner (below), the file is removed or a new root configured (on the default SQLite root, that is the logged warning above for the server, which keeps CreateBackup on the in-catalog path, while `backup sync` handles that root as configured, so `--adopt-store` adopts it). The default root, `backups` beside the database file, is shared by every database file in that directory: `--adopt-store` on it takes the root from whichever catalog wrote its file, so give each database its own `[backup.store]` when several share a directory. A server on an in-memory SQLite database writes the file with `ephemeral_catalog: true`; at its next start it adopts a root whose file names another catalog and carries that marker, and no other, so it never takes a durable catalog's root, and then syncs the root's backups into its new catalog (the `path` row above: until that sync's log line, the earlier backups answer `BackupNotFoundException`). A catalog whose root another catalog adopted (by `--adopt-store`, or a second in-memory server's start) keeps acting on the root until a later check that can read the catalog finds it owned by another catalog, at the earliest one reconciler interval after the cycle in progress ends (a check whose catalog read fails leaves the health as it was), with one exception, its deletion fence: right before it deletes a backup's objects (DeleteBackup, the reconciler's deletions, a failed backup job's cleanup, `purge`), removes a FAILED or DELETED row and writes a CreateBackup's first object, and also before DeleteBackup and `purge` change the row, a server or `purge` re-reads `extenddb-store.json`, and when the file names another catalog, names this catalog under another store id than the one the store's check settled (a root replaced by another root of this catalog), or the root has none, it deletes nothing and refuses the request; a file naming another catalog also marks the store unhealthy (`owned by another catalog`) at once, and one of another store id `identity mismatch`. DeleteBackup takes a file it cannot read, or a root with no file, at the read before the row is changed as it takes a store that does not answer: it writes DELETING and answers `DELETED`, the read before the deletion refuses it, and the row stays DELETING for the reconciler, whose deletion runs the fence again. `--prune-missing`'s removal of a row is not fenced: it deletes no object. A refusal at the read before the row is changed (the file names another catalog or another store id) leaves the row as it was, so DescribeBackup still answers AVAILABLE. One at the read before the deletion (an adoption landed between the two) leaves an AVAILABLE row DELETING with nothing deleted (DeleteBackup and DescribeBackup there answer `DELETED`), which this catalog's reconciler finishes only if this catalog owns the root again, and a dead CREATING row that `purge` already removed has its prefix kept. The read narrows the window to the gap between it and the deletion; on S3 that gap is a listing and a `DeleteObjects`, each under the SDK's retries, so seconds when the store throttles. A check that read the file before the adoption and ends after a fence's refusal records the store healthy again until the next cycle's check: each deletion stays fenced in that time, but a restore is accepted. A backup job of the previous owner that fails after the adoption leaves its partial prefix (no manifest, and no row of the new owner) under the root, which no duty or command of the new owner removes. A write that has already passed that read still lands, so stop every server of the catalog a file names before adopting its root. While the catalog has no AVAILABLE backup for a root, two host-local directories at one path cannot be told apart (each instance creates its own file and id, and a file copied from one directory to the other gives both one id), and the manifest probe has nothing to probe: a backup one instance completed under its own directory looks, to the other instance's reconciler, like a dead owner's row with no manifest, which it marks FAILED, and the first instance's restart then deletes the only copy. Point every instance at one shared root before the first backup; copying `extenddb-store.json` between directories defeats the check rather than satisfying it. The identity file is read up to 64 KiB; a larger object under its key is reported unreadable. + +`extenddb destroy` removes the databases and leaves the store untouched: it is the copy that survives destroy. `extenddb backup list [--failed]` prints the catalog's rows for the configured store's root, every store id they carry, with each row's `store_root` and `store_id`; it is a read that runs beside a server (on SQLite it takes no serve lock), reads the identity file without creating one, and runs on an unhealthy store, naming the problem in its header. It lists backups only, never restore jobs: a restore's record is the job's log line, not a `backup list` row. `extenddb backup purge --confirm` deletes every backup of the configured store, each backup's objects (its own prefix, never a whole account prefix, so a prefix the catalog has no row for survives) and its catalog row outright, leaving no tombstone, and leaves `extenddb-store.json` and the `exports/` component; it needs the store healthy. It takes the catalog-level maintenance claim first and heartbeats it for the command's whole run, waits up to `2 * stale_job_after_secs` until no store-backed CreateBackup is running on any instance (a CREATING or FAILED row with a heartbeat within `stale_job_after_secs` on the catalog database's clock) and no restore is RUNNING, refuses and releases the claim if they have not finished, deletes a CREATING row the wait saw no live owner for as it deletes any other row, finishes any DeleteBackup left in progress as part of the purge (a backup a DeleteBackup finishes while the purge runs is counted as deleted, not left), and logs each deletion under the `extenddb::audit` target with the account and the ARN. While the claim is held, CreateBackup and RestoreTableFromBackup on every instance answer `LimitExceededException` saying that a backup maintenance command holds the catalog's maintenance claim (the message does not say which command; the second command's refusal and the claim's row do), and a second maintenance command fails at once naming the holder. Without `--confirm` it prints the plan, changes nothing and exits 0. `--config` is accepted before or after the subcommand. None of the backup commands runs the startup recovery a server runs (on PostgreSQL the vector index rebuild, the control-plane recovery and the GSI queue workers; on MongoDB the key-repair pass; on Cassandra the control-plane transitions; on SQLite the interrupted-restore and index passes): that work belongs to the servers, and a command that ran it could exit with it half done. RUNNING restores are read by status, with no age, so a restore whose record was stamped this instant holds the wait too. On SQLite `purge --confirm` opens the database as `serve` does and so takes the serve lock: stop the server first (while one runs on the file the command refuses with exit 6, as a second `serve` does); under that lock it first resolves what the stopped server left (a RUNNING restore is failed and its target removed, the CREATING target of a FAILED restore removed, an in-catalog CREATING backup removed, a DELETING row finished) and deletes a CREATING or FAILED row whatever its heartbeat says, a heartbeat stamped ahead of the clock by a backward clock step included (the server that wrote it is stopped, so the zero window the command passes selects by status alone), so such a job never makes it wait or refuse. That includes a RUNNING restore from a backup store, which a server would resume: under the serve lock the command fails it and removes its target, as for an in-catalog restore (a resume is a job, which the maintenance claim excludes), except one that recorded no progress while its target holds items, which is failed `target_not_empty` and its target left for `extenddb backup remove-target`. A server starting on the file removes a claim a killed purge left, so no job is refused for the stale window, as a purge run again after the killed one takes its claim over. On the other backends it runs beside live servers, and a claim whose holder died expires after `stale_job_after_secs`. On PostgreSQL, MongoDB and Cassandra the backup commands (`list`, `purge`, and `sync` and `remove-target` below) open the catalog as a server does and hold a serving registration for their run, named after the command (`extenddb backup list`, `extenddb backup purge`, `extenddb backup sync`, `extenddb backup remove-target`: on PostgreSQL the session's `application_name`, on MongoDB the `command` field of its `extenddb_servers` document, on Cassandra the `release` column of its `extenddb_servers` row, after the release and ` as `), and release it when they exit: `extenddb migrate --yes` waits for them as for a server and names them while it waits, and while a migration runs they refuse at once (exit 6), as they do on a catalog at another migration than the binary's (exit 3, with the `extenddb migrate --yes` advice `serve` prints). On every backend Ctrl-C or `SIGTERM` makes the command release what it holds and exit: the serving registration after the open, if the signal came during it; the claim at once while it waits, and before its next backup while it deletes (the release itself is bounded by 10 s, past which the claim is left to the stale rule), so an interrupted purge leaves no half-deleted backup and no claim to wait out; run it again for the rest. The dry run on an unhealthy store lists the rows and says that `--confirm` refuses until the store's check passes. + +`extenddb backup sync [--adopt-store] [--prune-missing]` rebuilds the catalog's rows from the configured store: after a lost or re-created catalog, or after a backup's prefix was copied under the root from another deployment or another backend. It reads the catalog's rows for the root (of every status and every store id they carry), then lists the whole root and handles each `/
    //` prefix as the listing passes it, checking it against those rows before it reads anything under it. Sync holds one prefix's object list at a time and the prefix of each manifest it saw; a filesystem store builds its listing of the whole root in memory first (about 72 bytes plus the key per object), an S3 store lists a page at a time. The report keeps an entry per prefix for the whole run, and it lists every prefix the catalog already has a row for among the skipped ones, so on a root of small backups the per-prefix term dominates: about 350 bytes per prefix at one object per prefix (62 MiB of peak RSS at 50,000 such prefixes and 112 MiB at 200,000, against 45 MiB for an empty root; about 165 bytes per object at 100 objects per prefix), measured on a SQLite debug build with a filesystem root and no rows. Of the catalog's rows it keeps each row's status, store id and table, and the key of each AVAILABLE row of the configured store, but every backend reads the rows whole before sync sees the first, so the read is the larger cost and grows with the rows of the root, tombstones within their retention included: about 22.5 KiB per row of a table with five GSIs (1,452 bytes of source table description per row; 610 MiB of peak RSS at 25,000 rows and 1,144 MiB at 50,000, against 47 MiB with none; 148 MiB at 50,000 rows whose description is `{}`), measured on a SQLite debug build on a 16-core x86_64 Linux host with glibc's default arenas. `backup list` peaks the same over those rows. A prefix with no row gets an AVAILABLE row under the configured store's identity when its manifest and sidecar verify, every data file the manifest lists is under the prefix at its recorded size, and the manifest's ARN names the prefix's account, table and backup id; an ARN of another region is adopted and logged (requests find it by its ARN as written), and one of another partition than `aws` is skipped, since every request refuses such an ARN. The row carries the manifest's backup name, size, item count and source table, and the backup id's instant as its creation time. It never adopts a prefix with a row of any status (CREATING, DELETING, DELETED, FAILED or AVAILABLE), one whose manifest, sidecar or data files are missing or do not match, one whose manifest names another backup, or anything under `exports/` or a table's `changelog/`, and prints each with its reason. Before the insert it reads the row under both spellings of the ARN (this server's region and the manifest's), and the insert happens only if no row exists: a CreateBackup whose row landed after sync read the rows keeps its row, as does a row of another root, account or table holding the backup id, and sync changes no existing row. It also prints the AVAILABLE rows whose manifest the listing does not hold. It exits 0 once it ran, skipped prefixes included, and 1 when a prefix was skipped because a store read or a catalog call failed (the run is incomplete; a second run adopts it), when the listing failed part-way (the backups adopted before the failure keep their rows), when a signal stopped it, or when `--prune-missing` left a row it may remove on a later run (a store or catalog call failed, its manifest's `head` included, or a restore is reading the backup: run the command again once the restore ends). A row `--prune-missing` leaves because its manifest is back or another writer moved it is reported, and the exit stays 0: + +``` +=== extenddb backup sync === +Store: filesystem (store id ) + +--- Adopted (an AVAILABLE row inserted for each): + + +--- Skipped (not adopted): + /
    //: not a complete backup: no manifest + +--- AVAILABLE rows whose manifest is not in the store (the rows were AVAILABLE when the catalog was read): + (none) + +=== extenddb backup sync complete: 1 backup(s) adopted, 1 prefix(es) skipped, 0 AVAILABLE row(s) whose manifest is gone === +``` + +Sync needs the store healthy: on an unhealthy store it refuses, exit 1, naming the problem and changing no row. That covers an empty root under existing rows, a root whose file carries another store id and a root without the newest AVAILABLE row's manifest, which is what an unmounted volume shows. When the newest AVAILABLE row's own manifest is gone, the store is unhealthy for the same reason, so plain sync and `--prune-missing` both refuse; put the prefix (or its manifest) back and run the command again. On PostgreSQL, MongoDB and Cassandra every sync takes the maintenance claim and heartbeats it, so no `purge` runs between its read of the rows and its insert; plain sync waits for no job, and while it holds the claim CreateBackup and RestoreTableFromBackup are refused, as during `purge`. A sync killed while it holds the claim (`SIGKILL`, a lost host) leaves it held until its heartbeat is `stale_job_after_secs` old (300 s by default): until then every instance refuses CreateBackup and RestoreTableFromBackup. On PostgreSQL the claim is a session's advisory lock, so the next maintenance command (the sync or `purge` run again) takes a killed command's claim over at once and ends that window; MongoDB and Cassandra take it over only once it is stale. `--prune-missing` and `--adopt-store` also wait for running jobs as `purge` does. A first Ctrl-C or `SIGTERM` stops any sync before its next prefix (or the wait at once), releases the claim it holds and exits 1; the backups adopted before the stop keep their rows. `--prune-missing` removes each AVAILABLE row whose manifest is gone outright, one line per row, and deletes nothing in the store: no DELETING or DELETED row is left for a deletion duty to act on, so on host-local roots (each instance's root at one path holding other bytes) a prune on the instance that lacks a backup's bytes removes its row for every instance of the catalog and leaves the bytes on the other host: the backup answers `BackupNotFoundException` on every instance, the one holding the bytes included, until a sync on that instance adopts them again. A row a restore is reading, or one another writer moved meanwhile, is reported and left (the first exits 1). The objects a removed row's prefix still holds stay until an operator deletes the prefix by hand: `purge` deletes only the prefixes of rows, and every later sync reports the prefix as not a complete backup. `--adopt-store` is the way a durable catalog takes a root whose `extenddb-store.json` names another catalog (plain sync refuses such a root, naming the option; a server on an in-memory database adopts, at startup, a root another in-memory catalog wrote, as the identity paragraph says): under the claim it logs its intent under `extenddb::audit`, rewrites the file's catalog id and backend, keeping the store id and creation time, reads the file back and logs the outcome (adopted, or the race lost when another adopter's write landed between the two), syncs, and then reads the file once more and fails, naming the owner, when another catalog adopted the root during the sync. A write that lands after both reads is found by this catalog's next check. It refuses when this catalog's rows for the root carry another store id, and on a root this catalog already owns it says so and syncs. Stop every server of the catalog the file names before running it: the previous owner finds the store owned by another catalog only at a later check that can read its catalog, and until then its deletion fence (the identity paragraph) refuses each deletion and CreateBackup once it re-reads the file, but a write already past that read lands. A second maintenance command while one holds the claim fails at once. On SQLite every form takes the serve lock: stop the server first (beside a running server it exits 6); under the lock it first resolves what the stopped server left, as `purge` does. The rows of other roots are reported in part: a prefix whose backup id a row of another root holds is skipped naming that root; rows of other roots as such are not listed. + +Every `extenddb backup` command refuses an in-memory SQLite database (`[storage.sqlite] path = ":memory:"`, or no path on a `sqlite-memory` build) before it opens the catalog, exit 1: it would open a new, empty catalog that ends with the command. A server on such a database syncs with its store at every start instead (the `path` row above). + +`extenddb backup remove-target --account
    ` removes a restore target that `DeleteTable` refuses (a `CREATING` table) together with its restore record: the target of a `FAILED` restore whatever the reason (`target_not_empty` included: the state a resume leaves when it finds no recorded progress and more items in the target than the backup holds, which a catalog restored from a dump can produce when it names a restore that finished and was written to since; an ordinary crash before the first progress record on Cassandra leaves no more items than the backup holds and is resumed, not failed; until the command runs a `target_not_empty` target keeps its backup in use, so `DeleteBackup` and a second restore of the backup are `BackupInUseException` and the name is held; `owner dead: taken over 5 times by one instance without the restore ending` is the reason of a record one server's reconciler resumed five times without the restore ending, whose target the reconciler removes itself), the target of a `RUNNING` restore whose owner is dead (no heartbeat within `stale_job_after_secs` on the database's clock, the listing the reconciler uses; on SQLite under the serve lock every owner is dead), and a legacy (in-catalog) restore's target the same way. It refuses a `RUNNING` restore with a live owner ("stop it or wait for it"), a `DONE` restore and a table no restore record names, the latter two as "not a restore target", and exits 1. It needs no store (it runs whether or not `[backup.store]` is configured, and creates neither the default root nor its identity file), takes the catalog-level maintenance claim first as `purge` does and releases it before it returns, logs the removal under `extenddb::audit` with the account, the table and the backup's ARN, and prints what it did: + +``` +=== extenddb backup remove-target === +Taking the maintenance claim as "extenddb backup remove-target on (pid )". +removed the restore target
    (account , table id
    ) and its record: its store-backed restore from was FAILED (target_not_empty) +The maintenance claim is released. +``` + +The third line's tail names the record's state: `was FAILED ()` or `was RUNNING with a dead owner`, after `its store-backed restore from ` or `its in-catalog restore from ` for the record's kind. A refusal is one `Error:` line on stderr (`
    (account ) is not a restore target: no restore record names it`; `... its restore from is DONE (the table is ACTIVE, or awaits its index backfill)`; `the restore into
    (account ) from is RUNNING with a live owner (a heartbeat within stale_job_after_secs); stop it or wait for it`; on PostgreSQL, `the restore into
    (account ) from is a CREATING restore target (table id ) whose owner's lock is still held: its owner's session is alive though its heartbeat is stale; wait for the owner to stop, or stop it, and run the command again`, the owner's claim session holding the target's advisory lock, a frozen process for instance; `the maintenance claim is held by ""; one maintenance command runs at a time, and a claim whose holder died expires after stale_job_after_secs ( s)`). On SQLite it opens the database as `serve` does and takes the serve lock: stop the server first, as for `purge`; under that lock every owner is dead, so a `RUNNING` record's target is removed too. After it, `DescribeTable` of the name is `ResourceNotFoundException`, `DeleteBackup` of the backup succeeds and a new restore into the name runs. + #### [auth] | Key | Default | Description | @@ -873,9 +964,14 @@ JSON metrics endpoint with DynamoDB CloudWatch-style metric names and dimensions ```bash curl --cacert ~/.extenddb/tls/cert.pem https://127.0.0.1:18443/health -# {"status":"healthy"} +# {"status":"healthy","components":{}} +# (the key order depends on the build; the earlier Health Check section +# shows the components a backup store adds) ``` +The `components` object carries the `backup_store` component when a backup +store is configured (see [Health Check](#health-check) under Server Lifecycle). + ## Troubleshooting ### Vector Index Support (PostgreSQL) diff --git a/docs/manuals/06-developer-test-guide.md b/docs/manuals/06-developer-test-guide.md index 4e97f24b0..0ed1d9283 100755 --- a/docs/manuals/06-developer-test-guide.md +++ b/docs/manuals/06-developer-test-guide.md @@ -150,6 +150,83 @@ devtools/run-tests --extenddb --pytest --filter test_put_item 1. A running extenddb server: `./target/debug/extenddb serve --config extenddb.toml` 2. `EXTENDDB_TEST_ENDPOINT=https://localhost:18443` 3. `EXTENDDB_ADMIN_PASSWORD=` +4. `EXTENDDB_TEST_BACKUP_STORE=`, required for `--pytest` + (and so for `--all`) and read by the Rust integration suite's backup + tests: where the server under test writes its backups. `filesystem` for a + server with a `[backup.store]` filesystem root, which a file-backed SQLite + server without the section has by default (`backups/` beside its database + file); `s3` for an S3 store; `none` for a server without a store + (`kind = "disabled"`, or PostgreSQL, MongoDB and Cassandra without the + section). With a configured store CreateBackup is asynchronous and + RestoreTableFromBackup of its backups is the store-backed restore job + (the response is `CREATING` and the table is loaded in the background), + so the backup tests derive their expected outcome from the variable and + fail rather than guess when it is unset; `run-tests --pytest` refuses before any test runs with + `error: --extenddb --pytest requires EXTENDDB_TEST_BACKUP_STORE to be set.` + The suites check the value against the server's `/health` `backup_store` + component, so a wrong value fails too. + +**The backup store and job suites** (`tests/test_backup_store_config.py` +and `tests/test_backup_job.py`) start servers of their own, so `run-tests` +excludes them and they run by name: +`python3 -m pytest tests/test_backup_store_config.py tests/test_backup_job.py -v`. +`EXTENDDB_BINARY` names the release build under test and decides which +deployment the cells use: a SQLite build (the default root beside the +database file) or a PostgreSQL build, which also needs +`EXTENDDB_TEST_PG_CONNECTION_STRING` and +`EXTENDDB_TEST_PG_ADMIN_CONNECTION_STRING`. The S3 cells (the store +configuration suite's S3 rows and the job suite's store-outage cell) need a +moto server at `EXTENDDB_TEST_S3_ENDPOINT` (`http://127.0.0.1:`); +without it they skip. They set the credentials moto accepts +(`testing`/`testing`, `us-east-1`) themselves, in the server's environment +and on their own client, so nothing else needs to be exported. The job suite's restore cells size their +backup from `EXTENDDB_TEST_RESTORE_ITEMS` (default 10,000; at least two data +files are needed), and the multi-part-key cell enables +`[limits] allow_multipart_table_keys` on its own deployment. CI runs both +suites on the PostgreSQL job and the two SQLite jobs; the PostgreSQL job +and the SQLite S3 job start moto, so the outage cell runs there and skips on +the plain SQLite job (`.github/workflows/integration.yml`). + +**The MongoDB backup cells** of `tests/test_mongodb_backup_restore.py` that +read the store's root and the server's log skip unless +`EXTENDDB_TEST_BACKUP_ROOT` (the server's filesystem store root) and +`EXTENDDB_TEST_SERVER_LOG` (the file the server logs to) are set; +`devtools/run-mongodb-tests` serves a filesystem store and exports both +(and `EXTENDDB_TEST_BACKUP_STORE=filesystem`), so run them through it: +`devtools/run-mongodb-tests -- --pytest --filter test_mongodb_backup_restore`. +The suite's legacy LSI fallback cell edits an in-catalog backup and runs +only on a server without a store: +`devtools/run-mongodb-tests --backup-store none -- --pytest --filter test_mongodb_backup_restore` +(the runner then writes no `[backup.store]` and exports +`EXTENDDB_TEST_BACKUP_STORE=none`; the store-backed cells skip under it). +The CI jobs run the suite with `filesystem`. + +**The cross-backend restore matrix** is `devtools/backup-restore-matrix`: +one cell per invocation (`--source --target +`, plus `--duplicate-key`, `--vector` or +`--adopt-store`), each starting a source and a target server from the +per-backend release binaries (`--bin-`, defaulting to +`target/rel-/release/extenddb`) against PostgreSQL at +`127.0.0.1:55432`, MongoDB at `127.0.0.1:27117` and Cassandra at +`127.0.0.1:9042`; its MongoDB and Cassandra cells need `pymongo` and +`cassandra-driver` (`requirements-matrix.txt`). A cell copies the source's +backup prefix under the target's root and adopts it with `extenddb backup +sync` on the target (with the target's server stopped when it is SQLite); +`--adopt-store` points the target's store at the source's root instead, +adopts the root with `backup sync --adopt-store` and checks that the +source's server then reports the store owned by another catalog. The +nightly workflow `.github/workflows/matrix-nightly.yml` builds the four +binaries, starts the three services and runs the twelve plain cells, the +four duplicate-key cells, the eight vector cells and two `--adopt-store` +cells (SQLite to PostgreSQL and back); `workflow_dispatch` runs it on +demand. The same workflow runs the restore kill points of every backend in +four jobs (`restore-kill-points-`), each building its backend's +binary with `extenddb-app/backup-test-hooks` and starting its backend as +the matrix job does, through `devtools/backup-kill-points.py restore + --bin --work ` (the Cassandra job builds a +plain SQLite binary too, the `--source-bin` whose backup the Cassandra +target restores); every point ends ACTIVE with the exact base and GSI +counts, except the tampered-file point, whose target is removed. The `run-tests` script automatically: - Performs health check on the target endpoint @@ -185,8 +262,8 @@ The `run-tests` script automatically: The primary test suite is in Python, testing extenddb through the same AWS SDK interface that customers use. ```bash -# Via run-tests script (recommended) -devtools/run-tests --extenddb --pytest +# Via run-tests script (recommended; EXTENDDB_TEST_BACKUP_STORE as above) +EXTENDDB_TEST_BACKUP_STORE=filesystem devtools/run-tests --extenddb --pytest # Run in parallel (uses 1/3 of CPU cores by default) devtools/run-tests --extenddb --pytest --parallel diff --git a/docs/manuals/07-upgrade-manual.md b/docs/manuals/07-upgrade-manual.md index b1e3295e3..09835c730 100755 --- a/docs/manuals/07-upgrade-manual.md +++ b/docs/manuals/07-upgrade-manual.md @@ -4,7 +4,9 @@ ## Current Status -This release versions the PostgreSQL, SQLite, Cassandra and MongoDB catalogs by a migration ledger and carries catalog migrations 1 to 5, which `extenddb migrate --list` prints. **Every deployment created by a 0.1.x release runs `extenddb migrate --yes` once** before a server of this release starts on it: the server refuses a catalog with pending migrations. Migration 1 brings the catalog of every 0.1.x release to the v1.0 schema, and migration 5 drops `continuous_backups`, a table and on MongoDB a collection (see [Catalog 0.0.5](#catalog-005)). +This release versions the PostgreSQL, SQLite, Cassandra and MongoDB catalogs by a migration ledger and carries catalog migrations 1 to 7, which `extenddb migrate --list` prints. **Every deployment created by a 0.1.x release runs `extenddb migrate --yes` once** before a server of this release starts on it: the server refuses a catalog with pending migrations. Migration 1 brings the catalog of every 0.1.x release to the v1.0 schema, migration 5 drops `continuous_backups`, a table and on MongoDB a collection (see [Catalog 0.0.5](#catalog-005)), migration 6 adds the backup catalog index and the restore job records ([below](#migration-6-the-backup-catalog-index)), and migration 7 adds the `restore_progress` table to the PostgreSQL data database (it does not apply on the other backends). + +Configure `[backup.store]` before the release that stops creating in-catalog backups. From this release CreateBackup writes to the configured store on PostgreSQL, SQLite and MongoDB, and is asynchronous there as on the service: the response is `CREATING` and the backup becomes `AVAILABLE` once the upload ends, so a client that deletes a backup it just created has to poll DescribeBackup first (the previous release returned `AVAILABLE`). A store-backed backup is listed, described, deleted and restored like any other. `RestoreTableFromBackup` of a backup in the configured store runs the restore job from the store on every backend (PostgreSQL, SQLite, MongoDB and Cassandra); in-catalog backups, those of the previous release included, restore as before. An S3 store needs `spool_path` set (the restore's download area) on PostgreSQL, MongoDB and Cassandra, and that must be a directory of its own, not the database directory; a filesystem store needs no spool. Migration 7 adds the `restore_progress` table to the PostgreSQL data database; run `extenddb migrate --yes` before starting the new binary. The progress rows of a restore in flight live in the data database, so a data database restored from a dump taken before migration 7 has none of them and no `restore_progress` table: the new binary refuses to serve it until `extenddb migrate --yes` has run again, and its reconciler then finds no progress for the restore and resumes it from its first data file into the (empty) target, or `FAILED`s it as `target_not_empty` if the target already holds items, in which case `extenddb backup remove-target --account
    ` (the admin guide's `backup` section) removes the target and its record and the backup restores again. The same `target_not_empty` state is what a server stopped inside a restore's first data file on MongoDB, or inside its first batch of 10,000 items or 32 MiB on Cassandra, leaves on restart, since both record progress after the write; the command is the remedy there too. `backup remove-target` writes the catalog, so on SQLite the server is stopped for it, as for `purge`. The previous binary resumes no restore (a store-backed restore in flight at a rollback is lost with the catalog rows, as the rollback paragraph below says). Quiesce restores before a downgrade. `kind = "disabled"` is no longer needed to keep a deployment able to restore the backups it takes. Without a store CreateBackup writes the in-catalog format and logs a deprecation warning naming `[backup.store]` on each call; a later release's CreateBackup refuses without one. Cassandra writes the in-catalog format with or without a store. `kind = "disabled"` under `[backup.store]` is the opt-out that keeps a deployment on the in-catalog format on purpose and silences the warning. On file-backed SQLite an absent section is a filesystem store at `backups/` beside the database file, created `0700` at startup; that root must be owned by the extenddb user with mode `0700`, under directories owned by root or that user which others cannot write to. A default root that breaks the rule (a database file in a group-writable directory, say), or whose `extenddb-store.json` names another catalog (a catalog re-created by `destroy` and `init` under the old root), is logged at startup as a warning (`backup store: the default root ... failed its check`) and leaves the store unconfigured, not unhealthy, so the upgrade never stops a server that never configured backups and CreateBackup keeps working on it; the same root configured explicitly is a startup error or an unhealthy store. The warning applies only while the catalog has no backup rows for the default root (an upgrade, a fresh deployment, a catalog re-created by `destroy` and `init`): once a server has written rows for it, the default root is handled as a configured one, so its backups never vanish behind a warning (a failed identity check is an unhealthy store, reported on `/health`; a root that can no longer be opened is a startup error naming it). Instances that share a catalog must share one store root, an S3 bucket and prefix or a shared mount: a second instance pointed at a host-local directory under the same configured path starts with the store unhealthy (`/health` reports `backup_store` unhealthy; the admin guide's `[backup.store]` section says what the identity file and the manifest probe check). `extenddb destroy` leaves the store; `extenddb backup purge --confirm` is what deletes its backups, and a catalog re-created with `init` needs the old root's `extenddb-store.json` removed or a new root, since the file names the old catalog. A backup that a 0.1.x release took can restore without its global and local secondary indexes: a SQLite backup taken by 0.1.14 records only the table's key schema and attribute definitions, and its restore creates the table with its items and no index. Take a new backup of each table after upgrading to v1.0, and keep the 0.1.x backups for their items only. @@ -39,7 +41,15 @@ Exit status 6 means the migration lock, or a running server, was still held when `verify` also checks the connection to the data database. When that check fails, `verify` exits 1 on a Behind catalog instead of 3, because `migrate --yes` cannot apply the pending migrations while the data database cannot be reached. It still exits 4 or 5 for the statuses that need a person. -A migration is additive when the previous release serves the catalog correctly with it applied, and breaking otherwise. Migrations 1 to 5 are breaking: do not run a 0.1.x server against a migrated catalog. A 0.1.x binary that writes to a migrated catalog (its `migrate --yes`, or a SQLite dev server) leaves it WrittenByOlderRelease. `migrate --yes --reconverge` repairs it by running the convergent migrations again. Migration 2, which rewrites stored policies, is not convergent: the repair skips it and prints a line that says so (see the v0.1.14 notes below). +A migration is additive when the previous release serves the catalog correctly with it applied, and breaking otherwise. Migrations 1 to 6 are breaking. Migration 7 creates a table in the PostgreSQL data database that the previous release never reads; on PostgreSQL it is declared breaking only because the job that proves an additive declaration against the old binary does not yet exist (CONTRIBUTING, step 4 of adding a migration), and on SQLite, MongoDB and Cassandra it does not apply (`migrate --list` prints it as not applicable, and `migrate` counts it as additive). Do not run a 0.1.x server against a migrated catalog. A 0.1.x binary that writes to a migrated catalog (its `migrate --yes`, or a SQLite dev server) leaves it WrittenByOlderRelease. `migrate --yes --reconverge` repairs it by running the convergent migrations again. Migration 2, which rewrites stored policies, is not convergent: the repair skips it and prints a line that says so (see the v0.1.14 notes below). + +### Migration 6: the backup catalog index + +A backup of a table that holds an `N` value the wire did not validate, or an attribute name over the limit, fails on the store-backed path where the previous release backed it up. Four write paths stored such a value (`1E126`, `abc`) unvalidated (#618: BatchWriteItem, TransactWriteItems, UpdateItem on an absent item in the key it creates, ImportTable), UpdateItem and a TransactWriteItems Update of releases before #433 stored an attribute name over 65,535 bytes unchecked, and the store-backed job validates every item with the rule the restore applies, so no backup that no restore can load is reported `AVAILABLE`. This meets file-backed SQLite without a `[backup.store]` section (the defaulted root) and any deployment once a store is configured. The client sees `CREATING` and then `BackupNotFound`; the reason reaches the operator only: `extenddb backup list --failed` prints the FAILED row with the count of refused items and the first one, and the server log names each refused item's key and attribute path beside the backup ARN (up to 1,000 per backup), which a tenant's support case can cite (every tenant-supplied text in the line is written as a JSON string's content, control characters and the Unicode line separators U+0085, U+2028 and U+2029 escaped, so a newline or a line separator in an attribute name or a number value cannot forge a second log record). The way out is to rewrite or delete each named item through the wire, which validates the value (a bad range key cannot be rewritten, since a PutItem with that key is refused; `DeleteItem` removes it); the in-catalog create of a server without a store still backs such a table up. + +Migration 6 adds the catalog index of backups kept in a backup store, turns each restore registration into a restore job record, adds the maintenance claim record (`backup_maintenance`) and writes the catalog's random id (`settings.catalog_id`). On SQLite and PostgreSQL the index is the `backup_index` table and the job record is new columns of `table_restores`; on MongoDB the index is the `backup_index` collection and the job record a `restore_job` field of each restore target's `tables` document; on Cassandra the index is new columns of `backups_by_arn` and the `backup_index_by_status` table, and the job record new `restore_*` columns of `tables`. Stop every server built before this migration before running `extenddb migrate --yes`: such a server neither reads the index nor marks the restore registrations it writes, so a restore it ran during the upgrade would be owned by nobody. A restore in flight at that stop is left as a RUNNING job record whose owner is dead; the first reconciler pass of the new server (within `backup.reconcile_interval_secs`, default 60 s) marks it FAILED and removes its CREATING target, so the name is free again and the table has to be restored anew. On Cassandra the backfill reads `tables` with a filtered scan and writes each registration with a lightweight transaction, which takes seconds on a catalog of ordinary size. On Cassandra and MongoDB, which have no transaction around a body, every write of the migration is safe to send again: a `migrate --yes` that fails part way leaves migration 6 unrecorded, `serve` keeps refusing the catalog, and the next `migrate --yes` runs the body again and finishes. + +A rollback to the previous binary needs two things: the catalog restored from the pre-upgrade backup (the previous binary refuses a catalog carrying migration 6, exit status 4), and the `[backup]` section removed from the configuration file, which the previous binary refuses as an unknown field before it reaches the catalog (`unknown field 'backup'`). The pre-upgrade backup predates every backup this release wrote to the store, so a restored catalog has no `backup_index` row for them while the store keeps their bytes under its root; the previous binary serves neither (it does not read the store). After the upgrade is run again, `extenddb backup sync --adopt-store` gives them rows: the store's identity file names the pre-rollback catalog, so the re-upgraded catalog finds the store owned by another catalog until the command makes it the owner (on SQLite's default root the server instead logs a warning and leaves the store unconfigured, so `/health` has no `backup_store` component until then), and the command then adopts every backup under the root. On SQLite `backup sync`, with or without its options, takes the serve lock as `backup purge` does, so the server is stopped while it runs; on PostgreSQL, MongoDB and Cassandra it runs beside the servers under the maintenance claim. In-catalog backups taken after it are lost with the rows, as before. ## Writing a New Migration @@ -200,7 +210,7 @@ Adds the table definition a backup records and the registration of a running res A backup taken before this upgrade has no recorded definition. It restores with its keys, items, and billing mode but no secondary indexes; a provisioned one restores with 5/5 throughput on PostgreSQL and SQLite and with its recorded throughput on Cassandra. On PostgreSQL a pre-upgrade backup of a table with vector indexes is refused at restore; on SQLite it restores without the vector indexes. Tables restored before this upgrade have no registration and report no `RestoreSummary`. -The upgrade is not reversible in place: a 0.0.3 PostgreSQL or SQLite binary refuses to start against a 0.0.4 catalog (`Catalog version mismatch: expected 0.0.3, found 0.0.4`). Roll back by restoring what you saved before the upgrade. Backups live in the catalog, so that also removes every backup created after the upgrade. +The upgrade is not reversible in place: a 0.0.3 PostgreSQL or SQLite binary refuses to start against a 0.0.4 catalog (`Catalog version mismatch: expected 0.0.3, found 0.0.4`). Roll back by restoring what you saved before the upgrade. Backups live in the catalog, so that also removes every backup created after the upgrade (this is the in-catalog format; a backup in a `[backup.store]` store of a later catalog is outside the database, and a catalog restored from a pre-upgrade dump loses its `backup_index` rows while the store keeps the bytes). #### PostgreSQL @@ -248,7 +258,7 @@ On SQLite, `extenddb serve` reads the catalog's migration ledger before it opens The `extenddb migrate` command that `serve`, `migrate` and `verify` advise names the config file by its absolute path, quoted for a POSIX shell, so it can be pasted as printed. A relative `path` in `[storage.sqlite]`, such as the `extenddb.sqlite` that `init` writes, resolves against the directory a command runs in. So when the config file names the database by a relative `path`, or names none, the advised command starts with `EXTENDDB__STORAGE__SQLITE__PATH=`, and it reaches the same database from any directory. -On startup the SQLite server removes any restore target left in `CREATING` by a server that stopped mid-copy, which is only safe because one server at a time can hold the file. A backup keeps a WAL read snapshot open while it copies; checkpoints cannot reclaim pages needed by that snapshot, so allow disk headroom for WAL growth proportional to the backup duration and concurrent write volume. Set `pool_size` to 3 or more (see the Admin Guide); the server runs at most `pool_size / 2 - 1` backups at once (4 with the default of 10) and answers `LimitExceededException` to a `CreateBackup` that waited 30 seconds for a slot. Running backups hold one connection each for their read snapshot plus one shared writer (`n` backups hold `n + 1` connections), so from `pool_size` 4 up a flood of backups leaves at least half the pool to other requests; with a `pool_size` of 2 or 3 the one admitted backup holds two of them. A GSI added by `UpdateTable` with `ProvisionedThroughput` on a `PROVISIONED` table made every `DescribeTable` of that table fail on earlier builds (`Internal("index pt: missing field NumberOfDecreasesToday")`); this build writes the index's throughput in the shape `DescribeTable` reads and reads the shape earlier builds wrote (as the same units with `NumberOfDecreasesToday` 0), so such a table is describable again after the upgrade without any change to the database file. +On startup the SQLite server ages every RUNNING restore record to the epoch, which is only safe because one server at a time can hold the file, and its reconciler's first cycle then fails an in-catalog restore that a server stopped mid-copy and removes its `CREATING` target (the cycle runs beside the listener, so a request can still see the target `CREATING` until it ends); a restore from a backup store is resumed instead, from its recorded progress, and its target stays `CREATING` until it finishes. A backup keeps a WAL read snapshot open while it copies; checkpoints cannot reclaim pages needed by that snapshot, so allow disk headroom for WAL growth proportional to the backup duration and concurrent write volume. Set `pool_size` to 3 or more (see the Admin Guide); the server runs at most `pool_size / 2 - 1` backups at once (4 with the default of 10) and answers `LimitExceededException` at once to a `CreateBackup` that finds no slot free. Running backups hold one connection each for their read snapshot plus one shared writer (`n` backups hold `n + 1` connections), so from `pool_size` 4 up a flood of backups leaves at least half the pool to other requests; with a `pool_size` of 2 or 3 the one admitted backup holds two of them. A GSI added by `UpdateTable` with `ProvisionedThroughput` on a `PROVISIONED` table made every `DescribeTable` of that table fail on earlier builds (`Internal("index pt: missing field NumberOfDecreasesToday")`); this build writes the index's throughput in the shape `DescribeTable` reads and reads the shape earlier builds wrote (as the same units with `NumberOfDecreasesToday` 0), so such a table is describable again after the upgrade without any change to the database file. #### Cassandra @@ -262,7 +272,7 @@ extenddb serve --config extenddb.toml `` is the role `init` used (`--cassandra-user`). `EXTENDDB_CASSANDRA_PASSWORD` keeps the password out of process arguments; `--cassandra-pass` still works and wins if both are given. Without a user and password `migrate` connects as the operating-system user with no password, which a cluster using `PasswordAuthenticator` refuses; the error reports a connection timeout. -There is no startup version gate on Cassandra. A 0.0.4 server started before `extenddb migrate --yes` serves every table and item operation normally; CreateBackup, RestoreTableFromBackup, and DeleteBackup are refused with a `ValidationException` ("Backups are unavailable until the ExtendDB catalog is migrated: ... Run `extenddb migrate --yes` with the admin credentials.") until the catalog is migrated; a running server notices the migration without a restart, and the server logs an abandoned-restore sweep error (`Undefined column name restore_source_backup_arn`) at startup and every 15 seconds. ListBackups and DescribeBackup work and show backups taken before the upgrade. V005 only adds columns. This manual has no tested rollback for Cassandra; take a snapshot of the catalog and account keyspaces with your usual Cassandra tooling before migrating. +The 0.0.4 server had no startup version gate on Cassandra. A 0.0.4 server started before `extenddb migrate --yes` serves every table and item operation normally; CreateBackup, RestoreTableFromBackup, and DeleteBackup are refused with a `ValidationException` ("Backups are unavailable until the ExtendDB catalog is migrated: ... Run `extenddb migrate --yes` with the admin credentials.") until the catalog is migrated; a running server notices the migration without a restart, and the 0.0.4 server logs an abandoned-restore sweep error (`Undefined column name restore_source_backup_arn`) at startup and every 15 seconds (this release's server has no such sweep, and does not start on a catalog with pending migrations: see [Current Status](#current-status)). ListBackups and DescribeBackup work and show backups taken before the upgrade. V005 only adds columns. This manual has no tested rollback for Cassandra; take a snapshot of the catalog and account keyspaces with your usual Cassandra tooling before migrating. Upgrade every server to 0.0.4 before restoring a backup. A 0.0.3 server still running against the migrated catalog does not know about restores in progress: its DeleteTable deletes a table that is being restored and its DeleteBackup deletes a backup a restore is copying from, and that restore then fails and its target table is removed. @@ -561,12 +571,14 @@ region the caller wrote, so after a change a statement must name the new region admin guide covers this). Keep `server.region` within `[a-z0-9-]`. The `BackupArn` parser accepts only -that class, as the service does, so a region such as `LOCAL`, `us_east_1`, or -`dev.local` makes every backup ARN the server issues `ValidationException` -"Invalid Request: BackupArn is not valid" on `DescribeBackup`, `DeleteBackup`, -and `RestoreTableFromBackup`, and `ListBackups` refuses the cursor it hands out, -so every listing stops at its first page. The server starts with such a region -and logs a warning naming the consequence at startup. +that class, as the service does, so under a region such as `LOCAL`, `us_east_1`, +or `dev.local` no backup ARN can be formed: `CreateBackup` is refused with +`ValidationException` naming `server.region`, every backup an earlier release +issued under such a region keeps an ARN that is `ValidationException` "Invalid +Request: BackupArn is not valid" on `DescribeBackup`, `DeleteBackup`, and +`RestoreTableFromBackup`, and `ListBackups` refuses a cursor that is such an ARN +(a listing of backups created under a valid region pages). The server starts +with such a region and logs a warning naming the consequence at startup. ### v0.1.6 — IAM: bare operators on multivalued condition keys are no-ops (BR-7085) diff --git a/docs/manuals/08-install-linux.md b/docs/manuals/08-install-linux.md index 928ba957d..2b868522e 100755 --- a/docs/manuals/08-install-linux.md +++ b/docs/manuals/08-install-linux.md @@ -118,7 +118,7 @@ Expected: ``` === extenddb verify === ... - OK: Current: the catalog is at migration 5 + OK: Current: the catalog is at migration 6 ... === HEALTHY: All checks passed === ``` diff --git a/docs/manuals/09-install-macos.md b/docs/manuals/09-install-macos.md index 85ba0f1e8..b1ca6de2d 100755 --- a/docs/manuals/09-install-macos.md +++ b/docs/manuals/09-install-macos.md @@ -96,7 +96,7 @@ Expected: ``` === extenddb verify === ... - OK: Current: the catalog is at migration 5 + OK: Current: the catalog is at migration 6 ... === HEALTHY: All checks passed === ``` diff --git a/docs/manuals/11-deployment-guide.md b/docs/manuals/11-deployment-guide.md index 6950552fe..198c6e781 100755 --- a/docs/manuals/11-deployment-guide.md +++ b/docs/manuals/11-deployment-guide.md @@ -120,7 +120,13 @@ exec extenddb serve --foreground --config "$CONFIG" `extenddb stop` cannot stop a foreground server, because there is no PID file to read; stop the container instead, which delivers `SIGTERM` and triggers the same -graceful shutdown. Outside a container, where you may want `stop` to work, add +graceful shutdown. With a backup job running the shutdown takes up to 35 +seconds before the workers stop, and up to 76 seconds in all when the database +is slow to answer (the admin guide's Stopping section); a runtime whose grace +period is shorter (Docker's default is 10 s) kills the process during the job +drain, and the reconciler resolves the job's row as it does after a crash. The +repository's compose files set `stop_grace_period: 80s`, above that sum; a +compose file of your own needs the same. Outside a container, where you may want `stop` to work, add `--write-pid-file` — it writes the PID file to the usual `run_dir` path, which then has to be writable. @@ -221,7 +227,7 @@ extenddb init \ ### Database (PostgreSQL) - [ ] Use a dedicated PostgreSQL user for extenddb with minimal privileges -- [ ] Configure PostgreSQL `max_connections` ≥ extenddb `pool_size` + 3 +- [ ] Configure PostgreSQL `max_connections` ≥ `2 * pool_size + catalog_pool_size + max_concurrent_jobs`, plus one per overlapping vector index build and one during a schema migration - [ ] Enable PostgreSQL TLS - [ ] Set up automated backups (pg_dump, WAL archiving, or managed service snapshots) - [ ] Monitor PostgreSQL disk usage, connection count, and query performance @@ -261,6 +267,13 @@ User=extenddb Group=extenddb Restart=on-failure RestartSec=5 +# On a host that backs up tables of large items, one malloc arena keeps a +# backup job's high-water mark at the pipeline's terms whatever the number of +# large items it reads (the admin guide's [backup] max_concurrent_jobs row +# gives the figures). It costs the request path a few percent (3.5 to 9.2% +# on GetItem, Query and Scan on SQLite, about 2% on PostgreSQL (the scans, +# 2.1%)), so the line is left to the operator: +#Environment=MALLOC_ARENA_MAX=1 [Install] WantedBy=multi-user.target @@ -277,7 +290,8 @@ Run one extenddb instance per catalog. Multiple instances behind a load balancer - Credentials, policies, and table metadata are cached in each process (see the cache TTLs in the admin guide). Nothing invalidates a cache on another instance, so a key revoked or a policy changed through one instance stays in force on the others until the TTL expires. - Every instance runs every background worker (TTL expiry, GSI backfill, stream retention, control-plane transitions). The workers are safe to run twice, but the work is duplicated. -- `/health` is a liveness check that does not query the database, so a load balancer cannot use it to take an instance with a broken database connection out of rotation. +- `/health` is a liveness check that does not query the database, so a load balancer cannot use it to take an instance with a broken database connection out of rotation. It does report the backup store: `components.backup_store` is `unhealthy` and `status` is `degraded` while the configured store cannot be reached or fails its identity check, with HTTP 200 kept, so a store outage never drains a replica. +- Instances that share a catalog must share one backup store root (`[backup.store]`: one S3 bucket and prefix, or one shared mount that supports hard links, which the store's conditional create needs; some CIFS and FUSE mounts refuse `link(2)` with `EPERM`, and the server then reports the store as having failed validation, with the `link` error in the log line). A host-local directory configured at the same path on each host is not one root: once the catalog has rows for the root, the second instance finds the root's identity file missing or the newest backup's manifest absent and starts with the store unhealthy. While the catalog has no rows for the root (a fresh deployment, or one whose backups were purged), nothing tells the two directories apart: each instance creates its own identity file and store id and both start healthy, and the first backups then split the rows between two ids, which the next check reports as an identity mismatch that only `backup purge` resolves. Point every instance at one root before the first backup. Each instance on a host needs its own `spool_path`; the spool's `.lock` refuses a second instance. An existing spool directory must be owned by the server's user with mode `0700`, as the root must; a spool created `0755` stops startup naming it. On SQLite this limit is enforced: `extenddb serve` holds an exclusive lock on `.lock` and a second server on the same file refuses to start. On PostgreSQL, MongoDB, and Cassandra it is not enforced. On PostgreSQL, if you start a second instance anyway, size `pool_size × instance_count + 3 × instance_count ≤ PostgreSQL max_connections`. @@ -296,7 +310,7 @@ pool_size = 50 # Increase for higher concurrency Ensure PostgreSQL `max_connections` accommodates the total pool size plus overhead. -Backup and restore work is admitted against the pool so it cannot take every connection from ordinary requests. On PostgreSQL and MongoDB one server runs at most 4 `RestoreTableFromBackup` copies per account and `min(16, pool/2)` in total (PostgreSQL: `pool_size`; MongoDB: the engine's driver pool, `maxPoolSize` in the connection string or `max_connections`, default 10, so 5 restores); a restore that waits more than 30 seconds for a slot is refused with `LimitExceededException` and creates nothing. On SQLite one server runs at most `pool_size / 2 - 1` `CreateBackup` copies at once (between 1 and 16; 4 for the default `pool_size` of 10), each holding its snapshot connection for its duration, and refuses a backup that waited 30 seconds with `LimitExceededException`. A MongoDB restore request that is not registered within 60 seconds answers an internal error while the restore goes on in the background; the MongoDB driver does not enforce `waitQueueTimeoutMS`, so the admission bound is what keeps a flood of restores from holding every pooled connection. +Backup and restore work is bounded per server by `[backup] max_concurrent_jobs` (default 4): `CreateBackup` and `RestoreTableFromBackup` jobs together, on every backend, and a request that finds every slot held waits up to 30 s for one, holding nothing meanwhile, and is then refused with `LimitExceededException` and creates nothing. On PostgreSQL each running job holds one pooled connection for its copy plus one dedicated connection outside the pool for its claim, and startup raises `pool_size` and `catalog_pool_size` to at least `2 * max_concurrent_jobs + 10` with a warning naming both numbers (the admin guide's `[backup]` table). On SQLite a `CreateBackup` also holds its snapshot connection for its duration, and a server admits at most `pool_size / 2 - 1` of them at once (between 1 and 16; 4 for the default `pool_size` of 10), so that half the pool stays free for ordinary requests, and raises a smaller `pool_size` to `2 * (max_concurrent_jobs + 1)`, at most 34, so that bound refuses no job `max_concurrent_jobs` admits (the admin guide's `max_concurrent_jobs` row; above 16 jobs the slot refuses the seventeenth); a `CreateBackup` over that bound is refused at once with the same `LimitExceededException`. A MongoDB restore request that is not registered within 60 seconds answers an internal error while the restore goes on in the background; the MongoDB driver does not enforce `waitQueueTimeoutMS`, so the admission bound is what keeps a flood of restores from holding every pooled connection. ### Cassandra Connection Pool @@ -321,7 +335,7 @@ Key PostgreSQL settings for extenddb workloads: - `shared_buffers`: 25% of available RAM - `effective_cache_size`: 75% of available RAM - `work_mem`: 64MB (for sort operations in Query/Scan) -- `max_connections`: ≥ `2 * pool_size + catalog_pool_size`, plus one per vector index build you expect to overlap and one during a schema migration. `pool_size` sizes both the catalog and the data pool, and `max_connections` is per cluster, so the default pool sizes need 60 of PostgreSQL's default 100 +- `max_connections`: ≥ `2 * pool_size + catalog_pool_size + max_concurrent_jobs`, plus one per vector index build you expect to overlap and one during a schema migration. `pool_size` sizes both the catalog and the data pool, each backup or restore job holds one claim connection outside every pool, and `max_connections` is per cluster, so the default pool sizes and the default `max_concurrent_jobs` of 4 need 64 of PostgreSQL's default 100 ### Monitoring Queries diff --git a/docs/schema/cassandra.txt b/docs/schema/cassandra.txt index bc7ee91cc..b07cc42bf 100644 --- a/docs/schema/cassandra.txt +++ b/docs/schema/cassandra.txt @@ -2,6 +2,8 @@ table access_keys table accounts table admin_users +table backup_index_by_status +table backup_maintenance table backups_by_account table backups_by_arn table backups_by_table @@ -36,6 +38,17 @@ column accounts.created_at: timestamp regular column admin_users.admin_name: text partition_key 0 column admin_users.created_at: timestamp regular column admin_users.password_hash: text regular +column backup_index_by_status.store_root: text partition_key 0 +column backup_index_by_status.backup_status: text partition_key 1 +column backup_index_by_status.backup_arn: text clustering 0 asc +column backup_index_by_status.account_id: text regular +column backup_index_by_status.store_id: text regular +column backup_index_by_status.store_kind: text regular +column backup_maintenance.claim: text partition_key 0 +column backup_maintenance.claimed_at: timestamp regular +column backup_maintenance.heartbeat_at: timestamp regular +column backup_maintenance.holder: text regular +column backup_maintenance.owner_generation: bigint regular column backups_by_account.account_id: text partition_key 0 column backups_by_account.created_at: timestamp clustering 0 desc column backups_by_account.backup_arn: text clustering 1 asc @@ -48,16 +61,25 @@ column backups_by_arn.account_id: text partition_key 0 column backups_by_arn.backup_arn: text partition_key 1 column backups_by_arn.active_restores: map regular column backups_by_arn.attribute_definitions: text regular +column backups_by_arn.backup_id: text regular column backups_by_arn.backup_name: text regular column backups_by_arn.backup_size_bytes: bigint regular column backups_by_arn.backup_status: text regular column backups_by_arn.backup_type: text regular column backups_by_arn.billing_mode: text regular +column backups_by_arn.completed_at: timestamp regular column backups_by_arn.created_at: timestamp regular column backups_by_arn.definition: text regular +column backups_by_arn.failure_reason: text regular +column backups_by_arn.heartbeat_at: timestamp regular column backups_by_arn.item_count: bigint regular column backups_by_arn.key_schema: text regular +column backups_by_arn.owner_generation: bigint regular column backups_by_arn.provisioned_throughput: text regular +column backups_by_arn.source_table_details_json: text regular +column backups_by_arn.store_id: text regular +column backups_by_arn.store_kind: text regular +column backups_by_arn.store_root: text regular column backups_by_arn.stream_specification: text regular column backups_by_arn.table_arn: text regular column backups_by_arn.table_created_at: timestamp regular @@ -172,11 +194,23 @@ column tables.item_count: bigint regular column tables.key_schema: text regular column tables.on_demand_throughput: text regular column tables.provisioned_throughput: text regular +column tables.restore_completed_at: timestamp regular column tables.restore_date_time: timestamp regular +column tables.restore_failure_reason: text regular +column tables.restore_file_index: int regular column tables.restore_heartbeat: timestamp regular +column tables.restore_heartbeat_at: timestamp regular +column tables.restore_items_applied: bigint regular +column tables.restore_kind: text regular column tables.restore_owner: uuid regular +column tables.restore_owner_generation: bigint regular column tables.restore_source_backup_arn: text regular column tables.restore_source_table_arn: text regular +column tables.restore_spool_host: text regular +column tables.restore_status: text regular +column tables.restore_store_id: text regular +column tables.restore_store_kind: text regular +column tables.restore_store_root: text regular column tables.sse_specification: text regular column tables.status_transition_at: timestamp regular column tables.stream_label: text regular diff --git a/docs/schema/mongodb.txt b/docs/schema/mongodb.txt index 490403643..14488e8fb 100644 --- a/docs/schema/mongodb.txt +++ b/docs/schema/mongodb.txt @@ -6,6 +6,14 @@ index extenddb_catalog.accounts _id_ key {"_id":1} options {} index extenddb_catalog.accounts account_name_1 key {"account_name":1} options {"unique":true} collection extenddb_catalog.admin_users type collection options {} index extenddb_catalog.admin_users _id_ key {"_id":1} options {} +collection extenddb_catalog.backup_index type collection options {} +index extenddb_catalog.backup_index _id_ key {"_id":1} options {} +index extenddb_catalog.backup_index account_id_1_backup_id_1 key {"account_id":1,"backup_id":1} options {"unique":true} +index extenddb_catalog.backup_index account_id_1_table_name_1_created_at_1 key {"account_id":1,"table_name":1,"created_at":1} options {} +index extenddb_catalog.backup_index backup_id_1 key {"backup_id":1} options {"unique":true} +index extenddb_catalog.backup_index store_kind_1_store_root_1_store_id_1_status_1 key {"store_kind":1,"store_root":1,"store_id":1,"status":1} options {} +collection extenddb_catalog.backup_maintenance type collection options {} +index extenddb_catalog.backup_maintenance _id_ key {"_id":1} options {} collection extenddb_catalog.backups type collection options {} index extenddb_catalog.backups _id_ key {"_id":1} options {} index extenddb_catalog.backups account_id_1_table_name_1 key {"account_id":1,"table_name":1} options {} @@ -45,6 +53,7 @@ collection extenddb_catalog.settings type collection options {} index extenddb_catalog.settings _id_ key {"_id":1} options {} collection extenddb_catalog.tables type collection options {} index extenddb_catalog.tables _id_ key {"_id":1} options {} +index extenddb_catalog.tables restore_job.status_1_restore_job.heartbeat_at_1 key {"restore_job.status":1,"restore_job.heartbeat_at":1} options {} index extenddb_catalog.tables table_id_1 key {"table_id":1} options {"unique":true} collection extenddb_catalog.tags type collection options {} index extenddb_catalog.tags _id_ key {"_id":1} options {} diff --git a/docs/schema/postgres.txt b/docs/schema/postgres.txt index cee98bf9b..f14e9a125 100644 --- a/docs/schema/postgres.txt +++ b/docs/schema/postgres.txt @@ -3,7 +3,9 @@ table access_keys table accounts table admin_users table backup_definitions +table backup_index table backup_items +table backup_maintenance table backups table extenddb_migrations table iam_group_members @@ -41,10 +43,32 @@ column admin_users.password_hash: text NOT NULL column admin_users.created_at: timestamp with time zone NOT NULL DEFAULT now() column backup_definitions.backup_arn: text NOT NULL column backup_definitions.definition: jsonb NOT NULL +column backup_index.account_id: text NOT NULL +column backup_index.backup_id: text NOT NULL +column backup_index.table_name: text NOT NULL +column backup_index.backup_arn: text NOT NULL COLLATE "C" +column backup_index.backup_name: text NOT NULL +column backup_index.status: text NOT NULL +column backup_index.store_kind: text NOT NULL +column backup_index.store_root: text NOT NULL +column backup_index.store_id: text NOT NULL +column backup_index.size_bytes: bigint NOT NULL DEFAULT 0 +column backup_index.item_count: bigint NOT NULL DEFAULT 0 +column backup_index.created_at: timestamp with time zone NOT NULL +column backup_index.completed_at: timestamp with time zone +column backup_index.failure_reason: text +column backup_index.source_table_details_json: jsonb NOT NULL +column backup_index.owner_generation: bigint NOT NULL +column backup_index.heartbeat_at: timestamp with time zone NOT NULL column backup_items.backup_arn: text NOT NULL column backup_items.pk: text NOT NULL column backup_items.sk: text column backup_items.item_data: jsonb NOT NULL +column backup_maintenance.singleton: boolean NOT NULL DEFAULT true +column backup_maintenance.holder: text NOT NULL +column backup_maintenance.owner_generation: bigint NOT NULL +column backup_maintenance.claimed_at: timestamp with time zone NOT NULL DEFAULT now() +column backup_maintenance.heartbeat_at: timestamp with time zone NOT NULL DEFAULT now() column backups.backup_arn: text NOT NULL column backups.backup_name: text NOT NULL column backups.table_id: text NOT NULL @@ -160,6 +184,18 @@ column table_restores.table_id: text NOT NULL column table_restores.source_backup_arn: text NOT NULL column table_restores.source_table_arn: text NOT NULL column table_restores.restore_date_time: timestamp with time zone NOT NULL +column table_restores.status: text NOT NULL +column table_restores.kind: text NOT NULL +column table_restores.store_kind: text +column table_restores.store_root: text +column table_restores.store_id: text +column table_restores.owner_generation: bigint NOT NULL +column table_restores.heartbeat_at: timestamp with time zone NOT NULL +column table_restores.spool_host: text +column table_restores.file_index: bigint +column table_restores.items_applied: bigint +column table_restores.failure_reason: text +column table_restores.completed_at: timestamp with time zone column tables.account_id: text NOT NULL column tables.table_name: text NOT NULL column tables.key_schema: jsonb NOT NULL @@ -202,6 +238,8 @@ index accounts_account_name_key: CREATE UNIQUE INDEX accounts_account_name_key O index accounts_pkey: CREATE UNIQUE INDEX accounts_pkey ON public.accounts USING btree (account_id) index admin_users_pkey: CREATE UNIQUE INDEX admin_users_pkey ON public.admin_users USING btree (admin_name) index backup_definitions_pkey: CREATE UNIQUE INDEX backup_definitions_pkey ON public.backup_definitions USING btree (backup_arn) +index backup_index_pkey: CREATE UNIQUE INDEX backup_index_pkey ON public.backup_index USING btree (account_id, backup_id) +index backup_maintenance_pkey: CREATE UNIQUE INDEX backup_maintenance_pkey ON public.backup_maintenance USING btree (singleton) index backups_pkey: CREATE UNIQUE INDEX backups_pkey ON public.backups USING btree (backup_arn) index extenddb_migrations_pkey: CREATE UNIQUE INDEX extenddb_migrations_pkey ON public.extenddb_migrations USING btree (id) index iam_group_members_pkey: CREATE UNIQUE INDEX iam_group_members_pkey ON public.iam_group_members USING btree (account_id, group_name, user_name) @@ -218,6 +256,9 @@ index iam_user_tags_pkey: CREATE UNIQUE INDEX iam_user_tags_pkey ON public.iam_u index iam_users_pkey: CREATE UNIQUE INDEX iam_users_pkey ON public.iam_users USING btree (account_id, user_name) index iam_users_user_arn_key: CREATE UNIQUE INDEX iam_users_user_arn_key ON public.iam_users USING btree (user_arn) index idempotency_tokens_pkey: CREATE UNIQUE INDEX idempotency_tokens_pkey ON public.idempotency_tokens USING btree (token) +index idx_backup_index_account_table: CREATE INDEX idx_backup_index_account_table ON public.backup_index USING btree (account_id, table_name) +index idx_backup_index_backup_id: CREATE UNIQUE INDEX idx_backup_index_backup_id ON public.backup_index USING btree (backup_id) +index idx_backup_index_store_status: CREATE INDEX idx_backup_index_store_status ON public.backup_index USING btree (store_kind, store_root, store_id, status) index idx_backup_items_arn: CREATE INDEX idx_backup_items_arn ON public.backup_items USING btree (backup_arn) index idx_backups_table: CREATE INDEX idx_backups_table ON public.backups USING btree (account_id, table_name) index idx_idempotency_tokens_created: CREATE INDEX idx_idempotency_tokens_created ON public.idempotency_tokens USING btree (created_at) @@ -227,6 +268,7 @@ index idx_metrics_bucket: CREATE INDEX idx_metrics_bucket ON public.metrics USIN index idx_stream_records_created: CREATE INDEX idx_stream_records_created ON public.stream_records USING btree (created_at) index idx_stream_shards_table: CREATE INDEX idx_stream_shards_table ON public.stream_shards USING btree (table_id) index idx_table_restores_backup: CREATE INDEX idx_table_restores_backup ON public.table_restores USING btree (source_backup_arn) +index idx_table_restores_status: CREATE INDEX idx_table_restores_status ON public.table_restores USING btree (status, heartbeat_at) index idx_tables_pending_transition: CREATE INDEX idx_tables_pending_transition ON public.tables USING btree (status_transition_at) WHERE (status_transition_at IS NOT NULL) index idx_vector_indexes_index_id: CREATE UNIQUE INDEX idx_vector_indexes_index_id ON public.vector_indexes USING btree (index_id) index indexes_pkey: CREATE UNIQUE INDEX indexes_pkey ON public.indexes USING btree (table_id, index_name) @@ -247,7 +289,10 @@ constraint accounts.accounts_pkey: PRIMARY KEY (account_id) constraint admin_users.admin_users_pkey: PRIMARY KEY (admin_name) constraint backup_definitions.backup_definitions_backup_arn_fkey: FOREIGN KEY (backup_arn) REFERENCES backups(backup_arn) ON DELETE CASCADE constraint backup_definitions.backup_definitions_pkey: PRIMARY KEY (backup_arn) +constraint backup_index.backup_index_pkey: PRIMARY KEY (account_id, backup_id) constraint backup_items.backup_items_backup_arn_fkey: FOREIGN KEY (backup_arn) REFERENCES backups(backup_arn) ON DELETE CASCADE +constraint backup_maintenance.backup_maintenance_pkey: PRIMARY KEY (singleton) +constraint backup_maintenance.backup_maintenance_singleton_check: CHECK (singleton) constraint backups.backups_pkey: PRIMARY KEY (backup_arn) constraint extenddb_migrations.extenddb_migrations_pkey: PRIMARY KEY (id) constraint iam_group_members.iam_group_members_account_id_group_name_fkey: FOREIGN KEY (account_id, group_name) REFERENCES iam_groups(account_id, group_name) ON DELETE CASCADE @@ -301,6 +346,7 @@ sequence stream_seq: bigint START 1 INCREMENT 1 table extenddb_migrations table gsi_pending table idempotency_tokens +table restore_progress table schema_history table stream_records table stream_shards @@ -324,6 +370,9 @@ column idempotency_tokens.account_id: text NOT NULL column idempotency_tokens.token: text NOT NULL column idempotency_tokens.fingerprint: text NOT NULL column idempotency_tokens.created_at: timestamp with time zone NOT NULL DEFAULT now() +column restore_progress.table_id: text NOT NULL +column restore_progress.file_index: bigint NOT NULL +column restore_progress.items_applied: bigint NOT NULL column schema_history.filename: text NOT NULL column schema_history.applied_at: timestamp with time zone NOT NULL DEFAULT now() column stream_records.shard_id: text NOT NULL @@ -349,6 +398,7 @@ index idx_idempotency_tokens_created: CREATE INDEX idx_idempotency_tokens_create index idx_stream_records_created: CREATE INDEX idx_stream_records_created ON public.stream_records USING btree (created_at) index idx_stream_shards_table: CREATE INDEX idx_stream_shards_table ON public.stream_shards USING btree (table_id) index idx_vector_index_holds_table: CREATE INDEX idx_vector_index_holds_table ON public.vector_index_holds USING btree (table_id) +index restore_progress_pkey: CREATE UNIQUE INDEX restore_progress_pkey ON public.restore_progress USING btree (table_id, file_index) index schema_history_pkey: CREATE UNIQUE INDEX schema_history_pkey ON public.schema_history USING btree (filename) index stream_records_pkey: CREATE UNIQUE INDEX stream_records_pkey ON public.stream_records USING btree (shard_id, sequence_number) index stream_shards_pkey: CREATE UNIQUE INDEX stream_shards_pkey ON public.stream_shards USING btree (shard_id) @@ -356,6 +406,7 @@ index vector_index_holds_pkey: CREATE UNIQUE INDEX vector_index_holds_pkey ON pu constraint extenddb_migrations.extenddb_migrations_pkey: PRIMARY KEY (id) constraint gsi_pending.gsi_pending_pkey: PRIMARY KEY (id) constraint idempotency_tokens.idempotency_tokens_pkey: PRIMARY KEY (account_id, token) +constraint restore_progress.restore_progress_pkey: PRIMARY KEY (table_id, file_index) constraint schema_history.schema_history_pkey: PRIMARY KEY (filename) constraint stream_records.stream_records_pkey: PRIMARY KEY (shard_id, sequence_number) constraint stream_records.stream_records_shard_id_fkey: FOREIGN KEY (shard_id) REFERENCES stream_shards(shard_id) ON DELETE CASCADE diff --git a/docs/schema/sqlite.txt b/docs/schema/sqlite.txt index c74145d0f..24e052ba3 100644 --- a/docs/schema/sqlite.txt +++ b/docs/schema/sqlite.txt @@ -1,3 +1,6 @@ +index idx_backup_index_account_table on backup_index: CREATE INDEX idx_backup_index_account_table ON backup_index (account_id, table_name) +index idx_backup_index_backup_id on backup_index: CREATE UNIQUE INDEX idx_backup_index_backup_id ON backup_index (backup_id) +index idx_backup_index_store_status on backup_index: CREATE INDEX idx_backup_index_store_status ON backup_index (store_kind, store_root, store_id, status) index idx_backup_items_arn on backup_items: CREATE INDEX idx_backup_items_arn ON backup_items (backup_arn) index idx_backups_table on backups: CREATE INDEX idx_backups_table ON backups (account_id, table_name) index idx_gsi_pending_claim on gsi_pending: CREATE INDEX idx_gsi_pending_claim ON gsi_pending (worker_partition, ready_at, id) @@ -8,13 +11,16 @@ index idx_metrics_bucket on metrics: CREATE INDEX idx_metrics_bucket ON metrics index idx_stream_records_created on stream_records: CREATE INDEX idx_stream_records_created ON stream_records (created_at) index idx_stream_shards_table on stream_shards: CREATE INDEX idx_stream_shards_table ON stream_shards (table_id) index idx_table_restores_backup on table_restores: CREATE INDEX idx_table_restores_backup ON table_restores (source_backup_arn) +index idx_table_restores_status on table_restores: CREATE INDEX idx_table_restores_status ON table_restores (status, heartbeat_at) index idx_tables_pending_transition on tables: CREATE INDEX idx_tables_pending_transition ON tables (status_transition_at) WHERE status_transition_at IS NOT NULL index idx_vector_indexes_index_id on vector_indexes: CREATE UNIQUE INDEX idx_vector_indexes_index_id ON vector_indexes (index_id) table access_keys on access_keys: CREATE TABLE access_keys ( access_key_id TEXT PRIMARY KEY, secret_key_encrypted BLOB NOT NULL, account_id TEXT NOT NULL, user_name TEXT NOT NULL, is_active INTEGER NOT NULL DEFAULT 1, created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), FOREIGN KEY (account_id, user_name) REFERENCES iam_users(account_id, user_name) ON DELETE CASCADE ) table accounts on accounts: CREATE TABLE accounts ( account_id TEXT PRIMARY KEY, account_name TEXT NOT NULL UNIQUE, created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) ) table admin_users on admin_users: CREATE TABLE admin_users ( admin_name TEXT PRIMARY KEY, password_hash TEXT NOT NULL, created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) ) table backup_definitions on backup_definitions: CREATE TABLE backup_definitions ( backup_arn TEXT PRIMARY KEY REFERENCES backups(backup_arn) ON DELETE CASCADE, definition TEXT NOT NULL ) +table backup_index on backup_index: CREATE TABLE backup_index ( account_id TEXT NOT NULL, backup_id TEXT NOT NULL, table_name TEXT NOT NULL, backup_arn TEXT NOT NULL, backup_name TEXT NOT NULL, status TEXT NOT NULL, store_kind TEXT NOT NULL, store_root TEXT NOT NULL, store_id TEXT NOT NULL, size_bytes INTEGER NOT NULL DEFAULT 0, item_count INTEGER NOT NULL DEFAULT 0, created_at TEXT NOT NULL, completed_at TEXT, failure_reason TEXT, source_table_details_json TEXT NOT NULL, owner_generation INTEGER NOT NULL, heartbeat_at TEXT NOT NULL, PRIMARY KEY (account_id, backup_id) ) table backup_items on backup_items: CREATE TABLE backup_items ( id INTEGER PRIMARY KEY AUTOINCREMENT, backup_arn TEXT NOT NULL REFERENCES backups(backup_arn) ON DELETE CASCADE, pk TEXT NOT NULL, sk TEXT, item_data TEXT NOT NULL ) +table backup_maintenance on backup_maintenance: CREATE TABLE backup_maintenance ( singleton INTEGER PRIMARY KEY CHECK (singleton = 1), holder TEXT NOT NULL, owner_generation INTEGER NOT NULL, claimed_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), heartbeat_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) ) table backups on backups: CREATE TABLE backups ( backup_arn TEXT PRIMARY KEY, backup_name TEXT NOT NULL, table_id TEXT NOT NULL, table_name TEXT NOT NULL, account_id TEXT NOT NULL, backup_status TEXT NOT NULL DEFAULT 'AVAILABLE', backup_type TEXT NOT NULL DEFAULT 'USER', backup_size_bytes INTEGER NOT NULL DEFAULT 0, item_count INTEGER NOT NULL DEFAULT 0, key_schema TEXT NOT NULL, attribute_definitions TEXT NOT NULL, billing_mode TEXT NOT NULL DEFAULT 'PAY_PER_REQUEST', provisioned_throughput TEXT, stream_specification TEXT, created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) ) table extenddb_migrations on extenddb_migrations: CREATE TABLE extenddb_migrations (id INTEGER PRIMARY KEY, name TEXT NOT NULL, kind TEXT NOT NULL, checksum TEXT NOT NULL, outcome TEXT NOT NULL, notes TEXT NOT NULL DEFAULT '', applied_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), applied_by TEXT NOT NULL) table gsi_pending on gsi_pending: CREATE TABLE gsi_pending ( id INTEGER PRIMARY KEY AUTOINCREMENT, table_id TEXT NOT NULL, worker_partition INTEGER NOT NULL, old_item TEXT, new_item TEXT, index_context TEXT NOT NULL, ready_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) ) @@ -36,7 +42,7 @@ table seq_counters on seq_counters: CREATE TABLE seq_counters ( name TEXT PRIMAR table settings on settings: CREATE TABLE settings ( key TEXT PRIMARY KEY, value TEXT NOT NULL ) table stream_records on stream_records: CREATE TABLE stream_records ( shard_id TEXT NOT NULL REFERENCES stream_shards(shard_id) ON DELETE CASCADE, sequence_number TEXT NOT NULL, table_id TEXT NOT NULL, event_name TEXT NOT NULL, record_data TEXT NOT NULL, created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), PRIMARY KEY (shard_id, sequence_number) ) table stream_shards on stream_shards: CREATE TABLE stream_shards ( shard_id TEXT PRIMARY KEY, table_id TEXT NOT NULL REFERENCES tables(table_id) ON DELETE CASCADE, parent_shard_id TEXT, starting_sequence_number TEXT NOT NULL, ending_sequence_number TEXT, created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) ) -table table_restores on table_restores: CREATE TABLE table_restores ( table_id TEXT PRIMARY KEY REFERENCES tables(table_id) ON DELETE CASCADE, source_backup_arn TEXT NOT NULL, source_table_arn TEXT, restore_date_time TEXT NOT NULL ) +table table_restores on table_restores: CREATE TABLE table_restores ( table_id TEXT PRIMARY KEY REFERENCES tables(table_id) ON DELETE CASCADE, source_backup_arn TEXT NOT NULL, source_table_arn TEXT, restore_date_time TEXT NOT NULL , status TEXT, kind TEXT, store_kind TEXT, store_root TEXT, store_id TEXT, owner_generation INTEGER, heartbeat_at TEXT, spool_host TEXT, file_index INTEGER, items_applied INTEGER, failure_reason TEXT, completed_at TEXT) table tables on tables: CREATE TABLE tables ( account_id TEXT NOT NULL REFERENCES accounts(account_id) ON DELETE CASCADE, table_name TEXT NOT NULL, key_schema TEXT NOT NULL, attribute_definitions TEXT NOT NULL, billing_mode TEXT NOT NULL DEFAULT 'PAY_PER_REQUEST', provisioned_throughput TEXT, stream_specification TEXT, table_status TEXT NOT NULL DEFAULT 'CREATING', creation_date_time TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), table_size_bytes INTEGER NOT NULL DEFAULT 0, item_count INTEGER NOT NULL DEFAULT 0, table_arn TEXT NOT NULL, table_id TEXT NOT NULL, ttl_attribute TEXT, deletion_protection_enabled INTEGER NOT NULL DEFAULT 0, status_transition_at TEXT, stream_label TEXT, ttl_index_ready INTEGER NOT NULL DEFAULT 0, table_class TEXT, sse_specification TEXT, on_demand_throughput TEXT, PRIMARY KEY (account_id, table_name), CONSTRAINT tables_table_id_unique UNIQUE (table_id) ) table tags on tags: CREATE TABLE tags ( resource_arn TEXT NOT NULL, tag_key TEXT NOT NULL, tag_value TEXT NOT NULL, PRIMARY KEY (resource_arn, tag_key) ) table vector_indexes on vector_indexes: CREATE TABLE vector_indexes ( table_id TEXT NOT NULL, index_id TEXT NOT NULL, index_name TEXT NOT NULL, dimensions INTEGER NOT NULL, distance_function TEXT NOT NULL, vector_attribute TEXT NOT NULL, search_schema TEXT, projection TEXT NOT NULL, index_status TEXT NOT NULL DEFAULT 'CREATING', backfilling INTEGER, -- Items the backfill skipped because their stored bytes cannot enter the -- index (unparseable row, malformed or wrong-dimension vector). NULL until -- a backfill has completed; 0 afterwards when nothing was skipped. Kept so -- an operator can see that an ACTIVE index deliberately omits rows, rather -- than the build looping forever on them or dying part-way. skipped_item_count INTEGER, PRIMARY KEY (table_id, index_name), CONSTRAINT vector_indexes_table_id_fkey FOREIGN KEY (table_id) REFERENCES tables(table_id) ON DELETE CASCADE, CONSTRAINT chk_vector_dimensions_positive CHECK (dimensions > 0), CONSTRAINT chk_vector_backfilling_bool CHECK (backfilling IS NULL OR backfilling IN (0, 1)), -- An ACTIVE index must not carry the member at all, which is what the -- service does. Enforced here as well as in core, so a bug in the backend -- cannot persist a state the wire contract forbids. CONSTRAINT chk_vector_active_has_no_backfilling CHECK (index_status <> 'ACTIVE' OR backfilling IS NULL) ) diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index f22a2c4d1..2151440b3 100755 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -240,9 +240,9 @@ journalctl -t extenddb -f **Fix:** If permission denied, run `extenddb stop` as the same user that started extenddb. If the process already exited, the stale PID file will be cleaned up on the next `extenddb stop` invocation. -### `extenddb (pid ) did not exit within 10s after SIGTERM` +### `extenddb (pid ) did not exit within 76s after SIGTERM` -**Cause:** The server received SIGTERM but did not shut down within the 10-second timeout. It may be stuck draining long-running requests or waiting on a database operation. +**Cause:** The server received SIGTERM but did not shut down within the 76-second timeout, the sum of its shutdown bounds (5 s to drain connections, 30 s for running backup jobs to end, 36 s to stop its background workers and give up its serving registration, 5 s for its runtime). It may be stuck draining long-running requests or waiting on a database operation. **Fix:** Send SIGKILL to force termination: `kill -9 `. Check PostgreSQL for long-running queries that may be blocking shutdown. diff --git a/extenddb.sample.toml b/extenddb.sample.toml index 91869028a..d4ff16b12 100755 --- a/extenddb.sample.toml +++ b/extenddb.sample.toml @@ -53,13 +53,15 @@ # one against the data database, which `init` # always creates separately. Total PostgreSQL # connections used: 2 * pool_size + - # catalog_pool_size, plus one per vector index + # catalog_pool_size + [backup] max_concurrent_jobs + # (one claim connection per running backup or + # restore job), plus one per vector index # build running on this server and one while a # schema migration runs. max_connections is # per cluster, not per database, so both pools - # draw on the same budget. The build session and - # the migration connection sit OUTSIDE every - # pool, for two different reasons. + # draw on the same budget. The claim connections, + # the build session and the migration connection + # sit OUTSIDE every pool, for different reasons. # A build's ownership lock is SESSION-scoped, so # it needs a session that ends when the build # ends: a pooled connection returns to the pool @@ -71,10 +73,10 @@ # otherwise let the lock move between backends. # Increase for higher concurrency; ensure # PostgreSQL max_connections >= 2 * pool_size - # + catalog_pool_size + the number of vector - # index builds you expect to overlap. The - # defaults already need 60 of PostgreSQL's - # default 100. + # + catalog_pool_size + max_concurrent_jobs + + # the number of vector index builds you expect + # to overlap. The defaults already need 64 of + # PostgreSQL's default 100. # catalog_pool_size = 20 # Maximum concurrent connections for the # management/catalog pool (authz, IAM, console). # Defaults to pool_size if not set. @@ -173,6 +175,45 @@ # Read logs with: journalctl -t extenddb # No configuration needed — output always goes to syslog. +# [backup] +# Backup and restore jobs and the reconciler. Every key has a default; the +# section may be left out. stale_job_after_secs must be at least three +# heartbeat periods, the larger of heartbeat_interval_secs and the 15 s the +# in-catalog jobs heartbeat at (45 at the least); max_concurrent_jobs is +# at most 1024; on PostgreSQL pool_size and catalog_pool_size are raised to +# at least 2 * max_concurrent_jobs + 10. +# data_file_target_bytes = 67108864 # Uncompressed bytes per data file (minimum 32 MiB) +# heartbeat_interval_secs = 15 # Job heartbeat timer +# stale_job_after_secs = 300 # Heartbeat age after which a job's owner is dead +# failed_row_retention_secs = 604800 # FAILED tombstones kept for the operator (7 days) +# reconcile_interval_secs = 60 # Reconciler cycle +# max_concurrent_jobs = 4 # Backup and restore jobs running at once on this instance +# spool_path = "/var/lib/extenddb/spool" # Restore download area for an S3 store; required +# # with an S3 store except on file-backed SQLite, where +# # it defaults to backup-spool/ beside the database file + +# [backup.store] +# Where CreateBackup writes its backups on PostgreSQL, SQLite and MongoDB +# (CreateBackup answers CREATING and a job of the server uploads; Cassandra +# writes in-catalog backups with or without a store), and where restores and +# deletes of store-backed backups read. Absent on file-backed SQLite: a filesystem +# store at backups/ beside the database file. Absent on PostgreSQL, MongoDB, +# Cassandra and an in-memory SQLite database: no store, and CreateBackup +# writes in-catalog backups with a deprecation warning. kind = "disabled" +# opts out on any backend. Instances that share a catalog must share one +# store root (an S3 bucket and prefix, or a shared mount). +# kind = "filesystem" # "filesystem" | "s3" | "disabled" +# [backup.store.filesystem] +# path = "/var/lib/extenddb/backups" # Must exist, owned by the extenddb user, mode 0700 +# [backup.store.s3] +# bucket = "my-extenddb-backups" +# prefix = "extenddb/" # Optional key prefix +# region = "us-east-1" # Optional; the AWS SDK's chain otherwise +# endpoint = "http://127.0.0.1:9000" # Optional, for an S3-compatible server +# force_path_style = true # Required with an http:// endpoint +# upload_part_size_bytes = 8388608 # 5 MiB to 5 GiB +# max_concurrent_uploads = 4 + # Import/Export file operations are disabled by default (secure default). # Configure allowed directories separately for imports and exports. # All file paths are canonicalized and must resolve under one of the diff --git a/requirements-matrix.txt b/requirements-matrix.txt new file mode 100644 index 000000000..73033be99 --- /dev/null +++ b/requirements-matrix.txt @@ -0,0 +1,9 @@ +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 +# The drivers devtools/backup-restore-matrix uses to set up and tear down its +# MongoDB and Cassandra deployments (.github/workflows/matrix-nightly.yml +# installs this file beside requirements.txt). Separate from requirements.txt +# because nothing else in the test suites talks to those services directly +# and cassandra-driver builds a C extension on install. +pymongo==4.7.3 +cassandra-driver==3.29.3 diff --git a/tests/rust/src/backup_arn_scoping.rs b/tests/rust/src/backup_arn_scoping.rs index 42e5a2905..6dc8714f6 100644 --- a/tests/rust/src/backup_arn_scoping.rs +++ b/tests/rust/src/backup_arn_scoping.rs @@ -12,8 +12,11 @@ //! //! These tests drive the ARN directly rather than provisioning a second account, //! so they exercise the resolution rule without needing a second set of -//! credentials. +//! credentials. Every backup they create is waited to AVAILABLE before it is +//! described or deleted (`backup_restore::create_available_backup`: a +//! store-backed ExtendDB creates asynchronously) and deleted at the end. +use crate::backup_restore::create_available_backup; use crate::test_base::*; use aws_sdk_dynamodb::types::{ AttributeDefinition, BillingMode, KeySchemaElement, KeyType, ScalarAttributeType, @@ -126,14 +129,7 @@ async fn describe_backup_with_own_arn_still_resolves() { let table = format!("BackupScopeOwn_{}", ts()); make_table(&table).await; - let create = c - .create_backup() - .table_name(&table) - .backup_name("scope-own") - .send() - .await - .unwrap(); - let arn = create.backup_details().unwrap().backup_arn().to_string(); + let arn = create_available_backup(c, &table, "scope-own").await; let resp = c.describe_backup().backup_arn(&arn).send().await.unwrap(); assert_eq!( @@ -159,16 +155,12 @@ async fn describe_backup_with_own_nonexistent_arn_is_backup_not_found() { let table = format!("BackupScopeMiss_{}", ts()); make_table(&table).await; - let create = c - .create_backup() - .table_name(&table) - .backup_name("scope-miss") - .send() - .await - .unwrap(); - let real = create.backup_details().unwrap().backup_arn().to_string(); + let real = create_available_backup(c, &table, "scope-miss").await; // Same account/region/table prefix, a backup id that was never issued. - let missing = format!("{}/00000000000000-deadbeef", real.rsplit_once('/').unwrap().0); + let missing = format!( + "{}/00000000000000-deadbeef", + real.rsplit_once('/').unwrap().0 + ); let err = c .describe_backup() @@ -194,18 +186,16 @@ async fn describe_backup_after_delete_is_backup_not_found() { let table = format!("BackupScopeDel_{}", ts()); make_table(&table).await; - let create = c - .create_backup() - .table_name(&table) - .backup_name("scope-del") - .send() - .await - .unwrap(); - let arn = create.backup_details().unwrap().backup_arn().to_string(); + let arn = create_available_backup(c, &table, "scope-del").await; c.delete_backup().backup_arn(&arn).send().await.unwrap(); - let err = c.describe_backup().backup_arn(&arn).send().await.unwrap_err(); + let err = c + .describe_backup() + .backup_arn(&arn) + .send() + .await + .unwrap_err(); assert_eq!( err_code(&err), Some("BackupNotFoundException"), @@ -214,6 +204,7 @@ async fn describe_backup_after_delete_is_backup_not_found() { c.delete_table().table_name(&table).send().await.ok(); } + #[tokio::test] async fn backup_ids_are_not_derived_from_the_timestamp_alone() { let c = client(); @@ -222,14 +213,7 @@ async fn backup_ids_are_not_derived_from_the_timestamp_alone() { let mut arns = Vec::new(); for i in 0..3 { - let create = c - .create_backup() - .table_name(&table) - .backup_name(format!("scope-id-{i}")) - .send() - .await - .unwrap(); - arns.push(create.backup_details().unwrap().backup_arn().to_string()); + arns.push(create_available_backup(c, &table, &format!("scope-id-{i}")).await); } for arn in &arns { diff --git a/tests/rust/src/backup_restore.rs b/tests/rust/src/backup_restore.rs index e7b93884a..640b3482d 100755 --- a/tests/rust/src/backup_restore.rs +++ b/tests/rust/src/backup_restore.rs @@ -2,12 +2,104 @@ // SPDX-License-Identifier: Apache-2.0 //! Backup and restore integration tests — mirrors Java `BackupRestoreTests`. +//! +//! The service creates a backup asynchronously; so does ExtendDB with a +//! configured `[backup.store]` (CreateBackup answers CREATING and a job +//! uploads it). A backup still CREATING refuses DeleteBackup +//! (`BackupInUseException`) and restore, so every test that goes on to +//! describe, delete or restore a backup it just created waits for AVAILABLE +//! first ([`wait_for_backup_available`]), and deletes the backups it made so +//! the server's store does not keep them. Where a backup lives decides what +//! RestoreTableFromBackup answers this release; the tests read it from +//! `EXTENDDB_TEST_BACKUP_STORE` ([`backup_store`]), as the Python suite does. use crate::test_base::*; use aws_sdk_dynamodb::types::{ AttributeDefinition, BillingMode, KeySchemaElement, KeyType, PointInTimeRecoverySpecification, ScalarAttributeType, }; +use aws_sdk_dynamodb::Client; +use std::time::{Duration, Instant}; + +/// Where the server under test writes its backups, from +/// `EXTENDDB_TEST_BACKUP_STORE`: `filesystem` or `s3` (a configured +/// `[backup.store]`, under which a backup is in the store and the restore +/// job restores it) or `none` (the in-catalog create, which the legacy +/// reader restores). Against the service the variable is +/// not read (the restore runs). Against ExtendDB a test whose outcome depends +/// on it fails when it is unset, with the text the Python fixture prints, +/// rather than accept either outcome. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum BackupStore { + Filesystem, + S3, + None, +} + +pub fn backup_store() -> BackupStore { + const VALUES: [&str; 3] = ["filesystem", "s3", "none"]; + if is_real_dynamodb() { + return BackupStore::None; + } + let raw = std::env::var("EXTENDDB_TEST_BACKUP_STORE").unwrap_or_default(); + match raw.trim() { + "filesystem" => BackupStore::Filesystem, + "s3" => BackupStore::S3, + "none" => BackupStore::None, + "" => panic!( + "EXTENDDB_TEST_BACKUP_STORE is unset; set it to one of {VALUES:?} to say where the \ + server under test writes its backups" + ), + other => panic!( + "EXTENDDB_TEST_BACKUP_STORE is {other:?}; set it to one of {VALUES:?} to say where \ + the server under test writes its backups" + ), + } +} + +/// Poll DescribeBackup until the backup is AVAILABLE. Any other status than +/// CREATING is a failure: a backup that fails is BackupNotFound, never +/// FAILED, on the wire, so the describe itself errors then. +pub async fn wait_for_backup_available(c: &Client, arn: &str) { + let deadline = Instant::now() + Duration::from_secs(120); + loop { + let resp = c + .describe_backup() + .backup_arn(arn) + .send() + .await + .unwrap_or_else(|e| { + panic!("describe_backup of {arn} while waiting for AVAILABLE: {e:?}") + }); + let status = resp + .backup_description() + .and_then(|b| b.backup_details()) + .map(|d| d.backup_status().as_str().to_owned()) + .unwrap_or_default(); + if status == "AVAILABLE" { + return; + } + assert_eq!(status, "CREATING", "backup {arn} is {status}"); + assert!( + Instant::now() < deadline, + "backup {arn} not AVAILABLE in time" + ); + tokio::time::sleep(Duration::from_millis(250)).await; + } +} + +/// CreateBackup, then wait for AVAILABLE; the backup's ARN. +pub async fn create_available_backup(c: &Client, table: &str, backup_name: &str) -> String { + let create = create_backup_with_retry(c, table, backup_name).await; + let arn = create.backup_details().unwrap().backup_arn().to_string(); + wait_for_backup_available(c, &arn).await; + arn +} + +/// DeleteBackup after the test; a backup already gone is not a failure. +pub async fn delete_backup_quietly(c: &Client, arn: &str) { + c.delete_backup().backup_arn(arn).send().await.ok(); +} async fn make_table(name: &str) { let c = client(); @@ -71,8 +163,27 @@ async fn create_backup_happy_case() { let resp = create_backup_with_retry(c, &table, "test-backup-1").await; let details = resp.backup_details().unwrap(); assert!(!details.backup_arn().is_empty()); - assert_eq!(details.backup_status().as_str(), "AVAILABLE"); + // The in-catalog create answers AVAILABLE; the service and a store-backed + // ExtendDB answer CREATING and the backup reaches AVAILABLE later. + let status = details.backup_status().as_str(); + assert!( + status == "AVAILABLE" || status == "CREATING", + "CreateBackup answered {status}" + ); + let arn = details.backup_arn().to_string(); + wait_for_backup_available(c, &arn).await; + let desc = c.describe_backup().backup_arn(&arn).send().await.unwrap(); + assert_eq!( + desc.backup_description() + .unwrap() + .backup_details() + .unwrap() + .backup_status() + .as_str(), + "AVAILABLE" + ); + delete_backup_quietly(c, &arn).await; c.delete_table().table_name(&table).send().await.ok(); } @@ -101,13 +212,13 @@ async fn describe_backup() { let table = format!("DescBackup_{}", ts()); make_table(&table).await; - let create = create_backup_with_retry(c, &table, "desc-backup").await; - let arn = create.backup_details().unwrap().backup_arn().to_string(); + let arn = create_available_backup(c, &table, "desc-backup").await; let resp = c.describe_backup().backup_arn(&arn).send().await.unwrap(); let desc = resp.backup_description().unwrap(); assert_eq!(desc.backup_details().unwrap().backup_arn(), arn.as_str()); + delete_backup_quietly(c, &arn).await; c.delete_table().table_name(&table).send().await.ok(); } @@ -129,12 +240,14 @@ async fn list_backups() { let table = format!("ListBackup_{}", ts()); make_table(&table).await; - create_backup_with_retry(c, &table, "list-1").await; - create_backup_with_retry(c, &table, "list-2").await; + let first = create_available_backup(c, &table, "list-1").await; + let second = create_available_backup(c, &table, "list-2").await; let resp = c.list_backups().table_name(&table).send().await.unwrap(); assert!(resp.backup_summaries().len() >= 2); + delete_backup_quietly(c, &first).await; + delete_backup_quietly(c, &second).await; c.delete_table().table_name(&table).send().await.ok(); } @@ -156,8 +269,7 @@ async fn delete_backup() { let table = format!("DelBackup_{}", ts()); make_table(&table).await; - let create = create_backup_with_retry(c, &table, "del-backup").await; - let arn = create.backup_details().unwrap().backup_arn().to_string(); + let arn = create_available_backup(c, &table, "del-backup").await; let resp = c.delete_backup().backup_arn(&arn).send().await.unwrap(); let desc = resp.backup_description().unwrap(); @@ -185,31 +297,50 @@ async fn restore_table_from_backup() { .unwrap(); } - let create = create_backup_with_retry(c, &table, "restore-backup").await; - let arn = create.backup_details().unwrap().backup_arn().to_string(); + let arn = create_available_backup(c, &table, "restore-backup").await; let restored = format!("Restored_{}", ts()); - let resp = c + let restore = c .restore_table_from_backup() .target_table_name(&restored) .backup_arn(&arn) .send() - .await - .unwrap(); - assert_eq!( - resp.table_description() - .unwrap() - .table_status() - .unwrap() - .as_str(), - "CREATING" - ); + .await; + // The round trip on either path: a backup in the configured store is + // restored by the restore job, an in-catalog one by the legacy reader. + let resp = restore.unwrap(); + let desc = resp.table_description().unwrap(); + assert_eq!(desc.table_status().unwrap().as_str(), "CREATING"); + let summary = desc + .restore_summary() + .expect("a CREATING restore target carries its RestoreSummary"); + assert_eq!(summary.source_backup_arn(), Some(arn.as_str())); + assert!(summary.restore_in_progress()); wait_for_active(c, &restored).await; let scan = c.scan().table_name(&restored).send().await.unwrap(); assert_eq!(scan.count(), 5); - + let after = c + .describe_table() + .table_name(&restored) + .send() + .await + .unwrap(); + assert!( + after.table().unwrap().restore_summary().is_none(), + "RestoreSummary is reported only while CREATING" + ); + println!( + "restore of a {} backup completed with {} items", + match backup_store() { + BackupStore::Filesystem | BackupStore::S3 => "store-backed", + BackupStore::None => "in-catalog", + }, + scan.count() + ); c.delete_table().table_name(&restored).send().await.ok(); + + delete_backup_quietly(c, &arn).await; c.delete_table().table_name(&table).send().await.ok(); } diff --git a/tests/rust/src/restore_active_completeness.rs b/tests/rust/src/restore_active_completeness.rs index 83644ddc0..bab83e8f1 100644 --- a/tests/rust/src/restore_active_completeness.rs +++ b/tests/rust/src/restore_active_completeness.rs @@ -23,12 +23,18 @@ //! The dataset is sized so the copy takes longer than the transition window on //! a loaded server. On very fast or idle hardware, raising `ITEMS` widens the //! window. +//! +//! On a server with a configured `[backup.store]` (`EXTENDDB_TEST_BACKUP_STORE` +//! at `filesystem` or `s3`) the restore is the restore job's, from the store, +//! and the race runs against it as against the in-catalog copy. +use crate::backup_restore::{ + backup_store, delete_backup_quietly, wait_for_backup_available, BackupStore, +}; use crate::test_base::*; use aws_sdk_dynamodb::types::{ - AttributeDefinition, AttributeValue, BillingMode, GlobalSecondaryIndex, - KeySchemaElement, KeyType, Projection, ProjectionType, PutRequest, ScalarAttributeType, - Select, WriteRequest, + AttributeDefinition, AttributeValue, BillingMode, GlobalSecondaryIndex, KeySchemaElement, + KeyType, Projection, ProjectionType, PutRequest, ScalarAttributeType, Select, WriteRequest, }; use std::time::Duration; @@ -159,20 +165,16 @@ async fn restored_table_has_all_items_when_first_active() { .await .unwrap(); let arn = backup.backup_details().unwrap().backup_arn().to_string(); - // Wait until the backup is AVAILABLE. - for _ in 0..240 { - let d = c.describe_backup().backup_arn(&arn).send().await.unwrap(); - if d.backup_description() - .and_then(|b| b.backup_details()) - .map(|b| b.backup_status().as_str() == "AVAILABLE") - .unwrap_or(false) - { - break; - } - tokio::time::sleep(std::time::Duration::from_millis(250)).await; - } + wait_for_backup_available(c, &arn).await; let dst = format!("RestoreRaceDst_{}", ts()); + println!( + "restore of {} backup: racing the observer", + match backup_store() { + BackupStore::Filesystem | BackupStore::S3 => "a store-backed", + BackupStore::None => "an in-catalog", + } + ); // Race an observer against the in-flight restore: the moment it sees // ACTIVE, it counts. Returns None if ACTIVE never arrives within the // bound, which means the restore itself failed. @@ -254,6 +256,7 @@ async fn restored_table_has_all_items_when_first_active() { // above was still in flight (or just after, if the copy was fast). let observed = observer.await.unwrap(); + delete_backup_quietly(c, &arn).await; c.delete_table().table_name(&src).send().await.ok(); c.delete_table().table_name(&dst).send().await.ok(); diff --git a/tests/test_abac.py b/tests/test_abac.py index b56b85036..1e10a80b2 100755 --- a/tests/test_abac.py +++ b/tests/test_abac.py @@ -35,6 +35,7 @@ from conftest import wait_for_active, wait_for_deleted from management_helpers import ManagementClient +from test_backup_restore_fidelity import _delete_backup_when_available def _require_auth_env() -> tuple[str, str, str]: endpoint = os.environ.get("EXTENDDB_TEST_ENDPOINT", "").strip() admin_user = os.environ.get("EXTENDDB_ADMIN_USER", "").strip() @@ -1446,10 +1447,10 @@ def describe_backup(): "BackupDescription"]["BackupDetails"]["BackupArn"] == backup_arn finally: if backup_arn is not None: - try: - admin_client.delete_backup(BackupArn=backup_arn) - except Exception: - pass + # On a server with a backup store the backup is CREATING for + # the job's duration and DeleteBackup answers BackupInUse; + # deleting at once leaked a backup on every store-backed run. + _delete_backup_when_available(admin_client, backup_arn) try: admin_client.delete_table(TableName=table) except Exception: diff --git a/tests/test_auth_permissions.py b/tests/test_auth_permissions.py index 157f4cec3..6cf89d714 100755 --- a/tests/test_auth_permissions.py +++ b/tests/test_auth_permissions.py @@ -18,6 +18,7 @@ from __future__ import annotations import os +import time import uuid import boto3 @@ -1355,6 +1356,14 @@ def source_backup(self): backup_arn = admin_client.create_backup( TableName=table, BackupName="authz", )["BackupDetails"]["BackupArn"] + # A store-backed backup is CREATING until its job ends; a restore + # of a CREATING backup is BackupInUse, so the cells wait here. + deadline = time.monotonic() + 120 + while admin_client.describe_backup( + BackupArn=backup_arn + )["BackupDescription"]["BackupDetails"]["BackupStatus"] == "CREATING": + assert time.monotonic() < deadline, f"backup {backup_arn} still CREATING" + time.sleep(0.2) yield admin_client, table, backup_arn finally: for t in self.created: @@ -1364,6 +1373,13 @@ def source_backup(self): pass if backup_arn is not None: try: + # A store-backed backup is CREATING until its job ends + # and refuses DeleteBackup until then. + deadline = time.monotonic() + 120 + while time.monotonic() < deadline and admin_client.describe_backup( + BackupArn=backup_arn + )["BackupDescription"]["BackupDetails"]["BackupStatus"] == "CREATING": + time.sleep(0.2) admin_client.delete_backup(BackupArn=backup_arn) except Exception: pass diff --git a/tests/test_authz_resource_arn.py b/tests/test_authz_resource_arn.py index 4d81c5955..6c3b07808 100644 --- a/tests/test_authz_resource_arn.py +++ b/tests/test_authz_resource_arn.py @@ -38,6 +38,7 @@ from conftest import wait_for_active from management_helpers import ManagementClient +from test_backup_restore_fidelity import _delete_backup_when_available def _require_auth_env() -> tuple[str, str, str]: @@ -122,7 +123,13 @@ def _user_with_policy(mgmt: ManagementClient, account_id: str, policy: dict) -> @pytest.fixture(scope="module") def table(auth_env, mgmt, account_id, region): """A table with a GSI, a stream, one item, and a backup. Returns the - canonical table, index, stream, and backup ARNs.""" + canonical table, index, stream, and backup ARNs. The cells deny + DeleteBackup and DeleteTable to other users, so the backup and the table + are deleted here once the module is done: on a server with a configured + `[backup.store]` the backup is a complete set of objects under the store, + and one that is left behind stays until someone deletes it (CI runners + die with theirs; a developer's server keeps them). The backup is deleted + once AVAILABLE, since a backup still CREATING refuses DeleteBackup.""" endpoint = auth_env[0] ak, sk = _user_with_policy(mgmt, account_id, ALLOW_ALL) admin = _client(endpoint, ak, sk, region) @@ -147,13 +154,18 @@ def table(auth_env, mgmt, account_id, region): desc = admin.describe_table(TableName=name)["Table"] table_arn = desc["TableArn"] backup_arn = admin.create_backup(TableName=name, BackupName="arnbk")["BackupDetails"]["BackupArn"] - return { + yield { "name": name, "table_arn": table_arn, "index_arn": table_arn + "/index/gsi1", "stream_arn": desc.get("LatestStreamArn"), "backup_arn": backup_arn, } + _delete_backup_when_available(admin, backup_arn) + try: + admin.delete_table(TableName=name) + except ClientError: + pass class TestExplicitDenyOnResourceArn: diff --git a/tests/test_backup_job.py b/tests/test_backup_job.py new file mode 100644 index 000000000..0e48aeeec --- /dev/null +++ b/tests/test_backup_job.py @@ -0,0 +1,2528 @@ +# SPDX-License-Identifier: Apache-2.0 + +"""The store-backed CreateBackup job at the server's boundary. + +Each test starts its own `extenddb serve --foreground` on an isolated +deployment with a filesystem backup store (SQLite: the default root beside +the database file; PostgreSQL: a root under the test's directory) and reads +the API's answers, the server's log, `extenddb backup list --failed` and the +store's root: + +- a second CreateBackup while one job runs with `max_concurrent_jobs = 1` + waits for the slot and is answered only once the first job has ended (the + first backup AVAILABLE), and a third is accepted at once afterwards; a wait + that outlives the 30 s bound is LimitExceededException naming the bound; +- a table holding an `N` value the wire let in (#618: `1E126` or `abc` in a + non-key attribute, `1E126` in the range key of a key-ordered table) written + through each of the four paths that store it unvalidated (BatchWriteItem, + TransactWriteItems, UpdateItem on an absent item in the key it creates, + ImportTable) gives a FAILED row, no manifest under the root, three log + lines naming each item's key and attribute, and a FAILED reason giving the + count and the first item and naming #618; +- an item over `max_item_size_bytes` (stored under the default limit, the + server restarted with a lower one) gives a reason that does not name #618; +- a refused item with a UUID hash key and a ten-digit epoch `N` range key is + named by its whole key, and the wire's reason for its value, in the log + line and in `backup list --failed`. + +The store-backed RestoreTableFromBackup job, on both deployments: + +- a data file tampered with under the root fails the restore before any item + is loaded, the target gone and the log naming the file; +- SIGTERM mid-restore leaves the record RUNNING and the restarted server + resumes it (PostgreSQL after the stale window, SQLite at once); +- an S3 store (moto) cut mid-download leaves the record RUNNING and the + restore completes once the store answers; +- an unordered backup holding one key in two files, and a key-ordered file + leaving its `key_range` (both hand-built under the root), fail the restore + with the target gone; +- a table keyed on two HASH attributes (the `allow_multipart_table_keys` + preview, enabled on the cell's own deployment) backs up and restores with + the source's item set, key schema and attribute definitions; +- (SQLite) a RUNNING restore record with no recorded progress whose target + holds more items than the backup, the state a catalog restored from a dump + leaves when it names a restore that finished and was written to since, + seeded at the catalog, is FAILED `target_not_empty` + by the restarted server's reconciler, the target left CREATING and + `DeleteTable`, `DeleteBackup` and a second restore refused, and `extenddb + backup remove-target` removes the target and its record, after which + `DescribeTable` is `ResourceNotFoundException` and `DeleteBackup` succeeds. + +`extenddb backup sync`, on the SQLite and PostgreSQL deployments: + +- after the catalog loses four rows, sync gives an AVAILABLE row to the one + whole prefix, and DescribeBackup, ListBackups and a restore of it work; a + prefix without its manifest, with a sidecar that does not match, moved + under another backup id, under `changelog/` or `exports/`, or with a + FAILED, DELETING or CREATING row is reported and not adopted (on SQLite, + where sync takes the serve lock and refuses beside a server, the + dead-owner resolutions run first: the DELETING row is finished and the + CREATING row resolved AVAILABLE); +- a row whose manifest is gone is reported and stays AVAILABLE, and + `--prune-missing` removes the row and deletes no object; an empty root, a + root holding only the identity file and one with another store id each + make the store unhealthy, and sync and `--prune-missing` refuse with no + row changed and nothing written; so does a missing manifest behind the + newest AVAILABLE row, until the manifest is back; +- a row-less prefix whose manifest the store cannot read is reported and + the run exits 1 as incomplete; once it can be read, the next sync adopts + it; +- (PostgreSQL) while `--prune-missing` holds the maintenance claim, `backup + purge` and `--adopt-store` fail at once, and `--adopt-store` on a root + the catalog owns waits for a running job as `--prune-missing` does; +- (PostgreSQL) on host-local roots, `--prune-missing` on the host that + lacks a backup's bytes removes its row only, the other host keeps the + bytes, and a sync there adopts them again; +- (PostgreSQL) while purge holds the claim a plain sync fails at once; +- `--adopt-store` from a second catalog configured with the root makes that + catalog the owner, its server restores the adopted backup, and the first + catalog's server finds the store owned by another catalog; +- an identity file rewritten to name a third catalog while an adopting sync + is in its prefix loop ends the command with exit 1, naming that catalog; +- (PostgreSQL) a DeleteBackup on the first catalog's server, still running + with its last check healthy after the adoption, is refused by its + deletion fence and leaves the bytes, and the adopting catalog's store + stays healthy. + +The binary at `EXTENDDB_BINARY` decides which deployment runs: a SQLite build, +a PostgreSQL build (with `EXTENDDB_TEST_PG_CONNECTION_STRING`) or a MongoDB +build (with `EXTENDDB_TEST_MONGODB_URI`, a replica set of this suite's own: the +catalog's database names are fixed, so each test destroys the previous test's +catalog and initializes its own); the restore cells run on the SQLite and +PostgreSQL deployments. Excluded from the backend-agnostic suite; run by name. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +import shutil +import subprocess +import threading +import time +import uuid +from concurrent.futures import ThreadPoolExecutor + +import boto3 +import pytest +import requests +from botocore.auth import SigV4Auth +from botocore.awsrequest import AWSRequest +from botocore.exceptions import ClientError + +import lifecycle_helpers as lh +from test_backup_store_config import ( + ADMIN_PASSWORD, + EXTENDDB_BINARY, + _append, + _client, + _Forwarder, + _free_port, + _health, + _moto_or_skip, + _pg_build, + _provision, + _s3_env, + _Server, + _sqlite_build, + _SqliteDeployment, + _wait_backup_available, + _wait_health, + moto_bucket, # noqa: F401 - the fixture of the store-outage cell +) + +ACCOUNT = "123456789012" +# The job's bound in the slot-wait cell: items of this count and padding keep +# the first job running for longer than the second request's round trip (0.4 s +# for 30,000 400-byte items on a SQLite build of this host), so the second +# request is sent while the first job holds the one slot and has to wait for +# it. The passing branch is the second request admitted once the first job +# ended (the first backup AVAILABLE at its answer, and seen CREATING while it +# waited); a second answered while the first is still CREATING is a broken +# bound; a refusal at the wait's bound is the slow-host branch, which the cell +# accepts; a first backup already AVAILABLE before the second is sent is a +# probe that came too late, where the cell asks for more items. +BUSY_ITEMS = int(os.environ.get("EXTENDDB_TEST_BACKUP_BUSY_ITEMS", "30000")) +LIMIT_TEXT = ( + "Too many backup and restore jobs are running on this server " + "(at most 1 at once, the [backup] max_concurrent_jobs bound); retry later" +) + + +class _Deployment: + """A deployment with a filesystem store, its config, its root and its + catalog (`catalog`: the SQLite database file, or the PostgreSQL catalog + database's connection string), where a cell edits a `backup_index` row + it rewrote the prefix of.""" + + def __init__(self, backend: str, config: str, port: int, root: str, cert: str, cwd: str, catalog: str = ""): + self.backend = backend + self.config = config + self.port = port + self.root = root + self.cert = cert + self.cwd = cwd + self.catalog = catalog + + def run(self, *args): + import subprocess + + return subprocess.run( + [EXTENDDB_BINARY, args[0], "--config", self.config, *args[1:]], + cwd=self.cwd, + capture_output=True, + text=True, + stdin=subprocess.DEVNULL, + timeout=180, + ) + + def set_limit(self, key: str, value: str) -> None: + """Set a key of the init template's `[limits]` table.""" + with open(self.config) as f: + text = f.read() + assert text.count("[limits]\n") == 1, text + text = text.replace("[limits]\n", f"[limits]\n{key} = {value}\n", 1) + with open(self.config, "w") as f: + f.write(text) + + +def _mongodb_build() -> bool: + if not os.path.isfile(EXTENDDB_BINARY): + return False + res = subprocess.run([EXTENDDB_BINARY, "version"], capture_output=True, text=True, timeout=30) + return "(mongodb)" in res.stdout + + +def _mongodb_deployment(base: str, uri: str) -> tuple[str, int]: + """A catalog on the replica set at `uri`, initialized afresh: the config + stub names the connection string `init` reads, the previous test's + catalog (the database names are fixed per replica set) is destroyed first, + and the generated config is given a free port.""" + config = os.path.join(base, "extenddb.toml") + with open(config, "w") as f: + f.write(f'[storage]\nbackend = "mongodb"\n[storage.mongodb]\nconnection_string = "{uri}"\n') + env = os.environ.copy() + env["EXTENDDB_ADMIN_PASSWORD"] = ADMIN_PASSWORD + common = dict(cwd=base, capture_output=True, text=True, stdin=subprocess.DEVNULL, timeout=120, env=env) + # Nothing to destroy on the first test; `destroy` then exits non-zero. + subprocess.run([EXTENDDB_BINARY, "destroy", "--config", config, "--yes"], **common) + res = subprocess.run( + [EXTENDDB_BINARY, "init", "--backend", "mongodb", "--config", config, "--overwrite"], **common + ) + assert res.returncode == 0, res.stdout + res.stderr + port = lh._find_free_port() + lh._patch_config_port(config, port) + return config, port + + +@pytest.fixture() +def deployment(request, tmp_path) -> _Deployment: + """The build's deployment: SQLite with the defaulted root, or PostgreSQL + or MongoDB with a filesystem root under the test's directory. All read + `reconcile_interval_secs = 1`, `max_concurrent_jobs = 1` and import from + the test's directory (ImportTable reads `//`).""" + base = os.path.realpath(str(tmp_path)) + if _sqlite_build(): + d = _SqliteDeployment(str(tmp_path)) + config, port, cert = d.config, d.port, d.cert + root = os.path.join(base, "backups") + store = "" + backend = "sqlite" + catalog = d.db_path + elif _pg_build(): + cli_env = request.getfixturevalue("cli_env") + result = lh._run_extenddb( + "init", *lh._init_args(cli_env), config=cli_env["config_path"], + env_override={"EXTENDDB_ADMIN_PASSWORD": ADMIN_PASSWORD}, + ) + assert result.returncode == 0, result.stdout + result.stderr + lh._patch_config_port(cli_env["config_path"], cli_env["port"]) + config, port = cli_env["config_path"], cli_env["port"] + cert = os.path.join(cli_env["tls_dir"], "cert.pem") + root = os.path.join(base, "backups") + os.mkdir(root, 0o700) + store = f'[backup.store]\nkind = "filesystem"\n[backup.store.filesystem]\npath = "{root}"\n' + backend = "postgres" + catalog = cli_env["conn_string"] + elif _mongodb_build(): + uri = os.environ.get("EXTENDDB_TEST_MONGODB_URI") + if not uri: + pytest.skip("EXTENDDB_TEST_MONGODB_URI is unset (a replica set of this suite's own)") + config, port = _mongodb_deployment(base, uri) + cert = os.path.expanduser("~/.extenddb/tls/cert.pem") + root = os.path.join(base, "backups") + os.mkdir(root, 0o700) + store = f'[backup.store]\nkind = "filesystem"\n[backup.store.filesystem]\npath = "{root}"\n' + backend = "mongodb" + # The restore cells that edit the catalog run on SQLite and PostgreSQL. + catalog = "" + else: + pytest.skip("EXTENDDB_BINARY is neither a SQLite, a PostgreSQL nor a MongoDB build") + os.mkdir(os.path.join(base, ACCOUNT)) + _append( + config, + "[backup]\nreconcile_interval_secs = 1\nmax_concurrent_jobs = 1\n" + + store + + f'[import]\npaths = ["{base}"]\n', + ) + return _Deployment(backend, config, port, root, cert, base, catalog) + + +def _started(d: _Deployment, log_path: str) -> tuple[_Server, object, dict]: + """The server started on the deployment, a client and its credentials.""" + server = _Server(d.config, log_path) + try: + _wait_health( + d.port, + lambda h: h.get("components", {}).get("backup_store", {}).get("status") == "healthy", + server=server, + ) + creds = _provision(d.port, d.cert, ADMIN_PASSWORD) + except BaseException: + # A server whose store never came up (or whose provisioning failed) + # would otherwise keep running after the test's failure. + server.stop() + raise + return server, _client(d.port, creds), creds + + +def _signed_post(d: _Deployment, creds: dict, operation: str, body: dict) -> requests.Response: + """A hand-signed request for an operation botocore's model would refuse + (ImportTable's `FileSource`).""" + body_bytes = json.dumps(body).encode("utf-8") + headers = { + "X-Amz-Target": f"DynamoDB_20120810.{operation}", + "Content-Type": "application/x-amz-json-1.0", + } + endpoint = f"https://127.0.0.1:{d.port}" + frozen = boto3.Session( + aws_access_key_id=creds["AWS_ACCESS_KEY_ID"], + aws_secret_access_key=creds["AWS_SECRET_ACCESS_KEY"], + ).get_credentials().get_frozen_credentials() + aws_req = AWSRequest(method="POST", url=endpoint, data=body_bytes, headers=headers) + SigV4Auth(frozen, "dynamodb", "us-east-1").add_auth(aws_req) + return requests.post(endpoint, data=body_bytes, headers=dict(aws_req.headers), verify=False) + + +def _wait_active(ddb, name: str, timeout: float = 60.0) -> None: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if ddb.describe_table(TableName=name)["Table"]["TableStatus"] == "ACTIVE": + return + time.sleep(0.05) + raise AssertionError(f"{name} did not become ACTIVE") + + +def _create_composite_table(ddb, name: str, gsi: bool = False) -> None: + """A key-ordered table: `pk` S hash key, `sk` N range key.""" + attrs = [ + {"AttributeName": "pk", "AttributeType": "S"}, + {"AttributeName": "sk", "AttributeType": "N"}, + ] + kwargs: dict = {} + if gsi: + attrs.append({"AttributeName": "g", "AttributeType": "S"}) + kwargs["GlobalSecondaryIndexes"] = [ + { + "IndexName": "by-g", + "KeySchema": [{"AttributeName": "g", "KeyType": "HASH"}], + "Projection": {"ProjectionType": "ALL"}, + } + ] + ddb.create_table( + TableName=name, + KeySchema=[ + {"AttributeName": "pk", "KeyType": "HASH"}, + {"AttributeName": "sk", "KeyType": "RANGE"}, + ], + AttributeDefinitions=attrs, + BillingMode="PAY_PER_REQUEST", + **kwargs, + ) + _wait_active(ddb, name) + + +def _wait_backup_gone(ddb, arn: str, timeout: float = 60.0) -> None: + """Poll DescribeBackup until the backup is BackupNotFound (its row FAILED + or removed); an AVAILABLE backup fails the test.""" + deadline = time.monotonic() + timeout + while True: + try: + status = ddb.describe_backup(BackupArn=arn)["BackupDescription"]["BackupDetails"]["BackupStatus"] + except ClientError as e: + assert e.response["Error"]["Code"] == "BackupNotFoundException", e.response["Error"] + return + assert status == "CREATING", f"backup {arn} is {status}" + assert time.monotonic() < deadline, f"backup {arn} still CREATING after {timeout} s" + time.sleep(0.1) + + +_RUST_ESCAPES = {"n": "\n", "r": "\r", "t": "\t", "0": "\0", "\\": "\\", '"': '"', "'": "'"} + + +def _rust_debug_string(quoted: str) -> str: + """The string a Rust `{:?}` of a `String` printed: the quotes removed and + its escapes undone (`\\n`, `\\"`, `\\\\` and `\\u{hex}` for a character + `Debug` does not print as itself). `json.loads` refuses the `\\u{..}` + form, so a reason holding such a character (a key the client wrote with + one) is decoded here.""" + assert quoted.startswith('"') and quoted.endswith('"'), quoted + out: list[str] = [] + i, body = 0, quoted[1:-1] + while i < len(body): + c = body[i] + if c != "\\": + out.append(c) + i += 1 + continue + e = body[i + 1] + if e == "u": + close = body.index("}", i + 3) + out.append(chr(int(body[i + 3 : close], 16))) + i = close + 1 + else: + out.append(_RUST_ESCAPES[e]) + i += 2 + return "".join(out) + + +def _failed_rows(d: _Deployment) -> dict[str, str]: + """`extenddb backup list --failed`: each FAILED row's ARN to its reason.""" + res = d.run("backup", "list", "--failed") + assert res.returncode == 0, res.stdout + res.stderr + rows: dict[str, str] = {} + for line in res.stdout.splitlines(): + m = re.match(r"^FAILED\s.*?\s(arn:\S+)\s.*failure_reason=(\".*\")$", line) + if m: + rows[m.group(1)] = _rust_debug_string(m.group(2)) + return rows + + +def _prefix(d: _Deployment, table: str, arn: str) -> str: + return os.path.join(d.root, ACCOUNT, table, arn.rsplit("/", 1)[1]) + + +def _assert_no_manifest(d: _Deployment, table: str, arn: str) -> None: + prefix = _prefix(d, table, arn) + if os.path.isdir(prefix): + left = os.listdir(prefix) + assert "extenddb-manifest.json" not in left, left + + +def _restored_item_lines(log: str, arn: str) -> list[str]: + return [ + line for line in log.splitlines() if f"backup {arn}: item cannot be restored: " in line + ] + + +def test_second_create_backup_over_the_job_bound_waits_for_the_slot(deployment, tmp_path): + """With `max_concurrent_jobs = 1`, a CreateBackup sent while one job runs + waits for the slot (up to 30 s) and is answered only once the first job + has ended: when its answer arrives the first backup is AVAILABLE, never + still CREATING (a second accepted while the first is CREATING fails the + cell: the bound did not hold). A first backup already AVAILABLE when the + second is sent means the probe came too late and asks for more items. A + wait that outlives the bound (a slow host) is LimitExceededException with + the bound's text, which the cell accepts and reports. The first job + finishes AVAILABLE with every item and a GSI in its + SourceTableFeatureDetails, and a third CreateBackup is accepted at once + (the slot is free).""" + d = deployment + server, ddb, creds = _started(d, str(tmp_path / "serve.log")) + try: + name = f"busy-{uuid.uuid4().hex[:8]}" + _create_composite_table(ddb, name, gsi=True) + _load(d, creds, name, BUSY_ITEMS) + first = ddb.create_backup(TableName=name, BackupName="first")["BackupDetails"] + assert first["BackupStatus"] == "CREATING", first + before = ddb.describe_backup(BackupArn=first["BackupArn"])["BackupDescription"]["BackupDetails"]["BackupStatus"] + if before == "AVAILABLE": + pytest.fail( + "the first backup was already AVAILABLE before the second CreateBackup was sent, " + "so the probe checks nothing; raise EXTENDDB_TEST_BACKUP_BUSY_ITEMS" + ) + + # The second request from its own client and thread: it blocks on the + # slot while the main thread watches the first backup. + def second_request(): + client = _client(d.port, creds) + sent = time.monotonic() + try: + return client.create_backup(TableName=name, BackupName="second")["BackupDetails"], None, time.monotonic() - sent + except ClientError as e: + return None, e.response["Error"], time.monotonic() - sent + + with ThreadPoolExecutor(max_workers=1) as pool: + pending = pool.submit(second_request) + # The first backup is CREATING for as long as the second waits + # (a poll that sees it AVAILABLE is the first job's end, after + # which the second is admitted at once). + seen_creating = 0 + while not pending.done(): + status = ddb.describe_backup(BackupArn=first["BackupArn"])["BackupDescription"]["BackupDetails"]["BackupStatus"] + seen_creating += status == "CREATING" + time.sleep(0.05) + second, refused, waited = pending.result() + status_at_answer = ddb.describe_backup(BackupArn=first["BackupArn"])["BackupDescription"]["BackupDetails"]["BackupStatus"] + if refused is not None: + # The slow-host branch: the first job outlived the 30 s wait. + assert refused["Code"] == "LimitExceededException", refused + assert refused["Message"] == LIMIT_TEXT, refused + assert waited >= 25, f"a refusal before the wait's bound: {waited:.1f} s" + print(f"branch: the second request waited {waited:.1f} s and was refused at the bound ({d.backend})") + else: + assert status_at_answer == "AVAILABLE", ( + f"the second CreateBackup was answered while the first job was still running " + f"(the first backup is {status_at_answer}): the max_concurrent_jobs bound did not hold" + ) + # The second request waited: the first was CREATING while it was + # pending (a first job that ended before the thread connected + # would pass the branch above without any wait). + assert seen_creating > 0, ( + "the first backup was AVAILABLE before the second request was sent, so the probe " + "checked nothing; raise EXTENDDB_TEST_BACKUP_BUSY_ITEMS" + ) + assert second["BackupStatus"] in ("CREATING", "AVAILABLE"), second + print( + f"branch: the second request waited {waited:.1f} s for the slot and was admitted; " + f"the first was seen CREATING {seen_creating} time(s) meanwhile ({d.backend})" + ) + _wait_backup_available(ddb, second["BackupArn"], timeout=120) + _wait_backup_available(ddb, first["BackupArn"], timeout=120) + desc = ddb.describe_backup(BackupArn=first["BackupArn"])["BackupDescription"] + assert desc["SourceTableDetails"]["ItemCount"] == BUSY_ITEMS, desc["SourceTableDetails"] + gsis = desc["SourceTableFeatureDetails"]["GlobalSecondaryIndexes"] + assert [g["IndexName"] for g in gsis] == ["by-g"], gsis + assert os.path.isfile(os.path.join(_prefix(d, name, first["BackupArn"]), "extenddb-manifest.json")) + third = ddb.create_backup(TableName=name, BackupName="third")["BackupDetails"] + assert third["BackupStatus"] == "CREATING", third + _wait_backup_available(ddb, third["BackupArn"], timeout=120) + listed = sorted(b["BackupName"] for b in ddb.list_backups(TableName=name)["BackupSummaries"]) + expected = ["first", "third"] if refused is not None else ["first", "second", "third"] + assert listed == expected, listed + finally: + server.stop() + + +def _load(d: _Deployment, creds: dict, name: str, n: int) -> None: + """Write n composite-key items with BatchWriteItem from 8 threads, one + client each: a boto3 client shared across threads fails at random with + TLS record-layer errors that look like server faults.""" + local = threading.local() + + def write(start: int) -> None: + if not hasattr(local, "client"): + local.client = _client(d.port, creds) + requests_ = [ + { + "PutRequest": { + "Item": { + "pk": {"S": f"p{i % 500:04d}"}, + "sk": {"N": str(i)}, + "g": {"S": f"g{i % 50}"}, + "pad": {"S": "x" * 400}, + } + } + } + for i in range(start, min(start + 25, n)) + ] + pending = {name: requests_} + while pending: + pending = local.client.batch_write_item(RequestItems=pending).get("UnprocessedItems") or {} + if pending: + time.sleep(0.05) + + with ThreadPoolExecutor(max_workers=8) as pool: + list(pool.map(write, range(0, n, 25))) + + +# The three items each path stores: `1E126` and `abc` in a non-key attribute +# and `1E126` in the range key (an `abc` range key is refused by the number +# column on PostgreSQL and SQLite, so no path stores one). UpdateItem +# validates the values its expression sets, so its three items carry the bad +# value in the key it creates. +BAD_ITEMS = [ + {"pk": {"S": "a"}, "sk": {"N": "1"}, "amount": {"N": "1E126"}}, + {"pk": {"S": "a"}, "sk": {"N": "2"}, "note": {"N": "abc"}}, + {"pk": {"S": "b"}, "sk": {"N": "1E126"}}, +] +BAD_KEYS = [ + {"pk": {"S": "a"}, "sk": {"N": "1E126"}}, + {"pk": {"S": "b"}, "sk": {"N": "1E127"}}, + {"pk": {"S": "c"}, "sk": {"N": "1E128"}}, +] +GOOD_ITEMS = [{"pk": {"S": "z"}, "sk": {"N": str(i)}, "amount": {"N": str(i)}} for i in range(5)] + + +def _write_batch(d, ddb, creds, name): + _create_composite_table(ddb, name) + ddb.batch_write_item( + RequestItems={name: [{"PutRequest": {"Item": i}} for i in BAD_ITEMS + GOOD_ITEMS]} + ) + return BAD_ITEMS + + +def _write_transact(d, ddb, creds, name): + _create_composite_table(ddb, name) + ddb.transact_write_items( + TransactItems=[{"Put": {"TableName": name, "Item": i}} for i in BAD_ITEMS + GOOD_ITEMS] + ) + return BAD_ITEMS + + +def _write_update(d, ddb, creds, name): + _create_composite_table(ddb, name) + for key in BAD_KEYS: + ddb.update_item( + TableName=name, + Key=key, + UpdateExpression="SET note = :v", + ExpressionAttributeValues={":v": {"S": "created from the key"}}, + ) + for item in GOOD_ITEMS: + ddb.put_item(TableName=name, Item=item) + return BAD_KEYS + + +def _write_import(d, ddb, creds, name): + path = os.path.join(d.cwd, ACCOUNT, f"{name}.json") + with open(path, "w") as f: + for item in BAD_ITEMS + GOOD_ITEMS: + f.write(json.dumps({"Item": item}) + "\n") + resp = _signed_post( + d, + creds, + "ImportTable", + { + "FileSource": {"Path": path}, + "InputFormat": "DYNAMODB_JSON", + "TableCreationParameters": { + "TableName": name, + "AttributeDefinitions": [ + {"AttributeName": "pk", "AttributeType": "S"}, + {"AttributeName": "sk", "AttributeType": "N"}, + ], + "KeySchema": [ + {"AttributeName": "pk", "KeyType": "HASH"}, + {"AttributeName": "sk", "KeyType": "RANGE"}, + ], + "BillingMode": "PAY_PER_REQUEST", + }, + }, + ) + assert resp.status_code == 200, resp.text + assert resp.json()["ImportTableDescription"]["ImportStatus"] == "COMPLETED", resp.text + _wait_active(ddb, name) + return BAD_ITEMS + + +WRITE_PATHS = { + "batch_write_item": _write_batch, + "transact_write_items": _write_transact, + "update_item": _write_update, + "import_table": _write_import, +} + + +def _key_json(item: dict) -> str: + return json.dumps({"pk": item["pk"], "sk": item["sk"]}, separators=(",", ":")) + + +@pytest.mark.parametrize("path", sorted(WRITE_PATHS)) +def test_an_item_the_wire_let_in_fails_the_backup(deployment, tmp_path, path): + """Three items with an `N` value #618 let in, stored through one write + path, and five that restore: CreateBackup answers CREATING, the job + validates every item (three log lines, one per item, with its key and + attribute path), the row is FAILED with a reason giving the count and + the first item and naming #618, DescribeBackup is BackupNotFound, + ListBackups omits it, and no manifest is under the root.""" + d = deployment + server, ddb, creds = _started(d, str(tmp_path / "serve.log")) + try: + name = f"n618-{path.replace('_', '')}-{uuid.uuid4().hex[:6]}" + bad = WRITE_PATHS[path](d, ddb, creds, name) + stored = ddb.scan(TableName=name, ConsistentRead=True)["Items"] + assert len(stored) == len(bad) + len(GOOD_ITEMS), stored + details = ddb.create_backup(TableName=name, BackupName="nightly")["BackupDetails"] + assert details["BackupStatus"] == "CREATING", details + arn = details["BackupArn"] + _wait_backup_gone(ddb, arn) + assert ddb.list_backups(TableName=name)["BackupSummaries"] == [] + _assert_no_manifest(d, name, arn) + reason = _failed_rows(d)[arn] + first = sorted(bad, key=lambda i: (i["pk"]["S"], float(i["sk"]["N"])))[0] + assert reason.startswith(f"{len(bad)} item(s) cannot be restored; first: key {_key_json(first)}, attribute "), reason + assert reason.endswith(" (#618)"), reason + if "sk" in first and first["sk"]["N"] == "1E126": + assert ", attribute sk: " in reason, reason + else: + assert ", attribute amount: " in reason, reason + log = server.log_text() + lines = _restored_item_lines(log, arn) + assert len(lines) == len(bad), lines + logged_keys = {re.search(r"key (\{.*?\}\}), attribute", line).group(1) for line in lines} + assert logged_keys == {_key_json(i) for i in bad}, logged_keys + assert all(re.search(r"attribute (sk|amount|note): ", line) for line in lines), lines + assert f"backup {arn}: FAILED ({reason}); its objects were deleted" in log, log + assert "more item(s) cannot be restored" not in log + finally: + server.stop() + + +def test_a_refused_item_with_a_uuid_and_epoch_key_is_named_whole(deployment, tmp_path): + """One refused item with a realistic key, a UUID hash key and a ten-digit + epoch range key, and `amount: abc`: the log line and the FAILED reason + carry the whole key as DynamoDB JSON (`{"pk":{"S":""},"sk":{"N": + "1761955200"}}`), not a shortened form, and the wire's reason for the + value, so an operator can find the item from either.""" + d = deployment + server, ddb, _ = _started(d, str(tmp_path / "serve.log")) + try: + name = f"n618-key-{uuid.uuid4().hex[:6]}" + _create_composite_table(ddb, name) + item = {"pk": {"S": str(uuid.uuid4())}, "sk": {"N": "1761955200"}, "amount": {"N": "abc"}} + ddb.batch_write_item(RequestItems={name: [{"PutRequest": {"Item": i}} for i in [item] + GOOD_ITEMS]}) + arn = ddb.create_backup(TableName=name, BackupName="nightly")["BackupDetails"]["BackupArn"] + _wait_backup_gone(ddb, arn) + key = _key_json(item) + assert f'"pk":{{"S":"{item["pk"]["S"]}"}}' in key and '"sk":{"N":"1761955200"}' in key, key + value_reason = "attribute amount: The parameter cannot be converted to a numeric value: abc" + reason = _failed_rows(d)[arn] + assert reason == f"1 item(s) cannot be restored; first: key {key}, {value_reason} (#618)", reason + lines = _restored_item_lines(server.log_text(), arn) + assert len(lines) == 1, lines + assert f"backup {arn}: item cannot be restored: key {key}, {value_reason}" in lines[0], lines[0] + finally: + server.stop() + + +def test_an_item_over_max_item_size_bytes_fails_the_backup_without_naming_618(deployment, tmp_path): + """A 300 KB item stored under the default `max_item_size_bytes`, the + server restarted with `[limits] max_item_size_bytes = 65536`: the backup + is FAILED with a reason naming the item's size and the limit and not + #618, and the server's own PutItem refuses such an item.""" + d = deployment + name = f"big-{uuid.uuid4().hex[:8]}" + server, ddb, _ = _started(d, str(tmp_path / "serve.log")) + try: + _create_composite_table(ddb, name) + ddb.put_item(TableName=name, Item={"pk": {"S": "big"}, "sk": {"N": "1"}, "pad": {"S": "x" * 300000}}) + ddb.put_item(TableName=name, Item={"pk": {"S": "small"}, "sk": {"N": "1"}, "pad": {"S": "x"}}) + # The same table backs up whole under the default limit. + arn = ddb.create_backup(TableName=name, BackupName="under-default")["BackupDetails"]["BackupArn"] + _wait_backup_available(ddb, arn) + finally: + server.stop() + d.set_limit("max_item_size_bytes", "65536") + server, ddb, _ = _started(d, str(tmp_path / "serve2.log")) + try: + with pytest.raises(ClientError) as refused: + ddb.put_item(TableName=name, Item={"pk": {"S": "big2"}, "sk": {"N": "1"}, "pad": {"S": "x" * 100000}}) + assert refused.value.response["Error"]["Code"] == "ValidationException" + arn = ddb.create_backup(TableName=name, BackupName="under-lower")["BackupDetails"]["BackupArn"] + _wait_backup_gone(ddb, arn) + _assert_no_manifest(d, name, arn) + reason = _failed_rows(d)[arn] + assert reason.startswith('1 item(s) cannot be restored; first: key {"pk":{"S":"big"},"sk":{"N":"1"}}: item size '), reason + assert reason.endswith(" bytes exceeds the limit of 65536 bytes"), reason + assert "#618" not in reason, reason + lines = _restored_item_lines(server.log_text(), arn) + assert len(lines) == 1 and "exceeds the limit of 65536 bytes" in lines[0], lines + # The first backup, taken under the default limit, is untouched. + listed = [b["BackupName"] for b in ddb.list_backups(TableName=name)["BackupSummaries"]] + assert listed == ["under-default"], listed + finally: + server.stop() + + +# --- the restore job --------------------------------------------------------------- + +# Items of this count and padding make a backup of two data files (the knob's +# floor is 32 MiB of uncompressed lines) whose restore runs for long enough, +# on a PostgreSQL build of this host, that a SIGTERM sent right after the +# restore's response finds the job inside its download or its first file's +# load. The cells check the precondition they rely on (the target still +# CREATING when the signal is sent) rather than assume it. +RESTORE_ITEMS = int(os.environ.get("EXTENDDB_TEST_RESTORE_ITEMS", "10000")) +RESTORE_PAD = 4000 + + +# The `[backup]` keys the restore cells add to the deployment's: data files +# close at the knob's floor, so RESTORE_ITEMS padded items make two of them. +RESTORE_BACKUP_SETTINGS = "data_file_target_bytes = 33554432\n" + + +def _add_backup_settings(d: _Deployment, extra: str) -> None: + """Add `extra` to the deployment's `[backup]` table before the server + starts (TOML refuses a second `[backup]` header).""" + with open(d.config) as f: + text = f.read() + head = "[backup]\nreconcile_interval_secs = 1\n" + assert text.count(head) == 1, text + with open(d.config, "w") as f: + f.write(text.replace(head, head + extra)) + + +def _load_padded(d: _Deployment, creds: dict, name: str, n: int, pad: int) -> None: + """`_load` with items of `pad` bytes of padding and a `g` the GSI keys + on, from 8 threads with one client each.""" + local = threading.local() + + def write(start: int) -> None: + if not hasattr(local, "client"): + local.client = _client(d.port, creds) + requests_ = [ + { + "PutRequest": { + "Item": { + "pk": {"S": f"p{i % 500:04d}"}, + "sk": {"N": str(i)}, + "g": {"S": f"g{i % 50}"}, + "pad": {"S": "x" * pad}, + } + } + } + for i in range(start, min(start + 25, n)) + ] + pending = {name: requests_} + while pending: + pending = local.client.batch_write_item(RequestItems=pending).get("UnprocessedItems") or {} + if pending: + time.sleep(0.05) + + with ThreadPoolExecutor(max_workers=8) as pool: + list(pool.map(write, range(0, n, 25))) + + +def _count(ddb, name: str) -> int: + total, start = 0, None + while True: + kwargs = {"TableName": name, "Select": "COUNT", "ConsistentRead": True} + if start: + kwargs["ExclusiveStartKey"] = start + resp = ddb.scan(**kwargs) + total += resp["Count"] + start = resp.get("LastEvaluatedKey") + if not start: + return total + + +def _wait_table_gone(ddb, name: str, timeout: float = 120.0) -> None: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + try: + status = ddb.describe_table(TableName=name)["Table"]["TableStatus"] + except ClientError as e: + assert e.response["Error"]["Code"] == "ResourceNotFoundException", e.response["Error"] + return + assert status == "CREATING", f"{name} is {status}, it was never writable" + time.sleep(0.2) + raise AssertionError(f"{name} still exists after {timeout} s") + + +def _backed_up_table(d: _Deployment, ddb, creds: dict, prefix: str) -> tuple[str, str]: + """A composite-key table with a GSI holding RESTORE_ITEMS padded items, + backed up to AVAILABLE in two data files; the name and the ARN.""" + name = f"{prefix}-{uuid.uuid4().hex[:8]}" + _create_composite_table(ddb, name, gsi=True) + _load_padded(d, creds, name, RESTORE_ITEMS, RESTORE_PAD) + arn = ddb.create_backup(TableName=name, BackupName="nightly")["BackupDetails"]["BackupArn"] + _wait_backup_available(ddb, arn, timeout=180) + files = sorted(os.listdir(os.path.join(_prefix(d, name, arn), "data"))) + assert len(files) >= 2, f"the backup has one data file ({files}); raise EXTENDDB_TEST_RESTORE_ITEMS" + return name, arn + + + + +def _restore_cell(d: _Deployment) -> None: + """The store-backed restore cells run on the SQLite and PostgreSQL + deployments; on MongoDB the store-backed restore is covered by + tests/test_mongodb_backup_restore.py and the crate's live tests.""" + if d.backend == "mongodb": + pytest.skip("the restore cells run on the SQLite and PostgreSQL deployments") + +def test_a_tampered_data_file_fails_the_restore_before_any_item(deployment, tmp_path): + """A byte of the backup's second data file flipped under the root: the + restore request is accepted (CREATING with a RestoreSummary), the target + is removed once the job fails, and the server log names the file. That + the job verified every file before it loaded any is pinned by the FAILED + reason's text: "does not match the manifest" is the hashing pass's, which + runs over every file before the first `bulk_load`; a mismatch found while + a file is loaded reads "data file : data file sha256 checksum + mismatch" or, when the flipped byte sits in the gzip trailer so the + decoder's own CRC fires first, "data file : data file decode error: + corrupt gzip stream does not have a matching checksum" instead. A + CREATING target answers no read whatever the job did, so the GetItem + probe below only confirms the target never became readable. The backup + itself stays AVAILABLE.""" + d = deployment + _restore_cell(d) + _add_backup_settings(d, RESTORE_BACKUP_SETTINGS) + server, ddb, creds = _started(d, str(tmp_path / "serve.log")) + try: + name, arn = _backed_up_table(d, ddb, creds, "tamper") + path = os.path.join(_prefix(d, name, arn), "data", "000002.json.gz") + with open(path, "r+b") as f: + f.seek(-1, os.SEEK_END) + last = f.read(1) + f.seek(-1, os.SEEK_END) + f.write(bytes([last[0] ^ 0xFF])) + target = f"{name}-r" + resp = ddb.restore_table_from_backup(TargetTableName=target, BackupArn=arn)["TableDescription"] + assert resp["TableStatus"] == "CREATING", resp + assert resp["RestoreSummary"]["SourceBackupArn"] == arn, resp + readable = [] + deadline = time.monotonic() + 120 + while time.monotonic() < deadline: + try: + got = ddb.get_item(TableName=target, Key={"pk": {"S": "p0000"}, "sk": {"N": "0"}}, ConsistentRead=True) + if "Item" in got: + readable.append(got["Item"]) + except ClientError: + pass + try: + ddb.describe_table(TableName=target) + except ClientError as e: + assert e.response["Error"]["Code"] == "ResourceNotFoundException", e.response["Error"] + break + time.sleep(0.2) + else: + raise AssertionError(f"{target} was not removed within 120 s") + assert readable == [], f"an item of the failed restore was readable: {readable[:1]}" + log = server.log_text() + failed = [line for line in log.splitlines() if "FAILED (data file data/000002.json.gz does not match the manifest" in line] + assert len(failed) == 1, log[-4000:] + assert "the target was removed" in failed[0], failed[0] + assert ddb.describe_backup(BackupArn=arn)["BackupDescription"]["BackupDetails"]["BackupStatus"] == "AVAILABLE" + finally: + server.stop() + + +def test_sigterm_leaves_the_restore_running_and_it_finishes_after_restart(deployment, tmp_path): + """SIGTERM while the restore runs: the server drains, the job stops at + its next file or chunk with no status write, the record stays RUNNING + (the target CREATING with its RestoreSummary across the restart), and the + restarted server's reconciler resumes it: on PostgreSQL once the dead + owner's heartbeat is older than `stale_job_after_secs` (45 s, the floor, + on the database's clock); on SQLite at the first cycle, since the restart + frees the record's slot and ages every RUNNING record. The target reaches + ACTIVE with every item and the GSI answers.""" + d = deployment + _restore_cell(d) + _add_backup_settings(d, RESTORE_BACKUP_SETTINGS + "stale_job_after_secs = 45\n") + server, ddb, creds = _started(d, str(tmp_path / "serve.log")) + try: + name, arn = _backed_up_table(d, ddb, creds, "sigterm") + target = f"{name}-r" + resp = ddb.restore_table_from_backup(TargetTableName=target, BackupArn=arn)["TableDescription"] + assert resp["TableStatus"] == "CREATING", resp + status = ddb.describe_table(TableName=target)["Table"]["TableStatus"] + assert status == "CREATING", ( + f"the restore finished ({status}) before the signal was sent; raise EXTENDDB_TEST_RESTORE_ITEMS" + ) + server.proc.terminate() + code = server.wait_exit(timeout=60) + assert code == 0, server.log_text()[-4000:] + log = server.log_text() + assert "stopped by the server's shutdown; the record stays RUNNING" in log, log[-4000:] + assert ": FAILED (" not in log, log[-4000:] + finally: + server.stop() + stopped_at = time.monotonic() + server, ddb, creds = _started(d, str(tmp_path / "serve2.log")) + try: + table = ddb.describe_table(TableName=target)["Table"] + if d.backend == "postgres": + # PostgreSQL resumes only once the dead owner's heartbeat is + # stale (45 s), so right after the restart the target is still + # CREATING, carries its RestoreSummary, and a retry into the name + # is refused as a restore in progress. On SQLite the restart frees + # the record's slot and the first reconciler cycle resumes at + # once, so the target may already be ACTIVE here; the + # resume-timing assertion below pins the at-once behaviour. + assert table["TableStatus"] == "CREATING", table + assert table["RestoreSummary"]["SourceBackupArn"] == arn, table + with pytest.raises(ClientError) as refused: + ddb.restore_table_from_backup(TargetTableName=target, BackupArn=arn) + assert refused.value.response["Error"]["Code"] == "TableInUseException", refused.value.response + _wait_active(ddb, target, timeout=240) + resumed_after = time.monotonic() - stopped_at + if d.backend == "postgres": + assert resumed_after >= 45 - 5, f"the resume waited for the stale window ({resumed_after:.1f} s)" + else: + assert resumed_after < 45 - 5, f"the restart's slot rule resumes at once ({resumed_after:.1f} s)" + assert _count(ddb, target) == RESTORE_ITEMS + by_g = ddb.query( + TableName=target, + IndexName="by-g", + KeyConditionExpression="g = :g", + ExpressionAttributeValues={":g": {"S": "g7"}}, + Select="COUNT", + )["Count"] + assert by_g == RESTORE_ITEMS // 50, by_g + log = server.log_text() + assert "taken over by this instance and resumed" in log, log[-4000:] + assert ": ACTIVE, " in log, log[-4000:] + finally: + server.stop() + + +def test_a_store_outage_mid_download_leaves_the_restore_running_and_it_completes(request, tmp_path, moto_bucket): + """An S3 store (moto) reached through a TCP forwarder, with a spool: the + forwarder is cut right after the restore's response, so the job's data + file downloads fail as transport errors; the record stays RUNNING (the + target CREATING) while the job retries with backoff; once the forwarder + is back the download completes and the target reaches ACTIVE with every + item, the spool entry removed. On PostgreSQL the spool is `spool_path`; + on SQLite the defaulted `backup-spool/` beside the database file.""" + moto_host, moto_port = _moto_or_skip() + bucket = moto_bucket + base = os.path.realpath(str(tmp_path)) + port = _free_port() + store = ( + '[backup.store]\nkind = "s3"\n[backup.store.s3]\n' + f'bucket = "{bucket}"\nendpoint = "http://127.0.0.1:{port}"\n' + 'force_path_style = true\nregion = "us-east-1"\n' + ) + if _sqlite_build(): + sd = _SqliteDeployment(str(tmp_path)) + spool = os.path.join(base, "backup-spool") + _append(sd.config, "[backup]\nreconcile_interval_secs = 1\nmax_concurrent_jobs = 1\n" + RESTORE_BACKUP_SETTINGS + store) + d = _Deployment("sqlite", sd.config, sd.port, "", sd.cert, base) + elif _pg_build(): + cli_env = request.getfixturevalue("cli_env") + result = lh._run_extenddb( + "init", *lh._init_args(cli_env), config=cli_env["config_path"], + env_override={"EXTENDDB_ADMIN_PASSWORD": ADMIN_PASSWORD}, + ) + assert result.returncode == 0, result.stdout + result.stderr + lh._patch_config_port(cli_env["config_path"], cli_env["port"]) + spool = os.path.join(base, "spool") + _append( + cli_env["config_path"], + "[backup]\nreconcile_interval_secs = 1\nmax_concurrent_jobs = 1\n" + + RESTORE_BACKUP_SETTINGS + + f'spool_path = "{spool}"\n' + + store, + ) + d = _Deployment("postgres", cli_env["config_path"], cli_env["port"], "", os.path.join(cli_env["tls_dir"], "cert.pem"), base) + else: + pytest.skip("EXTENDDB_BINARY is neither a SQLite nor a PostgreSQL build") + forwarder = _Forwarder(port, (moto_host, moto_port)) + server = _Server(d.config, str(tmp_path / "serve.log"), env=_s3_env()) + try: + _wait_health( + d.port, + lambda h: h.get("components", {}).get("backup_store", {}).get("status") == "healthy", + timeout=60, + server=server, + ) + creds = _provision(d.port, d.cert, ADMIN_PASSWORD) + ddb = _client(d.port, creds) + name = f"outage-{uuid.uuid4().hex[:8]}" + _create_composite_table(ddb, name, gsi=True) + _load_padded(d, creds, name, RESTORE_ITEMS, RESTORE_PAD) + arn = ddb.create_backup(TableName=name, BackupName="nightly")["BackupDetails"]["BackupArn"] + _wait_backup_available(ddb, arn, timeout=180) + target = f"{name}-r" + resp = ddb.restore_table_from_backup(TargetTableName=target, BackupArn=arn)["TableDescription"] + assert resp["TableStatus"] == "CREATING", resp + # The cut: every data file get fails until the forwarder is back. + forwarder.stop() + cut_at = time.monotonic() + while time.monotonic() - cut_at < 8: + table = ddb.describe_table(TableName=target)["Table"] + assert table["TableStatus"] == "CREATING", table + assert table["RestoreSummary"]["RestoreInProgress"] is True, table + time.sleep(0.5) + log = server.log_text() + retried = [line for line in log.splitlines() if "retried in" in line and "data/" in line] + assert retried, "no download attempt was retried during the cut:\n" + log[-4000:] + forwarder = _Forwarder(port, (moto_host, moto_port)) + _wait_active(ddb, target, timeout=240) + assert _count(ddb, target) == RESTORE_ITEMS + entries = [e for e in os.listdir(spool) if e.startswith("restore-")] + assert entries == [], f"the spool entry stays: {entries}" + log = server.log_text() + assert ": ACTIVE, " in log and ": FAILED (" not in log, log[-4000:] + finally: + server.stop() + forwarder.stop() + + +# --- `backup remove-target`: the target_not_empty state ------------------------------ + + +def _seed_target_not_empty(d: _Deployment, target: str) -> str: + """Put a finished restore (table ACTIVE, record DONE) back into the state + a catalog restored from a dump leaves when it names a restore that + finished and was written to since: the target CREATING with its items + (one more than the backup holds, written after the restore) and a RUNNING + manifest record with no progress, which the restarted server's reconciler + ages, takes over, and fails `target_not_empty` since the target holds + more items than the backup's total (a count at or below the total is a + first data file applied in part and is resumed). No kill produces the + state on any backend; the catalog is edited with the server stopped. + Returns the target's table id.""" + try: + import pysqlite3 as sqlite3 + except ImportError: + import sqlite3 + conn = sqlite3.connect(d.catalog, timeout=10) + try: + row = conn.execute( + "SELECT table_id FROM tables WHERE account_id = ? AND table_name = ?", (ACCOUNT, target) + ).fetchone() + assert row, f"no catalog row for {target}" + table_id = row[0] + n = conn.execute( + "UPDATE tables SET table_status = 'CREATING' WHERE table_id = ?", (table_id,) + ).rowcount + assert n == 1, n + n = conn.execute( + "UPDATE table_restores SET status = 'RUNNING', file_index = NULL, items_applied = NULL, " + "failure_reason = NULL, completed_at = NULL, heartbeat_at = '1970-01-01T00:00:00.000Z' " + "WHERE table_id = ? AND kind = 'manifest' AND status = 'DONE'", + (table_id,), + ).rowcount + assert n == 1, n + conn.commit() + finally: + conn.close() + return table_id + + +def test_remove_target_removes_a_target_not_empty_restore(deployment, tmp_path): + """The operator's remedy for a resume that found no progress and more + items than the backup holds. A backup restored to ACTIVE, with one item + written to the target since, is seeded back into the `target_not_empty` + state (`_seed_target_not_empty`: the shape a catalog restored from a dump + leaves; a catalog edit is the one way to produce it). The restarted server's + reconciler fails the record `target_not_empty` and leaves the target + CREATING, which `DeleteTable` refuses (`ResourceInUseException`) and + which keeps the backup in use (`BackupInUseException` to `DeleteBackup` + and to a second restore). `extenddb backup remove-target`, run with the + server stopped (on SQLite it takes the serve lock), removes the target + and its record and prints what it did; the restarted server then answers + `ResourceNotFoundException` for the target, `DeleteBackup` succeeds, and + a new restore into the name runs to ACTIVE with every item.""" + d = deployment + _restore_cell(d) + if d.backend != "sqlite": + pytest.skip("the seeded target_not_empty state is built on the SQLite catalog (the PostgreSQL command runs beside a live server and is the engine tests')") + server, ddb, creds = _started(d, str(tmp_path / "serve.log")) + try: + name, arn = _small_backed_up_table(ddb, "rmt") + target = f"{name}-r" + resp = ddb.restore_table_from_backup(TargetTableName=target, BackupArn=arn)["TableDescription"] + assert resp["TableStatus"] == "CREATING", resp + _wait_active(ddb, target) + assert _count(ddb, target) == HAND_BUILT_ITEMS + # Written to since: one item the backup does not hold, so the seeded + # record's target holds more than the backup's total. + ddb.put_item(TableName=target, Item=_hand_item(HAND_BUILT_ITEMS)) + assert _count(ddb, target) == HAND_BUILT_ITEMS + 1 + finally: + server.stop() + table_id = _seed_target_not_empty(d, target) + + server, ddb, creds = _started(d, str(tmp_path / "serve2.log")) + try: + deadline = time.monotonic() + 60 + while "target_not_empty" not in server.log_text(): + assert time.monotonic() < deadline, server.log_text()[-4000:] + time.sleep(0.2) + log = server.log_text() + failed = [line for line in log.splitlines() if "target_not_empty" in line] + assert any(target in line for line in failed), failed + assert ddb.describe_table(TableName=target)["Table"]["TableStatus"] == "CREATING" + with pytest.raises(ClientError) as refused: + ddb.delete_table(TableName=target) + assert refused.value.response["Error"]["Code"] == "ResourceInUseException", refused.value.response + with pytest.raises(ClientError) as refused: + ddb.delete_backup(BackupArn=arn) + assert refused.value.response["Error"]["Code"] == "BackupInUseException", refused.value.response + with pytest.raises(ClientError) as refused: + ddb.restore_table_from_backup(TargetTableName=f"{name}-again", BackupArn=arn) + assert refused.value.response["Error"]["Code"] == "BackupInUseException", refused.value.response + # The reconciler's next cycles leave it: the FAILED-target duty skips + # this reason, and the target is still CREATING after two more cycles. + time.sleep(2.5) + assert ddb.describe_table(TableName=target)["Table"]["TableStatus"] == "CREATING" + finally: + server.stop() + + res = d.run("backup", "remove-target", "--account", ACCOUNT, target) + assert res.returncode == 0, res.stdout + res.stderr + assert "=== extenddb backup remove-target ===" in res.stdout, res.stdout + res.stderr + assert 'Taking the maintenance claim as "extenddb backup remove-target on ' in res.stdout, res.stdout + assert ( + f"removed the restore target {target} (account {ACCOUNT}, table id {table_id}) and its record: " + f"its store-backed restore from {arn} was FAILED (target_not_empty)" + ) in res.stdout, res.stdout + res.stderr + assert "The maintenance claim is released." in res.stdout, res.stdout + # A second run finds no record for the name and refuses, exit 1. + res = d.run("backup", "remove-target", "--account", ACCOUNT, target) + assert res.returncode == 1, res.stdout + res.stderr + assert f"{target} (account {ACCOUNT}) is not a restore target: no restore record names it" in res.stderr, ( + res.stdout + res.stderr + ) + + server, ddb, creds = _started(d, str(tmp_path / "serve3.log")) + try: + with pytest.raises(ClientError) as gone: + ddb.describe_table(TableName=target) + assert gone.value.response["Error"]["Code"] == "ResourceNotFoundException", gone.value.response + # The name is free and the backup no longer in use: a new restore runs. + resp = ddb.restore_table_from_backup(TargetTableName=target, BackupArn=arn)["TableDescription"] + assert resp["TableStatus"] == "CREATING", resp + _wait_active(ddb, target) + assert _count(ddb, target) == HAND_BUILT_ITEMS + ddb.delete_table(TableName=target) + _wait_table_gone_any_status(ddb, target) + ddb.delete_backup(BackupArn=arn) + _wait_backup_gone(ddb, arn) + log = server.log_text() + assert "target_not_empty" not in log, log[-4000:] + finally: + server.stop() + + +def _seed_running_with_progress(d: _Deployment, target: str) -> None: + """Put a finished restore (table ACTIVE, record DONE) back into the state + a server killed after the restore's first data file committed leaves + (the `restore_after_file_commit` kill point): the target CREATING with + its items and a RUNNING manifest record whose progress names the file + and its item count. Edited at the catalog with the server stopped, as + `_seed_target_not_empty` does.""" + try: + import pysqlite3 as sqlite3 + except ImportError: + import sqlite3 + conn = sqlite3.connect(d.catalog, timeout=10) + try: + row = conn.execute( + "SELECT table_id FROM tables WHERE account_id = ? AND table_name = ?", (ACCOUNT, target) + ).fetchone() + assert row, f"no catalog row for {target}" + table_id = row[0] + n = conn.execute( + "UPDATE tables SET table_status = 'CREATING' WHERE table_id = ?", (table_id,) + ).rowcount + assert n == 1, n + n = conn.execute( + "UPDATE table_restores SET status = 'RUNNING', file_index = 0, items_applied = ?, " + "failure_reason = NULL, completed_at = NULL " + "WHERE table_id = ? AND kind = 'manifest' AND status = 'DONE'", + (HAND_BUILT_ITEMS, table_id), + ).rowcount + assert n == 1, n + conn.commit() + finally: + conn.close() + + +def test_purge_fails_a_killed_servers_store_backed_restore_rather_than_resuming_it(deployment, tmp_path): + """`backup purge --confirm` under the SQLite serve lock, after a server + that was running a store-backed restore stopped (the record RUNNING with + recorded progress, `_seed_running_with_progress`): the command's own + reconciler cycle holds the maintenance claim, so it fails the record and + removes its target, as it does a legacy restore, and resumes nothing (no + `taken over by this instance` line, no restore job in the command). The + restarted server finds no target and no backup.""" + d = deployment + _restore_cell(d) + if d.backend != "sqlite": + pytest.skip("the purge's own reconciler cycle runs under the SQLite serve lock only") + server, ddb, creds = _started(d, str(tmp_path / "serve.log")) + try: + name, arn = _small_backed_up_table(ddb, "prg") + target = f"{name}-r" + resp = ddb.restore_table_from_backup(TargetTableName=target, BackupArn=arn)["TableDescription"] + assert resp["TableStatus"] == "CREATING", resp + _wait_active(ddb, target) + assert _count(ddb, target) == HAND_BUILT_ITEMS + finally: + server.stop() + _seed_running_with_progress(d, target) + + res = d.run("backup", "purge", "--confirm") + out = res.stdout + res.stderr + assert res.returncode == 0, out + assert "taken over by this instance" not in out, out + assert "resumed by this instance" not in out, out + assert "a maintenance claim is held; it is failed rather than resumed" in out, out + assert "Waiting for running jobs" not in out, out + + server, ddb, creds = _started(d, str(tmp_path / "serve2.log")) + try: + with pytest.raises(ClientError) as gone: + ddb.describe_table(TableName=target) + assert gone.value.response["Error"]["Code"] == "ResourceNotFoundException", gone.value.response + with pytest.raises(ClientError) as gone: + ddb.describe_backup(BackupArn=arn) + assert gone.value.response["Error"]["Code"] == "BackupNotFoundException", gone.value.response + log = server.log_text() + assert "taken over by this instance" not in log, log[-4000:] + finally: + server.stop() + + +def _wait_table_gone_any_status(ddb, name: str, timeout: float = 60.0) -> None: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + try: + ddb.describe_table(TableName=name) + except ClientError as e: + assert e.response["Error"]["Code"] == "ResourceNotFoundException", e.response["Error"] + return + time.sleep(0.2) + raise AssertionError(f"{name} still exists after {timeout} s") + + +# --- hand-built prefixes: the format's rules met at the server's boundary ---------- + +HAND_BUILT_ITEMS = 12 + + +def _hand_item(i: int) -> dict: + """Item `i` of a hand-built data file, in the shape `_create_composite_table` + keys on (`pk` S, `sk` N) with the GSI's `g`.""" + return {"pk": {"S": f"p{i % 4:04d}"}, "sk": {"N": str(i)}, "g": {"S": f"g{i % 3}"}, "v": {"S": f"value {i}"}} + + +def _gzip_lines(rows: list[dict]) -> tuple[bytes, int]: + """The data file of `rows`: one `{"Item": ..}` line each, gzipped; the + compressed bytes and the uncompressed length.""" + import gzip + import io + + lines = b"".join(json.dumps({"Item": r}, separators=(",", ":")).encode() + b"\n" for r in rows) + buffer = io.BytesIO() + with gzip.GzipFile(fileobj=buffer, mode="wb", mtime=0) as gz: + gz.write(lines) + return buffer.getvalue(), len(lines) + + +def _write_manifest(prefix: str, manifest: dict) -> None: + """Write the manifest and its sha256 sidecar under `prefix`, the files + private as the store writes them, and the two DynamoDB export manifests + for the manifest's data files, so the prefix stays a whole backup in the + format's sense (one `md5Checksum` and one `itemCount` per file, agreeing + with the extenddb manifest by construction).""" + import hashlib + + text = json.dumps(manifest, separators=(",", ":")).encode() + path = os.path.join(prefix, "extenddb-manifest.json") + with open(path, "wb") as f: + f.write(text) + os.chmod(path, 0o600) + sidecar = os.path.join(prefix, "extenddb-manifest.sha256") + with open(sidecar, "w") as f: + f.write(f"{hashlib.sha256(text).hexdigest()} extenddb-manifest.json\n") + os.chmod(sidecar, 0o600) + summary_path = os.path.join(prefix, "manifest-summary.json") + with open(summary_path) as f: + summary = json.load(f) + s3_prefix = summary["s3Prefix"] + files_path = os.path.join(prefix, "manifest-files.json") + with open(files_path, "w") as f: + for entry in manifest["data_files"]: + line = { + "itemCount": entry["item_count"], + "md5Checksum": entry["md5"], + "dataFileS3Key": f"{s3_prefix}/{entry['key']}" if s3_prefix else entry["key"], + } + f.write(json.dumps(line, separators=(",", ":")) + "\n") + os.chmod(files_path, 0o600) + summary["itemCount"] = sum(entry["item_count"] for entry in manifest["data_files"]) + summary["billedSizeBytes"] = manifest["backup_size_bytes"] + with open(summary_path, "w") as f: + f.write(json.dumps(summary, separators=(",", ":"))) + os.chmod(summary_path, 0o600) + + +def _write_data_files(prefix: str, files: list[list[dict]]) -> list[dict]: + """Replace the data files under `prefix` with one file per list of rows + (unordered: no `key_range`); the manifest entries.""" + import base64 + import hashlib + import shutil + + data_dir = os.path.join(prefix, "data") + shutil.rmtree(data_dir) + os.mkdir(data_dir, 0o700) + entries = [] + for index, rows in enumerate(files, start=1): + compressed, uncompressed = _gzip_lines(rows) + key = f"data/{index:06d}.json.gz" + path = os.path.join(prefix, key) + with open(path, "wb") as f: + f.write(compressed) + os.chmod(path, 0o600) + entries.append({ + "key": key, "item_count": len(rows), + "sha256": hashlib.sha256(compressed).hexdigest(), + "md5": base64.b64encode(hashlib.md5(compressed).digest()).decode(), + "bytes": len(compressed), "uncompressed_bytes": uncompressed, "key_range": None, + }) + return entries + + +def _update_row(d: _Deployment, arn: str, size_bytes: int, item_count: int) -> None: + """Set the `backup_index` row's `size_bytes` and `item_count` to the + rewritten prefix's, as `backup sync` would record them.""" + backup_id = arn.rsplit("/", 1)[1] + if d.backend == "sqlite": + try: + import pysqlite3 as sqlite3 + except ImportError: + import sqlite3 + conn = sqlite3.connect(d.catalog, timeout=10) + try: + n = conn.execute( + "UPDATE backup_index SET size_bytes = ?, item_count = ? WHERE account_id = ? AND backup_id = ?", + (size_bytes, item_count, ACCOUNT, backup_id), + ).rowcount + conn.commit() + finally: + conn.close() + else: + import psycopg2 + + conn = psycopg2.connect(d.catalog) + try: + with conn.cursor() as cur: + cur.execute( + "UPDATE backup_index SET size_bytes = %s, item_count = %s WHERE account_id = %s AND backup_id = %s", + (size_bytes, item_count, ACCOUNT, backup_id), + ) + n = cur.rowcount + conn.commit() + finally: + conn.close() + assert n == 1, f"{n} backup_index rows match {arn}" + + +def _small_backed_up_table(ddb, prefix: str) -> tuple[str, str]: + """A composite-key table with a GSI holding HAND_BUILT_ITEMS items, + backed up to AVAILABLE; the name and the ARN.""" + name = f"{prefix}-{uuid.uuid4().hex[:8]}" + _create_composite_table(ddb, name, gsi=True) + pending = {name: [{"PutRequest": {"Item": _hand_item(i)}} for i in range(HAND_BUILT_ITEMS)]} + while pending: + pending = ddb.batch_write_item(RequestItems=pending).get("UnprocessedItems") or {} + arn = ddb.create_backup(TableName=name, BackupName="nightly")["BackupDetails"]["BackupArn"] + _wait_backup_available(ddb, arn, timeout=60) + return name, arn + + +def _restore_fails(server: _Server, ddb, target: str, arn: str, *needles: str) -> str: + """The restore of `arn` into `target` is accepted CREATING and the target + is gone once the job fails, no item of it ever readable; the server log's + FAILED line, which must carry every needle. The job logs the line right + after it removes the target, so the log is polled for a moment.""" + resp = ddb.restore_table_from_backup(TargetTableName=target, BackupArn=arn)["TableDescription"] + assert resp["TableStatus"] == "CREATING", resp + assert resp["RestoreSummary"]["SourceBackupArn"] == arn, resp + _wait_table_gone(ddb, target) + deadline = time.monotonic() + 15 + while True: + failed = [line for line in server.log_text().splitlines() if ": FAILED (" in line and f" into {target} " in line] + if failed or time.monotonic() > deadline: + break + time.sleep(0.25) + assert len(failed) == 1, server.log_text()[-4000:] + for needle in needles: + assert needle in failed[0], failed[0] + assert "the target was removed" in failed[0], failed[0] + assert ddb.describe_backup(BackupArn=arn)["BackupDescription"]["BackupDetails"]["BackupStatus"] == "AVAILABLE" + return failed[0] + + +DUPLICATE_KEY_TEXT = "holds a primary key the target already has (the backup holds a key twice)" + + +def test_an_unordered_backup_holding_one_key_in_two_files_fails_the_restore(deployment, tmp_path): + """The backup's prefix rewritten under the root as an unordered backup of + two data files that share item 0's key (the manifest and its sidecar + rewritten, the row's size and count updated): the restore is accepted, + the job fails the second file on the duplicate, the target is gone and + the log names the reason. On PostgreSQL and SQLite, the two deployments + here, the detector is the base row's plain INSERT and the reason is its + text (`DUPLICATE_KEY_TEXT`); the engine's final count against the + manifest's total ("the manifest lists") is the detector on a backend + whose write is an upsert (Cassandra, through the matrix). A backend with + neither would restore 12 of 13 items.""" + d = deployment + _restore_cell(d) + server, ddb, creds = _started(d, str(tmp_path / "serve.log")) + try: + name, arn = _small_backed_up_table(ddb, "dup") + prefix = _prefix(d, name, arn) + with open(os.path.join(prefix, "extenddb-manifest.json")) as f: + manifest = json.load(f) + rows = [_hand_item(i) for i in range(HAND_BUILT_ITEMS)] + entries = _write_data_files(prefix, [rows[:6], rows[6:] + [rows[0]]]) + manifest["item_order"] = "unordered" + manifest["data_files"] = entries + manifest["backup_size_bytes"] = sum(e["bytes"] for e in entries) + manifest["source_table"]["item_count"] = HAND_BUILT_ITEMS + 1 + _write_manifest(prefix, manifest) + _update_row(d, arn, manifest["backup_size_bytes"], HAND_BUILT_ITEMS + 1) + line = _restore_fails(server, ddb, f"{name}-r", arn, "data/000002.json.gz", DUPLICATE_KEY_TEXT) + assert "the manifest lists" not in line, line + finally: + server.stop() + + +def test_a_key_ordered_file_leaving_its_key_range_fails_the_restore(deployment, tmp_path): + """The backup's manifest rewritten so its first data file's `key_range` + ends at its first item: the reader's key-range check ends the file's + stream at the second item, the restore is FAILED naming the file and the + rule, and the target is gone.""" + d = deployment + _restore_cell(d) + server, ddb, creds = _started(d, str(tmp_path / "serve.log")) + try: + name, arn = _small_backed_up_table(ddb, "range") + prefix = _prefix(d, name, arn) + with open(os.path.join(prefix, "extenddb-manifest.json")) as f: + manifest = json.load(f) + assert manifest["item_order"] == "key_ordered", manifest["item_order"] + first = manifest["data_files"][0] + assert first["item_count"] >= 2, first + first["key_range"]["last"] = first["key_range"]["first"] + _write_manifest(prefix, manifest) + _restore_fails(server, ddb, f"{name}-r", arn, "data/000001.json.gz", "key_range") + finally: + server.stop() + + +# --- the multi-part-key preview ------------------------------------------------------ + +MULTIPART_ITEMS = 60 + + +def _scan_items(ddb, name: str) -> list[dict]: + items: list[dict] = [] + kwargs: dict = {"TableName": name, "ConsistentRead": True} + while True: + resp = ddb.scan(**kwargs) + items += resp["Items"] + if "LastEvaluatedKey" not in resp: + return items + kwargs["ExclusiveStartKey"] = resp["LastEvaluatedKey"] + + +def _canonical(item: dict) -> str: + return json.dumps(item, sort_keys=True, separators=(",", ":")) + + +def test_a_multi_part_key_table_restores_with_the_sources_item_set(deployment, tmp_path): + """A table keyed on two HASH attributes (`[limits] + allow_multipart_table_keys = true` on this cell's deployment, so a + server that refuses the second HASH key fails the cell rather than + skipping it) backs up to the store as an `unordered` backup and + restores: the restored table's `Scan` holds what the source's held, + with the source's key schema and attribute definitions. The comparison + is with the source's `Scan`, not with the items written: the preview's + item paths address such a table by its first HASH attribute on every + backend (on SQLite the source keeps one item per `h1` value; on + PostgreSQL a GetItem by the whole key finds nothing on the source + either), and that is the preview's behaviour, not the restore's; for the + same reason the cell makes no GetItem by the whole key.""" + d = deployment + _restore_cell(d) + d.set_limit("allow_multipart_table_keys", "true") + server, ddb, creds = _started(d, str(tmp_path / "serve.log")) + try: + name = f"mpk-{uuid.uuid4().hex[:8]}" + key_schema = [ + {"AttributeName": "h1", "KeyType": "HASH"}, + {"AttributeName": "h2", "KeyType": "HASH"}, + ] + attr_defs = [ + {"AttributeName": "h1", "AttributeType": "S"}, + {"AttributeName": "h2", "AttributeType": "N"}, + ] + ddb.create_table(TableName=name, KeySchema=key_schema, AttributeDefinitions=attr_defs, BillingMode="PAY_PER_REQUEST") + _wait_active(ddb, name) + for i in range(MULTIPART_ITEMS): + ddb.put_item( + TableName=name, + Item={"h1": {"S": f"tenant-{i % 5}"}, "h2": {"N": str(i)}, "v": {"S": f"value-{i}"}, "n": {"N": str(i * 1.5)}}, + ) + expected = sorted(_canonical(i) for i in _scan_items(ddb, name)) + assert expected, "the source holds no item" + print(f"branch: the source holds {len(expected)} of {MULTIPART_ITEMS} items written ({d.backend})") + arn = ddb.create_backup(TableName=name, BackupName="nightly")["BackupDetails"]["BackupArn"] + _wait_backup_available(ddb, arn, timeout=60) + with open(os.path.join(_prefix(d, name, arn), "extenddb-manifest.json")) as f: + manifest = json.load(f) + assert manifest["item_order"] == "unordered", manifest["item_order"] + assert manifest["source_table"]["key_schema"] == key_schema, manifest["source_table"]["key_schema"] + target = f"{name}-r" + resp = ddb.restore_table_from_backup(TargetTableName=target, BackupArn=arn)["TableDescription"] + assert resp["TableStatus"] == "CREATING", resp + assert resp["RestoreSummary"]["SourceBackupArn"] == arn, resp + _wait_active(ddb, target, timeout=120) + table = ddb.describe_table(TableName=target)["Table"] + assert table["KeySchema"] == key_schema, table["KeySchema"] + assert sorted(table["AttributeDefinitions"], key=lambda a: a["AttributeName"]) == attr_defs, table["AttributeDefinitions"] + assert "RestoreSummary" not in table, table + got = sorted(_canonical(i) for i in _scan_items(ddb, target)) + assert got == expected, f"{len(got)} restored items against {len(expected)} in the source" + log = server.log_text() + assert f" into {target} " in log and ": ACTIVE, " in log and ": FAILED (" not in log, log[-4000:] + finally: + server.stop() + + +# --- extenddb backup sync ----------------------------------------------------------- + + +def _sync_cell(d: _Deployment) -> None: + """The sync cells run on the SQLite and PostgreSQL deployments: they drop + and rewrite `backup_index` rows with SQL (`_catalog`), which a MongoDB + catalog does not take; MongoDB's insert and removal are the crate's live + tests.""" + if d.backend == "mongodb": + pytest.skip("the sync cells run on the SQLite and PostgreSQL deployments") + + +def _catalog(d: _Deployment, statement: str, params: tuple) -> int: + """Run one statement on the deployment's catalog and return its row count. + The statement uses `?` placeholders and `{now}` for the database's clock, + which become `%s` and `now()` on PostgreSQL.""" + if d.backend == "sqlite": + try: + import pysqlite3 as sqlite3 + except ImportError: + import sqlite3 + conn = sqlite3.connect(d.catalog, timeout=10) + try: + n = conn.execute(statement.format(now="strftime('%Y-%m-%dT%H:%M:%fZ', 'now')"), params).rowcount + conn.commit() + finally: + conn.close() + return n + import psycopg2 + + conn = psycopg2.connect(d.catalog) + try: + with conn.cursor() as cur: + cur.execute(statement.format(now="now()").replace("?", "%s"), params) + n = cur.rowcount + conn.commit() + finally: + conn.close() + return n + + +def _backup_id(arn: str) -> str: + return arn.rsplit("/", 1)[1] + + +def _drop_row(d: _Deployment, arn: str) -> None: + """Delete the backup's `backup_index` row: the catalog lost it.""" + n = _catalog(d, "DELETE FROM backup_index WHERE account_id = ? AND backup_id = ?", (ACCOUNT, _backup_id(arn))) + assert n == 1, f"{n} backup_index rows match {arn}" + + +def _set_status(d: _Deployment, arn: str, status: str) -> None: + """Set the backup's row to `status`, its heartbeat and (except for + CREATING) its completion stamped now on the database's clock: a FAILED + row inside its retention, a CREATING row whose owner beat just now.""" + n = _catalog( + d, + "UPDATE backup_index SET status = ?, heartbeat_at = {now}, " + "completed_at = CASE WHEN ? = 'CREATING' THEN NULL ELSE {now} END " + "WHERE account_id = ? AND backup_id = ?", + (status, status, ACCOUNT, _backup_id(arn)), + ) + assert n == 1, f"{n} backup_index rows match {arn}" + + +def _rows(d: _Deployment) -> dict[str, str]: + """`extenddb backup list`: each row's ARN to its status.""" + res = d.run("backup", "list") + assert res.returncode == 0, res.stdout + res.stderr + rows: dict[str, str] = {} + for line in res.stdout.splitlines(): + m = re.match(r"^([A-Z]+)\s+\d{12}\s+\S+\s+\d+\s+\d+\s+(arn:\S+)\s", line) + if m: + rows[m.group(2)] = m.group(1) + return rows + + +def _key_prefix(table: str, arn: str) -> str: + """The prefix as the store's keys, and sync's report, spell it.""" + return f"{ACCOUNT}/{table}/{_backup_id(arn)}/" + + +def _store_id(root: str) -> str: + with open(os.path.join(root, "extenddb-store.json")) as f: + return json.load(f)["store_id"] + + +def _private_file(path: str, data: bytes) -> None: + """Write `path` 0600 with its missing parents 0700, as the store's own + objects are.""" + parent = os.path.dirname(path) + missing = [] + while not os.path.isdir(parent): + missing.append(parent) + parent = os.path.dirname(parent) + for directory in reversed(missing): + os.mkdir(directory, 0o700) + with open(path, "wb") as f: + f.write(data) + os.chmod(path, 0o600) + + +def test_sync_rebuilds_dropped_rows_and_adopts_only_verified_prefixes(deployment, tmp_path): + """The catalog loses the rows of four backups; `extenddb backup sync` + gives an AVAILABLE row to the one whose prefix is whole, after which + DescribeBackup (the manifest's name and item count), ListBackups and a + restore of it work. Not adopted, each reported with its reason and exit + 0: the prefix whose manifest was removed, the prefix whose sidecar no + longer matches, a prefix moved under another backup id (its manifest + names the backup id it was written under), a file under the table's + `changelog/` and one under `exports/`, and the prefixes whose rows are + FAILED, DELETING and CREATING. On PostgreSQL plain sync takes the + maintenance claim, waits for no job, and reads those three rows and + changes none. On SQLite sync runs under the serve lock (it + refuses while the server runs) and after the dead-owner resolutions a + stopped server leaves, so a DELETING row reaches sync finished (DELETED, + its prefix gone) and a CREATING row with a complete prefix resolved + AVAILABLE; neither is adopted. A second sync adopts nothing.""" + _sync_cell(deployment) + d = deployment + server, ddb, creds = _started(d, str(tmp_path / "serve.log")) + try: + name, kept = _small_backed_up_table(ddb, "sync") + others: dict[str, str] = {} + for label in ("no-manifest", "bad-sidecar", "moved", "failed", "deleting", "creating"): + arn = ddb.create_backup(TableName=name, BackupName=label)["BackupDetails"]["BackupArn"] + _wait_backup_available(ddb, arn, timeout=60) + others[label] = arn + if d.backend == "sqlite": + # The serve lock held is the catalog refusal `serve` gives for + # it: exit 6, before any store or row is read. + for args in (("sync",), ("sync", "--prune-missing")): + beside = d.run("backup", *args) + assert beside.returncode == 6, beside.stdout + beside.stderr + assert "another extenddb process" in beside.stdout + beside.stderr, beside.stdout + beside.stderr + assert "=== extenddb backup sync" not in beside.stdout, beside.stdout + assert server.proc.poll() is None, "the server kept running" + finally: + server.stop() + root = d.root + store_id = _store_id(root) + for arn in (kept, others["no-manifest"], others["bad-sidecar"], others["moved"]): + _drop_row(d, arn) + os.remove(os.path.join(_prefix(d, name, others["no-manifest"]), "extenddb-manifest.json")) + with open(os.path.join(_prefix(d, name, others["bad-sidecar"]), "extenddb-manifest.sha256"), "w") as f: + f.write("0" * 64 + " extenddb-manifest.json\n") + moved_id = _backup_id(others["moved"]) + millis, suffix = moved_id.split("-") + moved_to = f"{int(millis) + 1:014d}-{suffix}" + os.rename(_prefix(d, name, others["moved"]), os.path.join(root, ACCOUNT, name, moved_to)) + _private_file(os.path.join(root, ACCOUNT, name, "changelog", "000001.json"), b"{}\n") + _private_file(os.path.join(root, "exports", "01", "part-0.json"), b"{}\n") + _set_status(d, others["failed"], "FAILED") + _set_status(d, others["deleting"], "DELETING") + _set_status(d, others["creating"], "CREATING") + + res = d.run("backup", "sync") + out = res.stdout + res.stderr + print(f"branch: {d.backend}\n{res.stdout}") + assert res.returncode == 0, out + assert f"Store: filesystem {os.path.realpath(root)} (store id {store_id})" in res.stdout, res.stdout + took_claim = 'Took the maintenance claim as "extenddb backup sync on ' in res.stdout + assert took_claim == (d.backend == "postgres"), res.stdout + assert "Waiting for running jobs" not in res.stdout, res.stdout + adopted = res.stdout.split("--- Adopted", 1)[1].split("--- Skipped", 1)[0] + assert adopted.splitlines()[1:-1] == [f" {kept}"], adopted + assert "1 backup(s) adopted" in res.stdout, res.stdout + skipped = res.stdout.split("--- Skipped (not adopted):", 1)[1].split("--- AVAILABLE rows", 1)[0] + expected = [ + f"{_key_prefix(name, others['no-manifest'])}: not a complete backup: no manifest", + f"{_key_prefix(name, others['bad-sidecar'])}: not a complete backup: the sidecar does not match", + f"{ACCOUNT}/{name}/{moved_to}/: the manifest names {others['moved']}, whose account, table and backup id are " + "not the prefix's", + f"{ACCOUNT}/{name}/changelog/: reserved for the change-log archive", + "exports/: reserved for exports", + f"{_key_prefix(name, others['failed'])}: the catalog has a row for it (FAILED, store id {store_id})", + ] + if d.backend == "postgres": + expected += [ + f"{_key_prefix(name, others['deleting'])}: the catalog has a row for it (DELETING, store id {store_id})", + f"{_key_prefix(name, others['creating'])}: the catalog has a row for it (CREATING, store id {store_id})", + ] + else: + expected.append( + f"{_key_prefix(name, others['creating'])}: the catalog has a row for it (AVAILABLE, store id {store_id})" + ) + for line in expected: + assert f" {line}" in skipped, f"{line!r} not in\n{skipped}" + assert len([line for line in skipped.splitlines() if line.startswith(" ")]) == len(expected), skipped + rows = _rows(d) + assert rows[kept] == "AVAILABLE", rows + for label in ("no-manifest", "bad-sidecar", "moved"): + assert others[label] not in rows, (label, rows) + assert rows[others["failed"]] == "FAILED", rows + if d.backend == "postgres": + assert rows[others["deleting"]] == "DELETING", rows + assert rows[others["creating"]] == "CREATING", rows + for label in ("deleting", "creating"): + assert os.path.isfile(os.path.join(_prefix(d, name, others[label]), "extenddb-manifest.json")), label + else: + assert rows[others["deleting"]] == "DELETED", rows + assert not os.path.exists(_prefix(d, name, others["deleting"])), "the finished deletion left its prefix" + assert rows[others["creating"]] == "AVAILABLE", rows + # Nothing under a skipped prefix was changed or removed. + assert os.path.isfile(os.path.join(_prefix(d, name, others["bad-sidecar"]), "extenddb-manifest.json")) + assert os.path.isfile(os.path.join(root, ACCOUNT, name, moved_to, "extenddb-manifest.json")) + assert os.path.isfile(os.path.join(root, ACCOUNT, name, "changelog", "000001.json")) + + again = d.run("backup", "sync") + assert again.returncode == 0, again.stdout + again.stderr + assert "0 backup(s) adopted" in again.stdout, again.stdout + assert f" {_key_prefix(name, kept)}: the catalog has a row for it (AVAILABLE, store id {store_id})" in again.stdout + + for label in ("failed", "deleting", "creating"): + if rows[others[label]] in ("FAILED", "DELETING", "CREATING"): + _set_status(d, others[label], "AVAILABLE") + server, ddb, creds = _started(d, str(tmp_path / "serve2.log")) + try: + details = ddb.describe_backup(BackupArn=kept)["BackupDescription"] + assert details["BackupDetails"]["BackupStatus"] == "AVAILABLE", details + assert details["BackupDetails"]["BackupName"] == "nightly", details + assert details["SourceTableDetails"]["TableName"] == name, details + assert details["SourceTableDetails"]["ItemCount"] == HAND_BUILT_ITEMS, details + listed = [s["BackupArn"] for s in ddb.list_backups(TableName=name)["BackupSummaries"]] + assert kept in listed, listed + for label in ("no-manifest", "bad-sidecar", "moved"): + with pytest.raises(ClientError) as gone: + ddb.describe_backup(BackupArn=others[label]) + assert gone.value.response["Error"]["Code"] == "BackupNotFoundException", (label, gone.value.response) + target = f"{name}-r" + resp = ddb.restore_table_from_backup(TargetTableName=target, BackupArn=kept)["TableDescription"] + assert resp["TableStatus"] == "CREATING", resp + _wait_active(ddb, target) + assert _count(ddb, target) == HAND_BUILT_ITEMS + finally: + server.stop() + + +def _with_root_moved_away(d: _Deployment): + """The store volume unmounted: the root renamed away and an empty + directory of the same path in its place; returns the restore callable.""" + away = d.root + ".away" + os.rename(d.root, away) + os.mkdir(d.root, 0o700) + + def restore() -> None: + shutil.rmtree(d.root) + os.rename(away, d.root) + + return away, restore + + +def _tree(root: str) -> list[str]: + out = [] + for dirpath, dirnames, filenames in os.walk(root): + for f in filenames: + path = os.path.join(dirpath, f) + with open(path, "rb") as fh: + out.append(f"{os.path.relpath(path, root)} {hashlib.sha256(fh.read()).hexdigest()}") + return sorted(out) + + +def test_prune_missing_and_the_unhealthy_store_refusals(deployment, tmp_path): + """A row whose manifest is gone (not the newest AVAILABLE one): plain + sync reports it with the `--prune-missing` hint and leaves it AVAILABLE; + `--prune-missing` takes the maintenance claim and removes the row + outright (no tombstone, BackupNotFound), deleting no object: the rest of + its prefix stays. An unhealthy store refuses plain sync and + `--prune-missing`, exit 1, with no row changed and nothing written to the + root, for each shape an unmounted volume gives: an empty root under the + rows, a root holding only the identity file (the newest AVAILABLE row's + manifest absent), and a root whose identity file carries another store + id. Last, the newest AVAILABLE row's manifest + removed: the store is unhealthy (its manifest probe), so sync and + `--prune-missing` both refuse and the row stays AVAILABLE until the + manifest is put back, after which sync runs again. PostgreSQL runs the + commands beside its server; SQLite stops it (the serve lock).""" + _sync_cell(deployment) + d = deployment + server, ddb, creds = _started(d, str(tmp_path / "serve.log")) + try: + name, older = _small_backed_up_table(ddb, "prune") + newest = ddb.create_backup(TableName=name, BackupName="second")["BackupDetails"]["BackupArn"] + _wait_backup_available(ddb, newest, timeout=60) + if d.backend == "sqlite": + server.stop() + print(f"branch: {d.backend}, the server {'running' if d.backend == 'postgres' else 'stopped'}") + + os.remove(os.path.join(_prefix(d, name, older), "extenddb-manifest.json")) + res = d.run("backup", "sync") + assert res.returncode == 0, res.stdout + res.stderr + missing = res.stdout.split("--- AVAILABLE rows whose manifest is not in the store (the rows were AVAILABLE when the catalog was read):", 1)[1] + missing = missing.split("=== extenddb backup sync complete", 1)[0] + assert older in missing and newest not in missing, missing + assert "(left AVAILABLE; run with --prune-missing to remove the rows, which deletes no object)" in missing, missing + assert ("Took the maintenance claim" in res.stdout) == (d.backend == "postgres"), res.stdout + assert _rows(d)[older] == "AVAILABLE" + left_behind = _tree(_prefix(d, name, older)) + assert left_behind, "the older prefix keeps its other objects" + + res = d.run("backup", "sync", "--prune-missing") + assert res.returncode == 0, res.stdout + res.stderr + assert 'Took the maintenance claim as "extenddb backup sync on ' in res.stdout, res.stdout + assert f" {older}: removed: the row is gone, and no object was deleted" in res.stdout, res.stdout + assert "(1 removed)" in res.stdout, res.stdout + rows = _rows(d) + assert older not in rows and rows[newest] == "AVAILABLE", rows + assert _tree(_prefix(d, name, older)) == left_behind, "prune deleted an object" + if d.backend == "postgres": + with pytest.raises(ClientError) as gone: + ddb.describe_backup(BackupArn=older) + assert gone.value.response["Error"]["Code"] == "BackupNotFoundException", gone.value.response + + before = _rows(d) + with open(os.path.join(d.root, "extenddb-store.json"), "rb") as f: + identity_file = f.read() + other_id = json.loads(identity_file) + other_id["store_id"] = str(uuid.uuid4()) + shapes = { + "an empty root": None, + "the identity file alone": identity_file, + "another store id": (json.dumps(other_id, indent=2) + "\n").encode(), + } + for shape, content in shapes.items(): + away, put_back = _with_root_moved_away(d) + try: + if content is not None: + _private_file(os.path.join(d.root, "extenddb-store.json"), content) + tree = _tree(d.root) + for args in (("sync",), ("sync", "--prune-missing")): + res = d.run("backup", *args) + out = res.stdout + res.stderr + assert res.returncode == 1, (shape, args, out) + assert f"the filesystem backup store {os.path.realpath(d.root)} is unhealthy" in out, (shape, out) + assert "sync changes no row while the store is unhealthy" in out, (shape, out) + assert "Took the maintenance claim" not in out, (shape, out) + assert _tree(d.root) == tree, f"{shape}: the command wrote to the root" + print(f"{shape} {' '.join(args)}: {out.strip().splitlines()[-1]}") + finally: + put_back() + assert _rows(d) == before, shape + + manifest = os.path.join(_prefix(d, name, newest), "extenddb-manifest.json") + os.rename(manifest, manifest + ".aside") + try: + for args in (("sync",), ("sync", "--prune-missing")): + res = d.run("backup", *args) + out = res.stdout + res.stderr + assert res.returncode == 1 and "is unhealthy" in out and "manifest" in out, out + assert _rows(d) == before + finally: + os.rename(manifest + ".aside", manifest) + res = d.run("backup", "sync") + assert res.returncode == 0, res.stdout + res.stderr + assert "0 backup(s) adopted" in res.stdout and "0 AVAILABLE row(s) whose manifest is gone" in res.stdout, res.stdout + finally: + server.stop() + + +def test_a_second_maintenance_command_fails_at_once(deployment, tmp_path): + """`backup sync --prune-missing` takes the maintenance claim and waits + for a CREATING row whose owner beat just now (the row edited at the + catalog, its heartbeat the database's now); while it waits, `backup + purge --confirm` and `backup sync --adopt-store` each fail within + seconds, exit 1, naming the one-at-a-time rule, and change nothing. The + first command, signalled, stops before any change and releases the + claim. `--adopt-store` on the root this catalog owns then takes the + claim and waits for the same live job ("Waiting for running jobs to + finish"), and a signal stops it before any change; after that a plain + sync and `--prune-missing` run. SQLite has + no second command to refuse: every writing command takes the serve lock + first, which the sync cell shows.""" + _sync_cell(deployment) + d = deployment + if d.backend != "postgres": + pytest.skip("on SQLite the serve lock refuses a second writing command before any claim is read") + import subprocess + + server, ddb, creds = _started(d, str(tmp_path / "serve.log")) + try: + name, busy = _small_backed_up_table(ddb, "claim") + finally: + server.stop() + _set_status(d, busy, "CREATING") + out_path = str(tmp_path / "first.out") + with open(out_path, "wb") as out: + first = subprocess.Popen( + [EXTENDDB_BINARY, "backup", "--config", d.config, "sync", "--prune-missing"], + cwd=d.cwd, stdout=out, stderr=subprocess.STDOUT, stdin=subprocess.DEVNULL, + ) + try: + deadline = time.monotonic() + 60 + while "Waiting for running jobs to finish" not in open(out_path).read(): + assert first.poll() is None, open(out_path).read() + assert time.monotonic() < deadline, open(out_path).read() + time.sleep(0.2) + assert busy in open(out_path).read() or name in open(out_path).read(), open(out_path).read() + for args in (("purge", "--confirm"), ("sync", "--adopt-store")): + started = time.monotonic() + res = d.run("backup", *args) + took = time.monotonic() - started + out = res.stdout + res.stderr + assert res.returncode == 1, (args, out) + assert "one maintenance command runs at a time" in out, (args, out) + assert took < 15, f"{args} took {took:.1f} s to refuse" + print(f"{' '.join(args)}: refused in {took:.1f} s") + assert first.poll() is None, "the first command stopped while the others ran" + first.send_signal(15) + assert first.wait(timeout=30) == 1 + text = open(out_path).read() + assert "stopped before any change: SIGTERM received while waiting" in text, text + finally: + if first.poll() is None: + first.kill() + first.wait(timeout=10) + assert _rows(d)[busy] == "CREATING" + # `--adopt-store` on a root this catalog owns takes the claim and waits + # for the same live job before its sync, as `--prune-missing` does. + adopt_path = str(tmp_path / "adopt.out") + with open(adopt_path, "wb") as out: + adopting = subprocess.Popen( + [EXTENDDB_BINARY, "backup", "--config", d.config, "sync", "--adopt-store"], + cwd=d.cwd, stdout=out, stderr=subprocess.STDOUT, stdin=subprocess.DEVNULL, + ) + try: + deadline = time.monotonic() + 60 + while "Waiting for running jobs to finish" not in open(adopt_path).read(): + assert adopting.poll() is None, open(adopt_path).read() + assert time.monotonic() < deadline, open(adopt_path).read() + time.sleep(0.2) + text = open(adopt_path).read() + assert "there is nothing to adopt" in text and 'Took the maintenance claim as "extenddb backup sync on ' in text, text + assert "--- Adopted" not in text, text + adopting.send_signal(15) + assert adopting.wait(timeout=30) == 1 + text = open(adopt_path).read() + assert "stopped before any change: SIGTERM received while waiting" in text, text + finally: + if adopting.poll() is None: + adopting.kill() + adopting.wait(timeout=10) + assert _rows(d)[busy] == "CREATING" + _set_status(d, busy, "AVAILABLE") + res = d.run("backup", "sync", "--prune-missing") + assert res.returncode == 0, res.stdout + res.stderr + assert "Took the maintenance claim" in res.stdout, res.stdout + + +def _adopted_section(stdout: str) -> str: + return stdout.split("--- Adopted", 1)[1].split("--- Skipped", 1)[0] + + +def _rows_present(d: _Deployment, arns: list[str]) -> int: + """How many of `arns` have a `backup_index` row, read at the catalog + (a command holding SQLite's serve lock does not stop the read).""" + ids = tuple(_backup_id(a) for a in arns) + marks = ", ".join("?" for _ in ids) + statement = f"SELECT count(*) FROM backup_index WHERE account_id = ? AND backup_id IN ({marks})" + if d.backend == "sqlite": + try: + import pysqlite3 as sqlite3 + except ImportError: + import sqlite3 + conn = sqlite3.connect(d.catalog, timeout=10) + try: + return conn.execute(statement, (ACCOUNT, *ids)).fetchone()[0] + finally: + conn.close() + import psycopg2 + + conn = psycopg2.connect(d.catalog) + try: + with conn.cursor() as cur: + cur.execute(statement.replace("?", "%s"), (ACCOUNT, *ids)) + return cur.fetchone()[0] + finally: + conn.close() + + +def test_sigterm_during_the_prefix_loop_stops_sync_and_keeps_its_rows(deployment, tmp_path): + """Two backups whose rows the catalog lost sort first under the root, + followed by many copies of the first under later backup ids (each read + and skipped: its manifest names another backup id). SIGTERM once both + rows are back, while sync is still in the prefix loop: the command exits + 1 saying it stopped, the rows it inserted stay AVAILABLE, no + missing-manifest section is examined, and the next sync runs at once + (on PostgreSQL it takes the claim the first released) and adopts + nothing more.""" + _sync_cell(deployment) + d = deployment + import signal + import subprocess + + server, ddb, creds = _started(d, str(tmp_path / "serve.log")) + try: + name, first = _small_backed_up_table(ddb, "sigterm") + second = ddb.create_backup(TableName=name, BackupName="second")["BackupDetails"]["BackupArn"] + _wait_backup_available(ddb, second, timeout=60) + finally: + server.stop() + for arn in (first, second): + _drop_row(d, arn) + src = _prefix(d, name, first) + millis, suffix = max(_backup_id(first), _backup_id(second)).split("-") + copies = 10000 + for i in range(copies): + shutil.copytree(src, os.path.join(d.root, ACCOUNT, name, f"{int(millis) + 1 + i:014d}-{suffix}"), + copy_function=os.link) + out_path = str(tmp_path / "sync.out") + with open(out_path, "wb") as out: + sync = subprocess.Popen( + [EXTENDDB_BINARY, "backup", "--config", d.config, "sync"], + cwd=d.cwd, stdout=out, stderr=subprocess.STDOUT, stdin=subprocess.DEVNULL, + ) + try: + deadline = time.monotonic() + 60 + while _rows_present(d, [first, second]) < 2: + assert sync.poll() is None, open(out_path).read()[-4000:] + assert time.monotonic() < deadline, open(out_path).read()[-4000:] + time.sleep(0.05) + assert sync.poll() is None, "the sync ended before the signal" + sync.send_signal(signal.SIGTERM) + assert sync.wait(timeout=60) == 1, open(out_path).read()[-4000:] + finally: + if sync.poll() is None: + sync.kill() + sync.wait(timeout=10) + text = open(out_path).read() + print("\n".join(line for line in text.splitlines() if not line.startswith(" "))[-3000:]) + assert "stopped: SIGTERM received. 2 backup(s) were adopted before the stop" in text, text[-4000:] + assert " (not examined)" in text, text[-4000:] + assert "=== extenddb backup sync complete" not in text, text[-4000:] + rows = _rows(d) + assert rows[first] == "AVAILABLE" and rows[second] == "AVAILABLE", rows + started = time.monotonic() + res = d.run("backup", "sync") + took = time.monotonic() - started + out = res.stdout + res.stderr + assert res.returncode == 0, out[-4000:] + if d.backend == "postgres": + assert "Took the maintenance claim" in res.stdout, res.stdout[:2000] + assert _adopted_section(res.stdout).splitlines()[1:-1] == [" (none)"], res.stdout[:2000] + assert f"0 backup(s) adopted, {copies + 2} prefix(es) skipped" in res.stdout, res.stdout[-2000:] + print(f"the next sync: exit 0 in {took:.1f} s") + + +def test_prune_missing_on_a_host_local_root_deletes_no_object(deployment, tmp_path): + """Host-local roots on two instances of one catalog, one path: backups + K1 < K2 < K3, and the second host's root lacks K2 (its bytes are on the + first host), while it holds the newest, K3, so its store is healthy. + `--prune-missing` there removes K2's row outright and deletes nothing. + Back on the first host, the reconciler's cycles leave K2's prefix whole + (no tombstone names it), DescribeBackup answers BackupNotFound, and a + plain sync beside the server adopts K2 again, after which it is + AVAILABLE. A SQLite catalog has one instance, so the cell is + PostgreSQL's.""" + _sync_cell(deployment) + d = deployment + if d.backend != "postgres": + pytest.skip("host-local roots need two instances of one catalog") + server, ddb, creds = _started(d, str(tmp_path / "serve.log")) + try: + name, k1 = _small_backed_up_table(ddb, "hostlocal") + k2 = ddb.create_backup(TableName=name, BackupName="second")["BackupDetails"]["BackupArn"] + _wait_backup_available(ddb, k2, timeout=60) + k3 = ddb.create_backup(TableName=name, BackupName="third")["BackupDetails"]["BackupArn"] + _wait_backup_available(ddb, k3, timeout=60) + finally: + server.stop() + k2_tree = _tree(_prefix(d, name, k2)) + host1 = d.root + ".host1" + os.rename(d.root, host1) + try: + shutil.copytree(host1, d.root) + shutil.rmtree(_prefix(d, name, k2)) + res = d.run("backup", "sync", "--prune-missing") + out = res.stdout + res.stderr + print(out) + assert res.returncode == 0, out + assert f" {k2}: removed: the row is gone, and no object was deleted" in res.stdout, out + rows = _rows(d) + assert k2 not in rows and rows[k1] == "AVAILABLE" and rows[k3] == "AVAILABLE", rows + finally: + shutil.rmtree(d.root, ignore_errors=True) + os.rename(host1, d.root) + server, ddb, creds = _started(d, str(tmp_path / "serve2.log")) + try: + time.sleep(5) # several reconciler cycles (reconcile_interval_secs = 1) + assert _tree(_prefix(d, name, k2)) == k2_tree, "a deletion duty acted on K2's bytes" + with pytest.raises(ClientError) as gone: + ddb.describe_backup(BackupArn=k2) + assert gone.value.response["Error"]["Code"] == "BackupNotFoundException", gone.value.response + res = d.run("backup", "sync") + assert res.returncode == 0, res.stdout + res.stderr + assert _adopted_section(res.stdout).splitlines()[1:-1] == [f" {k2}"], res.stdout + details = ddb.describe_backup(BackupArn=k2)["BackupDescription"]["BackupDetails"] + assert details["BackupStatus"] == "AVAILABLE" and details["BackupName"] == "second", details + finally: + server.stop() + + +def test_sync_is_refused_while_purge_holds_the_claim(deployment, tmp_path): + """Beside servers every sync takes the maintenance claim, so no sync + runs while purge does. A CREATING row whose job died after its manifest + (its prefix complete, its heartbeat an hour old) and a CREATING row whose + job beat just now: `purge --confirm` takes the claim and waits for the + live job, and while it holds the claim a plain sync fails at once (exit + 1, one maintenance command at a time) and writes nothing. The job ends; + purge removes the dead row first and then fails to delete its prefix + (made read-only), reports it left and exits 1. The prefix is then a + complete backup with no row, which a later sync adopts and the next + purge deletes. SQLite runs both commands under the serve lock.""" + _sync_cell(deployment) + d = deployment + if d.backend != "postgres": + pytest.skip("on SQLite the serve lock orders sync and purge") + import subprocess + + server, ddb, creds = _started(d, str(tmp_path / "serve.log")) + try: + name, dead = _small_backed_up_table(ddb, "gap") + live = ddb.create_backup(TableName=name, BackupName="live")["BackupDetails"]["BackupArn"] + _wait_backup_available(ddb, live, timeout=60) + finally: + server.stop() + n = _catalog( + d, + "UPDATE backup_index SET status = 'CREATING', completed_at = NULL, " + "heartbeat_at = {now} - interval '1 hour' WHERE account_id = ? AND backup_id = ?", + (ACCOUNT, _backup_id(dead)), + ) + assert n == 1 + _set_status(d, live, "CREATING") + dead_prefix = _prefix(d, name, dead) + os.chmod(dead_prefix, 0o500) + out_path = str(tmp_path / "purge.out") + with open(out_path, "wb") as out: + purge = subprocess.Popen( + [EXTENDDB_BINARY, "backup", "--config", d.config, "purge", "--confirm"], + cwd=d.cwd, stdout=out, stderr=subprocess.STDOUT, stdin=subprocess.DEVNULL, + ) + try: + deadline = time.monotonic() + 60 + while "Waiting for running jobs to finish" not in open(out_path).read(): + assert purge.poll() is None, open(out_path).read() + assert time.monotonic() < deadline, open(out_path).read() + time.sleep(0.2) + before = _rows(d) + started = time.monotonic() + res = d.run("backup", "sync") + took = time.monotonic() - started + out = res.stdout + res.stderr + print(f"sync beside purge: exit {res.returncode} in {took:.1f} s: {out.strip().splitlines()[-1]}") + assert res.returncode == 1, out + assert "one maintenance command runs at a time" in out, out + assert "--- Adopted" not in res.stdout, out + assert took < 15, f"sync took {took:.1f} s to refuse" + assert _rows(d) == before + _set_status(d, live, "AVAILABLE") + assert purge.wait(timeout=90) == 1, open(out_path).read() + text = open(out_path).read() + print(text) + assert f"left: {dead}" in text, text + finally: + if purge.poll() is None: + purge.kill() + purge.wait(timeout=10) + os.chmod(dead_prefix, 0o700) + assert _rows(d) == {}, "purge removed both rows" + assert os.path.isfile(os.path.join(dead_prefix, "extenddb-manifest.json")) + res = d.run("backup", "sync") + assert res.returncode == 0, res.stdout + res.stderr + assert _adopted_section(res.stdout).splitlines()[1:-1] == [f" {dead}"], res.stdout + res = d.run("backup", "purge", "--confirm") + assert res.returncode == 0, res.stdout + res.stderr + assert not os.path.exists(dead_prefix) + + +@pytest.fixture() +def other_catalog(request, tmp_path, deployment): + """A second catalog of the build's backend (its own `init`), whose + `[backup.store]` names the first deployment's root; its databases are + dropped after the test.""" + _sync_cell(deployment) + base = os.path.join(os.path.realpath(str(tmp_path)), "other") + os.mkdir(base) + store = ( + "[backup]\nreconcile_interval_secs = 1\n" + f'[backup.store]\nkind = "filesystem"\n[backup.store.filesystem]\npath = "{deployment.root}"\n' + ) + if deployment.backend == "sqlite": + o = _SqliteDeployment(base) + _append(o.config, store) + yield _Deployment("sqlite", o.config, o.port, deployment.root, o.cert, base, o.db_path) + return + db_name = f"extenddb_test_{uuid.uuid4().hex[:8]}_catalog" + from urllib.parse import urlparse + + parsed = urlparse(lh.PG_CONN) + env = {"db_name": db_name, "pg_host": parsed.hostname or "localhost", "pg_port": str(parsed.port or 5432)} + config = os.path.join(base, "extenddb.toml") + port = _free_port() + try: + result = lh._run_extenddb( + "init", *lh._init_args(env), config=config, + env_override={"EXTENDDB_ADMIN_PASSWORD": ADMIN_PASSWORD}, + ) + assert result.returncode == 0, result.stdout + result.stderr + lh._patch_config_port(config, port) + _append(config, store) + cert = os.path.join(os.path.expanduser("~/.extenddb/tls"), "cert.pem") + yield _Deployment("postgres", config, port, deployment.root, cert, base, f"{lh.PG_CONN}/{db_name}") + finally: + lh._run_extenddb("destroy", "--yes", *lh._pg_args(), config=config, check=False, timeout=30) + lh._drop_database(db_name) + lh._drop_database(db_name[: -len("_catalog")]) + + +def test_adopt_store_makes_this_catalog_the_owner(deployment, other_catalog, tmp_path): + """A second catalog configured with the first deployment's root: plain + sync refuses the root another catalog owns, naming `--adopt-store`, and + leaves the identity file as it was. `--adopt-store` takes the claim, + rewrites the file's owner (the store id and creation time kept), logs + the audit line and adopts the root's backup; the first catalog's + server finds the store owned by another catalog at its next check (on + PostgreSQL the server kept running beside the adoption; on SQLite it was + stopped for the serve lock and the check is its restart's), and its own + sync now refuses. The second catalog's server describes, lists and + restores the adopted backup.""" + _sync_cell(deployment) + d, o = deployment, other_catalog + server, ddb, creds = _started(d, str(tmp_path / "serve.log")) + try: + name, arn = _small_backed_up_table(ddb, "adopt") + if d.backend == "sqlite": + server.stop() + identity = os.path.join(d.root, "extenddb-store.json") + with open(identity) as f: + before = json.load(f) + res = o.run("backup", "sync") + out = res.stdout + res.stderr + assert res.returncode == 1, out + assert "is owned by another catalog" in out and "run with --adopt-store to make this catalog its owner" in out, out + with open(identity) as f: + assert json.load(f) == before, "the refused sync changed the identity file" + + res = o.run("backup", "sync", "--adopt-store") + out = res.stdout + res.stderr + assert res.returncode == 0, out + assert 'Took the maintenance claim as "extenddb backup sync on ' in res.stdout, res.stdout + assert f"Adopted the root: extenddb-store.json named catalog {before['catalog_id']} and now names" in res.stdout + assert f" {arn}" in res.stdout and "1 backup(s) adopted" in res.stdout, res.stdout + with open(identity) as f: + after = json.load(f) + assert after["catalog_id"] != before["catalog_id"], (before, after) + assert after["store_id"] == before["store_id"] and after["created_at_ms"] == before["created_at_ms"], (before, after) + assert after["backend"] == o.backend and after.get("ephemeral_catalog") is False, after + assert "extenddb::audit" in out and before["catalog_id"] in out and after["catalog_id"] in out, out + + if d.backend == "sqlite": + server = _Server(d.config, str(tmp_path / "serve-previous-owner.log")) + health = _wait_health( + d.port, + lambda h: h.get("components", {}).get("backup_store", {}).get("reason") == "owned by another catalog", + server=server, + ) + assert health["components"]["backup_store"]["status"] == "unhealthy", health + print(f"branch: {d.backend}: the previous owner's store: {health['components']['backup_store']}") + finally: + server.stop() + res = d.run("backup", "sync") + assert res.returncode == 1 and "is owned by another catalog" in res.stdout + res.stderr, res.stdout + res.stderr + + server, ddb, creds = _started(o, str(tmp_path / "serve-other.log")) + try: + details = ddb.describe_backup(BackupArn=arn)["BackupDescription"]["BackupDetails"] + assert details["BackupStatus"] == "AVAILABLE" and details["BackupName"] == "nightly", details + assert arn in [s["BackupArn"] for s in ddb.list_backups()["BackupSummaries"]] + target = f"{name}-r" + resp = ddb.restore_table_from_backup(TargetTableName=target, BackupArn=arn)["TableDescription"] + assert resp["TableStatus"] == "CREATING", resp + _wait_active(ddb, target) + assert _count(ddb, target) == HAND_BUILT_ITEMS + finally: + server.stop() + + +def test_the_previous_owner_deletes_nothing_under_an_adopted_root(deployment, other_catalog, tmp_path): + """The first catalog's server keeps running, its next check an hour + away (`reconcile_interval_secs = 3600`), while a second catalog adopts + its root with `--adopt-store`. Its `/health` still reports the store + healthy, as its last check left it; DeleteBackup of the adopted backup + there is refused as an unhealthy store's, its deletion fence having + re-read the identity file before the row was changed: the backup's + bytes stay whole and DescribeBackup there still answers AVAILABLE; + `/health` + then reports the store owned by another catalog at once. The second + catalog's server finds its store healthy and the backup AVAILABLE, and + restores it. SQLite stops the first server for the serve lock, so the + cell is PostgreSQL's.""" + _sync_cell(deployment) + d, o = deployment, other_catalog + if d.backend != "postgres": + pytest.skip("the previous owner runs beside the adoption on PostgreSQL") + with open(d.config) as f: + text = f.read() + assert text.count("reconcile_interval_secs = 1\n") == 1, text + with open(d.config, "w") as f: + f.write(text.replace("reconcile_interval_secs = 1\n", "reconcile_interval_secs = 3600\n")) + server, ddb, creds = _started(d, str(tmp_path / "serve.log")) + try: + name, arn = _small_backed_up_table(ddb, "fence") + before = _tree(_prefix(d, name, arn)) + res = o.run("backup", "sync", "--adopt-store") + out = res.stdout + res.stderr + assert res.returncode == 0, out + assert f" {arn}" in res.stdout and "1 backup(s) adopted" in res.stdout, out + assert "Stop every such server before an adoption" in res.stdout, out + health = _health(d.port) + assert health["components"]["backup_store"]["status"] == "healthy", health + with pytest.raises(ClientError) as refused: + ddb.delete_backup(BackupArn=arn) + error = refused.value.response["Error"] + print(f"DeleteBackup on the previous owner: {error}") + assert error["Code"] == "ValidationException", error + assert "unhealthy (owned by another catalog)" in error["Message"], error + assert _tree(_prefix(d, name, arn)) == before, "the previous owner deleted bytes" + details = ddb.describe_backup(BackupArn=arn)["BackupDescription"]["BackupDetails"] + assert details["BackupStatus"] == "AVAILABLE", f"the refused request moved the row: {details}" + assert _rows(d)[arn] == "AVAILABLE" + component = _health(d.port)["components"]["backup_store"] + assert component["status"] == "unhealthy", component + assert component["reason"] == "owned by another catalog", component + log = server.log_text() + assert "refused before the row was changed; nothing was deleted" in log, log[-4000:] + finally: + server.stop() + + server, ddb, creds = _started(o, str(tmp_path / "serve-other.log")) + try: + details = ddb.describe_backup(BackupArn=arn)["BackupDescription"]["BackupDetails"] + assert details["BackupStatus"] == "AVAILABLE", details + target = f"{name}-r" + ddb.restore_table_from_backup(TargetTableName=target, BackupArn=arn) + _wait_active(ddb, target) + assert _count(ddb, target) == HAND_BUILT_ITEMS + finally: + server.stop() + + +def test_a_prefix_the_store_cannot_read_leaves_the_sync_incomplete(deployment, tmp_path): + """A backup whose row the catalog lost, its manifest made unreadable + (mode 000): `extenddb backup sync` reports the prefix as one the store + could not be read for and exits 1 saying the run is incomplete, with no + row inserted. Once the mode is restored, the next sync adopts the + backup and exits 0.""" + _sync_cell(deployment) + d = deployment + if os.geteuid() == 0: + pytest.skip("a mode-000 file is readable by root") + server, ddb, creds = _started(d, str(tmp_path / "serve.log")) + try: + name, arn = _small_backed_up_table(ddb, "unreadable") + finally: + server.stop() + _drop_row(d, arn) + manifest = os.path.join(_prefix(d, name, arn), "extenddb-manifest.json") + os.chmod(manifest, 0o000) + try: + res = d.run("backup", "sync") + out = res.stdout + res.stderr + print(f"branch: {d.backend}: {out.strip().splitlines()[-1]}") + assert res.returncode == 1, out + assert f" {_key_prefix(name, arn)}: the store could not be read: " in res.stdout, out + assert "the run is incomplete: 1 prefix(es) were skipped for a store or catalog error" in out, out + assert _adopted_section(res.stdout).splitlines()[1:-1] == [" (none)"], res.stdout + assert arn not in _rows(d) + finally: + os.chmod(manifest, 0o600) + res = d.run("backup", "sync") + out = res.stdout + res.stderr + assert res.returncode == 0, out + assert _adopted_section(res.stdout).splitlines()[1:-1] == [f" {arn}"], res.stdout + assert _rows(d)[arn] == "AVAILABLE" + + +def test_an_adopter_whose_put_lands_during_the_sync_ends_it_with_exit_1(deployment, other_catalog, tmp_path): + """A second catalog adopts the first one's root with `--adopt-store`, + the root holding one backup and many copies of it under later backup + ids (each read and skipped: its manifest names another backup id). Once + the command has printed its adoption and while its sync is still in the + prefix loop, the identity file is rewritten to name a third catalog + under the same store id, as a second adopter's put would leave it. The + command's last read of the file finds it: exit 1, naming the third + catalog and the store id. The first server is stopped throughout.""" + _sync_cell(deployment) + d, o = deployment, other_catalog + import subprocess + + server, ddb, creds = _started(d, str(tmp_path / "serve.log")) + try: + name, arn = _small_backed_up_table(ddb, "race") + finally: + server.stop() + src = _prefix(d, name, arn) + millis, suffix = _backup_id(arn).split("-") + copies = 10000 + for i in range(copies): + shutil.copytree(src, os.path.join(d.root, ACCOUNT, name, f"{int(millis) + 1 + i:014d}-{suffix}"), + copy_function=os.link) + identity = os.path.join(d.root, "extenddb-store.json") + with open(identity) as f: + before = json.load(f) + out_path = str(tmp_path / "adopt.out") + with open(out_path, "wb") as out: + adopting = subprocess.Popen( + [EXTENDDB_BINARY, "backup", "--config", o.config, "sync", "--adopt-store"], + cwd=o.cwd, stdout=out, stderr=subprocess.STDOUT, stdin=subprocess.DEVNULL, + ) + try: + deadline = time.monotonic() + 60 + while "Adopted the root:" not in open(out_path).read(): + assert adopting.poll() is None, open(out_path).read()[-4000:] + assert time.monotonic() < deadline, open(out_path).read()[-4000:] + time.sleep(0.05) + third = dict(before, catalog_id=str(uuid.uuid4())) + staged = identity + ".staged" + _private_file(staged, (json.dumps(third, indent=2) + "\n").encode()) + os.rename(staged, identity) + assert "--- Adopted" not in open(out_path).read(), "the sync had ended before the rewrite" + assert adopting.poll() is None, "the command ended before the rewrite" + assert adopting.wait(timeout=180) == 1, open(out_path).read()[-4000:] + finally: + if adopting.poll() is None: + adopting.kill() + adopting.wait(timeout=10) + text = open(out_path).read() + print("\n".join(line for line in text.splitlines() if not line.startswith(" "))[-3000:]) + assert ( + f"after the sync, extenddb-store.json names catalog {third['catalog_id']} (store id {before['store_id']}): " + "another catalog adopted the root during the sync" + ) in text, text[-4000:] + assert f" {arn}" in _adopted_section(text), text[-4000:] + assert "=== extenddb backup sync complete" not in text, text[-4000:] diff --git a/tests/test_backup_restore_fidelity.py b/tests/test_backup_restore_fidelity.py index 35032df5e..12da741c7 100644 --- a/tests/test_backup_restore_fidelity.py +++ b/tests/test_backup_restore_fidelity.py @@ -6,8 +6,9 @@ Every backend; hash-only keys and composite keys with S, N, and B sort keys, plus secondary indexes and provisioned throughput. A restore must reach ACTIVE with the source's key schema, attribute definitions, and exactly the source's -items, attribute for attribute. (Multi-part base keys are a preview that -restore refuses; that refusal is tested at the storage level.) Comparing whole items rather than counts is what catches a +items, attribute for attribute. (Multi-part base keys are a preview: a +store-backed backup of such a table restores through the store, an in-catalog +one is refused, and the refusal is tested at the storage level.) Comparing whole items rather than counts is what catches a backend that restores the right number of rows under the wrong keys. Readiness assessment P0-2: on PostgreSQL every table with a sort key backed up @@ -24,6 +25,7 @@ import boto3 import pytest +import requests from botocore.config import Config from botocore.exceptions import ClientError, ReadTimeoutError @@ -41,13 +43,84 @@ BACKUP_TIMEOUT_S = 900.0 RESTORE_TIMEOUT_S = 1800.0 +# Where the server under test writes its backups: `filesystem` or `s3` (a +# configured `[backup.store]`, under which CreateBackup is asynchronous and +# backs up vector tables, and RestoreTableFromBackup runs the restore job +# from the store) or `none` (the in-catalog create). The CI jobs set it +# beside EXTENDDB_TEST_ENDPOINT; the one test whose expected outcome depends +# on it fails when it is unset rather than accept either outcome. Every cell +# that restores a backup it created waits for AVAILABLE first +# (`_create_backup`), on either path. +BACKUP_STORE = os.environ.get("EXTENDDB_TEST_BACKUP_STORE", "").strip() or None +BACKUP_STORE_VALUES = ("filesystem", "s3", "none") + + +@pytest.fixture(scope="session") +def backup_store_component(endpoint_url): + """The server's `backup_store` health component, read once from + `/health`, checked against EXTENDDB_TEST_BACKUP_STORE: a configured store + (`filesystem`, `s3`) must be reported with that kind and `healthy`, and + `none` must report no component. A variable the harness cannot observe + is not a parameter: a check that only reads it asserts nothing about the + server, and without this, `s3` against a server with a filesystem store + passed every cell.""" + if BACKUP_STORE not in BACKUP_STORE_VALUES: + pytest.fail( + f"EXTENDDB_TEST_BACKUP_STORE is {'unset' if BACKUP_STORE is None else repr(BACKUP_STORE)}; " + f"set it to one of {BACKUP_STORE_VALUES} to say where the server under test writes " + "its backups" + ) + assert endpoint_url, "the backup store component is ExtendDB's; no endpoint is set" + health = requests.get(f"{endpoint_url.rstrip('/')}/health", verify=False, timeout=10).json() + component = health.get("components", {}).get("backup_store") + if BACKUP_STORE == "none": + assert component is None, ( + f"EXTENDDB_TEST_BACKUP_STORE=none but the server reports a backup store: {component}" + ) + else: + assert component is not None, ( + f"EXTENDDB_TEST_BACKUP_STORE={BACKUP_STORE} but the server reports no backup store: " + f"{health}" + ) + assert component.get("kind") == BACKUP_STORE, ( + f"EXTENDDB_TEST_BACKUP_STORE={BACKUP_STORE} but the server's store is {component}" + ) + assert component.get("status") == "healthy", ( + f"the server's {BACKUP_STORE} store is not healthy: {component}" + ) + print(f"backup store under test: {component if component else 'none'}") + return component + + +def wait_for_backup_available(client, backup_arn: str, timeout: float = BACKUP_TIMEOUT_S) -> None: + """Poll DescribeBackup until the backup is AVAILABLE. + + The service creates a backup asynchronously; so does ExtendDB with a + configured `[backup.store]` (CreateBackup answers CREATING and the job + uploads). A backup still CREATING refuses DeleteBackup (BackupInUse) and + restore, so every caller that goes on to delete, restore or read the + size of a backup it just created waits here first. Any other status is a + failure: a backup that fails is BackupNotFound, never FAILED, on the wire. + """ + deadline = time.monotonic() + timeout + while True: + status = client.describe_backup(BackupArn=backup_arn)["BackupDescription"][ + "BackupDetails" + ]["BackupStatus"] + if status == "AVAILABLE": + return + assert status == "CREATING", f"backup {backup_arn} is {status}" + assert time.monotonic() < deadline, f"backup {backup_arn} not AVAILABLE in time" + time.sleep(_poll_interval()) + def _create_backup(client, table_name: str) -> str: """CreateBackup, then wait for the backup to be AVAILABLE. The service creates a backup asynchronously and refuses to restore one - that is still CREATING; ExtendDB returns it AVAILABLE. A just-created - table can also refuse CreateBackup for a short while, which is retried. + that is still CREATING; so does ExtendDB with a configured store. A + just-created table can also refuse CreateBackup for a short while, which + is retried. """ deadline = time.monotonic() + 60 while True: @@ -62,26 +135,25 @@ def _create_backup(client, table_name: str) -> str: if not retryable or time.monotonic() >= deadline: raise time.sleep(0.5) - deadline = time.monotonic() + BACKUP_TIMEOUT_S try: - while True: - status = client.describe_backup(BackupArn=arn)["BackupDescription"][ - "BackupDetails" - ]["BackupStatus"] - if status == "AVAILABLE": - return arn - assert status == "CREATING", f"backup {arn} is {status}" - assert time.monotonic() < deadline, f"backup {arn} not AVAILABLE in time" - time.sleep(_poll_interval()) + wait_for_backup_available(client, arn) + return arn except BaseException: # The caller never learns the ARN, so the backup is removed here. - try: - client.delete_backup(BackupArn=arn) - except ClientError: - pass + _delete_backup_when_available(client, arn) raise +def _delete_backup_when_available(client, backup_arn: str) -> None: + """DeleteBackup once the backup is no longer CREATING; a backup that is + gone (failed or deleted) is left as it is.""" + try: + wait_for_backup_available(client, backup_arn) + client.delete_backup(BackupArn=backup_arn) + except (ClientError, AssertionError): + pass + + def _backup_creation_time(client, backup_arn: str): return client.describe_backup(BackupArn=backup_arn)["BackupDescription"][ "BackupDetails" @@ -256,7 +328,11 @@ def _index_items(client, table: str, index: str, hash_attr: str, values: list) - def _cleanup(client, backup_arn: str | None, *tables: str) -> None: - """Delete tables and the backup; each step runs even if an earlier one fails.""" + """Delete tables and the backup; each step runs even if an earlier one fails. + + A backup still CREATING (the asynchronous create) is waited for first: + DeleteBackup of one is BackupInUseException. + """ if tables: try: _drop(client, tables[0]) @@ -265,6 +341,7 @@ def _cleanup(client, backup_arn: str | None, *tables: str) -> None: return if backup_arn: try: + wait_for_backup_available(client, backup_arn) client.delete_backup(BackupArn=backup_arn) except client.exceptions.BackupNotFoundException: pass @@ -1903,19 +1980,45 @@ def test_on_demand_throughput_override_is_applied(dynamodb_client): @extenddb_only -def test_create_backup_of_a_vector_table_is_refused(dynamodb_client): - """Refused at CreateBackup, not at restore time.""" +def test_vector_table_backup_refused_in_catalog_and_restored_from_store(dynamodb_client, backup_store_component): + """CreateBackup of a table with vector indexes: the outcome of the + server's configured path, read from EXTENDDB_TEST_BACKUP_STORE and + checked against the server's `/health` (`backup_store_component`). + + `none` (no `[backup.store]`, or the in-catalog create): refused at + CreateBackup with the ValidationException main gives, and no backup is + created. `filesystem` or `s3`: the store-backed create backs the table + up (the manifest records its vector indexes): CreateBackup answers + CREATING, DescribeBackup reaches AVAILABLE with the table's features, + ListBackups lists it, and RestoreTableFromBackup of it restores the + table with its vector index: the response is CREATING with a + RestoreSummary, the target reaches ACTIVE, DescribeTable shows the + vector index ACTIVE, and SearchVectors on it answers the item. The test + fails rather than accept either outcome when the variable is unset (the + fixture fails first), and prints which branch ran. + """ from test_vector_index_query_scan import _error_message, _signed_post client = dynamodb_client name = f"restore-vec-{uuid.uuid4().hex[:10]}" + restored = f"{name}-r" resp = _signed_post( "CreateTable", { "TableName": name, - "AttributeDefinitions": [{"AttributeName": "pk", "AttributeType": "S"}], + "AttributeDefinitions": [ + {"AttributeName": "pk", "AttributeType": "S"}, + {"AttributeName": "g", "AttributeType": "S"}, + ], "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}], "BillingMode": "PAY_PER_REQUEST", + "GlobalSecondaryIndexes": [ + { + "IndexName": "by-g", + "KeySchema": [{"AttributeName": "g", "KeyType": "HASH"}], + "Projection": {"ProjectionType": "KEYS_ONLY"}, + } + ], "VectorIndexes": [ { "IndexName": "vidx", @@ -1930,23 +2033,82 @@ def test_create_backup_of_a_vector_table_is_refused(dynamodb_client): if resp.status_code == 400 and "not supported" in _error_message(resp): pytest.skip("target backend does not support vector indexes") assert resp.status_code == 200, resp.text + backup_arn = None try: wait_for_active(client, name, timeout=180) vector = _signed_post("DescribeTable", {"TableName": name}).json()["Table"] if not vector.get("VectorIndexes"): pytest.skip("target accepted CreateTable but did not create the vector index") - with pytest.raises(ClientError) as exc_info: - client.create_backup(TableName=name, BackupName=f"{name}-bkp") - error = exc_info.value.response["Error"] - assert error["Code"] == "ValidationException" - assert error["Message"] == ( - f"table {name} has 1 vector index(es); restoring a table with vector " - "indexes is not supported by this storage backend" + client.put_item( + TableName=name, + Item={"pk": {"S": "a"}, "g": {"S": "g1"}, "emb": {"L": [{"N": "0.1"}] * 4}}, ) + if BACKUP_STORE == "none": + print("branch: the in-catalog create refuses a vector table") + # A plain call, so a create that is accepted against expectation + # leaves its ARN for the cleanup rather than a backup nobody + # deletes. + try: + accepted = client.create_backup(TableName=name, BackupName=f"{name}-bkp") + except ClientError as e: + error = e.response["Error"] + else: + backup_arn = accepted["BackupDetails"]["BackupArn"] + pytest.fail( + f"the in-catalog create accepted a vector table: {accepted['BackupDetails']}" + ) + assert error["Code"] == "ValidationException" + assert error["Message"] == ( + f"table {name} has 1 vector index(es); restoring a table with vector " + "indexes is not supported by this storage backend" + ) + listed = client.list_backups(TableName=name)["BackupSummaries"] + assert listed == [], listed + return + print(f"branch: the {BACKUP_STORE} store backs a vector table up") + details = client.create_backup(TableName=name, BackupName=f"{name}-bkp")["BackupDetails"] + backup_arn = details["BackupArn"] + assert details["BackupStatus"] == "CREATING", details + wait_for_backup_available(client, backup_arn) + desc = client.describe_backup(BackupArn=backup_arn)["BackupDescription"] + assert desc["BackupDetails"]["BackupStatus"] == "AVAILABLE", desc + assert desc["SourceTableDetails"]["TableName"] == name + assert desc["SourceTableDetails"]["ItemCount"] == 1, desc["SourceTableDetails"] + gsis = desc["SourceTableFeatureDetails"]["GlobalSecondaryIndexes"] + assert [g["IndexName"] for g in gsis] == ["by-g"], desc["SourceTableFeatureDetails"] listed = client.list_backups(TableName=name)["BackupSummaries"] - assert listed == [], listed + assert [b["BackupArn"] for b in listed] == [backup_arn], listed + # The round trip: the restore runs from the store and the target + # comes back with its vector index. + resp = client.restore_table_from_backup(TargetTableName=restored, BackupArn=backup_arn) + table = resp["TableDescription"] + assert table["TableStatus"] == "CREATING", table + assert table["RestoreSummary"]["SourceBackupArn"] == backup_arn, table + assert table["RestoreSummary"]["RestoreInProgress"] is True, table + wait_for_active(client, restored, timeout=RESTORE_TIMEOUT_S) + desc = _signed_post("DescribeTable", {"TableName": restored}).json()["Table"] + assert "RestoreSummary" not in desc, desc + indexes = desc.get("VectorIndexes") or [] + assert [v["IndexName"] for v in indexes] == ["vidx"], desc + assert indexes[0]["IndexStatus"] == "ACTIVE", indexes + assert indexes[0]["Dimensions"] == 4, indexes + assert [g["IndexName"] for g in desc["GlobalSecondaryIndexes"]] == ["by-g"], desc + search = _signed_post( + "SearchVectors", + { + "TableName": restored, + "IndexName": "vidx", + "SearchVector": [{"N": "0.1"}] * 4, + "TopK": 1, + }, + ) + assert search.status_code == 200, search.text + results = search.json()["SearchResults"] + assert [r["Item"]["pk"] for r in results] == [{"S": "a"}], results + got = client.get_item(TableName=restored, Key={"pk": {"S": "a"}}, ConsistentRead=True) + assert got["Item"]["g"] == {"S": "g1"}, got finally: - _cleanup(client, None, name) + _cleanup(client, backup_arn, name, restored) def test_update_table_on_a_restored_table_reports_no_restore_summary(dynamodb_client): diff --git a/tests/test_backup_store_config.py b/tests/test_backup_store_config.py new file mode 100644 index 000000000..cddfc0c11 --- /dev/null +++ b/tests/test_backup_store_config.py @@ -0,0 +1,1706 @@ +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 + +"""The `[backup.store]` configuration at the server's boundary. + +Each test starts its own `extenddb serve --foreground` on an isolated +deployment and reads `/health`, the server's log and the API's answers: + +- the component `backup_store` on `/health`: absent without a store, healthy + with a filesystem store, `checking` while an S3 store's first check is in + flight, unhealthy (status `degraded`) while an S3 endpoint refuses + connections or stalls, and healthy again once it answers, without a restart; +- the restore and delete of a store-backed backup, and CreateBackup (which + writes to the store), refused with a ValidationException naming the + store's kind and the problem's category and not its root while the store + is checking or unhealthy, and CreateBackup writing a backup under the + store's root once it is healthy; +- a stalled S3 endpoint holding the listener for no longer than the server's + own startup, and each reconciler cycle for no longer than the check's bound; +- the configuration errors that stop startup with the key, path or bucket in + the message: an S3 store without `spool_path` on PostgreSQL, a `spool_path` + that is the database directory, a filesystem root that does not exist, and + a defaulted SQLite root under a group-writable parent configured explicitly + (the same root defaulted is a warning that leaves the store unconfigured + while the catalog has no rows for it; with rows for it the defaulted root + is handled as a configured one: unhealthy, reported, its rows refused); +- the default root beside a SQLite database file created `0700` with + `extenddb-store.json` in it, and a relative root resolved against the + shell's directory in daemon mode; +- `extenddb backup list` and `backup purge --confirm` on an empty store, and + `backup purge` refused while a server holds the SQLite serve lock (exit 6); +- an in-memory SQLite server restarted onto the root it used: the second + start adopts the root the first start's catalog owned and syncs before it + listens, so the first start's backup is described and restored; every + `backup` command refuses the in-memory database; a root whose file names + a durable catalog stays unhealthy; +- on PostgreSQL, the backup commands ending with the exit code `serve` gives + for a catalog the factory refuses: 6 while a migration holds the serving + key, 3 on a catalog behind the binary. + +The binary at `EXTENDDB_BINARY` decides which cells run: a SQLite build runs +the SQLite cells, a PostgreSQL build (with +`EXTENDDB_TEST_PG_CONNECTION_STRING`) the PostgreSQL ones. The S3 cells need +moto at `EXTENDDB_TEST_S3_ENDPOINT` (an `http://127.0.0.1:` endpoint) +and skip with a reason otherwise. Excluded from the backend-agnostic suite: +`devtools/run-tests` runs this file after it, in its own step on a SQLite +build (CI starts moto for it) and in the CLI lifecycle list on PostgreSQL. +""" + +from __future__ import annotations + +import json +import os +import re +import shutil +import signal +import socket +import ssl +import subprocess +import threading +import time +import urllib.request +import uuid + +import boto3 +import pytest +from botocore.exceptions import ClientError + +import lifecycle_helpers as lh + +# Absolute, since the deployments run the binary with `cwd` set to their own +# directory (the relative-root cell needs the shell's directory): CI names +# the binary as `target/release/extenddb`, which would otherwise be resolved +# against that directory. +EXTENDDB_BINARY = os.path.abspath(lh.EXTENDDB_BINARY) +S3_ENDPOINT = os.environ.get("EXTENDDB_TEST_S3_ENDPOINT", "").strip() +ADMIN_PASSWORD = "TestPass1!" +_DEVTOOLS = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "devtools") + + +def _health(port: int) -> dict: + ctx = ssl.create_default_context() + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + req = urllib.request.Request(f"https://127.0.0.1:{port}/health") + with urllib.request.urlopen(req, context=ctx, timeout=5) as resp: + assert resp.status == 200 + return json.loads(resp.read()) + + +def _wait_health(port: int, predicate, timeout: float = 20.0, server: "_Server | None" = None) -> dict: + deadline = time.monotonic() + timeout + last = None + while time.monotonic() < deadline: + if server is not None and server.proc.poll() is not None: + raise AssertionError(f"the server exited {server.proc.returncode}:\n{server.log_text()}") + try: + last = _health(port) + if predicate(last): + return last + except Exception: # noqa: BLE001 - the server may not listen yet + pass + time.sleep(0.25) + log = server.log_text() if server is not None else "" + raise AssertionError(f"/health did not reach the expected state; last: {last!r}\n{log}") + + +def _append(config_path: str, toml: str) -> None: + with open(config_path, "a") as f: + f.write("\n" + toml + "\n") + + +def _free_port() -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + + +class _Server: + """A `serve --foreground` process with its log in a file.""" + + def __init__(self, config_path: str, log_path: str, env: dict | None = None): + self.log_path = log_path + self.log = open(log_path, "wb") # noqa: SIM115 - closed in stop() + # A SQLite deployment's paths are relative to the config's directory. + self.proc = subprocess.Popen( + [EXTENDDB_BINARY, "serve", "--foreground", "--config", config_path], + stdout=self.log, + stderr=subprocess.STDOUT, + stdin=subprocess.DEVNULL, + env=env, + cwd=os.path.dirname(os.path.abspath(config_path)), + ) + + def log_text(self) -> str: + self.log.flush() + with open(self.log_path, errors="replace") as f: + return f.read() + + def wait_exit(self, timeout: float = 20.0) -> int: + return self.proc.wait(timeout=timeout) + + def stop(self) -> None: + if self.proc.poll() is None: + self.proc.terminate() + try: + self.proc.wait(timeout=20) + except subprocess.TimeoutExpired: + self.proc.kill() + self.proc.wait(timeout=10) + self.log.close() + + +def _provision(port: int, cert_path: str, admin_password: str) -> dict: + env = os.environ.copy() + env.update( + { + "EXTENDDB_TEST_ENDPOINT": f"https://127.0.0.1:{port}", + "EXTENDDB_ADMIN_USER": "admin", + "EXTENDDB_ADMIN_PASSWORD": admin_password, + "EXTENDDB_CA_CERT": cert_path, + } + ) + res = subprocess.run( + ["python3", os.path.join(_DEVTOOLS, "provision-test-credentials")], + capture_output=True, + text=True, + env=env, + check=True, + timeout=60, + ) + creds = {} + for line in res.stdout.splitlines(): + line = line.strip() + if line.startswith("export "): + k, _, v = line[len("export "):].partition("=") + creds[k] = v.strip().strip('"').strip("'") + assert "AWS_ACCESS_KEY_ID" in creds, f"provisioning failed: {res.stdout}\n{res.stderr}" + return creds + + +def _client(port: int, creds: dict): + return boto3.client( + "dynamodb", + endpoint_url=f"https://127.0.0.1:{port}", + region_name="us-east-1", + aws_access_key_id=creds["AWS_ACCESS_KEY_ID"], + aws_secret_access_key=creds["AWS_SECRET_ACCESS_KEY"], + verify=False, + ) + + +def _wait_backup_available(ddb, arn: str, timeout: float = 60.0) -> None: + """Poll DescribeBackup to AVAILABLE; any other status fails.""" + deadline = time.monotonic() + timeout + while True: + status = ddb.describe_backup(BackupArn=arn)["BackupDescription"]["BackupDetails"]["BackupStatus"] + if status == "AVAILABLE": + return + assert status == "CREATING", f"backup {arn} is {status}" + assert time.monotonic() < deadline, f"backup {arn} not AVAILABLE in {timeout} s" + time.sleep(0.1) + + +def _create_table(ddb, name: str) -> None: + ddb.create_table( + TableName=name, + KeySchema=[{"AttributeName": "pk", "KeyType": "HASH"}], + AttributeDefinitions=[{"AttributeName": "pk", "AttributeType": "S"}], + BillingMode="PAY_PER_REQUEST", + ) + deadline = time.monotonic() + 60 + while time.monotonic() < deadline: + if ddb.describe_table(TableName=name)["Table"]["TableStatus"] == "ACTIVE": + return + time.sleep(0.05) + raise AssertionError("table did not become ACTIVE") + + +class _Forwarder: + """A TCP forwarder from a local port to moto, started when the test wants + the S3 endpoint to start answering.""" + + def __init__(self, listen_port: int, upstream: tuple[str, int]): + self.listen_port = listen_port + self.upstream = upstream + self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + self.sock.bind(("127.0.0.1", listen_port)) + self.sock.listen(16) + self.stopping = threading.Event() + self.thread = threading.Thread(target=self._accept, daemon=True) + self.thread.start() + + def _pump(self, a: socket.socket, b: socket.socket) -> None: + try: + while not self.stopping.is_set(): + data = a.recv(65536) + if not data: + break + b.sendall(data) + except OSError: + pass + finally: + for s in (a, b): + try: + s.shutdown(socket.SHUT_RDWR) + except OSError: + pass + + def _accept(self) -> None: + self.sock.settimeout(0.5) + while not self.stopping.is_set(): + try: + client, _ = self.sock.accept() + except socket.timeout: + continue + except OSError: + break + try: + upstream = socket.create_connection(self.upstream, timeout=5) + except OSError: + client.close() + continue + threading.Thread(target=self._pump, args=(client, upstream), daemon=True).start() + threading.Thread(target=self._pump, args=(upstream, client), daemon=True).start() + + def stop(self) -> None: + self.stopping.set() + self.sock.close() + + +def _moto_or_skip() -> tuple[str, int]: + if not S3_ENDPOINT.startswith("http://"): + pytest.skip("EXTENDDB_TEST_S3_ENDPOINT is not an http:// moto endpoint") + host_port = S3_ENDPOINT[len("http://"):].rstrip("/") + host, _, port = host_port.partition(":") + return host, int(port) + + +def _s3_env() -> dict: + env = os.environ.copy() + env.update( + { + "AWS_ACCESS_KEY_ID": "testing", + "AWS_SECRET_ACCESS_KEY": "testing", + "AWS_REGION": "us-east-1", + "AWS_EC2_METADATA_DISABLED": "true", + } + ) + env.pop("AWS_SESSION_TOKEN", None) + env.pop("AWS_PROFILE", None) + return env + + +def _s3_client(): + return boto3.client( + "s3", + endpoint_url=S3_ENDPOINT, + region_name="us-east-1", + aws_access_key_id="testing", + aws_secret_access_key="testing", + ) + + +@pytest.fixture() +def moto_bucket(): + """A bucket of this test's own on the shared moto, emptied and deleted at + teardown so the endpoint does not accumulate one per run.""" + _moto_or_skip() + bucket = f"extenddb-wire-{uuid.uuid4().hex[:12]}" + s3 = _s3_client() + s3.create_bucket(Bucket=bucket) + yield bucket + try: + listed = s3.list_objects_v2(Bucket=bucket).get("Contents", []) + if listed: + s3.delete_objects(Bucket=bucket, Delete={"Objects": [{"Key": o["Key"]} for o in listed]}) + for upload in s3.list_multipart_uploads(Bucket=bucket).get("Uploads", []): + s3.abort_multipart_upload(Bucket=bucket, Key=upload["Key"], UploadId=upload["UploadId"]) + s3.delete_bucket(Bucket=bucket) + except ClientError: + pass + + +def _assert_store_refusal(exc: ClientError, kind: str, root_fragment: str, category: str = "unreachable") -> None: + """The refusal of a store-backed backup's restore or delete: a + ValidationException naming the store's kind and the problem's category + (`unreachable`, `manifest probe failed`, `first check is pending`) and + never the root.""" + err = exc.response["Error"] + assert err["Code"] == "ValidationException", err + assert f"{kind} backup store" in err["Message"], err + assert category in err["Message"], err + assert root_fragment not in err["Message"], err + + +_SOURCE_TABLE_JSON = json.dumps({ + "table_name": "Music", "table_id": "t-1", "table_arn": "arn:aws:dynamodb:us-east-1:123456789012:table/Music", + "key_schema": [{"AttributeName": "pk", "KeyType": "HASH"}], + "attribute_definitions": [{"AttributeName": "pk", "AttributeType": "S"}], + "billing_mode": "PAY_PER_REQUEST", "provisioned_throughput": None, "on_demand_throughput": None, + "global_secondary_indexes": [], "local_secondary_indexes": [], "vector_indexes": None, + "stream_specification": None, "time_to_live": None, "sse_description": None, + "item_count": 0, "table_size_bytes": 0, "table_creation_date_time": 0.0, +}) + + +def _sqlite_module(): + try: + # The catalog's schema needs a SQLite newer than some hosts' stdlib + # module links; the wheel carries a current library. + import pysqlite3 as sqlite3 + except ImportError: + import sqlite3 + return sqlite3 + + +def _seed_backup_row(db_path: str, backup_id: str, status: str, store_kind: str, store_root: str, store_id: str) -> str: + """One `backup_index` row of account 123456789012's `Music` table under + the given store, as a server that used the store leaves it; returns the + backup's ARN.""" + account = "123456789012" + arn = f"arn:aws:dynamodb:us-east-1:{account}:table/Music/backup/{backup_id}" + conn = _sqlite_module().connect(db_path, timeout=10) + try: + conn.execute( + "INSERT INTO backup_index (account_id, backup_id, table_name, backup_arn, backup_name, status, " + "store_kind, store_root, store_id, size_bytes, item_count, created_at, source_table_details_json, " + "owner_generation, heartbeat_at) VALUES (?, ?, 'Music', ?, 'nightly', ?, ?, ?, ?, 1, 1, " + "'2026-10-01T00:00:00.000Z', ?, 7, '2026-10-01T00:00:00.000Z')", + (account, backup_id, arn, status, store_kind, store_root, store_id, _SOURCE_TABLE_JSON), + ) + conn.commit() + finally: + conn.close() + return arn + + +# --- SQLite --------------------------------------------------------------------------- + + +def _sqlite_build() -> bool: + if not os.path.isfile(EXTENDDB_BINARY): + return False + res = subprocess.run( + [EXTENDDB_BINARY, "version"], capture_output=True, text=True, timeout=30 + ) + return "(sqlite)" in res.stdout + + +class _SqliteDeployment: + def __init__(self, tmp_dir: str): + self.dir = tmp_dir + self.config = os.path.join(tmp_dir, "extenddb.toml") + self.db_path = os.path.join(tmp_dir, "extenddb.sqlite") + self.port = _free_port() + env = os.environ.copy() + env["EXTENDDB_ADMIN_PASSWORD"] = ADMIN_PASSWORD + subprocess.run( + [EXTENDDB_BINARY, "init", "--backend", "sqlite", "--config", self.config], + cwd=tmp_dir, + capture_output=True, + text=True, + stdin=subprocess.DEVNULL, + timeout=60, + check=True, + env=env, + ) + lh._patch_config_port(self.config, self.port) + self.cert = os.path.expanduser("~/.extenddb/tls/cert.pem") + + def run(self, *args, env=None): + return subprocess.run( + [EXTENDDB_BINARY, args[0], "--config", self.config, *args[1:]], + cwd=self.dir, + capture_output=True, + text=True, + stdin=subprocess.DEVNULL, + timeout=180, + env=env, + ) + + +@pytest.fixture() +def sqlite_deployment(tmp_path): + if not _sqlite_build(): + pytest.skip("EXTENDDB_BINARY is not a SQLite build") + return _SqliteDeployment(str(tmp_path)) + + +class TestSqlite: + def test_default_root_is_created_private_and_reported_healthy(self, sqlite_deployment, tmp_path): + d = sqlite_deployment + server = _Server(d.config, str(tmp_path / "serve.log")) + try: + health = _wait_health(d.port, lambda h: "components" in h, server=server) + assert health["status"] == "healthy", health + component = health["components"]["backup_store"] + assert component == {"status": "healthy", "kind": "filesystem"}, component + root = os.path.join(d.dir, "backups") + assert os.path.isdir(root) + assert (os.stat(root).st_mode & 0o777) == 0o700 + identity = os.path.join(root, "extenddb-store.json") + with open(identity) as f: + file = json.load(f) + assert file["format_version"] == 1 + assert file["backend"] == "sqlite" + assert file["store_id"] and file["catalog_id"] + assert (os.stat(identity).st_mode & 0o777) == 0o600 + log = server.log_text() + assert "backup store: filesystem" in log and root in log, log + assert f"store id {file['store_id']}" in log, log + # The console banner names the store beside the storage line. + assert " storage: sqlite" in log, log + # In the spelling the log line and the rows use (the canonical + # directory), with the configured spelling only where it differs. + canonical = os.path.realpath(root) + configured = f" (configured as {root})" if canonical != root else "" + assert f" backup store: filesystem {canonical}{configured} (the default beside the database file)" in log, log + assert f"backup store: filesystem {canonical} (store id" in log, log + # CreateBackup writes to the store: CREATING answered, the job's + # objects under the root, the row AVAILABLE once the manifest is + # written, and no deprecation warning. + creds = _provision(d.port, d.cert, ADMIN_PASSWORD) + ddb = _client(d.port, creds) + _create_table(ddb, "Music") + ddb.put_item(TableName="Music", Item={"pk": {"S": "a"}, "n": {"N": "1"}}) + details = ddb.create_backup(TableName="Music", BackupName="nightly")["BackupDetails"] + assert details["BackupStatus"] == "CREATING", details + arn = details["BackupArn"] + _wait_backup_available(ddb, arn) + backup_id = arn.rsplit("/", 1)[1] + prefix = os.path.join(root, "123456789012", "Music", backup_id) + assert sorted(os.listdir(prefix)) == [ + "data", "extenddb-manifest.json", "extenddb-manifest.sha256", + "manifest-files.json", "manifest-summary.json", + ], os.listdir(prefix) + assert (os.stat(os.path.join(prefix, "extenddb-manifest.json")).st_mode & 0o777) == 0o600 + with open(os.path.join(prefix, "extenddb-manifest.json")) as f: + manifest = json.load(f) + assert manifest["backup_arn"] == arn and manifest["source_backend"] == "sqlite", manifest + described = ddb.describe_backup(BackupArn=arn)["BackupDescription"]["BackupDetails"] + assert described["BackupSizeBytes"] == manifest["backup_size_bytes"] > 0, described + log = server.log_text() + assert f"backup {arn}: started, " in log and "in the table as last counted by the stats pass, key-ordered" in log, log + # The completion line carries the backup's own count, and its + # bytes count the documents beside the data files. + written = re.search(rf"backup {re.escape(arn)}: AVAILABLE, (\d+) items, (\d+) bytes written in", log) + assert written and int(written.group(1)) == 1, log + assert int(written.group(2)) > manifest["backup_size_bytes"], log + assert "in-catalog backup format" not in log, log + assert "configure [backup.store]" not in log, log + finally: + server.stop() + + def test_disabled_leaves_no_component_and_the_legacy_create_warns(self, sqlite_deployment, tmp_path): + d = sqlite_deployment + _append(d.config, '[backup.store]\nkind = "disabled"\n') + server = _Server(d.config, str(tmp_path / "serve.log")) + try: + health = _wait_health(d.port, lambda h: "components" in h, server=server) + assert health["status"] == "healthy" + assert "backup_store" not in health["components"], health + assert not os.path.exists(os.path.join(d.dir, "backups")) + assert 'backup store: disabled' in server.log_text() + creds = _provision(d.port, d.cert, ADMIN_PASSWORD) + ddb = _client(d.port, creds) + _create_table(ddb, "Music") + ddb.create_backup(TableName="Music", BackupName="nightly") + assert "configure [backup.store] to write backups to a backup store" in server.log_text() + finally: + server.stop() + + def test_an_in_memory_database_with_a_filesystem_store_and_without_one(self, sqlite_deployment, tmp_path): + """`sqlite-memory`'s shape (`[storage.sqlite] path = ":memory:"`, the + default of that build) on the SQLite build: with `[backup.store.filesystem] + path` set the store is configured and healthy, its identity file is + written, and a backup created there is written to the store (CREATING, + then AVAILABLE with its manifest under the root) and restores from it + with its item; without + `[backup.store]` there is no default root (no database file to put it + beside), no component, and CreateBackup writes a legacy backup with the + warning naming `[backup.store]`.""" + d = sqlite_deployment + env = os.environ.copy() + env["EXTENDDB__STORAGE__SQLITE__PATH"] = ":memory:" + env["EXTENDDB_ADMIN_PASSWORD"] = ADMIN_PASSWORD + plain_config = os.path.join(d.dir, "plain.toml") + shutil.copyfile(d.config, plain_config) + root = tmp_path / "memory-root" + root.mkdir(mode=0o700) + _append(d.config, f'[backup.store]\nkind = "filesystem"\n\n[backup.store.filesystem]\npath = "{root}"\n') + + server = _Server(d.config, str(tmp_path / "serve-memory-store.log"), env=env) + try: + health = _wait_health(d.port, lambda h: "backup_store" in h.get("components", {}), server=server) + assert health["components"]["backup_store"] == {"status": "healthy", "kind": "filesystem"}, health + assert " storage: sqlite (:memory:)" in server.log_text(), "the server runs on an in-memory database" + assert os.path.isfile(root / "extenddb-store.json") + creds = _provision(d.port, d.cert, ADMIN_PASSWORD) + ddb = _client(d.port, creds) + _create_table(ddb, "Music") + ddb.put_item(TableName="Music", Item={"pk": {"S": "a"}, "v": {"N": "1"}}) + details = ddb.create_backup(TableName="Music", BackupName="nightly")["BackupDetails"] + assert details["BackupStatus"] == "CREATING", details + arn = details["BackupArn"] + _wait_backup_available(ddb, arn) + prefix = os.path.join(root, "123456789012", "Music", arn.rsplit("/", 1)[1]) + assert os.path.isfile(os.path.join(prefix, "extenddb-manifest.json")), os.listdir(root) + restored = ddb.restore_table_from_backup(TargetTableName="Restored", BackupArn=arn) + assert restored["TableDescription"]["TableStatus"] == "CREATING", restored + deadline = time.monotonic() + 60 + while ddb.describe_table(TableName="Restored")["Table"]["TableStatus"] != "ACTIVE": + assert time.monotonic() < deadline, "the restore did not finish" + time.sleep(0.1) + item = ddb.get_item(TableName="Restored", Key={"pk": {"S": "a"}})["Item"] + assert item == {"pk": {"S": "a"}, "v": {"N": "1"}}, item + assert "restore of " + arn in server.log_text(), "the restore ran as a store-backed job" + assert "configure [backup.store]" not in server.log_text(), server.log_text() + finally: + server.stop() + + server = _Server(plain_config, str(tmp_path / "serve-memory-plain.log"), env=env) + try: + health = _wait_health(d.port, lambda h: "components" in h, server=server) + assert "backup_store" not in health["components"], health + assert " storage: sqlite (:memory:)" in server.log_text(), server.log_text() + assert not os.path.exists(os.path.join(d.dir, "backups")), "no default root beside an in-memory database" + creds = _provision(d.port, d.cert, ADMIN_PASSWORD) + ddb = _client(d.port, creds) + _create_table(ddb, "Music") + details = ddb.create_backup(TableName="Music", BackupName="nightly")["BackupDetails"] + assert details["BackupStatus"] in ("CREATING", "AVAILABLE"), details + assert "configure [backup.store] to write backups to a backup store" in server.log_text(), server.log_text() + finally: + server.stop() + + def test_an_in_memory_server_restarted_onto_its_root_serves_the_previous_starts_backups( + self, sqlite_deployment, tmp_path + ): + """`sqlite-memory`'s shape with a filesystem store, restarted: the + catalog (and its `catalog_id`) is new at every start, so the first + start writes `extenddb-store.json` with `ephemeral_catalog: true`, + and the second adopts the root that file names (a new `catalog_id`, + the store id and the marker kept) and runs the startup sync before + it listens, so DescribeBackup, ListBackups and a restore of the first + start's backup answer from the first request. Between the two starts + every `extenddb backup` command refuses the in-memory database before + it opens a catalog, and the file is unchanged. A file whose marker is + false (a durable catalog's root) is left alone: the in-memory server + reports the store owned by another catalog.""" + d = sqlite_deployment + env = os.environ.copy() + env["EXTENDDB__STORAGE__SQLITE__PATH"] = ":memory:" + env["EXTENDDB_ADMIN_PASSWORD"] = ADMIN_PASSWORD + root = tmp_path / "memory-root" + root.mkdir(mode=0o700) + identity = root / "extenddb-store.json" + _append(d.config, f'[backup]\nreconcile_interval_secs = 1\n[backup.store]\nkind = "filesystem"\n\n[backup.store.filesystem]\npath = "{root}"\n') + healthy = lambda h: h.get("components", {}).get("backup_store", {}).get("status") == "healthy" # noqa: E731 + + server = _Server(d.config, str(tmp_path / "serve-1.log"), env=env) + try: + _wait_health(d.port, healthy, server=server) + first = json.loads(identity.read_text()) + assert first["ephemeral_catalog"] is True, first + ddb = _client(d.port, _provision(d.port, d.cert, ADMIN_PASSWORD)) + _create_table(ddb, "Music") + for k in ("a", "b"): + ddb.put_item(TableName="Music", Item={"pk": {"S": k}, "v": {"N": "1"}}) + arn = ddb.create_backup(TableName="Music", BackupName="nightly")["BackupDetails"]["BackupArn"] + _wait_backup_available(ddb, arn) + finally: + server.stop() + + for args in ( + ("backup", "list"), + ("backup", "sync"), + ("backup", "sync", "--adopt-store"), + ("backup", "purge", "--confirm"), + ("backup", "remove-target", "--account", "123456789012", "Music"), + ): + res = d.run(*args, env=env) + out = res.stdout + res.stderr + assert res.returncode == 1, (args, out) + assert "the SQLite database is in memory ([storage.sqlite] path names no file)" in out, (args, out) + assert json.loads(identity.read_text()) == first, "a refused command changed the identity file" + + server = _Server(d.config, str(tmp_path / "serve-2.log"), env=env) + try: + _wait_health(d.port, healthy, server=server) + second = json.loads(identity.read_text()) + assert second["catalog_id"] != first["catalog_id"], (first, second) + assert second["store_id"] == first["store_id"] and second["created_at_ms"] == first["created_at_ms"], (first, second) + assert second["ephemeral_catalog"] is True, second + lines = server.log_text().splitlines() + synced = [i for i, line in enumerate(lines) if "startup sync of the in-memory catalog: 1 backup(s) adopted" in line] + listening = [i for i, line in enumerate(lines) if "extenddb listening on" in line] + assert len(synced) == 1 and listening and synced[0] < listening[0], "\n".join(lines[-40:]) + ddb = _client(d.port, _provision(d.port, d.cert, ADMIN_PASSWORD)) + details = ddb.describe_backup(BackupArn=arn)["BackupDescription"]["BackupDetails"] + assert details["BackupStatus"] == "AVAILABLE" and details["BackupName"] == "nightly", details + assert arn in [s["BackupArn"] for s in ddb.list_backups()["BackupSummaries"]] + restored = ddb.restore_table_from_backup(TargetTableName="Restored", BackupArn=arn) + assert restored["TableDescription"]["TableStatus"] == "CREATING", restored + deadline = time.monotonic() + 60 + while ddb.describe_table(TableName="Restored")["Table"]["TableStatus"] != "ACTIVE": + assert time.monotonic() < deadline, "the restore did not finish" + time.sleep(0.1) + assert ddb.scan(TableName="Restored", Select="COUNT")["Count"] == 2 + finally: + server.stop() + + durable = dict(second, ephemeral_catalog=False) + identity.write_text(json.dumps(durable, indent=2) + "\n") + os.chmod(identity, 0o600) + before = identity.read_bytes() + server = _Server(d.config, str(tmp_path / "serve-3.log"), env=env) + try: + health = _wait_health(d.port, lambda h: "backup_store" in h.get("components", {}), server=server) + assert health["components"]["backup_store"] == { + "status": "unhealthy", "kind": "filesystem", "reason": "owned by another catalog", + }, health + assert identity.read_bytes() == before, "an in-memory server rewrote a durable catalog's file" + assert "startup sync of the in-memory catalog" not in server.log_text(), server.log_text()[-3000:] + finally: + server.stop() + + def test_default_root_under_a_group_writable_parent_warns_and_the_same_root_explicit_fails( + self, sqlite_deployment, tmp_path + ): + d = sqlite_deployment + os.chmod(d.dir, 0o775) + server = _Server(d.config, str(tmp_path / "serve.log")) + try: + health = _wait_health(d.port, lambda h: "components" in h, server=server) + assert "backup_store" not in health["components"], health + log = server.log_text() + # `open` applies the ancestor rule; `validate` would report the + # same root the same way. Either is the warning path. + assert ("is not usable" in log or "failed its validation" in log) and "unconfigured" in log, log + assert "writable by group or others" in log, log + assert "kind = \"disabled\" to silence this" in log, log + creds = _provision(d.port, d.cert, ADMIN_PASSWORD) + ddb = _client(d.port, creds) + _create_table(ddb, "Music") + ddb.create_backup(TableName="Music", BackupName="nightly") + assert "configure [backup.store] to write backups to a backup store" in server.log_text() + finally: + server.stop() + # The same root configured explicitly is a startup error naming it. + root = os.path.join(d.dir, "backups") + _append(d.config, f'[backup.store]\nkind = "filesystem"\n[backup.store.filesystem]\npath = "{root}"\n') + server = _Server(d.config, str(tmp_path / "serve2.log")) + try: + assert server.wait_exit() != 0 + log = server.log_text() + assert "cannot be opened" in log or "failed its startup validation" in log, log + assert "writable by group or others" in log and d.dir in log, log + finally: + server.stop() + + def test_a_missing_explicit_root_and_a_spool_in_the_database_directory_stop_startup( + self, sqlite_deployment, tmp_path + ): + d = sqlite_deployment + missing = os.path.join(d.dir, "nowhere") + _append(d.config, f'[backup.store]\nkind = "filesystem"\n[backup.store.filesystem]\npath = "{missing}"\n') + server = _Server(d.config, str(tmp_path / "serve.log")) + try: + assert server.wait_exit() != 0 + log = server.log_text() + assert "cannot be opened" in log and missing in log, log + finally: + server.stop() + # A spool that is the database directory. + (tmp_path / "second").mkdir() + d2 = _SqliteDeployment(str(tmp_path / "second")) + _append( + d2.config, + f'[backup]\nspool_path = "{d2.dir}"\n[backup.store]\nkind = "s3"\n[backup.store.s3]\nbucket = "b"\n', + ) + server = _Server(d2.config, str(tmp_path / "serve2.log")) + try: + assert server.wait_exit() != 0 + log = server.log_text() + assert "backup.spool_path" in log and "database directory" in log, log + finally: + server.stop() + + def test_an_unreachable_s3_endpoint_starts_the_server_unhealthy_and_recovers( + self, sqlite_deployment, tmp_path, moto_bucket + ): + d = sqlite_deployment + moto_host, moto_port = _moto_or_skip() + bucket = moto_bucket + dead_port = _free_port() + _append( + d.config, + "[backup]\nreconcile_interval_secs = 1\n" + '[backup.store]\nkind = "s3"\n[backup.store.s3]\n' + f'bucket = "{bucket}"\nendpoint = "http://127.0.0.1:{dead_port}"\n' + 'force_path_style = true\nregion = "us-east-1"\n', + ) + server = _Server(d.config, str(tmp_path / "serve.log"), env=_s3_env()) + forwarder = None + try: + # The first check runs after the listener is up: the component is + # `checking` first, then `unhealthy` when the connection is refused. + health = _wait_health( + d.port, lambda h: h.get("components", {}).get("backup_store", {}).get("status") == "unhealthy", + server=server, + ) + assert health["status"] == "degraded", health + component = health["components"]["backup_store"] + assert component["status"] == "unhealthy" and component["kind"] == "s3", component + assert component["reason"] == "unreachable", component + assert bucket not in json.dumps(component) + spool = os.path.join(d.dir, "backup-spool") + assert (os.stat(spool).st_mode & 0o777) == 0o700 + assert (os.stat(os.path.join(spool, ".lock")).st_mode & 0o777) == 0o600 + creds = _provision(d.port, d.cert, ADMIN_PASSWORD) + ddb = _client(d.port, creds) + _create_table(ddb, "Music") + # CreateBackup writes to the store, so it is refused while the + # store is unhealthy, naming the kind and the category, not the + # bucket; no row is created. + with pytest.raises(ClientError) as refused: + ddb.create_backup(TableName="Music", BackupName="nightly") + _assert_store_refusal(refused.value, "s3", bucket, category="unreachable") + assert ddb.list_backups(TableName="Music")["BackupSummaries"] == [] + log = server.log_text() + assert "is unhealthy: the store could not be reached" in log, log + assert bucket in log, "the operator's line names the bucket" + + # The endpoint starts answering: the next reconciler cycle recovers. + forwarder = _Forwarder(dead_port, (moto_host, moto_port)) + health = _wait_health( + d.port, lambda h: h["components"]["backup_store"]["status"] == "healthy", timeout=30, server=server + ) + assert health["status"] == "healthy" + assert "is healthy again" in server.log_text() + # The identity file was created on the first passing check, and + # CreateBackup now writes a backup into the bucket. + body = _s3_client().get_object(Bucket=bucket, Key="extenddb-store.json")["Body"].read() + assert json.loads(body)["backend"] == "sqlite" + ddb.put_item(TableName="Music", Item={"pk": {"S": "a"}, "n": {"N": "1"}}) + details = ddb.create_backup(TableName="Music", BackupName="nightly2")["BackupDetails"] + assert details["BackupStatus"] == "CREATING", details + arn = details["BackupArn"] + _wait_backup_available(ddb, arn) + keys = sorted( + o["Key"].rsplit("/", 1)[1] + for o in _s3_client().list_objects_v2( + Bucket=bucket, Prefix=f"123456789012/Music/{arn.rsplit('/', 1)[1]}/" + )["Contents"] + ) + assert keys == [ + "000001.json.gz", "extenddb-manifest.json", "extenddb-manifest.sha256", + "manifest-files.json", "manifest-summary.json", + ], keys + + # And degrades again once the endpoint is gone. + forwarder.stop() + forwarder = None + health = _wait_health( + d.port, lambda h: h["components"]["backup_store"]["status"] == "unhealthy", timeout=30, server=server + ) + assert health["status"] == "degraded" + with pytest.raises(ClientError) as refused: + ddb.create_backup(TableName="Music", BackupName="again") + _assert_store_refusal(refused.value, "s3", bucket, category="unreachable") + finally: + if forwarder is not None: + forwarder.stop() + server.stop() + + def test_a_stalled_s3_endpoint_holds_neither_the_listener_nor_the_reconciler(self, sqlite_deployment, tmp_path): + """An endpoint that accepts connections and never answers. Checked: + the server binds and serves before the store's first check, which runs + after the listener is up (`/health` answers well within the SDK's + attempt timeout with the component `checking`); CreateBackup and the + restore of a seeded store-backed row are refused naming the pending + check; the check is abandoned at its + 30 s bound, the component turns `unhealthy` (`unreachable`) without a + restart and the refusal names that; and the reconciler keeps cycling + while the endpoint stalls (the next cycle's check opens a new + connection to the tarpit within seconds of the abandoned one).""" + d = sqlite_deployment + tarpit = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + tarpit.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + tarpit.bind(("127.0.0.1", 0)) + tarpit.listen(16) + tarpit_port = tarpit.getsockname()[1] + held = [] + accepted_at = [] + + def accept_and_hold(): + while True: + try: + conn, _ = tarpit.accept() + except OSError: + return + held.append(conn) + accepted_at.append(time.monotonic()) + + threading.Thread(target=accept_and_hold, daemon=True).start() + # A row of the store, as a server that used it leaves one; the root + # label is the bucket and the endpoint in the spelling the store uses. + store_root = f"s3://tarpit-bucket/?endpoint=http://127.0.0.1:{tarpit_port}" + arn = _seed_backup_row(d.db_path, "01791976431438-00000001", "AVAILABLE", "s3", store_root, "store-seeded") + _append( + d.config, + "[backup]\nreconcile_interval_secs = 1\n" + '[backup.store]\nkind = "s3"\n[backup.store.s3]\n' + f'bucket = "tarpit-bucket"\nendpoint = "http://127.0.0.1:{tarpit_port}"\n' + 'force_path_style = true\nregion = "us-east-1"\n', + ) + started = time.monotonic() + server = _Server(d.config, str(tmp_path / "serve.log"), env=_s3_env()) + try: + health = _wait_health(d.port, lambda h: "components" in h, timeout=30, server=server) + answered_after = time.monotonic() - started + # The first answer is the server's own startup (TLS, the catalog + # open), about a second here; a check awaited before the listener + # would add the SDK's 60 s attempt timeout at the least. + assert answered_after < 30.0, f"/health first answered after {answered_after:.1f} s" + assert health["status"] == "healthy", health + assert health["components"]["backup_store"] == {"status": "checking", "kind": "s3"}, health + deadline = time.monotonic() + 20 + while not held and time.monotonic() < deadline: + time.sleep(0.1) + assert held, "the store's check reached the endpoint" + log = server.log_text() + assert "listening on" in log and "run after the listener is up" in log, log + assert "backup store: s3" in log and "tarpit-bucket" in log, log + # Requests are served while the check hangs: the table is + # created, and CreateBackup and the seeded row's restore are + # refused naming the pending check and the store's kind, not the + # bucket. + creds = _provision(d.port, d.cert, ADMIN_PASSWORD) + ddb = _client(d.port, creds) + _create_table(ddb, "Music") + with pytest.raises(ClientError) as refused: + ddb.create_backup(TableName="Music", BackupName="nightly") + _assert_store_refusal(refused.value, "s3", "tarpit-bucket", category="first check is pending") + with pytest.raises(ClientError) as refused: + ddb.restore_table_from_backup(TargetTableName="Restored", BackupArn=arn) + _assert_store_refusal(refused.value, "s3", "tarpit-bucket", category="first check is pending") + assert "Restored" not in [t for t in ddb.list_tables()["TableNames"]] + + # The check is abandoned at its bound (30 s, under the SDK's 60 s + # attempt timeout): the component turns unhealthy without a + # restart, the log says the check was abandoned and the cycle's + # duties ran. + health = _wait_health( + d.port, lambda h: h["components"]["backup_store"]["status"] == "unhealthy", timeout=45, server=server + ) + assert health["status"] == "degraded", health + assert health["components"]["backup_store"]["reason"] == "unreachable", health + log = server.log_text() + assert "check did not finish within 30 s and was abandoned" in log, log + assert "is unhealthy: the store could not be reached: the check did not finish within 30 s" in log, log + with pytest.raises(ClientError) as refused: + ddb.restore_table_from_backup(TargetTableName="Restored", BackupArn=arn) + _assert_store_refusal(refused.value, "s3", "tarpit-bucket", category="unreachable") + # The reconciler keeps cycling: the next cycle's check opens a new + # connection to the tarpit within seconds of the abandoned one + # (the SDK's own retry would come 60 s after the first). + first = accepted_at[0] + deadline = time.monotonic() + 15 + while len(accepted_at) < 2 and time.monotonic() < deadline: + time.sleep(0.2) + assert len(accepted_at) >= 2, "the next cycle's check reached the endpoint" + assert accepted_at[1] - first < 45.0, f"the second connection came {accepted_at[1] - first:.1f} s after the first" + assert "is unhealthy" in log and log.count("is unhealthy: the store could not be reached") == 1, \ + "the transition is logged once" + with pytest.raises(ClientError) as refused: + ddb.create_backup(TableName="Music", BackupName="nightly2") + _assert_store_refusal(refused.value, "s3", "tarpit-bucket", category="unreachable") + finally: + server.stop() + tarpit.close() + for conn in held: + conn.close() + # Shutdown did not wait on the stalled check. + with open(server.log_path, errors="replace") as f: + final_log = f.read() + assert "did not drain" not in final_log, final_log + + def test_a_default_root_with_rows_is_unhealthy_rather_than_unconfigured(self, sqlite_deployment, tmp_path): + """The default root's leniency ends with the catalog's first row for + it: a row seeded under the default root with its manifest missing + starts the server with the store configured and unhealthy (`/health` + degraded, `manifest probe failed`), the row's restore refused naming + the kind and the category and not the root, recovery when the manifest + is put back, and `backup purge --confirm` and `backup list` reaching + the store (the purge refuses on the unhealthy store, not for want of + one).""" + d = sqlite_deployment + _append(d.config, "[backup]\nreconcile_interval_secs = 1\n") + server = _Server(d.config, str(tmp_path / "serve.log")) + try: + _wait_health(d.port, lambda h: "backup_store" in h.get("components", {}), server=server) + finally: + server.stop() + root = os.path.realpath(os.path.join(d.dir, "backups")) + with open(os.path.join(root, "extenddb-store.json")) as f: + store_id = json.load(f)["store_id"] + backup_id = "01791976431438-00000001" + arn = _seed_backup_row(d.db_path, backup_id, "AVAILABLE", "filesystem", root, store_id) + manifest_dir = os.path.join(root, "123456789012", "Music", backup_id) + manifest = os.path.join(manifest_dir, "extenddb-manifest.json") + + server = _Server(d.config, str(tmp_path / "serve2.log")) + try: + health = _wait_health(d.port, lambda h: "backup_store" in h.get("components", {}), server=server) + assert health["status"] == "degraded", health + assert health["components"]["backup_store"] == { + "status": "unhealthy", "kind": "filesystem", "reason": "manifest probe failed", + }, health + log = server.log_text() + assert "is unhealthy: the manifest of " + arn + " is not under the root" in log, log + assert "backup rows for it, so it is handled as a configured root" in log, log + assert "unconfigured" not in log, log + creds = _provision(d.port, d.cert, ADMIN_PASSWORD) + ddb = _client(d.port, creds) + with pytest.raises(ClientError) as refused: + ddb.restore_table_from_backup(TargetTableName="Restored", BackupArn=arn) + _assert_store_refusal(refused.value, "filesystem", root, category="manifest probe failed") + with pytest.raises(ClientError) as refused: + ddb.delete_backup(BackupArn=arn) + _assert_store_refusal(refused.value, "filesystem", root, category="manifest probe failed") + # CreateBackup is refused on the unhealthy store too. + _create_table(ddb, "Music") + with pytest.raises(ClientError) as refused: + ddb.create_backup(TableName="Music", BackupName="nightly") + _assert_store_refusal(refused.value, "filesystem", root, category="manifest probe failed") + # The manifest put back: the next cycle recovers the store. + for level in (os.path.dirname(os.path.dirname(manifest_dir)), os.path.dirname(manifest_dir), manifest_dir): + if not os.path.isdir(level): + os.mkdir(level, 0o700) + with open(manifest, "wb") as f: + f.write(b"{}") + health = _wait_health( + d.port, lambda h: h["components"]["backup_store"]["status"] == "healthy", timeout=30, server=server + ) + assert health["status"] == "healthy", health + assert "is healthy again" in server.log_text() + finally: + server.stop() + + # The commands reach the store: with the manifest gone again, the + # listing (a read of the file and the rows, no probe) shows the row + # under the store, and the purge, which runs the check, refuses on + # the unhealthy store and deletes nothing. + os.remove(manifest) + res = d.run("backup", "list") + assert res.returncode == 0, res.stdout + res.stderr + assert backup_id in res.stdout and f"backups in the filesystem store {root}" in res.stdout, res.stdout + res = d.run("backup", "purge", "--confirm") + out = res.stdout + res.stderr + assert res.returncode != 0, out + assert "is unhealthy" in out and "manifest" in out, out + assert "no backup store is configured" not in out, out + res = d.run("backup", "list") + assert backup_id in res.stdout, "nothing was deleted: " + res.stdout + + def test_a_relative_root_resolves_against_the_shell_in_daemon_mode(self, sqlite_deployment, tmp_path): + """A relative `[backup.store.filesystem] path` is made absolute at + config load, against the shell's directory, so the daemonized child + (whose working directory is `/`) opens the same root the operator + named: the root beside the config gets the identity file and nothing + appears under `/`.""" + d = sqlite_deployment + os.mkdir(os.path.join(d.dir, "rel-backups"), 0o700) + run_dir = os.path.join(d.dir, "run") + with open(d.config) as f: + text = f.read() + import re + text, n = re.subn(r'(?m)^[ \t]*#?[ \t]*run_dir[ \t]*=.*$', f'run_dir = "{run_dir}"', text, count=1) + assert n == 1, text + with open(d.config, "w") as f: + f.write(text) + _append(d.config, '[backup.store]\nkind = "filesystem"\n[backup.store.filesystem]\npath = "rel-backups"\n') + res = subprocess.run( + [EXTENDDB_BINARY, "serve", "--config", d.config], cwd=d.dir, capture_output=True, text=True, + stdin=subprocess.DEVNULL, timeout=60, + ) + try: + assert res.returncode == 0, res.stdout + res.stderr + # Absolute at load; the banner spells it as the rows do (the + # canonical directory), the configured spelling beside it only + # where the two differ. + absolute = os.path.join(d.dir, "rel-backups") + canonical = os.path.realpath(absolute) + configured = f" (configured as {absolute})" if canonical != absolute else "" + assert f"backup store: filesystem {canonical}{configured}\n" in res.stdout, res.stdout + _wait_health(d.port, lambda h: h.get("components", {}).get("backup_store", {}).get("status") == "healthy") + assert os.path.isfile(os.path.join(d.dir, "rel-backups", "extenddb-store.json")) + assert not os.path.exists("/rel-backups") + finally: + # Stop the daemon whatever the outcome above; a daemon that never + # started has no PID file and `stop` says so. + subprocess.run( + [EXTENDDB_BINARY, "stop", "--config", d.config], cwd=d.dir, capture_output=True, text=True, + stdin=subprocess.DEVNULL, timeout=60, + ) + deadline = time.monotonic() + 30 + while time.monotonic() < deadline: + try: + _health(d.port) + except Exception: # noqa: BLE001 - the listener is gone + break + time.sleep(0.25) + + def test_a_catalog_re_created_after_destroy_warns_about_the_old_default_root(self, sqlite_deployment, tmp_path): + """`destroy` then `init --overwrite` on the same directory: the default + root's identity file names the old catalog. The server starts with the + store unconfigured and a warning naming the file (not degraded, not a + startup error), CreateBackup takes the in-catalog path with its + deprecation warning, and `destroy`'s output named `backup sync + --adopt-store`.""" + d = sqlite_deployment + server = _Server(d.config, str(tmp_path / "serve.log")) + try: + _wait_health(d.port, lambda h: "backup_store" in h.get("components", {}), server=server) + finally: + server.stop() + identity = os.path.join(d.dir, "backups", "extenddb-store.json") + with open(identity) as f: + old_catalog = json.load(f)["catalog_id"] + res = d.run("destroy", "--yes") + assert res.returncode == 0, res.stdout + res.stderr + assert "backup sync --adopt-store" in res.stdout, res.stdout + assert os.path.isfile(identity) + env = os.environ.copy() + env["EXTENDDB_ADMIN_PASSWORD"] = ADMIN_PASSWORD + subprocess.run( + [EXTENDDB_BINARY, "init", "--backend", "sqlite", "--config", d.config, "--overwrite"], + cwd=d.dir, capture_output=True, text=True, stdin=subprocess.DEVNULL, timeout=60, check=True, env=env, + ) + lh._patch_config_port(d.config, d.port) + server = _Server(d.config, str(tmp_path / "serve2.log")) + try: + health = _wait_health(d.port, lambda h: "components" in h, server=server) + assert health == {"status": "healthy", "components": {}}, health + log = server.log_text() + assert "failed its check" in log and "extenddb-store.json names catalog" in log, log + assert old_catalog in log and "destroy" in log and "unconfigured" in log, log + creds = _provision(d.port, d.cert, ADMIN_PASSWORD) + ddb = _client(d.port, creds) + _create_table(ddb, "Music") + details = ddb.create_backup(TableName="Music", BackupName="nightly")["BackupDetails"] + assert details["BackupStatus"] in ("CREATING", "AVAILABLE") + assert "configure [backup.store] to write backups to a backup store" in server.log_text() + finally: + server.stop() + + def test_backup_purge_deletes_each_rows_prefix_and_row_and_leaves_the_rest(self, sqlite_deployment, tmp_path): + """Rows of every status seeded in the catalog with objects under their + prefixes: purge deletes each row's objects and the row outright; a + prefix the catalog has no row for, the identity file and `exports/` + stay; `backup list` shows the rows before and none after. + """ + try: + # The catalog's schema needs a SQLite newer than some hosts' + # stdlib module links; the wheel carries a current library. + import pysqlite3 as sqlite3 + except ImportError: + import sqlite3 + + d = sqlite_deployment + # The shortest window the config accepts (three of the 15 s the jobs + # heartbeat at), so a purge that waited on a dead job would fail + # this test in 90 s rather than the default 600. + _append(d.config, "[backup]\nstale_job_after_secs = 45\n") + server = _Server(d.config, str(tmp_path / "serve.log")) + try: + _wait_health(d.port, lambda h: "backup_store" in h.get("components", {}), server=server) + finally: + server.stop() + root = os.path.join(d.dir, "backups") + with open(os.path.join(root, "extenddb-store.json")) as f: + store_id = json.load(f)["store_id"] + source_table = json.dumps({ + "table_name": "Music", "table_id": "t-1", "table_arn": "arn:aws:dynamodb:us-east-1:123456789012:table/Music", + "key_schema": [{"AttributeName": "pk", "KeyType": "HASH"}], + "attribute_definitions": [{"AttributeName": "pk", "AttributeType": "S"}], + "billing_mode": "PAY_PER_REQUEST", "provisioned_throughput": None, "on_demand_throughput": None, + "global_secondary_indexes": [], "local_secondary_indexes": [], "vector_indexes": None, + "stream_specification": None, "time_to_live": None, "sse_description": None, + "item_count": 0, "table_size_bytes": 0, "table_creation_date_time": 0.0, + }) + account = "123456789012" + rows = [ + ("01791976431438-00000001", "AVAILABLE"), + ("01791976431439-00000002", "FAILED"), + ("01791976431440-00000003", "DELETING"), + # A CREATING row whose owner is dead (its heartbeat is old). + ("01791976431441-00000004", "CREATING"), + # A FAILED and a CREATING row whose heartbeats are young (five + # seconds old): the stopped server's jobs wrote them, and under + # the serve lock those jobs are dead, so the command neither + # waits on them nor leaves them. + ("01791976431442-00000005", "FAILED"), + ("01791976431443-00000006", "CREATING"), + ] + young = {"01791976431442-00000005", "01791976431443-00000006"} + + def prefix(backup_id): + return os.path.join(root, account, "Music", backup_id) + + def private_dirs(path): + # Every level 0700: the store refuses a directory under its root + # that another user can reach (`makedirs` leaves the intermediate + # levels at the umask's mode). + parts = [] + while not os.path.isdir(path): + parts.append(path) + path = os.path.dirname(path) + for p in reversed(parts): + os.mkdir(p, 0o700) + + conn = sqlite3.connect(d.db_path, timeout=10) + try: + for backup_id, status in rows: + arn = f"arn:aws:dynamodb:us-east-1:{account}:table/Music/backup/{backup_id}" + heartbeat = ( + "strftime('%Y-%m-%dT%H:%M:%fZ', 'now', '-5 seconds')" if backup_id in young + else "'2026-10-01T00:00:00.000Z'" + ) + conn.execute( + "INSERT INTO backup_index (account_id, backup_id, table_name, backup_arn, backup_name, status, " + "store_kind, store_root, store_id, size_bytes, item_count, created_at, source_table_details_json, " + "owner_generation, heartbeat_at) VALUES (?, ?, 'Music', ?, 'nightly', ?, 'filesystem', ?, ?, 1, 1, " + f"'2026-10-01T00:00:00.000Z', ?, 7, {heartbeat})", + (account, backup_id, arn, status, root, store_id, source_table), + ) + # A restore the stopped server left RUNNING (its target CREATING): + # under the serve lock its owner is dead, and the command resolves + # it before the wait instead of waiting on it. + conn.execute( + "INSERT INTO accounts (account_id, account_name) SELECT ?, 'purge-test' WHERE NOT EXISTS " + "(SELECT 1 FROM accounts WHERE account_id = ?)", + (account, account), + ) + conn.execute( + "INSERT INTO tables (account_id, table_name, key_schema, attribute_definitions, table_status, " + "table_arn, table_id) VALUES (?, 'Restored', '[]', '[]', 'CREATING', " + "'arn:aws:dynamodb:us-east-1:123456789012:table/Restored', 'tid-restored')", + (account,), + ) + conn.execute( + "INSERT INTO table_restores (table_id, source_backup_arn, restore_date_time, status, kind, " + "owner_generation, heartbeat_at) VALUES ('tid-restored', " + "'arn:aws:dynamodb:us-east-1:123456789012:table/Music/backup/01791976431400-00000000', " + "'2026-10-01T00:00:00.000Z', 'RUNNING', 'legacy', 9, '2026-10-01T00:00:00.000Z')", + ) + conn.commit() + finally: + conn.close() + for backup_id, status in rows: + private_dirs(os.path.join(prefix(backup_id), "data")) + # A CREATING backup has its data files and no manifest yet (the + # writer puts the manifest last), so nothing adopts a prefix a + # failure between purge's two steps leaves. + names = ["data/00001.json.gz"] if status == "CREATING" else ["extenddb-manifest.json", "data/00001.json.gz"] + for name in names: + with open(os.path.join(prefix(backup_id), name), "wb") as f: + f.write(b"x") + # A prefix with no row (another catalog's, or a copy) and exports/. + orphan = prefix("01791976431499-0000beef") + private_dirs(orphan) + with open(os.path.join(orphan, "extenddb-manifest.json"), "wb") as f: + f.write(b"x") + private_dirs(os.path.join(root, "exports", "e-1")) + with open(os.path.join(root, "exports", "e-1", "manifest-summary.json"), "wb") as f: + f.write(b"x") + + res = d.run("backup", "list") + assert res.returncode == 0, res.stdout + res.stderr + for backup_id, status in rows: + assert backup_id in res.stdout and status in res.stdout, res.stdout + assert f"store_root={root}" in res.stdout and f"store_id={store_id}" in res.stdout, res.stdout + assert "6 row(s)" in res.stdout, res.stdout + res = d.run("backup", "list", "--failed") + assert res.returncode == 0 and "2 row(s)" in res.stdout and "FAILED" in res.stdout, res.stdout + + # The dry run prints the plan, changes nothing and exits 0. + res = d.run("backup", "purge") + assert res.returncode == 0, res.stdout + res.stderr + assert "Dry run: nothing was deleted" in res.stdout and "--confirm" in res.stdout, res.stdout + for backup_id, _ in rows: + assert os.path.isfile(os.path.join(prefix(backup_id), "data", "00001.json.gz")), "nothing deleted without --confirm" + # `--config` after the subcommand is accepted too. + res = subprocess.run( + [EXTENDDB_BINARY, "backup", "purge", "--config", d.config], cwd=d.dir, capture_output=True, text=True, + stdin=subprocess.DEVNULL, timeout=120, + ) + assert res.returncode == 0 and "Dry run" in res.stdout, res.stdout + res.stderr + + res = d.run("backup", "purge", "--confirm") + assert res.returncode == 0, res.stdout + res.stderr + assert "6 backup(s) deleted, 0 left" in res.stdout, res.stdout + # The interrupted restore was resolved before the wait, which then + # had nothing to wait for (the default window would be 600 s); the + # young FAILED and CREATING rows were neither waited on (the window + # is zero under the serve lock) nor left. + assert "removed the restore target Restored" in res.stdout, res.stdout + assert "Waiting for running jobs" not in res.stdout, "purge waited on a dead job: " + res.stdout + for backup_id, _ in rows: + assert not os.path.exists(prefix(backup_id)), f"{backup_id}'s prefix is gone" + assert f"deleted arn:aws:dynamodb:us-east-1:{account}:table/Music/backup/{backup_id}" in res.stdout + assert os.path.isfile(os.path.join(orphan, "extenddb-manifest.json")), "a prefix with no row stays" + assert os.path.isfile(os.path.join(root, "exports", "e-1", "manifest-summary.json")), "exports/ stays" + assert os.path.isfile(os.path.join(root, "extenddb-store.json")), "the identity file stays" + out = res.stdout + res.stderr + assert "extenddb::audit" in out and out.count("deleted backup objects and catalog row") == 6, out + conn = sqlite3.connect(d.db_path, timeout=10) + try: + assert conn.execute("SELECT count(*) FROM backup_index").fetchone()[0] == 0, "rows gone, no tombstones" + assert conn.execute("SELECT count(*) FROM backup_maintenance").fetchone()[0] == 0, "the claim is released" + assert conn.execute("SELECT count(*) FROM tables WHERE table_name = 'Restored'").fetchone()[0] == 0, \ + "the dead restore's target is gone" + finally: + conn.close() + res = d.run("backup", "list") + assert res.returncode == 0 and "(none)" in res.stdout, res.stdout + + # The wait's limit and its refusal: purge waits the window twice, + # refuses naming the job, releases the claim and deletes nothing. + # The lever is a RUNNING restore of kind `manifest` (a later release's + # job, which this release's reconciler leaves for its resumer): the + # wait lists RUNNING records by status, with no age, so a record no + # heartbeat ages out of holds it. Beside it a FAILED row whose + # heartbeat is stamped 120 s ahead of the clock (a backward clock + # step): under the zero window the serve lock gives every owner is + # dead whatever its heartbeat says, so the row is not waited on, and + # once the restore record is gone it is deleted outright. The rule + # the wait applies on the other backends, a FAILED row with a + # heartbeat within the window held while a job that has not yet seen + # its claim lost may still write under its prefix, is tested on + # PostgreSQL in `crates/storage-postgres/tests/backup_restore.rs` + # (`the_wait_passes_a_failed_record_and_holds_on_a_failed_row_with_a_live_heartbeat`). + def seed_manifest_restore(): + conn = sqlite3.connect(d.db_path, timeout=10) + try: + conn.execute( + "INSERT INTO tables (account_id, table_name, key_schema, attribute_definitions, table_status, " + "table_arn, table_id) VALUES (?, 'Resumed', '[]', '[]', 'CREATING', " + "'arn:aws:dynamodb:us-east-1:123456789012:table/Resumed', 'tid-resumed')", + (account,), + ) + conn.execute( + "INSERT INTO table_restores (table_id, source_backup_arn, restore_date_time, status, kind, " + "owner_generation, heartbeat_at) VALUES ('tid-resumed', " + "'arn:aws:dynamodb:us-east-1:123456789012:table/Music/backup/01791976431400-00000000', " + "'2026-10-01T00:00:00.000Z', 'RUNNING', 'manifest', 9, strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))", + ) + conn.commit() + finally: + conn.close() + + def remove_manifest_restore(): + conn = sqlite3.connect(d.db_path, timeout=10) + try: + conn.execute("DELETE FROM table_restores WHERE table_id = 'tid-resumed'") + conn.execute("DELETE FROM tables WHERE table_id = 'tid-resumed'") + conn.commit() + finally: + conn.close() + + live_failed = "01791976431450-00000005" + seed_manifest_restore() + conn = sqlite3.connect(d.db_path, timeout=10) + try: + conn.execute( + "INSERT INTO backup_index (account_id, backup_id, table_name, backup_arn, backup_name, status, " + "store_kind, store_root, store_id, size_bytes, item_count, created_at, source_table_details_json, " + "owner_generation, heartbeat_at) VALUES (?, ?, 'Music', ?, 'nightly', 'FAILED', 'filesystem', ?, ?, 1, 1, " + "'2026-10-01T00:00:00.000Z', ?, 7, strftime('%Y-%m-%dT%H:%M:%fZ', 'now', '+120 seconds'))", + (account, live_failed, f"arn:aws:dynamodb:us-east-1:{account}:table/Music/backup/{live_failed}", root, + store_id, source_table), + ) + conn.commit() + finally: + conn.close() + private_dirs(prefix(live_failed)) + with open(os.path.join(prefix(live_failed), "extenddb-manifest.json"), "wb") as f: + f.write(b"x") + started = time.monotonic() + res = d.run("backup", "purge", "--confirm") + waited = time.monotonic() - started + assert res.returncode != 0, res.stdout + res.stderr + out = res.stdout + res.stderr + assert f"Waiting for running jobs to finish: a restore into Resumed (account {account}) is RUNNING" in out, out + assert "the FAILED backup" not in out, "a heartbeat ahead of the clock is not a live owner under the zero window: " + out + assert "jobs were still running after waiting" in out and "(2 * stale_job_after_secs)" in out, out + assert waited >= 90, f"the wait is twice the window: {waited:.1f} s" + assert os.path.isfile(os.path.join(prefix(live_failed), "extenddb-manifest.json")), "nothing deleted" + conn = sqlite3.connect(d.db_path, timeout=10) + try: + assert conn.execute("SELECT count(*) FROM backup_index").fetchone()[0] == 1 + assert conn.execute("SELECT count(*) FROM backup_maintenance").fetchone()[0] == 0, "the claim is released" + assert conn.execute("SELECT status FROM table_restores WHERE table_id = 'tid-resumed'").fetchone()[0] \ + == "RUNNING", "a manifest restore is left for its resumer" + finally: + conn.close() + # With the restore gone, the FAILED row stamped ahead of the clock is + # deleted outright, with no wait: under the serve lock its owner is + # dead whatever its heartbeat says. + remove_manifest_restore() + res = d.run("backup", "purge", "--confirm") + assert res.returncode == 0, res.stdout + res.stderr + assert "Waiting for running jobs" not in res.stdout, res.stdout + assert "1 backup(s) deleted, 0 left" in res.stdout, res.stdout + assert not os.path.exists(prefix(live_failed)) + conn = sqlite3.connect(d.db_path, timeout=10) + try: + assert conn.execute("SELECT count(*) FROM backup_index").fetchone()[0] == 0 + finally: + conn.close() + + # A signal while the command waits releases the claim at once and + # deletes nothing; a SIGKILL leaves the claim's row, which the next + # server start clears, so CreateBackup is not refused for the + # window. The lever is the RUNNING manifest restore again, beside a + # FAILED row that must stay. + waited_on = "01791976431460-00000007" + seed_manifest_restore() + conn = sqlite3.connect(d.db_path, timeout=10) + try: + conn.execute( + "INSERT INTO backup_index (account_id, backup_id, table_name, backup_arn, backup_name, status, " + "store_kind, store_root, store_id, size_bytes, item_count, created_at, source_table_details_json, " + "owner_generation, heartbeat_at) VALUES (?, ?, 'Music', ?, 'nightly', 'FAILED', 'filesystem', ?, ?, 1, 1, " + "'2026-10-01T00:00:00.000Z', ?, 7, '2026-10-01T00:00:00.000Z')", + (account, waited_on, f"arn:aws:dynamodb:us-east-1:{account}:table/Music/backup/{waited_on}", root, + store_id, source_table), + ) + conn.commit() + finally: + conn.close() + private_dirs(prefix(waited_on)) + with open(os.path.join(prefix(waited_on), "extenddb-manifest.json"), "wb") as f: + f.write(b"x") + + def waiting_purge(): + proc = subprocess.Popen( + [EXTENDDB_BINARY, "backup", "--config", d.config, "purge", "--confirm"], cwd=d.dir, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, stdin=subprocess.DEVNULL, text=True, + ) + lines = [] + deadline = time.monotonic() + 60 + while time.monotonic() < deadline: + line = proc.stdout.readline() + if not line: + break + lines.append(line) + if "Waiting for running jobs to finish" in line: + return proc, lines + proc.kill() + raise AssertionError("purge did not reach its wait: " + "".join(lines)) + + def claim_rows(): + conn = sqlite3.connect(d.db_path, timeout=10) + try: + return conn.execute("SELECT count(*) FROM backup_maintenance").fetchone()[0] + finally: + conn.close() + + proc, lines = waiting_purge() + assert claim_rows() == 1, "the claim is held during the wait" + signalled = time.monotonic() + proc.send_signal(signal.SIGTERM) + rest = proc.communicate(timeout=30)[0] + out = "".join(lines) + rest + released_after = time.monotonic() - signalled + assert proc.returncode != 0, out + assert "SIGTERM received" in out and "stopped before any deletion: SIGTERM received while waiting" in out, out + assert "The maintenance claim is released" in out, out + assert released_after < 5, f"the signal ended the wait at once, not at the limit: {released_after:.1f} s" + assert claim_rows() == 0, "the claim is released on SIGTERM" + assert os.path.isfile(os.path.join(prefix(waited_on), "extenddb-manifest.json")), "nothing deleted" + + proc, lines = waiting_purge() + proc.kill() + proc.communicate(timeout=30) + assert claim_rows() == 1, "a killed purge leaves its claim's row" + # A second purge right after it proceeds to its wait: under the serve + # lock the killed holder is dead whatever its heartbeat says, so the + # claim is taken, not refused for the stale window. + proc, lines = waiting_purge() + out = "".join(lines) + assert "Took the maintenance claim" in out and "is held by" not in out, out + proc.kill() + proc.communicate(timeout=30) + assert claim_rows() == 1, "the second killed purge leaves its claim's row" + # The server starts on the file and clears the row; the first + # CreateBackup is accepted at once (it would be LimitExceeded for + # the 45 s window otherwise). + server = _Server(d.config, str(tmp_path / "serve-after-kill.log")) + try: + _wait_health(d.port, lambda h: "backup_store" in h.get("components", {}), server=server) + assert claim_rows() == 0, "the server start cleared the left-behind claim" + assert "maintenance claim held by extenddb backup purge" in server.log_text() \ + and "was left behind; cleared" in server.log_text(), server.log_text() + creds = _provision(d.port, d.cert, ADMIN_PASSWORD) + ddb = _client(d.port, creds) + _create_table(ddb, "Music") + details = ddb.create_backup(TableName="Music", BackupName="after-kill")["BackupDetails"] + assert details["BackupStatus"] in ("CREATING", "AVAILABLE"), details + assert "LimitExceededException" not in server.log_text(), server.log_text() + finally: + server.stop() + conn = sqlite3.connect(d.db_path, timeout=10) + try: + conn.execute("DELETE FROM backup_index") + conn.commit() + finally: + conn.close() + remove_manifest_restore() + # The server's reconciler may already have deleted the stale FAILED + # row's objects (its heartbeat is old): the prefix is gone or ours to + # remove. + shutil.rmtree(prefix(waited_on), ignore_errors=True) + + # The server starts again against the purged root, healthy. + server = _Server(d.config, str(tmp_path / "serve2.log")) + try: + health = _wait_health(d.port, lambda h: "backup_store" in h.get("components", {}), server=server) + assert health["components"]["backup_store"]["status"] == "healthy", health + finally: + server.stop() + + def test_remove_target_creates_neither_the_default_root_nor_its_identity_file(self, sqlite_deployment): + # `backup remove-target` needs no store: on a fresh deployment it + # refuses a table no restore record names and leaves the default root + # as it found it, missing, then present and empty. + d = sqlite_deployment + root = os.path.join(d.dir, "backups") + res = d.run("backup", "remove-target", "--account", "000000000000", "nosuch") + assert res.returncode == 1, res.stdout + res.stderr + assert "nosuch (account 000000000000) is not a restore target" in res.stderr, res.stderr + assert not os.path.exists(root), "remove-target created the default root" + os.mkdir(root, 0o700) + res = d.run("backup", "remove-target", "--account", "000000000000", "nosuch") + assert res.returncode == 1, res.stdout + res.stderr + assert "is not a restore target" in res.stderr, res.stderr + assert os.listdir(root) == [], "remove-target wrote into the default root" + + def test_backup_list_and_purge_on_an_empty_store_and_the_serve_lock(self, sqlite_deployment, tmp_path): + d = sqlite_deployment + # Create the default root with a server once, then stop it. + # Before any server: `list` is a read that creates neither the default + # root nor its identity file, and says the root does not exist yet. + res = d.run("backup", "list") + assert res.returncode == 0, res.stdout + res.stderr + assert "does not exist yet" in res.stdout and "(none)" in res.stdout, res.stdout + assert not os.path.exists(os.path.join(d.dir, "backups")), "a read creates nothing" + # The dry run is the same read: it prints an empty plan naming the + # reason and exits 0. + res = d.run("backup", "purge") + assert res.returncode == 0, res.stdout + res.stderr + assert "Store: none (the default root" in res.stdout and "does not exist yet" in res.stdout, res.stdout + assert "(none)" in res.stdout and "Dry run: nothing was deleted" in res.stdout, res.stdout + assert not os.path.exists(os.path.join(d.dir, "backups")), "a read creates nothing" + # The root created by hand, empty: `list` and the dry run read it and + # leave it without an identity file. + os.mkdir(os.path.join(d.dir, "backups"), 0o700) + res = d.run("backup", "list") + assert res.returncode == 0, res.stdout + res.stderr + assert "has no extenddb-store.json yet" in res.stdout and "(none)" in res.stdout, res.stdout + res = d.run("backup", "purge") + assert res.returncode == 0, res.stdout + res.stderr + assert "Dry run: nothing was deleted" in res.stdout and "no extenddb-store.json yet" in res.stdout, res.stdout + assert not os.path.exists(os.path.join(d.dir, "backups", "extenddb-store.json")), "a read creates nothing" + server = _Server(d.config, str(tmp_path / "serve.log")) + try: + _wait_health(d.port, lambda h: "backup_store" in h.get("components", {}), server=server) + # While the server runs, `list` and the dry run are reads beside + # it (no serve lock); `purge --confirm` opens the catalog as a + # server does and refuses. + res = d.run("backup", "list") + assert res.returncode == 0, res.stdout + res.stderr + assert "backups in the filesystem store" in res.stdout and "(none)" in res.stdout, res.stdout + res = d.run("backup", "purge") + assert res.returncode == 0, res.stdout + res.stderr + assert "Dry run: nothing was deleted" in res.stdout, res.stdout + res = d.run("backup", "purge", "--confirm") + # The serve lock held is the catalog refusal `serve` gives for + # it: exit 6, the code `migrate` uses for a locked catalog. + assert res.returncode == 6, res.stdout + res.stderr + assert "another extenddb process" in (res.stdout + res.stderr), res.stdout + res.stderr + assert server.proc.poll() is None, "the server kept running" + finally: + server.stop() + res = d.run("backup", "list") + assert res.returncode == 0, res.stdout + res.stderr + assert "backups in the filesystem store" in res.stdout and "(none)" in res.stdout, res.stdout + res = d.run("backup", "list", "--failed") + assert res.returncode == 0 and "(none)" in res.stdout, res.stdout + res.stderr + res = d.run("backup", "purge") + assert res.returncode == 0, res.stdout + res.stderr + assert "Dry run: nothing was deleted" in res.stdout, res.stdout + res = d.run("backup", "purge", "--confirm") + assert res.returncode == 0, res.stdout + res.stderr + assert "0 backup(s) deleted" in res.stdout, res.stdout + assert "Took the maintenance claim" in res.stdout, res.stdout + assert os.path.isfile(os.path.join(d.dir, "backups", "extenddb-store.json")) + # `destroy` names the store as left untouched. + res = d.run("destroy", "--yes") + assert res.returncode == 0, res.stdout + res.stderr + assert "backup store ([backup.store]) is left untouched" in res.stdout, res.stdout + assert os.path.isfile(os.path.join(d.dir, "backups", "extenddb-store.json")) + + +# --- PostgreSQL ----------------------------------------------------------------------- + + +def _pg_build() -> bool: + if not os.path.isfile(EXTENDDB_BINARY): + return False + res = subprocess.run([EXTENDDB_BINARY, "version"], capture_output=True, text=True, timeout=30) + return "(postgres)" in res.stdout + + +@pytest.fixture() +def pg_env(request): + if not _pg_build(): + pytest.skip("EXTENDDB_BINARY is not a PostgreSQL build") + # `cli_env` requires PostgreSQL and a binary; it is requested only once + # the build is known to be the PostgreSQL one, so the other build skips. + cli_env = request.getfixturevalue("cli_env") + result = lh._run_extenddb( + "init", *lh._init_args(cli_env), config=cli_env["config_path"], + env_override={"EXTENDDB_ADMIN_PASSWORD": ADMIN_PASSWORD}, + ) + assert result.returncode == 0, result.stdout + result.stderr + lh._patch_config_port(cli_env["config_path"], cli_env["port"]) + return cli_env + + +class TestPostgres: + def test_without_a_store_there_is_no_component_and_create_backup_warns(self, pg_env, tmp_path): + server = _Server(pg_env["config_path"], str(tmp_path / "serve.log")) + try: + health = _wait_health(pg_env["port"], lambda h: "components" in h, server=server) + assert health == {"status": "healthy", "components": {}}, health + log = server.log_text() + assert "backup store: none configured" in log, log + creds = _provision(pg_env["port"], os.path.join(pg_env["tls_dir"], "cert.pem"), ADMIN_PASSWORD) + ddb = _client(pg_env["port"], creds) + _create_table(ddb, "Music") + arn = ddb.create_backup(TableName="Music", BackupName="nightly")["BackupDetails"]["BackupArn"] + assert "configure [backup.store] to write backups to a backup store" in server.log_text() + # The in-catalog backup is served. + assert ddb.describe_backup(BackupArn=arn)["BackupDescription"]["BackupDetails"]["BackupArn"] == arn + listed = ddb.list_backups(TableName="Music")["BackupSummaries"] + assert [b["BackupArn"] for b in listed] == [arn] + finally: + server.stop() + + def test_an_s3_store_without_a_spool_stops_startup(self, pg_env, tmp_path): + _append(pg_env["config_path"], '[backup.store]\nkind = "s3"\n[backup.store.s3]\nbucket = "b"\n') + server = _Server(pg_env["config_path"], str(tmp_path / "serve.log"), env=_s3_env()) + try: + assert server.wait_exit() != 0 + log = server.log_text() + assert "backup.spool_path is required" in log, log + finally: + server.stop() + + def test_a_filesystem_store_is_healthy_and_a_bad_bucket_name_stops_startup(self, pg_env, tmp_path): + root = tmp_path / "backups" + root.mkdir(mode=0o700) + _append( + pg_env["config_path"], + f'[backup.store]\nkind = "filesystem"\n[backup.store.filesystem]\npath = "{root}"\n', + ) + server = _Server(pg_env["config_path"], str(tmp_path / "serve.log")) + try: + health = _wait_health(pg_env["port"], lambda h: "components" in h, server=server) + assert health["components"]["backup_store"] == {"status": "healthy", "kind": "filesystem"} + with open(root / "extenddb-store.json") as f: + assert json.load(f)["backend"] == "postgres" + finally: + server.stop() + # A bucket name the SDK routes to S3 Express is a configuration error + # naming the bucket. + with open(pg_env["config_path"]) as f: + text = f.read() + text = text[: text.index("[backup.store]")] + with open(pg_env["config_path"], "w") as f: + f.write(text) + _append( + pg_env["config_path"], + f'[backup]\nspool_path = "{tmp_path / "spool"}"\n' + '[backup.store]\nkind = "s3"\n[backup.store.s3]\nbucket = "my-bucket--usw2-az1--x-s3"\n', + ) + server = _Server(pg_env["config_path"], str(tmp_path / "serve2.log"), env=_s3_env()) + try: + assert server.wait_exit() != 0 + log = server.log_text() + assert "cannot be opened" in log and "my-bucket--usw2-az1--x-s3" in log, log + finally: + server.stop() + + def test_a_refused_catalog_ends_the_backup_commands_with_the_serve_exit_codes(self, pg_env, tmp_path): + """Every `backup` command opens the catalog through the factory and + holds a serving registration, so a catalog the factory refuses ends + it with the message and exit code `serve` gives: 6 while a migration + holds the serving key (naming the migration's session, without the + server's advice to start the server), 3 on a catalog behind the + binary (with the `extenddb migrate --yes` advice).""" + import psycopg2 + + config = pg_env["config_path"] + + def command(*args): + return lh._run_extenddb(*args, config=config, check=False, timeout=60) + + # A migration's hold of the serving key, as `migrate --yes` takes it: + # the exclusive session lock on (ExtendDB's namespace, serving key). + conn = psycopg2.connect(lh.PG_ADMIN_CONN + "/" + pg_env["db_name"]) + conn.autocommit = True + try: + with conn.cursor() as cur: + cur.execute("SET application_name = 'holds the serving key'") + cur.execute("SELECT pg_advisory_lock(%s, 2)", (0x00454442,)) + for args in ( + ("backup", "list"), + ("backup", "purge"), + ("backup", "purge", "--confirm"), + ("backup", "remove-target", "--account", "000000000000", "nosuch"), + ("backup", "sync"), + ): + res = command(*args) + out = res.stdout + res.stderr + assert res.returncode == 6, f"{args}: {out}" + assert "a migration is running on this catalog" in out, out + assert "holds the serving key" in out, out + assert "start the server" not in out, out + assert "Retry once it has finished" in out, out + finally: + conn.close() + + res = command("migrate", "--forget", "6", *lh._pg_args()) + assert res.returncode == 0, res.stdout + res.stderr + for args in ( + ("backup", "list"), + ("backup", "purge", "--confirm"), + ("backup", "remove-target", "--account", "000000000000", "nosuch"), + ("backup", "sync"), + ): + res = command(*args) + out = res.stdout + res.stderr + assert res.returncode == 3, f"{args}: {out}" + assert "the catalog is at migration 5" in out, out + assert "extenddb migrate --config" in out and "--yes" in out, out + res = command("migrate", "--yes", *lh._pg_args()) + assert res.returncode == 0, res.stdout + res.stderr + res = command("backup", "list") + assert res.returncode == 0, res.stdout + res.stderr diff --git a/tests/test_mongodb_backup_restore.py b/tests/test_mongodb_backup_restore.py index c68c34cdb..e3209023c 100644 --- a/tests/test_mongodb_backup_restore.py +++ b/tests/test_mongodb_backup_restore.py @@ -5,7 +5,10 @@ The table operations are exercised through the DynamoDB API. These tests are gated by the MongoDB runner because the backup metadata being checked is part -of the MongoDB implementation; other backends retain the shared API tests. +of the MongoDB implementation; other backends retain the shared API tests. The +runner's server has a filesystem backup store, so CreateBackup is the +store-backed job and RestoreTableFromBackup the store-backed restore; the +cells at the end meet the MongoDB target's own rules through that path. """ from __future__ import annotations @@ -26,10 +29,27 @@ def mongodb_container() -> str: container = os.environ.get("EXTENDDB_TEST_MONGODB_CONTAINER", "").strip() if not container: pytest.skip("requires devtools/run-mongodb-tests") + # These cells read the in-catalog create's documents (`backups`) and + # restore its backups at once; on a server with a `[backup.store]` + # CreateBackup is the store-backed job (CREATING, then AVAILABLE in the + # store) and RestoreTableFromBackup of its backups is refused until the + # restore job lands (PR 9a), which brings these cells back there. + store = os.environ.get("EXTENDDB_TEST_BACKUP_STORE", "").strip() + if store in ("filesystem", "s3"): + pytest.skip( + f"EXTENDDB_TEST_BACKUP_STORE={store}: these cells read and restore the " + "in-catalog create's backups; the store-backed restore is PR 9a's" + ) return container +BACKUP_STORE = os.environ.get("EXTENDDB_TEST_BACKUP_STORE", "").strip() or None + + def _create_backup(client, table_name: str, backup_name: str) -> str: + """CreateBackup, waited for AVAILABLE: with the runner's filesystem store + the create is the asynchronous job, and a restore or a delete of a + CREATING backup is BackupInUseException.""" deadline = time.monotonic() + 30 while True: try: @@ -37,7 +57,8 @@ def _create_backup(client, table_name: str, backup_name: str) -> str: TableName=table_name, BackupName=backup_name, ) - return response["BackupDetails"]["BackupArn"] + arn = response["BackupDetails"]["BackupArn"] + break except ClientError as exc: if ( exc.response["Error"]["Code"] @@ -46,6 +67,8 @@ def _create_backup(client, table_name: str, backup_name: str) -> str: ): raise time.sleep(0.2) + _wait_backup_available(client, arn) + return arn def _remove_legacy_capacity_metadata(container: str, backup_arn: str) -> None: @@ -239,7 +262,16 @@ def test_restore_preserves_provisioned_throughput_and_secondary_indexes( def test_restore_legacy_backup_falls_back_and_populates_lsi( dynamodb_client, unique_table_name, mongodb_container ): - """A pre-throughput backup still restores and its LSI remains queryable.""" + """A pre-throughput backup still restores and its LSI remains queryable. + The backup edited is an in-catalog one (the legacy `backups` document), + so the cell runs only on a server without a backup store: + `devtools/run-mongodb-tests --backup-store none -- --pytest --filter + test_mongodb_backup_restore` (the CI jobs run the suite with a filesystem + store, where this cell skips). With a store, CreateBackup writes the + store and the fallback is pinned by the crate's + `restore_uses_legacy_fallback_when_capacity_metadata_is_missing`.""" + if BACKUP_STORE != "none": + pytest.skip("the legacy throughput fallback is a rule of in-catalog backups (EXTENDDB_TEST_BACKUP_STORE is not none)") source = unique_table_name restored = f"{source}-restored" dynamodb_client.create_table( @@ -310,3 +342,250 @@ def test_restore_legacy_backup_falls_back_and_populates_lsi( _cleanup(dynamodb_client, restored, source) if backup_arn: dynamodb_client.delete_backup(BackupArn=backup_arn) + + +# --- the store-backed restore's MongoDB rules --------------------------------------- +# +# These cells rewrite a backup's prefix under the runner's filesystem store +# (EXTENDDB_TEST_BACKUP_ROOT) and read the restore job's log lines from the +# server's log (EXTENDDB_TEST_SERVER_LOG); devtools/run-mongodb-tests exports +# both. A rule the backend's write path enforces (an `N` sort key beyond +# Decimal128) is met inside the job, so the restore is FAILED with that text +# as its reason and the client sees the target disappear; a rule the request +# checks (vector indexes into a backend without vector support) is a +# synchronous ValidationException before any table exists. + +ACCOUNT = "123456789012" + + +@pytest.fixture() +def backup_root(mongodb_container) -> str: + root = os.environ.get("EXTENDDB_TEST_BACKUP_ROOT", "").strip() + log = os.environ.get("EXTENDDB_TEST_SERVER_LOG", "").strip() + if not root or not log: + pytest.skip("requires devtools/run-mongodb-tests (EXTENDDB_TEST_BACKUP_ROOT, EXTENDDB_TEST_SERVER_LOG)") + return root + + +def _wait_backup_available(client, arn: str, timeout: float = 120.0) -> None: + deadline = time.monotonic() + timeout + while True: + status = client.describe_backup(BackupArn=arn)["BackupDescription"]["BackupDetails"]["BackupStatus"] + if status == "AVAILABLE": + return + assert status == "CREATING", status + assert time.monotonic() < deadline, f"backup {arn} still CREATING after {timeout} s" + time.sleep(0.2) + + +def _prefix(root: str, table: str, arn: str) -> str: + return os.path.join(root, ACCOUNT, table, arn.rsplit("/", 1)[1]) + + +def _read_manifest(prefix: str) -> dict: + with open(os.path.join(prefix, "extenddb-manifest.json")) as f: + return json.load(f) + + +def _write_manifest(prefix: str, manifest: dict) -> None: + """The manifest and its sha256 sidecar rewritten under `prefix`, private + as the store writes them, and the two DynamoDB export manifests for the + manifest's data files (one `md5Checksum` and one `itemCount` per file, + agreeing with the extenddb manifest by construction), so the prefix + stays a whole backup in the format's sense, as `tests/test_backup_job.py` + and `devtools/backup-restore-matrix` rewrite theirs.""" + import hashlib + + text = json.dumps(manifest, separators=(",", ":")).encode() + path = os.path.join(prefix, "extenddb-manifest.json") + with open(path, "wb") as f: + f.write(text) + os.chmod(path, 0o600) + sidecar = os.path.join(prefix, "extenddb-manifest.sha256") + with open(sidecar, "w") as f: + f.write(f"{hashlib.sha256(text).hexdigest()} extenddb-manifest.json\n") + os.chmod(sidecar, 0o600) + summary_path = os.path.join(prefix, "manifest-summary.json") + with open(summary_path) as f: + summary = json.load(f) + s3_prefix = summary["s3Prefix"] + files_path = os.path.join(prefix, "manifest-files.json") + with open(files_path, "w") as f: + for entry in manifest["data_files"]: + line = { + "itemCount": entry["item_count"], + "md5Checksum": entry["md5"], + "dataFileS3Key": f"{s3_prefix}/{entry['key']}" if s3_prefix else entry["key"], + } + f.write(json.dumps(line, separators=(",", ":")) + "\n") + os.chmod(files_path, 0o600) + summary["itemCount"] = sum(entry["item_count"] for entry in manifest["data_files"]) + summary["billedSizeBytes"] = manifest["backup_size_bytes"] + with open(summary_path, "w") as f: + f.write(json.dumps(summary, separators=(",", ":"))) + os.chmod(summary_path, 0o600) + + +def _write_unordered_file(prefix: str, rows: list[dict]) -> dict: + """The one data file of an unordered backup holding `rows`; its manifest + entry.""" + import base64 + import gzip + import hashlib + import io + + lines = b"".join(json.dumps({"Item": r}, separators=(",", ":")).encode() + b"\n" for r in rows) + buffer = io.BytesIO() + with gzip.GzipFile(fileobj=buffer, mode="wb", mtime=0) as gz: + gz.write(lines) + compressed = buffer.getvalue() + path = os.path.join(prefix, "data", "000001.json.gz") + for name in os.listdir(os.path.join(prefix, "data")): + os.remove(os.path.join(prefix, "data", name)) + with open(path, "wb") as f: + f.write(compressed) + os.chmod(path, 0o600) + return { + "key": "data/000001.json.gz", "item_count": len(rows), + "sha256": hashlib.sha256(compressed).hexdigest(), + "md5": base64.b64encode(hashlib.md5(compressed).digest()).decode(), + "bytes": len(compressed), "uncompressed_bytes": len(lines), "key_range": None, + } + + +def _composite_table(client, name: str) -> None: + client.create_table( + TableName=name, + AttributeDefinitions=[ + {"AttributeName": "pk", "AttributeType": "S"}, + {"AttributeName": "sk", "AttributeType": "N"}, + ], + KeySchema=[ + {"AttributeName": "pk", "KeyType": "HASH"}, + {"AttributeName": "sk", "KeyType": "RANGE"}, + ], + BillingMode="PAY_PER_REQUEST", + ) + wait_for_active(client, name) + + +def _delete_backup_once_available(client, arn: str | None) -> None: + if arn: + _wait_backup_available(client, arn) + client.delete_backup(BackupArn=arn) + + +def test_a_vector_manifest_is_refused_before_any_table(dynamodb_client, unique_table_name, backup_root): + """A backup whose manifest records a vector index (the prefix of a real + backup rewritten under the root) is refused by RestoreTableFromBackup + with the vector ValidationException, and no table of the target's name + exists afterwards: the request checks the manifest against the backend + before it registers anything.""" + client = dynamodb_client + source = unique_table_name + target = f"{source}-restored" + backup_arn = None + try: + _composite_table(client, source) + client.put_item(TableName=source, Item={"pk": {"S": "a"}, "sk": {"N": "1"}, "emb": {"L": [{"N": "0.1"}]}}) + backup_arn = _create_backup(client, source, f"{source}-backup") + _wait_backup_available(client, backup_arn) + prefix = _prefix(backup_root, source, backup_arn) + manifest = _read_manifest(prefix) + manifest["source_table"]["vector_indexes"] = [{ + "IndexName": "by-emb", "Dimensions": 3, "DistanceFunction": "COSINE", + "VectorAttribute": {"AttributeName": "emb"}, + }] + _write_manifest(prefix, manifest) + with pytest.raises(ClientError) as refused: + client.restore_table_from_backup(TargetTableName=target, BackupArn=backup_arn) + err = refused.value.response["Error"] + assert err["Code"] == "ValidationException", err + assert err["Message"] == ( + f"backup {backup_arn} has 1 vector index(es); restoring a table with " + "vector indexes is not supported by this storage backend" + ), err + with pytest.raises(client.exceptions.ResourceNotFoundException): + client.describe_table(TableName=target) + finally: + _cleanup(client, target, source) + _delete_backup_once_available(client, backup_arn) + + +def test_a_sort_key_beyond_decimal128_fails_the_restore(dynamodb_client, unique_table_name, backup_root): + """A data file holding an `N` sort key of 35 significant digits (more + than Decimal128's 34, which the sort-key column is): the restore is + accepted CREATING, the job fails the file with the write path's + ValidationException text, the target is gone, and the server log carries + the Decimal128 text as the FAILED reason.""" + client = dynamodb_client + source = unique_table_name + target = f"{source}-restored" + backup_arn = None + try: + _composite_table(client, source) + for i in range(3): + client.put_item(TableName=source, Item={"pk": {"S": "a"}, "sk": {"N": str(i)}}) + backup_arn = _create_backup(client, source, f"{source}-backup") + _wait_backup_available(client, backup_arn) + prefix = _prefix(backup_root, source, backup_arn) + manifest = _read_manifest(prefix) + long_key = "12345678901234567890123456789012345" + rows = [{"pk": {"S": "a"}, "sk": {"N": str(i)}} for i in range(3)] + rows.append({"pk": {"S": "a"}, "sk": {"N": long_key}}) + entry = _write_unordered_file(prefix, rows) + manifest["item_order"] = "unordered" + manifest["data_files"] = [entry] + manifest["backup_size_bytes"] = entry["bytes"] + manifest["source_table"]["item_count"] = len(rows) + _write_manifest(prefix, manifest) + resp = client.restore_table_from_backup(TargetTableName=target, BackupArn=backup_arn)["TableDescription"] + assert resp["TableStatus"] == "CREATING", resp + deadline = time.monotonic() + 120 + while True: + try: + status = client.describe_table(TableName=target)["Table"]["TableStatus"] + except client.exceptions.ResourceNotFoundException: + break + assert status in ("CREATING", "DELETING"), f"{target} reached {status}" + assert time.monotonic() < deadline, f"{target} still exists after 120 s" + time.sleep(0.2) + deadline = time.monotonic() + 15 + while True: + with open(os.environ["EXTENDDB_TEST_SERVER_LOG"], errors="replace") as f: + failed = [line for line in f if ": FAILED (" in line and f" into {target} " in line] + if failed or time.monotonic() > deadline: + break + time.sleep(0.25) + assert len(failed) == 1, failed + assert "Decimal128" in failed[0] and long_key in failed[0], failed[0] + assert "the target was removed" in failed[0], failed[0] + finally: + _cleanup(client, target, source) + _delete_backup_once_available(client, backup_arn) + + +def test_a_non_key_number_of_38_digits_restores_intact(dynamodb_client, unique_table_name, backup_root): + """A non-key `N` of 38 significant digits (the service's precision, past + Decimal128's) is stored as the item's text and comes back from a restore + equal to what the source serves.""" + client = dynamodb_client + source = unique_table_name + target = f"{source}-restored" + backup_arn = None + big = "12345678901234567890123456789012345678" + try: + _composite_table(client, source) + client.put_item(TableName=source, Item={"pk": {"S": "a"}, "sk": {"N": "1"}, "big": {"N": big}}) + key = {"pk": {"S": "a"}, "sk": {"N": "1"}} + served = client.get_item(TableName=source, Key=key, ConsistentRead=True)["Item"] + assert served["big"] == {"N": big}, served + backup_arn = _create_backup(client, source, f"{source}-backup") + _wait_backup_available(client, backup_arn) + client.restore_table_from_backup(TargetTableName=target, BackupArn=backup_arn) + wait_for_active(client, target) + restored = client.get_item(TableName=target, Key=key, ConsistentRead=True)["Item"] + assert restored == served, (restored, served) + finally: + _cleanup(client, target, source) + _delete_backup_once_available(client, backup_arn) diff --git a/tests/test_resource_tag_migration.py b/tests/test_resource_tag_migration.py index 4a1252ee4..a1ee3d9ef 100644 --- a/tests/test_resource_tag_migration.py +++ b/tests/test_resource_tag_migration.py @@ -293,14 +293,31 @@ def deployment(tmp_path): server.stop() +BACKUP_CATALOG_INDEX_COLUMNS = ( + "status", "kind", "store_kind", "store_root", "store_id", "owner_generation", + "heartbeat_at", "spool_host", "file_index", "items_applied", "failure_reason", + "completed_at", +) + + def _as_released_catalog(sqlite_path: str) -> None: """Make a catalog this binary created read as one a release before 1.0 left: no ledger rows (the runner reads an empty ledger as it reads a - missing one), a dotted catalog_version, and the marker row every released - baseline seeded. The runner then applies every migration, the - condition-key rewrite included, as it does on a released catalog.""" + missing one), a dotted catalog_version, the marker row every released + baseline seeded, and the baseline's shape for every table migration 1 + checks: the objects migration 6 (backup_catalog_index) adds are removed, + since no release before 1.0 had them and the check after migration 1 + refuses a `table_restores` carrying its twelve job columns. The runner + then applies every migration, the condition-key rewrite included, as it + does on a released catalog.""" conn = sqlite3.connect(sqlite_path) try: + conn.execute("DROP INDEX IF EXISTS idx_table_restores_status") + for column in BACKUP_CATALOG_INDEX_COLUMNS: + conn.execute(f"ALTER TABLE table_restores DROP COLUMN {column}") + conn.execute("DROP TABLE IF EXISTS backup_index") + conn.execute("DROP TABLE IF EXISTS backup_maintenance") + conn.execute("DELETE FROM settings WHERE key = 'catalog_id'") conn.execute("DELETE FROM extenddb_migrations") conn.execute( "UPDATE settings SET value = '0.0.3' WHERE key = 'catalog_version'"