diff --git a/.github/workflows/admin-ui.yml b/.github/workflows/admin-ui.yml new file mode 100644 index 000000000..0992f048e --- /dev/null +++ b/.github/workflows/admin-ui.yml @@ -0,0 +1,42 @@ +name: Admin UI regressions + +on: + pull_request: + paths: + - 'awa-ui/frontend/**' + - '.github/workflows/admin-ui.yml' + push: + branches: [main] + paths: + - 'awa-ui/frontend/**' + - '.github/workflows/admin-ui.yml' + workflow_dispatch: + +concurrency: + group: admin-ui-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + operations: + name: Synthetic operations browser tests + runs-on: ubuntu-latest + timeout-minutes: 10 + defaults: + run: + working-directory: awa-ui/frontend + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-node@v6 + with: + node-version: '22' + cache: npm + cache-dependency-path: awa-ui/frontend/package-lock.json + - run: npm ci + - run: npx playwright install --with-deps chromium + - run: npm run test:operations + - uses: actions/upload-artifact@v7 + if: failure() + with: + name: admin-ui-regressions-${{ github.sha }} + path: awa-ui/frontend/test-results/ + retention-days: 7 diff --git a/.github/workflows/nightly-chaos.yml b/.github/workflows/nightly-chaos.yml index 9c69c0889..a57f22aa2 100644 --- a/.github/workflows/nightly-chaos.yml +++ b/.github/workflows/nightly-chaos.yml @@ -197,6 +197,17 @@ jobs: cargo test --release --package awa --test concurrent_lifecycle_test \ test_concurrent_queue_sweep -- --exact --ignored --nocapture \ | tee artifacts/nightly/rust-queue-sweep.txt + - name: Run per-lane enqueue contention benchmark + if: always() + timeout-minutes: 5 + run: | + set -o pipefail + # Many producers batch-inserting into one queue lane. Gates the + # per-lane rate and p99 through benchmarks/baseline.json, and the + # lane-lock hold time relative to the per-row insert function. + cargo test --release --package awa --test enqueue_lane_contention_test \ + test_enqueue_lane_contention_gate -- --exact --ignored --nocapture \ + | tee artifacts/nightly/rust-enqueue-lane.txt - name: Run MVCC horizon overlap benchmark if: always() timeout-minutes: 10 diff --git a/CHANGELOG.md b/CHANGELOG.md index 065d50db1..3d25dc4c1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,8 +4,118 @@ Notable changes between releases. Detailed migration notes for storage transitio ## [Unreleased] +### Added + +- **`insert_many_skip_unique_conflicts` inserts a batch and skips the jobs + whose unique key is already claimed.** `insert_many` fails the whole batch + with a unique violation, so a producer that replays work had to insert the + batch again one job at a time, taking the lane lock once per job. The new + function returns the inserted rows in input order, with skipped jobs absent. + It needs migration v055 and returns a validation error on an older schema; + `insert_many_copy` (and Python's `insert_many_copy`) skips unique conflicts on + every schema. + +- **The maintenance leader keeps planner statistics fresh on the queue-storage + tables autovacuum never analyses.** Autovacuum does not analyse partitioned + tables, so the thirteen ring parents (`ready_entries`, `done_entries`, + `leases`, ...) had no inherited statistics, and the two small head tables + (`queue_claim_heads`, `queue_enqueue_heads`) never reach the autoanalyze + threshold, so the planner priced lane joins against them at its defaults + while they drove joins over hundreds of thousands of retained rows. The + leader now runs `ANALYZE` on them when it is elected and then every five + minutes. `ClientBuilder::statistics_refresh_interval` (Python + `statistics_refresh_interval_ms`) changes the cadence; zero disables it. One + pass takes under a second on a small ring and a few seconds on a large one. + It analyses one relation per transaction with a 2 s `lock_timeout` (a call that + times out waiting for a lock is retried next interval and recorded as + `skipped_lock`) and never makes prune wait: a prune attempt that + lands while ANALYZE samples the oldest slot's tables returns `blocked` and + retries on its normal backoff. Results are in + `awa.maintenance.analyze.attempts` and `awa.maintenance.analyze.duration` + (scope `heads`, `ring_parent` or `pass`; outcome `ok`, `skipped_privilege`, + `skipped_lock`, `unavailable` or `failed`). +- **Migration v054 adds `awa.analyze_queue_storage_relation(schema, relation)`, + so a runtime role that does not own the tables can refresh statistics.** + `ANALYZE` skips a table its caller may not analyse, and the documented + runtime grants do not include `MAINTAIN`, so the refresh runs through this + `SECURITY DEFINER` function owned by the migration role. It analyses only a + head table or ring parent of a schema that `awa.queue_storage_schemas()` + reports as trusted. It is not executable by `PUBLIC`, because the lock it + takes lasts until the caller's transaction ends and would block ring prune: + the migration grants `EXECUTE` to the roles that already hold INSERT, UPDATE + and DELETE on the head tables (the runtime grantees), and the function + requires the calling session user to hold them. A role created later needs + the documented `GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA awa`. Run + `awa migrate` as the schema owner (see + [database roles](docs/security/database-roles.md)). A binary running ahead of + the migration probes for the function once per pass and, like one whose + function's owner may not analyse the tables, logs one warning naming what is + missing and skips the refresh without making failing calls or affecting job + processing. Earlier binaries never call the function. + Missing EXECUTE or head-table privileges are checked before each pass, + avoiding a failing helper call for every relation. + +### Changed + +- **COPY and opt-in skip-conflict batch enqueues hold lane locks only for reservation, ready-row writes and commit.** Migration v055 adds `awa.insert_jobs_compat_batch`, which allocates job ids, claims unique keys in key order, registers lanes and prepares rows before reserving each lane's sequence range in lane order. `insert_many_copy` probes its own connection on every call and uses the function when installed, falling back to per-row inserts on older schemas. `insert_many_skip_unique_conflicts` requires the function. Generic `insert_many` keeps its single-statement per-row path on every supported schema, preserving caller transactions when one process writes to databases at different migration levels. All paths return rows in input order and share the same per-lane lock with older binaries. + ### Fixed +- **`awa.jobs` keeps running jobs visible after priority aging changes their + effective priority** ([#569](https://github.com/hardbyte/awa/issues/569)). + Migration v057 joins receipt claims and materialized leases to the exact + job attempt rather than comparing effective and lane priorities, and + excludes tombstoned and older-attempt ready rows so a job is not listed + twice. Receipt and materialized-lease rows report the claimed effective + priority. The migration only replaces the compatibility function; storage + and runtime write protocols are unchanged. +- **A queue lane is usable by every runtime role, whichever role creates it + first (migration v053).** A lane's enqueue and claim sequences belonged to + the role that first enqueued into, claimed from or promoted into it, so with + several runtime roles sharing a group role, any role other than the creator + whose default privileges did not cover it failed with `permission denied for + sequence`. That included the maintenance leader promoting into a shard + created after `enqueue_shards` was raised. Lanes are now created through two + `SECURITY DEFINER` functions per queue-storage schema, `provision_lane` and + `provision_lane_sequences`, owned by the schema owner. Each new sequence gets + the privileges of the head table it backs, explicitly rather than through + default privileges. A runtime without schema `CREATE` can therefore create + lanes itself, and `awa storage prepare-queue` is optional and may run as any + runtime role. After v053 commits, `awa migrate` gives existing lane + sequences to the schema owner and adds their missing grants, one sequence per + transaction with a 300 ms lock wait, and only where it is allowed to, trying + the effective role and then the login role. A sequence an open transaction + holds is skipped and reported as busy; one it may not change is reported + with the roles that can repair it. Neither fails the migration. The new + read-only `awa storage lane-sequences` (or + `awa.lane_sequence_problems(schema)`) lists what remains, with the statements + to run and the roles to run them as, and exits 2 while any remain; + `--repair` (or `awa.repair_lane_sequence(schema, sequence)` per sequence) + repairs what the connected role may. `awa migrate` prints a summary of the leftovers. A binary + running ahead of v053 detects that `provision_lane` is missing and creates + lanes as before, and older binaries create correctly owned lanes once v053 + has refreshed the helpers they call. See "Lane ownership and v053" in the + 0.6 to 0.7 upgrade guide. + +- **Ring prune and terminal-count rollup report `Blocked` when any of their + statements times out on a lock.** Prune sets a short `lock_timeout` (10 ms + by default) before it takes `ACCESS EXCLUSIVE` on a slot's partitions, but + only those `LOCK TABLE` statements and the final `TRUNCATE` turned a + timeout into `PruneOutcome::Blocked`. The timeout covers every later + statement in the transaction too: the post-lock liveness checks read the + lease and claim partitions, the failed-row carry inserts into the current + slot's `done_entries` and terminal-count deltas, and the fold appends + `queue_terminal_rollup_deltas` and deletes `queue_terminal_live_counts` + rows. When one of those waited out the timeout, for example behind + autovacuum truncating a relation or a hosted Postgres's own background + workers, prune failed with a `lock_not_available` (`55P03`) database error, + was logged as an error and skipped the prune backoff. A lock timeout + anywhere in a queue, lease or claim prune, or in a terminal-count rollup + slot, now rolls the whole transaction back and reports `Blocked` + (`blocked_slots` for rollup), so maintenance backs off and retries; nothing + the attempt wrote survives the rollback. A rollup pass stops at its first + blocked slot rather than waiting out the timeout again on each remaining + one. - **Batch enqueues take their lane locks in one order, so they no longer deadlock with each other.** On queue storage every enqueue holds a transaction-scoped advisory lock on its `(queue, priority, shard)` lane @@ -15,10 +125,7 @@ Notable changes between releases. Detailed migration notes for storage transitio a `QueueStorage` enqueue could deadlock with it (`40P01`). The two insert paths now sort rows by `(queue, priority, shard)`, queues compared bytewise, before they reach `awa.insert_job_compat`, the order `QueueStorage` already - used, and still return rows in input order. Job ids within a batch that - spans several lanes are now allocated lane by lane rather than in input - order, and when an `insert_many_copy` batch carries the same unique key in - more than one lane, the row in the lowest lane is the one inserted. A + used, and still return rows in input order. A transaction that enqueues with separate statements still takes locks in statement order; see "`deadlock detected` while enqueueing" in the troubleshooting guide. @@ -230,7 +337,7 @@ Notable changes between releases. Detailed migration notes for storage transitio ### Upgrading -This release adds migrations v041 to v052. Run `awa migrate` as the migrator +This release adds migrations v041 to v058. Run `awa migrate` as the migrator role; the [0.6→0.7 upgrade guide](docs/upgrade-0.6-to-0.7.md) has the details for each step. @@ -242,9 +349,9 @@ for each step. entry below has the staged expand, flip and contract steps. - **Rollout order.** Health-gated workers report ready only once the schema reaches their version, so migrate before rolling them out. Producers and - workers without a readiness gate can roll before v050. On PostgreSQL 17 and - earlier, pre-v050 binaries scan every ready partition during maintenance - once v050 is applied, so replace them promptly. + workers without a readiness gate can roll before v050 and v056. On + PostgreSQL 17 and earlier, pre-v050 binaries scan every ready partition + during maintenance once v050 is applied, so replace them promptly. - **Runtime roles no longer need `CREATE` (v049).** Provision lanes with `awa storage prepare-queue` as the migrator before deploying, and grant the runtime role `USAGE, SELECT, UPDATE` on the lane sequences. Existing @@ -434,6 +541,22 @@ for each step. `awa.migration_lock_key()` expose the runner's advisory lock key. - Python prints a diagnostic to stderr if native shutdown has stalled for five seconds. +- **The admin overview answers "is it keeping up?" (v058, + [#543](https://github.com/hardbyte/awa/pull/543)).** The maintenance leader + writes one row per queue per minute to `awa.queue_history` (depth by state, + oldest wait, and claimed/completed/failed/retried/rate-limited counts) and + one fleet row to `awa.fleet_history` (live and healthy instances, leader, + running versions), keeping 48 hours by default. Runtimes publish the + per-queue outcome counters this needs in their existing snapshots. The + overview, Queues page and queue detail classify each queue as not being + worked, falling behind, waiting long, paused, draining, steady or idle, + say what is holding waiting work back (no workers, rate limit, worker + capacity), show measured completion and arrival rates and an hour of + history, and mark rollouts, leader changes and lost instances on the chart. + `GET /api/health/queues` returns the same verdicts for alerting + (`?fail_on=critical` or `warning` answers 503 at that level). + Configure retention with `ClientBuilder::queue_history_retention` or + Python's `queue_history_retention_hours`; zero stops sampling. - **`QueueStorage::rescue_stale_heartbeats_with_dlq`, `rescue_expired_deadlines_with_dlq` and `rescue_expired_callbacks_with_dlq`** take a per-queue predicate and dead-letter exhausted attempts in the rescue @@ -595,6 +718,22 @@ for each step. session sets no `lock_timeout`, each lock wait is capped at 2 s; on a timeout (`55P03`, usually behind a long `awa.jobs` read or a backup) the migration rolls back and can be re-run. +- **Enqueueing a batch of ready jobs takes one round trip (v056, + [#522](https://github.com/hardbyte/awa/issues/522)).** + `QueueStorage::enqueue_params_batch`, and `enqueue_params_copy` below its + 2,048-row COPY threshold, send a batch of ready jobs without uniqueness + claims as one call to the new `{schema}.enqueue_ready_rows` function instead + of ten round trips. Batches with uniqueness claims or scheduled jobs still + run in a transaction but use the function for most of the work, and + reserving a lane's sequence range now writes the sequence once instead of + twice. On an earlier revision, at 800 jobs/s in 20-job calls, producer call + p50 fell 19-62% and WAL per job by about 5 bytes, with throughput unchanged. + On a schema without the function, or without `EXECUTE` on it and on + `ring_cursor`, binaries keep using the multi-statement path, which is also + shorter now, and look for the function again after a minute. If a cached + function disappears or loses EXECUTE, the same call rolls back any transaction + it owns and retries its original input once without the function; healthy + cached calls keep the one-statement path. - **Retry-heavy workloads leave far fewer dead tuples.** After the ring authority flip, awa keeps one terminal-count row per lane in `queue_terminal_live_counts` instead of up to 256, and a batch of finished @@ -685,6 +824,11 @@ for each step. ### Fixed +- Queue history retains a departing worker's final published counters once, + keeps missing intervals unknown, and attributes striped queue outcomes to + their logical dispatcher. The overview no longer treats unknown callback + rates as zero when no worker runtime serves the queue. + - **JSON floats read back from Postgres exactly.** Postgres prints `jsonb` numbers in plain decimal, and serde_json's default parser could round a long one to the neighbouring `f64`, so a value like `5.130112436774387e-96` in job diff --git a/README.md b/README.md index 5c364f6e4..9c03d19ab 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ Awa (Māori: river) fills the gap between Postgres event queues that are too narrow for real job-queue behavior and language-specific job frameworks (River, Oban, Sidekiq) that couple you to one ecosystem. If you run Rust or Python (or both) on Postgres and want priorities, cron, DLQ, and transactional enqueue without Redis or RabbitMQ, Awa is built for you. -![AWA Web UI — Jobs (dark mode)](https://raw.githubusercontent.com/hardbyte/awa/main/docs/images/awa-ui-dark.png) +![Awa admin overview during an incident: one queue is not being worked, another is falling behind its rate limit, with recurring failures and fleet changes alongside](https://raw.githubusercontent.com/hardbyte/awa/main/docs/images/awa-ui-overview-incident-dark.png) ## Core Concepts @@ -49,7 +49,7 @@ Awa (Māori: river) fills the gap between Postgres event queues that are too nar - **Operator descriptors** — code-declared queue and job-kind names/descriptions with stale/drift visibility in the UI. - **Postgres-only** — one dependency you already have; no Redis, no RabbitMQ, no separate scheduler. -![AWA Web UI — Queue detail (dark mode)](https://raw.githubusercontent.com/hardbyte/awa/main/docs/images/awa-ui-queue-detail-dark.png) +![Awa admin overview during recovery: three queues draining their backlog with an estimate of when all will be clear, after a hotfix rollout shown in the fleet changes](https://raw.githubusercontent.com/hardbyte/awa/main/docs/images/awa-ui-overview-recovery-dark.png) ## Correctness diff --git a/awa-cli/src/main.rs b/awa-cli/src/main.rs index 405fc3023..e111c3574 100644 --- a/awa-cli/src/main.rs +++ b/awa-cli/src/main.rs @@ -273,6 +273,23 @@ fn sanitize_filename(description: &str) -> String { /// mutating commands require an explicit `--context` or `--database-url` /// once more than one context is defined, and are confirmation-gated on /// contexts marked `production = true`. +/// After a migration, say which lane sequences still need repair and how to +/// finish it. Read-only. +async fn print_lane_sequence_leftovers( + pool: &sqlx::PgPool, +) -> Result<(), Box> { + let mut problems = Vec::new(); + for (schema, _) in awa_model::storage::lane_sequence_problem_counts(pool).await? { + problems.extend(awa_model::storage::lane_sequence_problems(pool, &schema).await?); + } + for next_step in awa_model::storage::lane_sequence_next_steps( + problems.iter().map(|problem| (problem, false)), + ) { + eprintln!("warning: {next_step}"); + } + Ok(()) +} + fn access_for(command: &Commands) -> context::Access { use context::Access::{Mutating, ReadOnly}; match command { @@ -328,11 +345,13 @@ fn access_for(command: &Commands) -> context::Access { }, Commands::Storage { command } => match command { StorageCommands::Status => ReadOnly, + StorageCommands::LaneSequences { repair: false, .. } => ReadOnly, StorageCommands::Finalize { check: true, .. } => ReadOnly, StorageCommands::FlipRingAuthority { check: true, .. } => ReadOnly, StorageCommands::Prepare { .. } | StorageCommands::PrepareQueueStorageSchema { .. } | StorageCommands::PrepareQueue { .. } + | StorageCommands::LaneSequences { .. } | StorageCommands::Abort | StorageCommands::EnterMixedTransition { .. } | StorageCommands::Finalize { .. } @@ -811,7 +830,7 @@ enum StorageCommands { #[arg(long)] reset: bool, }, - /// Provision lane sequences as the migrator, before starting a runtime without DDL + /// Provision every priority and shard of a queue ahead of its first enqueue PrepareQueue { /// Queue-storage schema to provision #[arg(long, default_value = "awa")] @@ -826,6 +845,17 @@ enum StorageCommands { #[arg(long, default_value_t = 1)] queue_stripe_count: usize, }, + /// List lane sequences not owned by the schema owner or missing runtime + /// grants, with the statements that fix them. Exits 2 while any remain. + LaneSequences { + /// Queue-storage schema to check + #[arg(long, default_value = "awa")] + schema: String, + /// Reassign and grant first, as far as the connected role (or, failing + /// that, its login role) is allowed to + #[arg(long)] + repair: bool, + }, /// Abort a prepared or mixed-transition storage rollout before final activation Abort, /// Enter mixed transition and begin routing new writes to the prepared engine @@ -1182,6 +1212,7 @@ async fn run() -> Result<(), Box> { }; awa_model::migrations::run_with_options(&pool, options).await?; println!("Migrations applied successfully."); + print_lane_sequence_leftovers(&pool).await?; } } @@ -1933,6 +1964,39 @@ async fn run() -> Result<(), Box> { }))? ); } + StorageCommands::LaneSequences { schema, repair } => { + let next_steps = if repair { + let mut conn = pool.acquire().await?; + let repairs = + awa_model::storage::repair_lane_sequences(&mut conn, &schema) + .await?; + println!( + "{}", + serde_json::to_string_pretty(&serde_json::json!({ + "schema": schema, "results": repairs, + }))? + ); + awa_model::storage::lane_sequence_repair_next_steps(&repairs) + } else { + let problems = + awa_model::storage::lane_sequence_problems(&pool, &schema).await?; + println!( + "{}", + serde_json::to_string_pretty(&serde_json::json!({ + "schema": schema, "problems": problems, + }))? + ); + awa_model::storage::lane_sequence_next_steps( + problems.iter().map(|problem| (problem, false)), + ) + }; + if !next_steps.is_empty() { + for next_step in next_steps { + eprintln!("{next_step}"); + } + std::process::exit(2); + } + } StorageCommands::Abort => { awa_model::storage::abort(&pool).await?; let report = awa_model::storage::status_report(&pool).await?; diff --git a/awa-metrics/src/lib.rs b/awa-metrics/src/lib.rs index 1a1c9097b..3f1eee208 100644 --- a/awa-metrics/src/lib.rs +++ b/awa-metrics/src/lib.rs @@ -77,6 +77,8 @@ pub mod names { pub const MAINTENANCE_PRUNE_ATTEMPTS: &str = "awa.maintenance.prune.attempts"; pub const MAINTENANCE_PRUNE_SKIPPED_ROWS: &str = "awa.maintenance.prune.skipped_rows"; pub const MAINTENANCE_PRUNE_DURATION: &str = "awa.maintenance.prune.duration"; + pub const MAINTENANCE_ANALYZE_ATTEMPTS: &str = "awa.maintenance.analyze.attempts"; + pub const MAINTENANCE_ANALYZE_DURATION: &str = "awa.maintenance.analyze.duration"; pub const STORAGE_TRANSITION_READY: &str = "awa.storage.transition_ready"; pub const STORAGE_CANONICAL_LIVE_BACKLOG: &str = "awa.storage.canonical_live_backlog"; pub const STORAGE_LIVE_RUNTIME_CAPABILITY: &str = "awa.storage.live_runtime_capability"; @@ -277,6 +279,13 @@ pub struct AwaMetrics { /// rather than one per tick. Fleets alert on this rather than /// scraping the matching `tracing::warn!` line. Issue #242. pub maintenance_branch_overrun_total: Counter, + /// Planner-statistics `ANALYZE` calls by scope (`heads`, `ring_parent`, `pass`) + /// and outcome (`ok`, `skipped_privilege`, `skipped_lock`, + /// `unavailable`, `failed`). One increment per relation per refresh pass. + pub maintenance_analyze_attempts: Counter, + /// Wall-clock duration of one planner-statistics `ANALYZE` call, by scope + /// and outcome. + pub maintenance_analyze_duration_seconds: Histogram, /// Current ring `current_slot` per ring, sampled from each rotate call. /// The slot number itself isn't meaningful but the rate of advance is — /// dashboards plot `rate(slot_changes)` to see whether rotation is @@ -620,6 +629,21 @@ impl AwaMetrics { ) .with_unit("{episode}") .build(), + maintenance_analyze_attempts: meter + .u64_counter(names::MAINTENANCE_ANALYZE_ATTEMPTS) + .with_description( + "Planner-statistics ANALYZE calls by scope (heads/ring_parent) and outcome", + ) + .with_unit("{attempt}") + .build(), + maintenance_analyze_duration_seconds: meter + .f64_histogram(names::MAINTENANCE_ANALYZE_DURATION) + .with_description( + "Duration of one planner-statistics ANALYZE call by scope and outcome", + ) + .with_unit("s") + .with_boundaries(WAIT_DURATION_BUCKETS_SECONDS.to_vec()) + .build(), ring_current_slot: meter .i64_gauge(names::RING_CURRENT_SLOT) .with_description("Current slot index per ring (queue/lease/claim)") @@ -1280,6 +1304,24 @@ impl AwaMetrics { self.maintenance_branch_overrun_total.add(1, &attrs); } + /// Record one planner-statistics `ANALYZE` call. `scope` is `"heads"`, + /// `"ring_parent"` or `"pass"` (a pass-level probe or failure); `outcome` is `"ok"`, `"skipped_privilege"`, + /// `"skipped_lock"`, `"unavailable"` or `"failed"`. + pub fn record_statistics_analyze( + &self, + scope: &'static str, + outcome: &'static str, + duration: Duration, + ) { + let attrs = [ + opentelemetry::KeyValue::new("awa.analyze.scope", scope), + opentelemetry::KeyValue::new("awa.analyze.outcome", outcome), + ]; + self.maintenance_analyze_attempts.add(1, &attrs); + self.maintenance_analyze_duration_seconds + .record(duration.as_secs_f64(), &attrs); + } + /// Record a ring prune outcome. pub fn record_prune_outcome(&self, ring: &'static str, outcome: &awa_model::PruneOutcome) { let (label, reason, count) = match outcome { @@ -1457,7 +1499,7 @@ mod tests { .build(); let metrics = AwaMetrics::new(&provider.meter("awa-metrics-test")); - let seconds: [(&str, &Histogram, &[f64]); 9] = [ + let seconds: [(&str, &Histogram, &[f64]); 10] = [ ( names::ENQUEUE_DURATION, &metrics.enqueue_duration_seconds, @@ -1478,6 +1520,11 @@ mod tests { &metrics.maintenance_branch_duration_seconds, &WAIT_DURATION_BUCKETS_SECONDS, ), + ( + names::MAINTENANCE_ANALYZE_DURATION, + &metrics.maintenance_analyze_duration_seconds, + &WAIT_DURATION_BUCKETS_SECONDS, + ), ( names::DISPATCH_WAKE_TO_CLAIM_DURATION, &metrics.dispatch_wake_to_claim_seconds, diff --git a/awa-model/migrations/v023_install_queue_storage_substrate.sql b/awa-model/migrations/v023_install_queue_storage_substrate.sql index d852b2ac5..006b3704b 100644 --- a/awa-model/migrations/v023_install_queue_storage_substrate.sql +++ b/awa-model/migrations/v023_install_queue_storage_substrate.sql @@ -42,6 +42,12 @@ LANGUAGE plpgsql SECURITY INVOKER SET search_path = pg_catalog AS $refresh$ +DECLARE + v_schema_owner OID; + v_function REGPROCEDURE; + v_function_owner OID; + v_set_mode TEXT := CASE WHEN current_setting('server_version_num')::int >= 160000 + THEN 'SET' ELSE 'MEMBER' END; BEGIN EXECUTE format( $ddl$ @@ -69,28 +75,40 @@ BEGIN ); BEGIN IF to_regclass(format('%%I.%%I', %1$L, v_enqueue_seq)) IS NULL THEN - IF NOT has_schema_privilege(current_user, %1$L, 'CREATE') THEN + IF has_function_privilege( + to_regprocedure(format('%%I.provision_lane_sequences(text,smallint,smallint)', %1$L)), + 'EXECUTE' + ) THEN + PERFORM %1$I.provision_lane_sequences(p_queue, p_priority, p_enqueue_shard); + ELSIF NOT has_schema_privilege(current_user, %1$L, 'CREATE') THEN RAISE EXCEPTION 'lane sequence %%.%% has not been provisioned', %1$L, v_enqueue_seq USING ERRCODE = '42501', - HINT = 'Run awa storage prepare-queue as the migrator before starting producers or workers.'; + HINT = 'Run awa migrate, or awa storage prepare-queue as the migrator, before starting producers or workers.'; + ELSE + EXECUTE format( + 'CREATE SEQUENCE IF NOT EXISTS %%I.%%I AS bigint START WITH 1 MINVALUE 1 CACHE 1', + %1$L, + v_enqueue_seq + ); END IF; - EXECUTE format( - 'CREATE SEQUENCE IF NOT EXISTS %%I.%%I AS bigint START WITH 1 MINVALUE 1 CACHE 1', - %1$L, - v_enqueue_seq - ); END IF; IF to_regclass(format('%%I.%%I', %1$L, v_claim_seq)) IS NULL THEN - IF NOT has_schema_privilege(current_user, %1$L, 'CREATE') THEN + IF has_function_privilege( + to_regprocedure(format('%%I.provision_lane_sequences(text,smallint,smallint)', %1$L)), + 'EXECUTE' + ) THEN + PERFORM %1$I.provision_lane_sequences(p_queue, p_priority, p_enqueue_shard); + ELSIF NOT has_schema_privilege(current_user, %1$L, 'CREATE') THEN RAISE EXCEPTION 'lane sequence %%.%% has not been provisioned', %1$L, v_claim_seq USING ERRCODE = '42501', - HINT = 'Run awa storage prepare-queue as the migrator before starting producers or workers.'; + HINT = 'Run awa migrate, or awa storage prepare-queue as the migrator, before starting producers or workers.'; + ELSE + EXECUTE format( + 'CREATE SEQUENCE IF NOT EXISTS %%I.%%I AS bigint START WITH 1 MINVALUE 1 CACHE 1', + %1$L, + v_claim_seq + ); END IF; - EXECUTE format( - 'CREATE SEQUENCE IF NOT EXISTS %%I.%%I AS bigint START WITH 1 MINVALUE 1 CACHE 1', - %1$L, - v_claim_seq - ); END IF; UPDATE %1$I.queue_enqueue_heads @@ -131,16 +149,22 @@ BEGIN v_start BIGINT; BEGIN IF to_regclass(format('%%I.%%I', %1$L, v_seq_name)) IS NULL THEN - IF NOT has_schema_privilege(current_user, %1$L, 'CREATE') THEN + IF has_function_privilege( + to_regprocedure(format('%%I.provision_lane_sequences(text,smallint,smallint)', %1$L)), + 'EXECUTE' + ) THEN + PERFORM %1$I.provision_lane_sequences(NEW.queue, NEW.priority, NEW.enqueue_shard); + ELSIF NOT has_schema_privilege(current_user, %1$L, 'CREATE') THEN RAISE EXCEPTION 'lane sequence %%.%% has not been provisioned', %1$L, v_seq_name USING ERRCODE = '42501', - HINT = 'Run awa storage prepare-queue as the migrator before starting producers or workers.'; + HINT = 'Run awa migrate, or awa storage prepare-queue as the migrator, before starting producers or workers.'; + ELSE + EXECUTE format( + 'CREATE SEQUENCE IF NOT EXISTS %%I.%%I AS bigint START WITH 1 MINVALUE 1 CACHE 1', + %1$L, + v_seq_name + ); END IF; - EXECUTE format( - 'CREATE SEQUENCE IF NOT EXISTS %%I.%%I AS bigint START WITH 1 MINVALUE 1 CACHE 1', - %1$L, - v_seq_name - ); END IF; NEW.seq_name := v_seq_name; @@ -197,16 +221,22 @@ BEGIN ); BEGIN IF to_regclass(format('%%I.%%I', %1$L, v_seq_name)) IS NULL THEN - IF NOT has_schema_privilege(current_user, %1$L, 'CREATE') THEN + IF has_function_privilege( + to_regprocedure(format('%%I.provision_lane_sequences(text,smallint,smallint)', %1$L)), + 'EXECUTE' + ) THEN + PERFORM %1$I.provision_lane_sequences(NEW.queue, NEW.priority, NEW.enqueue_shard); + ELSIF NOT has_schema_privilege(current_user, %1$L, 'CREATE') THEN RAISE EXCEPTION 'lane sequence %%.%% has not been provisioned', %1$L, v_seq_name USING ERRCODE = '42501', - HINT = 'Run awa storage prepare-queue as the migrator before starting producers or workers.'; + HINT = 'Run awa migrate, or awa storage prepare-queue as the migrator, before starting producers or workers.'; + ELSE + EXECUTE format( + 'CREATE SEQUENCE IF NOT EXISTS %%I.%%I AS bigint START WITH 1 MINVALUE 1 CACHE 1', + %1$L, + v_seq_name + ); END IF; - EXECUTE format( - 'CREATE SEQUENCE IF NOT EXISTS %%I.%%I AS bigint START WITH 1 MINVALUE 1 CACHE 1', - %1$L, - v_seq_name - ); END IF; NEW.seq_name := v_seq_name; @@ -221,11 +251,657 @@ BEGIN p_schema ); + -- Lane sequences are created by the schema owner through these two + -- SECURITY DEFINER functions, whichever role first touches the lane, so + -- every runtime role can use them. Their grants mirror the head table each + -- sequence backs (see awa.lane_sequence_problems); they never depend on + -- the creating role's default privileges. + EXECUTE format( + $ddl$ + CREATE OR REPLACE FUNCTION %1$I.provision_lane_sequences( + p_queue TEXT, + p_priority SMALLINT, + p_enqueue_shard SMALLINT + ) + RETURNS VOID + LANGUAGE plpgsql + SECURITY DEFINER + SET search_path = pg_catalog, pg_temp + AS $func$ + DECLARE + v_seq_name TEXT; + v_grant TEXT; + BEGIN + IF p_queue IS NULL + OR p_priority IS NULL + OR p_enqueue_shard IS NULL OR p_enqueue_shard NOT BETWEEN 0 AND 63 THEN + RAISE EXCEPTION 'invalid queue lane: queue %%, priority %%, enqueue shard %%', + quote_nullable(p_queue), p_priority, p_enqueue_shard + USING ERRCODE = '22023'; + END IF; + + -- The lane lock QueueStorage takes before it writes lane rows, so + -- concurrent first touches of one lane create its sequences once. + PERFORM pg_advisory_xact_lock(hashtextextended( + format('%%s:%%s:%%s:%%s', %1$L, p_queue, p_priority, p_enqueue_shard), + 0 + )); + + FOREACH v_seq_name IN ARRAY ARRAY[ + %1$I.queue_lane_sequence_name('queue_enqueue_seq', p_queue, p_priority, p_enqueue_shard), + %1$I.queue_lane_sequence_name('queue_claim_seq', p_queue, p_priority, p_enqueue_shard) + ] LOOP + IF to_regclass(format('%%I.%%I', %1$L, v_seq_name)) IS NULL THEN + EXECUTE format( + 'CREATE SEQUENCE IF NOT EXISTS %%I.%%I AS bigint START WITH 1 MINVALUE 1 CACHE 1', + %1$L, + v_seq_name + ); + FOR v_grant IN + SELECT unnest(problem.missing_grants) + FROM awa.lane_sequence_problems(%1$L, v_seq_name) AS problem + WHERE problem.owner = current_user + LOOP + EXECUTE v_grant; + END LOOP; + END IF; + END LOOP; + END; + $func$ + $ddl$, + p_schema + ); + + EXECUTE format( + $ddl$ + CREATE OR REPLACE FUNCTION %1$I.provision_lane( + p_queue TEXT, + p_priority SMALLINT, + p_enqueue_shard SMALLINT + ) + RETURNS VOID + LANGUAGE plpgsql + SECURITY DEFINER + SET search_path = pg_catalog, pg_temp + AS $func$ + BEGIN + -- Validates the lane and takes its lock before any row is written. + PERFORM %1$I.provision_lane_sequences(p_queue, p_priority, p_enqueue_shard); + + INSERT INTO %1$I.queue_lanes (queue, priority) + VALUES (p_queue, p_priority) + ON CONFLICT (queue, priority) DO NOTHING; + + -- The head triggers fire before ON CONFLICT and set the lane's + -- sequence, so rows that already exist are skipped outright. + INSERT INTO %1$I.queue_enqueue_heads (queue, priority, enqueue_shard) + SELECT p_queue, p_priority, p_enqueue_shard + WHERE NOT EXISTS ( + SELECT 1 FROM %1$I.queue_enqueue_heads AS heads + WHERE heads.queue = p_queue + AND heads.priority = p_priority + AND heads.enqueue_shard = p_enqueue_shard + ) + ON CONFLICT (queue, priority, enqueue_shard) DO NOTHING; + + INSERT INTO %1$I.queue_claim_heads (queue, priority, enqueue_shard) + SELECT p_queue, p_priority, p_enqueue_shard + WHERE NOT EXISTS ( + SELECT 1 FROM %1$I.queue_claim_heads AS heads + WHERE heads.queue = p_queue + AND heads.priority = p_priority + AND heads.enqueue_shard = p_enqueue_shard + ) + ON CONFLICT (queue, priority, enqueue_shard) DO NOTHING; + + PERFORM %1$I.ensure_lane_sequences(p_queue, p_priority, p_enqueue_shard); + END; + $func$ + $ddl$, + p_schema + ); + + SELECT namespace.nspowner INTO v_schema_owner + FROM pg_namespace AS namespace + WHERE namespace.nspname = p_schema; + + FOREACH v_function IN ARRAY ARRAY[ + format('%I.provision_lane_sequences(text,smallint,smallint)', p_schema)::regprocedure, + format('%I.provision_lane(text,smallint,smallint)', p_schema)::regprocedure + ] LOOP + EXECUTE format('GRANT EXECUTE ON FUNCTION %s TO PUBLIC', v_function); + + SELECT routine.proowner INTO v_function_owner + FROM pg_proc AS routine + WHERE routine.oid = v_function; + + IF v_function_owner <> v_schema_owner THEN + IF pg_has_role(current_user, v_function_owner, 'USAGE') + AND pg_has_role(current_user, v_schema_owner, v_set_mode) THEN + EXECUTE format( + 'ALTER FUNCTION %s OWNER TO %I', + v_function, + pg_get_userbyid(v_schema_owner) + ); + ELSE + RAISE WARNING '% is owned by %, not the owner % of schema %; lane sequences it creates will be owned by % too', + v_function, pg_get_userbyid(v_function_owner), pg_get_userbyid(v_schema_owner), + p_schema, pg_get_userbyid(v_function_owner) + USING HINT = format( + 'Run awa migrate as a role that can SET ROLE to %s, or as %s itself.', + quote_ident(pg_get_userbyid(v_schema_owner)), + quote_ident(pg_get_userbyid(v_schema_owner)) + ); + END IF; + END IF; + END LOOP; + END; $refresh$; REVOKE EXECUTE ON FUNCTION awa.refresh_lane_sequence_helpers(TEXT) FROM PUBLIC; +-- Enqueue-path functions. Function replacement only: no table DDL and no +-- trigger changes, so a migration can refresh a live substrate through it. +CREATE OR REPLACE FUNCTION awa.refresh_enqueue_functions(p_schema TEXT) +RETURNS VOID +LANGUAGE plpgsql +SECURITY INVOKER +SET search_path = pg_catalog +AS $refresh$ +BEGIN + EXECUTE format( + $ddl$ + CREATE OR REPLACE FUNCTION %1$I.reserve_enqueue_seq( + p_queue TEXT, + p_priority SMALLINT, + p_enqueue_shard SMALLINT, + p_count BIGINT + ) + RETURNS BIGINT + LANGUAGE plpgsql + SET search_path = pg_catalog + AS $func$ + DECLARE + v_seq_name TEXT; + v_qualified_seq TEXT; + v_start BIGINT; + BEGIN + SELECT seq_name + INTO v_seq_name + FROM %1$I.queue_enqueue_heads + WHERE queue = p_queue + AND priority = p_priority + AND enqueue_shard = p_enqueue_shard; + + IF p_count IS NULL THEN + RAISE EXCEPTION 'reserve_enqueue_seq: count must not be NULL' + USING ERRCODE = '22004'; + END IF; + + IF p_count <= 0 THEN + RETURN %1$I.sequence_next_value(v_seq_name); + END IF; + + -- The head trigger names the sequence when the lane row is + -- inserted; only a lane row written before that trigger existed + -- still needs its sequences provisioned here. + IF v_seq_name IS NULL THEN + PERFORM %1$I.ensure_lane_sequences(p_queue, p_priority, p_enqueue_shard); + + SELECT seq_name + INTO v_seq_name + FROM %1$I.queue_enqueue_heads + WHERE queue = p_queue + AND priority = p_priority + AND enqueue_shard = p_enqueue_shard; + + IF v_seq_name IS NULL THEN + RAISE EXCEPTION 'missing enqueue lane sequence for queue %%, priority %%, shard %%', + p_queue, p_priority, p_enqueue_shard + USING ERRCODE = '55000'; + END IF; + END IF; + + v_qualified_seq := format('%%I.%%I', %1$L, v_seq_name); + -- Hold the lane reservation lock until the surrounding enqueue + -- transaction ends. Sequence movement is non-transactional: if a + -- later producer could reserve and commit lane N+1 while an + -- earlier producer still holds uncommitted lane N, claim could + -- advance past N and strand the earlier job when it eventually + -- commits. The transaction-scoped lock preserves visibility order + -- within a lane while aborted reservations still become harmless + -- sequence gaps. + PERFORM pg_advisory_xact_lock(hashtextextended(v_qualified_seq, 0)); + + -- Every writer of a lane sequence holds this key (set_sequence_next + -- and the head trigger take it too), so a read followed by one + -- setval reserves the range with a single sequence write. + IF p_count = 1 THEN + v_start := nextval(v_qualified_seq::regclass); + ELSE + v_start := COALESCE( + pg_sequence_last_value(v_qualified_seq::regclass) + 1, + %1$I.sequence_next_value(v_seq_name) + ); + PERFORM setval(v_qualified_seq::regclass, v_start + p_count - 1, true); + END IF; + + RETURN v_start; + END; + $func$ + $ddl$, + p_schema + ); + + -- One-call ready enqueue. Rows arrive grouped by lane, lanes in the + -- caller's lock order, each lane's rows contiguous. Per lane: provision + -- the lane rows if the head is missing (same lane lock and inserts as the + -- Rust ensure_lane path), then reserve through reserve_enqueue_seq. Then + -- read the queue-ring cursor, append ready rows before their segments, + -- and notify. A NULL job id takes the next value of the schema's job id + -- sequence. Segment offsets are relative to the lane's reserved start. + EXECUTE format( + $ddl$ + CREATE OR REPLACE FUNCTION %1$I.enqueue_ready_rows( + p_lane_queues TEXT[], + p_lane_priorities SMALLINT[], + p_lane_shards SMALLINT[], + p_lane_counts BIGINT[], + p_job_ids BIGINT[], + p_kinds TEXT[], + p_args JSONB[], + p_attempts SMALLINT[], + p_run_leases BIGINT[], + p_max_attempts SMALLINT[], + p_run_ats TIMESTAMPTZ[], + p_attempted_ats TIMESTAMPTZ[], + p_created_ats TIMESTAMPTZ[], + p_unique_keys BYTEA[], + p_unique_states TEXT[], + p_payloads JSONB[], + p_segment_lanes INT[], + p_segment_first_offsets BIGINT[], + p_segment_next_offsets BIGINT[], + p_segment_run_ats TIMESTAMPTZ[], + p_notify_channels TEXT[] + ) + RETURNS BIGINT + LANGUAGE plpgsql + SET search_path = pg_catalog + AS $func$ + DECLARE + v_lane_total INT := COALESCE(cardinality(p_lane_queues), 0); + v_row_total INT := COALESCE(cardinality(p_kinds), 0); + v_segment_total INT := COALESCE(cardinality(p_segment_lanes), 0); + v_lane INT; + v_start BIGINT; + v_lane_starts BIGINT[] := ARRAY[]::BIGINT[]; + v_row_lanes INT[] := ARRAY[]::INT[]; + v_row_lane_seqs BIGINT[] := ARRAY[]::BIGINT[]; + v_ready_slot INT; + v_ready_generation BIGINT; + v_inserted BIGINT; + BEGIN + IF COALESCE(cardinality(p_lane_priorities), 0) <> v_lane_total + OR COALESCE(cardinality(p_lane_shards), 0) <> v_lane_total + OR COALESCE(cardinality(p_lane_counts), 0) <> v_lane_total + OR COALESCE(cardinality(p_job_ids), 0) <> v_row_total + OR COALESCE(cardinality(p_args), 0) <> v_row_total + OR COALESCE(cardinality(p_attempts), 0) <> v_row_total + OR COALESCE(cardinality(p_run_leases), 0) <> v_row_total + OR COALESCE(cardinality(p_max_attempts), 0) <> v_row_total + OR COALESCE(cardinality(p_run_ats), 0) <> v_row_total + OR COALESCE(cardinality(p_attempted_ats), 0) <> v_row_total + OR COALESCE(cardinality(p_created_ats), 0) <> v_row_total + OR COALESCE(cardinality(p_unique_keys), 0) <> v_row_total + OR COALESCE(cardinality(p_unique_states), 0) <> v_row_total + OR COALESCE(cardinality(p_payloads), 0) <> v_row_total + OR COALESCE(cardinality(p_segment_first_offsets), 0) <> v_segment_total + OR COALESCE(cardinality(p_segment_next_offsets), 0) <> v_segment_total + OR COALESCE(cardinality(p_segment_run_ats), 0) <> v_segment_total + THEN + RAISE EXCEPTION 'enqueue_ready_rows: array lengths disagree' + USING ERRCODE = '22023'; + END IF; + + FOR v_lane IN 1..v_lane_total LOOP + IF p_lane_counts[v_lane] < 1 THEN + RAISE EXCEPTION 'enqueue_ready_rows: lane %% has no rows', v_lane + USING ERRCODE = '22023'; + END IF; + + PERFORM 1 + FROM %1$I.queue_enqueue_heads + WHERE queue = p_lane_queues[v_lane] + AND priority = p_lane_priorities[v_lane] + AND enqueue_shard = p_lane_shards[v_lane]; + + IF NOT FOUND THEN + PERFORM pg_advisory_xact_lock(hashtextextended( + format('%%s:%%s:%%s:%%s', %1$L, p_lane_queues[v_lane], + p_lane_priorities[v_lane], p_lane_shards[v_lane]), + 0 + )); + INSERT INTO %1$I.queue_lanes (queue, priority) + VALUES (p_lane_queues[v_lane], p_lane_priorities[v_lane]) + ON CONFLICT (queue, priority) DO NOTHING; + INSERT INTO %1$I.queue_enqueue_heads (queue, priority, enqueue_shard) + VALUES (p_lane_queues[v_lane], p_lane_priorities[v_lane], p_lane_shards[v_lane]) + ON CONFLICT (queue, priority, enqueue_shard) DO NOTHING; + INSERT INTO %1$I.queue_claim_heads (queue, priority, enqueue_shard) + VALUES (p_lane_queues[v_lane], p_lane_priorities[v_lane], p_lane_shards[v_lane]) + ON CONFLICT (queue, priority, enqueue_shard) DO NOTHING; + PERFORM %1$I.ensure_lane_sequences( + p_lane_queues[v_lane], p_lane_priorities[v_lane], p_lane_shards[v_lane] + ); + END IF; + + v_start := %1$I.reserve_enqueue_seq( + p_lane_queues[v_lane], + p_lane_priorities[v_lane], + p_lane_shards[v_lane], + p_lane_counts[v_lane] + ); + v_lane_starts := v_lane_starts || v_start; + v_row_lanes := v_row_lanes || array_fill(v_lane, ARRAY[p_lane_counts[v_lane]::INT]); + v_row_lane_seqs := v_row_lane_seqs + || ARRAY(SELECT generate_series(v_start, v_start + p_lane_counts[v_lane] - 1)); + END LOOP; + + IF cardinality(v_row_lane_seqs) <> v_row_total THEN + RAISE EXCEPTION 'enqueue_ready_rows: lane counts cover %% rows, got %%', + cardinality(v_row_lane_seqs), v_row_total + USING ERRCODE = '22023'; + END IF; + + IF v_row_total = 0 THEN + RETURN 0; + END IF; + + SELECT cursor.slot, cursor.generation + INTO v_ready_slot, v_ready_generation + FROM %1$I.ring_cursor('queue') AS cursor; + + INSERT INTO %1$I.ready_entries ( + ready_slot, ready_generation, job_id, kind, queue, args, priority, + attempt, run_lease, max_attempts, lane_seq, enqueue_shard, run_at, + attempted_at, created_at, unique_key, unique_states, payload + ) + SELECT + v_ready_slot, + v_ready_generation, + COALESCE(input.job_id, nextval(%2$L::regclass)), + input.kind, + p_lane_queues[input.lane], + input.args, + p_lane_priorities[input.lane], + input.attempt, + input.run_lease, + input.max_attempts, + input.lane_seq, + p_lane_shards[input.lane], + input.run_at, + input.attempted_at, + input.created_at, + input.unique_key, + input.unique_states, + input.payload + FROM unnest( + v_row_lanes, v_row_lane_seqs, p_job_ids, p_kinds, p_args, p_attempts, + p_run_leases, p_max_attempts, p_run_ats, p_attempted_ats, p_created_ats, + p_unique_keys, p_unique_states, p_payloads + ) AS input( + lane, lane_seq, job_id, kind, args, attempt, run_lease, max_attempts, + run_at, attempted_at, created_at, unique_key, unique_states, payload + ); + GET DIAGNOSTICS v_inserted = ROW_COUNT; + + INSERT INTO %1$I.ready_segments ( + ready_slot, ready_generation, queue, priority, enqueue_shard, + first_lane_seq, next_lane_seq, first_run_at + ) + SELECT + v_ready_slot, + v_ready_generation, + p_lane_queues[segment.lane], + p_lane_priorities[segment.lane], + p_lane_shards[segment.lane], + v_lane_starts[segment.lane] + segment.first_offset, + v_lane_starts[segment.lane] + segment.next_offset, + segment.run_at + FROM unnest( + p_segment_lanes, p_segment_first_offsets, p_segment_next_offsets, p_segment_run_ats + ) AS segment(lane, first_offset, next_offset, run_at) + ON CONFLICT (ready_slot, ready_generation, queue, priority, enqueue_shard, first_lane_seq) + DO NOTHING; + + PERFORM pg_notify(channel, '') + FROM unnest(p_notify_channels) AS channel; + + RETURN v_inserted; + END; + $func$ + $ddl$, + p_schema, + format('%I.job_id_seq', p_schema) + ); +END; +$refresh$; + +REVOKE EXECUTE ON FUNCTION awa.refresh_enqueue_functions(TEXT) FROM PUBLIC; + +-- Lane sequences in p_schema (or the one named p_sequence) that are not owned +-- by the schema owner or lack a runtime grant. A lane sequence carries a head +-- table's cursor, so whoever may write that head table (INSERT or UPDATE) gets +-- USAGE, SELECT and UPDATE on it, and whoever may read it gets SELECT: +-- queue_enqueue_seq_* mirror queue_enqueue_heads, queue_claim_seq_* mirror +-- queue_claim_heads. missing_grants holds the statements to run as the +-- sequence owner; reassign_sql, when the owner is wrong, needs a role that +-- holds the current owner's privileges and can SET ROLE to the schema owner. +-- repair_roles lists the login roles, superusers aside, that can run both. +-- Read-only. +CREATE OR REPLACE FUNCTION awa.lane_sequence_problems(p_schema TEXT, p_sequence TEXT DEFAULT NULL) +RETURNS TABLE ( + sequence_name TEXT, + owner TEXT, + expected_owner TEXT, + missing_grants TEXT[], + reassign_sql TEXT, + repair_roles TEXT[] +) +LANGUAGE sql +STABLE +SECURITY INVOKER +SET search_path = pg_catalog, pg_temp +AS $problems$ + WITH target AS ( + SELECT namespace.oid, namespace.nspowner + FROM pg_namespace AS namespace + WHERE namespace.nspname = p_schema + ), + lane_sequence AS ( + SELECT sequence.oid, + sequence.relname::text AS relname, + sequence.relowner, + sequence.relacl, + target.nspowner, + CASE WHEN starts_with(sequence.relname::text, 'queue_enqueue_seq_') + THEN 'queue_enqueue_heads' ELSE 'queue_claim_heads' END AS head_table + FROM target + JOIN pg_class AS sequence ON sequence.relnamespace = target.oid + WHERE sequence.relkind = 'S' + AND sequence.relname ~ '^queue_(enqueue|claim)_seq_[0-9a-f]{32}$' + AND (p_sequence IS NULL OR sequence.relname = p_sequence) + ), + wanted AS ( + SELECT DISTINCT lane_sequence.oid, head_privilege.grantee, sequence_privilege.privilege_type + FROM lane_sequence + JOIN pg_class AS head + ON head.relnamespace = (SELECT target.oid FROM target) + AND head.relname = lane_sequence.head_table + CROSS JOIN LATERAL aclexplode(COALESCE(head.relacl, acldefault('r', head.relowner))) AS head_privilege + CROSS JOIN LATERAL unnest( + CASE head_privilege.privilege_type + WHEN 'SELECT' THEN ARRAY['SELECT'] + WHEN 'INSERT' THEN ARRAY['USAGE', 'SELECT', 'UPDATE'] + WHEN 'UPDATE' THEN ARRAY['USAGE', 'SELECT', 'UPDATE'] + ELSE ARRAY[]::TEXT[] + END + ) AS sequence_privilege(privilege_type) + WHERE head_privilege.grantee <> lane_sequence.nspowner + ), + missing AS ( + SELECT wanted.oid, wanted.grantee, wanted.privilege_type + FROM wanted + JOIN lane_sequence ON lane_sequence.oid = wanted.oid + WHERE NOT EXISTS ( + SELECT 1 + FROM aclexplode(COALESCE(lane_sequence.relacl, acldefault('S', lane_sequence.relowner))) AS held + WHERE held.grantee = wanted.grantee + AND held.privilege_type = wanted.privilege_type + ) + ) + SELECT lane_sequence.relname, + pg_get_userbyid(lane_sequence.relowner)::text, + pg_get_userbyid(lane_sequence.nspowner)::text, + ARRAY( + SELECT format( + 'GRANT %s ON SEQUENCE %I.%I TO %s', + string_agg(missing.privilege_type, ', ' ORDER BY missing.privilege_type DESC), + p_schema, + lane_sequence.relname, + CASE missing.grantee WHEN 0 THEN 'PUBLIC' + ELSE quote_ident(pg_get_userbyid(missing.grantee)) END + ) + FROM missing + WHERE missing.oid = lane_sequence.oid + GROUP BY missing.grantee + ORDER BY 1 + ), + CASE WHEN lane_sequence.relowner <> lane_sequence.nspowner THEN format( + 'ALTER SEQUENCE %I.%I OWNER TO %I', + p_schema, lane_sequence.relname, pg_get_userbyid(lane_sequence.nspowner) + ) END, + ARRAY( + SELECT candidate.rolname::text + FROM pg_roles AS candidate + WHERE candidate.rolcanlogin + AND NOT candidate.rolsuper + AND pg_has_role(candidate.oid, lane_sequence.relowner, 'USAGE') + AND (lane_sequence.relowner = lane_sequence.nspowner + OR (pg_has_role(candidate.oid, lane_sequence.nspowner, 'USAGE') + AND pg_has_role( + candidate.oid, + lane_sequence.nspowner, + CASE WHEN current_setting('server_version_num')::int >= 160000 + THEN 'SET' ELSE 'MEMBER' END + ))) + ORDER BY candidate.rolname + ) + FROM lane_sequence + WHERE lane_sequence.relowner <> lane_sequence.nspowner + OR EXISTS (SELECT 1 FROM missing WHERE missing.oid = lane_sequence.oid) + ORDER BY lane_sequence.relname +$problems$; + +-- Give one lane sequence to the schema owner and add its missing runtime +-- grants, doing only what the calling role is allowed to: a sequence it +-- cannot reassign keeps its owner and still gets the grants when the caller +-- holds that owner's privileges. Never raises for a sequence it cannot +-- repair. Reassigning takes the sequence's ACCESS EXCLUSIVE lock, which an +-- open enqueue or claim on the lane conflicts with, so the lock wait is +-- bounded: on timeout nothing changes and the outcome is 'busy'. The lock is +-- held until the caller's transaction ends, so callers repair one sequence +-- per transaction. Returns no row when the sequence needs nothing, otherwise +-- one row: outcome is 'repaired', 'owner_unchanged' (grants complete, owner +-- still wrong), 'unrepaired' or 'busy', with the statements still to run. +CREATE OR REPLACE FUNCTION awa.repair_lane_sequence(p_schema TEXT, p_sequence TEXT) +RETURNS TABLE ( + sequence_name TEXT, + owner TEXT, + expected_owner TEXT, + outcome TEXT, + missing_grants TEXT[], + reassign_sql TEXT, + repair_roles TEXT[] +) +LANGUAGE plpgsql +SECURITY INVOKER +SET search_path = pg_catalog, pg_temp +SET lock_timeout = '300ms' +AS $repair$ +DECLARE + v_problem RECORD; + v_remaining RECORD; + v_busy BOOLEAN := false; + v_set_mode TEXT := CASE WHEN current_setting('server_version_num')::int >= 160000 + THEN 'SET' ELSE 'MEMBER' END; + v_statement TEXT; +BEGIN + SELECT * INTO v_problem + FROM awa.lane_sequence_problems(p_schema, p_sequence) AS problem; + IF NOT FOUND THEN + RETURN; + END IF; + + BEGIN + -- ALTER ... OWNER TO needs the current owner's privileges, SET ROLE to + -- the new owner, and the new owner's CREATE on the schema (which a + -- schema owner has). Check first so an unrelated owner is skipped + -- without waiting for the lock. + IF v_problem.reassign_sql IS NOT NULL + AND pg_has_role(current_user, to_regrole(quote_ident(v_problem.owner)), 'USAGE') + AND pg_has_role(current_user, to_regrole(quote_ident(v_problem.expected_owner)), v_set_mode) THEN + BEGIN + EXECUTE v_problem.reassign_sql; + EXCEPTION WHEN insufficient_privilege THEN + NULL; + END; + END IF; + + SELECT * INTO v_remaining + FROM awa.lane_sequence_problems(p_schema, p_sequence) AS problem; + IF FOUND + AND cardinality(v_remaining.missing_grants) > 0 + AND pg_has_role(current_user, to_regrole(quote_ident(v_remaining.owner)), 'USAGE') THEN + FOREACH v_statement IN ARRAY v_remaining.missing_grants LOOP + BEGIN + EXECUTE v_statement; + EXCEPTION WHEN insufficient_privilege THEN + NULL; + END; + END LOOP; + END IF; + EXCEPTION WHEN lock_not_available THEN + v_busy := true; + END; + + SELECT * INTO v_remaining + FROM awa.lane_sequence_problems(p_schema, p_sequence) AS problem; + sequence_name := v_problem.sequence_name; + expected_owner := v_problem.expected_owner; + IF FOUND THEN + owner := v_remaining.owner; + missing_grants := v_remaining.missing_grants; + reassign_sql := v_remaining.reassign_sql; + repair_roles := v_remaining.repair_roles; + outcome := CASE WHEN v_busy THEN 'busy' + WHEN cardinality(v_remaining.missing_grants) = 0 THEN 'owner_unchanged' + ELSE 'unrepaired' END; + ELSE + owner := v_problem.expected_owner; + missing_grants := ARRAY[]::TEXT[]; + reassign_sql := NULL; + repair_roles := ARRAY[]::TEXT[]; + outcome := 'repaired'; + END IF; + RETURN NEXT; +END +$repair$; + +REVOKE EXECUTE ON FUNCTION awa.repair_lane_sequence(TEXT, TEXT) FROM PUBLIC; + -- Retained job output (ADR-048): add the output columns to an existing -- substrate and expose them through terminal_jobs. Called by the installer -- and by migration v052, which does not replay the installer. Each column is @@ -1414,78 +2090,7 @@ BEGIN PERFORM awa.refresh_lane_sequence_helpers(p_schema); - EXECUTE format( - $ddl$ - CREATE OR REPLACE FUNCTION %1$I.reserve_enqueue_seq( - p_queue TEXT, - p_priority SMALLINT, - p_enqueue_shard SMALLINT, - p_count BIGINT - ) - RETURNS BIGINT - LANGUAGE plpgsql - SET search_path = pg_catalog - AS $func$ - DECLARE - v_seq_name TEXT; - v_lock_key TEXT; - v_start BIGINT; - BEGIN - IF p_count <= 0 THEN - RETURN %1$I.sequence_next_value(( - SELECT seq_name - FROM %1$I.queue_enqueue_heads - WHERE queue = p_queue - AND priority = p_priority - AND enqueue_shard = p_enqueue_shard - )); - END IF; - - PERFORM %1$I.ensure_lane_sequences(p_queue, p_priority, p_enqueue_shard); - - SELECT seq_name - INTO v_seq_name - FROM %1$I.queue_enqueue_heads - WHERE queue = p_queue - AND priority = p_priority - AND enqueue_shard = p_enqueue_shard; - - IF v_seq_name IS NULL THEN - RAISE EXCEPTION 'missing enqueue lane sequence for queue %%, priority %%, shard %%', - p_queue, p_priority, p_enqueue_shard - USING ERRCODE = '55000'; - END IF; - - v_lock_key := format('%%I.%%I', %1$L, v_seq_name); - -- Hold the lane reservation lock until the surrounding enqueue - -- transaction ends. Sequence movement is non-transactional: if a - -- later producer could reserve and commit lane N+1 while an - -- earlier producer still holds uncommitted lane N, claim could - -- advance past N and strand the earlier job when it eventually - -- commits. The transaction-scoped lock preserves visibility order - -- within a lane while aborted reservations still become harmless - -- sequence gaps. - PERFORM pg_advisory_xact_lock(hashtextextended(v_lock_key, 0)); - EXECUTE format( - 'SELECT nextval(%%L::regclass)::bigint', - format('%%I.%%I', %1$L, v_seq_name) - ) - INTO v_start; - - IF p_count > 1 THEN - EXECUTE format( - 'SELECT setval(%%L::regclass, %%s, true)', - format('%%I.%%I', %1$L, v_seq_name), - v_start + p_count - 1 - ); - END IF; - - RETURN v_start; - END; - $func$ - $ddl$, - p_schema - ); + PERFORM awa.refresh_enqueue_functions(p_schema); EXECUTE format( 'DROP TRIGGER IF EXISTS queue_enqueue_heads_sequence_sync ON %I.queue_enqueue_heads', diff --git a/awa-model/migrations/v053_definer_lane_provisioning.sql b/awa-model/migrations/v053_definer_lane_provisioning.sql new file mode 100644 index 000000000..ced1a7642 --- /dev/null +++ b/awa-model/migrations/v053_definer_lane_provisioning.sql @@ -0,0 +1,58 @@ +-- v053: Create queue lanes as the schema owner, whichever role touches them first. +-- Tracking: https://github.com/hardbyte/awa/pull/637 +-- Lane sequences used to belong to the role that first enqueued into, claimed +-- from or promoted into a lane, and only that role (plus whoever its default +-- privileges named) could use them. refresh_lane_sequence_helpers now installs +-- two SECURITY DEFINER functions, provision_lane and provision_lane_sequences, +-- owned by the schema owner, and the lane helpers create a missing sequence +-- through them. Each new sequence gets the grants of the head table it backs. +-- This migration does not reassign existing lane sequences: ALTER SEQUENCE +-- OWNER takes the sequence's ACCESS EXCLUSIVE lock until commit, and the +-- migration runs in one transaction. After it commits, `awa migrate` repairs +-- each lane sequence in its own short transaction with a bounded lock wait +-- (awa.repair_lane_sequence). Another runner can do the same with +-- `awa storage lane-sequences --repair`. +-- +-- N-1 binaries call the same helpers with the same arguments and cursor +-- semantics, so lanes they create are owned by the schema owner too. A binary +-- that predates provision_lane keeps using the helpers; this binary probes for +-- provision_lane and uses the helpers when it is missing. +-- +-- Live load: function replacement only. Reads the catalog; no table DDL, row +-- scans or sequence locks. + +DO $$ +DECLARE + v_schema TEXT; + v_count BIGINT; +BEGIN + FOR v_schema IN + SELECT discovered.schema_name FROM awa.queue_storage_schemas() AS discovered + LOOP + IF to_regclass(format('%I.queue_enqueue_heads', v_schema)) IS NULL + OR to_regclass(format('%I.queue_claim_heads', v_schema)) IS NULL + OR to_regclass(format('%I.queue_ring_state', v_schema)) IS NULL + OR to_regclass(format('%I.lease_ring_state', v_schema)) IS NULL + OR to_regclass(format('%I.claim_ring_state', v_schema)) IS NULL THEN + CONTINUE; + END IF; + + PERFORM awa.refresh_lane_sequence_helpers(v_schema); + + SELECT count(*) INTO v_count + FROM awa.lane_sequence_problems(v_schema) AS problem; + + IF v_count > 0 THEN + RAISE NOTICE '% lane sequence(s) in schema % are not owned by the schema owner or lack runtime grants', v_count, v_schema + USING HINT = format( + 'awa migrate attempts repair after this commits; review its warnings for any sequences left unrepaired. To retry, run awa storage lane-sequences --schema %s --repair as the migration role itself, without SET ROLE.', + v_schema + ); + END IF; + END LOOP; +END; +$$; + +INSERT INTO awa.schema_version (version, description) +VALUES (53, 'Create queue lanes as the schema owner') +ON CONFLICT (version) DO NOTHING; diff --git a/awa-model/migrations/v054_analyze_queue_storage_statistics.sql b/awa-model/migrations/v054_analyze_queue_storage_statistics.sql new file mode 100644 index 000000000..ef2ecfc1e --- /dev/null +++ b/awa-model/migrations/v054_analyze_queue_storage_statistics.sql @@ -0,0 +1,180 @@ +-- v054: Planner statistics for the queue-storage tables autovacuum never analyses (#631) +-- +-- Autovacuum does not analyse partitioned tables, so the ring parents +-- (ready_entries, done_entries, leases, ...) have no inherited statistics. It +-- also never reaches the autoanalyze threshold on the two small head tables +-- (queue_claim_heads, queue_enqueue_heads), so the planner prices lane joins +-- against them at its defaults (320 rows, n_distinct 200) while they drive +-- joins over hundreds of thousands of retained rows. +-- +-- The maintenance leader refreshes both on an interval. Runtimes usually +-- connect as a role that does not own the tables, and ANALYZE skips with a +-- warning a table its caller may not analyse, so the refresh runs through +-- this SECURITY DEFINER function owned by the migration role, which owns the +-- tables. +-- +-- The function analyses one relation per call and the caller runs each call as +-- its own transaction, so the locks ANALYZE takes on a partitioned table's +-- children are held for that table rather than for the whole refresh. +-- +-- Locking on a partitioned table, per the ANALYZE documentation: SKIP_LOCKED +-- applies when ANALYZE starts work on a relation, so a child that ring prune +-- holds ACCESS EXCLUSIVE is skipped during the per-child analysis, but +-- sampling the children for the parent's statistics can still wait for it. +-- That wait is bounded by the function's lock_timeout and fails the call with +-- SQLSTATE 55P03, which the caller records as skipped_lock. SKIP_LOCKED also +-- skips every partition silently when the parent itself is locked, so the +-- function takes the parent's SHARE UPDATE EXCLUSIVE lock first and lets that +-- wait fail instead. +-- +-- Definer boundary: only a head table or ring parent of a schema that +-- awa.queue_storage_schemas() reports is analysed, which requires the schema's +-- ownership, grants and triggers to be trusted, and SQL is built from nothing +-- but quoted identifiers. ANALYZE writes statistics and nothing else. +-- Only the fixed list of ring parents and the two head tables is accepted; +-- a migration that adds a partitioned table to the substrate must extend it. +-- +-- Who may call it: the lock above lasts until the caller's transaction ends +-- and blocks ring prune of that table, so EXECUTE is not granted to PUBLIC. It +-- goes to the roles that already hold INSERT, UPDATE and DELETE on the head +-- tables of the awa schema (the runtime grantees), and the function itself +-- requires the calling session user to hold those privileges on the target +-- schema's queue_claim_heads. A role created later needs the documented +-- `GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA awa` (or default privileges). +-- +-- Returns 'analyzed'; 'skipped_privilege' when the function owner may not +-- analyse the relation; or 'missing' when the relation does not exist. +-- +-- N-1 binaries never call it. A binary newer than this schema probes for the +-- function, logs once that it is missing and skips the refresh. Creating or +-- replacing it takes no table lock and completes in milliseconds; calling it +-- takes SHARE UPDATE EXCLUSIVE on the relation until the caller's transaction +-- ends. + +CREATE OR REPLACE FUNCTION awa.analyze_queue_storage_relation( + p_schema TEXT, + p_relation TEXT +) +RETURNS TEXT +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = pg_catalog, pg_temp +SET lock_timeout = '2s' +AS $$ +DECLARE + v_heads REGCLASS; + v_relid OID; + v_relkind "char"; + v_relowner OID; + v_may_analyze BOOLEAN; +BEGIN + -- awa_statistics_helper_v1: bump when the definition or grants change. + IF NOT EXISTS ( + SELECT 1 FROM awa.queue_storage_schemas() AS substrate + WHERE substrate.schema_name = p_schema + ) THEN + RAISE EXCEPTION '% is not a queue-storage schema', p_schema + USING ERRCODE = '22023'; + END IF; + + IF p_relation <> ALL (ARRAY[ + 'queue_claim_heads', + 'queue_enqueue_heads', + 'done_entries', + 'lease_claim_batches', + 'lease_claim_closure_batches', + 'lease_claim_closures', + 'lease_claims', + 'leases', + 'queue_terminal_count_deltas', + 'ready_claim_attempt_batches', + 'ready_entries', + 'ready_segments', + 'ready_tombstones', + 'receipt_completion_batches', + 'receipt_completion_tombstones' + ]) THEN + RAISE EXCEPTION '%.% is not a ring parent or head table of a queue-storage schema', + p_schema, p_relation + USING ERRCODE = '22023'; + END IF; + + v_heads := to_regclass(format('%I.queue_claim_heads', p_schema)); + IF v_heads IS NULL + OR NOT has_table_privilege(session_user, v_heads, 'INSERT') + OR NOT has_table_privilege(session_user, v_heads, 'UPDATE') + OR NOT has_table_privilege(session_user, v_heads, 'DELETE') THEN + RAISE EXCEPTION 'permission denied: % may not maintain queue storage in schema %', + session_user, p_schema + USING ERRCODE = '42501'; + END IF; + + v_relid := to_regclass(format('%I.%I', p_schema, p_relation)); + IF v_relid IS NULL THEN + RETURN 'missing'; + END IF; + + SELECT class.relkind, class.relowner INTO v_relkind, v_relowner + FROM pg_class AS class WHERE class.oid = v_relid; + + -- ANALYZE skips, with a warning rather than an error, a table the caller + -- may not analyse: the owner, a MAINTAIN holder (PostgreSQL 17+), a + -- member of pg_maintain, or the database owner may analyse it. + SELECT + pg_has_role(current_user, v_relowner, 'USAGE') + OR pg_has_role( + current_user, + (SELECT database.datdba FROM pg_database AS database + WHERE database.datname = current_database()), + 'USAGE' + ) + OR CASE + WHEN current_setting('server_version_num')::int >= 170000 + THEN has_table_privilege(v_relid, 'MAINTAIN') + ELSE FALSE + END + INTO v_may_analyze; + + IF NOT v_may_analyze THEN + RETURN 'skipped_privilege'; + END IF; + + IF v_relkind = 'p' THEN + EXECUTE format( + 'LOCK TABLE ONLY %I.%I IN SHARE UPDATE EXCLUSIVE MODE', p_schema, p_relation + ); + EXECUTE format('ANALYZE (SKIP_LOCKED) %I.%I', p_schema, p_relation); + ELSE + EXECUTE format('ANALYZE %I.%I', p_schema, p_relation); + END IF; + RETURN 'analyzed'; +END +$$; + +REVOKE EXECUTE ON FUNCTION awa.analyze_queue_storage_relation(TEXT, TEXT) FROM PUBLIC; + +DO $grant$ +DECLARE + v_grantee TEXT; +BEGIN + FOR v_grantee IN + SELECT pg_get_userbyid(privilege.grantee) + FROM pg_class AS heads + CROSS JOIN LATERAL aclexplode(heads.relacl) AS privilege + WHERE heads.oid = to_regclass('awa.queue_claim_heads') + AND privilege.grantee <> 0 + AND privilege.privilege_type IN ('INSERT', 'UPDATE', 'DELETE') + GROUP BY privilege.grantee + HAVING count(DISTINCT privilege.privilege_type) = 3 + LOOP + EXECUTE format( + 'GRANT EXECUTE ON FUNCTION awa.analyze_queue_storage_relation(text, text) TO %I', + v_grantee + ); + END LOOP; +END +$grant$; + +INSERT INTO awa.schema_version (version, description) +VALUES (54, 'Planner statistics refresh for queue-storage tables autovacuum never analyses') +ON CONFLICT (version) DO NOTHING; diff --git a/awa-model/migrations/v055_insert_jobs_batch_compat.sql b/awa-model/migrations/v055_insert_jobs_batch_compat.sql new file mode 100644 index 000000000..9fcca8610 --- /dev/null +++ b/awa-model/migrations/v055_insert_jobs_batch_compat.sql @@ -0,0 +1,580 @@ +-- v055: Batch producer insert that holds each enqueue lane lock only for the +-- lane write. +-- Tracking: https://github.com/hardbyte/awa/pull/635 +-- +-- awa.insert_job_compat takes the (queue, priority, shard) lane lock in +-- reserve_enqueue_seq on a batch's first row and, because the lock is +-- transaction-scoped, holds it while every remaining row is processed. Many +-- concurrent producers on one lane therefore queue behind whole batches. This +-- migration adds awa.insert_jobs_compat_batch, which does the lane-independent +-- work first (job ids, unique claims in key order, lane rows, payloads), then +-- reserves each lane's sequence range once, in (queue, priority, shard) order, +-- and writes the ready rows and segments in two statements. The lock is held +-- across the reservation, those two statements and the commit. +-- +-- The column arrays are parallel and must have equal length. Every element is +-- required except run_at, unique_key, unique_states and ordering_hash; +-- insert_job_compat's per-row defaults are the caller's to apply. +-- +-- p_skip_unique_conflicts inserts a batch while skipping rows whose unique key +-- is already claimed (the rows are absent from the result) instead of failing +-- the whole batch with a unique violation. +-- +-- N-1 binaries: unchanged. They keep calling awa.insert_job_compat, whose +-- definition this migration does not touch, and the two paths share the lane +-- lock, so producers on either version serialise exactly as before. +-- Binaries that know the function: insert_many_copy probes its own connection +-- and uses the per-row function until it exists, picking up the migration +-- without a restart. Generic insert_many always uses the per-row function; +-- insert_many_skip_unique_conflicts requires the batch function. +-- External runners: apply this file; nothing else reads it. +-- Live load: one CREATE OR REPLACE FUNCTION, no table DDL. + +CREATE OR REPLACE FUNCTION awa.insert_jobs_compat_batch( + p_kind TEXT[], + p_queue TEXT[], + p_args JSONB[], + p_state TEXT[], + p_priority SMALLINT[], + p_max_attempts SMALLINT[], + p_run_at TIMESTAMPTZ[], + p_metadata JSONB[], + p_tags JSONB[], + p_unique_key BYTEA[], + p_unique_states TEXT[], + p_ordering_hash TEXT[], + p_skip_unique_conflicts BOOLEAN DEFAULT FALSE +) +RETURNS SETOF awa.jobs +AS $$ +DECLARE + v_count INT := COALESCE(cardinality(p_kind), 0); + v_schema TEXT; + v_old_search_path TEXT; + v_now TIMESTAMPTZ := clock_timestamp(); + v_job_ids BIGINT[]; + v_skipped BIGINT[] := ARRAY[]::BIGINT[]; + v_lane RECORD; + v_lane_queues TEXT[] := ARRAY[]::TEXT[]; + v_lane_priorities SMALLINT[] := ARRAY[]::SMALLINT[]; + v_lane_shards SMALLINT[] := ARRAY[]::SMALLINT[]; + v_lane_starts BIGINT[] := ARRAY[]::BIGINT[]; + v_start BIGINT; + v_ready_slot INT; + v_ready_generation BIGINT; + v_row RECORD; + v_shards SMALLINT[]; +BEGIN + IF v_count = 0 THEN + RETURN; + END IF; + + IF cardinality(p_queue) IS DISTINCT FROM v_count + OR cardinality(p_args) IS DISTINCT FROM v_count + OR cardinality(p_state) IS DISTINCT FROM v_count + OR cardinality(p_priority) IS DISTINCT FROM v_count + OR cardinality(p_max_attempts) IS DISTINCT FROM v_count + OR cardinality(p_run_at) IS DISTINCT FROM v_count + OR cardinality(p_metadata) IS DISTINCT FROM v_count + OR cardinality(p_tags) IS DISTINCT FROM v_count + OR cardinality(p_unique_key) IS DISTINCT FROM v_count + OR cardinality(p_unique_states) IS DISTINCT FROM v_count + OR cardinality(p_ordering_hash) IS DISTINCT FROM v_count + THEN + RAISE EXCEPTION 'insert_jobs_compat_batch: column arrays must have equal length' + USING ERRCODE = '22023'; + END IF; + + IF EXISTS ( + SELECT 1 + FROM unnest(p_kind, p_queue, p_args, p_state, p_priority, p_max_attempts, p_metadata, p_tags) + AS input(kind, queue, args, state, priority, max_attempts, metadata, tags) + WHERE input.kind IS NULL OR input.queue IS NULL OR input.args IS NULL + OR input.state IS NULL OR input.priority IS NULL OR input.max_attempts IS NULL + OR input.metadata IS NULL OR input.tags IS NULL + ) THEN + RAISE EXCEPTION 'insert_jobs_compat_batch: kind, queue, args, state, priority, max_attempts, metadata and tags must not be NULL' + USING ERRCODE = '22004'; + END IF; + + IF EXISTS (SELECT 1 FROM unnest(p_kind) AS kind WHERE length(kind) > 200) THEN + RAISE EXCEPTION 'job kind length must be <= 200 characters' + USING ERRCODE = '23514'; + END IF; + + IF EXISTS (SELECT 1 FROM unnest(p_queue) AS queue WHERE length(queue) > 200) THEN + RAISE EXCEPTION 'queue name length must be <= 200 characters' + USING ERRCODE = '23514'; + END IF; + + IF EXISTS ( + SELECT 1 FROM unnest(p_priority) AS priority + WHERE priority < 1 OR priority > 4 + ) THEN + RAISE EXCEPTION 'priority must be between 1 and 4' + USING ERRCODE = '23514'; + END IF; + + IF EXISTS ( + SELECT 1 FROM unnest(p_max_attempts) AS max_attempts + WHERE max_attempts < 1 OR max_attempts > 1000 + ) THEN + RAISE EXCEPTION 'max_attempts must be between 1 and 1000' + USING ERRCODE = '23514'; + END IF; + + IF EXISTS ( + SELECT 1 FROM unnest(p_tags) AS tags + WHERE jsonb_array_length(tags) > 20 + ) THEN + RAISE EXCEPTION 'job tags must contain at most 20 values' + USING ERRCODE = '23514'; + END IF; + + v_schema := awa.active_queue_storage_schema(); + + -- Canonical storage has no lane lock to shorten: insert row by row. + IF v_schema IS NULL THEN + IF NOT p_skip_unique_conflicts THEN + RETURN QUERY + SELECT inserted.* + FROM unnest( + p_kind, p_queue, p_args, p_state, p_priority, p_max_attempts, + p_run_at, p_metadata, p_tags, p_unique_key, p_unique_states + ) WITH ORDINALITY AS input( + kind, queue, args, state, priority, max_attempts, run_at, + metadata, tags, unique_key, unique_states, ord + ) + CROSS JOIN LATERAL awa.insert_job_compat( + input.kind, + input.queue, + input.args, + input.state::awa.job_state, + input.priority, + input.max_attempts, + input.run_at, + input.metadata, + ARRAY( + SELECT tag.value + FROM jsonb_array_elements_text(input.tags) WITH ORDINALITY AS tag(value, tag_ord) + ORDER BY tag.tag_ord + ), + input.unique_key, + input.unique_states::bit(8), + NULL + ) AS inserted + ORDER BY input.ord; + RETURN; + END IF; + + -- Skipping needs a subtransaction per row. + FOR v_row IN + SELECT + input.ord, + input.kind, + input.queue, + input.args, + input.state, + input.priority, + input.max_attempts, + input.run_at, + input.metadata, + ARRAY( + SELECT tag.value + FROM jsonb_array_elements_text(input.tags) WITH ORDINALITY AS tag(value, tag_ord) + ORDER BY tag.tag_ord + ) AS tags, + input.unique_key, + input.unique_states + FROM unnest( + p_kind, p_queue, p_args, p_state, p_priority, p_max_attempts, + p_run_at, p_metadata, p_tags, p_unique_key, p_unique_states + ) WITH ORDINALITY AS input( + kind, queue, args, state, priority, max_attempts, run_at, + metadata, tags, unique_key, unique_states, ord + ) + ORDER BY input.ord + LOOP + BEGIN + RETURN NEXT awa.insert_job_compat( + v_row.kind, + v_row.queue, + v_row.args, + v_row.state::awa.job_state, + v_row.priority, + v_row.max_attempts, + v_row.run_at, + v_row.metadata, + v_row.tags, + v_row.unique_key, + v_row.unique_states::bit(8), + NULL + ); + EXCEPTION WHEN unique_violation THEN + NULL; + END; + END LOOP; + RETURN; + END IF; + + IF EXISTS ( + SELECT 1 FROM unnest(p_state) AS state + WHERE state NOT IN ('available', 'scheduled', 'retryable') + ) THEN + RAISE EXCEPTION 'queue storage does not support initial state %', + (SELECT state FROM unnest(p_state) AS state + WHERE state NOT IN ('available', 'scheduled', 'retryable') LIMIT 1) + USING ERRCODE = '22023'; + END IF; + + v_old_search_path := current_setting('search_path'); + PERFORM set_config('search_path', format('%I,awa,public', v_schema), true); + + -- Everything up to the lane reservation is lane-independent and runs + -- without a lane lock. + EXECUTE format( + 'SELECT array_agg(nextval(%L::regclass)::bigint) FROM generate_series(1, $1)', + format('%I.job_id_seq', v_schema) + ) + INTO v_job_ids + USING v_count; + + -- Rows that enqueue as `available` go to a lane. Their shard is derived as + -- insert_job_compat derives it: ordering-key rows from the key hash, the + -- rest from the backend pid. + SELECT array_agg( + CASE WHEN input.state = 'available' THEN + MOD( + CASE WHEN input.ordering_hash IS NULL + THEN pg_backend_pid()::numeric + ELSE input.ordering_hash::numeric + END, + COALESCE(meta.enqueue_shards, 1)::numeric + )::smallint + END + ORDER BY input.ord + ) + INTO v_shards + FROM unnest(p_queue, p_state, p_ordering_hash) + WITH ORDINALITY AS input(queue, state, ordering_hash, ord) + LEFT JOIN awa.queue_meta AS meta ON meta.queue = input.queue; + + -- Claims go in unique-key order, so two producers with overlapping keys + -- cannot wait on each other in opposite orders. + IF p_skip_unique_conflicts THEN + WITH claimed AS ( + INSERT INTO awa.job_unique_claims (unique_key, job_id) + SELECT input.unique_key, v_job_ids[input.ord] + FROM unnest(p_unique_key, p_unique_states, p_state) + WITH ORDINALITY AS input(unique_key, unique_states, state, ord) + WHERE input.unique_key IS NOT NULL + AND input.unique_states IS NOT NULL + AND awa.job_state_in_bitmask(input.unique_states::bit(8), input.state::awa.job_state) + ORDER BY input.unique_key, input.ord + ON CONFLICT (unique_key) DO NOTHING + RETURNING job_id + ) + SELECT COALESCE(array_agg(wanted.job_id), ARRAY[]::BIGINT[]) + INTO v_skipped + FROM ( + SELECT v_job_ids[input.ord] AS job_id + FROM unnest(p_unique_key, p_unique_states, p_state) + WITH ORDINALITY AS input(unique_key, unique_states, state, ord) + WHERE input.unique_key IS NOT NULL + AND input.unique_states IS NOT NULL + AND awa.job_state_in_bitmask(input.unique_states::bit(8), input.state::awa.job_state) + ) AS wanted + WHERE NOT EXISTS (SELECT 1 FROM claimed WHERE claimed.job_id = wanted.job_id); + ELSE + INSERT INTO awa.job_unique_claims (unique_key, job_id) + SELECT input.unique_key, v_job_ids[input.ord] + FROM unnest(p_unique_key, p_unique_states, p_state) + WITH ORDINALITY AS input(unique_key, unique_states, state, ord) + WHERE input.unique_key IS NOT NULL + AND input.unique_states IS NOT NULL + AND awa.job_state_in_bitmask(input.unique_states::bit(8), input.state::awa.job_state) + ORDER BY input.unique_key, input.ord; + END IF; + + -- Register lanes this batch is the first to use. The head tables carry a + -- row trigger that takes the lane lock before the conflict check, so only + -- missing lanes are proposed: a batch for an existing lane must not queue + -- behind that lane's producers before it has any work to show for it. + INSERT INTO queue_lanes (queue, priority) + SELECT lanes.queue, lanes.priority + FROM ( + SELECT DISTINCT input.queue, input.priority + FROM unnest(p_queue, p_priority, p_state) + WITH ORDINALITY AS input(queue, priority, state, ord) + WHERE input.state = 'available' + AND v_job_ids[input.ord] <> ALL (v_skipped) + ) AS lanes + WHERE NOT EXISTS ( + SELECT 1 FROM queue_lanes AS existing + WHERE existing.queue = lanes.queue AND existing.priority = lanes.priority + ) + ORDER BY lanes.queue COLLATE "C", lanes.priority + ON CONFLICT ON CONSTRAINT queue_lanes_pkey DO NOTHING; + + INSERT INTO queue_enqueue_heads (queue, priority, enqueue_shard) + SELECT lanes.queue, lanes.priority, lanes.shard + FROM ( + SELECT DISTINCT input.queue, input.priority, v_shards[input.ord] AS shard + FROM unnest(p_queue, p_priority, p_state) + WITH ORDINALITY AS input(queue, priority, state, ord) + WHERE input.state = 'available' + AND v_job_ids[input.ord] <> ALL (v_skipped) + ) AS lanes + WHERE NOT EXISTS ( + SELECT 1 FROM queue_enqueue_heads AS existing + WHERE existing.queue = lanes.queue + AND existing.priority = lanes.priority + AND existing.enqueue_shard = lanes.shard + ) + ORDER BY lanes.queue COLLATE "C", lanes.priority, lanes.shard + ON CONFLICT (queue, priority, enqueue_shard) DO NOTHING; + + INSERT INTO queue_claim_heads (queue, priority, enqueue_shard) + SELECT lanes.queue, lanes.priority, lanes.shard + FROM ( + SELECT DISTINCT input.queue, input.priority, v_shards[input.ord] AS shard + FROM unnest(p_queue, p_priority, p_state) + WITH ORDINALITY AS input(queue, priority, state, ord) + WHERE input.state = 'available' + AND v_job_ids[input.ord] <> ALL (v_skipped) + ) AS lanes + WHERE NOT EXISTS ( + SELECT 1 FROM queue_claim_heads AS existing + WHERE existing.queue = lanes.queue + AND existing.priority = lanes.priority + AND existing.enqueue_shard = lanes.shard + ) + ORDER BY lanes.queue COLLATE "C", lanes.priority, lanes.shard + ON CONFLICT (queue, priority, enqueue_shard) DO NOTHING; + + -- Re-enqueues that are not yet due carry no lane. + INSERT INTO deferred_jobs ( + job_id, kind, queue, args, state, priority, attempt, run_lease, + max_attempts, run_at, attempted_at, finalized_at, created_at, + unique_key, unique_states, payload + ) + SELECT + v_job_ids[input.ord], + input.kind, + input.queue, + input.args, + input.state::awa.job_state, + input.priority, + 0, + 0, + input.max_attempts, + COALESCE(input.run_at, v_now), + NULL, + NULL, + v_now, + input.unique_key, + input.unique_states, + jsonb_build_object( + 'metadata', input.metadata, + 'tags', input.tags, + 'errors', '[]'::jsonb, + 'progress', NULL + ) + FROM unnest( + p_kind, p_queue, p_args, p_state, p_priority, p_max_attempts, p_run_at, + p_metadata, p_tags, p_unique_key, p_unique_states + ) WITH ORDINALITY AS input( + kind, queue, args, state, priority, max_attempts, run_at, + metadata, tags, unique_key, unique_states, ord + ) + WHERE input.state <> 'available' + AND v_job_ids[input.ord] <> ALL (v_skipped); + + -- The result rows do not depend on the lanes, so build them before the + -- lock window opens. + RETURN QUERY + SELECT + v_job_ids[input.ord], + input.kind, + input.queue, + input.args, + input.state::awa.job_state, + input.priority, + 0::smallint, + input.max_attempts, + COALESCE(input.run_at, v_now), + NULL::timestamptz, + NULL::timestamptz, + NULL::timestamptz, + NULL::timestamptz, + v_now, + ARRAY[]::jsonb[], + input.metadata, + ARRAY( + SELECT tag.value + FROM jsonb_array_elements_text(input.tags) WITH ORDINALITY AS tag(value, tag_ord) + ORDER BY tag.tag_ord + ), + input.unique_key, + input.unique_states::bit(8), + NULL::uuid, + NULL::timestamptz, + NULL::text, + NULL::text, + NULL::text, + NULL::text, + 0::bigint, + NULL::jsonb + FROM unnest( + p_kind, p_queue, p_args, p_state, p_priority, p_max_attempts, p_run_at, + p_metadata, p_tags, p_unique_key, p_unique_states + ) WITH ORDINALITY AS input( + kind, queue, args, state, priority, max_attempts, run_at, + metadata, tags, unique_key, unique_states, ord + ) + WHERE v_job_ids[input.ord] <> ALL (v_skipped) + ORDER BY input.ord; + + -- Lane lock window: reserve each lane's sequence range once, in + -- (queue, priority, shard) order, then write the ready rows. The locks are + -- held to the end of the transaction, which keeps the rows of a lane + -- visible in lane_seq order. + FOR v_lane IN + SELECT lanes.queue, lanes.priority, lanes.shard, lanes.row_count + FROM ( + SELECT input.queue, input.priority, v_shards[input.ord] AS shard, + count(*) AS row_count + FROM unnest(p_queue, p_priority, p_state) + WITH ORDINALITY AS input(queue, priority, state, ord) + WHERE input.state = 'available' + AND v_job_ids[input.ord] <> ALL (v_skipped) + GROUP BY input.queue, input.priority, v_shards[input.ord] + ) AS lanes + ORDER BY lanes.queue COLLATE "C", lanes.priority, lanes.shard + LOOP + EXECUTE format('SELECT %I.reserve_enqueue_seq($1, $2, $3, $4)', v_schema) + INTO v_start + USING v_lane.queue, v_lane.priority, v_lane.shard, v_lane.row_count::BIGINT; + + v_lane_queues := v_lane_queues || v_lane.queue; + v_lane_priorities := v_lane_priorities || v_lane.priority; + v_lane_shards := v_lane_shards || v_lane.shard; + v_lane_starts := v_lane_starts || v_start; + END LOOP; + + IF cardinality(v_lane_queues) > 0 THEN + SELECT rc.slot, rc.generation + INTO v_ready_slot, v_ready_generation + FROM ring_cursor('queue') AS rc; + + INSERT INTO ready_entries ( + ready_slot, ready_generation, job_id, kind, queue, args, priority, + attempt, run_lease, max_attempts, lane_seq, enqueue_shard, run_at, + attempted_at, created_at, unique_key, unique_states, payload + ) + SELECT + v_ready_slot, + v_ready_generation, + v_job_ids[placed.ord], + placed.kind, + placed.queue, + placed.args, + placed.priority, + 0, + 0, + placed.max_attempts, + lane.start_seq + placed.lane_rank - 1, + lane.shard, + COALESCE(placed.run_at, v_now), + NULL, + v_now, + placed.unique_key, + placed.unique_states, + jsonb_build_object( + 'metadata', placed.metadata, + 'tags', placed.tags, + 'errors', '[]'::jsonb, + 'progress', NULL + ) + FROM ( + SELECT + input.*, + v_shards[input.ord] AS shard, + row_number() OVER ( + PARTITION BY input.queue, input.priority, v_shards[input.ord] + ORDER BY input.ord + ) AS lane_rank + FROM unnest( + p_kind, p_queue, p_args, p_state, p_priority, p_max_attempts, p_run_at, + p_metadata, p_tags, p_unique_key, p_unique_states + ) WITH ORDINALITY AS input( + kind, queue, args, state, priority, max_attempts, run_at, + metadata, tags, unique_key, unique_states, ord + ) + WHERE input.state = 'available' + AND v_job_ids[input.ord] <> ALL (v_skipped) + ) AS placed + JOIN unnest(v_lane_queues, v_lane_priorities, v_lane_shards, v_lane_starts) + AS lane(queue, priority, shard, start_seq) + ON lane.queue = placed.queue + AND lane.priority = placed.priority + AND lane.shard = placed.shard; + + -- One segment per run of consecutive lane_seq values sharing a run_at. + INSERT INTO ready_segments ( + ready_slot, ready_generation, queue, priority, enqueue_shard, + first_lane_seq, next_lane_seq, first_run_at + ) + SELECT + v_ready_slot, + v_ready_generation, + runs.queue, + runs.priority, + runs.shard, + min(runs.lane_seq), + max(runs.lane_seq) + 1, + runs.run_at + FROM ( + SELECT + placed.*, + placed.lane_seq - row_number() OVER ( + PARTITION BY placed.queue, placed.priority, placed.shard, placed.run_at + ORDER BY placed.lane_seq + ) AS run_id + FROM ( + SELECT + input.queue, + input.priority, + v_shards[input.ord] AS shard, + COALESCE(input.run_at, v_now) AS run_at, + lane.start_seq + row_number() OVER ( + PARTITION BY input.queue, input.priority, v_shards[input.ord] + ORDER BY input.ord + ) - 1 AS lane_seq + FROM unnest(p_queue, p_priority, p_state, p_run_at) + WITH ORDINALITY AS input(queue, priority, state, run_at, ord) + JOIN unnest(v_lane_queues, v_lane_priorities, v_lane_shards, v_lane_starts) + AS lane(queue, priority, shard, start_seq) + ON lane.queue = input.queue + AND lane.priority = input.priority + AND lane.shard = v_shards[input.ord] + WHERE input.state = 'available' + AND v_job_ids[input.ord] <> ALL (v_skipped) + ) AS placed + ) AS runs + GROUP BY runs.queue, runs.priority, runs.shard, runs.run_at, runs.run_id + ON CONFLICT (ready_slot, ready_generation, queue, priority, enqueue_shard, first_lane_seq) + DO NOTHING; + + PERFORM pg_notify('awa:' || notified.queue, '') + FROM (SELECT DISTINCT unnest(v_lane_queues) AS queue) AS notified; + END IF; + + PERFORM set_config('search_path', v_old_search_path, true); +END; +$$ LANGUAGE plpgsql VOLATILE +SET search_path = pg_catalog, awa, public; + +INSERT INTO awa.schema_version (version, description) +VALUES (55, 'Batch producer insert with a short enqueue lane lock window') +ON CONFLICT (version) DO NOTHING; diff --git a/awa-model/migrations/v056_enqueue_round_trips.sql b/awa-model/migrations/v056_enqueue_round_trips.sql new file mode 100644 index 000000000..89a8c8b60 --- /dev/null +++ b/awa-model/migrations/v056_enqueue_round_trips.sql @@ -0,0 +1,34 @@ +-- v056: Fewer round trips and sequence writes per queue-storage enqueue. +-- Tracking: https://github.com/hardbyte/awa/issues/522 +-- reserve_enqueue_seq keeps its signature, lock key and return value. It +-- provisions lane sequences only for a lane row without a sequence name and +-- reserves a multi-row range with one sequence write instead of two. +-- enqueue_ready_rows is new: one ready batch's lane reservations, ring +-- cursor read, ready and segment appends and NOTIFY in a single call. +-- N-1 binaries call reserve_enqueue_seq unchanged and never call the new +-- function; current binaries probe for it and keep the multi-statement path +-- on a schema without it. +-- Live load: function replacement only, with no table DDL or row scans. +-- Function lock waits and schema count determine wall time. + +DO $$ +DECLARE + v_schema TEXT; +BEGIN + FOR v_schema IN + SELECT discovered.schema_name FROM awa.queue_storage_schemas() AS discovered + LOOP + IF to_regclass(format('%I.queue_enqueue_heads', v_schema)) IS NULL + OR to_regclass(format('%I.queue_claim_heads', v_schema)) IS NULL + OR to_regclass(format('%I.queue_ring_state', v_schema)) IS NULL THEN + CONTINUE; + END IF; + + PERFORM awa.refresh_enqueue_functions(v_schema); + END LOOP; +END; +$$; + +INSERT INTO awa.schema_version (version, description) +VALUES (56, 'Fewer round trips and sequence writes per queue-storage enqueue') +ON CONFLICT (version) DO NOTHING; diff --git a/awa-model/migrations/v057_jobs_compat_priority_aging.sql b/awa-model/migrations/v057_jobs_compat_priority_aging.sql new file mode 100644 index 000000000..3785f6968 --- /dev/null +++ b/awa-model/migrations/v057_jobs_compat_priority_aging.sql @@ -0,0 +1,564 @@ +-- v057: keep aged running jobs visible through awa.jobs. +-- Issue: https://github.com/hardbyte/awa/issues/569. +-- +-- Claims and leases store effective priority after aging; ready entries keep +-- their original lane priority. Match the job and claimed attempt instead of +-- comparing those two priority domains. Exclude tombstoned ready entries so +-- reprioritization cannot duplicate a running row, and match run_lease so a +-- retained ready entry from an earlier attempt cannot supply its body. +-- +-- N-1 compatibility: only refreshes awa.jobs_compat(), preserving its return +-- type and all storage/write protocols. Existing binaries keep using the same +-- view and claim functions; no new runtime capability or preflight is needed. +-- +-- Lock impact: brief function-catalog and schema-version writes only, with no +-- hot-table DDL or data scan. Execution time is independent of queue depth. +-- Re-runnable and transaction-safe. External runners apply this bundle normally. + +CREATE OR REPLACE FUNCTION awa.jobs_compat() +RETURNS TABLE ( + id BIGINT, + kind TEXT, + queue TEXT, + args JSONB, + state awa.job_state, + priority SMALLINT, + attempt SMALLINT, + max_attempts SMALLINT, + run_at TIMESTAMPTZ, + heartbeat_at TIMESTAMPTZ, + deadline_at TIMESTAMPTZ, + attempted_at TIMESTAMPTZ, + finalized_at TIMESTAMPTZ, + created_at TIMESTAMPTZ, + errors JSONB[], + metadata JSONB, + tags TEXT[], + unique_key BYTEA, + unique_states BIT(8), + callback_id UUID, + callback_timeout_at TIMESTAMPTZ, + callback_filter TEXT, + callback_on_complete TEXT, + callback_on_fail TEXT, + callback_transform TEXT, + run_lease BIGINT, + progress JSONB +) +LANGUAGE plpgsql +STABLE +SET search_path = pg_catalog, awa, public +AS $$ +DECLARE + v_schema TEXT; +BEGIN + v_schema := awa.active_queue_storage_schema(); + + IF v_schema IS NULL THEN + RETURN QUERY + SELECT + j.id, + j.kind, + j.queue, + j.args, + j.state, + j.priority, + j.attempt, + j.max_attempts, + j.run_at, + j.heartbeat_at, + j.deadline_at, + j.attempted_at, + j.finalized_at, + j.created_at, + j.errors, + j.metadata, + j.tags, + j.unique_key, + j.unique_states, + j.callback_id, + j.callback_timeout_at, + j.callback_filter, + j.callback_on_complete, + j.callback_on_fail, + j.callback_transform, + j.run_lease, + j.progress + FROM awa.jobs_hot AS j + UNION ALL + SELECT + j.id, + j.kind, + j.queue, + j.args, + j.state, + j.priority, + j.attempt, + j.max_attempts, + j.run_at, + j.heartbeat_at, + j.deadline_at, + j.attempted_at, + j.finalized_at, + j.created_at, + j.errors, + j.metadata, + j.tags, + j.unique_key, + j.unique_states, + j.callback_id, + j.callback_timeout_at, + j.callback_filter, + j.callback_on_complete, + j.callback_on_fail, + j.callback_transform, + j.run_lease, + j.progress + FROM awa.scheduled_jobs AS j; + RETURN; + END IF; + + RETURN QUERY EXECUTE format( + $sql$ + WITH claim_cursors AS MATERIALIZED ( + -- One catalog read per lane instead of one per ready row + -- (#422). sequence_next_value is VOLATILE, so spelled inline in + -- the available-branch filter the planner could neither use it + -- as an index bound nor cache it across rows: every + -- ready_entries child was seq-scanned in full and each surviving + -- row paid a dynamic-SQL catalog round-trip (~15x slower on a + -- 20k-row backlog). + SELECT + claims.queue, + claims.priority, + claims.enqueue_shard, + %1$I.sequence_next_value(claims.seq_name) AS claim_seq + FROM %1$I.queue_claim_heads AS claims + ), + current_available AS ( + SELECT + ready.job_id AS id, + ready.kind, + ready.queue, + ready.args, + 'available'::awa.job_state AS state, + ready.priority, + ready.attempt, + ready.max_attempts, + ready.run_at, + NULL::timestamptz AS heartbeat_at, + NULL::timestamptz AS deadline_at, + ready.attempted_at, + NULL::timestamptz AS finalized_at, + ready.created_at, + awa.queue_storage_payload_errors(ready.payload) AS errors, + COALESCE(NULLIF(ready.payload->'metadata', 'null'::jsonb), '{}'::jsonb) AS metadata, + awa.queue_storage_payload_tags(ready.payload) AS tags, + ready.unique_key, + CASE + WHEN ready.unique_states IS NULL THEN NULL::bit(8) + ELSE ready.unique_states::bit(8) + END AS unique_states, + NULL::uuid AS callback_id, + NULL::timestamptz AS callback_timeout_at, + NULL::text AS callback_filter, + NULL::text AS callback_on_complete, + NULL::text AS callback_on_fail, + NULL::text AS callback_transform, + ready.run_lease, + NULLIF(ready.payload->'progress', 'null'::jsonb) AS progress + FROM %1$I.ready_entries AS ready + JOIN claim_cursors AS claims + ON claims.queue = ready.queue + AND claims.priority = ready.priority + AND claims.enqueue_shard = ready.enqueue_shard + WHERE ready.lane_seq >= claims.claim_seq + AND NOT EXISTS ( + SELECT 1 + FROM %1$I.ready_tombstones AS tomb + WHERE tomb.ready_slot = ready.ready_slot + AND tomb.ready_generation = ready.ready_generation + AND tomb.queue = ready.queue + AND tomb.priority = ready.priority + AND tomb.enqueue_shard = ready.enqueue_shard + AND tomb.lane_seq = ready.lane_seq + ) + ) + SELECT + current_available.id, + current_available.kind, + current_available.queue, + current_available.args, + current_available.state, + current_available.priority, + current_available.attempt, + current_available.max_attempts, + current_available.run_at, + current_available.heartbeat_at, + current_available.deadline_at, + current_available.attempted_at, + current_available.finalized_at, + current_available.created_at, + current_available.errors, + current_available.metadata, + current_available.tags, + current_available.unique_key, + current_available.unique_states, + current_available.callback_id, + current_available.callback_timeout_at, + current_available.callback_filter, + current_available.callback_on_complete, + current_available.callback_on_fail, + current_available.callback_transform, + current_available.run_lease, + current_available.progress + FROM current_available + UNION ALL + -- Receipt claims that have not materialised into `leases` are + -- running too (#246 / #416 / #422): legacy row-local `lease_claims` + -- below, default compact `lease_claim_batches` members next. Each + -- leg anti-joins every durable closure/supersession shape so a job + -- reports at most one state. Mirrors admin::state_counts (#410) and + -- queue_storage::open_receipt_running_claims_sql. + SELECT + claims.job_id AS id, + ready.kind, + ready.queue, + ready.args, + 'running'::awa.job_state AS state, + claims.priority, + claims.attempt, + claims.max_attempts, + ready.run_at, + attempt.heartbeat_at, + claims.deadline_at, + claims.claimed_at AS attempted_at, + NULL::timestamptz AS finalized_at, + ready.created_at, + awa.queue_storage_payload_errors(ready.payload) AS errors, + COALESCE(NULLIF(ready.payload->'metadata', 'null'::jsonb), '{}'::jsonb) AS metadata, + awa.queue_storage_payload_tags(ready.payload) AS tags, + ready.unique_key, + CASE + WHEN ready.unique_states IS NULL THEN NULL::bit(8) + ELSE ready.unique_states::bit(8) + END AS unique_states, + NULL::uuid AS callback_id, + NULL::timestamptz AS callback_timeout_at, + NULL::text AS callback_filter, + NULL::text AS callback_on_complete, + NULL::text AS callback_on_fail, + NULL::text AS callback_transform, + claims.run_lease, + COALESCE(NULLIF(attempt.progress, 'null'::jsonb), NULLIF(ready.payload->'progress', 'null'::jsonb)) + FROM %1$I.lease_claims AS claims + JOIN %1$I.ready_entries AS ready + ON ready.ready_slot = claims.ready_slot + AND ready.ready_generation = claims.ready_generation + AND ready.queue = claims.queue + AND ready.run_lease + 1 = claims.run_lease + AND ready.enqueue_shard = claims.enqueue_shard + AND ready.lane_seq = claims.lane_seq + AND ready.job_id = claims.job_id + AND NOT EXISTS ( + SELECT 1 FROM %1$I.ready_tombstones AS tomb + WHERE tomb.ready_slot = ready.ready_slot + AND tomb.ready_generation = ready.ready_generation + AND tomb.queue = ready.queue + AND tomb.priority = ready.priority + AND tomb.enqueue_shard = ready.enqueue_shard + AND tomb.lane_seq = ready.lane_seq + ) + LEFT JOIN %1$I.attempt_state AS attempt + ON attempt.job_id = claims.job_id + AND attempt.run_lease = claims.run_lease + WHERE claims.closed_at IS NULL + AND NOT EXISTS ( + SELECT 1 FROM %1$I.lease_claim_closures AS cx + WHERE cx.claim_slot = claims.claim_slot + AND cx.job_id = claims.job_id + AND cx.run_lease = claims.run_lease + ) + AND NOT EXISTS ( + SELECT 1 FROM %1$I.lease_claim_closure_batches AS cb + WHERE cb.receipt_ranges @> claims.receipt_id + ) + AND NOT EXISTS ( + SELECT 1 FROM %1$I.leases AS lease + WHERE lease.job_id = claims.job_id + AND lease.run_lease = claims.run_lease + ) + AND NOT EXISTS ( + SELECT 1 FROM %1$I.done_entries AS done + WHERE done.job_id = claims.job_id + AND done.run_lease = claims.run_lease + ) + AND NOT EXISTS ( + SELECT 1 FROM %1$I.deferred_jobs AS deferred + WHERE deferred.job_id = claims.job_id + AND deferred.run_lease = claims.run_lease + ) + AND NOT EXISTS ( + SELECT 1 FROM %1$I.dlq_entries AS dlq + WHERE dlq.job_id = claims.job_id + AND dlq.run_lease = claims.run_lease + ) + UNION ALL + SELECT + items.job_id AS id, + ready.kind, + ready.queue, + ready.args, + 'running'::awa.job_state AS state, + batches.priority, + batches.attempts[items.ord], + batches.max_attempts[items.ord], + ready.run_at, + attempt.heartbeat_at, + batches.deadline_at, + batches.claimed_at AS attempted_at, + NULL::timestamptz AS finalized_at, + ready.created_at, + awa.queue_storage_payload_errors(ready.payload) AS errors, + COALESCE(NULLIF(ready.payload->'metadata', 'null'::jsonb), '{}'::jsonb) AS metadata, + awa.queue_storage_payload_tags(ready.payload) AS tags, + ready.unique_key, + CASE + WHEN ready.unique_states IS NULL THEN NULL::bit(8) + ELSE ready.unique_states::bit(8) + END AS unique_states, + NULL::uuid AS callback_id, + NULL::timestamptz AS callback_timeout_at, + NULL::text AS callback_filter, + NULL::text AS callback_on_complete, + NULL::text AS callback_on_fail, + NULL::text AS callback_transform, + items.run_lease, + COALESCE(NULLIF(attempt.progress, 'null'::jsonb), NULLIF(ready.payload->'progress', 'null'::jsonb)) + FROM %1$I.lease_claim_batches AS batches + CROSS JOIN LATERAL unnest( + batches.job_ids, + batches.run_leases, + batches.receipt_ids, + batches.lane_seqs + ) WITH ORDINALITY AS items(job_id, run_lease, receipt_id, lane_seq, ord) + JOIN %1$I.ready_entries AS ready + ON ready.ready_slot = batches.ready_slot + AND ready.ready_generation = batches.ready_generation + AND ready.queue = batches.queue + AND ready.run_lease + 1 = items.run_lease + AND ready.enqueue_shard = batches.enqueue_shard + AND ready.lane_seq = items.lane_seq + AND ready.job_id = items.job_id + AND NOT EXISTS ( + SELECT 1 FROM %1$I.ready_tombstones AS tomb + WHERE tomb.ready_slot = ready.ready_slot + AND tomb.ready_generation = ready.ready_generation + AND tomb.queue = ready.queue + AND tomb.priority = ready.priority + AND tomb.enqueue_shard = ready.enqueue_shard + AND tomb.lane_seq = ready.lane_seq + ) + LEFT JOIN %1$I.attempt_state AS attempt + ON attempt.job_id = items.job_id + AND attempt.run_lease = items.run_lease + WHERE NOT EXISTS ( + SELECT 1 FROM %1$I.lease_claim_closures AS cx + WHERE cx.claim_slot = batches.claim_slot + AND cx.job_id = items.job_id + AND cx.run_lease = items.run_lease + ) + AND NOT EXISTS ( + SELECT 1 FROM %1$I.lease_claim_closure_batches AS cb + WHERE cb.claim_slot = batches.claim_slot + AND cb.receipt_ranges @> items.receipt_id + ) + AND NOT EXISTS ( + SELECT 1 FROM %1$I.leases AS lease + WHERE lease.job_id = items.job_id + AND lease.run_lease = items.run_lease + ) + AND NOT EXISTS ( + SELECT 1 FROM %1$I.done_entries AS done + WHERE done.job_id = items.job_id + AND done.run_lease = items.run_lease + ) + AND NOT EXISTS ( + SELECT 1 FROM %1$I.deferred_jobs AS deferred + WHERE deferred.job_id = items.job_id + AND deferred.run_lease = items.run_lease + ) + AND NOT EXISTS ( + SELECT 1 FROM %1$I.dlq_entries AS dlq + WHERE dlq.job_id = items.job_id + AND dlq.run_lease = items.run_lease + ) + UNION ALL + SELECT + deferred.job_id AS id, + deferred.kind, + deferred.queue, + deferred.args, + deferred.state, + deferred.priority, + deferred.attempt, + deferred.max_attempts, + deferred.run_at, + NULL::timestamptz AS heartbeat_at, + NULL::timestamptz AS deadline_at, + deferred.attempted_at, + deferred.finalized_at, + deferred.created_at, + awa.queue_storage_payload_errors(deferred.payload) AS errors, + COALESCE(NULLIF(deferred.payload->'metadata', 'null'::jsonb), '{}'::jsonb) AS metadata, + awa.queue_storage_payload_tags(deferred.payload) AS tags, + deferred.unique_key, + CASE + WHEN deferred.unique_states IS NULL THEN NULL::bit(8) + ELSE deferred.unique_states::bit(8) + END AS unique_states, + NULL::uuid AS callback_id, + NULL::timestamptz AS callback_timeout_at, + NULL::text AS callback_filter, + NULL::text AS callback_on_complete, + NULL::text AS callback_on_fail, + NULL::text AS callback_transform, + deferred.run_lease, + NULLIF(deferred.payload->'progress', 'null'::jsonb) AS progress + FROM %1$I.deferred_jobs AS deferred + UNION ALL + SELECT + leases.job_id AS id, + ready.kind, + ready.queue, + ready.args, + leases.state, + leases.priority, + leases.attempt, + leases.max_attempts, + ready.run_at, + leases.heartbeat_at, + leases.deadline_at, + leases.attempted_at, + NULL::timestamptz AS finalized_at, + ready.created_at, + awa.queue_storage_payload_errors(ready.payload) AS errors, + CASE + WHEN attempt.callback_result IS NULL + THEN COALESCE(NULLIF(ready.payload->'metadata', 'null'::jsonb), '{}'::jsonb) + ELSE COALESCE(NULLIF(ready.payload->'metadata', 'null'::jsonb), '{}'::jsonb) + || jsonb_build_object('_awa_callback_result', attempt.callback_result) + END AS metadata, + awa.queue_storage_payload_tags(ready.payload) AS tags, + ready.unique_key, + CASE + WHEN ready.unique_states IS NULL THEN NULL::bit(8) + ELSE ready.unique_states::bit(8) + END AS unique_states, + leases.callback_id, + leases.callback_timeout_at, + attempt.callback_filter, + attempt.callback_on_complete, + attempt.callback_on_fail, + attempt.callback_transform, + leases.run_lease, + COALESCE( + NULLIF(attempt.progress, 'null'::jsonb), + NULLIF(ready.payload->'progress', 'null'::jsonb) + ) AS progress + FROM %1$I.leases AS leases + JOIN %1$I.ready_entries AS ready + ON ready.ready_slot = leases.ready_slot + AND ready.ready_generation = leases.ready_generation + AND ready.queue = leases.queue + AND ready.run_lease + 1 = leases.run_lease + AND ready.enqueue_shard = leases.enqueue_shard + AND ready.lane_seq = leases.lane_seq + AND ready.job_id = leases.job_id + AND NOT EXISTS ( + SELECT 1 FROM %1$I.ready_tombstones AS tomb + WHERE tomb.ready_slot = ready.ready_slot + AND tomb.ready_generation = ready.ready_generation + AND tomb.queue = ready.queue + AND tomb.priority = ready.priority + AND tomb.enqueue_shard = ready.enqueue_shard + AND tomb.lane_seq = ready.lane_seq + ) + LEFT JOIN %1$I.attempt_state AS attempt + ON attempt.job_id = leases.job_id + AND attempt.run_lease = leases.run_lease + UNION ALL + SELECT + done.job_id AS id, + done.kind, + done.queue, + done.args, + done.state, + done.priority, + done.attempt, + done.max_attempts, + done.run_at, + NULL::timestamptz AS heartbeat_at, + NULL::timestamptz AS deadline_at, + done.attempted_at, + done.finalized_at, + done.created_at, + awa.queue_storage_payload_errors(done.payload) AS errors, + COALESCE(NULLIF(done.payload->'metadata', 'null'::jsonb), '{}'::jsonb) AS metadata, + awa.queue_storage_payload_tags(done.payload) AS tags, + done.unique_key, + CASE + WHEN done.unique_states IS NULL THEN NULL::bit(8) + ELSE done.unique_states::bit(8) + END AS unique_states, + NULL::uuid AS callback_id, + NULL::timestamptz AS callback_timeout_at, + NULL::text AS callback_filter, + NULL::text AS callback_on_complete, + NULL::text AS callback_on_fail, + NULL::text AS callback_transform, + done.run_lease, + NULLIF(done.payload->'progress', 'null'::jsonb) AS progress + FROM %1$I.terminal_jobs AS done + UNION ALL + SELECT + dlq.job_id AS id, + dlq.kind, + dlq.queue, + dlq.args, + dlq.state, + dlq.priority, + dlq.attempt, + dlq.max_attempts, + dlq.run_at, + NULL::timestamptz AS heartbeat_at, + NULL::timestamptz AS deadline_at, + dlq.attempted_at, + dlq.finalized_at, + dlq.created_at, + awa.queue_storage_payload_errors(dlq.payload) AS errors, + COALESCE(NULLIF(dlq.payload->'metadata', 'null'::jsonb), '{}'::jsonb) AS metadata, + awa.queue_storage_payload_tags(dlq.payload) AS tags, + dlq.unique_key, + CASE + WHEN dlq.unique_states IS NULL THEN NULL::bit(8) + ELSE dlq.unique_states::bit(8) + END AS unique_states, + NULL::uuid AS callback_id, + NULL::timestamptz AS callback_timeout_at, + NULL::text AS callback_filter, + NULL::text AS callback_on_complete, + NULL::text AS callback_on_fail, + NULL::text AS callback_transform, + dlq.run_lease, + NULLIF(dlq.payload->'progress', 'null'::jsonb) AS progress + FROM %1$I.dlq_entries AS dlq + $sql$, + v_schema + ); +END; +$$; + +INSERT INTO awa.schema_version (version, description) +VALUES (57, 'Keep aged running jobs visible in jobs_compat()') +ON CONFLICT (version) DO NOTHING; diff --git a/awa-model/migrations/v058_queue_history.sql b/awa-model/migrations/v058_queue_history.sql new file mode 100644 index 000000000..dc91234f1 --- /dev/null +++ b/awa-model/migrations/v058_queue_history.sql @@ -0,0 +1,66 @@ +-- v058: per-minute queue and fleet history for the admin overview +-- +-- Job tables record current state, not how a backlog moved, so the admin UI +-- could not tell a queue that is draining from one falling behind. The +-- maintenance leader now writes one row per queue per minute (depth by state, +-- oldest available wait, and claim/completion/failure deltas differenced from +-- the per-queue counters runtimes publish in their snapshots) plus one fleet +-- row per minute (instance health, leader, running versions). The UI reads +-- these instead of sampling in each `awa serve` process. +-- +-- Both tables are additive and written only by the elected maintenance +-- leader, never by triggers or the claim/finalize path, so ADR-045 does not +-- apply and enqueue/claim cost is unchanged. Rows older than the configured +-- retention (default 48 hours) are deleted by the same leader. +-- +-- N-1 binaries: never read or write these tables, and a 0.7.x leader simply +-- does not populate them; the UI then reports no history. Counter columns are +-- NULL for a minute when any live runtime serving the queue predates the +-- counters, so partial sums are never presented as totals. +-- +-- Locks and time: two CREATE TABLE IF NOT EXISTS on new relations; no +-- existing table is touched. Effectively instant. +-- Runtime roles need SELECT, INSERT and DELETE on both tables; the default +-- privileges in docs/security/database-roles.md grant that. +-- Tracking: https://github.com/hardbyte/awa/pull/543 + +CREATE TABLE IF NOT EXISTS awa.queue_history ( + queue TEXT NOT NULL, + bucket TIMESTAMPTZ NOT NULL, + available BIGINT NOT NULL, + running BIGINT NOT NULL, + scheduled BIGINT NOT NULL, + retryable BIGINT NOT NULL, + failed BIGINT NOT NULL, + waiting_external BIGINT NOT NULL, + lag_seconds DOUBLE PRECISION, + paused BOOLEAN NOT NULL DEFAULT FALSE, + -- Deltas since the previous sample. NULL when unknown: the first sample + -- after a leader change, or a live runtime without counters. + claimed BIGINT, + completed BIGINT, + failed_attempts BIGINT, + retried BIGINT, + rate_limited BIGINT, + claim_wait_ms BIGINT, + -- Live runtimes serving the queue and their summed in-flight count. + live_workers INT NOT NULL DEFAULT 0, + capacity BIGINT, + PRIMARY KEY (queue, bucket) +); + +CREATE INDEX IF NOT EXISTS idx_awa_queue_history_bucket + ON awa.queue_history (bucket); + +CREATE TABLE IF NOT EXISTS awa.fleet_history ( + bucket TIMESTAMPTZ PRIMARY KEY, + live_instances INT NOT NULL, + healthy_instances INT NOT NULL, + stale_instances INT NOT NULL, + leader_instance_id UUID, + versions TEXT[] NOT NULL DEFAULT ARRAY[]::TEXT[] +); + +INSERT INTO awa.schema_version (version, description) +VALUES (58, 'Per-minute queue and fleet history') +ON CONFLICT (version) DO NOTHING; diff --git a/awa-model/src/admin.rs b/awa-model/src/admin.rs index 71cd067ea..979144427 100644 --- a/awa-model/src/admin.rs +++ b/awa-model/src/admin.rs @@ -2498,6 +2498,36 @@ pub struct QueueRuntimeSnapshot { pub in_flight: u32, pub overflow_held: Option, pub config: QueueRuntimeConfigSnapshot, + /// Cumulative outcome counters since this instance started (#543). + /// `None` from runtimes that predate them, so readers can tell "not + /// reported" from zero. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub counters: Option, +} + +/// Monotonic per-queue counters a runtime publishes with each snapshot. +/// +/// They reset when the instance restarts; consumers difference consecutive +/// readings of the same `instance_id` and treat a decrease as a reset. +#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize, PartialEq, Eq)] +pub struct QueueOutcomeCounters { + /// Jobs this instance claimed from the queue. + pub claimed: u64, + pub completed: u64, + /// Attempts that ended the job as failed (terminal). + pub failed: u64, + /// Attempts that failed and were scheduled for another try. + pub retried: u64, + /// Dispatch polls skipped because the queue's rate limit had no tokens. + pub rate_limited: u64, + /// Sum over claimed jobs of the time between becoming runnable and + /// being claimed, in milliseconds. Divide by `claimed` for a mean. + pub claim_wait_ms: u64, + /// Attempts that parked waiting for an external callback. Whatever + /// resolves the callback may run in another process, so the eventual + /// outcome is not counted here. + #[serde(default)] + pub parked: u64, } #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Hash)] diff --git a/awa-model/src/insert.rs b/awa-model/src/insert.rs index e221b0fa7..6bb26f47b 100644 --- a/awa-model/src/insert.rs +++ b/awa-model/src/insert.rs @@ -11,13 +11,21 @@ use sqlx::{PgExecutor, PgPool}; // On queue storage every available-state insert holds a transaction-scoped // advisory lock on its `(queue, priority, enqueue_shard)` lane until the // enclosing transaction ends, so two transactions that enqueue into the same -// lanes in opposite orders can deadlock. Every multi-row enqueue therefore -// takes its lane locks in one canonical order: `(queue, priority, shard)` -// ascending, queues compared bytewise (`COLLATE "C"`, the order of Rust's -// `str`). The batch inserts here sort rows by that key, stably, before they -// reach `awa.insert_job_compat`, and put the rows back in input order for the -// caller. `QueueStorage` batch enqueues group rows in a `BTreeMap` keyed the -// same way. +// lanes in opposite orders can deadlock. `insert_many_copy` and +// `insert_many_skip_unique_conflicts` use `awa.insert_jobs_compat_batch` +// (migration v055), which does the +// lane-independent work first and then reserves its lanes in one canonical +// order: `(queue, priority, shard)` ascending, queues compared bytewise +// (`COLLATE "C"`, the order of Rust's `str`). `QueueStorage` batch enqueues +// group rows in a `BTreeMap` keyed the same way. Holding the lanes only across +// the reservation and the ready-row writes keeps a lane's lock window +// independent of the batch's unique-claim and payload work. +// +// Generic `insert_many` always uses the per-row path, as does +// `insert_many_copy` before the migration: rows are sorted by the same key before +// they reach `awa.insert_job_compat`, and returned in input order. Both paths +// take the same lane lock, so producers on either, and older binaries, share a +// lane without reordering it. // // The shard is derived as `insert_job_compat` derives it: rows with an // ordering key use `ordering_key_hash64 % enqueue_shards`, other rows @@ -211,11 +219,35 @@ impl PreparedJobInsert { /// job, because sqlx has no array encoding for the enum, the bit string, or a /// ragged array of arrays. `ordering_hash` carries each row's /// [`ordering_key_hash64`] as decimal text (the value exceeds `bigint`) so the -/// statement can place keyed rows on the shard `insert_job_compat` will pick. +/// function can place keyed rows on the shard `insert_job_compat` would pick. /// -/// Rows are inserted in canonical lane order (see "Lane lock order" above) and -/// returned in input order. -const INSERT_MANY_SQL: &str = r#" +/// `$13` skips rows whose unique key is already claimed instead of failing the +/// batch. Rows come back in input order; skipped rows are absent. +const INSERT_MANY_BATCH_SQL: &str = r#" + SELECT * + FROM awa.insert_jobs_compat_batch( + $1::text[], + $2::text[], + $3::jsonb[], + $4::text[], + $5::smallint[], + $6::smallint[], + $7::timestamptz[], + $8::jsonb[], + $9::jsonb[], + $10::bytea[], + $11::text[], + $12::text[], + $13 + ) +"#; + +/// The per-row form of [`INSERT_MANY_BATCH_SQL`], for schemas without +/// `awa.insert_jobs_compat_batch`: one `awa.insert_job_compat` call per row, +/// rows inserted in canonical lane order (see "Lane lock order" above) and +/// returned in input order. It works before and after the batch migration, +/// including when one process writes to databases at different schema versions. +const INSERT_MANY_ROW_FUNCTION_SQL: &str = r#" WITH input AS MATERIALIZED ( SELECT raw.*, @@ -476,7 +508,17 @@ fn ordering_hash_text(row: &PreparedRow) -> Option { /// Insert multiple jobs in a single statement. /// /// Supports uniqueness constraints — jobs with `unique` opts will have their -/// `unique_key` and `unique_states` computed and included. +/// `unique_key` and `unique_states` computed and included. A job whose unique +/// key is already claimed fails the whole batch with a unique violation +/// (SQLSTATE `23505`, as [`AwaError::Database`]); use +/// [`insert_many_skip_unique_conflicts`] to insert the rest instead. +/// +/// Calls `awa.insert_job_compat` per row, in canonical lane order, on every +/// supported schema. A generic executor is consumed by one statement and +/// cannot be probed separately, so this path never assumes another database +/// has the same schema. [`insert_many_copy`] probes its concrete connection +/// and uses the batch function when available; +/// [`insert_many_skip_unique_conflicts`] requires that function. #[tracing::instrument( skip(executor, jobs), fields( @@ -495,9 +537,119 @@ where if jobs.is_empty() { return Ok(Vec::new()); } + let rows = precompute_rows(jobs)?; + Ok(bind_row_function_columns( + sqlx::query_as::<_, JobRow>(INSERT_MANY_ROW_FUNCTION_SQL), + &rows, + ) + .fetch_all(executor) + .await?) +} +/// Insert multiple jobs in a single statement, skipping those whose unique key +/// is already claimed. +/// +/// Returns the rows that were inserted, in input order; a skipped job has no +/// entry in the result. Jobs without `unique` opts are always inserted. A key +/// repeated inside the batch inserts its first job. Unlike [`insert_many`], +/// which fails the batch with a unique violation, this is what a caller that +/// replays work wants, and it takes each lane lock once for the batch rather +/// than once per job as a row-by-row retry does. +/// +/// Requires migration v055 (`awa migrate`); on an older schema it returns +/// [`AwaError::Validation`]. [`insert_many_copy`] skips unique conflicts on +/// every schema, through a connection, and is the Python +/// `insert_many_copy`. +#[tracing::instrument( + skip(executor, jobs), + fields( + job.count = jobs.len(), + otel.name = "send", + otel.kind = "producer", + messaging.system = "awa", + messaging.operation.r#type = "send", + messaging.batch.message_count = jobs.len(), + ) +)] +pub async fn insert_many_skip_unique_conflicts<'e, E>( + executor: E, + jobs: &[InsertParams], +) -> Result, AwaError> +where + E: PgExecutor<'e>, +{ + if jobs.is_empty() { + return Ok(Vec::new()); + } let rows = precompute_rows(jobs)?; - let results = sqlx::query_as::<_, JobRow>(INSERT_MANY_SQL) + match bind_batch_columns(sqlx::query_as::<_, JobRow>(INSERT_MANY_BATCH_SQL), &rows) + .bind(true) + .fetch_all(executor) + .await + { + Ok(results) => Ok(results), + Err(err) if is_missing_batch_function(&err) => Err(AwaError::Validation( + "insert_many_skip_unique_conflicts needs migration v055; run `awa migrate`".to_string(), + )), + Err(err) => Err(err.into()), + } +} + +/// `undefined_function` naming the batch function: the schema predates +/// migration v055 or the function was dropped. +fn is_missing_batch_function(err: &sqlx::Error) -> bool { + matches!( + err, + sqlx::Error::Database(db_err) + if db_err.code().as_deref() == Some("42883") + && db_err.message().contains("insert_jobs_compat_batch") + ) +} + +fn bind_batch_columns<'q, O>( + query: sqlx::query::QueryAs<'q, sqlx::Postgres, O, sqlx::postgres::PgArguments>, + rows: &'q [PreparedRow], +) -> sqlx::query::QueryAs<'q, sqlx::Postgres, O, sqlx::postgres::PgArguments> { + query + .bind(rows.iter().map(|row| row.kind.as_str()).collect::>()) + .bind( + rows.iter() + .map(|row| row.queue.as_str()) + .collect::>(), + ) + .bind(rows.iter().map(|row| &row.args).collect::>()) + .bind( + rows.iter() + .map(|row| row.state_db_str()) + .collect::>(), + ) + .bind(rows.iter().map(|row| row.priority).collect::>()) + .bind(rows.iter().map(|row| row.max_attempts).collect::>()) + .bind(rows.iter().map(|row| row.run_at).collect::>()) + .bind(rows.iter().map(|row| &row.metadata).collect::>()) + .bind( + rows.iter() + .map(|row| serde_json::json!(row.tags)) + .collect::>(), + ) + .bind( + rows.iter() + .map(|row| row.unique_key.as_deref()) + .collect::>(), + ) + .bind( + rows.iter() + .map(|row| row.unique_states.as_deref()) + .collect::>(), + ) + .bind(rows.iter().map(ordering_hash_text).collect::>()) +} + +fn bind_row_function_columns<'q>( + query: sqlx::query::QueryAs<'q, sqlx::Postgres, JobRow, sqlx::postgres::PgArguments>, + rows: &'q [PreparedRow], +) -> sqlx::query::QueryAs<'q, sqlx::Postgres, JobRow, sqlx::postgres::PgArguments> { + query .bind(rows.iter().map(|row| row.kind.as_str()).collect::>()) .bind( rows.iter() @@ -535,46 +687,14 @@ where .collect::>(), ) .bind(rows.iter().map(ordering_hash_text).collect::>()) - .fetch_all(executor) - .await?; - - Ok(results) -} - -/// CTEs that number the staged rows in staging order and re-sort them into -/// canonical lane order (see "Lane lock order" above); `staged_in_lane_order` -/// carries the staging-order position as `ord`. -macro_rules! staged_in_lane_order_ctes { - () => { - r#" - staged AS ( - SELECT *, row_number() OVER () AS ord - FROM pg_temp.awa_copy_staging - ), - staged_in_lane_order AS MATERIALIZED ( - SELECT - staged.*, - MOD( - CASE WHEN staged.ordering_hash IS NULL - THEN pg_backend_pid()::numeric - ELSE staged.ordering_hash::numeric - END, - COALESCE( - (SELECT meta.enqueue_shards FROM awa.queue_meta AS meta WHERE meta.queue = staged.queue), - 1 - )::numeric - ) AS lane_shard - FROM staged - ORDER BY staged.queue COLLATE "C", staged.priority, lane_shard, staged.ord - ) - "# - }; } /// Insert many jobs through the compatibility COPY surface. /// /// Uses a temp staging table with no constraints for fast COPY ingestion, -/// then routes staged rows through `awa.insert_job_compat`. Accepts +/// then inserts them through `awa.insert_jobs_compat_batch`, or, on a schema +/// without migration v055, through `awa.insert_job_compat` row by row. Jobs +/// whose unique key is already claimed are skipped. Accepts /// `&mut PgConnection` so callers can use pool connections or transactions /// (Transaction derefs to PgConnection). For high-volume queue-storage /// producers, prefer [`QueueStorage::enqueue_params_copy`](crate::QueueStorage::enqueue_params_copy) @@ -646,10 +766,152 @@ pub async fn insert_many_copy( copy_in.finish().await?; } - // 3. INSERT...SELECT from staging into real table + // 3. Insert the staged rows. With the batch function, unique conflicts are + // skipped in the one statement; without it, each unique row goes through + // the per-row function in its own savepoint. let has_unique = rows.iter().any(|r| r.unique_key.is_some()); + let results = if batch_function_available(conn).await? { + insert_staged_rows_with_batch_function(conn, has_unique).await? + } else { + insert_staged_rows_per_row(conn, has_unique).await? + }; + + // Keep the session-local staging table reusable across multiple COPY calls + // within the same outer transaction. + sqlx::query("DELETE FROM pg_temp.awa_copy_staging") + .execute(&mut *conn) + .await?; - let results = if has_unique { + Ok(results) +} + +/// Convenience wrapper that acquires a connection from the pool. +/// +/// Wraps the operation in a transaction so the staging rows are cleaned up at +/// commit time even if the caller does not reuse the connection afterward. +#[tracing::instrument( + skip(pool, jobs), + fields( + job.count = jobs.len(), + otel.name = "send", + otel.kind = "producer", + messaging.system = "awa", + messaging.operation.r#type = "send", + messaging.batch.message_count = jobs.len(), + ) +)] +pub async fn insert_many_copy_from_pool( + pool: &PgPool, + jobs: &[InsertParams], +) -> Result, AwaError> { + if jobs.is_empty() { + return Ok(Vec::new()); + } + + let mut tx = pool.begin().await?; + let results = insert_many_copy(&mut tx, jobs).await?; + tx.commit().await?; + + Ok(results) +} + +/// Probe this connection on every COPY call. A process can write to databases +/// at different migration levels, and the generic executor path cannot safely +/// seed a shared answer for them. +async fn batch_function_available(conn: &mut PgConnection) -> Result { + Ok(sqlx::query_scalar( + "SELECT to_regprocedure('awa.insert_jobs_compat_batch(text[],text[],jsonb[],text[],smallint[],smallint[],timestamptz[],jsonb[],jsonb[],bytea[],text[],text[],boolean)') IS NOT NULL", + ) + .fetch_one(&mut *conn) + .await?) +} + +async fn insert_staged_rows_with_batch_function( + conn: &mut PgConnection, + skip_unique_conflicts: bool, +) -> Result, AwaError> { + let result = sqlx::query_as::<_, JobRow>( + r#" + WITH staged AS ( + SELECT *, row_number() OVER () AS ord + FROM pg_temp.awa_copy_staging + ), + columns AS ( + SELECT + array_agg(kind ORDER BY ord) AS kinds, + array_agg(queue ORDER BY ord) AS queues, + array_agg(args ORDER BY ord) AS args, + array_agg(state::text ORDER BY ord) AS states, + array_agg(priority ORDER BY ord) AS priorities, + array_agg(max_attempts ORDER BY ord) AS max_attempts, + array_agg(run_at ORDER BY ord) AS run_ats, + array_agg(metadata ORDER BY ord) AS metadata, + array_agg(to_jsonb(tags) ORDER BY ord) AS tags, + array_agg(unique_key ORDER BY ord) AS unique_keys, + array_agg(unique_states::text ORDER BY ord) AS unique_states, + array_agg(ordering_hash ORDER BY ord) AS ordering_hashes + FROM staged + ) + SELECT inserted.* + FROM columns + CROSS JOIN LATERAL awa.insert_jobs_compat_batch( + columns.kinds, + columns.queues, + columns.args, + columns.states, + columns.priorities, + columns.max_attempts, + columns.run_ats, + columns.metadata, + columns.tags, + columns.unique_keys, + columns.unique_states, + columns.ordering_hashes, + $1 + ) AS inserted + "#, + ) + .bind(skip_unique_conflicts) + .fetch_all(&mut *conn) + .await?; + Ok(result) +} + +/// CTEs that number the staged rows in staging order and re-sort them into +/// canonical lane order (see "Lane lock order" above); `staged_in_lane_order` +/// carries the staging-order position as `ord`. +macro_rules! staged_in_lane_order_ctes { + () => { + r#" + staged AS ( + SELECT *, row_number() OVER () AS ord + FROM pg_temp.awa_copy_staging + ), + staged_in_lane_order AS MATERIALIZED ( + SELECT + staged.*, + MOD( + CASE WHEN staged.ordering_hash IS NULL + THEN pg_backend_pid()::numeric + ELSE staged.ordering_hash::numeric + END, + COALESCE( + (SELECT meta.enqueue_shards FROM awa.queue_meta AS meta WHERE meta.queue = staged.queue), + 1 + )::numeric + ) AS lane_shard + FROM staged + ORDER BY staged.queue COLLATE "C", staged.priority, lane_shard, staged.ord + ) + "# + }; +} + +async fn insert_staged_rows_per_row( + conn: &mut PgConnection, + has_unique: bool, +) -> Result, AwaError> { + let results: Vec = if has_unique { // The compatibility `awa.jobs` surface is now a view backed by hot and // deferred tables, so the old `ON CONFLICT` path is no longer available // here. Keep COPY for staging/parsing, then insert unique rows one at a @@ -790,43 +1052,6 @@ pub async fn insert_many_copy( .fetch_all(&mut *conn) .await? }; - - // Keep the session-local staging table reusable across multiple COPY calls - // within the same outer transaction. - sqlx::query("DELETE FROM pg_temp.awa_copy_staging") - .execute(&mut *conn) - .await?; - - Ok(results) -} - -/// Convenience wrapper that acquires a connection from the pool. -/// -/// Wraps the operation in a transaction so the staging rows are cleaned up at -/// commit time even if the caller does not reuse the connection afterward. -#[tracing::instrument( - skip(pool, jobs), - fields( - job.count = jobs.len(), - otel.name = "send", - otel.kind = "producer", - messaging.system = "awa", - messaging.operation.r#type = "send", - messaging.batch.message_count = jobs.len(), - ) -)] -pub async fn insert_many_copy_from_pool( - pool: &PgPool, - jobs: &[InsertParams], -) -> Result, AwaError> { - if jobs.is_empty() { - return Ok(Vec::new()); - } - - let mut tx = pool.begin().await?; - let results = insert_many_copy(&mut tx, jobs).await?; - tx.commit().await?; - Ok(results) } diff --git a/awa-model/src/lib.rs b/awa-model/src/lib.rs index 66a5dd4ee..7b4a4c63a 100644 --- a/awa-model/src/lib.rs +++ b/awa-model/src/lib.rs @@ -14,6 +14,7 @@ pub mod kind; pub mod migrations; pub mod output; pub mod partitioned_queue; +pub mod queue_history; pub mod queue_storage; pub mod reschedule; pub mod storage; @@ -45,7 +46,8 @@ pub use cron::{CronJobRow, CronMissedFirePolicy, PeriodicJob, PeriodicJobBuilder pub use dlq::{DlqMetadata, DlqRow, ListDlqFilter, RetryFromDlqOpts}; pub use error::{map_sqlx_error, AwaError}; pub use insert::{ - insert, insert_many, insert_many_copy, insert_many_copy_from_pool, insert_prepared, insert_with, + insert, insert_many, insert_many_copy, insert_many_copy_from_pool, + insert_many_skip_unique_conflicts, insert_prepared, insert_with, }; pub use job::{InsertOpts, InsertParams, JobRow, JobState, UniqueOpts}; pub use output::{AttemptError, JobOutput, OutputDescriptor, OutputError, MAX_OUTPUT_BYTES}; diff --git a/awa-model/src/migrations.rs b/awa-model/src/migrations.rs index 385035b65..10970880f 100644 --- a/awa-model/src/migrations.rs +++ b/awa-model/src/migrations.rs @@ -5,7 +5,7 @@ use sqlx::{Connection, PgPool}; use tracing::{info, warn}; /// Current schema version. -pub const CURRENT_VERSION: i32 = 52; +pub const CURRENT_VERSION: i32 = 58; /// Migrations that require an exclusive (no-live-runtime) upgrade window. /// @@ -292,6 +292,36 @@ const MIGRATIONS: &[(i32, &str, &[&str])] = &[ ), (51, "Per-slot claim functions", &[V23_UP, V51_UP]), (52, "Retained job output", &[V23_UP, V52_UP]), + ( + 53, + "Create queue lanes as the schema owner", + &[V23_UP, V53_UP], + ), + ( + 54, + "Planner statistics refresh for queue-storage tables autovacuum never analyses", + &[V23_UP, V54_UP], + ), + ( + 55, + "Batch producer insert with a short enqueue lane lock window", + &[V55_UP], + ), + ( + 56, + "Fewer round trips and sequence writes per queue-storage enqueue", + &[V23_UP, V56_UP], + ), + ( + 57, + "Keep aged running jobs visible in jobs_compat()", + &[V57_UP], + ), + ( + 58, + "Per-minute queue and fleet history", + &[SCHEMA_DISCOVERY_UP, V58_UP], + ), ]; const V1_UP: &str = include_str!("../migrations/v001_canonical_schema.sql"); @@ -352,6 +382,12 @@ const V49_UP: &str = include_str!("../migrations/v049_prepared_lane_sequences.sq const V50_UP: &str = include_str!("../migrations/v050_drop_ready_lane_shard_index.sql"); const V51_UP: &str = include_str!("../migrations/v051_claim_slot_functions.sql"); const V52_UP: &str = include_str!("../migrations/v052_retained_job_output.sql"); +const V53_UP: &str = include_str!("../migrations/v053_definer_lane_provisioning.sql"); +const V54_UP: &str = include_str!("../migrations/v054_analyze_queue_storage_statistics.sql"); +const V55_UP: &str = include_str!("../migrations/v055_insert_jobs_batch_compat.sql"); +const V56_UP: &str = include_str!("../migrations/v056_enqueue_round_trips.sql"); +const V57_UP: &str = include_str!("../migrations/v057_jobs_compat_priority_aging.sql"); +const V58_UP: &str = include_str!("../migrations/v058_queue_history.sql"); /// Old version numbers from pre-0.4 releases that used V3/V4/V5 numbering. /// Also tolerates the unreleased inline-V6 branch numbering used during review. @@ -417,9 +453,9 @@ pub async fn run_with_options(pool: &PgPool, options: MigrateOptions) -> Result< // only sound because every migration is transaction-safe (no // `CREATE INDEX CONCURRENTLY`, `VACUUM`, or explicit transaction control). let mut retry = 0; - loop { + let version_before = loop { match run_migration_transaction(pool, &options, lock_key).await { - Ok(()) => break, + Ok(version_before) => break version_before, Err(error) if is_migration_deadlock(&error) && retry < MIGRATION_DEADLOCK_RETRIES => { retry += 1; let delay = std::time::Duration::from_millis(50 * (1 << (retry - 1))); @@ -433,6 +469,10 @@ pub async fn run_with_options(pool: &PgPool, options: MigrateOptions) -> Result< } Err(error) => return Err(error), } + }; + + if version_before < LANE_PROVISIONING_VERSION { + repair_lane_sequences_after_migration(pool).await; } // Best-effort admin-metadata cache warmup, deliberately outside the @@ -443,29 +483,73 @@ pub async fn run_with_options(pool: &PgPool, options: MigrateOptions) -> Result< Ok(()) } +/// Returns the schema version the transaction started from. async fn run_migration_transaction( pool: &PgPool, options: &MigrateOptions, lock_key: i64, -) -> Result<(), AwaError> { +) -> Result { let mut tx = pool.begin().await?; sqlx::query("SELECT pg_advisory_xact_lock($1)") .bind(lock_key) .execute(&mut *tx) .await?; - apply_migrations(&mut tx, options).await?; + let version_before = apply_migrations(&mut tx, options).await?; tx.commit().await?; - Ok(()) + Ok(version_before) +} + +/// The migration that makes lane sequences belong to the schema owner. +const LANE_PROVISIONING_VERSION: i32 = 53; + +/// Repair existing lane sequences once v053 has committed. +/// +/// `ALTER SEQUENCE ... OWNER` takes the sequence's ACCESS EXCLUSIVE lock until +/// its transaction ends, so it cannot run inside the migration transaction +/// without stalling every enqueue and claim on those lanes until the whole +/// upgrade commits. Each sequence is repaired in its own short transaction +/// with a bounded lock wait instead. Best-effort: the schema is committed, and +/// whatever is left is reported with the command that finishes it. +/// +/// Uses a detached connection for the reason given on +/// [`warm_admin_metadata_cache`]. +async fn repair_lane_sequences_after_migration(pool: &PgPool) { + let mut conn = match pool.acquire().await { + Ok(conn) => conn.detach(), + Err(error) => { + warn!(%error, "Could not connect to repair lane sequences; run `awa storage lane-sequences --repair`"); + return; + } + }; + match crate::storage::repair_lane_sequences_in_every_schema(&mut conn).await { + Ok(repairs) => { + let repaired = repairs + .iter() + .filter(|repair| repair.outcome == crate::storage::LaneSequenceOutcome::Repaired) + .count(); + if repaired > 0 { + info!(repaired, "Gave lane sequences to the schema owner"); + } + for next_step in crate::storage::lane_sequence_repair_next_steps(&repairs) { + warn!("{next_step}"); + } + } + Err(error) => { + warn!(%error, "Could not repair lane sequences; run `awa storage lane-sequences --repair`"); + } + } + let _ = conn.close().await; } fn is_migration_deadlock(error: &AwaError) -> bool { error.is_deadlock() } +/// Returns the schema version before this run. async fn apply_migrations( conn: &mut PgConnection, options: &MigrateOptions, -) -> Result<(), AwaError> { +) -> Result { let has_schema: bool = sqlx::query_scalar("SELECT EXISTS(SELECT 1 FROM pg_namespace WHERE nspname = 'awa')") .fetch_one(&mut *conn) @@ -558,7 +642,7 @@ async fn apply_migrations( info!(version = current, "Schema is up to date"); } - Ok(()) + Ok(current) } /// Warm the admin-metadata cache after migrations commit. diff --git a/awa-model/src/queue_history.rs b/awa-model/src/queue_history.rs new file mode 100644 index 000000000..1a5528a99 --- /dev/null +++ b/awa-model/src/queue_history.rs @@ -0,0 +1,871 @@ +//! Per-minute queue and fleet history (v058, #543). +//! +//! The elected maintenance leader samples queue depth and differences the +//! cumulative per-queue counters runtimes publish in their snapshots +//! ([`crate::admin::QueueOutcomeCounters`]) into per-minute deltas. The admin +//! UI reads the result to judge whether each queue is keeping up. +//! +//! Counter deltas are `None` ("unknown") rather than a guess whenever the +//! leader cannot account for every live runtime serving a queue: on its first +//! sample, for a runtime it has not seen before that started earlier, for +//! runtimes that predate the counters, and when the interval since the last +//! sample is not about a minute. Completed, failed and retried counts are also +//! unknown wherever a callback may have resolved a parked job, since that can +//! happen in any process. A departing runtime contributes its final published +//! counters once; its unpublished tail is lost. A participant disappearing +//! without a final reading makes that interval unknown. + +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; + +use chrono::{DateTime, TimeDelta, Utc}; +use serde::Serialize; +use sqlx::PgPool; +use uuid::Uuid; + +use crate::admin::{QueueOutcomeCounters, QueueOverview, QueueRuntimeMode, RuntimeInstance}; +use crate::error::AwaError; + +/// Counter deltas for one queue over one sample interval. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize)] +pub struct QueueFlow { + pub claimed: i64, + pub completed: i64, + pub failed_attempts: i64, + pub retried: i64, + pub rate_limited: i64, + pub claim_wait_ms: i64, + pub parked: i64, +} + +/// How far a sample interval may stray from a minute and still be stored as +/// one. Workers publish counters on their own snapshot cadence, so some jitter +/// is inherent; an interval well off a minute (the first one after taking +/// leadership, or a delayed sample) would misstate a per-minute rate. +const MINUTE_TOLERANCE: TimeDelta = TimeDelta::seconds(15); + +impl QueueFlow { + fn between(previous: Option<&QueueOutcomeCounters>, current: &QueueOutcomeCounters) -> Self { + let delta = |now: u64, before: u64| i64::try_from(now - before).unwrap_or(i64::MAX); + match previous { + // Any counter going backwards means the runtime restarted (or its + // counters did); everything it reports now is new since then. + Some(prev) + if current.claimed >= prev.claimed + && current.completed >= prev.completed + && current.failed >= prev.failed + && current.retried >= prev.retried + && current.rate_limited >= prev.rate_limited + && current.claim_wait_ms >= prev.claim_wait_ms + && current.parked >= prev.parked => + { + Self { + claimed: delta(current.claimed, prev.claimed), + completed: delta(current.completed, prev.completed), + failed_attempts: delta(current.failed, prev.failed), + retried: delta(current.retried, prev.retried), + rate_limited: delta(current.rate_limited, prev.rate_limited), + claim_wait_ms: delta(current.claim_wait_ms, prev.claim_wait_ms), + parked: delta(current.parked, prev.parked), + } + } + _ => Self::between(Some(&QueueOutcomeCounters::default()), current), + } + } + + fn add(&mut self, other: Self) { + self.claimed = self.claimed.saturating_add(other.claimed); + self.completed = self.completed.saturating_add(other.completed); + self.failed_attempts = self.failed_attempts.saturating_add(other.failed_attempts); + self.retried = self.retried.saturating_add(other.retried); + self.rate_limited = self.rate_limited.saturating_add(other.rate_limited); + self.claim_wait_ms = self.claim_wait_ms.saturating_add(other.claim_wait_ms); + self.parked = self.parked.saturating_add(other.parked); + } +} + +/// What the leader remembers between samples. Lost on leader change, which +/// makes the next sample's counter deltas unknown. +#[derive(Debug, Default)] +pub struct CounterBaseline { + sampled_at: Option>, + readings: HashMap<(Uuid, String), Option>, + waiting_external: HashMap, +} + +/// Per-queue runtime coverage for one sample. +#[derive(Debug, Clone, Default, PartialEq)] +pub struct QueueCoverage { + pub flow: Option, + pub live_workers: i32, + /// Summed `max_workers` across live hard-reserved runtimes. `None` when + /// any live runtime runs the queue in weighted mode, where capacity is + /// shared and cannot be attributed to one queue. + pub capacity: Option, +} + +impl CounterBaseline { + /// Difference participating runtimes' counters against the previous sample. + /// A departing runtime's final published snapshot contributes once, without + /// counting it as live capacity. A participant missing entirely makes the + /// interval unknown rather than turning its unobserved work into zero. + pub fn advance( + &mut self, + now: DateTime, + instances: &[RuntimeInstance], + ) -> HashMap { + let mut coverage: HashMap = HashMap::new(); + let mut unknown: BTreeSet = BTreeSet::new(); + let about_a_minute = self.sampled_at.is_none_or(|sampled_at| { + ((now - sampled_at) - TimeDelta::minutes(1)).abs() <= MINUTE_TOLERANCE + }); + let mut readings = HashMap::new(); + let mut observed = HashSet::new(); + + for instance in instances { + for snapshot in &instance.queues { + let key = (instance.instance_id, snapshot.queue.clone()); + if instance.stale + && !self.readings.contains_key(&key) + && self.sampled_at.is_none_or(|at| instance.last_seen_at < at) + { + // This final snapshot was already consumed, or predates our + // first baseline. It cannot contribute to this interval. + continue; + } + observed.insert(key.clone()); + let entry = + coverage + .entry(snapshot.queue.clone()) + .or_insert_with(|| QueueCoverage { + flow: Some(QueueFlow::default()), + live_workers: 0, + capacity: Some(0), + }); + if !instance.stale { + entry.live_workers += 1; + entry.capacity = match (entry.capacity, snapshot.config.mode) { + (Some(total), QueueRuntimeMode::HardReserved) => { + Some(total + i64::from(snapshot.config.max_workers.unwrap_or(0))) + } + _ => None, + }; + // Retain missing counters too: disappearance of an older + // participant cannot make the next interval known zero. + readings.insert(key.clone(), snapshot.counters); + } + + let Some(current) = snapshot.counters else { + unknown.insert(snapshot.queue.clone()); + continue; + }; + let previous = self.readings.get(&key).and_then(Option::as_ref); + let started_since_last_sample = self + .sampled_at + .is_some_and(|sampled_at| instance.started_at >= sampled_at); + if !about_a_minute || (previous.is_none() && !started_since_last_sample) { + // Counts from before our baseline: we cannot tell how many + // fall in this interval. + unknown.insert(snapshot.queue.clone()); + } else if let Some(flow) = entry.flow.as_mut() { + flow.add(QueueFlow::between(previous, ¤t)); + } + } + } + for key in self.readings.keys().filter(|key| !observed.contains(*key)) { + unknown.insert(key.1.clone()); + } + for queue in unknown { + coverage.entry(queue).or_default().flow = None; + } + self.sampled_at = Some(now); + self.readings = readings; + coverage + } + + /// Queues where a callback may have completed, failed or retried a job + /// since the last sample: a job was parked during the interval, or one was + /// already waiting at its start. Callbacks resolve from any process (an + /// HTTP receiver, an admin call), which no runtime counts. + pub fn callback_queues( + &mut self, + queues: &[QueueOverview], + coverage: &HashMap, + ) -> BTreeSet { + let waiting: HashMap = queues + .iter() + .map(|queue| (queue.queue.clone(), queue.waiting_external)) + .collect(); + let affected = queues + .iter() + .filter(|queue| { + // With no earlier reading, assume what is waiting now was + // waiting all interval. + let waited = self + .waiting_external + .get(&queue.queue) + .copied() + .unwrap_or(queue.waiting_external) + > 0; + let parked = coverage + .get(&queue.queue) + .and_then(|c| c.flow) + .is_some_and(|flow| flow.parked > 0); + waited || parked + }) + .map(|queue| queue.queue.clone()) + .collect(); + self.waiting_external = waiting; + affected + } +} + +/// One row of `awa.queue_history`. +#[derive(Debug, Clone, PartialEq, Serialize, sqlx::FromRow)] +pub struct QueueHistoryRow { + pub queue: String, + pub bucket: DateTime, + pub available: i64, + pub running: i64, + pub scheduled: i64, + pub retryable: i64, + pub failed: i64, + pub waiting_external: i64, + pub lag_seconds: Option, + pub paused: bool, + pub claimed: Option, + pub completed: Option, + pub failed_attempts: Option, + pub retried: Option, + pub rate_limited: Option, + pub claim_wait_ms: Option, + pub live_workers: i32, + pub capacity: Option, +} + +/// One row of `awa.fleet_history`. +#[derive(Debug, Clone, PartialEq, Serialize, sqlx::FromRow)] +pub struct FleetHistoryRow { + pub bucket: DateTime, + pub live_instances: i32, + pub healthy_instances: i32, + pub stale_instances: i32, + pub leader_instance_id: Option, + pub versions: Vec, +} + +/// Build the rows for one sample from current queue depth and runtime state. +pub fn build_sample( + bucket: DateTime, + queues: &[QueueOverview], + coverage: &HashMap, + callback_queues: &BTreeSet, + instances: &[RuntimeInstance], +) -> (Vec, FleetHistoryRow) { + let rows = queues + .iter() + .map(|queue| { + let covered = coverage.get(&queue.queue); + // No participant over this interval is a measured zero. A coverage + // entry with no flow is unknown, even if no workers remain live. + let flow = covered.map_or(Some(QueueFlow::default()), |c| c.flow); + QueueHistoryRow { + queue: queue.queue.clone(), + bucket, + available: queue.available, + running: queue.running, + scheduled: queue.scheduled, + retryable: queue.retryable, + failed: queue.failed, + waiting_external: queue.waiting_external, + lag_seconds: queue.lag_seconds, + paused: queue.paused, + claimed: flow.map(|f| f.claimed), + completed: flow.map(|f| f.completed), + failed_attempts: flow.map(|f| f.failed_attempts), + retried: flow.map(|f| f.retried), + rate_limited: flow.map(|f| f.rate_limited), + claim_wait_ms: flow.map(|f| f.claim_wait_ms), + live_workers: covered.map_or(0, |c| c.live_workers), + capacity: covered.and_then(|c| c.capacity), + } + }) + .map(|mut row| { + if callback_queues.contains(&row.queue) { + row.completed = None; + row.failed_attempts = None; + row.retried = None; + } + row + }) + .collect(); + + let live: Vec<&RuntimeInstance> = instances.iter().filter(|i| !i.stale).collect(); + let versions: BTreeSet = live.iter().map(|i| i.version.clone()).collect(); + let fleet = FleetHistoryRow { + bucket, + live_instances: live.len() as i32, + healthy_instances: live.iter().filter(|i| i.healthy).count() as i32, + stale_instances: (instances.len() - live.len()) as i32, + leader_instance_id: live.iter().find(|i| i.leader).map(|i| i.instance_id), + versions: versions.into_iter().collect(), + }; + (rows, fleet) +} + +/// Whether v058 is applied. Lets callers report "no history" on an older +/// schema instead of failing. +pub async fn history_installed(pool: &PgPool) -> Result { + let installed: bool = sqlx::query_scalar( + "SELECT to_regclass('awa.queue_history') IS NOT NULL \ + AND to_regclass('awa.fleet_history') IS NOT NULL", + ) + .fetch_one(pool) + .await?; + Ok(installed) +} + +/// Write one sample. A bucket that already has rows (a second leader in the +/// same minute after failover) keeps the first write. +pub async fn record_sample( + pool: &PgPool, + rows: &[QueueHistoryRow], + fleet: &FleetHistoryRow, +) -> Result<(), AwaError> { + let mut tx = pool.begin().await?; + if !rows.is_empty() { + let column = + |f: fn(&QueueHistoryRow) -> Option| rows.iter().map(f).collect::>(); + sqlx::query( + r#" + INSERT INTO awa.queue_history ( + queue, bucket, available, running, scheduled, retryable, failed, + waiting_external, lag_seconds, paused, claimed, completed, + failed_attempts, retried, rate_limited, claim_wait_ms, + live_workers, capacity + ) + SELECT * FROM UNNEST( + $1::text[], $2::timestamptz[], $3::bigint[], $4::bigint[], $5::bigint[], + $6::bigint[], $7::bigint[], $8::bigint[], $9::float8[], $10::bool[], + $11::bigint[], $12::bigint[], $13::bigint[], $14::bigint[], $15::bigint[], + $16::bigint[], $17::int[], $18::bigint[] + ) + ON CONFLICT (queue, bucket) DO NOTHING + "#, + ) + .bind(rows.iter().map(|r| r.queue.clone()).collect::>()) + .bind(rows.iter().map(|r| r.bucket).collect::>()) + .bind(rows.iter().map(|r| r.available).collect::>()) + .bind(rows.iter().map(|r| r.running).collect::>()) + .bind(rows.iter().map(|r| r.scheduled).collect::>()) + .bind(rows.iter().map(|r| r.retryable).collect::>()) + .bind(rows.iter().map(|r| r.failed).collect::>()) + .bind(rows.iter().map(|r| r.waiting_external).collect::>()) + .bind(rows.iter().map(|r| r.lag_seconds).collect::>()) + .bind(rows.iter().map(|r| r.paused).collect::>()) + .bind(column(|r| r.claimed)) + .bind(column(|r| r.completed)) + .bind(column(|r| r.failed_attempts)) + .bind(column(|r| r.retried)) + .bind(column(|r| r.rate_limited)) + .bind(column(|r| r.claim_wait_ms)) + .bind(rows.iter().map(|r| r.live_workers).collect::>()) + .bind(column(|r| r.capacity)) + .execute(&mut *tx) + .await?; + } + sqlx::query( + r#" + INSERT INTO awa.fleet_history ( + bucket, live_instances, healthy_instances, stale_instances, + leader_instance_id, versions + ) + VALUES ($1, $2, $3, $4, $5, $6) + ON CONFLICT (bucket) DO NOTHING + "#, + ) + .bind(fleet.bucket) + .bind(fleet.live_instances) + .bind(fleet.healthy_instances) + .bind(fleet.stale_instances) + .bind(fleet.leader_instance_id) + .bind(&fleet.versions) + .execute(&mut *tx) + .await?; + tx.commit().await?; + Ok(()) +} + +/// Delete history older than `retention`. Returns rows removed. +pub async fn prune_history(pool: &PgPool, retention: std::time::Duration) -> Result { + let secs = i64::try_from(retention.as_secs()).unwrap_or(i64::MAX); + let queues = sqlx::query( + "DELETE FROM awa.queue_history WHERE bucket < now() - make_interval(secs => $1)", + ) + .bind(secs as f64) + .execute(pool) + .await? + .rows_affected(); + let fleet = sqlx::query( + "DELETE FROM awa.fleet_history WHERE bucket < now() - make_interval(secs => $1)", + ) + .bind(secs as f64) + .execute(pool) + .await? + .rows_affected(); + Ok(queues + fleet) +} + +/// Queue rows since `since`, oldest first, grouped by queue. +pub async fn queue_history_since( + pool: &PgPool, + since: DateTime, +) -> Result>, AwaError> { + let rows: Vec = sqlx::query_as( + r#" + SELECT queue, bucket, available, running, scheduled, retryable, failed, + waiting_external, lag_seconds, paused, claimed, completed, + failed_attempts, retried, rate_limited, claim_wait_ms, + live_workers, capacity + FROM awa.queue_history + WHERE bucket >= $1 + ORDER BY queue, bucket + "#, + ) + .bind(since) + .fetch_all(pool) + .await?; + let mut grouped: BTreeMap> = BTreeMap::new(); + for row in rows { + grouped.entry(row.queue.clone()).or_default().push(row); + } + Ok(grouped) +} + +/// Fleet rows since `since`, oldest first. +pub async fn fleet_history_since( + pool: &PgPool, + since: DateTime, +) -> Result, AwaError> { + Ok(sqlx::query_as( + r#" + SELECT bucket, live_instances, healthy_instances, stale_instances, + leader_instance_id, versions + FROM awa.fleet_history + WHERE bucket >= $1 + ORDER BY bucket + "#, + ) + .bind(since) + .fetch_all(pool) + .await?) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::admin::{QueueRuntimeConfigSnapshot, QueueRuntimeSnapshot, StorageCapability}; + + fn at(seconds: i64) -> DateTime { + DateTime::from_timestamp(1_800_000_000 + seconds, 0).unwrap() + } + + fn counters(claimed: u64, completed: u64) -> QueueOutcomeCounters { + QueueOutcomeCounters { + claimed, + completed, + claim_wait_ms: claimed * 10, + ..Default::default() + } + } + + fn instance( + id: u128, + started_at: DateTime, + queues: Vec<(&str, Option, QueueRuntimeMode)>, + ) -> RuntimeInstance { + RuntimeInstance { + instance_id: Uuid::from_u128(id), + hostname: None, + pid: 1, + version: "0.7.0".into(), + storage_capability: StorageCapability::QueueStorage, + transition_role: Default::default(), + started_at, + last_seen_at: started_at, + snapshot_interval_ms: 10_000, + stale: false, + healthy: true, + postgres_connected: true, + poll_loop_alive: true, + heartbeat_alive: true, + maintenance_alive: true, + shutting_down: false, + leader: id == 1, + global_max_workers: None, + queues: queues + .into_iter() + .map(|(queue, counters, mode)| QueueRuntimeSnapshot { + queue: queue.into(), + in_flight: 0, + overflow_held: None, + config: QueueRuntimeConfigSnapshot { + mode, + max_workers: Some(8), + min_workers: None, + weight: None, + global_max_workers: None, + poll_interval_ms: 200, + deadline_duration_secs: 300, + priority_aging_interval_secs: 60, + claimers: None, + claim_batch_size: None, + dlq_enabled: None, + rate_limit: None, + }, + counters, + }) + .collect(), + } + } + + const HARD: QueueRuntimeMode = QueueRuntimeMode::HardReserved; + + #[test] + fn first_sample_after_leader_start_is_unknown() { + let mut baseline = CounterBaseline::default(); + let coverage = baseline.advance( + at(60), + &[instance( + 1, + at(0), + vec![("email", Some(counters(5, 5)), HARD)], + )], + ); + assert_eq!(coverage["email"].flow, None); + assert_eq!(coverage["email"].live_workers, 1); + assert_eq!(coverage["email"].capacity, Some(8)); + } + + #[test] + fn later_samples_difference_and_sum_across_instances() { + let mut baseline = CounterBaseline::default(); + baseline.advance( + at(60), + &[ + instance(1, at(0), vec![("email", Some(counters(5, 5)), HARD)]), + instance(2, at(0), vec![("email", Some(counters(1, 0)), HARD)]), + ], + ); + let coverage = baseline.advance( + at(120), + &[ + instance(1, at(0), vec![("email", Some(counters(15, 14)), HARD)]), + instance(2, at(0), vec![("email", Some(counters(4, 4)), HARD)]), + ], + ); + let flow = coverage["email"].flow.unwrap(); + assert_eq!(flow.claimed, 13); + assert_eq!(flow.completed, 13); + assert_eq!(flow.claim_wait_ms, 130); + assert_eq!(coverage["email"].capacity, Some(16)); + } + + #[test] + fn a_restart_counts_from_zero() { + let mut baseline = CounterBaseline::default(); + baseline.advance( + at(60), + &[instance( + 1, + at(0), + vec![("email", Some(counters(50, 50)), HARD)], + )], + ); + let coverage = baseline.advance( + at(120), + &[instance( + 1, + at(0), + vec![("email", Some(counters(3, 2)), HARD)], + )], + ); + assert_eq!(coverage["email"].flow.unwrap().claimed, 3); + } + + #[test] + fn departing_runtime_contributes_its_final_snapshot_once() { + let mut baseline = CounterBaseline::default(); + baseline.advance( + at(60), + &[ + instance(1, at(0), vec![("email", Some(counters(5, 5)), HARD)]), + instance(2, at(0), vec![("email", Some(counters(10, 10)), HARD)]), + ], + ); + let mut departed = instance(2, at(0), vec![("email", Some(counters(18, 18)), HARD)]); + departed.last_seen_at = at(100); + departed.stale = true; + let coverage = baseline.advance( + at(120), + &[ + instance(1, at(0), vec![("email", Some(counters(10, 10)), HARD)]), + departed.clone(), + ], + ); + assert_eq!(coverage["email"].flow.unwrap().completed, 13); + assert_eq!(coverage["email"].live_workers, 1); + assert_eq!(coverage["email"].capacity, Some(8)); + + let coverage = baseline.advance( + at(180), + &[ + instance(1, at(0), vec![("email", Some(counters(11, 11)), HARD)]), + departed, + ], + ); + assert_eq!(coverage["email"].flow.unwrap().completed, 1); + } + + #[test] + fn a_runtime_that_starts_and_dies_between_samples_contributes_once() { + let mut baseline = CounterBaseline::default(); + baseline.advance(at(60), &[]); + let mut departed = instance(2, at(90), vec![("email", Some(counters(7, 6)), HARD)]); + departed.last_seen_at = at(100); + departed.stale = true; + let coverage = baseline.advance(at(120), &[departed.clone()]); + assert_eq!(coverage["email"].flow.unwrap().completed, 6); + assert_eq!(coverage["email"].live_workers, 0); + assert_eq!(coverage["email"].capacity, Some(0)); + assert!(baseline.advance(at(180), &[departed]).is_empty()); + } + + #[test] + fn a_missing_participant_makes_the_interval_unknown_even_with_no_live_workers() { + let mut baseline = CounterBaseline::default(); + baseline.advance( + at(60), + &[instance( + 1, + at(0), + vec![("email", Some(counters(5, 5)), HARD)], + )], + ); + let coverage = baseline.advance(at(120), &[]); + assert_eq!(coverage["email"].flow, None); + let (rows, _) = build_sample( + at(120), + &[overview("email", 0, 0)], + &coverage, + &BTreeSet::new(), + &[], + ); + assert_eq!(rows[0].live_workers, 0); + assert_eq!(rows[0].claimed, None); + assert_eq!(rows[0].completed, None); + assert!(baseline.advance(at(180), &[]).is_empty()); + } + + #[test] + fn a_departed_runtime_without_counters_cannot_turn_into_known_zero() { + let mut baseline = CounterBaseline::default(); + baseline.advance( + at(60), + &[instance( + 1, + at(0), + vec![("email", Some(counters(5, 5)), HARD)], + )], + ); + let mut departed = instance(1, at(0), vec![("email", None, HARD)]); + departed.last_seen_at = at(100); + departed.stale = true; + let coverage = baseline.advance(at(120), &[departed]); + assert_eq!(coverage["email"].flow, None); + let (rows, _) = build_sample( + at(120), + &[overview("email", 0, 0)], + &coverage, + &BTreeSet::new(), + &[], + ); + assert_eq!(rows[0].completed, None); + } + + #[test] + fn new_instances_count_only_if_they_started_after_the_baseline() { + let mut baseline = CounterBaseline::default(); + baseline.advance( + at(60), + &[instance( + 1, + at(0), + vec![("email", Some(counters(1, 1)), HARD)], + )], + ); + let fresh = baseline.advance( + at(120), + &[ + instance(1, at(0), vec![("email", Some(counters(2, 2)), HARD)]), + instance(2, at(90), vec![("email", Some(counters(7, 7)), HARD)]), + ], + ); + assert_eq!(fresh["email"].flow.unwrap().completed, 8); + + // An instance the leader first sees, but which predates the baseline, + // makes the minute unknown rather than double counting. + let unseen = baseline.advance( + at(180), + &[ + instance(1, at(0), vec![("email", Some(counters(3, 3)), HARD)]), + instance(2, at(90), vec![("email", Some(counters(7, 7)), HARD)]), + instance(3, at(10), vec![("email", Some(counters(900, 900)), HARD)]), + ], + ); + assert_eq!(unseen["email"].flow, None); + } + + #[test] + fn runtimes_without_counters_make_the_queue_unknown() { + let mut baseline = CounterBaseline::default(); + baseline.advance( + at(60), + &[instance( + 1, + at(0), + vec![("email", Some(counters(1, 1)), HARD)], + )], + ); + let coverage = baseline.advance( + at(120), + &[ + instance(1, at(0), vec![("email", Some(counters(2, 2)), HARD)]), + instance(2, at(0), vec![("email", None, HARD)]), + ], + ); + assert_eq!(coverage["email"].flow, None); + assert_eq!(coverage["email"].live_workers, 2); + } + + #[test] + fn weighted_queues_have_no_attributable_capacity() { + let mut baseline = CounterBaseline::default(); + let coverage = baseline.advance( + at(60), + &[instance( + 1, + at(0), + vec![("email", None, QueueRuntimeMode::Weighted)], + )], + ); + assert_eq!(coverage["email"].capacity, None); + } + + fn overview(name: &str, available: i64, waiting_external: i64) -> QueueOverview { + QueueOverview { + queue: name.into(), + display_name: None, + description: None, + owner: None, + docs_url: None, + tags: vec![], + extra: serde_json::Value::Null, + descriptor_last_seen_at: None, + descriptor_stale: false, + descriptor_mismatch: false, + total_queued: available + waiting_external, + scheduled: 0, + available, + retryable: 0, + running: 0, + failed: 0, + waiting_external, + completed_last_hour: 0, + lag_seconds: Some(120.0), + paused: false, + } + } + + #[test] + fn intervals_well_off_a_minute_are_unknown() { + let mut baseline = CounterBaseline::default(); + let worker = |claimed| { + [instance( + 1, + at(0), + vec![("email", Some(counters(claimed, claimed)), HARD)], + )] + }; + baseline.advance(at(57), &worker(10)); + // Leadership taken at :57, next sample just after the minute: three + // seconds of counts must not be stored as a minute's worth. + assert_eq!(baseline.advance(at(62), &worker(11))["email"].flow, None); + let on_time = baseline.advance(at(121), &worker(71)); + assert_eq!(on_time["email"].flow.unwrap().claimed, 60); + // A sample delayed well past the minute is unknown too. + assert_eq!(baseline.advance(at(230), &worker(171))["email"].flow, None); + } + + #[test] + fn callbacks_make_outcomes_unknown_but_not_claims() { + let mut baseline = CounterBaseline::default(); + let worker = |claimed, parked| { + [instance( + 1, + at(0), + vec![( + "hooks", + Some(QueueOutcomeCounters { + parked, + ..counters(claimed, 0) + }), + HARD, + )], + )] + }; + let quiet = [overview("hooks", 0, 0), overview("plain", 3, 0)]; + let coverage = baseline.advance(at(60), &worker(5, 0)); + assert!(baseline.callback_queues(&quiet, &coverage).is_empty()); + + // Parked during the interval: whatever resolves it is not counted. + let coverage = baseline.advance(at(120), &worker(9, 4)); + let parked = baseline.callback_queues(&[overview("hooks", 0, 4)], &coverage); + assert!(parked.contains("hooks")); + let (rows, _) = build_sample(at(120), &[overview("hooks", 0, 4)], &coverage, &parked, &[]); + assert_eq!(rows[0].claimed, Some(4)); + assert_eq!(rows[0].completed, None); + assert_eq!(rows[0].failed_attempts, None); + assert_eq!(rows[0].retried, None); + + // Nothing parked this minute, but jobs were waiting at its start. + let coverage = baseline.advance(at(180), &worker(9, 4)); + let resolved = baseline.callback_queues(&[overview("hooks", 0, 0)], &coverage); + assert!(resolved.contains("hooks")); + + // Once nothing waited at the start and nothing parked, counts return. + let coverage = baseline.advance(at(240), &worker(9, 4)); + assert!(baseline + .callback_queues(&[overview("hooks", 0, 0)], &coverage) + .is_empty()); + } + + #[test] + fn unserved_callback_queues_are_unknown_not_zero() { + // No worker serves the queue, yet callbacks can still finish its jobs. + let queue = overview("receiver_only", 0, 2); + let callback = BTreeSet::from(["receiver_only".to_string()]); + let (rows, _) = build_sample(at(60), &[queue], &HashMap::new(), &callback, &[]); + assert_eq!(rows[0].claimed, Some(0)); + assert_eq!(rows[0].completed, None); + } + + #[test] + fn unserved_queues_record_zero_flow() { + let queue = overview("orphan", 4, 0); + let (rows, fleet) = build_sample(at(60), &[queue], &HashMap::new(), &BTreeSet::new(), &[]); + assert_eq!(rows[0].live_workers, 0); + assert_eq!(rows[0].claimed, Some(0)); + assert_eq!(fleet.live_instances, 0); + assert!(fleet.versions.is_empty()); + } +} diff --git a/awa-model/src/queue_storage.rs b/awa-model/src/queue_storage.rs index 339001705..ae205f7e9 100644 --- a/awa-model/src/queue_storage.rs +++ b/awa-model/src/queue_storage.rs @@ -13,7 +13,7 @@ use sqlx::{PgPool, Postgres}; use std::collections::hash_map::DefaultHasher; use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; use std::hash::{Hash, Hasher}; -use std::sync::atomic::{AtomicU16, AtomicUsize, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU16, AtomicUsize, Ordering}; use std::sync::Mutex; use std::time::{Duration, Instant}; use uuid::Uuid; @@ -713,6 +713,35 @@ async fn set_prune_lock_timeout_tx( Ok(()) } +/// Commit or roll back a maintenance transaction that has set the prune +/// `lock_timeout`. +/// +/// The timeout applies to every later statement in the transaction, not only +/// the explicit `LOCK TABLE`: an insert, update or read of another relation can +/// also queue behind a conflicting lock (autovacuum's heap truncation takes +/// `ACCESS EXCLUSIVE`, for example). A lock timeout anywhere therefore rolls the +/// whole transaction back and reports `blocked`, so the caller backs off and +/// retries instead of surfacing an error. +async fn settle_prune_tx( + tx: sqlx::Transaction<'_, Postgres>, + result: Result, + blocked: T, +) -> Result { + match result { + Ok(outcome) => { + tx.commit().await.map_err(map_sqlx_error)?; + Ok(outcome) + } + Err(err) => { + let _ = tx.rollback().await; + match err { + AwaError::Database(ref db_err) if is_lock_contention_error(db_err) => Ok(blocked), + err => Err(err), + } + } + } +} + fn validate_ident(ident: &str) -> Result<(), AwaError> { let mut chars = ident.chars(); match chars.next() { @@ -2847,9 +2876,161 @@ struct ReadySegmentInsert { first_run_at: DateTime, } +/// Argument types of `{schema}.enqueue_ready_rows`, as `to_regprocedure` +/// spells them. +const ENQUEUE_READY_ROWS_ARGUMENT_TYPES: &str = "text[], smallint[], smallint[], bigint[], \ + bigint[], text[], jsonb[], smallint[], bigint[], smallint[], timestamptz[], \ + timestamptz[], timestamptz[], bytea[], text[], jsonb[], integer[], bigint[], bigint[], \ + timestamptz[], text[]"; + +/// How long an "enqueue_ready_rows is unavailable" observation stands before +/// the store probes again, so a runtime started before v056 picks the +/// function up once the migration lands. +const ENQUEUE_FUNCTION_REPROBE: Duration = Duration::from_secs(60); + +fn enqueue_function_unavailable(error: &AwaError) -> bool { + matches!(error, AwaError::Database(error) if matches!( + error.as_database_error().and_then(|db| db.code()).as_deref(), + Some("42883" | "42501") + )) +} + +/// One ready row as `{schema}.enqueue_ready_rows` takes it. Queue, priority +/// and shard come from the row's lane; `lane_seq` from the lane's +/// reservation; a `None` job id from the schema's job id sequence. +struct EnqueueReadyRow<'r> { + job_id: Option, + kind: &'r str, + args: &'r serde_json::Value, + attempt: i16, + run_lease: i64, + max_attempts: i16, + run_at: DateTime, + attempted_at: Option>, + created_at: DateTime, + unique_key: Option<&'r [u8]>, + unique_states: Option<&'r str>, + payload: &'r serde_json::Value, +} + +impl<'r> EnqueueReadyRow<'r> { + fn from_ready_row(row: &'r RuntimeReadyRow) -> Self { + Self { + job_id: row.job_id, + kind: &row.kind, + args: &row.args, + attempt: row.attempt, + run_lease: row.run_lease, + max_attempts: row.max_attempts, + run_at: row.run_at, + attempted_at: row.attempted_at, + created_at: row.created_at, + unique_key: row.unique_key.as_deref(), + unique_states: row.unique_states.as_deref(), + payload: &row.payload, + } + } + + fn from_insert(row: &'r RuntimeReadyInsert) -> Self { + Self { + job_id: Some(row.job_id), + kind: &row.kind, + args: &row.args, + attempt: row.attempt, + run_lease: row.run_lease, + max_attempts: row.max_attempts, + run_at: row.run_at, + attempted_at: row.attempted_at, + created_at: row.created_at, + unique_key: row.unique_key.as_deref(), + unique_states: row.unique_states.as_deref(), + payload: &row.payload, + } + } +} + +/// Column arrays for one `{schema}.enqueue_ready_rows` call. Lanes are pushed +/// in lock order with their rows contiguous; segment offsets are relative to +/// the start the function reserves for the lane. +#[derive(Default)] +struct EnqueueReadyRowsCall<'r> { + lane_queues: Vec<&'r str>, + lane_priorities: Vec, + lane_shards: Vec, + lane_counts: Vec, + job_ids: Vec>, + kinds: Vec<&'r str>, + args: Vec<&'r serde_json::Value>, + attempts: Vec, + run_leases: Vec, + max_attempts: Vec, + run_ats: Vec>, + attempted_ats: Vec>>, + created_ats: Vec>, + unique_keys: Vec>, + unique_states: Vec>, + payloads: Vec>, + segment_lanes: Vec, + segment_first_offsets: Vec, + segment_next_offsets: Vec, + segment_run_ats: Vec>, +} + +impl<'r> EnqueueReadyRowsCall<'r> { + fn push_lane( + &mut self, + queue: &'r str, + priority: i16, + enqueue_shard: i16, + rows: impl IntoIterator>, + ) { + let first_row = self.run_ats.len(); + for row in rows { + self.job_ids.push(row.job_id); + self.kinds.push(row.kind); + self.args.push(row.args); + self.attempts.push(row.attempt); + self.run_leases.push(row.run_lease); + self.max_attempts.push(row.max_attempts); + self.run_ats.push(row.run_at); + self.attempted_ats.push(row.attempted_at); + self.created_ats.push(row.created_at); + self.unique_keys.push(row.unique_key); + self.unique_states.push(row.unique_states); + self.payloads.push(storage_payload(row.payload)); + } + let count = self.run_ats.len() - first_row; + if count == 0 { + return; + } + self.lane_queues.push(queue); + self.lane_priorities.push(priority); + self.lane_shards.push(enqueue_shard); + self.lane_counts.push(count as i64); + + // Same split as `ready_segments_from_rows` within one lane, whose + // sequence numbers are contiguous: a new segment at each `run_at` + // change. + let lane_number = self.lane_queues.len() as i32; + let mut segment_start = 0usize; + for offset in 1..=count { + if offset == count + || self.run_ats[first_row + offset] != self.run_ats[first_row + segment_start] + { + self.segment_lanes.push(lane_number); + self.segment_first_offsets.push(segment_start as i64); + self.segment_next_offsets.push(offset as i64); + self.segment_run_ats + .push(self.run_ats[first_row + segment_start]); + segment_start = offset; + } + } + } +} + #[cfg(test)] mod ready_segment_tests { - use super::{QueueStorage, RuntimeReadyInsert}; + use super::{EnqueueReadyRow, EnqueueReadyRowsCall, QueueStorage, RuntimeReadyInsert}; use chrono::{Duration, TimeZone, Utc}; fn ready_row(lane_seq: i64, run_at: chrono::DateTime) -> RuntimeReadyInsert { @@ -2901,6 +3082,52 @@ mod ready_segment_tests { assert_eq!(ranges, vec![(1, 3, first), (3, 4, second), (4, 5, first)]); } + + #[test] + fn enqueue_function_segments_match_rust_segments_after_reservation() { + let first = Utc + .with_ymd_and_hms(2026, 6, 14, 12, 0, 0) + .single() + .expect("valid test timestamp"); + let second = first + Duration::seconds(1); + let reserved_start = 41; + let rows: Vec<_> = [first, first, second, first, first, second] + .into_iter() + .enumerate() + .map(|(offset, run_at)| ready_row(reserved_start + offset as i64, run_at)) + .collect(); + + let mut call = EnqueueReadyRowsCall::default(); + call.push_lane( + "segment-q", + 2, + 0, + rows.iter().map(EnqueueReadyRow::from_insert), + ); + let function_ranges: Vec<_> = (0..call.segment_lanes.len()) + .map(|index| { + assert_eq!(call.segment_lanes[index], 1); + ( + reserved_start + call.segment_first_offsets[index], + reserved_start + call.segment_next_offsets[index], + call.segment_run_ats[index], + ) + }) + .collect(); + let rust_ranges: Vec<_> = QueueStorage::ready_segments_from_rows(&rows) + .iter() + .map(|segment| { + ( + segment.first_lane_seq, + segment.next_lane_seq, + segment.first_run_at, + ) + }) + .collect(); + + assert_eq!(function_ranges, rust_ranges); + assert_eq!(call.lane_counts, vec![6]); + } } #[derive(Debug, Clone, sqlx::FromRow)] @@ -3461,8 +3688,22 @@ pub struct QueueStorage { /// TRUNCATEs queue_lanes; `advance_enqueue_head` repairs a stale entry /// (head row gone after a rolled-back ensure_lane). ensured_lanes: Mutex>, + /// Whether `{schema}.provision_lane` (migration v053) was callable the + /// last time a lane was created. Only a positive answer is kept: lane + /// creation is rare, and the probe rides on the lane-lock round trip. + lane_provisioner_present: AtomicBool, + /// Set once a provisioner failure has been logged. + lane_provisioner_warned: AtomicBool, slot_fill_observations: Mutex>, prune_lock_timeout: Duration, + /// Last probe of whether the schema has an executable + /// `enqueue_ready_rows` (v056), and when it ran. A missing function is + /// probed again after `ENQUEUE_FUNCTION_REPROBE`; an available one until + /// a call fails for want of it. An owned enqueue transaction then rolls + /// back and retries its original input once without the function. Cleared + /// by `prepare_schema` and `reset`. + enqueue_function: Mutex>, + enqueue_function_disabled: bool, cancel_phase_hook: CancelPhaseHookSlot, /// Whether this schema has migration v052's output columns (ADR-048), /// and when that was last checked. @@ -3516,13 +3757,25 @@ impl QueueStorage { enqueue_shards_cache: Mutex::new(HashMap::new()), claimer_gates: Mutex::new(HashMap::new()), ensured_lanes: Mutex::new(HashSet::new()), + lane_provisioner_present: AtomicBool::new(false), + lane_provisioner_warned: AtomicBool::new(false), slot_fill_observations: Mutex::new(HashMap::new()), prune_lock_timeout: DEFAULT_PRUNE_LOCK_TIMEOUT, + enqueue_function: Mutex::new(None), + enqueue_function_disabled: false, cancel_phase_hook: CancelPhaseHookSlot(None), job_output_columns: Mutex::new(None), }) } + /// Enqueue through the multi-statement path even when the schema has + /// `enqueue_ready_rows`, as a binary without the function would. + #[doc(hidden)] + pub fn without_enqueue_function(mut self) -> Self { + self.enqueue_function_disabled = true; + self + } + /// Whether this schema can store job outputs (ADR-048, migration v052). /// /// A positive answer is cached for the life of the store. A negative one @@ -3993,6 +4246,7 @@ impl QueueStorage { /// upgrade get built; the rest are no-ops. #[tracing::instrument(skip(self, pool), name = "queue_storage.prepare_schema")] pub async fn prepare_schema(&self, pool: &PgPool) -> Result<(), AwaError> { + self.forget_schema_observations(); let schema = self.schema(); let install_lock_name = format!("awa.queue_storage.install:{schema}"); let mut install_tx = pool.begin().await.map_err(map_sqlx_error)?; @@ -4539,6 +4793,7 @@ impl QueueStorage { // Clear the caches so the next ensure_lane / shard_for_enqueue / // claim calls re-observe DB state. self.clear_lane_cache(); + self.forget_schema_observations(); self.enqueue_shards_cache .lock() .expect("enqueue_shards_cache poisoned") @@ -4594,9 +4849,27 @@ impl QueueStorage { .await } - /// Provision every priority and shard of a queue before deploying a runtime - /// role without schema CREATE. Run as the schema owner/migrator. This does - /// not change enqueue routing or the configured shard count. + /// Lane rows for a caller that reserves through `advance_enqueue_head` + /// next. A cached lane needs no visibility probe here: the reserve reads + /// the head row itself and repairs a stale cache entry when it is missing. + async fn ensure_lane_for_reserve<'a>( + &self, + tx: &mut sqlx::Transaction<'a, sqlx::Postgres>, + queue: &str, + priority: i16, + enqueue_shard: i16, + ) -> Result<(), AwaError> { + if self.lane_is_cached(queue, priority, enqueue_shard) { + return Ok(()); + } + self.ensure_lane_inserts(tx, queue, priority, enqueue_shard) + .await + } + + /// Provision every priority and shard of a queue ahead of its first + /// enqueue. From migration v053 any runtime role may run it: the lanes are + /// created as the schema owner. This does not change enqueue routing or + /// the configured shard count. pub async fn prepare_queue( &self, pool: &PgPool, @@ -4637,11 +4910,35 @@ impl QueueStorage { ) -> Result<(), AwaError> { let schema = self.schema(); let lane_lock_key = format!("{schema}:{queue}:{priority}:{enqueue_shard}"); - sqlx::query("SELECT pg_advisory_xact_lock(hashtextextended($1, 0))") - .bind(lane_lock_key) - .execute(tx.as_mut()) + let provisioner_callable = if self.lane_provisioner_present.load(Ordering::Relaxed) { + sqlx::query("SELECT pg_advisory_xact_lock(hashtextextended($1, 0))") + .bind(&lane_lock_key) + .execute(tx.as_mut()) + .await + .map_err(map_sqlx_error)?; + true + } else { + sqlx::query_scalar( + r#" + SELECT COALESCE(has_function_privilege(to_regprocedure($2), 'EXECUTE'), false) + FROM (SELECT pg_advisory_xact_lock(hashtextextended($1, 0))) AS lane_lock + "#, + ) + .bind(&lane_lock_key) + .bind(format!("{schema}.provision_lane(text,smallint,smallint)")) + .fetch_one(tx.as_mut()) .await - .map_err(map_sqlx_error)?; + .map_err(map_sqlx_error)? + }; + + if provisioner_callable + && self + .provision_lane(tx, queue, priority, enqueue_shard) + .await? + { + self.mark_lane_ensured(queue, priority, enqueue_shard); + return Ok(()); + } sqlx::query(audited_sql(format!( r#" @@ -4712,6 +5009,63 @@ impl QueueStorage { Ok(()) } + /// Create the lane through `{schema}.provision_lane`, which runs as the + /// schema owner, inside a savepoint. Returns false, with the transaction + /// as it was, when the function has gone (`42883`) or the schema owner + /// may not write the lane (`42501`); the caller then writes the rows + /// itself, which is what a binary that predates the function does. + async fn provision_lane<'a>( + &self, + tx: &mut sqlx::Transaction<'a, sqlx::Postgres>, + queue: &str, + priority: i16, + enqueue_shard: i16, + ) -> Result { + let schema = self.schema(); + sqlx::query("SAVEPOINT awa_provision_lane") + .execute(tx.as_mut()) + .await + .map_err(map_sqlx_error)?; + let provisioned = sqlx::query(audited_sql(format!( + "SELECT {schema}.provision_lane($1, $2, $3)" + ))) + .bind(queue) + .bind(priority) + .bind(enqueue_shard) + .execute(tx.as_mut()) + .await; + match provisioned { + Ok(_) => { + sqlx::query("RELEASE SAVEPOINT awa_provision_lane") + .execute(tx.as_mut()) + .await + .map_err(map_sqlx_error)?; + self.lane_provisioner_present.store(true, Ordering::Relaxed); + Ok(true) + } + Err(sqlx::Error::Database(db_err)) + if matches!(db_err.code().as_deref(), Some("42883" | "42501")) => + { + sqlx::query("ROLLBACK TO SAVEPOINT awa_provision_lane") + .execute(tx.as_mut()) + .await + .map_err(map_sqlx_error)?; + self.lane_provisioner_present + .store(false, Ordering::Relaxed); + if !self.lane_provisioner_warned.swap(true, Ordering::Relaxed) { + tracing::warn!( + schema, + code = db_err.code().as_deref().unwrap_or_default(), + error = %db_err.message(), + "Lane provisioning function failed; creating lane rows as the connected role" + ); + } + Ok(false) + } + Err(err) => Err(map_sqlx_error(err)), + } + } + /// Pick the enqueue shard for a row on `queue`. /// /// Reads `awa.queue_meta.enqueue_shards` (default 1, cached @@ -4803,6 +5157,139 @@ impl QueueStorage { .remove(&(queue.to_string(), priority, enqueue_shard)); } + fn forget_schema_observations(&self) { + *self + .enqueue_function + .lock() + .expect("enqueue function observation mutex") = None; + } + + async fn enqueue_function_available( + &self, + executor: impl sqlx::PgExecutor<'_>, + ) -> Result { + if self.enqueue_function_disabled { + return Ok(false); + } + let observed = *self + .enqueue_function + .lock() + .expect("enqueue function observation mutex"); + match observed { + Some((true, _)) => return Ok(true), + Some((false, at)) if at.elapsed() < ENQUEUE_FUNCTION_REPROBE => return Ok(false), + _ => {} + } + // The function calls ring_cursor, whose EXECUTE is revoked from PUBLIC. + let available: bool = sqlx::query_scalar( + "SELECT COALESCE(has_function_privilege(to_regprocedure($1), 'EXECUTE'), FALSE) \ + AND COALESCE(has_function_privilege(to_regprocedure($2), 'EXECUTE'), FALSE)", + ) + .bind(format!( + "{}.enqueue_ready_rows({ENQUEUE_READY_ROWS_ARGUMENT_TYPES})", + self.schema() + )) + .bind(format!("{}.ring_cursor(text)", self.schema())) + .fetch_one(executor) + .await + .map_err(map_sqlx_error)?; + *self + .enqueue_function + .lock() + .expect("enqueue function observation mutex") = Some((available, Instant::now())); + Ok(available) + } + + async fn execute_enqueue_ready_rows( + &self, + connection: &mut sqlx::PgConnection, + call: &EnqueueReadyRowsCall<'_>, + notify_channels: &[String], + ) -> Result { + let schema = self.schema(); + let inserted: i64 = sqlx::query_scalar(audited_sql(format!( + r#" + SELECT {schema}.enqueue_ready_rows( + $1::text[], $2::smallint[], $3::smallint[], $4::bigint[], + $5::bigint[], $6::text[], $7::jsonb[], $8::smallint[], $9::bigint[], + $10::smallint[], $11::timestamptz[], $12::timestamptz[], $13::timestamptz[], + $14::bytea[], $15::text[], $16::jsonb[], + $17::integer[], $18::bigint[], $19::bigint[], $20::timestamptz[], + $21::text[] + ) + "# + ))) + .bind(&call.lane_queues) + .bind(&call.lane_priorities) + .bind(&call.lane_shards) + .bind(&call.lane_counts) + .bind(&call.job_ids) + .bind(&call.kinds) + .bind(&call.args) + .bind(&call.attempts) + .bind(&call.run_leases) + .bind(&call.max_attempts) + .bind(&call.run_ats) + .bind(&call.attempted_ats) + .bind(&call.created_ats) + .bind(&call.unique_keys) + .bind(&call.unique_states) + .bind(&call.payloads) + .bind(&call.segment_lanes) + .bind(&call.segment_first_offsets) + .bind(&call.segment_next_offsets) + .bind(&call.segment_run_ats) + .bind(notify_channels) + .fetch_one(connection) + .await + .map_err(|err| { + // The probe can have run in a transaction that created the + // function and rolled back, or privileges can since have been + // revoked; forget it so the next enqueue probes again. + if matches!( + err.as_database_error().and_then(|db| db.code()).as_deref(), + Some("42883" | "42501") + ) { + self.forget_schema_observations(); + } + map_sqlx_error(err) + })?; + Ok(inserted as usize) + } + + /// Ready-only enqueue with no uniqueness claims in one autocommit + /// statement: lane reservation, ring cursor, ready rows, segments and + /// NOTIFY all run inside `enqueue_ready_rows`. + async fn enqueue_ready_rows_autocommit( + &self, + pool: &PgPool, + rows: Vec, + ) -> Result { + let notify_channels = self.notify_channels(rows.iter().map(|row| row.queue.clone())); + let mut connection = pool.acquire().await.map_err(map_sqlx_error)?; + let grouped = self + .group_ready_rows_by_shard(&mut connection, rows) + .await?; + let mut call = EnqueueReadyRowsCall::default(); + for ((queue, priority, enqueue_shard), lane_rows) in &grouped { + call.push_lane( + queue, + *priority, + *enqueue_shard, + lane_rows.iter().map(EnqueueReadyRow::from_ready_row), + ); + } + self.execute_enqueue_ready_rows(&mut connection, &call, ¬ify_channels) + .await + } + + fn ready_rows_take_unique_claims(rows: &[RuntimeReadyRow]) -> bool { + rows.iter().any(|row| { + row.unique_key.is_some() + && unique_state_claims(row.unique_states.as_deref(), JobState::Available) + }) + } + fn clear_lane_cache(&self) { self.ensured_lanes .lock() @@ -5109,16 +5596,41 @@ impl QueueStorage { } } - /// Current queue-ring cursor for the Rust-side enqueue paths, honouring - /// the schema's authority (#371 staged upgrade). Reads the authority row - /// (single cached page) then the O(1) cursor; callers that already hold - /// an authority for the transaction should use `ring_cursor_tx` directly. + /// Current queue-ring cursor for the Rust-side enqueue paths, under the + /// authority in force for this statement (#371 staged upgrade), read in + /// one round trip. Callers that already hold an authority for the + /// transaction should use `ring_cursor_tx` directly. async fn current_queue_ring<'a>( &self, tx: &mut sqlx::Transaction<'a, sqlx::Postgres>, ) -> Result<(i32, i64), AwaError> { - let authority = self.ring_authority_tx(tx).await?; - self.ring_cursor_tx(tx, RingFamily::Queue, authority).await + let schema = self.schema(); + sqlx::query_as(audited_sql(format!( + r#" + WITH authority AS ( + SELECT authority = 'ledger' AS ledger + FROM {schema}.ring_cursor_authority + WHERE singleton = TRUE + ) + ( + SELECT ledger.slot, ledger.generation + FROM {schema}.queue_ring_rotations AS ledger + WHERE (SELECT authority.ledger FROM authority) + ORDER BY ledger.generation DESC + LIMIT 1 + ) + UNION ALL + ( + SELECT state.current_slot, state.generation + FROM {schema}.queue_ring_state AS state + WHERE state.singleton = TRUE + AND NOT (SELECT authority.ledger FROM authority) + ) + "# + ))) + .fetch_one(tx.as_mut()) + .await + .map_err(map_sqlx_error) } async fn next_job_ids<'a>( @@ -5602,12 +6114,13 @@ impl QueueStorage { tx: &mut sqlx::Transaction<'a, sqlx::Postgres>, rows: Vec, unique_claims_synced: bool, + allow_enqueue_function: bool, ) -> Result { if rows.is_empty() { return Ok(0); } - let grouped = self.group_ready_rows_by_shard(tx, rows).await?; + let grouped = self.group_ready_rows_by_shard(tx.as_mut(), rows).await?; let total_rows: usize = grouped.values().map(Vec::len).sum(); let missing_job_ids = grouped .values() @@ -5664,8 +6177,13 @@ impl QueueStorage { self.sync_ready_enqueue_unique_claims(tx, &ready_rows) .await?; } + if allow_enqueue_function && self.enqueue_function_available(tx.as_mut()).await? { + return self + .enqueue_lane_ranges_tx(tx, &lane_ranges, &ready_rows) + .await; + } for (queue, priority, enqueue_shard, range_start, range_end) in lane_ranges { - self.ensure_lane(tx, &queue, priority, enqueue_shard) + self.ensure_lane_for_reserve(tx, &queue, priority, enqueue_shard) .await?; let count = (range_end - range_start) as i64; @@ -5683,6 +6201,27 @@ impl QueueStorage { Ok(total_rows) } + async fn enqueue_lane_ranges_tx<'a>( + &self, + tx: &mut sqlx::Transaction<'a, sqlx::Postgres>, + lane_ranges: &[(String, i16, i16, usize, usize)], + ready_rows: &[RuntimeReadyInsert], + ) -> Result { + let mut call = EnqueueReadyRowsCall::default(); + for (queue, priority, enqueue_shard, range_start, range_end) in lane_ranges { + call.push_lane( + queue, + *priority, + *enqueue_shard, + ready_rows[*range_start..*range_end] + .iter() + .map(EnqueueReadyRow::from_insert), + ); + } + self.execute_enqueue_ready_rows(tx.as_mut(), &call, &[]) + .await + } + /// Re-group rows by `(queue, priority, enqueue_shard)` so each /// resulting bucket targets exactly one head row. /// @@ -5708,9 +6247,9 @@ impl QueueStorage { /// order callers reserve lanes in. It is the canonical lane-lock order /// that `insert_many` and `insert_many_copy` also follow, so batches /// from either path cannot deadlock on each other's lane locks. - async fn group_ready_rows_by_shard<'a>( + async fn group_ready_rows_by_shard( &self, - tx: &mut sqlx::Transaction<'a, sqlx::Postgres>, + connection: &mut sqlx::PgConnection, rows: Vec, ) -> Result>, AwaError> { // Partition by (queue, priority) first so the rotor pick for @@ -5730,7 +6269,7 @@ impl QueueStorage { for row in bucket { if row.ordering_key.is_some() { let shard = self - .shard_for_enqueue(tx.as_mut(), &queue, row.ordering_key.as_deref()) + .shard_for_enqueue(&mut *connection, &queue, row.ordering_key.as_deref()) .await?; grouped .entry((queue.clone(), priority, shard)) @@ -5741,7 +6280,9 @@ impl QueueStorage { } } if !rotor_rows.is_empty() { - let shard = self.shard_for_enqueue(tx.as_mut(), &queue, None).await?; + let shard = self + .shard_for_enqueue(&mut *connection, &queue, None) + .await?; grouped .entry((queue.clone(), priority, shard)) .or_default() @@ -5756,12 +6297,13 @@ impl QueueStorage { tx: &mut sqlx::Transaction<'a, sqlx::Postgres>, rows: Vec, unique_claims_synced: bool, + allow_enqueue_function: bool, ) -> Result { if rows.is_empty() { return Ok(0); } - let grouped = self.group_ready_rows_by_shard(tx, rows).await?; + let grouped = self.group_ready_rows_by_shard(tx.as_mut(), rows).await?; let total_rows: usize = grouped.values().map(Vec::len).sum(); let missing_job_ids = grouped @@ -5819,8 +6361,16 @@ impl QueueStorage { self.sync_ready_enqueue_unique_claims(tx, &ready_rows) .await?; } + if allow_enqueue_function + && ready_rows.len() < COPY_MIN_ROWS + && self.enqueue_function_available(tx.as_mut()).await? + { + return self + .enqueue_lane_ranges_tx(tx, &lane_ranges, &ready_rows) + .await; + } for (queue, priority, enqueue_shard, range_start, range_end) in lane_ranges { - self.ensure_lane(tx, &queue, priority, enqueue_shard) + self.ensure_lane_for_reserve(tx, &queue, priority, enqueue_shard) .await?; let count = (range_end - range_start) as i64; @@ -5887,7 +6437,7 @@ impl QueueStorage { // retries must re-enqueue with the original key from the // application layer. let enqueue_shard = self.shard_for_enqueue(tx.as_mut(), &queue, None).await?; - self.ensure_lane(tx, &queue, priority, enqueue_shard) + self.ensure_lane_for_reserve(tx, &queue, priority, enqueue_shard) .await?; let count = lane_rows.len() as i64; @@ -6780,15 +7330,36 @@ impl QueueStorage { &self, pool: &PgPool, rows: Vec, + ) -> Result { + match self.enqueue_runtime_rows_attempt(pool, &rows, true).await { + Err(error) if enqueue_function_unavailable(&error) => { + self.enqueue_runtime_rows_attempt(pool, &rows, false).await + } + result => result, + } + } + + async fn enqueue_runtime_rows_attempt( + &self, + pool: &PgPool, + rows: &[RuntimeReadyRow], + allow_enqueue_function: bool, ) -> Result { if rows.is_empty() { return Ok(0); } let mut tx = pool.begin().await.map_err(map_sqlx_error)?; - let total_rows = self - .insert_ready_rows_tx(&mut tx, rows.clone(), false) - .await?; + let inserted = self + .insert_ready_rows_tx(&mut tx, rows.to_vec(), false, allow_enqueue_function) + .await; + let total_rows = match inserted { + Ok(count) => count, + Err(error) => { + tx.rollback().await.map_err(map_sqlx_error)?; + return Err(error); + } + }; let queues_to_notify: Vec = rows.iter().map(|row| row.queue.clone()).collect(); self.notify_queues_tx(&mut tx, queues_to_notify).await?; @@ -6843,6 +7414,20 @@ impl QueueStorage { &self, pool: &PgPool, jobs: &[InsertParams], + ) -> Result { + match self.enqueue_params_batch_attempt(pool, jobs, true).await { + Err(error) if enqueue_function_unavailable(&error) => { + self.enqueue_params_batch_attempt(pool, jobs, false).await + } + result => result, + } + } + + async fn enqueue_params_batch_attempt( + &self, + pool: &PgPool, + jobs: &[InsertParams], + allow_enqueue_function: bool, ) -> Result { if jobs.is_empty() { return Ok(0); @@ -6904,6 +7489,14 @@ impl QueueStorage { } } + if allow_enqueue_function + && deferred_rows.is_empty() + && !Self::ready_rows_take_unique_claims(&ready_rows) + && self.enqueue_function_available(pool).await? + { + return self.enqueue_ready_rows_autocommit(pool, ready_rows).await; + } + let queues_to_notify: Vec = ready_rows.iter().map(|row| row.queue.clone()).collect(); @@ -6926,11 +7519,18 @@ impl QueueStorage { let mut total = 0usize; if !ready_rows.is_empty() { - total += self - .insert_ready_rows_tx(&mut tx, ready_rows.clone(), true) - .await?; - } - if !deferred_rows.is_empty() { + let inserted = self + .insert_ready_rows_tx(&mut tx, ready_rows.clone(), true, allow_enqueue_function) + .await; + total += match inserted { + Ok(count) => count, + Err(error) => { + tx.rollback().await.map_err(map_sqlx_error)?; + return Err(error); + } + }; + } + if !deferred_rows.is_empty() { total += self .execute_deferred_inserts_tx(&mut tx, &deferred_rows) .await?; @@ -6953,6 +7553,20 @@ impl QueueStorage { &self, pool: &PgPool, jobs: &[InsertParams], + ) -> Result { + match self.enqueue_params_copy_attempt(pool, jobs, true).await { + Err(error) if enqueue_function_unavailable(&error) => { + self.enqueue_params_copy_attempt(pool, jobs, false).await + } + result => result, + } + } + + async fn enqueue_params_copy_attempt( + &self, + pool: &PgPool, + jobs: &[InsertParams], + allow_enqueue_function: bool, ) -> Result { if jobs.is_empty() { return Ok(0); @@ -7014,6 +7628,15 @@ impl QueueStorage { } } + if allow_enqueue_function + && deferred_rows.is_empty() + && ready_rows.len() < COPY_MIN_ROWS + && !Self::ready_rows_take_unique_claims(&ready_rows) + && self.enqueue_function_available(pool).await? + { + return self.enqueue_ready_rows_autocommit(pool, ready_rows).await; + } + let queues_to_notify: Vec = ready_rows.iter().map(|row| row.queue.clone()).collect(); @@ -7036,9 +7659,16 @@ impl QueueStorage { .await?; if !ready_rows.is_empty() { - total += self - .insert_ready_rows_copy_tx(&mut tx, ready_rows, true) - .await?; + let inserted = self + .insert_ready_rows_copy_tx(&mut tx, ready_rows, true, allow_enqueue_function) + .await; + total += match inserted { + Ok(count) => count, + Err(error) => { + tx.rollback().await.map_err(map_sqlx_error)?; + return Err(error); + } + }; } if !deferred_rows.is_empty() { total += self @@ -11666,20 +12296,25 @@ impl QueueStorage { .map_err(map_sqlx_error) } + /// Logical-queue NOTIFY channels, deduplicated so a transaction never + /// emits the same NOTIFY twice. + fn notify_channels(&self, queues: impl IntoIterator) -> Vec { + queues + .into_iter() + .map(|queue| format!("awa:{}", self.logical_queue_name(&queue))) + .collect::>() + .into_iter() + .collect() + } + async fn notify_queues_tx<'a>( &self, tx: &mut sqlx::Transaction<'a, sqlx::Postgres>, queues: impl IntoIterator, ) -> Result<(), AwaError> { - // BTreeSet dedups so we never emit the same NOTIFY twice per - // transaction. Multi-queue enqueues fold into a single round-trip via + // Multi-queue enqueues fold into a single round-trip via // `unnest($1::text[])` rather than one round-trip per channel. - let channels: Vec = queues - .into_iter() - .map(|queue| format!("awa:{}", self.logical_queue_name(&queue))) - .collect::>() - .into_iter() - .collect(); + let channels = self.notify_channels(queues); if channels.is_empty() { return Ok(()); } @@ -18163,7 +18798,10 @@ impl QueueStorage { break; } TerminalDeltaSlotRollup::Blocked => { + // The next slot would most likely wait out the same lock + // timeout, and the leader loop is blocked while it does. outcome.blocked_slots += 1; + break; } } } @@ -18350,237 +18988,215 @@ impl QueueStorage { return Ok(TerminalDeltaSlotRollup::SkippedActive); } - set_prune_lock_timeout_tx(&mut tx, self.prune_lock_timeout).await?; - - let lock_delta = sqlx::query(audited_sql(format!( - "LOCK TABLE {delta_child} IN ACCESS EXCLUSIVE MODE" - ))) - .execute(tx.as_mut()) - .await; - - match lock_delta { - Ok(_) => {} - Err(err) if is_lock_contention_error(&err) => { - let _ = tx.rollback().await; - return Ok(TerminalDeltaSlotRollup::Blocked); - } - Err(err) => { - let _ = tx.rollback().await; - return Err(map_sqlx_error(err)); - } - } - - let active_leases = - queue_prune_has_active_leases_tx(&mut tx, schema, slot, generation).await?; + let locked: Result = async { + set_prune_lock_timeout_tx(&mut tx, self.prune_lock_timeout).await?; - if active_leases { - tx.commit().await.map_err(map_sqlx_error)?; - return Ok(TerminalDeltaSlotRollup::SkippedActive); - } + sqlx::query(audited_sql(format!( + "LOCK TABLE {delta_child} IN ACCESS EXCLUSIVE MODE" + ))) + .execute(tx.as_mut()) + .await + .map_err(map_sqlx_error)?; - let unclosed_claim_refs = - queue_prune_has_unclosed_claim_refs_tx(&mut tx, schema, slot, generation).await?; + let active_leases = + queue_prune_has_active_leases_tx(&mut tx, schema, slot, generation).await?; - if unclosed_claim_refs { - tx.commit().await.map_err(map_sqlx_error)?; - return Ok(TerminalDeltaSlotRollup::SkippedActive); - } + if active_leases { + return Ok(TerminalDeltaSlotRollup::SkippedActive); + } - let delta_rows: i64 = sqlx::query_scalar(audited_sql(format!( - r#" - SELECT count(*)::bigint - FROM {delta_child} - WHERE ready_generation = $1 - "# - ))) - .bind(generation) - .fetch_one(tx.as_mut()) - .await - .map_err(map_sqlx_error)?; + let unclosed_claim_refs = + queue_prune_has_unclosed_claim_refs_tx(&mut tx, schema, slot, generation).await?; - if delta_rows == 0 { - tx.commit().await.map_err(map_sqlx_error)?; - return Ok(TerminalDeltaSlotRollup::Empty); - } + if unclosed_claim_refs { + return Ok(TerminalDeltaSlotRollup::SkippedActive); + } - // In ledger authority each touched lane collapses to one bucket-0 row - // holding its old total (summed over every bucket) plus the signed - // delta, which keeps a negative delta exact when the matching positive - // count sits in another bucket. Before the flip, N-1 maintenance can - // still win the election and folds per job-id bucket, so the fold - // keeps that shape until the flip fences it (ADR-041). - let grouped_keys: i64 = match authority { - RingAuthority::Ledger => sqlx::query_scalar(audited_sql(format!( - r#" - WITH grouped AS MATERIALIZED ( - SELECT - ready_slot, - queue, - priority, - enqueue_shard, - SUM(terminal_delta)::bigint AS delta - FROM {delta_child} - WHERE ready_generation = $1 - GROUP BY ready_slot, queue, priority, enqueue_shard - HAVING SUM(terminal_delta) <> 0 - ), - collapsed AS ( - DELETE FROM {schema}.queue_terminal_live_counts AS counts - USING grouped - WHERE counts.ready_slot = grouped.ready_slot - AND counts.queue = grouped.queue - AND counts.priority = grouped.priority - AND counts.enqueue_shard = grouped.enqueue_shard - RETURNING - counts.ready_slot, - counts.queue, - counts.priority, - counts.enqueue_shard, - counts.live_terminal_count - ), - folded AS ( - SELECT - grouped.ready_slot, - grouped.queue, - grouped.priority, - grouped.enqueue_shard, - (grouped.delta + COALESCE(SUM(collapsed.live_terminal_count), 0))::bigint - AS live_terminal_count - FROM grouped - LEFT JOIN collapsed - ON collapsed.ready_slot = grouped.ready_slot - AND collapsed.queue = grouped.queue - AND collapsed.priority = grouped.priority - AND collapsed.enqueue_shard = grouped.enqueue_shard - GROUP BY - grouped.ready_slot, - grouped.queue, - grouped.priority, - grouped.enqueue_shard, - grouped.delta - ), - inserted AS ( - INSERT INTO {schema}.queue_terminal_live_counts ( - ready_slot, - queue, - priority, - enqueue_shard, - counter_bucket, - live_terminal_count - ) - SELECT - ready_slot, - queue, - priority, - enqueue_shard, - 0::smallint, - live_terminal_count - FROM folded - WHERE live_terminal_count <> 0 - ORDER BY ready_slot, queue, priority, enqueue_shard - RETURNING 1 - ) - SELECT count(*)::bigint FROM grouped - "# - ))) - .bind(generation) - .fetch_one(tx.as_mut()) - .await - .map_err(map_sqlx_error)?, - RingAuthority::Columns => sqlx::query_scalar(audited_sql(format!( + let delta_rows: i64 = sqlx::query_scalar(audited_sql(format!( r#" - WITH grouped AS MATERIALIZED ( - SELECT - ready_slot, - queue, - priority, - enqueue_shard, - counter_bucket, - SUM(terminal_delta)::bigint AS delta + SELECT count(*)::bigint FROM {delta_child} WHERE ready_generation = $1 - GROUP BY ready_slot, queue, priority, enqueue_shard, counter_bucket - HAVING SUM(terminal_delta) <> 0 - ), - updated AS ( - UPDATE {schema}.queue_terminal_live_counts AS counts - SET live_terminal_count = GREATEST(0, counts.live_terminal_count + grouped.delta) - FROM grouped - WHERE counts.ready_slot = grouped.ready_slot - AND counts.queue = grouped.queue - AND counts.priority = grouped.priority - AND counts.enqueue_shard = grouped.enqueue_shard - AND counts.counter_bucket = grouped.counter_bucket - RETURNING - counts.ready_slot, - counts.queue, - counts.priority, - counts.enqueue_shard, - counts.counter_bucket - ), - inserted AS ( - INSERT INTO {schema}.queue_terminal_live_counts AS counts ( - ready_slot, - queue, - priority, - enqueue_shard, - counter_bucket, - live_terminal_count - ) - SELECT - grouped.ready_slot, - grouped.queue, - grouped.priority, - grouped.enqueue_shard, - grouped.counter_bucket, - grouped.delta - FROM grouped - WHERE grouped.delta > 0 - AND NOT EXISTS ( - SELECT 1 - FROM updated - WHERE updated.ready_slot = grouped.ready_slot - AND updated.queue = grouped.queue - AND updated.priority = grouped.priority - AND updated.enqueue_shard = grouped.enqueue_shard - AND updated.counter_bucket = grouped.counter_bucket - ) - ORDER BY ready_slot, queue, priority, enqueue_shard, counter_bucket - ON CONFLICT (ready_slot, queue, priority, enqueue_shard, counter_bucket) DO UPDATE - SET live_terminal_count = - GREATEST(0, counts.live_terminal_count + EXCLUDED.live_terminal_count) - RETURNING 1 - ) - SELECT count(*)::bigint FROM grouped - "# + "# ))) .bind(generation) .fetch_one(tx.as_mut()) .await - .map_err(map_sqlx_error)?, - }; - - let truncate_delta = sqlx::query(audited_sql(format!("TRUNCATE TABLE {delta_child}"))) - .execute(tx.as_mut()) - .await; + .map_err(map_sqlx_error)?; - match truncate_delta { - Ok(_) => { - tx.commit().await.map_err(map_sqlx_error)?; - Ok(TerminalDeltaSlotRollup::Rolled { - delta_rows, - grouped_keys, - }) - } - Err(err) if is_lock_contention_error(&err) => { - let _ = tx.rollback().await; - Ok(TerminalDeltaSlotRollup::Blocked) - } - Err(err) => { - let _ = tx.rollback().await; - Err(map_sqlx_error(err)) + if delta_rows == 0 { + return Ok(TerminalDeltaSlotRollup::Empty); } + + // In ledger authority each touched lane collapses to one bucket-0 row + // holding its old total (summed over every bucket) plus the signed + // delta, which keeps a negative delta exact when the matching positive + // count sits in another bucket. Before the flip, N-1 maintenance can + // still win the election and folds per job-id bucket, so the fold + // keeps that shape until the flip fences it (ADR-041). + let grouped_keys: i64 = match authority { + RingAuthority::Ledger => sqlx::query_scalar(audited_sql(format!( + r#" + WITH grouped AS MATERIALIZED ( + SELECT + ready_slot, + queue, + priority, + enqueue_shard, + SUM(terminal_delta)::bigint AS delta + FROM {delta_child} + WHERE ready_generation = $1 + GROUP BY ready_slot, queue, priority, enqueue_shard + HAVING SUM(terminal_delta) <> 0 + ), + collapsed AS ( + DELETE FROM {schema}.queue_terminal_live_counts AS counts + USING grouped + WHERE counts.ready_slot = grouped.ready_slot + AND counts.queue = grouped.queue + AND counts.priority = grouped.priority + AND counts.enqueue_shard = grouped.enqueue_shard + RETURNING + counts.ready_slot, + counts.queue, + counts.priority, + counts.enqueue_shard, + counts.live_terminal_count + ), + folded AS ( + SELECT + grouped.ready_slot, + grouped.queue, + grouped.priority, + grouped.enqueue_shard, + (grouped.delta + COALESCE(SUM(collapsed.live_terminal_count), 0))::bigint + AS live_terminal_count + FROM grouped + LEFT JOIN collapsed + ON collapsed.ready_slot = grouped.ready_slot + AND collapsed.queue = grouped.queue + AND collapsed.priority = grouped.priority + AND collapsed.enqueue_shard = grouped.enqueue_shard + GROUP BY + grouped.ready_slot, + grouped.queue, + grouped.priority, + grouped.enqueue_shard, + grouped.delta + ), + inserted AS ( + INSERT INTO {schema}.queue_terminal_live_counts ( + ready_slot, + queue, + priority, + enqueue_shard, + counter_bucket, + live_terminal_count + ) + SELECT + ready_slot, + queue, + priority, + enqueue_shard, + 0::smallint, + live_terminal_count + FROM folded + WHERE live_terminal_count <> 0 + ORDER BY ready_slot, queue, priority, enqueue_shard + RETURNING 1 + ) + SELECT count(*)::bigint FROM grouped + "# + ))) + .bind(generation) + .fetch_one(tx.as_mut()) + .await + .map_err(map_sqlx_error)?, + RingAuthority::Columns => sqlx::query_scalar(audited_sql(format!( + r#" + WITH grouped AS MATERIALIZED ( + SELECT + ready_slot, + queue, + priority, + enqueue_shard, + counter_bucket, + SUM(terminal_delta)::bigint AS delta + FROM {delta_child} + WHERE ready_generation = $1 + GROUP BY ready_slot, queue, priority, enqueue_shard, counter_bucket + HAVING SUM(terminal_delta) <> 0 + ), + updated AS ( + UPDATE {schema}.queue_terminal_live_counts AS counts + SET live_terminal_count = GREATEST(0, counts.live_terminal_count + grouped.delta) + FROM grouped + WHERE counts.ready_slot = grouped.ready_slot + AND counts.queue = grouped.queue + AND counts.priority = grouped.priority + AND counts.enqueue_shard = grouped.enqueue_shard + AND counts.counter_bucket = grouped.counter_bucket + RETURNING + counts.ready_slot, + counts.queue, + counts.priority, + counts.enqueue_shard, + counts.counter_bucket + ), + inserted AS ( + INSERT INTO {schema}.queue_terminal_live_counts AS counts ( + ready_slot, + queue, + priority, + enqueue_shard, + counter_bucket, + live_terminal_count + ) + SELECT + grouped.ready_slot, + grouped.queue, + grouped.priority, + grouped.enqueue_shard, + grouped.counter_bucket, + grouped.delta + FROM grouped + WHERE grouped.delta > 0 + AND NOT EXISTS ( + SELECT 1 + FROM updated + WHERE updated.ready_slot = grouped.ready_slot + AND updated.queue = grouped.queue + AND updated.priority = grouped.priority + AND updated.enqueue_shard = grouped.enqueue_shard + AND updated.counter_bucket = grouped.counter_bucket + ) + ORDER BY ready_slot, queue, priority, enqueue_shard, counter_bucket + ON CONFLICT (ready_slot, queue, priority, enqueue_shard, counter_bucket) DO UPDATE + SET live_terminal_count = + GREATEST(0, counts.live_terminal_count + EXCLUDED.live_terminal_count) + RETURNING 1 + ) + SELECT count(*)::bigint FROM grouped + "# + ))) + .bind(generation) + .fetch_one(tx.as_mut()) + .await + .map_err(map_sqlx_error)?, + }; + + sqlx::query(audited_sql(format!("TRUNCATE TABLE {delta_child}"))) + .execute(tx.as_mut()) + .await + .map_err(map_sqlx_error)?; + Ok(TerminalDeltaSlotRollup::Rolled { + delta_rows, + grouped_keys, + }) } + .await; + + settle_prune_tx(tx, locked, TerminalDeltaSlotRollup::Blocked).await } /// Fold pending `queue_terminal_rollup_deltas` rows into the @@ -18830,172 +19446,119 @@ impl QueueStorage { }); } - set_prune_lock_timeout_tx(&mut tx, self.prune_lock_timeout).await?; + let locked: Result = async { + set_prune_lock_timeout_tx(&mut tx, self.prune_lock_timeout).await?; - let lock_started = Instant::now(); - // Claims read segments before ready rows, while producers write in - // the opposite order. Queue behind segment readers, then abandon - // the transaction rather than wait with an exclusive child held. - let lock_tables = async { - sqlx::query(audited_sql(format!( - "LOCK TABLE {segment_child} IN ACCESS EXCLUSIVE MODE" - ))) - .execute(tx.as_mut()) - .await?; - sqlx::query(audited_sql(format!( - "LOCK TABLE {ready_child}, {claim_attempt_child}, {done_child}, {tomb_child}, {receipt_batch_child}, {receipt_tomb_child}, {delta_child} IN ACCESS EXCLUSIVE MODE NOWAIT" - ))) - .execute(tx.as_mut()) + let lock_started = Instant::now(); + // Claims read segments before ready rows, while producers write in + // the opposite order. Queue behind segment readers, then abandon + // the transaction rather than wait with an exclusive child held. + async { + sqlx::query(audited_sql(format!( + "LOCK TABLE {segment_child} IN ACCESS EXCLUSIVE MODE" + ))) + .execute(tx.as_mut()) + .await?; + sqlx::query(audited_sql(format!( + "LOCK TABLE {ready_child}, {claim_attempt_child}, {done_child}, {tomb_child}, {receipt_batch_child}, {receipt_tomb_child}, {delta_child} IN ACCESS EXCLUSIVE MODE NOWAIT" + ))) + .execute(tx.as_mut()) + .await + } .await - } - .await; - let lock_duration = lock_started.elapsed(); + .map_err(map_sqlx_error)?; + let lock_duration = lock_started.elapsed(); + + // The pre-lock gates are only an MVCC snapshot. If an in-flight + // producer or claimer committed while the exclusive partition + // lock waited, these post-lock gates see it before TRUNCATE. + let active_leases = + queue_prune_has_active_leases_tx(&mut tx, schema, slot, generation).await?; + if active_leases { + return Ok(PruneOutcome::SkippedActive { + slot, + reason: SkipReason::QueueActiveLeases, + count: busy_indicator(active_leases), + }); + } - if let Err(err) = lock_tables { - let _ = tx.rollback().await; - if is_lock_contention_error(&err) { - return Ok(PruneOutcome::Blocked { slot }); + let pending = queue_prune_has_pending_ready_tx(&mut tx, schema, slot, generation).await?; + if pending { + return Ok(PruneOutcome::SkippedActive { + slot, + reason: SkipReason::QueuePendingReady, + count: busy_indicator(pending), + }); } - return Err(map_sqlx_error(err)); - } - // The pre-lock gates are only an MVCC snapshot. If an in-flight - // producer or claimer committed while the exclusive partition - // lock waited, these post-lock gates see it before TRUNCATE. - let active_leases = - queue_prune_has_active_leases_tx(&mut tx, schema, slot, generation).await?; - if active_leases { - tx.commit().await.map_err(map_sqlx_error)?; - return Ok(PruneOutcome::SkippedActive { - slot, - reason: SkipReason::QueueActiveLeases, - count: busy_indicator(active_leases), - }); - } + let unclosed_claim_refs = + queue_prune_has_unclosed_claim_refs_tx(&mut tx, schema, slot, generation).await?; + if unclosed_claim_refs { + return Ok(PruneOutcome::SkippedActive { + slot, + reason: SkipReason::QueueUnclosedClaimRefs, + count: busy_indicator(unclosed_claim_refs), + }); + } - let pending = queue_prune_has_pending_ready_tx(&mut tx, schema, slot, generation).await?; - if pending { - tx.commit().await.map_err(map_sqlx_error)?; - return Ok(PruneOutcome::SkippedActive { - slot, - reason: SkipReason::QueuePendingReady, - count: busy_indicator(pending), - }); - } - - let unclosed_claim_refs = - queue_prune_has_unclosed_claim_refs_tx(&mut tx, schema, slot, generation).await?; - if unclosed_claim_refs { - tx.commit().await.map_err(map_sqlx_error)?; - return Ok(PruneOutcome::SkippedActive { - slot, - reason: SkipReason::QueueUnclosedClaimRefs, - count: busy_indicator(unclosed_claim_refs), - }); - } - - let failed_retention_secs = i64::try_from(failed_retention.as_secs()).unwrap_or(i64::MAX); - let retention_floor = failed_retention_secs > 0; - - // #337: failed terminal rows still inside the retention floor - // are re-homed into the live done segment before the TRUNCATE - // so they stay retryable. The ring rotation lock is held above, - // which serializes against rotate — the current slot (and its - // generation, read from the ledger cursor) cannot move while - // carried rows are re-inserted into it. - let carried_failed_rows = if retention_floor { - // Carried rows are widened to the self-contained synthetic - // done-row shape (the COALESCE chain mirrors - // `done_row_projection`) because the ready rows they would - // otherwise hydrate from are truncated with this slot. - // Deliberately no ON CONFLICT: lane_seq comes from - // never-resetting sequences, so a PK collision means - // corrupted terminal state and must abort the prune loudly - // rather than silently drop a terminal fact. - let carried = sqlx::query(audited_sql(format!( - r#" - INSERT INTO {schema}.done_entries ( - ready_slot, ready_generation, job_id, kind, queue, - args, state, priority, attempt, run_lease, - max_attempts, lane_seq, enqueue_shard, run_at, - attempted_at, finalized_at, created_at, unique_key, - unique_states, payload - ) - SELECT - $2, - $3, - done.job_id, - done.kind, - done.queue, - COALESCE(done.args, ready.args, '{{}}'::jsonb), - done.state, - done.priority, - done.attempt, - done.run_lease, - COALESCE(done.max_attempts, ready.max_attempts, 25::smallint), - done.lane_seq, - done.enqueue_shard, - COALESCE(done.run_at, ready.run_at, done.finalized_at), - COALESCE(done.attempted_at, ready.attempted_at), - done.finalized_at, - COALESCE(done.created_at, ready.created_at, done.finalized_at), - COALESCE(done.unique_key, ready.unique_key), - COALESCE(done.unique_states, ready.unique_states), - COALESCE(done.payload, ready.payload, '{{}}'::jsonb) - FROM {done_child} AS done - LEFT JOIN {ready_child} AS ready - ON ready.ready_slot = done.ready_slot - AND ready.ready_generation = done.ready_generation - AND ready.queue = done.queue - AND ready.priority = done.priority - AND ready.enqueue_shard = done.enqueue_shard - AND ready.lane_seq = done.lane_seq - WHERE done.state = 'failed' - AND done.finalized_at >= now() - make_interval(secs => $1::bigint) - "# - ))) - .bind(failed_retention_secs) - .bind(state.0) - .bind(current_generation) - .execute(tx.as_mut()) - .await - .map_err(map_sqlx_error)? - .rows_affected(); - - if carried > 0 { - // This slot's live counter rows and delta segment are - // wiped below, so the carried rows' exact-count - // evidence moves with them: re-append positive deltas - // under the current slot, grouped exactly like the - // completion path's delta append. - sqlx::query(audited_sql(format!( + let failed_retention_secs = i64::try_from(failed_retention.as_secs()).unwrap_or(i64::MAX); + let retention_floor = failed_retention_secs > 0; + + // #337: failed terminal rows still inside the retention floor + // are re-homed into the live done segment before the TRUNCATE + // so they stay retryable. The ring rotation lock is held above, + // which serializes against rotate — the current slot (and its + // generation, read from the ledger cursor) cannot move while + // carried rows are re-inserted into it. + let carried_failed_rows = if retention_floor { + // Carried rows are widened to the self-contained synthetic + // done-row shape (the COALESCE chain mirrors + // `done_row_projection`) because the ready rows they would + // otherwise hydrate from are truncated with this slot. + // Deliberately no ON CONFLICT: lane_seq comes from + // never-resetting sequences, so a PK collision means + // corrupted terminal state and must abort the prune loudly + // rather than silently drop a terminal fact. + let carried = sqlx::query(audited_sql(format!( r#" - INSERT INTO {schema}.queue_terminal_count_deltas ( - ready_slot, - ready_generation, - queue, - priority, - enqueue_shard, - counter_bucket, - terminal_delta + INSERT INTO {schema}.done_entries ( + ready_slot, ready_generation, job_id, kind, queue, + args, state, priority, attempt, run_lease, + max_attempts, lane_seq, enqueue_shard, run_at, + attempted_at, finalized_at, created_at, unique_key, + unique_states, payload ) SELECT $2, $3, - queue, - priority, - enqueue_shard, - mod( - mod(job_id, {TERMINAL_COUNTER_BUCKETS}::bigint) - + {TERMINAL_COUNTER_BUCKETS}::bigint, - {TERMINAL_COUNTER_BUCKETS}::bigint - )::smallint AS counter_bucket, - count(*)::bigint - FROM {done_child} - WHERE state = 'failed' - AND finalized_at >= now() - make_interval(secs => $1::bigint) - GROUP BY queue, priority, enqueue_shard, counter_bucket - ORDER BY queue, priority, enqueue_shard, counter_bucket + done.job_id, + done.kind, + done.queue, + COALESCE(done.args, ready.args, '{{}}'::jsonb), + done.state, + done.priority, + done.attempt, + done.run_lease, + COALESCE(done.max_attempts, ready.max_attempts, 25::smallint), + done.lane_seq, + done.enqueue_shard, + COALESCE(done.run_at, ready.run_at, done.finalized_at), + COALESCE(done.attempted_at, ready.attempted_at), + done.finalized_at, + COALESCE(done.created_at, ready.created_at, done.finalized_at), + COALESCE(done.unique_key, ready.unique_key), + COALESCE(done.unique_states, ready.unique_states), + COALESCE(done.payload, ready.payload, '{{}}'::jsonb) + FROM {done_child} AS done + LEFT JOIN {ready_child} AS ready + ON ready.ready_slot = done.ready_slot + AND ready.ready_generation = done.ready_generation + AND ready.queue = done.queue + AND ready.priority = done.priority + AND ready.enqueue_shard = done.enqueue_shard + AND ready.lane_seq = done.lane_seq + WHERE done.state = 'failed' + AND done.finalized_at >= now() - make_interval(secs => $1::bigint) "# ))) .bind(failed_retention_secs) @@ -19003,187 +19566,232 @@ impl QueueStorage { .bind(current_generation) .execute(tx.as_mut()) .await - .map_err(map_sqlx_error)?; - } - carried - } else { - 0 - }; - - // #290: scan the about-to-be-truncated partition for the rollup - // fold. The rollup column is *permanent* state, so we MUST fold - // from ground truth (the `{done_child}` partition itself), not - // from `queue_terminal_live_counts` — the counter can drift - // briefly during a rolling upgrade from a pre-counter binary - // and folding drift into the rollup would bake it in forever. - // The counter rows for this slot are still cleaned up after the - // truncate; reads from the counter (queue_counts_exact in #305) - // may transiently disagree with the rollup until the operator - // runs `awa storage rebuild-terminal-counters`, but the rollup - // itself stays authoritative. See PR #304 reviewer finding - // "High: A1 can persist stale counter state before the - // read-switch PR" for the trade-off. - // - // Carried failed rows are excluded from both columns: they are - // still live in `done_entries`, so folding them into the - // permanent rollup would double-count them. - let pruned_terminal_counts: Vec<(String, i16, i64, i64)> = - sqlx::query_as(audited_sql(format!( - r#" - WITH done_counts AS ( - SELECT - queue, - priority, - (count(*) FILTER (WHERE state <> 'failed'))::bigint AS completed, - (count(*) FILTER ( + .map_err(map_sqlx_error)? + .rows_affected(); + + if carried > 0 { + // This slot's live counter rows and delta segment are + // wiped below, so the carried rows' exact-count + // evidence moves with them: re-append positive deltas + // under the current slot, grouped exactly like the + // completion path's delta append. + sqlx::query(audited_sql(format!( + r#" + INSERT INTO {schema}.queue_terminal_count_deltas ( + ready_slot, + ready_generation, + queue, + priority, + enqueue_shard, + counter_bucket, + terminal_delta + ) + SELECT + $2, + $3, + queue, + priority, + enqueue_shard, + mod( + mod(job_id, {TERMINAL_COUNTER_BUCKETS}::bigint) + + {TERMINAL_COUNTER_BUCKETS}::bigint, + {TERMINAL_COUNTER_BUCKETS}::bigint + )::smallint AS counter_bucket, + count(*)::bigint + FROM {done_child} WHERE state = 'failed' - AND ( - $2::boolean IS FALSE - OR finalized_at < now() - make_interval(secs => $1::bigint) - ) - ))::bigint AS failed - FROM {done_child} - GROUP BY queue, priority - ), - -- Scan the entire partition being truncated (all generations), - -- mirroring done_counts' whole-child scan above. The rotate guard - -- keeps a partition to one generation at prune time, so this is the - -- same set today; counting the whole child keeps the rollup fold - -- conservative even if that invariant is ever weakened (a stray - -- generation's batches would otherwise be truncated without being - -- folded into the rollup -> terminal-count undercount). - batch_rows AS ( - SELECT - batch.ready_generation, - batch.queue, - batch.priority, - item.job_id, - item.run_lease - FROM {receipt_batch_child} AS batch - CROSS JOIN LATERAL unnest( - batch.job_ids, - batch.run_leases - ) AS item(job_id, run_lease) - ), - batch_counts AS ( + AND finalized_at >= now() - make_interval(secs => $1::bigint) + GROUP BY queue, priority, enqueue_shard, counter_bucket + ORDER BY queue, priority, enqueue_shard, counter_bucket + "# + ))) + .bind(failed_retention_secs) + .bind(state.0) + .bind(current_generation) + .execute(tx.as_mut()) + .await + .map_err(map_sqlx_error)?; + } + carried + } else { + 0 + }; + + // #290: scan the about-to-be-truncated partition for the rollup + // fold. The rollup column is *permanent* state, so we MUST fold + // from ground truth (the `{done_child}` partition itself), not + // from `queue_terminal_live_counts` — the counter can drift + // briefly during a rolling upgrade from a pre-counter binary + // and folding drift into the rollup would bake it in forever. + // The counter rows for this slot are still cleaned up after the + // truncate; reads from the counter (queue_counts_exact in #305) + // may transiently disagree with the rollup until the operator + // runs `awa storage rebuild-terminal-counters`, but the rollup + // itself stays authoritative. See PR #304 reviewer finding + // "High: A1 can persist stale counter state before the + // read-switch PR" for the trade-off. + // + // Carried failed rows are excluded from both columns: they are + // still live in `done_entries`, so folding them into the + // permanent rollup would double-count them. + let pruned_terminal_counts: Vec<(String, i16, i64, i64)> = + sqlx::query_as(audited_sql(format!( + r#" + WITH done_counts AS ( + SELECT + queue, + priority, + (count(*) FILTER (WHERE state <> 'failed'))::bigint AS completed, + (count(*) FILTER ( + WHERE state = 'failed' + AND ( + $2::boolean IS FALSE + OR finalized_at < now() - make_interval(secs => $1::bigint) + ) + ))::bigint AS failed + FROM {done_child} + GROUP BY queue, priority + ), + -- Scan the entire partition being truncated (all generations), + -- mirroring done_counts' whole-child scan above. The rotate guard + -- keeps a partition to one generation at prune time, so this is the + -- same set today; counting the whole child keeps the rollup fold + -- conservative even if that invariant is ever weakened (a stray + -- generation's batches would otherwise be truncated without being + -- folded into the rollup -> terminal-count undercount). + batch_rows AS ( + SELECT + batch.ready_generation, + batch.queue, + batch.priority, + item.job_id, + item.run_lease + FROM {receipt_batch_child} AS batch + CROSS JOIN LATERAL unnest( + batch.job_ids, + batch.run_leases + ) AS item(job_id, run_lease) + ), + batch_counts AS ( + SELECT + batch_rows.queue, + batch_rows.priority, + count(*)::bigint AS completed + FROM batch_rows + WHERE NOT EXISTS ( + SELECT 1 + FROM {receipt_tomb_child} AS tomb + WHERE tomb.ready_generation = batch_rows.ready_generation + AND tomb.job_id = batch_rows.job_id + AND tomb.run_lease = batch_rows.run_lease + ) + GROUP BY batch_rows.queue, batch_rows.priority + ), + keys AS ( + SELECT queue, priority FROM done_counts + UNION + SELECT queue, priority FROM batch_counts + ) SELECT - batch_rows.queue, - batch_rows.priority, - count(*)::bigint AS completed - FROM batch_rows - WHERE NOT EXISTS ( - SELECT 1 - FROM {receipt_tomb_child} AS tomb - WHERE tomb.ready_generation = batch_rows.ready_generation - AND tomb.job_id = batch_rows.job_id - AND tomb.run_lease = batch_rows.run_lease + keys.queue, + keys.priority, + ( + COALESCE(done_counts.completed, 0) + + COALESCE(batch_counts.completed, 0) + )::bigint AS pruned_completed_count, + COALESCE(done_counts.failed, 0)::bigint AS pruned_failed_count + FROM keys + LEFT JOIN done_counts USING (queue, priority) + LEFT JOIN batch_counts USING (queue, priority) + "# + ))) + .bind(failed_retention_secs) + .bind(retention_floor) + .fetch_all(tx.as_mut()) + .await + .map_err(map_sqlx_error)?; + + let truncate_started = Instant::now(); + self + .truncate_used_children_tx( + &mut tx, + &[ + &ready_child, + &claim_attempt_child, + &done_child, + &tomb_child, + &segment_child, + &receipt_batch_child, + &receipt_tomb_child, + &delta_child, + ], ) - GROUP BY batch_rows.queue, batch_rows.priority - ), - keys AS ( - SELECT queue, priority FROM done_counts - UNION - SELECT queue, priority FROM batch_counts - ) - SELECT - keys.queue, - keys.priority, - ( - COALESCE(done_counts.completed, 0) - + COALESCE(batch_counts.completed, 0) - )::bigint AS pruned_completed_count, - COALESCE(done_counts.failed, 0)::bigint AS pruned_failed_count - FROM keys - LEFT JOIN done_counts USING (queue, priority) - LEFT JOIN batch_counts USING (queue, priority) - "# + .await + .map_err(map_sqlx_error)?; + let truncate_duration = truncate_started.elapsed(); + if !pruned_terminal_counts.is_empty() { + // #371: append the pruned counts as delta rows instead + // of upserting `queue_terminal_rollups` here — the + // upsert under a pinned MVCC horizon left one + // unreclaimable dead rollup version per prune. The + // maintenance leader folds the deltas + // (`fold_terminal_rollup_deltas`) when the horizon is + // clear; exact readers add unfolded delta sums to + // their rollup reads. + self.append_terminal_rollup_deltas_tx( + &mut tx, + pruned_terminal_counts.into_iter(), + ) + .await?; + } + + // #290: the live counter rows for this slot are about to + // be orphans (their underlying done_entries rows just got + // truncated). Delete them in the same transaction. The + // rollup fold above already captured ground-truth from + // the partition scan; this just cleans up the counter + // index entries so a future insert into a re-rotated + // slot starts from zero. + sqlx::query(audited_sql(format!( + "DELETE FROM {schema}.queue_terminal_live_counts WHERE ready_slot = $1" ))) - .bind(failed_retention_secs) - .bind(retention_floor) - .fetch_all(tx.as_mut()) + .bind(slot) + .execute(tx.as_mut()) .await .map_err(map_sqlx_error)?; + Ok(PruneOutcome::Pruned { + slot, + generation, + carried_failed_rows, + durations: PruneDurations { + lock: lock_duration, + truncate: truncate_duration, + // Set once the transaction commits. + commit: Duration::ZERO, + }, + }) + } + .await; - let truncate_started = Instant::now(); - let truncate = self - .truncate_used_children_tx( - &mut tx, - &[ - &ready_child, - &claim_attempt_child, - &done_child, - &tomb_child, - &segment_child, - &receipt_batch_child, - &receipt_tomb_child, - &delta_child, - ], - ) - .await; - let truncate_duration = truncate_started.elapsed(); - - match truncate { - Ok(_) => { - if !pruned_terminal_counts.is_empty() { - // #371: append the pruned counts as delta rows instead - // of upserting `queue_terminal_rollups` here — the - // upsert under a pinned MVCC horizon left one - // unreclaimable dead rollup version per prune. The - // maintenance leader folds the deltas - // (`fold_terminal_rollup_deltas`) when the horizon is - // clear; exact readers add unfolded delta sums to - // their rollup reads. - self.append_terminal_rollup_deltas_tx( - &mut tx, - pruned_terminal_counts.into_iter(), - ) - .await?; - } - - // #290: the live counter rows for this slot are about to - // be orphans (their underlying done_entries rows just got - // truncated). Delete them in the same transaction. The - // rollup fold above already captured ground-truth from - // the partition scan; this just cleans up the counter - // index entries so a future insert into a re-rotated - // slot starts from zero. - sqlx::query(audited_sql(format!( - "DELETE FROM {schema}.queue_terminal_live_counts WHERE ready_slot = $1" - ))) - .bind(slot) - .execute(tx.as_mut()) - .await - .map_err(map_sqlx_error)?; - let commit_started = Instant::now(); - tx.commit().await.map_err(map_sqlx_error)?; - let commit_duration = commit_started.elapsed(); - if carried_failed_rows > 0 { - tracing::info!( - slot, - carried_failed_rows, - "Carried failed terminal rows inside the retention floor to the live done segment" - ); - } - Ok(PruneOutcome::Pruned { + let commit_started = Instant::now(); + let mut outcome = settle_prune_tx(tx, locked, PruneOutcome::Blocked { slot }).await?; + if let PruneOutcome::Pruned { + carried_failed_rows, + durations, + .. + } = &mut outcome + { + durations.commit = commit_started.elapsed(); + let carried_failed_rows = *carried_failed_rows; + if carried_failed_rows > 0 { + tracing::info!( slot, - generation, carried_failed_rows, - durations: PruneDurations { - lock: lock_duration, - truncate: truncate_duration, - commit: commit_duration, - }, - }) - } - Err(err) if is_lock_contention_error(&err) => { - let _ = tx.rollback().await; - Ok(PruneOutcome::Blocked { slot }) - } - Err(err) => { - let _ = tx.rollback().await; - Err(map_sqlx_error(err)) + "Carried failed terminal rows inside the retention floor to the live done segment" + ); } } + Ok(outcome) } #[tracing::instrument( @@ -19259,82 +19867,68 @@ impl QueueStorage { let lease_child = lease_child_name(schema, slot as usize); - set_prune_lock_timeout_tx(&mut tx, self.prune_lock_timeout).await?; + let locked: Result = async { + set_prune_lock_timeout_tx(&mut tx, self.prune_lock_timeout).await?; - let lock_started = Instant::now(); - let lock_table = sqlx::query(audited_sql(format!( - "LOCK TABLE {lease_child} IN ACCESS EXCLUSIVE MODE" - ))) - .execute(tx.as_mut()) - .await; - let lock_duration = lock_started.elapsed(); + let lock_started = Instant::now(); + sqlx::query(audited_sql(format!( + "LOCK TABLE {lease_child} IN ACCESS EXCLUSIVE MODE" + ))) + .execute(tx.as_mut()) + .await + .map_err(map_sqlx_error)?; + let lock_duration = lock_started.elapsed(); - if let Err(err) = lock_table { - let _ = tx.rollback().await; - if is_lock_contention_error(&err) { - return Ok(PruneOutcome::Blocked { slot }); - } - return Err(map_sqlx_error(err)); - } + // The ring rotation serialization held above keeps the cursor + // stable, but keep the explicit recheck to document the truncate + // precondition. + let (current_slot, _) = self + .ring_cursor_tx(&mut tx, RingFamily::Lease, authority) + .await?; - // The ring rotation serialization held above keeps the cursor - // stable, but keep the explicit recheck to document the truncate - // precondition. - let (current_slot, _) = self - .ring_cursor_tx(&mut tx, RingFamily::Lease, authority) - .await?; + if current_slot == slot { + return Ok(PruneOutcome::SkippedActive { + slot, + reason: SkipReason::LeaseCurrent, + count: 0, + }); + } - if current_slot == slot { - tx.commit().await.map_err(map_sqlx_error)?; - return Ok(PruneOutcome::SkippedActive { - slot, - reason: SkipReason::LeaseCurrent, - count: 0, - }); - } + let active_leases = Self::relation_has_rows_tx(&mut tx, &lease_child).await?; - let active_leases = Self::relation_has_rows_tx(&mut tx, &lease_child).await?; + if active_leases { + return Ok(PruneOutcome::SkippedActive { + slot, + reason: SkipReason::LeaseActive, + count: busy_indicator(active_leases), + }); + } - if active_leases { - tx.commit().await.map_err(map_sqlx_error)?; - return Ok(PruneOutcome::SkippedActive { + let truncate_started = Instant::now(); + self.truncate_used_children_tx(&mut tx, &[&lease_child]) + .await + .map_err(map_sqlx_error)?; + let truncate_duration = truncate_started.elapsed(); + Ok(PruneOutcome::Pruned { slot, - reason: SkipReason::LeaseActive, - count: busy_indicator(active_leases), - }); + generation, + carried_failed_rows: 0, + durations: PruneDurations { + lock: lock_duration, + truncate: truncate_duration, + // Set once the transaction commits. + commit: Duration::ZERO, + }, + }) } + .await; - let truncate_started = Instant::now(); - let truncate = self - .truncate_used_children_tx(&mut tx, &[&lease_child]) - .await; - let truncate_duration = truncate_started.elapsed(); - - match truncate { - Ok(_) => { - let commit_started = Instant::now(); - tx.commit().await.map_err(map_sqlx_error)?; - let commit_duration = commit_started.elapsed(); - Ok(PruneOutcome::Pruned { - slot, - generation, - carried_failed_rows: 0, - durations: PruneDurations { - lock: lock_duration, - truncate: truncate_duration, - commit: commit_duration, - }, - }) - } - Err(err) if is_lock_contention_error(&err) => { - let _ = tx.rollback().await; - Ok(PruneOutcome::Blocked { slot }) - } - Err(err) => { - let _ = tx.rollback().await; - Err(map_sqlx_error(err)) - } + let commit_started = Instant::now(); + let mut outcome = settle_prune_tx(tx, locked, PruneOutcome::Blocked { slot }).await?; + if let PruneOutcome::Pruned { durations, .. } = &mut outcome { + durations.commit = commit_started.elapsed(); } + Ok(outcome) } pub async fn vacuum_leases(&self, pool: &PgPool) -> Result<(), AwaError> { @@ -19638,116 +20232,103 @@ impl QueueStorage { }); } - set_prune_lock_timeout_tx(&mut tx, self.prune_lock_timeout).await?; - - let lock_started = Instant::now(); - let lock_tables = sqlx::query(audited_sql(format!( - "LOCK TABLE {claim_child}, {claim_batch_child}, {closure_child}, {closure_batch_child} IN ACCESS EXCLUSIVE MODE" - ))) - .execute(tx.as_mut()) - .await; - let lock_duration = lock_started.elapsed(); - - if let Err(err) = lock_tables { - let _ = tx.rollback().await; - if is_lock_contention_error(&err) { - return Ok(PruneOutcome::Blocked { slot }); - } - return Err(map_sqlx_error(err)); - } + let locked: Result = async { + set_prune_lock_timeout_tx(&mut tx, self.prune_lock_timeout).await?; - // After taking ACCESS EXCLUSIVE, recheck that the slot is still - // not the current one. The ring rotation serialization should - // already make this stable, but keeping the explicit gate documents - // the truncate precondition. - let (current_slot, _) = self - .ring_cursor_tx(&mut tx, RingFamily::Claim, authority) - .await?; + let lock_started = Instant::now(); + sqlx::query(audited_sql(format!( + "LOCK TABLE {claim_child}, {claim_batch_child}, {closure_child}, {closure_batch_child} IN ACCESS EXCLUSIVE MODE" + ))) + .execute(tx.as_mut()) + .await + .map_err(map_sqlx_error)?; + let lock_duration = lock_started.elapsed(); - if current_slot == slot { - tx.commit().await.map_err(map_sqlx_error)?; - return Ok(PruneOutcome::SkippedActive { - slot, - reason: SkipReason::ClaimCurrent, - count: 0, - }); - } + // After taking ACCESS EXCLUSIVE, recheck that the slot is still + // not the current one. The ring rotation serialization should + // already make this stable, but keeping the explicit gate documents + // the truncate precondition. + let (current_slot, _) = self + .ring_cursor_tx(&mut tx, RingFamily::Claim, authority) + .await?; - let open_claims = claim_prune_has_open_claims_tx( - &mut tx, - schema, - &claim_child, - &claim_batch_child, - &closure_child, - &closure_batch_child, - ) - .await?; - if open_claims { - tx.commit().await.map_err(map_sqlx_error)?; - return Ok(PruneOutcome::SkippedActive { - slot, - reason: SkipReason::ClaimOpen, - count: busy_indicator(open_claims), - }); - } + if current_slot == slot { + return Ok(PruneOutcome::SkippedActive { + slot, + reason: SkipReason::ClaimCurrent, + count: 0, + }); + } - let truncate_started = Instant::now(); - let truncate = self - .truncate_used_children_tx( + let open_claims = claim_prune_has_open_claims_tx( &mut tx, - &[ - &claim_child, - &claim_batch_child, - &closure_child, - &closure_batch_child, - ], + schema, + &claim_child, + &claim_batch_child, + &closure_child, + &closure_batch_child, ) - .await; - let truncate_duration = truncate_started.elapsed(); + .await?; + if open_claims { + return Ok(PruneOutcome::SkippedActive { + slot, + reason: SkipReason::ClaimOpen, + count: busy_indicator(open_claims), + }); + } - match truncate { - Ok(_) => { - sqlx::query(audited_sql(format!( - r#" - UPDATE {schema}.claim_ring_slots - SET rescue_cursor_claimed_at = '-infinity'::timestamptz, - rescue_cursor_job_id = 0, - rescue_cursor_run_lease = 0, - deadline_cursor_deadline_at = '-infinity'::timestamptz, - deadline_cursor_job_id = 0, - deadline_cursor_run_lease = 0, - batch_deadline_cursor_deadline_at = '-infinity'::timestamptz, - batch_deadline_cursor_batch_id = 0 - WHERE slot = $1 - "# - ))) - .bind(slot) - .execute(tx.as_mut()) + let truncate_started = Instant::now(); + self + .truncate_used_children_tx( + &mut tx, + &[ + &claim_child, + &claim_batch_child, + &closure_child, + &closure_batch_child, + ], + ) .await .map_err(map_sqlx_error)?; - let commit_started = Instant::now(); - tx.commit().await.map_err(map_sqlx_error)?; - let commit_duration = commit_started.elapsed(); - Ok(PruneOutcome::Pruned { - slot, - generation, - carried_failed_rows: 0, - durations: PruneDurations { - lock: lock_duration, - truncate: truncate_duration, - commit: commit_duration, - }, - }) - } - Err(err) if is_lock_contention_error(&err) => { - let _ = tx.rollback().await; - Ok(PruneOutcome::Blocked { slot }) - } - Err(err) => { - let _ = tx.rollback().await; - Err(map_sqlx_error(err)) - } + let truncate_duration = truncate_started.elapsed(); + sqlx::query(audited_sql(format!( + r#" + UPDATE {schema}.claim_ring_slots + SET rescue_cursor_claimed_at = '-infinity'::timestamptz, + rescue_cursor_job_id = 0, + rescue_cursor_run_lease = 0, + deadline_cursor_deadline_at = '-infinity'::timestamptz, + deadline_cursor_job_id = 0, + deadline_cursor_run_lease = 0, + batch_deadline_cursor_deadline_at = '-infinity'::timestamptz, + batch_deadline_cursor_batch_id = 0 + WHERE slot = $1 + "# + ))) + .bind(slot) + .execute(tx.as_mut()) + .await + .map_err(map_sqlx_error)?; + Ok(PruneOutcome::Pruned { + slot, + generation, + carried_failed_rows: 0, + durations: PruneDurations { + lock: lock_duration, + truncate: truncate_duration, + // Set once the transaction commits. + commit: Duration::ZERO, + }, + }) } + .await; + + let commit_started = Instant::now(); + let mut outcome = settle_prune_tx(tx, locked, PruneOutcome::Blocked { slot }).await?; + if let PruneOutcome::Pruned { durations, .. } = &mut outcome { + durations.commit = commit_started.elapsed(); + } + Ok(outcome) } fn job_id_sequence(&self) -> String { diff --git a/awa-model/src/storage.rs b/awa-model/src/storage.rs index b7a0ff1ad..14770c40a 100644 --- a/awa-model/src/storage.rs +++ b/awa-model/src/storage.rs @@ -2,7 +2,7 @@ use crate::admin; use crate::error::AwaError; use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; -use sqlx::{FromRow, PgExecutor, PgPool}; +use sqlx::{Connection, FromRow, PgConnection, PgExecutor, PgPool}; use std::collections::BTreeMap; #[derive(Debug, Clone, Serialize, Deserialize, FromRow, PartialEq)] @@ -469,3 +469,272 @@ pub async fn status_report(pool: &PgPool) -> Result, + pub reassign_sql: Option, + /// Login roles, superusers aside, that can run every statement above. + /// Empty when no such role exists yet. + pub repair_roles: Vec, +} + +/// What a repair did to one lane sequence. +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Hash)] +#[serde(rename_all = "snake_case")] +pub enum LaneSequenceOutcome { + /// Owned by the schema owner with every runtime grant. + Repaired, + /// Grants complete, but the role could not reassign the sequence. + OwnerUnchanged, + /// The role could neither reassign the sequence nor add its grants. + Unrepaired, + /// An open transaction held the sequence's lock past the repair's lock + /// timeout, so nothing changed. Rerun the repair. + Busy, +} + +impl TryFrom for LaneSequenceOutcome { + type Error = String; + + fn try_from(outcome: String) -> Result { + match outcome.as_str() { + "repaired" => Ok(Self::Repaired), + "owner_unchanged" => Ok(Self::OwnerUnchanged), + "unrepaired" => Ok(Self::Unrepaired), + "busy" => Ok(Self::Busy), + _ => Err(format!("unknown lane sequence repair outcome {outcome:?}")), + } + } +} + +/// What [`repair_lane_sequences`] did to one lane sequence that had a problem, +/// with what is still wrong afterwards. +#[derive(Debug, Clone, Serialize, Deserialize, FromRow, PartialEq, Eq)] +pub struct LaneSequenceRepair { + pub schema: String, + pub sequence_name: String, + /// Owner after the repair. + pub owner: String, + pub expected_owner: String, + #[sqlx(try_from = "String")] + pub outcome: LaneSequenceOutcome, + pub missing_grants: Vec, + pub reassign_sql: Option, + /// Login roles, superusers aside, that can finish the repair. + pub repair_roles: Vec, +} + +/// Lane sequences in `schema` that need repair. Read-only; any role may run it. +pub async fn lane_sequence_problems<'e, E>( + executor: E, + schema: &str, +) -> Result, AwaError> +where + E: PgExecutor<'e>, +{ + Ok(sqlx::query_as( + "SELECT $1::text AS schema, sequence_name, owner, expected_owner, missing_grants, \ + reassign_sql, repair_roles \ + FROM awa.lane_sequence_problems($1)", + ) + .bind(schema) + .fetch_all(executor) + .await?) +} + +/// Give each lane sequence in `schema` to the schema owner and add its missing +/// grants, as far as the connection's roles are allowed to. A sequence that +/// cannot be repaired is returned with the statements still to run and the +/// roles that can run them, never an error. +/// +/// Each sequence is repaired by its own statement, outside any transaction +/// the caller holds, so a reassignment's ACCESS EXCLUSIVE lock is released +/// before the next sequence and waits at most the repair function's lock +/// timeout. A sequence locked for longer by an open enqueue or claim comes +/// back [`LaneSequenceOutcome::Busy`]. +/// +/// Reassigning needs the current owner's privileges. A migration role that +/// connects as itself and sets its role to the schema owner (`role` in the +/// connection options) loses its own privileges while acting as the owner, +/// so a sequence the current role cannot repair is retried once as the login +/// role. +pub async fn repair_lane_sequences( + conn: &mut PgConnection, + schema: &str, +) -> Result, AwaError> { + let (current_role, login_role): (String, String) = + sqlx::query_as("SELECT current_user::text, session_user::text") + .fetch_one(&mut *conn) + .await?; + let problems = lane_sequence_problems(&mut *conn, schema).await?; + let mut repairs = Vec::with_capacity(problems.len()); + for problem in problems { + let mut repair: Option = sqlx::query_as(REPAIR_LANE_SEQUENCE_SQL) + .bind(schema) + .bind(&problem.sequence_name) + .fetch_optional(&mut *conn) + .await?; + let role_was_the_obstacle = matches!( + repair.as_ref().map(|repair| repair.outcome), + Some(LaneSequenceOutcome::OwnerUnchanged | LaneSequenceOutcome::Unrepaired) + ); + if role_was_the_obstacle && login_role != current_role { + if let Some(retried) = + repair_lane_sequence_as_login_role(conn, schema, &problem.sequence_name).await? + { + repair = retried; + } + } + repairs.extend(repair); + } + Ok(repairs) +} + +const REPAIR_LANE_SEQUENCE_SQL: &str = + "SELECT $1::text AS schema, sequence_name, owner, expected_owner, outcome, missing_grants, \ + reassign_sql, repair_roles \ + FROM awa.repair_lane_sequence($1, $2)"; + +/// Repair one sequence with the login role as the current role, for this +/// transaction only. `None` when the login role may not run the repair. +async fn repair_lane_sequence_as_login_role( + conn: &mut PgConnection, + schema: &str, + sequence_name: &str, +) -> Result>, AwaError> { + let mut transaction = conn.begin().await?; + sqlx::query("SET LOCAL ROLE NONE") + .execute(&mut *transaction) + .await?; + let repair = sqlx::query_as(REPAIR_LANE_SEQUENCE_SQL) + .bind(schema) + .bind(sequence_name) + .fetch_optional(&mut *transaction) + .await; + match repair { + Ok(repair) => { + transaction.commit().await?; + Ok(Some(repair)) + } + Err(sqlx::Error::Database(error)) if error.code().as_deref() == Some("42501") => { + transaction.rollback().await?; + Ok(None) + } + Err(error) => Err(error.into()), + } +} + +/// [`repair_lane_sequences`] for every queue-storage schema with a lane +/// sequence that needs repair. +pub async fn repair_lane_sequences_in_every_schema( + conn: &mut PgConnection, +) -> Result, AwaError> { + let mut repairs = Vec::new(); + for (schema, _) in lane_sequence_problem_counts(&mut *conn).await? { + repairs.extend(repair_lane_sequences(conn, &schema).await?); + } + Ok(repairs) +} + +/// One line per group of lane sequences that still need repair, saying what +/// to do next. `busy` marks sequences a repair found locked. +pub fn lane_sequence_next_steps<'a>( + leftovers: impl IntoIterator, +) -> Vec { + let mut groups: BTreeMap<(String, bool, String, String, Vec), usize> = BTreeMap::new(); + for (problem, busy) in leftovers { + *groups + .entry(( + problem.schema.clone(), + busy, + problem.owner.clone(), + problem.expected_owner.clone(), + problem.repair_roles.clone(), + )) + .or_default() += 1; + } + groups + .into_iter() + .map( + |((schema, busy, owner, expected_owner, repair_roles), count)| { + let rerun = format!("awa storage lane-sequences --schema {schema} --repair"); + if busy { + format!( + "{count} lane sequence(s) in schema {schema} owned by {owner} are busy: an \ + open transaction held the lock, so nothing changed. Rerun `{rerun}`." + ) + } else if repair_roles.is_empty() { + format!( + "{count} lane sequence(s) in schema {schema} owned by {owner} still need \ + repair, and no login role can both act as {owner} and SET ROLE to \ + {expected_owner}. Grant {owner} to the migration role, then rerun `{rerun}` \ + connected as the migration role, or run the statements from \ + `awa storage lane-sequences --schema {schema}` as {owner}." + ) + } else { + format!( + "{count} lane sequence(s) in schema {schema} owned by {owner} still need \ + repair. Rerun `{rerun}` connected as one of: {}.", + repair_roles.join(", ") + ) + } + }, + ) + .collect() +} + +impl LaneSequenceRepair { + /// The problem left after this repair, or `None` when it was repaired. + pub fn remaining_problem(&self) -> Option { + (self.outcome != LaneSequenceOutcome::Repaired).then(|| LaneSequenceProblem { + schema: self.schema.clone(), + sequence_name: self.sequence_name.clone(), + owner: self.owner.clone(), + expected_owner: self.expected_owner.clone(), + missing_grants: self.missing_grants.clone(), + reassign_sql: self.reassign_sql.clone(), + repair_roles: self.repair_roles.clone(), + }) + } +} + +/// [`lane_sequence_next_steps`] for the sequences a repair left behind. +pub fn lane_sequence_repair_next_steps(repairs: &[LaneSequenceRepair]) -> Vec { + let leftovers: Vec<(LaneSequenceProblem, bool)> = repairs + .iter() + .filter_map(|repair| { + repair + .remaining_problem() + .map(|problem| (problem, repair.outcome == LaneSequenceOutcome::Busy)) + }) + .collect(); + lane_sequence_next_steps(leftovers.iter().map(|(problem, busy)| (problem, *busy))) +} + +/// How many lane sequences need repair in each queue-storage schema that has +/// any. Read-only. +pub async fn lane_sequence_problem_counts<'e, E>( + executor: E, +) -> Result, AwaError> +where + E: PgExecutor<'e>, +{ + Ok(sqlx::query_as( + "SELECT discovered.schema_name, count(*) \ + FROM awa.queue_storage_schemas() AS discovered \ + CROSS JOIN LATERAL awa.lane_sequence_problems(discovered.schema_name) AS problem \ + GROUP BY discovered.schema_name \ + ORDER BY discovered.schema_name", + ) + .fetch_all(executor) + .await?) +} diff --git a/awa-model/tests/least_privilege_runtime_test.rs b/awa-model/tests/least_privilege_runtime_test.rs index 99e69d536..f06542c99 100644 --- a/awa-model/tests/least_privilege_runtime_test.rs +++ b/awa-model/tests/least_privilege_runtime_test.rs @@ -147,6 +147,10 @@ async fn exercise(options: PgConnectOptions, owner: &str, runtime: &str) -> Test assert_eq!(before, after); } assert_no_head_table_ddl_locks(&owner_pool).await?; + for schema in SCHEMAS { + assert_enqueue_function_privileges_fall_back(&owner_pool, &runtime_pool, schema, runtime) + .await?; + } runtime_pool.close().await; owner_pool.close().await; Ok(()) @@ -250,9 +254,29 @@ async fn check_runtime(pool: &PgPool, schema: &str) -> TestResult { INSERT INTO {schema}.queue_claim_heads(queue, priority, enqueue_shard) VALUES ('prepared_queue', 2, 0) ON CONFLICT DO NOTHING; SELECT {schema}.reserve_enqueue_seq('prepared_queue', 2::smallint, 0::smallint, 3); "#)))).await?; + // An unprepared lane is created through the schema owner's provisioning + // function, so the runtime still issues no DDL of its own. + pool.execute(sqlx::raw_sql(awa_model::audited_sql(format!( + "SELECT {schema}.ensure_lane_sequences('unprepared_queue', 2::smallint, 0::smallint); \ + SELECT {schema}.provision_lane('unprepared_queue', 2::smallint, 0::smallint);" + )))) + .await?; + let foreign_owned: i64 = sqlx::query_scalar(awa_model::audited_sql(format!( + "SELECT count(*) FROM pg_class \ + WHERE relnamespace = '{schema}'::regnamespace AND relkind = 'S' \ + AND relowner <> (SELECT nspowner FROM pg_namespace WHERE nspname = '{schema}')" + ))) + .fetch_one(pool) + .await?; + assert_eq!(foreign_owned, 0); + sqlx::query_scalar::<_, i64>(awa_model::audited_sql(format!( + "SELECT {schema}.reserve_enqueue_seq('unprepared_queue', 2::smallint, 0::smallint, 1)" + ))) + .fetch_one(pool) + .await?; let error = pool .execute(sqlx::raw_sql(awa_model::audited_sql(format!( - "SELECT {schema}.ensure_lane_sequences('unprepared_queue', 2::smallint, 0::smallint)" + "CREATE TABLE {schema}.must_not_create(id integer)" )))) .await .unwrap_err(); @@ -260,21 +284,84 @@ async fn check_runtime(pool: &PgPool, schema: &str) -> TestResult { error.as_database_error().unwrap().code().as_deref(), Some("42501") ); - assert!(error - .as_database_error() - .unwrap() - .message() - .contains("has not been provisioned")); - let error = pool + Ok(()) +} + +/// The one-call enqueue needs EXECUTE on enqueue_ready_rows and on +/// ring_cursor. Without either the store keeps the multi-statement path; a +/// grant revoked after the store has used the function falls back within +/// the same call, after rolling back the store-owned transaction. +async fn assert_enqueue_function_privileges_fall_back( + owner_pool: &PgPool, + runtime_pool: &PgPool, + schema: &str, + runtime: &str, +) -> TestResult { + let enqueue = |pool: PgPool| async move { + QueueStorage::new(QueueStorageConfig { + schema: schema.into(), + ..Default::default() + }) + .expect("store") + .enqueue_batch(&pool, "prepared_queue", 2, 1) + .await + }; + owner_pool .execute(sqlx::raw_sql(awa_model::audited_sql(format!( - "CREATE TABLE {schema}.must_not_create(id integer)" + "REVOKE EXECUTE ON FUNCTION {schema}.ring_cursor(text) FROM {runtime}" )))) - .await - .unwrap_err(); + .await?; assert_eq!( - error.as_database_error().unwrap().code().as_deref(), - Some("42501") + enqueue(runtime_pool.clone()).await?, + 1, + "without ring_cursor EXECUTE the store keeps the multi-statement path" + ); + owner_pool + .execute(sqlx::raw_sql(awa_model::audited_sql(format!( + "GRANT EXECUTE ON FUNCTION {schema}.ring_cursor(text) TO {runtime}" + )))) + .await?; + + let store = QueueStorage::new(QueueStorageConfig { + schema: schema.into(), + ..Default::default() + })?; + assert_eq!( + store + .enqueue_batch(runtime_pool, "prepared_queue", 2, 1) + .await?, + 1 + ); + let signature: String = sqlx::query_scalar(awa_model::audited_sql(format!( + "SELECT oid::regprocedure::text FROM pg_proc \ + WHERE pronamespace = '{schema}'::regnamespace AND proname = 'enqueue_ready_rows'" + ))) + .fetch_one(owner_pool) + .await?; + owner_pool + .execute(sqlx::raw_sql(awa_model::audited_sql(format!( + "REVOKE EXECUTE ON FUNCTION {signature} FROM PUBLIC, {runtime}" + )))) + .await?; + assert_eq!( + store + .enqueue_batch(runtime_pool, "prepared_queue", 2, 1) + .await?, + 1, + "the same call falls back after EXECUTE is revoked" ); + assert_eq!( + store + .enqueue_batch(runtime_pool, "prepared_queue", 2, 1) + .await?, + 1, + "the next call probes again and uses the multi-statement path" + ); + owner_pool + .execute(sqlx::raw_sql(awa_model::audited_sql(format!( + "GRANT EXECUTE ON FUNCTION {signature} TO {runtime}" + )))) + .await?; Ok(()) } @@ -331,10 +418,14 @@ async fn exercise_lazy_lane(options: PgConnectOptions, owner: &str, runtime: &st "GRANT USAGE, CREATE ON SCHEMA awa TO {runtime}; \ GRANT SELECT, INSERT, UPDATE, DELETE, TRUNCATE ON ALL TABLES IN SCHEMA awa TO {runtime}; \ GRANT USAGE, SELECT, UPDATE ON ALL SEQUENCES IN SCHEMA awa TO {runtime}; \ - GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA awa TO {runtime};" + GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA awa TO {runtime}; \ + REVOKE EXECUTE ON FUNCTION awa.provision_lane_sequences(text, smallint, smallint) \ + FROM PUBLIC, {runtime};" ), ) .await?; + // Without the provisioning function the runtime creates the lane's + // sequences itself and owns them, as a 0.6 runtime with CREATE did. let runtime_pool = PgPoolOptions::new() .max_connections(2) .connect_with(options.username(runtime)) diff --git a/awa-model/tests/schema_discovery_owner_test.rs b/awa-model/tests/schema_discovery_owner_test.rs index 3dfc52bbd..914414343 100644 --- a/awa-model/tests/schema_discovery_owner_test.rs +++ b/awa-model/tests/schema_discovery_owner_test.rs @@ -556,8 +556,15 @@ async fn assert_configured_substrates_fail_closed( ) .fetch_one(migrator_pool) .await?; - sqlx::query("DELETE FROM awa.schema_version WHERE version = $1") - .bind(migrations::CURRENT_VERSION) + // The newest migration that runs the discovery preflight. Later ones + // that touch no custom substrate carry no preflight to refuse. + let (preflight_version, _, pending_sql) = migrations::migration_sql() + .into_iter() + .rev() + .find(|(_, _, sql)| sql.contains("unsafe configured queue-storage schema")) + .expect("a migration that runs the discovery preflight"); + sqlx::query("DELETE FROM awa.schema_version WHERE version >= $1") + .bind(preflight_version) .execute(migrator_pool) .await?; let previous_version: i32 = sqlx::query_scalar("SELECT max(version) FROM awa.schema_version") @@ -602,7 +609,6 @@ async fn assert_configured_substrates_fail_closed( "CREATE SCHEMA awa_exp AUTHORIZATION {outsider}" )))) .await?; - let (_, _, pending_sql) = migrations::migration_sql().pop().unwrap(); for (details, expected_schema) in [ ( serde_json::json!({"schema": "injected_custom"}), diff --git a/awa-python/python/awa/_awa.pyi b/awa-python/python/awa/_awa.pyi index 2219bfc3b..ea507ec81 100644 --- a/awa-python/python/awa/_awa.pyi +++ b/awa-python/python/awa/_awa.pyi @@ -596,6 +596,7 @@ class Client: completed_retention_hours: float | None = None, failed_retention_hours: float | None = None, descriptor_retention_days: float | None = None, + queue_history_retention_hours: float | None = None, cleanup_batch_size: int | None = None, leader_election_interval_ms: int | None = None, heartbeat_interval_ms: int | None = None, @@ -605,6 +606,7 @@ class Client: deadline_rescue_interval_ms: int | None = None, callback_rescue_interval_ms: int | None = None, terminal_count_rollup_interval_ms: int | None = None, + statistics_refresh_interval_ms: int | None = None, queue_storage_schema: str | None = None, queue_storage_queue_slot_count: int = 16, queue_storage_lease_slot_count: int = 8, diff --git a/awa-python/python/awa/client.py b/awa-python/python/awa/client.py index f734ca679..b44007969 100644 --- a/awa-python/python/awa/client.py +++ b/awa-python/python/awa/client.py @@ -913,6 +913,7 @@ async def start( completed_retention_hours: float | None = None, failed_retention_hours: float | None = None, descriptor_retention_days: float | None = None, + queue_history_retention_hours: float | None = None, cleanup_batch_size: int | None = None, leader_election_interval_ms: int | None = None, heartbeat_interval_ms: int | None = None, @@ -922,6 +923,7 @@ async def start( deadline_rescue_interval_ms: int | None = None, callback_rescue_interval_ms: int | None = None, terminal_count_rollup_interval_ms: int | None = None, + statistics_refresh_interval_ms: int | None = None, queue_storage_schema: str | None = None, queue_storage_queue_slot_count: int = DEFAULT_QUEUE_STORAGE_QUEUE_SLOT_COUNT, queue_storage_lease_slot_count: int = DEFAULT_QUEUE_STORAGE_LEASE_SLOT_COUNT, @@ -946,6 +948,10 @@ async def start( ``awa.job_kind_descriptors``. Defaults to 30 days; pass ``0`` to disable retention (useful if you manage the catalog externally). + ``queue_history_retention_hours`` controls how long the maintenance + leader keeps the per-minute queue and fleet history behind the admin + UI's trends. Defaults to 48 hours; pass ``0`` to stop sampling. + Workers are queue-storage-capable by default, but during a storage transition ``storage_transition_role`` controls how this runtime participates: @@ -967,6 +973,11 @@ async def start( counters. Exact reads include pending deltas, so the default favors fewer hot counter updates over immediate folding. + ``statistics_refresh_interval_ms`` controls how often the maintenance + leader runs ``ANALYZE`` on the queue-storage head tables and + partitioned ring parents, which autovacuum never analyses (default 5 + minutes; ``0`` disables the refresh). + Queue dicts also accept ``claimers`` and ``claim_batch_size`` for hot queue-storage queues. ``claimers`` adds dispatcher/claimer loops that share the queue's worker permits; benchmark ``2`` or ``4`` before @@ -979,6 +990,7 @@ async def start( completed_retention_hours=completed_retention_hours, failed_retention_hours=failed_retention_hours, descriptor_retention_days=descriptor_retention_days, + queue_history_retention_hours=queue_history_retention_hours, cleanup_batch_size=cleanup_batch_size, leader_election_interval_ms=leader_election_interval_ms, heartbeat_interval_ms=heartbeat_interval_ms, @@ -988,6 +1000,7 @@ async def start( deadline_rescue_interval_ms=deadline_rescue_interval_ms, callback_rescue_interval_ms=callback_rescue_interval_ms, terminal_count_rollup_interval_ms=terminal_count_rollup_interval_ms, + statistics_refresh_interval_ms=statistics_refresh_interval_ms, queue_storage_schema=queue_storage_schema, queue_storage_queue_slot_count=queue_storage_queue_slot_count, queue_storage_lease_slot_count=queue_storage_lease_slot_count, diff --git a/awa-python/src/client.rs b/awa-python/src/client.rs index f141508a7..336e3ddd6 100644 --- a/awa-python/src/client.rs +++ b/awa-python/src/client.rs @@ -1796,7 +1796,7 @@ impl PyClient { }) } - #[pyo3(signature = (queues=None, *, poll_interval_ms=200, global_max_workers=None, completed_retention_hours=None, failed_retention_hours=None, descriptor_retention_days=None, cleanup_batch_size=None, leader_election_interval_ms=None, heartbeat_interval_ms=None, promote_interval_ms=None, heartbeat_rescue_interval_ms=None, heartbeat_staleness_ms=None, deadline_rescue_interval_ms=None, callback_rescue_interval_ms=None, terminal_count_rollup_interval_ms=None, queue_storage_schema=None, queue_storage_queue_slot_count=16, queue_storage_lease_slot_count=8, queue_storage_claim_slot_count=8, queue_storage_queue_stripe_count=1, queue_storage_queue_rotate_interval_ms=1000, queue_storage_lease_rotate_interval_ms=1000, queue_storage_claim_rotate_interval_ms=None, queue_storage_rotate_min_slot_age_ms=None, queue_storage_rotate_target_slot_bytes=None, queue_storage_rotate_min_slot_rows=None, queue_storage_rotate_max_slot_age_ms=None, storage_transition_role=None))] + #[pyo3(signature = (queues=None, *, poll_interval_ms=200, global_max_workers=None, completed_retention_hours=None, failed_retention_hours=None, descriptor_retention_days=None, queue_history_retention_hours=None, cleanup_batch_size=None, leader_election_interval_ms=None, heartbeat_interval_ms=None, promote_interval_ms=None, heartbeat_rescue_interval_ms=None, heartbeat_staleness_ms=None, deadline_rescue_interval_ms=None, callback_rescue_interval_ms=None, terminal_count_rollup_interval_ms=None, statistics_refresh_interval_ms=None, queue_storage_schema=None, queue_storage_queue_slot_count=16, queue_storage_lease_slot_count=8, queue_storage_claim_slot_count=8, queue_storage_queue_stripe_count=1, queue_storage_queue_rotate_interval_ms=1000, queue_storage_lease_rotate_interval_ms=1000, queue_storage_claim_rotate_interval_ms=None, queue_storage_rotate_min_slot_age_ms=None, queue_storage_rotate_target_slot_bytes=None, queue_storage_rotate_min_slot_rows=None, queue_storage_rotate_max_slot_age_ms=None, storage_transition_role=None))] #[allow(clippy::too_many_arguments)] fn start<'py>( &self, @@ -1807,6 +1807,7 @@ impl PyClient { completed_retention_hours: Option, failed_retention_hours: Option, descriptor_retention_days: Option, + queue_history_retention_hours: Option, cleanup_batch_size: Option, leader_election_interval_ms: Option, heartbeat_interval_ms: Option, @@ -1816,6 +1817,7 @@ impl PyClient { deadline_rescue_interval_ms: Option, callback_rescue_interval_ms: Option, terminal_count_rollup_interval_ms: Option, + statistics_refresh_interval_ms: Option, queue_storage_schema: Option, queue_storage_queue_slot_count: u32, queue_storage_lease_slot_count: u32, @@ -1901,6 +1903,14 @@ impl PyClient { } builder = builder.descriptor_retention(Duration::from_secs_f64(days * 86400.0)); } + if let Some(hours) = queue_history_retention_hours { + if !hours.is_finite() || hours < 0.0 { + return Err(pyo3::exceptions::PyValueError::new_err( + "queue_history_retention_hours must be a non-negative finite number (0 disables)", + )); + } + builder = builder.queue_history_retention(Duration::from_secs_f64(hours * 3600.0)); + } if let Some(batch_size) = cleanup_batch_size { if batch_size <= 0 { return Err(pyo3::exceptions::PyValueError::new_err( @@ -1950,6 +1960,9 @@ impl PyClient { } builder = builder.terminal_count_rollup_interval(Duration::from_millis(ms)); } + if let Some(ms) = statistics_refresh_interval_ms { + builder = builder.statistics_refresh_interval(Duration::from_millis(ms)); + } if queue_storage_queue_slot_count == 0 { return Err(pyo3::exceptions::PyValueError::new_err( "queue_storage_queue_slot_count must be > 0", diff --git a/awa-python/tests/test_start_config.py b/awa-python/tests/test_start_config.py index 302df0fc2..76319f656 100644 --- a/awa-python/tests/test_start_config.py +++ b/awa-python/tests/test_start_config.py @@ -294,11 +294,24 @@ async def handle(job): [(queue, 10)], completed_retention_hours=1.0, failed_retention_hours=168.0, + queue_history_retention_hours=24.0, cleanup_batch_size=500, ) await client.shutdown() +@pytest.mark.asyncio +async def test_negative_queue_history_retention_raises(client): + queue = "cfg_history" + + @client.task(ConfigTestJob, queue=queue) + async def handle(job): + return None + + with pytest.raises(ValueError, match="queue_history_retention_hours"): + await client.start([(queue, 1)], queue_history_retention_hours=-1.0) + + @pytest.mark.asyncio async def test_maintenance_interval_kwargs_accepted(client): """start() accepts all maintenance interval kwargs including heartbeat_staleness_ms.""" @@ -316,12 +329,26 @@ async def handle(job): deadline_rescue_interval_ms=1_000, callback_rescue_interval_ms=1_000, terminal_count_rollup_interval_ms=1_000, + statistics_refresh_interval_ms=1_000, leader_election_interval_ms=200, promote_interval_ms=500, ) await client.shutdown() +@pytest.mark.asyncio +async def test_statistics_refresh_can_be_disabled(client): + """statistics_refresh_interval_ms=0 disables the refresh; start() accepts it.""" + queue = "cfg_statistics_refresh_off" + + @client.task(ConfigTestJob, queue=queue) + async def handle(job): + return None + + await client.start([(queue, 1)], statistics_refresh_interval_ms=0) + await client.shutdown() + + @pytest.mark.asyncio async def test_per_queue_retention_in_dict_config(client): """Dict form with per-queue retention config starts successfully.""" diff --git a/awa-ui/frontend/e2e/dashboard.spec.ts b/awa-ui/frontend/e2e/dashboard.spec.ts index 332398950..d4758f623 100644 --- a/awa-ui/frontend/e2e/dashboard.spec.ts +++ b/awa-ui/frontend/e2e/dashboard.spec.ts @@ -1,190 +1,92 @@ -import { test, expect, type Response } from "@playwright/test"; - -function isQueuesListResponse(response: Response): boolean { - const url = new URL(response.url()); - return response.ok() && response.request().method() === "GET" && url.pathname === "/api/queues"; +import { test, expect, type Page } from "@playwright/test"; + +/** The overview lists only queues needing attention until expanded; seeded + * queues may or may not qualify depending on their age, so expand first. */ +async function showAllQueues(page: Page) { + await expect(page.getByTestId("verdict")).not.toHaveText("Checking queues…"); + const expand = page.getByRole("button", { + name: /^Show \d+ queues? keeping up$/, + }); + if (await expand.isVisible()) await expand.click(); + return page.getByRole("table", { + name: /^(All queues by risk|Queues needing attention)$/, + }); } -test.describe("Dashboard page", () => { - test("page loads and shows Dashboard heading", async ({ page }) => { +test.describe("Operations overview", () => { + test("answers whether the system is keeping up", async ({ page }) => { await page.goto("/"); - await expect( - page.getByRole("heading", { name: "Dashboard" }) - ).toBeVisible(); - }); - - test("state counter cards are visible with counts", async ({ page }) => { - await Promise.all([ - page.waitForResponse((r) => r.url().includes("/api/stats") && r.ok()), - page.goto("/"), - ]); - - // Counter cards should show numeric values (seeded data exists) - const counterCards = page.locator('a[href*="/jobs"]'); - const count = await counterCards.count(); - expect(count).toBeGreaterThanOrEqual(3); + await expect(page.getByRole("heading", { name: "Overview" })).toBeVisible(); + await expect(page.getByTestId("verdict")).not.toHaveText( + "Checking queues…", + ); + await expect(page.getByText("Waiting across all queues")).toBeVisible(); + // The e2e server runs no workers, so no leader has recorded history; + // the page says why trends are missing instead of inventing them. + await expect(page.getByText(/No trends( yet)?:/)).toBeVisible(); }); - test("queues card renders", async ({ page }) => { - await Promise.all([ - page.waitForResponse(isQueuesListResponse), - page.goto("/"), - ]); - - await expect( - page.locator('[data-slot="card-header"]', { hasText: "Queues" }) - ).toBeVisible(); - - const queueTable = page.getByRole("grid", { name: "Queue summary" }); - for (const header of ["Queue", "Total queued", "Scheduled", "Retryable"]) { + test("queue table includes trend columns and state drilldowns", async ({ + page, + }) => { + await page.goto("/"); + const table = await showAllQueues(page); + await expect(table).toBeVisible(); + for (const header of [ + "Queue", + "Status", + "Waiting", + "Last hour", + "Trend", + "Oldest wait", + "Limited by", + ]) { await expect( - queueTable.getByRole("columnheader", { name: header, exact: true }) + table.getByRole("columnheader", { name: header, exact: true }), ).toBeVisible(); } - - // Seeded e2e_test queue should appear in the queue table - await expect(page.getByText("e2e_test").first()).toBeVisible(); - }); - - test("recent failures section shows seeded failed job", async ({ page }) => { - await Promise.all([ - page.waitForResponse( - (r) => - r.url().includes("/api/jobs") && - r.url().includes("failed") && - r.ok() - ), - page.goto("/"), - ]); - - await expect( - page.locator('[data-slot="card-header"]', { - hasText: "Recent Failures", - }) - ).toBeVisible(); - - // Seeded failed e2e_job should appear - await expect(page.getByText("e2e_job").first()).toBeVisible(); - }); - - test("counter card click navigates to /jobs with state filter", async ({ page }) => { - await Promise.all([ - page.waitForResponse((r) => r.url().includes("/api/stats") && r.ok()), - page.goto("/"), - ]); - - // Click the "available" counter card — it links to /jobs?state=available - const availableCard = page.locator('a[href*="state=available"]').first(); - await expect(availableCard).toBeVisible(); - await availableCard.click(); - - await page.waitForURL(/\/jobs/); - expect(page.url()).toContain("state=available"); - await expect(page.getByRole("heading", { name: /Jobs/ })).toBeVisible(); - }); - - test("failed counter card navigates to /jobs?state=failed", async ({ page }) => { - await Promise.all([ - page.waitForResponse((r) => r.url().includes("/api/stats") && r.ok()), - page.goto("/"), - ]); - - const failedCard = page.locator('a[href*="state=failed"]').first(); - await expect(failedCard).toBeVisible(); - await failedCard.click(); - - await page.waitForURL(/\/jobs/); - expect(page.url()).toContain("state=failed"); - }); - - test("queue row click navigates to /jobs with queue filter", async ({ page }) => { - await Promise.all([ - page.waitForResponse(isQueuesListResponse), - page.goto("/"), - ]); - - // The dashboard queue table has clickable rows with onAction - const queueTable = page.getByRole("grid", { name: "Queue summary" }); - await expect(queueTable).toBeVisible(); - - // Click the e2e_test row (the row itself, not the link inside) - const e2eRow = queueTable.getByRole("row", { name: /e2e_test/ }); - await expect(e2eRow).toBeVisible(); - await e2eRow.click(); - - await page.waitForURL(/\/jobs/); - expect(page.url()).toContain("queue"); - expect(page.url()).toContain("e2e_test"); + const row = table.getByRole("row", { name: /E2E Queue/ }); + await row.locator('a[href*="state=available"]').click(); + await expect(page).toHaveURL(/state=available/); + await expect(page).toHaveURL(/e2e_test/); }); - test("runtime card renders with stat counters", async ({ page }) => { - await Promise.all([ - page.waitForResponse( - (r) => r.url().includes("/api/runtime") && r.ok() - ), - page.goto("/"), - ]); - - // Runtime card header should be visible - await expect( - page.locator('[data-slot="card-header"]', { hasText: "Runtime" }) - ).toBeVisible(); - - // Stat counter labels should be present - for (const label of ["Live", "Healthy", "Leader", "Stale"]) { - await expect(page.getByText(label, { exact: true }).first()).toBeVisible(); - } - }); + for (const [label, state] of [ + ["Retrying", "retryable"], + ["Failed", "failed"], + ["Running", "running"], + ] as const) { + test(`${label} counter links to its job state`, async ({ page }) => { + await page.goto("/"); + await page + .getByRole("navigation", { name: "Job states" }) + .getByRole("link", { name: new RegExp(`^${label}`) }) + .click(); + await expect(page).toHaveURL(new RegExp(`state=${state}`)); + await expect(page.getByRole("heading", { name: "Jobs" })).toBeVisible(); + }); + } - test("runtime card shows empty state when no workers running", async ({ + test("worker summary reports problems or an explicit healthy state", async ({ page, }) => { - await Promise.all([ - page.waitForResponse( - (r) => r.url().includes("/api/runtime") && r.ok() - ), - page.goto("/"), - ]); - - // When no workers are running, expect either the table or the empty state text - const runtimeCard = page.locator('[data-slot="card-header"]', { - hasText: "Runtime", - }); - await expect(runtimeCard).toBeVisible(); - - // Either instances are shown (table) or the empty/loading message appears - const emptyText = page.getByText( - "No runtime snapshots yet" - ); - const table = page.getByRole("grid", { name: "Runtime instances" }); - const hasTable = await table.isVisible().catch(() => false); - const hasEmptyText = await emptyText.isVisible().catch(() => false); - // One or the other should be present (if a worker happens to be running, table shows) - expect(hasTable || hasEmptyText).toBeTruthy(); - }); - - test("recent failure row click navigates to job detail", async ({ page }) => { - await Promise.all([ - page.waitForResponse( - (r) => - r.url().includes("/api/jobs") && - r.url().includes("failed") && - r.ok() - ), - page.goto("/"), - ]); - - const failuresTable = page.getByRole("grid", { name: "Recent failures" }); - await expect(failuresTable).toBeVisible(); - - // Click the first failure row - const firstRow = failuresTable.getByRole("row").nth(1); - await expect(firstRow).toBeVisible(); - await firstRow.click(); - - await page.waitForURL(/\/jobs\/\d+/); + await page.goto("/"); + await expect(page.getByRole("link", { name: "All workers" })).toBeVisible(); await expect( - page.getByRole("heading", { name: /Job #\d+/ }) + page + .getByText( + /Workers needing attention|reporting instances are healthy|No runtime snapshots/, + ) + .or(page.getByRole("list", { name: "Workers needing attention" })), ).toBeVisible(); }); + + test("failure group opens the latest retained failure", async ({ page }) => { + await page.goto("/"); + const groups = page.getByRole("list", { name: "Failure groups" }); + await expect(groups).toBeVisible(); + await groups.getByRole("link").first().click(); + await expect(page).toHaveURL(/\/jobs\/\d+/); + await expect(page.getByRole("heading", { name: /Job #\d+/ })).toBeVisible(); + }); }); diff --git a/awa-ui/frontend/e2e/navigation.spec.ts b/awa-ui/frontend/e2e/navigation.spec.ts index 030eca046..01ecc19e5 100644 --- a/awa-ui/frontend/e2e/navigation.spec.ts +++ b/awa-ui/frontend/e2e/navigation.spec.ts @@ -25,9 +25,9 @@ test.describe("Shell navigation", () => { await expect(page.getByRole("heading", { name: "Queues" })).toBeVisible(); // Click Runtime nav link - await nav.getByRole("link", { name: "Runtime" }).click(); + await nav.getByRole("link", { name: "Workers" }).click(); await expect(page).toHaveURL(/\/runtime/); - await expect(page.getByRole("heading", { name: "Runtime" })).toBeVisible(); + await expect(page.getByRole("heading", { name: "Workers" })).toBeVisible(); // Click Cron nav link (use exact match to avoid matching queue names like "cron_...") await nav.getByRole("link", { name: "Cron", exact: true }).click(); @@ -37,10 +37,10 @@ test.describe("Shell navigation", () => { ).toBeVisible(); // Click Dashboard nav link - await nav.getByRole("link", { name: "Dashboard" }).click(); + await nav.getByRole("link", { name: "Overview" }).click(); await expect(page).toHaveURL(/\/$/); await expect( - page.getByRole("heading", { name: "Dashboard" }) + page.getByRole("heading", { name: "Overview" }) ).toBeVisible(); }); @@ -55,7 +55,7 @@ test.describe("Shell navigation", () => { await logoLink.click(); await expect(page).toHaveURL(/\/$/); await expect( - page.getByRole("heading", { name: "Dashboard" }) + page.getByRole("heading", { name: "Overview" }) ).toBeVisible(); }); @@ -69,7 +69,7 @@ test.describe("Shell navigation", () => { // Dashboard should NOT be marked as current const dashboardLink = page - .getByRole("link", { name: "Dashboard" }) + .getByRole("link", { name: "Overview" }) .first(); await expect(dashboardLink).not.toHaveAttribute("aria-current", "page"); diff --git a/awa-ui/frontend/e2e/operations-fixture.ts b/awa-ui/frontend/e2e/operations-fixture.ts new file mode 100644 index 000000000..c7ae6da6b --- /dev/null +++ b/awa-ui/frontend/e2e/operations-fixture.ts @@ -0,0 +1,673 @@ +import type { Page } from "@playwright/test"; +import type { + JobRow, + QueueAssessment, + QueueHealthKind, + QueueStats, + RuntimeInstance, +} from "../src/lib/api"; +import { formatDuration, formatRate } from "../src/lib/operations"; + +/** + * Three moments in one synthetic fleet: + * + * - `normal`: every queue keeps up; nothing needs a human. + * - `incident`: a 0.7.1 rollout left image-processing with no worker + * claiming it, email-delivery is falling behind its rate limit, + * invoice-export was paused, and search-index drains a burst. + * - `recovery`: a 0.7.2 hotfix is rolled out, the email rate limit raised + * and invoice-export resumed; the backlogs drain. + */ +export type Phase = "normal" | "incident" | "recovery"; +type Scenario = Phase | "empty" | "error" | "loading"; + +const NAMES = [ + "email-delivery", + "invoice-export", + "webhook-dispatch", + "image-processing", + "search-index", + "daily-reports", +]; +const EMAIL = 0; +const INVOICE = 1; +const WEBHOOK = 2; +const IMAGE = 3; +const SEARCH = 4; + +/** What a queue looks like now, and how its waiting count got there. */ +interface QueueStory { + available: number; + running: number; + retryable: number; + failed: number; + lag: number; + paused?: boolean; + /** Waiting jobs `minutesAgo` minutes ago. */ + depth: (minutesAgo: number) => number; + /** The server's verdict while work is waiting (otherwise steady/idle). */ + verdict?: { + health: QueueHealthKind; + perMinute: number; + limitedBy?: QueueAssessment["limited_by"]; + reason?: string; + }; + claimed: number; + arrivals: number; + claimWait: number; + failing?: number; + rateLimited?: boolean; +} + +function story(phase: Phase, index: number): QueueStory { + const quiet = 180 + index * 7; + const steady = (available: number): QueueStory => ({ + available, + running: 40 + index, + retryable: phase === "normal" ? 2 : 10, + failed: 0, + lag: phase === "normal" ? 1.2 : 2.4, + depth: (m) => available + ((m * 7 + index) % 5) * 4, + claimed: 2140, + arrivals: 2140, + claimWait: 1.2, + }); + if (phase === "normal") { + if (index === SEARCH) return { ...steady(0), depth: () => 0 }; + if (index === EMAIL) + return { ...steady(1840), retryable: 12, failed: 3, lag: 2.1 }; + return steady(quiet); + } + if (phase === "incident") { + if (index === EMAIL) + return { + ...steady(184200), + retryable: 1840, + failed: 236, + lag: 458, + depth: (m) => (m <= 20 ? 184200 - m * 2500 : 134200 + (m % 3) * 300), + verdict: { + health: "behind", + perMinute: 2500, + limitedBy: "rate_limit", + }, + arrivals: 4640, + claimWait: 410, + failing: 4, + rateLimited: true, + }; + if (index === INVOICE) + return { + ...steady(2117), + lag: 135, + paused: true, + depth: (m) => (m <= 12 ? 2117 : 40 + (m % 4) * 5), + }; + if (index === IMAGE) + return { + ...steady(2151), + running: 0, + lag: 95, + depth: (m) => (m <= 2 ? 2151 - m * 900 : 30 + (m % 2) * 10), + verdict: { + health: "stalled", + perMinute: 415, + limitedBy: "nothing", + reason: "Nothing running or claimed; oldest has waited 1m 35s", + }, + claimed: 0, + arrivals: 415, + }; + if (index === SEARCH) + return { + ...steady(12168), + depth: (m) => (m <= 15 ? 12168 + m * 2000 : 4000), + verdict: { health: "draining", perMinute: -2000 }, + arrivals: 140, + }; + return steady(2100 + index * 17); + } + // recovery + if (index === EMAIL) + return { + ...steady(62000), + retryable: 310, + failed: 236, + lag: 210, + depth: (m) => + m <= 14 + ? 62000 + m * 8700 + : m <= 34 + ? 184200 - (m - 14) * 2500 + : 134200 + (m % 3) * 300, + verdict: { health: "draining", perMinute: -8700 }, + claimed: 10840, + arrivals: 2140, + claimWait: 190, + }; + if (index === INVOICE) + return { + ...steady(640), + lag: 20, + depth: (m) => (m <= 6 ? 640 + m * 246 : m <= 26 ? 2117 : 40), + verdict: { health: "draining", perMinute: -246 }, + }; + if (index === IMAGE) + return { + ...steady(380), + lag: 40, + depth: (m) => (m <= 10 ? 380 + m * 177 : m <= 22 ? 2151 : 30), + verdict: { health: "draining", perMinute: -177 }, + }; + if (index === SEARCH) return { ...steady(0), depth: () => 0 }; + return steady(2100 + index * 17); +} + +const RANK: Record = { + stalled: 0, + behind: 1, + slow: 2, + paused: 3, + draining: 4, + unknown: 5, + steady: 6, + idle: 7, +}; + +const LABEL: Record = { + stalled: "Not being worked", + behind: "Falling behind", + slow: "Long wait", + paused: "Paused", + draining: "Draining", + unknown: "No trend yet", + steady: "Steady", + idle: "Idle", +}; + +function plural(count: number, noun: string): string { + return `${count} ${noun}${count === 1 ? "" : "s"}`; +} + +export async function mockOperations( + page: Page, + scenario: Scenario = "incident", + readOnly = false, +) { + const phase: Phase = + scenario === "normal" || scenario === "recovery" ? scenario : "incident"; + const now = Date.now(); + const ago = (seconds: number) => new Date(now - seconds * 1000).toISOString(); + const stories = Array.from({ length: 24 }, (_, index) => story(phase, index)); + const queues: QueueStats[] = stories.map((s, index) => ({ + queue: NAMES[index] ?? `tenant-sync-${index}`, + display_name: null, + description: "Synthetic queue for local operator validation", + owner: "Platform", + docs_url: null, + tags: [], + extra: {}, + descriptor_last_seen_at: ago(8), + descriptor_stale: false, + descriptor_mismatch: false, + total_queued: 0, + available: s.available, + running: s.running, + retryable: s.retryable, + failed: s.failed, + scheduled: 850 + index, + waiting_external: + index === WEBHOOK ? (phase === "normal" ? 1400 : 14400) : 0, + completed_last_hour: s.claimed * 60, + lag_seconds: s.lag, + paused: s.paused ?? false, + })); + for (const q of queues) + q.total_queued = + q.available + q.running + q.retryable + q.scheduled + q.waiting_external; + + const config = { + mode: "hard_reserved", + max_workers: 64, + min_workers: null, + weight: null, + global_max_workers: null, + poll_interval_ms: 200, + deadline_duration_secs: 300, + priority_aging_interval_secs: 60, + dlq_enabled: true, + rate_limit: null, + }; + const version = { normal: "0.7.1", incident: "0.7.1", recovery: "0.7.2" }[ + phase + ]; + const instances: RuntimeInstance[] = Array.from( + { length: 48 }, + (_, index) => { + const troubled = phase === "incident"; + return { + instance_id: `00000000-0000-4000-8000-${String(index).padStart(12, "0")}`, + hostname: `worker-eu-${String(index + 1).padStart(2, "0")}`, + pid: 1300 + index, + version, + storage_capability: "queue_storage", + transition_role: "auto", + started_at: ago(phase === "recovery" ? 600 : 86400), + last_seen_at: ago(troubled && index === 0 ? 170 : 4), + snapshot_interval_ms: 10000, + stale: troubled && index === 0, + healthy: !troubled || index > 1, + postgres_connected: true, + poll_loop_alive: !troubled || index !== 1, + heartbeat_alive: true, + maintenance_alive: true, + shutting_down: false, + leader: index === 2, + global_max_workers: 64, + queues: queues.slice(0, 6).map((q) => ({ + queue: q.queue, + in_flight: 8, + overflow_held: 0, + config: config as RuntimeInstance["queues"][number]["config"], + })), + }; + }, + ); + const live = instances.filter((i) => !i.stale); + + // Per-minute history the way the leader records it: an hour of buckets, + // a leader that started 50 minutes ago, and a three-minute gap where + // sampling failed. + const BUCKETS = 60; + const minuteStart = Math.floor(now / 60_000) * 60_000; + const bucketAt = (bucket: number) => + new Date(minuteStart - (BUCKETS - 1 - bucket) * 60_000).toISOString(); + const sampled = (bucket: number) => + bucket >= 10 && ![33, 34, 35].includes(bucket); + const series = (value: (bucket: number) => T) => + Array.from({ length: BUCKETS }, (_, bucket) => + sampled(bucket) ? value(bucket) : null, + ); + const events = { + normal: [], + incident: [ + [41, "version_change", "Versions 0.7.0 → 0.7.0, 0.7.1"], + [44, "version_change", "Versions 0.7.0, 0.7.1 → 0.7.1"], + [47, "instances_lost", "1 instance stopped reporting (48 → 47 live)"], + ], + recovery: [ + [27, "version_change", "Versions 0.7.0 → 0.7.0, 0.7.1"], + [30, "version_change", "Versions 0.7.0, 0.7.1 → 0.7.1"], + [32, "instances_lost", "1 instance stopped reporting (48 → 47 live)"], + [48, "version_change", "Versions 0.7.1 → 0.7.1, 0.7.2"], + [50, "leader_change", "Leader 3f2a91c4 → 8d07b1e2"], + [51, "version_change", "Versions 0.7.1, 0.7.2 → 0.7.2"], + ], + }[phase] as [number, string, string][]; + const queueHistory = { + installed: true, + bucket_secs: 60, + minutes: BUCKETS, + observed_since: bucketAt(10), + buckets: Array.from({ length: BUCKETS }, (_, bucket) => bucketAt(bucket)), + queues: Object.fromEntries( + queues.map((q, index) => { + const s = stories[index]!; + return [ + q.queue, + { + available: series((b) => Math.max(0, s.depth(BUCKETS - 1 - b))), + running: series(() => q.running), + lag_seconds: series(() => q.lag_seconds), + claimed: series(() => s.claimed), + completed: series(() => s.claimed), + failed_attempts: series(() => s.failing ?? 0), + rate_limited: series(() => (s.rateLimited ? 30 : 0)), + }, + ]; + }), + ), + fleet: { + live_instances: series(() => live.length), + healthy_instances: series(() => live.filter((i) => i.healthy).length), + }, + events: events.map(([bucket, kind, detail]) => ({ + at: bucketAt(bucket), + kind, + detail, + })), + }; + + // The server classifies queues (awa-ui/src/assessment.rs); the fixture + // reproduces its verdicts for each story from the live queue state, so + // pause, resume and drain change the report as they would for real. + const assess = (q: QueueStats, index: number): QueueAssessment => { + const s = stories[index]!; + const base = { + queue: q.queue, + flow: { + claimed_per_minute: s.claimed, + completed_per_minute: s.claimed, + failed_per_minute: s.failing ?? 0, + retried_per_minute: 0, + arrivals_per_minute: s.arrivals, + mean_claim_wait_seconds: s.claimWait, + rate_limited: s.rateLimited ?? false, + }, + live_workers: live.length, + in_flight: q.running, + capacity: 64 * live.length, + clear_seconds: null as number | null, + trend: { per_minute: 0, change: 0, minutes: 5 }, + limited_by: null as QueueAssessment["limited_by"], + }; + const make = ( + health: QueueHealthKind, + reason: string, + extra: Partial = {}, + ): QueueAssessment => ({ + ...base, + health, + label: LABEL[health], + reason, + needs_attention: + health === "stalled" || + health === "behind" || + health === "slow" || + (health === "paused" && q.available > 0), + rank: health === "paused" && q.available === 0 ? 7 : RANK[health], + ...extra, + }); + if (q.paused) + return q.available + ? make("paused", `${q.available} waiting while paused`, { + limited_by: "paused", + }) + : make("paused", "Nothing waiting"); + if (q.available === 0) return make("idle", "Nothing waiting"); + const verdict = s.verdict; + if (!verdict) return make("steady", "Backlog holding level"); + const trend = { + per_minute: verdict.perMinute, + change: verdict.perMinute * 5, + minutes: 5, + }; + if (verdict.health === "draining") { + const clear = (q.available / -verdict.perMinute) * 60; + return make( + "draining", + `Clears in about ${formatDuration(clear)} at this pace`, + { trend, clear_seconds: clear }, + ); + } + return make( + verdict.health, + verdict.reason ?? + `Backlog ${formatRate(verdict.perMinute, true)} over 5m`, + { trend, limited_by: verdict.limitedBy ?? null }, + ); + }; + const healthReport = () => { + const assessed = queues + .map((q, index) => assess(q, index)) + .sort( + (a, b) => + a.rank - b.rank || + queues.find((q) => q.queue === b.queue)!.available - + queues.find((q) => q.queue === a.queue)!.available || + a.queue.localeCompare(b.queue), + ); + const count = (h: QueueHealthKind) => + assessed.filter((a) => a.health === h); + const stalled = count("stalled"); + const behind = count("behind").length; + const paused = assessed.filter( + (a) => a.health === "paused" && a.needs_attention, + ).length; + const recovering = count("draining").filter( + (a) => (a.clear_seconds ?? 0) >= 120, + ); + const others = [ + behind && `${plural(behind, "queue")} falling behind`, + paused && `${plural(paused, "paused queue")} holding work`, + ].filter(Boolean); + const longest = Math.max(...recovering.map((a) => a.clear_seconds ?? 0)); + const [status, title, detail] = stalled.length + ? [ + "critical", + `${plural(stalled.length, "queue")} not being worked`, + [ + `Waiting work nobody is claiming: ${stalled.map((a) => a.queue).join(", ")}.`, + ...others, + ].join(" · "), + ] + : behind + ? ["warning", `${plural(behind, "queue")} falling behind`, ""] + : paused + ? [ + "ok", + `${plural(paused, "paused queue")} holding work`, + "Every running queue is keeping up. Resume paused queues when ready.", + ] + : recovering.length + ? [ + "ok", + "Recovering", + `${plural(recovering.length, "queue")} draining a backlog (${recovering + .slice(0, 3) + .map((a) => a.queue) + .join( + ", ", + )}); all clear in about ${formatDuration(longest)} at this pace.`, + ] + : [ + "ok", + "Keeping up", + `${plural(assessed.length, "queue")} steady, draining, or idle.`, + ]; + return { + status, + title, + detail, + generated_at: new Date(now).toISOString(), + history_available: true, + history_since: bucketAt(10), + queues: assessed, + }; + }; + + const failureCount = phase === "normal" ? 3 : 60; + const jobs: JobRow[] = Array.from({ length: failureCount }, (_, index) => { + const image = phase !== "normal" && index % 7 === 3; + // After recovery the failures stopped about 14 minutes ago. + const finalized = + phase === "normal" + ? 1200 + index * 380 + : index * 12 + (phase === "recovery" ? 840 : 8); + return { + id: 984213 - index, + queue: image ? "image-processing" : "email-delivery", + kind: image ? "ResizeImage" : "SendTransactionalEmail", + args: { template: "receipt", synthetic: true }, + state: "failed", + priority: 2, + original_priority: 2, + attempt: 5, + max_attempts: 5, + run_at: ago(finalized + 900), + heartbeat_at: null, + deadline_at: null, + attempted_at: ago(finalized + 100), + finalized_at: ago(finalized), + created_at: ago(finalized + 1000), + errors: [ + { + error: + phase === "normal" + ? "Mailbox unavailable (550 5.1.1)" + : image + ? `Out of memory decoding upload ${7_310_000 + index}` + : "Provider request timed out after 30 seconds", + attempt: 5, + at: ago(finalized), + }, + ], + metadata: {}, + tags: ["synthetic"], + progress: null, + callback_id: null, + callback_timeout_at: null, + callback_filter: null, + callback_on_complete: null, + callback_on_fail: null, + callback_transform: null, + queue_descriptor: null, + kind_descriptor: null, + }; + }); + + const mutations: string[] = []; + let failReads = scenario === "error"; + await page.route("**/api/**", async (route) => { + const request = route.request(); + const url = new URL(request.url()); + const path = url.pathname.slice(4); + if (request.method() !== "GET") { + mutations.push(`${request.method()} ${path}`); + const queue = queues.find((q) => path.includes(q.queue)); + if (queue && path.endsWith("/pause")) queue.paused = true; + if (queue && path.endsWith("/resume")) queue.paused = false; + if (queue && path.endsWith("/drain")) { + queue.available = 0; + queue.retryable = 0; + queue.scheduled = 0; + queue.waiting_external = 0; + queue.total_queued = queue.running; + } + const job = jobs.find((entry) => path.startsWith(`/jobs/${entry.id}/`)); + if (job && path.endsWith("/retry")) { + job.state = "available"; + job.attempt = 0; + } + if (job && path.endsWith("/cancel")) job.state = "cancelled"; + return route.fulfill({ json: job ?? { ok: true, drained: 184200 } }); + } + if (path === "/capabilities") + return route.fulfill({ + json: { + read_only: readOnly, + poll_interval_ms: 60000, + instance_name: "Synthetic / local", + instance_color: "#23745a", + peers: [], + }, + }); + if (scenario === "loading") + await new Promise((resolve) => setTimeout(resolve, 2500)); + if ( + failReads && + [ + "/stats", + "/queues", + "/stats/queue-history", + "/health/queues", + "/runtime", + "/jobs", + "/dlq/depth", + ].includes(path) + ) + return route.fulfill({ + status: 503, + json: { error: "Synthetic API unavailable" }, + }); + const empty = scenario === "empty"; + const sum = (pick: (q: QueueStats) => number) => + queues.reduce((total, q) => total + pick(q), 0); + let data: unknown = []; + if (path === "/stats") + data = empty + ? {} + : { + available: sum((q) => q.available), + running: sum((q) => q.running), + retryable: sum((q) => q.retryable), + failed: sum((q) => q.failed), + scheduled: sum((q) => q.scheduled), + waiting_external: sum((q) => q.waiting_external), + completed: 14820000, + }; + else if (path === "/queues") data = empty ? [] : queues; + else if (path === "/health/queues") + data = empty + ? { ...healthReport(), status: "ok", title: "Keeping up", queues: [] } + : healthReport(); + else if (path === "/stats/queue-history") + data = empty + ? { ...queueHistory, observed_since: null, queues: {}, events: [] } + : queueHistory; + else if (path === "/queues/runtime") + data = empty + ? [] + : queues.map((q) => ({ + queue: q.queue, + instance_count: instances.length, + live_instances: live.length, + stale_instances: instances.length - live.length, + healthy_instances: live.filter((i) => i.healthy).length, + total_in_flight: q.running, + overflow_held_total: 0, + config_mismatch: false, + config, + })); + else if (path.startsWith("/queues/")) + data = queues.find((q) => q.queue === decodeURIComponent(path.slice(8))); + else if (path === "/runtime") + data = { + total_instances: empty ? 0 : instances.length, + live_instances: empty ? 0 : live.length, + stale_instances: empty ? 0 : instances.length - live.length, + healthy_instances: empty ? 0 : live.filter((i) => i.healthy).length, + leader_instances: empty ? 0 : 1, + instances: empty ? [] : instances, + }; + else if (path === "/dlq/depth") + data = { + total: empty ? 0 : phase === "normal" ? 12 : 1240, + by_queue: empty + ? [] + : [ + { + queue: "email-delivery", + count: phase === "normal" ? 12 : 1240, + }, + ], + }; + else if (path === "/jobs") + data = empty + ? [] + : jobs + .filter( + (job) => + !url.searchParams.has("queue") || + job.queue === url.searchParams.get("queue"), + ) + .map((job) => ({ + ...job, + state: url.searchParams.get("state") ?? job.state, + })) + .slice(0, Number(url.searchParams.get("limit") ?? 50)); + else if (/^\/jobs\/\d+$/.test(path)) + data = jobs.find((job) => job.id === Number(path.slice(6))); + else if (path === "/stats/kinds") data = ["SendTransactionalEmail"]; + else if (path === "/stats/queues") data = queues.map((q) => q.queue); + else if (path === "/storage") + return route.fulfill({ status: 404, json: {} }); + return route.fulfill({ json: data }); + }); + return { + mutations, + setReadFailure: (value: boolean) => { + failReads = value; + }, + }; +} diff --git a/awa-ui/frontend/e2e/operations.spec.ts b/awa-ui/frontend/e2e/operations.spec.ts new file mode 100644 index 000000000..74006ee43 --- /dev/null +++ b/awa-ui/frontend/e2e/operations.spec.ts @@ -0,0 +1,435 @@ +import { test, expect } from "@playwright/test"; +import { mockOperations } from "./operations-fixture"; + +const artifacts = + process.env.AWA_REVIEW_ARTIFACT_DIR ?? "test-results/operations-review"; +test("incident overview leads with the problem and opens targeted drilldowns", async ({ + page, +}) => { + const fixture = await mockOperations(page); + const errors: string[] = []; + page.on("pageerror", (error) => errors.push(error.message)); + await page.setViewportSize({ width: 1440, height: 1050 }); + await page.goto("/"); + await expect(page.getByRole("heading", { name: "Overview" })).toBeVisible(); + await expect(page.getByTestId("verdict")).toHaveText( + "1 queue not being worked", + ); + await expect(page.getByText(/1 queue falling behind/)).toBeVisible(); + const attention = page + .getByRole("table", { name: "Queues needing attention" }) + .locator("tbody tr"); + // Three queues need attention; the draining one stays in view. + await expect(attention).toHaveCount(4); + await expect(attention.nth(0)).toContainText("image-processing"); + await expect(attention.nth(0)).toContainText("Not being worked"); + await expect(attention.nth(1)).toContainText("email-delivery"); + await expect(attention.nth(1)).toContainText("Falling behind"); + await expect(attention.nth(1)).toContainText("+2,500/min"); + await expect(attention.nth(1)).toContainText("7m 38s"); + await expect(attention.nth(1)).toContainText("Rate limit"); + await expect(attention.nth(0)).toContainText("No visible limit"); + // Fleet changes are listed and marked on the total-backlog chart. + await expect( + page.getByRole("list", { name: "Fleet changes" }).getByRole("listitem"), + ).toHaveCount(3); + await expect(page.locator("[data-marker]")).toHaveCount(3); + await expect(page.getByText(/arriving .*\/min · last 5m/)).toBeVisible(); + await expect(attention.nth(2)).toContainText("Paused"); + await expect(attention.nth(3)).toContainText("search-index"); + await expect(attention.nth(3)).toContainText("Draining"); + await page.getByRole("button", { name: "Show 20 queues keeping up" }).click(); + const everything = page + .getByRole("table", { name: "All queues by risk" }) + .locator("tbody tr"); + await expect(everything).toHaveCount(24); + await expect(everything.nth(3)).toContainText("search-index"); + await expect(everything.nth(3)).toContainText("Draining"); + await expect( + page.getByRole("navigation", { name: "Job states" }), + ).toContainText("Retrying2,070"); + const workers = page.getByRole("list", { name: "Workers needing attention" }); + await expect(workers.getByRole("listitem")).toHaveCount(2); + await expect(workers).toContainText("health unknown"); + const failures = page.getByRole("list", { name: "Failure groups" }); + await expect(failures.getByRole("listitem")).toHaveCount(2); + await expect(failures.getByRole("listitem").first()).toContainText("×43"); + await page.screenshot({ + path: `${artifacts}/overview-desktop.png`, + fullPage: true, + }); + await page.getByRole("link", { name: "184,200", exact: true }).click(); + await expect(page).toHaveURL(/state=available/); + await expect(page).toHaveURL(/queue%3Aemail-delivery/); + await expect( + page.getByRole("button", { name: "available 184,200" }), + ).toHaveAttribute("aria-pressed", "true"); + await expect(page.getByText("State counts cover this queue")).toBeVisible(); + for (let index = 0; index < 3; index++) { + await page.goto("/"); + await page.getByRole("link", { name: "worker-eu-01" }).click(); + await expect(page).toHaveURL(/runtime\/00000000/); + await page.goBack(); + await expect(page.getByRole("heading", { name: "Overview" })).toBeVisible(); + } + expect(fixture.mutations).toEqual([]); + expect(errors).toEqual([]); +}); + +test("queue filters and core controls use only the synthetic API", async ({ + page, +}) => { + const fixture = await mockOperations(page); + await page.goto("/queues"); + await page.getByRole("searchbox", { name: "Find a queue" }).fill("invoice"); + const rows = page.getByRole("table", { name: "Queues" }).locator("tbody tr"); + await expect(rows).toHaveCount(1); + await expect(rows).toContainText("max 64"); + await page.getByRole("button", { name: "Resume", exact: true }).click(); + await expect( + page.getByRole("button", { name: "Pause", exact: true }), + ).toBeVisible(); + await page.getByRole("button", { name: "Pause", exact: true }).click(); + await expect( + page.getByRole("button", { name: "Resume", exact: true }), + ).toBeVisible(); + await page.getByRole("button", { name: "Drain…" }).click(); + await expect(page.getByRole("alertdialog")).toContainText( + "Running jobs will not be affected", + ); + await page.getByRole("button", { name: "Cancel", exact: true }).click(); + expect(fixture.mutations).toHaveLength(2); + await page.getByRole("button", { name: "Drain…" }).click(); + await page.getByRole("button", { name: "Drain queue", exact: true }).click(); + await expect(rows.locator("td").nth(1)).toHaveText("0"); + await expect(page).toHaveURL(/q=invoice/); + expect(fixture.mutations).toContain("POST /queues/invoice-export/drain"); + await page.getByRole("searchbox").fill("not-a-real-queue"); + await expect(page.getByText("No queues match these filters.")).toBeVisible(); + await page.getByRole("searchbox").fill(""); + await page + .getByRole("combobox", { name: "Sort queues" }) + .selectOption("failed"); + await expect(rows.first()).toContainText("email-delivery"); + // The drain above emptied the paused queue, so only the stalled and the + // falling-behind queue still need attention. + await page.getByText("Needs attention only").click(); + await expect(rows).toHaveCount(2); + await page.reload(); + await expect(rows).toHaveCount(2); + await expect(page.getByRole("combobox", { name: "Sort queues" })).toHaveValue( + "failed", + ); + await page.setViewportSize({ width: 1440, height: 1050 }); + await page.screenshot({ + path: `${artifacts}/queues-desktop.png`, + fullPage: true, + }); + await page.goto("/jobs/984213"); + await page.getByRole("button", { name: "Retry", exact: true }).click(); + await expect + .poll(() => fixture.mutations) + .toContain("POST /jobs/984213/retry"); + await page.getByRole("button", { name: "Cancel", exact: true }).click(); + await expect + .poll(() => fixture.mutations) + .toContain("POST /jobs/984213/cancel"); +}); + +test("mobile navigation, controls, job drilldown and layout at narrow sizes", async ({ + page, +}) => { + test.setTimeout(60_000); + await mockOperations(page); + await page.setViewportSize({ width: 390, height: 844 }); + await page.goto("/"); + await expect(page.getByTestId("verdict")).toHaveText( + "1 queue not being worked", + ); + await page.screenshot({ + path: `${artifacts}/overview-mobile.png`, + fullPage: true, + }); + await page.screenshot({ + path: `${artifacts}/overview-mobile-top.png`, + fullPage: false, + }); + for (let index = 0; index < 3; index++) { + await page.getByRole("button", { name: "Toggle Sidebar" }).click(); + await page.getByRole("link", { name: "Queues", exact: true }).click(); + await expect(page.getByRole("searchbox")).toBeVisible(); + await expect(page.getByRole("dialog")).toHaveCount(0); + await page.getByRole("button", { name: "Toggle Sidebar" }).click(); + await page.getByRole("link", { name: "Overview", exact: true }).click(); + await expect(page.getByRole("heading", { name: "Overview" })).toBeVisible(); + await expect(page.getByRole("dialog")).toHaveCount(0); + } + await page.goto("/queues"); + await page.getByRole("searchbox").fill("email"); + await page.screenshot({ + path: `${artifacts}/queues-mobile.png`, + fullPage: true, + }); + const pauseBounds = await page + .getByRole("button", { name: "Pause", exact: true }) + .boundingBox(); + expect(pauseBounds?.height).toBeGreaterThanOrEqual(44); + await page.getByRole("link", { name: "236", exact: true }).click(); + await page.screenshot({ + path: `${artifacts}/jobs-mobile.png`, + fullPage: false, + }); + await page + .getByRole("link", { name: /SendTransactionalEmail/ }) + .first() + .click(); + await expect(page).toHaveURL(/jobs\/984213/); + await page.screenshot({ path: `${artifacts}/job-detail-mobile.png` }); + for (const width of [320, 390, 768, 1024, 1440]) { + await page.setViewportSize({ width, height: 900 }); + for (const route of [ + "/", + "/queues", + "/jobs", + "/queues/email-delivery", + "/runtime", + ]) { + await page.goto(route); + const heading = + route === "/" + ? "Overview" + : route === "/queues" + ? "Queues" + : route === "/jobs" + ? "Jobs" + : route === "/runtime" + ? "Workers" + : "email-delivery"; + await expect( + page.getByRole("heading", { name: heading, exact: true }), + ).toBeVisible(); + if (route === "/jobs") + await expect( + page + .getByText("SendTransactionalEmail") + .filter({ visible: true }) + .first(), + ).toBeVisible(); + if (route === "/runtime") + await expect( + page + .getByText("worker-eu-02", { exact: true }) + .filter({ visible: true }) + .first(), + ).toBeVisible(); + if (route === "/queues") + await expect( + page.getByRole("link", { name: "email-delivery", exact: true }), + ).toBeVisible(); + if (route === "/") + await expect(page.getByTestId("verdict")).toHaveText( + "1 queue not being worked", + ); + await page.evaluate(() => document.fonts.ready); + if ( + (width === 390 || width === 1440) && + (route === "/runtime" || route.startsWith("/queues/")) + ) { + const view = route === "/runtime" ? "workers" : "queue-detail"; + await page.screenshot({ + path: `${artifacts}/${view}-${width === 390 ? "mobile" : "desktop"}.png`, + }); + } + if (width === 768 && route === "/jobs") + await page.screenshot({ path: `${artifacts}/jobs-tablet.png` }); + await expect + .poll( + () => + page.evaluate( + () => document.documentElement.scrollWidth <= innerWidth, + ), + { message: `${route} at ${width}px should fit the viewport` }, + ) + .toBe(true); + } + } +}); + +for (const scenario of ["empty", "error", "loading"] as const) { + test(`${scenario} state is explicit without invented zeroes`, async ({ + page, + }) => { + await mockOperations(page, scenario); + await page.setViewportSize({ width: 1440, height: 1000 }); + await page.goto("/"); + if (scenario === "empty") + await expect( + page.getByText("No queues yet. Queues appear when registered or used."), + ).toBeVisible(); + if (scenario === "error") { + await expect(page.getByRole("alert")).toBeVisible({ timeout: 15000 }); + await expect( + page.getByRole("navigation", { name: "Job states" }), + ).toContainText("Running—"); + } + if (scenario === "loading") + await expect(page.getByText("Loading snapshots…")).toBeVisible(); + await page.screenshot({ + path: `${artifacts}/${scenario}-desktop.png`, + fullPage: true, + }); + }); +} + +test("failed refresh retains known values and labels them out of date", async ({ + page, +}) => { + const fixture = await mockOperations(page); + await page.goto("/"); + const verdict = page.getByTestId("verdict"); + await expect(verdict).toHaveText("1 queue not being worked"); + fixture.setReadFailure(true); + await page.getByRole("button", { name: "Refresh data", exact: true }).click(); + await expect(page.getByText(/Refresh failed ·/)).toBeVisible({ + timeout: 15000, + }); + await expect(verdict).toHaveText("1 queue not being worked"); + await page.screenshot({ + path: `${artifacts}/stale-desktop.png`, + fullPage: true, + }); + fixture.setReadFailure(false); + await page.getByRole("button", { name: "Refresh data", exact: true }).click(); + await expect(page.getByText(/Snapshots received/)).toBeVisible(); +}); + +test("read-only controls remain disabled and dark mode stays legible", async ({ + page, +}) => { + const fixture = await mockOperations(page, "incident", true); + await page.addInitScript(() => localStorage.setItem("theme", "dark")); + await page.setViewportSize({ width: 1440, height: 1050 }); + await page.goto("/"); + await expect( + page.getByText("Connected to a read-only database."), + ).toBeVisible(); + await expect(page.locator("html")).toHaveClass(/dark/); + await page.screenshot({ + path: `${artifacts}/overview-dark.png`, + fullPage: true, + }); + await page.goto("/queues"); + await expect( + page.getByRole("button", { name: "Pause", exact: true }).first(), + ).toBeDisabled(); + await expect( + page.getByRole("button", { name: "Drain…" }).first(), + ).toBeDisabled(); + await page.goto("/jobs/984213"); + await expect( + page.getByRole("button", { name: "Retry", exact: true }), + ).toBeDisabled(); + expect(fixture.mutations).toEqual([]); +}); + +test("mobile drawer closes promptly with both motion preferences", async ({ + page, +}) => { + await mockOperations(page); + await page.setViewportSize({ width: 390, height: 844 }); + for (const reducedMotion of ["reduce", "no-preference"] as const) { + await page.emulateMedia({ reducedMotion }); + await page.goto("/"); + for (const name of ["Queues", "Overview", "Workers", "Overview"]) { + await page.getByRole("button", { name: "Toggle Sidebar" }).click(); + await page.getByRole("link", { name, exact: true }).click(); + await expect(page.getByRole("dialog", { name: "Sidebar" })).toHaveCount( + 0, + ); + await expect( + page.getByRole("heading", { + name, + exact: true, + }), + ).toBeVisible(); + } + await page.getByRole("button", { name: "Toggle Sidebar" }).click(); + await page.getByRole("button", { name: "Close", exact: true }).click(); + await expect(page.getByRole("dialog", { name: "Sidebar" })).toHaveCount(0); + await expect( + page.getByRole("button", { name: "Toggle Sidebar" }), + ).toBeFocused(); + } +}); + +test("normal operations, an incident, and the recovery each read at a glance", async ({ + page, +}) => { + await page.setViewportSize({ width: 1440, height: 1000 }); + + await mockOperations(page, "normal"); + await page.goto("/"); + await expect(page.getByTestId("verdict")).toHaveText("Keeping up"); + await expect( + page.getByRole("table", { name: "Queues needing attention" }), + ).toHaveCount(0); + await expect( + page.getByRole("button", { name: "Show 24 queues keeping up" }), + ).toBeVisible(); + await expect( + page.getByText("All 48 reporting instances are healthy."), + ).toBeVisible(); + await expect(page.getByRole("list", { name: "Fleet changes" })).toHaveCount( + 0, + ); + await page.screenshot({ path: `${artifacts}/phase-normal-desktop.png` }); + + await page.unrouteAll({ behavior: "ignoreErrors" }); + await mockOperations(page, "incident"); + await page.goto("/"); + await expect(page.getByTestId("verdict")).toHaveText( + "1 queue not being worked", + ); + await expect( + page + .getByRole("table", { name: "Queues needing attention" }) + .locator("tbody tr") + .first(), + ).toContainText("image-processing"); + await page.screenshot({ path: `${artifacts}/phase-incident-desktop.png` }); + await page.goto("/queues/email-delivery"); + await expect(page.getByText("Limited by: Rate limit")).toBeVisible(); + await page.screenshot({ + path: `${artifacts}/phase-incident-queue-desktop.png`, + }); + + await page.unrouteAll({ behavior: "ignoreErrors" }); + await mockOperations(page, "recovery"); + await page.goto("/"); + await expect(page.getByTestId("verdict")).toHaveText("Recovering"); + await expect(page.getByText(/all clear in about 7m/)).toBeVisible(); + await expect(page.getByText("Recovering (3)", { exact: true })).toBeVisible(); + const draining = page + .getByRole("table", { name: "Queues needing attention" }) + .locator("tbody tr"); + await expect(draining).toHaveCount(3); + await expect(draining.first()).toContainText("email-delivery"); + await expect(draining.first()).toContainText("−8,700/min"); + await expect( + page.getByRole("list", { name: "Fleet changes" }).getByRole("listitem"), + ).toHaveCount(5); + await expect(page.getByText("Versions 0.7.1, 0.7.2 → 0.7.2")).toBeVisible(); + await page.screenshot({ path: `${artifacts}/phase-recovery-desktop.png` }); + + await page.setViewportSize({ width: 390, height: 844 }); + await page.unrouteAll({ behavior: "ignoreErrors" }); + await mockOperations(page, "incident"); + await page.goto("/"); + await expect(page.getByTestId("verdict")).toHaveText( + "1 queue not being worked", + ); + await page.screenshot({ + path: `${artifacts}/phase-incident-mobile.png`, + fullPage: true, + }); +}); diff --git a/awa-ui/frontend/e2e/queues.spec.ts b/awa-ui/frontend/e2e/queues.spec.ts index 0cc373a7e..38aedf464 100644 --- a/awa-ui/frontend/e2e/queues.spec.ts +++ b/awa-ui/frontend/e2e/queues.spec.ts @@ -2,7 +2,11 @@ import { test, expect, type Page, type Response } from "@playwright/test"; function isQueuesListResponse(response: Response): boolean { const url = new URL(response.url()); - return response.ok() && response.request().method() === "GET" && url.pathname === "/api/queues"; + return ( + response.ok() && + response.request().method() === "GET" && + url.pathname === "/api/queues" + ); } async function loadQueuesPage(page: Page) { @@ -22,59 +26,66 @@ test.describe("Queues page", () => { await loadQueuesPage(page); // Desktop table (visible at default viewport) - const queueTable = page.getByRole("grid", { name: "Queues" }); + const queueTable = page.getByRole("table", { name: "Queues" }); await expect(queueTable).toBeVisible(); - // Columns were consolidated: Scheduled/Available fold into Queued; - // Mode/RateLimit fold into Capacity; Waiting is a subline under Queue. for (const header of [ "Queue", - "Queued", - "Running", - "Retry", - "Failed", - "Completed/hr", - "Capacity", "Status", - "Actions", + "Waiting", + "Trend", + "Oldest wait", + "Running", + "Done/min", + "Retrying / failed", + "Controls", ]) { await expect( - queueTable.getByRole("columnheader", { name: header, exact: true }) + queueTable.getByRole("columnheader", { name: header, exact: true }), ).toBeVisible(); } + // Capacity joins the table only where there is room for it. + const capacity = queueTable.getByRole("columnheader", { + name: "Capacity", + exact: true, + }); + await expect(capacity).toBeHidden(); + await page.setViewportSize({ width: 1920, height: 1080 }); + await expect(capacity).toBeVisible(); + await page.setViewportSize({ width: 1280, height: 720 }); + // Descriptor-backed queue shows display name and raw key await expect(queueTable.getByText("E2E Queue")).toBeVisible(); await expect(queueTable.getByText("e2e_test")).toBeVisible(); await expect( - queueTable.getByRole("link", { name: "legacy_queue", exact: true }) + queueTable.getByRole("link", { name: "legacy_queue", exact: true }), ).toBeVisible(); }); - test("queue table shows descriptor description", async ({ page }) => { + test("queue table carries the descriptor description", async ({ page }) => { await loadQueuesPage(page); - const queueTable = page.getByRole("grid", { name: "Queues" }); + const queueTable = page.getByRole("table", { name: "Queues" }); await expect( - queueTable.getByText("End-to-end queue used for UI coverage") - ).toBeVisible(); + queueTable.getByRole("link", { name: "E2E Queue" }), + ).toHaveAttribute("title", "End-to-end queue used for UI coverage"); }); - test("queue table surfaces consolidated capacity and actions columns", async ({ page }) => { + test("queue table surfaces available wait and controls", async ({ page }) => { await loadQueuesPage(page); - - const queueTable = page.getByRole("grid", { name: "Queues" }); - await expect(queueTable).toBeVisible(); - - // Mode / rate limit / waiting folded into the Capacity and Queue cells. - for (const header of ["Capacity", "Status", "Actions"]) { - await expect( - queueTable.getByRole("columnheader", { name: header, exact: true }) - ).toBeVisible(); - } + const queueTable = page.getByRole("table", { name: "Queues" }); + await expect( + queueTable.getByRole("columnheader", { name: "Oldest wait" }), + ).toBeVisible(); + await expect( + queueTable.getByRole("columnheader", { name: "Controls" }), + ).toBeVisible(); }); - test("queue table surfaces DLQ policy from runtime config", async ({ page }) => { + test("queue table surfaces DLQ policy from runtime config", async ({ + page, + }) => { await page.route("**/api/queues", async (route) => { await route.fulfill({ contentType: "application/json", @@ -136,38 +147,52 @@ test.describe("Queues page", () => { await page.goto("/queues"); - const queueTable = page.getByRole("grid", { name: "Queues" }); - await expect(queueTable.getByRole("rowheader", { name: /DLQ Off Queue/ })).toBeVisible(); - await expect(queueTable.getByText("DLQ off", { exact: true })).toBeVisible(); + const queueTable = page.getByRole("table", { name: "Queues" }); + await expect( + queueTable.getByRole("rowheader", { name: /DLQ Off Queue/ }), + ).toBeVisible(); + await expect( + queueTable.getByText("DLQ off", { exact: true }), + ).toBeVisible(); }); test("click queue navigates to queue detail", async ({ page }) => { await loadQueuesPage(page); - const queueTable = page.getByRole("grid", { name: "Queues" }); + const queueTable = page.getByRole("table", { name: "Queues" }); const queueLink = queueTable.getByRole("link", { name: "E2E Queue" }); await expect(queueLink).toBeVisible(); await queueLink.click(); await page.waitForURL(/\/queues\/e2e_test/); - await expect(page.getByRole("heading", { name: "E2E Queue" })).toBeVisible(); + await expect( + page.getByRole("heading", { name: "E2E Queue" }), + ).toBeVisible(); }); test("queue detail renders descriptor metadata", async ({ page }) => { await Promise.all([ - page.waitForResponse((r) => /\/api\/queues\/e2e_test$/.test(r.url()) && r.ok()), + page.waitForResponse( + (r) => /\/api\/queues\/e2e_test$/.test(r.url()) && r.ok(), + ), page.goto("/queues/e2e_test"), ]); - await expect(page.getByRole("heading", { name: "E2E Queue" })).toBeVisible(); + await expect( + page.getByRole("heading", { name: "E2E Queue" }), + ).toBeVisible(); await expect(page.getByText("e2e_test").first()).toBeVisible(); - await expect(page.getByText("End-to-end queue used for UI coverage")).toBeVisible(); + await expect( + page.getByText("End-to-end queue used for UI coverage"), + ).toBeVisible(); await expect(page.getByText("qa-platform")).toBeVisible(); await expect(page.getByText("critical")).toBeVisible(); }); - test("queue detail links to filtered DLQ rows when DLQ is enabled", async ({ page }) => { + test("queue detail links to filtered DLQ rows when DLQ is enabled", async ({ + page, + }) => { await page.route("**/api/queues/payments", async (route) => { await route.fulfill({ contentType: "application/json", @@ -231,7 +256,10 @@ test.describe("Queues page", () => { }); }); await page.route("**/api/dlq**", async (route) => { - await route.fulfill({ contentType: "application/json", body: JSON.stringify([]) }); + await route.fulfill({ + contentType: "application/json", + body: JSON.stringify([]), + }); }); await page.goto("/queues/payments"); @@ -244,16 +272,22 @@ test.describe("Queues page", () => { test("legacy queue still renders without descriptors", async ({ page }) => { await Promise.all([ - page.waitForResponse((r) => /\/api\/queues\/legacy_queue$/.test(r.url()) && r.ok()), + page.waitForResponse( + (r) => /\/api\/queues\/legacy_queue$/.test(r.url()) && r.ok(), + ), page.goto("/queues/legacy_queue"), ]); - await expect(page.getByRole("heading", { name: "legacy_queue" })).toBeVisible(); + await expect( + page.getByRole("heading", { name: "legacy_queue" }), + ).toBeVisible(); await expect(page.getByText("Descriptor health")).toBeVisible(); await expect(page.getByText("Not declared by a live worker")).toBeVisible(); }); - test("route-mocked descriptor stale and drift badges render", async ({ page }) => { + test("route-mocked descriptor stale and drift badges render", async ({ + page, + }) => { await page.route("**/api/queues", async (route) => { await route.fulfill({ contentType: "application/json", @@ -291,111 +325,73 @@ test.describe("Queues page", () => { }); await page.goto("/queues"); - const queueTable = page.getByRole("grid", { name: "Queues" }); - await expect(queueTable.getByText("Descriptor stale", { exact: true })).toBeVisible(); - await expect(queueTable.getByText("Descriptor drift", { exact: true })).toBeVisible(); - await expect(queueTable.getByText("Descriptor drift across live runtimes")).toBeVisible(); + const queueTable = page.getByRole("table", { name: "Queues" }); + await expect( + queueTable.getByText("Descriptor stale", { exact: true }), + ).toBeVisible(); + await expect( + queueTable.getByText("Descriptor drift", { exact: true }), + ).toBeVisible(); }); - async function openActionsMenu(page: Page, queueName: string) { - const queueTable = page.getByRole("grid", { name: "Queues" }); - const row = queueTable.getByRole("row", { name: new RegExp(queueName) }); - await expect(row).toBeVisible(); - await row - .getByRole("button", { name: new RegExp(`Actions for ${queueName}`) }) - .click(); + function queueRow(page: Page) { + return page + .getByRole("table", { name: "Queues" }) + .getByRole("row", { name: /E2E Queue/ }); } - test("Pause/Resume and Drain appear in the Actions menu", async ({ page }) => { + test("Pause or Resume and Drain controls are available", async ({ page }) => { await loadQueuesPage(page); - await openActionsMenu(page, "e2e_test"); - - // Menu renders either "Pause queue" or "Resume queue" depending on state, - // plus Drain. We just need one of pause/resume visible. - const pauseItem = page.getByRole("menuitem", { name: "Pause queue" }); - const resumeItem = page.getByRole("menuitem", { name: "Resume queue" }); - const pauseVisible = await pauseItem.isVisible().catch(() => false); - const resumeVisible = await resumeItem.isVisible().catch(() => false); - expect(pauseVisible || resumeVisible).toBeTruthy(); - - await expect(page.getByRole("menuitem", { name: /^Drain queue/ })).toBeVisible(); + await expect( + queueRow(page).getByRole("button", { name: /^(Pause|Resume)$/ }), + ).toBeVisible(); + await expect( + queueRow(page).getByRole("button", { name: "Drain…" }), + ).toBeVisible(); }); - test("Pause executes mutation from the Actions menu", async ({ page }) => { + test("Pause and Resume execute mutations", async ({ page }) => { await loadQueuesPage(page); - - // Ensure the queue is active so "Pause" is the available verb. - await openActionsMenu(page, "e2e_test"); - const resumeItem = page.getByRole("menuitem", { name: "Resume queue" }); - const isCurrentlyPaused = await resumeItem.isVisible().catch(() => false); - if (isCurrentlyPaused) { - await Promise.all([ - page.waitForResponse( - (r) => r.url().includes("/resume") && r.request().method() === "POST" - ), - resumeItem.click(), - ]); - await page.waitForResponse(isQueuesListResponse); - await openActionsMenu(page, "e2e_test"); - } else { - await page.keyboard.press("Escape"); - await openActionsMenu(page, "e2e_test"); + const row = queueRow(page); + if ( + await row.getByRole("button", { name: "Resume", exact: true }).isVisible() + ) { + await row.getByRole("button", { name: "Resume", exact: true }).click(); } - - const pauseItem = page.getByRole("menuitem", { name: "Pause queue" }); - await expect(pauseItem).toBeVisible(); + await expect( + row.getByRole("button", { name: "Pause", exact: true }), + ).toBeVisible(); const [pauseResponse] = await Promise.all([ page.waitForResponse( - (r) => r.url().includes("/pause") && r.request().method() === "POST" + (response) => + response.url().endsWith("/pause") && + response.request().method() === "POST", ), - pauseItem.click(), + row.getByRole("button", { name: "Pause", exact: true }).click(), ]); expect(pauseResponse.ok()).toBeTruthy(); - }); - - test("Resume executes mutation from the Actions menu", async ({ page }) => { - await loadQueuesPage(page); - - await openActionsMenu(page, "e2e_test"); - const pauseItem = page.getByRole("menuitem", { name: "Pause queue" }); - const isCurrentlyActive = await pauseItem.isVisible().catch(() => false); - if (isCurrentlyActive) { - await Promise.all([ - page.waitForResponse( - (r) => r.url().includes("/pause") && r.request().method() === "POST" - ), - pauseItem.click(), - ]); - await page.waitForResponse(isQueuesListResponse); - await openActionsMenu(page, "e2e_test"); - } else { - await page.keyboard.press("Escape"); - await openActionsMenu(page, "e2e_test"); - } - - const resumeItem = page.getByRole("menuitem", { name: "Resume queue" }); - await expect(resumeItem).toBeVisible(); + await expect( + row.getByRole("button", { name: "Resume", exact: true }), + ).toBeVisible(); const [resumeResponse] = await Promise.all([ page.waitForResponse( - (r) => r.url().includes("/resume") && r.request().method() === "POST" + (response) => + response.url().endsWith("/resume") && + response.request().method() === "POST", ), - resumeItem.click(), + row.getByRole("button", { name: "Resume", exact: true }).click(), ]); expect(resumeResponse.ok()).toBeTruthy(); - await page.waitForResponse(isQueuesListResponse); }); - test("Drain shows confirmation dialog from the Actions menu", async ({ page }) => { + test("Drain requires confirmation and can be cancelled", async ({ page }) => { await loadQueuesPage(page); - await openActionsMenu(page, "e2e_test"); - - await page.getByRole("menuitem", { name: /^Drain queue/ }).click(); - + await queueRow(page).getByRole("button", { name: "Drain…" }).click(); const dialog = page.getByRole("alertdialog"); await expect(dialog).toBeVisible(); - await expect(dialog.getByRole("heading", { name: /Drain queue/ })).toBeVisible(); - await expect(dialog.getByText("Running jobs will not be affected")).toBeVisible(); - + await expect( + dialog.getByText(/Running jobs will not be affected/), + ).toBeVisible(); await dialog.getByRole("button", { name: "Cancel", exact: true }).click(); await expect(dialog).not.toBeVisible(); }); diff --git a/awa-ui/frontend/e2e/readme-images.spec.ts b/awa-ui/frontend/e2e/readme-images.spec.ts new file mode 100644 index 000000000..35e80a50a --- /dev/null +++ b/awa-ui/frontend/e2e/readme-images.spec.ts @@ -0,0 +1,24 @@ +import { test, expect } from "@playwright/test"; +import { mockOperations, type Phase } from "./operations-fixture"; + +// Regenerates the README screenshots from the synthetic fleet: +// AWA_README_IMAGES_DIR=../../docs/images npm run test:operations -- readme-images +const out = process.env.AWA_README_IMAGES_DIR; + +test.use({ viewport: { width: 1440, height: 1180 }, deviceScaleFactor: 2 }); + +for (const [phase, verdict] of [ + ["incident", "1 queue not being worked"], + ["recovery", "Recovering"], +] as [Phase, string][]) { + test(`README image: overview during ${phase}`, async ({ page }) => { + test.skip(!out, "set AWA_README_IMAGES_DIR to regenerate README images"); + await page.addInitScript(() => localStorage.setItem("theme", "dark")); + await mockOperations(page, phase); + await page.goto("/"); + await expect(page.getByTestId("verdict")).toHaveText(verdict); + await expect(page.locator("tbody tr").first()).toBeVisible(); + await page.evaluate(() => document.fonts.ready); + await page.screenshot({ path: `${out}/awa-ui-overview-${phase}-dark.png` }); + }); +} diff --git a/awa-ui/frontend/e2e/runtime.spec.ts b/awa-ui/frontend/e2e/runtime.spec.ts index 6c3999c52..8138a0654 100644 --- a/awa-ui/frontend/e2e/runtime.spec.ts +++ b/awa-ui/frontend/e2e/runtime.spec.ts @@ -3,7 +3,7 @@ import { test, expect } from "@playwright/test"; test.describe("Runtime page", () => { test("navigate to /runtime, heading visible", async ({ page }) => { await page.goto("/runtime"); - await expect(page.getByRole("heading", { name: "Runtime" })).toBeVisible(); + await expect(page.getByRole("heading", { name: "Workers" })).toBeVisible(); }); test("renders empty states when no runtime snapshots exist", async ({ page }) => { diff --git a/awa-ui/frontend/package.json b/awa-ui/frontend/package.json index 28e1c758d..03d58f643 100644 --- a/awa-ui/frontend/package.json +++ b/awa-ui/frontend/package.json @@ -7,7 +7,8 @@ "dev": "vite", "build": "tsc -b && vite build", "lint": "tsc --noEmit", - "preview": "vite preview" + "preview": "vite preview", + "test:operations": "playwright test --config playwright.operations.config.ts" }, "dependencies": { "@heroicons/react": "^2", diff --git a/awa-ui/frontend/playwright.config.ts b/awa-ui/frontend/playwright.config.ts index 3a3349c7f..8eb0ccd1f 100644 --- a/awa-ui/frontend/playwright.config.ts +++ b/awa-ui/frontend/playwright.config.ts @@ -11,6 +11,7 @@ const databaseUrl = export default defineConfig({ globalSetup: "./e2e/seed.ts", testDir: "./e2e", + testIgnore: ["**/operations.spec.ts", "**/readme-images.spec.ts"], fullyParallel: true, forbidOnly: !!process.env.CI, retries: process.env.CI ? 2 : 0, diff --git a/awa-ui/frontend/playwright.operations.config.ts b/awa-ui/frontend/playwright.operations.config.ts new file mode 100644 index 000000000..553278579 --- /dev/null +++ b/awa-ui/frontend/playwright.operations.config.ts @@ -0,0 +1,23 @@ +import { defineConfig } from "@playwright/test"; +export default defineConfig({ + testDir: "./e2e", + testMatch: /(operations|readme-images)\.spec\.ts/, + workers: 1, + forbidOnly: !!process.env.CI, + reporter: "list", + use: { + baseURL: "http://127.0.0.1:4173", + reducedMotion: "reduce", + trace: "retain-on-failure", + screenshot: "only-on-failure", + launchOptions: { + executablePath: process.env.PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH, + }, + }, + webServer: { + command: + "npm run dev -- --mode operations --host 127.0.0.1 --port 4173 --strictPort", + url: "http://127.0.0.1:4173", + reuseExistingServer: false, + }, +}); diff --git a/awa-ui/frontend/src/__tests__/operations.test.ts b/awa-ui/frontend/src/__tests__/operations.test.ts new file mode 100644 index 000000000..81a49f6de --- /dev/null +++ b/awa-ui/frontend/src/__tests__/operations.test.ts @@ -0,0 +1,247 @@ +import { describe, it, expect } from "vitest"; +import type { + HealthReport, + JobRow, + QueueAssessment, + QueueHistory, + QueueStats, +} from "@/lib/api"; +import { + eventIndexes, + formatDuration, + formatRate, + groupFailures, + joinQueues, + totalBacklog, + totalRate, +} from "@/lib/operations"; + +function queue(name: string, available = 10): QueueStats { + return { + queue: name, + display_name: null, + description: null, + owner: null, + docs_url: null, + tags: [], + extra: {}, + descriptor_last_seen_at: null, + descriptor_stale: false, + descriptor_mismatch: false, + total_queued: available, + scheduled: 0, + available, + retryable: 0, + running: 1, + failed: 0, + waiting_external: 0, + completed_last_hour: 600, + lag_seconds: 5, + paused: false, + }; +} + +function assessment( + name: string, + rank: number, + flow: Partial = {}, + liveWorkers = 1, +): QueueAssessment { + return { + queue: name, + health: "steady", + label: "Steady", + reason: "Backlog holding level", + needs_attention: false, + rank, + limited_by: null, + trend: null, + clear_seconds: null, + flow: { + claimed_per_minute: null, + completed_per_minute: null, + failed_per_minute: null, + retried_per_minute: null, + arrivals_per_minute: null, + mean_claim_wait_seconds: null, + rate_limited: false, + ...flow, + }, + live_workers: liveWorkers, + in_flight: 0, + capacity: null, + }; +} + +function report(queues: QueueAssessment[]): HealthReport { + return { + status: "ok", + title: "Keeping up", + detail: "", + generated_at: "2026-10-25T00:00:00Z", + history_available: true, + history_since: null, + queues, + }; +} + +function history( + queues: Record, + events: QueueHistory["events"] = [], +): QueueHistory { + const length = Object.values(queues)[0]?.length ?? 3; + const empty = () => Array.from({ length }, () => null); + return { + installed: true, + bucket_secs: 60, + minutes: length, + observed_since: null, + buckets: Array.from( + { length }, + (_, index) => `2026-10-25T00:0${index}:00Z`, + ), + queues: Object.fromEntries( + Object.entries(queues).map(([name, available]) => [ + name, + { + available, + running: empty(), + lag_seconds: empty(), + claimed: empty(), + completed: empty(), + failed_attempts: empty(), + rate_limited: empty(), + }, + ]), + ), + fleet: { live_instances: empty(), healthy_instances: empty() }, + events, + }; +} + +describe("joinQueues", () => { + it("follows the report's risk order and appends queues it does not know", () => { + const rows = joinQueues( + [queue("a"), queue("new"), queue("b")], + report([assessment("b", 0), assessment("a", 6)]), + history({ a: [1, 2, 3] }), + ); + expect(rows.map((row) => row.queue.queue)).toEqual(["b", "a", "new"]); + expect(rows[1]?.series?.available).toEqual([1, 2, 3]); + expect(rows[2]?.assessment).toBeUndefined(); + }); +}); + +describe("totalBacklog", () => { + it("sums per bucket and keeps buckets with no samples null", () => { + expect( + totalBacklog(history({ a: [null, 1, 2], b: [null, null, 3] })), + ).toEqual([null, 1, 5]); + }); +}); + +describe("totalRate", () => { + it("sums known rates", () => { + const total = totalRate( + report([ + assessment("a", 0, { completed_per_minute: 10 }), + assessment("b", 0, { completed_per_minute: 5 }), + ]), + (flow) => flow.completed_per_minute, + ); + expect(total).toBe(15); + }); + + it("refuses a partial total while a served queue is unknown", () => { + const total = totalRate( + report([ + assessment("a", 0, { completed_per_minute: 10 }), + assessment("b", 0, {}, 2), + ]), + (flow) => flow.completed_per_minute, + ); + expect(total).toBeNull(); + }); + + it("keeps receiver-only completion rates unknown without worker runtimes", () => { + const total = totalRate( + report([ + assessment("a", 0, { completed_per_minute: 10 }), + assessment("callback-receiver", 0, {}, 0), + ]), + (flow) => flow.completed_per_minute, + ); + expect(total).toBeNull(); + }); + + it("includes a known zero on an idle queue without worker runtimes", () => { + expect(totalRate( + report([ + assessment("active", 0, { completed_per_minute: 10 }), + assessment("idle", 0, { completed_per_minute: 0 }, 0), + ]), + (flow) => flow.completed_per_minute, + )).toBe(10); + }); +}); + +describe("eventIndexes", () => { + it("maps fleet events onto bucket positions", () => { + const indexes = eventIndexes( + history({ a: [1, 2, 3] }, [ + { at: "2026-10-25T00:02:00Z", kind: "version_change", detail: "" }, + { at: "2026-10-24T23:00:00Z", kind: "leader_change", detail: "" }, + ]), + ); + expect(indexes).toEqual([2]); + }); +}); + +describe("groupFailures", () => { + function failed( + id: number, + message: string, + finalized: string, + kind = "SendEmail", + ): JobRow { + return { + id, + kind, + queue: "email", + errors: [{ error: message }], + finalized_at: finalized, + } as unknown as JobRow; + } + + it("groups by kind, queue and error, masking per-job identifiers", () => { + const groups = groupFailures([ + failed(1, "timed out calling order 18273645", "2026-10-25T00:00:01Z"), + failed(2, "timed out calling order 99887766", "2026-10-25T00:00:03Z"), + failed( + 3, + "timed out calling order 12345678", + "2026-10-25T00:00:02Z", + "Other", + ), + ]); + expect(groups).toHaveLength(2); + expect(groups[0]).toMatchObject({ kind: "SendEmail", count: 2 }); + expect(groups[0]!.latest.id).toBe(2); + expect(groups[0]!.message).toBe("timed out calling order 99887766"); + }); +}); + +describe("formatting", () => { + it("formats durations consistently across units", () => { + expect(formatDuration(2.4)).toBe("2s"); + expect(formatDuration(458)).toBe("7m 38s"); + expect(formatDuration(3720)).toBe("1h 2m"); + expect(formatDuration(null)).toBe("—"); + }); + + it("formats signed rates", () => { + expect(formatRate(1234, true)).toBe("+1,234/min"); + expect(formatRate(-0.5, true)).toBe("−0.5/min"); + expect(formatRate(52_000)).toBe("52K/min"); + }); +}); diff --git a/awa-ui/frontend/src/app.css b/awa-ui/frontend/src/app.css index 874caa0b1..0e4a424c0 100644 --- a/awa-ui/frontend/src/app.css +++ b/awa-ui/frontend/src/app.css @@ -307,3 +307,15 @@ If you want to use the typography plugin (prose), uncomment the line below. outline: 1px solid var(--primary); transform: translateZ(0); } + +/* Touch targets on phones: header controls and in-page buttons stay at + least 44px, and long unbroken identifiers wrap instead of widening the page. */ +@media (max-width: 767px) { + [data-slot=sidebar-trigger], header button, main button { min-height: 44px; min-width: 44px; } + [data-slot=sidebar-item] { min-height: 44px; } + main { overflow-wrap: anywhere; } +} +@media (prefers-reduced-motion: reduce) { + *, *::before, *::after { scroll-behavior: auto !important; animation: none !important; transition: none !important; } +} + diff --git a/awa-ui/frontend/src/components/HistorySpark.tsx b/awa-ui/frontend/src/components/HistorySpark.tsx new file mode 100644 index 000000000..cd64cf9d0 --- /dev/null +++ b/awa-ui/frontend/src/components/HistorySpark.tsx @@ -0,0 +1,108 @@ +/** + * Sparkline for sampled queue history. Unlike `Sparkline`, points keep their + * bucket position (so a gap in sampling shows as a gap, not a shorter line) + * and the y axis starts at zero: a large, steady backlog stays flat instead + * of min-max scaling turning a 1% wobble into a cliff. + */ +export function HistorySpark({ + data, + width = 120, + height = 24, + label, + className, + markers = [], +}: { + data: ReadonlyArray; + width?: number; + height?: number; + label: string; + className?: string; + /** Bucket indexes to mark with a vertical tick, e.g. fleet changes. */ + markers?: ReadonlyArray; +}) { + const values = data.filter((value): value is number => value !== null); + if (values.length === 0) { + return ( + + ); + } + const max = Math.max(...values) || 1; + const step = data.length > 1 ? width / (data.length - 1) : 0; + const pad = 1.5; + const y = (value: number) => pad + (1 - value / max) * (height - pad * 2); + + const segments: string[][] = []; + let current: string[] = []; + data.forEach((value, index) => { + if (value === null) { + if (current.length) segments.push(current); + current = []; + return; + } + current.push(`${(index * step).toFixed(1)},${y(value).toFixed(1)}`); + }); + if (current.length) segments.push(current); + + return ( + + + {markers.map((index) => ( + + ))} + {segments.map((points, index) => + points.length === 1 ? ( + + ) : ( + + ), + )} + + ); +} diff --git a/awa-ui/frontend/src/components/QueueFlowTable.tsx b/awa-ui/frontend/src/components/QueueFlowTable.tsx new file mode 100644 index 000000000..024070617 --- /dev/null +++ b/awa-ui/frontend/src/components/QueueFlowTable.tsx @@ -0,0 +1,504 @@ +import type { ReactNode } from "react"; +import { Link } from "@tanstack/react-router"; +import type { + QueueHealthKind, + QueueRuntimeSummary, + QueueStats, +} from "@/lib/api"; +import { Badge } from "@/components/ui/badge"; +import { HistorySpark } from "@/components/HistorySpark"; +import { + capacityLabel, + formatCount, + formatDuration, + formatRate, + healthOf, + LIMITED_BY_HINT, + LIMITED_BY_LABEL, + type AssessedQueue, +} from "@/lib/operations"; + +export const HEALTH_BADGE: Record< + QueueHealthKind, + "danger" | "warning" | "info" | "secondary" | "outline" +> = { + stalled: "danger", + behind: "warning", + slow: "warning", + paused: "secondary", + draining: "info", + unknown: "outline", + steady: "outline", + idle: "outline", +}; + +/** Text tint for the backlog sparkline, so the trend reads at a glance. */ +export const SPARK_TONE: Record = { + stalled: "text-danger", + behind: "text-warning", + slow: "text-warning", + paused: "text-muted-fg", + draining: "text-info", + unknown: "text-muted-fg", + steady: "text-muted-fg", + idle: "text-muted-fg", +}; + +function queueName(queue: QueueStats): string { + return queue.display_name?.trim() || queue.queue; +} + +function JobsLink({ + queue, + state, + value, + className = "", +}: { + queue: string; + state: string; + value: number; + className?: string; +}) { + return ( + + {value.toLocaleString()} + + ); +} + +function Capacity({ summary }: { summary: QueueRuntimeSummary | undefined }) { + if (summary && summary.live_instances === 0) { + return No live workers; + } + const title = summary + ? `${summary.healthy_instances}/${summary.live_instances} live instances healthy${summary.config_mismatch ? " · config differs between instances" : ""}` + : "No runtime has reported this queue"; + return ( + + {capacityLabel(summary)} + + ); +} + +/** + * Rare conditions worth seeing without opening the queue. They sit by the + * name rather than in a column so they survive when narrow tables drop one. + */ +function QueueFlags({ + queue, + summary, +}: { + queue: QueueStats; + summary: QueueRuntimeSummary | undefined; +}) { + return ( + <> + {summary?.config_mismatch && ( + + Config mismatch + + )} + {summary?.config?.dlq_enabled === false && ( + + DLQ off + + )} + {queue.descriptor_stale && ( + + Descriptor stale + + )} + {queue.descriptor_mismatch && ( + + Descriptor drift + + )} + + ); +} + +function TrendText({ row }: { row: AssessedQueue }) { + const trend = row.assessment?.trend; + if (!trend) return —; + // Below the noise threshold the fitted slope is batching jitter; printing + // "-0.6/min" would invite reading meaning into it. + const health = healthOf(row); + if (health === "steady" || health === "idle" || health === "paused") { + return flat; + } + const clear = row.assessment?.clear_seconds ?? null; + return ( + + {formatRate(trend.per_minute, true)} + {clear !== null && ( + · ~{formatDuration(clear)} + )} + + ); +} + +function LimitedBy({ row }: { row: AssessedQueue }) { + const limited = row.assessment?.limited_by; + if (!limited) return —; + const tone = + limited === "no_workers" + ? "text-danger-subtle-fg" + : limited === "rate_limit" || limited === "worker_capacity" + ? "text-warning-subtle-fg" + : ""; + return ( + + {LIMITED_BY_LABEL[limited]} + + ); +} + +/** Measured completions per minute over the last 5 minutes, or the last + * hour's average when the runtimes have not reported counters. */ +function DoneRate({ row }: { row: AssessedQueue }) { + const measured = row.assessment?.flow.completed_per_minute; + if (measured !== null && measured !== undefined) { + return ( + + {formatCount(Math.round(measured))} + + ); + } + return ( + + {formatCount(Math.round(row.queue.completed_last_hour / 60))} + + ); +} + +/** + * One line per queue, ordered by the caller. Switches to stacked cards when + * its own container is narrow (not the viewport: the docked sidebar takes a + * fixed share of a tablet screen). + */ +export function QueueFlowTable({ + rows, + controls, + runtime, + label, +}: { + rows: ReadonlyArray; + controls?: (queue: QueueStats) => ReactNode; + /** When given, adds a capacity column from live runtime config. */ + runtime?: ReadonlyMap; + label: string; +}) { + return ( +
+
+ + + + + + + + + + + + {runtime && ( + + )} + + + {controls && ( + + )} + + + + {rows.map((row) => { + const { queue, assessment } = row; + const health = healthOf(row); + return ( + + + + + + + + + + {runtime && ( + + )} + + + {controls && ( + + )} + + ); + })} + +
+ Queue + Status + Waiting + Last hour + Trend + + Oldest wait + + Running + + Limited by + + Capacity + + Done/min + + Retrying / failed + + Controls +
+ + {queueName(queue)} + + {queue.display_name?.trim() && ( + + {queue.queue} + + )} + + + + {assessment?.label ?? "…"} + + + + + + + + + {queue.available > 0 ? ( + formatDuration(queue.lag_seconds) + ) : ( + — + )} + + + + + + + + + + + + + + {controls(queue)} +
+
+ +
    + {rows.map((row) => { + const { queue, assessment } = row; + const health = healthOf(row); + return ( +
  • +
    +
    + + {queueName(queue)} + +

    {assessment?.reason}

    + {(queue.descriptor_stale || queue.descriptor_mismatch) && ( +
    + +
    + )} +
    + + {assessment?.label ?? "…"} + +
    +
    + + + + +
    +
    +
    +
    Waiting
    +
    + +
    +
    +
    +
    Oldest wait
    +
    + {queue.available > 0 + ? formatDuration(queue.lag_seconds) + : "—"} +
    +
    +
    +
    Running
    +
    + +
    +
    +
    +
    Done/min
    +
    + +
    +
    +
    +
    Retrying
    +
    + +
    +
    +
    +
    Failed
    +
    + +
    +
    +
    +
    Limited by
    +
    + +
    +
    + {runtime && ( +
    +
    Capacity
    +
    + +
    +
    + )} +
    + {controls &&
    {controls(queue)}
    } +
    • + ); + })} +
+
+ ); +} diff --git a/awa-ui/frontend/src/components/Shell.tsx b/awa-ui/frontend/src/components/Shell.tsx index a760aae31..4f586f3c2 100644 --- a/awa-ui/frontend/src/components/Shell.tsx +++ b/awa-ui/frontend/src/components/Shell.tsx @@ -1,4 +1,4 @@ -import { useEffect, useState } from "react"; +import { useEffect, useLayoutEffect, useRef, useState } from "react"; import { Outlet, useRouterState, useNavigate } from "@tanstack/react-router"; import { Link as AriaLink, @@ -228,13 +228,13 @@ function useSidebarOpenState(): [boolean, (value: boolean) => void] { // Closes the mobile sidebar sheet on route change. No-op on desktop. function CloseMobileOnNavigate({ path }: { path: string }) { - const { isMobile, isOpenOnMobile, setIsOpenOnMobile } = useSidebar(); - useEffect(() => { - if (isMobile && isOpenOnMobile) { - setIsOpenOnMobile(false); - } - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [path]); + const { isMobile, setIsOpenOnMobile } = useSidebar(); + const previousPath = useRef(path); + useLayoutEffect(() => { + if (previousPath.current === path) return; + previousPath.current = path; + if (isMobile) setIsOpenOnMobile(false); + }, [path, isMobile, setIsOpenOnMobile]); return null; } @@ -341,11 +341,12 @@ const IconBatchOps = () => ( ); const NAV_ITEMS = [ - { to: "/", label: "Dashboard", Icon: IconDashboard }, + { to: "/", label: "Overview", Icon: IconDashboard }, { to: "/jobs", label: "Jobs", Icon: IconJobs }, - { to: "/kinds", label: "Kinds", Icon: IconKinds }, { to: "/queues", label: "Queues", Icon: IconQueues }, - { to: "/runtime", label: "Runtime", Icon: IconRuntime }, + { to: "/dlq", label: "Dead letters", Icon: IconJobs }, + { to: "/kinds", label: "Kinds", Icon: IconKinds }, + { to: "/runtime", label: "Workers", Icon: IconRuntime }, { to: "/cron", label: "Cron", Icon: IconCron }, { to: "/batch-ops", label: "Batch Ops", Icon: IconBatchOps }, ] as const; @@ -419,6 +420,7 @@ export function Shell() { onOpenChange={setSidebarOpen} style={{ "--sidebar-width": "13.5rem" } as React.CSSProperties} > + - - + + + Skip to content + {/* Opaque: any translucency lets scrolled table text ghost through the sticky bar and fight the header content for legibility. */}
{instanceName && ( - {instanceName} + {instanceName} )} {peers.length > 0 && ( @@ -504,14 +508,18 @@ export function Shell() {
-
+
{showReadOnlyBanner && ( -
- Connected to a read-only database. Dashboard queries work, but admin actions are disabled. +
+ Connected to a read-only database. Live queries work; admin actions are disabled.
)} -
+ diff --git a/awa-ui/frontend/src/components/SnapshotStatus.tsx b/awa-ui/frontend/src/components/SnapshotStatus.tsx new file mode 100644 index 000000000..ea8e8eebb --- /dev/null +++ b/awa-ui/frontend/src/components/SnapshotStatus.tsx @@ -0,0 +1,48 @@ +import { useEffect, useState } from "react"; +import { usePollInterval } from "@/hooks/use-poll-interval"; + +interface SnapshotSource { + dataUpdatedAt: number; + isError: boolean; + isPending: boolean; +} +export function SnapshotStatus({ sources }: { sources: SnapshotSource[] }) { + const poll = usePollInterval(); + const [now, setNow] = useState(Date.now); + useEffect(() => { + const timer = setInterval(() => setNow(Date.now()), 1000); + return () => clearInterval(timer); + }, []); + const updated = sources.map((source) => source.dataUpdatedAt); + const oldest = Math.min(...updated); + const failed = sources.some((source) => source.isError); + const pending = sources.some((source) => source.isPending); + const age = oldest ? Math.max(0, Math.floor((now - oldest) / 1000)) : null; + const delayed = + age !== null && age * 1000 > Math.max(30000, poll.interval * 2); + return ( +
+
+ ); +} diff --git a/awa-ui/frontend/src/components/ui/sidebar.tsx b/awa-ui/frontend/src/components/ui/sidebar.tsx index e7dfe11fb..7454f8c3f 100644 --- a/awa-ui/frontend/src/components/ui/sidebar.tsx +++ b/awa-ui/frontend/src/components/ui/sidebar.tsx @@ -216,19 +216,21 @@ const Sidebar = ({ return ( <> - - {children} - + {isOpenOnMobile && ( + + {children} + + )} ) } diff --git a/awa-ui/frontend/src/lib/api.ts b/awa-ui/frontend/src/lib/api.ts index 2b0cfcce7..f6dcc7b7a 100644 --- a/awa-ui/frontend/src/lib/api.ts +++ b/awa-ui/frontend/src/lib/api.ts @@ -617,6 +617,101 @@ export function fetchTimeseries( return apiFetch(`/stats/timeseries${qs}`); } +/** Per-minute history written by the maintenance leader (v051). */ +export interface QueueHistory { + /** False on a schema before v051. */ + installed: boolean; + bucket_secs: number; + minutes: number; + observed_since: string | null; + /** Start of each one-minute bucket, oldest first. */ + buckets: string[]; + queues: Record; + fleet: { + live_instances: (number | null)[]; + healthy_instances: (number | null)[]; + }; + events: FleetEvent[]; +} + +/** Aligned to `QueueHistory.buckets`; `null` where nothing was recorded or a + * counter was unknown. Counter series are per-minute deltas. */ +export interface QueueSeries { + available: (number | null)[]; + running: (number | null)[]; + lag_seconds: (number | null)[]; + claimed: (number | null)[]; + completed: (number | null)[]; + failed_attempts: (number | null)[]; + rate_limited: (number | null)[]; +} + +export interface FleetEvent { + at: string; + kind: "version_change" | "leader_change" | "no_leader" | "instances_lost"; + detail: string; +} + +export type QueueHealthKind = + | "stalled" + | "behind" + | "slow" + | "paused" + | "draining" + | "unknown" + | "steady" + | "idle"; + +export type LimitedBy = + | "paused" + | "no_workers" + | "rate_limit" + | "worker_capacity" + | "nothing"; + +export interface QueueAssessment { + queue: string; + health: QueueHealthKind; + label: string; + reason: string; + needs_attention: boolean; + rank: number; + limited_by: LimitedBy | null; + trend: { per_minute: number; change: number; minutes: number } | null; + clear_seconds: number | null; + flow: { + claimed_per_minute: number | null; + completed_per_minute: number | null; + failed_per_minute: number | null; + retried_per_minute: number | null; + arrivals_per_minute: number | null; + mean_claim_wait_seconds: number | null; + rate_limited: boolean; + }; + live_workers: number; + in_flight: number; + capacity: number | null; +} + +/** `/api/health/queues`: is every queue keeping up, worst first. */ +export interface HealthReport { + status: "ok" | "unknown" | "warning" | "critical"; + title: string; + detail: string; + generated_at: string; + history_available: boolean; + history_since: string | null; + queues: QueueAssessment[]; +} + +export function fetchQueueHistory(minutes = 60): Promise { + return apiFetch(`/stats/queue-history?minutes=${minutes}`); +} + +export function fetchQueueHealth(): Promise { + return apiFetch("/health/queues"); +} + export function fetchDistinctKinds(): Promise { return apiFetch("/stats/kinds"); } diff --git a/awa-ui/frontend/src/lib/operations.ts b/awa-ui/frontend/src/lib/operations.ts new file mode 100644 index 000000000..041820e34 --- /dev/null +++ b/awa-ui/frontend/src/lib/operations.ts @@ -0,0 +1,226 @@ +import type { + HealthReport, + JobRow, + LimitedBy, + QueueAssessment, + QueueHealthKind, + QueueHistory, + QueueRuntimeSummary, + QueueSeries, + QueueStats, + RuntimeInstance, +} from "./api"; + +export function formatDuration(seconds: number | null): string { + if (seconds === null || !Number.isFinite(seconds)) return "—"; + const value = Math.max(0, Math.round(seconds)); + if (value < 60) return `${value}s`; + if (value < 3600) return `${Math.floor(value / 60)}m ${value % 60}s`; + if (value < 86400) + return `${Math.floor(value / 3600)}h ${Math.floor((value % 3600) / 60)}m`; + return `${Math.floor(value / 86400)}d ${Math.floor((value % 86400) / 3600)}h`; +} + +const compact = new Intl.NumberFormat("en", { + notation: "compact", + maximumFractionDigits: 1, +}); + +export function formatCount(value: number): string { + return Math.abs(value) < 10_000 + ? value.toLocaleString() + : compact.format(value); +} + +export function formatRate(perMinute: number, signed = false): string { + const magnitude = Math.abs(perMinute); + const rounded = + magnitude >= 10 ? formatCount(Math.round(magnitude)) : magnitude.toFixed(1); + const sign = !signed ? "" : perMinute > 0 ? "+" : perMinute < 0 ? "−" : ""; + return `${sign}${rounded}/min`; +} + +/** A queue joined with the server's assessment and its history series. */ +export interface AssessedQueue { + queue: QueueStats; + /** Missing only while the health report is loading or failed. */ + assessment: QueueAssessment | undefined; + series: QueueSeries | undefined; +} + +/** Join snapshots to the health report, in the report's risk order. Queues + * the report does not know yet (created since it was built) go last. */ +export function joinQueues( + queues: ReadonlyArray, + report: HealthReport | undefined, + history: QueueHistory | undefined, +): AssessedQueue[] { + const order = new Map(report?.queues.map((a, index) => [a.queue, index])); + const byName = new Map(report?.queues.map((a) => [a.queue, a])); + return queues + .map((queue) => ({ + queue, + assessment: byName.get(queue.queue), + series: history?.queues[queue.queue], + })) + .sort( + (left, right) => + (order.get(left.queue.queue) ?? Infinity) - + (order.get(right.queue.queue) ?? Infinity) || + left.queue.queue.localeCompare(right.queue.queue), + ); +} + +export const LIMITED_BY_LABEL: Record = { + paused: "Paused", + no_workers: "No workers", + rate_limit: "Rate limit", + worker_capacity: "Worker capacity", + nothing: "No visible limit", +}; + +export const LIMITED_BY_HINT: Record = { + paused: "The queue is paused, so nothing is claimed.", + no_workers: "No live worker runtime declares this queue.", + rate_limit: + "Dispatch skipped polls in the last 5 minutes because the queue's rate limit had no tokens.", + worker_capacity: "Every worker permit for this queue is in use.", + nothing: + "Workers had free permits. If work still waits, look at claim latency, the database, or job duration.", +}; + +export function needsAttention(row: AssessedQueue): boolean { + return row.assessment?.needs_attention ?? false; +} + +export function healthOf(row: AssessedQueue): QueueHealthKind { + return row.assessment?.health ?? "unknown"; +} + +/** Per-bucket total of waiting jobs across queues; null where no queue was + * sampled. */ +export function totalBacklog( + history: QueueHistory | undefined, +): (number | null)[] { + if (!history) return []; + const totals: (number | null)[] = history.buckets.map(() => null); + for (const series of Object.values(history.queues)) { + series.available.forEach((value, index) => { + if (value === null) return; + totals[index] = (totals[index] ?? 0) + value; + }); + } + return totals; +} + +/** Sum a per-queue rate, or null when any queue's rate is unknown. A callback + * receiver can complete work without worker runtimes; missing coverage is not zero. */ +export function totalRate( + report: HealthReport | undefined, + pick: (flow: QueueAssessment["flow"]) => number | null, +): number | null { + if (!report) return null; + let total = 0; + for (const queue of report.queues) { + const value = pick(queue.flow); + if (value === null) return null; + total += value; + } + return total; +} + +/** Bucket index of each fleet event, for chart markers. */ +export function eventIndexes(history: QueueHistory | undefined): number[] { + if (!history) return []; + const starts = history.buckets.map((bucket) => new Date(bucket).getTime()); + return history.events + .map((event) => starts.indexOf(new Date(event.at).getTime())) + .filter((index) => index >= 0); +} + +export function lastErrorMessage(job: JobRow): string { + const last = job.errors?.[job.errors.length - 1]; + if (typeof last === "object" && last && "error" in last) { + return String((last as { error: unknown }).error); + } + return last ? JSON.stringify(last) : "No error details recorded"; +} + +export interface FailureGroup { + key: string; + kind: string; + queue: string; + message: string; + count: number; + latest: JobRow; +} + +/** Group retained failed jobs by kind, queue, and error text so one recurring + * fault reads as one line with a count. Long digit runs and UUIDs are masked + * so per-job identifiers do not split a group. */ +export function groupFailures(jobs: ReadonlyArray): FailureGroup[] { + const groups = new Map(); + for (const job of jobs) { + const message = lastErrorMessage(job); + const signature = message + .replace( + /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/gi, + "…", + ) + .replace(/\d{4,}/g, "…"); + const key = `${job.kind}\u0000${job.queue}\u0000${signature}`; + const group = groups.get(key); + if (group) { + group.count += 1; + if ((job.finalized_at ?? "") > (group.latest.finalized_at ?? "")) { + group.latest = job; + group.message = message; + } + } else { + groups.set(key, { + key, + kind: job.kind, + queue: job.queue, + message, + count: 1, + latest: job, + }); + } + } + return [...groups.values()].sort( + (left, right) => + right.count - left.count || + (right.latest.finalized_at ?? "").localeCompare( + left.latest.finalized_at ?? "", + ), + ); +} + +/** Concurrency and rate limit as configured by the live runtimes, e.g. + * "max 64 · 2/s". Rate limits are the usual reason a queue backs up while + * workers sit idle, so they belong next to the backlog. */ +export function capacityLabel( + summary: QueueRuntimeSummary | undefined, +): string { + const config = summary?.config; + if (!config) return "—"; + const concurrency = + config.mode === "weighted" + ? `min ${config.min_workers ?? 0} · weight ${config.weight ?? 1}` + : `max ${config.max_workers ?? 0}`; + return config.rate_limit + ? `${concurrency} · ${config.rate_limit.max_rate}/s` + : concurrency; +} + +export function runtimeConcern(instance: RuntimeInstance): string { + if (instance.stale) return "Snapshot stale · health unknown"; + if (instance.shutting_down) return "Shutting down"; + const problems = [ + !instance.postgres_connected && "Postgres disconnected", + !instance.poll_loop_alive && "Poll loop stopped", + !instance.heartbeat_alive && "Heartbeat loop stopped", + !instance.maintenance_alive && "Maintenance loop stopped", + ].filter(Boolean); + return problems.join(" · ") || (instance.healthy ? "Healthy" : "Degraded"); +} diff --git a/awa-ui/frontend/src/routes/index.tsx b/awa-ui/frontend/src/routes/index.tsx index 6400e1d91..e5ea58fc5 100644 --- a/awa-ui/frontend/src/routes/index.tsx +++ b/awa-ui/frontend/src/routes/index.tsx @@ -1,586 +1,510 @@ -import { useMemo } from "react"; +import { useState } from "react"; import { useQuery } from "@tanstack/react-query"; -import { Link, useNavigate } from "@tanstack/react-router"; +import { Link } from "@tanstack/react-router"; import { fetchStats, fetchQueues, + fetchQueueHealth, + fetchQueueHistory, fetchJobs, fetchRuntime, - fetchTimeseries, + fetchDlqDepth, } from "@/lib/api"; -import type { - StateCounts, - QueueStats, - JobRow, - RuntimeOverview, - TimeseriesBucket, -} from "@/lib/api"; -import { StateBadge } from "@/components/StateBadge"; -import { Sparkline } from "@/components/Sparkline"; -import { Heading } from "@/components/ui/heading"; -import { Card, CardAction, CardContent, CardHeader } from "@/components/ui/card"; +import type { HealthReport, QueueHistory } from "@/lib/api"; import { - Table, - TableHeader, - TableBody, - TableRow, - TableCell, - TableColumn, -} from "@/components/ui/table"; -import { Badge } from "@/components/ui/badge"; -import { LagValue } from "@/components/LagValue"; + eventIndexes, + formatCount, + formatRate, + groupFailures, + joinQueues, + healthOf, + needsAttention, + runtimeConcern, + totalBacklog, + totalRate, +} from "@/lib/operations"; import { timeAgo } from "@/lib/time"; -import { DASHBOARD_QUEUE_LIMIT } from "@/lib/constants"; import { usePollInterval } from "@/hooks/use-poll-interval"; +import { Heading } from "@/components/ui/heading"; +import { + Card, + CardAction, + CardContent, + CardHeader, +} from "@/components/ui/card"; +import { Button } from "@/components/ui/button"; +import { HistorySpark } from "@/components/HistorySpark"; +import { QueueFlowTable } from "@/components/QueueFlowTable"; +import { SnapshotStatus } from "@/components/SnapshotStatus"; + +const FAILURE_SAMPLE = 50; -/** Background tint per state for counter cards */ -const STATE_CARD_BG: Record = { - available: "bg-info-subtle", - running: "bg-success-subtle", - failed: "bg-danger-subtle", - scheduled: "bg-secondary/50", - waiting_external: "bg-[oklch(0.87_0.07_280)]/30", +const VERDICT_TONE: Record = { + critical: "border-l-danger", + warning: "border-l-warning", + unknown: "border-l-border", + ok: "border-l-success", }; -/** Headline metrics — includes deferred and callback-blocked work */ -const COUNTER_KEYS = ["available", "running", "failed", "scheduled", "waiting_external"] as const; +/** Shown only when trends are missing, so the page explains a gap rather + * than narrating what it shows. */ +function historyGap(history: QueueHistory | undefined): string | null { + if (!history) return null; + if (!history.installed) + return "No trends: queue history needs schema v058 (run awa migrate)."; + if (!history.observed_since) + return "No trends yet: the maintenance leader records one sample a minute."; + return null; +} export function DashboardPage() { - const navigate = useNavigate(); const poll = usePollInterval(); + const polling = { refetchInterval: poll.interval, staleTime: poll.staleTime }; + const [showAll, setShowAll] = useState(false); - const statsQuery = useQuery({ + const stats = useQuery({ queryKey: ["stats"], queryFn: fetchStats, - refetchInterval: poll.interval, staleTime: poll.staleTime, + ...polling, }); - - // Timeseries powers the inline sparklines on the state counter cards. - // Returns an empty array on pre-v10 deployments / seed-free databases, in - // which case Sparkline renders an empty slot of its reserved size so the - // card layout doesn't jump. - const timeseriesQuery = useQuery({ - queryKey: ["stats", "timeseries", 60], - queryFn: () => fetchTimeseries(60), - refetchInterval: poll.interval, - staleTime: poll.staleTime, - }); - - /** Pivot the flat timeseries response into a per-state numeric series - * suitable for . Missing buckets are filled as 0. */ - const seriesByState = useMemo(() => { - const buckets = timeseriesQuery.data ?? []; - const byBucket = new Map>(); - const ordered: string[] = []; - for (const point of buckets) { - if (!byBucket.has(point.bucket)) { - byBucket.set(point.bucket, new Map()); - ordered.push(point.bucket); - } - byBucket.get(point.bucket)!.set(point.state, point.count); - } - ordered.sort(); - const result: Record = {}; - for (const bucket of ordered) { - const stateMap = byBucket.get(bucket)!; - for (const state of stateMap.keys()) { - if (!result[state]) result[state] = []; - } - } - for (const bucket of ordered) { - const stateMap = byBucket.get(bucket)!; - for (const state of Object.keys(result)) { - const series = result[state]; - if (series) series.push(stateMap.get(state) ?? 0); - } - } - return result; - }, [timeseriesQuery.data]); - - const queuesQuery = useQuery({ + const queues = useQuery({ queryKey: ["queues"], queryFn: fetchQueues, - refetchInterval: poll.interval, staleTime: poll.staleTime, + ...polling, }); - - const failedQuery = useQuery({ - queryKey: ["jobs", { state: "failed", limit: 10 }], - queryFn: () => fetchJobs({ state: "failed", limit: 10 }), - refetchInterval: poll.interval, staleTime: poll.staleTime, + const history = useQuery({ + queryKey: ["queue-history", 60], + queryFn: () => fetchQueueHistory(60), + ...polling, }); - - const runtimeQuery = useQuery({ + const health = useQuery({ + queryKey: ["queue-health"], + queryFn: fetchQueueHealth, + ...polling, + }); + const runtime = useQuery({ queryKey: ["runtime"], queryFn: fetchRuntime, - refetchInterval: poll.interval, staleTime: poll.staleTime, + ...polling, + }); + const failures = useQuery({ + queryKey: ["jobs", { state: "failed", limit: FAILURE_SAMPLE }], + queryFn: () => fetchJobs({ state: "failed", limit: FAILURE_SAMPLE }), + ...polling, + }); + const dlq = useQuery({ + queryKey: ["dlq-depth"], + queryFn: fetchDlqDepth, + refetchInterval: Math.max(poll.interval, 60_000), + staleTime: Math.max(poll.staleTime, 60_000), }); - const completedPerHour = queuesQuery.data - ? queuesQuery.data.reduce((sum, q) => sum + q.completed_last_hour, 0) - : null; + const report = health.data; + const assessed = joinQueues(queues.data ?? [], report, history.data); + const attention = assessed.filter(needsAttention); + // Draining queues stay in view so recovery is visible, not just declared. + const focus = assessed.filter( + (row) => needsAttention(row) || healthOf(row) === "draining", + ); + const draining = focus.length - attention.length; + const rest = assessed.filter((row) => !focus.includes(row)); - const totalJobs = statsQuery.data - ? Object.values(statsQuery.data).reduce((a, b) => a + b, 0) + const backlogSeries = totalBacklog(history.data); + const markers = eventIndexes(history.data); + const backlog = queues.data?.reduce((sum, queue) => sum + queue.available, 0); + // Measured rates when every served queue has counters, otherwise the + // completed-last-hour average so the number never silently drops. + const measuredCompleted = totalRate(report, (f) => f.completed_per_minute); + const completedPerMinute = + measuredCompleted ?? + (queues.data + ? queues.data.reduce((sum, queue) => sum + queue.completed_last_hour, 0) / + 60 + : undefined); + const arrivalsPerMinute = totalRate(report, (f) => f.arrivals_per_minute); + const backlogTrend = report + ? report.queues.reduce( + (sum, queue) => + queue.trend && sum !== null ? sum + queue.trend.per_minute : sum, + 0, + ) : null; + const recentEvents = [...(history.data?.events ?? [])].reverse().slice(0, 5); + + const instances = runtime.data?.instances ?? []; + const troubled = instances.filter( + (instance) => instance.stale || !instance.healthy, + ); + const failureGroups = groupFailures(failures.data ?? []); - // Sort queues by visible workload so deferred-heavy queues stay prominent. - const topQueues = queuesQuery.data - ? [...queuesQuery.data] - .sort( - (a, b) => b.total_queued + b.failed - (a.total_queued + a.failed) - ) - .slice(0, DASHBOARD_QUEUE_LIMIT) - : []; + const errors = [ + ["Job counts", stats], + ["Queues", queues], + ["Queue history", history], + ["Queue health", health], + ["Workers", runtime], + ["Failed jobs", failures], + ["Dead letter queue", dlq], + ] + .filter(([, query]) => (query as { isError: boolean }).isError) + .map(([name]) => name as string); - const hasMoreQueues = - queuesQuery.data && queuesQuery.data.length > DASHBOARD_QUEUE_LIMIT; + const counters = [ + { label: "Running", state: "running", value: stats.data?.running }, + { + label: "Retrying", + state: "retryable", + value: stats.data?.retryable, + tone: "text-warning-subtle-fg", + }, + { + label: "Failed", + state: "failed", + value: stats.data?.failed, + tone: "text-danger-subtle-fg", + }, + { label: "Scheduled", state: "scheduled", value: stats.data?.scheduled }, + { + label: "Waiting external", + state: "waiting_external", + value: stats.data?.waiting_external, + }, + ]; return (
- Dashboard +
+ Overview + + {historyGap(history.data) && ( +

{historyGap(history.data)}

+ )} +
- {/* Headline counter cards */} -
- {COUNTER_KEYS.map((key) => { - const count = statsQuery.data?.[key]; - const bg = STATE_CARD_BG[key] ?? ""; - const series = seriesByState[key] ?? []; - return ( - - 0 && ( +
+ Could not refresh: {errors.join(", ")}. Previous values stay visible + where available. +
+ )} + + + +
+
+

- -
- {statsQuery.isLoading ? ( - - ) : ( - (count ?? 0).toLocaleString() - )} -
-
- - -
- - - - ); - })} - - - -
- {queuesQuery.isLoading ? ( - - ) : completedPerHour != null ? ( - completedPerHour.toLocaleString() - ) : ( - "—" - )} + {report?.title ?? + (health.isError + ? "Queue health unavailable" + : "Checking queues…")} +

+ {report && ( +

{report.detail}

+ )} +
+
+
+

+ Waiting across all queues +

+ + {backlog?.toLocaleString() ?? "—"} + +

+ {report?.history_available && backlogTrend !== null + ? // Same noise floor the server applies per queue. + Math.abs(backlogTrend) * 5 < + Math.max(20, (backlog ?? 0) * 0.05) + ? "flat over 5m" + : `${formatRate(backlogTrend, true)} over 5m` + : "No trend yet"} +

-
completed/hr
- - - -
+ +
+

Finishing

+

+ {completedPerMinute === undefined + ? "—" + : formatRate(completedPerMinute)} +

+

+ {measuredCompleted !== null + ? arrivalsPerMinute !== null + ? `arriving ${formatRate(arrivalsPerMinute)} · last 5m` + : "measured, last 5m" + : "average, last hour"} +

+
+
+
+ + - {/* Total jobs count */} - {totalJobs != null && ( -

- {totalJobs.toLocaleString()} total jobs across{" "} - {queuesQuery.data?.length ?? 0} queues -

- )} + 0 - ? `${runtimeQuery.data.instances.length} instance(s) · snapshots every ${ - (runtimeQuery.data.instances[0]?.snapshot_interval_ms ?? 10000) >= 1000 - ? `${Math.round((runtimeQuery.data.instances[0]?.snapshot_interval_ms ?? 10000) / 1000)}s` - : `${runtimeQuery.data.instances[0]?.snapshot_interval_ms ?? 0}ms` - }` - : "Worker instances, leader health, and current runtime topology" + title={ + attention.length + ? `Needs attention (${attention.length})` + + (draining ? ` · recovering (${draining})` : "") + : draining + ? `Recovering (${draining})` + : "Queues" } > - - Open runtime + + All queues - -
-
-
Live
-
- {runtimeQuery.data?.live_instances ?? "—"} -
-
-
-
Healthy
-
- {runtimeQuery.data?.healthy_instances ?? "—"} -
-
-
-
Leader
-
- {runtimeQuery.data?.leader_instances ?? "—"} -
-
-
-
Stale
-
- {runtimeQuery.data?.stale_instances ?? "—"} -
-
-
- - {/* Mobile runtime cards */} - {(runtimeQuery.data?.instances.length ?? 0) === 0 ? ( -
- {runtimeQuery.isLoading - ? "Loading runtime…" - : runtimeQuery.isError - ? "Failed to load runtime data." - : "No runtime snapshots yet. Start a worker to populate this view."} -
+ + {assessed.length === 0 ? ( +

+ {queues.isPending + ? "Loading queues…" + : queues.isError + ? "Queue data unavailable." + : "No queues yet. Queues appear when registered or used."} +

) : ( -
- {runtimeQuery.data!.instances.map((instance) => { - const label = instance.hostname ?? `pid ${instance.pid}`; - const healthLabel = instance.stale - ? "Stale" - : instance.healthy - ? "Healthy" - : "Degraded"; - const healthIntent = instance.stale - ? ("warning" as const) - : instance.healthy - ? ("success" as const) - : ("danger" as const); - return ( -
-
-
- {label} - - {instance.version} - -
- {healthLabel} -
-
- - poll - - - hb - - - maint - - {instance.leader && ( - Leader - )} -
-
- {instance.queues.length} queue(s) · seen {timeAgo(instance.last_seen_at)} -
-
- ); - })} -
- )} - - {/* Desktop runtime table */} - - - Instance - Health - Loops - Role - Queues - Seen - - ( -
- {runtimeQuery.isLoading - ? "Loading runtime…" - : runtimeQuery.isError - ? "Failed to load runtime data." - : "No runtime snapshots yet. Start a worker to populate this view."} + <> + {(focus.length > 0 || showAll) && ( + + )} + {rest.length > 0 && ( +
+
)} - > - {(runtimeQuery.data?.instances ?? []).map((instance) => { - const label = instance.hostname ?? `pid ${instance.pid}`; - const healthLabel = instance.stale - ? "Stale" - : instance.healthy - ? "Healthy" - : "Degraded"; - const healthIntent = instance.stale - ? "warning" - : instance.healthy - ? "success" - : "danger"; - return ( - - -
{label}
-
- {instance.version} · pid {instance.pid} -
- - - {healthLabel} - - -
- - poll - - - heartbeat - - - maintenance - -
- - - {instance.leader ? ( - Leader - ) : ( - Worker - )} - - - {instance.queues.length} - - {timeAgo(instance.last_seen_at)} - - ); - })} - -
+ + )} - {/* Queue summary — top N by activity */} - - - - - - Queue - Total queued - Scheduled - Available - Retryable - Running - Failed - Waiting - Completed/hr - Lag (s) - Status - - ( -
- {queuesQuery.isLoading - ? "Loading queues…" - : queuesQuery.isError - ? "Failed to load queues." - : "No queues found."} -
- )} - > - {topQueues.map((q) => ( - - void navigate({ - to: "/jobs", - search: { q: `queue:${q.queue}` }, - }) - } +
+ + + + + All failed + + + + + {failureGroups.length ? ( +
    + {failureGroups.slice(0, 6).map((group) => ( +
  • + - -
    - - {q.display_name ?? q.queue} - - {q.display_name && ( -
    - {q.queue} -
    - )} -
    -     - - {q.total_queued.toLocaleString()} - - - {q.scheduled.toLocaleString()} - - - {q.available.toLocaleString()} - - - {q.retryable.toLocaleString()} - - - {q.running.toLocaleString()} - - - 0 ? "text-danger" : ""}> - {q.failed.toLocaleString()} + + ×{group.count} + + + + {group.latest.kind_descriptor?.display_name?.trim() || + group.kind} + + {" "} + · {group.queue} + - - - {q.waiting_external > 0 - ? q.waiting_external.toLocaleString() - : "-"} - - - {q.completed_last_hour.toLocaleString()} - - - - - - {q.paused ? ( - Paused - ) : ( - Active - )} - - - ))} - -
- {hasMoreQueues && topQueues.length > 0 && ( -
+ + {group.message} + + + + {group.latest.finalized_at + ? timeAgo(group.latest.finalized_at) + : "—"} + + + + ))} + + ) : ( +

+ {failures.isPending + ? "Loading failed jobs…" + : failures.isError + ? "Failed-job data unavailable." + : "No failed jobs in retained data."} +

+ )} + + + + + + - View all {queuesQuery.data?.length} queues → + All workers -
- )} - - - - {/* Recent failures */} - - - - - - ID - Kind - Queue - Attempt - Failed - Error - - ( -
- {failedQuery.isLoading - ? "Loading failures…" - : failedQuery.isError - ? "Failed to load recent failures." - : "No recent failures."} -
- )} - > - {(failedQuery.data ?? []).map((job) => { - const lastErr = - job.errors && job.errors.length > 0 - ? job.errors[job.errors.length - 1] - : null; - const errMsg = - lastErr && - typeof lastErr === "object" && - lastErr !== null - ? String( - (lastErr as Record)["error"] ?? "" - ) - : ""; - return ( - - void navigate({ - to: "/jobs/$id", - params: { id: String(job.id) }, - }) - } + + + + {troubled.length ? ( +
    + {troubled.slice(0, 6).map((instance) => ( +
  • + - - {job.id} - - {job.kind} - {job.queue} - - {job.attempt}/{job.max_attempts} - - - {job.finalized_at - ? timeAgo(job.finalized_at) - : "-"} - - - {errMsg || "-"} - - - ); - })} - -
- - + + + {instance.hostname || `pid ${instance.pid}`} + + + {runtimeConcern(instance)} + + + + Seen {timeAgo(instance.last_seen_at)} + {instance.version} + + + + ))} + {troubled.length > 6 && ( +
  • + and {troubled.length - 6} more on the Workers page +
    • + )} + + ) : ( +

    + {runtime.isPending + ? "Loading workers…" + : runtime.isError + ? "Worker data unavailable." + : instances.length + ? `All ${instances.length} reporting instances are healthy.` + : "No runtime snapshots. Start a worker to report health."} +

    + )} + {recentEvents.length > 0 && ( +
    +

    + Fleet changes, last hour +

    +
      + {recentEvents.map((event) => ( +
    • + {event.detail} + + {timeAgo(event.at)} + +
      • + ))} +
    +
    + )} + + +
    ); } diff --git a/awa-ui/frontend/src/routes/jobs.tsx b/awa-ui/frontend/src/routes/jobs.tsx index 57202d6bf..2286c9096 100644 --- a/awa-ui/frontend/src/routes/jobs.tsx +++ b/awa-ui/frontend/src/routes/jobs.tsx @@ -4,7 +4,7 @@ import { useQueryClient, } from "@tanstack/react-query"; import { useState, useCallback, useMemo } from "react"; -import { useNavigate, useSearch } from "@tanstack/react-router"; +import { Link, useNavigate, useSearch } from "@tanstack/react-router"; import { fetchJobs, fetchStats, @@ -39,6 +39,7 @@ import { RETRYABLE_STATES, TERMINAL_STATES, } from "@/lib/constants"; +import { SnapshotStatus } from "@/components/SnapshotStatus"; import { usePollInterval } from "@/hooks/use-poll-interval"; const STATES = [ @@ -279,24 +280,30 @@ export function JobsPage() { )} + {hasSel ? ( +

    Job refresh paused while rows are selected.

    + ) : ( + + )} + {/* Queue context banner — shown when filtering by a single queue */} {activeQueue && queueStats && (
    {queueStats.paused ? ( Paused ) : ( - Active + Claims enabled )} {queueStats.description && ( {queueStats.description} )} - {queueStats.available} available ·{" "} - {queueStats.running} running ·{" "} + {queueStats.available.toLocaleString()} available ·{" "} + {queueStats.running.toLocaleString()} running ·{" "} 0 ? "text-danger" : ""}> - {queueStats.failed} failed + {queueStats.failed.toLocaleString()} failed {" "} - · {queueStats.completed_last_hour}/hr + · {queueStats.completed_last_hour.toLocaleString()} completed in the last hour
    {queueStats.paused ? ( @@ -334,12 +341,17 @@ export function JobsPage() { {/* Compact state filter pills + search on same row */}
    {STATES.map((s) => { - const count = s === "all" ? total : statsQuery.data?.[s]; + const count = activeQueue + ? s === "all" || s === "completed" || s === "cancelled" + ? undefined + : queueStats?.[s as keyof QueueStats] + : s === "all" ? total : statsQuery.data?.[s]; const isActive = filters.state === s; return (