diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 10a813d..c71deea 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -179,6 +179,14 @@ jobs: --notes-file "$notes" gh release upload "$RELEASE_TAG" "${assets[@]}" fi + published_artifacts=$(mktemp -d) + trap 'rm -rf "$published_artifacts"' EXIT + gh release download "$RELEASE_TAG" --dir "$published_artifacts" + cmp "release/SHA256SUMS-$version" "$published_artifacts/SHA256SUMS-$version" + ( + cd "$published_artifacts" + sha256sum --check "SHA256SUMS-$version" + ) gh release edit "$RELEASE_TAG" \ --title "RustQueue $version" \ --notes-file "$notes" \ diff --git a/.github/workflows/performance-qualification.yml b/.github/workflows/performance-qualification.yml new file mode 100644 index 0000000..0717d69 --- /dev/null +++ b/.github/workflows/performance-qualification.yml @@ -0,0 +1,59 @@ +name: performance-qualification + +on: + pull_request: + paths: + - .github/workflows/performance-qualification.yml + - scripts/benchmark-qualify.sh + workflow_dispatch: + inputs: + baseline_ref: + description: Published baseline version tag + required: true + default: v0.10.0 + +permissions: + contents: read + +concurrency: + group: performance-${{ github.ref }} + cancel-in-progress: true + +jobs: + qualify: + runs-on: ubuntu-24.04-arm + timeout-minutes: 360 + env: + QUALIFICATION_ENVIRONMENT: github_actions + WARMUP_SECONDS: 30 + MEASUREMENT_SECONDS: 120 + DRAIN_TIMEOUT_SECONDS: 1800 + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + fetch-depth: 0 + - name: Resolve exact candidate and release version + run: | + echo "CANDIDATE_REF=$(git rev-parse HEAD)" >> "$GITHUB_ENV" + version=$(awk '/^\[workspace.package\]$/ {p=1;next} /^\[/ {p=0} p && $1=="version" {gsub(/"/,"",$3);print $3;exit}' Cargo.toml) + echo "RELEASE=$version" >> "$GITHUB_ENV" + - name: Same-binary calibration before comparison + env: + PAIRS: 2 + CALIBRATION_ONLY: 1 + EVIDENCE_OUTPUT: ${{ github.workspace }}/benchmarks/results/aa-calibration.json + run: BASELINE_REF="$CANDIDATE_REF" ./scripts/benchmark-qualify.sh + - name: Compare published baseline and candidate + env: + BASELINE_REF: ${{ inputs.baseline_ref || 'v0.10.0' }} + PAIRS: 10 + EVIDENCE_OUTPUT: ${{ github.workspace }}/benchmarks/results/ab-qualification.json + run: ./scripts/benchmark-qualify.sh + - uses: actions/upload-artifact@v4 + if: always() + with: + name: performance-qualification-${{ github.run_id }} + path: benchmarks/results/** + if-no-files-found: error + retention-days: 7 diff --git a/Cargo.lock b/Cargo.lock index 26974c9..61baeb1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2315,7 +2315,7 @@ dependencies = [ [[package]] name = "rustqueue-bench" -version = "0.10.0" +version = "0.10.1" dependencies = [ "anyhow", "clap", @@ -2327,7 +2327,7 @@ dependencies = [ [[package]] name = "rustqueue-console" -version = "0.10.0" +version = "0.10.1" dependencies = [ "anyhow", "axum", @@ -2355,7 +2355,7 @@ dependencies = [ [[package]] name = "rustqueue-discovery" -version = "0.10.0" +version = "0.10.1" dependencies = [ "anyhow", "axum", @@ -2379,7 +2379,7 @@ dependencies = [ [[package]] name = "rustqueue-operator" -version = "0.10.0" +version = "0.10.1" dependencies = [ "anyhow", "axum", @@ -2406,7 +2406,7 @@ dependencies = [ [[package]] name = "rustqueue-protocol" -version = "0.10.0" +version = "0.10.1" dependencies = [ "bytes", "serde", @@ -2416,7 +2416,7 @@ dependencies = [ [[package]] name = "rustqueue-proxy" -version = "0.10.0" +version = "0.10.1" dependencies = [ "anyhow", "axum", @@ -2437,7 +2437,7 @@ dependencies = [ [[package]] name = "rustqueue-queue" -version = "0.10.0" +version = "0.10.1" dependencies = [ "anyhow", "bytes", @@ -2459,7 +2459,7 @@ dependencies = [ [[package]] name = "rustqueue-server" -version = "0.10.0" +version = "0.10.1" dependencies = [ "anyhow", "async-compression", @@ -2495,7 +2495,7 @@ dependencies = [ [[package]] name = "rustqueue-storage" -version = "0.10.0" +version = "0.10.1" dependencies = [ "anyhow", "crc32c", @@ -2508,14 +2508,14 @@ dependencies = [ [[package]] name = "rustqueue-telemetry" -version = "0.10.0" +version = "0.10.1" dependencies = [ "serde", ] [[package]] name = "rustqueuectl" -version = "0.10.0" +version = "0.10.1" dependencies = [ "anyhow", "clap", @@ -2523,8 +2523,10 @@ dependencies = [ "k8s-openapi", "kube", "reqwest", + "rustls", "rustqueue-operator", "serde_json", + "tempfile", "tokio", ] diff --git a/Cargo.toml b/Cargo.toml index 147851a..bbdce90 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,7 +16,7 @@ members = [ exclude = ["fuzz"] [workspace.package] -version = "0.10.0" +version = "0.10.1" edition = "2021" license = "Apache-2.0" rust-version = "1.88" diff --git a/README.md b/README.md index 1865bf0..0900fe9 100644 --- a/README.md +++ b/README.md @@ -11,14 +11,14 @@ [NSQ performance boundaries](docs/architecture/nsq-performance.md) · [Kubernetes operations](docs/operations/kubernetes.md) · [Console operations](docs/operations/console.md) · -[v0.10.0 release](https://github.com/SamuelSupe/rustqueue/releases/tag/v0.10.0) +[v0.10.1 release](https://github.com/SamuelSupe/rustqueue/releases/tag/v0.10.1) -RustQueue 0.10.0 is a Kubernetes-native, NSQ V2-compatible message queue for +RustQueue 0.10.1 is a Kubernetes-native, NSQ V2-compatible message queue for trusted internal networks. It is written in Rust and uses a deliberately simple share-nothing model: each Broker owns one durable RWO PVC, while Kubernetes provides scheduling, rollout and discovery. -> Current release: [v0.10.0](https://github.com/SamuelSupe/rustqueue/releases/tag/v0.10.0). +> Current release: [v0.10.1](https://github.com/SamuelSupe/rustqueue/releases/tag/v0.10.1). > RustQueue is a production candidate for workloads that accept single-PVC > durability and at-least-once delivery. It does not replicate messages between > Brokers and is not an HA replacement for a replicated log. @@ -45,7 +45,13 @@ messages stored on that Broker are lost. Configure disk pressure protection, monitor the exported metrics, and choose PVC/storage failure policies that fit your workload before deploying to production. -## What's new in 0.10.0 +## What's new in 0.10.1 + +This patch fixes delayed-message memory growth, legacy checkpoint recovery, +slow-consumer metadata budgets, heartbeat timing, recovery status, TLS CLI +startup, and producer-proxy rollout continuity. The optional WebSocket live +subscription introduced in 0.10.0 remains default-off. + - **TTL live WebSocket subscriptions.** The default-off WebSocket endpoint streams only existing `TTL_DISCARD` Topics. It is a best-effort feed with no @@ -59,29 +65,29 @@ your workload before deploying to production. capability, while NSQ TCP/HTTP publishing, ACK modes, storage format v7 and existing `RELIABLE` behavior remain unchanged. -See the [v0.10.0 release notes](docs/releases/v0.10.0.md) and [NSQ performance +See the [v0.10.1 release notes](docs/releases/v0.10.1.md) and [NSQ performance boundaries](docs/architecture/nsq-performance.md) for the contract and benchmark interpretation. -## Download 0.10.0 +## Download 0.10.1 Every release contains native Linux binaries, the Console UI, source, the Helm Chart and a checksum manifest: | Asset | Contents | | --- | --- | -| `rustqueue-0.10.0-linux-x86_64.tar.gz` | Linux x86_64 binaries, Console UI and example configuration | -| `rustqueue-0.10.0-linux-aarch64.tar.gz` | Linux ARM64 binaries, Console UI and example configuration | -| `rustqueue-0.10.0-source.tar.gz` | Source archive for the tagged commit | -| `rustqueue-0.10.0.tgz` | Helm Chart | -| `SHA256SUMS-0.10.0` | SHA-256 checksums for every downloadable artifact | +| `rustqueue-0.10.1-linux-x86_64.tar.gz` | Linux x86_64 binaries, Console UI and example configuration | +| `rustqueue-0.10.1-linux-aarch64.tar.gz` | Linux ARM64 binaries, Console UI and example configuration | +| `rustqueue-0.10.1-source.tar.gz` | Source archive for the tagged commit | +| `rustqueue-0.10.1.tgz` | Helm Chart | +| `SHA256SUMS-0.10.1` | SHA-256 checksums for every downloadable artifact | ```sh arch="$(uname -m)" -curl -LO "https://github.com/SamuelSupe/rustqueue/releases/download/v0.10.0/rustqueue-0.10.0-linux-${arch}.tar.gz" -curl -LO "https://github.com/SamuelSupe/rustqueue/releases/download/v0.10.0/SHA256SUMS-0.10.0" -sha256sum --check --ignore-missing SHA256SUMS-0.10.0 -tar -xzf "rustqueue-0.10.0-linux-${arch}.tar.gz" +curl -LO "https://github.com/SamuelSupe/rustqueue/releases/download/v0.10.1/rustqueue-0.10.1-linux-${arch}.tar.gz" +curl -LO "https://github.com/SamuelSupe/rustqueue/releases/download/v0.10.1/SHA256SUMS-0.10.1" +sha256sum --check --ignore-missing SHA256SUMS-0.10.1 +tar -xzf "rustqueue-0.10.1-linux-${arch}.tar.gz" ``` ## Architecture @@ -144,7 +150,15 @@ operator -> eligible nodes -> StatefulSet ordinal + retained RWO PVC buffer, and admission charges the input plus bounded encoding metadata. - Backlog has no message-count ceiling. Sealed segments keep only constant-size summaries in RAM; fixed-size message metadata is paged through one bounded - Broker cache. Publish admission is therefore governed by the configured PVC + Broker cache. Sealing a segment releases unused message-metadata and record- + location buffer capacity before returning its cache budget, so cold Topics + do not retain buffers from their former active tails. Consumer scans retain + one rewind position and wakeup time per Channel for future DPUB messages, + instead of copying every skipped deadline into Channel requeue state. + Recovery compacts legacy DPUB scan entries against bounded metadata pages, + preserving explicit REQ targets and lost-position fences. Explicit client + REQ deadlines remain durable. Publish admission is + governed by the configured PVC watermarks, not by an arbitrary number of messages. - GC locates retention boundaries by monotonic indexes instead of rescanning a full backlog every five seconds. A normal tick rotates across at most 128 @@ -152,9 +166,17 @@ operator -> eligible nodes -> StatefulSet ordinal + retained RWO PVC reclaimable. Scrub snapshots immutable files under the Topic lock, then verifies them lock-free with a default 64 MiB/s I/O limit. - Slow consumers share a 512 MiB node-wide delivery working-set budget; each - connection can request at most 32 MiB of payload. Cache misses charge both the - file-read buffer and delivered body before I/O, and cancelled reads return - their memory permits and in-flight reservations. + connection can request at most 32 MiB of payload. A separate node-wide + in-flight message budget defaults to 262,144 slots + (`limits.node_delivery_inflight_messages`) and can be overridden with + `RUSTQUEUE_NODE_DELIVERY_INFLIGHT_MESSAGES`. Each reserved message consumes + one slot, including an empty message. A slot returns only after both the + queue reservation and consumer-session unacknowledged metadata are released; + FIN/REQ completion, timeout, cancellation, disconnect, and TTL cleanup + release their respective holds. At the limit, later reservations + pause until a slot is released. Cache misses charge both the file-read buffer + and delivered body before I/O, and cancelled reads return their memory + permits and in-flight reservations. ## Components @@ -337,6 +359,10 @@ This covers official Go and Python clients across direct publishing and consumption, lookup, MPUB, DPUB, REQ, TOUCH, RDY, fan-out, sampling, ephemeral channels, Snappy, Deflate, TLS, mTLS and external AUTH. +TCP command processing and response writes have a bounded progress timeout. +Their elapsed time does not consume the client's heartbeat idle window, which +restarts when the command completes. Idle clients and blocked writes still time out. + ## Kubernetes ### Standard deployment @@ -354,7 +380,7 @@ kubectl label node worker-1 rustqueue.io/eligible=true helm upgrade --install rustqueue deploy/helm/rustqueue \ --namespace rustqueue --create-namespace \ - --set queue.image=registry.example/rustqueue:0.10.0 \ + --set queue.image=registry.example/rustqueue:0.10.1 \ --set queue.storageClassName=ssd-rwo ``` @@ -389,6 +415,9 @@ required hostname anti-affinity. The profile therefore needs three schedulable nodes. Each Gateway requests one CPU and 768 MiB, is not CPU-limited, and is limited to 1 GiB memory so its fixed 512 MiB in-flight body budget is scheduler-backed. +Custom Kodo Gateway configurations must reserve at least 136 MiB of ingress +budget: a 128 MiB MPUB body plus 8 MiB of bounded per-message parsing metadata. +The standard profile's 512 MiB budget already includes this working set. The reviewed Kodo source enforces an application maximum of 104857500 bytes (100 MiB minus 100 bytes) and sets the go-nsq producer write deadline to three seconds. A maximum-size publish therefore needs more than roughly 34 MiB/s of @@ -548,6 +577,10 @@ the exact target and stage; the last 20 replaced operations remain in Examples using a locally built `rustqueuectl`: +Kubernetes connection failures return an error and a nonzero exit status. +`stats` and `scrub` also contact Broker Pod IPs directly, so run them where the +Pod network is reachable. + ```sh rustqueuectl --namespace rustqueue status rustqueuectl --namespace rustqueue brokers @@ -591,7 +624,25 @@ test-only direct Pod placement; production anti-affinity is unchanged. A unit fixture covers discovery indexing for 500 brokers. No 500-broker deployment or load test is part of the functional gate. -The v0.10.0 CI/CD workflow publishes a Release only after the non-Kubernetes +For a long-running message ledger, run the Go client in `tests/soak` with the +Discovery HTTP endpoint and direct Broker TCP endpoints. It creates the new +Topic's subscriptions directly before waiting for Discovery to index them, so +startup does not depend on a producer publishing first. For example, with +three locally exposed Brokers: + +```sh +cd tests/soak +go run . -lookup localhost:4161 \ + -tcp localhost:4150,localhost:5150,localhost:6150 -duration 24h -rate 10 +``` + +The client retains message IDs until the final report, so its memory grows with +the duration and rate. A low rate bounds that client cost and validates +long-term integrity; it does not qualify saturation throughput. Inspect the +report's duplicates, publish errors and unconfirmed deliveries as well as its +exit status, missing messages and final Broker backlog. + +The v0.10.1 CI/CD workflow publishes a Release only after the non-Kubernetes release gate, both native Linux builds, packaging and checksum verification succeed. The v0.8.0 Kodo compatibility baseline additionally passed the unmodified Kodo source replay, an exact 104,857,500-byte `PUB`/`DPUB` with one @@ -678,8 +729,10 @@ queueing, publish and channel ACK, payload reads, scrub/GC, proxy backend calls, and discovery registry polling. Queue aggregates have fixed cardinality by default; `[metrics].detailed_queue_metrics` enables bounded per-topic/channel series up to `max_detailed_series`. -Delivery-budget bytes, waiters and cumulative waits are exported as bounded -aggregate gauges/counters. +Delivery-budget bytes, in-flight message slots, waiters and cumulative waits are +exported as bounded aggregate gauges/counters. Native observation includes +`delivery_budget.in_flight_messages`, and Prometheus exports the slot count as +`rustqueue_delivery_inflight_messages`. `rustqueue_ttl_discarded_messages_total` is label-free. When detailed queue metrics are enabled, `rustqueue_topic_ttl_discarded_messages_total` uses the same global series budget as the other per-Topic metrics. @@ -736,11 +789,19 @@ The on-disk layout is: ``` Tail short writes are truncated during startup. Sealed segments reopen from an -atomic recovery index by reading its CRC-protected fixed header and first/last -queue entries; individual metadata entries carry their own CRC and are paged on -demand. Full sidecar checksums run in background scrub, so startup is O(segment -count), not O(message count). Missing or invalid indexes fall back to a full -segment scan. +atomic recovery index. Storage record locations use a CRC-protected fixed header; +queue metadata is validated in bounded pages to recover message ranges and the +minimum/maximum delivery deadline for each segment. This scans sealed queue +metadata at startup, but does not retain per-message delayed indexes or read +sealed payloads. Individual metadata entries carry their own CRC and delivery +loads pages on demand. Full sidecar checksums run in background scrub. Missing or +invalid indexes fall back to a full segment scan. +Delayed backlog statistics use these segment deadline summaries and read bounded +metadata pages for mixed deadlines. They do not copy a backlog-sized position +set. The Rust `Broker::stats`, `filtered_stats`, and `metrics_stats` APIs return +`Result`; metadata read or integrity failures return an error and isolate storage +rather than report partial counters. HTTP statistics and metrics return an error +response on such failures; successful response schemas are unchanged. Cold payload corruption is isolated by payload-read CRC or background scrub. Automatic scrub and normal GC start after a configurable 30-second quiet period so a large PVC can become Ready without immediately competing with maintenance diff --git a/console-ui/package.json b/console-ui/package.json index 04922f5..da4666a 100644 --- a/console-ui/package.json +++ b/console-ui/package.json @@ -1,7 +1,7 @@ { "name": "rustqueue-console-ui", "private": true, - "version": "0.10.0", + "version": "0.10.1", "packageManager": "pnpm@11.9.0", "type": "module", "scripts": { diff --git a/console-ui/src/api/useSnapshot.test.tsx b/console-ui/src/api/useSnapshot.test.tsx new file mode 100644 index 0000000..e6b196c --- /dev/null +++ b/console-ui/src/api/useSnapshot.test.tsx @@ -0,0 +1,149 @@ +// @vitest-environment jsdom + +import { StrictMode } from 'react'; +import { act, cleanup, render, screen } from '@testing-library/react'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import type { Snapshot } from './types'; +import { useSnapshot } from './useSnapshot'; + +function deferred() { + let resolve!: (value: T) => void; + const promise = new Promise((resolvePromise) => { + resolve = resolvePromise; + }); + return { promise, resolve }; +} + +function response(snapshot: Snapshot): Response { + return { + ok: true, + status: 200, + json: vi.fn().mockResolvedValue(snapshot), + } as unknown as Response; +} + +function errorResponse(message: string): Response { + return { + ok: false, + status: 503, + json: vi.fn().mockResolvedValue({ message }), + } as unknown as Response; +} + +function snapshot(schemaVersion: number): Snapshot { + return { schema_version: schemaVersion } as Snapshot; +} + +function Probe() { + const { snapshot: value, error, loading } = useSnapshot(); + return ( +
+ {value?.schema_version ?? ''} + {loading ? 'loading' : 'ready'} + {error ?? ''} +
+ ); +} + +describe('useSnapshot polling', () => { + beforeEach(() => { + vi.useFakeTimers(); + }); + + afterEach(() => { + cleanup(); + vi.useRealTimers(); + vi.unstubAllGlobals(); + }); + + it('keeps a slow request alive and refreshes after it completes', async () => { + const requests: Array<{ + signal: AbortSignal; + deferred: ReturnType>; + }> = []; + const fetchMock = vi.fn((_input: RequestInfo | URL, init?: RequestInit) => { + const request = { signal: init?.signal as AbortSignal, deferred: deferred() }; + requests.push(request); + return request.deferred.promise; + }); + vi.stubGlobal('fetch', fetchMock); + + render(); + expect(fetchMock).toHaveBeenCalledTimes(1); + + await act(async () => { + await vi.advanceTimersByTimeAsync(6000); + }); + expect(fetchMock).toHaveBeenCalledTimes(1); + expect(requests[0].signal.aborted).toBe(false); + + await act(async () => { + requests[0].deferred.resolve(response(snapshot(1))); + await Promise.resolve(); + await Promise.resolve(); + }); + expect(screen.getByTestId('schema').textContent).toBe('1'); + expect(screen.getByTestId('state').textContent).toBe('ready'); + + await act(async () => { + await vi.advanceTimersByTimeAsync(2000); + }); + expect(fetchMock).toHaveBeenCalledTimes(2); + + await act(async () => { + requests[1].deferred.resolve(response(snapshot(2))); + await Promise.resolve(); + await Promise.resolve(); + }); + expect(screen.getByTestId('schema').textContent).toBe('2'); + }); + + it.each(['success', 'failure'] as const)( + 'does not let a StrictMode-aborted %s clear the replacement request', + async (lateResult) => { + const requests: Array<{ + signal: AbortSignal; + deferred: ReturnType>; + }> = []; + const fetchMock = vi.fn((_input: RequestInfo | URL, init?: RequestInit) => { + const request = { signal: init?.signal as AbortSignal, deferred: deferred() }; + requests.push(request); + return request.deferred.promise; + }); + vi.stubGlobal('fetch', fetchMock); + + render( + + + , + ); + expect(fetchMock).toHaveBeenCalledTimes(2); + expect(requests[0].signal.aborted).toBe(true); + expect(requests[1].signal.aborted).toBe(false); + + await act(async () => { + requests[0].deferred.resolve( + lateResult === 'success' + ? response(snapshot(1)) + : errorResponse('stale backend failure'), + ); + await Promise.resolve(); + await Promise.resolve(); + }); + expect(screen.getByTestId('schema').textContent).toBe(''); + expect(screen.getByTestId('error').textContent).toBe(''); + await act(async () => { + await vi.advanceTimersByTimeAsync(2000); + }); + expect(fetchMock).toHaveBeenCalledTimes(2); + + await act(async () => { + requests[1].deferred.resolve(response(snapshot(2))); + await Promise.resolve(); + await Promise.resolve(); + }); + expect(screen.getByTestId('schema').textContent).toBe('2'); + expect(screen.getByTestId('error').textContent).toBe(''); + }, + ); +}); diff --git a/console-ui/src/api/useSnapshot.ts b/console-ui/src/api/useSnapshot.ts index f29bef9..927d409 100644 --- a/console-ui/src/api/useSnapshot.ts +++ b/console-ui/src/api/useSnapshot.ts @@ -8,7 +8,7 @@ export function useSnapshot() { const request = useRef(undefined); const refresh = useCallback(async () => { - request.current?.abort(); + if (request.current && !request.current.signal.aborted) return; const controller = new AbortController(); request.current = controller; try { @@ -20,11 +20,16 @@ export function useSnapshot() { const body = await response.json().catch(() => ({})); throw new Error(body.message || `HTTP ${response.status}`); } - setSnapshot((await response.json()) as Snapshot); + const next = (await response.json()) as Snapshot; + if (controller.signal.aborted) return; + setSnapshot(next); setError(undefined); } catch (reason) { - if ((reason as Error).name !== 'AbortError') setError((reason as Error).message); + if (!controller.signal.aborted && (reason as Error).name !== 'AbortError') { + setError((reason as Error).message); + } } finally { + if (request.current === controller) request.current = undefined; if (!controller.signal.aborted) setLoading(false); } }, []); diff --git a/console-ui/src/i18n.tsx b/console-ui/src/i18n.tsx index e2a0c9e..7f22d01 100644 --- a/console-ui/src/i18n.tsx +++ b/console-ui/src/i18n.tsx @@ -16,6 +16,8 @@ const messages: Record> = { 'action.language': '切换语言', 'action.theme': '切换主题', 'action.skipToContent': '跳到主要内容', + 'action.openNavigation': '打开导航', + 'action.closeNavigation': '关闭导航', 'action.copy': '复制到剪贴板', 'action.copied': '已复制', 'action.showMore': '展开', @@ -243,6 +245,8 @@ const messages: Record> = { 'action.language': 'Change language', 'action.theme': 'Change theme', 'action.skipToContent': 'Skip to main content', + 'action.openNavigation': 'Open navigation', + 'action.closeNavigation': 'Close navigation', 'action.copy': 'Copy to clipboard', 'action.copied': 'Copied', 'action.showMore': 'Show more', diff --git a/console-ui/src/layout/AppShell.test.tsx b/console-ui/src/layout/AppShell.test.tsx new file mode 100644 index 0000000..e58ca72 --- /dev/null +++ b/console-ui/src/layout/AppShell.test.tsx @@ -0,0 +1,64 @@ +// @vitest-environment jsdom + +import { cleanup, fireEvent, render, screen } from '@testing-library/react'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { I18nProvider } from '../i18n'; +import { AppShell } from './AppShell'; + +describe('compact navigation', () => { + beforeEach(() => { + localStorage.setItem('rustqueue-language', 'en'); + vi.stubGlobal('matchMedia', vi.fn((media: string) => ({ + matches: false, + media, + addEventListener: vi.fn(), + removeEventListener: vi.fn(), + }))); + }); + + afterEach(() => { + cleanup(); + localStorage.clear(); + vi.unstubAllGlobals(); + }); + + function renderShell() { + const onPage = vi.fn(); + const view = render( + + +

Cluster overview

+ + , + ); + return { ...view, onPage }; + } + + it('opens navigation and closes it after selecting a page', () => { + const { onPage } = renderShell(); + const menu = screen.getByRole('button', { name: 'Open navigation' }); + expect(menu.getAttribute('aria-expanded')).toBe('false'); + fireEvent.click(menu); + expect(screen.getByRole('button', { name: 'Close navigation' }).getAttribute('aria-expanded')).toBe('true'); + + fireEvent.click(screen.getByRole('link', { name: 'Configuration' })); + expect(onPage).toHaveBeenCalledWith('configuration'); + expect(screen.getByRole('button', { name: 'Open navigation' }).getAttribute('aria-expanded')).toBe('false'); + }); + + it.each(['menu', 'overlay', 'Escape'] as const)('dismisses navigation with %s', (action) => { + const { container } = renderShell(); + fireEvent.click(screen.getByRole('button', { name: 'Open navigation' })); + if (action === 'menu') { + const menu = screen.getByRole('button', { name: 'Close navigation' }); + fireEvent.blur(screen.getByRole('navigation'), { relatedTarget: menu }); + fireEvent.click(menu); + } else if (action === 'overlay') { + fireEvent.click(container.querySelector('.cds--side-nav__overlay')!); + } else { + fireEvent.keyDown(screen.getByRole('navigation'), { key: 'Escape' }); + expect(document.activeElement).toBe(screen.getByRole('button', { name: 'Open navigation' })); + } + expect(screen.getByRole('button', { name: 'Open navigation' }).getAttribute('aria-expanded')).toBe('false'); + }); +}); diff --git a/console-ui/src/layout/AppShell.tsx b/console-ui/src/layout/AppShell.tsx index 41dfa02..d6fd0d3 100644 --- a/console-ui/src/layout/AppShell.tsx +++ b/console-ui/src/layout/AppShell.tsx @@ -2,13 +2,15 @@ import { Header, HeaderGlobalAction, HeaderGlobalBar, + HeaderMenuButton, SideNav, SideNavItems, SideNavLink, SkipToContent, } from '@carbon/react'; import { Dashboard, DataBase, Language, Moon, Renew, ServerProxy, Settings, StoragePool, Task, Sun } from '@carbon/icons-react'; -import type { ReactNode } from 'react'; +import { useState, type ReactNode } from 'react'; +import { version } from '../../package.json'; import { useI18n } from '../i18n'; export type Page = 'overview' | 'brokers' | 'topics' | 'storage' | 'operations' | 'configuration'; @@ -31,10 +33,19 @@ export function AppShell({ page, onPage, dark, onTheme, onRefresh, children }: { children: ReactNode; }) { const { language, setLanguage, t } = useI18n(); + const [navigationOpen, setNavigationOpen] = useState(false); return (
{t('action.skipToContent')} + setNavigationOpen((open) => !open)} + /> RustQueue @@ -47,17 +58,31 @@ export function AppShell({ page, onPage, dark, onTheme, onRefresh, children }: { {dark ? : }
- + { + if (typeof event !== 'boolean') { + // Preserve the open state until the menu button receives its click. + if ('relatedTarget' in event && event.relatedTarget instanceof HTMLElement && event.relatedTarget.id === 'navigation-toggle') return; + if ('key' in event && event.key === 'Escape') document.getElementById('navigation-toggle')?.focus(); + } + setNavigationOpen(false); + }} + onOverlayClick={() => setNavigationOpen(false)} + > {items.map((item) => { event.preventDefault(); onPage(item.id); }} + onClick={(event) => { event.preventDefault(); onPage(item.id); setNavigationOpen(false); }} >{t(`nav.${item.id}`)})} -
{t('app.readOnly')}v0.10.0
+
{t('app.readOnly')}v{version}
{children}
diff --git a/console-ui/src/styles.scss b/console-ui/src/styles.scss index c037740..8192a9f 100644 --- a/console-ui/src/styles.scss +++ b/console-ui/src/styles.scss @@ -21,7 +21,7 @@ button, input { font: inherit; } .header-brand__product { color: var(--cds-text-secondary); } .header-mode { color: var(--cds-text-secondary); font-size: .75rem; letter-spacing: .02em; padding-left: 1rem; } .main-content { margin-left: 16rem; padding: 4.5rem 2rem 3rem; min-height: 100vh; max-width: 1920px; } -.cds--side-nav { width: 16rem; top: 3rem; height: calc(100vh - 3rem); border-right: 1px solid var(--cds-border-subtle); background: var(--cds-layer-01); } +.cds--side-nav { top: 3rem; height: calc(100vh - 3rem); border-right: 1px solid var(--cds-border-subtle); background: var(--cds-layer-01); } .cds--side-nav__items { padding-top: 1rem; } .cds--side-nav__link { margin: .125rem .75rem; width: calc(100% - 1.5rem); border-radius: 8px; } .cds--side-nav__link--current::before { border-radius: 8px 0 0 8px; } @@ -120,13 +120,20 @@ button, input { font: inherit; } .loading-state__tile { width: 100%; height: 6rem; } .chart-panel .cds--chart-holder { margin: 0 -.5rem -.5rem; } -@media (max-width: 1056px) { - .main-content { margin-left: 16rem; padding-left: 1.25rem; padding-right: 1.25rem; } +@media (width < 66rem) { + .main-content { margin-left: 0; padding-left: 1.25rem; padding-right: 1.25rem; } + .header-mode { display: none; } .channel-card dl { grid-template-columns: repeat(2, 1fr); } .page-header { align-items: flex-start; flex-direction: column; } .page-header__meta { justify-content: flex-start; } } +@media (max-width: 480px) { + .header-brand { padding: 0 .5rem; } + .header-brand__product { display: none; } + .config-table > div { grid-template-columns: minmax(0, 1fr); gap: .5rem; } +} + @media (prefers-reduced-motion: reduce) { *, *::before, *::after { scroll-behavior: auto !important; animation-duration: .01ms !important; animation-iteration-count: 1 !important; transition-duration: .01ms !important; } } diff --git a/crates/console/src/collector/observe.rs b/crates/console/src/collector/observe.rs index 570ab4f..3799a3a 100644 --- a/crates/console/src/collector/observe.rs +++ b/crates/console/src/collector/observe.rs @@ -154,7 +154,7 @@ async fn fetch_observation( fences: &rustqueue_queue::ManagementFenceSnapshot, fence_revision: &str, ) -> Result { - let origin = format!("http://{}:{port}", broker.pod_ip); + let origin = broker.http_origin(port); let head_response = http .get(format!("{origin}/v1/observe/head")) .bearer_auth(token) @@ -251,6 +251,52 @@ fn requires_full( #[cfg(test)] mod tests { use super::*; + use axum::extract::State; + use axum::http::StatusCode; + use axum::routing::{get, post}; + use axum::{Json, Router}; + use serde_json::{json, Value}; + use tokio::sync::Mutex as AsyncMutex; + + struct FakeBroker { + requests: AsyncMutex>, + synced_fences: AsyncMutex>, + head: BrokerObservationHead, + observation: BrokerObservation, + } + + fn head_json(head: &BrokerObservationHead) -> Value { + json!({ + "schema_version": head.schema_version, + "collected_at_ms": head.collected_at_ms, + "registry_revision": head.registry_revision, + "node": &head.node, + "readiness": &head.readiness, + "disk": &head.disk, + "runtime": &head.runtime, + "delivery_budget": &head.delivery_budget, + "limits": &head.limits, + }) + } + + async fn fake_head(State(state): State>) -> Json { + state.requests.lock().await.push("head"); + Json(head_json(&state.head)) + } + + async fn fake_observation(State(state): State>) -> Json { + state.requests.lock().await.push("full"); + Json(state.observation.clone()) + } + + async fn fake_fence_sync( + State(state): State>, + Json(fences): Json, + ) -> StatusCode { + state.requests.lock().await.push("fence_sync"); + state.synced_fences.lock().await.push(fences); + StatusCode::NO_CONTENT + } fn fixture() -> (BrokerView, BrokerObservationHead, CachedObservation) { let broker = BrokerView { @@ -322,4 +368,92 @@ mod tests { assert!(requires_fence_sync(Some(&cached), &broker, "fence-2")); assert!(requires_fence_sync(None, &broker, "fence-1")); } + + #[tokio::test] + async fn ipv6_fetch_observation_runs_head_full_and_fence_sync_requests() { + let mut head = BrokerObservationHead { + schema_version: 1, + collected_at_ms: 11, + registry_revision: 4, + ..Default::default() + }; + head.node.version = "0.8.1".into(); + let mut observation = BrokerObservation { + schema_version: 1, + collected_at_ms: 22, + registry_revision: 4, + ..Default::default() + }; + observation.node.version = "0.8.1".into(); + let state = Arc::new(FakeBroker { + requests: AsyncMutex::new(Vec::new()), + synced_fences: AsyncMutex::new(Vec::new()), + head, + observation, + }); + let app = Router::new() + .route("/v1/observe/head", get(fake_head)) + .route("/v1/observe", get(fake_observation)) + .route("/v1/manage/fences/sync", post(fake_fence_sync)) + .with_state(Arc::clone(&state)); + let listener = tokio::net::TcpListener::bind("[::1]:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + let server = tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }); + + let broker = BrokerView { + uid: "pod-1".into(), + name: "queue-0".into(), + pod_ip: "::1".into(), + ..Default::default() + }; + let fences = rustqueue_queue::ManagementFenceSnapshot { + revision: "fence-1".into(), + ..Default::default() + }; + let http = reqwest::Client::builder() + .no_proxy() + .timeout(std::time::Duration::from_secs(2)) + .build() + .unwrap(); + let cached = fetch_observation( + &broker, + None, + &http, + "console-token", + port, + true, + std::time::Duration::from_secs(30), + &fences, + "fence-1", + ) + .await + .unwrap(); + assert_eq!(cached.observation.collected_at_ms, 22); + + let merged = fetch_observation( + &broker, + Some(cached), + &http, + "console-token", + port, + true, + std::time::Duration::from_secs(30), + &fences, + "fence-1", + ) + .await + .unwrap(); + assert_eq!(merged.observation.collected_at_ms, 11); + + assert_eq!( + state.requests.lock().await.as_slice(), + ["head", "fence_sync", "full", "head"] + ); + assert_eq!(state.synced_fences.lock().await.as_slice(), [fences]); + + server.abort(); + let _ = server.await; + } } diff --git a/crates/console/src/management/backend.rs b/crates/console/src/management/backend.rs index 0352bf5..a23ae66 100644 --- a/crates/console/src/management/backend.rs +++ b/crates/console/src/management/backend.rs @@ -137,8 +137,8 @@ async fn post( let response = state .http .post(format!( - "http://{}:{}{}", - broker.pod_ip, state.config.broker_http_port, path + "{}{path}", + broker.http_origin(state.config.broker_http_port) )) .bearer_auth(token) .timeout(Duration::from_secs(15)) diff --git a/crates/console/src/management/resource_state.rs b/crates/console/src/management/resource_state.rs index 349c2bf..92cd48f 100644 --- a/crates/console/src/management/resource_state.rs +++ b/crates/console/src/management/resource_state.rs @@ -139,7 +139,7 @@ pub async fn begin_topic( }; ensure_topic_children_idle(state, &request.topic).await?; resource.spec.owners = challenge.owners.clone(); - if request.action == "configure" { + if matches!(request.action.as_str(), "create" | "configure") { resource.spec.delivery_mode = request_topic_mode(request)?; resource.spec.message_ttl_seconds = request.message_ttl_seconds; } diff --git a/crates/console/src/model.rs b/crates/console/src/model.rs index e170bfd..d2b4f5a 100644 --- a/crates/console/src/model.rs +++ b/crates/console/src/model.rs @@ -197,6 +197,16 @@ pub struct BrokerView { pub error: Option, } +impl BrokerView { + pub fn http_origin(&self, port: u16) -> String { + if self.pod_ip.contains(':') && !self.pod_ip.starts_with('[') { + format!("http://[{}]:{port}", self.pod_ip) + } else { + format!("http://{}:{port}", self.pod_ip) + } + } +} + #[derive(Clone, Debug, Default, Serialize)] pub struct PvcView { pub name: String, diff --git a/crates/operator/src/controller/mod.rs b/crates/operator/src/controller/mod.rs index 0aa4fd5..c688b2a 100644 --- a/crates/operator/src/controller/mod.rs +++ b/crates/operator/src/controller/mod.rs @@ -8,6 +8,9 @@ mod operations; mod preflight; mod status; mod storage; +mod validation; + +use validation::{validate, with_storage_feature_floor}; use crate::resources::{self, BuildInput}; use crate::RustQueue; @@ -1253,50 +1256,6 @@ fn deployment_runs_without_kodo(deployment: Option<&Deployment>) -> bool { && deployment_is_fully_ready(deployment) } -fn with_storage_feature_floor( - cluster: Arc, - active_feature_floor: u32, -) -> Arc { - let effective = - effective_storage_feature_level(cluster.spec.storage_feature_level, active_feature_floor); - let (connection_delivery, node_delivery) = effective_delivery_limits( - effective, - cluster.spec.max_message_bytes, - cluster.spec.connection_delivery_inflight_bytes, - cluster.spec.node_delivery_inflight_bytes, - ); - if effective == cluster.spec.storage_feature_level - && connection_delivery == cluster.spec.connection_delivery_inflight_bytes - && node_delivery == cluster.spec.node_delivery_inflight_bytes - { - return cluster; - } - let mut adjusted = cluster.as_ref().clone(); - adjusted.spec.storage_feature_level = effective; - adjusted.spec.connection_delivery_inflight_bytes = connection_delivery; - adjusted.spec.node_delivery_inflight_bytes = node_delivery; - Arc::new(adjusted) -} - -fn effective_storage_feature_level(requested: u32, active: u32) -> u32 { - requested.max(active).max(1) -} - -fn effective_delivery_limits( - storage_feature_level: u32, - max_message_bytes: usize, - connection: usize, - node: usize, -) -> (usize, usize) { - let retained_message_bound = if storage_feature_level >= 2 { - 100 * 1024 * 1024 - } else { - max_message_bytes - }; - let connection = connection.max(retained_message_bound); - (connection, node.max(connection.saturating_mul(2))) -} - async fn record_reconcile_error(context: &ContextData, cluster: &RustQueue, error: &anyhow::Error) { if let Some(namespace) = cluster.namespace() { match auth::ensure(&context.client, cluster, &namespace).await { @@ -1332,251 +1291,12 @@ async fn record_reconcile_error(context: &ContextData, cluster: &RustQueue, erro } } -fn validate(cluster: &RustQueue, active_feature_floor: u32) -> anyhow::Result<()> { - if cluster.spec.image.trim().is_empty() { - bail!("spec.image is required"); - } - if cluster.spec.min_brokers < 1 || cluster.spec.max_brokers < cluster.spec.min_brokers { - bail!("broker limits must satisfy 1 <= minBrokers <= maxBrokers"); - } - if cluster.spec.storage_class_name.trim().is_empty() - || cluster.spec.storage_size.trim().is_empty() - { - bail!("storageClassName and storageSize are required"); - } - if cluster.spec.storage_feature_level == 0 { - bail!("storageFeatureLevel must be greater than zero"); - } - if cluster.spec.disk_low_watermark_percent >= cluster.spec.disk_high_watermark_percent - || cluster.spec.disk_high_watermark_percent > 100 - { - bail!("disk watermarks must satisfy low < high <= 100"); - } - if cluster.spec.bootstrap_retention_seconds == 0 - || cluster.spec.max_message_bytes == 0 - || cluster.spec.max_message_bytes > 100 * 1024 * 1024 - || cluster.spec.message_index_cache_bytes == 0 - || cluster.spec.connection_delivery_inflight_bytes < cluster.spec.max_message_bytes - || cluster - .spec - .connection_delivery_inflight_bytes - .checked_mul(2) - .is_none_or(|minimum| cluster.spec.node_delivery_inflight_bytes < minimum) - || cluster.spec.node_delivery_inflight_bytes > u32::MAX as usize - || cluster.spec.max_topics == 0 - || cluster.spec.max_publish_workers == 0 - || cluster.spec.publish_worker_idle_seconds == 0 - || (cluster.spec.publish_ack_mode != "durable" - && (cluster.spec.relaxed_sync_messages == 0 - || cluster.spec.relaxed_sync_bytes < 4096 - || cluster.spec.relaxed_sync_interval_ms == 0)) - || cluster.spec.max_detailed_metric_series == 0 - { - bail!("queue limits are outside the stable v7 contract"); - } - if !matches!( - cluster.spec.publish_ack_mode.as_str(), - "durable" | "write_ack" | "nsq_relaxed" - ) { - bail!("publishAckMode must be durable, write_ack, or nsq_relaxed"); - } - if cluster.spec.websocket.max_connections == 0 - || cluster.spec.websocket.max_connections > tokio::sync::Semaphore::MAX_PERMITS - || cluster.spec.websocket.max_connections_per_topic == 0 - || cluster.spec.websocket.max_connections_per_topic > cluster.spec.websocket.max_connections - || cluster.spec.websocket.frame_inflight_bytes - < cluster.spec.max_message_bytes.saturating_add(24) - || cluster.spec.websocket.frame_inflight_bytes > u32::MAX as usize - { - bail!("WebSocket limits must fit one maximum message and satisfy per-Topic <= global connections"); - } - if cluster.spec.websocket.allowed_origins.iter().any(|origin| { - let authority = origin - .strip_prefix("http://") - .or_else(|| origin.strip_prefix("https://")); - origin == "*" - || authority.is_none_or(|authority| { - authority.is_empty() - || authority.contains('/') - || authority.contains('?') - || authority.contains('#') - || authority.contains(',') - || authority.trim() != authority - }) - }) { - bail!("WebSocket allowedOrigins must be exact HTTP origins without paths or wildcards"); - } - validate_message_storage_contract( - cluster.spec.max_message_bytes, - effective_storage_feature_level(cluster.spec.storage_feature_level, active_feature_floor), - )?; - if cluster.spec.kodo_compatibility.enabled - && (cluster.spec.min_brokers != 3 - || cluster.spec.max_brokers != 3 - || cluster.spec.storage_feature_level != 2 - || cluster.spec.bootstrap_retention_seconds < KODO_BOOTSTRAP_RETENTION_SECONDS - || cluster.spec.max_message_bytes != 100 * 1024 * 1024 - || cluster.spec.connection_delivery_inflight_bytes < 128 * 1024 * 1024 - || cluster.spec.node_delivery_inflight_bytes < 512 * 1024 * 1024 - || cluster.spec.publish_ack_mode != "durable") - { - bail!( - "Kodo compatibility requires exactly 3 brokers, storageFeatureLevel 2, \ - bootstrapRetentionSeconds >= 180, \ - maxMessageBytes 104857600, connectionDeliveryInflightBytes >= 134217728, \ - nodeDeliveryInflightBytes >= 536870912, and publishAckMode durable" - ); - } - if !(630..=86_400).contains(&cluster.spec.kodo_compatibility.cutover_grace_seconds) { - bail!("Kodo compatibility cutoverGraceSeconds must be between 630 and 86400"); - } - if cluster.spec.kodo_compatibility.enabled { - if cluster.spec.kodo_compatibility.decommission_confirmed { - bail!("decommissionConfirmed must be false while Kodo compatibility is enabled"); - } - if cluster.spec.kodo_compatibility.cleanup_enabled { - bail!( - "Kodo automatic cleanup is disabled until cluster-wide atomic deletion is available" - ); - } - let target_image = cluster - .spec - .rollout - .rollback_to_image - .as_deref() - .unwrap_or(&cluster.spec.image); - if cluster.spec.image_pull_policy != "Never" && !has_sha256_digest(target_image) { - bail!( - "Kodo compatibility requires an immutable @sha256 image or imagePullPolicy Never" - ); - } - if cluster - .spec - .kodo_compatibility - .allowed_pod_selector - .is_empty() - || cluster - .spec - .kodo_compatibility - .allowed_pod_selector - .iter() - .any(|(key, value)| key.trim().is_empty() || value.trim().is_empty()) - || cluster - .spec - .kodo_compatibility - .allowed_namespace_selector - .iter() - .any(|(key, value)| key.trim().is_empty() || value.trim().is_empty()) - { - bail!("Kodo compatibility requires a non-empty allowedPodSelector and valid selector labels"); - } - let memory_request = storage::parse_quantity(&cluster.spec.broker_resources.memory_request) - .context("parse broker memory request")?; - if memory_request < 2_u128 << 30 { - bail!("Kodo compatibility requires brokerResources.memoryRequest >= 2Gi"); - } - let cpu_request = parse_cpu_millis(&cluster.spec.broker_resources.cpu_request) - .context("parse broker CPU request")?; - if cpu_request < 1_000.0 { - bail!("Kodo compatibility requires brokerResources.cpuRequest >= 1 CPU"); - } - if let Some(cpu_limit) = cluster.spec.broker_resources.cpu_limit.as_deref() { - let cpu_limit = parse_cpu_millis(cpu_limit).context("parse broker CPU limit")?; - if cpu_limit < cpu_request { - bail!("brokerResources.cpuLimit must be greater than or equal to cpuRequest"); - } - } - if let Some(memory_limit) = cluster.spec.broker_resources.memory_limit.as_deref() { - let memory_limit = - storage::parse_quantity(memory_limit).context("parse broker memory limit")?; - if memory_limit < memory_request { - bail!("brokerResources.memoryLimit must be greater than or equal to memoryRequest"); - } - } - } - if cluster.spec.rollout.timeout_seconds == 0 || cluster.spec.rollout.timeout_seconds > 86_400 { - bail!("rollout timeoutSeconds must be between 1 and 86400"); - } - if cluster - .spec - .rollout - .rollback_to_image - .as_ref() - .is_some_and(|image| image.trim().is_empty()) - { - bail!("rollout rollbackToImage cannot be empty"); - } - if cluster - .spec - .broker_scheduling - .topology_key - .trim() - .is_empty() - || cluster.spec.broker_resources.cpu_request.trim().is_empty() - || cluster - .spec - .broker_resources - .memory_request - .trim() - .is_empty() - { - bail!("broker scheduling and resource requests cannot be empty"); - } - Ok(()) -} - fn ordinary_desired_brokers(eligible: usize, minimum: i32, maximum: i32) -> i32 { i32::try_from(eligible) .unwrap_or(i32::MAX) .clamp(minimum, maximum) } -fn parse_cpu_millis(value: &str) -> anyhow::Result { - let value = value.trim(); - let (number, multiplier) = if let Some(number) = value.strip_suffix('m') { - (number, 1.0) - } else if let Some(number) = value.strip_suffix('u') { - (number, 0.001) - } else if let Some(number) = value.strip_suffix('n') { - (number, 0.000_001) - } else { - (value, 1_000.0) - }; - let number: f64 = number.parse()?; - anyhow::ensure!( - number.is_finite() && number >= 0.0, - "invalid CPU quantity {value}" - ); - Ok(number * multiplier) -} - -fn validate_message_storage_contract( - max_message_bytes: usize, - storage_feature_level: u32, -) -> anyhow::Result<()> { - const LEGACY_MAX_RECORD_BYTES: usize = 72 * 1024 * 1024; - const SINGLE_MESSAGE_ENVELOPE_BYTES: usize = 24; - const MPUB_ENTRY_BYTES: usize = 16; - const MAX_MPUB_MESSAGES: usize = 65_536; - let max_body_bytes = (64 * 1024 * 1024).max(max_message_bytes); - let maximum_record = max_message_bytes - .saturating_add(SINGLE_MESSAGE_ENVELOPE_BYTES) - .max(max_body_bytes.saturating_add(MPUB_ENTRY_BYTES.saturating_mul(MAX_MPUB_MESSAGES))); - if maximum_record > LEGACY_MAX_RECORD_BYTES && storage_feature_level < 2 { - bail!("messages above the v7 legacy record bound require storageFeatureLevel 2"); - } - Ok(()) -} - -fn has_sha256_digest(image: &str) -> bool { - image.rsplit_once("@sha256:").is_some_and(|(_, digest)| { - digest.len() == 64 - && digest - .bytes() - .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) - }) -} - fn error_policy( cluster: Arc, error: &ReconcileError, @@ -1987,12 +1707,6 @@ mod tests { assert!(!condition_true_for(&cluster, "KodoGatewaysAdvertised", 630)); } - #[test] - fn active_storage_feature_level_is_a_monotonic_floor() { - assert_eq!(effective_storage_feature_level(1, 2), 2); - assert_eq!(effective_storage_feature_level(2, 1), 2); - } - #[test] fn ordinary_scaling_honors_the_configured_floor_and_ceiling() { assert_eq!(ordinary_desired_brokers(0, 3, 10), 3); @@ -2000,85 +1714,4 @@ mod tests { assert_eq!(ordinary_desired_brokers(12, 3, 10), 10); assert_eq!(ordinary_desired_brokers(usize::MAX, 3, 10), 10); } - - #[test] - fn cpu_quantities_compare_in_millicores() { - assert_eq!(parse_cpu_millis("1").unwrap(), 1_000.0); - assert_eq!(parse_cpu_millis("1000m").unwrap(), 1_000.0); - assert_eq!(parse_cpu_millis("500m").unwrap(), 500.0); - assert_eq!(parse_cpu_millis("500000u").unwrap(), 500.0); - assert!(parse_cpu_millis("-1").is_err()); - } - - #[test] - fn feature_two_retains_large_message_delivery_capacity() { - assert_eq!( - effective_delivery_limits(2, 20 * 1024 * 1024, 32 * 1024 * 1024, 64 * 1024 * 1024), - (100 * 1024 * 1024, 200 * 1024 * 1024) - ); - } - - #[test] - fn legacy_feature_rejects_messages_that_cross_the_record_bound() { - let maximum = 71 * 1024 * 1024; - assert!(validate_message_storage_contract(maximum, 1).is_ok()); - assert!(validate_message_storage_contract(maximum + 1, 1).is_err()); - assert!(validate_message_storage_contract(100 * 1024 * 1024, 2).is_ok()); - } - - #[test] - fn immutable_image_detection_requires_a_complete_sha256_digest() { - let digest = "a".repeat(64); - assert!(has_sha256_digest(&format!( - "registry/rustqueue@sha256:{digest}" - ))); - assert!(!has_sha256_digest(&format!( - "registry/rustqueue@sha256:{}", - "A".repeat(64) - ))); - assert!(!has_sha256_digest("registry/rustqueue:latest")); - assert!(!has_sha256_digest("registry/rustqueue@sha256:abcd")); - } - - #[test] - fn kodo_contract_requires_a_second_lookup_poll_retention_window() { - let mut cluster: RustQueue = serde_json::from_value(serde_json::json!({ - "apiVersion": "rustqueue.io/v1alpha1", - "kind": "RustQueue", - "metadata": {"name": "queue", "namespace": "test"}, - "spec": { - "image": "rustqueue:test", - "imagePullPolicy": "Never", - "minBrokers": 3, - "maxBrokers": 3, - "storageFeatureLevel": 2, - "bootstrapRetentionSeconds": 180, - "maxMessageBytes": 104857600, - "connectionDeliveryInflightBytes": 134217728, - "nodeDeliveryInflightBytes": 536870912, - "brokerResources": {"cpuRequest": "1", "memoryRequest": "2Gi"}, - "kodoCompatibility": {"enabled": true} - } - })) - .unwrap(); - - assert!(validate(&cluster, 2).is_ok()); - cluster.spec.broker_resources.cpu_request = "999m".into(); - assert!(validate(&cluster, 2) - .unwrap_err() - .to_string() - .contains("cpuRequest >= 1 CPU")); - cluster.spec.broker_resources.cpu_request = "1".into(); - cluster.spec.bootstrap_retention_seconds = KODO_BOOTSTRAP_RETENTION_SECONDS - 1; - assert!(validate(&cluster, 2) - .unwrap_err() - .to_string() - .contains("bootstrapRetentionSeconds >= 180")); - cluster.spec.bootstrap_retention_seconds = KODO_BOOTSTRAP_RETENTION_SECONDS; - cluster.spec.publish_ack_mode = "nsq_relaxed".into(); - assert!(validate(&cluster, 2) - .unwrap_err() - .to_string() - .contains("publishAckMode durable")); - } } diff --git a/crates/operator/src/controller/status.rs b/crates/operator/src/controller/status.rs index a9a4f8d..5d0fe29 100644 --- a/crates/operator/src/controller/status.rs +++ b/crates/operator/src/controller/status.rs @@ -148,11 +148,14 @@ impl<'a> StatusBuilder<'a> { } fn resume_resets_operation_clock(previous: &str, next: &str) -> bool { - matches!(previous, "Paused" | "AwaitingCanaryApproval" | "Blocked") - && !matches!( - next, - "Paused" | "AwaitingCanaryApproval" | "Blocked" | "Completed" | "Failed" - ) + // Pod recovery can reopen a completed revision long after its rollout ended. + matches!( + previous, + "Paused" | "AwaitingCanaryApproval" | "Blocked" | "Completed" + ) && !matches!( + next, + "Paused" | "AwaitingCanaryApproval" | "Blocked" | "Completed" | "Failed" + ) } pub(super) fn operation_id(kind: &str, target: &str, revision: &str) -> String { @@ -310,46 +313,42 @@ mod tests { } #[test] - fn resuming_a_human_gate_restarts_the_operation_timeout_clock() { - let mut resource = cluster(); - resource.status = Some( - StatusBuilder::new(&resource, 3, 3, 1) - .operation(OperationUpdate { - id: "rollout-1", - kind: "Rollout", - phase: "Paused", - target: "queue:v2", - revision: "r2", - message: "paused", - previous_image: Some("queue:v1".into()), - current_broker: None, - }) - .build(), - ); - resource - .status - .as_mut() - .unwrap() - .current_operation - .as_mut() - .unwrap() - .started_at = "2000-01-01T00:00:00Z".into(); + fn resuming_an_inactive_operation_restarts_the_timeout_clock() { + let update = |phase| OperationUpdate { + id: "rollout-1", + kind: "Rollout", + phase, + target: "queue:v2", + revision: "r2", + message: "broker readiness changed", + previous_image: Some("queue:v1".into()), + current_broker: Some("queue-2".into()), + }; + for phase in ["Paused", "AwaitingCanaryApproval", "Blocked", "Completed"] { + let mut resource = cluster(); + let mut inactive = StatusBuilder::new(&resource, 3, 3, 1) + .operation(update(phase)) + .build(); + inactive.current_operation.as_mut().unwrap().started_at = "2000-01-01T00:00:00Z".into(); + resource.status = Some(inactive); - let resumed = StatusBuilder::new(&resource, 3, 3, 1) - .operation(OperationUpdate { - id: "rollout-1", - kind: "Rollout", - phase: "Draining", - target: "queue:v2", - revision: "r2", - message: "running", - previous_image: Some("queue:v1".into()), - current_broker: Some("queue-2".into()), - }) - .build() - .current_operation - .unwrap(); - assert_ne!(resumed.started_at, "2000-01-01T00:00:00Z"); + let mut resumed = StatusBuilder::new(&resource, 3, 2, 1) + .operation(update("WaitingForReady")) + .build(); + let operation = resumed.current_operation.as_mut().unwrap(); + assert_ne!(operation.started_at, "2000-01-01T00:00:00Z", "{phase}"); + assert!(operation.completed_at.is_none(), "{phase}"); + operation.started_at = "2001-01-01T00:00:00Z".into(); + resource.status = Some(resumed); + + let waiting = StatusBuilder::new(&resource, 3, 2, 1) + .operation(update("WaitingForReady")) + .build(); + assert_eq!( + waiting.current_operation.unwrap().started_at, + "2001-01-01T00:00:00Z" + ); + } assert!(!resume_resets_operation_clock("Draining", "Replacing")); } } diff --git a/crates/operator/src/controller/validation.rs b/crates/operator/src/controller/validation.rs new file mode 100644 index 0000000..c1ac3f8 --- /dev/null +++ b/crates/operator/src/controller/validation.rs @@ -0,0 +1,379 @@ +use super::{storage, KODO_BOOTSTRAP_RETENTION_SECONDS}; +use crate::RustQueue; +use anyhow::{bail, Context as _}; +use std::sync::Arc; + +pub(super) fn with_storage_feature_floor( + cluster: Arc, + active_feature_floor: u32, +) -> Arc { + let effective = + effective_storage_feature_level(cluster.spec.storage_feature_level, active_feature_floor); + let (connection_delivery, node_delivery) = effective_delivery_limits( + effective, + cluster.spec.max_message_bytes, + cluster.spec.connection_delivery_inflight_bytes, + cluster.spec.node_delivery_inflight_bytes, + ); + if effective == cluster.spec.storage_feature_level + && connection_delivery == cluster.spec.connection_delivery_inflight_bytes + && node_delivery == cluster.spec.node_delivery_inflight_bytes + { + return cluster; + } + let mut adjusted = cluster.as_ref().clone(); + adjusted.spec.storage_feature_level = effective; + adjusted.spec.connection_delivery_inflight_bytes = connection_delivery; + adjusted.spec.node_delivery_inflight_bytes = node_delivery; + Arc::new(adjusted) +} + +fn effective_storage_feature_level(requested: u32, active: u32) -> u32 { + requested.max(active).max(1) +} + +fn effective_delivery_limits( + storage_feature_level: u32, + max_message_bytes: usize, + connection: usize, + node: usize, +) -> (usize, usize) { + let retained_message_bound = if storage_feature_level >= 2 { + 100 * 1024 * 1024 + } else { + max_message_bytes + }; + let connection = connection.max(retained_message_bound); + (connection, node.max(connection.saturating_mul(2))) +} + +pub(super) fn validate(cluster: &RustQueue, active_feature_floor: u32) -> anyhow::Result<()> { + if cluster.spec.image.trim().is_empty() { + bail!("spec.image is required"); + } + if cluster.spec.min_brokers < 1 || cluster.spec.max_brokers < cluster.spec.min_brokers { + bail!("broker limits must satisfy 1 <= minBrokers <= maxBrokers"); + } + if cluster.spec.storage_class_name.trim().is_empty() + || cluster.spec.storage_size.trim().is_empty() + { + bail!("storageClassName and storageSize are required"); + } + if cluster.spec.storage_feature_level == 0 { + bail!("storageFeatureLevel must be greater than zero"); + } + if cluster.spec.disk_low_watermark_percent >= cluster.spec.disk_high_watermark_percent + || cluster.spec.disk_high_watermark_percent > 100 + { + bail!("disk watermarks must satisfy low < high <= 100"); + } + if cluster.spec.bootstrap_retention_seconds == 0 + || cluster.spec.max_message_bytes == 0 + || cluster.spec.max_message_bytes > 100 * 1024 * 1024 + || cluster.spec.message_index_cache_bytes == 0 + || cluster.spec.connection_delivery_inflight_bytes < cluster.spec.max_message_bytes + || cluster + .spec + .connection_delivery_inflight_bytes + .checked_mul(2) + .is_none_or(|minimum| cluster.spec.node_delivery_inflight_bytes < minimum) + || cluster.spec.node_delivery_inflight_bytes > u32::MAX as usize + || cluster.spec.max_topics == 0 + || cluster.spec.max_publish_workers == 0 + || cluster.spec.publish_worker_idle_seconds == 0 + || (cluster.spec.publish_ack_mode != "durable" + && (cluster.spec.relaxed_sync_messages == 0 + || cluster.spec.relaxed_sync_bytes < 4096 + || cluster.spec.relaxed_sync_interval_ms == 0)) + || cluster.spec.max_detailed_metric_series == 0 + { + bail!("queue limits are outside the stable v7 contract"); + } + if !matches!( + cluster.spec.publish_ack_mode.as_str(), + "durable" | "write_ack" | "nsq_relaxed" + ) { + bail!("publishAckMode must be durable, write_ack, or nsq_relaxed"); + } + if cluster.spec.websocket.max_connections == 0 + || cluster.spec.websocket.max_connections > tokio::sync::Semaphore::MAX_PERMITS + || cluster.spec.websocket.max_connections_per_topic == 0 + || cluster.spec.websocket.max_connections_per_topic > cluster.spec.websocket.max_connections + || cluster.spec.websocket.frame_inflight_bytes + < cluster.spec.max_message_bytes.saturating_add(24) + || cluster.spec.websocket.frame_inflight_bytes > u32::MAX as usize + { + bail!("WebSocket limits must fit one maximum message and satisfy per-Topic <= global connections"); + } + if cluster.spec.websocket.allowed_origins.iter().any(|origin| { + let authority = origin + .strip_prefix("http://") + .or_else(|| origin.strip_prefix("https://")); + origin == "*" + || authority.is_none_or(|authority| { + authority.is_empty() + || authority.contains('/') + || authority.contains('?') + || authority.contains('#') + || authority.contains(',') + || authority.trim() != authority + }) + }) { + bail!("WebSocket allowedOrigins must be exact HTTP origins without paths or wildcards"); + } + validate_message_storage_contract( + cluster.spec.max_message_bytes, + effective_storage_feature_level(cluster.spec.storage_feature_level, active_feature_floor), + )?; + if cluster.spec.kodo_compatibility.enabled + && (cluster.spec.min_brokers != 3 + || cluster.spec.max_brokers != 3 + || cluster.spec.storage_feature_level != 2 + || cluster.spec.bootstrap_retention_seconds < KODO_BOOTSTRAP_RETENTION_SECONDS + || cluster.spec.max_message_bytes != 100 * 1024 * 1024 + || cluster.spec.connection_delivery_inflight_bytes < 128 * 1024 * 1024 + || cluster.spec.node_delivery_inflight_bytes < 512 * 1024 * 1024 + || cluster.spec.publish_ack_mode != "durable") + { + bail!( + "Kodo compatibility requires exactly 3 brokers, storageFeatureLevel 2, \ + bootstrapRetentionSeconds >= 180, \ + maxMessageBytes 104857600, connectionDeliveryInflightBytes >= 134217728, \ + nodeDeliveryInflightBytes >= 536870912, and publishAckMode durable" + ); + } + if !(630..=86_400).contains(&cluster.spec.kodo_compatibility.cutover_grace_seconds) { + bail!("Kodo compatibility cutoverGraceSeconds must be between 630 and 86400"); + } + if cluster.spec.kodo_compatibility.enabled { + if cluster.spec.kodo_compatibility.decommission_confirmed { + bail!("decommissionConfirmed must be false while Kodo compatibility is enabled"); + } + if cluster.spec.kodo_compatibility.cleanup_enabled { + bail!( + "Kodo automatic cleanup is disabled until cluster-wide atomic deletion is available" + ); + } + let target_image = cluster + .spec + .rollout + .rollback_to_image + .as_deref() + .unwrap_or(&cluster.spec.image); + if cluster.spec.image_pull_policy != "Never" && !has_sha256_digest(target_image) { + bail!( + "Kodo compatibility requires an immutable @sha256 image or imagePullPolicy Never" + ); + } + if cluster + .spec + .kodo_compatibility + .allowed_pod_selector + .is_empty() + || cluster + .spec + .kodo_compatibility + .allowed_pod_selector + .iter() + .any(|(key, value)| key.trim().is_empty() || value.trim().is_empty()) + || cluster + .spec + .kodo_compatibility + .allowed_namespace_selector + .iter() + .any(|(key, value)| key.trim().is_empty() || value.trim().is_empty()) + { + bail!("Kodo compatibility requires a non-empty allowedPodSelector and valid selector labels"); + } + let memory_request = storage::parse_quantity(&cluster.spec.broker_resources.memory_request) + .context("parse broker memory request")?; + if memory_request < 2_u128 << 30 { + bail!("Kodo compatibility requires brokerResources.memoryRequest >= 2Gi"); + } + let cpu_request = parse_cpu_millis(&cluster.spec.broker_resources.cpu_request) + .context("parse broker CPU request")?; + if cpu_request < 1_000.0 { + bail!("Kodo compatibility requires brokerResources.cpuRequest >= 1 CPU"); + } + if let Some(cpu_limit) = cluster.spec.broker_resources.cpu_limit.as_deref() { + let cpu_limit = parse_cpu_millis(cpu_limit).context("parse broker CPU limit")?; + if cpu_limit < cpu_request { + bail!("brokerResources.cpuLimit must be greater than or equal to cpuRequest"); + } + } + if let Some(memory_limit) = cluster.spec.broker_resources.memory_limit.as_deref() { + let memory_limit = + storage::parse_quantity(memory_limit).context("parse broker memory limit")?; + if memory_limit < memory_request { + bail!("brokerResources.memoryLimit must be greater than or equal to memoryRequest"); + } + } + } + if cluster.spec.rollout.timeout_seconds == 0 || cluster.spec.rollout.timeout_seconds > 86_400 { + bail!("rollout timeoutSeconds must be between 1 and 86400"); + } + if cluster + .spec + .rollout + .rollback_to_image + .as_ref() + .is_some_and(|image| image.trim().is_empty()) + { + bail!("rollout rollbackToImage cannot be empty"); + } + if cluster + .spec + .broker_scheduling + .topology_key + .trim() + .is_empty() + || cluster.spec.broker_resources.cpu_request.trim().is_empty() + || cluster + .spec + .broker_resources + .memory_request + .trim() + .is_empty() + { + bail!("broker scheduling and resource requests cannot be empty"); + } + Ok(()) +} + +fn parse_cpu_millis(value: &str) -> anyhow::Result { + let value = value.trim(); + let (number, multiplier) = if let Some(number) = value.strip_suffix('m') { + (number, 1.0) + } else if let Some(number) = value.strip_suffix('u') { + (number, 0.001) + } else if let Some(number) = value.strip_suffix('n') { + (number, 0.000_001) + } else { + (value, 1_000.0) + }; + let number: f64 = number.parse()?; + anyhow::ensure!( + number.is_finite() && number >= 0.0, + "invalid CPU quantity {value}" + ); + Ok(number * multiplier) +} + +fn validate_message_storage_contract( + max_message_bytes: usize, + storage_feature_level: u32, +) -> anyhow::Result<()> { + const LEGACY_MAX_RECORD_BYTES: usize = 72 * 1024 * 1024; + const SINGLE_MESSAGE_ENVELOPE_BYTES: usize = 24; + const MPUB_ENTRY_BYTES: usize = 16; + const MAX_MPUB_MESSAGES: usize = 65_536; + let max_body_bytes = (64 * 1024 * 1024).max(max_message_bytes); + let maximum_record = max_message_bytes + .saturating_add(SINGLE_MESSAGE_ENVELOPE_BYTES) + .max(max_body_bytes.saturating_add(MPUB_ENTRY_BYTES.saturating_mul(MAX_MPUB_MESSAGES))); + if maximum_record > LEGACY_MAX_RECORD_BYTES && storage_feature_level < 2 { + bail!("messages above the v7 legacy record bound require storageFeatureLevel 2"); + } + Ok(()) +} + +fn has_sha256_digest(image: &str) -> bool { + image.rsplit_once("@sha256:").is_some_and(|(_, digest)| { + digest.len() == 64 + && digest + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn active_storage_feature_level_is_a_monotonic_floor() { + assert_eq!(effective_storage_feature_level(1, 2), 2); + assert_eq!(effective_storage_feature_level(2, 1), 2); + } + + #[test] + fn cpu_quantities_compare_in_millicores() { + assert_eq!(parse_cpu_millis("1").unwrap(), 1_000.0); + assert_eq!(parse_cpu_millis("1000m").unwrap(), 1_000.0); + assert_eq!(parse_cpu_millis("500m").unwrap(), 500.0); + assert_eq!(parse_cpu_millis("500000u").unwrap(), 500.0); + assert!(parse_cpu_millis("-1").is_err()); + } + + #[test] + fn feature_two_retains_large_message_delivery_capacity() { + assert_eq!( + effective_delivery_limits(2, 20 * 1024 * 1024, 32 * 1024 * 1024, 64 * 1024 * 1024), + (100 * 1024 * 1024, 200 * 1024 * 1024) + ); + } + + #[test] + fn legacy_feature_rejects_messages_that_cross_the_record_bound() { + let maximum = 71 * 1024 * 1024; + assert!(validate_message_storage_contract(maximum, 1).is_ok()); + assert!(validate_message_storage_contract(maximum + 1, 1).is_err()); + assert!(validate_message_storage_contract(100 * 1024 * 1024, 2).is_ok()); + } + + #[test] + fn immutable_image_detection_requires_a_complete_sha256_digest() { + let digest = "a".repeat(64); + assert!(has_sha256_digest(&format!( + "registry/rustqueue@sha256:{digest}" + ))); + assert!(!has_sha256_digest(&format!( + "registry/rustqueue@sha256:{}", + "A".repeat(64) + ))); + assert!(!has_sha256_digest("registry/rustqueue:latest")); + assert!(!has_sha256_digest("registry/rustqueue@sha256:abcd")); + } + + #[test] + fn kodo_contract_requires_a_second_lookup_poll_retention_window() { + let mut cluster: RustQueue = serde_json::from_value(serde_json::json!({ + "apiVersion": "rustqueue.io/v1alpha1", + "kind": "RustQueue", + "metadata": {"name": "queue", "namespace": "test"}, + "spec": { + "image": "rustqueue:test", + "imagePullPolicy": "Never", + "minBrokers": 3, + "maxBrokers": 3, + "storageFeatureLevel": 2, + "bootstrapRetentionSeconds": 180, + "maxMessageBytes": 104857600, + "connectionDeliveryInflightBytes": 134217728, + "nodeDeliveryInflightBytes": 536870912, + "brokerResources": {"cpuRequest": "1", "memoryRequest": "2Gi"}, + "kodoCompatibility": {"enabled": true} + } + })) + .unwrap(); + + assert!(validate(&cluster, 2).is_ok()); + cluster.spec.broker_resources.cpu_request = "999m".into(); + assert!(validate(&cluster, 2) + .unwrap_err() + .to_string() + .contains("cpuRequest >= 1 CPU")); + cluster.spec.broker_resources.cpu_request = "1".into(); + cluster.spec.bootstrap_retention_seconds = KODO_BOOTSTRAP_RETENTION_SECONDS - 1; + assert!(validate(&cluster, 2) + .unwrap_err() + .to_string() + .contains("bootstrapRetentionSeconds >= 180")); + cluster.spec.bootstrap_retention_seconds = KODO_BOOTSTRAP_RETENTION_SECONDS; + cluster.spec.publish_ack_mode = "nsq_relaxed".into(); + assert!(validate(&cluster, 2) + .unwrap_err() + .to_string() + .contains("publishAckMode durable")); + } +} diff --git a/crates/operator/src/resources.rs b/crates/operator/src/resources.rs index 01bd6f8..cb5e234 100644 --- a/crates/operator/src/resources.rs +++ b/crates/operator/src/resources.rs @@ -356,6 +356,10 @@ pub fn build(input: BuildInput<'_>) -> anyhow::Result { "metadata": metadata(&proxy_name, "proxy"), "spec": { "selector": {"matchLabels": labels_for(&labels, "proxy")}, + "updateStrategy": { + "type": "RollingUpdate", + "rollingUpdate": {"maxUnavailable": 0, "maxSurge": 1} + }, "template": { "metadata": {"labels": labels_for(&labels, "proxy"), "annotations": {"rustqueue.io/revision": revision}}, "spec": { @@ -852,6 +856,13 @@ mod tests { && variable.value.as_deref() == Some("300") }))); let proxy_value = serde_json::to_value(&resources.proxy).unwrap(); + assert_eq!( + proxy_value["spec"]["updateStrategy"], + json!({ + "type": "RollingUpdate", + "rollingUpdate": {"maxUnavailable": 0, "maxSurge": 1} + }) + ); assert_eq!( proxy_value["spec"]["template"]["spec"]["containers"][0]["livenessProbe"]["httpGet"] ["path"], diff --git a/crates/queue/src/broker.rs b/crates/queue/src/broker.rs index a41f6f4..cf03078 100644 --- a/crates/queue/src/broker.rs +++ b/crates/queue/src/broker.rs @@ -113,6 +113,7 @@ pub struct BrokerConfig { pub payload_read_workers: usize, pub payload_read_queue: usize, pub delivery_inflight_bytes: usize, + pub delivery_inflight_messages: usize, pub scrub_bytes_per_second: u64, pub storage_feature_level: u32, pub require_management_fence_sync: bool, @@ -140,6 +141,7 @@ impl Default for BrokerConfig { payload_read_workers: 0, payload_read_queue: 4096, delivery_inflight_bytes: 512 * 1024 * 1024, + delivery_inflight_messages: 262_144, scrub_bytes_per_second: 64 * 1024 * 1024, storage_feature_level: BASE_STORAGE_FEATURE_LEVEL, require_management_fence_sync: false, @@ -273,6 +275,13 @@ impl Broker { )); } let now = Instant::now(); + if config.delivery_inflight_messages == 0 + || config.delivery_inflight_messages > tokio::sync::Semaphore::MAX_PERMITS + { + return Err(BrokerError::InvalidRecord( + "delivery message budget must be positive and fit the runtime semaphore".into(), + )); + } if now.checked_add(config.message_timeout).is_none() || now.checked_add(config.publish_worker_idle).is_none() || now.checked_add(config.relaxed_sync_interval).is_none() @@ -393,7 +402,10 @@ impl Broker { config.max_publish_workers, config.publish_worker_idle, ); - let delivery_budget = DeliveryBudget::new(config.delivery_inflight_bytes); + let delivery_budget = DeliveryBudget::new( + config.delivery_inflight_bytes, + config.delivery_inflight_messages, + ); let broker = Self { inner: Arc::new(BrokerInner { config, diff --git a/crates/queue/src/broker/channel_commit.rs b/crates/queue/src/broker/channel_commit.rs index 75edbec..a52cb88 100644 --- a/crates/queue/src/broker/channel_commit.rs +++ b/crates/queue/src/broker/channel_commit.rs @@ -95,10 +95,10 @@ impl ChannelGroups { topic: &str, ) -> Result, BrokerError> { let mut senders = self.senders.lock(); - senders.retain(|_, entry| !entry.sender.is_closed()); - if let Some(entry) = senders.get(topic) { + if let Some(entry) = senders.get(topic).filter(|entry| !entry.sender.is_closed()) { return Ok(entry.sender.clone()); } + senders.retain(|_, entry| !entry.sender.is_closed()); if senders.len() >= self.max_workers { self.rejected_workers.fetch_add(1, Ordering::Relaxed); return Err(BrokerError::ChannelWorkerLimit); diff --git a/crates/queue/src/broker/delivery.rs b/crates/queue/src/broker/delivery.rs index 10fab53..1a02a4b 100644 --- a/crates/queue/src/broker/delivery.rs +++ b/crates/queue/src/broker/delivery.rs @@ -135,6 +135,7 @@ impl Broker { remaining_messages, max_bytes.saturating_sub(batch.payload_bytes()).max(1), timeout, + &self.inner.delivery_budget, )?; let mut paths = match &action { ReserveBatch::Ready(reserved) => reserved diff --git a/crates/queue/src/broker/group_commit.rs b/crates/queue/src/broker/group_commit.rs index 1df969d..2f12887 100644 --- a/crates/queue/src/broker/group_commit.rs +++ b/crates/queue/src/broker/group_commit.rs @@ -70,10 +70,10 @@ impl PublishGroups { topic: &str, ) -> Result, BrokerError> { let mut senders = self.senders.lock(); - senders.retain(|_, entry| !entry.sender.is_closed()); - if let Some(entry) = senders.get(topic) { + if let Some(entry) = senders.get(topic).filter(|entry| !entry.sender.is_closed()) { return Ok(entry.sender.clone()); } + senders.retain(|_, entry| !entry.sender.is_closed()); if senders.len() >= self.max_workers { self.rejected_workers.fetch_add(1, Ordering::Relaxed); return Err(BrokerError::PublishWorkerLimit); diff --git a/crates/queue/src/broker/io.rs b/crates/queue/src/broker/io.rs index 0a0bb81..fc091bd 100644 --- a/crates/queue/src/broker/io.rs +++ b/crates/queue/src/broker/io.rs @@ -550,7 +550,7 @@ mod tests { publisher.join().unwrap(), Err(BrokerError::TopicTombstoned) )); - assert_eq!(broker.stats().topics[0].message_count, 0); + assert_eq!(broker.stats().unwrap().topics[0].message_count, 0); } #[tokio::test] @@ -575,11 +575,11 @@ mod tests { ) .await .unwrap(); - assert_eq!(broker.stats().topics[0].unsynced_messages, 1); + assert_eq!(broker.stats().unwrap().topics[0].unsynced_messages, 1); broker .publish_durable_body_sync("events", &[Bytes::from_static(b"durable")], Duration::ZERO) .unwrap(); - let stats = broker.stats(); + let stats = broker.stats().unwrap(); assert_eq!(stats.topics[0].last_durable_position, 2); assert_eq!(stats.topics[0].unsynced_messages, 0); } diff --git a/crates/queue/src/broker/stats.rs b/crates/queue/src/broker/stats.rs index 8cc81c0..4c23b0a 100644 --- a/crates/queue/src/broker/stats.rs +++ b/crates/queue/src/broker/stats.rs @@ -2,32 +2,42 @@ use super::*; use crate::model::{BrokerStats, QueueAggregateStats, TopicStats}; impl Broker { - pub fn stats(&self) -> BrokerStats { + /// Returns an exact snapshot, or a storage error if deferred metadata cannot + /// be read. Integrity errors also isolate the Broker through storage health. + pub fn stats(&self) -> Result { self.filtered_stats(None, None) } - pub fn filtered_stats(&self, topic: Option<&str>, channel: Option<&str>) -> BrokerStats { - let mut topics = self.topic_stats(topic); + pub fn filtered_stats( + &self, + topic: Option<&str>, + channel: Option<&str>, + ) -> Result { + let mut topics = self.observe_storage_result(self.topic_stats(topic))?; if let Some(channel) = channel { for topic in &mut topics { topic.channels.retain(|candidate| candidate.name == channel); } } topics.sort_by(|left, right| left.name.cmp(&right.name)); - self.snapshot(topics) + Ok(self.snapshot(topics)) } - pub fn metrics_stats(&self, detailed: bool, max_series: usize) -> BrokerStats { + pub fn metrics_stats( + &self, + detailed: bool, + max_series: usize, + ) -> Result { let handles: Vec<_> = self.inner.topics.read().values().cloned().collect(); let mut aggregate = QueueAggregateStats::default(); let mut topics = Vec::new(); let mut remaining = max_series; for handle in handles { let _commit_gate = handle.commit_gate.lock(); - let mut topic = handle.state.lock(); - topic.add_aggregate_stats(&mut aggregate); + let topic = handle.state.lock(); + let mut stats = self.observe_storage_result(topic.stats())?; + aggregate.add_topic(&stats); if detailed && remaining >= 6 { - let mut stats = topic.stats(); remaining = remaining.saturating_sub(6); let channel_limit = remaining / 4; stats.channels.truncate(channel_limit); @@ -36,13 +46,13 @@ impl Broker { } } topics.sort_by(|left, right| left.name.cmp(&right.name)); - BrokerStats { + Ok(BrokerStats { aggregate, ..self.snapshot(topics) - } + }) } - fn topic_stats(&self, topic: Option<&str>) -> Vec { + fn topic_stats(&self, topic: Option<&str>) -> Result, BrokerError> { let handles: Vec<_> = match topic { Some(name) => self .inner @@ -101,7 +111,9 @@ mod tests { broker.create_channel("events", "audit").await.unwrap(); broker.create_channel("other", "workers").await.unwrap(); - let stats = broker.filtered_stats(Some("events"), Some("workers")); + let stats = broker + .filtered_stats(Some("events"), Some("workers")) + .unwrap(); assert_eq!(stats.aggregate.topic_count, 1); assert_eq!(stats.aggregate.channel_count, 1); assert_eq!(stats.topics.len(), 1); @@ -111,6 +123,7 @@ mod tests { assert!(broker .filtered_stats(Some("missing"), None) + .unwrap() .topics .is_empty()); } @@ -125,7 +138,7 @@ mod tests { .unwrap(); broker.get_or_create_topic("events").unwrap(); - assert!(broker.metrics_stats(true, 5).topics.is_empty()); - assert_eq!(broker.metrics_stats(true, 6).topics.len(), 1); + assert!(broker.metrics_stats(true, 5).unwrap().topics.is_empty()); + assert_eq!(broker.metrics_stats(true, 6).unwrap().topics.len(), 1); } } diff --git a/crates/queue/src/broker_tests.rs b/crates/queue/src/broker_tests.rs index b75152b..dca82df 100644 --- a/crates/queue/src/broker_tests.rs +++ b/crates/queue/src/broker_tests.rs @@ -181,7 +181,7 @@ async fn queued_publish_rechecks_storage_health_before_append() { Err(BrokerError::StorageUnavailable) )); assert_eq!( - broker.filtered_stats(Some("events"), None).topics[0].message_count, + broker.filtered_stats(Some("events"), None).unwrap().topics[0].message_count, 0 ); } @@ -208,7 +208,7 @@ async fn write_ack_hides_the_unsynced_tail_until_flush() { ) .await .unwrap(); - let stats = broker.filtered_stats(Some("events"), None); + let stats = broker.filtered_stats(Some("events"), None).unwrap(); assert_eq!(stats.topics[0].last_durable_position, 0); assert_eq!(stats.topics[0].unsynced_messages, 1); assert!(stats.topics[0].unsynced_bytes > 0); @@ -226,7 +226,7 @@ async fn write_ack_hides_the_unsynced_tail_until_flush() { .unwrap(); assert_eq!(delivery.id, ids[0]); assert_eq!(&*delivery.body, b"pending"); - let stats = broker.filtered_stats(Some("events"), None); + let stats = broker.filtered_stats(Some("events"), None).unwrap(); assert_eq!(stats.topics[0].last_durable_position, 1); assert_eq!(stats.topics[0].unsynced_messages, 0); assert_eq!(stats.topics[0].unsynced_bytes, 0); @@ -254,7 +254,7 @@ async fn nsq_relaxed_exposes_the_unsynced_tail_immediately() { ) .await .unwrap(); - let stats = broker.filtered_stats(Some("events"), None); + let stats = broker.filtered_stats(Some("events"), None).unwrap(); assert_eq!(stats.topics[0].last_durable_position, 0); assert_eq!(stats.topics[0].unsynced_messages, 1); let delivery = broker @@ -303,7 +303,7 @@ async fn write_ack_timer_makes_the_tail_durable_and_visible() { .unwrap(); assert_eq!(&*delivery.body, b"timer"); assert_eq!( - broker.filtered_stats(Some("events"), None).topics[0].unsynced_messages, + broker.filtered_stats(Some("events"), None).unwrap().topics[0].unsynced_messages, 0 ); } @@ -580,7 +580,7 @@ async fn blocked_dlq_target_does_not_prevent_broker_restart() { .count(), 1 ); - let stats = broker.stats(); + let stats = broker.stats().unwrap(); let source = stats .topics .iter() @@ -630,7 +630,7 @@ async fn startup_does_not_replay_outbox_before_required_fence_sync() { .count(), 1 ); - let stats = broker.stats(); + let stats = broker.stats().unwrap(); let source = stats .topics .iter() @@ -677,7 +677,7 @@ async fn concurrent_dlq_moves_publish_only_one_copy() { 1 ); - let stats = broker.stats(); + let stats = broker.stats().unwrap(); let source = stats .topics .iter() @@ -727,7 +727,7 @@ async fn cancelled_dlq_move_keeps_the_transaction_serialized_until_completion() tokio::time::timeout(Duration::from_secs(2), async { loop { - let stats = broker.stats(); + let stats = broker.stats().unwrap(); let source_depth = stats .topics .iter() @@ -762,6 +762,7 @@ async fn cancelled_dlq_move_keeps_the_transaction_serialized_until_completion() assert_eq!( broker .stats() + .unwrap() .topics .iter() .find(|topic| topic.name == "events.workers.DLQ") @@ -816,6 +817,7 @@ async fn completed_dlq_outbox_is_not_published_again_after_restart() { assert!(!outbox_path.exists()); let target = broker .stats() + .unwrap() .topics .into_iter() .find(|topic| topic.name == target) @@ -860,6 +862,7 @@ async fn dlq_outbox_recovers_when_the_source_was_removed() { ); let target = broker .stats() + .unwrap() .topics .into_iter() .find(|topic| topic.name == target) @@ -887,7 +890,7 @@ async fn stats_settle_expired_in_flight_messages_without_another_fetch() { .is_some()); broker.expire_in_flight().await.unwrap(); - let stats = broker.stats(); + let stats = broker.stats().unwrap(); let channel = &stats.topics[0].channels[0]; assert_eq!(channel.in_flight_count, 0); assert_eq!(channel.timeout_count, 1); @@ -953,17 +956,87 @@ async fn stale_delivery_token_cannot_mutate_a_redelivery() { Err(BrokerError::MessageNotInFlight) )); broker.release_deliveries("events", "workers", &[(id, stale_token)]); - assert_eq!(broker.stats().topics[0].channels[0].in_flight_count, 1); + assert_eq!( + broker.stats().unwrap().topics[0].channels[0].in_flight_count, + 1 + ); broker .finish_delivery("events", "workers", id, current_token) .await .unwrap(); - assert_eq!(broker.stats().topics[0].channels[0].depth, 0); + assert_eq!(broker.stats().unwrap().topics[0].channels[0].depth, 0); +} + +#[tokio::test] +async fn channel_recreation_rejects_old_tokens_and_guard_cancellation() { + let root = tempdir().unwrap(); + let broker = Broker::open(BrokerConfig { + data_path: root.path().into(), + bootstrap_retention: Duration::from_secs(90), + ..BrokerConfig::default() + }) + .unwrap(); + broker.create_channel("events", "anchor").await.unwrap(); + broker.create_channel("events", "workers").await.unwrap(); + broker + .publish("events", vec![b"body".to_vec()], Duration::ZERO) + .await + .unwrap(); + + let old_batch = broker + .fetch_batch_retained("events", "workers", 1, usize::MAX, Duration::ZERO, None) + .await + .unwrap(); + let (old_messages, old_guard) = old_batch.into_parts(); + let id = old_messages[0].id; + let old_token = old_guard.token(id).unwrap(); + drop(old_messages); + + broker.delete_channel("events", "workers").await.unwrap(); + broker.create_channel("events", "workers").await.unwrap(); + + let new_batch = broker + .fetch_batch_retained("events", "workers", 1, usize::MAX, Duration::ZERO, None) + .await + .unwrap(); + let (new_messages, new_guard) = new_batch.into_parts(); + assert_eq!(new_messages[0].id, id); + let new_token = new_guard.token(id).unwrap(); + assert_ne!(old_token, new_token); + + assert!(matches!( + broker + .finish_delivery("events", "workers", id, old_token) + .await, + Err(BrokerError::MessageNotInFlight) + )); + + drop(old_guard); + let stats = broker + .filtered_stats(Some("events"), Some("workers")) + .unwrap(); + let workers = &stats.topics[0].channels[0]; + assert_eq!(workers.in_flight_count, 1); + assert_eq!(broker.delivery_budget_stats().in_flight_messages, 1); + + broker + .finish_delivery("events", "workers", id, new_token) + .await + .unwrap(); + let stats = broker + .filtered_stats(Some("events"), Some("workers")) + .unwrap(); + let workers = &stats.topics[0].channels[0]; + assert_eq!((workers.depth, workers.in_flight_count), (0, 0)); + + drop(new_messages); + drop(new_guard); + assert_eq!(broker.delivery_budget_stats().in_flight_messages, 0); } #[tokio::test] -async fn current_delivery_tokens_can_renew_an_expiring_batch() { +async fn batch_touch_extends_delivery_deadlines_before_redelivery() { let root = tempdir().unwrap(); let broker = Broker::open(BrokerConfig { data_path: root.path().into(), @@ -987,7 +1060,7 @@ async fn current_delivery_tokens_can_renew_an_expiring_batch() { 2, usize::MAX, Duration::ZERO, - Some(Duration::ZERO), + Some(Duration::from_millis(100)), ) .await .unwrap(); @@ -997,25 +1070,200 @@ async fn current_delivery_tokens_can_renew_an_expiring_batch() { .map(|delivery| (delivery.id, guard.token(delivery.id).unwrap())) .collect(); broker - .touch_deliveries("events", "workers", &tokens, Some(Duration::from_secs(30))) + .touch_deliveries("events", "workers", &tokens, Some(Duration::from_secs(2))) .unwrap(); for (id, token) in &tokens { assert_eq!(guard.accept_with_token(*id), Some(*token)); } - broker - .expire_channel_in_flight("events", "workers") + tokio::time::sleep(Duration::from_millis(200)).await; + let before_new_deadline = broker + .fetch_batch("events", "workers", 2, usize::MAX, Duration::ZERO, None) + .await + .unwrap(); + assert!(before_new_deadline.is_empty()); + assert_eq!( + broker.stats().unwrap().topics[0].channels[0].in_flight_count, + 2 + ); + + tokio::time::sleep(Duration::from_secs(2)).await; + let redeliveries = broker + .fetch_batch("events", "workers", 2, usize::MAX, Duration::ZERO, None) .await .unwrap(); - assert_eq!(broker.stats().topics[0].channels[0].in_flight_count, 2); - for (id, token) in tokens { + assert_eq!(redeliveries.len(), 2); + assert_eq!( + redeliveries + .iter() + .map(|delivery| delivery.id) + .collect::>(), + tokens.iter().map(|(id, _)| *id).collect::>() + ); + for delivery in redeliveries { broker - .finish_delivery("events", "workers", id, token) + .finish("events", "workers", delivery.id) .await .unwrap(); } } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn delivery_message_budget_is_shared_and_released_after_fetch_lifecycle() { + let root = tempdir().unwrap(); + let broker = Broker::open(BrokerConfig { + data_path: root.path().into(), + max_message_bytes: 32, + delivery_inflight_bytes: 64, + delivery_inflight_messages: 2, + ..BrokerConfig::default() + }) + .unwrap(); + broker.create_channel("events", "alpha").await.unwrap(); + broker.create_channel("other", "beta").await.unwrap(); + broker + .publish("events", vec![vec![0x5a; 32]], Duration::ZERO) + .await + .unwrap(); + + // The public publish contract rejects an empty body, but a zero-length + // persisted payload is still a valid storage shape. Keep this path in the + // broker-level lease regression because it has no byte-budget hold. + let empty_id = { + let id = broker.reserve_ids(1).unwrap(); + let mut metadata = broker.reserve_message_metadata(1).unwrap(); + let handle = broker.topic("events").unwrap(); + let _commit_gate = handle.commit_gate.lock(); + let mut state = handle.state.lock(); + let bodies = [bytes::Bytes::new()]; + let batch = crate::batch::encode(state.next_position(), id, &bodies).unwrap(); + state + .append_batch(id, now_ns(), now_ms(), batch, true, &mut metadata) + .unwrap()[0] + }; + broker + .publish( + "other", + vec![vec![0x6b; 16], vec![0x6c; 16]], + Duration::ZERO, + ) + .await + .unwrap(); + + let first = broker + .fetch_batch_retained("events", "alpha", 1, 32, Duration::ZERO, None) + .await + .unwrap(); + let (first_messages, first_guard) = first.into_parts(); + assert_eq!(first_messages.len(), 1); + assert_eq!(broker.delivery_budget_stats().in_flight_messages, 1); + assert_eq!(broker.delivery_budget_stats().in_flight_bytes, 64); + + let pending_fetch = tokio::spawn({ + let broker = broker.clone(); + async move { + broker + .fetch_batch_retained("other", "beta", 1, 32, Duration::ZERO, None) + .await + } + }); + tokio::time::timeout(Duration::from_secs(1), async { + while { + let stats = broker.delivery_budget_stats(); + stats.waiters == 0 || stats.in_flight_messages != 2 + } { + tokio::task::yield_now().await; + } + }) + .await + .expect("the second topic must reserve a message before waiting on bytes"); + pending_fetch.abort(); + let join_error = pending_fetch + .await + .err() + .expect("the pending fetch must be cancelled"); + assert!(join_error.is_cancelled()); + tokio::time::timeout(Duration::from_secs(1), async { + while broker.delivery_budget_stats().in_flight_messages != 1 { + tokio::task::yield_now().await; + } + }) + .await + .expect("cancelling a fetch must release its message reservation"); + assert_eq!(broker.delivery_budget_stats().in_flight_bytes, 64); + + drop(first_messages); + drop(first_guard); + assert_eq!(broker.delivery_budget_stats().in_flight_messages, 0); + assert_eq!(broker.delivery_budget_stats().in_flight_bytes, 0); + + let second = broker + .fetch_batch_retained("events", "alpha", 2, 32, Duration::ZERO, None) + .await + .unwrap(); + let (second_messages, mut second_guard) = second.into_parts(); + assert_eq!(second_messages.len(), 2); + assert!(second_messages + .iter() + .any(|delivery| delivery.id == empty_id && delivery.body.is_empty())); + assert_eq!(broker.delivery_budget_stats().in_flight_messages, 2); + + let (empty_token, empty_lease) = second_guard + .accept_with_lease(empty_id) + .expect("the zero-length delivery must carry a consumer lease"); + drop(second_messages); + broker + .finish_delivery("events", "alpha", empty_id, empty_token) + .await + .unwrap(); + assert_eq!( + broker.delivery_budget_stats().in_flight_messages, + 2, + "FIN cannot release the consumer-owned lease" + ); + drop(second_guard); + assert_eq!(broker.delivery_budget_stats().in_flight_messages, 1); + assert_eq!(broker.delivery_budget_stats().in_flight_bytes, 0); + + let other = broker + .fetch_batch_retained("other", "beta", 2, 32, Duration::ZERO, None) + .await + .unwrap(); + let (other_messages, mut other_guard) = other.into_parts(); + assert_eq!( + other_messages.len(), + 1, + "the node-wide message budget leaves one slot while FIN's lease is held" + ); + let other_id = other_messages[0].id; + let (other_token, other_lease) = other_guard.accept_with_lease(other_id).unwrap(); + drop(other_messages); + drop(other_guard); + broker + .requeue_delivery("other", "beta", other_id, other_token, Duration::ZERO) + .await + .unwrap(); + assert_eq!( + broker.delivery_budget_stats().in_flight_messages, + 2, + "REQ cannot release the consumer-owned lease" + ); + drop(other_lease); + assert_eq!(broker.delivery_budget_stats().in_flight_messages, 1); + + drop(empty_lease); + assert_eq!(broker.delivery_budget_stats().in_flight_messages, 0); + let recovered = broker + .fetch_batch("other", "beta", 2, 32, Duration::ZERO, None) + .await + .unwrap(); + assert_eq!(recovered.len(), 2); + for delivery in recovered { + broker.finish("other", "beta", delivery.id).await.unwrap(); + } + assert_eq!(broker.delivery_budget_stats().in_flight_messages, 0); +} + #[tokio::test] async fn kodo_channel_counters_are_monotonic_across_empty_and_restart() { if rustqueue_storage::MAX_WRITER_FEATURE_LEVEL < 2 { @@ -1058,14 +1306,14 @@ async fn kodo_channel_counters_are_monotonic_across_empty_and_restart() { .await .unwrap(); - let channel = &broker.stats().topics[0].channels[0]; + let channel = &broker.stats().unwrap().topics[0].channels[0]; assert_eq!(channel.message_count, 3); assert_eq!(channel.requeue_count, 1); assert_eq!(channel.timeout_count, 1); drop(broker); let reopened = Broker::open(config).unwrap(); - let channel = &reopened.stats().topics[0].channels[0]; + let channel = &reopened.stats().unwrap().topics[0].channels[0]; assert_eq!(channel.message_count, 3); assert_eq!(channel.requeue_count, 1); assert_eq!(channel.timeout_count, 1); @@ -1086,7 +1334,7 @@ async fn durable_batch_accepts_the_full_protocol_message_count() { .unwrap(); assert_eq!(ids.len(), rustqueue_protocol::MAX_MPUB_MESSAGES); assert_eq!( - broker.stats().topics[0].published_count, + broker.stats().unwrap().topics[0].published_count, rustqueue_protocol::MAX_MPUB_MESSAGES as u64 ); } @@ -1299,14 +1547,14 @@ async fn concurrent_publishes_share_a_durable_group_commit() { .collect(); assert_eq!(ids.len(), 32); - let commit = broker.stats().publish_group_commit; + let commit = broker.stats().unwrap().publish_group_commit; assert_eq!(commit.requests, 32); assert!(commit.commits < commit.requests); assert!(commit.max_batch_requests > 1); drop(broker); let broker = Broker::open(config).unwrap(); - assert_eq!(broker.stats().topics[0].message_count, 32); + assert_eq!(broker.stats().unwrap().topics[0].message_count, 32); } #[tokio::test] @@ -1353,7 +1601,7 @@ async fn concurrent_fin_and_req_share_a_durable_group_commit() { for result in join_all(commands).await { result.unwrap(); } - let commit = broker.stats().channel_group_commit; + let commit = broker.stats().unwrap().channel_group_commit; assert_eq!(commit.requests, 16); assert!(commit.commits < commit.requests); assert!(commit.max_batch_requests > 1); @@ -1399,7 +1647,7 @@ async fn idle_publish_workers_retire_and_capacity_is_reusable() { .publish("second", vec![b"two".to_vec()], Duration::ZERO) .await .unwrap(); - let stats = broker.stats().publish_group_commit; + let stats = broker.stats().unwrap().publish_group_commit; assert!(stats.retired_workers >= 1); assert_eq!(stats.rejected_workers, 1); assert!(stats.active_workers <= 1); @@ -1472,7 +1720,7 @@ async fn sealed_backlog_uses_bounded_metadata_residency() { broker.compact().await.unwrap(); let handle = broker.topic("events").unwrap(); let (active, sealed) = handle.state.lock().index_residency(); - assert_eq!(broker.stats().topics[0].message_count, 3_264); + assert_eq!(broker.stats().unwrap().topics[0].message_count, 3_264); assert!(sealed >= 100); assert!( active <= 64, @@ -1483,7 +1731,7 @@ async fn sealed_backlog_uses_bounded_metadata_residency() { let broker = Broker::open(config).unwrap(); let handle = broker.topic("events").unwrap(); let (active, sealed) = handle.state.lock().index_residency(); - assert_eq!(broker.stats().topics[0].message_count, 3_264); + assert_eq!(broker.stats().unwrap().topics[0].message_count, 3_264); assert!(sealed >= 100); assert!(active <= 64); let deliveries = broker @@ -1520,7 +1768,7 @@ async fn idle_gc_does_not_seal_a_channel_blocked_active_segment() { for _ in 0..4 { assert_eq!(broker.compact().await.unwrap(), 0); } - let stats = broker.stats(); + let stats = broker.stats().unwrap(); assert_eq!(stats.topics[0].segment_count, 1); assert_eq!(stats.topics[0].message_count, 1); } @@ -1531,6 +1779,7 @@ async fn deferred_stats_are_exact_without_a_consumer_and_after_restart() { let config = BrokerConfig { data_path: root.path().into(), max_ack_gap: 2, + bootstrap_retention: Duration::from_nanos(1), ..BrokerConfig::default() }; let broker = Broker::open(config.clone()).unwrap(); @@ -1551,13 +1800,134 @@ async fn deferred_stats_are_exact_without_a_consumer_and_after_restart() { ) .await .unwrap(); - let channel = &broker.stats().topics[0].channels[0]; + let channel = &broker.stats().unwrap().topics[0].channels[0]; assert_eq!((channel.depth, channel.deferred_count), (4, 1)); + broker + .topic("events") + .unwrap() + .state + .lock() + .spill_message_metadata() + .unwrap(); + broker.create_channel("events", "tail").await.unwrap(); + broker + .publish( + "events", + vec![b"tail-deferred".to_vec()], + Duration::from_secs(60), + ) + .await + .unwrap(); + let ready = broker + .publish("events", vec![b"tail-ready".to_vec()], Duration::ZERO) + .await + .unwrap()[0]; + broker + .topic("events") + .unwrap() + .state + .lock() + .spill_message_metadata() + .unwrap(); + let stats = broker + .filtered_stats(Some("events"), Some("workers")) + .unwrap(); + assert_eq!( + ( + stats.topics[0].channels[0].depth, + stats.topics[0].channels[0].deferred_count + ), + (6, 2) + ); + let deliveries = broker + .fetch_batch("events", "workers", 8, usize::MAX, Duration::ZERO, None) + .await + .unwrap(); + assert_eq!(deliveries.len(), 2); + broker + .finish("events", "workers", deliveries[1].id) + .await + .unwrap(); + let tail = broker + .fetch_batch("events", "tail", 8, usize::MAX, Duration::ZERO, None) + .await + .unwrap(); + assert_eq!(tail.len(), 1); + assert_eq!(tail[0].id, ready); + let stats = broker.filtered_stats(Some("events"), Some("tail")).unwrap(); + assert_eq!( + ( + stats.topics[0].channels[0].in_flight_count, + stats.topics[0].channels[0].deferred_count + ), + (1, 1) + ); + broker + .requeue("events", "tail", ready, Duration::from_secs(60)) + .await + .unwrap(); drop(broker); let reopened = Broker::open(config).unwrap(); - let channel = &reopened.stats().topics[0].channels[0]; - assert_eq!((channel.depth, channel.deferred_count), (4, 1)); + let workers = reopened + .filtered_stats(Some("events"), Some("workers")) + .unwrap(); + assert_eq!( + ( + workers.topics[0].channels[0].depth, + workers.topics[0].channels[0].deferred_count + ), + (5, 2) + ); + let tail = reopened + .filtered_stats(Some("events"), Some("tail")) + .unwrap(); + assert_eq!( + ( + tail.topics[0].channels[0].depth, + tail.topics[0].channels[0].deferred_count + ), + (2, 2) + ); +} + +#[tokio::test] +async fn deferred_stats_integrity_failure_is_reported_and_isolates_storage() { + let root = tempdir().unwrap(); + let broker = Broker::open(BrokerConfig { + data_path: root.path().into(), + ..BrokerConfig::default() + }) + .unwrap(); + broker.create_channel("events", "workers").await.unwrap(); + broker + .publish("events", vec![b"ready".to_vec()], Duration::ZERO) + .await + .unwrap(); + broker + .publish("events", vec![b"delayed".to_vec()], Duration::from_secs(60)) + .await + .unwrap(); + let handle = broker.topic("events").unwrap(); + handle.state.lock().spill_message_metadata().unwrap(); + let index = std::fs::read_dir( + root.path() + .join("topics") + .join(hex::encode("events")) + .join("segments"), + ) + .unwrap() + .map(|entry| entry.unwrap().path()) + .find(|path| { + path.extension() + .is_some_and(|extension| extension == "rqidx") + }) + .unwrap(); + let mut bytes = std::fs::read(&index).unwrap(); + *bytes.last_mut().unwrap() ^= 1; + std::fs::write(&index, bytes).unwrap(); + assert!(matches!(broker.stats(), Err(BrokerError::InvalidRecord(_)))); + assert!(!broker.storage_healthy()); } #[tokio::test] @@ -1577,9 +1947,12 @@ async fn deferred_stats_promote_messages_when_the_deadline_passes() { ) .await .unwrap(); - assert_eq!(broker.stats().topics[0].channels[0].deferred_count, 1); + assert_eq!( + broker.stats().unwrap().topics[0].channels[0].deferred_count, + 1 + ); tokio::time::sleep(Duration::from_millis(40)).await; - let channel = &broker.stats().topics[0].channels[0]; + let channel = &broker.stats().unwrap().topics[0].channels[0]; assert_eq!((channel.depth, channel.deferred_count), (1, 0)); } @@ -1614,6 +1987,7 @@ async fn bounded_gc_rotates_across_topics() { assert_eq!( broker .stats() + .unwrap() .topics .iter() .map(|topic| topic.message_count) @@ -1624,6 +1998,7 @@ async fn bounded_gc_rotates_across_topics() { assert_eq!( broker .stats() + .unwrap() .topics .iter() .map(|topic| topic.message_count) @@ -1655,7 +2030,7 @@ async fn delivery_budget_bounds_slow_consumers_and_cancellation_releases_reserva .unwrap(); let (alpha_messages, alpha_hold) = alpha.into_parts(); assert_eq!(alpha_messages.len(), 1); - assert_eq!(broker.stats().delivery_budget.in_flight_bytes, 64); + assert_eq!(broker.stats().unwrap().delivery_budget.in_flight_bytes, 64); let blocked = tokio::time::timeout( Duration::from_millis(20), @@ -1663,8 +2038,8 @@ async fn delivery_budget_bounds_slow_consumers_and_cancellation_releases_reserva ) .await; assert!(blocked.is_err()); - assert_eq!(broker.stats().delivery_budget.waiters, 0); - assert!(broker.stats().delivery_budget.waits_total >= 1); + assert_eq!(broker.stats().unwrap().delivery_budget.waiters, 0); + assert!(broker.stats().unwrap().delivery_budget.waits_total >= 1); drop(alpha_messages); drop(alpha_hold); @@ -1676,7 +2051,7 @@ async fn delivery_budget_bounds_slow_consumers_and_cancellation_releases_reserva assert_eq!(beta_messages.len(), 1); drop(beta_messages); drop(beta_hold); - assert_eq!(broker.stats().delivery_budget.in_flight_bytes, 0); + assert_eq!(broker.stats().unwrap().delivery_budget.in_flight_bytes, 0); } #[tokio::test] @@ -1763,6 +2138,7 @@ async fn broker_metadata_budget_spills_active_tails_across_topics() { assert_eq!( broker .stats() + .unwrap() .topics .iter() .map(|topic| topic.message_count) @@ -1770,12 +2146,25 @@ async fn broker_metadata_budget_spills_active_tails_across_topics() { 3_200 ); assert!(broker.inner.message_index_cache.resident_bytes() <= 64 * 1024); + let active_metadata_allocations: usize = broker + .inner + .topics + .read() + .values() + .map(|topic| topic.state.lock().active_metadata_allocated_bytes()) + .sum(); + assert!( + active_metadata_allocations <= config.message_index_cache_bytes, + "active metadata buffers retain {active_metadata_allocations} bytes with a {} byte budget", + config.message_index_cache_bytes, + ); drop(broker); let broker = Broker::open(config).unwrap(); assert_eq!( broker .stats() + .unwrap() .topics .iter() .map(|topic| topic.message_count) @@ -1804,7 +2193,7 @@ async fn metadata_spill_wakes_a_write_ack_consumer() { .await .unwrap(); assert_eq!( - broker.filtered_stats(Some("events"), None).topics[0].unsynced_messages, + broker.filtered_stats(Some("events"), None).unwrap().topics[0].unsynced_messages, 64 ); @@ -1828,7 +2217,9 @@ async fn metadata_spill_wakes_a_write_ack_consumer() { .publish("other", vec![vec![2u8]; 1_024], Duration::ZERO) .await .unwrap(); - let stats = broker.filtered_stats(Some("events"), Some("workers")); + let stats = broker + .filtered_stats(Some("events"), Some("workers")) + .unwrap(); assert_eq!(stats.topics[0].unsynced_messages, 0); assert_eq!(stats.topics[0].last_durable_position, 64); assert_eq!(stats.topics[0].channels[0].depth, 64); @@ -1871,6 +2262,7 @@ async fn concurrent_publishes_wait_for_metadata_spill_instead_of_rejecting() { assert_eq!( broker .stats() + .unwrap() .topics .iter() .map(|topic| topic.message_count) @@ -1969,7 +2361,7 @@ async fn waiting_fetch_rechecks_channel_fence_before_reserving() { .unwrap(), Err(BrokerError::ChannelTombstoned) )); - let channel = &broker.stats().topics[0].channels[0]; + let channel = &broker.stats().unwrap().topics[0].channels[0]; assert_eq!(channel.depth, 1); assert_eq!(channel.in_flight_count, 0); } @@ -2330,7 +2722,7 @@ async fn management_rejects_stale_registry_revisions() { actual } if expected == stale && actual == created.revision )); - assert!(!broker.stats().topics[0].paused); + assert!(!broker.stats().unwrap().topics[0].paused); } #[tokio::test] @@ -2872,7 +3264,7 @@ async fn ttl_discard_removes_unrouted_backlog_across_restart() { .await .unwrap() .is_empty()); - let topic = &broker.stats().topics[0]; + let topic = &broker.stats().unwrap().topics[0]; assert_eq!(topic.message_count, 0); assert_eq!(topic.ttl_discarded_messages, 1); } @@ -2902,7 +3294,7 @@ async fn ttl_policy_changes_apply_to_backlog_without_reviving_expired_messages() ) .await .unwrap(); - assert_eq!(broker.stats().topics[0].message_count, 1); + assert_eq!(broker.stats().unwrap().topics[0].message_count, 1); broker .configure_topic_policy( @@ -2955,7 +3347,7 @@ async fn ttl_policy_changes_apply_to_backlog_without_reviving_expired_messages() .await .unwrap() .is_none()); - assert_eq!(broker.stats().topics[0].ttl_discarded_messages, 1); + assert_eq!(broker.stats().unwrap().topics[0].ttl_discarded_messages, 1); } #[tokio::test] @@ -3098,7 +3490,7 @@ async fn ttl_discard_evicts_durable_ephemeral_deferred_requeued_and_in_flight_st .await .unwrap() .remove(0); - assert!(broker.stats().topics[0] + assert!(broker.stats().unwrap().topics[0] .channels .iter() .any(|channel| channel.deferred_count > 0)); @@ -3106,7 +3498,7 @@ async fn ttl_discard_evicts_durable_ephemeral_deferred_requeued_and_in_flight_st tokio::time::sleep(Duration::from_millis(1_100)).await; while !broker.expire_due_topics(128).await.unwrap().is_empty() {} - let topic = &broker.stats().topics[0]; + let topic = &broker.stats().unwrap().topics[0]; assert_eq!(topic.message_count, 0); assert_eq!(topic.ttl_discarded_messages, 3); assert!(topic.channels.iter().all(|channel| { diff --git a/crates/queue/src/channel.rs b/crates/queue/src/channel.rs index bff5b90..b7d94b6 100644 --- a/crates/queue/src/channel.rs +++ b/crates/queue/src/channel.rs @@ -3,9 +3,14 @@ use crate::model::ChannelStats; use crate::BrokerError; use serde::{Deserialize, Serialize}; use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; use std::time::{Duration, Instant}; +// Old sessions and guards can outlive deletion of a Channel. Tokens must stay +// distinct when that Channel name is recreated and delivers the same message. +static NEXT_DELIVERY_TOKEN: AtomicU64 = AtomicU64::new(1); + #[derive(Clone, Debug, Deserialize, Serialize)] pub(crate) enum ChannelCommand { Finish { @@ -59,6 +64,7 @@ struct InFlight { timestamp_ns: i64, deadline: Instant, token: u64, + _lease: Option, } pub(crate) struct ChannelRuntime { @@ -77,12 +83,14 @@ pub(crate) struct ChannelState { pub ephemeral: bool, message_count_origin_position: u64, next_position: u64, + // DPUB deadlines already live in shared Topic metadata. One rewind point + // avoids copying every skipped deadline into each Channel's requeue state. + deferred_scan: Option<(u64, i64)>, in_flight: HashMap, in_flight_ids: HashMap, in_flight_deadlines: BTreeSet<(Instant, u64, u64)>, redelivery: BTreeSet, attempts: HashMap, - next_token: u64, max_ack_gap: usize, requeue_count: u64, timeout_count: u64, @@ -115,12 +123,12 @@ impl ChannelState { ephemeral, message_count_origin_position: barrier_position, next_position: barrier_position.saturating_add(1), + deferred_scan: None, in_flight: HashMap::new(), in_flight_ids: HashMap::new(), in_flight_deadlines: BTreeSet::new(), redelivery: BTreeSet::new(), attempts: HashMap::new(), - next_token: 1, max_ack_gap: max_ack_gap.max(1), requeue_count: 0, timeout_count: 0, @@ -155,12 +163,12 @@ impl ChannelState { ephemeral: checkpoint.ephemeral, message_count_origin_position, next_position, + deferred_scan: None, in_flight: HashMap::new(), in_flight_ids: HashMap::new(), in_flight_deadlines: BTreeSet::new(), redelivery, attempts: checkpoint.attempts.into_iter().collect(), - next_token: 1, max_ack_gap: max_ack_gap.max(1), requeue_count: checkpoint.requeue_count, timeout_count: checkpoint.timeout_count, @@ -168,6 +176,28 @@ impl ChannelState { }) } + pub(crate) fn compact_recovered_dpubs( + &mut self, + mut available_at: impl FnMut(u64) -> Result, BrokerError>, + ) -> Result<(), BrokerError> { + let mut cursor = std::ops::Bound::Unbounded; + while let Some((position, until)) = self + .requeued_until + .range((cursor, std::ops::Bound::Unbounded)) + .next() + .map(|(position, until)| (*position, *until)) + { + cursor = std::ops::Bound::Excluded(position); + // Old DPUB scans copied the immutable deadline without recording + // a delivery attempt. Explicit REQ state keeps its own deadline. + if !self.attempts.contains_key(&position) && available_at(position)? == Some(until) { + self.requeued_until.remove(&position); + self.redelivery.remove(&position); + } + } + Ok(()) + } + pub fn checkpoint(&self) -> ChannelCheckpoint { ChannelCheckpoint { format: 7, @@ -249,9 +279,22 @@ impl ChannelState { if self.paused { return NextCandidate::None; } - let mut absent = Vec::new(); - let redelivery: Vec<_> = self.redelivery.iter().copied().collect(); - for position in redelivery { + if let Some((first, until)) = self.deferred_scan { + if until <= now_ms { + self.next_position = self + .next_position + .min(first.max(self.ack_floor_position.saturating_add(1))); + self.deferred_scan = None; + } + } + let mut cursor = std::ops::Bound::Unbounded; + while let Some(position) = self + .redelivery + .range((cursor, std::ops::Bound::Unbounded)) + .next() + .copied() + { + cursor = std::ops::Bound::Excluded(position); if self.is_absent(position) { self.redelivery.remove(&position); self.requeued_until.remove(&position); @@ -276,16 +319,11 @@ impl ChannelState { } MessageAvailability::Missing => return NextCandidate::Load(position), MessageAvailability::Absent => { - absent.push(position); self.acknowledge(position); } MessageAvailability::Ready(_) => {} } } - for position in absent { - self.redelivery.remove(&position); - self.requeued_until.remove(&position); - } while self.next_position <= last_position { if let Some(end) = self.absent_range_end(self.next_position) { self.next_position = end.saturating_add(1); @@ -315,8 +353,10 @@ impl ChannelState { }; self.next_position = self.next_position.saturating_add(1); if available > now_ms { - self.redelivery.insert(position); - self.requeued_until.insert(position, available); + self.deferred_scan = Some(match self.deferred_scan { + Some((first, until)) => (first.min(position), until.min(available)), + None => (position, available), + }); continue; } return NextCandidate::Ready(position); @@ -355,7 +395,7 @@ impl ChannelState { #[cfg(test)] pub fn reserve(&mut self, position: u64, id: u64, timeout: Duration) -> (u64, u16) { - self.reserve_timestamped(position, id, 0, timeout) + self.reserve_timestamped(position, id, 0, timeout, None) } pub fn reserve_timestamped( @@ -364,9 +404,9 @@ impl ChannelState { id: u64, timestamp_ns: i64, timeout: Duration, + lease: Option, ) -> (u64, u16) { - let token = self.next_token; - self.next_token = self.next_token.wrapping_add(1).max(1); + let token = NEXT_DELIVERY_TOKEN.fetch_add(1, Ordering::Relaxed); let attempts = self.attempts.entry(position).or_insert(0); *attempts = attempts.saturating_add(1); let deadline = Instant::now() + timeout; @@ -377,6 +417,7 @@ impl ChannelState { timestamp_ns, deadline, token, + _lease: lease, }, ); self.in_flight_deadlines.insert((deadline, position, token)); @@ -500,11 +541,11 @@ impl ChannelState { pub fn stats( &self, last_position: u64, - scheduled: &BTreeSet, + scheduled_without_requeues: u64, now_ms: i64, ) -> ChannelStats { let (depth, in_flight_count, deferred_count, ack_gap) = - self.metric_counts(last_position, scheduled, now_ms); + self.metric_counts(last_position, scheduled_without_requeues, now_ms); ChannelStats { name: self.name.clone(), depth, @@ -523,7 +564,7 @@ impl ChannelState { pub fn metric_counts( &self, last_position: u64, - scheduled: &BTreeSet, + scheduled_without_requeues: u64, now_ms: i64, ) -> (u64, u64, u64, u64) { let total = last_position @@ -531,27 +572,51 @@ impl ChannelState { .saturating_sub( self.absent_count(self.ack_floor_position.saturating_add(1), last_position), ); - let scheduled_count = scheduled - .iter() - .filter(|position| self.is_outstanding(**position, last_position)) - .count() as u64; let requeued_count = self .requeued_until .iter() .filter(|(position, until)| { - **until > now_ms - && !scheduled.contains(position) - && self.is_outstanding(**position, last_position) + **until > now_ms && self.is_outstanding(**position, last_position) }) .count() as u64; ( total.saturating_sub(self.acknowledged.len() as u64), self.in_flight.len() as u64, - scheduled_count.saturating_add(requeued_count), + scheduled_without_requeues.saturating_add(requeued_count), self.acknowledged.len() as u64, ) } + pub(crate) fn scheduled_count_in_range(&self, first: u64, last: u64, now_ms: i64) -> u64 { + let first = first.max(self.ack_floor_position.saturating_add(1)); + if first > last || last <= self.ack_floor_position { + return 0; + } + let acknowledged = self + .acknowledged + .range(first..=last) + .filter(|position| !self.is_absent(**position)) + .count() as u64; + let in_flight = self + .in_flight + .keys() + .filter(|position| (first..=last).contains(position) && !self.is_absent(**position)) + .count() as u64; + // Legacy checkpoints can contain DPUB scan deadlines. Exclude these + // and any overlapping explicit REQ deadline from the Topic count. + let requeued = self + .requeued_until + .range(first..=last) + .filter(|(position, until)| **until > now_ms && self.is_outstanding(**position, last)) + .count() as u64; + last.saturating_sub(first) + .saturating_add(1) + .saturating_sub(self.absent_count(first, last)) + .saturating_sub(acknowledged) + .saturating_sub(in_flight) + .saturating_sub(requeued) + } + fn is_outstanding(&self, position: u64, last_position: u64) -> bool { position > self.ack_floor_position && position <= last_position @@ -685,11 +750,128 @@ impl ChannelRuntime { mod tests { use super::*; + #[test] + fn deferred_scan_window_stays_compact_and_preserves_requeue_after_recovery() { + let last = 65_536; + let availability = + |position| MessageAvailability::Ready(if position == last { 150 } else { 200 }); + let mut channel = ChannelState::new("workers".into(), 0, false, last as usize); + assert!(matches!( + channel.next_candidate(100, last, availability), + NextCandidate::None + )); + let checkpoint = serde_json::to_vec(&channel.checkpoint()).unwrap(); + assert!( + checkpoint.len() < 1024, + "scanning DPUB must not expand the Channel checkpoint: {} bytes", + checkpoint.len() + ); + let mut channel = ChannelState::from_checkpoint( + serde_json::from_slice(&checkpoint).unwrap(), + last as usize, + ) + .unwrap(); + assert!(matches!( + channel.next_candidate(100, last, availability), + NextCandidate::None + )); + assert!(matches!( + channel.next_candidate(150, last, availability), + NextCandidate::Ready(position) if position == last + )); + channel.reserve(last, last, Duration::from_secs(30)); + channel.apply(&ChannelCommand::Requeue { + position: last, + message_id: last, + available_at_ms: 250, + attempts: 1, + cumulative_count: Some(1), + }); + let mut channel = + ChannelState::from_checkpoint(channel.checkpoint(), last as usize).unwrap(); + assert!(matches!( + channel.next_candidate(150, last, availability), + NextCandidate::None + )); + for position in 1..last { + assert!(matches!( + channel.next_candidate(200, last, availability), + NextCandidate::Ready(candidate) if candidate == position + )); + channel.reserve(position, position, Duration::from_secs(30)); + channel.apply(&ChannelCommand::Finish { + position, + message_id: position, + }); + } + assert!(matches!( + channel.next_candidate(249, last, availability), + NextCandidate::None + )); + assert!(matches!( + channel.next_candidate(250, last, availability), + NextCandidate::Ready(position) if position == last + )); + assert_eq!(channel.reserve(last, last, Duration::from_secs(30)).1, 2); + channel.apply(&ChannelCommand::Finish { + position: last, + message_id: last, + }); + let stats = channel.stats(last, 0, 250); + assert_eq!( + (stats.ack_cursor, stats.depth, stats.requeue_count), + (last, 0, 1) + ); + } + + #[test] + fn deferred_scan_rewinds_across_a_cache_miss_and_empty() { + let mut channel = ChannelState::new("workers".into(), 10, false, 8); + assert!(matches!( + channel.next_candidate(0, 14, |position| match position { + 11 => MessageAvailability::Ready(30), + 12 => MessageAvailability::Missing, + _ => panic!("scan must stop at the cache miss"), + }), + NextCandidate::Load(12) + )); + assert!(matches!( + channel.next_candidate(35, 14, |position| match position { + 11 => MessageAvailability::Ready(30), + _ => panic!("due message must be revisited before the missing page"), + }), + NextCandidate::Ready(11) + )); + channel.reserve(11, 11, Duration::from_secs(30)); + channel.apply(&ChannelCommand::Finish { + position: 11, + message_id: 11, + }); + assert!(matches!( + channel.next_candidate(35, 14, |position| match position { + 12 => MessageAvailability::Ready(40), + 13 => MessageAvailability::Ready(0), + _ => panic!("scan must stop at the ready message"), + }), + NextCandidate::Ready(13) + )); + channel.apply(&ChannelCommand::Empty { + through_position: 13, + }); + assert!(matches!( + channel.next_candidate(45, 14, |position| { + assert_eq!(position, 14); + MessageAvailability::Ready(0) + }), + NextCandidate::Ready(14) + )); + } + #[test] fn recovered_position_gaps_do_not_consume_the_ack_window() { let mut channel = ChannelState::new("workers".into(), 0, false, 2); channel.set_absent_ranges(Arc::from(vec![(2, 10)])); - let stats = channel.stats(11, &BTreeSet::new(), i64::MAX); + let stats = channel.stats(11, 0, i64::MAX); assert_eq!(stats.depth, 2); assert_eq!(stats.message_count, 11); @@ -718,7 +900,7 @@ mod tests { message_id: 100, }); - let stats = channel.stats(11, &BTreeSet::new(), i64::MAX); + let stats = channel.stats(11, 0, i64::MAX); assert_eq!(stats.ack_cursor, 11); assert_eq!(stats.ack_gap, 0); assert_eq!(stats.depth, 0); @@ -763,10 +945,7 @@ mod tests { assert_eq!(runtime.expire_in_flight().unwrap(), 1); let channel = runtime.state; assert_eq!(channel.in_flight_position(10), None); - assert_eq!( - channel.stats(1, &BTreeSet::new(), i64::MAX).timeout_count, - 1 - ); + assert_eq!(channel.stats(1, 0, i64::MAX).timeout_count, 1); } #[test] @@ -807,13 +986,13 @@ mod tests { runtime.state.apply(&ChannelCommand::Empty { through_position: 2, }); - let current = runtime.state.stats(2, &BTreeSet::new(), i64::MAX); + let current = runtime.state.stats(2, 0, i64::MAX); assert_eq!(current.message_count, 2); assert_eq!(current.requeue_count, 1); assert_eq!(current.timeout_count, 1); let recovered = ChannelState::from_checkpoint(runtime.state.checkpoint(), 16).unwrap(); - let restarted = recovered.stats(4, &BTreeSet::new(), i64::MAX); + let restarted = recovered.stats(4, 0, i64::MAX); assert_eq!(restarted.message_count, 4); assert_eq!(restarted.requeue_count, 1); assert_eq!(restarted.timeout_count, 1); @@ -833,10 +1012,10 @@ mod tests { })) .unwrap(); let mut state = ChannelState::from_checkpoint(checkpoint, 16).unwrap(); - assert_eq!(state.stats(7, &BTreeSet::new(), i64::MAX).message_count, 2); + assert_eq!(state.stats(7, 0, i64::MAX).message_count, 2); state.apply(&ChannelCommand::Empty { through_position: 7, }); - assert_eq!(state.stats(7, &BTreeSet::new(), i64::MAX).message_count, 2); + assert_eq!(state.stats(7, 0, i64::MAX).message_count, 2); } } diff --git a/crates/queue/src/channel_store.rs b/crates/queue/src/channel_store.rs index 866816a..d2bd1a6 100644 --- a/crates/queue/src/channel_store.rs +++ b/crates/queue/src/channel_store.rs @@ -524,7 +524,7 @@ mod tests { .path() .join(format!("{}.checkpoint", hex::encode("workers"))); let (state, _) = ChannelStore::open(&checkpoint, 65_536).unwrap(); - let stats = state.stats(1, &Default::default(), i64::MAX); + let stats = state.stats(1, 0, i64::MAX); assert_eq!(stats.requeue_count, 1); assert_eq!(stats.timeout_count, 2); } @@ -558,7 +558,7 @@ mod tests { .path() .join(format!("{}.checkpoint", hex::encode("workers"))); let (mut recovered, _) = ChannelStore::open(&checkpoint, 65_536).unwrap(); - let stats = recovered.stats(1, &Default::default(), 0); + let stats = recovered.stats(1, 0, 0); assert_eq!(stats.requeue_count, 1); assert_eq!(stats.timeout_count, 2); assert!(matches!( diff --git a/crates/queue/src/delivery_budget.rs b/crates/queue/src/delivery_budget.rs index a30e635..bffca50 100644 --- a/crates/queue/src/delivery_budget.rs +++ b/crates/queue/src/delivery_budget.rs @@ -5,6 +5,13 @@ use tokio::sync::{OwnedSemaphorePermit, Semaphore}; const PAYLOAD_READ_WORKING_SET_FACTOR: usize = 2; +/// Keeps one node-wide metadata slot occupied until both the queue reservation +/// and any consumer-side bookkeeping have been released. +#[derive(Clone, Debug)] +pub struct DeliveryLease { + _permit: Arc, +} + pub struct DeliveryHold { _permit: Option, bytes: u64, @@ -14,16 +21,20 @@ pub struct DeliveryHold { pub(crate) struct DeliveryBudget { capacity: usize, permits: Arc, + message_capacity: usize, + messages: Arc, in_flight: Arc, waiters: Arc, waits_total: AtomicU64, } impl DeliveryBudget { - pub(crate) fn new(capacity: usize) -> Self { + pub(crate) fn new(capacity: usize, message_capacity: usize) -> Self { Self { capacity, permits: Arc::new(Semaphore::new(capacity)), + message_capacity, + messages: Arc::new(Semaphore::new(message_capacity)), in_flight: Arc::new(AtomicU64::new(0)), waiters: Arc::new(AtomicUsize::new(0)), waits_total: AtomicU64::new(0), @@ -34,6 +45,15 @@ impl DeliveryBudget { self.capacity / PAYLOAD_READ_WORKING_SET_FACTOR } + pub(crate) fn try_reserve_message(&self) -> Option { + Arc::clone(&self.messages) + .try_acquire_owned() + .ok() + .map(|permit| DeliveryLease { + _permit: Arc::new(permit), + }) + } + pub(crate) async fn acquire(&self, payload_bytes: usize) -> Result { if payload_bytes == 0 { return Ok(DeliveryHold { @@ -82,6 +102,10 @@ impl DeliveryBudget { pub(crate) fn snapshot(&self) -> crate::model::DeliveryBudgetStats { crate::model::DeliveryBudgetStats { in_flight_bytes: self.in_flight.load(Ordering::Acquire), + in_flight_messages: self + .message_capacity + .saturating_sub(self.messages.available_permits()) + as u64, waiters: self.waiters.load(Ordering::Acquire) as u64, waits_total: self.waits_total.load(Ordering::Relaxed), } @@ -115,7 +139,7 @@ mod tests { #[tokio::test] async fn oversized_request_fails_instead_of_waiting_forever() { - let budget = DeliveryBudget::new(8); + let budget = DeliveryBudget::new(8, 1); let error = match budget.acquire(5).await { Ok(_) => panic!("oversized delivery request was accepted"), Err(error) => error, diff --git a/crates/queue/src/delivery_guard.rs b/crates/queue/src/delivery_guard.rs index 3bfd84f..89c371a 100644 --- a/crates/queue/src/delivery_guard.rs +++ b/crates/queue/src/delivery_guard.rs @@ -1,4 +1,4 @@ -use crate::delivery_budget::DeliveryHold; +use crate::delivery_budget::{DeliveryHold, DeliveryLease}; use crate::model::ReservedDelivery; use crate::topic::TopicHandle; use std::sync::Arc; @@ -39,12 +39,19 @@ impl DeliveryGuard { } pub fn accept_with_token(&mut self, id: u64) -> Option { + self.accept_with_lease(id).map(|(token, _lease)| token) + } + + /// Transfers the consumer's metadata hold together with the delivery token. + /// Keep the lease until FIN/REQ completion, timeout, or disconnect cleanup. + pub fn accept_with_lease(&mut self, id: u64) -> Option<(u64, DeliveryLease)> { if let Some(index) = self .reservations .iter() .position(|reservation| reservation.id == id) { - return Some(self.reservations.swap_remove(index).token); + let reservation = self.reservations.swap_remove(index); + return Some((reservation.token, reservation.lease)); } None } diff --git a/crates/queue/src/lib.rs b/crates/queue/src/lib.rs index a3b6b39..b7bd1c9 100644 --- a/crates/queue/src/lib.rs +++ b/crates/queue/src/lib.rs @@ -18,7 +18,7 @@ pub use broker::{ Broker, BrokerConfig, BrokerError, InvalidPublishAckMode, PublishAckMode, RELAXED_SYNC_MIN_BYTES, }; -pub use delivery_budget::DeliveryHold; +pub use delivery_budget::{DeliveryHold, DeliveryLease}; pub use delivery_guard::DeliveryGuard; pub use eviction::ProtectiveEviction; pub use management::{ diff --git a/crates/queue/src/model.rs b/crates/queue/src/model.rs index 5fafe79..38142fb 100644 --- a/crates/queue/src/model.rs +++ b/crates/queue/src/model.rs @@ -83,6 +83,8 @@ impl DeliveryBatch { #[derive(Clone, Debug, Default, Deserialize, Serialize)] pub struct DeliveryBudgetStats { pub in_flight_bytes: u64, + #[serde(default)] + pub in_flight_messages: u64, pub waiters: u64, pub waits_total: u64, } @@ -244,5 +246,6 @@ pub(crate) struct ReservedDelivery { pub timestamp_ns: i64, pub attempts: u16, pub token: u64, + pub lease: crate::delivery_budget::DeliveryLease, pub payload: PayloadRef, } diff --git a/crates/queue/src/payload_reader.rs b/crates/queue/src/payload_reader.rs index 64417b6..293da28 100644 --- a/crates/queue/src/payload_reader.rs +++ b/crates/queue/src/payload_reader.rs @@ -573,7 +573,7 @@ mod tests { }; let (reader, _) = reader(1, 1); let lease = reader.retain(vec![payload]); - let budget = DeliveryBudget::new(2); + let budget = DeliveryBudget::new(2, 1); let hold = budget.acquire(1).await.unwrap(); assert_eq!(budget.snapshot().in_flight_bytes, 2); diff --git a/crates/queue/src/topic.rs b/crates/queue/src/topic.rs index 9016d4a..28d813f 100644 --- a/crates/queue/src/topic.rs +++ b/crates/queue/src/topic.rs @@ -300,12 +300,44 @@ impl Topic { } fn recover_channels(&mut self) -> Result<(), BrokerError> { + let mut channel_next = 1u64; for path in checkpoint_paths(&self.directory.join("channels"))? { - let (state, store) = ChannelStore::open(&path, self.max_ack_gap)?; + let (mut state, store) = ChannelStore::open(&path, self.max_ack_gap)?; if state.ephemeral { store.remove()?; continue; } + // Fence recorded positions before removing lost relaxed entries. + let recorded_last = state.recovered_position_high_watermark(); + channel_next = channel_next.max(recorded_last.checked_add(1).ok_or_else(|| { + BrokerError::InvalidRecord("channel position range is exhausted".into()) + })?); + let last = recorded_last.max(self.manifest.next_position.saturating_sub(1)); + state.set_absent_ranges(Arc::from(self.messages.position_gaps(last))); + let mut page_key = None; + let mut metadata_page = Vec::new(); + state.compact_recovered_dpubs(|position| match self.messages.lookup(position) { + index::Lookup::Found(message) => Ok(Some(message.available_at_ms)), + index::Lookup::Absent => Ok(None), + index::Lookup::Load(request) => { + if page_key.as_ref() != Some(&request.key) { + metadata_page = recovery::read_page( + &request.metadata, + request.first_ordinal, + request.count, + )?; + page_key = Some(request.key); + } + let ordinal = metadata_page + .binary_search_by_key(&position, |message| message.position) + .map_err(|_| { + BrokerError::InvalidRecord( + "recovered channel metadata position is missing".into(), + ) + })?; + Ok(Some(metadata_page[ordinal].available_at_ms)) + } + })?; let name = state.name.clone(); if self.channels.contains_key(&name) { return Err(BrokerError::InvalidRecord(format!( @@ -325,16 +357,6 @@ impl Topic { .messages .last_position() .map_or(1, |position| position.saturating_add(1)); - let channel_next = self - .channels - .values() - .map(|channel| channel.state.recovered_position_high_watermark()) - .max() - .unwrap_or(0) - .checked_add(1) - .ok_or_else(|| { - BrokerError::InvalidRecord("channel position range is exhausted".into()) - })?; if self.manifest.next_position < channel_next { self.manifest.next_position = channel_next; store_atomic(&self.manifest_path, &self.manifest)?; @@ -480,6 +502,11 @@ impl Topic { self.messages.active_count() } + #[cfg(test)] + pub(crate) fn active_metadata_allocated_bytes(&self) -> usize { + self.messages.active_metadata_allocated_bytes() + } + fn persist_segment_index(&mut self, path: &Path) -> Result<(), BrokerError> { let (_first_index, _last_index) = self .log @@ -570,7 +597,6 @@ impl Topic { self.persist_channel(&channel, ChannelCommand::Evict { through_position })?; } } - self.messages.discard_scheduled_through(through_position); Ok(()) } @@ -720,16 +746,21 @@ impl Topic { pub fn channel_counts(&mut self, channel: &str) -> Result<(u64, u64, u64), BrokerError> { let now_ms = now_ms(); - let scheduled = self.messages.deferred_positions(now_ms); let last_position = self.deliverable_position; let channel = self .channels .get(channel) .ok_or(BrokerError::ChannelNotFound)?; + let mut scheduled = 0u64; + self.messages + .for_each_deferred_range(now_ms, last_position, |first, last| { + scheduled = scheduled + .saturating_add(channel.state.scheduled_count_in_range(first, last, now_ms)); + })?; let (depth, in_flight, deferred, _) = channel .state - .metric_counts(last_position, &scheduled, now_ms); + .metric_counts(last_position, scheduled, now_ms); Ok((depth, in_flight, deferred)) } diff --git a/crates/queue/src/topic/delivery.rs b/crates/queue/src/topic/delivery.rs index a2a02f4..7c8bf74 100644 --- a/crates/queue/src/topic/delivery.rs +++ b/crates/queue/src/topic/delivery.rs @@ -19,6 +19,7 @@ impl Topic { max_messages: usize, max_bytes: usize, timeout: Duration, + budget: &crate::delivery_budget::DeliveryBudget, ) -> Result { if self.manifest.deleted { return Err(BrokerError::TopicNotFound); @@ -60,11 +61,16 @@ impl Topic { channel.state.defer_candidate(position); break; } + let Some(lease) = budget.try_reserve_message() else { + channel.state.defer_candidate(position); + break; + }; let (token, attempts) = channel.state.reserve_timestamped( position, message.id, message.timestamp_ns, timeout, + Some(lease.clone()), ); bytes = bytes.saturating_add(next_bytes); reserved.push(ReservedDelivery { @@ -73,6 +79,7 @@ impl Topic { timestamp_ns: message.timestamp_ns, attempts, token, + lease, payload: message.payload.clone(), }); } @@ -264,7 +271,8 @@ impl Topic { .collect::>()?; let deadline = Instant::now() + timeout; for position in positions { - debug_assert!(channel.state.touch_until(position, deadline)); + let touched = channel.state.touch_until(position, deadline); + debug_assert!(touched); } Ok(()) } diff --git a/crates/queue/src/topic/index.rs b/crates/queue/src/topic/index.rs index 625fd60..ade19d6 100644 --- a/crates/queue/src/topic/index.rs +++ b/crates/queue/src/topic/index.rs @@ -4,7 +4,7 @@ use super::recovery; use crate::model::MessageMeta; use crate::BrokerError; use rustqueue_storage::RecoveryMetadataRef; -use std::collections::{BTreeMap, BTreeSet, VecDeque}; +use std::collections::{BTreeSet, VecDeque}; use std::path::{Path, PathBuf}; use std::sync::Arc; @@ -27,12 +27,13 @@ struct SealedMessages { first_timestamp_ns: i64, last_timestamp_ns: i64, last_log_index: u64, + min_available_at_ms: i64, + max_available_at_ms: i64, } pub(crate) struct MessageIndex { sealed: VecDeque, active: VecDeque, - scheduled: BTreeMap>, total_count: u64, cache: Arc, } @@ -42,7 +43,6 @@ impl MessageIndex { Self { sealed: VecDeque::new(), active: VecDeque::new(), - scheduled: BTreeMap::new(), total_count: 0, cache, } @@ -52,11 +52,8 @@ impl MessageIndex { &mut self, metadata: RecoveryMetadataRef, ) -> Result<(), BrokerError> { - let summary = recovery::inspect(&metadata, unix_ms())?; - let scheduled = summary.scheduled.clone(); - self.push_sealed(SealedMessages::from_summary(metadata, summary))?; - self.extend_scheduled(scheduled); - Ok(()) + let summary = recovery::inspect(&metadata)?; + self.push_sealed(SealedMessages::from_summary(metadata, summary)) } pub(crate) fn recover_active(&mut self, messages: Vec) -> Result<(), BrokerError> { @@ -102,8 +99,18 @@ impl MessageIndex { .find(|message| message.payload.path.as_ref() == path) .cloned() .expect("non-empty metadata"); + let (min_available_at_ms, max_available_at_ms) = + self.active_for_path(path) + .fold((i64::MAX, i64::MIN), |(min, max), message| { + ( + min.min(message.available_at_ms), + max.max(message.available_at_ms), + ) + }); self.active .retain(|message| message.payload.path.as_ref() != path); + // Cold topics must not retain uncharged buffers after their metadata spills. + self.active.shrink_to_fit(); self.cache.release_active(count); self.total_count = self.total_count.saturating_sub(count as u64); self.push_sealed(SealedMessages { @@ -116,6 +123,8 @@ impl MessageIndex { first_timestamp_ns: first.timestamp_ns, last_timestamp_ns: last.timestamp_ns, last_log_index: last.log_index, + min_available_at_ms, + max_available_at_ms, }) } @@ -199,25 +208,53 @@ impl MessageIndex { self.total_count } - pub(crate) fn deferred_positions(&mut self, now_ms: i64) -> BTreeSet { - self.scheduled = self.scheduled.split_off(&now_ms.saturating_add(1)); - self.scheduled - .values() - .flat_map(|positions| positions.iter().copied()) - .collect() - } - - pub(crate) fn discard_scheduled_through(&mut self, through_position: u64) { - self.scheduled.retain(|_, positions| { - positions.retain(|position| *position > through_position); - !positions.is_empty() - }); + pub(crate) fn for_each_deferred_range( + &self, + now_ms: i64, + last_position: u64, + mut visit: impl FnMut(u64, u64), + ) -> Result<(), BrokerError> { + for segment in &self.sealed { + if segment.first_position > last_position { + break; + } + if segment.max_available_at_ms <= now_ms { + continue; + } + let last = segment.last_position.min(last_position); + if segment.min_available_at_ms > now_ms { + visit(segment.first_position, last); + continue; + } + // Mixed deadlines require one bounded page, never a backlog-sized set. + let count = last - segment.first_position + 1; + let mut ordinal = 0; + while ordinal < count { + let length = (count - ordinal).min(PAGE_MESSAGES) as usize; + let page = recovery::read_page(&segment.metadata, ordinal, length)?; + visit_deferred_ranges(page.iter(), now_ms, &mut visit); + ordinal += length as u64; + } + } + visit_deferred_ranges( + self.active + .iter() + .take_while(|message| message.position <= last_position), + now_ms, + &mut visit, + ); + Ok(()) } pub(crate) fn active_count(&self) -> usize { self.active.len() } + #[cfg(test)] + pub(crate) fn active_metadata_allocated_bytes(&self) -> usize { + self.active.capacity() * std::mem::size_of::() + } + pub(crate) fn active_last_position(&self) -> Option { self.active.back().map(|message| message.position) } @@ -491,14 +528,6 @@ impl MessageIndex { self.total_count = self.total_count.saturating_sub(*count); self.cache.invalidate(path); } - self.scheduled.retain(|_, positions| { - positions.retain(|position| { - !removed - .iter() - .any(|(_, _, first, last)| (*first..=*last).contains(position)) - }); - !positions.is_empty() - }); self.sealed .retain(|segment| existing.contains(segment.metadata.segment_path())); } @@ -537,26 +566,10 @@ impl MessageIndex { expected = Some(message.position.saturating_add(1)); } self.total_count = self.total_count.saturating_add(messages.len() as u64); - let now_ms = unix_ms(); - self.extend_scheduled( - messages - .iter() - .filter(|message| message.available_at_ms > now_ms) - .map(|message| (message.available_at_ms, message.position)), - ); self.active.extend(messages); Ok(()) } - fn extend_scheduled(&mut self, scheduled: impl IntoIterator) { - for (available_at_ms, position) in scheduled { - self.scheduled - .entry(available_at_ms) - .or_default() - .insert(position); - } - } - fn load_ordinal( &self, segment: &SealedMessages, @@ -602,12 +615,26 @@ impl MessageIndex { } } -fn unix_ms() -> i64 { - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap_or_default() - .as_millis() - .min(i64::MAX as u128) as i64 +fn visit_deferred_ranges<'a>( + messages: impl Iterator, + now_ms: i64, + visit: &mut impl FnMut(u64, u64), +) { + let mut range: Option<(u64, u64)> = None; + for message in messages { + if message.available_at_ms > now_ms { + if let Some((_, last)) = range.as_mut() { + *last = message.position; + } else { + range = Some((message.position, message.position)); + } + } else if let Some((first, last)) = range.take() { + visit(first, last); + } + } + if let Some((first, last)) = range { + visit(first, last); + } } impl Drop for MessageIndex { @@ -631,6 +658,8 @@ impl SealedMessages { first_timestamp_ns: summary.first.timestamp_ns, last_timestamp_ns: summary.last.timestamp_ns, last_log_index: summary.last.log_index, + min_available_at_ms: summary.min_available_at_ms, + max_available_at_ms: summary.max_available_at_ms, } } } diff --git a/crates/queue/src/topic/maintenance.rs b/crates/queue/src/topic/maintenance.rs index 8cb0478..0f16763 100644 --- a/crates/queue/src/topic/maintenance.rs +++ b/crates/queue/src/topic/maintenance.rs @@ -1,6 +1,6 @@ use super::*; use crate::eviction::{self, ProtectiveEviction}; -use crate::model::{QueueAggregateStats, TopicStats}; +use crate::model::TopicStats; use rustqueue_storage::ScrubTarget; use std::collections::BTreeSet; use std::sync::Arc; @@ -43,19 +43,31 @@ impl Topic { self.persist_channel(channel, ChannelCommand::Pause { paused }) } - pub fn stats(&mut self) -> TopicStats { + pub fn stats(&self) -> Result { let last = self.deliverable_position; let now_ms = now_ms(); - let scheduled = self.messages.deferred_positions(now_ms); let (segment_count, segment_bytes) = self.log.storage_usage(); let pending_sync = self.pending_sync(); - let mut channels: Vec<_> = self + let channels: Vec<_> = self .channels - .values_mut() - .map(|channel| channel.state.stats(last, &scheduled, now_ms)) + .values() + .map(|channel| &channel.state) + .collect(); + let mut scheduled = vec![0u64; channels.len()]; + self.messages + .for_each_deferred_range(now_ms, last, |first, last| { + for (channel, count) in channels.iter().zip(&mut scheduled) { + *count = + count.saturating_add(channel.scheduled_count_in_range(first, last, now_ms)); + } + })?; + let mut channels: Vec<_> = channels + .into_iter() + .zip(scheduled) + .map(|(channel, count)| channel.stats(last, count, now_ms)) .collect(); channels.sort_by(|left, right| left.name.cmp(&right.name)); - TopicStats { + Ok(TopicStats { name: self.name.clone(), paused: self.manifest.paused, delivery_mode: self.manifest.delivery_mode, @@ -74,38 +86,7 @@ impl Topic { pending.since.elapsed().as_millis().min(u64::MAX as u128) as u64 }), channels, - } - } - - pub fn add_aggregate_stats(&mut self, aggregate: &mut QueueAggregateStats) { - let last = self.deliverable_position; - let now_ms = now_ms(); - let scheduled = self.messages.deferred_positions(now_ms); - let (segment_count, segment_bytes) = self.log.storage_usage(); - aggregate.topic_count = aggregate.topic_count.saturating_add(1); - aggregate.message_count = aggregate.message_count.saturating_add( - self.messages - .count_after_position(self.manifest.expired_through_position), - ); - aggregate.segment_count = aggregate.segment_count.saturating_add(segment_count); - aggregate.segment_bytes = aggregate.segment_bytes.saturating_add(segment_bytes); - if let Some(pending) = self.pending_sync() { - aggregate.unsynced_messages = - aggregate.unsynced_messages.saturating_add(pending.messages); - aggregate.unsynced_bytes = aggregate.unsynced_bytes.saturating_add(pending.bytes); - aggregate.sync_lag_ms = aggregate - .sync_lag_ms - .max(pending.since.elapsed().as_millis().min(u64::MAX as u128) as u64); - } - for channel in self.channels.values_mut() { - let (depth, in_flight, deferred, ack_gap) = - channel.state.metric_counts(last, &scheduled, now_ms); - aggregate.channel_count = aggregate.channel_count.saturating_add(1); - aggregate.channel_depth = aggregate.channel_depth.saturating_add(depth); - aggregate.channel_in_flight = aggregate.channel_in_flight.saturating_add(in_flight); - aggregate.channel_deferred = aggregate.channel_deferred.saturating_add(deferred); - aggregate.channel_ack_gap = aggregate.channel_ack_gap.saturating_add(ack_gap); - } + }) } pub fn oldest_message_timestamp(&self) -> Option { @@ -284,7 +265,6 @@ impl Topic { } store_atomic(&self.manifest_path, &manifest)?; self.manifest = manifest; - self.messages.discard_scheduled_through(through_position); Ok(()) } diff --git a/crates/queue/src/topic/recovery.rs b/crates/queue/src/topic/recovery.rs index 867996e..a6fbcf4 100644 --- a/crates/queue/src/topic/recovery.rs +++ b/crates/queue/src/topic/recovery.rs @@ -14,7 +14,8 @@ pub(super) struct Summary { pub count: u64, pub first: MessageMeta, pub last: MessageMeta, - pub scheduled: Vec<(i64, u64)>, + pub min_available_at_ms: i64, + pub max_available_at_ms: i64, } pub(super) fn encode<'a>(messages: impl Iterator + Clone) -> Vec { @@ -38,10 +39,7 @@ pub(super) fn encode<'a>(messages: impl Iterator + Clone bytes } -pub(super) fn inspect( - reference: &RecoveryMetadataRef, - now_ms: i64, -) -> Result { +pub(super) fn inspect(reference: &RecoveryMetadataRef) -> Result { let header = reference.read_range(0, HEADER_LEN)?; let count = decode_header(&header)?; if count == 0 { @@ -58,7 +56,8 @@ pub(super) fn inspect( "topic recovery index length mismatch".into(), )); } - let mut scheduled = Vec::new(); + let mut min_available_at_ms = i64::MAX; + let mut max_available_at_ms = i64::MIN; let mut first = None; let mut last: Option = None; let mut ordinal = 0; @@ -72,9 +71,8 @@ pub(super) fn inspect( "topic recovery index range is invalid".into(), )); } - if message.available_at_ms > now_ms { - scheduled.push((message.available_at_ms, message.position)); - } + min_available_at_ms = min_available_at_ms.min(message.available_at_ms); + max_available_at_ms = max_available_at_ms.max(message.available_at_ms); first.get_or_insert_with(|| message.clone()); last = Some(message); } @@ -84,7 +82,8 @@ pub(super) fn inspect( count, first: first.expect("non-empty recovery index"), last: last.expect("non-empty recovery index"), - scheduled, + min_available_at_ms, + max_available_at_ms, }) } @@ -174,3 +173,177 @@ fn decode_entry( }, }) } + +#[cfg(test)] +mod checkpoint_tests { + use crate::topic::{MessageIndexCache, Topic}; + use crate::{Broker, BrokerConfig}; + use std::sync::{atomic::AtomicBool, Arc}; + use std::time::Duration; + use tempfile::tempdir; + + #[tokio::test] + async fn legacy_dpub_checkpoints_recover_without_duplicating_shared_deadlines() { + let root = tempdir().unwrap(); + let config = BrokerConfig { + data_path: root.path().into(), + ..BrokerConfig::default() + }; + let broker = Broker::open(config.clone()).unwrap(); + broker.create_channel("events", "workers").await.unwrap(); + broker + .publish("events", vec![b"first".to_vec()], Duration::from_secs(60)) + .await + .unwrap(); + let ready = broker + .publish("events", vec![b"ready".to_vec()], Duration::ZERO) + .await + .unwrap()[0]; + broker + .publish("events", vec![b"last".to_vec()], Duration::from_secs(60)) + .await + .unwrap(); + assert_eq!( + broker + .next_message("events", "workers", None) + .await + .unwrap() + .unwrap() + .id, + ready + ); + broker + .requeue("events", "workers", ready, Duration::from_secs(120)) + .await + .unwrap(); + drop(broker); + let directory = root.path().join("topics").join(hex::encode("events")); + let cache = MessageIndexCache::new( + config.message_index_cache_bytes, + 1, + 16, + Arc::new(AtomicBool::new(true)), + ); + let mut topic = Topic::open( + &directory, + config.max_segment_bytes, + config.max_ack_gap, + config.storage_feature_level, + Arc::clone(&cache), + ) + .unwrap(); + topic.recover_channels().unwrap(); + for position in [1, 3] { + let crate::topic::index::Lookup::Found(message) = topic.messages.lookup(position) + else { + panic!("active metadata must be resident"); + }; + topic + .channels + .get_mut("workers") + .unwrap() + .state + .requeued_until + .insert(position, message.available_at_ms); + } + topic.spill_message_metadata().unwrap(); + topic.checkpoint_channels().unwrap(); + drop(topic); + let mut topic = Topic::open( + &directory, + config.max_segment_bytes, + config.max_ack_gap, + config.storage_feature_level, + cache, + ) + .unwrap(); + topic.recover_channels().unwrap(); + let checkpoint = topic.channels["workers"].state.checkpoint(); + assert_eq!(checkpoint.requeued_until.len(), 1); + assert!(checkpoint.requeued_until.contains_key(&2)); + assert_eq!(checkpoint.attempts.get(&2), Some(&1)); + drop(topic); + let reopened = Broker::open(config).unwrap(); + let stats = reopened.stats().unwrap(); + let channel = &stats.topics[0].channels[0]; + assert_eq!( + (channel.depth, channel.deferred_count, channel.requeue_count), + (3, 3, 1) + ); + } + + #[tokio::test] + async fn legacy_dpub_checkpoint_gaps_still_fence_lost_positions() { + let root = tempdir().unwrap(); + let config = BrokerConfig { + data_path: root.path().into(), + ..BrokerConfig::default() + }; + let broker = Broker::open(config.clone()).unwrap(); + broker.create_channel("events", "workers").await.unwrap(); + broker + .publish("events", vec![b"kept".to_vec()], Duration::ZERO) + .await + .unwrap(); + drop(broker); + let directory = root.path().join("topics").join(hex::encode("events")); + let cache = MessageIndexCache::new( + config.message_index_cache_bytes, + 1, + 16, + Arc::new(AtomicBool::new(true)), + ); + let mut topic = Topic::open( + &directory, + config.max_segment_bytes, + config.max_ack_gap, + config.storage_feature_level, + Arc::clone(&cache), + ) + .unwrap(); + topic.recover_channels().unwrap(); + topic + .channels + .get_mut("workers") + .unwrap() + .state + .requeued_until + .insert(4, 100); + topic.checkpoint_channels().unwrap(); + drop(topic); + let mut topic = Topic::open( + &directory, + config.max_segment_bytes, + config.max_ack_gap, + config.storage_feature_level, + cache, + ) + .unwrap(); + topic.recover_channels().unwrap(); + assert_eq!(topic.next_position(), 5); + drop(topic); + let reopened = Broker::open(config).unwrap(); + reopened + .publish("events", vec![b"next".to_vec()], Duration::ZERO) + .await + .unwrap(); + let deliveries = reopened + .fetch_batch( + "events", + "workers", + 8, + usize::MAX, + Duration::from_secs(30), + None, + ) + .await + .unwrap(); + assert_eq!( + deliveries + .iter() + .map(|delivery| delivery.body.as_ref()) + .collect::>(), + vec![b"kept".as_slice(), b"next".as_slice()] + ); + } +} diff --git a/crates/queue/tests/crash_boundaries.rs b/crates/queue/tests/crash_boundaries.rs index 023aa8d..0deb9c6 100644 --- a/crates/queue/tests/crash_boundaries.rs +++ b/crates/queue/tests/crash_boundaries.rs @@ -219,6 +219,7 @@ async fn sigkill_ttl_boundaries_never_redeliver_or_double_count() { let broker = Broker::open(config(root.path())).unwrap(); let topic = broker .stats() + .unwrap() .topics .into_iter() .find(|topic| topic.name == "events") diff --git a/crates/queue/tests/recovery.rs b/crates/queue/tests/recovery.rs index 7ca0e9b..1027893 100644 --- a/crates/queue/tests/recovery.rs +++ b/crates/queue/tests/recovery.rs @@ -166,7 +166,9 @@ async fn nsq_relaxed_preserves_a_gap_after_an_acknowledged_tail_is_lost() { .await .unwrap()[0]; assert_ne!(replacement_id, lost_id); - let lagging = broker.filtered_stats(Some("events"), Some("lagging")); + let lagging = broker + .filtered_stats(Some("events"), Some("lagging")) + .unwrap(); assert_eq!(lagging.topics[0].channels[0].depth, 1); let replacement = broker .next_message("events", "workers", None) @@ -264,7 +266,10 @@ async fn messages_without_a_durable_channel_survive_gc_and_a_legacy_restart() { broker.flush().await.unwrap(); assert_eq!(broker.compact().await.unwrap(), 0); - assert_eq!(broker.stats().topics[0].message_count, ids.len() as u64); + assert_eq!( + broker.stats().unwrap().topics[0].message_count, + ids.len() as u64 + ); drop(broker); let manifest_path = root @@ -330,7 +335,10 @@ async fn messages_after_the_last_durable_channel_is_deleted_survive_gc_and_resta let broker = Broker::open(cfg).unwrap(); assert!(broker.compact().await.unwrap() > 0); - assert_eq!(broker.stats().topics[0].message_count, fresh.len() as u64); + assert_eq!( + broker.stats().unwrap().topics[0].message_count, + fresh.len() as u64 + ); broker .create_channel("events", "replacement") .await @@ -427,7 +435,7 @@ async fn gc_preserves_positions_and_an_empty_log_reopens_at_the_next_index() { .unwrap(); } assert!(broker.compact().await.unwrap() > 0); - assert_eq!(broker.stats().topics[0].message_count, 0); + assert_eq!(broker.stats().unwrap().topics[0].message_count, 0); drop(broker); let broker = Broker::open(cfg).unwrap(); @@ -466,7 +474,7 @@ async fn protective_eviction_persists_channel_gap_and_audit_before_deleting() { assert_eq!(report.topic, "events"); assert_eq!(report.messages, 1); assert_eq!(report.through_position, 1); - let channel = &broker.stats().topics[0].channels[0]; + let channel = &broker.stats().unwrap().topics[0].channels[0]; assert_eq!(channel.ack_cursor, 1); assert_eq!(channel.message_count, 3); assert_eq!( @@ -478,7 +486,10 @@ async fn protective_eviction_persists_channel_gap_and_audit_before_deleting() { drop(broker); let broker = Broker::open(config(root.path())).unwrap(); - assert_eq!(broker.stats().topics[0].channels[0].message_count, 3); + assert_eq!( + broker.stats().unwrap().topics[0].channels[0].message_count, + 3 + ); let next = broker .next_message("events", "workers", None) .await diff --git a/crates/rustqueuectl/Cargo.toml b/crates/rustqueuectl/Cargo.toml index 2a1b6df..61068c3 100644 --- a/crates/rustqueuectl/Cargo.toml +++ b/crates/rustqueuectl/Cargo.toml @@ -12,10 +12,14 @@ futures.workspace = true k8s-openapi.workspace = true kube.workspace = true reqwest.workspace = true +rustls.workspace = true rustqueue-operator = { path = "../operator" } serde_json.workspace = true tokio.workspace = true +[dev-dependencies] +tempfile.workspace = true + [[bin]] name = "rustqueuectl" path = "src/main.rs" diff --git a/crates/rustqueuectl/src/main.rs b/crates/rustqueuectl/src/main.rs index 1f4a863..0cc01f1 100644 --- a/crates/rustqueuectl/src/main.rs +++ b/crates/rustqueuectl/src/main.rs @@ -62,6 +62,7 @@ enum RolloutAction { #[tokio::main] async fn main() -> anyhow::Result<()> { + let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); let cli = Cli::parse(); let client = Client::try_default().await?; let clusters = Api::::namespaced(client.clone(), &cli.namespace); diff --git a/crates/rustqueuectl/tests/startup.rs b/crates/rustqueuectl/tests/startup.rs new file mode 100644 index 0000000..b924777 --- /dev/null +++ b/crates/rustqueuectl/tests/startup.rs @@ -0,0 +1,40 @@ +use std::process::Command; + +#[test] +fn unavailable_cluster_returns_an_error_without_panicking_during_tls_setup() { + let directory = tempfile::tempdir().unwrap(); + let kubeconfig = directory.path().join("config"); + std::fs::write( + &kubeconfig, + r#"apiVersion: v1 +kind: Config +current-context: audit +clusters: +- name: audit + cluster: + server: https://127.0.0.1:0 + insecure-skip-tls-verify: true +contexts: +- name: audit + context: + cluster: audit + user: audit +users: +- name: audit + user: {} +"#, + ) + .unwrap(); + let output = Command::new(env!("CARGO_BIN_EXE_rustqueuectl")) + .env("KUBECONFIG", kubeconfig) + .env("NO_PROXY", "127.0.0.1") + .args(["status"]) + .output() + .unwrap(); + assert_eq!( + output.status.code(), + Some(1), + "{}", + String::from_utf8_lossy(&output.stderr) + ); +} diff --git a/crates/server/src/config.rs b/crates/server/src/config.rs index 0d94352..b10fd47 100644 --- a/crates/server/src/config.rs +++ b/crates/server/src/config.rs @@ -148,6 +148,7 @@ pub struct LimitsConfig { pub node_publish_inflight_bytes: usize, pub connection_publish_inflight_bytes: usize, pub node_delivery_inflight_bytes: usize, + pub node_delivery_inflight_messages: usize, pub connection_delivery_inflight_bytes: usize, pub max_rdy_count: u64, pub max_connections: usize, @@ -299,6 +300,7 @@ impl Default for LimitsConfig { node_publish_inflight_bytes: 512 * 1024 * 1024, connection_publish_inflight_bytes: 80 * 1024 * 1024, node_delivery_inflight_bytes: 512 * 1024 * 1024, + node_delivery_inflight_messages: 262_144, connection_delivery_inflight_bytes: 32 * 1024 * 1024, max_rdy_count: 2500, max_connections: 10_000, diff --git a/crates/server/src/config/environment.rs b/crates/server/src/config/environment.rs index e692912..60d16cf 100644 --- a/crates/server/src/config/environment.rs +++ b/crates/server/src/config/environment.rs @@ -132,6 +132,10 @@ impl Config { "RUSTQUEUE_NODE_DELIVERY_INFLIGHT_BYTES", &mut self.limits.node_delivery_inflight_bytes, )?; + set_from_env( + "RUSTQUEUE_NODE_DELIVERY_INFLIGHT_MESSAGES", + &mut self.limits.node_delivery_inflight_messages, + )?; set_from_env( "RUSTQUEUE_CONNECTION_DELIVERY_INFLIGHT_BYTES", &mut self.limits.connection_delivery_inflight_bytes, diff --git a/crates/server/src/config/validation.rs b/crates/server/src/config/validation.rs index 8443b30..ebc9e28 100644 --- a/crates/server/src/config/validation.rs +++ b/crates/server/src/config/validation.rs @@ -97,6 +97,11 @@ impl Config { } pub(super) fn validate_runtime_limits(&self) -> anyhow::Result<()> { + if self.limits.node_delivery_inflight_messages == 0 + || self.limits.node_delivery_inflight_messages > tokio::sync::Semaphore::MAX_PERMITS + { + bail!("limits.node_delivery_inflight_messages must be positive and fit the runtime semaphore"); + } if self.limits.max_connections > tokio::sync::Semaphore::MAX_PERMITS { bail!("limits.max_connections exceeds the runtime semaphore capacity"); } diff --git a/crates/server/src/http/compat.rs b/crates/server/src/http/compat.rs index aac00b6..67239a7 100644 --- a/crates/server/src/http/compat.rs +++ b/crates/server/src/http/compat.rs @@ -132,10 +132,13 @@ pub(super) async fn metrics_handler(State(state): State) -> Response { return ApiError::from(error).into_response(); } let mut output = state.metrics.render(); - let queue_stats = state.broker.metrics_stats( + let queue_stats = match state.broker.metrics_stats( state.config.metrics.detailed_queue_metrics, state.config.metrics.max_detailed_series, - ); + ) { + Ok(stats) => stats, + Err(error) => return ApiError::from(error).into_response(), + }; output.push_str(&crate::metrics::render_broker( &queue_stats, &state.config.metrics, diff --git a/crates/server/src/http/kodo_compat.rs b/crates/server/src/http/kodo_compat.rs index f55286e..28254d9 100644 --- a/crates/server/src/http/kodo_compat.rs +++ b/crates/server/src/http/kodo_compat.rs @@ -89,7 +89,7 @@ mod tests { .await; assert_eq!(response.status(), StatusCode::OK); - assert!(broker.stats().topics[0].channels.is_empty()); + assert!(broker.stats().unwrap().topics[0].channels.is_empty()); let missing = delete_idle_channel_compat( State(state), diff --git a/crates/server/src/http/native.rs b/crates/server/src/http/native.rs index 2f1ac79..246afc4 100644 --- a/crates/server/src/http/native.rs +++ b/crates/server/src/http/native.rs @@ -66,7 +66,7 @@ pub(super) async fn registry( "registry or console", )?; state.broker.expire_in_flight().await?; - let stats = state.broker.stats(); + let stats = state.broker.stats()?; let topics = registry_topics(&stats); let (stored_messages, depth, in_flight) = backlog(&stats); let process_ready = state.accepting.load(Ordering::Acquire); @@ -172,7 +172,7 @@ pub(super) async fn drain_status( "registry or console", )?; state.broker.expire_in_flight().await?; - let stats = state.broker.metrics_stats(false, 0); + let stats = state.broker.metrics_stats(false, 0)?; let (stored_messages, depth, in_flight) = backlog(&stats); let draining = !state.accepting.load(Ordering::Acquire); let delivery_frozen = !state.delivering.load(Ordering::Acquire); @@ -215,7 +215,7 @@ pub(super) async fn set_drain( "broker drain state changed" ); state.broker.expire_in_flight().await?; - let stats = state.broker.metrics_stats(false, 0); + let stats = state.broker.metrics_stats(false, 0)?; let (stored_messages, depth, in_flight) = backlog(&stats); let publish_inflight_bytes = state.metrics.publish_inflight_bytes.load(Ordering::Acquire); let delivery_frozen = !state.delivering.load(Ordering::Acquire); @@ -247,7 +247,7 @@ pub(super) async fn native_stats( state.broker.expire_in_flight().await?; let filtered = state .broker - .filtered_stats(query.topic.as_deref(), query.channel.as_deref()); + .filtered_stats(query.topic.as_deref(), query.channel.as_deref())?; Ok(Json(json!({ "complete": true, "node_id": state.config.node.id, @@ -262,7 +262,7 @@ pub(super) async fn observe( ) -> Result, ApiError> { authorize(&headers, &state.tokens.console, "console")?; state.broker.expire_in_flight().await?; - let stats = state.broker.stats(); + let stats = state.broker.stats()?; let segment_count = stats.aggregate.segment_count; let segment_bytes = stats.aggregate.segment_bytes; let mut value = observation_head(&state); diff --git a/crates/server/src/http/nsq_stats.rs b/crates/server/src/http/nsq_stats.rs index f098724..5ab60a3 100644 --- a/crates/server/src/http/nsq_stats.rs +++ b/crates/server/src/http/nsq_stats.rs @@ -46,9 +46,13 @@ pub(super) async fn stats( if let Err(error) = state.broker.expire_in_flight().await { return ApiError::from(error).into_response(); } - let filtered = state + let filtered = match state .broker - .filtered_stats(query.topic.as_deref(), query.channel.as_deref()); + .filtered_stats(query.topic.as_deref(), query.channel.as_deref()) + { + Ok(stats) => stats, + Err(error) => return ApiError::from(error).into_response(), + }; let include_clients = query.include_clients.unwrap_or(true); if query.format.as_deref() == Some("json") { let topics = filtered diff --git a/crates/server/src/http/websocket.rs b/crates/server/src/http/websocket.rs index dea7794..2e4f83d 100644 --- a/crates/server/src/http/websocket.rs +++ b/crates/server/src/http/websocket.rs @@ -1,120 +1,32 @@ +mod admission; +mod auth; +mod delivery; + +pub(super) use admission::WebSocketRuntime; +use admission::{AdmissionError, ConnectionAdmission}; +use auth::{ + authenticate_header, ensure_authorized, observe_auth_failure, read_auth, refresh_auth, + SessionAuth, +}; +use delivery::{deliver_one, reliable_policy_changed, FRAME_HEADER_BYTES}; + use super::*; -use crate::auth::{AuthError, AuthSession, Authenticator}; use crate::subscriptions::ClientIdentity; use crate::tcp::EphemeralConsumers; use axum::extract::connect_info::ConnectInfo; use axum::extract::ws::{CloseFrame, Message, Utf8Bytes, WebSocket, WebSocketUpgrade}; use axum::extract::Path; -use bytes::{BufMut, BytesMut}; -use parking_lot::Mutex; -use rustqueue_queue::{BrokerError, Delivery, DeliveryGuard, DeliveryMode, TopicPolicy}; -use serde::Deserialize; -use std::collections::HashMap; +use rustqueue_queue::{BrokerError, DeliveryMode}; use std::net::SocketAddr; -use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::atomic::Ordering; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; -use tokio::sync::{OwnedSemaphorePermit, Semaphore}; use tower_http::cors::{AllowOrigin, CorsLayer}; use tracing::debug; const SUBPROTOCOL: &str = "rustqueue.live.v1"; -const AUTH_CHANNEL: &str = "websocket#ephemeral"; -const FRAME_MAGIC: &[u8; 4] = b"RQW1"; -const FRAME_HEADER_BYTES: usize = 24; const MAX_CONTROL_BYTES: usize = 64 * 1024; const CONTROL_WRITE_TIMEOUT: Duration = Duration::from_secs(1); -#[derive(Clone)] -pub(super) struct WebSocketRuntime { - connections: Arc, - frame_bytes: Arc, - topics: Arc>>, - max_connections_per_topic: usize, - next_id: Arc, -} - -pub(super) struct ConnectionAdmission { - runtime: WebSocketRuntime, - topic: String, - _connection: OwnedSemaphorePermit, -} - -#[derive(Debug)] -enum AdmissionError { - Global, - Topic, -} - -#[derive(Deserialize)] -struct AuthControl { - #[serde(rename = "type")] - kind: String, - secret: String, -} - -struct SessionAuth { - session: Option, - secret: Option, -} - -struct ControlAuthError { - code: &'static str, - detail: String, -} - -impl WebSocketRuntime { - pub(super) fn new(config: &crate::config::WebSocketConfig) -> Self { - Self { - connections: Arc::new(Semaphore::new(config.max_connections)), - frame_bytes: Arc::new(Semaphore::new(config.frame_inflight_bytes)), - topics: Arc::new(Mutex::new(HashMap::new())), - max_connections_per_topic: config.max_connections_per_topic, - next_id: Arc::new(AtomicU64::new(0)), - } - } - - fn admit(&self, topic: &str) -> Result<(u64, ConnectionAdmission), AdmissionError> { - let connection = Arc::clone(&self.connections) - .try_acquire_owned() - .map_err(|_| AdmissionError::Global)?; - let mut topics = self.topics.lock(); - let count = topics.entry(topic.to_owned()).or_default(); - if *count >= self.max_connections_per_topic { - return Err(AdmissionError::Topic); - } - *count += 1; - let id = self.next_id.fetch_add(1, Ordering::Relaxed).wrapping_add(1); - Ok(( - id, - ConnectionAdmission { - runtime: self.clone(), - topic: topic.to_owned(), - _connection: connection, - }, - )) - } - - fn reserve_frame(&self, bytes: usize) -> Option { - let permits = u32::try_from(bytes.max(1)).ok()?; - Arc::clone(&self.frame_bytes) - .try_acquire_many_owned(permits) - .ok() - } -} - -impl Drop for ConnectionAdmission { - fn drop(&mut self) { - let mut topics = self.runtime.topics.lock(); - let remove = topics.get_mut(&self.topic).is_some_and(|count| { - *count = count.saturating_sub(1); - *count == 0 - }); - if remove { - topics.remove(&self.topic); - } - } -} - pub(super) fn lookup_cors(allowed_origins: &[String]) -> Option { let origins = allowed_origins .iter() @@ -235,47 +147,6 @@ fn ensure_origin_allowed(allowed: &[String], headers: &HeaderMap) -> Result<(), } } -async fn authenticate_header( - state: &AppState, - peer: SocketAddr, - topic: &str, - headers: &HeaderMap, -) -> Result { - let Some(authenticator) = state.authenticator.as_deref() else { - return Ok(SessionAuth { - session: None, - secret: None, - }); - }; - let Some(value) = headers.get(header::AUTHORIZATION) else { - return Ok(SessionAuth { - session: None, - secret: None, - }); - }; - let secret = value - .to_str() - .ok() - .and_then(|value| value.strip_prefix("Bearer ")) - .filter(|secret| !secret.is_empty()) - .ok_or_else(|| { - observe_auth_failure(state); - ApiError { - status: StatusCode::UNAUTHORIZED, - code: "E_AUTH_FAILED", - detail: "Authorization must use a non-empty Bearer token".into(), - } - })? - .to_owned(); - let session = authenticate(authenticator, peer, topic, &secret) - .await - .map_err(|error| auth_api_error(state, error))?; - Ok(SessionAuth { - session: Some(session), - secret: Some(secret), - }) -} - #[allow(clippy::too_many_arguments)] async fn run_session( mut socket: WebSocket, @@ -575,299 +446,6 @@ async fn run_session_inner( Ok(()) } -#[allow(clippy::too_many_arguments)] -async fn deliver_one( - socket: &mut WebSocket, - state: &AppState, - topic: &str, - channel: &Arc, - delivery: Delivery, - guard: &mut DeliveryGuard, - policy_changes: &mut tokio::sync::watch::Receiver, - reliable_policy_epoch: u64, -) -> anyhow::Result { - let Some(expiration_ns) = state - .broker - .delivery_expiration_ns(topic, delivery.timestamp_ns)? - else { - anyhow::bail!("Topic stopped using TTL_DISCARD"); - }; - if remaining_until(expiration_ns).is_zero() { - return Ok(false); - } - let frame_bytes = FRAME_HEADER_BYTES.saturating_add(delivery.body.len()); - let Some(_frame_hold) = state.websocket.reserve_frame(frame_bytes) else { - state - .metrics - .websocket_capacity_rejections - .fetch_add(1, Ordering::Relaxed); - close_control( - socket, - "E_WS_CAPACITY", - "WebSocket frame memory capacity is exhausted", - 1013, - ) - .await; - anyhow::bail!("WebSocket frame memory capacity is exhausted"); - }; - let frame = encode_delivery(&delivery); - let write_deadline = Instant::now() - .checked_add(Duration::from_millis( - state.config.limits.tcp_command_timeout_ms, - )) - .unwrap_or_else(Instant::now); - let send = socket.send(Message::Binary(frame)); - tokio::pin!(send); - let mut shutdown = state.shutdown.clone(); - loop { - let expiration_ns = state - .broker - .delivery_expiration_ns(topic, delivery.timestamp_ns)? - .ok_or_else(|| anyhow::anyhow!("Topic stopped using TTL_DISCARD"))?; - let ttl_remaining = remaining_until(expiration_ns); - let socket_remaining = write_deadline.saturating_duration_since(Instant::now()); - if ttl_remaining.is_zero() { - anyhow::bail!("Message TTL expired during WebSocket frame write"); - } - if socket_remaining.is_zero() { - state - .metrics - .websocket_slow_disconnects - .fetch_add(1, Ordering::Relaxed); - anyhow::bail!("WebSocket frame write timed out"); - } - let timeout = ttl_remaining.min(socket_remaining); - tokio::select! { - biased; - changed = shutdown.changed() => { - let _ = changed; - anyhow::bail!("Broker shut down during WebSocket frame write"); - } - changed = policy_changes.changed() => { - if changed.is_err() { - anyhow::bail!("Topic closed during WebSocket frame write"); - } - if reliable_policy_changed(state, topic, reliable_policy_epoch)? { - anyhow::bail!("Topic switched to RELIABLE during WebSocket frame write"); - } - if policy_changes.borrow_and_update().delivery_mode != DeliveryMode::TtlDiscard { - anyhow::bail!("Topic stopped using TTL_DISCARD during WebSocket frame write"); - } - continue; - } - result = &mut send => { - result?; - break; - } - _ = tokio::time::sleep(timeout) => { - if socket_remaining <= ttl_remaining { - state.metrics.websocket_slow_disconnects.fetch_add(1, Ordering::Relaxed); - anyhow::bail!("WebSocket frame write timed out"); - } - anyhow::bail!("Message TTL expired during WebSocket frame write"); - } - } - } - let Some(token) = guard.accept_with_token(delivery.id) else { - anyhow::bail!("WebSocket delivery reservation disappeared"); - }; - state - .broker - .finish_delivery_shared(topic, Arc::clone(channel), delivery.id, token) - .await?; - state - .metrics - .websocket_messages - .fetch_add(1, Ordering::Relaxed); - state - .metrics - .websocket_bytes - .fetch_add(delivery.body.len() as u64, Ordering::Relaxed); - Ok(true) -} - -fn reliable_policy_changed( - state: &AppState, - topic: &str, - expected_epoch: u64, -) -> Result { - Ok(state.broker.topic_reliable_policy_epoch(topic)? != expected_epoch) -} - -fn encode_delivery(delivery: &Delivery) -> bytes::Bytes { - let mut frame = BytesMut::with_capacity(FRAME_HEADER_BYTES + delivery.body.len()); - frame.extend_from_slice(FRAME_MAGIC); - frame.put_u16(FRAME_HEADER_BYTES as u16); - frame.put_u16(delivery.attempts); - frame.put_i64(delivery.timestamp_ns); - frame.put_u64(delivery.id); - frame.extend_from_slice(&delivery.body); - frame.freeze() -} - -async fn read_auth( - socket: &mut WebSocket, - state: &AppState, - peer: SocketAddr, - topic: &str, -) -> Result { - let message = tokio::time::timeout( - Duration::from_millis(state.config.limits.auth_timeout_ms), - socket.recv(), - ) - .await - .map_err(|_| ControlAuthError { - code: "E_AUTH_TIMEOUT", - detail: "WebSocket AUTH timed out".into(), - })? - .ok_or_else(|| ControlAuthError { - code: "E_AUTH_FAILED", - detail: "WebSocket closed before AUTH".into(), - })? - .map_err(|error| ControlAuthError { - code: "E_AUTH_FAILED", - detail: error.to_string(), - })?; - let Message::Text(text) = message else { - return Err(ControlAuthError { - code: "E_BAD_AUTH", - detail: "WebSocket AUTH must be a text control frame".into(), - }); - }; - let request: AuthControl = - serde_json::from_str(text.as_str()).map_err(|error| ControlAuthError { - code: "E_BAD_AUTH", - detail: error.to_string(), - })?; - if request.kind != "auth" || request.secret.is_empty() { - return Err(ControlAuthError { - code: "E_BAD_AUTH", - detail: "WebSocket AUTH frame is invalid".into(), - }); - } - let authenticator = state - .authenticator - .as_deref() - .ok_or_else(|| ControlAuthError { - code: "E_AUTH_DISABLED", - detail: "WebSocket AUTH is disabled".into(), - })?; - let session = authenticate(authenticator, peer, topic, &request.secret) - .await - .map_err(|error| ControlAuthError { - code: auth_error_code(&error), - detail: "WebSocket AUTH failed".into(), - })?; - Ok(SessionAuth { - session: Some(session), - secret: Some(request.secret), - }) -} - -async fn refresh_auth( - auth: &mut SessionAuth, - state: &AppState, - peer: SocketAddr, - topic: &str, -) -> Result<(), ControlAuthError> { - let Some(session) = auth.session.as_ref() else { - return Ok(()); - }; - if !session.is_expired() { - ensure_authorized(Some(session), topic).map_err(|_| ControlAuthError { - code: "E_UNAUTHORIZED", - detail: "WebSocket subscription is no longer authorized".into(), - })?; - return Ok(()); - } - let authenticator = state - .authenticator - .as_deref() - .ok_or_else(|| ControlAuthError { - code: "E_AUTH_FAILED", - detail: "WebSocket AUTH configuration is unavailable".into(), - })?; - let secret = auth.secret.as_deref().ok_or_else(|| ControlAuthError { - code: "E_AUTH_FAILED", - detail: "WebSocket AUTH state is unavailable".into(), - })?; - let session = authenticate(authenticator, peer, topic, secret) - .await - .map_err(|error| { - observe_auth_failure(state); - ControlAuthError { - code: auth_error_code(&error), - detail: "WebSocket AUTH refresh failed".into(), - } - })?; - auth.session = Some(session); - Ok(()) -} - -async fn authenticate( - authenticator: &Authenticator, - peer: SocketAddr, - topic: &str, - secret: &str, -) -> Result { - let session = authenticator - .authenticate(&peer.ip().to_string(), false, "", secret.as_bytes()) - .await?; - if !session.can_subscribe(topic, AUTH_CHANNEL) { - return Err(AuthError::Unauthorized); - } - Ok(session) -} - -fn ensure_authorized(session: Option<&AuthSession>, topic: &str) -> anyhow::Result<()> { - if session.is_some_and(|session| !session.can_subscribe(topic, AUTH_CHANNEL)) { - anyhow::bail!("WebSocket subscription is not authorized"); - } - Ok(()) -} - -fn auth_api_error(state: &AppState, error: AuthError) -> ApiError { - observe_auth_failure(state); - let (status, code, detail) = match error { - AuthError::Unauthorized => ( - StatusCode::FORBIDDEN, - "E_UNAUTHORIZED", - "AUTH does not permit this Topic subscription", - ), - AuthError::Overloaded => ( - StatusCode::SERVICE_UNAVAILABLE, - "E_AUTH_OVERLOADED", - "AUTH capacity is exhausted", - ), - _ => ( - StatusCode::SERVICE_UNAVAILABLE, - "E_AUTH_FAILED", - "AUTH service failed", - ), - }; - ApiError { - status, - code, - detail: detail.into(), - } -} - -fn observe_auth_failure(state: &AppState) { - state - .metrics - .websocket_auth_failures - .fetch_add(1, Ordering::Relaxed); - state.metrics.auth_failures.fetch_add(1, Ordering::Relaxed); -} - -fn auth_error_code(error: &AuthError) -> &'static str { - match error { - AuthError::Unauthorized => "E_UNAUTHORIZED", - AuthError::Overloaded => "E_AUTH_OVERLOADED", - _ => "E_AUTH_FAILED", - } -} - async fn handle_incoming( socket: &mut WebSocket, incoming: Option>, @@ -981,61 +559,6 @@ impl Drop for EphemeralCleanup { mod tests { use super::*; - fn runtime(max_connections: usize, max_connections_per_topic: usize) -> WebSocketRuntime { - let config = crate::config::WebSocketConfig { - max_connections, - max_connections_per_topic, - ..Default::default() - }; - WebSocketRuntime::new(&config) - } - - #[test] - fn encodes_rqw1_golden_frame() { - let delivery = Delivery { - id: 0x0102_0304_0506_0708, - timestamp_ns: -0x0102_0304_0506_0708, - attempts: 0x090a, - body: Arc::<[u8]>::from(&b"hello"[..]), - }; - - let frame = encode_delivery(&delivery); - - assert_eq!( - frame.as_ref(), - &[ - b'R', b'Q', b'W', b'1', 0, 24, 9, 10, 0xfe, 0xfd, 0xfc, 0xfb, 0xfa, 0xf9, 0xf8, - 0xf8, 1, 2, 3, 4, 5, 6, 7, 8, b'h', b'e', b'l', b'l', b'o', - ] - ); - } - - #[test] - fn admission_enforces_global_and_per_topic_limits_and_releases() { - let runtime = runtime(2, 1); - let (_, first) = runtime.admit("events").unwrap(); - assert!(matches!( - runtime.admit("events"), - Err(AdmissionError::Topic) - )); - let (_, second) = runtime.admit("other").unwrap(); - assert!(matches!( - runtime.admit("third"), - Err(AdmissionError::Global) - )); - - drop(first); - let (_, replacement) = runtime.admit("events").unwrap(); - assert!(matches!( - runtime.admit("other"), - Err(AdmissionError::Global) - )); - - drop(second); - drop(replacement); - assert!(runtime.admit("events").is_ok()); - } - #[test] fn origin_check_allows_missing_and_exact_origin_only() { let allowed = ["https://app.example".to_owned()]; diff --git a/crates/server/src/http/websocket/admission.rs b/crates/server/src/http/websocket/admission.rs new file mode 100644 index 0000000..dfe70d9 --- /dev/null +++ b/crates/server/src/http/websocket/admission.rs @@ -0,0 +1,119 @@ +use parking_lot::Mutex; +use std::collections::HashMap; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Arc; +use tokio::sync::{OwnedSemaphorePermit, Semaphore}; + +#[derive(Clone)] +pub(in crate::http) struct WebSocketRuntime { + connections: Arc, + frame_bytes: Arc, + topics: Arc>>, + max_connections_per_topic: usize, + next_id: Arc, +} + +pub(super) struct ConnectionAdmission { + runtime: WebSocketRuntime, + topic: String, + _connection: OwnedSemaphorePermit, +} + +#[derive(Debug)] +pub(super) enum AdmissionError { + Global, + Topic, +} + +impl WebSocketRuntime { + pub(in crate::http) fn new(config: &crate::config::WebSocketConfig) -> Self { + Self { + connections: Arc::new(Semaphore::new(config.max_connections)), + frame_bytes: Arc::new(Semaphore::new(config.frame_inflight_bytes)), + topics: Arc::new(Mutex::new(HashMap::new())), + max_connections_per_topic: config.max_connections_per_topic, + next_id: Arc::new(AtomicU64::new(0)), + } + } + + pub(super) fn admit(&self, topic: &str) -> Result<(u64, ConnectionAdmission), AdmissionError> { + let connection = Arc::clone(&self.connections) + .try_acquire_owned() + .map_err(|_| AdmissionError::Global)?; + let mut topics = self.topics.lock(); + let count = topics.entry(topic.to_owned()).or_default(); + if *count >= self.max_connections_per_topic { + return Err(AdmissionError::Topic); + } + *count += 1; + let id = self.next_id.fetch_add(1, Ordering::Relaxed).wrapping_add(1); + Ok(( + id, + ConnectionAdmission { + runtime: self.clone(), + topic: topic.to_owned(), + _connection: connection, + }, + )) + } + + pub(super) fn reserve_frame(&self, bytes: usize) -> Option { + let permits = u32::try_from(bytes.max(1)).ok()?; + Arc::clone(&self.frame_bytes) + .try_acquire_many_owned(permits) + .ok() + } +} + +impl Drop for ConnectionAdmission { + fn drop(&mut self) { + let mut topics = self.runtime.topics.lock(); + let remove = topics.get_mut(&self.topic).is_some_and(|count| { + *count = count.saturating_sub(1); + *count == 0 + }); + if remove { + topics.remove(&self.topic); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn runtime(max_connections: usize, max_connections_per_topic: usize) -> WebSocketRuntime { + let config = crate::config::WebSocketConfig { + max_connections, + max_connections_per_topic, + ..Default::default() + }; + WebSocketRuntime::new(&config) + } + + #[test] + fn admission_enforces_global_and_per_topic_limits_and_releases() { + let runtime = runtime(2, 1); + let (_, first) = runtime.admit("events").unwrap(); + assert!(matches!( + runtime.admit("events"), + Err(AdmissionError::Topic) + )); + let (_, second) = runtime.admit("other").unwrap(); + assert!(matches!( + runtime.admit("third"), + Err(AdmissionError::Global) + )); + + drop(first); + let (_, replacement) = runtime.admit("events").unwrap(); + assert!(matches!( + runtime.admit("other"), + Err(AdmissionError::Global) + )); + + drop(second); + drop(replacement); + assert!(runtime.admit("events").is_ok()); + } +} diff --git a/crates/server/src/http/websocket/auth.rs b/crates/server/src/http/websocket/auth.rs new file mode 100644 index 0000000..da9a782 --- /dev/null +++ b/crates/server/src/http/websocket/auth.rs @@ -0,0 +1,231 @@ +use super::{ApiError, AppState}; +use crate::auth::{AuthError, AuthSession, Authenticator}; +use axum::extract::ws::{Message, WebSocket}; +use axum::http::{header, HeaderMap, StatusCode}; +use serde::Deserialize; +use std::net::SocketAddr; +use std::sync::atomic::Ordering; +use std::time::Duration; + +const AUTH_CHANNEL: &str = "websocket#ephemeral"; + +#[derive(Deserialize)] +struct AuthControl { + #[serde(rename = "type")] + kind: String, + secret: String, +} + +pub(super) struct SessionAuth { + pub(super) session: Option, + pub(super) secret: Option, +} + +pub(super) struct ControlAuthError { + pub(super) code: &'static str, + pub(super) detail: String, +} + +pub(super) async fn authenticate_header( + state: &AppState, + peer: SocketAddr, + topic: &str, + headers: &HeaderMap, +) -> Result { + let Some(authenticator) = state.authenticator.as_deref() else { + return Ok(SessionAuth { + session: None, + secret: None, + }); + }; + let Some(value) = headers.get(header::AUTHORIZATION) else { + return Ok(SessionAuth { + session: None, + secret: None, + }); + }; + let secret = value + .to_str() + .ok() + .and_then(|value| value.strip_prefix("Bearer ")) + .filter(|secret| !secret.is_empty()) + .ok_or_else(|| { + observe_auth_failure(state); + ApiError { + status: StatusCode::UNAUTHORIZED, + code: "E_AUTH_FAILED", + detail: "Authorization must use a non-empty Bearer token".into(), + } + })? + .to_owned(); + let session = authenticate(authenticator, peer, topic, &secret) + .await + .map_err(|error| auth_api_error(state, error))?; + Ok(SessionAuth { + session: Some(session), + secret: Some(secret), + }) +} + +pub(super) async fn read_auth( + socket: &mut WebSocket, + state: &AppState, + peer: SocketAddr, + topic: &str, +) -> Result { + let message = tokio::time::timeout( + Duration::from_millis(state.config.limits.auth_timeout_ms), + socket.recv(), + ) + .await + .map_err(|_| ControlAuthError { + code: "E_AUTH_TIMEOUT", + detail: "WebSocket AUTH timed out".into(), + })? + .ok_or_else(|| ControlAuthError { + code: "E_AUTH_FAILED", + detail: "WebSocket closed before AUTH".into(), + })? + .map_err(|error| ControlAuthError { + code: "E_AUTH_FAILED", + detail: error.to_string(), + })?; + let Message::Text(text) = message else { + return Err(ControlAuthError { + code: "E_BAD_AUTH", + detail: "WebSocket AUTH must be a text control frame".into(), + }); + }; + let request: AuthControl = + serde_json::from_str(text.as_str()).map_err(|error| ControlAuthError { + code: "E_BAD_AUTH", + detail: error.to_string(), + })?; + if request.kind != "auth" || request.secret.is_empty() { + return Err(ControlAuthError { + code: "E_BAD_AUTH", + detail: "WebSocket AUTH frame is invalid".into(), + }); + } + let authenticator = state + .authenticator + .as_deref() + .ok_or_else(|| ControlAuthError { + code: "E_AUTH_DISABLED", + detail: "WebSocket AUTH is disabled".into(), + })?; + let session = authenticate(authenticator, peer, topic, &request.secret) + .await + .map_err(|error| ControlAuthError { + code: auth_error_code(&error), + detail: "WebSocket AUTH failed".into(), + })?; + Ok(SessionAuth { + session: Some(session), + secret: Some(request.secret), + }) +} + +pub(super) async fn refresh_auth( + auth: &mut SessionAuth, + state: &AppState, + peer: SocketAddr, + topic: &str, +) -> Result<(), ControlAuthError> { + let Some(session) = auth.session.as_ref() else { + return Ok(()); + }; + if !session.is_expired() { + ensure_authorized(Some(session), topic).map_err(|_| ControlAuthError { + code: "E_UNAUTHORIZED", + detail: "WebSocket subscription is no longer authorized".into(), + })?; + return Ok(()); + } + let authenticator = state + .authenticator + .as_deref() + .ok_or_else(|| ControlAuthError { + code: "E_AUTH_FAILED", + detail: "WebSocket AUTH configuration is unavailable".into(), + })?; + let secret = auth.secret.as_deref().ok_or_else(|| ControlAuthError { + code: "E_AUTH_FAILED", + detail: "WebSocket AUTH state is unavailable".into(), + })?; + let session = authenticate(authenticator, peer, topic, secret) + .await + .map_err(|error| { + observe_auth_failure(state); + ControlAuthError { + code: auth_error_code(&error), + detail: "WebSocket AUTH refresh failed".into(), + } + })?; + auth.session = Some(session); + Ok(()) +} + +async fn authenticate( + authenticator: &Authenticator, + peer: SocketAddr, + topic: &str, + secret: &str, +) -> Result { + let session = authenticator + .authenticate(&peer.ip().to_string(), false, "", secret.as_bytes()) + .await?; + if !session.can_subscribe(topic, AUTH_CHANNEL) { + return Err(AuthError::Unauthorized); + } + Ok(session) +} + +pub(super) fn ensure_authorized(session: Option<&AuthSession>, topic: &str) -> anyhow::Result<()> { + if session.is_some_and(|session| !session.can_subscribe(topic, AUTH_CHANNEL)) { + anyhow::bail!("WebSocket subscription is not authorized"); + } + Ok(()) +} + +fn auth_api_error(state: &AppState, error: AuthError) -> ApiError { + observe_auth_failure(state); + let (status, code, detail) = match error { + AuthError::Unauthorized => ( + StatusCode::FORBIDDEN, + "E_UNAUTHORIZED", + "AUTH does not permit this Topic subscription", + ), + AuthError::Overloaded => ( + StatusCode::SERVICE_UNAVAILABLE, + "E_AUTH_OVERLOADED", + "AUTH capacity is exhausted", + ), + _ => ( + StatusCode::SERVICE_UNAVAILABLE, + "E_AUTH_FAILED", + "AUTH service failed", + ), + }; + ApiError { + status, + code, + detail: detail.into(), + } +} + +pub(super) fn observe_auth_failure(state: &AppState) { + state + .metrics + .websocket_auth_failures + .fetch_add(1, Ordering::Relaxed); + state.metrics.auth_failures.fetch_add(1, Ordering::Relaxed); +} + +fn auth_error_code(error: &AuthError) -> &'static str { + match error { + AuthError::Unauthorized => "E_UNAUTHORIZED", + AuthError::Overloaded => "E_AUTH_OVERLOADED", + _ => "E_AUTH_FAILED", + } +} diff --git a/crates/server/src/http/websocket/delivery.rs b/crates/server/src/http/websocket/delivery.rs new file mode 100644 index 0000000..47b4aa7 --- /dev/null +++ b/crates/server/src/http/websocket/delivery.rs @@ -0,0 +1,165 @@ +use super::{close_control, remaining_until, AppState}; +use axum::extract::ws::{Message, WebSocket}; +use bytes::{BufMut, BytesMut}; +use rustqueue_queue::{BrokerError, Delivery, DeliveryGuard, DeliveryMode, TopicPolicy}; +use std::sync::atomic::Ordering; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +const FRAME_MAGIC: &[u8; 4] = b"RQW1"; +pub(super) const FRAME_HEADER_BYTES: usize = 24; + +#[allow(clippy::too_many_arguments)] +pub(super) async fn deliver_one( + socket: &mut WebSocket, + state: &AppState, + topic: &str, + channel: &Arc, + delivery: Delivery, + guard: &mut DeliveryGuard, + policy_changes: &mut tokio::sync::watch::Receiver, + reliable_policy_epoch: u64, +) -> anyhow::Result { + let Some(expiration_ns) = state + .broker + .delivery_expiration_ns(topic, delivery.timestamp_ns)? + else { + anyhow::bail!("Topic stopped using TTL_DISCARD"); + }; + if remaining_until(expiration_ns).is_zero() { + return Ok(false); + } + let frame_bytes = FRAME_HEADER_BYTES.saturating_add(delivery.body.len()); + let Some(_frame_hold) = state.websocket.reserve_frame(frame_bytes) else { + state + .metrics + .websocket_capacity_rejections + .fetch_add(1, Ordering::Relaxed); + close_control( + socket, + "E_WS_CAPACITY", + "WebSocket frame memory capacity is exhausted", + 1013, + ) + .await; + anyhow::bail!("WebSocket frame memory capacity is exhausted"); + }; + let frame = encode_delivery(&delivery); + let write_deadline = Instant::now() + .checked_add(Duration::from_millis( + state.config.limits.tcp_command_timeout_ms, + )) + .unwrap_or_else(Instant::now); + let send = socket.send(Message::Binary(frame)); + tokio::pin!(send); + let mut shutdown = state.shutdown.clone(); + loop { + let expiration_ns = state + .broker + .delivery_expiration_ns(topic, delivery.timestamp_ns)? + .ok_or_else(|| anyhow::anyhow!("Topic stopped using TTL_DISCARD"))?; + let ttl_remaining = remaining_until(expiration_ns); + let socket_remaining = write_deadline.saturating_duration_since(Instant::now()); + if ttl_remaining.is_zero() { + anyhow::bail!("Message TTL expired during WebSocket frame write"); + } + if socket_remaining.is_zero() { + state + .metrics + .websocket_slow_disconnects + .fetch_add(1, Ordering::Relaxed); + anyhow::bail!("WebSocket frame write timed out"); + } + let timeout = ttl_remaining.min(socket_remaining); + tokio::select! { + biased; + changed = shutdown.changed() => { + let _ = changed; + anyhow::bail!("Broker shut down during WebSocket frame write"); + } + changed = policy_changes.changed() => { + if changed.is_err() { + anyhow::bail!("Topic closed during WebSocket frame write"); + } + if reliable_policy_changed(state, topic, reliable_policy_epoch)? { + anyhow::bail!("Topic switched to RELIABLE during WebSocket frame write"); + } + if policy_changes.borrow_and_update().delivery_mode != DeliveryMode::TtlDiscard { + anyhow::bail!("Topic stopped using TTL_DISCARD during WebSocket frame write"); + } + continue; + } + result = &mut send => { + result?; + break; + } + _ = tokio::time::sleep(timeout) => { + if socket_remaining <= ttl_remaining { + state.metrics.websocket_slow_disconnects.fetch_add(1, Ordering::Relaxed); + anyhow::bail!("WebSocket frame write timed out"); + } + anyhow::bail!("Message TTL expired during WebSocket frame write"); + } + } + } + let Some((token, _lease)) = guard.accept_with_lease(delivery.id) else { + anyhow::bail!("WebSocket delivery reservation disappeared"); + }; + state + .broker + .finish_delivery_shared(topic, Arc::clone(channel), delivery.id, token) + .await?; + state + .metrics + .websocket_messages + .fetch_add(1, Ordering::Relaxed); + state + .metrics + .websocket_bytes + .fetch_add(delivery.body.len() as u64, Ordering::Relaxed); + Ok(true) +} + +pub(super) fn reliable_policy_changed( + state: &AppState, + topic: &str, + expected_epoch: u64, +) -> Result { + Ok(state.broker.topic_reliable_policy_epoch(topic)? != expected_epoch) +} + +fn encode_delivery(delivery: &Delivery) -> bytes::Bytes { + let mut frame = BytesMut::with_capacity(FRAME_HEADER_BYTES + delivery.body.len()); + frame.extend_from_slice(FRAME_MAGIC); + frame.put_u16(FRAME_HEADER_BYTES as u16); + frame.put_u16(delivery.attempts); + frame.put_i64(delivery.timestamp_ns); + frame.put_u64(delivery.id); + frame.extend_from_slice(&delivery.body); + frame.freeze() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn encodes_rqw1_golden_frame() { + let delivery = Delivery { + id: 0x0102_0304_0506_0708, + timestamp_ns: -0x0102_0304_0506_0708, + attempts: 0x090a, + body: Arc::<[u8]>::from(&b"hello"[..]), + }; + + let frame = encode_delivery(&delivery); + + assert_eq!( + frame.as_ref(), + &[ + b'R', b'Q', b'W', b'1', 0, 24, 9, 10, 0xfe, 0xfd, 0xfc, 0xfb, 0xfa, 0xf9, 0xf8, + 0xf8, 1, 2, 3, 4, 5, 6, 7, 8, b'h', b'e', b'l', b'l', b'o', + ] + ); + } +} diff --git a/crates/server/src/main.rs b/crates/server/src/main.rs index 94c5ba7..7468490 100644 --- a/crates/server/src/main.rs +++ b/crates/server/src/main.rs @@ -83,6 +83,7 @@ async fn main() -> anyhow::Result<()> { payload_read_workers: config.storage.payload_read_workers, payload_read_queue: config.storage.payload_read_queue, delivery_inflight_bytes: config.limits.node_delivery_inflight_bytes, + delivery_inflight_messages: config.limits.node_delivery_inflight_messages, scrub_bytes_per_second: config.storage.scrub_bytes_per_second, storage_feature_level: config.storage.feature_level, require_management_fence_sync: config.security.console_management_enabled, diff --git a/crates/server/src/metrics.rs b/crates/server/src/metrics.rs index a53c360..15190ee 100644 --- a/crates/server/src/metrics.rs +++ b/crates/server/src/metrics.rs @@ -134,6 +134,8 @@ pub fn render_broker(stats: &BrokerStats, config: &MetricsConfig) -> String { rustqueue_channel_ack_gap_total {channel_ack_gap}\n\ # TYPE rustqueue_delivery_inflight_bytes gauge\n\ rustqueue_delivery_inflight_bytes {}\n\ + # TYPE rustqueue_delivery_inflight_messages gauge\n\ + rustqueue_delivery_inflight_messages {}\n\ # TYPE rustqueue_delivery_budget_waiters gauge\n\ rustqueue_delivery_budget_waiters {}\n\ # TYPE rustqueue_delivery_budget_waits_total counter\n\ @@ -154,6 +156,7 @@ pub fn render_broker(stats: &BrokerStats, config: &MetricsConfig) -> String { stats.aggregate.unsynced_messages, stats.aggregate.unsynced_bytes, stats.delivery_budget.in_flight_bytes, + stats.delivery_budget.in_flight_messages, stats.delivery_budget.waiters, stats.delivery_budget.waits_total, ); diff --git a/crates/server/src/tcp.rs b/crates/server/src/tcp.rs index 694b7ab..e7cc9ed 100644 --- a/crates/server/src/tcp.rs +++ b/crates/server/src/tcp.rs @@ -123,10 +123,11 @@ struct RemoteDelivery { ttl_discard: bool, } -#[derive(Clone, Copy, Debug)] +#[derive(Clone, Debug)] struct InFlightDelivery { deadline: Instant, token: u64, + _lease: Option, } struct SessionState { @@ -173,17 +174,17 @@ impl SessionState { if self.pending_channel_ops.contains(&id) { return None; } - self.in_flight.get(&id).copied() + self.in_flight.get(&id).cloned() } fn record_delivery(&mut self, id: u64, delivery: InFlightDelivery) { + let deadline = (delivery.deadline, id, delivery.token); if let Some(previous) = self.in_flight.insert(id, delivery) { self.in_flight_deadlines .remove(&(previous.deadline, id, previous.token)); } if !self.pending_channel_ops.contains(&id) { - self.in_flight_deadlines - .insert((delivery.deadline, id, delivery.token)); + self.in_flight_deadlines.insert(deadline); } } diff --git a/crates/server/src/tcp/channel_ops.rs b/crates/server/src/tcp/channel_ops.rs index 8ed2a83..64b0d99 100644 --- a/crates/server/src/tcp/channel_ops.rs +++ b/crates/server/src/tcp/channel_ops.rs @@ -1,5 +1,6 @@ use super::*; use futures::stream::{FuturesUnordered, StreamExt}; +use parking_lot::Mutex; use tokio::sync::mpsc; use tokio::task::JoinHandle; @@ -64,7 +65,14 @@ pub(super) struct ChannelOpCompletion { #[derive(Clone)] pub(super) struct ChannelOpSender { - sender: mpsc::UnboundedSender, + sender: mpsc::Sender, +} + +pub(super) struct ChannelOps { + pub(super) sender: ChannelOpSender, + pub(super) results: mpsc::UnboundedReceiver, + pub(super) task: JoinHandle<()>, + pub(super) active: Arc>>, } impl ChannelOpSender { @@ -119,36 +127,41 @@ impl ChannelOpSender { fn send(&self, operation: ChannelOp) -> Result<(), BrokerError> { self.sender - .send(operation) - .map_err(|_| BrokerError::StorageUnavailable) + .try_send(operation) + .map_err(|error| match error { + mpsc::error::TrySendError::Full(_) => BrokerError::ChannelWorkerLimit, + mpsc::error::TrySendError::Closed(_) => BrokerError::StorageUnavailable, + }) } } -pub(super) fn start_channel_ops( - broker: Broker, -) -> ( - ChannelOpSender, - mpsc::UnboundedReceiver, - JoinHandle<()>, -) { +pub(super) fn start_channel_ops(broker: Broker, capacity: usize) -> ChannelOps { // The session admits at most one operation per in-flight message, and // in-flight messages are capped by max_rdy_count. - let (operation_tx, operation_rx) = mpsc::unbounded_channel(); + let (operation_tx, operation_rx) = mpsc::channel(capacity.max(1)); let (completion_tx, completion_rx) = mpsc::unbounded_channel(); - let task = tokio::spawn(run_channel_ops(broker, operation_rx, completion_tx)); - ( - ChannelOpSender { + let active = Arc::new(Mutex::new(HashSet::new())); + let task = tokio::spawn(run_channel_ops( + broker, + operation_rx, + completion_tx, + Arc::clone(&active), + )); + ChannelOps { + sender: ChannelOpSender { sender: operation_tx, }, - completion_rx, + results: completion_rx, task, - ) + active, + } } async fn run_channel_ops( broker: Broker, - mut operations: mpsc::UnboundedReceiver, + mut operations: mpsc::Receiver, completions: mpsc::UnboundedSender, + active: Arc>>, ) { let mut pending = FuturesUnordered::new(); let mut receiving = true; @@ -157,6 +170,7 @@ async fn run_channel_ops( operation = operations.recv(), if receiving && pending.len() < MAX_PENDING_CHANNEL_OPS => { match operation { Some(operation) => { + active.lock().insert(operation.id()); let broker = broker.clone(); pending.push(execute_channel_op(broker, operation)); } @@ -165,6 +179,7 @@ async fn run_channel_ops( } completion = pending.next(), if !pending.is_empty() => { if let Some(completion) = completion { + active.lock().remove(&completion.id); let _ = completions.send(completion); } } @@ -231,7 +246,12 @@ mod tests { .unwrap(); let (deliveries, mut guard) = batch.into_parts(); - let (sender, mut completions, task) = start_channel_ops(broker.clone()); + let ChannelOps { + sender, + results: mut completions, + task, + .. + } = start_channel_ops(broker.clone(), 32); for delivery in deliveries { let token = guard.accept_with_token(delivery.id).unwrap(); sender @@ -247,7 +267,7 @@ mod tests { } task.await.unwrap(); - let stats = broker.stats().channel_group_commit; + let stats = broker.stats().unwrap().channel_group_commit; assert_eq!(completed, 32); assert_eq!(stats.requests, 32); assert!(stats.commits < stats.requests); diff --git a/crates/server/src/tcp/dead_letter.rs b/crates/server/src/tcp/dead_letter.rs index e6cc351..e0aa213 100644 --- a/crates/server/src/tcp/dead_letter.rs +++ b/crates/server/src/tcp/dead_letter.rs @@ -176,7 +176,7 @@ mod tests { .await .unwrap()); - let stats = broker.stats(); + let stats = broker.stats().unwrap(); let source = stats .topics .iter() diff --git a/crates/server/src/tcp/session.rs b/crates/server/src/tcp/session.rs index 6be934c..bd90449 100644 --- a/crates/server/src/tcp/session.rs +++ b/crates/server/src/tcp/session.rs @@ -77,7 +77,16 @@ pub(super) async fn run_session( let mut last_command = Instant::now(); let mut pending_fetch: Option> = None; let mut subscription_policy_changes = None; - let (channel_ops, mut channel_op_results, channel_ops_task) = start_channel_ops(broker.clone()); + let capacity = usize::try_from(config.limits.max_rdy_count) + .unwrap_or(usize::MAX) + .min(config.limits.node_delivery_inflight_messages); + let ChannelOps { + sender: channel_ops, + results: mut channel_op_results, + task: channel_ops_task, + active: active_channel_ops, + } = start_channel_ops(broker.clone(), capacity); + let _channel_ops_task = AbortOnDrop(channel_ops_task.abort_handle()); let session_result: anyhow::Result<()> = async { loop { @@ -162,7 +171,6 @@ pub(super) async fn run_session( .await?; continue; } - last_command = Instant::now(); let progress_timeout = connection_progress_timeout(state.heartbeat); let keep_open = tokio::time::timeout( progress_timeout, @@ -182,6 +190,9 @@ pub(super) async fn run_session( ) .await .map_err(|_| anyhow::anyhow!("client command timed out"))??; + // Broker work and response I/O must not consume the peer's + // heartbeat window before it can react to the response. + last_command = Instant::now(); if !keep_open { break; } @@ -264,7 +275,6 @@ pub(super) async fn run_session( write_error_timed(&mut writer, state.heartbeat, "E_CLOSING", "node is shutting down").await?; continue; } - last_command = Instant::now(); let progress_timeout = connection_progress_timeout(state.heartbeat); let keep_open = tokio::time::timeout( progress_timeout, @@ -284,6 +294,7 @@ pub(super) async fn run_session( ) .await .map_err(|_| anyhow::anyhow!("client command timed out"))??; + last_command = Instant::now(); if !keep_open { break; } @@ -440,10 +451,10 @@ pub(super) async fn run_session( InFlightDelivery { deadline, token, + _lease: delivery_guard.accept_with_lease(delivery.id).map(|(_, lease)| lease), }, ); state.mark_channel_operation_pending(delivery.id); - delivery_guard.accept(delivery.id); continue; } let header = encode_message_header( @@ -464,8 +475,8 @@ pub(super) async fn run_session( &mut policy_changes, ) .await?; - let accepted_token = delivery_guard - .accept_with_token(delivery.id) + let (accepted_token, lease) = delivery_guard + .accept_with_lease(delivery.id) .ok_or_else(|| anyhow::anyhow!("delivery token is missing"))?; debug_assert_eq!(accepted_token, token); let mut delivery_deadline = handoff_deadline; @@ -478,6 +489,7 @@ pub(super) async fn run_session( InFlightDelivery { deadline: delivery_deadline, token, + _lease: Some(lease), }, ); handed_off.push((delivery.id, token)); @@ -497,7 +509,7 @@ pub(super) async fn run_session( ) .map_err(|error| anyhow::anyhow!(error.to_string()))?; for (id, token) in handed_off { - if let Some(delivery) = state.in_flight.get(&id).copied() { + if let Some(delivery) = state.in_flight.get(&id).cloned() { debug_assert_eq!(delivery.token, token); state.record_delivery( id, @@ -586,8 +598,10 @@ pub(super) async fn run_session( // the matching delivery tokens. drop(pending_fetch.take()); if let Some(subscription) = &state.subscription { + // Only active futures may have reached the durable worker. Operations + // still in the session queue cannot commit after the task is aborted. let deliveries = - releaseable_in_flight_deliveries(state.in_flight, &state.pending_channel_ops); + releaseable_in_flight_deliveries(state.in_flight, &active_channel_ops.lock()); if !deliveries.is_empty() { broker.release_deliveries(&subscription.topic, &subscription.channel, &deliveries); } @@ -798,9 +812,15 @@ async fn wait_for_in_flight_deadline(deadline: Option) { #[cfg(test)] mod tests { use super::*; + use rustqueue_queue::BrokerConfig; + use tempfile::tempdir; fn delivery(deadline: Instant, token: u64) -> InFlightDelivery { - InFlightDelivery { deadline, token } + InFlightDelivery { + deadline, + token, + _lease: None, + } } fn session_state() -> SessionState { @@ -827,6 +847,75 @@ mod tests { } } + #[tokio::test] + async fn slow_command_response_does_not_consume_the_client_heartbeat_window() { + for initial_command in [true, false] { + let root = tempdir().unwrap(); + let broker = Broker::open(BrokerConfig { + data_path: root.path().into(), + ..BrokerConfig::default() + }) + .unwrap(); + let heartbeat = Duration::from_millis(100); + let mut state = session_state(); + state.heartbeat = Some(heartbeat); + let command = ParsedCommand { + command: Command::Publish { + topic: "events".into(), + }, + body: Some(Bytes::from_static(b"body")), + publish_reservation: None, + }; + let (mut peer, server) = tokio::io::duplex(1); + let task = tokio::spawn(async move { + run_session( + Box::new(server), + initial_command.then_some(command), + "127.0.0.1:10000".parse().unwrap(), + &Config::default(), + &broker, + &Metrics::default(), + None, + EphemeralConsumers::default(), + Arc::new(AtomicBool::new(true)), + Arc::new(AtomicBool::new(true)), + Arc::new(PublishAdmission::new(1024, Arc::new(Metrics::default()))), + Arc::new(ConnectionBudget::new(1024)), + SubscriptionRegistry::default(), + state, + ) + .await + }); + if !initial_command { + peer.write_all(b"PUB events\n\0\0\0\x04body").await.unwrap(); + } + let mut response = [0u8; 10]; + peer.read_exact(&mut response[..1]).await.unwrap(); + tokio::time::sleep(heartbeat * 3).await; + peer.read_exact(&mut response[1..]).await.unwrap(); + assert_eq!(response.as_slice(), encode_frame(FrameType::Response, OK)); + + let mut length = [0u8; 4]; + tokio::time::timeout(heartbeat * 3, peer.read_exact(&mut length)) + .await + .unwrap() + .expect("completed response must leave time to answer a heartbeat"); + let mut frame = vec![0u8; u32::from_be_bytes(length) as usize]; + peer.read_exact(&mut frame).await.unwrap(); + assert_eq!(&frame[4..], HEARTBEAT); + + // An unresponsive peer must still be evicted after the response. + let drain = async { + let mut remaining = Vec::new(); + peer.read_to_end(&mut remaining).await.unwrap(); + task.await.unwrap().unwrap(); + }; + tokio::time::timeout(Duration::from_secs(2), drain) + .await + .expect("heartbeat timeout must still bound an idle connection"); + } + } + #[tokio::test] async fn pending_fetch_survives_an_unrelated_ready_branch() { let request = FetchRequest { @@ -910,16 +999,122 @@ mod tests { ); } - #[test] - fn unresolved_channel_operations_are_not_released_on_disconnect() { - let now = Instant::now(); - let in_flight = HashMap::from([(7, delivery(now, 70)), (8, delivery(now, 80))]); - let pending = HashSet::from([7]); + #[tokio::test(flavor = "current_thread")] + async fn queued_finish_and_requeue_are_released_and_redelivered_on_disconnect() { + let root = tempdir().unwrap(); + let broker = Broker::open(BrokerConfig { + data_path: root.path().into(), + ..BrokerConfig::default() + }) + .unwrap(); + broker.create_channel("events", "workers").await.unwrap(); + broker + .publish( + "events", + vec![b"first".to_vec(), b"second".to_vec()], + Duration::ZERO, + ) + .await + .unwrap(); + let message_timeout = Duration::from_secs(15 * 60); + let batch = broker + .fetch_batch_retained( + "events", + "workers", + 2, + usize::MAX, + Duration::ZERO, + Some(message_timeout), + ) + .await + .unwrap(); + let (deliveries, mut guard) = batch.into_parts(); + assert_eq!(deliveries.len(), 2); + let expected = deliveries + .iter() + .map(|delivery| (delivery.id, delivery.body.clone())) + .collect::>(); + + let mut state = session_state(); + state.message_timeout = message_timeout; + let mut operations = Vec::with_capacity(deliveries.len()); + for delivery in deliveries { + let (token, lease) = guard.accept_with_lease(delivery.id).unwrap(); + state.record_delivery( + delivery.id, + InFlightDelivery { + deadline: Instant::now() + message_timeout, + token, + _lease: Some(lease), + }, + ); + state.mark_channel_operation_pending(delivery.id); + operations.push((delivery.id, token)); + } + drop(guard); + + let ChannelOps { + sender, + task, + active, + .. + } = start_channel_ops(broker.clone(), operations.len()); + sender + .finish( + "events".into(), + "workers".into(), + operations[0].0, + operations[0].1, + ) + .unwrap(); + sender + .requeue( + "events".into(), + "workers".into(), + operations[1].0, + operations[1].1, + Duration::ZERO, + ) + .unwrap(); + assert!(active.lock().is_empty()); + drop(sender); + task.abort(); + let _ = task.await; + + let active = active.lock().clone(); + let releaseable = + releaseable_in_flight_deliveries(std::mem::take(&mut state.in_flight), &active); assert_eq!( - releaseable_in_flight_deliveries(in_flight, &pending), - vec![(8, 80)] + releaseable, + operations + .iter() + .map(|(id, token)| (*id, *token)) + .collect::>() ); + broker.release_deliveries("events", "workers", &releaseable); + drop(state); + + let redelivery = tokio::time::timeout( + Duration::from_secs(1), + broker.fetch_batch_retained( + "events", + "workers", + 2, + usize::MAX, + Duration::ZERO, + Some(Duration::ZERO), + ), + ) + .await + .expect("released deliveries should be immediately fetchable") + .unwrap(); + let (redeliveries, _redelivery_guard) = redelivery.into_parts(); + let actual = redeliveries + .iter() + .map(|delivery| (delivery.id, delivery.body.clone())) + .collect::>(); + assert_eq!(actual, expected); } #[test] diff --git a/crates/storage/src/segment.rs b/crates/storage/src/segment.rs index 3efc5c4..d5c61c7 100644 --- a/crates/storage/src/segment.rs +++ b/crates/storage/src/segment.rs @@ -524,6 +524,7 @@ impl SegmentLog { self.sealed_indexes.insert(path.to_path_buf(), summary); self.resident_records .retain(|location| location.segment.as_ref() != path); + self.resident_records.shrink_to_fit(); Ok(()) } diff --git a/crates/storage/src/segment/append.rs b/crates/storage/src/segment/append.rs index 63b702e..0b73fba 100644 --- a/crates/storage/src/segment/append.rs +++ b/crates/storage/src/segment/append.rs @@ -14,8 +14,9 @@ pub(super) fn write_parts( .collect(); let mut part_index = 0usize; let mut part_offset = 0usize; + let mut vectors = Vec::with_capacity(MAX_VECTORS); while part_index < parts.len() { - let mut vectors = Vec::with_capacity(MAX_VECTORS); + vectors.clear(); vectors.push(IoSlice::new(&parts[part_index][part_offset..])); vectors.extend( parts[part_index + 1..] @@ -23,7 +24,10 @@ pub(super) fn write_parts( .take(MAX_VECTORS - 1) .map(|part| IoSlice::new(part)), ); - let written = writer.write_vectored(&vectors)?; + let written = match writer.write_vectored(&vectors) { + Err(error) if error.kind() == io::ErrorKind::Interrupted => continue, + result => result?, + }; if written == 0 { return Err(io::Error::new( io::ErrorKind::WriteZero, @@ -50,6 +54,59 @@ pub(super) fn write_parts( mod tests { use super::*; + struct InterruptingShortWriter { + output: Vec, + calls: usize, + interrupted_calls: usize, + short_writes: usize, + } + + impl InterruptingShortWriter { + fn new() -> Self { + Self { + output: Vec::new(), + calls: 0, + interrupted_calls: 0, + short_writes: 0, + } + } + } + + impl Write for InterruptingShortWriter { + fn write(&mut self, bytes: &[u8]) -> io::Result { + self.output.extend_from_slice(bytes); + Ok(bytes.len()) + } + + fn write_vectored(&mut self, buffers: &[IoSlice<'_>]) -> io::Result { + self.calls += 1; + if self.calls % 2 == 1 { + self.interrupted_calls += 1; + return Err(io::Error::new(io::ErrorKind::Interrupted, "retry")); + } + + let available = buffers.iter().map(|buffer| buffer.len()).sum::(); + let limit = available.min(2); + let mut remaining = limit; + for buffer in buffers { + let written = buffer.len().min(remaining); + self.output.extend_from_slice(&buffer[..written]); + remaining -= written; + if remaining == 0 { + break; + } + } + if limit < available { + self.short_writes += 1; + } + Ok(limit) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } + } + #[test] fn writes_all_parts_in_order() { let header = [7u8; HEADER_LEN]; @@ -58,4 +115,18 @@ mod tests { assert_eq!(&output[..HEADER_LEN], &header); assert_eq!(&output[HEADER_LEN..], b"onetwo"); } + + #[test] + fn retries_interrupted_vectored_writes_and_preserves_part_order() { + let header = [7u8; HEADER_LEN]; + let mut writer = InterruptingShortWriter::new(); + + write_parts(&mut writer, &header, &[b"one", b"", b"two", b"three"]).unwrap(); + + let mut expected = header.to_vec(); + expected.extend_from_slice(b"onetwothree"); + assert_eq!(writer.output, expected); + assert!(writer.interrupted_calls > 0); + assert!(writer.short_writes > 0); + } } diff --git a/crates/storage/src/segment/tests.rs b/crates/storage/src/segment/tests.rs index 372e970..10419ce 100644 --- a/crates/storage/src/segment/tests.rs +++ b/crates/storage/src/segment/tests.rs @@ -53,22 +53,33 @@ fn multipart_append_preserves_the_record_wire_format() { #[test] fn sealed_segments_recover_from_the_sidecar_index() { let directory = tempdir().unwrap(); - let mut log = SegmentLog::open(directory.path(), 100).unwrap(); - log.append(record(0, &[1; 20]), true).unwrap(); + let mut log = SegmentLog::open(directory.path(), 128 * 1024).unwrap(); + for _ in 0..1024 { + log.append(record(0, &[1; 20]), false).unwrap(); + } + log.seal().unwrap(); log.append(record(0, &[2; 20]), true).unwrap(); let sealed = log.segment_paths().unwrap()[0].clone(); log.persist_recovery_index(&sealed, b"queue-metadata".to_vec()) .unwrap(); + assert!( + log.resident_records.capacity() <= 4, + "sealed records retain capacity for {} locations with only {} active location(s)", + log.resident_records.capacity(), + log.resident_records.len(), + ); drop(log); - let log = SegmentLog::open(directory.path(), 100).unwrap(); - assert_eq!(log.recovery_report().indexed_records, 1); + let log = SegmentLog::open(directory.path(), 128 * 1024).unwrap(); + assert_eq!(log.recovery_report().indexed_records, 1024); assert_eq!(log.recovery_report().scanned_records, 1); assert_eq!( log.load_recovery_metadata(&sealed).unwrap(), Some(b"queue-metadata".to_vec()) ); assert_eq!(log.read(1).unwrap().unwrap().payload, vec![1; 20]); + assert_eq!(log.read(1024).unwrap().unwrap().payload, vec![1; 20]); + assert_eq!(log.read(1025).unwrap().unwrap().payload, vec![2; 20]); } #[test] diff --git a/deploy/helm/rustqueue/Chart.yaml b/deploy/helm/rustqueue/Chart.yaml index 38df640..533995e 100644 --- a/deploy/helm/rustqueue/Chart.yaml +++ b/deploy/helm/rustqueue/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: rustqueue description: Kubernetes-native share-nothing NSQ-compatible message queue type: application -version: 0.10.0 -appVersion: "0.10.0" +version: 0.10.1 +appVersion: "0.10.1" kubeVersion: ">=1.28.0-0" keywords: - messaging diff --git a/deploy/helm/rustqueue/values.yaml b/deploy/helm/rustqueue/values.yaml index 8729276..bf1a0f9 100644 --- a/deploy/helm/rustqueue/values.yaml +++ b/deploy/helm/rustqueue/values.yaml @@ -5,7 +5,7 @@ operator: replicas: 2 image: repository: rustqueue-operator - tag: "0.10.0" + tag: "0.10.1" pullPolicy: IfNotPresent log: rustqueue_operator=info resources: @@ -20,7 +20,7 @@ operator: queue: create: true name: rustqueue - image: rustqueue:0.10.0 + image: rustqueue:0.10.1 imagePullPolicy: IfNotPresent minBrokers: 1 maxBrokers: 500 diff --git a/docs/architecture/nsq-performance.md b/docs/architecture/nsq-performance.md index 11dba9d..7069f1f 100644 --- a/docs/architecture/nsq-performance.md +++ b/docs/architecture/nsq-performance.md @@ -24,6 +24,26 @@ serialized, and reservation stops at a durable position that advances only after the sync succeeds. Existing durable messages can therefore continue to flow while the next group waits for its acknowledged-durability boundary. +Delivery admission has a second, node-wide bound for in-flight message count. +`limits.node_delivery_inflight_messages` defaults to 262,144 and is passed to +`BrokerConfig.delivery_inflight_messages`; set +`RUSTQUEUE_NODE_DELIVERY_INFLIGHT_MESSAGES` to override it. This slot budget is +separate from the 512 MiB node-wide payload working-set budget: it covers the +queue reservation and consumer-session metadata for unacknowledged messages, +and a slot returns only after both holders release it. Empty messages consume a +slot as well. When the count reaches its limit, later reservations pause, so +throughput under saturation depends on the configured capacity and how long +consumers retain their deliveries. FIN/REQ +completion, timeout, cancellation or disconnect cleanup, and TTL cleanup +release slots according to that lifecycle. The slot count is visible in +`delivery_budget.in_flight_messages` and as the Prometheus gauge +`rustqueue_delivery_inflight_messages`. + +TCP command processing and response writes have a bounded progress timeout. +After a command completes, the peer receives a fresh heartbeat idle window; +time spent waiting on Broker work or response I/O does not consume that window. +Idle peers and blocked writes still time out. + ## Material differences from NSQ | Area | NSQ v1.3.0 | RustQueue | Performance consequence | @@ -63,6 +83,42 @@ Track `rustqueue_publish_unsynced_messages`, `rustqueue_publish_unsynced_bytes`, and `rustqueue_publish_sync_lag_seconds` alongside throughput and ACK latency. +## Release qualification + +Release comparisons use `scripts/benchmark-qualify.sh`. OrbStack remains the +default environment. When shared host paging prevents calibration, the +`performance-qualification` GitHub workflow uses a fresh Linux ARM64 runner and +records that environment separately. It first runs two interleaved pairs of +the exact same Broker image in all three profiles, with 30 seconds of warmup +and 120 seconds of measurement. Each profile must stay within 5 percent +throughput spread and 10 percent PUB-ACK p99 spread before the ten-pair +baseline/candidate comparison runs. Broker and load generator each receive +two CPUs and 2 GiB with swap disabled. Delivery integrity and complete-drain +checks remain required; calibration failure stops comparison. +The Linux runner fixes the Broker to CPUs 0-1 and the load generator to CPUs 2-3, +so they do not compete for the same cores. Their CPU quotas and effective CPU +sets are recorded; OrbStack keeps its default CPU placement. Both A/A and A/B +use the same placement without changing the profiles or acceptance thresholds. +The workflow allows six hours for calibration, all ten comparison pairs and +complete consumer drains. + +The workflow runs when its definition or the qualification script changes in +a PR, and can also be dispatched with an explicit published baseline tag. +Its artifacts contain raw measurements, source commits, binary hashes, +environment details, calibration and the existing regression verdict. +On the Linux runner, RSS is read from the Broker's host PID instead of spawning +`docker exec` inside its CPU-limited cgroup every second. Before/after cgroup CPU, +I/O and memory-event counters, host VM counters, CPU utilization counters, +block-device counters, load averages and PSI are saved with each trial. +The named data volume's filesystem, device and free capacity are recorded; +RAM-backed data filesystems are rejected. The load generator's cgroup +CPU usage and throttling counters are sampled every second from the host. +Unavailable pressure counters are recorded explicitly. +These distinguish generator CPU limits from Broker limits and host contention; +they do not change the measurement windows or acceptance thresholds. +Comparisons apply only to the recorded host and architecture. A hosted runner +is not assumed suitable until its same-binary calibration passes. + ## Next candidates to measure before changing semantics 1. Profile Channel WAL `FIN`/`REQ` commit time against reservation latency; diff --git a/docs/architecture/share-nothing-v7.md b/docs/architecture/share-nothing-v7.md index c671d37..6ec3087 100644 --- a/docs/architecture/share-nothing-v7.md +++ b/docs/architecture/share-nothing-v7.md @@ -357,7 +357,6 @@ discard is a separate, audited administrative action. channels/ .checkpoint .wal - delayed.index dlq-outbox/ audit/ ``` @@ -372,15 +371,26 @@ discard is a separate, audited administrative action. - The payload is stored once per broker regardless of channel count. - Tail short writes are truncated during recovery. - Each sealed segment has an atomic CRC-protected recovery index containing - record locations and fixed-size, per-entry-CRC queue metadata. Startup reads - only the fixed header and first/last entries; pages are loaded on demand into - a Broker-wide byte-bounded cache, while background scrub validates the full + record locations and fixed-size, per-entry-CRC queue metadata. Storage locations + recover from the fixed header; queue metadata is validated in bounded pages at + startup to recover ranges and per-segment delivery-deadline bounds. Delivery + pages are loaded on demand into a Broker-wide byte-bounded cache, while + background scrub validates the full sidecar checksum. A missing or bad index safely falls back to a full scan and is rebuilt. - There is no backlog message-count limit. Sealed segments retain one summary in memory, so cold metadata residency scales with segment count rather than message count; disk high-watermark and minimum-free-space policy controls - publish rejection. + publish rejection. Sealing also releases unused message-metadata and record- + location buffer capacity before returning its Broker-wide cache budget; + spilled Topics cannot retain buffers from their former active tails. +- DPUB deadlines stay in queue metadata. Sealed segments retain only minimum and + maximum deadlines, rather than one in-memory position per delayed message. + Statistics count fully deferred segments by range and stream bounded pages for + mixed deadlines, excluding ACKs, in-flight deliveries, and duplicate requeue + bookkeeping. Metadata read failures propagate from the Rust statistics APIs + as `Result` errors, fail HTTP statistics/metrics, and isolate storage; they + never produce a successful partial snapshot. - The active tail is always fully scanned. Cold indexed payload corruption is detected and isolated on payload read or by background scrub. Automatic scrub and normal GC wait for the configured startup quiet period (30 seconds by @@ -398,6 +408,15 @@ discard is a separate, audited administrative action. - Each local channel stores barrier/cursor, ACK floor, bounded sparse ACK, requeue targets, attempts that reached a durable `REQ`, paused state, and retention cursor. +- Scanning future DPUB messages retains one rewind position and the earliest + wakeup time per Channel. When that deadline passes, delivery revisits shared + Topic metadata from the rewind position; no per-message DPUB scan entries + are added to the Channel checkpoint. This scan cursor is transient and + recovery rescans from the ACK floor. Explicit REQ targets and attempts keep + their existing WAL and checkpoint semantics. + Recovery removes legacy scan deadlines that match shared metadata and have + no delivery attempt, reading at most one metadata page at a time per Channel. + Recorded position fences are captured before absent state is discarded. - State mutations append to a per-channel WAL. - Concurrent FIN/REQ mutations for one topic are combined into groups of at most 64 requests or 1 ms. Each touched durable channel is fsynced once before diff --git a/docs/operations/console.md b/docs/operations/console.md index cea0efe..9fe7cce 100644 --- a/docs/operations/console.md +++ b/docs/operations/console.md @@ -41,6 +41,10 @@ The interface follows the browser language and operating-system theme on first use. Both can be overridden from the header and are stored locally in the browser. Chinese and English are supported. +Below the 1056px desktop breakpoint, navigation starts closed and the header +menu button opens it. Selecting a page, clicking the overlay or pressing Escape +inside navigation closes it so the page can use the full viewport width. + ## Live trends The backend polls a lightweight runtime/revision head every two seconds and @@ -97,6 +101,8 @@ preview and require the exact resource name. Deletion first persists a tombstone, then synchronizes the fence to all brokers before removing data. The default tombstone lifetime is 10 minutes and can be changed with `console.management.tombstoneSeconds`. +Explicitly recreating a deleted Topic applies the newly selected delivery mode +and TTL. Choosing `RELIABLE` clears the previous TTL. The Console selects a healthy, non-maintenance, disk-eligible broker for new Topics. A mutation first persists an operation ID and owner progress in the diff --git a/docs/operations/kubernetes.md b/docs/operations/kubernetes.md index e373562..c506f46 100644 --- a/docs/operations/kubernetes.md +++ b/docs/operations/kubernetes.md @@ -23,6 +23,11 @@ The chart starts two Operator replicas. They coordinate through the reconciles RustQueue resources. The Broker and discovery PDBs are owned by the RustQueue resource; the Operator PDB is owned by Helm. +The Proxy DaemonSet keeps the existing Ready Pod until its replacement is +Ready, using `maxUnavailable: 0` and `maxSurge: 1`. Reserve capacity for one +additional Proxy Pod during an update. This also preserves a publish endpoint +when `proxyNodeSelector` selects only one node. + ## Reading cluster state ```sh @@ -163,6 +168,10 @@ maintenance request exists per RustQueue, which prevents conflicting drains. ## Rolling an image +When a completed rollout re-enters readiness recovery after a Pod restart, +its timeout clock starts again. Repeated checks during that recovery keep the +same start time, so a stalled recovery still reaches the configured timeout. + Set the image through Helm or patch the RustQueue. The Operator performs target image capability preflight, validates every current Broker, quiesces publishing and freezes new delivery, waits for already-issued leases and delivery buffers @@ -178,7 +187,7 @@ Canary approval is optional: ```sh helm upgrade rustqueue deploy/helm/rustqueue \ --namespace rustqueue \ - --set queue.image=registry.example/rustqueue:0.10.0 \ + --set queue.image=registry.example/rustqueue:0.10.1 \ --set queue.rollout.requireCanaryApproval=true rustqueuectl -n rustqueue rollout approve @@ -258,6 +267,16 @@ Permanent PVC loss permanently loses that Broker's unconsumed messages. K8S_ACCEPTANCE=1 ./scripts/release-gate.sh ``` +The acceptance scripts apply the checked-in CRDs before installing the chart +and use Helm's `--skip-crds` option. This keeps CRD field ownership with the +explicit bootstrap step and avoids a conflicting second apply from Helm 4. +Read-only health probes allow up to 60 seconds for Service routing to converge +after Pod readiness; publishing and management requests are not retried by +this probe policy. +The multi-Broker rollout check starts a phase watch before canary approval and +reads readiness and observed generation from the same API response. This +captures short rolling phases without combining status from different updates. + The second command is destructive only inside the dedicated OrbStack test namespaces. It validates real A/B binaries, canary pause, Lease failover, online PVC expansion, rollback fencing and a Go-client `missing=0` ledger. It diff --git a/docs/releases/v0.10.1.md b/docs/releases/v0.10.1.md new file mode 100644 index 0000000..87dc7a6 --- /dev/null +++ b/docs/releases/v0.10.1.md @@ -0,0 +1,59 @@ +# RustQueue 0.10.1 + +RustQueue 0.10.1 fixes queue memory and recovery failures, NSQ heartbeat timing, +completed recovery status, TLS CLI startup, and producer-proxy rollout continuity. +The optional WebSocket live subscription introduced in 0.10.0 remains default-off. + +## Queue memory and recovery + +- Sealing releases unused metadata and record-location buffer capacity before + returning cache credits. Cold Topics no longer retain their former active buffers. +- Delayed-message deadlines stay in queue metadata. Statistics use segment ranges + and bounded pages, and consumer scans keep one rewind position per Channel. +- Recovery compacts older DPUB scan entries one Channel at a time. Explicit REQ + deadlines and attempts remain durable; lost relaxed positions are fenced before + absent state is removed and are never reused. +- A node-wide in-flight message limit bounds empty-message and consumer-session + metadata as well as the existing payload-byte budget. + +Rust statistics APIs return `Result` so metadata read failures can propagate. +HTTP statistics/metrics fail and isolate uncertain storage rather than returning +successful partial counts. Rust callers must handle these errors; successful +HTTP response schemas remain unchanged. + +## Runtime and Console fixes + +- NSQ heartbeat send timing advances after awaited response writes complete. +- Completed Operator recovery stops using the expired recovery deadline while + waiting for a replacement Broker to become Ready. +- The CLI installs its TLS crypto provider before parsing commands. +- Producer-proxy DaemonSets retain an endpoint while rolling. +- Recreating a deleted Topic applies the requested delivery mode and TTL instead + of inheriting its old policy; choosing reliable delivery clears the old TTL. +- Console navigation supports mobile menu dismissal and focus restoration, and + snapshot failures remain visible to the user. The footer reads the packaged + version instead of displaying a hard-coded previous release. + +## Upgrade + +The v7 data format, RCC7 Channel checkpoints and RQTMv2 metadata remain compatible. +No Topic data migration is required. Apply the CRDs before upgrading an existing +Operator installation, then update Broker and Operator images to 0.10.1. + +## Release assets + +- `rustqueue-0.10.1-linux-x86_64.tar.gz` +- `rustqueue-0.10.1-linux-aarch64.tar.gz` +- `rustqueue-0.10.1-source.tar.gz` +- `rustqueue-0.10.1.tgz` +- `SHA256SUMS-0.10.1` + +The production-gate workflow builds the release from the commit on `main`, runs +core, compatibility, UI, Helm and fuzz checks, builds both Linux architectures, +and verifies the published asset checksums. Local scoped runtime checks are +recorded separately from throughput and latency calibration; memory observations +are not a calibrated performance comparison. +The Linux performance workflow gives Broker and load generator separate CPU +sets within their existing two-CPU limits, and records the effective placement. +Calibration still gates the comparison using the original measurement windows +and spread thresholds. diff --git a/fuzz/Cargo.lock b/fuzz/Cargo.lock index 2a4e745..6421472 100644 --- a/fuzz/Cargo.lock +++ b/fuzz/Cargo.lock @@ -2071,7 +2071,7 @@ dependencies = [ [[package]] name = "rustqueue-discovery" -version = "0.10.0" +version = "0.10.1" dependencies = [ "anyhow", "axum", @@ -2110,7 +2110,7 @@ dependencies = [ [[package]] name = "rustqueue-protocol" -version = "0.10.0" +version = "0.10.1" dependencies = [ "bytes", "serde", @@ -2120,7 +2120,7 @@ dependencies = [ [[package]] name = "rustqueue-proxy" -version = "0.10.0" +version = "0.10.1" dependencies = [ "anyhow", "axum", @@ -2140,7 +2140,7 @@ dependencies = [ [[package]] name = "rustqueue-queue" -version = "0.10.0" +version = "0.10.1" dependencies = [ "anyhow", "bytes", @@ -2160,7 +2160,7 @@ dependencies = [ [[package]] name = "rustqueue-storage" -version = "0.10.0" +version = "0.10.1" dependencies = [ "anyhow", "crc32c", @@ -2172,7 +2172,7 @@ dependencies = [ [[package]] name = "rustqueue-telemetry" -version = "0.10.0" +version = "0.10.1" dependencies = [ "serde", ] diff --git a/rustqueue.example.toml b/rustqueue.example.toml index 789a2ae..41e0859 100644 --- a/rustqueue.example.toml +++ b/rustqueue.example.toml @@ -77,6 +77,8 @@ node_publish_inflight_bytes = 536870912 connection_publish_inflight_bytes = 83886080 # Payload bodies held by slow consumers are bounded separately from the read cache. node_delivery_inflight_bytes = 536870912 +# Covers queue and consumer metadata until acknowledgement or release. +node_delivery_inflight_messages = 262144 connection_delivery_inflight_bytes = 33554432 max_rdy_count = 2500 max_connections = 10000 diff --git a/scripts/acceptance-k8s.sh b/scripts/acceptance-k8s.sh index 7840f91..2d51d3e 100755 --- a/scripts/acceptance-k8s.sh +++ b/scripts/acceptance-k8s.sh @@ -320,6 +320,36 @@ run_console_management_acceptance() { run_curl console-publish-after-recreate -fsS -X POST --data-binary recreated \ "http://$QUEUE-proxy:4151/pub?topic=$topic" >/dev/null + local mode ttl_seconds owner + for mode in TTL_DISCARD RELIABLE; do + ttl_seconds="" + [[ "$mode" != TTL_DISCARD ]] || ttl_seconds=30 + preview=$(console_preview "$base" "$origin" "$cookie" "$csrf" topic delete "$topic" "") + token=$(jq -er '.action_token' <<<"$preview") + curl -fsS -b "$cookie" -H "Origin: $origin" -H 'Content-Type: application/json' \ + -H "X-RustQueue-CSRF: $csrf" \ + --data "$(console_apply_body topic delete "$topic" "" "$token" "$topic")" \ + "$base/api/v1/management/apply" >/dev/null + wait_managed_resource rustqueuetopics "$topic" "" TOMBSTONED + wait_console_topic "$base" "$topic" TOMBSTONED + + preview=$(console_preview "$base" "$origin" "$cookie" "$csrf" topic create "$topic" "" "$mode" "$ttl_seconds") + token=$(jq -er '.action_token' <<<"$preview") + owner=$(jq -er '.impact.owners[0]' <<<"$preview") + curl -fsS -b "$cookie" -H "Origin: $origin" -H 'Content-Type: application/json' \ + -H "X-RustQueue-CSRF: $csrf" \ + --data "$(console_apply_body topic create "$topic" "" "$token" "" "$mode" "$ttl_seconds")" \ + "$base/api/v1/management/apply" >/dev/null + wait_managed_resource rustqueuetopics "$topic" "" ACTIVE + wait_console_topic "$base" "$topic" ACTIVE + kubectl -n "$NAMESPACE" exec "$owner" -c broker -- \ + curl -fsS "http://127.0.0.1:4151/v1/stats?topic=$topic" | \ + jq -e --arg topic "$topic" --arg mode "$mode" --arg ttl "$ttl_seconds" \ + 'any(.topics[]; .name == $topic and .delivery_mode == $mode + and .message_ttl_seconds == (if $ttl == "" then null else ($ttl | tonumber) end))' \ + >/dev/null + done + # Simulate a Console crash after persisting the operation but before calling a broker. local operation_now resource operation_now=$(date +%s000) @@ -374,18 +404,24 @@ run_console_management_acceptance() { console_preview() { local base=$1 origin=$2 cookie=$3 csrf=$4 kind=$5 action=$6 topic=$7 channel=$8 + local mode=${9:-} ttl=${10:-} curl -fsS -b "$cookie" -H "Origin: $origin" -H 'Content-Type: application/json' \ -H "X-RustQueue-CSRF: $csrf" \ --data "$(jq -cn --arg kind "$kind" --arg action "$action" --arg topic "$topic" \ - --arg channel "$channel" '{kind:$kind,action:$action,topic:$topic,channel:(if $channel == "" then null else $channel end)}')" \ + --arg channel "$channel" --arg mode "$mode" --arg ttl "$ttl" \ + '{kind:$kind,action:$action,topic:$topic,channel:(if $channel == "" then null else $channel end)} + + (if $mode == "" then {} else {delivery_mode:$mode,message_ttl_seconds:(if $ttl == "" then null else ($ttl | tonumber) end)} end)')" \ "$base/api/v1/management/preview" } console_apply_body() { local kind=$1 action=$2 topic=$3 channel=$4 token=$5 confirmation=$6 + local mode=${7:-} ttl=${8:-} jq -cn --arg kind "$kind" --arg action "$action" --arg topic "$topic" \ --arg channel "$channel" --arg token "$token" --arg confirmation "$confirmation" \ - '{kind:$kind,action:$action,topic:$topic,channel:(if $channel == "" then null else $channel end),action_token:$token,confirmation:$confirmation}' + --arg mode "$mode" --arg ttl "$ttl" \ + '{kind:$kind,action:$action,topic:$topic,channel:(if $channel == "" then null else $channel end),action_token:$token,confirmation:$confirmation} + + (if $mode == "" then {} else {delivery_mode:$mode,message_ttl_seconds:(if $ttl == "" then null else ($ttl | tonumber) end)} end)' } require kubectl @@ -422,6 +458,7 @@ kubectl wait --for=condition=Established crd/rustqueues.rustqueue.io --timeout=6 kubectl wait --for=condition=Established crd/rustqueuetopics.rustqueue.io --timeout=60s kubectl wait --for=condition=Established crd/rustqueuechannels.rustqueue.io --timeout=60s helm upgrade --install "$RELEASE" "$CHART" \ + --skip-crds \ --namespace "$NAMESPACE" \ --set-string operator.image.repository="${OPERATOR_IMAGE%:*}" \ --set-string operator.image.tag="${OPERATOR_IMAGE##*:}" \ @@ -446,8 +483,10 @@ kubectl -n "$NAMESPACE" wait --for=condition=Ready pod/"$QUEUE-0" --timeout=180s PVC_BEFORE=$(kubectl -n "$NAMESPACE" get pvc -l app.kubernetes.io/instance="$QUEUE",app.kubernetes.io/component=broker -o jsonpath='{range .items[*]}{.metadata.name}={.metadata.uid}{"\n"}{end}') [[ -n "$PVC_BEFORE" ]] || { echo "operator did not create the broker PVC" >&2; exit 1; } -run_curl discovery-health -fsS "http://$QUEUE-discovery:4161/v1/health" -run_curl proxy-health -fsS "http://$QUEUE-proxy:4151/v1/health" +run_curl discovery-health -fsS --max-time 5 --retry 30 --retry-delay 1 \ + --retry-max-time 60 --retry-connrefused "http://$QUEUE-discovery:4161/v1/health" +run_curl proxy-health -fsS --max-time 5 --retry 30 --retry-delay 1 \ + --retry-max-time 60 --retry-connrefused "http://$QUEUE-proxy:4151/v1/health" CONSOLE_SNAPSHOT=$(kubectl -n "$NAMESPACE" exec "$QUEUE-0" -c broker -- \ curl -fsS "http://$QUEUE-console:4180/api/v1/snapshot") [[ "$(jq -r '.complete' <<<"$CONSOLE_SNAPSHOT")" == "true" ]] || { @@ -546,6 +585,7 @@ kubectl -n "$NAMESPACE" patch rustqueue "$QUEUE" --type=merge \ wait_queue_ready 300 kubectl -n "$NAMESPACE" rollout status deployment/"$QUEUE-discovery" --timeout=180s kubectl -n "$NAMESPACE" rollout status daemonset/"$QUEUE-proxy" --timeout=180s -run_curl proxy-health-after-recovery -fsS "http://$QUEUE-proxy:4151/v1/health" +run_curl proxy-health-after-recovery -fsS --max-time 5 --retry 30 --retry-delay 1 \ + --retry-max-time 60 --retry-connrefused "http://$QUEUE-proxy:4151/v1/health" echo "OrbStack Kubernetes share-nothing v7 acceptance passed" diff --git a/scripts/acceptance-multi-broker-k8s.sh b/scripts/acceptance-multi-broker-k8s.sh index 1aa3efe..b1115ab 100755 --- a/scripts/acceptance-multi-broker-k8s.sh +++ b/scripts/acceptance-multi-broker-k8s.sh @@ -18,6 +18,8 @@ NODE_KEEPER="" STORAGE_CLASS="" STORAGE_CLASS_CREATED=0 CONSOLE_FORWARD_PID="" +ROLLOUT_WATCH_PID="" +ROLLOUT_WATCH_FILE="" source "$(dirname "$0")/lib/console-multi-owner.sh" source "$(dirname "$0")/lib/token-rotation.sh" @@ -32,6 +34,7 @@ diagnostics() { kubectl -n "$NAMESPACE" logs -l app.kubernetes.io/component=operator --tail=300 || true kubectl -n "$NAMESPACE" logs -l app.kubernetes.io/component=console --tail=300 || true kubectl -n "$NAMESPACE" logs -l app.kubernetes.io/component=discovery --tail=300 || true + kubectl -n "$NAMESPACE" logs -l app.kubernetes.io/component=proxy --tail=300 --prefix || true kubectl -n "$NAMESPACE" logs -l app.kubernetes.io/component=broker --tail=300 --prefix || true kubectl -n "$NAMESPACE" logs default-lookup-bootstrap --tail=300 || true kubectl -n "$NAMESPACE" logs operational-ledger --tail=300 || true @@ -39,6 +42,13 @@ diagnostics() { cleanup() { code=$? + if [[ -n "$ROLLOUT_WATCH_PID" ]]; then + kill "$ROLLOUT_WATCH_PID" >/dev/null 2>&1 || true + wait "$ROLLOUT_WATCH_PID" 2>/dev/null || true + fi + if [[ -n "$ROLLOUT_WATCH_FILE" ]]; then + rm -f "$ROLLOUT_WATCH_FILE" + fi if [[ -n "$CONSOLE_FORWARD_PID" ]]; then kill "$CONSOLE_FORWARD_PID" >/dev/null 2>&1 || true wait "$CONSOLE_FORWARD_PID" 2>/dev/null || true @@ -373,6 +383,7 @@ kubectl create namespace "$NAMESPACE" >/dev/null kubectl wait --for=condition=Established crd/rustqueuetopics.rustqueue.io --timeout=60s kubectl wait --for=condition=Established crd/rustqueuechannels.rustqueue.io --timeout=60s helm upgrade --install "$RELEASE" "$CHART" \ + --skip-crds \ --namespace "$NAMESPACE" \ --set-string operator.image.repository="${OPERATOR_IMAGE%:*}" \ --set-string operator.image.tag="${OPERATOR_IMAGE##*:}" \ @@ -505,19 +516,32 @@ kubectl -n "$NAMESPACE" delete pod "$leader" --wait=false >/dev/null wait_operator_failover "$leader" 90 wait_queue_phase RolloutAwaitingApproval 30 approved_revision=$(kubectl -n "$NAMESPACE" get rustqueue "$QUEUE" -o jsonpath='{.status.currentOperation.revision}') -kubectl -n "$NAMESPACE" patch rustqueue "$QUEUE" --type=merge \ - -p "{\"spec\":{\"rollout\":{\"approvedRevision\":\"$approved_revision\"}}}" >/dev/null +ROLLOUT_WATCH_FILE=$(mktemp) +kubectl -n "$NAMESPACE" get rustqueue "$QUEUE" --watch \ + -o 'jsonpath={.status.phase}{"\n"}' >"$ROLLOUT_WATCH_FILE" & +ROLLOUT_WATCH_PID=$! +deadline=$((SECONDS + 30)) +while [[ ! -s "$ROLLOUT_WATCH_FILE" ]] && (( SECONDS < deadline )); do sleep .1; done +[[ -s "$ROLLOUT_WATCH_FILE" ]] || { echo "rollout phase watch did not start" >&2; exit 1; } +approval_generation=$(kubectl -n "$NAMESPACE" patch rustqueue "$QUEUE" --type=merge \ + -p "{\"spec\":{\"rollout\":{\"approvedRevision\":\"$approved_revision\"}}}" \ + -o json | jq -r '.metadata.generation') saw_rolling=0 deadline=$((SECONDS + 420)) while (( SECONDS < deadline )); do - phase=$(kubectl -n "$NAMESPACE" get rustqueue "$QUEUE" -o jsonpath='{.status.phase}' 2>/dev/null || true) - observed=$(kubectl -n "$NAMESPACE" get rustqueue "$QUEUE" -o jsonpath='{.status.observedGeneration}' 2>/dev/null || true) - generation=$(kubectl -n "$NAMESPACE" get rustqueue "$QUEUE" -o jsonpath='{.metadata.generation}' 2>/dev/null || true) - [[ "$phase" == "Rolling" ]] && saw_rolling=1 - if [[ "$phase" == "Ready" && "$observed" == "$generation" ]]; then break; fi + rollout_status=$(kubectl -n "$NAMESPACE" get rustqueue "$QUEUE" -o json 2>/dev/null | \ + jq -r '[.status.phase, .status.observedGeneration] | @tsv' || true) + read -r phase observed <<<"$rollout_status" + grep -q '^Rolling$' "$ROLLOUT_WATCH_FILE" && saw_rolling=1 + if [[ "$phase" == "Ready" && "$observed" == "$approval_generation" && "$saw_rolling" == 1 ]]; then break; fi sleep 2 done +kill "$ROLLOUT_WATCH_PID" >/dev/null 2>&1 || true +wait "$ROLLOUT_WATCH_PID" 2>/dev/null || true +ROLLOUT_WATCH_PID="" +printf 'Observed rollout phases:\n' +cat "$ROLLOUT_WATCH_FILE" [[ "$saw_rolling" == "1" && "$phase" == "Ready" ]] || { echo "operator did not complete a visible multi-broker rolling replacement" >&2 exit 1 diff --git a/scripts/benchmark-qualify.sh b/scripts/benchmark-qualify.sh index 29c168c..11a9de8 100755 --- a/scripts/benchmark-qualify.sh +++ b/scripts/benchmark-qualify.sh @@ -14,6 +14,8 @@ DRAIN_TIMEOUT_SECONDS="${DRAIN_TIMEOUT_SECONDS:-1800}" CASES="${CASES:-raw_write sustainable low_load_latency}" QUALIFICATION_DEV="${QUALIFICATION_DEV:-0}" KEEP_IMAGES="${KEEP_IMAGES:-0}" +CALIBRATION_ONLY="${CALIBRATION_ONLY:-0}" +QUALIFICATION_ENVIRONMENT="${QUALIFICATION_ENVIRONMENT:-orbstack}" RESULT_ROOT="$ROOT/benchmarks/results" EVIDENCE_OUTPUT="${EVIDENCE_OUTPUT:-$ROOT/benchmarks/qualifications/v0.8.4-orbstack.json}" RUN_ID="$(date -u +%Y%m%dT%H%M%SZ)-$$" @@ -34,8 +36,10 @@ BASELINE_IMAGE="rustqueue:qualify-baseline-$$" CANDIDATE_IMAGE="rustqueue:qualify-candidate-$$" TOOLS_IMAGE="rustqueue:qualify-tools-$$" ACTIVE_BROKER="" +ACTIVE_LOAD_GENERATOR="" ACTIVE_VOLUME="" SAMPLER_PID="" +CPU_SAMPLER_PID="" SEQUENCE=0 FINAL_DRAIN_ATTEMPTS=30 @@ -50,11 +54,20 @@ require_positive_integer() { } cleanup_run() { + if [[ -n "$CPU_SAMPLER_PID" ]]; then + kill "$CPU_SAMPLER_PID" >/dev/null 2>&1 || true + wait "$CPU_SAMPLER_PID" >/dev/null 2>&1 || true + CPU_SAMPLER_PID="" + fi if [[ -n "$SAMPLER_PID" ]]; then kill "$SAMPLER_PID" >/dev/null 2>&1 || true wait "$SAMPLER_PID" >/dev/null 2>&1 || true SAMPLER_PID="" fi + if [[ -n "$ACTIVE_LOAD_GENERATOR" ]]; then + docker rm -f "$ACTIVE_LOAD_GENERATOR" >/dev/null 2>&1 || true + ACTIVE_LOAD_GENERATOR="" + fi if [[ -n "$ACTIVE_BROKER" ]]; then docker rm -f "$ACTIVE_BROKER" >/dev/null 2>&1 || true ACTIVE_BROKER="" @@ -100,7 +113,24 @@ done docker_context="$(docker context show)" docker_os="$(docker info --format '{{.OperatingSystem}}')" -if [[ "$docker_context" != "orbstack" && "$docker_os" != *OrbStack* ]]; then +docker_cpus="$(docker info --format '{{.NCPU}}')" +broker_args=(--cpus 2 --memory 2g --memory-swap 2g) +load_generator_args=(--cpus 2 --memory 2g --memory-swap 2g) +broker_cpuset="" +load_generator_cpuset="" +platform="OrbStack on macOS" +if [[ "$QUALIFICATION_ENVIRONMENT" == github_actions ]]; then + [[ "${GITHUB_ACTIONS:-}" == true && "${RUNNER_ENVIRONMENT:-}" == github-hosted ]] || + die "github_actions qualification requires a GitHub-hosted runner" + [[ "$docker_cpus" -ge 4 ]] || die "GitHub qualification requires at least four CPUs" + broker_cpuset="0-1" + load_generator_cpuset="2-3" + broker_args+=(--cpuset-cpus "$broker_cpuset") + load_generator_args+=(--cpuset-cpus "$load_generator_cpuset") + platform="GitHub Actions on Linux" +elif [[ "$QUALIFICATION_ENVIRONMENT" != orbstack ]]; then + die "unknown qualification environment $QUALIFICATION_ENVIRONMENT" +elif [[ "$docker_context" != "orbstack" && "$docker_os" != *OrbStack* ]]; then die "Docker must use OrbStack (context=$docker_context, os=$docker_os)" fi @@ -108,7 +138,15 @@ baseline_commit="$(git -C "$ROOT" rev-parse --verify "$BASELINE_REF^{commit}")" candidate_commit="$(git -C "$ROOT" rev-parse --verify "$CANDIDATE_REF^{commit}")" BASELINE_TARGET="$TARGET_ROOT/$baseline_commit" CANDIDATE_TARGET="$TARGET_ROOT/$candidate_commit" -if [[ "$QUALIFICATION_DEV" == 0 ]]; then +if [[ "$CALIBRATION_ONLY" == 1 ]]; then + [[ "$baseline_commit" == "$candidate_commit" ]] || die "A/A requires the same source commit" + [[ "$PAIRS" == 2 && "$WARMUP_SECONDS" == 30 && "$MEASUREMENT_SECONDS" == 120 \ + && "$DRAIN_TIMEOUT_SECONDS" == 1800 \ + && "$CASES" == "raw_write sustainable low_load_latency" ]] || + die "A/A requires all three profiles, two pairs, 30s warmup, 120s measurement and full drain" +elif [[ "$CALIBRATION_ONLY" != 0 ]]; then + die "CALIBRATION_ONLY must be 0 or 1" +elif [[ "$QUALIFICATION_DEV" == 0 ]]; then [[ "$BASELINE_REF" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]] || die "baseline reference must be a version tag" tag_commit="$(git -C "$ROOT" rev-parse --verify "refs/tags/$BASELINE_REF^{commit}")" @@ -158,6 +196,7 @@ baseline_version="$(read_workspace_version "$BASELINE_SOURCE")" candidate_version="$(read_workspace_version "$CANDIDATE_SOURCE")" if [[ "$QUALIFICATION_DEV" == 0 ]]; then expected_baseline_version="${BASELINE_REF#v}" + [[ "$CALIBRATION_ONLY" != 1 ]] || expected_baseline_version="$RELEASE" [[ "$baseline_version" == "$expected_baseline_version" ]] || die "baseline workspace version is $baseline_version, expected $expected_baseline_version" [[ "$candidate_version" == "$RELEASE" ]] || @@ -183,6 +222,7 @@ build_release() { done docker run --rm \ -e RUSTUP_TOOLCHAIN=1.88.0 \ + -e CARGO_BUILD_JOBS=2 \ -e CARGO_INCREMENTAL=0 \ -e CARGO_TARGET_DIR=/target \ -e "RUSTQUEUE_BUILD_VERSION=$version" \ @@ -195,21 +235,23 @@ build_release() { cargo build --locked --release "${binary_args[@]}" } -printf 'Compiling exact baseline Broker %s (%s)\n' "$BASELINE_REF" "$baseline_commit" -build_release "$BASELINE_SOURCE" "$BASELINE_TARGET" "$baseline_version" rustqueued -cp "$BASELINE_TARGET/release/rustqueued" "$BASELINE_RUNTIME/rustqueued" -if command -v shasum >/dev/null 2>&1; then - baseline_binary_sha256="$( - shasum -a 256 "$BASELINE_RUNTIME/rustqueued" | awk '{print $1}' - )" -else - baseline_binary_sha256="$( - sha256sum "$BASELINE_RUNTIME/rustqueued" | awk '{print $1}' - )" +if [[ "$CALIBRATION_ONLY" != 1 ]]; then + printf 'Compiling exact baseline Broker %s (%s)\n' "$BASELINE_REF" "$baseline_commit" + build_release "$BASELINE_SOURCE" "$BASELINE_TARGET" "$baseline_version" rustqueued + cp "$BASELINE_TARGET/release/rustqueued" "$BASELINE_RUNTIME/rustqueued" + if command -v shasum >/dev/null 2>&1; then + baseline_binary_sha256="$( + shasum -a 256 "$BASELINE_RUNTIME/rustqueued" | awk '{print $1}' + )" + else + baseline_binary_sha256="$( + sha256sum "$BASELINE_RUNTIME/rustqueued" | awk '{print $1}' + )" + fi + docker build --target broker \ + -f "$ROOT/benchmarks/Dockerfile.qualify" \ + -t "$BASELINE_IMAGE" "$BASELINE_RUNTIME" fi -docker build --target broker \ - -f "$ROOT/benchmarks/Dockerfile.qualify" \ - -t "$BASELINE_IMAGE" "$BASELINE_RUNTIME" printf 'Compiling candidate Broker %s (%s)\n' "$CANDIDATE_REF" "$candidate_commit" build_release \ @@ -234,6 +276,10 @@ fi docker build --target broker \ -f "$ROOT/benchmarks/Dockerfile.qualify" \ -t "$CANDIDATE_IMAGE" "$CANDIDATE_RUNTIME" +if [[ "$CALIBRATION_ONLY" == 1 ]]; then + docker tag "$CANDIDATE_IMAGE" "$BASELINE_IMAGE" + baseline_binary_sha256="$candidate_binary_sha256" +fi printf 'Building common load generator and qualification evaluator\n' docker build --target tools \ -f "$ROOT/benchmarks/Dockerfile.qualify" \ @@ -243,13 +289,23 @@ docker network create "$NETWORK" >/dev/null sample_rss() { local output=$1 broker=$2 + local pid="" + if [[ "$QUALIFICATION_ENVIRONMENT" == github_actions ]]; then + pid=$(docker inspect "$broker" --format '{{.State.Pid}}') + fi { printf 'timestamp_utc\trss_bytes\n' - while docker inspect "$broker" >/dev/null 2>&1; do - rss="$( - docker exec "$broker" awk '/VmRSS:/ { print $2 * 1024 }' /proc/1/status \ - 2>/dev/null || true - )" + while true; do + if [[ -n "$pid" ]]; then + [[ -r "/proc/$pid/status" && "$(cat "/proc/$pid/comm")" == rustqueued ]] || break + rss=$(awk '/VmRSS:/ { print $2 * 1024 }' "/proc/$pid/status") + else + docker inspect "$broker" >/dev/null 2>&1 || break + rss="$( + docker exec "$broker" awk '/VmRSS:/ { print $2 * 1024 }' /proc/1/status \ + 2>/dev/null || true + )" + fi if [[ "$rss" =~ ^[0-9]+$ ]]; then printf '%s\t%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$rss" fi @@ -258,6 +314,54 @@ sample_rss() { } >"$output" } +capture_host_counters() { + [[ "$QUALIFICATION_ENVIRONMENT" == github_actions ]] || return 0 + local stem=$1 broker=$2 pid group metric + pid=$(docker inspect "$broker" --format '{{.State.Pid}}') + group=$(awk -F: '$1 == "0" { print $3 }' "/proc/$pid/cgroup") + [[ -n "$group" ]] || die "GitHub qualification requires readable cgroup v2 counters" + for metric in cpu.stat cpu.max cpuset.cpus.effective io.stat memory.events; do + cat "/sys/fs/cgroup$group/$metric" >"$stem.$metric" + done + if [[ -r "/sys/fs/cgroup$group/cpu.pressure" ]]; then + cat "/sys/fs/cgroup$group/cpu.pressure" >"$stem.cpu.pressure" + else + printf 'unavailable\n' >"$stem.cpu.pressure" + fi + cat /proc/vmstat >"$stem.host-vmstat" + cat /proc/stat >"$stem.host-stat" + cat /proc/loadavg >"$stem.host-loadavg" + cat /proc/diskstats >"$stem.host-diskstats" + for metric in cpu io memory; do + if [[ -r "/proc/pressure/$metric" ]]; then + cat "/proc/pressure/$metric" >"$stem.host-$metric-pressure" + else + printf 'unavailable\n' >"$stem.host-$metric-pressure" + fi + done +} + +sample_load_generator_cpu() { + local output=$1 container=$2 pid group + pid=$(docker inspect "$container" --format '{{.State.Pid}}') + group=$(awk -F: '$1 == "0" { print $3 }' "/proc/$pid/cgroup") + [[ -n "$group" ]] || return 1 + cat "/sys/fs/cgroup$group/cpu.max" >"${output%.tsv}.cpu.max" + cat "/sys/fs/cgroup$group/cpuset.cpus.effective" >"${output%.tsv}.cpuset.cpus.effective" + { + printf 'timestamp_utc\tusage_usec\tuser_usec\tsystem_usec\tnr_periods\tnr_throttled\tthrottled_usec\n' + while [[ -r "/sys/fs/cgroup$group/cpu.stat" ]]; do + awk -v timestamp="$(date -u +%Y-%m-%dT%H:%M:%SZ)" ' + { counters[$1] = $2 } + END { printf "%s\t%.0f\t%.0f\t%.0f\t%.0f\t%.0f\t%.0f\n", timestamp, + counters["usage_usec"], counters["user_usec"], counters["system_usec"], + counters["nr_periods"], counters["nr_throttled"], counters["throttled_usec"] } + ' "/sys/fs/cgroup$group/cpu.stat" || break + sleep 1 + done + } >"$output" +} + wait_for_broker() { local broker=$1 for _ in $(seq 1 60); do @@ -273,7 +377,7 @@ wait_for_broker() { run_variant() { local scenario=$1 pair=$2 position=$3 variant=$4 - local image commit producers consumers batch rate attempt + local image commit producers consumers batch rate attempt data_path case "$variant" in baseline) image="$BASELINE_IMAGE" @@ -321,11 +425,21 @@ run_variant() { docker volume create "$ACTIVE_VOLUME" >/dev/null docker run -d --name "$ACTIVE_BROKER" --network "$NETWORK" \ - --cpus 2 --memory 2g \ + "${broker_args[@]}" \ -e RUSTQUEUE_DATA_PATH=/data \ -v "$ACTIVE_VOLUME:/data" \ "$image" >/dev/null wait_for_broker "$ACTIVE_BROKER" + if [[ "$QUALIFICATION_ENVIRONMENT" == github_actions ]]; then + data_path=$(docker volume inspect "$ACTIVE_VOLUME" --format '{{.Mountpoint}}') + sudo -n findmnt --json --target "$data_path" \ + --output TARGET,SOURCE,FSTYPE,OPTIONS,MAJ:MIN >"$RUN_DIR/$label.data-filesystem.json" + sudo -n df -B1 "$data_path" >"$RUN_DIR/$label.data-filesystem-space.txt" + jq -e '.filesystems | length == 1 and all(.[]; .fstype != "tmpfs" and .fstype != "ramfs")' \ + "$RUN_DIR/$label.data-filesystem.json" >/dev/null || + die "qualification data must use a disk-backed filesystem" + fi + capture_host_counters "$RUN_DIR/$label.before" "$ACTIVE_BROKER" sample_rss "$rss_file" "$ACTIVE_BROKER" & SAMPLER_PID=$! @@ -347,16 +461,32 @@ run_variant() { if [[ -n "$rate" ]]; then bench_args+=(--rate "$rate") fi - set +e - docker run --rm --network "$NETWORK" --cpus 2 --memory 2g \ + ACTIVE_LOAD_GENERATOR="$PREFIX-load$SEQUENCE" + docker run -d --name "$ACTIVE_LOAD_GENERATOR" --network "$NETWORK" \ + "${load_generator_args[@]}" \ --entrypoint /usr/local/bin/rustqueue-bench "$TOOLS_IMAGE" \ - "${bench_args[@]}" >"$report" 2>"$error_log" - benchmark_status=$? - set -e + "${bench_args[@]}" >/dev/null + if [[ "$QUALIFICATION_ENVIRONMENT" == github_actions ]]; then + sample_load_generator_cpu "$RUN_DIR/$label.load-generator-cpu.tsv" \ + "$ACTIVE_LOAD_GENERATOR" & + CPU_SAMPLER_PID=$! + fi + benchmark_status=$(docker wait "$ACTIVE_LOAD_GENERATOR") + docker logs "$ACTIVE_LOAD_GENERATOR" >"$report" 2>"$error_log" + if [[ -n "$CPU_SAMPLER_PID" ]]; then + kill "$CPU_SAMPLER_PID" >/dev/null 2>&1 || true + wait "$CPU_SAMPLER_PID" >/dev/null 2>&1 || true + CPU_SAMPLER_PID="" + awk 'END { exit (NR < 2) }' "$RUN_DIR/$label.load-generator-cpu.tsv" || { + cleanup_run + die "$label did not capture load generator CPU counters" + } + fi kill "$SAMPLER_PID" >/dev/null 2>&1 || true wait "$SAMPLER_PID" >/dev/null 2>&1 || true SAMPLER_PID="" + capture_host_counters "$RUN_DIR/$label.after" "$ACTIVE_BROKER" if [[ "$benchmark_status" -ne 0 ]]; then cat "$error_log" >&2 cleanup_run @@ -544,19 +674,24 @@ baseline_image_id="$(docker image inspect "$BASELINE_IMAGE" --format '{{.Id}}')" candidate_image_id="$(docker image inspect "$CANDIDATE_IMAGE" --format '{{.Id}}')" docker_server_version="$(docker version --format '{{.Server.Version}}')" docker_architecture="$(docker info --format '{{.Architecture}}')" -docker_cpus="$(docker info --format '{{.NCPU}}')" docker_memory_bytes="$(docker info --format '{{.MemTotal}}')" docker_storage_driver="$(docker info --format '{{.Driver}}')" orbstack_version="$(orbctl version 2>/dev/null | head -n 1 || printf unknown)" macos_version="$(sw_vers -productVersion 2>/dev/null || printf unknown)" hardware_model="$(sysctl -n hw.model 2>/dev/null || printf unknown)" +rss_observer="docker exec inside Broker cgroup" +[[ "$QUALIFICATION_ENVIRONMENT" != github_actions ]] || rss_observer="Linux host /proc; outside Broker cgroup" jq -n \ + --arg platform "$platform" \ + --arg rss_observer "$rss_observer" \ --arg docker_context "$docker_context" \ --arg docker_os "$docker_os" \ --arg docker_server_version "$docker_server_version" \ --arg docker_architecture "$docker_architecture" \ --argjson docker_cpus "$docker_cpus" \ + --arg broker_cpuset "$broker_cpuset" \ + --arg load_generator_cpuset "$load_generator_cpuset" \ --argjson docker_memory_bytes "$docker_memory_bytes" \ --arg docker_storage_driver "$docker_storage_driver" \ --arg orbstack_version "$orbstack_version" \ @@ -564,8 +699,9 @@ jq -n \ --arg hardware_model "$hardware_model" \ --arg tool_source "$tool_source" \ '{ - platform: "OrbStack on macOS", + platform: $platform, comparison_scope: "same-host relative only", + rss_observer: $rss_observer, macos_version: $macos_version, hardware_model: $hardware_model, orbstack_version: $orbstack_version, @@ -579,8 +715,14 @@ jq -n \ storage_driver: $docker_storage_driver }, resource_limits: { - broker: {cpus: 2, memory_bytes: 2147483648}, - load_generator: {cpus: 2, memory_bytes: 2147483648} + broker: { + cpus: 2, memory_bytes: 2147483648, memory_swap_bytes: 2147483648, + cpuset_cpus: (if $broker_cpuset == "" then null else $broker_cpuset end) + }, + load_generator: { + cpus: 2, memory_bytes: 2147483648, memory_swap_bytes: 2147483648, + cpuset_cpus: (if $load_generator_cpuset == "" then null else $load_generator_cpuset end) + } }, tool_source: $tool_source }' >"$RUN_DIR/environment-core.json" @@ -701,6 +843,34 @@ set -e mkdir -p "$(dirname "$EVIDENCE_OUTPUT")" cp "$EVIDENCE_FILE" "$EVIDENCE_OUTPUT" +if [[ "$CALIBRATION_ONLY" == 1 ]]; then + # A valid evaluator document proves the original integrity and drain checks ran. + jq -e '.schema_version == 1 and (.runs | length) == 12 + and (.verdict.hard_failures | length) == 0 + and .verdict.status == "pass"' "$EVIDENCE_FILE" >/dev/null || + die "A/A evaluator rejected integrity or regression checks" + jq ' + def spread: ((max / min) - 1) * 100; + if .baseline.binary_sha256 != .candidate.binary_sha256 + or .baseline.image_id != .candidate.image_id then error("A/A artifacts differ") else . end + | [.runs | group_by(.case)[] | . as $runs | { + profile: .[0].case, runs: length, + throughput_spread_percent: ([.[] | if .case == "sustainable" + then .metrics.receive_messages_per_second else .metrics.publish_messages_per_second end] | spread), + p99_spread_percent: ([.[].metrics.pub_ack_p99_us] | spread) + }] + | {limits_percent: {throughput: 5, p99: 10}, profiles: ., + status: (if length == 3 and all(.[]; + .runs == 4 and .throughput_spread_percent <= 5 and .p99_spread_percent <= 10) + then "pass" else "fail" end)} + ' "$INPUT_FILE" >"$RUN_DIR/calibration.json" + cp "$RUN_DIR/calibration.json" "$EVIDENCE_OUTPUT" + if jq -e '.status == "pass"' "$EVIDENCE_OUTPUT" >/dev/null; then + qualification_status=0 + else + qualification_status=1 + fi +fi printf 'Raw qualification artifacts: %s\n' "$RUN_DIR" printf 'Compact qualification evidence: %s\n' "$EVIDENCE_OUTPUT" exit "$qualification_status" diff --git a/tests/kodo-gateway/compose.yaml b/tests/kodo-gateway/compose.yaml index 287994a..aa7d0eb 100644 --- a/tests/kodo-gateway/compose.yaml +++ b/tests/kodo-gateway/compose.yaml @@ -43,7 +43,9 @@ services: RUSTQUEUE_KODO_GATEWAY_ORDINAL: "0" RUSTQUEUE_PROXY_MAX_MESSAGE_BYTES: "104857600" RUSTQUEUE_PROXY_MAX_BODY_BYTES: "134217728" - RUSTQUEUE_PROXY_MAX_INFLIGHT_BYTES: "134217728" + # 128 MiB MPUB body plus 8 MiB parsing metadata. Holding 40 MiB still + # prevents a 100 MiB PUB and exercises the unchanged failover assertion. + RUSTQUEUE_PROXY_MAX_INFLIGHT_BYTES: "142606336" RUSTQUEUE_PROXY_TCP_COMMAND_TIMEOUT_MS: "120000" RUSTQUEUE_PROXY_TCP_MAX_CONNECTION_AGE_SECONDS: "0" depends_on: diff --git a/tests/soak/main.go b/tests/soak/main.go index 6b81da9..842b6fd 100644 --- a/tests/soak/main.go +++ b/tests/soak/main.go @@ -39,7 +39,7 @@ type report struct { func main() { lookup := flag.String("lookup", "host.docker.internal:4151", "lookupd-compatible HTTP address") - tcp := flag.String("tcp", "host.docker.internal:4150,host.docker.internal:5150,host.docker.internal:6150", "comma-separated NSQ TCP addresses") + tcp := flag.String("tcp", "host.docker.internal:4150,host.docker.internal:5150,host.docker.internal:6150", "comma-separated direct Broker TCP addresses") duration := flag.Duration("duration", 24*time.Hour, "publish duration") grace := flag.Duration("grace", 2*time.Minute, "final drain timeout") rate := flag.Int("rate", 1000, "target messages per second; zero means saturated") @@ -81,6 +81,9 @@ func main() { return nil })) check(consumer.ConnectToNSQLookupd(*lookup)) + for _, address := range strings.Split(*tcp, ",") { + check(consumer.ConnectToNSQD(strings.TrimSpace(address))) + } waitConsumer(consumer, *lookup, topic, channel) if *readyFile != "" { check(os.WriteFile(*readyFile, []byte(topic), 0o600))