diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..fb0c87fb --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +# Preserve PDF fixture bytes, including fixed-width cross-reference records. +tests/fixtures/media/*.pdf binary diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 389eda7b..6cc69c7e 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -98,7 +98,7 @@ body: id: data_involvement attributes: label: Database or media involvement - description: State whether SQLite, migrations, uploads, ffmpeg/ffprobe, boards, backups, or restore is involved. Do not upload real data. + description: State whether SQLite, migrations, uploads, media processing, FFmpeg video, boards, backups, or restore is involved. Do not upload real data. placeholder: "SQLite involved; reproduced on a new disposable database. No media involved." validations: required: true diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 33400b32..3bf50e06 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,7 +33,7 @@ jobs: - name: Install Rust toolchain uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable with: - toolchain: 1.91.0 + toolchain: 1.99.0 components: clippy, rustfmt - name: Cache Rust artifacts @@ -50,8 +50,16 @@ jobs: - name: Check formatting run: cargo fmt --all --check + - name: Check all targets and features + run: cargo check --locked --workspace --all-targets --all-features + - name: Run Clippy - run: cargo clippy --locked --workspace --all-targets --all-features + run: cargo clippy --locked --workspace --all-targets --all-features -- -D warnings + + - name: Build documentation + run: cargo doc --locked --workspace --all-features --no-deps + env: + RUSTDOCFLAGS: -D warnings - name: Enforce locked dependency policy run: cargo deny --locked check @@ -68,7 +76,7 @@ jobs: - name: Install Rust toolchain uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable with: - toolchain: 1.91.0 + toolchain: 1.99.0 - name: Cache Rust artifacts uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2 @@ -104,7 +112,7 @@ jobs: - name: Install Rust toolchain uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable with: - toolchain: 1.91.0 + toolchain: 1.99.0 targets: ${{ matrix.target }} - name: Cache Rust artifacts diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4348104b..420f28df 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -31,7 +31,7 @@ jobs: - name: Install Rust toolchain uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable with: - toolchain: 1.91.0 + toolchain: 1.99.0 components: clippy, rustfmt - name: Cache Rust artifacts @@ -71,8 +71,16 @@ jobs: - name: Check formatting run: cargo fmt --all --check + - name: Check all targets and features + run: cargo check --locked --workspace --all-targets --all-features + - name: Run Clippy - run: cargo clippy --locked --workspace --all-targets --all-features + run: cargo clippy --locked --workspace --all-targets --all-features -- -D warnings + + - name: Build documentation + run: cargo doc --locked --workspace --all-features --no-deps + env: + RUSTDOCFLAGS: -D warnings - name: Test native update packaging and signing run: python3 -m unittest discover -s tools -p "test_update_package.py" @@ -121,7 +129,7 @@ jobs: - name: Install Rust toolchain uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable with: - toolchain: 1.91.0 + toolchain: 1.99.0 targets: ${{ matrix.target }} - name: Cache Rust artifacts diff --git a/.gitignore b/.gitignore index 5cd651cc..9932abf4 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,19 @@ Desktop.ini *.orig *.rej +# Local tests. +tests/ +# Migration unit tests compile these sanitized fixtures into the binary. +!tests/ +tests/* +!tests/fixtures/ +tests/fixtures/* +!tests/fixtures/media/ +!tests/e2e/ +tests/e2e/* +!tests/e2e/non-video-media.spec.ts +!tests/e2e/settings-restart.spec.ts + # Build output, dependency caches, coverage, and profiling output. target/ node_modules/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 26125054..bbe23ac8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,14 @@ All notable changes to RustChan will be documented in this file. +## RustChan 1.6.5 — 2026-10-03 + +- Moved image processing, supported audio decoding, media probing and PDF previews into Rust, with optional FFmpeg fallback for video and uncovered audio codecs. +- Added verified administrator settings restarts, configuration rollback and recovery handling. +- Improved SQLite connection ownership and measured query performance; CLI startup proceeds once one usable database connection is ready. +- Fixed media timing, multistream audio, bounded previews, Speex waveform layouts, AV1 WebM conversion and Windows restart builds. +- Hardened the Rust 1.99 lint policy and consolidated console, request and test utilities. + ## RustChan 1.6.0 — 2026-10-01 This release includes every integrated change since the published v1.5.0 release: @@ -10,7 +18,7 @@ upgrades, public posting and reading improvements, and verified native software ### Upgrade notes -- Back up the complete data directory before upgrading. Existing database structures are verified and stamped for the running 1.6.0 package; Rust 1.91 remains the minimum supported compiler. +- Back up the complete data directory before upgrading. Existing database structures are verified and stamped for the running 1.6.0 package; Rust 1.99 is the minimum supported compiler. - Native self-updates require one-time operator setup on Linux GNU x86_64 or ARM64: separate service identities, the supplied systemd/polkit configuration, a protected updater executable and independently provisioned trusted public key. The updater is operator-managed and is not replaced by application updates. See [Software Updates](docs/software-updates.md). - Containers, macOS, Windows and custom installations use their deployment tools to install releases. Their admin panel can check releases but does not offer native installation. Keep container data volumes when recreating an instance. - `/readyz` additionally reports the running package version. It does not expose release availability, updater transactions or retained backup metadata. @@ -33,6 +41,8 @@ upgrades, public posting and reading improvements, and verified native software ### Fixed +- Let CLI startup proceed once one fully initialized database connection is ready while spare connections fill in the background; retain the pool capacity, one-second checkout bound, and database validation. + - Completed native updates clear the availability notice; stale discovery records cannot offer installation of the running or an older version. Rollbacks retain notice of a newer release. - Banned identities can no longer submit reports or poll votes; those mutations share the existing posting ban gate and enforce CSRF before authorization results are disclosed. Long ban reasons wrap on narrow screens. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a5901e43..ff604de7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -51,7 +51,7 @@ has already been coordinated under [SECURITY.md](SECURITY.md). ## Rust and feature policy -The workspace currently declares Rust `1.91` as its minimum supported Rust +The workspace currently declares Rust `1.99` as its minimum supported Rust version (MSRV). Keep `rust-version` and the `cargo-msrv` metadata aligned; do not raise the MSRV incidentally. @@ -72,20 +72,22 @@ Rust checks are: ```sh cargo fmt --all --check -cargo clippy --locked --workspace --all-targets --all-features +cargo check --locked --workspace --all-targets --all-features +cargo clippy --locked --workspace --all-targets --all-features -- -D warnings cargo test --locked --workspace --all-features +RUSTDOCFLAGS="-D warnings" cargo doc --locked --workspace --all-features --no-deps ``` Use the Playwright harness under `tests/e2e/` for public UI, admin UI, media, backup and restore, moderation, Tor/proxy, and no-JavaScript changes. The maintained harness source is tracked: `playwright.config.ts`, `package.json`, -`package-lock.json`, `tests/e2e/**`, and `.github/workflows/e2e.yml`. Generated -evidence is not: keep reports, traces, screenshots, videos, storage state, +`package-lock.json`, and `tests/e2e/**`. Browser validation runs locally. +Generated evidence is not: keep reports, traces, screenshots, videos, storage state, runtime databases, preserved fixture roots, audit output, and other browser artifacts under the matching `.gitignore` rules and out of commits. See `tests/e2e/README.md` for the normal regression and deep audit commands. Run focused scenarios first and the available browser matrix for broad changes. -Some media tests require `ffmpeg`, `ffprobe`, and specific codecs; do not replace +Some media tests require `ffmpeg` and specific codecs; do not replace deterministic fixtures with private uploads from a live site. Permanent Rust unit and integration tests remain normal repository source. diff --git a/Cargo.lock b/Cargo.lock index 626d5433..7879c2c1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -740,6 +740,20 @@ name = "bytemuck" version = "1.25.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" +dependencies = [ + "bytemuck_derive", +] + +[[package]] +name = "bytemuck_derive" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a1f896587b6f2c069c73d2f0913e2d590c3990285cd2f0b6aa02b786b4c679c" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] [[package]] name = "byteorder" @@ -909,6 +923,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "color" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ec7c5eb7a16992b1904d76c517d170ab353b0e0b3d5a0c81a8a0cd1037893cf" +dependencies = [ + "bytemuck", +] + [[package]] name = "color_quant" version = "1.1.0" @@ -1440,7 +1463,7 @@ dependencies = [ "quote", "sha3 0.12.0", "strum", - "syn 2.0.119", + "syn 3.0.6", "unicode-ident", "void", ] @@ -1738,6 +1761,15 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "euclid" +version = "0.22.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1a05365e3b1c6d1650318537c7460c6923f1abdd272ad6842baa2b509957a06" +dependencies = [ + "num-traits", +] + [[package]] name = "event-listener" version = "5.4.2" @@ -1769,6 +1801,12 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "extended" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af9673d8203fcb076b19dfd17e38b3d4ae9f44959416ea532ce72415a6020365" + [[package]] name = "fallible-iterator" version = "0.3.0" @@ -1802,6 +1840,12 @@ dependencies = [ "simd-adler32", ] +[[package]] +name = "fearless_simd" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97b65636e5b9ef369943878ac74335ba1c55c1cb6adbf1e2c293c624248d693" + [[package]] name = "ff" version = "0.13.1" @@ -1888,6 +1932,15 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" +[[package]] +name = "font-types" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b38ad915f6dadd993ced50848a8291a543bd41ca62bc10740d5e64e2ab4cfd7" +dependencies = [ + "bytemuck", +] + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -2126,6 +2179,23 @@ dependencies = [ "weezl", ] +[[package]] +name = "glifo" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d99fc21d493812643aae86d53b7bbd02f376434a90317e8a790bc209fdd6605e" +dependencies = [ + "bytemuck", + "foldhash 0.2.0", + "hashbrown 0.17.1", + "log", + "peniko", + "png 0.18.1", + "skrifa", + "smallvec", + "vello_common 0.0.9", +] + [[package]] name = "glob-match" version = "0.2.1" @@ -2155,6 +2225,15 @@ dependencies = [ "xxhash-rust", ] +[[package]] +name = "guillotiere" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b17e70c989c36bad147b27a58d148c0741c51448aa5653436547323e524d0ab" +dependencies = [ + "euclid", +] + [[package]] name = "h2" version = "0.4.19" @@ -2237,12 +2316,107 @@ dependencies = [ "hashbrown 0.16.1", ] +[[package]] +name = "hayro" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4caa128ab87fd48ffb7490617cf93f77f606820dffbc9fd9ef6ab0ed077f56d" +dependencies = [ + "bytemuck", + "hayro-interpret", + "image 0.25.10", + "kurbo", + "pic-scale", + "vello_cpu", +] + +[[package]] +name = "hayro-ccitt" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f4d0e94ddd48749f06bbe4e5389fb9799a0c45bcaf00495042076ef05e3241a" + +[[package]] +name = "hayro-cmap" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d285dc30731c8485de5fa732fbdf2b3affdf01e4da7c2135022ca6fa664bf6" +dependencies = [ + "hayro-postscript", +] + +[[package]] +name = "hayro-interpret" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2613d0406898995042d0794c4245b2f2fba1246490c8ac593769bba6551129d" +dependencies = [ + "bitflags 2.13.2", + "hayro-cmap", + "hayro-syntax", + "kurbo", + "moxcms", + "phf 0.13.1", + "rustc-hash", + "siphasher", + "skrifa", + "smallvec", + "yoke", +] + +[[package]] +name = "hayro-jbig2" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69374b3668dd45aeb3d3145cda68f2c7b4f223aaa2511e67d076f1c7d741388d" +dependencies = [ + "fearless_simd", + "hayro-ccitt", +] + +[[package]] +name = "hayro-jpeg2000" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c75ab947623ef4ccaa7acf0579edf7cbb5a73838e3839a7be73335e522f433a1" +dependencies = [ + "fearless_simd", +] + +[[package]] +name = "hayro-postscript" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "885c5ef0654933139a9b9546fc2c69e18d37f38aa2520f079092b1be18f1fcaa" + +[[package]] +name = "hayro-syntax" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0edeafd70aa2db743de8ede8637d07ec87db05efe69cde371d03f1b185fcef27" +dependencies = [ + "flate2", + "hayro-ccitt", + "hayro-jbig2", + "hayro-jpeg2000", + "memchr", + "rustc-hash", + "smallvec", + "zune-jpeg", +] + [[package]] name = "heck" version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "heic-rs" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adf9dde634bb6d5b718bacd01bef80192f2334306b2f785519a0136cca7b9618" + [[package]] name = "hermit-abi" version = "0.5.3" @@ -2932,6 +3106,18 @@ dependencies = [ "libc", ] +[[package]] +name = "kurbo" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b60dfc32f652b926df6192e55525b16d186c69d47876c3ead4da5cc9f8450e2" +dependencies = [ + "arrayvec", + "euclid", + "polycool", + "smallvec", +] + [[package]] name = "lazy_static" version = "1.5.0" @@ -2982,6 +3168,12 @@ dependencies = [ "bitflags 2.13.2", ] +[[package]] +name = "linebender_resource_handle" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4a5ff6bcca6c4867b1c4fd4ef63e4db7436ef363e0ad7531d1558856bae64f4" + [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -3278,6 +3470,15 @@ dependencies = [ "zeroize", ] +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + [[package]] name = "num-conv" version = "0.2.2" @@ -3405,6 +3606,12 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" +[[package]] +name = "opus-pure" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4de7c2ad44427a636b27bf0b03eb29e713775ba1b45ff31a0618c19ee5adb215" + [[package]] name = "ordered-float" version = "2.10.1" @@ -3578,23 +3785,56 @@ dependencies = [ "base64ct", ] +[[package]] +name = "peniko" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "839c8299360d2e998bdb106dc0a6cd71dcc5f4df51df1b620361bf50e283cca6" +dependencies = [ + "bytemuck", + "color", + "kurbo", + "linebender_resource_handle", + "smallvec", +] + [[package]] name = "percent-encoding" version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "phf" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf" +dependencies = [ + "phf_macros 0.13.1", + "phf_shared 0.13.1", +] + [[package]] name = "phf" version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "010378780309880b08997fae13be7834dba947d36393bd372f2b1556deb2a2f6" dependencies = [ - "phf_macros", - "phf_shared", + "phf_macros 0.14.0", + "phf_shared 0.14.0", "serde", ] +[[package]] +name = "phf_generator" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "135ace3a761e564ec88c03a77317a7c6b80bb7f7135ef2544dbe054243b89737" +dependencies = [ + "fastrand", + "phf_shared 0.13.1", +] + [[package]] name = "phf_generator" version = "0.14.0" @@ -3602,7 +3842,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aeb62e0959d5a1bebc965f4d15d9e2b7cea002b6b0f5ba8cde6cc26738467100" dependencies = [ "fastrand", - "phf_shared", + "phf_shared 0.14.0", +] + +[[package]] +name = "phf_macros" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "812f032b54b1e759ccd5f8b6677695d5268c588701effba24601f6932f8269ef" +dependencies = [ + "phf_generator 0.13.1", + "phf_shared 0.13.1", + "proc-macro2", + "quote", + "syn 2.0.119", ] [[package]] @@ -3611,13 +3864,22 @@ version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5fa8d0ca26d424d27630da600c6624696e7dec8bf7b3b492b383c5dc49e5e085" dependencies = [ - "phf_generator", - "phf_shared", + "phf_generator 0.14.0", + "phf_shared 0.14.0", "proc-macro2", "quote", "syn 2.0.119", ] +[[package]] +name = "phf_shared" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e57fef6bc5981e38c2ce2d63bfa546861309f875b8a75f092d1d54ae2d64f266" +dependencies = [ + "siphasher", +] + [[package]] name = "phf_shared" version = "0.14.0" @@ -3627,6 +3889,16 @@ dependencies = [ "siphasher", ] +[[package]] +name = "pic-scale" +version = "0.7.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6edb0ff645d35accaec252de081d55984a0d6c952cb1e5dc2b9afa7c78f07288" +dependencies = [ + "num-traits", + "pxfm", +] + [[package]] name = "pin-project" version = "1.1.13" @@ -3731,6 +4003,15 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "polycool" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50596ddc09eb5ad5f75cacd40209568e66df71baf86e1499a0e99c4cff12a5a6" +dependencies = [ + "arrayvec", +] + [[package]] name = "polyval" version = "0.7.3" @@ -4112,6 +4393,16 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "read-fonts" +version = "0.39.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4ed38b89c2c77ff968c524145ad65fb010f38af5c7a224b53b81d47ac2daa81" +dependencies = [ + "bytemuck", + "font-types", +] + [[package]] name = "redox_syscall" version = "0.5.18" @@ -4174,6 +4465,12 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "regex-lite" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973" + [[package]] name = "regex-syntax" version = "0.8.11" @@ -4315,6 +4612,12 @@ dependencies = [ "time", ] +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + [[package]] name = "rustc_version" version = "0.4.1" @@ -4326,7 +4629,7 @@ dependencies = [ [[package]] name = "rustchan" -version = "1.6.0" +version = "1.6.5" dependencies = [ "anyhow", "argon2", @@ -4343,12 +4646,16 @@ dependencies = [ "flate2", "futures", "getrandom 0.4.3", + "gif", + "hayro", + "heic-rs", "hex", "hyper", "hyper-util", "image 0.25.10", "ipnet", "kamadak-exif", + "opus-pure", "parking_lot", "pem-rfc7468 1.0.0", "r2d2", @@ -4371,6 +4678,7 @@ dependencies = [ "sha2 0.11.0", "sha3 0.12.0", "subtle", + "symphonia", "tar", "tempfile", "thiserror 2.0.20", @@ -4952,6 +5260,16 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" +[[package]] +name = "skrifa" +version = "0.42.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c34617370ae968efb7161bb2beb517d9084659aae19e24b89e3db25b46e4564" +dependencies = [ + "bytemuck", + "read-fonts", +] + [[package]] name = "slab" version = "0.4.12" @@ -5140,6 +5458,189 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7973cce6668464ea31f176d85b13c7ab3bba2cb3b77a2ed26abd7801688010a" +[[package]] +name = "symphonia" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7edef6a96b696d4e0cab5ee9ebb7ca155ed95f30a6b45bbb8b97d2727f02424" +dependencies = [ + "lazy_static", + "symphonia-bundle-flac", + "symphonia-bundle-mp3", + "symphonia-codec-aac", + "symphonia-codec-adpcm", + "symphonia-codec-alac", + "symphonia-codec-pcm", + "symphonia-codec-vorbis", + "symphonia-core", + "symphonia-format-isomp4", + "symphonia-format-mkv", + "symphonia-format-ogg", + "symphonia-format-riff", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-bundle-flac" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6405d5c34ff6f8f7ca08a4101efe66d21e180f7322ef9359900a0e55698a7892" +dependencies = [ + "log", + "symphonia-common", + "symphonia-core", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-bundle-mp3" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98ea5ffc8716bff677dfb3b01b420c7b758de901a72b8c330bf2040ab74b4add" +dependencies = [ + "lazy_static", + "log", + "symphonia-core", +] + +[[package]] +name = "symphonia-codec-aac" +version = "0.6.1" +dependencies = [ + "lazy_static", + "log", + "symphonia-common", + "symphonia-core", +] + +[[package]] +name = "symphonia-codec-adpcm" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "445932ecb0c59362fde9c082dd63a75380650c5077fdb8b80b8cac850442a74c" +dependencies = [ + "log", + "symphonia-core", +] + +[[package]] +name = "symphonia-codec-alac" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "920a78f96f3cf62932d0c497959c0cc2100ef36b03c7ca1417b260f432a482d8" +dependencies = [ + "log", + "symphonia-common", + "symphonia-core", +] + +[[package]] +name = "symphonia-codec-pcm" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e04ba75686acbe43542fdd374571195f0530c0b7785ca25cc6840e9c6c4b6eea" +dependencies = [ + "log", + "symphonia-core", +] + +[[package]] +name = "symphonia-codec-vorbis" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73d90b4fcf796137cc683c538282804ff9629f8ad9dbfd881fcbba331ac4e986" +dependencies = [ + "log", + "symphonia-common", + "symphonia-core", +] + +[[package]] +name = "symphonia-common" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acc3fcc18ec9b8cdd48614e259c4cf0d27b71d41e5d9b120b42c5adab12d7c4" +dependencies = [ + "log", + "symphonia-core", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-core" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01c412864d599d4750d0c3d684d7e093ec05e5309681ef5252cc1096a437f6e0" +dependencies = [ + "bitflags 2.13.2", + "bytemuck", + "lazy_static", + "log", + "num-complex", + "smallvec", +] + +[[package]] +name = "symphonia-format-isomp4" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e681a70e1870d34e02abf1dbc51e4267c3f1827801474e8870be8c689fc4dc3" +dependencies = [ + "log", + "symphonia-common", + "symphonia-core", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-format-mkv" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d015c5c0558864665894b3f4cbd95e10abb01b9c868e751c72670f326a56360e" +dependencies = [ + "lazy_static", + "log", + "symphonia-common", + "symphonia-core", +] + +[[package]] +name = "symphonia-format-ogg" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5495e7f7e3c7035328d82b6d6e377eef289bb0c4105bdeb557fc93a833f994" +dependencies = [ + "log", + "symphonia-common", + "symphonia-core", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-format-riff" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ff70929083a8c1a5f6cd7c904b6071c7914ad04739b510c2f7239dfc9b7dabe" +dependencies = [ + "extended", + "log", + "symphonia-core", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-metadata" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83713a97705d77bdef7cdbc0768fd6e5a54e4cd7e48d60a806ae85639e2c87c6" +dependencies = [ + "lazy_static", + "log", + "regex-lite", + "smallvec", + "symphonia-core", +] + [[package]] name = "syn" version = "1.0.109" @@ -6387,7 +6888,7 @@ dependencies = [ "itertools 0.15.0", "memchr", "paste", - "phf", + "phf 0.14.0", "rand 0.10.2", "rangemap", "saturating-time", @@ -6972,6 +7473,53 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" +[[package]] +name = "vello_common" +version = "0.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3361bff7f7d82c0c496b92048db83846691f0e844cc28dee92b1c824291b55ee" +dependencies = [ + "bytemuck", + "fearless_simd", + "guillotiere", + "hashbrown 0.17.1", + "log", + "peniko", + "png 0.18.1", + "smallvec", + "thiserror 2.0.20", +] + +[[package]] +name = "vello_common" +version = "0.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d672facaa2d697285a786cd9d44d614cd2ce54cdc022504bf339f8fff3b750" +dependencies = [ + "bytemuck", + "fearless_simd", + "guillotiere", + "hashbrown 0.17.1", + "log", + "peniko", + "png 0.18.1", + "smallvec", + "thiserror 2.0.20", +] + +[[package]] +name = "vello_cpu" +version = "0.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d8ded630e8316bb94a55881256506d1f3b9947b5f66db8a7d32ca7ba02decd0" +dependencies = [ + "bytemuck", + "glifo", + "hashbrown 0.17.1", + "png 0.18.1", + "vello_common 0.0.8", +] + [[package]] name = "version_check" version = "0.9.5" diff --git a/Cargo.toml b/Cargo.toml index 80d59473..3960654a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,9 +1,8 @@ [package] name = "rustchan" -version = "1.6.0" +version = "1.6.5" edition = "2021" -# Keep the package MSRV and cargo-msrv metadata aligned with Arti's Rust 1.91 -# requirement. +# Rust 1.99 is the supported minimum for the package and build tooling. rust-version.workspace = true license = "MIT" description = "Self-contained imageboard — single binary, zero runtime dependencies" @@ -13,13 +12,14 @@ categories = ["web-programming", "multimedia::images"] # Keep `cargo-msrv` metadata aligned with `rust-version`. [package.metadata] -msrv = "1.91" +msrv = "1.99" [workspace] resolver = "2" +exclude = ["vendor/symphonia-codec-aac"] [workspace.package] -rust-version = "1.91" +rust-version = "1.99" [profile.release] opt-level = 3 @@ -81,6 +81,17 @@ rand_core = "0.10.1" getrandom = { version = "0.4.3", features = ["sys_rng"] } image = { version = "0.25.10", default-features = false, features = ["jpeg", "png", "gif", "webp", "bmp", "tiff", "ico"] } +# Already used by image; inspect GIF repeat/disposal metadata without retaining frames. +gif = { version = "0.14.2", default-features = false, features = ["std", "color_quant"] } +# Container inspection only for video; no video decoders are enabled. +symphonia = { version = "0.6.1", default-features = false, features = ["aac", "adpcm", "alac", "flac", "mp1", "mp2", "mp3", "pcm", "vorbis", "isomp4", "mkv", "ogg", "wav", "exp-video-codecs"] } +# Symphonia supplies Opus packets; this decoder avoids native libopus. +opus-pure = "0.2.2" +# HEVC intra still pictures only; no native library or video framework. +heic-rs = { version = "0.1.1", default-features = false, features = ["std"] } + +# Pure-Rust first-page PDF previews, with embedded standard fonts. +hayro = { version = "0.7.1", default-features = false, features = ["embed-fonts"] } # Apply JPEG Orientation tags before thumbnailing so phone photos display upright. kamadak-exif = "0.6.1" @@ -186,32 +197,37 @@ warnings = { level = "deny", priority = -3 } future_incompatible = { level = "deny", priority = -2 } rust_2018_idioms = { level = "deny", priority = -2 } rust_2024_compatibility = { level = "deny", priority = -2 } +nonstandard_style = { level = "deny", priority = -2 } +let_underscore = { level = "deny", priority = -2 } unused = { level = "deny", priority = -2 } unsafe_code = "forbid" +unsafe_op_in_unsafe_fn = "deny" +raw_borrows_via_references = "deny" missing_docs = "deny" missing_debug_implementations = "deny" non_ascii_idents = "deny" unreachable_pub = "deny" +unused_crate_dependencies = "deny" +unused_results = "deny" +macro_use_extern_crate = "deny" +redundant_imports = "deny" +redundant_lifetimes = "deny" +single_use_lifetimes = "deny" +trivial_casts = "deny" +trivial_numeric_casts = "deny" +unit_bindings = "deny" +unused_import_braces = "deny" unused_lifetimes = "deny" unused_qualifications = "deny" let_underscore_drop = "deny" -# This rustc lint can report false positives for dependencies shared across -# library, binary, example, test, benchmark, and feature-specific targets. -# Keep it visible, but do not force fake imports or dependency distortions. -unused_crate_dependencies = "warn" - [workspace.lints.clippy] all = { level = "deny", priority = -3 } pedantic = { level = "deny", priority = -2 } +nursery = { level = "deny", priority = -2 } cargo = { level = "deny", priority = -2 } -# This can represent unavoidable transitive version splits, especially in -# a dependency graph as large as Arti's. Audit every report without making -# unavoidable upstream duplication a hard failure. -multiple_crate_versions = "warn" - # Panic-driven control flow. unwrap_used = "deny" expect_used = "deny" @@ -232,6 +248,9 @@ print_stderr = "deny" undocumented_unsafe_blocks = "deny" multiple_unsafe_ops_per_block = "deny" missing_safety_doc = "deny" +unnecessary_safety_comment = "deny" +unnecessary_safety_doc = "deny" +default_union_representation = "deny" # Suppressions must be narrow, intentional, and documented. allow_attributes = "deny" @@ -239,6 +258,7 @@ allow_attributes_without_reason = "deny" # Numeric conversions must be explicit and reviewed. as_conversions = "deny" +arithmetic_side_effects = "deny" cast_lossless = "deny" cast_possible_truncation = "deny" cast_possible_wrap = "deny" @@ -248,16 +268,26 @@ cast_sign_loss = "deny" char_lit_as_u8 = "deny" fn_to_numeric_cast_any = "deny" lossy_float_literal = "deny" +default_numeric_fallback = "deny" +suspicious_xor_used_as_pow = "deny" +precedence_bits = "deny" # Avoid unchecked indexing and UTF-8 string slicing. indexing_slicing = "deny" string_slice = "deny" +# Ignored values and error mappings must preserve their meaning. +let_underscore_must_use = "deny" +let_underscore_untyped = "deny" +map_err_ignore = "deny" + # Ownership and resource-management restrictions. clone_on_ref_ptr = "deny" mem_forget = "deny" rc_buffer = "deny" rc_mutex = "deny" +string_add = "deny" +str_to_string = "deny" # Documentation, testing, and maintainability. missing_docs_in_private_items = "deny" @@ -265,3 +295,18 @@ missing_assert_message = "deny" large_include_file = "deny" verbose_file_reads = "deny" wildcard_dependencies = "deny" +mixed_read_write_in_expression = "deny" +shadow_unrelated = "deny" +wildcard_enum_match_arm = "deny" +partial_pub_fields = "deny" +redundant_type_annotations = "deny" +rest_pattern_accessible_field = "deny" +unnecessary_rest_pattern = "deny" +doc_include_without_cfg = "deny" +doc_paragraphs_missing_punctuation = "deny" +tests_outside_test_module = "deny" +host_endian_bytes = "deny" + +# Reject detected in-band SBR instead of publishing AAC-LC-only HE-AAC audio. +[patch.crates-io] +symphonia-codec-aac = { path = "vendor/symphonia-codec-aac" } diff --git a/Dockerfile b/Dockerfile index b373058c..bf91df5c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,14 @@ # syntax=docker/dockerfile:1.7 -FROM rust:1.91.0-bookworm AS builder +FROM rust:1.99.0-bookworm AS builder WORKDIR /build +ARG CARGO_BUILD_JOBS=2 +ENV CARGO_BUILD_JOBS=${CARGO_BUILD_JOBS} # BuildKit retains compiled dependencies between builds without carrying Cargo # caches or the source tree into the final image. COPY Cargo.toml Cargo.lock ./ +COPY vendor ./vendor COPY src ./src COPY static ./static RUN --mount=type=cache,target=/usr/local/cargo/registry,sharing=locked \ @@ -19,10 +22,12 @@ LABEL org.opencontainers.image.title="RustChan" \ org.opencontainers.image.source="https://github.com/csd113/RustChan" \ org.opencontainers.image.licenses="MIT" -# FFmpeg and ffprobe enable RustChan's complete media pipeline. curl provides +# FFmpeg handles video and uncovered audio codecs. Rust handles other media. +# The standalone ffprobe executable is unnecessary. curl provides # a bounded readiness probe; ca-certificates support outbound TLS and Arti. RUN apt-get update && apt-get install -y --no-install-recommends \ ca-certificates curl ffmpeg && \ + rm -f /usr/bin/ffprobe && \ rm -rf /var/lib/apt/lists/* && \ groupadd --system --gid 10001 rustchan && \ useradd --system --uid 10001 --gid rustchan --home-dir /data \ diff --git a/README.md b/README.md index 7f7693a4..f8182193 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ Conversations are organized into **boards** (topics such as photography or techn RustChan runs as a single app, with a built-in database. You do not need to set up a separate database server. -**Source version:** v1.6.0. **Downloads:** [latest published release](https://github.com/csd113/RustChan/releases/latest). **Docker image:** `ghcr.io/csd113/rustchan:latest`. +**Source version:** v1.6.5. **Downloads:** [latest published release](https://github.com/csd113/RustChan/releases/latest). **Docker image:** `ghcr.io/csd113/rustchan:latest`. ## Quick start @@ -28,7 +28,7 @@ Choose the option that suits you: |---|---|---| | [Docker](#start-with-docker) | Running the ready-made app with its media tools included | Docker running on your computer or server | | [Download the app](#download-the-app) | Running RustChan directly, without Docker or compiling code | A supported Linux, macOS, or Windows computer | -| [Build from source](#build-from-source) | Developers who want to build the app themselves | Git and Rust 1.91 or newer | +| [Build from source](#build-from-source) | Developers who want to build the app themselves | Git and Rust 1.99 or newer | ### Start with Docker @@ -62,7 +62,7 @@ docker stop rustchan # Stop the site docker start rustchan # Start it again ``` -The image includes `ffmpeg` and `ffprobe` for video thumbnails, audio waveforms, and supported media conversion. Tor is off by default in Docker. +Images, animated GIF/WebP conversion, HEIC/HEIF, container inspection, common audio waveforms, and supported PDF previews run in Rust. The image includes `ffmpeg` for video thumbnails, WebM transcoding, and audio codecs the Rust decoders do not cover. Tor is off by default in Docker. For **Docker Compose, updates, backups, or a public website**, follow the [container guide](docs/containers.md). Public sites need HTTPS and suitable network settings; complete setup before allowing visitors. The guide also explains how to pin an image to a specific revision instead of following `latest`. @@ -70,14 +70,14 @@ For **Docker Compose, updates, backups, or a public website**, follow the [conta Open the [latest release](https://github.com/csd113/RustChan/releases/latest), download the ZIP for your operating system, and extract it into a folder where the app can save files. -ZIP names include the release version; the v1.6.0 names are shown below. Choose the matching files from the published release you download. +ZIP names include the release version; the v1.6.5 names are shown below. Choose the matching files from the published release you download. | Your computer | Download | |---|---| -| Linux, Intel or AMD 64-bit | `rustchan-cli-v1.6.0-linux-x86_64.zip` | -| Linux, ARM64 | `rustchan-cli-v1.6.0-linux-arm64.zip` | -| macOS, Apple silicon | `rustchan-cli-v1.6.0-macos-apple-silicon.zip` | -| Windows, Intel or AMD 64-bit | `rustchan-cli-v1.6.0-windows-x86_64.zip` | +| Linux, Intel or AMD 64-bit | `rustchan-cli-v1.6.5-linux-x86_64.zip` | +| Linux, ARM64 | `rustchan-cli-v1.6.5-linux-arm64.zip` | +| macOS, Apple silicon | `rustchan-cli-v1.6.5-macos-apple-silicon.zip` | +| Windows, Intel or AMD 64-bit | `rustchan-cli-v1.6.5-windows-x86_64.zip` | From a terminal in the extracted folder, run: @@ -89,11 +89,11 @@ chmod +x rustchan-cli On Windows, run `rustchan-cli.exe` (or `./rustchan-cli.exe` in PowerShell). Follow the terminal's first-run prompts to create your administrator and, optionally, your first board. You can also create boards in [the admin panel](http://localhost:8080/admin). Keep the app running while you use the site. -RustChan creates a `rustchan-data` folder next to the app. Keep that folder when updating. Installing `ffmpeg` and `ffprobe` adds the enhanced media features; see [SETUP.md](SETUP.md) for installation and deployment details. +RustChan creates a `rustchan-data` folder next to the app. Keep that folder when updating. Installing `ffmpeg` enables video processing and the audio compatibility fallback; see [SETUP.md](SETUP.md) for installation and deployment details. ### Build from source -With Git and Rust **1.91 or newer** installed: +With Git and Rust **1.99 or newer** installed: ```bash git clone https://github.com/csd113/RustChan.git @@ -176,7 +176,7 @@ Common settings: | `port` | The HTTP port; normally `8080` | | `enable_tor_support` | Whether the built-in Tor onion service runs | | `tor_only` | Whether the site is served through Tor with a local-only listener | -| `require_ffmpeg` | Whether startup requires the enhanced media tools | +| `require_ffmpeg` | Whether startup requires FFmpeg for video and uncovered audio codecs | | `auto_full_backup_interval_hours` | How often automatic full-site backups run | | `backup_directory` | An optional absolute path for saved backups | | `[tls].enabled` | Whether the app's built-in HTTPS listener runs | @@ -211,14 +211,24 @@ As a site owner, keep settings, secrets, databases, backups, TLS keys, and Tor i RustChan is written in Rust using Axum, Tokio, bundled SQLite, server-rendered templates, Rustls, and Arti. Docker is optional for development. -Run the Rust checks before submitting changes: +Use Rust 1.99.0 and run the Rust checks before submitting changes: ```bash cargo fmt --all --check -cargo clippy --locked --workspace --all-targets --all-features +cargo check --locked --workspace --all-targets --all-features +cargo clippy --locked --workspace --all-targets --all-features -- -D warnings cargo test --locked --workspace --all-features +RUSTDOCFLAGS="-D warnings" cargo doc --locked --workspace --all-features --no-deps ``` +The workspace lint tables in `Cargo.toml` enforce the Rust and Clippy policy for +every RustChan target. They deny the main Clippy groups (`all`, `pedantic`, +`nursery`, and `cargo`) and explicit checks for panic paths, unchecked arithmetic +and indexing, conversions, ignored results, documentation, and resource safety. +Unsafe Rust is forbidden. New workspace packages must inherit this policy with +`[lints] workspace = true`. The patched upstream AAC dependency under `vendor/` +is excluded from the workspace and retains its upstream lint configuration. + Browser tests run locally; their harness and npm files are tracked, but GitHub Actions does not run browser checks. Keep browser reports and temporary screenshots out of commits; the demonstration screenshots in `docs/screenshots/` are published documentation. See [CONTRIBUTING.md](CONTRIBUTING.md) for contribution guidance. ## Documentation diff --git a/SETUP.md b/SETUP.md index eb5feff4..7a435964 100644 --- a/SETUP.md +++ b/SETUP.md @@ -2,12 +2,13 @@ Current setup and deployment guide for Linux, macOS, and Windows. -Current version: `1.5.0`. +Current version: `1.6.5`. This guide reflects the current RustChan architecture: - Tor onion hosting is built in via Arti. You do not install or manage a separate `tor` service. -- `ffmpeg` is optional, but strongly recommended if you want WebP thumbnails, WebM transcoding, video thumbnails, and audio waveforms. +- Images, GIF/WebP animation, HEIC/HEIF, container metadata, common audio waveforms, and supported PDF previews run internally in Rust. Unsupported PDF previews use the built-in SVG placeholder. +- `ffmpeg` is optional, but recommended for video thumbnails, WebM transcoding, and uncovered audio codecs. Standalone `ffprobe` and external PDF renderers are not required. - The post edit form and self-delete flow share a 60-second self-action window after posting. ## Contents @@ -33,9 +34,9 @@ This guide reflects the current RustChan architecture: RustChan is a single Rust binary. A basic install only needs: -- Rust toolchain to build it +- Rust 1.99 or newer to build it - a writable runtime data directory (next to the binary by default, or selected with `--data-dir`) -- `ffmpeg` if you want the enhanced media pipeline +- `ffmpeg` for video processing and uncovered audio codecs RustChan does not require: @@ -96,11 +97,10 @@ cargo --version When `ffmpeg` is available, RustChan can: - extract video thumbnails -- generate audio waveform thumbnails -- convert supported image thumbnails to WebP +- generate waveform thumbnails for uncovered audio codecs (for example AC-3 or Speex) - transcode MP4 uploads to WebM when VP9 and Opus are available -Without `ffmpeg`, RustChan still runs, but video and audio handling degrades gracefully. +Without `ffmpeg`, images, animation, metadata inspection, supported PDF previews, and common audio waveforms still work. Video processing and uncovered-codec waveform previews use their existing placeholders. ### Debian / Ubuntu / Raspberry Pi OS @@ -135,7 +135,6 @@ Then make sure the FFmpeg `bin` directory is on `PATH`. ```bash ffmpeg -version -ffprobe -version ``` If you want RustChan to refuse startup when `ffmpeg` is missing, set: @@ -146,34 +145,36 @@ require_ffmpeg = true ## Verify WebP and WebM Support -RustChan checks more than just whether `ffmpeg` exists. It also checks whether your build includes: +WebP images, GIF → animated WebP, and animated WebP validation are built into RustChan through Rust crates and remain available without FFmpeg. Video thumbnails use FFmpeg to extract a PNG frame and Rust to encode WebP. + +For video/WebM conversion, RustChan checks the installed decoder, encoder and muxer lists independently: -- `libwebp` for WebP image thumbnails and conversions - `libvpx-vp9` for WebM video encoding - `libopus` for WebM audio encoding +- the `webm` muxer for output +- AV1 decoders for AV1 inputs (AV1 encoder availability is reported separately; output remains VP9 + Opus) Use these commands: ```bash -ffmpeg -encoders | rg libwebp ffmpeg -encoders | rg libvpx-vp9 ffmpeg -encoders | rg libopus +ffmpeg -muxers | rg webm +ffmpeg -decoders | rg av1 ``` If you do not have `rg`, use: ```bash -ffmpeg -encoders | grep libwebp ffmpeg -encoders | grep libvpx-vp9 ffmpeg -encoders | grep libopus ``` -You want all three to appear. +WebM conversion needs both selected encoders and the WebM muxer. AV1 input also needs an AV1 decoder; an AV1 encoder is optional and is not selected for RustChan output. ### What Each Encoder Enables -- `libwebp`: WebP thumbnail and image conversion support -- `libvpx-vp9` + `libopus`: MP4 to WebM transcoding support +- `libvpx-vp9` + `libopus`: MP4/Matroska and WebM/AV1 to VP9/Opus WebM conversion. Compatible VP8/VP9 WebM with Opus/Vorbis audio is preserved. ### Linux Notes @@ -181,7 +182,7 @@ On Debian-family systems, the usual install is: ```bash sudo apt update -sudo apt install -y ffmpeg libwebp-dev libvpx-dev libopus-dev +sudo apt install -y ffmpeg libvpx-dev libopus-dev ``` The important part is still the actual `ffmpeg -encoders` output. Package names alone do not guarantee your installed FFmpeg binary was built with every encoder enabled. @@ -191,7 +192,7 @@ The important part is still the actual `ffmpeg -encoders` output. Package names Most Homebrew FFmpeg installs are fine, but verify with: ```bash -ffmpeg -encoders | rg 'libwebp|libvpx-vp9|libopus' +ffmpeg -encoders | rg 'libvpx-vp9|libopus' ``` If one is missing, reinstall FFmpeg from a build source that includes that codec set. @@ -201,7 +202,6 @@ If one is missing, reinstall FFmpeg from a build source that includes that codec Use a full FFmpeg build rather than a minimal one, then verify with: ```powershell -ffmpeg -encoders | Select-String libwebp ffmpeg -encoders | Select-String libvpx-vp9 ffmpeg -encoders | Select-String libopus ``` @@ -210,8 +210,9 @@ ffmpeg -encoders | Select-String libopus RustChan will log warnings and continue: -- missing `libwebp`: image thumbnails stay in original-friendly formats where needed -- missing VP9 or Opus: MP4 uploads are stored as MP4 instead of transcoded to WebM +- missing VP9, Opus or the WebM muxer: video uploads retain their originals +- missing an AV1 decoder: AV1 conversion fails with a useful diagnostic and retains the original +- missing FFmpeg: Rust-native still and animated WebP continue to work These warnings appear in the console at startup and in `rustchan-data/logs/`. @@ -329,7 +330,6 @@ enable_tor_support = true require_ffmpeg = false # ffmpeg_path = "/usr/local/bin/ffmpeg" -# ffprobe_path = "/usr/local/bin/ffprobe" ffmpeg_timeout_secs = 600 [tls] @@ -657,7 +657,7 @@ Run: ```bash ffmpeg -version -ffmpeg -encoders | rg 'libwebp|libvpx-vp9|libopus' +ffmpeg -encoders | rg 'libvpx-vp9|libopus' ``` If one of those encoders is missing, RustChan will still run but some media features will be downgraded. @@ -705,3 +705,21 @@ Back that directory up if the onion address matters. ### Optional administrator-controlled software updates For immutable native Linux version directories, a separate restricted updater service, signing trust setup and automatic backup/rollback, follow [Software Updates](docs/software-updates.md). The existing single-binary service remains deployment-managed until explicitly converted. + +### Internal media processing + +RustChan handles JPEG/PNG/BMP/TIFF, static and animated WebP, GIF conversion, +HEIC/HEIF still pictures, container metadata, and common audio waveforms in Rust. +PDF previews cover a small bounded vector/text subset; compressed streams, +embedded images/fonts and other unbudgeted features use the existing SVG preview +while the original PDF remains available. + +FFmpeg remains the video backend. It also preserves waveform support for audio +variants the internal decoders do not cover, including AC-3, Speex, unsupported +AAC profiles and Opus multistream/surround. See the +[media migration report](docs/non-video-media-migration.md) for format coverage, +resource limits and verification. + +## Applying administrator configuration + +Live controls keep applying immediately. Other controls show their active, saved and next-start values. Saving never restarts RustChan; managed Linux and opted-in supervised containers expose **Restart RustChan** for pending settings. See [settings restarts](docs/settings-restarts.md) for the complete classification, supervisor setup, graceful shutdown and failure recovery. diff --git a/compose.yaml b/compose.yaml index e0d1c8a7..e49b5c54 100644 --- a/compose.yaml +++ b/compose.yaml @@ -8,9 +8,10 @@ services: CHAN_HOST: 0.0.0.0 CHAN_PORT: "8080" CHAN_TOR_SUPPORT: "false" + RUSTCHAN_RESTART_ON_EXIT: "1" volumes: - rustchan-data:/data - stop_grace_period: 30s + stop_grace_period: 90s volumes: rustchan-data: diff --git a/deploy/systemd/rustchan.service b/deploy/systemd/rustchan.service index 8db76773..c56b8fb1 100644 --- a/deploy/systemd/rustchan.service +++ b/deploy/systemd/rustchan.service @@ -17,6 +17,8 @@ ExecStart=/opt/rustchan/current/rustchan-cli --data-dir /var/lib/rustchan serve Restart=on-failure RestartSec=5 TimeoutStopSec=90 +# Configuration recovery must never replace files underneath an unresponsive process. +SendSIGKILL=no UMask=0077 NoNewPrivileges=true ProtectSystem=strict diff --git a/docs/admin-redesign-coverage.md b/docs/admin-redesign-coverage.md index 22ad3cb8..695525b4 100644 --- a/docs/admin-redesign-coverage.md +++ b/docs/admin-redesign-coverage.md @@ -50,7 +50,6 @@ Every inventory leaf is listed below. All 70 leaves have working controls or a j | Master arbitrary-file upload gate | enable_any_file_uploads_feature | `#media` → `#setting-enable_any_file_uploads_feature` | Atomic root TOML; active/saved/source/next state; environment/CLI overrides; restart | 24 config-admin tests pass; 38 new JS/no-JS browser cases pass (save/restart/override/negative cases, proxy simulations, all nine themes) | | Require media tools at startup | require_ffmpeg | `#media` → `#setting-require_ffmpeg` | Atomic root TOML; active/saved/source/next state; environment/CLI overrides; restart | 24 config-admin tests pass; 38 new JS/no-JS browser cases pass (save/restart/override/negative cases, proxy simulations, all nine themes) | | FFmpeg executable | ffmpeg_path | `#media` → `#setting-ffmpeg_path` | Atomic root TOML; active/saved/source/next state; environment/CLI overrides; restart | 24 config-admin tests pass; 38 new JS/no-JS browser cases pass (save/restart/override/negative cases, proxy simulations, all nine themes) | -| FFprobe executable | ffprobe_path | `#media` → `#setting-ffprobe_path` | Atomic root TOML; active/saved/source/next state; environment/CLI overrides; restart | 24 config-admin tests pass; 38 new JS/no-JS browser cases pass (save/restart/override/negative cases, proxy simulations, all nine themes) | | Generated thumbnail dimension | thumb_size | `#media` → `#setting-thumb_size` | Atomic root TOML; active/saved/source/next state; environment/CLI overrides; restart | 24 config-admin tests pass; 38 new JS/no-JS browser cases pass (save/restart/override/negative cases, proxy simulations, all nine themes) | | Background job queue capacity | job_queue_capacity | `#media` → `#setting-job_queue_capacity` | Atomic root TOML; active/saved/source/next state; environment/CLI overrides; restart | 24 config-admin tests pass; 38 new JS/no-JS browser cases pass (save/restart/override/negative cases, proxy simulations, all nine themes) | | Thumbnail/waveform cache budget | waveform_cache_max_mb | `#media` → `#setting-waveform_cache_max_mb` | Atomic root TOML; active/saved/source/next state; environment/CLI overrides; restart | 24 config-admin tests pass; 38 new JS/no-JS browser cases pass (save/restart/override/negative cases, proxy simulations, all nine themes) | diff --git a/docs/containers.md b/docs/containers.md index 77520f01..d9ccbe7e 100644 --- a/docs/containers.md +++ b/docs/containers.md @@ -1,15 +1,16 @@ # Container deployment -RustChan's container listens on HTTP port **8080**. The image includes `ffmpeg`, `ffprobe`, and CA certificates. It runs as the unprivileged `rustchan` user (UID/GID 10001). Templates and static assets are embedded in the server binary. +RustChan's container listens on HTTP port **8080**. The image includes `ffmpeg` and CA certificates. It runs as the unprivileged `rustchan` user (UID/GID 10001). Templates and static assets are embedded in the server binary. ## Pull and run ```bash docker pull ghcr.io/csd113/rustchan:latest docker volume create rustchan-data -docker run -d --name rustchan --restart unless-stopped \ +docker run -d --name rustchan --restart unless-stopped --stop-timeout 90 \ -p 8080:8080 \ -e CHAN_HOST=0.0.0.0 -e CHAN_PORT=8080 -e CHAN_TOR_SUPPORT=false \ + -e RUSTCHAN_RESTART_ON_EXIT=1 \ -v rustchan-data:/data \ ghcr.io/csd113/rustchan:latest ``` @@ -68,3 +69,7 @@ GHCR image: `ghcr.io/csd113/rustchan`. The publishing workflow builds `linux/amd ### Software update checks Administrators can check stable RustChan releases in Software Updates. Container images remain deployment-managed; there is no Install control. Pull the chosen image and recreate the container while retaining the whole data volume. `/readyz` includes the running package `version`, without latest-release or update state. See [Software Updates](software-updates.md). + +## Applying saved settings + +Compose opts into graceful administrator settings restarts using its existing restart policy. The docker run command above pairs the same opt-in with `--restart unless-stopped`. Saving never restarts the container; choose **Restart RustChan** after reviewing pending settings. A bounded readiness check commits the replacement, and failed startup restores the last healthy configuration before the next startup. Keep the loopback HTTP health listener on CHAN_PORT available. Do not set RUSTCHAN_RESTART_ON_EXIT without an automatic process restart policy. See [settings restarts](settings-restarts.md) for classification, shutdown and recovery details. diff --git a/docs/media-capabilities.md b/docs/media-capabilities.md new file mode 100644 index 00000000..792436e3 --- /dev/null +++ b/docs/media-capabilities.md @@ -0,0 +1,74 @@ +# Media capabilities after the Rust follow-up + +This is the current capability audit for `ffmpeg-replace-with-rust`, superseding the historical conclusions in [the original migration report](non-video-media-migration.md). Full removal of FFmpeg except WebM transcoding is **not achieved**. The remaining audio/video compatibility paths are explicit and retained to avoid discarding functionality. No OxiMedia or OxideAV crate is used; the user declined OxideAV. + +## Old behavior and current paths + +“Old” means the pre-migration call sites, with the required executable and codec available. It is not a claim that every file of a named format worked. Unsupported previews preserve the original valid upload; corrupt uploads still follow the existing validation/error contract. + +| Operation / input | Old backend and behavior | Current backend | Verified coverage / remaining limit | +| --- | --- | --- | --- | +| JPEG, PNG, BMP, TIFF, ICO, static WebP | FFmpeg conversion or existing image fallback; resizing, orientation, WebP/PNG size choice | `image`, EXIF orientation, lossless WebP | Decodable output, exact lossless pixels, alpha, orientation, dimensions, metadata stripping, smaller-original PNG policy; hostile/truncated and large transparent images | +| Animated GIF and animated WebP; banners | FFmpeg and existing first-frame fallbacks | Rust compositor, resizer and bounded animated WebP writer | Every composited frame, partial rectangles, disposal, transparency, delays and loop count. GIF 65,535 repeats require 65,536 plays, beyond WebP's finite u16 loop field: preserve the complete original GIF, with its actual MIME and extension, rather than alter its animation | +| HEIC/HEIF still images | External/native image decoder where installed, or image fallback | `heic-rs`, no native HEVC library | Real tiled HEIC, container rotation, auxiliary alpha and iPhone primary SDR picture; all EXIF/ICC/HDR combinations are not claimed. HDR gain-map rendering is outside the established thumbnail behavior | +| Image first-frame thumbnails | FFmpeg or image fallback | Rust image/animation/HEIC decoding | Bounded dimensions, composited first frame, alpha, orientation and original retention | +| MP4/M4A, Matroska/WebM, Ogg and common audio classification / primary codec | FFprobe or bounded FFmpeg stream maps | Symphonia container parsers; small bounded Ogg validators | Content signatures override misleading suffixes, audio-only WebM and multiple tracks, codec names, CRC/sequence/EOS, truncated/ambiguous rejection. Unsupported audio-container mappings still use the explicit FFmpeg probe compatibility path | +| PCM, MP1/2/3, FLAC, Vorbis, ALAC, AAC-LC waveforms | FFmpeg `showwavespic` | Symphonia PCM with streaming mean-absolute channel envelope | WAV/MP3/FLAC/Ogg/ADTS/M4A/Matroska/WebM fixtures, silence, tiny/long/out-of-phase/surround audio, finite sample and packet bounds, cancellation/deadline and atomic publication | +| AAC-LC movie edit lists | FFmpeg applied encoder delay, trim and empty edits | Bounded Rust MP4 metadata walker and streaming presentation timeline | M4A fixture: **5,760** presented 48 kHz frames, versus 7,168 raw decoded frames; independent stereo PCM envelope error ≤0.002. Unit-rate ordered forward ranges and empty edits are supported. Valid dwell, rate changes, repeated/backward edits and edited MP4 Opus use explicit compatibility; every edit is validated before compatibility selection, including malformed entries after a valid unsupported edit | +| Opus voice, stereo and surround | FFmpeg/libopus; earlier Rust surround fallback | `opus-pure` plus Rust multistream framing/mapping | Ogg families 0, 1 and single-stream-container family 255; SILK/CELT, self-delimited VBR/CBR/padding, gain, pre-skip and end trim. Three six-channel golden clips have **11,520** frames and per-channel PCM error ≤0.00004; channel-mean envelope compared independently. Public family 255 probing and decoding are tested | +| Other Opus mappings / containers | FFmpeg accepted mappings supported by its build | Audited compatibility where the existing demuxer exposes the unsupported mapping | Family 2 packet reconstruction is understood by the adapter, but its public Ogg mapping is not implemented. Chained/multiplexed family 255 containers require independent presentation state and are explicitly rejected. No claim of complete arbitrary Opus-container parity | +| HE-AAC / SBR / parametric stereo | FFmpeg full reconstruction where supported | **FFmpeg compatibility retained** | Explicit and sync-extension configurations select compatibility. Patched existing AAC decoder also catches in-band SBR after validating the complete raw-data block; it never publishes the former half-rate AAC-LC-only result. Truncated SBR payloads do not select compatibility | +| Speex NB/WB/UWB waveforms | FFmpeg Speex decode | **FFmpeg compatibility retained**; Rust Ogg framing/metadata validation | Single, all-Speex chained, all-Speex multiplexed and chained multiplexed fixture containers accepted, every logical stream validated (≤64 streams). Mixed-codec containers are rejected. Debian FFmpeg 5.1 requires explicit mono/stereo layout negotiation before its waveform filter; both channel counts are covered. Independent FFmpeg itself reports two decode/timestamp errors on the chained fixture; complete chained waveform parity is not claimed | +| AC-3 and other uncovered audio codecs | FFmpeg codecs installed on the host | **FFmpeg compatibility retained** | AC-3 fixture and precise unsupported-error routing. Generic malformed packet/container, resource, I/O, cancellation and decoder-internal errors cannot trigger fallback | +| Small vector/text PDF first page | `pdftoppm`, `mutool` or `qlmanage` where installed, else SVG | Hayro in Rust, embedded standard fonts | Original vector/text fixtures and a Flate-compressed vector fixture render correctly; compressed old-Poppler output independently confirms the former capability. Aggregate source/expanded content ≤2 MiB, flat page tree ≤4,096 objects, syntax/numeric/depth/work limits, decompression checksum/trailing-data validation | +| Inline-image, nested-resource, embedded-font, encrypted or other complex PDFs | Some documents rendered by an available old external renderer; otherwise SVG | **SVG preview; original PDF retained** | A valid inline RGB image fixture is verified separately with old Poppler. Current gate rejects inline image operators, decompression bombs, recursive/indirect trees and unbudgeted resources before Hayro. This is a remaining preview capability regression, not full PDF parity | +| Video first-frame thumbnail, including MP4/MKV/WebM | FFmpeg video decoding and WebP encoding | **FFmpeg retained** | Standalone H.264 candidate decodes ordinary High 8-bit CABAC, but genuine High 10-bit and High 4:4:4 fixtures fail while the old decoder succeeds. Removing the path would lose existing thumbnail capability. No broad video framework was added | +| Video → VP9/Opus **WebM** | FFmpeg/libvpx/libopus | **FFmpeg retained, intended exception** | Existing arguments, color handling, timeout/cancellation, atomic durable jobs, generated container validation and stale-original redirects | + +## Exact subprocess boundary + +There is no production `ffprobe` command and no external image/PDF renderer command. The runtime image removes `/usr/bin/ffprobe`. FFmpeg commands remain at: + +1. `src/media/ffmpeg.rs::ffmpeg_thumbnail`, called only by the **video** thumbnail branch. +2. `src/workers/mod.rs::render_waveform_with_compatibility`, using the existing bounded waveform command only for audited unsupported audio capabilities. +3. `src/media/ffmpeg.rs::probe_uncovered_audio`, two bounded zero-frame audio/video maps after an explicitly unsupported audio container result. +4. `src/workers/mod.rs::transcode_video` and `build_vp9_transcode_args`, for VP9/Opus **WebM output**. +5. Startup/admin executable `-version` and `-encoders` capability checks for that retained backend. + +`src/media/process.rs` bounds subprocess output, deadlines and process-group cleanup; `/bin/kill` is process cleanup, not codec processing. Retaining calls 1–3 conflicts with the requested WebM-only boundary. They remain visible because removing them under the current constraints would drop working capabilities. + +## Dependency and candidate review + +- Replace `opus-decoder` 0.1.1 with **`opus-pure` 0.2.2**, BSD-3-Clause, MSRV 1.88, no dependencies, build script, C or FFI. RustChan's MSRV remains **1.99**. +- Vendor the already-used **`symphonia-codec-aac` 0.6.1**, MPL-2.0, for one explicit SBR error after full block parsing. No decoder reconstruction or dependency closure is added. See [patch provenance](../vendor/symphonia-codec-aac/RUSTCHAN-PATCH.md); upstream MSRV 1.85 is below the application requirement. +- PDF expansion reuses existing `flate2` and its Rust backend; no new PDF dependency. +- `fdk-aac-rust` 0.2.3 was evaluated outside the application: strict decoding rejects the legitimate in-band fixture with `NonZeroTrailingBits`; its standard API returns 24 kHz core-sized PCM with unusable comparison error. It was not adopted. +- [`rusty_aac` 0.5.0](https://docs.rs/crate/rusty_aac/0.5.0) explicitly provides core-only HE-AAC, without SBR/PS reconstruction. Configuration recognition cannot establish PCM parity. +- [`speexdsp-rs`](https://github.com/rust-av/speexdsp-rs) is DSP/resampling, not a Speex codec. Available native Speex bindings violate the requested implementation boundary; OxideAV was declined. No suitable alternative decoder reached validation. +- `rusty_h264-decoder` 0.16.0 was tested on 8-bit High, High 10-bit and High 4:4:4 synthetic streams. Ordinary High/CABAC works, so stale claims that it cannot decode CABAC were discarded. The other two fail on bit depth/chroma support. A partial H.264 path would not replace all existing video-thumbnail codecs/profiles. +- Hayro lacks a general allocation/work or cooperative cancellation interface for arbitrary PDF programs ([upstream issue](https://github.com/LaurenzV/hayro/issues/1052)). The bounded subset is intentional; enabling arbitrary resource graphs would not satisfy the input/resource contract. + +## Validation evidence + +Validated on 2026-10-02 with native Apple Silicon Rust **1.99.0** (`b940084d7`, 2026-09-28) and an actual Linux arm64 Docker build using the official `rust:1.99.0-bookworm` image. The previously reported unavailable Docker tag is no longer a blocker. Build parallelism was limited to two Cargo jobs per environment. + +```sh +cargo fmt --all --check +cargo check --locked --workspace --all-targets --all-features +cargo clippy --locked --workspace --all-targets --all-features +CHAN_FFMPEG_PATH=/nonexistent/rustchan-validation-ffmpeg CHAN_REQUIRE_FFMPEG=0 \ + cargo test --locked --workspace --all-features -- --test-threads=4 +cargo doc --locked --workspace --all-features --no-deps +cargo deny --locked check +cargo build --release --locked --workspace --all-features +docker build --build-arg CARGO_BUILD_JOBS=2 -t rustchan-media-validation:local . +``` + +- The integrated no-tools workspace run passed **2,619** tests: **1,306** in each library/CLI test binary, five CLI integration tests, one UI fixture test and one rustdoc test. Seven Python update-packaging tests also passed. Formatting, check, canonical strict Clippy, documentation, locked dependency policy and release builds passed. +- A process-spy executable recorded **zero invocations** for all 16 Rust audio tests. The worker routing test recorded exactly eight `-version` capability checks for the eight uncovered audio fixtures, and no codec command. Repeating that test with the installed FFmpeg build containing Speex passed all eight real compatibility waveform paths. A separately built validation FFmpeg without Speex failed this check, demonstrating that fallback still requires the actual codec rather than merely an executable. +- Local Chromium and Chromium without JavaScript passed the media/upload/error suites: **14 tests passed, four intentional JavaScript-only skips**, including 55 JavaScript and 54 no-JavaScript upload-validation scenarios and 14 common audio formats per browser context. The separate restart suite passed **four** tests, and the opt-in real FFmpeg toolchain suite passed **one** test. Browser tests remain local; no hosted browser Actions were introduced. +- The final Linux runtime ran as UID **10001**, with `ffmpeg` and `ffprobe` physically absent from executable search paths and the saved FFmpeg file mode `000`. **42** real upload → processing → preview → download journeys passed across JavaScript and no-JavaScript contexts. These included all 14 common audio fixtures, rotated/alpha HEIC, simple/compressed/unsupported PDF and both converted and maximum-loop GIFs. Original unsupported PDFs and the unrepresentable-loop GIF were retained byte for byte. Independent Pillow checks read generated previews and every animation frame; no FFmpeg participates in these production paths. +- Separately enabled Docker FFmpeg passed two real MP4 → VP9/Opus WebM browser journeys, with WebP thumbnails and stale-original redirects where the conversion raced the first response. Two actual Docker restart-policy journeys verified replacement process identities, readiness, saved settings application, JavaScript reconnect and no-JavaScript recovery. Separate uploads also verify the eight unsupported-audio compatibility fixtures in both browser contexts. The runtime uses Debian FFmpeg **5.1.9** for the explicitly retained compatibility/video backend. +- Synthetic fixtures and independent PCM reference files are checked in under `tests/fixtures/media`. Private browser data, candidate experiments, screenshots, logs and preserved runnable binaries are kept outside `target` and outside the Git deliverable. The final handoff gives their location and the exact commit, image and required-CI results. + +Native systemd restart operation remains unverified on this macOS host; the previous owner's managed-Linux tests used mocked service control. The first required Windows build exposed an unused Unix updater helper in the restart handoff; matching its definition to its existing Unix caller fixes the build without relaxing lints. Configured Windows/Linux/macOS build coverage is reported in the final required GitHub platform checks. Full HE-AAC/Speex reconstruction, arbitrary video thumbnails and complex PDF rendering remain the precise functional limits above; a successful test run does not establish the requested WebM-only FFmpeg boundary. diff --git a/docs/non-video-media-migration.md b/docs/non-video-media-migration.md new file mode 100644 index 00000000..4e88a8d4 --- /dev/null +++ b/docs/non-video-media-migration.md @@ -0,0 +1,365 @@ +# Non-video media backend migration + +Current video pipeline note (2026-10-02): FFmpeg extracts PNG video frames; Rust encodes their WebP thumbnails. FFmpeg WebP encoder detection and installation requirements have been removed. WebM conversion retains VP9 + Opus with independent AV1 decoder/encoder diagnostics. Historical validation results below describe the implementation at the time of that audit. + + +> Historical report for the initial migration at `16f4b91`. The current [capability matrix and follow-up evidence](media-capabilities.md) supersede the uncommitted status, Opus compatibility limitations and unavailable Docker-base conclusions below. Remaining compatibility calls and PDF limitations are still explicit. + +Implemented backend migration, with the explicitly authorized FFmpeg compatibility fallback for uncovered audio codecs/variants. All changes remain uncommitted. Rust 1.99 is now the minimum in Cargo, CI, release builds, Docker and setup documentation. Actual video frame decoding and VP9/Opus WebM transcoding remain on FFmpeg. + +**Final verification conclusion:** suitable for review, but production readiness is not established. The final pass below records passing validation, local fixes, remaining AAC/Speex compatibility limits, and the failed Docker base-image resolution. Do not treat the initial migration results as release approval. + +## Pre-implementation audit + +| Original operation / call site | Class | Required behavior | Implemented replacement / retained backend | Semantic regression coverage | +| --- | --- | --- | --- | --- | +| `media/ffmpeg.rs::ffmpeg_image_to_webp`, used by `convert.rs` | IMAGE | JPEG, BMP, TIFF, HEIC/HEIF to metadata-free WebP; PNG retained only when smaller | Existing `image` lossless WebP encoder; `heic-rs` still decoder | Decode output, dimensions, orientation, alpha, metadata and PNG size policy | +| Same helper with GIF input | IMAGE | Animated WebP, composited frames, delays, transparency and repeat behavior | Existing `image` GIF compositor and lossless WebP encoder, bounded animation container writer | Decode every output frame and compare pixels, delays and loops | +| `ffmpeg_image_to_webp_scaled`, used by `banner.rs` | IMAGE | GIF and animated WebP banners scaled without flattening | Same animation pipeline, existing image WebP decoder | Partial frames, disposal, bounds and stored animation compatibility | +| `thumbnail.rs` image branch -> `ffmpeg_thumbnail` | IMAGE | First composited frame thumbnail, alpha, aspect ratio, no upload mutation | Existing `image` decoder/resizer/encoder plus HEIC still decoder | Thumbnail dimensions/pixels with FFmpeg absent | +| `thumbnail.rs` video branch -> `ffmpeg_thumbnail` | VIDEO | Decode compressed video frame, fit thumbnail; SVG fallback on failure | FFmpeg extracts a PNG frame; Rust encodes WebP (updated by the AV1 follow-up) | Existing video thumbnail tests | +| `probe_stream_kind`, called from upload validation and worker output validation | CONTAINER PROBING | Audio/video classification; MP4/M4A, WebM/MKV, standalone audio; malformed/ambiguous handling | Symphonia ISOBMFF/Matroska/audio format parsers | Old/new stream-kind and malformed fixture comparison | +| `probe_stream_kind_with_ffmpeg` -> two zero-frame stream-map commands | CONTAINER PROBING | Establish ambiguous WebM kind without decoding, neutral download if ambiguous | Same Rust container parsers; unsupported audio containers retain bounded zero-frame FFmpeg maps | Audio WebM, video WebM and ambiguous policy tests | +| `probe_video_codec`, called by transcode preparation/output validation | CONTAINER PROBING | First video codec, skip existing VP9, verify generated VP9 | Same container parsers; actual video processing retained | MP4 H.264 and WebM VP9 before/after | +| `probe_audio_codec`, called by MIME canonicalization | CONTAINER PROBING | Primary codec determines canonical audio MIME | Audio/container parser metadata | MP1/2/3, PCM, Vorbis, FLAC, AAC, Opus and M4A | +| `workers::generate_waveform` / `waveform_prepare` -> `showwavespic` | AUDIO | PNG of thumb width × half-height, streaming decode, deterministic envelope, atomic job persistence, timeout/cancellation | Symphonia plus focused pure-Rust Opus decoder; unavailable codecs/variants use the approved FFmpeg fallback | Supported formats, silence, short/stereo audio, malformed input, job persistence and no executable | +| `thumbnail.rs`: `pdftoppm`, `mutool draw`, `qlmanage -t` | PDF | Bounded first-page WebP or existing static SVG placeholder; rendering failure does not reject valid PDF | Hayro behind a conservative input/work gate; static fallback retained | Simple render, malformed/unsupported PDF fallback | +| `workers::transcode_video` -> `build_vp9_transcode_args` | VIDEO TRANSCODING | Existing libvpx-vp9/libopus flags, color normalization, tuning, timeouts, cancellation, atomicity and queue behavior | Retain existing FFmpeg backend | Existing transcode argument, integration and job tests unchanged | +| `detect.rs`, `media/ffmpeg.rs`: executable `-version` and `-encoders` probes | Capability reporting | Accurate startup/admin status without obsolete non-video tool requirements | Retain FFmpeg/video encoder detection; standalone FFprobe and image/PDF process detection removed | Startup with nonexistent executable paths and reporting tests | + +Related audit surfaces: `media/process.rs` owns bounded subprocess output and process-group cleanup (retained for video); `utils/files/storage.rs` owns validation, orientation, size policy and atomic upload handoff; `workers/mod.rs` owns durable jobs and waveform persistence; application state, posting handlers, maintenance reports, configuration, Dockerfile, README, SETUP and browser media tests carry executable capability assumptions. SVG upload rejection and no-JS/UI hooks remain unchanged. + +## Dependencies and implementation choices + +| Direct dependency | Why selected | Enabled scope / native-library audit | +| --- | --- | --- | +| Existing `image` 0.25.10 | Reuse the existing mature codecs, resizer, orientation support and lossless WebP encoder | JPEG/PNG/GIF/WebP/BMP/TIFF/ICO; no new native image library | +| `gif` 0.14.2 (already transitive) | Read repeat metadata without retaining the whole animation | `default-features = false`; `std`, `color_quant`; pure Rust | +| [Symphonia 0.6.1](https://github.com/pdeljanov/Symphonia) | Established incremental audio decoders and content-based container parsers | Only required audio codecs and ISOBMFF/MKV/Ogg/WAV readers; experimental video **declarations** enable metadata IDs, with no video decoder registered | +| [opus-decoder 0.1.1](https://docs.rs/opus-decoder/0.1.1/opus_decoder/) | Symphonia supplies Opus packets but lacks an Opus decoder | No defaults, no native libopus, no build script; mono/stereo CELT and low-bitrate voice fixtures verified | +| [heic-rs 0.1.1](https://github.com/tbraun96/heic-rs) | Focused HEVC-intra still-picture decoder with container transforms and pixel bounds | `std` only, one decoding thread, no libheif/FFmpeg/system HEVC library or build script | +| [Hayro 0.7.1](https://github.com/LaurenzV/hayro) | First-page vector/text PDF rendering without a native PDF executable/library | No defaults; embedded standard fonts; Rust font, raster, compression and image dependencies | + +`webpkit` was evaluated; its Rust 1.96 requirement is now covered by the requested Rust 1.99 minimum. It would duplicate existing codecs. A small checked RIFF animation writer reuses `image`'s lossless VP8L encoding instead. It writes full composited frames to temporary output without retaining encoded frames for the entire animation. + +`cargo tree` and `cargo tree --edges features` were inspected. The added media dependency closures contain no FFmpeg/libwebp/libheif/native-Opus bindings or multimedia `*-sys` build dependencies. Existing SQLite/TLS/native dependencies elsewhere in the program remain. The lockfile adds 50 package versions and removes no previous package versions. `cargo deny --locked check` passed advisories, bans, licenses and sources (existing duplicate-version and `captcha` manifest-license warnings remain). + +HEIC and Opus decoders are recent focused implementations. Their selected behavior is covered by representative real/generated files, rather than a claim of universal codec conformance. In particular, the Opus crate's multistream splitter rejects a valid standard libopus 5.1 fixture; RustChan deliberately does not use that implementation. Valid mapped Opus streams use the approved compatibility backend. The pinned Opus version maps both unimplemented features and internal transform failures to public `InternalError`. That error now fails closed; only validated multistream metadata explicitly selects compatibility. + +## Before/after format coverage + +| Format/operation | Previous behavior | Current behavior / verified evidence | +| --- | --- | --- | +| JPEG, BMP, TIFF | FFmpeg WebP conversion when available; image-crate fallback previews | Rust lossless WebP; exact decoded pixel equality, dimensions, thumbnails, stripped output metadata | +| PNG, transparent PNG | Keep converted WebP only when smaller | Same size policy in Rust; alpha and decoded pixel equality preserved | +| GIF | FFmpeg animated WebP when codec/tool available | Rust animated WebP; four partial/composited frames, Keep/Previous/Background disposal, transparent pixels, 30/110/230/70 ms delays, infinite and finite loops decoded and compared | +| Static WebP | Preserve upload and generate preview | Existing upload compatibility retained; Rust preview and alpha checks | +| Animated WebP | Preserve stored animation; optional scaled animation for banners | Source animation remains intact; scaled banner output is decoded to prove multiple frames, alpha and timing | +| HEIC/HEIF | FFmpeg still-picture conversion/preview | Rust tiled HEIC and iPhone primary-image decode, alpha and container rotation; exact decoded-pixel equality after lossless WebP encoding and bounded thumbnails | +| MP4/M4A, WebM, MKV | Separate FFprobe, or zero-frame FFmpeg stream inspection | Rust content-based tracks/primary codec inspection; nineteen fixtures compared with FFprobe during final verification, plus malformed input and misleading extensions | +| MP3, FLAC, WAV/PCM, Vorbis, AAC-LC, M4A/ALAC, common Opus, audio WebM/MKV | FFmpeg waveform; placeholder without FFmpeg | Streaming Rust audio and width-sized envelope; MP3/FLAC/WAV/Vorbis/AAC/M4A/Opus/audio-WebM fixtures verified; MKV FLAC track inspection verified | +| Opus mono/stereo, voice/SILK | FFmpeg | Rust, including exact encoder-delay/end-padding handling and header gain | +| Uncovered audio: AC-3, Speex, unsupported AAC profiles, mapped Opus/multistream | FFmpeg when present, otherwise placeholder | Approved FFmpeg compatibility path, selected only for unavailable formats/codecs/variants; AC-3 M4A and standard 5.1 Opus waveform output verified | +| PDF | External Poppler/MuPDF/QuickLook preview when available, otherwise SVG | Hayro renders a small bounded uncompressed vector/text subset; broader or unsafe rendering uses the existing SVG without changing/rejecting the PDF download | +| Video frame thumbnails | FFmpeg/libwebp or SVG | FFmpeg PNG extraction and Rust WebP encoding; SVG fallback on failure | +| Video → WebM | FFmpeg libvpx-vp9/libopus | Retained encoder arguments, color normalization, CPU tuning, timeouts, cancellation, atomic DB/file lifecycle; actual browser transcode, WebM playback markup and stale-MP4 redirect pass | + +The compatibility fallback was explicitly authorized after reproducing AC-3 M4A failure in Rust and success with the original `showwavespic` command. It overrides the objective's earlier strict no-audio-FFmpeg rule. Common codecs never consult an executable to render their waveforms. + +## Equivalence, privacy and resource limits + +- Static lossless WebP pixels match the decoded source exactly; EXIF, ICC and XMP are absent from newly encoded WebP. JPEG orientation 6 remains correct through conversion and thumbnailing. The existing original-PNG size policy and original-file error fallback remain. +- GIF semantic tests decode every WebP frame and compare pixels, order, delays and repeat behavior against the original GIF compositor. A finite GIF repeat count is mapped to WebP total plays. Existing animated WebP scaling remains animated. +- HEIC fixtures include a 2048 × 1536 tiled Apple-encoded scene, 48 × 48 auxiliary alpha, 40 × 64 rotated color corners, and an actual 1512 × 850 iPhone HDR file. HEIF extension handling uses content. Pixel comparisons with FFmpeg gave mean absolute 8-bit channel differences of about 0.19 for the generated tiled scene and 4.90 for the phone primary SDR image; exact encoded bytes/colorspace rounding are not promised. HDR gain-map rendering is outside the existing primary-thumbnail behavior. Provenance and licenses are in `tests/fixtures/media/README.md`. +- The old Poppler renderer and Hayro produce the same 100 × 80 first-page geometry and red interior for the two-page vector PDF fixture. A valid compressed PDF is accepted unchanged and receives SVG. **PDF rendering coverage is deliberately narrower:** flat page trees, an object ceiling before parsing, and a source-size × viewport-pixel work ceiling precede rendering. Compressed streams, embedded images/fonts, recursion-capable resources and other unbudgeted features use placeholders because Hayro exposes no general allocation/work cancellation budget. +- Decoding is off the async executor. Existing upload limits remain; images have a 40-million-pixel budget and allocation limits. Animation processes frames incrementally, checks dimensions, limits frame count to 10,000, and uses the operator deadline. Output RIFF lengths/durations are checked, animated output is capped at 256 MiB before each frame write, and publication uses temporary files. +- HEIC source reads are bounded to the shared allocation limit; primary/alpha coded SPS dimensions are checked before plane allocation, tile counts are bounded, and the sum of coded tile areas is bounded. Codec-level work is cooperative at existing operation boundaries. +- Audio uses two streaming passes (count frames, then fill peak buckets), a 4-million-interleaved-sample packet ceiling, finite-sample checks, deterministic mean absolute channel amplitude, and existing cancellation/deadline checks. Tests cover silence, a single sample frame, opposite-phase six-channel PCM, 120-second PCM, malformed input, unavailable-codec selection and Opus trimming. No full-clip PCM buffer is retained. +- Symphonia container readers cache regular-file length and enforce cancellation, a deadline, a 64 MiB header-read ceiling, a 16 MiB packet-read ceiling, and a repeated-read budget before progressive parser allocations. Metadata tag/artwork sizes are capped. Malformed input, I/O errors, cancellation and resource failures do not select the compatibility decoder. +- Waveform persistence keeps validated paths, temporary output, database transactions, hashes, rollback cleanup and durable job recovery. Faster waveform jobs exposed a stale-SVG request race; protected-board/path checks now precede a temporary redirect to an existing completed PNG, including the race where the file disappears just before `ServeFile` opens it. + +Useful debug measurements on this Apple-silicon host: isolated HEIC decode was about 1.52 s for the tiled scene, 0.97 s for the phone photo, and 1–2 ms for the small alpha/rotation fixtures. These are debug measurements and include no release-performance claim. + +## Remaining process responsibilities + +- `media/ffmpeg.rs::ffmpeg_thumbnail`: actual video-frame decoding to PNG. Rust now encodes the WebP preview; the AV1 follow-up removed FFmpeg WebP encoder detection. +- `workers::transcode_video` / `media/ffmpeg.rs::build_vp9_transcode_args`: retained video WebM transcoder. Both the argument builder and CPU profile selection were compared with HEAD and are unchanged. +- `workers::render_waveform_with_compatibility`: original bounded `showwavespic` command for uncovered audio only, sharing the worker deadline, process-group shutdown and atomic publication. +- `media/ffmpeg.rs::probe_uncovered_audio`: bounded zero-frame stream maps only after an unsupported audio container/parser result, preserving accepted codec/container variants. +- FFmpeg version/encoder detection and configuration executable validation remain for the retained backend. No standalone FFprobe or external PDF renderer process wrapper remains. Docker removes the unneeded standalone `/usr/bin/ffprobe` after installing FFmpeg. + +## Initial migration validation (superseded by final verification below) + +All checks use Rust/cargo 1.99.0. Tests include the existing video argument/lifecycle tests, newly generated media fixtures, strong image/animation pixel checks, real HEIC files, primary-codec/stream comparisons, common audio/voice decoding, multistream compatibility regression, PDF rendering/fallback, malformed inputs and atomic waveform persistence. Browser fixtures now contain real containers rather than header-only fake media; byte-exact PNG assertions allow the original smaller-WebP retention policy. + +| Command / pass | Result | +| --- | --- | +| `cargo fmt --all --check` | Passed | +| `cargo check --workspace --all-targets --all-features` | Passed | +| `cargo clippy --workspace --all-targets --all-features -- -D warnings -D clippy::all -D clippy::pedantic -D clippy::nursery -D clippy::cargo` | Passed | +| `cargo test --workspace --all-features` | Passed: 1,266 tests in each library/application suite, 5 CLI tests, 1 UI fixture test and 1 doc test | +| `cargo tree`; `cargo tree --edges features` | Inspected | +| `cargo deny --locked check` | Passed all policies; existing duplicate-version warnings | +| `CHAN_FFMPEG_PATH=__rustchan_non_video_tools_unavailable__ CHAN_REQUIRE_FFMPEG=0 cargo test --workspace --all-features media::` | Passed: 91 media tests in each library/application suite with FFmpeg unavailable | +| `cargo test --all-features waveform_common_codecs_work_without_tools_and_uncovered_codecs_use_fallback` | Passed for common Opus with tool disabled, AC-3/Opus 5.1 with tool restored, and malformed-input rejection | +| `npx playwright test tests/e2e/upload-validation.spec.ts --project=chromium --project=chromium-nojs --workers=2` | Passed: 55 scenarios with JS, 54 with no JS and one JS-only scenario skipped | +| `npx playwright test tests/e2e/non-video-media.spec.ts --project=chromium --project=chromium-nojs --workers=2` | Passed: all eight common formats complete waveform jobs without FFmpeg in both modes | +| Final combined upload-validation and non-video-media browser run, both Chromium projects | Passed: all 4 tests with the final implementation | +| `RUSTCHAN_E2E_FFMPEG_PATH=/tmp/rustchan-video-validation/ffmpeg-9.0.2/ffmpeg npm run test:e2e:media` | Passed: actual video WebP thumbnail, VP9/Opus WebM, redirect, audio waveform, internal PDF preview and responsive media layouts | +| `git diff --check` | Passed | + +The installed FFmpeg lacked libwebp, so a temporary source build with libwebp/libvpx/libopus was used to run the unchanged strict video browser test. It lives only under `/tmp`; no system FFmpeg installation or runtime video architecture was replaced. Docker was subsequently attempted in the final verification below. Hosted CI/release jobs were not run remotely. + +## Files changed + +- `.github/workflows/ci.yml` +- `.github/workflows/release.yml` +- `Cargo.lock` +- `Cargo.toml` +- `Dockerfile` +- `README.md` +- `SETUP.md` +- `docs/admin-redesign-coverage.md` +- `docs/containers.md` +- `docs/non-video-media-migration.md` +- `docs/strict-lint-exceptions.tsv` +- `src/banner.rs` +- `src/config.rs` +- `src/config/admin.rs` +- `src/config/admin/runtime.rs` +- `src/config/template.rs` +- `src/detect.rs` +- `src/handlers/admin/mod.rs` +- `src/handlers/board/access_preferences.rs` +- `src/handlers/board/create_thread.rs` +- `src/handlers/board/media.rs` +- `src/handlers/mod.rs` +- `src/handlers/posting.rs` +- `src/handlers/setup.rs` +- `src/handlers/thread.rs` +- `src/media/audio.rs` +- `src/media/convert.rs` +- `src/media/ffmpeg.rs` +- `src/media/heif.rs` +- `src/media/images.rs` +- `src/media/images/tests.rs` +- `src/media/mod.rs` +- `src/media/pdf.rs` +- `src/media/probe.rs` +- `src/media/thumbnail.rs` +- `src/middleware/state.rs` +- `src/pending_fs.rs` +- `src/server/server.rs` +- `src/templates/admin.rs` +- `src/templates/admin/maintenance.rs` +- `src/templates/admin/network.rs` +- `src/templates/admin/site_health.rs` +- `src/test_fixtures.rs` +- `src/utils/files/storage.rs` +- `src/workers/mod.rs` +- `tests/cli_runtime.rs` +- `tests/e2e/README.md` +- `tests/e2e/helpers.ts` +- `tests/e2e/media-toolchain-check.mjs` +- `tests/e2e/media-toolchain.spec.ts` +- `tests/e2e/non-video-media.spec.ts` +- `tests/e2e/upload-regressions.spec.ts` +- `tests/e2e/upload-validation.spec.ts` +- `tests/fixtures/media/README.md` +- `tests/fixtures/media/ac3.m4a` +- `tests/fixtures/media/alpha.heic` +- `tests/fixtures/media/audio.mkv` +- `tests/fixtures/media/audio.webm` +- `tests/fixtures/media/compressed.pdf` +- `tests/fixtures/media/irot90.heic` +- `tests/fixtures/media/phone-hdr-LICENSE` +- `tests/fixtures/media/phone-hdr.heic` +- `tests/fixtures/media/photo-LICENSE-APACHE` +- `tests/fixtures/media/photo-LICENSE-MIT` +- `tests/fixtures/media/photo.heic` +- `tests/fixtures/media/simple.pdf` +- `tests/fixtures/media/speech-mode.opus` +- `tests/fixtures/media/surround.opus` +- `tests/fixtures/media/tone.aac` +- `tests/fixtures/media/tone.flac` +- `tests/fixtures/media/tone.m4a` +- `tests/fixtures/media/tone.mp3` +- `tests/fixtures/media/tone.ogg` +- `tests/fixtures/media/tone.opus` +- `tests/fixtures/media/tone.wav` +- `tests/fixtures/media/video.mkv` +- `tests/fixtures/media/video.mp4` +- `tests/fixtures/media/video.webm` + +## Final verification and polish pass + +The current candidate, including changed and untracked source, configuration, templates, documentation, tests and fixtures, was reviewed against the pre-migration HEAD. Existing user changes were preserved. No commit, branch, push, tag, release, backup deletion or known-good replacement was performed. No application dependency was added during this pass. + +### Issues found and local fixes + +- The broad `tests/` ignore hid new compile-time media fixtures and the new browser specification. Narrow exceptions now expose those inputs to Git while generated browser output remains ignored. These untracked files must accompany the candidate when it is eventually committed. +- Unsupported-codec routing was too broad: malformed container errors, resource failures and Opus internal decoder errors could select FFmpeg. Routing now recognizes explicit audited unsupported variants; malformed input, I/O, cancellation, deadlines and limits fail closed. AAC configuration checks distinguish valid unsupported profiles and explicit/implicit SBR metadata from malformed configurations. +- Standard Ogg Speex could disappear inside Symphonia's unsupported Ogg mapping. A bounded page validator now checks identification metadata, framing, CRC, sequence, continuation and completion before selecting compatibility. Real Speex, HE-AAC M4A, AC-3 and mapped surround Opus exercise the worker fallback. +- Container parsing could allocate from hostile packet lengths before the later sample limit. File reads now enforce cancellation/deadlines and header/packet read ceilings before progressive parser allocation. ADTS validation scans fixed headers and seeks over payload instead of buffering the file; reserved frequency indices are rejected. +- PDF page-tree recursion could occur before the existing post-parse object limit. Preflight now rejects recursive/indirect page trees, duplicate objects, revisions and unbudgeted stream/resource structures before Hayro parsing. A checked source-byte × viewport-pixel work ceiling also precedes rendering. Unsupported PDFs keep their original download and deterministic SVG. +- Animated WebP checked the RIFF limit only after encoding the entire animation. The writer now checks a 256 MiB output budget before each frame is published to its temporary output; failures preserve the original upload. +- Completed MKV transcodes could leave stale original URLs returning 404. Validated MP4/MKV URLs now redirect to the completed WebM, including the file-open race. Protected-board and path/symlink checks still precede redirect handling. +- A global test-only PDF renderer override could leak between concurrent tests. The override is now thread-local and guarded; production behavior is unchanged. +- Browser fixtures/assertions assumed obsolete MIME choices, optional local audio encoders and a no-JS mouse gesture that stalled Chromium. Tests now follow the stored media URL, reuse real bundled audio fixtures, assert exact intentional negative-upload diagnostics, and check no-JS resize behavior through CSS. Common-audio browser coverage adds ALAC and SILK/voice Opus. +- Added representative large transparent image, truncated image, malformed container, bounded-read, PDF vector/text and resource regressions. Refreshed the narrow lint-exception ledger, fixture provenance, setup version and stale FFprobe wording. + +### Final validation + +All local Rust checks used Rust/cargo 1.99.0. The following final checks passed after the production changes: + +| Command / pass | Final result | +| --- | --- | +| `cargo fmt --all --check` | Passed | +| `cargo check --workspace --all-targets --all-features` | Passed | +| `cargo clippy --workspace --all-targets --all-features -- -D warnings -D clippy::all -D clippy::pedantic -D clippy::nursery -D clippy::cargo` | Passed | +| `cargo test --workspace --all-features` | 1,275 library + 1,275 application unit tests, 5 CLI integration tests, 1 UI fixture integration test and 1 doc test: **2,557 passed**, zero failures | +| `cargo tree`; `cargo tree --edges features` | Inspected; no new native multimedia requirement in the added dependency closures | +| `cargo deny --locked check` | Advisories, bans, licenses and sources passed; existing duplicate-version and `captcha` manifest-license warnings remain | +| `CHAN_FFMPEG_PATH=__rustchan_non_video_tools_unavailable__ CHAN_REQUIRE_FFMPEG=0 cargo test --workspace --all-features media::` | **100 + 100 media tests passed** with FFmpeg unavailable | +| Chromium + Chromium no-JS: upload validation, non-video media, media and phase-3 runtime | **14 passed, 4 intentional skips**; upload matrix covers 55 JS scenarios and 54 no-JS scenarios, with one JS-only scenario skipped; ten common audio formats complete jobs without FFmpeg | +| Chromium + Chromium no-JS: real media toolchain, auto-compression, opt-in upload regressions and phase-3 runtime | **14 passed, 8 intentional project/JS-only skips**; actual WebP video thumbnails, VP9/Opus WebM, redirects, playback markup and oversized-upload cleanup verified | +| Chromium + Chromium no-JS: affected PDF/media toolchain rerun after the final PDF work ceiling | **8 passed**, zero skips | +| `git diff --check` | Passed, including the final report edits | + +The combined browser command was: + +```sh +RUSTCHAN_AUDIT_OUTPUT=output/playwright/final-verification-browser-verified npx playwright test tests/e2e/upload-validation.spec.ts tests/e2e/non-video-media.spec.ts tests/e2e/media.spec.ts tests/e2e/phase3-media-runtime.spec.ts --project=chromium --project=chromium-nojs --workers=2 +``` + +The real-video command was: + +```sh +RUSTCHAN_UPLOAD_REGRESSION_E2E=1 RUSTCHAN_E2E_MEDIA_TOOLCHAIN=1 RUSTCHAN_E2E_FFMPEG_PATH=/tmp/rustchan-video-validation/ffmpeg-9.0.2/ffmpeg RUSTCHAN_AUDIT_OUTPUT=output/playwright/final-verification-real-media-verified npx playwright test tests/e2e/media-toolchain.spec.ts tests/e2e/auto-compress.spec.ts tests/e2e/upload-regressions.spec.ts tests/e2e/phase3-media-runtime.spec.ts --project=chromium --project=chromium-nojs --workers=2 +``` + +The final affected browser rerun was: + +```sh +RUSTCHAN_E2E_MEDIA_TOOLCHAIN=1 RUSTCHAN_E2E_FFMPEG_PATH=/tmp/rustchan-video-validation/ffmpeg-9.0.2/ffmpeg RUSTCHAN_AUDIT_OUTPUT=output/playwright/final-verification-pdf-budget npx playwright test tests/e2e/media-toolchain.spec.ts tests/e2e/media.spec.ts --project=chromium --project=chromium-nojs --workers=2 +``` + +Final logs are under `/tmp/rustchan-final-verification/`; browser results are in the ignored `output/playwright/final-verification-*` directories. The temporary FFmpeg build includes libwebp/libvpx/libopus and did not replace the installed executable. A temporary Speex fixture encoder was built from the [official Speex release source](https://www.speex.org/downloads/); it is not an application dependency. + +The retained `ffmpeg_thumbnail`, `build_vp9_transcode_args`, `vp9_encoding_profile`, `wait_for_ffmpeg_output`, `waveform_finalise` and `transcode_video_finalise` function bodies were compared with HEAD and remain unchanged. Nineteen real media fixtures were compared with FFprobe as an oracle, including multi-stream MP4, audio-only containers, M4A, WebM and MKV. FFprobe is not required for normal Rust probing. + +### Unresolved compatibility and release limits + +1. **In-band HE-AAC ADTS is a confirmed gap.** Remuxing the HE-AAC M4A fixture to ADTS preserves a valid 48 kHz, two-channel HE-AAC stream according to FFprobe. Symphonia 0.6.1 instead declares and decodes the 24 kHz AAC-LC core successfully, exposes no SBR profile/configuration through its public metadata, and does not return an unsupported error. The compatibility fallback is therefore not selected. Correct SBR reconstruction and waveform equivalence are not established. Header-signaled HE-AAC M4A now falls back correctly, but that does not repair this packet-signaled case. Accurate detection needs decoder support; a payload signature heuristic or routing all common AAC through FFmpeg would violate the intended routing policy. +2. **Speex coverage is deliberately limited to a validated single logical Ogg stream.** Chained/multiplexed Speex is rejected by the new validator. Other unvalidated AAC configurations and payload-level unsupported features also fail closed. These limits prevent a claim that every previously FFmpeg-supported upload remains equivalent. +3. **Docker build/runtime is unresolved.** Docker Desktop and the daemon were available. The actual `docker build --progress=plain --tag rustchan-final-verification:local .` attempt failed while resolving `docker.io/library/rust:1.99.0-bookworm`: `not found`. No builder step ran. The Linux production binary, runtime startup/health, non-root execution and in-container retained FFmpeg behavior remain unverified. An older Rust base image was not substituted and no image was pushed. +4. Image/HEIC codec calls remain cooperative at bounded operation boundaries; the wrapper cannot forcibly interrupt one library decode. Representative behavior and malformed-input/resource regressions passed, but this is not universal codec conformance or a formal bound on every decoder instruction. Hosted CI/release jobs were not run remotely. + +### Release gates + +| # | Gate | Assessment | +| --- | --- | --- | +| 1 | Common non-video media does not require FFmpeg | **Satisfied for the representative supported formats**, including ALAC and SILK/voice Opus | +| 2 | Video still works through FFmpeg | **Satisfied** by real thumbnails/transcoding/browser playback and unchanged argument builders | +| 3 | Approved uncommon-audio fallback works | **Partially satisfied**: AC-3, standard Speex, HE-AAC M4A and surround Opus pass; in-band HE-AAC ADTS remains uncovered | +| 4 | Malformed errors cannot incorrectly select compatibility fallback | **Satisfied for audited routing and regression cases**; explicit unsupported classification is separated from malformed/I/O/resource/cancellation errors | +| 5 | PDF fallback behaves intentionally | **Satisfied**: supported vector/text render; complex or unbudgeted inputs retain the original and receive SVG | +| 6 | No meaningful supported upload behavior was lost | **Not satisfied**: the AAC gap and Speex/other AAC limits prevent certification | +| 7 | Resource/security limits remain enforced | **Satisfied for audited wrappers and tested cases**, with cooperative library-decode limits disclosed above | +| 8 | Docker validates locally, or inability is explicitly reported | **Failure explicitly reported; build/runtime release validation remains unresolved** because the required base tag is unavailable | +| 9 | Rust 1.99 is consistent | **Satisfied** in active Cargo, CI, release, Docker and user/developer setup requirements; historical references are intentional | +| 10 | No unexplained working-tree changes | **Satisfied**: candidate changes reviewed and this pass's additions/fixes listed below | + +The working tree is safe to inspect and review. It is **not certified for a production-ready migration commit or release** until the compatibility gaps and Docker validation are resolved. All changes remain uncommitted. + +### Files changed during this final pass only + +The broader candidate file list above includes earlier migration changes. This pass touched these 24 files: + +- `.gitignore` +- `.github/ISSUE_TEMPLATE/bug_report.yml` +- `SETUP.md` +- `docs/non-video-media-migration.md` +- `docs/strict-lint-exceptions.tsv` +- `src/handlers/board/media.rs` +- `src/media/audio.rs` +- `src/media/images.rs` +- `src/media/images/tests.rs` +- `src/media/pdf.rs` +- `src/media/probe.rs` +- `src/media/probe/speex.rs` (new in this pass) +- `src/media/thumbnail.rs` +- `src/utils/files/storage.rs` +- `src/workers/mod.rs` +- `tests/e2e/non-video-media.spec.ts` +- `tests/e2e/phase3-media-runtime.spec.ts` +- `tests/e2e/upload-regressions.spec.ts` +- `tests/fixtures/media/README.md` +- `tests/fixtures/media/tone-alac.m4a` (new in this pass) +- `tests/fixtures/media/multiple-streams.mp4` (new in this pass) +- `tests/fixtures/media/he-aac.m4a` (new in this pass) +- `tests/fixtures/media/tone.spx` (new in this pass) +- `tests/fixtures/media/text.pdf` (new in this pass) + +### Final `git status --short` + +```text + M .github/ISSUE_TEMPLATE/bug_report.yml + M .github/workflows/ci.yml + M .github/workflows/release.yml + M .gitignore + M Cargo.lock + M Cargo.toml + M Dockerfile + M README.md + M SETUP.md + M docs/admin-redesign-coverage.md + M docs/containers.md + M docs/strict-lint-exceptions.tsv + M src/banner.rs + M src/config.rs + M src/config/admin.rs + M src/config/admin/runtime.rs + M src/config/template.rs + M src/detect.rs + M src/handlers/admin/mod.rs + M src/handlers/board/access_preferences.rs + M src/handlers/board/create_thread.rs + M src/handlers/board/media.rs + M src/handlers/mod.rs + M src/handlers/posting.rs + M src/handlers/setup.rs + M src/handlers/thread.rs + M src/media/convert.rs + M src/media/ffmpeg.rs + M src/media/mod.rs + M src/media/thumbnail.rs + M src/middleware/state.rs + M src/pending_fs.rs + M src/server/server.rs + M src/templates/admin.rs + M src/templates/admin/maintenance.rs + M src/templates/admin/network.rs + M src/templates/admin/site_health.rs + M src/test_fixtures.rs + M src/utils/files/storage.rs + M src/workers/mod.rs + M tests/cli_runtime.rs + M tests/e2e/README.md + M tests/e2e/helpers.ts + M tests/e2e/media-toolchain-check.mjs + M tests/e2e/media-toolchain.spec.ts + M tests/e2e/phase3-media-runtime.spec.ts + M tests/e2e/upload-regressions.spec.ts + M tests/e2e/upload-validation.spec.ts +?? docs/non-video-media-migration.md +?? src/media/audio.rs +?? src/media/heif.rs +?? src/media/images.rs +?? src/media/images/ +?? src/media/pdf.rs +?? src/media/probe.rs +?? src/media/probe/ +?? tests/e2e/non-video-media.spec.ts +?? tests/fixtures/ +``` diff --git a/docs/public-interface-polish-audit.md b/docs/public-interface-polish-audit.md index 8e47116b..bde29ad8 100644 --- a/docs/public-interface-polish-audit.md +++ b/docs/public-interface-polish-audit.md @@ -1,5 +1,8 @@ # Public interface polish audit — 2026-09-30 +Current video pipeline note (2026-10-02): FFmpeg extracts PNG video frames; Rust encodes their WebP thumbnails. FFmpeg WebP encoder detection and installation requirements have been removed. WebM conversion retains VP9 + Opus with independent AV1 decoder/encoder diagnostics. Historical validation results below describe the implementation at the time of that audit. + + Scope: home, board indexes, catalog, threads, search, archives, preferences, media, and polls. Admin changes were excluded; shared behavior was checked with the maintained regression harness. ChanNet is already removed from this branch; no obsolete infrastructure was restored. The initial `interface-audit` working tree was clean. At the audit handoff, all changes were local and uncommitted. No dependencies, framework, route renames, deployment, or visual redesign were introduced. diff --git a/docs/settings-restarts.md b/docs/settings-restarts.md new file mode 100644 index 00000000..23b3983e --- /dev/null +++ b/docs/settings-restarts.md @@ -0,0 +1,146 @@ +# Applying administrator settings + +Saving a form never restarts RustChan. File settings are validated together and +published by a private, synced, atomic replacement that preserves TOML comments. +The **Configuration restart** status is visible above the admin tasks. When saved, +effective startup settings differ from the running process it says: + +> Settings saved. Restart RustChan to apply these changes. + +Review the active/saved/next-start values, then choose **Restart RustChan**. This +separate POST action includes only the existing signed CSRF token. Unsaved form +edits are not included. Unchanged submissions and values masked by environment or +launcher overrides do not create restart warnings. Pending changes survive saves +in other sections. A trial remains in progress until the replacement passes +readiness; accepting a request alone never completes it. + +JavaScript disables repeated clicks and polls the administrator-only status +endpoint for up to four minutes, showing rejection, timeout, reconnect and final +verification/recovery messages. A lost acknowledgment is treated as uncertain: +check durable status before retrying. Without JavaScript the same POST form, +server-rendered status and refresh link work. Secret rotation can invalidate the +administrator session; sign in again to see the result. Listener/host changes may +require navigating to the new public address and updating external firewall or +proxy settings; a restart cannot change those deployment settings. + +## Classification + +The server's `SettingDefinition.application` registry is authoritative. Browser +flags never decide whether a restart is needed. Runtime controls are immutable +`CONFIG` reads or initialize the listener, executor, database pool, tool discovery, +logging or recurring worker schedule. Existing live controls retain their existing +save paths. + +| Application | Controls | +| --- | --- | +| Live | Site name/subtitle; activity badges; default NSFW visibility; default and enabled themes, custom themes and CSS; favicon; banner content, rotation and external-link policy; board identity, ordering, access, upload/posting/display/pruning policies; global active-media pruning and cap; FFmpeg subprocess timeout; automatic full-backup interval, retention, Tor-key inclusion, storage format and split-part size | +| Restart | All Network & Security controls: proxy trust/CIDRs, public hosts, secure-cookie policy, HTTP port/bind address, browsing allowance/window/policy, admin session lifetime, public readiness detail and metrics | +| Restart | All Tor controls: enablement, Tor-only mode, bootstrap timeout, concurrent streams and service nickname | +| Restart | File-backed Media controls: default image/video/audio limits, arbitrary-file global gate, archive-before-prune, FFmpeg requirement/path and capability detection, thumbnail size, job-queue capacity, waveform cache cap, every managed-media reconciliation toggle/interval/budget | +| Restart | Maintenance schedules: WAL checkpoint interval, vacuum interval, poll cleanup interval and database size warning threshold | +| Restart | Executor blocking threads and SQLite pool size | +| Restart | Every native HTTPS/TLS leaf: enablement, HTTPS-only policy, HTTPS/redirect ports, redirect toggle, ACME enablement/domains/email/staging/cache, manual certificate/key paths | +| Restart | Failed-login and board-password allowances/windows, board access-cookie lifetime and signed self-action window; board index page size/reply previews; log filter; normal read/write deadlines | +| Restart | Saved backup directory and cookie-secret rotation. Storage data/database/upload paths remain operator-managed and do not migrate data merely by changing a path. | + +Startup seed aliases for database-owned appearance/board/pruning controls do not +compete with existing database values. Environment and launcher overrides remain +authoritative at startup. Existing sessions retain their stored expiry when only +session lifetime changes. Manual database actions and content/moderation operations +continue to apply immediately. + +## Native managed Linux + +Use the existing `rustchan-updater` deployment in [Software Updates](software-updates.md). +Install the matching updater build (1.6.0 or newer with settings restart support), +fixed root-owned configuration, service units and polkit rule. No additional +privileged daemon, socket or service authority is introduced. The same +`/run/rustchan-updater/control.sock` accepts only the configured web UID, with +0660 socket permissions and bounded request/reply sizes and deadlines. + +The Restart operation has only the authenticated administrator ID and the running +process's server-generated identity. It cannot name a command, executable, +argument, service, environment or filesystem path. The helper still controls only +`rustchan.service` with fixed systemctl start/stop argv, without a shell. Existing +software-install password reauthentication, expiring one-use approvals, release +signatures and archive validation remain required for installation. + +The updater's existing `update.lock`, durable status phases and startup/recovery +admission also own settings restarts. A shared `.settings.lock` file serializes all +configuration saves with service transitions. Restart preflight closes HTTP write +admission; the existing maintenance gate prevents backup/restore overlap. Duplicate +requests, stale process identities and replay of an already consumed restart are +rejected. Software update and restart workers cannot own the service concurrently. +The readiness observation binds a new process identity to exact startup-file bytes; +its private digest is never returned in public health or admin status. + +The updater retains the last ready configuration in its own private state directory. +After gracefully stopping the service it starts the same executable, requires the +fixed loopback `/readyz` to report the expected version, healthy database and a new +process identity in the nonsecret `X-RustChan-Instance` readiness header, and checks that the replacement loaded the requested settings. +Startup and health checks are bounded to 60 seconds. Keep `health_port` aligned +with an HTTP application listener reachable at 127.0.0.1; the supplied unit pins +CHAN_HOST/CHAN_PORT to that endpoint. Changes that remove that probe path cannot +commit, including HTTPS-only configurations without a suitable local backend. +Use the reverse proxy for public TLS with this supplied service model. + +Failed or interrupted settings startup enters the existing updater recovery +barrier, stops the application before restoring the verified last healthy file, +then starts and checks the previous configuration. This is configuration-only +recovery: it does not revert SQLite, posts, media or software. Software updates +continue to use their complete persistent-state snapshot rollback. A settings +rollback restores the whole last healthy settings file, including file-owned +backup defaults; database-owned live changes remain persisted. If previous +configuration recovery also fails, admission stays closed and the private payload +is retained for operator recovery. Configuration files and rollback payloads must +be regular files without symlinks/hardlinks. + +## Containers + +The shipped Compose deployment sets `RUSTCHAN_RESTART_ON_EXIT=1` together with +`restart: unless-stopped` and a 90-second stop grace period. For `docker run`, pair +`--restart unless-stopped --stop-timeout 90` with +`-e RUSTCHAN_RESTART_ON_EXIT=1`. The image alone does not opt in: the process cannot +inspect Docker's restart policy without access to Docker administration, which it +never receives. Do not opt in without a supervisor that restarts the process after +normal exit. Keep the data volume mounted and its private permissions intact. + +On explicit admin restart the process writes private intent, blocks further file +saves, drains HTTP/workers and exits. Docker starts its replacement. Private state +under `runtime/settings-restart` gives the new instance one startup trial; it must +initialize the database/listeners and pass the existing loopback HTTP readiness +probe at CHAN_PORT before committing. A failed or interrupted trial restores the +last healthy file on the next supervised startup, before logging, immutable config +or the database is initialized. A failed recovery remains blocked for operator +repair rather than repeatedly trying an unverified candidate. Docker's existing +health check remains useful for deployment monitoring; a health label alone does +not restart containers. Ordinary `docker exec ... admin` commands do not consume +or roll back a pending startup trial. + +Containers keep software images deployment-managed. The settings restart action +never installs an image or invokes Docker. Native update IPC remains disabled +inside containers. Keep the fixed mapped HTTP health port available; HTTPS-only +cutovers or alternate unmapped listener ports fail the health check and recover. + +## Shutdown and unsupported deployments + +Shutdown closes admission to new work and stops listeners, allowing accepted HTTP +requests ten seconds to finish. Background media workers, scheduled tasks and Tor +receive their existing cancellation token and share one 60-second shutdown budget. +Tracked persistent operations drain within that budget; interrupted durable jobs +retain the existing startup reconciliation path. A final SQLite WAL checkpoint is +attempted, database resources are released, the Tokio blocking pool has a further +ten-second drain cap, and file logging guards flush queued output. SQLite's +transaction/WAL recovery remains authoritative if an operation exceeds the budget. + +The supplied systemd unit has `TimeoutStopSec=90` and `SendSIGKILL=no`. A stuck +service causes control/recovery to fail closed; the helper does not restore files +under an unconfirmed stopped process. No process-name matching, pkill, kill -9, +replacement spawning by the web process, or remote reboot operation is used. + +Standalone terminal invocations, macOS/Windows native runs and custom supervisors +without the explicit container contract show why automatic restart is unavailable. +Saving still works and active/saved settings remain visible. Use a supported +supervised deployment for administrator-triggered restart, or stop/start the +standalone process through its normal launcher. No unsupported service name or +shell-command input is offered as a fallback. diff --git a/docs/software-update-ui-release-audit.md b/docs/software-update-ui-release-audit.md index b64cc398..bcc5e732 100644 --- a/docs/software-update-ui-release-audit.md +++ b/docs/software-update-ui-release-audit.md @@ -1,5 +1,8 @@ # Software Updates UI and v1.6.0 release audit +Current video pipeline note (2026-10-02): FFmpeg extracts PNG video frames; Rust encodes their WebP thumbnails. FFmpeg WebP encoder detection and installation requirements have been removed. WebM conversion retains VP9 + Opus with independent AV1 decoder/encoder diagnostics. Historical validation results below describe the implementation at the time of that audit. + + Audit date: 2026-10-01. Candidate repository: `csd113/RustChan`. ## Prerequisite and scope diff --git a/docs/software-updates.md b/docs/software-updates.md index 644cc976..d20792ac 100644 --- a/docs/software-updates.md +++ b/docs/software-updates.md @@ -8,7 +8,7 @@ package version**. It never reports release discovery or update transactions. ## Boundary and supported deployments -The web process sends only Status, Check, Ready and Install operations over +The web process sends only Status, Check, Ready, Started, Restart and Install operations over `/run/rustchan-updater/control.sock`. Install contains a one-use opaque approval and an authenticated administrator ID; no request supplies a path, URL, command, service or environment. Existing session-scoped CSRF and same-origin checks apply @@ -184,3 +184,7 @@ immutable snapshots as the operator. Correct permissions, space, service/polkit snapshot integrity issues before restarting the updater. No web action can force an unverified restore or clear failed recovery. Keep independent manual/offsite backups; native rollback snapshots protect the local transaction, not disk loss. + +## Settings restarts + +[Administrator settings restarts](settings-restarts.md) reuse this same socket, service controller, transaction lock, journal and recovery admission. Their rollback restores configuration only. Install the matching updater build before enabling this UI; newly packaged releases require updater 1.6.0 or newer. Software installation retains its existing password/approval and complete snapshot protections. diff --git a/docs/sqlite-engineering.md b/docs/sqlite-engineering.md new file mode 100644 index 00000000..3bd82987 --- /dev/null +++ b/docs/sqlite-engineering.md @@ -0,0 +1,466 @@ +# SQLite engineering pass + +## Baseline audit (2026-10-02, before production changes) + +The starting tree was clean. Rust 1.99.0, rusqlite 0.39.0 (bundled SQLite, +backup support), r2d2 0.8.10 and r2d2_sqlite 0.34.0 are authoritative. No +backend replacement, extra service, new dependencies, or commits were introduced. + +### Ownership and atomicity map + +| Area | Baseline implementation | Ownership / correctness boundary | +| --- | --- | --- | +| Startup | `db/pool.rs`, `db/schema.rs`, `db/migrations.rs` | Every connection gets WAL, NORMAL, foreign keys, -32000 KiB cache, MEMORY temp store, 64 MiB mmap and 1000 ms busy timeout. Configurable pool defaults to 8; checkout timeout is 1 second. Schema installation/repair is transactional and structural verification rejects unexpected objects. | +| Thread / reply HTTP | `handlers/board/create_thread.rs`, `handlers/thread.rs`, `handlers/posting.rs` | Blocking workers hold a media semaphore through submission. A connection is checked out before submission and held through identity/body preparation, media probing/hash/thumbnail generation, DB commit, upload promotion, job enqueueing and configured pruning. | +| Thread / reply writes | `db/threads.rs` | BEGIN IMMEDIATE serializes token replay, dedup revalidation, thread/OP or reply/count updates, optional poll and filesystem intent. Reply locked/archived state is checked again under the write lock. Commit failures roll back before return. | +| Upload / dedup | `handlers/mod.rs`, `pending_fs.rs`, `db/fs_ops.rs` | Staged uploads get content hashes and a durable promotion record in the post transaction. Dedup reuse is board-scoped and checked again against current DB references. Promotion/replay retains the intent on failure. | +| Polls / moderation | `db/posts.rs`, `db/admin.rs`, `db/accounts.rs`, `db/boards.rs` | Unique poll voter and one-OP indexes plus schema domain/relationship triggers protect persisted invariants. Edit/delete/account mutations use immediate transactions when read-validation-write must serialize. | +| Jobs | `workers/mod.rs`, `db/posts.rs` | Atomic UPDATE RETURNING claim; bounded attempts; durable media state transitions, retry and interruption recovery. Heavy media prepare/render runs outside final completion transactions. Persistence retries handle pool/busy failures. | +| Pruning / filesystem deletion | `media/prune.rs`, `db/threads.rs`, `pending_fs.rs` | Short intent creation followed by revalidated finalization. Some finalization transactions intentionally include filesystem checks/deletion to exclude racing reference creation; removing these without a replacement protocol is unsafe. | +| Reconciliation | `media/reconcile.rs` | Consistent DB reference snapshot; filesystem inventory outside its read transaction, but the same connection stays borrowed. `data_version` comparisons are connection-local; checking on a newly borrowed connection is not equivalent. Repairs revalidate each candidate inside an immediate transaction. | +| Backups | `handlers/admin/backup/create.rs`, `downloads.rs` | VACUUM INTO includes committed WAL state. Full backup keeps its live pooled connection through file copies, hashing and manifest creation until late in the operation. Maintenance backup also retains it through packaging. | +| Restores | `handlers/admin/backup/restore_full.rs`, `restore_board.rs` | Preflight, snapshot, staged swaps, durable recovery markers and rollback preserve DB/filesystem coordination. Board restore holds a write transaction through its coordinated restore; restructure only with specific recovery evidence. | +| Updates | `updates/transaction.rs`, `updates/snapshot.rs` | Stopped-service VACUUM INTO and verified persistent-state inventory; transaction journal and interrupted-update recovery. No live pool is needed by the isolated updater. | +| FTS / repair | `db/schema.rs`, `db/admin.rs` | FTS5 external-content table and insert/update/delete triggers; explicit rebuild during repair. Backup precedes repair. Structural verification plus quick/integrity and foreign-key checks protect startup/restore. | +| WAL | `db/admin.rs`, maintenance handlers | SQLite default automatic checkpoint (1000 pages); operator TRUNCATE checkpoint reports busy/log/checkpointed. A recurring TRUNCATE checkpoint plus PRAGMA optimize runs hourly by default, staggered and skipped while requests/uploads/maintenance are busy; these two statements originally ran on an async worker. | + +### Findings, ranked + +1. **Avoidable pool occupancy during media preparation.** A scarce pooled + connection remains owned during independent expensive work. The dedup query + is the only DB access required by the primary-upload processor. +2. **Posting state can become stale.** Board access preflight occurs before + multipart parsing; final DB writes do not recheck board access, bans or + cooldown. The bump decision uses the pre-processing reply count and board + bump limit, permitting an extra bump after concurrent replies cross the + limit. Locked/archived reply state is already correctly revalidated. +3. **Board listing aggregates before pagination.** The thread SELECT joins all + replies, groups all matching board threads, then orders and limits. Even a + ten-thread page visits replies of threads outside that page. Profile before + changing it; retain exact preview/count/latest-post semantics. +4. **Backups retain live connections during packaging.** Snapshot-based metadata + extraction can potentially release the live pool earlier and also make + metadata correspond to the actual snapshot. +5. **Limited contention attribution.** Pool/busy errors map to retryable 503 + but there is little timing evidence distinguishing checkout, hold time, + write-lock acquisition and transaction work. +6. **Reconciliation lifetime is deliberate.** Its generation check requires + the same SQLite connection. Do not shorten ownership by simply reacquiring + and comparing `data_version` values from different connections. + +Existing hot reads and job claiming already use cached prepared statements; +preview posts and thread-file collection already batch work. File-reference +OR queries have partial indexes on all three paths. Schema repair already +removes two historical duplicate indexes. Extra indexes will require populated +query-plan/timing evidence and the existing additive migration path. + +### Durability interpretation + +WAL + NORMAL preserves consistency and recovery after application crashes; +some acknowledged transactions may be lost after power loss or OS failure. +FULL adds a WAL sync per commit for stronger commit durability when the storage +stack honors sync. Neither setting fixes faulty storage or application-level +filesystem coordination. Actual power-loss behavior cannot be established by +killing an application process. macOS fullfsync is a distinct setting, so a +local FULL timing alone must not be portrayed as a universal hardware guarantee. +See [SQLite synchronous](https://sqlite.org/pragma.html#pragma_synchronous) and +[WAL](https://sqlite.org/wal.html). Pool/cache/durability defaults remain unchanged +until representative measurements support a change. + +## 1. Baseline architecture + +The audit above records the original connection, transaction, filesystem and +recovery boundaries. SQLite remains bundled and embedded; no new dependencies, +services, unsafe code, database abstraction or backend were introduced. The +pool remains configurable through settings and `CHAN_DB_POOL_SIZE`. + +## 2. Problems found + +The highest-impact measured issue was repeated FTS **prefix** lookups per board +post: a common-term count took 11.2 seconds on the populated fixture. Uploads +held connections through independent processing and nested spam-job checkouts +could exhaust a two-connection pool. Board pages aggregated unrequested replies. +Posting cooldown, access, ban and bump decisions could become stale before the +write lock. Backups held live connections through packaging. Recurring session +purges and checkpoint/optimize calls ran directly on async workers; checkpoint +logs mislabeled SQLite's `busy` flag as pages backfilled. + +## 3. Changes implemented + +- Posting borrows for initial reads, dedup lookup, atomic insertion, promotion + metadata and job persistence independently. Media probing, hashes, encoding, + staged-file promotion and candidate filesystem scans release the pool. +- Transactional revalidation checks current board identity/access, admin session, + bans, cooldown, CAPTCHA/media/body policy and word filters. Changed preparation + policy returns a recoverable conflict; locked/archived threads remain rejected. +- Bump decisions use the current reply count and current board limit under the + immediate write lock, including simultaneous replies at the boundary. +- Thread pages materialize the ordered page before indexed per-thread aggregates. + Missing OPs are excluded before pagination; preview, image count and latest-post + semantics are covered against the original query, including timestamp ties + (active IDs descend, archived IDs ascend). +- FTS drives the prefix-result/count joins; cached statements retain board scope, + sanitization, ordering and pagination. No adaptive strategy remains. +- Two measured indexes use the existing transactional additive repair path: + `(thread_id,id)` for polling and `(board_id,ip_hash,created_at DESC)` for cooldown. +- Full and maintenance backup packaging reads the exact read-only VACUUM snapshot + after releasing the live connection. Authorization is rechecked before snapshot. +- Recurring checkpoint/optimize and session cleanup use blocking workers; checkpoint + logs now report `log_pages`, `checkpointed_pages` and `busy` accurately. +- Thresholded pool/SQL/transaction tracing, pool/file-size metrics and opt-in + offline workload, actual-upload and crash tests provide reproducible evidence. + +## 4. Connection lifetime + +| Phase | Before | After | +| --- | --- | --- | +| Initial validation | One submission-wide checkout | Scoped reads; release before independent preparation | +| Hash / decode / thumbnail | Live connection held | No connection; one scoped dedup lookup | +| Post commit | Same connection | Late checkout; revalidation and atomic commit; immediate release | +| Upload promotion / replay | Same connection through moves / verification | Filesystem work first; short metadata transaction only | +| Job persistence | Live checkout plus queue's nested checkout | Independent media-job scope; release before spam queue checkout | +| Media pruning | Connection held while inventory paths are checked | Collect rows; release; validate paths; borrow per mutation boundary | +| Full backup | Live checkout through packaging | Live VACUUM checkout only; standalone snapshot connection during packaging | + +The one-connection regression acquires the sole connection inside independent +preparation and successfully schedules work after submission. Existing recovery +and deletion protocols retain filesystem I/O inside transactions where a racing +new reference must be excluded. Reconciliation deliberately retains one connection +for `data_version`: comparisons across different connections are invalid. + +## 5. Transaction boundaries + +Posting retains BEGIN IMMEDIATE because token replay, policy checks, dedup paths, +thread flags, counters, bump limits, OP/poll rows and durable filesystem/prune +intents require one serialized decision. Idempotent replay precedes revalidation; +preparation from a losing request is cleaned without creating another post. No +media command/encoding runs inside these transactions. Existing commit rollback +handling is preserved, and typed validation errors survive the boundary. + +Upload finalization still commits hash metadata, optional-thumbnail repair and +intent removal together after promotion. Original-file pruning and deletion keep +their race-exclusion transaction. Restores/updater recovery protocols were audited +and retained. VACUUM INTO necessarily occupies its connection while SQLite makes +the consistent snapshot; independent packaging no longer occupies the live pool. + +## 6. PRAGMA decisions + +The production default changes from NORMAL to **FULL**. Across three optimized +runs, the pool-eight throughput medians show roughly 2%/6%/12.5% cost for +read-heavy/burst/mixed work. Actual media posting remains dominated by processing +CPU. This is a reasonable cost for stronger acknowledged-commit durability when +storage honors sync; it does not improve every failure mode or replace upload +recovery. The connection pool remains eight, configurable as before. + +| Setting | Value | Rationale | +| --- | --- | --- | +| journal_mode | WAL | Readers can proceed alongside one writer; committed WAL state is included in snapshots. | +| synchronous | FULL, every pooled connection | WAL sync at commit; modest measured cost. Process-crash recovery does not establish power-loss guarantees. | +| foreign_keys | ON, every pooled connection | Relationship constraints are connection-local and must remain enforced. | +| cache_size | -32000 | 32,000 KiB target per connection; allocated on demand, roughly 250 MiB across eight fully populated caches. No evidence supports enlarging it. | +| temp_store | MEMORY | Existing bounded page/sort workloads; removed large pre-page aggregates rather than enlarging memory settings. | +| mmap_size | 67108864 | Existing 64 MiB mapping ceiling; virtual/shared mapping is distinct from private per-connection cache. | +| busy_timeout | 1000 ms | Bounded write-lock wait; overload remains retryable rather than hanging. | +| wal_autocheckpoint | SQLite default 1000 pages | Retained alongside existing idle scheduled/operator checkpoints. | + +Pool checkout timeout remains one second. Pool warnings begin at 250 ms and are +rate-limited independently per category to one per 30 seconds per pool. Detailed +SQL, lock-wait and posting transaction timings require `db=debug`, use fixed +operation labels, and contain no SQL parameters or private request content. +Metrics add pool total/idle connections and physical DB/WAL bytes; `-1` means +unavailable. File-size sampling uses async filesystem APIs, and existing metrics +access control and deep schema-health semantics are unchanged. + +## 7. Pool-size findings + +Eight remains the default. Four sometimes achieves higher throughput and lower +p99, but eight greatly reduces checkout waiting and provides useful headroom for +background jobs. Twelve and sixteen do not consistently improve throughput or +latency; extra caches and competing writers make enlargement unjustified. + +Median operations/second across three optimized runs, with eight workers: + +| Pool | NORMAL read / burst / mixed | FULL read / burst / mixed | +| ---: | ---: | ---: | +| 2 | 3545 / 1861 / 2757 | 3598 / 1734 / 2734 | +| 4 | 5567 / 1900 / 3274 | 5699 / 1782 / 3018 | +| 8 | 4953 / 1914 / 3458 | 4854 / 1794 / 3027 | +| 12 | 5209 / 1878 / 3344 | 5092 / 1816 / 3030 | +| 16 | 4956 / 1894 / 3486 | 5064 / 1827 / 3171 | + +In the third run FULL-eight checkout p95 was 3–5 microseconds versus roughly +3–8 milliseconds at pools two/four; 0–1 of 3200 checkouts exceeded 100 microseconds +at eight. This is a measurable-wait proxy, not a claim that every shorter +checkout had no wait. The first FULL-eight read run was a 2496 ops/s outlier; +repeats reached 4854 and 5546. Raw runs retain that variation. + +## 8. Query-plan/index findings + +SQLite 3.51.3, eight boards, 800 threads and 40,000 posts; every query is prepared +against the populated real schema. Twenty-five cached repetitions, nearest-rank +percentiles, with result-ID/order equivalence checks. Times below are medians. + +| Production query | Before | After | Evidence | +| --- | ---: | ---: | --- | +| Board page, 10 threads | 5.372 ms | 0.144 ms | MATERIALIZE page; indexed OP lookup; bounded reply aggregates; removes temp GROUP BY and DISTINCT | +| Rare prefix search, full posts | 74.174 ms | 0.434 ms | One FTS scan, primary-key post lookups | +| Common prefix search, full posts | 56.798 ms | 13.300 ms | Preserved order/limit; eliminates repeated prefix evaluation | +| Missing prefix search, full posts | 38.956 ms | 0.076 ms | Empty FTS result exits before board scan | +| Rare prefix count | 157.500 ms | 0.258 ms | FTS-driven count | +| Common prefix count | 11241.770 ms | 8.623 ms | FTS-driven count | +| Missing prefix count | 38.762 ms | 0.010 ms | FTS-driven count | +| Live polling | 2.749 ms | 0.005 ms | thread/id range index; no order sort | +| Cooldown lookup | 0.154 ms | 0.002 ms | Covering board/IP/timestamp index | + +Exact-term exploratory measurements initially showed a limited-common-term +regression for an FTS-first join. RustChan actually uses sanitized quoted prefix +terms. The final acceptance comparison uses that **actual syntax** and complete +post projections; all three term classes improve. The rejected adaptive probe is +absent from production. CROSS JOIN explicitly preserves the measured join order; +see [SQLite's optimizer guidance](https://sqlite.org/optoverview.html#manual_control_of_query_plans_using_cross_join). + +Two indexes added 1,302,528 bytes (~10.7%) after compacting the dropped-index +fixture: 12,210,176 → 13,512,704 bytes. Isolated FULL write median/p95 changed +384/510 → 404/543 microseconds (~5%/~6.5%); the mixed/burst matrix is the +acceptance gate. Existing +thread/timestamp, cross-board IP and path indexes remain necessary. Historical +redundant indexes were already removed by repair; no speculative new latest-thread, +admin or dedup index was added. Reports, bans, jobs, path OR checks, file hashes +and admin logs already use useful indexes. Full backup/integrity inventories +necessarily scan their complete snapshot/data; metrics/admin deep checks remain +intentional expensive diagnostics. + +## 9. WAL/checkpoint findings + +The baseline sustained matrix retained WAL files up to ~174 MB while all final +PASSIVE checkpoints reported `busy=0` and backfilled all current frames. Physical +WAL bytes are a reusable high-water allocation, not a count of currently live +frames. The third optimized run retained at most 14.3 MB of physical WAL and completed +PASSIVE checkpoints in at most 3.263 ms, all with busy=0 and complete backfill. +The final acceptance run reached at most 15.80 MB WAL, +5.235 ms checkpoint latency and 123.1 MiB case-end RSS; +all final checkpoints completed with busy=0. No evidence supports another +checkpoint manager or changing the existing policy. + +A held read snapshot deterministically prevents complete TRUNCATE (`busy=1`, +checkpointed < log); a concurrent VACUUM snapshot still includes later committed +boards. Releasing the reader permits complete TRUNCATE. Normal WAL readers remain +usable during an uncommitted writer; another writer fails within the existing +busy bound and maps to retryable overload. No new checkpoint manager was added. + +## 10. Benchmark results + +The SQL matrix uses eight concurrent workers, 400 operations each, all pools +2/4/8/12/16, NORMAL/FULL and 5%/70%/25% write mixes: 96,000 operations per matrix. +Reads use production board/catalog/thread/polling/FTS/moderation/poll/dedup helpers. +Writes include replies, counters, token records, FTS triggers, poll votes, durable +job enqueue/claim/complete and job cleanup. Actual PNG submissions separately +exercise new-thread, filesystem-intent/promotion and job paths with two producers +and four readers. No network service, FFmpeg or new dependency is required. + +The tables below compare the original NORMAL/eight default with the final +FULL/eight default, thus including the durability cost. Total latency includes +checkout and, for writes, lock acquisition. Failed operations count in latency +but not successful throughput. The final acceptance matrix also verifies actual +non-OP row counts against every persisted thread reply counter. + +| Workload / default | Ops/s | Total p50 / p95 / p99, ms | Checkout p95, µs | Write wait p95, ms | Write hold p95, ms | Busy / timeout | RSS, MiB | WAL, MiB | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| read_heavy Before | 78 | 3.915 / 694.506 / 950.879 | 11 | 0.189 | 4.900 | 0 / 0 | 102.8 | 12.16 | +| read_heavy After | 5501 | 0.840 / 3.618 / 6.471 | 3 | 11.995 | 3.051 | 0 / 0 | 89.3 | 13.43 | +| write_burst Before | 224 | 2.070 / 57.696 / 807.621 | 5 | 4.654 | 3.199 | 0 / 0 | 90.5 | 165.82 | +| write_burst After | 1747 | 0.482 / 3.080 / 67.702 | 3 | 4.551 | 0.757 | 0 / 0 | 112.3 | 4.67 | +| mixed Before | 96 | 2.913 / 654.620 / 934.125 | 14 | 1.567 | 3.706 | 0 / 0 | 97.2 | 59.10 | +| mixed After | 3065 | 0.517 / 2.624 / 27.526 | 3 | 20.170 | 1.169 | 0 / 0 | 78.1 | 7.30 | + +| Final FULL-eight workload | Read p50 / p95 / p99, ms | Write p50 / p95 / p99, ms | CPU, seconds | Checkpoint, ms | Checkpoint busy / log / backfilled | +| --- | ---: | ---: | ---: | ---: | --- | +| read_heavy | 0.821 / 3.438 / 4.272 | 2.501 / 12.846 / 50.634 | 3.50 | 0.498 | 0 / 43 / 43 | +| write_burst | 0.423 / 1.680 / 1.964 | 0.488 / 5.043 / 108.172 | 1.70 | 2.699 | 0 / 763 / 763 | +| mixed | 0.419 / 1.735 / 1.969 | 0.634 / 20.784 / 95.912 | 1.92 | 2.054 | 0 / 695 / 695 | + +All 96,000 final acceptance operations succeeded across 30 cases: zero busy +errors and zero pool timeouts. Every case passed integrity, foreign keys, FTS, +schema and reply-counter checks. Across all four optimized matrices there was +one bounded busy error in 384,000 operations (the third run's experimental +NORMAL-eight burst); all FULL cases were error-free. The original 96,000-operation +matrix also had zero busy errors/timeouts. This supports the chosen default, +without claiming that experimental configurations never encounter saturation. + +Write-lock **hold** p95 improves in all three default comparisons. Acquisition +wait grows in the much faster read-heavy/mixed workloads because more writes compete +per second; it is separately reported rather than hidden in transaction time. +The longest waits remain governed by the existing one-second busy timeout plus +scheduling overhead. Shortening holds cannot guarantee writer fairness. +Case-end maximum RSS increased +from 102.8 to 123.1 MiB across the full matrix (~20 MiB), with two additional +indexes and more useful cached data. Observed RSS stayed within that range; +these case-end samples do not establish a peak-memory ceiling. + + +Actual upload ownership evidence (milliseconds, 8 PNG posts per case): + +| Pool | Before hold p95 / max | After FULL hold p95 / max | Reader checkout p95, before → FULL after | Pool timeouts, before → after | +| ---: | ---: | ---: | ---: | ---: | +| 2 | 1382.262 / 1382.262 | 1.626 / 4.428 | 374.829 → 0.023 | 20 → 0 | +| 4 | 357.447 / 357.447 | 1.991 / 5.394 | 0.058 → 0.005 | 0 → 0 | +| 8 | 695.155 / 695.155 | 1.670 / 2.451 | 0.047 → 0.005 | 0 → 0 | + +All upload cases created exactly eight posts, finalized every durable upload +intent, and had no pending filesystem records. The eight FULL upload posting +p50/p95 values are reported in raw evidence; processing dominates that timing. + +Raw evidence: [original matrix](sqlite-evidence/baseline.json), +[first optimized matrix](sqlite-evidence/optimized.json), +[second optimized matrix](sqlite-evidence/optimized-repeat.json), +[third optimized matrix](sqlite-evidence/final-matrix.json), +[final acceptance matrix](sqlite-evidence/acceptance-matrix.json), +[final query/index profile](sqlite-evidence/query-profile.json), +[exploratory index profile](sqlite-evidence/index-profile.json), +[original media](sqlite-evidence/media-baseline.json), +[final media](sqlite-evidence/media-optimized.json). The exploratory index profile's +exact-term FTS numbers do not represent production prefix syntax. CPU endpoints +in the first two optimized matrices include verification; use the last two +matrices for workload CPU comparisons. +The SQL matrix excludes HTTP and media CPU. PNG generation precedes timing; +production media processing is included. RSS is sampled at case end, not peak; +CPU time uses optional host `ps` and is available in the final optimized runs only; +timer endpoints exclude verification. +These are debug-build, shared-host measurements, not hardware-independent capacity +promises. Short media runs have only eight posts per case, so pool hold times +and timeout behavior carry more weight than their noisy posting percentiles. + +## 11. Durability/recovery results + +NORMAL and FULL both pass abrupt writer-process termination with 100 committed +replies plus an interrupted open transaction. Restart checks exact reply count, +integrity, foreign keys, synchronized FTS, autocommit/write-lock availability, +interrupted-job recovery and durable filesystem-intent replay. This verifies +application-process crash recovery, not storage power-cut durability. + +The full suite exercises repair/rebuild, insert/edit/delete/restore FTS, dedup +races, one OP, poll-voter uniqueness, failed commits, backup/restore, software-update +snapshots and interrupted updates. Browser validation covers manual/automated +backup, full restore, rejected corrupt restore, native updates/restarts, media, +moderation and seven browser/no-JS projects. + +## 12. Validation + +Formatting required a minimal correction to three existing comma placements in +`server/assets.rs`; no UI behavior changed. Four heavy harness cases per Rust +target are explicitly ignored for normal CI; bounded concurrency and parent crash +regressions run normally. No lint policy was weakened or suppression added. + +The seven-project browser matrix initially passed 104 applicable cases and failed +three copies of the same stale PNG-only assertion. The existing pipeline produced +lossless WebP, so the assertion now checks exact persisted MIME and PNG/WebP magic +bytes. The targeted seven-project rerun passed all three applicable cases; four +projects correctly skip that no-JS-only test. The other 75 matrix skips are +existing project applicability guards. No failing case was disabled. A separate +Chromium recovery/media/update suite passed 35 cases with three existing guards. +Together these are 142 passing browser executions. + +The initial full Rust run exposed a genuine listing tie-order regression, fixed +with active-descending/archive-ascending ID order and full-field equivalence +coverage. A later unrestricted parallel run had one configuration supervisor +lease test fail acquiring a released lease. Its isolated rerun and a complete +four-thread suite passed (1330 tests in each library/binary target, seven +integration/doc tests). The cause of that transient lease failure is not proven; +no lease behavior or assertion was weakened. The final unrestricted rerun instead +had one PDF fallback-format assertion fail; its isolated rerun passed, and the +complete suite was rerun with the four-thread limit documented in +`docs/media-capabilities.md`. Its transient cause is also unproven; the PDF +assertion/implementation were preserved. Final-tree command results follow. + +| Command / suite | Final result | +| --- | --- | +| `cargo fmt --all --check` | Passed | +| `cargo check --locked --workspace --all-targets --all-features` | Passed | +| `cargo clippy --locked --workspace --all-targets --all-features -- -D warnings -D clippy::all -D clippy::pedantic -D clippy::nursery -D clippy::cargo` | Passed | +| `cargo test --locked --workspace --all-features` | Final unrestricted run: 1329 passed, 1 PDF assertion failed; isolated rerun passed; complete four-thread rerun below | +| `cargo test --locked --workspace --all-features -- --test-threads=4` | Passed: 1330 library + 1330 CLI + 5 CLI integration + 1 native UI fixture + 1 doc = 2667; 8 explicitly invoked heavy cases ignored in normal suite | +| `cargo deny --locked check` | Passed advisories, bans, licenses and sources; existing reviewed warnings | +| `python3 -m unittest discover -s tools -p test_update_package.py` | 7 passed | +| `cargo test --locked --lib db:: -- --test-threads=4` | Focused DB regressions passed before final suite | +| `cargo test --locked --lib server::handlers::posting::tests:: -- --test-threads=4` | Posting policy/race/idempotency regressions passed before final suite | +| Ignored SQL matrix / query profile / PNG harness commands below | All passed with schema/integrity/FK/FTS checks; bounded experimental busy result documented | +| Chromium 11-spec posting/media/backup/maintenance/update suite | 35 passed, 3 applicability skips | +| Seven-project posting/password/no-JS/moderation matrix | 104 passed; 3 MIME assertion failures corrected, targeted rerun 3 passed; 75 applicability skips | +| Seven-project corrected upload-format case | 3 passed, 4 applicability skips | +| `git diff --check` | Passed | + +Browser commands (existing repository Playwright harness): + +```sh +RUSTCHAN_UPLOAD_REGRESSION_E2E=1 RUSTCHAN_AUDIT_OUTPUT=output/playwright/sqlite-engineering/chromium npx playwright test tests/e2e/posting-thread.spec.ts tests/e2e/submission-idempotency.spec.ts tests/e2e/upload-validation.spec.ts tests/e2e/upload-regressions.spec.ts tests/e2e/non-video-media.spec.ts tests/e2e/password-boards.spec.ts tests/e2e/moderation-phase2.spec.ts tests/e2e/backup-restore.spec.ts tests/e2e/maintenance-backup-phase2.spec.ts tests/e2e/software-updates.spec.ts tests/e2e/settings-restart.spec.ts --project=chromium --workers=2 --max-failures=5 +RUSTCHAN_AUDIT_OUTPUT=output/playwright/sqlite-engineering/matrix npx playwright test tests/e2e/posting-thread.spec.ts tests/e2e/password-boards.spec.ts tests/e2e/audit-nojs-parity.spec.ts tests/e2e/moderation-phase2.spec.ts --workers=2 --max-failures=5 +RUSTCHAN_E2E_SKIP_BUILD=1 RUSTCHAN_AUDIT_OUTPUT=output/playwright/sqlite-engineering/upload-format npx playwright test tests/e2e/audit-nojs-parity.spec.ts --grep 'file-upload path stores and serves' --workers=2 +``` + +Changed files: + +| Files | Purpose | +| --- | --- | +| `src/db/{mod,pool,diagnostics,workload}.rs` | FULL per connection, bounded pool diagnostics, concurrency/crash tests and offline matrix/profile | +| `src/db/{posts,threads,schema}.rs` | FTS/page queries, two additive indexes, atomic posting revalidation/bump logic, equivalence tests | +| `src/handlers/{mod,posting,thread}.rs`, `src/handlers/board/create_thread.rs`, `src/handlers/posting/performance.rs` | Scoped posting/media ownership, policy race tests, actual PNG harness | +| `src/handlers/admin/backup/create.rs` | Release live connection after authorized snapshot; package snapshot metadata | +| `src/{pending_fs.rs,media/prune.rs}` | Scoped promotion, replay/startup cleanup and configured pruning while preserving recovery boundaries | +| `src/server/server.rs`, `src/server/server/{observability,assets}.rs` | Blocking recurring DB work, accurate checkpoint/pool metrics; required comma formatting | +| `tests/e2e/audit-nojs-parity.spec.ts` | Verify stored upload format and actual bytes | +| `docs/sqlite-engineering.md`, `docs/sqlite-evidence/*.json` | Audit, measured decisions and compact reproducible raw evidence | + +Reproduce the offline evidence from the current tree (run serially to avoid +benchmark interference): + +```sh +RUSTCHAN_DB_EVIDENCE=/tmp/rustchan-db-matrix.json cargo test --locked --lib db::workload::benchmark -- --ignored --exact +RUSTCHAN_DB_QUERY_EVIDENCE=/tmp/rustchan-db-queries.json cargo test --locked --lib db::workload::query_profile -- --ignored --exact +RUSTCHAN_DB_POST_EVIDENCE=/tmp/rustchan-db-media.json cargo test --locked --lib server::handlers::posting::performance::posting_connection_workload -- --ignored --exact +``` + +The crash child is started and killed by its normal parent regression, not +manually invoked. Generated databases are temporary and are never committed. + +## 13. Remaining limitations + +SQLite WAL still has one writer. Short transactions and scoped connections cannot +remove that limit. Consider PostgreSQL only when measurements show sustained +write-lock p95/p99 approaching the one-second busy bound, persistent retryable busy +errors despite short indexed writes, a growing durable job backlog attributable +to write contention, or a real requirement for multiple application servers to +share transactional state. Filesystem coordination would still need redesign for +multiple servers; replacing SQLite alone does not solve it. + +A real storage power-loss campaign, release-build deployment-scale soak, sampled +peak RSS, diverse filesystems/devices and cross-platform timing are outside the +local evidence. Operators should watch pool wait/hold warnings, debug lock/SQL +labels, job queues and WAL growth under their actual workload. Existing deep +health/repair and exclusive restore work can be expensive by design. Nothing in +these measurements justifies a database migration today. + +## 14. CLI startup readiness follow-up + +The published working revision `39675da` had three Linux CLI integration failures +at database-pool construction in CI run `37048226995`. Both Rust test binaries +passed, but separate CLI processes reported a one-second pool timeout. Unchanged +local CLI tests and 60 fresh Linux-container startups did not reproduce that +host-specific scheduling failure; those container runs used the previously built +validation image, not this source revision. + +A deterministic regression exposed an unnecessarily strict startup requirement: +r2d2's checked builder waits for the whole configured pool by default, even when +one fully initialized connection is usable. Holding spare initializers behind a +gate made the former builder fail with the same timeout. Production startup now +keeps eager background filling and checks out one initialized connection before +installing and verifying the schema. Pool capacity, one-second request checkout, +WAL, FULL synchronization, foreign keys and busy timeout are unchanged. An +additional regression rejects a database path that cannot yield any usable +connection. No startup timeout was increased. + +With Rust 1.99, the repaired tree passed all-target/all-feature checking, strict +Clippy (all, pedantic, nursery and cargo), all five CLI integration tests, 2,671 +workspace test executions with four test threads, dependency policy and the +all-feature workspace release build. Four opt-in tests remain ignored in each +Rust test binary. The exact scheduling cause of the earlier hosted failure is +still unproven; exact-head hosted CI remains the final platform verification. diff --git a/docs/sqlite-evidence/acceptance-matrix.json b/docs/sqlite-evidence/acceptance-matrix.json new file mode 100644 index 00000000..a14d73a3 --- /dev/null +++ b/docs/sqlite-evidence/acceptance-matrix.json @@ -0,0 +1,1875 @@ +{ + "fixture": { + "boards": 8, + "posts": 40000, + "threads": 800 + }, + "limitations": "SQL/database workload; media CPU and HTTP excluded; RSS sampled at case end; ps optional; process-kill is not power loss", + "measurements": [ + { + "busy_errors": 0, + "checkout": { + "max_us": 27149, + "p50_us": 3, + "p95_us": 8033, + "p99_us": 14199 + }, + "checkout_over_100us": 1150, + "checkpoint": [ + 0, + 1233, + 1233 + ], + "checkpoint_us": 3243, + "cpu_seconds": 1.7, + "database_bytes": 14753792, + "failures": 0, + "latency": { + "max_us": 28069, + "p50_us": 700, + "p95_us": 8665, + "p99_us": 14763 + }, + "operations": 3200, + "operations_per_second": 3650.079952434896, + "pool_size": 2, + "pool_timeouts": 0, + "read_latency": { + "max_us": 27768, + "p50_us": 703, + "p95_us": 8624, + "p99_us": 14763 + }, + "rss_kib": 67952, + "synchronous": "NORMAL", + "transaction": { + "max_us": 8800, + "p50_us": 479, + "p95_us": 1068, + "p99_us": 8203 + }, + "wal_bytes": 5079992, + "workload": "read_heavy", + "write_latency": { + "max_us": 28069, + "p50_us": 593, + "p95_us": 8688, + "p99_us": 19888 + }, + "write_wait": { + "max_us": 1590, + "p50_us": 3, + "p95_us": 1352, + "p99_us": 1579 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 30565, + "p50_us": 2303, + "p95_us": 10982, + "p99_us": 17861 + }, + "checkout_over_100us": 1821, + "checkpoint": [ + 0, + 135, + 135 + ], + "checkpoint_us": 859, + "cpu_seconds": 1.71, + "database_bytes": 15200256, + "failures": 0, + "latency": { + "max_us": 114153, + "p50_us": 2844, + "p95_us": 12615, + "p99_us": 23983 + }, + "operations": 3200, + "operations_per_second": 1921.250493915227, + "pool_size": 2, + "pool_timeouts": 0, + "read_latency": { + "max_us": 28996, + "p50_us": 2835, + "p95_us": 12281, + "p99_us": 20738 + }, + "rss_kib": 61344, + "synchronous": "NORMAL", + "transaction": { + "max_us": 9068, + "p50_us": 443, + "p95_us": 716, + "p99_us": 4222 + }, + "wal_bytes": 5005832, + "workload": "write_burst", + "write_latency": { + "max_us": 114153, + "p50_us": 2844, + "p95_us": 12639, + "p99_us": 27587 + }, + "write_wait": { + "max_us": 113625, + "p50_us": 3, + "p95_us": 1543, + "p99_us": 26999 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 34851, + "p50_us": 3, + "p95_us": 9006, + "p99_us": 14422 + }, + "checkout_over_100us": 1360, + "checkpoint": [ + 0, + 252, + 252 + ], + "checkpoint_us": 1330, + "cpu_seconds": 1.8099999999999987, + "database_bytes": 14753792, + "failures": 0, + "latency": { + "max_us": 36365, + "p50_us": 1566, + "p95_us": 9779, + "p99_us": 15366 + }, + "operations": 3200, + "operations_per_second": 2838.8765646702172, + "pool_size": 2, + "pool_timeouts": 0, + "read_latency": { + "max_us": 36365, + "p50_us": 1547, + "p95_us": 10217, + "p99_us": 16035 + }, + "rss_kib": 61040, + "synchronous": "NORMAL", + "transaction": { + "max_us": 9512, + "p50_us": 449, + "p95_us": 882, + "p99_us": 4405 + }, + "wal_bytes": 5615592, + "workload": "mixed", + "write_latency": { + "max_us": 29937, + "p50_us": 1780, + "p95_us": 9298, + "p99_us": 14316 + }, + "write_wait": { + "max_us": 26429, + "p50_us": 3, + "p95_us": 1562, + "p99_us": 4576 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 18458, + "p50_us": 2, + "p95_us": 3844, + "p99_us": 7046 + }, + "checkout_over_100us": 810, + "checkpoint": [ + 0, + 23, + 23 + ], + "checkpoint_us": 445, + "cpu_seconds": 1.9800000000000004, + "database_bytes": 14753792, + "failures": 0, + "latency": { + "max_us": 26444, + "p50_us": 745, + "p95_us": 4830, + "p99_us": 8756 + }, + "operations": 3200, + "operations_per_second": 5754.384443533797, + "pool_size": 4, + "pool_timeouts": 0, + "read_latency": { + "max_us": 20128, + "p50_us": 735, + "p95_us": 4789, + "p99_us": 8522 + }, + "rss_kib": 80720, + "synchronous": "NORMAL", + "transaction": { + "max_us": 8860, + "p50_us": 771, + "p95_us": 1170, + "p99_us": 7571 + }, + "wal_bytes": 14160472, + "workload": "read_heavy", + "write_latency": { + "max_us": 26444, + "p50_us": 896, + "p95_us": 5532, + "p99_us": 15629 + }, + "write_wait": { + "max_us": 24071, + "p50_us": 3, + "p95_us": 1571, + "p99_us": 12125 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 20149, + "p50_us": 4, + "p95_us": 6542, + "p99_us": 11044 + }, + "checkout_over_100us": 1543, + "checkpoint": [ + 0, + 621, + 621 + ], + "checkpoint_us": 2676, + "cpu_seconds": 1.7300000000000004, + "database_bytes": 15208448, + "failures": 0, + "latency": { + "max_us": 594165, + "p50_us": 1681, + "p95_us": 9086, + "p99_us": 27270 + }, + "operations": 3200, + "operations_per_second": 1921.7697096814027, + "pool_size": 4, + "pool_timeouts": 0, + "read_latency": { + "max_us": 18091, + "p50_us": 1660, + "p95_us": 7814, + "p99_us": 12071 + }, + "rss_kib": 82736, + "synchronous": "NORMAL", + "transaction": { + "max_us": 8397, + "p50_us": 448, + "p95_us": 815, + "p99_us": 4211 + }, + "wal_bytes": 5347792, + "workload": "write_burst", + "write_latency": { + "max_us": 594165, + "p50_us": 1711, + "p95_us": 10227, + "p99_us": 47309 + }, + "write_wait": { + "max_us": 593277, + "p50_us": 3, + "p95_us": 4409, + "p99_us": 45799 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 18886, + "p50_us": 3, + "p95_us": 5396, + "p99_us": 10221 + }, + "checkout_over_100us": 1158, + "checkpoint": [ + 0, + 83, + 83 + ], + "checkpoint_us": 665, + "cpu_seconds": 1.9399999999999977, + "database_bytes": 14753792, + "failures": 0, + "latency": { + "max_us": 135716, + "p50_us": 884, + "p95_us": 7455, + "p99_us": 14588 + }, + "operations": 3200, + "operations_per_second": 3262.3293833718617, + "pool_size": 4, + "pool_timeouts": 0, + "read_latency": { + "max_us": 19109, + "p50_us": 824, + "p95_us": 6182, + "p99_us": 11736 + }, + "rss_kib": 78080, + "synchronous": "NORMAL", + "transaction": { + "max_us": 8490, + "p50_us": 489, + "p95_us": 1243, + "p99_us": 3388 + }, + "wal_bytes": 8149392, + "workload": "mixed", + "write_latency": { + "max_us": 135716, + "p50_us": 1809, + "p95_us": 12097, + "p99_us": 47023 + }, + "write_wait": { + "max_us": 135088, + "p50_us": 4, + "p95_us": 10518, + "p99_us": 46562 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 154, + "p50_us": 2, + "p95_us": 3, + "p99_us": 4 + }, + "checkout_over_100us": 1, + "checkpoint": [ + 0, + 1825, + 1825 + ], + "checkpoint_us": 8, + "cpu_seconds": 3.460000000000001, + "database_bytes": 14753792, + "failures": 0, + "latency": { + "max_us": 116031, + "p50_us": 829, + "p95_us": 3832, + "p99_us": 6279 + }, + "operations": 3200, + "operations_per_second": 5381.314107521391, + "pool_size": 8, + "pool_timeouts": 0, + "read_latency": { + "max_us": 11224, + "p50_us": 816, + "p95_us": 3654, + "p99_us": 4877 + }, + "rss_kib": 92352, + "synchronous": "NORMAL", + "transaction": { + "max_us": 10085, + "p50_us": 1530, + "p95_us": 3494, + "p99_us": 8633 + }, + "wal_bytes": 7519032, + "workload": "read_heavy", + "write_latency": { + "max_us": 116031, + "p50_us": 2102, + "p95_us": 12638, + "p99_us": 107337 + }, + "write_wait": { + "max_us": 114254, + "p50_us": 6, + "p95_us": 11564, + "p99_us": 105421 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 71, + "p50_us": 2, + "p95_us": 4, + "p99_us": 11 + }, + "checkout_over_100us": 0, + "checkpoint": [ + 0, + 132, + 132 + ], + "checkpoint_us": 709, + "cpu_seconds": 1.8200000000000003, + "database_bytes": 15200256, + "failures": 0, + "latency": { + "max_us": 1035134, + "p50_us": 463, + "p95_us": 5041, + "p99_us": 41292 + }, + "operations": 3200, + "operations_per_second": 1849.7705923182207, + "pool_size": 8, + "pool_timeouts": 0, + "read_latency": { + "max_us": 6219, + "p50_us": 440, + "p95_us": 1809, + "p99_us": 2978 + }, + "rss_kib": 116144, + "synchronous": "NORMAL", + "transaction": { + "max_us": 9817, + "p50_us": 456, + "p95_us": 1079, + "p99_us": 4089 + }, + "wal_bytes": 5409592, + "workload": "write_burst", + "write_latency": { + "max_us": 1035134, + "p50_us": 465, + "p95_us": 5486, + "p99_us": 77682 + }, + "write_wait": { + "max_us": 1033848, + "p50_us": 3, + "p95_us": 4591, + "p99_us": 76899 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 67, + "p50_us": 2, + "p95_us": 3, + "p99_us": 5 + }, + "checkout_over_100us": 0, + "checkpoint": [ + 0, + 575, + 575 + ], + "checkpoint_us": 2817, + "cpu_seconds": 1.9699999999999989, + "database_bytes": 14753792, + "failures": 0, + "latency": { + "max_us": 598539, + "p50_us": 542, + "p95_us": 3643, + "p99_us": 24197 + }, + "operations": 3200, + "operations_per_second": 3314.158252351143, + "pool_size": 8, + "pool_timeouts": 0, + "read_latency": { + "max_us": 10182, + "p50_us": 425, + "p95_us": 1755, + "p99_us": 2129 + }, + "rss_kib": 94912, + "synchronous": "NORMAL", + "transaction": { + "max_us": 8632, + "p50_us": 520, + "p95_us": 1481, + "p99_us": 3293 + }, + "wal_bytes": 12751432, + "workload": "mixed", + "write_latency": { + "max_us": 598539, + "p50_us": 742, + "p95_us": 12847, + "p99_us": 104105 + }, + "write_wait": { + "max_us": 597956, + "p50_us": 41, + "p95_us": 12098, + "p99_us": 103340 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 69, + "p50_us": 2, + "p95_us": 3, + "p99_us": 4 + }, + "checkout_over_100us": 0, + "checkpoint": [ + 0, + 851, + 851 + ], + "checkpoint_us": 5235, + "cpu_seconds": 3.6000000000000085, + "database_bytes": 14753792, + "failures": 0, + "latency": { + "max_us": 109325, + "p50_us": 855, + "p95_us": 3783, + "p99_us": 6434 + }, + "operations": 3200, + "operations_per_second": 5242.628932530368, + "pool_size": 12, + "pool_timeouts": 0, + "read_latency": { + "max_us": 11163, + "p50_us": 833, + "p95_us": 3662, + "p99_us": 4656 + }, + "rss_kib": 86688, + "synchronous": "NORMAL", + "transaction": { + "max_us": 12806, + "p50_us": 1662, + "p95_us": 3054, + "p99_us": 10754 + }, + "wal_bytes": 10749112, + "workload": "read_heavy", + "write_latency": { + "max_us": 109325, + "p50_us": 2238, + "p95_us": 13754, + "p99_us": 29293 + }, + "write_wait": { + "max_us": 107440, + "p50_us": 32, + "p95_us": 12144, + "p99_us": 27222 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 68, + "p50_us": 2, + "p95_us": 3, + "p99_us": 6 + }, + "checkout_over_100us": 0, + "checkpoint": [ + 0, + 216, + 216 + ], + "checkpoint_us": 909, + "cpu_seconds": 1.6999999999999886, + "database_bytes": 15192064, + "failures": 0, + "latency": { + "max_us": 809888, + "p50_us": 445, + "p95_us": 4936, + "p99_us": 72383 + }, + "operations": 3200, + "operations_per_second": 1970.4619041277326, + "pool_size": 12, + "pool_timeouts": 0, + "read_latency": { + "max_us": 9164, + "p50_us": 425, + "p95_us": 1713, + "p99_us": 1988 + }, + "rss_kib": 93184, + "synchronous": "NORMAL", + "transaction": { + "max_us": 8564, + "p50_us": 437, + "p95_us": 779, + "p99_us": 4459 + }, + "wal_bytes": 5759792, + "workload": "write_burst", + "write_latency": { + "max_us": 809888, + "p50_us": 447, + "p95_us": 5273, + "p99_us": 105280 + }, + "write_wait": { + "max_us": 809258, + "p50_us": 3, + "p95_us": 4582, + "p99_us": 104525 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 67, + "p50_us": 2, + "p95_us": 3, + "p99_us": 6 + }, + "checkout_over_100us": 0, + "checkpoint": [ + 0, + 68, + 68 + ], + "checkpoint_us": 583, + "cpu_seconds": 2.0, + "database_bytes": 14753792, + "failures": 0, + "latency": { + "max_us": 578485, + "p50_us": 553, + "p95_us": 5020, + "p99_us": 27582 + }, + "operations": 3200, + "operations_per_second": 3386.712867074695, + "pool_size": 12, + "pool_timeouts": 0, + "read_latency": { + "max_us": 12570, + "p50_us": 427, + "p95_us": 1766, + "p99_us": 2181 + }, + "rss_kib": 89952, + "synchronous": "NORMAL", + "transaction": { + "max_us": 8781, + "p50_us": 517, + "p95_us": 1339, + "p99_us": 3472 + }, + "wal_bytes": 15800232, + "workload": "mixed", + "write_latency": { + "max_us": 578485, + "p50_us": 800, + "p95_us": 25266, + "p99_us": 72704 + }, + "write_wait": { + "max_us": 577857, + "p50_us": 46, + "p95_us": 24682, + "p99_us": 72225 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 62, + "p50_us": 2, + "p95_us": 3, + "p99_us": 5 + }, + "checkout_over_100us": 0, + "checkpoint": [ + 0, + 83, + 83 + ], + "checkpoint_us": 1144, + "cpu_seconds": 3.530000000000001, + "database_bytes": 14753792, + "failures": 0, + "latency": { + "max_us": 80311, + "p50_us": 850, + "p95_us": 3665, + "p99_us": 6606 + }, + "operations": 3200, + "operations_per_second": 5309.300633585225, + "pool_size": 16, + "pool_timeouts": 0, + "read_latency": { + "max_us": 12853, + "p50_us": 820, + "p95_us": 3526, + "p99_us": 4442 + }, + "rss_kib": 86416, + "synchronous": "NORMAL", + "transaction": { + "max_us": 10221, + "p50_us": 1739, + "p95_us": 2915, + "p99_us": 9542 + }, + "wal_bytes": 13905032, + "workload": "read_heavy", + "write_latency": { + "max_us": 80311, + "p50_us": 2306, + "p95_us": 13655, + "p99_us": 78021 + }, + "write_wait": { + "max_us": 79736, + "p50_us": 71, + "p95_us": 12077, + "p99_us": 75820 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 71, + "p50_us": 2, + "p95_us": 3, + "p99_us": 6 + }, + "checkout_over_100us": 0, + "checkpoint": [ + 0, + 645, + 645 + ], + "checkpoint_us": 2550, + "cpu_seconds": 1.6899999999999977, + "database_bytes": 15192064, + "failures": 0, + "latency": { + "max_us": 813625, + "p50_us": 448, + "p95_us": 3475, + "p99_us": 46437 + }, + "operations": 3200, + "operations_per_second": 1904.5372521228326, + "pool_size": 16, + "pool_timeouts": 0, + "read_latency": { + "max_us": 6073, + "p50_us": 420, + "p95_us": 1699, + "p99_us": 2019 + }, + "rss_kib": 124976, + "synchronous": "NORMAL", + "transaction": { + "max_us": 8339, + "p50_us": 443, + "p95_us": 702, + "p99_us": 4531 + }, + "wal_bytes": 5417832, + "workload": "write_burst", + "write_latency": { + "max_us": 813625, + "p50_us": 450, + "p95_us": 5016, + "p99_us": 74327 + }, + "write_wait": { + "max_us": 811513, + "p50_us": 3, + "p95_us": 3878, + "p99_us": 73730 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 69, + "p50_us": 2, + "p95_us": 3, + "p99_us": 5 + }, + "checkout_over_100us": 0, + "checkpoint": [ + 0, + 303, + 303 + ], + "checkpoint_us": 1721, + "cpu_seconds": 2.0200000000000102, + "database_bytes": 14753792, + "failures": 0, + "latency": { + "max_us": 238312, + "p50_us": 600, + "p95_us": 4847, + "p99_us": 25798 + }, + "operations": 3200, + "operations_per_second": 3560.797512397481, + "pool_size": 16, + "pool_timeouts": 0, + "read_latency": { + "max_us": 12025, + "p50_us": 439, + "p95_us": 1783, + "p99_us": 2033 + }, + "rss_kib": 91248, + "synchronous": "NORMAL", + "transaction": { + "max_us": 8660, + "p50_us": 555, + "p95_us": 1441, + "p99_us": 3546 + }, + "wal_bytes": 11358872, + "workload": "mixed", + "write_latency": { + "max_us": 238312, + "p50_us": 784, + "p95_us": 24805, + "p99_us": 105116 + }, + "write_wait": { + "max_us": 237500, + "p50_us": 47, + "p95_us": 24056, + "p99_us": 104677 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 33116, + "p50_us": 3, + "p95_us": 7830, + "p99_us": 15332 + }, + "checkout_over_100us": 1168, + "checkpoint": [ + 0, + 800, + 800 + ], + "checkpoint_us": 3138, + "cpu_seconds": 1.6899999999999977, + "database_bytes": 14753792, + "failures": 0, + "latency": { + "max_us": 33458, + "p50_us": 704, + "p95_us": 8448, + "p99_us": 16080 + }, + "operations": 3200, + "operations_per_second": 3631.23366052655, + "pool_size": 2, + "pool_timeouts": 0, + "read_latency": { + "max_us": 33458, + "p50_us": 703, + "p95_us": 8341, + "p99_us": 15680 + }, + "rss_kib": 74752, + "synchronous": "FULL", + "transaction": { + "max_us": 9017, + "p50_us": 528, + "p95_us": 1526, + "p99_us": 8720 + }, + "wal_bytes": 5953432, + "workload": "read_heavy", + "write_latency": { + "max_us": 26528, + "p50_us": 719, + "p95_us": 10417, + "p99_us": 23649 + }, + "write_wait": { + "max_us": 10059, + "p50_us": 3, + "p95_us": 217, + "p99_us": 4576 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 31893, + "p50_us": 2472, + "p95_us": 10840, + "p99_us": 16138 + }, + "checkout_over_100us": 1894, + "checkpoint": [ + 0, + 638, + 638 + ], + "checkpoint_us": 1654, + "cpu_seconds": 1.6899999999999977, + "database_bytes": 15208448, + "failures": 0, + "latency": { + "max_us": 302950, + "p50_us": 3017, + "p95_us": 12289, + "p99_us": 21848 + }, + "operations": 3200, + "operations_per_second": 1839.2328930668255, + "pool_size": 2, + "pool_timeouts": 0, + "read_latency": { + "max_us": 24760, + "p50_us": 3195, + "p95_us": 11544, + "p99_us": 16356 + }, + "rss_kib": 77824, + "synchronous": "FULL", + "transaction": { + "max_us": 8674, + "p50_us": 472, + "p95_us": 683, + "p99_us": 3385 + }, + "wal_bytes": 4911072, + "workload": "write_burst", + "write_latency": { + "max_us": 302950, + "p50_us": 2921, + "p95_us": 12703, + "p99_us": 27447 + }, + "write_wait": { + "max_us": 300226, + "p50_us": 3, + "p95_us": 4, + "p99_us": 23726 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 29149, + "p50_us": 3, + "p95_us": 8763, + "p99_us": 15037 + }, + "checkout_over_100us": 1336, + "checkpoint": [ + 0, + 20, + 20 + ], + "checkpoint_us": 329, + "cpu_seconds": 1.789999999999992, + "database_bytes": 14753792, + "failures": 0, + "latency": { + "max_us": 50419, + "p50_us": 1452, + "p95_us": 9742, + "p99_us": 16103 + }, + "operations": 3200, + "operations_per_second": 2843.686024534678, + "pool_size": 2, + "pool_timeouts": 0, + "read_latency": { + "max_us": 29228, + "p50_us": 1365, + "p95_us": 9572, + "p99_us": 15613 + }, + "rss_kib": 76896, + "synchronous": "FULL", + "transaction": { + "max_us": 8357, + "p50_us": 481, + "p95_us": 902, + "p99_us": 3327 + }, + "wal_bytes": 5207712, + "workload": "mixed", + "write_latency": { + "max_us": 50419, + "p50_us": 1798, + "p95_us": 10274, + "p99_us": 17542 + }, + "write_wait": { + "max_us": 49616, + "p50_us": 3, + "p95_us": 1560, + "p99_us": 4563 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 26552, + "p50_us": 3, + "p95_us": 3869, + "p99_us": 7489 + }, + "checkout_over_100us": 835, + "checkpoint": [ + 0, + 23, + 23 + ], + "checkpoint_us": 509, + "cpu_seconds": 1.990000000000009, + "database_bytes": 14753792, + "failures": 0, + "latency": { + "max_us": 26923, + "p50_us": 747, + "p95_us": 4726, + "p99_us": 9305 + }, + "operations": 3200, + "operations_per_second": 5722.362594061626, + "pool_size": 4, + "pool_timeouts": 0, + "read_latency": { + "max_us": 26923, + "p50_us": 742, + "p95_us": 4567, + "p99_us": 8710 + }, + "rss_kib": 77280, + "synchronous": "FULL", + "transaction": { + "max_us": 9183, + "p50_us": 661, + "p95_us": 1383, + "p99_us": 6254 + }, + "wal_bytes": 8837432, + "workload": "read_heavy", + "write_latency": { + "max_us": 13458, + "p50_us": 863, + "p95_us": 7108, + "p99_us": 12686 + }, + "write_wait": { + "max_us": 12105, + "p50_us": 3, + "p95_us": 1662, + "p99_us": 12104 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 23500, + "p50_us": 598, + "p95_us": 6287, + "p99_us": 10059 + }, + "checkout_over_100us": 1643, + "checkpoint": [ + 0, + 461, + 461 + ], + "checkpoint_us": 1461, + "cpu_seconds": 1.740000000000009, + "database_bytes": 15200256, + "failures": 0, + "latency": { + "max_us": 588000, + "p50_us": 1710, + "p95_us": 8511, + "p99_us": 26381 + }, + "operations": 3200, + "operations_per_second": 1828.9674439182554, + "pool_size": 4, + "pool_timeouts": 0, + "read_latency": { + "max_us": 18283, + "p50_us": 1662, + "p95_us": 7368, + "p99_us": 12336 + }, + "rss_kib": 90080, + "synchronous": "FULL", + "transaction": { + "max_us": 8819, + "p50_us": 480, + "p95_us": 724, + "p99_us": 3364 + }, + "wal_bytes": 5121192, + "workload": "write_burst", + "write_latency": { + "max_us": 588000, + "p50_us": 1789, + "p95_us": 9141, + "p99_us": 46981 + }, + "write_wait": { + "max_us": 587067, + "p50_us": 3, + "p95_us": 1574, + "p99_us": 45571 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 20907, + "p50_us": 3, + "p95_us": 5376, + "p99_us": 8985 + }, + "checkout_over_100us": 1135, + "checkpoint": [ + 0, + 236, + 236 + ], + "checkpoint_us": 1139, + "cpu_seconds": 1.9199999999999875, + "database_bytes": 14753792, + "failures": 0, + "latency": { + "max_us": 130241, + "p50_us": 839, + "p95_us": 7078, + "p99_us": 12853 + }, + "operations": 3200, + "operations_per_second": 3232.2987452115262, + "pool_size": 4, + "pool_timeouts": 0, + "read_latency": { + "max_us": 21064, + "p50_us": 794, + "p95_us": 6132, + "p99_us": 9988 + }, + "rss_kib": 81168, + "synchronous": "FULL", + "transaction": { + "max_us": 8776, + "p50_us": 518, + "p95_us": 1137, + "p99_us": 3198 + }, + "wal_bytes": 7576712, + "workload": "mixed", + "write_latency": { + "max_us": 130241, + "p50_us": 1904, + "p95_us": 11741, + "p99_us": 47657 + }, + "write_wait": { + "max_us": 128660, + "p50_us": 4, + "p95_us": 9723, + "p99_us": 47130 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 35, + "p50_us": 2, + "p95_us": 3, + "p99_us": 4 + }, + "checkout_over_100us": 0, + "checkpoint": [ + 0, + 43, + 43 + ], + "checkpoint_us": 498, + "cpu_seconds": 3.5, + "database_bytes": 14753792, + "failures": 0, + "latency": { + "max_us": 111647, + "p50_us": 840, + "p95_us": 3618, + "p99_us": 6471 + }, + "operations": 3200, + "operations_per_second": 5501.155242600947, + "pool_size": 8, + "pool_timeouts": 0, + "read_latency": { + "max_us": 11511, + "p50_us": 821, + "p95_us": 3438, + "p99_us": 4272 + }, + "rss_kib": 91472, + "synchronous": "FULL", + "transaction": { + "max_us": 11056, + "p50_us": 1729, + "p95_us": 3051, + "p99_us": 8137 + }, + "wal_bytes": 14078072, + "workload": "read_heavy", + "write_latency": { + "max_us": 111647, + "p50_us": 2501, + "p95_us": 12846, + "p99_us": 50634 + }, + "write_wait": { + "max_us": 109849, + "p50_us": 96, + "p95_us": 11995, + "p99_us": 49590 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 88, + "p50_us": 3, + "p95_us": 3, + "p99_us": 5 + }, + "checkout_over_100us": 0, + "checkpoint": [ + 0, + 763, + 763 + ], + "checkpoint_us": 2699, + "cpu_seconds": 1.700000000000017, + "database_bytes": 15196160, + "failures": 0, + "latency": { + "max_us": 1034435, + "p50_us": 482, + "p95_us": 3080, + "p99_us": 67702 + }, + "operations": 3200, + "operations_per_second": 1746.521431569672, + "pool_size": 8, + "pool_timeouts": 0, + "read_latency": { + "max_us": 10375, + "p50_us": 423, + "p95_us": 1680, + "p99_us": 1964 + }, + "rss_kib": 115024, + "synchronous": "FULL", + "transaction": { + "max_us": 8145, + "p50_us": 477, + "p95_us": 757, + "p99_us": 3321 + }, + "wal_bytes": 4898712, + "workload": "write_burst", + "write_latency": { + "max_us": 1034435, + "p50_us": 488, + "p95_us": 5043, + "p99_us": 108172 + }, + "write_wait": { + "max_us": 1033877, + "p50_us": 3, + "p95_us": 4551, + "p99_us": 107027 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 71, + "p50_us": 2, + "p95_us": 3, + "p99_us": 5 + }, + "checkout_over_100us": 0, + "checkpoint": [ + 0, + 695, + 695 + ], + "checkpoint_us": 2054, + "cpu_seconds": 1.9199999999999875, + "database_bytes": 14753792, + "failures": 0, + "latency": { + "max_us": 793298, + "p50_us": 517, + "p95_us": 2624, + "p99_us": 27526 + }, + "operations": 3200, + "operations_per_second": 3065.4335960026747, + "pool_size": 8, + "pool_timeouts": 0, + "read_latency": { + "max_us": 11631, + "p50_us": 419, + "p95_us": 1735, + "p99_us": 1969 + }, + "rss_kib": 79952, + "synchronous": "FULL", + "transaction": { + "max_us": 8951, + "p50_us": 523, + "p95_us": 1169, + "p99_us": 3226 + }, + "wal_bytes": 7650872, + "workload": "mixed", + "write_latency": { + "max_us": 793298, + "p50_us": 634, + "p95_us": 20784, + "p99_us": 95912 + }, + "write_wait": { + "max_us": 792386, + "p50_us": 35, + "p95_us": 20170, + "p99_us": 95366 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 68, + "p50_us": 2, + "p95_us": 3, + "p99_us": 4 + }, + "checkout_over_100us": 0, + "checkpoint": [ + 0, + 46, + 46 + ], + "checkpoint_us": 548, + "cpu_seconds": 3.2700000000000102, + "database_bytes": 14753792, + "failures": 0, + "latency": { + "max_us": 74887, + "p50_us": 821, + "p95_us": 3441, + "p99_us": 6710 + }, + "operations": 3200, + "operations_per_second": 5622.350662714731, + "pool_size": 12, + "pool_timeouts": 0, + "read_latency": { + "max_us": 12940, + "p50_us": 803, + "p95_us": 3284, + "p99_us": 3969 + }, + "rss_kib": 107440, + "synchronous": "FULL", + "transaction": { + "max_us": 11576, + "p50_us": 1502, + "p95_us": 3124, + "p99_us": 6360 + }, + "wal_bytes": 14065712, + "workload": "read_heavy", + "write_latency": { + "max_us": 74887, + "p50_us": 2312, + "p95_us": 25952, + "p99_us": 51970 + }, + "write_wait": { + "max_us": 73849, + "p50_us": 90, + "p95_us": 24522, + "p99_us": 49783 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 39, + "p50_us": 3, + "p95_us": 4, + "p99_us": 6 + }, + "checkout_over_100us": 0, + "checkpoint": [ + 0, + 251, + 251 + ], + "checkpoint_us": 953, + "cpu_seconds": 1.710000000000008, + "database_bytes": 15200256, + "failures": 0, + "latency": { + "max_us": 709856, + "p50_us": 483, + "p95_us": 3158, + "p99_us": 66890 + }, + "operations": 3200, + "operations_per_second": 1872.531239453297, + "pool_size": 12, + "pool_timeouts": 0, + "read_latency": { + "max_us": 9407, + "p50_us": 428, + "p95_us": 1715, + "p99_us": 1940 + }, + "rss_kib": 126016, + "synchronous": "FULL", + "transaction": { + "max_us": 9196, + "p50_us": 477, + "p95_us": 717, + "p99_us": 3197 + }, + "wal_bytes": 5191232, + "workload": "write_burst", + "write_latency": { + "max_us": 709856, + "p50_us": 486, + "p95_us": 5071, + "p99_us": 106574 + }, + "write_wait": { + "max_us": 709214, + "p50_us": 3, + "p95_us": 4558, + "p99_us": 105965 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 265, + "p50_us": 3, + "p95_us": 5, + "p99_us": 10 + }, + "checkout_over_100us": 1, + "checkpoint": [ + 0, + 950, + 950 + ], + "checkpoint_us": 2305, + "cpu_seconds": 2.1899999999999977, + "database_bytes": 14753792, + "failures": 0, + "latency": { + "max_us": 369313, + "p50_us": 654, + "p95_us": 5109, + "p99_us": 37743 + }, + "operations": 3200, + "operations_per_second": 2977.345654182638, + "pool_size": 12, + "pool_timeouts": 0, + "read_latency": { + "max_us": 21597, + "p50_us": 464, + "p95_us": 1920, + "p99_us": 3395 + }, + "rss_kib": 83600, + "synchronous": "FULL", + "transaction": { + "max_us": 11496, + "p50_us": 606, + "p95_us": 1799, + "p99_us": 3621 + }, + "wal_bytes": 10032232, + "workload": "mixed", + "write_latency": { + "max_us": 369313, + "p50_us": 1221, + "p95_us": 25719, + "p99_us": 103672 + }, + "write_wait": { + "max_us": 368612, + "p50_us": 70, + "p95_us": 25212, + "p99_us": 103009 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 61, + "p50_us": 2, + "p95_us": 3, + "p99_us": 4 + }, + "checkout_over_100us": 0, + "checkpoint": [ + 0, + 23, + 23 + ], + "checkpoint_us": 455, + "cpu_seconds": 3.530000000000001, + "database_bytes": 14753792, + "failures": 0, + "latency": { + "max_us": 117519, + "p50_us": 848, + "p95_us": 3684, + "p99_us": 6660 + }, + "operations": 3200, + "operations_per_second": 5406.892985971436, + "pool_size": 16, + "pool_timeouts": 0, + "read_latency": { + "max_us": 13140, + "p50_us": 827, + "p95_us": 3554, + "p99_us": 4742 + }, + "rss_kib": 99040, + "synchronous": "FULL", + "transaction": { + "max_us": 11477, + "p50_us": 1526, + "p95_us": 3124, + "p99_us": 6922 + }, + "wal_bytes": 7774472, + "workload": "read_heavy", + "write_latency": { + "max_us": 117519, + "p50_us": 2229, + "p95_us": 12974, + "p99_us": 49962 + }, + "write_wait": { + "max_us": 115170, + "p50_us": 18, + "p95_us": 12105, + "p99_us": 48442 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 65, + "p50_us": 3, + "p95_us": 4, + "p99_us": 7 + }, + "checkout_over_100us": 0, + "checkpoint": [ + 0, + 931, + 931 + ], + "checkpoint_us": 2657, + "cpu_seconds": 1.7299999999999898, + "database_bytes": 15208448, + "failures": 0, + "latency": { + "max_us": 1030277, + "p50_us": 490, + "p95_us": 3848, + "p99_us": 45559 + }, + "operations": 3200, + "operations_per_second": 1869.068763625066, + "pool_size": 16, + "pool_timeouts": 0, + "read_latency": { + "max_us": 16892, + "p50_us": 430, + "p95_us": 1729, + "p99_us": 2254 + }, + "rss_kib": 110080, + "synchronous": "FULL", + "transaction": { + "max_us": 11224, + "p50_us": 483, + "p95_us": 744, + "p99_us": 3316 + }, + "wal_bytes": 4886352, + "workload": "write_burst", + "write_latency": { + "max_us": 1030277, + "p50_us": 494, + "p95_us": 5134, + "p99_us": 75572 + }, + "write_wait": { + "max_us": 1029713, + "p50_us": 3, + "p95_us": 4561, + "p99_us": 74938 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 62, + "p50_us": 2, + "p95_us": 4, + "p99_us": 6 + }, + "checkout_over_100us": 0, + "checkpoint": [ + 0, + 631, + 631 + ], + "checkpoint_us": 2772, + "cpu_seconds": 2.0500000000000114, + "database_bytes": 14753792, + "failures": 0, + "latency": { + "max_us": 588826, + "p50_us": 592, + "p95_us": 4978, + "p99_us": 27463 + }, + "operations": 3200, + "operations_per_second": 3215.504358138856, + "pool_size": 16, + "pool_timeouts": 0, + "read_latency": { + "max_us": 11548, + "p50_us": 441, + "p95_us": 1814, + "p99_us": 2105 + }, + "rss_kib": 88416, + "synchronous": "FULL", + "transaction": { + "max_us": 9129, + "p50_us": 575, + "p95_us": 1242, + "p99_us": 3697 + }, + "wal_bytes": 11074592, + "workload": "mixed", + "write_latency": { + "max_us": 588826, + "p50_us": 749, + "p95_us": 25554, + "p99_us": 109289 + }, + "write_wait": { + "max_us": 588273, + "p50_us": 46, + "p95_us": 23781, + "p99_us": 107634 + } + } + ], + "query_plans": [ + { + "details": [ + "MATERIALIZE page", + "SEARCH t USING INDEX idx_threads_board_sticky_bumped (board_id=?)", + "SEARCH op EXISTS USING COVERING INDEX idx_posts_one_op_per_thread (thread_id=?)", + "USE TEMP B-TREE FOR LAST TERM OF ORDER BY", + "SCAN t", + "SEARCH op USING INDEX idx_posts_one_op_per_thread (thread_id=?)", + "CORRELATED SCALAR SUBQUERY 3", + "SEARCH fp USING INDEX idx_posts_thread_live (thread_id=?)", + "CORRELATED SCALAR SUBQUERY 4", + "SEARCH fp USING COVERING INDEX idx_posts_thread (thread_id=?)" + ], + "name": "board_index" + }, + { + "details": [ + "MATERIALIZE page", + "SEARCH t USING INDEX idx_threads_board_sticky_bumped (board_id=?)", + "SEARCH op EXISTS USING COVERING INDEX idx_posts_one_op_per_thread (thread_id=?)", + "USE TEMP B-TREE FOR LAST TERM OF ORDER BY", + "SCAN t", + "SEARCH op USING INDEX idx_posts_one_op_per_thread (thread_id=?)", + "CORRELATED SCALAR SUBQUERY 3", + "SEARCH fp USING INDEX idx_posts_thread_live (thread_id=?)", + "CORRELATED SCALAR SUBQUERY 4", + "SEARCH fp USING COVERING INDEX idx_posts_thread (thread_id=?)" + ], + "name": "catalog" + }, + { + "details": [ + "SEARCH posts USING INDEX idx_posts_thread (thread_id=?)" + ], + "name": "thread_loading" + }, + { + "details": [ + "SEARCH posts USING COVERING INDEX idx_posts_board (board_id=?)", + "USE TEMP B-TREE FOR LAST TERM OF ORDER BY" + ], + "name": "latest_post" + }, + { + "details": [ + "SEARCH threads USING INTEGER PRIMARY KEY (rowid=?)", + "SCALAR SUBQUERY 1", + "SEARCH boards USING INTEGER PRIMARY KEY (rowid=?)" + ], + "name": "thread_bump" + }, + { + "details": [ + "SCAN CONSTANT ROW", + "SCALAR SUBQUERY 1", + "SCAN posts USING COVERING INDEX idx_posts_board", + "SCALAR SUBQUERY 2", + "SCAN threads USING COVERING INDEX idx_threads_archived", + "SCALAR SUBQUERY 3", + "SEARCH reports USING COVERING INDEX idx_reports_status (status=?)" + ], + "name": "admin_overview" + }, + { + "details": [ + "SEARCH reports USING COVERING INDEX idx_reports_open_unique (post_id=? AND reporter_hash=?)" + ], + "name": "report_lookup" + }, + { + "details": [ + "SEARCH posts USING INDEX idx_posts_board (board_id=?)", + "USE TEMP B-TREE FOR ORDER BY" + ], + "name": "backup_board_inventory" + }, + { + "details": [ + "SEARCH posts USING INDEX idx_posts_thread_live (thread_id=? AND id>?)" + ], + "name": "live_poll" + }, + { + "details": [ + "SEARCH posts USING COVERING INDEX idx_posts_board_ip_created (board_id=? AND ip_hash=?)" + ], + "name": "cooldown" + }, + { + "details": [ + "SEARCH threads USING INDEX idx_threads_archived (board_id=? AND archived=?)", + "USE TEMP B-TREE FOR ORDER BY" + ], + "name": "latest_thread" + }, + { + "details": [ + "SEARCH threads USING INDEX idx_threads_archived (board_id=? AND archived=?)" + ], + "name": "prune" + }, + { + "details": [ + "SCAN posts_fts VIRTUAL TABLE INDEX 0:M1", + "SEARCH p USING INTEGER PRIMARY KEY (rowid=?)", + "USE TEMP B-TREE FOR ORDER BY" + ], + "name": "search_selective" + }, + { + "details": [ + "SEARCH reports USING INDEX idx_reports_status (status=?)" + ], + "name": "reports" + }, + { + "details": [ + "SEARCH bans USING INDEX idx_bans_ip (ip_hash=?)" + ], + "name": "bans" + }, + { + "details": [ + "SEARCH background_jobs USING INDEX idx_jobs_pending (status=?)" + ], + "name": "job_claim" + }, + { + "details": [ + "MULTI-INDEX OR", + "INDEX 1", + "SEARCH posts USING INDEX idx_posts_file_path (file_path=?)", + "INDEX 2", + "SEARCH posts USING INDEX idx_posts_thumb_path (thumb_path=?)", + "INDEX 3", + "SEARCH posts USING INDEX idx_posts_audio_file_path (audio_file_path=?)" + ], + "name": "file_refs" + }, + { + "details": [ + "SEARCH file_hashes USING INDEX sqlite_autoindex_file_hashes_1 (sha256=?)" + ], + "name": "dedup" + }, + { + "details": [ + "SCAN mod_log USING INDEX idx_mod_log_created" + ], + "name": "mod_log" + } + ], + "sqlite_version": "3.51.3", + "workers": 8 +} \ No newline at end of file diff --git a/docs/sqlite-evidence/baseline.json b/docs/sqlite-evidence/baseline.json new file mode 100644 index 00000000..adca7f12 --- /dev/null +++ b/docs/sqlite-evidence/baseline.json @@ -0,0 +1,1391 @@ +{ + "fixture": { + "boards": 8, + "posts": 40000, + "threads": 800 + }, + "limitations": "SQL/database workload; media CPU and HTTP excluded; RSS sampled at case end; ps optional; process-kill is not power loss", + "measurements": [ + { + "busy_errors": 0, + "checkout": { + "max_us": 464568, + "p50_us": 20250, + "p95_us": 179150, + "p99_us": 273655 + }, + "checkout_over_100us": 2353, + "checkpoint": [ + 0, + 356, + 356 + ], + "database_bytes": 13066240, + "failures": 0, + "latency": { + "max_us": 533293, + "p50_us": 27230, + "p95_us": 234122, + "p99_us": 327467 + }, + "operations": 3200, + "operations_per_second": 108.92363673420175, + "pool_size": 2, + "pool_timeouts": 0, + "rss_kib": 64608, + "synchronous": "NORMAL", + "transaction": { + "max_us": 9411, + "p50_us": 688, + "p95_us": 1318, + "p99_us": 2937 + }, + "wal_bytes": 11284712, + "workload": "read_heavy", + "write_wait": { + "max_us": 1548, + "p50_us": 7, + "p95_us": 52, + "p99_us": 127 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 358504, + "p50_us": 5235, + "p95_us": 108697, + "p99_us": 157718 + }, + "checkout_over_100us": 2152, + "checkpoint": [ + 0, + 101, + 101 + ], + "database_bytes": 13463552, + "failures": 0, + "latency": { + "max_us": 383902, + "p50_us": 7464, + "p95_us": 128914, + "p99_us": 190777 + }, + "operations": 3200, + "operations_per_second": 289.70630166500194, + "pool_size": 2, + "pool_timeouts": 0, + "rss_kib": 57216, + "synchronous": "NORMAL", + "transaction": { + "max_us": 8978, + "p50_us": 497, + "p95_us": 1047, + "p99_us": 2039 + }, + "wal_bytes": 21580592, + "workload": "write_burst", + "write_wait": { + "max_us": 12108, + "p50_us": 4, + "p95_us": 58, + "p99_us": 1648 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 490567, + "p50_us": 15643, + "p95_us": 170101, + "p99_us": 255079 + }, + "checkout_over_100us": 2302, + "checkpoint": [ + 0, + 93, + 93 + ], + "database_bytes": 13066240, + "failures": 0, + "latency": { + "max_us": 603467, + "p50_us": 21002, + "p95_us": 212974, + "p99_us": 320300 + }, + "operations": 3200, + "operations_per_second": 126.22044860704878, + "pool_size": 2, + "pool_timeouts": 0, + "rss_kib": 61888, + "synchronous": "NORMAL", + "transaction": { + "max_us": 9027, + "p50_us": 529, + "p95_us": 1262, + "p99_us": 2709 + }, + "wal_bytes": 26289752, + "workload": "mixed", + "write_wait": { + "max_us": 1587, + "p50_us": 4, + "p95_us": 67, + "p99_us": 240 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 305115, + "p50_us": 13008, + "p95_us": 114621, + "p99_us": 173946 + }, + "checkout_over_100us": 2312, + "checkpoint": [ + 0, + 3095, + 3095 + ], + "database_bytes": 13066240, + "failures": 0, + "latency": { + "max_us": 518826, + "p50_us": 20684, + "p95_us": 226231, + "p99_us": 345693 + }, + "operations": 3200, + "operations_per_second": 140.50231131127484, + "pool_size": 4, + "pool_timeouts": 0, + "rss_kib": 82528, + "synchronous": "NORMAL", + "transaction": { + "max_us": 11778, + "p50_us": 999, + "p95_us": 1707, + "p99_us": 5164 + }, + "wal_bytes": 12751432, + "workload": "read_heavy", + "write_wait": { + "max_us": 160, + "p50_us": 7, + "p95_us": 123, + "p99_us": 158 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 145753, + "p50_us": 3469, + "p95_us": 39908, + "p99_us": 97550 + }, + "checkout_over_100us": 2465, + "checkpoint": [ + 0, + 100, + 100 + ], + "database_bytes": 13463552, + "failures": 0, + "latency": { + "max_us": 401769, + "p50_us": 5806, + "p95_us": 103074, + "p99_us": 223400 + }, + "operations": 3200, + "operations_per_second": 401.0883281103068, + "pool_size": 4, + "pool_timeouts": 0, + "rss_kib": 66064, + "synchronous": "NORMAL", + "transaction": { + "max_us": 11700, + "p50_us": 725, + "p95_us": 1199, + "p99_us": 2145 + }, + "wal_bytes": 173175992, + "workload": "write_burst", + "write_wait": { + "max_us": 49625, + "p50_us": 4, + "p95_us": 1553, + "p99_us": 4653 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 268401, + "p50_us": 11289, + "p95_us": 101325, + "p99_us": 157297 + }, + "checkout_over_100us": 2413, + "checkpoint": [ + 0, + 14986, + 14986 + ], + "database_bytes": 13066240, + "failures": 0, + "latency": { + "max_us": 467960, + "p50_us": 15643, + "p95_us": 215325, + "p99_us": 300229 + }, + "operations": 3200, + "operations_per_second": 173.29291267119606, + "pool_size": 4, + "pool_timeouts": 0, + "rss_kib": 74944, + "synchronous": "NORMAL", + "transaction": { + "max_us": 10135, + "p50_us": 814, + "p95_us": 1715, + "p99_us": 3041 + }, + "wal_bytes": 61742352, + "workload": "mixed", + "write_wait": { + "max_us": 4649, + "p50_us": 5, + "p95_us": 125, + "p99_us": 1600 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 67, + "p50_us": 4, + "p95_us": 11, + "p99_us": 16 + }, + "checkout_over_100us": 0, + "checkpoint": [ + 0, + 3095, + 3095 + ], + "database_bytes": 13066240, + "failures": 0, + "latency": { + "max_us": 1052826, + "p50_us": 3915, + "p95_us": 694506, + "p99_us": 950879 + }, + "operations": 3200, + "operations_per_second": 77.5297546295621, + "pool_size": 8, + "pool_timeouts": 0, + "rss_kib": 105280, + "synchronous": "NORMAL", + "transaction": { + "max_us": 20400, + "p50_us": 2527, + "p95_us": 4900, + "p99_us": 6397 + }, + "wal_bytes": 12751432, + "workload": "read_heavy", + "write_wait": { + "max_us": 1545, + "p50_us": 9, + "p95_us": 189, + "p99_us": 260 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 68, + "p50_us": 4, + "p95_us": 5, + "p99_us": 10 + }, + "checkout_over_100us": 0, + "checkpoint": [ + 0, + 41, + 41 + ], + "database_bytes": 13463552, + "failures": 0, + "latency": { + "max_us": 1175750, + "p50_us": 2070, + "p95_us": 57696, + "p99_us": 807621 + }, + "operations": 3200, + "operations_per_second": 224.14675037701357, + "pool_size": 8, + "pool_timeouts": 0, + "rss_kib": 92640, + "synchronous": "NORMAL", + "transaction": { + "max_us": 20041, + "p50_us": 1985, + "p95_us": 3199, + "p99_us": 4253 + }, + "wal_bytes": 173876392, + "workload": "write_burst", + "write_wait": { + "max_us": 138331, + "p50_us": 3, + "p95_us": 4654, + "p99_us": 42027 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 90, + "p50_us": 4, + "p95_us": 14, + "p99_us": 20 + }, + "checkout_over_100us": 0, + "checkpoint": [ + 0, + 15041, + 15041 + ], + "database_bytes": 13066240, + "failures": 0, + "latency": { + "max_us": 1181026, + "p50_us": 2913, + "p95_us": 654620, + "p99_us": 934125 + }, + "operations": 3200, + "operations_per_second": 95.83439756109111, + "pool_size": 8, + "pool_timeouts": 0, + "rss_kib": 99584, + "synchronous": "NORMAL", + "transaction": { + "max_us": 18247, + "p50_us": 2372, + "p95_us": 3706, + "p99_us": 4907 + }, + "wal_bytes": 61968952, + "workload": "mixed", + "write_wait": { + "max_us": 27216, + "p50_us": 8, + "p95_us": 1567, + "p99_us": 4654 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 192, + "p50_us": 5, + "p95_us": 16, + "p99_us": 21 + }, + "checkout_over_100us": 1, + "checkpoint": [ + 0, + 3095, + 3095 + ], + "database_bytes": 13066240, + "failures": 0, + "latency": { + "max_us": 1229563, + "p50_us": 4023, + "p95_us": 720485, + "p99_us": 986203 + }, + "operations": 3200, + "operations_per_second": 73.46966631140599, + "pool_size": 12, + "pool_timeouts": 0, + "rss_kib": 76368, + "synchronous": "NORMAL", + "transaction": { + "max_us": 20272, + "p50_us": 2717, + "p95_us": 4930, + "p99_us": 10450 + }, + "wal_bytes": 12751432, + "workload": "read_heavy", + "write_wait": { + "max_us": 273, + "p50_us": 9, + "p95_us": 186, + "p99_us": 222 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 128, + "p50_us": 4, + "p95_us": 10, + "p99_us": 22 + }, + "checkout_over_100us": 1, + "checkpoint": [ + 0, + 98, + 98 + ], + "database_bytes": 13467648, + "failures": 0, + "latency": { + "max_us": 1364444, + "p50_us": 2219, + "p95_us": 69133, + "p99_us": 858600 + }, + "operations": 3200, + "operations_per_second": 207.28200018893833, + "pool_size": 12, + "pool_timeouts": 0, + "rss_kib": 63248, + "synchronous": "NORMAL", + "transaction": { + "max_us": 22997, + "p50_us": 2080, + "p95_us": 3572, + "p99_us": 4650 + }, + "wal_bytes": 173612712, + "workload": "write_burst", + "write_wait": { + "max_us": 149846, + "p50_us": 4, + "p95_us": 4696, + "p99_us": 47282 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 103, + "p50_us": 5, + "p95_us": 19, + "p99_us": 28 + }, + "checkout_over_100us": 1, + "checkpoint": [ + 0, + 14989, + 14989 + ], + "database_bytes": 13066240, + "failures": 0, + "latency": { + "max_us": 1437463, + "p50_us": 3401, + "p95_us": 724692, + "p99_us": 1057465 + }, + "operations": 3200, + "operations_per_second": 84.05966668169796, + "pool_size": 12, + "pool_timeouts": 0, + "rss_kib": 74480, + "synchronous": "NORMAL", + "transaction": { + "max_us": 23697, + "p50_us": 2793, + "p95_us": 4387, + "p99_us": 5826 + }, + "wal_bytes": 61754712, + "workload": "mixed", + "write_wait": { + "max_us": 12174, + "p50_us": 11, + "p95_us": 1669, + "p99_us": 4675 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 234, + "p50_us": 5, + "p95_us": 20, + "p99_us": 36 + }, + "checkout_over_100us": 3, + "checkpoint": [ + 0, + 3095, + 3095 + ], + "database_bytes": 13066240, + "failures": 0, + "latency": { + "max_us": 1377211, + "p50_us": 4090, + "p95_us": 801741, + "p99_us": 1081945 + }, + "operations": 3200, + "operations_per_second": 67.8800452190142, + "pool_size": 16, + "pool_timeouts": 0, + "rss_kib": 79872, + "synchronous": "NORMAL", + "transaction": { + "max_us": 21835, + "p50_us": 2930, + "p95_us": 5302, + "p99_us": 9048 + }, + "wal_bytes": 12751432, + "workload": "read_heavy", + "write_wait": { + "max_us": 4661, + "p50_us": 10, + "p95_us": 260, + "p99_us": 4613 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 54, + "p50_us": 4, + "p95_us": 6, + "p99_us": 14 + }, + "checkout_over_100us": 0, + "checkpoint": [ + 0, + 49, + 49 + ], + "database_bytes": 13467648, + "failures": 0, + "latency": { + "max_us": 1314361, + "p50_us": 2229, + "p95_us": 64952, + "p99_us": 914552 + }, + "operations": 3200, + "operations_per_second": 199.9639044281275, + "pool_size": 16, + "pool_timeouts": 0, + "rss_kib": 73744, + "synchronous": "NORMAL", + "transaction": { + "max_us": 19137, + "p50_us": 2118, + "p95_us": 3536, + "p99_us": 4667 + }, + "wal_bytes": 173843432, + "workload": "write_burst", + "write_wait": { + "max_us": 243128, + "p50_us": 4, + "p95_us": 4669, + "p99_us": 43114 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 52, + "p50_us": 4, + "p95_us": 14, + "p99_us": 22 + }, + "checkout_over_100us": 0, + "checkpoint": [ + 0, + 14978, + 14978 + ], + "database_bytes": 13066240, + "failures": 0, + "latency": { + "max_us": 1269234, + "p50_us": 3029, + "p95_us": 685749, + "p99_us": 986730 + }, + "operations": 3200, + "operations_per_second": 91.3617257500745, + "pool_size": 16, + "pool_timeouts": 0, + "rss_kib": 81824, + "synchronous": "NORMAL", + "transaction": { + "max_us": 19824, + "p50_us": 2442, + "p95_us": 3894, + "p99_us": 6056 + }, + "wal_bytes": 61709392, + "workload": "mixed", + "write_wait": { + "max_us": 12117, + "p50_us": 9, + "p95_us": 1629, + "p99_us": 4678 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 508625, + "p50_us": 25477, + "p95_us": 215950, + "p99_us": 295290 + }, + "checkout_over_100us": 2443, + "checkpoint": [ + 0, + 252, + 252 + ], + "database_bytes": 13066240, + "failures": 0, + "latency": { + "max_us": 542751, + "p50_us": 38687, + "p95_us": 265163, + "p99_us": 389872 + }, + "operations": 3200, + "operations_per_second": 92.46357277579011, + "pool_size": 2, + "pool_timeouts": 0, + "rss_kib": 40864, + "synchronous": "FULL", + "transaction": { + "max_us": 11813, + "p50_us": 968, + "p95_us": 2099, + "p99_us": 9404 + }, + "wal_bytes": 11713192, + "workload": "read_heavy", + "write_wait": { + "max_us": 1677, + "p50_us": 10, + "p95_us": 84, + "p99_us": 150 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 330775, + "p50_us": 7439, + "p95_us": 120112, + "p99_us": 182594 + }, + "checkout_over_100us": 2454, + "checkpoint": [ + 0, + 674, + 674 + ], + "database_bytes": 13463552, + "failures": 0, + "latency": { + "max_us": 394040, + "p50_us": 10798, + "p95_us": 145266, + "p99_us": 216467 + }, + "operations": 3200, + "operations_per_second": 252.29865094585196, + "pool_size": 2, + "pool_timeouts": 0, + "rss_kib": 39648, + "synchronous": "FULL", + "transaction": { + "max_us": 14715, + "p50_us": 666, + "p95_us": 1281, + "p99_us": 2668 + }, + "wal_bytes": 25193832, + "workload": "write_burst", + "write_wait": { + "max_us": 45203, + "p50_us": 5, + "p95_us": 117, + "p99_us": 4530 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 384774, + "p50_us": 18025, + "p95_us": 178709, + "p99_us": 255125 + }, + "checkout_over_100us": 2461, + "checkpoint": [ + 0, + 13298, + 13298 + ], + "database_bytes": 13066240, + "failures": 0, + "latency": { + "max_us": 537567, + "p50_us": 24322, + "p95_us": 227560, + "p99_us": 313408 + }, + "operations": 3200, + "operations_per_second": 118.5801919314107, + "pool_size": 2, + "pool_timeouts": 0, + "rss_kib": 42368, + "synchronous": "FULL", + "transaction": { + "max_us": 9234, + "p50_us": 747, + "p95_us": 1292, + "p99_us": 3361 + }, + "wal_bytes": 54787792, + "workload": "mixed", + "write_wait": { + "max_us": 4590, + "p50_us": 6, + "p95_us": 97, + "p99_us": 1557 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 309896, + "p50_us": 14882, + "p95_us": 119106, + "p99_us": 188842 + }, + "checkout_over_100us": 2347, + "checkpoint": [ + 0, + 3095, + 3095 + ], + "database_bytes": 13066240, + "failures": 0, + "latency": { + "max_us": 604633, + "p50_us": 22669, + "p95_us": 244321, + "p99_us": 365405 + }, + "operations": 3200, + "operations_per_second": 130.8936484352161, + "pool_size": 4, + "pool_timeouts": 0, + "rss_kib": 56096, + "synchronous": "FULL", + "transaction": { + "max_us": 10860, + "p50_us": 1147, + "p95_us": 1999, + "p99_us": 5769 + }, + "wal_bytes": 12751432, + "workload": "read_heavy", + "write_wait": { + "max_us": 1627, + "p50_us": 6, + "p95_us": 134, + "p99_us": 204 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 303586, + "p50_us": 4204, + "p95_us": 45842, + "p99_us": 147391 + }, + "checkout_over_100us": 2579, + "checkpoint": [ + 0, + 238, + 238 + ], + "database_bytes": 13467648, + "failures": 0, + "latency": { + "max_us": 466526, + "p50_us": 6582, + "p95_us": 148317, + "p99_us": 297394 + }, + "operations": 3200, + "operations_per_second": 321.4653192916612, + "pool_size": 4, + "pool_timeouts": 0, + "rss_kib": 53888, + "synchronous": "FULL", + "transaction": { + "max_us": 49154, + "p50_us": 843, + "p95_us": 1449, + "p99_us": 2008 + }, + "wal_bytes": 172261352, + "workload": "write_burst", + "write_wait": { + "max_us": 95095, + "p50_us": 5, + "p95_us": 152, + "p99_us": 12049 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 309371, + "p50_us": 13341, + "p95_us": 122745, + "p99_us": 198738 + }, + "checkout_over_100us": 2487, + "checkpoint": [ + 0, + 14986, + 14986 + ], + "database_bytes": 13066240, + "failures": 0, + "latency": { + "max_us": 569003, + "p50_us": 19844, + "p95_us": 249923, + "p99_us": 357322 + }, + "operations": 3200, + "operations_per_second": 143.6189365957794, + "pool_size": 4, + "pool_timeouts": 0, + "rss_kib": 51136, + "synchronous": "FULL", + "transaction": { + "max_us": 11933, + "p50_us": 1032, + "p95_us": 1783, + "p99_us": 2394 + }, + "wal_bytes": 61742352, + "workload": "mixed", + "write_wait": { + "max_us": 12091, + "p50_us": 5, + "p95_us": 133, + "p99_us": 1600 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 156, + "p50_us": 5, + "p95_us": 17, + "p99_us": 23 + }, + "checkout_over_100us": 1, + "checkpoint": [ + 0, + 3095, + 3095 + ], + "database_bytes": 13066240, + "failures": 0, + "latency": { + "max_us": 1245326, + "p50_us": 4009, + "p95_us": 707043, + "p99_us": 994516 + }, + "operations": 3200, + "operations_per_second": 74.23347018831551, + "pool_size": 8, + "pool_timeouts": 0, + "rss_kib": 78704, + "synchronous": "FULL", + "transaction": { + "max_us": 21840, + "p50_us": 2815, + "p95_us": 5353, + "p99_us": 9732 + }, + "wal_bytes": 12751432, + "workload": "read_heavy", + "write_wait": { + "max_us": 4687, + "p50_us": 9, + "p95_us": 262, + "p99_us": 4638 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 68, + "p50_us": 4, + "p95_us": 10, + "p99_us": 19 + }, + "checkout_over_100us": 0, + "checkpoint": [ + 0, + 23, + 23 + ], + "database_bytes": 13463552, + "failures": 0, + "latency": { + "max_us": 1260073, + "p50_us": 2214, + "p95_us": 65979, + "p99_us": 865687 + }, + "operations": 3200, + "operations_per_second": 214.6013920999969, + "pool_size": 8, + "pool_timeouts": 0, + "rss_kib": 64880, + "synchronous": "FULL", + "transaction": { + "max_us": 19575, + "p50_us": 2085, + "p95_us": 3536, + "p99_us": 4378 + }, + "wal_bytes": 173662152, + "workload": "write_burst", + "write_wait": { + "max_us": 148270, + "p50_us": 4, + "p95_us": 4698, + "p99_us": 49610 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 604, + "p50_us": 6, + "p95_us": 23, + "p99_us": 41 + }, + "checkout_over_100us": 4, + "checkpoint": [ + 0, + 14998, + 14998 + ], + "database_bytes": 13066240, + "failures": 0, + "latency": { + "max_us": 1544055, + "p50_us": 3591, + "p95_us": 748626, + "p99_us": 1089856 + }, + "operations": 3200, + "operations_per_second": 83.70383506253467, + "pool_size": 8, + "pool_timeouts": 0, + "rss_kib": 76704, + "synchronous": "FULL", + "transaction": { + "max_us": 72702, + "p50_us": 2837, + "p95_us": 5089, + "p99_us": 12883 + }, + "wal_bytes": 61791792, + "workload": "mixed", + "write_wait": { + "max_us": 63117, + "p50_us": 11, + "p95_us": 1673, + "p99_us": 10499 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 101, + "p50_us": 6, + "p95_us": 21, + "p99_us": 36 + }, + "checkout_over_100us": 1, + "checkpoint": [ + 0, + 3095, + 3095 + ], + "database_bytes": 13066240, + "failures": 0, + "latency": { + "max_us": 1488034, + "p50_us": 4188, + "p95_us": 730465, + "p99_us": 1043183 + }, + "operations": 3200, + "operations_per_second": 71.77445111948836, + "pool_size": 12, + "pool_timeouts": 0, + "rss_kib": 78352, + "synchronous": "FULL", + "transaction": { + "max_us": 20697, + "p50_us": 2951, + "p95_us": 5176, + "p99_us": 10622 + }, + "wal_bytes": 12751432, + "workload": "read_heavy", + "write_wait": { + "max_us": 1623, + "p50_us": 10, + "p95_us": 232, + "p99_us": 273 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 208, + "p50_us": 4, + "p95_us": 12, + "p99_us": 22 + }, + "checkout_over_100us": 1, + "checkpoint": [ + 0, + 267, + 267 + ], + "database_bytes": 13467648, + "failures": 0, + "latency": { + "max_us": 1187878, + "p50_us": 2296, + "p95_us": 66211, + "p99_us": 888892 + }, + "operations": 3200, + "operations_per_second": 204.55967564742696, + "pool_size": 12, + "pool_timeouts": 0, + "rss_kib": 73776, + "synchronous": "FULL", + "transaction": { + "max_us": 20024, + "p50_us": 2171, + "p95_us": 3531, + "p99_us": 4490 + }, + "wal_bytes": 172702192, + "workload": "write_burst", + "write_wait": { + "max_us": 227910, + "p50_us": 4, + "p95_us": 4683, + "p99_us": 27337 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 106, + "p50_us": 5, + "p95_us": 19, + "p99_us": 34 + }, + "checkout_over_100us": 1, + "checkpoint": [ + 0, + 14984, + 14984 + ], + "database_bytes": 13066240, + "failures": 0, + "latency": { + "max_us": 1495203, + "p50_us": 3080, + "p95_us": 693122, + "p99_us": 1046100 + }, + "operations": 3200, + "operations_per_second": 88.61247979605173, + "pool_size": 12, + "pool_timeouts": 0, + "rss_kib": 79616, + "synchronous": "FULL", + "transaction": { + "max_us": 22725, + "p50_us": 2577, + "p95_us": 4254, + "p99_us": 11714 + }, + "wal_bytes": 61734112, + "workload": "mixed", + "write_wait": { + "max_us": 27418, + "p50_us": 9, + "p95_us": 1572, + "p99_us": 4718 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 64, + "p50_us": 5, + "p95_us": 20, + "p99_us": 22 + }, + "checkout_over_100us": 0, + "checkpoint": [ + 0, + 3095, + 3095 + ], + "database_bytes": 13066240, + "failures": 0, + "latency": { + "max_us": 1318983, + "p50_us": 4197, + "p95_us": 725376, + "p99_us": 1007996 + }, + "operations": 3200, + "operations_per_second": 72.34264708100383, + "pool_size": 16, + "pool_timeouts": 0, + "rss_kib": 76384, + "synchronous": "FULL", + "transaction": { + "max_us": 19297, + "p50_us": 2853, + "p95_us": 4925, + "p99_us": 9767 + }, + "wal_bytes": 12751432, + "workload": "read_heavy", + "write_wait": { + "max_us": 4605, + "p50_us": 11, + "p95_us": 202, + "p99_us": 1579 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 62, + "p50_us": 3, + "p95_us": 5, + "p99_us": 11 + }, + "checkout_over_100us": 0, + "checkpoint": [ + 0, + 331, + 331 + ], + "database_bytes": 13467648, + "failures": 0, + "latency": { + "max_us": 1137207, + "p50_us": 1968, + "p95_us": 55281, + "p99_us": 788142 + }, + "operations": 3200, + "operations_per_second": 226.71084103788579, + "pool_size": 16, + "pool_timeouts": 0, + "rss_kib": 56000, + "synchronous": "FULL", + "transaction": { + "max_us": 14651, + "p50_us": 1888, + "p95_us": 2971, + "p99_us": 3751 + }, + "wal_bytes": 172780472, + "workload": "write_burst", + "write_wait": { + "max_us": 140706, + "p50_us": 3, + "p95_us": 4664, + "p99_us": 46449 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 74, + "p50_us": 4, + "p95_us": 14, + "p99_us": 21 + }, + "checkout_over_100us": 0, + "checkpoint": [ + 0, + 92, + 92 + ], + "database_bytes": 13066240, + "failures": 0, + "latency": { + "max_us": 1242980, + "p50_us": 3015, + "p95_us": 658369, + "p99_us": 939466 + }, + "operations": 3200, + "operations_per_second": 93.16475825391993, + "pool_size": 16, + "pool_timeouts": 0, + "rss_kib": 79072, + "synchronous": "FULL", + "transaction": { + "max_us": 21313, + "p50_us": 2489, + "p95_us": 3793, + "p99_us": 5807 + }, + "wal_bytes": 61433352, + "workload": "mixed", + "write_wait": { + "max_us": 27375, + "p50_us": 10, + "p95_us": 1605, + "p99_us": 4646 + } + } + ], + "query_plans": [ + { + "details": [ + "SEARCH t USING INDEX idx_threads_archived (board_id=? AND archived=?)", + "SEARCH op USING INDEX idx_posts_one_op_per_thread (thread_id=?)", + "SEARCH fp USING INDEX idx_posts_thread (thread_id=?) LEFT-JOIN", + "USE TEMP B-TREE FOR GROUP BY", + "USE TEMP B-TREE FOR count(DISTINCT)", + "USE TEMP B-TREE FOR ORDER BY" + ], + "name": "board_index" + }, + { + "details": [ + "SEARCH posts USING INDEX idx_posts_board (board_id=?)", + "USE TEMP B-TREE FOR ORDER BY" + ], + "name": "live_poll" + }, + { + "details": [ + "SEARCH posts USING INDEX idx_posts_board (board_id=?)" + ], + "name": "cooldown" + }, + { + "details": [ + "SEARCH threads USING INDEX idx_threads_archived (board_id=? AND archived=?)", + "USE TEMP B-TREE FOR ORDER BY" + ], + "name": "latest_thread" + }, + { + "details": [ + "SEARCH threads USING INDEX idx_threads_archived (board_id=? AND archived=?)" + ], + "name": "prune" + }, + { + "details": [ + "SEARCH p USING COVERING INDEX idx_posts_board (board_id=?)", + "SCAN f VIRTUAL TABLE INDEX 0:=M1", + "USE TEMP B-TREE FOR LAST TERM OF ORDER BY" + ], + "name": "search" + }, + { + "details": [ + "SEARCH reports USING INDEX idx_reports_status (status=?)" + ], + "name": "reports" + }, + { + "details": [ + "SEARCH bans USING INDEX idx_bans_ip (ip_hash=?)" + ], + "name": "bans" + }, + { + "details": [ + "SEARCH background_jobs USING INDEX idx_jobs_pending (status=?)" + ], + "name": "job_claim" + }, + { + "details": [ + "MULTI-INDEX OR", + "INDEX 1", + "SEARCH posts USING INDEX idx_posts_file_path (file_path=?)", + "INDEX 2", + "SEARCH posts USING INDEX idx_posts_thumb_path (thumb_path=?)", + "INDEX 3", + "SEARCH posts USING INDEX idx_posts_audio_file_path (audio_file_path=?)" + ], + "name": "file_refs" + }, + { + "details": [ + "SEARCH file_hashes USING INDEX sqlite_autoindex_file_hashes_1 (sha256=?)" + ], + "name": "dedup" + }, + { + "details": [ + "SCAN mod_log USING INDEX idx_mod_log_created" + ], + "name": "mod_log" + } + ], + "sqlite_version": "3.51.3", + "workers": 8 +} \ No newline at end of file diff --git a/docs/sqlite-evidence/final-matrix.json b/docs/sqlite-evidence/final-matrix.json new file mode 100644 index 00000000..5527844e --- /dev/null +++ b/docs/sqlite-evidence/final-matrix.json @@ -0,0 +1,1875 @@ +{ + "fixture": { + "boards": 8, + "posts": 40000, + "threads": 800 + }, + "limitations": "SQL/database workload; media CPU and HTTP excluded; RSS sampled at case end; ps optional; process-kill is not power loss", + "measurements": [ + { + "busy_errors": 0, + "checkout": { + "max_us": 23849, + "p50_us": 3, + "p95_us": 8089, + "p99_us": 14975 + }, + "checkout_over_100us": 1127, + "checkpoint": [ + 0, + 23, + 23 + ], + "checkpoint_us": 488, + "cpu_seconds": 1.73, + "database_bytes": 14753792, + "failures": 0, + "latency": { + "max_us": 24506, + "p50_us": 710, + "p95_us": 8933, + "p99_us": 15398 + }, + "operations": 3200, + "operations_per_second": 3545.295698279347, + "pool_size": 2, + "pool_timeouts": 0, + "read_latency": { + "max_us": 24506, + "p50_us": 715, + "p95_us": 8933, + "p99_us": 15398 + }, + "rss_kib": 68096, + "synchronous": "NORMAL", + "transaction": { + "max_us": 9003, + "p50_us": 493, + "p95_us": 2105, + "p99_us": 8558 + }, + "wal_bytes": 5285992, + "workload": "read_heavy", + "write_latency": { + "max_us": 17964, + "p50_us": 588, + "p95_us": 8568, + "p99_us": 17312 + }, + "write_wait": { + "max_us": 4635, + "p50_us": 3, + "p95_us": 113, + "p99_us": 1605 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 29813, + "p50_us": 2247, + "p95_us": 10675, + "p99_us": 16328 + }, + "checkout_over_100us": 1834, + "checkpoint": [ + 0, + 726, + 726 + ], + "checkpoint_us": 3057, + "cpu_seconds": 1.7299999999999995, + "database_bytes": 15204352, + "failures": 0, + "latency": { + "max_us": 397696, + "p50_us": 2804, + "p95_us": 11863, + "p99_us": 20110 + }, + "operations": 3200, + "operations_per_second": 1861.4113333461153, + "pool_size": 2, + "pool_timeouts": 0, + "read_latency": { + "max_us": 31365, + "p50_us": 2811, + "p95_us": 11949, + "p99_us": 17895 + }, + "rss_kib": 61248, + "synchronous": "NORMAL", + "transaction": { + "max_us": 8439, + "p50_us": 446, + "p95_us": 684, + "p99_us": 4483 + }, + "wal_bytes": 5426072, + "workload": "write_burst", + "write_latency": { + "max_us": 397696, + "p50_us": 2801, + "p95_us": 11794, + "p99_us": 21820 + }, + "write_wait": { + "max_us": 397097, + "p50_us": 4, + "p95_us": 30, + "p99_us": 12075 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 22625, + "p50_us": 3, + "p95_us": 9223, + "p99_us": 15908 + }, + "checkout_over_100us": 1380, + "checkpoint": [ + 0, + 166, + 166 + ], + "checkpoint_us": 1143, + "cpu_seconds": 1.8300000000000018, + "database_bytes": 14753792, + "failures": 0, + "latency": { + "max_us": 74561, + "p50_us": 1614, + "p95_us": 10289, + "p99_us": 16794 + }, + "operations": 3200, + "operations_per_second": 2757.439269554338, + "pool_size": 2, + "pool_timeouts": 0, + "read_latency": { + "max_us": 24271, + "p50_us": 1483, + "p95_us": 10086, + "p99_us": 16578 + }, + "rss_kib": 61360, + "synchronous": "NORMAL", + "transaction": { + "max_us": 8219, + "p50_us": 454, + "p95_us": 1355, + "p99_us": 3641 + }, + "wal_bytes": 6105872, + "workload": "mixed", + "write_latency": { + "max_us": 74561, + "p50_us": 1985, + "p95_us": 11060, + "p99_us": 19046 + }, + "write_wait": { + "max_us": 73915, + "p50_us": 3, + "p95_us": 1569, + "p99_us": 4585 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 22770, + "p50_us": 2, + "p95_us": 3953, + "p99_us": 7352 + }, + "checkout_over_100us": 801, + "checkpoint": [ + 0, + 28, + 28 + ], + "checkpoint_us": 480, + "cpu_seconds": 2.020000000000003, + "database_bytes": 14753792, + "failures": 0, + "latency": { + "max_us": 24749, + "p50_us": 740, + "p95_us": 4987, + "p99_us": 8805 + }, + "operations": 3200, + "operations_per_second": 5566.67682875119, + "pool_size": 4, + "pool_timeouts": 0, + "read_latency": { + "max_us": 21390, + "p50_us": 728, + "p95_us": 4719, + "p99_us": 8623 + }, + "rss_kib": 77856, + "synchronous": "NORMAL", + "transaction": { + "max_us": 13756, + "p50_us": 757, + "p95_us": 1207, + "p99_us": 8760 + }, + "wal_bytes": 14139872, + "workload": "read_heavy", + "write_latency": { + "max_us": 24749, + "p50_us": 936, + "p95_us": 6460, + "p99_us": 24520 + }, + "write_wait": { + "max_us": 23910, + "p50_us": 3, + "p95_us": 1610, + "p99_us": 12093 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 16041, + "p50_us": 483, + "p95_us": 6751, + "p99_us": 10607 + }, + "checkout_over_100us": 1609, + "checkpoint": [ + 0, + 518, + 518 + ], + "checkpoint_us": 1912, + "cpu_seconds": 1.759999999999998, + "database_bytes": 15200256, + "failures": 0, + "latency": { + "max_us": 701059, + "p50_us": 1744, + "p95_us": 8915, + "p99_us": 26124 + }, + "operations": 3200, + "operations_per_second": 1900.1210288027173, + "pool_size": 4, + "pool_timeouts": 0, + "read_latency": { + "max_us": 17763, + "p50_us": 1684, + "p95_us": 7322, + "p99_us": 10997 + }, + "rss_kib": 88512, + "synchronous": "NORMAL", + "transaction": { + "max_us": 9543, + "p50_us": 451, + "p95_us": 762, + "p99_us": 4284 + }, + "wal_bytes": 5166512, + "workload": "write_burst", + "write_latency": { + "max_us": 701059, + "p50_us": 1782, + "p95_us": 9683, + "p99_us": 49594 + }, + "write_wait": { + "max_us": 700476, + "p50_us": 3, + "p95_us": 1595, + "p99_us": 47076 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 17908, + "p50_us": 3, + "p95_us": 5550, + "p99_us": 9382 + }, + "checkout_over_100us": 1187, + "checkpoint": [ + 0, + 168, + 168 + ], + "checkpoint_us": 1090, + "cpu_seconds": 1.980000000000004, + "database_bytes": 14753792, + "failures": 0, + "latency": { + "max_us": 288672, + "p50_us": 936, + "p95_us": 7323, + "p99_us": 13750 + }, + "operations": 3200, + "operations_per_second": 3214.506934787404, + "pool_size": 4, + "pool_timeouts": 0, + "read_latency": { + "max_us": 19506, + "p50_us": 871, + "p95_us": 6484, + "p99_us": 10402 + }, + "rss_kib": 66528, + "synchronous": "NORMAL", + "transaction": { + "max_us": 8444, + "p50_us": 510, + "p95_us": 1701, + "p99_us": 3942 + }, + "wal_bytes": 10506032, + "workload": "mixed", + "write_latency": { + "max_us": 288672, + "p50_us": 1825, + "p95_us": 12554, + "p99_us": 39675 + }, + "write_wait": { + "max_us": 288069, + "p50_us": 4, + "p95_us": 11325, + "p99_us": 39170 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 57, + "p50_us": 2, + "p95_us": 3, + "p99_us": 4 + }, + "checkout_over_100us": 0, + "checkpoint": [ + 0, + 3460, + 3460 + ], + "checkpoint_us": 10, + "cpu_seconds": 3.4499999999999957, + "database_bytes": 14753792, + "failures": 0, + "latency": { + "max_us": 71663, + "p50_us": 830, + "p95_us": 3695, + "p99_us": 6625 + }, + "operations": 3200, + "operations_per_second": 5361.656659955514, + "pool_size": 8, + "pool_timeouts": 0, + "read_latency": { + "max_us": 13099, + "p50_us": 804, + "p95_us": 3502, + "p99_us": 4826 + }, + "rss_kib": 78752, + "synchronous": "NORMAL", + "transaction": { + "max_us": 13503, + "p50_us": 1682, + "p95_us": 2988, + "p99_us": 10885 + }, + "wal_bytes": 14255232, + "workload": "read_heavy", + "write_latency": { + "max_us": 71663, + "p50_us": 2504, + "p95_us": 12581, + "p99_us": 51802 + }, + "write_wait": { + "max_us": 70904, + "p50_us": 54, + "p95_us": 11657, + "p99_us": 49694 + } + }, + { + "busy_errors": 1, + "checkout": { + "max_us": 58, + "p50_us": 2, + "p95_us": 3, + "p99_us": 6 + }, + "checkout_over_100us": 0, + "checkpoint": [ + 0, + 325, + 325 + ], + "checkpoint_us": 1830, + "cpu_seconds": 1.7700000000000031, + "database_bytes": 15204352, + "failures": 1, + "latency": { + "max_us": 1080606, + "p50_us": 458, + "p95_us": 5017, + "p99_us": 72699 + }, + "operations": 3200, + "operations_per_second": 1925.477142821244, + "pool_size": 8, + "pool_timeouts": 0, + "read_latency": { + "max_us": 6859, + "p50_us": 431, + "p95_us": 1759, + "p99_us": 2040 + }, + "rss_kib": 102960, + "synchronous": "NORMAL", + "transaction": { + "max_us": 9169, + "p50_us": 451, + "p95_us": 976, + "p99_us": 3970 + }, + "wal_bytes": 5928712, + "workload": "write_burst", + "write_latency": { + "max_us": 1080606, + "p50_us": 461, + "p95_us": 7942, + "p99_us": 105466 + }, + "write_wait": { + "max_us": 1080400, + "p50_us": 3, + "p95_us": 4609, + "p99_us": 104964 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 31, + "p50_us": 2, + "p95_us": 3, + "p99_us": 4 + }, + "checkout_over_100us": 0, + "checkpoint": [ + 0, + 330, + 330 + ], + "checkpoint_us": 2882, + "cpu_seconds": 2.039999999999999, + "database_bytes": 14753792, + "failures": 0, + "latency": { + "max_us": 386547, + "p50_us": 559, + "p95_us": 4659, + "p99_us": 27480 + }, + "operations": 3200, + "operations_per_second": 3463.3115342656574, + "pool_size": 8, + "pool_timeouts": 0, + "read_latency": { + "max_us": 13194, + "p50_us": 430, + "p95_us": 1795, + "p99_us": 2007 + }, + "rss_kib": 86224, + "synchronous": "NORMAL", + "transaction": { + "max_us": 9862, + "p50_us": 545, + "p95_us": 1528, + "p99_us": 3629 + }, + "wal_bytes": 11408312, + "workload": "mixed", + "write_latency": { + "max_us": 386547, + "p50_us": 765, + "p95_us": 25215, + "p99_us": 108468 + }, + "write_wait": { + "max_us": 386003, + "p50_us": 44, + "p95_us": 24639, + "p99_us": 107800 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 33, + "p50_us": 2, + "p95_us": 3, + "p99_us": 4 + }, + "checkout_over_100us": 0, + "checkpoint": [ + 0, + 26, + 26 + ], + "checkpoint_us": 587, + "cpu_seconds": 3.4399999999999977, + "database_bytes": 14753792, + "failures": 0, + "latency": { + "max_us": 81566, + "p50_us": 829, + "p95_us": 3801, + "p99_us": 7255 + }, + "operations": 3200, + "operations_per_second": 5208.924322422961, + "pool_size": 12, + "pool_timeouts": 0, + "read_latency": { + "max_us": 13570, + "p50_us": 806, + "p95_us": 3506, + "p99_us": 4667 + }, + "rss_kib": 107728, + "synchronous": "NORMAL", + "transaction": { + "max_us": 10012, + "p50_us": 1839, + "p95_us": 4424, + "p99_us": 7194 + }, + "wal_bytes": 14148112, + "workload": "read_heavy", + "write_latency": { + "max_us": 81566, + "p50_us": 2777, + "p95_us": 16184, + "p99_us": 51479 + }, + "write_wait": { + "max_us": 80043, + "p50_us": 104, + "p95_us": 12198, + "p99_us": 48423 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 22, + "p50_us": 2, + "p95_us": 3, + "p99_us": 6 + }, + "checkout_over_100us": 0, + "checkpoint": [ + 0, + 570, + 570 + ], + "checkpoint_us": 2417, + "cpu_seconds": 1.7600000000000051, + "database_bytes": 15200256, + "failures": 0, + "latency": { + "max_us": 610550, + "p50_us": 457, + "p95_us": 4547, + "p99_us": 49485 + }, + "operations": 3200, + "operations_per_second": 1902.9916784695365, + "pool_size": 12, + "pool_timeouts": 0, + "read_latency": { + "max_us": 7921, + "p50_us": 435, + "p95_us": 1779, + "p99_us": 2276 + }, + "rss_kib": 127328, + "synchronous": "NORMAL", + "transaction": { + "max_us": 9147, + "p50_us": 450, + "p95_us": 898, + "p99_us": 4307 + }, + "wal_bytes": 5846312, + "workload": "write_burst", + "write_latency": { + "max_us": 610550, + "p50_us": 459, + "p95_us": 5257, + "p99_us": 76745 + }, + "write_wait": { + "max_us": 609884, + "p50_us": 3, + "p95_us": 4570, + "p99_us": 76135 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 29, + "p50_us": 2, + "p95_us": 3, + "p99_us": 4 + }, + "checkout_over_100us": 0, + "checkpoint": [ + 0, + 109, + 109 + ], + "checkpoint_us": 881, + "cpu_seconds": 2.030000000000001, + "database_bytes": 14753792, + "failures": 0, + "latency": { + "max_us": 291727, + "p50_us": 562, + "p95_us": 4887, + "p99_us": 27227 + }, + "operations": 3200, + "operations_per_second": 3344.4141752781616, + "pool_size": 12, + "pool_timeouts": 0, + "read_latency": { + "max_us": 11975, + "p50_us": 428, + "p95_us": 1807, + "p99_us": 2050 + }, + "rss_kib": 73968, + "synchronous": "NORMAL", + "transaction": { + "max_us": 10487, + "p50_us": 551, + "p95_us": 1518, + "p99_us": 3783 + }, + "wal_bytes": 12640192, + "workload": "mixed", + "write_latency": { + "max_us": 291727, + "p50_us": 793, + "p95_us": 23091, + "p99_us": 101922 + }, + "write_wait": { + "max_us": 291145, + "p50_us": 46, + "p95_us": 22423, + "p99_us": 101489 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 33, + "p50_us": 2, + "p95_us": 3, + "p99_us": 4 + }, + "checkout_over_100us": 0, + "checkpoint": [ + 0, + 3458, + 3458 + ], + "checkpoint_us": 11, + "cpu_seconds": 3.5600000000000023, + "database_bytes": 14753792, + "failures": 0, + "latency": { + "max_us": 61147, + "p50_us": 861, + "p95_us": 3802, + "p99_us": 6652 + }, + "operations": 3200, + "operations_per_second": 5304.8888112960485, + "pool_size": 16, + "pool_timeouts": 0, + "read_latency": { + "max_us": 14500, + "p50_us": 830, + "p95_us": 3603, + "p99_us": 4744 + }, + "rss_kib": 96272, + "synchronous": "NORMAL", + "transaction": { + "max_us": 12877, + "p50_us": 1702, + "p95_us": 3406, + "p99_us": 11156 + }, + "wal_bytes": 14246992, + "workload": "read_heavy", + "write_latency": { + "max_us": 61147, + "p50_us": 2612, + "p95_us": 14092, + "p99_us": 46223 + }, + "write_wait": { + "max_us": 48266, + "p50_us": 107, + "p95_us": 12214, + "p99_us": 44066 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 42, + "p50_us": 2, + "p95_us": 3, + "p99_us": 5 + }, + "checkout_over_100us": 0, + "checkpoint": [ + 0, + 150, + 150 + ], + "checkpoint_us": 965, + "cpu_seconds": 1.7199999999999989, + "database_bytes": 15200256, + "failures": 0, + "latency": { + "max_us": 1112467, + "p50_us": 456, + "p95_us": 4957, + "p99_us": 49303 + }, + "operations": 3200, + "operations_per_second": 1919.5566304072916, + "pool_size": 16, + "pool_timeouts": 0, + "read_latency": { + "max_us": 6593, + "p50_us": 434, + "p95_us": 1739, + "p99_us": 2184 + }, + "rss_kib": 110064, + "synchronous": "NORMAL", + "transaction": { + "max_us": 8473, + "p50_us": 448, + "p95_us": 835, + "p99_us": 4021 + }, + "wal_bytes": 5578512, + "workload": "write_burst", + "write_latency": { + "max_us": 1112467, + "p50_us": 459, + "p95_us": 5195, + "p99_us": 90894 + }, + "write_wait": { + "max_us": 1111916, + "p50_us": 3, + "p95_us": 4571, + "p99_us": 90383 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 29, + "p50_us": 2, + "p95_us": 3, + "p99_us": 4 + }, + "checkout_over_100us": 0, + "checkpoint": [ + 0, + 414, + 414 + ], + "checkpoint_us": 2139, + "cpu_seconds": 2.030000000000001, + "database_bytes": 14753792, + "failures": 0, + "latency": { + "max_us": 283508, + "p50_us": 618, + "p95_us": 4979, + "p99_us": 27026 + }, + "operations": 3200, + "operations_per_second": 3486.4579908467167, + "pool_size": 16, + "pool_timeouts": 0, + "read_latency": { + "max_us": 12515, + "p50_us": 427, + "p95_us": 1796, + "p99_us": 2033 + }, + "rss_kib": 85104, + "synchronous": "NORMAL", + "transaction": { + "max_us": 8696, + "p50_us": 570, + "p95_us": 1410, + "p99_us": 3296 + }, + "wal_bytes": 11853272, + "workload": "mixed", + "write_latency": { + "max_us": 283508, + "p50_us": 849, + "p95_us": 25972, + "p99_us": 129948 + }, + "write_wait": { + "max_us": 283079, + "p50_us": 54, + "p95_us": 25075, + "p99_us": 129436 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 35149, + "p50_us": 3, + "p95_us": 8272, + "p99_us": 14254 + }, + "checkout_over_100us": 1166, + "checkpoint": [ + 0, + 722, + 722 + ], + "checkpoint_us": 2737, + "cpu_seconds": 1.789999999999992, + "database_bytes": 14753792, + "failures": 0, + "latency": { + "max_us": 35871, + "p50_us": 732, + "p95_us": 8979, + "p99_us": 15365 + }, + "operations": 3200, + "operations_per_second": 3416.955152715621, + "pool_size": 2, + "pool_timeouts": 0, + "read_latency": { + "max_us": 35871, + "p50_us": 732, + "p95_us": 8861, + "p99_us": 15047 + }, + "rss_kib": 76704, + "synchronous": "FULL", + "transaction": { + "max_us": 10082, + "p50_us": 601, + "p95_us": 1215, + "p99_us": 5977 + }, + "wal_bytes": 6641472, + "workload": "read_heavy", + "write_latency": { + "max_us": 22813, + "p50_us": 748, + "p95_us": 11023, + "p99_us": 19223 + }, + "write_wait": { + "max_us": 9320, + "p50_us": 3, + "p95_us": 1565, + "p99_us": 4621 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 40935, + "p50_us": 2847, + "p95_us": 11373, + "p99_us": 17688 + }, + "checkout_over_100us": 1995, + "checkpoint": [ + 0, + 920, + 920 + ], + "checkpoint_us": 2673, + "cpu_seconds": 1.9100000000000108, + "database_bytes": 15204352, + "failures": 0, + "latency": { + "max_us": 278449, + "p50_us": 3463, + "p95_us": 12971, + "p99_us": 21816 + }, + "operations": 3200, + "operations_per_second": 1605.6341703035953, + "pool_size": 2, + "pool_timeouts": 0, + "read_latency": { + "max_us": 41232, + "p50_us": 3492, + "p95_us": 12971, + "p99_us": 20249 + }, + "rss_kib": 78688, + "synchronous": "FULL", + "transaction": { + "max_us": 9026, + "p50_us": 520, + "p95_us": 792, + "p99_us": 3783 + }, + "wal_bytes": 4754512, + "workload": "write_burst", + "write_latency": { + "max_us": 278449, + "p50_us": 3439, + "p95_us": 12971, + "p99_us": 24596 + }, + "write_wait": { + "max_us": 273109, + "p50_us": 3, + "p95_us": 24, + "p99_us": 12111 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 36738, + "p50_us": 3, + "p95_us": 9192, + "p99_us": 16757 + }, + "checkout_over_100us": 1372, + "checkpoint": [ + 0, + 992, + 992 + ], + "checkpoint_us": 2936, + "cpu_seconds": 1.8400000000000034, + "database_bytes": 14753792, + "failures": 0, + "latency": { + "max_us": 49481, + "p50_us": 1567, + "p95_us": 10033, + "p99_us": 17866 + }, + "operations": 3200, + "operations_per_second": 2733.9587904838154, + "pool_size": 2, + "pool_timeouts": 0, + "read_latency": { + "max_us": 37083, + "p50_us": 1510, + "p95_us": 9608, + "p99_us": 17091 + }, + "rss_kib": 73456, + "synchronous": "FULL", + "transaction": { + "max_us": 9113, + "p50_us": 497, + "p95_us": 787, + "p99_us": 3621 + }, + "wal_bytes": 5582632, + "workload": "mixed", + "write_latency": { + "max_us": 49481, + "p50_us": 2008, + "p95_us": 10656, + "p99_us": 20085 + }, + "write_wait": { + "max_us": 48950, + "p50_us": 3, + "p95_us": 1592, + "p99_us": 4585 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 21648, + "p50_us": 2, + "p95_us": 4062, + "p99_us": 7876 + }, + "checkout_over_100us": 784, + "checkpoint": [ + 0, + 785, + 785 + ], + "checkpoint_us": 2717, + "cpu_seconds": 2.0, + "database_bytes": 14753792, + "failures": 0, + "latency": { + "max_us": 25794, + "p50_us": 743, + "p95_us": 5024, + "p99_us": 9152 + }, + "operations": 3200, + "operations_per_second": 5727.091132113926, + "pool_size": 4, + "pool_timeouts": 0, + "read_latency": { + "max_us": 22022, + "p50_us": 735, + "p95_us": 4870, + "p99_us": 8949 + }, + "rss_kib": 68512, + "synchronous": "FULL", + "transaction": { + "max_us": 9351, + "p50_us": 706, + "p95_us": 1257, + "p99_us": 6909 + }, + "wal_bytes": 11021032, + "workload": "read_heavy", + "write_latency": { + "max_us": 25794, + "p50_us": 945, + "p95_us": 7490, + "p99_us": 12064 + }, + "write_wait": { + "max_us": 25223, + "p50_us": 3, + "p95_us": 1598, + "p99_us": 9872 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 19272, + "p50_us": 609, + "p95_us": 6176, + "p99_us": 10195 + }, + "checkout_over_100us": 1616, + "checkpoint": [ + 0, + 24, + 24 + ], + "checkpoint_us": 414, + "cpu_seconds": 1.740000000000009, + "database_bytes": 15216640, + "failures": 0, + "latency": { + "max_us": 593160, + "p50_us": 1792, + "p95_us": 8843, + "p99_us": 46751 + }, + "operations": 3200, + "operations_per_second": 1853.5860124330882, + "pool_size": 4, + "pool_timeouts": 0, + "read_latency": { + "max_us": 15461, + "p50_us": 1714, + "p95_us": 6888, + "p99_us": 11052 + }, + "rss_kib": 86896, + "synchronous": "FULL", + "transaction": { + "max_us": 11236, + "p50_us": 479, + "p95_us": 733, + "p99_us": 3343 + }, + "wal_bytes": 5384872, + "workload": "write_burst", + "write_latency": { + "max_us": 593160, + "p50_us": 1852, + "p95_us": 10413, + "p99_us": 66414 + }, + "write_wait": { + "max_us": 592456, + "p50_us": 3, + "p95_us": 4554, + "p99_us": 65826 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 24922, + "p50_us": 3, + "p95_us": 5396, + "p99_us": 9175 + }, + "checkout_over_100us": 1072, + "checkpoint": [ + 0, + 20, + 20 + ], + "checkpoint_us": 316, + "cpu_seconds": 1.8700000000000045, + "database_bytes": 14753792, + "failures": 0, + "latency": { + "max_us": 823226, + "p50_us": 785, + "p95_us": 6968, + "p99_us": 12556 + }, + "operations": 3200, + "operations_per_second": 2995.898195009067, + "pool_size": 4, + "pool_timeouts": 0, + "read_latency": { + "max_us": 25499, + "p50_us": 750, + "p95_us": 6097, + "p99_us": 10488 + }, + "rss_kib": 80160, + "synchronous": "FULL", + "transaction": { + "max_us": 8408, + "p50_us": 506, + "p95_us": 1126, + "p99_us": 3581 + }, + "wal_bytes": 9331832, + "workload": "mixed", + "write_latency": { + "max_us": 823226, + "p50_us": 1530, + "p95_us": 9598, + "p99_us": 27161 + }, + "write_wait": { + "max_us": 822049, + "p50_us": 3, + "p95_us": 4569, + "p99_us": 26619 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 104, + "p50_us": 2, + "p95_us": 3, + "p99_us": 4 + }, + "checkout_over_100us": 1, + "checkpoint": [ + 0, + 3463, + 3463 + ], + "checkpoint_us": 3261, + "cpu_seconds": 3.3499999999999943, + "database_bytes": 14753792, + "failures": 0, + "latency": { + "max_us": 115560, + "p50_us": 817, + "p95_us": 3451, + "p99_us": 6692 + }, + "operations": 3200, + "operations_per_second": 5545.922403880066, + "pool_size": 8, + "pool_timeouts": 0, + "read_latency": { + "max_us": 13345, + "p50_us": 794, + "p95_us": 3307, + "p99_us": 4066 + }, + "rss_kib": 86832, + "synchronous": "FULL", + "transaction": { + "max_us": 11415, + "p50_us": 1570, + "p95_us": 3424, + "p99_us": 10332 + }, + "wal_bytes": 14267592, + "workload": "read_heavy", + "write_latency": { + "max_us": 115560, + "p50_us": 2128, + "p95_us": 13494, + "p99_us": 81868 + }, + "write_wait": { + "max_us": 114729, + "p50_us": 53, + "p95_us": 12164, + "p99_us": 79781 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 50, + "p50_us": 3, + "p95_us": 4, + "p99_us": 7 + }, + "checkout_over_100us": 0, + "checkpoint": [ + 0, + 736, + 736 + ], + "checkpoint_us": 2191, + "cpu_seconds": 1.7800000000000011, + "database_bytes": 15200256, + "failures": 0, + "latency": { + "max_us": 1047420, + "p50_us": 493, + "p95_us": 3270, + "p99_us": 45375 + }, + "operations": 3200, + "operations_per_second": 1793.6255385757938, + "pool_size": 8, + "pool_timeouts": 0, + "read_latency": { + "max_us": 12327, + "p50_us": 443, + "p95_us": 1745, + "p99_us": 1944 + }, + "rss_kib": 87056, + "synchronous": "FULL", + "transaction": { + "max_us": 11604, + "p50_us": 488, + "p95_us": 779, + "p99_us": 3601 + }, + "wal_bytes": 5022312, + "workload": "write_burst", + "write_latency": { + "max_us": 1047420, + "p50_us": 498, + "p95_us": 5120, + "p99_us": 50213 + }, + "write_wait": { + "max_us": 1046725, + "p50_us": 3, + "p95_us": 4556, + "p99_us": 49627 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 313, + "p50_us": 3, + "p95_us": 5, + "p99_us": 16 + }, + "checkout_over_100us": 1, + "checkpoint": [ + 0, + 872, + 872 + ], + "checkpoint_us": 3147, + "cpu_seconds": 2.280000000000001, + "database_bytes": 14753792, + "failures": 0, + "latency": { + "max_us": 512019, + "p50_us": 654, + "p95_us": 5431, + "p99_us": 28246 + }, + "operations": 3200, + "operations_per_second": 2625.7450687022424, + "pool_size": 8, + "pool_timeouts": 0, + "read_latency": { + "max_us": 12706, + "p50_us": 481, + "p95_us": 2002, + "p99_us": 5266 + }, + "rss_kib": 77872, + "synchronous": "FULL", + "transaction": { + "max_us": 15399, + "p50_us": 635, + "p95_us": 2151, + "p99_us": 6716 + }, + "wal_bytes": 9817992, + "workload": "mixed", + "write_latency": { + "max_us": 512019, + "p50_us": 942, + "p95_us": 26716, + "p99_us": 135339 + }, + "write_wait": { + "max_us": 511445, + "p50_us": 66, + "p95_us": 25976, + "p99_us": 134703 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 53, + "p50_us": 3, + "p95_us": 4, + "p99_us": 14 + }, + "checkout_over_100us": 0, + "checkpoint": [ + 0, + 49, + 49 + ], + "checkpoint_us": 890, + "cpu_seconds": 3.579999999999984, + "database_bytes": 14753792, + "failures": 0, + "latency": { + "max_us": 107255, + "p50_us": 922, + "p95_us": 4158, + "p99_us": 7918 + }, + "operations": 3200, + "operations_per_second": 4734.7854328407675, + "pool_size": 12, + "pool_timeouts": 0, + "read_latency": { + "max_us": 14414, + "p50_us": 870, + "p95_us": 3864, + "p99_us": 5094 + }, + "rss_kib": 81344, + "synchronous": "FULL", + "transaction": { + "max_us": 11154, + "p50_us": 1916, + "p95_us": 4434, + "p99_us": 9905 + }, + "wal_bytes": 14065712, + "workload": "read_heavy", + "write_latency": { + "max_us": 107255, + "p50_us": 2688, + "p95_us": 27281, + "p99_us": 100857 + }, + "write_wait": { + "max_us": 105378, + "p50_us": 75, + "p95_us": 25124, + "p99_us": 99106 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 25, + "p50_us": 3, + "p95_us": 5, + "p99_us": 11 + }, + "checkout_over_100us": 0, + "checkpoint": [ + 0, + 21, + 21 + ], + "checkpoint_us": 623, + "cpu_seconds": 2.0500000000000114, + "database_bytes": 15192064, + "failures": 0, + "latency": { + "max_us": 946846, + "p50_us": 578, + "p95_us": 5205, + "p99_us": 105659 + }, + "operations": 3200, + "operations_per_second": 1563.5547444085446, + "pool_size": 12, + "pool_timeouts": 0, + "read_latency": { + "max_us": 7275, + "p50_us": 507, + "p95_us": 1998, + "p99_us": 2412 + }, + "rss_kib": 81456, + "synchronous": "FULL", + "transaction": { + "max_us": 9513, + "p50_us": 565, + "p95_us": 1241, + "p99_us": 4189 + }, + "wal_bytes": 5228312, + "workload": "write_burst", + "write_latency": { + "max_us": 946846, + "p50_us": 583, + "p95_us": 11987, + "p99_us": 137035 + }, + "write_wait": { + "max_us": 946021, + "p50_us": 3, + "p95_us": 11276, + "p99_us": 136408 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 63, + "p50_us": 3, + "p95_us": 4, + "p99_us": 8 + }, + "checkout_over_100us": 0, + "checkpoint": [ + 0, + 599, + 599 + ], + "checkpoint_us": 2425, + "cpu_seconds": 2.1499999999999773, + "database_bytes": 14753792, + "failures": 0, + "latency": { + "max_us": 602536, + "p50_us": 616, + "p95_us": 5143, + "p99_us": 27726 + }, + "operations": 3200, + "operations_per_second": 2825.104200459125, + "pool_size": 12, + "pool_timeouts": 0, + "read_latency": { + "max_us": 12409, + "p50_us": 466, + "p95_us": 1911, + "p99_us": 2337 + }, + "rss_kib": 73056, + "synchronous": "FULL", + "transaction": { + "max_us": 8758, + "p50_us": 619, + "p95_us": 1508, + "p99_us": 3671 + }, + "wal_bytes": 12961552, + "workload": "mixed", + "write_latency": { + "max_us": 602536, + "p50_us": 887, + "p95_us": 26186, + "p99_us": 107093 + }, + "write_wait": { + "max_us": 601905, + "p50_us": 62, + "p95_us": 25603, + "p99_us": 106435 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 105, + "p50_us": 2, + "p95_us": 4, + "p99_us": 18 + }, + "checkout_over_100us": 1, + "checkpoint": [ + 0, + 412, + 412 + ], + "checkpoint_us": 2239, + "cpu_seconds": 3.4599999999999795, + "database_bytes": 14753792, + "failures": 0, + "latency": { + "max_us": 103559, + "p50_us": 849, + "p95_us": 3977, + "p99_us": 8875 + }, + "operations": 3200, + "operations_per_second": 4862.84867194843, + "pool_size": 16, + "pool_timeouts": 0, + "read_latency": { + "max_us": 14364, + "p50_us": 822, + "p95_us": 3652, + "p99_us": 5231 + }, + "rss_kib": 88704, + "synchronous": "FULL", + "transaction": { + "max_us": 11824, + "p50_us": 1625, + "p95_us": 4550, + "p99_us": 8829 + }, + "wal_bytes": 12557792, + "workload": "read_heavy", + "write_latency": { + "max_us": 103559, + "p50_us": 2566, + "p95_us": 26969, + "p99_us": 77162 + }, + "write_wait": { + "max_us": 101640, + "p50_us": 73, + "p95_us": 23178, + "p99_us": 75214 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 25, + "p50_us": 3, + "p95_us": 4, + "p99_us": 10 + }, + "checkout_over_100us": 0, + "checkpoint": [ + 0, + 994, + 994 + ], + "checkpoint_us": 3263, + "cpu_seconds": 1.9799999999999898, + "database_bytes": 15200256, + "failures": 0, + "latency": { + "max_us": 817040, + "p50_us": 540, + "p95_us": 5154, + "p99_us": 75740 + }, + "operations": 3200, + "operations_per_second": 1649.081758063573, + "pool_size": 16, + "pool_timeouts": 0, + "read_latency": { + "max_us": 7651, + "p50_us": 476, + "p95_us": 1931, + "p99_us": 2405 + }, + "rss_kib": 88816, + "synchronous": "FULL", + "transaction": { + "max_us": 11961, + "p50_us": 533, + "p95_us": 957, + "p99_us": 3969 + }, + "wal_bytes": 5562032, + "workload": "write_burst", + "write_latency": { + "max_us": 817040, + "p50_us": 544, + "p95_us": 5899, + "p99_us": 108367 + }, + "write_wait": { + "max_us": 815895, + "p50_us": 3, + "p95_us": 4622, + "p99_us": 107852 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 70, + "p50_us": 3, + "p95_us": 4, + "p99_us": 9 + }, + "checkout_over_100us": 0, + "checkpoint": [ + 0, + 977, + 977 + ], + "checkpoint_us": 2798, + "cpu_seconds": 2.1999999999999886, + "database_bytes": 14753792, + "failures": 0, + "latency": { + "max_us": 499719, + "p50_us": 638, + "p95_us": 4392, + "p99_us": 27164 + }, + "operations": 3200, + "operations_per_second": 2934.8208382261573, + "pool_size": 16, + "pool_timeouts": 0, + "read_latency": { + "max_us": 11853, + "p50_us": 476, + "p95_us": 1952, + "p99_us": 2273 + }, + "rss_kib": 69872, + "synchronous": "FULL", + "transaction": { + "max_us": 9114, + "p50_us": 624, + "p95_us": 1352, + "p99_us": 4747 + }, + "wal_bytes": 9455432, + "workload": "mixed", + "write_latency": { + "max_us": 499719, + "p50_us": 851, + "p95_us": 25858, + "p99_us": 105171 + }, + "write_wait": { + "max_us": 499093, + "p50_us": 67, + "p95_us": 25340, + "p99_us": 104568 + } + } + ], + "query_plans": [ + { + "details": [ + "MATERIALIZE page", + "SEARCH t USING INDEX idx_threads_board_sticky_bumped (board_id=?)", + "SEARCH op EXISTS USING COVERING INDEX idx_posts_one_op_per_thread (thread_id=?)", + "USE TEMP B-TREE FOR LAST TERM OF ORDER BY", + "SCAN t", + "SEARCH op USING INDEX idx_posts_one_op_per_thread (thread_id=?)", + "CORRELATED SCALAR SUBQUERY 3", + "SEARCH fp USING INDEX idx_posts_thread_live (thread_id=?)", + "CORRELATED SCALAR SUBQUERY 4", + "SEARCH fp USING COVERING INDEX idx_posts_thread (thread_id=?)" + ], + "name": "board_index" + }, + { + "details": [ + "MATERIALIZE page", + "SEARCH t USING INDEX idx_threads_board_sticky_bumped (board_id=?)", + "SEARCH op EXISTS USING COVERING INDEX idx_posts_one_op_per_thread (thread_id=?)", + "USE TEMP B-TREE FOR LAST TERM OF ORDER BY", + "SCAN t", + "SEARCH op USING INDEX idx_posts_one_op_per_thread (thread_id=?)", + "CORRELATED SCALAR SUBQUERY 3", + "SEARCH fp USING INDEX idx_posts_thread_live (thread_id=?)", + "CORRELATED SCALAR SUBQUERY 4", + "SEARCH fp USING COVERING INDEX idx_posts_thread (thread_id=?)" + ], + "name": "catalog" + }, + { + "details": [ + "SEARCH posts USING INDEX idx_posts_thread (thread_id=?)" + ], + "name": "thread_loading" + }, + { + "details": [ + "SEARCH posts USING COVERING INDEX idx_posts_board (board_id=?)", + "USE TEMP B-TREE FOR LAST TERM OF ORDER BY" + ], + "name": "latest_post" + }, + { + "details": [ + "SEARCH threads USING INTEGER PRIMARY KEY (rowid=?)", + "SCALAR SUBQUERY 1", + "SEARCH boards USING INTEGER PRIMARY KEY (rowid=?)" + ], + "name": "thread_bump" + }, + { + "details": [ + "SCAN CONSTANT ROW", + "SCALAR SUBQUERY 1", + "SCAN posts USING COVERING INDEX idx_posts_board", + "SCALAR SUBQUERY 2", + "SCAN threads USING COVERING INDEX idx_threads_archived", + "SCALAR SUBQUERY 3", + "SEARCH reports USING COVERING INDEX idx_reports_status (status=?)" + ], + "name": "admin_overview" + }, + { + "details": [ + "SEARCH reports USING COVERING INDEX idx_reports_open_unique (post_id=? AND reporter_hash=?)" + ], + "name": "report_lookup" + }, + { + "details": [ + "SEARCH posts USING INDEX idx_posts_board (board_id=?)", + "USE TEMP B-TREE FOR ORDER BY" + ], + "name": "backup_board_inventory" + }, + { + "details": [ + "SEARCH posts USING INDEX idx_posts_thread_live (thread_id=? AND id>?)" + ], + "name": "live_poll" + }, + { + "details": [ + "SEARCH posts USING COVERING INDEX idx_posts_board_ip_created (board_id=? AND ip_hash=?)" + ], + "name": "cooldown" + }, + { + "details": [ + "SEARCH threads USING INDEX idx_threads_archived (board_id=? AND archived=?)", + "USE TEMP B-TREE FOR ORDER BY" + ], + "name": "latest_thread" + }, + { + "details": [ + "SEARCH threads USING INDEX idx_threads_archived (board_id=? AND archived=?)" + ], + "name": "prune" + }, + { + "details": [ + "SCAN posts_fts VIRTUAL TABLE INDEX 0:M1", + "SEARCH p USING INTEGER PRIMARY KEY (rowid=?)", + "USE TEMP B-TREE FOR ORDER BY" + ], + "name": "search_selective" + }, + { + "details": [ + "SEARCH reports USING INDEX idx_reports_status (status=?)" + ], + "name": "reports" + }, + { + "details": [ + "SEARCH bans USING INDEX idx_bans_ip (ip_hash=?)" + ], + "name": "bans" + }, + { + "details": [ + "SEARCH background_jobs USING INDEX idx_jobs_pending (status=?)" + ], + "name": "job_claim" + }, + { + "details": [ + "MULTI-INDEX OR", + "INDEX 1", + "SEARCH posts USING INDEX idx_posts_file_path (file_path=?)", + "INDEX 2", + "SEARCH posts USING INDEX idx_posts_thumb_path (thumb_path=?)", + "INDEX 3", + "SEARCH posts USING INDEX idx_posts_audio_file_path (audio_file_path=?)" + ], + "name": "file_refs" + }, + { + "details": [ + "SEARCH file_hashes USING INDEX sqlite_autoindex_file_hashes_1 (sha256=?)" + ], + "name": "dedup" + }, + { + "details": [ + "SCAN mod_log USING INDEX idx_mod_log_created" + ], + "name": "mod_log" + } + ], + "sqlite_version": "3.51.3", + "workers": 8 +} \ No newline at end of file diff --git a/docs/sqlite-evidence/index-profile.json b/docs/sqlite-evidence/index-profile.json new file mode 100644 index 00000000..26bef657 --- /dev/null +++ b/docs/sqlite-evidence/index-profile.json @@ -0,0 +1,240 @@ +{ + "database_bytes_after": 14237696, + "database_bytes_before": 13066240, + "queries": [ + { + "after": { + "max_us": 615, + "p50_us": 265, + "p95_us": 289, + "p99_us": 615 + }, + "after_plan": [ + "SCAN posts_fts VIRTUAL TABLE INDEX 0:M1", + "SEARCH p USING INTEGER PRIMARY KEY (rowid=?)", + "USE TEMP B-TREE FOR ORDER BY" + ], + "before": { + "max_us": 24659, + "p50_us": 24246, + "p95_us": 24552, + "p99_us": 24659 + }, + "before_plan": [ + "SEARCH p USING COVERING INDEX idx_posts_board (board_id=?)", + "SCAN f VIRTUAL TABLE INDEX 0:=M1", + "USE TEMP B-TREE FOR LAST TERM OF ORDER BY" + ], + "query": "fts_p.id_needle" + }, + { + "after": { + "max_us": 377, + "p50_us": 238, + "p95_us": 303, + "p99_us": 377 + }, + "after_plan": [ + "SCAN posts_fts VIRTUAL TABLE INDEX 0:M1", + "SEARCH p USING INTEGER PRIMARY KEY (rowid=?)" + ], + "before": { + "max_us": 43439, + "p50_us": 42653, + "p95_us": 43424, + "p99_us": 43439 + }, + "before_plan": [ + "SEARCH p USING COVERING INDEX idx_posts_board (board_id=?)", + "SCAN f VIRTUAL TABLE INDEX 0:=M1" + ], + "query": "fts_COUNT(*)_needle" + }, + { + "after": { + "max_us": 9327, + "p50_us": 8878, + "p95_us": 9109, + "p99_us": 9327 + }, + "after_plan": [ + "SCAN posts_fts VIRTUAL TABLE INDEX 0:M1", + "SEARCH p USING INTEGER PRIMARY KEY (rowid=?)", + "USE TEMP B-TREE FOR ORDER BY" + ], + "before": { + "max_us": 272, + "p50_us": 244, + "p95_us": 270, + "p99_us": 272 + }, + "before_plan": [ + "SEARCH p USING COVERING INDEX idx_posts_board (board_id=?)", + "SCAN f VIRTUAL TABLE INDEX 0:=M1", + "USE TEMP B-TREE FOR LAST TERM OF ORDER BY" + ], + "query": "fts_p.id_representative" + }, + { + "after": { + "max_us": 7151, + "p50_us": 6779, + "p95_us": 6872, + "p99_us": 7151 + }, + "after_plan": [ + "SCAN posts_fts VIRTUAL TABLE INDEX 0:M1", + "SEARCH p USING INTEGER PRIMARY KEY (rowid=?)" + ], + "before": { + "max_us": 59522, + "p50_us": 58353, + "p95_us": 58803, + "p99_us": 59522 + }, + "before_plan": [ + "SEARCH p USING COVERING INDEX idx_posts_board (board_id=?)", + "SCAN f VIRTUAL TABLE INDEX 0:=M1" + ], + "query": "fts_COUNT(*)_representative" + }, + { + "after": { + "max_us": 30, + "p50_us": 8, + "p95_us": 13, + "p99_us": 30 + }, + "after_plan": [ + "SCAN posts_fts VIRTUAL TABLE INDEX 0:M1", + "SEARCH p USING INTEGER PRIMARY KEY (rowid=?)", + "USE TEMP B-TREE FOR ORDER BY" + ], + "before": { + "max_us": 18506, + "p50_us": 18177, + "p95_us": 18403, + "p99_us": 18506 + }, + "before_plan": [ + "SEARCH p USING COVERING INDEX idx_posts_board (board_id=?)", + "SCAN f VIRTUAL TABLE INDEX 0:=M1", + "USE TEMP B-TREE FOR LAST TERM OF ORDER BY" + ], + "query": "fts_p.id_missingword" + }, + { + "after": { + "max_us": 9, + "p50_us": 6, + "p95_us": 6, + "p99_us": 9 + }, + "after_plan": [ + "SCAN posts_fts VIRTUAL TABLE INDEX 0:M1", + "SEARCH p USING INTEGER PRIMARY KEY (rowid=?)" + ], + "before": { + "max_us": 18411, + "p50_us": 18219, + "p95_us": 18403, + "p99_us": 18411 + }, + "before_plan": [ + "SEARCH p USING COVERING INDEX idx_posts_board (board_id=?)", + "SCAN f VIRTUAL TABLE INDEX 0:=M1" + ], + "query": "fts_COUNT(*)_missingword" + }, + { + "after": { + "max_us": 149, + "p50_us": 139, + "p95_us": 149, + "p99_us": 149 + }, + "after_plan": [ + "MATERIALIZE page", + "SEARCH t USING INDEX idx_threads_board_sticky_bumped (board_id=?)", + "SEARCH op EXISTS USING COVERING INDEX idx_posts_one_op_per_thread (thread_id=?)", + "SCAN t", + "SEARCH op USING INDEX idx_posts_one_op_per_thread (thread_id=?)", + "CORRELATED SCALAR SUBQUERY 3", + "SEARCH fp USING INDEX idx_posts_thread (thread_id=?)", + "CORRELATED SCALAR SUBQUERY 4", + "SEARCH fp USING COVERING INDEX idx_posts_thread (thread_id=?)" + ], + "before": { + "max_us": 5533, + "p50_us": 5341, + "p95_us": 5446, + "p99_us": 5533 + }, + "before_plan": [ + "SEARCH t USING INDEX idx_threads_archived (board_id=? AND archived=?)", + "SEARCH op USING INDEX idx_posts_one_op_per_thread (thread_id=?)", + "SEARCH fp USING INDEX idx_posts_thread (thread_id=?) LEFT-JOIN", + "USE TEMP B-TREE FOR GROUP BY", + "USE TEMP B-TREE FOR count(DISTINCT)", + "USE TEMP B-TREE FOR ORDER BY" + ], + "query": "board_page" + }, + { + "after": { + "max_us": 54, + "p50_us": 5, + "p95_us": 7, + "p99_us": 54 + }, + "after_plan": [ + "SEARCH posts USING INDEX idx_posts_thread_live (thread_id=? AND id>?)" + ], + "before": { + "max_us": 2877, + "p50_us": 2728, + "p95_us": 2821, + "p99_us": 2877 + }, + "before_plan": [ + "SEARCH posts USING INDEX idx_posts_board (board_id=?)", + "USE TEMP B-TREE FOR ORDER BY" + ], + "query": "live_poll" + }, + { + "after": { + "max_us": 27, + "p50_us": 2, + "p95_us": 3, + "p99_us": 27 + }, + "after_plan": [ + "SEARCH posts USING COVERING INDEX idx_posts_board_ip_created (board_id=? AND ip_hash=?)" + ], + "before": { + "max_us": 181, + "p50_us": 155, + "p95_us": 167, + "p99_us": 181 + }, + "before_plan": [ + "SEARCH posts USING INDEX idx_posts_board (board_id=?)" + ], + "query": "cooldown" + } + ], + "sqlite": "3.51.3", + "write_after": { + "max_us": 3536, + "p50_us": 373, + "p95_us": 525, + "p99_us": 3064 + }, + "write_before": { + "max_us": 7394, + "p50_us": 321, + "p95_us": 491, + "p99_us": 3614 + } +} \ No newline at end of file diff --git a/docs/sqlite-evidence/media-baseline.json b/docs/sqlite-evidence/media-baseline.json new file mode 100644 index 00000000..179ada39 --- /dev/null +++ b/docs/sqlite-evidence/media-baseline.json @@ -0,0 +1,89 @@ +[ + { + "pending_fs_ops": 0, + "pool_size": 2, + "pool_timeouts": 20, + "posting": { + "max_us": 1382414, + "p50_us": 1369530, + "p95_us": 1382414, + "p99_us": 1382414, + "samples": 8 + }, + "posting_connection_holds": { + "max_us": 1382262, + "p50_us": 1357904, + "p95_us": 1382262, + "p99_us": 1382262, + "samples": 12 + }, + "posts": 8, + "reader_checkout": { + "max_us": 735546, + "p50_us": 7584, + "p95_us": 374829, + "p99_us": 735546, + "samples": 34 + }, + "reads": 34, + "seconds": 5.483942458 + }, + { + "pending_fs_ops": 0, + "pool_size": 4, + "pool_timeouts": 0, + "posting": { + "max_us": 357451, + "p50_us": 349984, + "p95_us": 357451, + "p99_us": 357451, + "samples": 8 + }, + "posting_connection_holds": { + "max_us": 357447, + "p50_us": 1335, + "p95_us": 357447, + "p99_us": 357447, + "samples": 16 + }, + "posts": 8, + "reader_checkout": { + "max_us": 189, + "p50_us": 8, + "p95_us": 58, + "p99_us": 93, + "samples": 2240 + }, + "reads": 2240, + "seconds": 1.412153167 + }, + { + "pending_fs_ops": 0, + "pool_size": 8, + "pool_timeouts": 0, + "posting": { + "max_us": 695160, + "p50_us": 598755, + "p95_us": 695160, + "p99_us": 695160, + "samples": 8 + }, + "posting_connection_holds": { + "max_us": 695155, + "p50_us": 166, + "p95_us": 695155, + "p99_us": 695155, + "samples": 16 + }, + "posts": 8, + "reader_checkout": { + "max_us": 20104, + "p50_us": 15, + "p95_us": 47, + "p99_us": 10037, + "samples": 3339 + }, + "reads": 3339, + "seconds": 2.4163724589999998 + } +] \ No newline at end of file diff --git a/docs/sqlite-evidence/media-optimized.json b/docs/sqlite-evidence/media-optimized.json new file mode 100644 index 00000000..28bee2c3 --- /dev/null +++ b/docs/sqlite-evidence/media-optimized.json @@ -0,0 +1,182 @@ +[ + { + "pending_fs_ops": 0, + "pool_size": 2, + "pool_timeouts": 0, + "posting": { + "max_us": 382820, + "p50_us": 346854, + "p95_us": 382820, + "p99_us": 382820, + "samples": 8 + }, + "posting_connection_holds": { + "max_us": 4657, + "p50_us": 45, + "p95_us": 1620, + "p99_us": 4657, + "samples": 64 + }, + "posts": 8, + "reader_checkout": { + "max_us": 1140, + "p50_us": 3, + "p95_us": 22, + "p99_us": 32, + "samples": 1930 + }, + "reads": 1930, + "seconds": 1.423081083, + "synchronous": "NORMAL" + }, + { + "pending_fs_ops": 0, + "pool_size": 4, + "pool_timeouts": 0, + "posting": { + "max_us": 349927, + "p50_us": 347987, + "p95_us": 349927, + "p99_us": 349927, + "samples": 8 + }, + "posting_connection_holds": { + "max_us": 2324, + "p50_us": 46, + "p95_us": 2203, + "p99_us": 2324, + "samples": 64 + }, + "posts": 8, + "reader_checkout": { + "max_us": 123, + "p50_us": 3, + "p95_us": 4, + "p99_us": 11, + "samples": 1876 + }, + "reads": 1876, + "seconds": 1.39533675, + "synchronous": "NORMAL" + }, + { + "pending_fs_ops": 0, + "pool_size": 8, + "pool_timeouts": 0, + "posting": { + "max_us": 352702, + "p50_us": 347864, + "p95_us": 352702, + "p99_us": 352702, + "samples": 8 + }, + "posting_connection_holds": { + "max_us": 4517, + "p50_us": 47, + "p95_us": 1598, + "p99_us": 4517, + "samples": 64 + }, + "posts": 8, + "reader_checkout": { + "max_us": 44, + "p50_us": 3, + "p95_us": 5, + "p99_us": 13, + "samples": 1894 + }, + "reads": 1894, + "seconds": 1.395383542, + "synchronous": "NORMAL" + }, + { + "pending_fs_ops": 0, + "pool_size": 2, + "pool_timeouts": 0, + "posting": { + "max_us": 352849, + "p50_us": 347823, + "p95_us": 352849, + "p99_us": 352849, + "samples": 8 + }, + "posting_connection_holds": { + "max_us": 4428, + "p50_us": 48, + "p95_us": 1626, + "p99_us": 4428, + "samples": 64 + }, + "posts": 8, + "reader_checkout": { + "max_us": 654, + "p50_us": 3, + "p95_us": 23, + "p99_us": 35, + "samples": 1912 + }, + "reads": 1912, + "seconds": 1.401341458, + "synchronous": "FULL" + }, + { + "pending_fs_ops": 0, + "pool_size": 4, + "pool_timeouts": 0, + "posting": { + "max_us": 356356, + "p50_us": 349336, + "p95_us": 356356, + "p99_us": 356356, + "samples": 8 + }, + "posting_connection_holds": { + "max_us": 5394, + "p50_us": 54, + "p95_us": 1991, + "p99_us": 5394, + "samples": 64 + }, + "posts": 8, + "reader_checkout": { + "max_us": 135, + "p50_us": 3, + "p95_us": 5, + "p99_us": 14, + "samples": 1910 + }, + "reads": 1910, + "seconds": 1.408157208, + "synchronous": "FULL" + }, + { + "pending_fs_ops": 0, + "pool_size": 8, + "pool_timeouts": 0, + "posting": { + "max_us": 351161, + "p50_us": 349132, + "p95_us": 351161, + "p99_us": 351161, + "samples": 8 + }, + "posting_connection_holds": { + "max_us": 2451, + "p50_us": 55, + "p95_us": 1670, + "p99_us": 2451, + "samples": 64 + }, + "posts": 8, + "reader_checkout": { + "max_us": 60, + "p50_us": 3, + "p95_us": 5, + "p99_us": 10, + "samples": 1896 + }, + "reads": 1896, + "seconds": 1.400743125, + "synchronous": "FULL" + } +] \ No newline at end of file diff --git a/docs/sqlite-evidence/optimized-repeat.json b/docs/sqlite-evidence/optimized-repeat.json new file mode 100644 index 00000000..b5b690f1 --- /dev/null +++ b/docs/sqlite-evidence/optimized-repeat.json @@ -0,0 +1,1453 @@ +{ + "fixture": { + "boards": 8, + "posts": 40000, + "threads": 800 + }, + "limitations": "SQL/database workload; media CPU and HTTP excluded; RSS sampled at case end; ps optional; process-kill is not power loss", + "measurements": [ + { + "busy_errors": 0, + "checkout": { + "max_us": 36258, + "p50_us": 3, + "p95_us": 8066, + "p99_us": 13619 + }, + "checkout_over_100us": 1122, + "checkpoint": [ + 0, + 295, + 295 + ], + "checkpoint_us": 2191, + "cpu_seconds": 3.8600000000000003, + "database_bytes": 14753792, + "failures": 0, + "latency": { + "max_us": 37774, + "p50_us": 700, + "p95_us": 8793, + "p99_us": 14162 + }, + "operations": 3200, + "operations_per_second": 3712.021562093888, + "pool_size": 2, + "pool_timeouts": 0, + "rss_kib": 68016, + "synchronous": "NORMAL", + "transaction": { + "max_us": 8606, + "p50_us": 468, + "p95_us": 1157, + "p99_us": 5126 + }, + "wal_bytes": 4614432, + "workload": "read_heavy", + "write_wait": { + "max_us": 1619, + "p50_us": 3, + "p95_us": 18, + "p99_us": 1611 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 37641, + "p50_us": 2343, + "p95_us": 11022, + "p99_us": 17506 + }, + "checkout_over_100us": 1821, + "checkpoint": [ + 0, + 72, + 72 + ], + "checkpoint_us": 603, + "cpu_seconds": 10.45, + "database_bytes": 15200256, + "failures": 0, + "latency": { + "max_us": 237764, + "p50_us": 2892, + "p95_us": 12331, + "p99_us": 24497 + }, + "operations": 3200, + "operations_per_second": 1820.5260475098087, + "pool_size": 2, + "pool_timeouts": 0, + "rss_kib": 71712, + "synchronous": "NORMAL", + "transaction": { + "max_us": 9820, + "p50_us": 453, + "p95_us": 725, + "p99_us": 4209 + }, + "wal_bytes": 5018192, + "workload": "write_burst", + "write_wait": { + "max_us": 232814, + "p50_us": 3, + "p95_us": 10, + "p99_us": 24896 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 26942, + "p50_us": 3, + "p95_us": 9307, + "p99_us": 15211 + }, + "checkout_over_100us": 1331, + "checkpoint": [ + 0, + 527, + 527 + ], + "checkpoint_us": 2634, + "cpu_seconds": 5.959999999999999, + "database_bytes": 14753792, + "failures": 0, + "latency": { + "max_us": 33427, + "p50_us": 1389, + "p95_us": 10065, + "p99_us": 16483 + }, + "operations": 3200, + "operations_per_second": 2864.551472191739, + "pool_size": 2, + "pool_timeouts": 0, + "rss_kib": 60656, + "synchronous": "NORMAL", + "transaction": { + "max_us": 8421, + "p50_us": 460, + "p95_us": 1053, + "p99_us": 3537 + }, + "wal_bytes": 5636192, + "workload": "mixed", + "write_wait": { + "max_us": 25973, + "p50_us": 3, + "p95_us": 1579, + "p99_us": 4591 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 16905, + "p50_us": 3, + "p95_us": 3987, + "p99_us": 7480 + }, + "checkout_over_100us": 822, + "checkpoint": [ + 0, + 797, + 797 + ], + "checkpoint_us": 3751, + "cpu_seconds": 4.25, + "database_bytes": 14753792, + "failures": 0, + "latency": { + "max_us": 26363, + "p50_us": 755, + "p95_us": 4896, + "p99_us": 9298 + }, + "operations": 3200, + "operations_per_second": 5544.51465571163, + "pool_size": 4, + "pool_timeouts": 0, + "rss_kib": 70528, + "synchronous": "NORMAL", + "transaction": { + "max_us": 8949, + "p50_us": 650, + "p95_us": 1775, + "p99_us": 6275 + }, + "wal_bytes": 5496112, + "workload": "read_heavy", + "write_wait": { + "max_us": 25802, + "p50_us": 4, + "p95_us": 1626, + "p99_us": 12130 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 17998, + "p50_us": 3, + "p95_us": 6293, + "p99_us": 10388 + }, + "checkout_over_100us": 1490, + "checkpoint": [ + 0, + 158, + 158 + ], + "checkpoint_us": 1030, + "cpu_seconds": 10.54, + "database_bytes": 15204352, + "failures": 0, + "latency": { + "max_us": 506398, + "p50_us": 1523, + "p95_us": 8449, + "p99_us": 30163 + }, + "operations": 3200, + "operations_per_second": 1922.5091150933379, + "pool_size": 4, + "pool_timeouts": 0, + "rss_kib": 81216, + "synchronous": "NORMAL", + "transaction": { + "max_us": 8508, + "p50_us": 453, + "p95_us": 718, + "p99_us": 4318 + }, + "wal_bytes": 5170632, + "workload": "write_burst", + "write_wait": { + "max_us": 498531, + "p50_us": 3, + "p95_us": 1560, + "p99_us": 45994 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 20924, + "p50_us": 3, + "p95_us": 5545, + "p99_us": 9324 + }, + "checkout_over_100us": 1112, + "checkpoint": [ + 0, + 908, + 908 + ], + "checkpoint_us": 4316, + "cpu_seconds": 6.049999999999997, + "database_bytes": 14753792, + "failures": 0, + "latency": { + "max_us": 227871, + "p50_us": 845, + "p95_us": 7175, + "p99_us": 13177 + }, + "operations": 3200, + "operations_per_second": 3274.4869384722156, + "pool_size": 4, + "pool_timeouts": 0, + "rss_kib": 71872, + "synchronous": "NORMAL", + "transaction": { + "max_us": 8623, + "p50_us": 489, + "p95_us": 1206, + "p99_us": 3569 + }, + "wal_bytes": 8190592, + "workload": "mixed", + "write_wait": { + "max_us": 227352, + "p50_us": 4, + "p95_us": 4629, + "p99_us": 40947 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 69, + "p50_us": 2, + "p95_us": 3, + "p99_us": 10 + }, + "checkout_over_100us": 0, + "checkpoint": [ + 0, + 1164, + 1164 + ], + "checkpoint_us": 8, + "cpu_seconds": 5.840000000000003, + "database_bytes": 14753792, + "failures": 0, + "latency": { + "max_us": 110191, + "p50_us": 864, + "p95_us": 4027, + "p99_us": 7063 + }, + "operations": 3200, + "operations_per_second": 4904.787100259105, + "pool_size": 8, + "pool_timeouts": 0, + "rss_kib": 106192, + "synchronous": "NORMAL", + "transaction": { + "max_us": 12140, + "p50_us": 1783, + "p95_us": 3547, + "p99_us": 11607 + }, + "wal_bytes": 9451312, + "workload": "read_heavy", + "write_wait": { + "max_us": 108605, + "p50_us": 98, + "p95_us": 12155, + "p99_us": 78217 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 27, + "p50_us": 2, + "p95_us": 3, + "p99_us": 5 + }, + "checkout_over_100us": 0, + "checkpoint": [ + 0, + 70, + 70 + ], + "checkpoint_us": 692, + "cpu_seconds": 11.119999999999997, + "database_bytes": 15204352, + "failures": 0, + "latency": { + "max_us": 502975, + "p50_us": 460, + "p95_us": 5089, + "p99_us": 102121 + }, + "operations": 3200, + "operations_per_second": 1913.9163775543961, + "pool_size": 8, + "pool_timeouts": 0, + "rss_kib": 112768, + "synchronous": "NORMAL", + "transaction": { + "max_us": 8930, + "p50_us": 456, + "p95_us": 907, + "p99_us": 4061 + }, + "wal_bytes": 5562032, + "workload": "write_burst", + "write_wait": { + "max_us": 502501, + "p50_us": 3, + "p95_us": 4637, + "p99_us": 122950 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 18, + "p50_us": 2, + "p95_us": 3, + "p99_us": 5 + }, + "checkout_over_100us": 0, + "checkpoint": [ + 0, + 637, + 637 + ], + "checkpoint_us": 3033, + "cpu_seconds": 6.160000000000004, + "database_bytes": 14753792, + "failures": 0, + "latency": { + "max_us": 482950, + "p50_us": 586, + "p95_us": 4941, + "p99_us": 27298 + }, + "operations": 3200, + "operations_per_second": 3458.4814270841716, + "pool_size": 8, + "pool_timeouts": 0, + "rss_kib": 86384, + "synchronous": "NORMAL", + "transaction": { + "max_us": 9368, + "p50_us": 565, + "p95_us": 1393, + "p99_us": 3899 + }, + "wal_bytes": 11655512, + "workload": "mixed", + "write_wait": { + "max_us": 482497, + "p50_us": 47, + "p95_us": 23952, + "p99_us": 106602 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 51, + "p50_us": 2, + "p95_us": 3, + "p99_us": 5 + }, + "checkout_over_100us": 0, + "checkpoint": [ + 0, + 3460, + 3460 + ], + "checkpoint_us": 7, + "cpu_seconds": 5.670000000000002, + "database_bytes": 14753792, + "failures": 0, + "latency": { + "max_us": 29414, + "p50_us": 850, + "p95_us": 3778, + "p99_us": 7956 + }, + "operations": 3200, + "operations_per_second": 5436.963747168464, + "pool_size": 12, + "pool_timeouts": 0, + "rss_kib": 100672, + "synchronous": "NORMAL", + "transaction": { + "max_us": 12489, + "p50_us": 1708, + "p95_us": 4133, + "p99_us": 11713 + }, + "wal_bytes": 14255232, + "workload": "read_heavy", + "write_wait": { + "max_us": 27098, + "p50_us": 108, + "p95_us": 22717, + "p99_us": 27079 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 26, + "p50_us": 2, + "p95_us": 3, + "p99_us": 6 + }, + "checkout_over_100us": 0, + "checkpoint": [ + 0, + 524, + 524 + ], + "checkpoint_us": 3041, + "cpu_seconds": 10.530000000000001, + "database_bytes": 15204352, + "failures": 0, + "latency": { + "max_us": 922656, + "p50_us": 459, + "p95_us": 5016, + "p99_us": 73583 + }, + "operations": 3200, + "operations_per_second": 1878.475913853393, + "pool_size": 12, + "pool_timeouts": 0, + "rss_kib": 111424, + "synchronous": "NORMAL", + "transaction": { + "max_us": 10755, + "p50_us": 454, + "p95_us": 930, + "p99_us": 4210 + }, + "wal_bytes": 6072912, + "workload": "write_burst", + "write_wait": { + "max_us": 921846, + "p50_us": 3, + "p95_us": 4585, + "p99_us": 110366 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 26, + "p50_us": 2, + "p95_us": 3, + "p99_us": 5 + }, + "checkout_over_100us": 0, + "checkpoint": [ + 0, + 719, + 719 + ], + "checkpoint_us": 2805, + "cpu_seconds": 6.180000000000007, + "database_bytes": 14753792, + "failures": 0, + "latency": { + "max_us": 223510, + "p50_us": 626, + "p95_us": 5040, + "p99_us": 26752 + }, + "operations": 3200, + "operations_per_second": 3522.40386451734, + "pool_size": 12, + "pool_timeouts": 0, + "rss_kib": 76784, + "synchronous": "NORMAL", + "transaction": { + "max_us": 8595, + "p50_us": 612, + "p95_us": 1520, + "p99_us": 3307 + }, + "wal_bytes": 18416432, + "workload": "mixed", + "write_wait": { + "max_us": 222804, + "p50_us": 47, + "p95_us": 23905, + "p99_us": 102926 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 82, + "p50_us": 2, + "p95_us": 3, + "p99_us": 11 + }, + "checkout_over_100us": 0, + "checkpoint": [ + 0, + 260, + 260 + ], + "checkpoint_us": 2250, + "cpu_seconds": 6.13000000000001, + "database_bytes": 14753792, + "failures": 0, + "latency": { + "max_us": 29173, + "p50_us": 998, + "p95_us": 4198, + "p99_us": 6974 + }, + "operations": 3200, + "operations_per_second": 4956.411006039621, + "pool_size": 16, + "pool_timeouts": 0, + "rss_kib": 115296, + "synchronous": "NORMAL", + "transaction": { + "max_us": 13828, + "p50_us": 1957, + "p95_us": 4012, + "p99_us": 11210 + }, + "wal_bytes": 6847472, + "workload": "read_heavy", + "write_wait": { + "max_us": 27347, + "p50_us": 124, + "p95_us": 12103, + "p99_us": 26397 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 17, + "p50_us": 2, + "p95_us": 4, + "p99_us": 5 + }, + "checkout_over_100us": 0, + "checkpoint": [ + 0, + 345, + 345 + ], + "checkpoint_us": 1781, + "cpu_seconds": 10.609999999999985, + "database_bytes": 15212544, + "failures": 0, + "latency": { + "max_us": 816170, + "p50_us": 478, + "p95_us": 5145, + "p99_us": 79186 + }, + "operations": 3200, + "operations_per_second": 1894.3771278058296, + "pool_size": 16, + "pool_timeouts": 0, + "rss_kib": 109504, + "synchronous": "NORMAL", + "transaction": { + "max_us": 10116, + "p50_us": 473, + "p95_us": 879, + "p99_us": 4270 + }, + "wal_bytes": 5772152, + "workload": "write_burst", + "write_wait": { + "max_us": 815619, + "p50_us": 3, + "p95_us": 10352, + "p99_us": 109491 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 36, + "p50_us": 2, + "p95_us": 3, + "p99_us": 4 + }, + "checkout_over_100us": 0, + "checkpoint": [ + 0, + 401, + 401 + ], + "checkpoint_us": 2279, + "cpu_seconds": 6.22999999999999, + "database_bytes": 14753792, + "failures": 0, + "latency": { + "max_us": 289201, + "p50_us": 659, + "p95_us": 5290, + "p99_us": 27657 + }, + "operations": 3200, + "operations_per_second": 3555.7652785425344, + "pool_size": 16, + "pool_timeouts": 0, + "rss_kib": 88048, + "synchronous": "NORMAL", + "transaction": { + "max_us": 10594, + "p50_us": 603, + "p95_us": 1611, + "p99_us": 3347 + }, + "wal_bytes": 19755432, + "workload": "mixed", + "write_wait": { + "max_us": 288579, + "p50_us": 78, + "p95_us": 26553, + "p99_us": 77515 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 33380, + "p50_us": 3, + "p95_us": 7913, + "p99_us": 15400 + }, + "checkout_over_100us": 1115, + "checkpoint": [ + 0, + 83, + 83 + ], + "checkpoint_us": 762, + "cpu_seconds": 3.8999999999999915, + "database_bytes": 14753792, + "failures": 0, + "latency": { + "max_us": 33695, + "p50_us": 703, + "p95_us": 8691, + "p99_us": 15910 + }, + "operations": 3200, + "operations_per_second": 3597.7499995300436, + "pool_size": 2, + "pool_timeouts": 0, + "rss_kib": 72448, + "synchronous": "FULL", + "transaction": { + "max_us": 9589, + "p50_us": 538, + "p95_us": 1038, + "p99_us": 8914 + }, + "wal_bytes": 5125312, + "workload": "read_heavy", + "write_wait": { + "max_us": 4568, + "p50_us": 3, + "p95_us": 155, + "p99_us": 4544 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 32006, + "p50_us": 2534, + "p95_us": 10466, + "p99_us": 16527 + }, + "checkout_over_100us": 1893, + "checkpoint": [ + 0, + 527, + 527 + ], + "checkpoint_us": 1795, + "cpu_seconds": 10.489999999999995, + "database_bytes": 15208448, + "failures": 0, + "latency": { + "max_us": 499878, + "p50_us": 3097, + "p95_us": 11725, + "p99_us": 19498 + }, + "operations": 3200, + "operations_per_second": 1733.722485115773, + "pool_size": 2, + "pool_timeouts": 0, + "rss_kib": 77888, + "synchronous": "FULL", + "transaction": { + "max_us": 10956, + "p50_us": 490, + "p95_us": 705, + "p99_us": 3302 + }, + "wal_bytes": 5491992, + "workload": "write_burst", + "write_wait": { + "max_us": 499283, + "p50_us": 3, + "p95_us": 5, + "p99_us": 12051 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 27070, + "p50_us": 3, + "p95_us": 9076, + "p99_us": 15224 + }, + "checkout_over_100us": 1373, + "checkpoint": [ + 0, + 869, + 869 + ], + "checkpoint_us": 2616, + "cpu_seconds": 5.959999999999994, + "database_bytes": 14753792, + "failures": 0, + "latency": { + "max_us": 27139, + "p50_us": 1535, + "p95_us": 9861, + "p99_us": 16026 + }, + "operations": 3200, + "operations_per_second": 2852.773803497367, + "pool_size": 2, + "pool_timeouts": 0, + "rss_kib": 66528, + "synchronous": "FULL", + "transaction": { + "max_us": 8888, + "p50_us": 490, + "p95_us": 832, + "p99_us": 3889 + }, + "wal_bytes": 5994632, + "workload": "mixed", + "write_wait": { + "max_us": 12075, + "p50_us": 3, + "p95_us": 1577, + "p99_us": 4566 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 15562, + "p50_us": 2, + "p95_us": 3776, + "p99_us": 7650 + }, + "checkout_over_100us": 812, + "checkpoint": [ + 0, + 26, + 26 + ], + "checkpoint_us": 471, + "cpu_seconds": 4.159999999999997, + "database_bytes": 14753792, + "failures": 0, + "latency": { + "max_us": 17207, + "p50_us": 735, + "p95_us": 4683, + "p99_us": 9534 + }, + "operations": 3200, + "operations_per_second": 5698.886427618992, + "pool_size": 4, + "pool_timeouts": 0, + "rss_kib": 83520, + "synchronous": "FULL", + "transaction": { + "max_us": 9566, + "p50_us": 685, + "p95_us": 1248, + "p99_us": 9239 + }, + "wal_bytes": 8862152, + "workload": "read_heavy", + "write_wait": { + "max_us": 4575, + "p50_us": 4, + "p95_us": 1609, + "p99_us": 4568 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 21147, + "p50_us": 4, + "p95_us": 6595, + "p99_us": 10826 + }, + "checkout_over_100us": 1490, + "checkpoint": [ + 0, + 440, + 440 + ], + "checkpoint_us": 1542, + "cpu_seconds": 10.589999999999975, + "database_bytes": 15208448, + "failures": 0, + "latency": { + "max_us": 606876, + "p50_us": 1574, + "p95_us": 9078, + "p99_us": 28731 + }, + "operations": 3200, + "operations_per_second": 1782.296684082468, + "pool_size": 4, + "pool_timeouts": 0, + "rss_kib": 96992, + "synchronous": "FULL", + "transaction": { + "max_us": 9043, + "p50_us": 494, + "p95_us": 1050, + "p99_us": 3385 + }, + "wal_bytes": 5351912, + "workload": "write_burst", + "write_wait": { + "max_us": 606290, + "p50_us": 3, + "p95_us": 1600, + "p99_us": 45795 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 17255, + "p50_us": 3, + "p95_us": 5348, + "p99_us": 8876 + }, + "checkout_over_100us": 1108, + "checkpoint": [ + 0, + 375, + 375 + ], + "checkpoint_us": 1535, + "cpu_seconds": 6.070000000000022, + "database_bytes": 14753792, + "failures": 0, + "latency": { + "max_us": 282516, + "p50_us": 866, + "p95_us": 6731, + "p99_us": 13310 + }, + "operations": 3200, + "operations_per_second": 3214.108328507986, + "pool_size": 4, + "pool_timeouts": 0, + "rss_kib": 74400, + "synchronous": "FULL", + "transaction": { + "max_us": 8723, + "p50_us": 535, + "p95_us": 1196, + "p99_us": 3523 + }, + "wal_bytes": 7659112, + "workload": "mixed", + "write_wait": { + "max_us": 282024, + "p50_us": 4, + "p95_us": 4581, + "p99_us": 27081 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 68, + "p50_us": 2, + "p95_us": 3, + "p99_us": 11 + }, + "checkout_over_100us": 0, + "checkpoint": [ + 0, + 148, + 148 + ], + "checkpoint_us": 1054, + "cpu_seconds": 5.329999999999984, + "database_bytes": 14753792, + "failures": 0, + "latency": { + "max_us": 229535, + "p50_us": 792, + "p95_us": 3365, + "p99_us": 6773 + }, + "operations": 3200, + "operations_per_second": 4853.844610790283, + "pool_size": 8, + "pool_timeouts": 0, + "rss_kib": 107232, + "synchronous": "FULL", + "transaction": { + "max_us": 11021, + "p50_us": 1489, + "p95_us": 3003, + "p99_us": 10506 + }, + "wal_bytes": 13645472, + "workload": "read_heavy", + "write_wait": { + "max_us": 228506, + "p50_us": 8, + "p95_us": 12220, + "p99_us": 43131 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 54, + "p50_us": 2, + "p95_us": 3, + "p99_us": 5 + }, + "checkout_over_100us": 0, + "checkpoint": [ + 0, + 209, + 209 + ], + "checkpoint_us": 1173, + "cpu_seconds": 10.53000000000003, + "database_bytes": 15204352, + "failures": 0, + "latency": { + "max_us": 908597, + "p50_us": 497, + "p95_us": 3803, + "p99_us": 50449 + }, + "operations": 3200, + "operations_per_second": 1877.147903757787, + "pool_size": 8, + "pool_timeouts": 0, + "rss_kib": 112528, + "synchronous": "FULL", + "transaction": { + "max_us": 9137, + "p50_us": 494, + "p95_us": 779, + "p99_us": 3436 + }, + "wal_bytes": 5277752, + "workload": "write_burst", + "write_wait": { + "max_us": 907953, + "p50_us": 3, + "p95_us": 4560, + "p99_us": 112124 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 35, + "p50_us": 3, + "p95_us": 3, + "p99_us": 5 + }, + "checkout_over_100us": 0, + "checkpoint": [ + 0, + 638, + 638 + ], + "checkpoint_us": 3806, + "cpu_seconds": 6.219999999999999, + "database_bytes": 14753792, + "failures": 0, + "latency": { + "max_us": 581880, + "p50_us": 594, + "p95_us": 4986, + "p99_us": 38429 + }, + "operations": 3200, + "operations_per_second": 3027.3250397676275, + "pool_size": 8, + "pool_timeouts": 0, + "rss_kib": 87104, + "synchronous": "FULL", + "transaction": { + "max_us": 10741, + "p50_us": 590, + "p95_us": 1481, + "p99_us": 3542 + }, + "wal_bytes": 11663752, + "workload": "mixed", + "write_wait": { + "max_us": 580893, + "p50_us": 56, + "p95_us": 24758, + "p99_us": 98586 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 49, + "p50_us": 2, + "p95_us": 3, + "p99_us": 4 + }, + "checkout_over_100us": 0, + "checkpoint": [ + 0, + 49, + 49 + ], + "checkpoint_us": 810, + "cpu_seconds": 5.72999999999999, + "database_bytes": 14753792, + "failures": 0, + "latency": { + "max_us": 87386, + "p50_us": 852, + "p95_us": 3678, + "p99_us": 7166 + }, + "operations": 3200, + "operations_per_second": 5255.176457198355, + "pool_size": 12, + "pool_timeouts": 0, + "rss_kib": 114448, + "synchronous": "FULL", + "transaction": { + "max_us": 10700, + "p50_us": 1769, + "p95_us": 3669, + "p99_us": 10254 + }, + "wal_bytes": 14053352, + "workload": "read_heavy", + "write_wait": { + "max_us": 77128, + "p50_us": 120, + "p95_us": 12100, + "p99_us": 49136 + } + }, + { + "busy_errors": 1, + "checkout": { + "max_us": 28, + "p50_us": 2, + "p95_us": 4, + "p99_us": 5 + }, + "checkout_over_100us": 0, + "checkpoint": [ + 0, + 587, + 587 + ], + "checkpoint_us": 2166, + "cpu_seconds": 10.54000000000002, + "database_bytes": 15196160, + "failures": 1, + "latency": { + "max_us": 1113361, + "p50_us": 506, + "p95_us": 3401, + "p99_us": 40005 + }, + "operations": 3200, + "operations_per_second": 1826.505396726793, + "pool_size": 12, + "pool_timeouts": 0, + "rss_kib": 98800, + "synchronous": "FULL", + "transaction": { + "max_us": 9757, + "p50_us": 500, + "p95_us": 779, + "p99_us": 3606 + }, + "wal_bytes": 5520832, + "workload": "write_burst", + "write_wait": { + "max_us": 1110626, + "p50_us": 3, + "p95_us": 4541, + "p99_us": 76621 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 20, + "p50_us": 2, + "p95_us": 3, + "p99_us": 5 + }, + "checkout_over_100us": 0, + "checkpoint": [ + 0, + 277, + 277 + ], + "checkpoint_us": 1070, + "cpu_seconds": 6.069999999999993, + "database_bytes": 14753792, + "failures": 0, + "latency": { + "max_us": 387484, + "p50_us": 535, + "p95_us": 3271, + "p99_us": 27571 + }, + "operations": 3200, + "operations_per_second": 3190.7448457318487, + "pool_size": 12, + "pool_timeouts": 0, + "rss_kib": 72784, + "synchronous": "FULL", + "transaction": { + "max_us": 8494, + "p50_us": 539, + "p95_us": 1154, + "p99_us": 3623 + }, + "wal_bytes": 8478992, + "workload": "mixed", + "write_wait": { + "max_us": 386899, + "p50_us": 42, + "p95_us": 24681, + "p99_us": 107559 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 46, + "p50_us": 2, + "p95_us": 3, + "p99_us": 12 + }, + "checkout_over_100us": 0, + "checkpoint": [ + 0, + 911, + 911 + ], + "checkpoint_us": 3452, + "cpu_seconds": 5.909999999999997, + "database_bytes": 14753792, + "failures": 0, + "latency": { + "max_us": 49369, + "p50_us": 906, + "p95_us": 4108, + "p99_us": 7264 + }, + "operations": 3200, + "operations_per_second": 5064.045008877556, + "pool_size": 16, + "pool_timeouts": 0, + "rss_kib": 110096, + "synchronous": "FULL", + "transaction": { + "max_us": 12503, + "p50_us": 1828, + "p95_us": 4211, + "p99_us": 11452 + }, + "wal_bytes": 10493672, + "workload": "read_heavy", + "write_wait": { + "max_us": 47792, + "p50_us": 98, + "p95_us": 12110, + "p99_us": 42915 + } + }, + { + "busy_errors": 1, + "checkout": { + "max_us": 13, + "p50_us": 3, + "p95_us": 4, + "p99_us": 5 + }, + "checkout_over_100us": 0, + "checkpoint": [ + 0, + 266, + 266 + ], + "checkpoint_us": 1156, + "cpu_seconds": 10.5, + "database_bytes": 15216640, + "failures": 1, + "latency": { + "max_us": 1113083, + "p50_us": 511, + "p95_us": 4963, + "p99_us": 67227 + }, + "operations": 3200, + "operations_per_second": 1826.623291769334, + "pool_size": 16, + "pool_timeouts": 0, + "rss_kib": 100256, + "synchronous": "FULL", + "transaction": { + "max_us": 9157, + "p50_us": 504, + "p95_us": 766, + "p99_us": 3680 + }, + "wal_bytes": 5141792, + "workload": "write_burst", + "write_wait": { + "max_us": 1113075, + "p50_us": 3, + "p95_us": 4572, + "p99_us": 78251 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 102, + "p50_us": 2, + "p95_us": 3, + "p99_us": 4 + }, + "checkout_over_100us": 1, + "checkpoint": [ + 0, + 385, + 385 + ], + "checkpoint_us": 1479, + "cpu_seconds": 6.150000000000006, + "database_bytes": 14753792, + "failures": 0, + "latency": { + "max_us": 224127, + "p50_us": 636, + "p95_us": 5189, + "p99_us": 44766 + }, + "operations": 3200, + "operations_per_second": 3400.7254095873004, + "pool_size": 16, + "pool_timeouts": 0, + "rss_kib": 70384, + "synchronous": "FULL", + "transaction": { + "max_us": 8665, + "p50_us": 587, + "p95_us": 1299, + "p99_us": 3311 + }, + "wal_bytes": 11816192, + "workload": "mixed", + "write_wait": { + "max_us": 223564, + "p50_us": 74, + "p95_us": 27083, + "p99_us": 106299 + } + } + ], + "query_plans": [ + { + "details": [ + "MATERIALIZE page", + "SEARCH t USING INDEX idx_threads_board_sticky_bumped (board_id=?)", + "SEARCH op EXISTS USING COVERING INDEX idx_posts_one_op_per_thread (thread_id=?)", + "SCAN t", + "SEARCH op USING INDEX idx_posts_one_op_per_thread (thread_id=?)", + "CORRELATED SCALAR SUBQUERY 3", + "SEARCH fp USING INDEX idx_posts_thread_live (thread_id=?)", + "CORRELATED SCALAR SUBQUERY 4", + "SEARCH fp USING COVERING INDEX idx_posts_thread (thread_id=?)" + ], + "name": "board_index" + }, + { + "details": [ + "SEARCH posts USING INDEX idx_posts_thread_live (thread_id=? AND id>?)" + ], + "name": "live_poll" + }, + { + "details": [ + "SEARCH posts USING COVERING INDEX idx_posts_board_ip_created (board_id=? AND ip_hash=?)" + ], + "name": "cooldown" + }, + { + "details": [ + "SEARCH threads USING INDEX idx_threads_archived (board_id=? AND archived=?)", + "USE TEMP B-TREE FOR ORDER BY" + ], + "name": "latest_thread" + }, + { + "details": [ + "SEARCH threads USING INDEX idx_threads_archived (board_id=? AND archived=?)" + ], + "name": "prune" + }, + { + "details": [ + "SCAN posts_fts VIRTUAL TABLE INDEX 0:M1", + "SEARCH p USING INTEGER PRIMARY KEY (rowid=?)", + "USE TEMP B-TREE FOR ORDER BY" + ], + "name": "search_selective" + }, + { + "details": [ + "SEARCH reports USING INDEX idx_reports_status (status=?)" + ], + "name": "reports" + }, + { + "details": [ + "SEARCH bans USING INDEX idx_bans_ip (ip_hash=?)" + ], + "name": "bans" + }, + { + "details": [ + "SEARCH background_jobs USING INDEX idx_jobs_pending (status=?)" + ], + "name": "job_claim" + }, + { + "details": [ + "MULTI-INDEX OR", + "INDEX 1", + "SEARCH posts USING INDEX idx_posts_file_path (file_path=?)", + "INDEX 2", + "SEARCH posts USING INDEX idx_posts_thumb_path (thumb_path=?)", + "INDEX 3", + "SEARCH posts USING INDEX idx_posts_audio_file_path (audio_file_path=?)" + ], + "name": "file_refs" + }, + { + "details": [ + "SEARCH file_hashes USING INDEX sqlite_autoindex_file_hashes_1 (sha256=?)" + ], + "name": "dedup" + }, + { + "details": [ + "SCAN mod_log USING INDEX idx_mod_log_created" + ], + "name": "mod_log" + } + ], + "sqlite_version": "3.51.3", + "workers": 8 +} \ No newline at end of file diff --git a/docs/sqlite-evidence/optimized.json b/docs/sqlite-evidence/optimized.json new file mode 100644 index 00000000..c1ec5be4 --- /dev/null +++ b/docs/sqlite-evidence/optimized.json @@ -0,0 +1,1453 @@ +{ + "fixture": { + "boards": 8, + "posts": 40000, + "threads": 800 + }, + "limitations": "SQL/database workload; media CPU and HTTP excluded; RSS sampled at case end; ps optional; process-kill is not power loss", + "measurements": [ + { + "busy_errors": 0, + "checkout": { + "max_us": 32531, + "p50_us": 3, + "p95_us": 7725, + "p99_us": 15506 + }, + "checkout_over_100us": 1181, + "checkpoint": [ + 0, + 89, + 89 + ], + "checkpoint_us": 1108, + "cpu_seconds": 3.9400000000000004, + "database_bytes": 14753792, + "failures": 0, + "latency": { + "max_us": 32757, + "p50_us": 744, + "p95_us": 8569, + "p99_us": 16495 + }, + "operations": 3200, + "operations_per_second": 3455.361121005532, + "pool_size": 2, + "pool_timeouts": 0, + "rss_kib": 68592, + "synchronous": "NORMAL", + "transaction": { + "max_us": 9085, + "p50_us": 556, + "p95_us": 1667, + "p99_us": 9060 + }, + "wal_bytes": 4935792, + "workload": "read_heavy", + "write_wait": { + "max_us": 12155, + "p50_us": 4, + "p95_us": 1537, + "p99_us": 1649 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 35486, + "p50_us": 2239, + "p95_us": 10715, + "p99_us": 16648 + }, + "checkout_over_100us": 1779, + "checkpoint": [ + 0, + 610, + 610 + ], + "checkpoint_us": 2445, + "cpu_seconds": 10.37, + "database_bytes": 15200256, + "failures": 0, + "latency": { + "max_us": 131181, + "p50_us": 2755, + "p95_us": 12070, + "p99_us": 22652 + }, + "operations": 3200, + "operations_per_second": 1927.907844197634, + "pool_size": 2, + "pool_timeouts": 0, + "rss_kib": 70768, + "synchronous": "NORMAL", + "transaction": { + "max_us": 10356, + "p50_us": 440, + "p95_us": 671, + "p99_us": 4254 + }, + "wal_bytes": 5393112, + "workload": "write_burst", + "write_wait": { + "max_us": 130577, + "p50_us": 4, + "p95_us": 40, + "p99_us": 12098 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 33033, + "p50_us": 3, + "p95_us": 9706, + "p99_us": 15775 + }, + "checkout_over_100us": 1335, + "checkpoint": [ + 0, + 1008, + 1008 + ], + "checkpoint_us": 8, + "cpu_seconds": 5.970000000000001, + "database_bytes": 14753792, + "failures": 0, + "latency": { + "max_us": 48149, + "p50_us": 1536, + "p95_us": 10459, + "p99_us": 17289 + }, + "operations": 3200, + "operations_per_second": 2687.5391392833735, + "pool_size": 2, + "pool_timeouts": 0, + "rss_kib": 61360, + "synchronous": "NORMAL", + "transaction": { + "max_us": 8601, + "p50_us": 476, + "p95_us": 1455, + "p99_us": 4398 + }, + "wal_bytes": 5990512, + "workload": "mixed", + "write_wait": { + "max_us": 47562, + "p50_us": 4, + "p95_us": 1593, + "p99_us": 4589 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 25528, + "p50_us": 3, + "p95_us": 3948, + "p99_us": 7223 + }, + "checkout_over_100us": 843, + "checkpoint": [ + 0, + 2202, + 2202 + ], + "checkpoint_us": 2517, + "cpu_seconds": 4.199999999999999, + "database_bytes": 14753792, + "failures": 0, + "latency": { + "max_us": 27327, + "p50_us": 752, + "p95_us": 4942, + "p99_us": 8803 + }, + "operations": 3200, + "operations_per_second": 5686.233200643719, + "pool_size": 4, + "pool_timeouts": 0, + "rss_kib": 84480, + "synchronous": "NORMAL", + "transaction": { + "max_us": 9466, + "p50_us": 680, + "p95_us": 1721, + "p99_us": 7812 + }, + "wal_bytes": 9072272, + "workload": "read_heavy", + "write_wait": { + "max_us": 12137, + "p50_us": 4, + "p95_us": 1675, + "p99_us": 12102 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 23168, + "p50_us": 715, + "p95_us": 6582, + "p99_us": 9974 + }, + "checkout_over_100us": 1662, + "checkpoint": [ + 0, + 682, + 682 + ], + "checkpoint_us": 2883, + "cpu_seconds": 10.440000000000001, + "database_bytes": 15187968, + "failures": 0, + "latency": { + "max_us": 305324, + "p50_us": 1783, + "p95_us": 8808, + "p99_us": 47643 + }, + "operations": 3200, + "operations_per_second": 1883.9719568584105, + "pool_size": 4, + "pool_timeouts": 0, + "rss_kib": 85952, + "synchronous": "NORMAL", + "transaction": { + "max_us": 9214, + "p50_us": 463, + "p95_us": 818, + "p99_us": 4203 + }, + "wal_bytes": 5104712, + "workload": "write_burst", + "write_wait": { + "max_us": 297979, + "p50_us": 3, + "p95_us": 4552, + "p99_us": 71019 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 18313, + "p50_us": 3, + "p95_us": 5226, + "p99_us": 9303 + }, + "checkout_over_100us": 1208, + "checkpoint": [ + 0, + 805, + 805 + ], + "checkpoint_us": 3163, + "cpu_seconds": 6.109999999999999, + "database_bytes": 14753792, + "failures": 0, + "latency": { + "max_us": 101874, + "p50_us": 1020, + "p95_us": 7089, + "p99_us": 14240 + }, + "operations": 3200, + "operations_per_second": 3380.9449339180474, + "pool_size": 4, + "pool_timeouts": 0, + "rss_kib": 67952, + "synchronous": "NORMAL", + "transaction": { + "max_us": 8339, + "p50_us": 522, + "p95_us": 1362, + "p99_us": 3359 + }, + "wal_bytes": 8034032, + "workload": "mixed", + "write_wait": { + "max_us": 101363, + "p50_us": 30, + "p95_us": 11134, + "p99_us": 26920 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 78, + "p50_us": 2, + "p95_us": 3, + "p99_us": 11 + }, + "checkout_over_100us": 0, + "checkpoint": [ + 0, + 3460, + 3460 + ], + "checkpoint_us": 7, + "cpu_seconds": 5.780000000000001, + "database_bytes": 14753792, + "failures": 0, + "latency": { + "max_us": 76034, + "p50_us": 859, + "p95_us": 4032, + "p99_us": 7040 + }, + "operations": 3200, + "operations_per_second": 4953.215969656797, + "pool_size": 8, + "pool_timeouts": 0, + "rss_kib": 91072, + "synchronous": "NORMAL", + "transaction": { + "max_us": 12965, + "p50_us": 1974, + "p95_us": 3619, + "p99_us": 9623 + }, + "wal_bytes": 14255232, + "workload": "read_heavy", + "write_wait": { + "max_us": 73519, + "p50_us": 114, + "p95_us": 12218, + "p99_us": 49796 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 14, + "p50_us": 2, + "p95_us": 3, + "p99_us": 5 + }, + "checkout_over_100us": 0, + "checkpoint": [ + 0, + 701, + 701 + ], + "checkpoint_us": 3180, + "cpu_seconds": 10.46, + "database_bytes": 15208448, + "failures": 0, + "latency": { + "max_us": 1044923, + "p50_us": 475, + "p95_us": 4846, + "p99_us": 77271 + }, + "operations": 3200, + "operations_per_second": 1863.0262813309948, + "pool_size": 8, + "pool_timeouts": 0, + "rss_kib": 115584, + "synchronous": "NORMAL", + "transaction": { + "max_us": 10351, + "p50_us": 471, + "p95_us": 823, + "p99_us": 4656 + }, + "wal_bytes": 5479632, + "workload": "write_burst", + "write_wait": { + "max_us": 1044004, + "p50_us": 3, + "p95_us": 4597, + "p99_us": 105968 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 82, + "p50_us": 2, + "p95_us": 3, + "p99_us": 5 + }, + "checkout_over_100us": 0, + "checkpoint": [ + 0, + 181, + 181 + ], + "checkpoint_us": 1217, + "cpu_seconds": 6.130000000000003, + "database_bytes": 14753792, + "failures": 0, + "latency": { + "max_us": 406916, + "p50_us": 579, + "p95_us": 4601, + "p99_us": 41080 + }, + "operations": 3200, + "operations_per_second": 3246.995249367926, + "pool_size": 8, + "pool_timeouts": 0, + "rss_kib": 75168, + "synchronous": "NORMAL", + "transaction": { + "max_us": 10342, + "p50_us": 589, + "p95_us": 1570, + "p99_us": 3831 + }, + "wal_bytes": 14045112, + "workload": "mixed", + "write_wait": { + "max_us": 406281, + "p50_us": 40, + "p95_us": 25595, + "p99_us": 103449 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 90, + "p50_us": 2, + "p95_us": 4, + "p99_us": 14 + }, + "checkout_over_100us": 0, + "checkpoint": [ + 0, + 927, + 927 + ], + "checkpoint_us": 5164, + "cpu_seconds": 6.030000000000001, + "database_bytes": 14753792, + "failures": 0, + "latency": { + "max_us": 115074, + "p50_us": 888, + "p95_us": 4224, + "p99_us": 7124 + }, + "operations": 3200, + "operations_per_second": 4681.820035592806, + "pool_size": 12, + "pool_timeouts": 0, + "rss_kib": 96848, + "synchronous": "NORMAL", + "transaction": { + "max_us": 13284, + "p50_us": 1993, + "p95_us": 4518, + "p99_us": 10837 + }, + "wal_bytes": 10419512, + "workload": "read_heavy", + "write_wait": { + "max_us": 112876, + "p50_us": 80, + "p95_us": 12069, + "p99_us": 48656 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 155, + "p50_us": 3, + "p95_us": 8, + "p99_us": 31 + }, + "checkout_over_100us": 2, + "checkpoint": [ + 0, + 938, + 938 + ], + "checkpoint_us": 3634, + "cpu_seconds": 11.440000000000012, + "database_bytes": 15196160, + "failures": 0, + "latency": { + "max_us": 796608, + "p50_us": 589, + "p95_us": 6630, + "p99_us": 123034 + }, + "operations": 3200, + "operations_per_second": 1405.7358304985698, + "pool_size": 12, + "pool_timeouts": 0, + "rss_kib": 87920, + "synchronous": "NORMAL", + "transaction": { + "max_us": 17679, + "p50_us": 575, + "p95_us": 2009, + "p99_us": 5203 + }, + "wal_bytes": 5590872, + "workload": "write_burst", + "write_wait": { + "max_us": 795282, + "p50_us": 3, + "p95_us": 11982, + "p99_us": 148757 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 55, + "p50_us": 3, + "p95_us": 4, + "p99_us": 6 + }, + "checkout_over_100us": 0, + "checkpoint": [ + 0, + 279, + 279 + ], + "checkpoint_us": 1629, + "cpu_seconds": 6.310000000000002, + "database_bytes": 14753792, + "failures": 0, + "latency": { + "max_us": 476162, + "p50_us": 630, + "p95_us": 4228, + "p99_us": 24757 + }, + "operations": 3200, + "operations_per_second": 3004.391693969342, + "pool_size": 12, + "pool_timeouts": 0, + "rss_kib": 70096, + "synchronous": "NORMAL", + "transaction": { + "max_us": 10130, + "p50_us": 630, + "p95_us": 1840, + "p99_us": 4752 + }, + "wal_bytes": 12945072, + "workload": "mixed", + "write_wait": { + "max_us": 475350, + "p50_us": 54, + "p95_us": 21012, + "p99_us": 102674 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 81, + "p50_us": 2, + "p95_us": 4, + "p99_us": 17 + }, + "checkout_over_100us": 0, + "checkpoint": [ + 0, + 1455, + 1455 + ], + "checkpoint_us": 9, + "cpu_seconds": 6.0800000000000125, + "database_bytes": 14753792, + "failures": 0, + "latency": { + "max_us": 148422, + "p50_us": 905, + "p95_us": 4668, + "p99_us": 7439 + }, + "operations": 3200, + "operations_per_second": 4310.519013429961, + "pool_size": 16, + "pool_timeouts": 0, + "rss_kib": 73696, + "synchronous": "NORMAL", + "transaction": { + "max_us": 12534, + "p50_us": 2072, + "p95_us": 4128, + "p99_us": 7628 + }, + "wal_bytes": 8260632, + "workload": "read_heavy", + "write_wait": { + "max_us": 146004, + "p50_us": 145, + "p95_us": 12194, + "p99_us": 130275 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 52, + "p50_us": 2, + "p95_us": 3, + "p99_us": 5 + }, + "checkout_over_100us": 0, + "checkpoint": [ + 0, + 42, + 42 + ], + "checkpoint_us": 568, + "cpu_seconds": 10.460000000000008, + "database_bytes": 15192064, + "failures": 0, + "latency": { + "max_us": 1026774, + "p50_us": 455, + "p95_us": 5030, + "p99_us": 53078 + }, + "operations": 3200, + "operations_per_second": 1839.1869085259677, + "pool_size": 16, + "pool_timeouts": 0, + "rss_kib": 102608, + "synchronous": "NORMAL", + "transaction": { + "max_us": 11093, + "p50_us": 450, + "p95_us": 816, + "p99_us": 4074 + }, + "wal_bytes": 5117072, + "workload": "write_burst", + "write_wait": { + "max_us": 1025964, + "p50_us": 3, + "p95_us": 4594, + "p99_us": 101544 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 43, + "p50_us": 2, + "p95_us": 3, + "p99_us": 4 + }, + "checkout_over_100us": 0, + "checkpoint": [ + 0, + 388, + 388 + ], + "checkpoint_us": 2018, + "cpu_seconds": 6.170000000000002, + "database_bytes": 14753792, + "failures": 0, + "latency": { + "max_us": 382063, + "p50_us": 635, + "p95_us": 5158, + "p99_us": 27214 + }, + "operations": 3200, + "operations_per_second": 3380.292254744385, + "pool_size": 16, + "pool_timeouts": 0, + "rss_kib": 76272, + "synchronous": "NORMAL", + "transaction": { + "max_us": 12267, + "p50_us": 609, + "p95_us": 1704, + "p99_us": 3293 + }, + "wal_bytes": 18152752, + "workload": "mixed", + "write_wait": { + "max_us": 381350, + "p50_us": 62, + "p95_us": 24983, + "p99_us": 95121 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 35880, + "p50_us": 2, + "p95_us": 7914, + "p99_us": 14466 + }, + "checkout_over_100us": 1100, + "checkpoint": [ + 0, + 69, + 69 + ], + "checkpoint_us": 752, + "cpu_seconds": 3.8400000000000034, + "database_bytes": 14753792, + "failures": 0, + "latency": { + "max_us": 35949, + "p50_us": 689, + "p95_us": 8585, + "p99_us": 15358 + }, + "operations": 3200, + "operations_per_second": 3677.993224187099, + "pool_size": 2, + "pool_timeouts": 0, + "rss_kib": 56640, + "synchronous": "FULL", + "transaction": { + "max_us": 8575, + "p50_us": 531, + "p95_us": 1095, + "p99_us": 5046 + }, + "wal_bytes": 5356032, + "workload": "read_heavy", + "write_wait": { + "max_us": 12093, + "p50_us": 3, + "p95_us": 1328, + "p99_us": 1595 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 29494, + "p50_us": 2534, + "p95_us": 10415, + "p99_us": 15716 + }, + "checkout_over_100us": 1904, + "checkpoint": [ + 0, + 257, + 257 + ], + "checkpoint_us": 1173, + "cpu_seconds": 10.560000000000002, + "database_bytes": 15200256, + "failures": 0, + "latency": { + "max_us": 179932, + "p50_us": 3090, + "p95_us": 11987, + "p99_us": 19831 + }, + "operations": 3200, + "operations_per_second": 1809.933679940648, + "pool_size": 2, + "pool_timeouts": 0, + "rss_kib": 51856, + "synchronous": "FULL", + "transaction": { + "max_us": 8397, + "p50_us": 491, + "p95_us": 717, + "p99_us": 3367 + }, + "wal_bytes": 4659752, + "workload": "write_burst", + "write_wait": { + "max_us": 179182, + "p50_us": 3, + "p95_us": 63, + "p99_us": 12110 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 31358, + "p50_us": 3, + "p95_us": 10103, + "p99_us": 16817 + }, + "checkout_over_100us": 1392, + "checkpoint": [ + 0, + 830, + 830 + ], + "checkpoint_us": 3094, + "cpu_seconds": 6.019999999999996, + "database_bytes": 14753792, + "failures": 0, + "latency": { + "max_us": 31996, + "p50_us": 1591, + "p95_us": 11377, + "p99_us": 18086 + }, + "operations": 3200, + "operations_per_second": 2633.843580662017, + "pool_size": 2, + "pool_timeouts": 0, + "rss_kib": 43408, + "synchronous": "FULL", + "transaction": { + "max_us": 8645, + "p50_us": 513, + "p95_us": 1259, + "p99_us": 3770 + }, + "wal_bytes": 5673272, + "workload": "mixed", + "write_wait": { + "max_us": 26704, + "p50_us": 3, + "p95_us": 1619, + "p99_us": 9685 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 28274, + "p50_us": 3, + "p95_us": 3801, + "p99_us": 7475 + }, + "checkout_over_100us": 838, + "checkpoint": [ + 0, + 1644, + 1644 + ], + "checkpoint_us": 2432, + "cpu_seconds": 4.199999999999989, + "database_bytes": 14753792, + "failures": 0, + "latency": { + "max_us": 28777, + "p50_us": 742, + "p95_us": 4853, + "p99_us": 8968 + }, + "operations": 3200, + "operations_per_second": 5656.393956441432, + "pool_size": 4, + "pool_timeouts": 0, + "rss_kib": 57168, + "synchronous": "FULL", + "transaction": { + "max_us": 8960, + "p50_us": 700, + "p95_us": 1690, + "p99_us": 8485 + }, + "wal_bytes": 7481952, + "workload": "read_heavy", + "write_wait": { + "max_us": 12170, + "p50_us": 3, + "p95_us": 1684, + "p99_us": 12072 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 21491, + "p50_us": 1613, + "p95_us": 6846, + "p99_us": 10549 + }, + "checkout_over_100us": 1869, + "checkpoint": [ + 0, + 647, + 647 + ], + "checkpoint_us": 1760, + "cpu_seconds": 10.539999999999992, + "database_bytes": 15200256, + "failures": 0, + "latency": { + "max_us": 401966, + "p50_us": 2200, + "p95_us": 9610, + "p99_us": 45851 + }, + "operations": 3200, + "operations_per_second": 1730.7645248371298, + "pool_size": 4, + "pool_timeouts": 0, + "rss_kib": 68528, + "synchronous": "FULL", + "transaction": { + "max_us": 8953, + "p50_us": 506, + "p95_us": 750, + "p99_us": 3550 + }, + "wal_bytes": 4882232, + "workload": "write_burst", + "write_wait": { + "max_us": 397701, + "p50_us": 3, + "p95_us": 3728, + "p99_us": 62936 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 18625, + "p50_us": 3, + "p95_us": 5439, + "p99_us": 9668 + }, + "checkout_over_100us": 1194, + "checkpoint": [ + 0, + 584, + 584 + ], + "checkpoint_us": 1844, + "cpu_seconds": 7.3300000000000125, + "database_bytes": 14753792, + "failures": 0, + "latency": { + "max_us": 151279, + "p50_us": 968, + "p95_us": 7566, + "p99_us": 17594 + }, + "operations": 3200, + "operations_per_second": 3018.384199984932, + "pool_size": 4, + "pool_timeouts": 0, + "rss_kib": 51344, + "synchronous": "FULL", + "transaction": { + "max_us": 10402, + "p50_us": 550, + "p95_us": 1163, + "p99_us": 3635 + }, + "wal_bytes": 7292432, + "workload": "mixed", + "write_wait": { + "max_us": 143932, + "p50_us": 5, + "p95_us": 12047, + "p99_us": 43980 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 4301, + "p50_us": 4, + "p95_us": 16, + "p99_us": 34 + }, + "checkout_over_100us": 12, + "checkpoint": [ + 0, + 3460, + 3460 + ], + "checkpoint_us": 35, + "cpu_seconds": 6.219999999999999, + "database_bytes": 14753792, + "failures": 0, + "latency": { + "max_us": 131627, + "p50_us": 1196, + "p95_us": 11074, + "p99_us": 30020 + }, + "operations": 3200, + "operations_per_second": 2495.9066156539757, + "pool_size": 8, + "pool_timeouts": 0, + "rss_kib": 83632, + "synchronous": "FULL", + "transaction": { + "max_us": 30673, + "p50_us": 2454, + "p95_us": 12382, + "p99_us": 23512 + }, + "wal_bytes": 14255232, + "workload": "read_heavy", + "write_wait": { + "max_us": 129314, + "p50_us": 57, + "p95_us": 40725, + "p99_us": 74943 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 59, + "p50_us": 3, + "p95_us": 4, + "p99_us": 9 + }, + "checkout_over_100us": 0, + "checkpoint": [ + 0, + 707, + 707 + ], + "checkpoint_us": 2099, + "cpu_seconds": 10.610000000000014, + "database_bytes": 15204352, + "failures": 0, + "latency": { + "max_us": 716857, + "p50_us": 529, + "p95_us": 5127, + "p99_us": 80081 + }, + "operations": 3200, + "operations_per_second": 1724.5564872123475, + "pool_size": 8, + "pool_timeouts": 0, + "rss_kib": 72784, + "synchronous": "FULL", + "transaction": { + "max_us": 11383, + "p50_us": 524, + "p95_us": 1121, + "p99_us": 3818 + }, + "wal_bytes": 5780392, + "workload": "write_burst", + "write_wait": { + "max_us": 716266, + "p50_us": 3, + "p95_us": 4628, + "p99_us": 130775 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 36, + "p50_us": 2, + "p95_us": 3, + "p99_us": 5 + }, + "checkout_over_100us": 0, + "checkpoint": [ + 0, + 820, + 820 + ], + "checkpoint_us": 2554, + "cpu_seconds": 6.140000000000015, + "database_bytes": 14753792, + "failures": 0, + "latency": { + "max_us": 385533, + "p50_us": 567, + "p95_us": 5103, + "p99_us": 27757 + }, + "operations": 3200, + "operations_per_second": 3163.647056835317, + "pool_size": 8, + "pool_timeouts": 0, + "rss_kib": 61024, + "synchronous": "FULL", + "transaction": { + "max_us": 8641, + "p50_us": 555, + "p95_us": 1161, + "p99_us": 3377 + }, + "wal_bytes": 8198832, + "workload": "mixed", + "write_wait": { + "max_us": 384641, + "p50_us": 57, + "p95_us": 26188, + "p99_us": 113147 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 68, + "p50_us": 2, + "p95_us": 3, + "p99_us": 14 + }, + "checkout_over_100us": 0, + "checkpoint": [ + 0, + 2267, + 2267 + ], + "checkpoint_us": 1337, + "cpu_seconds": 5.789999999999992, + "database_bytes": 14753792, + "failures": 0, + "latency": { + "max_us": 102243, + "p50_us": 876, + "p95_us": 4018, + "p99_us": 7157 + }, + "operations": 3200, + "operations_per_second": 5091.736455266197, + "pool_size": 12, + "pool_timeouts": 0, + "rss_kib": 68896, + "synchronous": "FULL", + "transaction": { + "max_us": 13715, + "p50_us": 1832, + "p95_us": 3679, + "p99_us": 7495 + }, + "wal_bytes": 9340072, + "workload": "read_heavy", + "write_wait": { + "max_us": 100333, + "p50_us": 85, + "p95_us": 12125, + "p99_us": 48728 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 16, + "p50_us": 3, + "p95_us": 3, + "p99_us": 4 + }, + "checkout_over_100us": 0, + "checkpoint": [ + 0, + 615, + 615 + ], + "checkpoint_us": 1831, + "cpu_seconds": 10.400000000000006, + "database_bytes": 15208448, + "failures": 0, + "latency": { + "max_us": 828124, + "p50_us": 496, + "p95_us": 3737, + "p99_us": 50032 + }, + "operations": 3200, + "operations_per_second": 1816.1688117559472, + "pool_size": 12, + "pool_timeouts": 0, + "rss_kib": 102608, + "synchronous": "FULL", + "transaction": { + "max_us": 8846, + "p50_us": 492, + "p95_us": 725, + "p99_us": 3348 + }, + "wal_bytes": 4890472, + "workload": "write_burst", + "write_wait": { + "max_us": 827564, + "p50_us": 3, + "p95_us": 4582, + "p99_us": 104970 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 58, + "p50_us": 3, + "p95_us": 4, + "p99_us": 5 + }, + "checkout_over_100us": 0, + "checkpoint": [ + 0, + 276, + 276 + ], + "checkpoint_us": 1768, + "cpu_seconds": 6.139999999999986, + "database_bytes": 14753792, + "failures": 0, + "latency": { + "max_us": 504422, + "p50_us": 589, + "p95_us": 5059, + "p99_us": 24492 + }, + "operations": 3200, + "operations_per_second": 3029.9143663161485, + "pool_size": 12, + "pool_timeouts": 0, + "rss_kib": 63136, + "synchronous": "FULL", + "transaction": { + "max_us": 8743, + "p50_us": 579, + "p95_us": 1250, + "p99_us": 3564 + }, + "wal_bytes": 10295912, + "workload": "mixed", + "write_wait": { + "max_us": 503871, + "p50_us": 59, + "p95_us": 12139, + "p99_us": 142615 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 49, + "p50_us": 2, + "p95_us": 3, + "p99_us": 12 + }, + "checkout_over_100us": 0, + "checkpoint": [ + 0, + 48, + 48 + ], + "checkpoint_us": 688, + "cpu_seconds": 5.6200000000000045, + "database_bytes": 14753792, + "failures": 0, + "latency": { + "max_us": 81503, + "p50_us": 811, + "p95_us": 3936, + "p99_us": 7202 + }, + "operations": 3200, + "operations_per_second": 5106.928982775856, + "pool_size": 16, + "pool_timeouts": 0, + "rss_kib": 102544, + "synchronous": "FULL", + "transaction": { + "max_us": 13323, + "p50_us": 1735, + "p95_us": 4812, + "p99_us": 9702 + }, + "wal_bytes": 14057472, + "workload": "read_heavy", + "write_wait": { + "max_us": 79225, + "p50_us": 85, + "p95_us": 23820, + "p99_us": 49968 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 30, + "p50_us": 3, + "p95_us": 4, + "p99_us": 4 + }, + "checkout_over_100us": 0, + "checkpoint": [ + 0, + 69, + 69 + ], + "checkpoint_us": 516, + "cpu_seconds": 10.469999999999999, + "database_bytes": 15192064, + "failures": 0, + "latency": { + "max_us": 1030743, + "p50_us": 508, + "p95_us": 5066, + "p99_us": 75838 + }, + "operations": 3200, + "operations_per_second": 1835.341084160354, + "pool_size": 16, + "pool_timeouts": 0, + "rss_kib": 97504, + "synchronous": "FULL", + "transaction": { + "max_us": 10341, + "p50_us": 502, + "p95_us": 771, + "p99_us": 3460 + }, + "wal_bytes": 5471392, + "workload": "write_burst", + "write_wait": { + "max_us": 1029972, + "p50_us": 3, + "p95_us": 4595, + "p99_us": 102290 + } + }, + { + "busy_errors": 0, + "checkout": { + "max_us": 37, + "p50_us": 2, + "p95_us": 3, + "p99_us": 4 + }, + "checkout_over_100us": 0, + "checkpoint": [ + 0, + 874, + 874 + ], + "checkpoint_us": 2532, + "cpu_seconds": 6.060000000000002, + "database_bytes": 14753792, + "failures": 0, + "latency": { + "max_us": 718727, + "p50_us": 556, + "p95_us": 5026, + "p99_us": 26297 + }, + "operations": 3200, + "operations_per_second": 3170.9820766540442, + "pool_size": 16, + "pool_timeouts": 0, + "rss_kib": 68112, + "synchronous": "FULL", + "transaction": { + "max_us": 8783, + "p50_us": 556, + "p95_us": 1162, + "p99_us": 3652 + }, + "wal_bytes": 9859192, + "workload": "mixed", + "write_wait": { + "max_us": 718130, + "p50_us": 44, + "p95_us": 12928, + "p99_us": 103761 + } + } + ], + "query_plans": [ + { + "details": [ + "MATERIALIZE page", + "SEARCH t USING INDEX idx_threads_board_sticky_bumped (board_id=?)", + "SEARCH op EXISTS USING COVERING INDEX idx_posts_one_op_per_thread (thread_id=?)", + "SCAN t", + "SEARCH op USING INDEX idx_posts_one_op_per_thread (thread_id=?)", + "CORRELATED SCALAR SUBQUERY 3", + "SEARCH fp USING INDEX idx_posts_thread_live (thread_id=?)", + "CORRELATED SCALAR SUBQUERY 4", + "SEARCH fp USING COVERING INDEX idx_posts_thread (thread_id=?)" + ], + "name": "board_index" + }, + { + "details": [ + "SEARCH posts USING INDEX idx_posts_thread_live (thread_id=? AND id>?)" + ], + "name": "live_poll" + }, + { + "details": [ + "SEARCH posts USING COVERING INDEX idx_posts_board_ip_created (board_id=? AND ip_hash=?)" + ], + "name": "cooldown" + }, + { + "details": [ + "SEARCH threads USING INDEX idx_threads_archived (board_id=? AND archived=?)", + "USE TEMP B-TREE FOR ORDER BY" + ], + "name": "latest_thread" + }, + { + "details": [ + "SEARCH threads USING INDEX idx_threads_archived (board_id=? AND archived=?)" + ], + "name": "prune" + }, + { + "details": [ + "SCAN posts_fts VIRTUAL TABLE INDEX 0:M1", + "SEARCH p USING INTEGER PRIMARY KEY (rowid=?)", + "USE TEMP B-TREE FOR ORDER BY" + ], + "name": "search_selective" + }, + { + "details": [ + "SEARCH reports USING INDEX idx_reports_status (status=?)" + ], + "name": "reports" + }, + { + "details": [ + "SEARCH bans USING INDEX idx_bans_ip (ip_hash=?)" + ], + "name": "bans" + }, + { + "details": [ + "SEARCH background_jobs USING INDEX idx_jobs_pending (status=?)" + ], + "name": "job_claim" + }, + { + "details": [ + "MULTI-INDEX OR", + "INDEX 1", + "SEARCH posts USING INDEX idx_posts_file_path (file_path=?)", + "INDEX 2", + "SEARCH posts USING INDEX idx_posts_thumb_path (thumb_path=?)", + "INDEX 3", + "SEARCH posts USING INDEX idx_posts_audio_file_path (audio_file_path=?)" + ], + "name": "file_refs" + }, + { + "details": [ + "SEARCH file_hashes USING INDEX sqlite_autoindex_file_hashes_1 (sha256=?)" + ], + "name": "dedup" + }, + { + "details": [ + "SCAN mod_log USING INDEX idx_mod_log_created" + ], + "name": "mod_log" + } + ], + "sqlite_version": "3.51.3", + "workers": 8 +} \ No newline at end of file diff --git a/docs/sqlite-evidence/query-profile.json b/docs/sqlite-evidence/query-profile.json new file mode 100644 index 00000000..ecd9f063 --- /dev/null +++ b/docs/sqlite-evidence/query-profile.json @@ -0,0 +1,286 @@ +{ + "database_bytes_after": 13512704, + "database_bytes_before": 12210176, + "queries": [ + { + "after": { + "max_us": 640, + "p50_us": 287, + "p95_us": 341, + "p99_us": 640 + }, + "after_plan": [ + "SCAN posts_fts VIRTUAL TABLE INDEX 0:M1", + "SEARCH p USING INTEGER PRIMARY KEY (rowid=?)", + "USE TEMP B-TREE FOR ORDER BY" + ], + "before": { + "max_us": 74905, + "p50_us": 74295, + "p95_us": 74806, + "p99_us": 74905 + }, + "before_plan": [ + "SEARCH p USING COVERING INDEX idx_posts_board (board_id=?)", + "SCAN f VIRTUAL TABLE INDEX 0:=M1", + "USE TEMP B-TREE FOR LAST TERM OF ORDER BY" + ], + "query": "fts_p.id_needle" + }, + { + "after": { + "max_us": 409, + "p50_us": 258, + "p95_us": 323, + "p99_us": 409 + }, + "after_plan": [ + "SCAN posts_fts VIRTUAL TABLE INDEX 0:M1", + "SEARCH p USING INTEGER PRIMARY KEY (rowid=?)" + ], + "before": { + "max_us": 159648, + "p50_us": 157500, + "p95_us": 158784, + "p99_us": 159648 + }, + "before_plan": [ + "SEARCH p USING COVERING INDEX idx_posts_board (board_id=?)", + "SCAN f VIRTUAL TABLE INDEX 0:=M1" + ], + "query": "fts_COUNT(*)_needle" + }, + { + "after": { + "max_us": 11181, + "p50_us": 10528, + "p95_us": 10724, + "p99_us": 11181 + }, + "after_plan": [ + "SCAN posts_fts VIRTUAL TABLE INDEX 0:M1", + "SEARCH p USING INTEGER PRIMARY KEY (rowid=?)", + "USE TEMP B-TREE FOR ORDER BY" + ], + "before": { + "max_us": 57525, + "p50_us": 56581, + "p95_us": 57194, + "p99_us": 57525 + }, + "before_plan": [ + "SEARCH p USING COVERING INDEX idx_posts_board (board_id=?)", + "SCAN f VIRTUAL TABLE INDEX 0:=M1", + "USE TEMP B-TREE FOR LAST TERM OF ORDER BY" + ], + "query": "fts_p.id_representative" + }, + { + "after": { + "max_us": 9061, + "p50_us": 8623, + "p95_us": 8952, + "p99_us": 9061 + }, + "after_plan": [ + "SCAN posts_fts VIRTUAL TABLE INDEX 0:M1", + "SEARCH p USING INTEGER PRIMARY KEY (rowid=?)" + ], + "before": { + "max_us": 19234274, + "p50_us": 11241770, + "p95_us": 11777815, + "p99_us": 19234274 + }, + "before_plan": [ + "SEARCH p USING COVERING INDEX idx_posts_board (board_id=?)", + "SCAN f VIRTUAL TABLE INDEX 0:=M1" + ], + "query": "fts_COUNT(*)_representative" + }, + { + "after": { + "max_us": 22, + "p50_us": 12, + "p95_us": 16, + "p99_us": 22 + }, + "after_plan": [ + "SCAN posts_fts VIRTUAL TABLE INDEX 0:M1", + "SEARCH p USING INTEGER PRIMARY KEY (rowid=?)", + "USE TEMP B-TREE FOR ORDER BY" + ], + "before": { + "max_us": 39554, + "p50_us": 38995, + "p95_us": 39379, + "p99_us": 39554 + }, + "before_plan": [ + "SEARCH p USING COVERING INDEX idx_posts_board (board_id=?)", + "SCAN f VIRTUAL TABLE INDEX 0:=M1", + "USE TEMP B-TREE FOR LAST TERM OF ORDER BY" + ], + "query": "fts_p.id_missingword" + }, + { + "after": { + "max_us": 16, + "p50_us": 10, + "p95_us": 11, + "p99_us": 16 + }, + "after_plan": [ + "SCAN posts_fts VIRTUAL TABLE INDEX 0:M1", + "SEARCH p USING INTEGER PRIMARY KEY (rowid=?)" + ], + "before": { + "max_us": 39571, + "p50_us": 38762, + "p95_us": 39389, + "p99_us": 39571 + }, + "before_plan": [ + "SEARCH p USING COVERING INDEX idx_posts_board (board_id=?)", + "SCAN f VIRTUAL TABLE INDEX 0:=M1" + ], + "query": "fts_COUNT(*)_missingword" + }, + { + "after": { + "max_us": 1138, + "p50_us": 434, + "p95_us": 524, + "p99_us": 1138 + }, + "before": { + "max_us": 119853, + "p50_us": 74174, + "p95_us": 75448, + "p99_us": 119853 + }, + "query": "production_fts_needle" + }, + { + "after": { + "max_us": 13872, + "p50_us": 13300, + "p95_us": 13671, + "p99_us": 13872 + }, + "before": { + "max_us": 57792, + "p50_us": 56798, + "p95_us": 57438, + "p99_us": 57792 + }, + "query": "production_fts_representative" + }, + { + "after": { + "max_us": 169, + "p50_us": 76, + "p95_us": 91, + "p99_us": 169 + }, + "before": { + "max_us": 39664, + "p50_us": 38956, + "p95_us": 39613, + "p99_us": 39664 + }, + "query": "production_fts_missingword" + }, + { + "after": { + "max_us": 282, + "p50_us": 144, + "p95_us": 156, + "p99_us": 282 + }, + "after_plan": [ + "MATERIALIZE page", + "SEARCH t USING INDEX idx_threads_board_sticky_bumped (board_id=?)", + "SEARCH op EXISTS USING COVERING INDEX idx_posts_one_op_per_thread (thread_id=?)", + "USE TEMP B-TREE FOR LAST TERM OF ORDER BY", + "SCAN t", + "SEARCH op USING INDEX idx_posts_one_op_per_thread (thread_id=?)", + "CORRELATED SCALAR SUBQUERY 3", + "SEARCH fp USING INDEX idx_posts_thread_live (thread_id=?)", + "CORRELATED SCALAR SUBQUERY 4", + "SEARCH fp USING COVERING INDEX idx_posts_thread (thread_id=?)" + ], + "before": { + "max_us": 6398, + "p50_us": 5372, + "p95_us": 5538, + "p99_us": 6398 + }, + "before_plan": [ + "SEARCH t USING INDEX idx_threads_archived (board_id=? AND archived=?)", + "SEARCH op USING INDEX idx_posts_one_op_per_thread (thread_id=?)", + "SEARCH fp USING INDEX idx_posts_thread_live (thread_id=?) LEFT-JOIN", + "USE TEMP B-TREE FOR GROUP BY", + "USE TEMP B-TREE FOR count(DISTINCT)", + "USE TEMP B-TREE FOR ORDER BY" + ], + "query": "board_page" + }, + { + "after": { + "max_us": 46, + "p50_us": 5, + "p95_us": 6, + "p99_us": 46 + }, + "after_plan": [ + "SEARCH posts USING INDEX idx_posts_thread_live (thread_id=? AND id>?)" + ], + "before": { + "max_us": 2945, + "p50_us": 2749, + "p95_us": 2847, + "p99_us": 2945 + }, + "before_plan": [ + "SEARCH posts USING INDEX idx_posts_board (board_id=?)", + "USE TEMP B-TREE FOR ORDER BY" + ], + "query": "live_poll" + }, + { + "after": { + "max_us": 24, + "p50_us": 2, + "p95_us": 3, + "p99_us": 24 + }, + "after_plan": [ + "SEARCH posts USING COVERING INDEX idx_posts_board_ip_created (board_id=? AND ip_hash=?)" + ], + "before": { + "max_us": 170, + "p50_us": 154, + "p95_us": 167, + "p99_us": 170 + }, + "before_plan": [ + "SEARCH posts USING INDEX idx_posts_board (board_id=?)" + ], + "query": "cooldown" + } + ], + "sqlite": "3.51.3", + "write_after": { + "max_us": 2480, + "p50_us": 404, + "p95_us": 543, + "p99_us": 1362 + }, + "write_before": { + "max_us": 7782, + "p50_us": 384, + "p95_us": 510, + "p99_us": 1568 + } +} \ No newline at end of file diff --git a/docs/strict-lint-exceptions.tsv b/docs/strict-lint-exceptions.tsv index 786f70ae..aa34f802 100644 --- a/docs/strict-lint-exceptions.tsv +++ b/docs/strict-lint-exceptions.tsv @@ -1,6 +1,6 @@ path line attribute lints reason scope src/banner.rs 1125 expect clippy::panic_in_result_fn test assertions intentionally panic on failure fn rejects_banner_gif_frame_bomb -src/banner.rs 1148 expect clippy::panic_in_result_fn test assertions intentionally panic on failure fn animated_gif_banner_falls_back_to_gif_without_ffmpeg +src/banner.rs 1148 expect clippy::panic_in_result_fn test assertions intentionally panic on failure fn animated_gif_banner_converts_to_animated_webp_without_ffmpeg src/banner.rs 1196 expect clippy::panic_in_result_fn test assertions intentionally panic on failure fn deleting_banner_asset_removes_webp_and_gif_siblings src/banner.rs 1235 expect clippy::panic_in_result_fn test assertions intentionally panic on failure fn animated_webp_banner_is_preserved_without_static_decode src/banner.rs 1264 expect clippy::panic_in_result_fn test assertions intentionally panic on failure fn rejects_oversized_banner_dimensions @@ -149,7 +149,7 @@ src/handlers/admin/settings/themes.rs 252 expect clippy::too_many_lines builder src/handlers/board/access_preferences.rs 959 expect clippy::too_many_lines rate limiting, credential checks, signed access grants, and redirect validation form one request fn unlock_board_access src/handlers/board/catalog.rs 26 expect clippy::too_many_lines access control, catalog loading, unread-state calculation, and rendering form one request fn catalog src/handlers/board/create_thread.rs 15 expect clippy::too_many_lines access checks, multipart validation, thread creation, and response cookies form one request fn create_thread -src/handlers/board/media.rs 91 expect clippy::too_many_lines path validation, access policy, legacy redirect handling, and file response form one request fn serve_board_media +src/handlers/board/media.rs 103 expect clippy::too_many_lines path validation, access policy, legacy redirect handling, and file response form one request fn serve_board_media src/handlers/board/pages.rs 37 expect clippy::too_many_lines home-page data loading, activity markers, banner selection, and rendering form one request fn index src/handlers/board/pages.rs 226 expect clippy::too_many_lines board access, pagination, activity markers, banner selection, and rendering form one request fn board_index src/handlers/board/tests.rs 432 expect clippy::too_many_lines the end-to-end test keeps its fixture setup and ordered assertions in one scenario fn post_password_board_write_actions_require_unlock @@ -189,11 +189,8 @@ src/media/prune.rs 524 expect clippy::panic_in_result_fn test assertions intenti src/media/prune.rs 595 expect clippy::panic_in_result_fn test assertions intentionally panic on failure fn pruning_removes_secondary_audio_original_with_combo_post src/media/prune.rs 657 expect clippy::panic_in_result_fn test assertions intentionally panic on failure fn pruning_refuses_symlink_media_paths src/media/prune.rs 711 expect clippy::panic_in_result_fn test assertions intentionally panic on failure fn pruning_refuses_symlink_parent_components -src/media/thumbnail.rs 383 expect clippy::cognitive_complexity the renderer fallback and fail-closed validation sequence is intentionally linear fn pdf_first_page_thumbnail -src/media/thumbnail.rs 387 expect clippy::too_many_lines PDF rendering keeps subprocess validation, fallback, and cleanup in one auditable sequence fn pdf_first_page_thumbnail -src/media/thumbnail.rs 830 expect clippy::panic_in_result_fn test assertions intentionally panic on failure fn write_pdf_placeholder_outputs_svg -src/media/thumbnail.rs 845 expect clippy::panic_in_result_fn test assertions intentionally panic on failure fn unsupported_image_thumbnail_falls_back_to_svg_placeholder_without_ffmpeg -src/media/thumbnail.rs 868 expect clippy::panic_in_result_fn test assertions intentionally panic on failure fn mutool_command_caps_render_dimensions +src/media/thumbnail.rs 497 expect clippy::panic_in_result_fn test assertions intentionally panic on failure fn write_pdf_placeholder_outputs_svg +src/media/thumbnail.rs 512 expect clippy::panic_in_result_fn test assertions intentionally panic on failure fn unsupported_image_thumbnail_falls_back_to_svg_placeholder_without_ffmpeg src/middleware/state.rs 342 expect clippy::struct_excessive_bools independent runtime capability flags are shared across handlers and workers struct AppState src/middleware/state.rs 435 expect clippy::panic_in_result_fn assertions provide precise regression failures after fallible test-database setup fn db_maintenance_jobs_assign_monotonic_job_ids_and_preserve_them_in_terminal_states src/middleware/state.rs 527 expect clippy::panic_in_result_fn assertions provide precise regression failures after fallible test-database setup fn db_maintenance_jobs_finished_updates_only_matching_running_job @@ -326,11 +323,10 @@ src/tls/self_signed.rs 283 expect clippy::panic_in_result_fn assertions intentio src/tls/self_signed.rs 342 expect clippy::panic_in_result_fn assertions intentionally report violations of missing-key recovery fn missing_key_triggers_regeneration_on_next_startup src/utils/crypto.rs 70 expect clippy::exit fail-closed randomness failures must terminate before issuing weak tokens fn fatal_randomness_error src/utils/crypto.rs 212 expect clippy::panic_in_result_fn test assertions intentionally panic on failure fn hash_and_verify_password -src/utils/files/storage.rs 34 expect clippy::struct_excessive_bools the booleans are independent upload-policy and media-capability flags struct SaveUploadOptions -src/utils/files/storage.rs 135 expect clippy::too_many_arguments the arguments mirror independent upload metadata and board limits at the handler boundary fn save_audio_with_image_thumb_from_path -src/utils/files/storage.rs 246 expect clippy::as_conversions,clippy::cast_precision_loss human-readable size display intentionally rounds integral byte counts to one decimal place fn format_file_size -src/utils/files/storage.rs 1499 expect clippy::print_stderr test skip diagnostics must remain visible when optional ffmpeg fixtures are unavailable fn valid_audio_uploads_accept_common_formats_with_ffprobe -src/utils/files/storage.rs 1554 expect clippy::print_stderr test skip diagnostics must remain visible when optional ffmpeg fixtures are unavailable fn valid_mkv_uploads_save_as_video_with_probe +src/utils/files/storage.rs 121 expect clippy::redundant_pub_crate this opaque validated-upload handoff is shared across sibling crate modules fn save_validated_upload_from_path +src/utils/files/storage.rs 247 expect clippy::as_conversions,clippy::cast_precision_loss human-readable size display intentionally rounds integral byte counts to one decimal place fn format_file_size +src/utils/files/storage.rs 349 expect clippy::redundant_pub_crate this opaque validation token crosses the private storage-module boundary struct ValidatedUpload +src/utils/files/storage.rs 385 expect clippy::redundant_pub_crate the posting handler must obtain the opaque token without exposing it publicly fn validate_upload_for_storage src/utils/sanitize.rs 29 expect clippy::expect_used the reply-reference regex is a source literal covered by sanitizer tests item RE_REPLY src/utils/sanitize.rs 37 expect clippy::expect_used the cross-board-link regex is a source literal covered by sanitizer tests item RE_CROSSLINK src/utils/sanitize.rs 46 expect clippy::expect_used the URL regex is a source literal covered by sanitizer tests item RE_URL @@ -338,31 +334,46 @@ src/utils/sanitize.rs 55 expect clippy::expect_used the bold-markup regex is a s src/utils/sanitize.rs 63 expect clippy::expect_used the italic-markup regex is a source literal covered by sanitizer tests item RE_ITALIC src/utils/sanitize.rs 71 expect clippy::expect_used the spoiler-markup regex is a source literal covered by sanitizer tests item RE_SPOILER src/utils/sanitize.rs 80 expect clippy::expect_used the dice-expression regex is a source literal covered by sanitizer tests item RE_DICE -src/workers/mod.rs 699 expect clippy::cognitive_complexity,clippy::too_many_lines claim, execution, completion persistence, retry, and shutdown form one worker state machine fn worker_loop -src/workers/mod.rs 864 expect clippy::cognitive_complexity the dispatcher mirrors the closed set of background job variants fn handle_job -src/workers/mod.rs 996 expect clippy::cognitive_complexity transcode process control and fail-closed cleanup share one lifecycle fn transcode_video -src/workers/mod.rs 1111 expect clippy::cognitive_complexity source validation and codec-specific preparation share one preflight boundary fn transcode_video_prepare -src/workers/mod.rs 1660 expect clippy::cognitive_complexity archive, prune, and filesystem finalization remain one consistency operation fn prune_threads -src/workers/mod.rs 1850 expect clippy::expect_used test database queries stop locally with fixture-specific failure context fn file_hash_count -src/workers/mod.rs 1875 expect clippy::expect_used constructing the constrained test pool is a prerequisite for completion-retry tests fn single_connection_test_pool -src/workers/mod.rs 1905 expect clippy::expect_used claiming the seeded job is a prerequisite for each queue-state assertion fn claim_job -src/workers/mod.rs 1918 expect clippy::expect_used test fixture failures stop the individual queue accounting test with local context fn retryable_failure_restores_queue_pending_count -src/workers/mod.rs 1948 expect clippy::expect_used test fixture failures stop the individual queue accounting test with local context fn permanent_failure_leaves_queue_pending_count_at_zero -src/workers/mod.rs 1984 expect clippy::expect_used the async retry test requires every timed and database fixture step to succeed fn completion_retries_the_same_job_after_pool_exhaustion -src/workers/mod.rs 2037 expect clippy::expect_used the shutdown recovery test requires every timed and database fixture step to succeed fn completion_stops_retrying_after_shutdown_grace_and_startup_can_recover -src/workers/mod.rs 2077 expect clippy::expect_used the completion recovery fixture must resolve within its explicit test timeout fn already_resolved_completion_error_does_not_wedge_worker -src/workers/mod.rs 2113 expect clippy::too_many_lines the regression keeps job seeding, terminal persistence, and replay checks in one end-to-end scenario fn non_transient_done_completion_is_terminal_and_never_replayed -src/workers/mod.rs 2231 expect clippy::expect_used the injected retry-state fixture requires each database and timeout step to succeed fn non_transient_failed_completion_preserves_bounded_attempt_count -src/workers/mod.rs 2349 expect clippy::expect_used temporary-file and database fixture setup must succeed before cleanup assertions fn stale_transcode_persist_removes_unattached_webm_and_writes_no_hash -src/workers/mod.rs 2385 expect clippy::expect_used temporary-file fixture setup must succeed before transcode output assertions fn transcode_finalise_skips_larger_output_and_cleans_temp_file -src/workers/mod.rs 2429 expect clippy::expect_used temporary upload fixture setup must succeed before path validation assertions fn transcode_source_validation_rejects_escape_paths -src/workers/mod.rs 2448 expect clippy::expect_used Unix symlink fixture setup must succeed before containment assertions fn thumb_cache_eviction_skips_symlinked_thumbs_dir -src/workers/mod.rs 2475 expect clippy::expect_used Unix symlink fixture setup must succeed before containment assertions fn thumb_cache_eviction_skips_symlinked_board_dir -src/workers/mod.rs 2502 expect clippy::expect_used Unix symlink fixture setup must succeed before containment assertions fn thumb_cache_eviction_skips_symlinked_thumb_file -src/workers/mod.rs 2526 expect clippy::expect_used temporary-file and database fixture setup must succeed before cleanup assertions fn stale_waveform_finalise_removes_unattached_png_and_writes_no_hash +src/workers/mod.rs 964 expect clippy::cognitive_complexity,clippy::too_many_lines claim, execution, completion persistence, retry, and shutdown form one worker state machine fn worker_loop +src/workers/mod.rs 1130 expect clippy::cognitive_complexity,clippy::too_many_arguments the dispatcher mirrors the closed set of background job variants fn handle_job +src/workers/mod.rs 1269 expect clippy::too_many_arguments transcode process control and fail-closed cleanup share one lifecycle fn transcode_video +src/workers/mod.rs 1380 expect clippy::cognitive_complexity source validation and codec-specific preparation share one preflight boundary fn transcode_video_prepare +src/workers/mod.rs 1551 expect clippy::too_many_arguments the finalisation boundary carries the durable job and all validated filesystem identities fn transcode_video_finalise +src/workers/mod.rs 1593 expect clippy::too_many_arguments output installation and atomic DB completion require the full validated transcode identity fn persist_transcoded_webm +src/workers/mod.rs 1934 expect clippy::too_many_arguments the finalisation boundary explicitly carries all validated paths and DB identity fn waveform_finalise +src/workers/mod.rs 2115 expect clippy::cognitive_complexity archive, prune, and filesystem finalization remain one consistency operation fn prune_threads +src/workers/mod.rs 2561 expect clippy::cognitive_complexity,clippy::too_many_lines the bounded legacy-state matrix is intentionally explicit fn reconcile_media_job_states +src/workers/mod.rs 3050 expect clippy::expect_used test database queries stop locally with fixture-specific failure context fn file_hash_count +src/workers/mod.rs 3227 expect clippy::expect_used fixture setup failures should stop this queue durability test fn required_thread_prune_bypasses_capacity_and_survives_queue_recreation +src/workers/mod.rs 3690 expect clippy::expect_used fixture setup failures should stop this coalescing test fn board_prune_coalesces_and_running_work_leaves_one_dirty_evaluation +src/workers/mod.rs 3891 expect clippy::too_many_lines the legacy reconciliation matrix remains explicit in one fixed-point test fn media_reconciliation_repairs_legacy_matrix_and_reaches_fixed_point +src/workers/mod.rs 4056 expect clippy::expect_used constructing the constrained test pool is a prerequisite for completion-retry tests fn single_connection_test_pool +src/workers/mod.rs 4086 expect clippy::expect_used claiming the seeded job is a prerequisite for each queue-state assertion fn claim_job +src/workers/mod.rs 4099 expect clippy::expect_used test fixture failures stop the individual queue accounting test with local context fn retryable_failure_restores_queue_pending_count +src/workers/mod.rs 4129 expect clippy::expect_used test fixture failures stop the individual queue accounting test with local context fn permanent_failure_leaves_queue_pending_count_at_zero +src/workers/mod.rs 4165 expect clippy::expect_used the async retry test requires every timed and database fixture step to succeed fn completion_retries_the_same_job_after_pool_exhaustion +src/workers/mod.rs 4218 expect clippy::expect_used the shutdown recovery test requires every timed and database fixture step to succeed fn completion_stops_retrying_after_shutdown_grace_and_startup_can_recover +src/workers/mod.rs 4258 expect clippy::expect_used the completion recovery fixture must resolve within its explicit test timeout fn already_resolved_completion_error_does_not_wedge_worker +src/workers/mod.rs 4294 expect clippy::too_many_lines the regression keeps job seeding, terminal persistence, and replay checks in one end-to-end scenario fn non_transient_done_completion_is_terminal_and_never_replayed +src/workers/mod.rs 4412 expect clippy::expect_used the injected retry-state fixture requires each database and timeout step to succeed fn non_transient_failed_completion_preserves_bounded_attempt_count +src/workers/mod.rs 4532 expect clippy::expect_used temporary-file and database fixture setup must succeed before cleanup assertions fn stale_transcode_persist_removes_unattached_webm_and_writes_no_hash +src/workers/mod.rs 4569 expect clippy::expect_used temporary-file fixture setup must succeed before transcode output assertions fn transcode_finalise_skips_larger_output_and_cleans_temp_file +src/workers/mod.rs 4614 expect clippy::expect_used temporary upload fixture setup must succeed before path validation assertions fn transcode_source_validation_rejects_escape_paths +src/workers/mod.rs 4633 expect clippy::expect_used Unix symlink fixture setup must succeed before containment assertions fn thumb_cache_eviction_skips_symlinked_thumbs_dir +src/workers/mod.rs 4663 expect clippy::expect_used Unix symlink fixture setup must succeed before containment assertions fn thumb_cache_eviction_skips_symlinked_board_dir +src/workers/mod.rs 4693 expect clippy::expect_used Unix symlink fixture setup must succeed before containment assertions fn thumb_cache_eviction_skips_symlinked_thumb_file +src/workers/mod.rs 4721 expect clippy::expect_used fixture setup must succeed before eviction assertions fn thumb_cache_evicts_only_unreferenced_files_across_all_media_states +src/workers/mod.rs 4805 expect clippy::expect_used temporary-file and database fixture setup must succeed before cleanup assertions fn stale_waveform_finalise_removes_unattached_png_and_writes_no_hash tests/cli_runtime.rs 1 expect unused_crate_dependencies Cargo exposes all package dependencies to this black-box integration target, which intentionally interacts only with the compiled CLI crate tests/cli_runtime.rs 263 expect clippy::panic_in_result_fn assertions intentionally report violated CLI filesystem invariants fn help_and_version_do_not_create_runtime_state tests/cli_runtime.rs 304 expect clippy::panic_in_result_fn assertions intentionally report violated CLI filesystem invariants fn explicit_data_dir_owns_all_initialized_runtime_state tests/cli_runtime.rs 356 expect clippy::panic_in_result_fn assertions intentionally report violated CLI filesystem invariants fn relative_data_dir_is_rejected_before_filesystem_mutation tests/cli_runtime.rs 396 expect clippy::too_many_lines one subprocess lifecycle verifies the complete direct-HTTPS topology and cookie contract fn direct_https_process_exposes_only_https_and_redirect_with_secure_login_cookies tests/cli_runtime.rs 400 expect clippy::panic_in_result_fn assertions intentionally report violated TLS topology and cookie invariants fn direct_https_process_exposes_only_https_and_redirect_with_secure_login_cookies +src/media/audio.rs 521 expect clippy::cast_possible_truncation,clippy::cast_sign_loss,clippy::as_conversions clamped normalized amplitudes map to a bounded pixel radius fn render_peaks +src/media/heif.rs 12 expect clippy::redundant_pub_crate keep codec helpers internal while allowing upload and banner callers fn decode +src/media/images.rs 21 expect clippy::redundant_pub_crate keep codec helpers internal while allowing upload and banner callers fn validate_dimensions +src/media/images.rs 35 expect clippy::redundant_pub_crate keep codec helpers internal while allowing upload and banner callers fn decode_still +src/media/images.rs 74 expect clippy::redundant_pub_crate keep codec helpers internal while allowing upload and banner callers fn image_to_webp +src/media/pdf.rs 31 expect clippy::redundant_pub_crate PDF rendering is an internal media helper fn render +src/media/pdf.rs 35 expect clippy::cast_possible_truncation,clippy::cast_sign_loss,clippy::as_conversions finite positive viewport dimensions are clamped to a checked u16 bound before rounding fn render diff --git a/playwright.restart.config.ts b/playwright.restart.config.ts new file mode 100644 index 00000000..63744042 --- /dev/null +++ b/playwright.restart.config.ts @@ -0,0 +1,16 @@ +import { defineConfig, devices } from '@playwright/test'; + +// Restart tests own disposable processes and have no dependency on the media audit harness. +export default defineConfig({ + testDir: './tests/e2e', + testMatch: 'settings-restart.spec.ts', + workers: 1, + timeout: 120_000, + expect: { timeout: 15_000 }, + outputDir: 'test-results/restart', + use: { ...devices['Desktop Chrome'], trace: 'retain-on-failure' }, + projects: [ + { name: 'chromium' }, + { name: 'chromium-nojs', use: { javaScriptEnabled: false } }, + ], +}); diff --git a/src/banner.rs b/src/banner.rs index 83286eb5..ac66109a 100644 --- a/src/banner.rs +++ b/src/banner.rs @@ -250,7 +250,10 @@ pub fn validate_banner_restore_entry_name(name: &str) -> Result { .next() .and_then(|component| match component { Component::Normal(value) => value.to_str(), - _ => None, + Component::Prefix(_) + | Component::RootDir + | Component::CurDir + | Component::ParentDir => None, }) .ok_or_else(|| anyhow::anyhow!("Banner restore entry name is not valid UTF-8."))?; if scope != "global" && scope != "home" { @@ -260,7 +263,10 @@ pub fn validate_banner_restore_entry_name(name: &str) -> Result { .next() .and_then(|component| match component { Component::Normal(value) => value.to_str(), - _ => None, + Component::Prefix(_) + | Component::RootDir + | Component::CurDir + | Component::ParentDir => None, }) .ok_or_else(|| anyhow::anyhow!("Banner restore entry name is not valid UTF-8."))?; let extension = Path::new(file_name) @@ -540,11 +546,16 @@ pub fn choose_active_banner( ); } if settings.rotation_interval_minutes > 0 { - let bucket = chrono::Utc::now().timestamp() - / settings - .rotation_interval_minutes - .saturating_mul(60) - .max(60); + let interval_seconds = settings + .rotation_interval_minutes + .saturating_mul(60) + .max(60); + let Some(bucket) = chrono::Utc::now().timestamp().checked_div(interval_seconds) else { + // The positive interval rules out both division by zero and the + // signed MIN / -1 overflow; retain an explicit fail-closed guard. + tracing::error!(interval_seconds, "Invalid banner rotation interval"); + return (None, "invalid-rotation".to_owned(), false); + }; let len = i64::try_from(candidates.len()).unwrap_or(1); let index = usize::try_from(bucket.rem_euclid(len)).unwrap_or(0); let asset = candidates.get(index).cloned(); @@ -557,7 +568,7 @@ pub fn choose_active_banner( let nonce = uuid::Uuid::new_v4().as_u128(); let len = u128::try_from(candidates.len()).unwrap_or(1); - let index = usize::try_from(nonce % len).unwrap_or(0); + let index = usize::try_from(nonce.rem_euclid(len)).unwrap_or(0); let asset = candidates.get(index).cloned(); let fragment = asset.as_ref().map_or_else( || "none".to_owned(), @@ -682,44 +693,17 @@ fn write_animated_gif_banner_asset_scaled( target_path: &Path, original_width: u32, original_height: u32, -) -> Result<(u32, u32)> { - write_animated_gif_banner_asset_scaled_with_caps( - bytes, - target_path, - original_width, - original_height, - crate::media::ffmpeg::detect_ffmpeg(), - crate::media::ffmpeg::check_webp_encoder(), - ) -} - -/// Store an animated GIF as scaled WebP or an unmodified GIF fallback. -fn write_animated_gif_banner_asset_scaled_with_caps( - bytes: &[u8], - target_path: &Path, - original_width: u32, - original_height: u32, - ffmpeg_available: bool, - ffmpeg_webp_available: bool, ) -> Result<(u32, u32)> { let (max_width, max_height) = maybe_shrink_dimensions(original_width, original_height); - if !ffmpeg_available || !ffmpeg_webp_available { - write_gif_banner_fallback(bytes, target_path)?; - return Ok((original_width, original_height)); - } let input_path = animated_gif_temp_path(target_path); std::fs::write(&input_path, bytes) .with_context(|| format!("write {}", input_path.display()))?; let webp_path = banner_webp_path(target_path); - let conversion = crate::media::ffmpeg::ffmpeg_image_to_webp_scaled( - &input_path, - &webp_path, - max_width, - max_height, - ) - .with_context(|| format!("convert animated gif banner {}", input_path.display())); + let conversion = + crate::media::images::image_to_webp(&input_path, &webp_path, Some((max_width, max_height))) + .with_context(|| format!("convert animated gif banner {}", input_path.display())); drop(std::fs::remove_file(&input_path)); match conversion { Ok(()) => { @@ -737,7 +721,7 @@ fn write_animated_gif_banner_asset_scaled_with_caps( } } -/// Store an animated WebP, scaling it when the external encoder is available. +/// Store an animated WebP, scaling its composited frames in Rust. fn write_animated_webp_banner_asset( bytes: &[u8], target_path: &Path, @@ -746,18 +730,14 @@ fn write_animated_webp_banner_asset( ) -> Result<(u32, u32)> { let (max_width, max_height) = maybe_shrink_dimensions(original_width, original_height); let should_scale = (max_width, max_height) != (original_width, original_height); - if should_scale - && crate::media::ffmpeg::detect_ffmpeg() - && crate::media::ffmpeg::check_webp_encoder() - { + if should_scale { let input_path = animated_webp_temp_path(target_path); std::fs::write(&input_path, bytes) .with_context(|| format!("write {}", input_path.display()))?; - let conversion = crate::media::ffmpeg::ffmpeg_image_to_webp_scaled( + let conversion = crate::media::images::image_to_webp( &input_path, target_path, - max_width, - max_height, + Some((max_width, max_height)), ) .with_context(|| format!("scale animated webp banner {}", input_path.display())); drop(std::fs::remove_file(&input_path)); @@ -928,7 +908,7 @@ fn count_gif_frames(bytes: &[u8]) -> usize { let mut frame_markers = 0usize; for window in bytes.windows(2) { if window == [0x21, 0xF9] { - frame_markers += 1; + frame_markers = frame_markers.saturating_add(1); } } frame_markers.max(1) @@ -941,10 +921,10 @@ fn is_animated_webp(bytes: &[u8]) -> bool { } let mut offset = 12usize; while offset.saturating_add(8) <= bytes.len() { - let Some(chunk_type) = bytes.get(offset..offset + 4) else { + let Some(chunk_type) = bytes.get(offset..offset.saturating_add(4)) else { return false; }; - let Some(size_bytes) = bytes.get(offset + 4..offset + 8) else { + let Some(size_bytes) = bytes.get(offset.saturating_add(4)..offset.saturating_add(8)) else { return false; }; let Ok(size_bytes) = <[u8; 4]>::try_from(size_bytes) else { @@ -967,16 +947,16 @@ fn is_animated_webp(bytes: &[u8]) -> bool { #[cfg(test)] mod tests { use super::{ - banner_admin_anchor, banner_asset_content_type, banner_gif_fallback_path, - banner_open_section, banner_storage_path, banner_target_draft, canonicalize_banner_bytes, - choose_active_banner, is_animated_webp, normalize_external_url, normalize_internal_path, - parse_banner_target, resolve_banner_href, safe_return_to, - validate_banner_restore_entry_name, validate_banner_storage_key, - write_animated_gif_banner_asset_scaled_with_caps, write_animated_webp_banner_asset, - DISPLAY_HEIGHT, DISPLAY_WIDTH, MAX_ANIMATED_GIF_FRAMES, + banner_admin_anchor, banner_gif_fallback_path, banner_open_section, banner_storage_path, + banner_target_draft, canonicalize_banner_bytes, choose_active_banner, is_animated_webp, + normalize_external_url, normalize_internal_path, parse_banner_target, resolve_banner_href, + safe_return_to, validate_banner_restore_entry_name, validate_banner_storage_key, + write_animated_gif_banner_asset_scaled, write_animated_webp_banner_asset, DISPLAY_HEIGHT, + DISPLAY_WIDTH, MAX_ANIMATED_GIF_FRAMES, }; use crate::models::{BannerAsset, BannerScope, BannerTargetType}; use anyhow::Result; + use image::AnimationDecoder as _; use image::{codecs::gif::GifEncoder, Delay, Frame, ImageBuffer, ImageFormat, Rgba}; use std::io::Cursor; @@ -1036,13 +1016,13 @@ mod tests { fn banner_target_draft_only_populates_selected_field() { let board = banner_target_draft(BannerTargetType::InternalBoard, "tech"); assert_eq!(board.board_value, "tech"); - assert!(board.thread_value.is_empty()); - assert!(board.external_url.is_empty()); + assert_eq!(board.thread_value.len(), 0); + assert_eq!(board.external_url, ""); let thread = banner_target_draft(BannerTargetType::InternalPath, "/tech/thread/42"); - assert!(thread.board_value.is_empty()); + assert_eq!(thread.board_value.len(), 0); assert_eq!(thread.thread_value, "/tech/thread/42"); - assert!(thread.external_url.is_empty()); + assert_eq!(thread.external_url, ""); } #[test] @@ -1149,7 +1129,7 @@ mod tests { clippy::panic_in_result_fn, reason = "test assertions intentionally panic on failure" )] - fn animated_gif_banner_falls_back_to_gif_without_ffmpeg() -> Result<()> { + fn animated_gif_banner_converts_to_animated_webp_without_ffmpeg() -> Result<()> { let mut bytes = Vec::new(); { let mut encoder = GifEncoder::new(&mut bytes); @@ -1161,34 +1141,31 @@ mod tests { } let target = std::env::temp_dir().join(format!("{}.webp", uuid::Uuid::new_v4().simple())); let gif_path = banner_gif_fallback_path(&target); - let dimensions = write_animated_gif_banner_asset_scaled_with_caps( - &bytes, - &target, - DISPLAY_WIDTH, - DISPLAY_HEIGHT, - false, - false, - )?; + let dimensions = + write_animated_gif_banner_asset_scaled(&bytes, &target, DISPLAY_WIDTH, DISPLAY_HEIGHT)?; assert_eq!( dimensions, (DISPLAY_WIDTH, DISPLAY_HEIGHT), "fallback must preserve source dimensions" ); + assert!(target.exists(), "WebP must be produced without FFmpeg"); + let decoder = image::codecs::webp::WebPDecoder::new(std::io::BufReader::new( + std::fs::File::open(&target)?, + ))?; assert!( - !target.exists(), - "the unavailable WebP output must not be published" + decoder.has_animation(), + "banner animation must survive conversion" ); assert_eq!( - std::fs::read(&gif_path)?, - bytes, - "the GIF fallback must preserve the source bytes" + decoder.into_frames().collect_frames()?.len(), + 2, + "both banner frames must survive" ); - assert_eq!( - banner_asset_content_type(&gif_path), - "image/gif", - "the fallback must be served as GIF" + assert!( + !gif_path.exists(), + "successful conversion must not leave a GIF fallback" ); - drop(std::fs::remove_file(gif_path)); + drop(std::fs::remove_file(target)); Ok(()) } diff --git a/src/cache.rs b/src/cache.rs index 934e3495..96248ce2 100644 --- a/src/cache.rs +++ b/src/cache.rs @@ -18,14 +18,14 @@ pub const CACHE_CONTROL_IMMUTABLE_MEDIA: &str = "public, max-age=31536000, immut /// Insert `Cache-Control` only when the response does not already define it. pub fn insert_cache_control_if_absent(headers: &mut HeaderMap, value: &'static str) { - headers + let _configured = headers .entry(header::CACHE_CONTROL) .or_insert(HeaderValue::from_static(value)); } /// Replace the response's `Cache-Control` header with `value`. pub fn set_cache_control(headers: &mut HeaderMap, value: &'static str) { - headers.insert(header::CACHE_CONTROL, HeaderValue::from_static(value)); + let _previous_value = headers.insert(header::CACHE_CONTROL, HeaderValue::from_static(value)); } /// Ensure a response's `Vary` header includes `Cookie`. @@ -34,7 +34,7 @@ pub fn insert_vary_cookie(headers: &mut HeaderMap) { .get(header::VARY) .and_then(|value| value.to_str().ok()) else { - headers.insert(header::VARY, HeaderValue::from_static("Cookie")); + let _previous_value = headers.insert(header::VARY, HeaderValue::from_static("Cookie")); return; }; @@ -47,6 +47,6 @@ pub fn insert_vary_cookie(headers: &mut HeaderMap) { let combined = format!("{existing}, Cookie"); if let Ok(value) = HeaderValue::from_str(&combined) { - headers.insert(header::VARY, value); + let _previous_value = headers.insert(header::VARY, value); } } diff --git a/src/captcha.rs b/src/captcha.rs index 4372f4ee..669f9c71 100644 --- a/src/captcha.rs +++ b/src/captcha.rs @@ -94,7 +94,7 @@ pub fn generate_captcha_image( prune_expired(chrono::Utc::now().timestamp()); let mut captcha = Captcha::new(); - captcha + let _configured = captcha .set_chars(CAPTCHA_CHARSET) .add_chars(CAPTCHA_ANSWER_LEN) .apply_filter(Noise::new(0.25)) @@ -163,7 +163,7 @@ pub fn verify_captcha( /// Hash and store one challenge when its generated answer is valid. fn store_challenge(board_short: &str, captcha_id: &str, answer: &str, now: i64) { if let Some(answer_hash) = answer_hash(board_short, captcha_id, answer) { - CAPTCHA_CHALLENGES.insert( + let _previous_value = CAPTCHA_CHALLENGES.insert( captcha_id.to_owned(), StoredCaptchaChallenge { board_short: board_short.to_owned(), @@ -232,7 +232,7 @@ pub mod testing { let Some(answer_hash) = super::answer_hash(board_short, captcha_id, answer) else { return; }; - super::CAPTCHA_CHALLENGES.insert( + let _previous_value = super::CAPTCHA_CHALLENGES.insert( captcha_id.to_owned(), super::StoredCaptchaChallenge { board_short: board_short.to_owned(), @@ -250,7 +250,7 @@ pub mod testing { #[cfg(test)] mod tests { - use super::{testing, *}; + use super::*; #[test] fn generated_captcha_is_png_and_stores_server_side_answer() { diff --git a/src/config.rs b/src/config.rs index 20c69257..03552477 100644 --- a/src/config.rs +++ b/src/config.rs @@ -185,9 +185,9 @@ pub fn configure_port_override(port: Option) -> anyhow::Result<()> { return Ok(()); }; anyhow::ensure!(port != 0, "--port must not be zero"); - CLI_PORT_OVERRIDE - .set(port) - .map_err(|_| anyhow::anyhow!("launcher port was already configured")) + CLI_PORT_OVERRIDE.set(port).map_err(|rejected_port| { + anyhow::anyhow!("launcher port {rejected_port} was already configured") + }) } /// Configure an explicit runtime data directory before configuration is loaded. @@ -201,9 +201,12 @@ pub fn configure_data_dir(path: Option<&Path>) -> anyhow::Result<()> { return Ok(()); }; let resolved = resolve_data_dir_override(path)?; - DATA_DIR_OVERRIDE - .set(resolved) - .map_err(|_| anyhow::anyhow!("runtime data directory was already configured")) + DATA_DIR_OVERRIDE.set(resolved).map_err(|rejected_path| { + anyhow::anyhow!( + "runtime data directory {} was already configured", + rejected_path.display() + ) + }) } #[must_use] @@ -535,12 +538,10 @@ struct SettingsFile { /// multiple instances that share the same storage to avoid key collisions. /// Default: "rustchan". tor_service_nickname: Option, - /// Whether startup fails when `FFmpeg` or `FFprobe` is unavailable. + /// Whether startup fails when `FFmpeg` is unavailable. require_ffmpeg: Option, /// Configured `FFmpeg` executable path. ffmpeg_path: Option, - /// Configured `FFprobe` executable path. - ffprobe_path: Option, /// Whether boards may enable arbitrary-file uploads. enable_any_file_uploads_feature: Option, /// How often to run PRAGMA `wal_checkpoint(TRUNCATE)`, in seconds. @@ -913,6 +914,10 @@ pub fn describe_timeout_secs(timeout_secs: u64) -> String { reason = "runtime configuration intentionally models independent operator-controlled feature switches" )] /// Fully resolved runtime configuration. +#[expect( + clippy::partial_pub_fields, + reason = "settings remain public, while startup provenance is maintained only inside the crate" +)] pub struct Config { /// Authentication, public paging and request-deadline policies. pub operator: operator::OperatorSettings, @@ -965,8 +970,6 @@ pub struct Config { pub require_ffmpeg: bool, /// Explicit ffmpeg binary path, or plain "ffmpeg" for PATH lookup. pub ffmpeg_path: String, - /// Explicit ffprobe binary path, or plain "ffprobe" for PATH lookup. - pub ffprobe_path: String, /// Global feature gate for arbitrary uploads. Boards can only enable the /// per-board toggle when this is true. pub enable_any_file_uploads_feature: bool, @@ -1103,7 +1106,6 @@ impl std::fmt::Debug for Config { .field("tor_service_nickname", &self.tor_service_nickname) .field("require_ffmpeg", &self.require_ffmpeg) .field("ffmpeg_path", &self.ffmpeg_path) - .field("ffprobe_path", &self.ffprobe_path) .field( "enable_any_file_uploads_feature", &self.enable_any_file_uploads_feature, @@ -1300,7 +1302,7 @@ impl Config { // CHAN_BIND's port. Resolve that before validation and state display. let bind_addr = cli_port.map_or_else( || bind_addr.clone(), - |port| format_bind_addr(bind_host_for_family(&bind_addr), port), + |override_port| format_bind_addr(bind_host_for_family(&bind_addr), override_port), ); let tor_only = environment.boolean("CHAN_TOR_ONLY", s.tor_only.unwrap_or(false)); let enable_tor_support = @@ -1387,11 +1389,6 @@ impl Config { .ok() .or(s.ffmpeg_path) .unwrap_or_else(|| "ffmpeg".to_owned()), - ffprobe_path: environment - .var("CHAN_FFPROBE_PATH") - .ok() - .or(s.ffprobe_path) - .unwrap_or_else(|| "ffprobe".to_owned()), enable_any_file_uploads_feature: environment.boolean( "CHAN_ENABLE_ANY_FILE_UPLOADS_FEATURE", s.enable_any_file_uploads_feature.unwrap_or(false), @@ -1638,21 +1635,23 @@ impl Config { } } for cidr in &self.trusted_proxy_cidrs { - cidr.parse::().map_err(|error| { - anyhow::anyhow!( + cidr.parse::() + .map_err(|error| { + anyhow::anyhow!( "CONFIG ERROR: trusted_proxy_cidrs entry '{cidr}' is not valid CIDR: {error}" ) - })?; + }) + .map(|_validated_value| ())?; } for host in &self.public_hosts { normalize_public_host(host).ok_or_else(|| { anyhow::anyhow!( "CONFIG ERROR: public_hosts entry '{host}' must be a bare hostname or IP literal." ) - })?; + }).map(|_operation_summary| ())?; } if let Some(path) = &self.backup_directory { - prepare_backup_directory(path, self)?; + prepare_backup_directory(path, self).map(|_validated_path| ())?; } // Verify the upload directory is writable. let upload_path = Path::new(&self.upload_dir); @@ -1700,7 +1699,7 @@ impl Config { ACME validation requires public HTTPS reachability, but tor_only binds RustChan to loopback." ); } - validate_ffmpeg_timeout_secs(self.ffmpeg_timeout_secs)?; + validate_ffmpeg_timeout_secs(self.ffmpeg_timeout_secs).map(|_operation_summary| ())?; Ok(()) } @@ -1750,7 +1749,7 @@ fn rewrite_settings_file_lines( let trimmed = line.trim_start(); for (key, value) in updates { if trimmed.starts_with(key) && line.contains('=') { - seen_keys.insert(*key); + let _inserted = seen_keys.insert(*key); return format!("{key} = {value}"); } } @@ -1787,7 +1786,7 @@ fn rewrite_settings_file_lines( if next_line.is_some_and(|line| !line.trim().is_empty()) { insertion_block.push(String::new()); } - updated_lines.splice(insert_idx..insert_idx, insertion_block); + drop(updated_lines.splice(insert_idx..insert_idx, insertion_block)); } let mut out = updated_lines.join("\n"); @@ -1808,20 +1807,20 @@ fn update_settings_file_entries_result( /// Persist a complete file update and compensate it if a prepared DB commit fails. fn update_settings_file_entries_with_commit( updates: &[(&str, String)], - insert_missing_before: Option<&str>, + _insert_missing_before: Option<&str>, commit: impl FnOnce() -> anyhow::Result<()>, ) -> anyhow::Result<()> { let _guard = admin::SETTINGS_WRITE_LOCK.lock(); - let _ = insert_missing_before; // Legacy anchors remain API-compatible; root insertion is structural. + // Legacy anchors remain API-compatible; root insertion is structural. let parsed = updates .iter() .map(|(key, value)| { let document: toml::Value = toml::from_str(&format!("value = {value}"))?; - let value = document + let parsed_value = document .get("value") .cloned() .context("missing settings value")?; - Ok(((*key).to_owned(), Some(value))) + Ok(((*key).to_owned(), Some(parsed_value))) }) .collect::>>()?; admin::save_root_and_commit_at(&settings_file_path(), &parsed, commit) @@ -2094,7 +2093,8 @@ pub fn check_cookie_secret_rotation(conn: &rusqlite::Connection) -> anyhow::Resu if h == ¤t_hash { return Ok(()); // Secret unchanged — nothing to do. } - tx.execute("DELETE FROM admin_sessions", [])?; + tx.execute("DELETE FROM admin_sessions", []) + .map(|_affected_rows| ())?; tracing::warn!( "SECURITY WARNING: cookie_secret has changed since the last run. \ All IP-based bans are now invalid because all IP hashes have changed. \ @@ -2108,13 +2108,14 @@ pub fn check_cookie_secret_rotation(conn: &rusqlite::Connection) -> anyhow::Resu "INSERT INTO site_settings (key, value) VALUES (?1, ?2) ON CONFLICT(key) DO UPDATE SET value = excluded.value", rusqlite::params![KEY, current_hash], - )?; + ) + .map(|_affected_rows| ())?; tx.commit()?; Ok(()) } /// Environment source used by startup and mutation-free configuration previews. -enum Environment<'a> { +pub(crate) enum Environment<'a> { /// Read the process environment, including non-Unicode path overrides. Process, /// Explicit overrides for isolated previews and tests. @@ -2527,7 +2528,6 @@ mod tests { tor_service_nickname: "rustchan".to_owned(), require_ffmpeg: false, ffmpeg_path: "ffmpeg".to_owned(), - ffprobe_path: "ffprobe".to_owned(), enable_any_file_uploads_feature: false, bind_addr: "0.0.0.0:8080".to_owned(), database_path: "chan.db".to_owned(), diff --git a/src/config/admin.rs b/src/config/admin.rs index 6ffbc6c3..047704c3 100644 --- a/src/config/admin.rs +++ b/src/config/admin.rs @@ -38,9 +38,24 @@ pub enum InputKind { OptionalText, } +/// How a validated administrator setting takes effect. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ApplicationMode { + /// Database or shared live state is updated by the existing save handler. + Live, + /// The process must reinitialize the affected resource or immutable policy. + Restart, +} + /// Definition shared by rendering, input validation and coverage checks. #[derive(Debug)] +#[expect( + clippy::partial_pub_fields, + reason = "renderers inspect metadata, while the config-only getter remains encapsulated" +)] pub struct SettingDefinition { + /// Whether this control uses live state or immutable process initialization. + pub application: ApplicationMode, /// Authoritative settings-file key. pub key: &'static str, /// Human-readable label and units. @@ -57,18 +72,18 @@ pub struct SettingDefinition { /// Network settings; default request policy preserves the historical behavior. pub static NETWORK_SETTINGS: &[SettingDefinition] = &[ - SettingDefinition { key: "behind_proxy", label: "Trust reverse-proxy headers", help: "For Serveo or another reverse proxy. Forwarded visitor identity and HTTPS are accepted only from trusted peers. Configure the proxy to replace client-supplied forwarding headers.", environment: "CHAN_BEHIND_PROXY", kind: InputKind::Boolean, value: |c| c.behind_proxy.to_string() }, - SettingDefinition { key: "trusted_proxy_cidrs", label: "Trusted proxy networks (CIDRs)", help: "One CIDR per line or separated by commas. Trust only your proxy peers; a broad allowlist lets those peers supply visitor identity for bans and request counters.", environment: "CHAN_TRUSTED_PROXY_CIDRS", kind: InputKind::List, value: |c| c.trusted_proxy_cidrs.join(", ") }, - SettingDefinition { key: "public_hosts", label: "Public hostnames", help: "Bare hostnames or IP literals, without scheme, port or path. The first hostname is the displayed public entry point. Used by host checks and HTTPS redirects.", environment: "CHAN_PUBLIC_HOSTS", kind: InputKind::List, value: |c| c.public_hosts.join(", ") }, - SettingDefinition { key: "https_cookies", label: "Secure authentication cookies", help: "Automatic enables the policy for proxy mode or native HTTPS. Secure cookies are issued for native HTTPS or trusted forwarded HTTPS, preserving plain HTTP access behavior.", environment: "CHAN_HTTPS_COOKIES", kind: InputKind::Boolean, value: |c| c.https_cookies.to_string() }, - SettingDefinition { key: "port", label: "Primary HTTP port", help: "CHAN_PORT overrides this default. An explicit listen address supplies its own port. A launcher --port/-p overrides the final listener port, including CHAN_BIND.", environment: "CHAN_PORT", kind: InputKind::Number(1, 65535), value: |c| c.port.to_string() }, - SettingDefinition { key: "bind_addr", label: "Listen address (IP:port)", help: "Leave blank for 0.0.0.0 with the primary HTTP port. Use [::]:8080 for IPv6. CHAN_BIND overrides this field; CHAN_HOST overrides the saved or automatic interface. A launcher --port/-p overrides this address’s port. Tor-only forces loopback. Listener changes require a service restart.", environment: "CHAN_BIND", kind: InputKind::Address, value: |c| c.bind_addr.clone() }, - SettingDefinition { key: "rate_limit_gets", label: "Browsing request allowance (per visitor/window)", help: "Raise this if visitors frequently see ‘slow down’. Counters use the trusted forwarded identity or the direct peer. Posting cooldowns and password-attempt protection remain separate.", environment: "CHAN_RATE_GETS", kind: InputKind::Number(1, 1_000_000), value: |c| c.rate_limit_gets.to_string() }, - SettingDefinition { key: "rate_limit_window", label: "Browsing window (seconds)", help: "The historical counter rolls over when elapsed whole seconds are greater than this window. Changing it takes effect on restart.", environment: "CHAN_RATE_WINDOW", kind: InputKind::Number(1, 86400), value: |c| c.rate_limit_window.to_string() }, - SettingDefinition { key: "rate_limit_policy", label: "Requests counted by the browsing limiter", help: "Legacy counts all methods, including writes, API polling, favicons and banners. Reads counts GET/HEAD only; write-specific protections still apply. Both exempt /static/, /theme-css/, /boards/, admin live logs and backup progress.", environment: "CHAN_RATE_POLICY", kind: InputKind::Policy, value: |c| c.rate_limit_policy.as_str().to_owned() }, - SettingDefinition { key: "session_duration", label: "Administrator session lifetime (seconds)", help: "Applies to sessions created after restart. Existing sessions keep their stored expiry. Default: 28800 seconds (8 hours).", environment: "CHAN_SESSION_SECS", kind: InputKind::Number(60, 2_592_000), value: |c| c.session_duration.to_string() }, - SettingDefinition { key: "public_readiness_details", label: "Public detailed readiness", help: "Expose operational details at /readyz. /healthz remains minimal and public.", environment: "CHAN_PUBLIC_READINESS_DETAILS", kind: InputKind::Boolean, value: |c| c.public_readiness_details.to_string() }, - SettingDefinition { key: "public_metrics_enabled", label: "Public Prometheus metrics", help: "Expose unauthenticated /metrics. Use a trusted scrape path if enabled.", environment: "CHAN_PUBLIC_METRICS_ENABLED", kind: InputKind::Boolean, value: |c| c.public_metrics_enabled.to_string() }, + SettingDefinition { application: ApplicationMode::Restart, key: "behind_proxy", label: "Trust reverse-proxy headers", help: "For Serveo or another reverse proxy. Forwarded visitor identity and HTTPS are accepted only from trusted peers. Configure the proxy to replace client-supplied forwarding headers.", environment: "CHAN_BEHIND_PROXY", kind: InputKind::Boolean, value: |c| c.behind_proxy.to_string() }, + SettingDefinition { application: ApplicationMode::Restart, key: "trusted_proxy_cidrs", label: "Trusted proxy networks (CIDRs)", help: "One CIDR per line or separated by commas. Trust only your proxy peers; a broad allowlist lets those peers supply visitor identity for bans and request counters.", environment: "CHAN_TRUSTED_PROXY_CIDRS", kind: InputKind::List, value: |c| c.trusted_proxy_cidrs.join(", ") }, + SettingDefinition { application: ApplicationMode::Restart, key: "public_hosts", label: "Public hostnames", help: "Bare hostnames or IP literals, without scheme, port or path. The first hostname is the displayed public entry point. Used by host checks and HTTPS redirects.", environment: "CHAN_PUBLIC_HOSTS", kind: InputKind::List, value: |c| c.public_hosts.join(", ") }, + SettingDefinition { application: ApplicationMode::Restart, key: "https_cookies", label: "Secure authentication cookies", help: "Automatic enables the policy for proxy mode or native HTTPS. Secure cookies are issued for native HTTPS or trusted forwarded HTTPS, preserving plain HTTP access behavior.", environment: "CHAN_HTTPS_COOKIES", kind: InputKind::Boolean, value: |c| c.https_cookies.to_string() }, + SettingDefinition { application: ApplicationMode::Restart, key: "port", label: "Primary HTTP port", help: "CHAN_PORT overrides this default. An explicit listen address supplies its own port. A launcher --port/-p overrides the final listener port, including CHAN_BIND.", environment: "CHAN_PORT", kind: InputKind::Number(1, 65535), value: |c| c.port.to_string() }, + SettingDefinition { application: ApplicationMode::Restart, key: "bind_addr", label: "Listen address (IP:port)", help: "Leave blank for 0.0.0.0 with the primary HTTP port. Use [::]:8080 for IPv6. CHAN_BIND overrides this field; CHAN_HOST overrides the saved or automatic interface. A launcher --port/-p overrides this address’s port. Tor-only forces loopback. Listener changes require a service restart.", environment: "CHAN_BIND", kind: InputKind::Address, value: |c| c.bind_addr.clone() }, + SettingDefinition { application: ApplicationMode::Restart, key: "rate_limit_gets", label: "Browsing request allowance (per visitor/window)", help: "Raise this if visitors frequently see ‘slow down’. Counters use the trusted forwarded identity or the direct peer. Posting cooldowns and password-attempt protection remain separate.", environment: "CHAN_RATE_GETS", kind: InputKind::Number(1, 1_000_000), value: |c| c.rate_limit_gets.to_string() }, + SettingDefinition { application: ApplicationMode::Restart, key: "rate_limit_window", label: "Browsing window (seconds)", help: "The historical counter rolls over when elapsed whole seconds are greater than this window. Changing it takes effect on restart.", environment: "CHAN_RATE_WINDOW", kind: InputKind::Number(1, 86400), value: |c| c.rate_limit_window.to_string() }, + SettingDefinition { application: ApplicationMode::Restart, key: "rate_limit_policy", label: "Requests counted by the browsing limiter", help: "Legacy counts all methods, including writes, API polling, favicons and banners. Reads counts GET/HEAD only; write-specific protections still apply. Both exempt /static/, /theme-css/, /boards/, admin live logs and backup progress.", environment: "CHAN_RATE_POLICY", kind: InputKind::Policy, value: |c| c.rate_limit_policy.as_str().to_owned() }, + SettingDefinition { application: ApplicationMode::Restart, key: "session_duration", label: "Administrator session lifetime (seconds)", help: "Applies to sessions created after restart. Existing sessions keep their stored expiry. Default: 28800 seconds (8 hours).", environment: "CHAN_SESSION_SECS", kind: InputKind::Number(60, 2_592_000), value: |c| c.session_duration.to_string() }, + SettingDefinition { application: ApplicationMode::Restart, key: "public_readiness_details", label: "Public detailed readiness", help: "Expose operational details at /readyz. /healthz remains minimal and public.", environment: "CHAN_PUBLIC_READINESS_DETAILS", kind: InputKind::Boolean, value: |c| c.public_readiness_details.to_string() }, + SettingDefinition { application: ApplicationMode::Restart, key: "public_metrics_enabled", label: "Public Prometheus metrics", help: "Expose unauthenticated /metrics. Use a trusted scrape path if enabled.", environment: "CHAN_PUBLIC_METRICS_ENABLED", kind: InputKind::Boolean, value: |c| c.public_metrics_enabled.to_string() }, ]; /// One control with saved, effective and active state; never includes secrets. @@ -114,8 +129,8 @@ fn snapshot_from( active: &Config, definitions: &'static [SettingDefinition], ) -> anyhow::Result> { - let raw: toml::Value = toml::from_str(content) - .map_err(|_| anyhow::anyhow!("settings.toml is invalid; repair it before saving"))?; + let raw: toml::Value = + toml::from_str(content).context("settings.toml is invalid; repair it before saving")?; let saved = resolve_file(content, &Environment::Values(&BTreeMap::new()))?; let next = resolve_file(content, &Environment::Process)?; Ok(definitions @@ -142,7 +157,8 @@ fn snapshot_from( SettingField { definition, input, - pending: active_value != next_start, + pending: definition.application == ApplicationMode::Restart + && active_value != next_start, overridden: saved_value != next_start, next_start, active: active_value, @@ -189,7 +205,6 @@ pub(super) fn startup_network_sources( } "require_ffmpeg" => settings.require_ffmpeg.is_some(), "ffmpeg_path" => settings.ffmpeg_path.is_some(), - "ffprobe_path" => settings.ffprobe_path.is_some(), "thumb_size" => settings.thumb_size.is_some(), "job_queue_capacity" => settings.job_queue_capacity.is_some(), "waveform_cache_max_mb" => settings.waveform_cache_max_mb.is_some(), @@ -303,7 +318,11 @@ pub(super) fn environment_override_is_valid( InputKind::OptionalText if definition.key == "log_filter" => { tracing_subscriber::EnvFilter::try_new(value).is_ok() } - _ => true, + InputKind::Boolean + | InputKind::List + | InputKind::Address + | InputKind::Text + | InputKind::OptionalText => true, } } @@ -322,12 +341,12 @@ fn setting_source(definition: &SettingDefinition, active: &Config) -> String { } /// Resolve file configuration without filesystem validation or secret exposure. -fn resolve_file(content: &str, environment: &Environment<'_>) -> anyhow::Result { +pub(crate) fn resolve_file(content: &str, environment: &Environment<'_>) -> anyhow::Result { let mut settings: SettingsFile = super::parse_settings_file_str(content) - .map_err(|_| anyhow::anyhow!("settings.toml is invalid; repair it before saving"))?; + .context("settings.toml is invalid; repair it before saving")?; // A preview does not use secrets. Avoid generating a fallback secret for a // manually authored file that omitted it; startup remains authoritative. - settings.cookie_secret.get_or_insert_with(|| "0".repeat(64)); + let _initialized_secret = settings.cookie_secret.get_or_insert_with(|| "0".repeat(64)); Ok(Config::from_settings(settings, environment)) } @@ -387,7 +406,8 @@ fn parse_settings_form( if field.key == "trusted_proxy_cidrs" { entry .parse::() - .with_context(|| format!("invalid trusted proxy CIDR: {entry}"))?; + .with_context(|| format!("invalid trusted proxy CIDR: {entry}")) + .map(|_validated_value| ())?; } else { ensure!(valid_public_host(entry), "invalid public hostname: {entry}"); } @@ -414,7 +434,8 @@ fn parse_settings_form( } InputKind::Policy => { text.parse::() - .map_err(anyhow::Error::msg)?; + .map_err(anyhow::Error::msg) + .map(|_validated_value| ())?; Some(toml::Value::String(text.to_owned())) } }; @@ -488,7 +509,8 @@ fn validate_network(config: &Config) -> anyhow::Result<()> { ); for cidr in &config.trusted_proxy_cidrs { cidr.parse::() - .context("invalid trusted proxy CIDR")?; + .context("invalid trusted proxy CIDR") + .map(|_validated_value| ())?; } for host in &config.public_hosts { ensure!(valid_public_host(host), "invalid public hostname: {host}"); @@ -525,6 +547,17 @@ pub(super) fn save_root_at( updates: &BTreeMap>, environment: &Environment<'_>, validate: impl Fn(&Config) -> anyhow::Result<()>, +) -> anyhow::Result<()> { + let _lease = settings_lease(path)?; + save_root_locked(path, updates, environment, validate) +} + +/// Validate and publish while holding the cross-process settings lease. +fn save_root_locked( + path: &Path, + updates: &BTreeMap>, + environment: &Environment<'_>, + validate: impl Fn(&Config) -> anyhow::Result<()>, ) -> anyhow::Result<()> { ensure!( !std::fs::symlink_metadata(path)?.file_type().is_symlink(), @@ -537,6 +570,13 @@ pub(super) fn save_root_at( &Environment::Values(&BTreeMap::new()), )?)?; validate(&resolve_file(&after, environment)?)?; + validate_restart_candidate(&after, environment)?; + if crate::restart::container_restart_enabled() { + ensure!( + !crate::restart::container_status()?.phase.active(), + "A restart is in progress; settings cannot be changed" + ); + } atomic_replace(path, &after) } @@ -547,8 +587,9 @@ pub(super) fn save_root_and_commit_at( updates: &BTreeMap>, commit: impl FnOnce() -> anyhow::Result<()>, ) -> anyhow::Result<()> { + let _lease = settings_lease(path)?; let before = std::fs::read_to_string(path).context("read setup settings before commit")?; - save_root_at(path, updates, &Environment::Process, |config| { + save_root_locked(path, updates, &Environment::Process, |config| { validate_network(config)?; config.operator.validate() })?; @@ -567,10 +608,9 @@ pub(super) fn rewrite_root_settings( if updates.keys().any(|key| key.contains('.')) { return certificates::rewrite_tls(content, updates); } - let spans: BTreeMap> = toml::from_str(content) - .map_err(|_| anyhow::anyhow!("settings.toml is invalid; no changes saved"))?; - let mut expected: toml::Value = - toml::from_str(content).map_err(|_| anyhow::anyhow!("invalid settings.toml"))?; + let spans: BTreeMap> = + toml::from_str(content).context("settings.toml is invalid; no changes saved")?; + let mut expected: toml::Value = toml::from_str(content).context("invalid settings.toml")?; let table = expected .as_table_mut() .context("settings root must be a table")?; @@ -578,9 +618,9 @@ pub(super) fn rewrite_root_settings( let mut missing = String::new(); for (key, value) in updates { if let Some(value) = value { - table.insert(key.clone(), value.clone()); + let _previous_value = table.insert(key.clone(), value.clone()); } else { - table.remove(key); + let _previous_value = table.remove(key); } if let Some(span) = spans.get(key) { let range = span.span(); @@ -590,7 +630,9 @@ pub(super) fn rewrite_root_settings( } } else { let prefix = content.get(..range.start).context("invalid TOML span")?; - let start = prefix.rfind('\n').map_or(0, |offset| offset + 1); + let start = prefix + .rfind('\n') + .map_or(0, |offset| offset.saturating_add(1)); edits.push((start..range.end, String::new())); } } else if let Some(value) = value { @@ -598,12 +640,13 @@ pub(super) fn rewrite_root_settings( } } if !missing.is_empty() { - let mut offset = 0; + // Split lines partition content, so these offsets remain bounded by its byte length. + let mut offset = 0_usize; let first_table = content .split_inclusive('\n') .find_map(|line| { let start = offset; - offset += line.len(); + offset = offset.saturating_add(line.len()); let within_value = spans .values() .any(|v| !v.get_ref().is_table() && v.span().contains(&start)); @@ -629,7 +672,7 @@ pub(super) fn rewrite_root_settings( output.replace_range(range, &replacement); } let actual: toml::Value = toml::from_str(&output) - .map_err(|_| anyhow::anyhow!("settings rewrite failed verification; no changes saved"))?; + .context("settings rewrite failed verification; no changes saved")?; ensure!( actual == expected, "settings rewrite changed unrelated values; no changes saved" @@ -638,7 +681,7 @@ pub(super) fn rewrite_root_settings( } /// Persist private, synced bytes before replacing the original settings file. -fn atomic_replace(path: &Path, content: &str) -> anyhow::Result<()> { +pub(crate) fn atomic_replace(path: &Path, content: &str) -> anyhow::Result<()> { let parent = path.parent().context("settings path needs a parent")?; let mut temporary = tempfile::Builder::new() .prefix(".settings_") @@ -653,10 +696,36 @@ fn atomic_replace(path: &Path, content: &str) -> anyhow::Result<()> { .as_file() .sync_all() .context("sync temporary settings file")?; - temporary - .persist(path) - .map_err(|e| e.error) - .context("atomically replace settings.toml")?; + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt as _; + match std::fs::symlink_metadata(path) { + Ok(metadata) => { + ensure!( + metadata.is_file() && metadata.nlink() == 1, + "configuration destination must be a regular file without hardlinks" + ); + let staged = temporary.as_file().metadata()?; + if (staged.uid(), staged.gid()) != (metadata.uid(), metadata.gid()) { + rustix::fs::fchown( + temporary.as_file(), + Some(rustix::fs::Uid::from_raw(metadata.uid())), + Some(rustix::fs::Gid::from_raw(metadata.gid())), + )?; + } + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(error.into()), + } + } + drop( + temporary + .persist(path) + .map_err(|e| e.error) + .context("atomically replace settings.toml")?, + ); + #[cfg(unix)] + std::fs::File::open(parent)?.sync_all()?; Ok(()) } @@ -725,7 +794,8 @@ mod tests { "CREATE TABLE sample(value TEXT); INSERT INTO sample VALUES ('original');", )?; let tx = database.transaction()?; - tx.execute("UPDATE sample SET value='pending'", [])?; + tx.execute("UPDATE sample SET value='pending'", []) + .map(|_affected_rows| ())?; let changes = BTreeMap::from([("port".to_owned(), Some(toml::Value::Integer(9000)))]); let result = super::save_root_and_commit_at(&path, &changes, move || { drop(tx); @@ -784,20 +854,20 @@ mod tests { ("rate_limit_policy", "none"), ] { let mut invalid = form.clone(); - invalid.insert(key.to_owned(), input.to_owned()); + drop(invalid.insert(key.to_owned(), input.to_owned())); ensure!( parse_network_form(&invalid).is_err(), "must reject {key}={input}" ); } let mut invalid = form.clone(); - invalid.insert("cookie_secret".to_owned(), "replacement".to_owned()); + drop(invalid.insert("cookie_secret".to_owned(), "replacement".to_owned())); ensure!( parse_network_form(&invalid).is_err(), "network form must not accept secret rotation" ); let mut missing = form; - missing.remove("trusted_proxy_cidrs"); + drop(missing.remove("trusted_proxy_cidrs")); ensure!( parse_network_form(&missing).is_err(), "partial form must not save" @@ -938,7 +1008,7 @@ mod tests { let before = "# preserved\nport = 8080\nrate_limit_gets = 60\nenable_tor_support = false\n[tls]\nenabled = true\nport = 8443\n"; std::fs::write(&path, before)?; let mut form = valid_form()?; - form.insert("port".to_owned(), "8443".to_owned()); + drop(form.insert("port".to_owned(), "8443".to_owned())); let invalid = parse_network_form(&form)?; let empty = BTreeMap::new(); ensure!( @@ -949,8 +1019,8 @@ mod tests { std::fs::read_to_string(&path)? == before, "failure must leave original bytes intact" ); - form.insert("port".to_owned(), "9000".to_owned()); - form.insert("rate_limit_gets".to_owned(), "120".to_owned()); + drop(form.insert("port".to_owned(), "9000".to_owned())); + drop(form.insert("rate_limit_gets".to_owned(), "120".to_owned())); let valid = parse_network_form(&form)?; let active = resolve_file(before, &Environment::Values(&empty))?; save_network_at(&path, &valid, &Environment::Values(&empty))?; @@ -1090,3 +1160,240 @@ mod tests { Ok(()) } } + +/// Lock the fixed configuration resource across web saves and supervisor transactions. +/// +/// # Errors +/// Rejects unsafe lock paths and concurrent saves, restarts or configuration recovery. +pub(crate) fn settings_lease(path: &Path) -> anyhow::Result { + let parent = path + .parent() + .context("settings path needs a parent")? + .canonicalize()?; + crate::utils::fs_security::reject_symlink_components(&parent)?; + let lock_path = parent.join(".settings.lock"); + if lock_path.exists() { + crate::utils::fs_security::assert_regular_file_no_symlink(&lock_path)?; + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt as _; + ensure!( + std::fs::metadata(&lock_path)?.nlink() == 1, + "settings lock must not be hardlinked" + ); + } + } + let mut options = std::fs::File::options(); + let _access_options = options.read(true).write(true).create(true).truncate(false); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt as _; + let _platform_options = options + .mode(0o600) + .custom_flags(i32::try_from(rustix::fs::OFlags::NOFOLLOW.bits())?); + } + let file = options.open(lock_path)?; + ensure!( + file.metadata()?.is_file(), + "settings lease must be a regular file" + ); + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt as _; + let metadata = file.metadata()?; + ensure!( + metadata.nlink() == 1, + "settings lease must not be hardlinked" + ); + // The updater may establish this inode during an upgrade from a pre-restart build. + // Keep the fixed lock accessible to the web-owned data directory after handoff. + let owner = std::fs::metadata(&parent)?; + if (metadata.uid(), metadata.gid()) != (owner.uid(), owner.gid()) { + rustix::fs::fchown( + &file, + Some(rustix::fs::Uid::from_raw(owner.uid())), + Some(rustix::fs::Gid::from_raw(owner.gid())), + )?; + } + } + file.try_lock().context( + "Another configuration save, update or restart is in progress. Retry after it completes.", + )?; + Ok(file) +} + +/// Compare effective startup controls against the running process, excluding live controls. +/// +/// # Errors +/// Rejects an unreadable or invalid settings file. +pub fn restart_pending() -> anyhow::Result { + let bytes = crate::restart::Store::read(&super::settings_file_path(), 4 * 1024 * 1024)?; + pending_from(std::str::from_utf8(&bytes)?, &CONFIG, &Environment::Process) +} + +/// Pure comparison seam for effective configuration and no-op save tests. +fn pending_from( + content: &str, + active: &Config, + environment: &Environment<'_>, +) -> anyhow::Result { + let next = resolve_file(content, environment)?; + Ok(NETWORK_SETTINGS + .iter() + .chain(runtime::all_definitions()) + .chain(application::SETTINGS) + .any(|field| { + field.application == ApplicationMode::Restart + && (field.value)(active) != (field.value)(&next) + }) + || active.cookie_secret != next.cookie_secret + || active.database_path != next.database_path + || active.upload_dir != next.upload_dir) +} + +/// Compare two saved files using the centralized immutable-setting registry. +/// +/// # Errors +/// Rejects invalid saved or previously initialized configuration. +#[cfg(unix)] +pub(crate) fn restart_files_differ(previous: &str, candidate: &str) -> anyhow::Result { + let active = resolve_file(previous, &Environment::Process)?; + pending_from(candidate, &active, &Environment::Process) +} + +/// Validate the entire saved and effective configuration without creating startup resources. +/// +/// # Errors +/// Rejects malformed configuration, invalid bounds, listener conflicts and unusable TLS material. +pub(crate) fn validate_restart_candidate( + content: &str, + environment: &Environment<'_>, +) -> anyhow::Result<()> { + for env in [&Environment::Values(&BTreeMap::new()), environment] { + let config = resolve_file(content, env)?; + for section in runtime::SECTIONS { + runtime::validate_section(*section, &config)?; + } + for definition in application::SETTINGS { + if let InputKind::Number(min, max) = definition.kind { + let value: u64 = (definition.value)(&config).parse()?; + ensure!( + (min..=max).contains(&value), + "{} is outside its allowed range", + definition.label + ); + } + } + if let Some(path) = &config.backup_directory { + super::backup_storage::validate_backup_directory(path, &config)?; + } + certificates::validate(&config)?; + ensure!( + config.cookie_secret.len() >= 64, + "Cookie secret must have at least 64 characters" + ); + } + Ok(()) +} + +#[cfg(test)] +/// Classification and cross-process write serialization tests. +mod restart_tests { + use super::*; + + /// Complete validation rejects unsafe related configuration before any save can publish it. + #[test] + fn complete_candidate_validation_rejects_invalid_related_paths_and_live_bounds( + ) -> anyhow::Result<()> { + let env = Environment::Values(&BTreeMap::new()); + for content in [ + "backup_directory = 'relative/backups'\n", + "ffmpeg_path = '../untrusted-executable'\n", + "auto_full_backup_interval_hours = 10000\n", + ] { + ensure!( + validate_restart_candidate(content, &env).is_err(), + "invalid related configuration must reject" + ); + } + validate_restart_candidate("ffmpeg_path = 'optional-unavailable-ffmpeg'\n", &env)?; + Ok(()) + } + + /// Live edits and equivalent effective values never create a restart warning. + #[test] + fn pending_state_uses_restart_metadata_and_actual_effective_changes() -> anyhow::Result<()> { + let empty = BTreeMap::new(); + let env = Environment::Values(&empty); + let active = resolve_file("enable_tor_support = false\n", &env)?; + ensure!(!pending_from("enable_tor_support = false\nforum_name = 'Live title'\nauto_full_backup_interval_hours = 48\n", &active, &env)?, "live settings must not request restart"); + ensure!( + !pending_from( + "enable_tor_support = false\nrate_limit_gets = 60\n", + &active, + &env + )?, + "unchanged startup values must not request restart" + ); + ensure!( + pending_from( + "enable_tor_support = false\nrate_limit_gets = 61\n", + &active, + &env + )?, + "changed immutable policy must remain pending" + ); + let overrides = BTreeMap::from([("CHAN_RATE_GETS".to_owned(), "60".to_owned())]); + ensure!( + !pending_from( + "enable_tor_support = false\nrate_limit_gets = 61\n", + &active, + &Environment::Values(&overrides) + )?, + "operator override must prevent ineffective restart warning" + ); + ensure!( + application::SETTINGS.iter().all(|f| f.application + == if f.key == "backup_directory" { + ApplicationMode::Restart + } else { + ApplicationMode::Live + }), + "application controls must declare their actual ownership" + ); + Ok(()) + } + /// A supervisor's lease rejects all configuration writers until it releases the resource. + #[test] + fn supervisor_lease_rejects_config_save_without_partial_mutation() -> anyhow::Result<()> { + let dir = tempfile::tempdir()?; + let path = dir.path().join("settings.toml"); + let before = "enable_tor_support = false\n"; + std::fs::write(&path, before)?; + let lease = settings_lease(&path)?; + let updates = + BTreeMap::from([("rate_limit_gets".to_owned(), Some(toml::Value::Integer(61)))]); + ensure!( + save_root_at( + &path, + &updates, + &Environment::Values(&BTreeMap::new()), + |_| Ok(()) + ) + .is_err(), + "settings save must reject during restart" + ); + ensure!( + std::fs::read_to_string(&path)? == before, + "rejected save must preserve original bytes" + ); + drop(lease); + save_root_at( + &path, + &updates, + &Environment::Values(&BTreeMap::new()), + |_| Ok(()), + )?; + Ok(()) + } +} diff --git a/src/config/admin/application.rs b/src/config/admin/application.rs index 328ed45a..77f7bd71 100644 --- a/src/config/admin/application.rs +++ b/src/config/admin/application.rs @@ -5,22 +5,22 @@ use anyhow::Context as _; /// The sixteen pre-existing runtime controls, with their startup getters. pub static SETTINGS: &[SettingDefinition] = &[ - SettingDefinition { key: "forum_name", label: "Site name", help: "Database-owned after seeding; settings.toml and CHAN_FORUM_NAME initialize an unconfigured site.", environment: "CHAN_FORUM_NAME", kind: InputKind::Text, value: |c| c.forum_name.clone() }, - SettingDefinition { key: "site_subtitle", label: "Homepage subtitle", help: "Database-owned after seeding.", environment: "CHAN_SITE_SUBTITLE", kind: InputKind::Text, value: |c| c.initial_site_subtitle.clone() }, - SettingDefinition { key: "homepage_new_thread_badges_enabled", label: "Homepage new-thread badges", help: "Database-owned after seeding; the legacy combined notification default remains a startup alias.", environment: "CHAN_HOMEPAGE_NEW_THREAD_BADGES", kind: InputKind::Boolean, value: |c| c.initial_homepage_new_thread_badges_enabled.to_string() }, - SettingDefinition { key: "homepage_new_reply_badges_enabled", label: "Homepage new-reply badges", help: "Database-owned after seeding.", environment: "CHAN_HOMEPAGE_NEW_REPLY_BADGES", kind: InputKind::Boolean, value: |c| c.initial_homepage_new_reply_badges_enabled.to_string() }, - SettingDefinition { key: "thread_new_reply_badges_enabled", label: "Thread new-reply badges", help: "Database-owned after seeding.", environment: "CHAN_THREAD_NEW_REPLY_BADGES", kind: InputKind::Boolean, value: |c| c.initial_thread_new_reply_badges_enabled.to_string() }, - SettingDefinition { key: "default_theme", label: "Default visitor theme", help: "Database-owned catalog default. Per-board and visitor selections keep precedence.", environment: "CHAN_DEFAULT_THEME", kind: InputKind::Text, value: |c| c.initial_default_theme.clone() }, - SettingDefinition { key: "enabled_builtin_themes", label: "Enabled built-in themes", help: "Database theme catalog is authoritative after seeding; custom themes are managed in Appearance.", environment: "", kind: InputKind::List, value: |c| c.initial_enabled_builtin_themes.join(", ") }, - SettingDefinition { key: "ffmpeg_timeout_secs", label: "FFmpeg job timeout (seconds)", help: "Saved with Media settings in one database transaction. CHAN_FFMPEG_TIMEOUT_SECS overrides on restart. Applies live to new subprocess calls.", environment: "CHAN_FFMPEG_TIMEOUT_SECS", kind: InputKind::Number(30,86400), value: |c| c.ffmpeg_timeout_secs.to_string() }, - SettingDefinition { key: "media_auto_prune_enabled", label: "Automatic post-media pruning", help: "Database-owned; saving may immediately run pruning. Global upload gates remain separate.", environment: "CHAN_MEDIA_AUTO_PRUNE_ENABLED", kind: InputKind::Boolean, value: |c| c.initial_media_auto_prune_enabled.to_string() }, - SettingDefinition { key: "media_max_active_content_size_bytes", label: "Active post-media cap (bytes)", help: "Database-owned; 0 is unset. Full-size media pruning retains thumbnails where possible.", environment: "CHAN_MEDIA_MAX_ACTIVE_CONTENT_SIZE_BYTES", kind: InputKind::Number(0, u64::MAX), value: |c| c.initial_media_max_active_content_size_bytes.to_string() }, - SettingDefinition { key: "backup_directory", label: "Saved backup directory", help: "File-owned, restart required; existing backups are not relocated.", environment: "CHAN_BACKUP_DIRECTORY", kind: InputKind::Text, value: |c| c.backup_directory.clone().unwrap_or_else(|| super::super::data_dir().join("backups")).display().to_string() }, - SettingDefinition { key: "auto_full_backup_interval_hours", label: "Automatic backup interval (hours)", help: "File-owned shared live scheduler; environment wins on restart.", environment: "CHAN_AUTO_FULL_BACKUP_HOURS", kind: InputKind::Number(0,8760), value: |c| c.auto_full_backup_interval_hours.to_string() }, - SettingDefinition { key: "auto_full_backup_copies_to_keep", label: "Automatic backup retention (copies)", help: "File-owned shared live scheduler; environment wins on restart.", environment: "CHAN_AUTO_FULL_BACKUP_COPIES", kind: InputKind::Number(1,1000), value: |c| c.auto_full_backup_copies_to_keep.to_string() }, - SettingDefinition { key: "auto_full_backup_include_tor_hidden_service_keys", label: "Include Tor identities in automatic backups", help: "File-owned shared live scheduler; protect archives containing private onion identities.", environment: "CHAN_AUTO_FULL_BACKUP_INCLUDE_TOR_KEYS", kind: InputKind::Boolean, value: |c| c.auto_full_backup_include_tor_hidden_service_keys.to_string() }, - SettingDefinition { key: "auto_full_backup_storage_mode", label: "Automatic backup format", help: "File-owned shared live scheduler; directory or split_zip.", environment: "CHAN_AUTO_FULL_BACKUP_STORAGE_MODE", kind: InputKind::Text, value: |c| c.auto_full_backup_storage_mode.clone() }, - SettingDefinition { key: "auto_full_backup_split_zip_part_size_gib", label: "Automatic ZIP part size (GiB)", help: "File-owned shared live scheduler; applies to subsequent backups.", environment: "CHAN_AUTO_FULL_BACKUP_SPLIT_ZIP_PART_SIZE_GIB", kind: InputKind::Number(1,64), value: |c| (c.auto_full_backup_split_zip_part_size_bytes / (1024*1024*1024)).to_string() }, + SettingDefinition { application: crate::config::admin::ApplicationMode::Live, key: "forum_name", label: "Site name", help: "Database-owned after seeding; settings.toml and CHAN_FORUM_NAME initialize an unconfigured site.", environment: "CHAN_FORUM_NAME", kind: InputKind::Text, value: |c| c.forum_name.clone() }, + SettingDefinition { application: crate::config::admin::ApplicationMode::Live, key: "site_subtitle", label: "Homepage subtitle", help: "Database-owned after seeding.", environment: "CHAN_SITE_SUBTITLE", kind: InputKind::Text, value: |c| c.initial_site_subtitle.clone() }, + SettingDefinition { application: crate::config::admin::ApplicationMode::Live, key: "homepage_new_thread_badges_enabled", label: "Homepage new-thread badges", help: "Database-owned after seeding; the legacy combined notification default remains a startup alias.", environment: "CHAN_HOMEPAGE_NEW_THREAD_BADGES", kind: InputKind::Boolean, value: |c| c.initial_homepage_new_thread_badges_enabled.to_string() }, + SettingDefinition { application: crate::config::admin::ApplicationMode::Live, key: "homepage_new_reply_badges_enabled", label: "Homepage new-reply badges", help: "Database-owned after seeding.", environment: "CHAN_HOMEPAGE_NEW_REPLY_BADGES", kind: InputKind::Boolean, value: |c| c.initial_homepage_new_reply_badges_enabled.to_string() }, + SettingDefinition { application: crate::config::admin::ApplicationMode::Live, key: "thread_new_reply_badges_enabled", label: "Thread new-reply badges", help: "Database-owned after seeding.", environment: "CHAN_THREAD_NEW_REPLY_BADGES", kind: InputKind::Boolean, value: |c| c.initial_thread_new_reply_badges_enabled.to_string() }, + SettingDefinition { application: crate::config::admin::ApplicationMode::Live, key: "default_theme", label: "Default visitor theme", help: "Database-owned catalog default. Per-board and visitor selections keep precedence.", environment: "CHAN_DEFAULT_THEME", kind: InputKind::Text, value: |c| c.initial_default_theme.clone() }, + SettingDefinition { application: crate::config::admin::ApplicationMode::Live, key: "enabled_builtin_themes", label: "Enabled built-in themes", help: "Database theme catalog is authoritative after seeding; custom themes are managed in Appearance.", environment: "", kind: InputKind::List, value: |c| c.initial_enabled_builtin_themes.join(", ") }, + SettingDefinition { application: crate::config::admin::ApplicationMode::Live, key: "ffmpeg_timeout_secs", label: "FFmpeg job timeout (seconds)", help: "Saved with Media settings in one database transaction. CHAN_FFMPEG_TIMEOUT_SECS overrides on restart. Applies live to new subprocess calls.", environment: "CHAN_FFMPEG_TIMEOUT_SECS", kind: InputKind::Number(30,86400), value: |c| c.ffmpeg_timeout_secs.to_string() }, + SettingDefinition { application: crate::config::admin::ApplicationMode::Live, key: "media_auto_prune_enabled", label: "Automatic post-media pruning", help: "Database-owned; saving may immediately run pruning. Global upload gates remain separate.", environment: "CHAN_MEDIA_AUTO_PRUNE_ENABLED", kind: InputKind::Boolean, value: |c| c.initial_media_auto_prune_enabled.to_string() }, + SettingDefinition { application: crate::config::admin::ApplicationMode::Live, key: "media_max_active_content_size_bytes", label: "Active post-media cap (bytes)", help: "Database-owned; 0 is unset. Full-size media pruning retains thumbnails where possible.", environment: "CHAN_MEDIA_MAX_ACTIVE_CONTENT_SIZE_BYTES", kind: InputKind::Number(0, u64::MAX), value: |c| c.initial_media_max_active_content_size_bytes.to_string() }, + SettingDefinition { application: crate::config::admin::ApplicationMode::Restart, key: "backup_directory", label: "Saved backup directory", help: "File-owned, restart required; existing backups are not relocated.", environment: "CHAN_BACKUP_DIRECTORY", kind: InputKind::Text, value: |c| c.backup_directory.clone().unwrap_or_else(|| super::super::data_dir().join("backups")).display().to_string() }, + SettingDefinition { application: crate::config::admin::ApplicationMode::Live, key: "auto_full_backup_interval_hours", label: "Automatic backup interval (hours)", help: "File-owned shared live scheduler; environment wins on restart.", environment: "CHAN_AUTO_FULL_BACKUP_HOURS", kind: InputKind::Number(0,8760), value: |c| c.auto_full_backup_interval_hours.to_string() }, + SettingDefinition { application: crate::config::admin::ApplicationMode::Live, key: "auto_full_backup_copies_to_keep", label: "Automatic backup retention (copies)", help: "File-owned shared live scheduler; environment wins on restart.", environment: "CHAN_AUTO_FULL_BACKUP_COPIES", kind: InputKind::Number(1,1000), value: |c| c.auto_full_backup_copies_to_keep.to_string() }, + SettingDefinition { application: crate::config::admin::ApplicationMode::Live, key: "auto_full_backup_include_tor_hidden_service_keys", label: "Include Tor identities in automatic backups", help: "File-owned shared live scheduler; protect archives containing private onion identities.", environment: "CHAN_AUTO_FULL_BACKUP_INCLUDE_TOR_KEYS", kind: InputKind::Boolean, value: |c| c.auto_full_backup_include_tor_hidden_service_keys.to_string() }, + SettingDefinition { application: crate::config::admin::ApplicationMode::Live, key: "auto_full_backup_storage_mode", label: "Automatic backup format", help: "File-owned shared live scheduler; directory or split_zip.", environment: "CHAN_AUTO_FULL_BACKUP_STORAGE_MODE", kind: InputKind::Text, value: |c| c.auto_full_backup_storage_mode.clone() }, + SettingDefinition { application: crate::config::admin::ApplicationMode::Live, key: "auto_full_backup_split_zip_part_size_gib", label: "Automatic ZIP part size (GiB)", help: "File-owned shared live scheduler; applies to subsequent backups.", environment: "CHAN_AUTO_FULL_BACKUP_SPLIT_ZIP_PART_SIZE_GIB", kind: InputKind::Number(1,64), value: |c| (c.auto_full_backup_split_zip_part_size_bytes / (1024*1024*1024)).to_string() }, ]; /// Report actual live values rather than immutable startup defaults. @@ -75,10 +75,12 @@ pub fn snapshot( "Startup file/environment/default; live atomic timeout" } .into(); - field.pending = field.active != field.next_start; + field.pending = field.definition.application == super::ApplicationMode::Restart + && field.active != field.next_start; field.overridden = field.saved != field.next_start; } else { - field.pending = field.active != field.next_start; + field.pending = field.definition.application == super::ApplicationMode::Restart + && field.active != field.next_start; field.source = backup_source(field.definition, &super::Environment::Process); } } diff --git a/src/config/admin/certificates.rs b/src/config/admin/certificates.rs index 0a1865d7..4daf0a6b 100644 --- a/src/config/admin/certificates.rs +++ b/src/config/admin/certificates.rs @@ -6,18 +6,18 @@ use std::{fmt::Write as _, path::Path, sync::Arc}; /// All twelve inventoried TLS leaves, using their authoritative TOML paths. pub static SETTINGS: &[SettingDefinition] = &[ - SettingDefinition { key: "tls.enabled", label: "Native HTTPS listener", environment: "", kind: InputKind::Boolean, value: |c| c.tls.enabled.to_string(), help: "Enable built-in HTTPS after choosing a certificate source below. Reverse-proxy HTTPS does not require this listener." }, - SettingDefinition { key: "tls.require_https", label: "Require native HTTPS", environment: "", kind: InputKind::Boolean, value: |c| c.tls.require_https.to_string(), help: "Disables public plaintext application access when native HTTPS is enabled. Verify the HTTPS listener is reachable before restarting." }, - SettingDefinition { key: "tls.port", label: "HTTPS port", environment: "", kind: InputKind::Number(1, 65535), value: |c| c.tls.port.to_string(), help: "Must differ from the primary HTTP and redirect ports. Your service launcher and firewall must allow this port." }, - SettingDefinition { key: "tls.redirect_http", label: "HTTP redirect listener", environment: "", kind: InputKind::Boolean, value: |c| c.tls.redirect_http.to_string(), help: "Adds a separate listener that redirects to HTTPS; requires a configured public hostname." }, - SettingDefinition { key: "tls.http_port", label: "HTTP redirect port", environment: "", kind: InputKind::Number(1, 65535), value: |c| c.tls.http_port.to_string(), help: "Separate from the primary application listener. Default: 8080." }, - SettingDefinition { key: "tls.acme.enabled", label: "Automatic ACME certificates", environment: "", kind: InputKind::Boolean, value: |c| c.tls.acme.enabled.to_string(), help: "Requires a tls-acme build, public DNS and reachable HTTPS for TLS-ALPN-01 validation. Issuance happens after restart; use staging first." }, - SettingDefinition { key: "tls.acme.domains", label: "ACME domain names", environment: "", kind: InputKind::List, value: |c| c.tls.acme.domains.join(", "), help: "DNS names only, one per line or separated by commas. These must resolve to your public HTTPS listener." }, - SettingDefinition { key: "tls.acme.email", label: "ACME contact email (optional)", environment: "", kind: InputKind::OptionalText, value: |c| c.tls.acme.email.clone().unwrap_or_default(), help: "Optional account contact. Leave blank to omit. No ACME request is sent by this form." }, - SettingDefinition { key: "tls.acme.staging", label: "ACME staging directory", environment: "", kind: InputKind::Boolean, value: |c| c.tls.acme.staging.to_string(), help: "Staging certificates are for testing and are not trusted by browsers. Preserve your current selection until public issuance is ready." }, - SettingDefinition { key: "tls.acme.cache_dir", label: "ACME private cache directory", environment: "", kind: InputKind::OptionalText, value: |c| c.tls.acme.cache_dir.clone(), help: "Absolute path or path relative to the data directory. Leave blank for the loader default. Contains account credentials; restrict service-user access." }, - SettingDefinition { key: "tls.manual_cert.cert_path", label: "Manual PEM certificate-chain path", environment: "", kind: InputKind::OptionalText, value: |c| c.tls.manual_cert.as_ref().map_or_else(String::new, |m| m.cert_path.clone()), help: "Install the PEM files on the service host first. Absolute path or relative to the active data directory; chain and key are checked before saving. No file contents are displayed." }, - SettingDefinition { key: "tls.manual_cert.key_path", label: "Manual PEM private-key path", environment: "", kind: InputKind::OptionalText, value: |c| c.tls.manual_cert.as_ref().map_or_else(String::new, |m| m.key_path.clone()), help: "Select both certificate and key, or leave both blank to remove the manual source. Manual certificates take priority over ACME. Renew files offline, then restart to reload." }, + SettingDefinition { application: crate::config::admin::ApplicationMode::Restart, key: "tls.enabled", label: "Native HTTPS listener", environment: "", kind: InputKind::Boolean, value: |c| c.tls.enabled.to_string(), help: "Enable built-in HTTPS after choosing a certificate source below. Reverse-proxy HTTPS does not require this listener." }, + SettingDefinition { application: crate::config::admin::ApplicationMode::Restart, key: "tls.require_https", label: "Require native HTTPS", environment: "", kind: InputKind::Boolean, value: |c| c.tls.require_https.to_string(), help: "Disables public plaintext application access when native HTTPS is enabled. Verify the HTTPS listener is reachable before restarting." }, + SettingDefinition { application: crate::config::admin::ApplicationMode::Restart, key: "tls.port", label: "HTTPS port", environment: "", kind: InputKind::Number(1, 65535), value: |c| c.tls.port.to_string(), help: "Must differ from the primary HTTP and redirect ports. Your service launcher and firewall must allow this port." }, + SettingDefinition { application: crate::config::admin::ApplicationMode::Restart, key: "tls.redirect_http", label: "HTTP redirect listener", environment: "", kind: InputKind::Boolean, value: |c| c.tls.redirect_http.to_string(), help: "Adds a separate listener that redirects to HTTPS; requires a configured public hostname." }, + SettingDefinition { application: crate::config::admin::ApplicationMode::Restart, key: "tls.http_port", label: "HTTP redirect port", environment: "", kind: InputKind::Number(1, 65535), value: |c| c.tls.http_port.to_string(), help: "Separate from the primary application listener. Default: 8080." }, + SettingDefinition { application: crate::config::admin::ApplicationMode::Restart, key: "tls.acme.enabled", label: "Automatic ACME certificates", environment: "", kind: InputKind::Boolean, value: |c| c.tls.acme.enabled.to_string(), help: "Requires a tls-acme build, public DNS and reachable HTTPS for TLS-ALPN-01 validation. Issuance happens after restart; use staging first." }, + SettingDefinition { application: crate::config::admin::ApplicationMode::Restart, key: "tls.acme.domains", label: "ACME domain names", environment: "", kind: InputKind::List, value: |c| c.tls.acme.domains.join(", "), help: "DNS names only, one per line or separated by commas. These must resolve to your public HTTPS listener." }, + SettingDefinition { application: crate::config::admin::ApplicationMode::Restart, key: "tls.acme.email", label: "ACME contact email (optional)", environment: "", kind: InputKind::OptionalText, value: |c| c.tls.acme.email.clone().unwrap_or_default(), help: "Optional account contact. Leave blank to omit. No ACME request is sent by this form." }, + SettingDefinition { application: crate::config::admin::ApplicationMode::Restart, key: "tls.acme.staging", label: "ACME staging directory", environment: "", kind: InputKind::Boolean, value: |c| c.tls.acme.staging.to_string(), help: "Staging certificates are for testing and are not trusted by browsers. Preserve your current selection until public issuance is ready." }, + SettingDefinition { application: crate::config::admin::ApplicationMode::Restart, key: "tls.acme.cache_dir", label: "ACME private cache directory", environment: "", kind: InputKind::OptionalText, value: |c| c.tls.acme.cache_dir.clone(), help: "Absolute path or path relative to the data directory. Leave blank for the loader default. Contains account credentials; restrict service-user access." }, + SettingDefinition { application: crate::config::admin::ApplicationMode::Restart, key: "tls.manual_cert.cert_path", label: "Manual PEM certificate-chain path", environment: "", kind: InputKind::OptionalText, value: |c| c.tls.manual_cert.as_ref().map_or_else(String::new, |m| m.cert_path.clone()), help: "Install the PEM files on the service host first. Absolute path or relative to the active data directory; chain and key are checked before saving. No file contents are displayed." }, + SettingDefinition { application: crate::config::admin::ApplicationMode::Restart, key: "tls.manual_cert.key_path", label: "Manual PEM private-key path", environment: "", kind: InputKind::OptionalText, value: |c| c.tls.manual_cert.as_ref().map_or_else(String::new, |m| m.key_path.clone()), help: "Select both certificate and key, or leave both blank to remove the manual source. Manual certificates take priority over ACME. Renew files offline, then restart to reload." }, ]; /// Save one complete certificate workflow, validating saved and effective listeners. @@ -65,7 +65,7 @@ fn confirm_cutover( } /// Validate certificate-source selection without creating files or contacting ACME. -fn validate(config: &Config) -> anyhow::Result<()> { +pub(super) fn validate(config: &Config) -> anyhow::Result<()> { super::validate_network(config)?; let tls = &config.tls; ensure!( @@ -94,7 +94,9 @@ fn validate(config: &Config) -> anyhow::Result<()> { "ACME requires a public DNS name" ); } - validate_path(&tls.acme.cache_dir)?; + if !tls.acme.cache_dir.is_empty() { + validate_path(&tls.acme.cache_dir)?; + } if let Some(manual) = &tls.manual_cert { validate_path(&manual.cert_path)?; validate_path(&manual.key_path)?; @@ -159,7 +161,8 @@ fn validate_pair(cert_path: &Path, key_path: &Path) -> anyhow::Result<()> { .context("TLS protocol configuration")? .with_no_client_auth() .with_single_cert(certificates, key) - .context("certificate and key do not form a usable pair")?; + .context("certificate and key do not form a usable pair") + .map(|_validated_value| ())?; Ok(()) } @@ -173,7 +176,9 @@ fn read_material(path: &Path) -> anyhow::Result> { "certificate/key must be a regular file no larger than 1 MiB" ); let mut bytes = Vec::new(); - file.take(1_048_577).read_to_end(&mut bytes)?; + file.take(1_048_577) + .read_to_end(&mut bytes) + .map(|_bytes_read| ())?; ensure!( bytes.len() <= 1_048_576, "certificate/key grew beyond 1 MiB" @@ -196,7 +201,7 @@ pub(super) fn rewrite_tls( matches!(table, "tls" | "tls.acme" | "tls.manual_cert"), "unknown TLS table" ); - groups + let _previous_value = groups .entry(table) .or_default() .insert(key.to_owned(), value.clone()); @@ -210,11 +215,13 @@ pub(super) fn rewrite_tls( /// Locate real headers by parsing them; ignore header-like lines in scalar values. fn table_range(content: &str, table: &str) -> anyhow::Result>> { let spans: BTreeMap> = toml::from_str(content)?; - let mut offset = 0; + let mut offset = 0_usize; let mut found = None; for line in content.split_inclusive('\n') { let start = offset; - offset += line.len(); + offset = offset + .checked_add(line.len()) + .context("TLS table offset overflow")?; if !line.trim_start().starts_with('[') || spans .values() @@ -260,14 +267,14 @@ fn rewrite_table( } for (key, value) in updates { if let Some(value) = value { - target.insert(key.clone(), value.clone()); + let _previous_value = target.insert(key.clone(), value.clone()); } else { - target.remove(key); + let _previous_value = target.remove(key); } } let remove_manual = table == "tls.manual_cert" && target.is_empty(); if remove_manual { - expected + let _previous_value = expected .get_mut("tls") .and_then(toml::Value::as_table_mut) .context("missing TLS")? @@ -276,7 +283,9 @@ fn rewrite_table( let mut output = content.to_owned(); if let Some(range) = table_range(content, table)? { let section = content.get(range.clone()).context("invalid TLS range")?; - let header_end = section.find('\n').map_or(section.len(), |p| p + 1); + let header_end = section + .find('\n') + .map_or(section.len(), |p| p.saturating_add(1)); let body = section.get(header_end..).context("invalid TLS header")?; // Parsing the isolated body ensures no nested/inline table gets silently overwritten. let parsed: BTreeMap> = @@ -361,7 +370,7 @@ mod tests { fn tls_roundtrip_preserves_comments_and_clears_optional_sources() -> anyhow::Result<()> { let before = "# keep root\nport = 3000\n[tls] # listener\nenabled = false # keep inline\n[tls.acme]\nstaging = true # test CA\ndomains = [\n 'example.test', # keep note\n]\n[unrelated]\nvalue = 42\n"; let mut form = form(before)?; - form.insert("tls.port".into(), "9443".into()); + let _previous_value = form.insert("tls.port".into(), "9443".into()); let updates = super::super::parse_settings_form(SETTINGS, &form)?; let after = rewrite_tls(before, &updates)?; for comment in [ @@ -399,7 +408,7 @@ mod tests { ("tls.acme.cache_dir", "../outside"), ] { let mut form = form(before)?; - form.insert(key.into(), value.into()); + let _previous_value = form.insert(key.into(), value.into()); let updates = super::super::parse_settings_form(SETTINGS, &form)?; ensure!( super::super::save_root_at( diff --git a/src/config/admin/runtime.rs b/src/config/admin/runtime.rs index 4899bc8f..c7676aac 100644 --- a/src/config/admin/runtime.rs +++ b/src/config/admin/runtime.rs @@ -81,7 +81,14 @@ impl RuntimeSection { pub const fn route_key(self) -> &'static str { match self { Self::Maintenance => "maintenance", - _ => self.key(), + Self::Tor + | Self::Media + | Self::System + | Self::Https + | Self::Access + | Self::Display + | Self::Logging + | Self::Timeouts => self.key(), } } } @@ -134,25 +141,10 @@ pub fn save_section( let after = super::rewrite_root_settings(&before, &updates)?; let saved = super::resolve_file(&after, &Environment::Values(&empty))?; validate_section(section, &saved)?; - if section == RuntimeSection::Media { - for (tool, old, new, required) in [ - ( - "ffmpeg", - &previous.ffmpeg_path, - &saved.ffmpeg_path, - saved.require_ffmpeg, - ), - ( - "ffprobe", - &previous.ffprobe_path, - &saved.ffprobe_path, - false, - ), - ] { - if old != new || required { - probe_tool(tool, new)?; - } - } + if section == RuntimeSection::Media + && (previous.ffmpeg_path != saved.ffmpeg_path || saved.require_ffmpeg) + { + probe_tool("ffmpeg", &saved.ffmpeg_path)?; } super::save_root_at(&path, &updates, &Environment::Process, |candidate| { validate_section(section, candidate) @@ -160,7 +152,7 @@ pub fn save_section( } /// Validate related section values without changing paths, workers or listeners. -fn validate_section(section: RuntimeSection, config: &Config) -> anyhow::Result<()> { +pub(super) fn validate_section(section: RuntimeSection, config: &Config) -> anyhow::Result<()> { super::validate_network(config)?; for definition in section.definitions() { if let InputKind::Number(min, max) = definition.kind { @@ -178,7 +170,20 @@ fn validate_section(section: RuntimeSection, config: &Config) -> anyhow::Result< config .tor_service_nickname .parse::() - .context("invalid Tor service nickname")?; + .context("invalid Tor service nickname") + .map(|_validated_value| ())?; + } + if section == RuntimeSection::Media { + // Existing operator overrides may name an unavailable optional PATH tool. + // Newly submitted tool paths still receive the stricter executable probe in save_section. + ensure!( + !config.ffmpeg_path.is_empty() + && !config.ffmpeg_path.chars().any(char::is_control) + && !std::path::Path::new(&config.ffmpeg_path) + .components() + .any(|part| part == std::path::Component::ParentDir), + "FFmpeg path must be nonempty without control characters or parent traversal" + ); } Ok(()) } @@ -227,46 +232,45 @@ fn probe_tool(tool: &str, path: &str) -> anyhow::Result<()> { /// Dedicated tor controls. static TOR_SETTINGS: &[SettingDefinition] = &[ - SettingDefinition { key: "enable_tor_support", label: "Built-in onion service", environment: "CHAN_TOR_SUPPORT", kind: InputKind::Boolean, value: |c| c.enable_tor_support.to_string(), help: "Enable the built-in Arti onion service. Disabling it removes onion access after restart." }, - SettingDefinition { key: "tor_only", label: "Tor-only listener binding", environment: "CHAN_TOR_ONLY", kind: InputKind::Boolean, value: |c| c.tor_only.to_string(), help: "Requires Tor support and forces the primary listener to loopback. Review current clearnet access before restarting. Cannot use ACME." }, - SettingDefinition { key: "tor_bootstrap_timeout_secs", label: "Tor bootstrap timeout (seconds)", environment: "CHAN_TOR_BOOTSTRAP_TIMEOUT", kind: InputKind::Number(1, 3600), value: |c| c.tor_bootstrap_timeout_secs.to_string(), help: "Deadline per bootstrap attempt; increase on censored networks. Default: 120 seconds." }, - SettingDefinition { key: "tor_max_concurrent_streams", label: "Concurrent Tor streams", environment: "CHAN_TOR_MAX_STREAMS", kind: InputKind::Number(1, 65536), value: |c| c.tor_max_concurrent_streams.to_string(), help: "Each stream holds a file descriptor. Excess streams are dropped. Default: 512." }, - SettingDefinition { key: "tor_service_nickname", label: "Onion-service identity nickname", environment: "CHAN_TOR_NICKNAME", kind: InputKind::Text, value: |c| c.tor_service_nickname.clone(), help: "Changing this selects a different identity in the Tor keystore and can change the onion address. Back up the identity first." }, + SettingDefinition { application: crate::config::admin::ApplicationMode::Restart, key: "enable_tor_support", label: "Built-in onion service", environment: "CHAN_TOR_SUPPORT", kind: InputKind::Boolean, value: |c| c.enable_tor_support.to_string(), help: "Enable the built-in Arti onion service. Disabling it removes onion access after restart." }, + SettingDefinition { application: crate::config::admin::ApplicationMode::Restart, key: "tor_only", label: "Tor-only listener binding", environment: "CHAN_TOR_ONLY", kind: InputKind::Boolean, value: |c| c.tor_only.to_string(), help: "Requires Tor support and forces the primary listener to loopback. Review current clearnet access before restarting. Cannot use ACME." }, + SettingDefinition { application: crate::config::admin::ApplicationMode::Restart, key: "tor_bootstrap_timeout_secs", label: "Tor bootstrap timeout (seconds)", environment: "CHAN_TOR_BOOTSTRAP_TIMEOUT", kind: InputKind::Number(1, 3600), value: |c| c.tor_bootstrap_timeout_secs.to_string(), help: "Deadline per bootstrap attempt; increase on censored networks. Default: 120 seconds." }, + SettingDefinition { application: crate::config::admin::ApplicationMode::Restart, key: "tor_max_concurrent_streams", label: "Concurrent Tor streams", environment: "CHAN_TOR_MAX_STREAMS", kind: InputKind::Number(1, 65536), value: |c| c.tor_max_concurrent_streams.to_string(), help: "Each stream holds a file descriptor. Excess streams are dropped. Default: 512." }, + SettingDefinition { application: crate::config::admin::ApplicationMode::Restart, key: "tor_service_nickname", label: "Onion-service identity nickname", environment: "CHAN_TOR_NICKNAME", kind: InputKind::Text, value: |c| c.tor_service_nickname.clone(), help: "Changing this selects a different identity in the Tor keystore and can change the onion address. Back up the identity first." }, ]; /// Dedicated media controls. static MEDIA_SETTINGS: &[SettingDefinition] = &[ - SettingDefinition { key: "max_image_size_mb", label: "New-board image upload default (MiB)", environment: "CHAN_MAX_IMAGE_MB", kind: InputKind::Number(1, 100), value: |c| (c.max_image_size / (1024 * 1024)).to_string(), help: "Upload limit for newly created boards. Existing boards retain their own caps. New-board PDFs also derive their cap from this default." }, - SettingDefinition { key: "max_video_size_mb", label: "New-board video upload default (MiB)", environment: "CHAN_MAX_VIDEO_MB", kind: InputKind::Number(1, 2048), value: |c| (c.max_video_size / (1024 * 1024)).to_string(), help: "Upload limit for newly created boards; edit existing per-board caps under Boards." }, - SettingDefinition { key: "max_audio_size_mb", label: "New-board audio upload default (MiB)", environment: "CHAN_MAX_AUDIO_MB", kind: InputKind::Number(1, 512), value: |c| (c.max_audio_size / (1024 * 1024)).to_string(), help: "Upload limit for newly created boards; existing boards retain their stored caps." }, - SettingDefinition { key: "enable_any_file_uploads_feature", label: "Master arbitrary-file upload gate", environment: "CHAN_ENABLE_ANY_FILE_UPLOADS_FEATURE", kind: InputKind::Boolean, value: |c| c.enable_any_file_uploads_feature.to_string(), help: "Both this global gate and the per-board arbitrary-file checkbox must be enabled. This does not change individual board preferences." }, - SettingDefinition { key: "require_ffmpeg", label: "Require FFmpeg at startup", environment: "CHAN_REQUIRE_FFMPEG", kind: InputKind::Boolean, value: |c| c.require_ffmpeg.to_string(), help: "Missing FFmpeg becomes a startup error. FFprobe is detected separately; this flag does not make missing FFprobe fatal." }, - SettingDefinition { key: "ffmpeg_path", label: "FFmpeg executable", environment: "CHAN_FFMPEG_PATH", kind: InputKind::Text, value: |c| c.ffmpeg_path.clone(), help: "Use ffmpeg for PATH lookup, or an absolute executable path. New tool paths are checked with a bounded version probe before saving." }, - SettingDefinition { key: "ffprobe_path", label: "FFprobe executable", environment: "CHAN_FFPROBE_PATH", kind: InputKind::Text, value: |c| c.ffprobe_path.clone(), help: "Use ffprobe for PATH lookup, or an absolute executable path. Missing FFprobe reduces media metadata support." }, - SettingDefinition { key: "thumb_size", label: "Generated thumbnail dimension (pixels)", environment: "CHAN_THUMB_SIZE", kind: InputKind::Number(16, 4096), value: |c| c.thumb_size.to_string(), help: "Applies to thumbnails generated after restart. Existing thumbnails are not regenerated." }, - SettingDefinition { key: "job_queue_capacity", label: "Background queue capacity (pending jobs)", environment: "CHAN_JOB_QUEUE_CAPACITY", kind: InputKind::Number(0, 1_000_000), value: |c| c.job_queue_capacity.to_string(), help: "0 is unlimited. Once full, the queue drops new media jobs with a warning. Current queue pressure is shown in Site Health." }, - SettingDefinition { key: "waveform_cache_max_mb", label: "Thumbnail/waveform cache budget (MiB)", environment: "CHAN_WAVEFORM_CACHE_MAX_MB", kind: InputKind::Number(0, 1_048_576), value: |c| (c.waveform_cache_max_bytes / (1024 * 1024)).to_string(), help: "Oldest cache files are evicted by a background task when the budget is exceeded. 0 disables eviction." }, - SettingDefinition { key: "archive_before_prune", label: "Always archive overflow threads", environment: "CHAN_ARCHIVE_BEFORE_PRUNE", kind: InputKind::Boolean, value: |c| c.archive_before_prune.to_string(), help: "Overrides per-board overflow deletion/archive policy when enabled. Boards still retain their stored allow_archive preference." }, - SettingDefinition { key: "media_reconcile_repair_enabled", label: "Permit safe reconciliation repairs", environment: "CHAN_MEDIA_RECONCILE_REPAIR_ENABLED", kind: InputKind::Boolean, value: |c| c.media_reconcile_repair_enabled.to_string(), help: "Separate repair permission from periodic audits. Only bounded, verified safe repairs are scheduled." }, - SettingDefinition { key: "media_reconcile_interval_hours", label: "Reconciliation audit interval (hours)", environment: "CHAN_MEDIA_RECONCILE_INTERVAL_HOURS", kind: InputKind::Number(0, 8760), value: |c| c.media_reconcile_interval_hours.to_string(), help: "0 disables periodic audit passes. Manual maintenance remains available." }, - SettingDefinition { key: "media_reconcile_files_per_pass", label: "Filesystem entries per audit pass", environment: "CHAN_MEDIA_RECONCILE_FILES_PER_PASS", kind: InputKind::Number(0, 1_000_000), value: |c| c.media_reconcile_files_per_pass.to_string(), help: "Bound the filesystem work performed by each reconciliation pass." }, - SettingDefinition { key: "media_reconcile_database_rows_per_pass", label: "Reference rows per audit pass", environment: "CHAN_MEDIA_RECONCILE_DATABASE_ROWS_PER_PASS", kind: InputKind::Number(0, 10_000_000), value: |c| c.media_reconcile_database_rows_per_pass.to_string(), help: "Bound the authoritative-reference snapshot used by each pass." }, - SettingDefinition { key: "media_reconcile_hash_bytes_per_pass", label: "Bytes hashed per audit pass", environment: "CHAN_MEDIA_RECONCILE_HASH_BYTES_PER_PASS", kind: InputKind::Number(0, 1_099_511_627_776), value: |c| c.media_reconcile_hash_bytes_per_pass.to_string(), help: "I/O budget in bytes. Default: 67108864 bytes (64 MiB)." }, - SettingDefinition { key: "media_reconcile_repairs_per_pass", label: "Repair attempts per audit pass", environment: "CHAN_MEDIA_RECONCILE_REPAIRS_PER_PASS", kind: InputKind::Number(0, 1_000_000), value: |c| c.media_reconcile_repairs_per_pass.to_string(), help: "Used when repair permission is enabled; caps repair work per pass." }, + SettingDefinition { application: crate::config::admin::ApplicationMode::Restart, key: "max_image_size_mb", label: "New-board image upload default (MiB)", environment: "CHAN_MAX_IMAGE_MB", kind: InputKind::Number(1, 100), value: |c| (c.max_image_size / (1024 * 1024)).to_string(), help: "Upload limit for newly created boards. Existing boards retain their own caps. New-board PDFs also derive their cap from this default." }, + SettingDefinition { application: crate::config::admin::ApplicationMode::Restart, key: "max_video_size_mb", label: "New-board video upload default (MiB)", environment: "CHAN_MAX_VIDEO_MB", kind: InputKind::Number(1, 2048), value: |c| (c.max_video_size / (1024 * 1024)).to_string(), help: "Upload limit for newly created boards; edit existing per-board caps under Boards." }, + SettingDefinition { application: crate::config::admin::ApplicationMode::Restart, key: "max_audio_size_mb", label: "New-board audio upload default (MiB)", environment: "CHAN_MAX_AUDIO_MB", kind: InputKind::Number(1, 512), value: |c| (c.max_audio_size / (1024 * 1024)).to_string(), help: "Upload limit for newly created boards; existing boards retain their stored caps." }, + SettingDefinition { application: crate::config::admin::ApplicationMode::Restart, key: "enable_any_file_uploads_feature", label: "Master arbitrary-file upload gate", environment: "CHAN_ENABLE_ANY_FILE_UPLOADS_FEATURE", kind: InputKind::Boolean, value: |c| c.enable_any_file_uploads_feature.to_string(), help: "Both this global gate and the per-board arbitrary-file checkbox must be enabled. This does not change individual board preferences." }, + SettingDefinition { application: crate::config::admin::ApplicationMode::Restart, key: "require_ffmpeg", label: "Require FFmpeg at startup", environment: "CHAN_REQUIRE_FFMPEG", kind: InputKind::Boolean, value: |c| c.require_ffmpeg.to_string(), help: "Missing FFmpeg becomes a startup error. Internal Rust image, metadata, PDF and common-audio processing does not need FFmpeg." }, + SettingDefinition { application: crate::config::admin::ApplicationMode::Restart, key: "ffmpeg_path", label: "FFmpeg executable", environment: "CHAN_FFMPEG_PATH", kind: InputKind::Text, value: |c| c.ffmpeg_path.clone(), help: "Use ffmpeg for PATH lookup, or an absolute executable path. New tool paths are checked with a bounded version probe before saving." }, + SettingDefinition { application: crate::config::admin::ApplicationMode::Restart, key: "thumb_size", label: "Generated thumbnail dimension (pixels)", environment: "CHAN_THUMB_SIZE", kind: InputKind::Number(16, 4096), value: |c| c.thumb_size.to_string(), help: "Applies to thumbnails generated after restart. Existing thumbnails are not regenerated." }, + SettingDefinition { application: crate::config::admin::ApplicationMode::Restart, key: "job_queue_capacity", label: "Background queue capacity (pending jobs)", environment: "CHAN_JOB_QUEUE_CAPACITY", kind: InputKind::Number(0, 1_000_000), value: |c| c.job_queue_capacity.to_string(), help: "0 is unlimited. Once full, the queue drops new media jobs with a warning. Current queue pressure is shown in Site Health." }, + SettingDefinition { application: crate::config::admin::ApplicationMode::Restart, key: "waveform_cache_max_mb", label: "Thumbnail/waveform cache budget (MiB)", environment: "CHAN_WAVEFORM_CACHE_MAX_MB", kind: InputKind::Number(0, 1_048_576), value: |c| (c.waveform_cache_max_bytes / (1024 * 1024)).to_string(), help: "Oldest cache files are evicted by a background task when the budget is exceeded. 0 disables eviction." }, + SettingDefinition { application: crate::config::admin::ApplicationMode::Restart, key: "archive_before_prune", label: "Always archive overflow threads", environment: "CHAN_ARCHIVE_BEFORE_PRUNE", kind: InputKind::Boolean, value: |c| c.archive_before_prune.to_string(), help: "Overrides per-board overflow deletion/archive policy when enabled. Boards still retain their stored allow_archive preference." }, + SettingDefinition { application: crate::config::admin::ApplicationMode::Restart, key: "media_reconcile_repair_enabled", label: "Permit safe reconciliation repairs", environment: "CHAN_MEDIA_RECONCILE_REPAIR_ENABLED", kind: InputKind::Boolean, value: |c| c.media_reconcile_repair_enabled.to_string(), help: "Separate repair permission from periodic audits. Only bounded, verified safe repairs are scheduled." }, + SettingDefinition { application: crate::config::admin::ApplicationMode::Restart, key: "media_reconcile_interval_hours", label: "Reconciliation audit interval (hours)", environment: "CHAN_MEDIA_RECONCILE_INTERVAL_HOURS", kind: InputKind::Number(0, 8760), value: |c| c.media_reconcile_interval_hours.to_string(), help: "0 disables periodic audit passes. Manual maintenance remains available." }, + SettingDefinition { application: crate::config::admin::ApplicationMode::Restart, key: "media_reconcile_files_per_pass", label: "Filesystem entries per audit pass", environment: "CHAN_MEDIA_RECONCILE_FILES_PER_PASS", kind: InputKind::Number(0, 1_000_000), value: |c| c.media_reconcile_files_per_pass.to_string(), help: "Bound the filesystem work performed by each reconciliation pass." }, + SettingDefinition { application: crate::config::admin::ApplicationMode::Restart, key: "media_reconcile_database_rows_per_pass", label: "Reference rows per audit pass", environment: "CHAN_MEDIA_RECONCILE_DATABASE_ROWS_PER_PASS", kind: InputKind::Number(0, 10_000_000), value: |c| c.media_reconcile_database_rows_per_pass.to_string(), help: "Bound the authoritative-reference snapshot used by each pass." }, + SettingDefinition { application: crate::config::admin::ApplicationMode::Restart, key: "media_reconcile_hash_bytes_per_pass", label: "Bytes hashed per audit pass", environment: "CHAN_MEDIA_RECONCILE_HASH_BYTES_PER_PASS", kind: InputKind::Number(0, 1_099_511_627_776), value: |c| c.media_reconcile_hash_bytes_per_pass.to_string(), help: "I/O budget in bytes. Default: 67108864 bytes (64 MiB)." }, + SettingDefinition { application: crate::config::admin::ApplicationMode::Restart, key: "media_reconcile_repairs_per_pass", label: "Repair attempts per audit pass", environment: "CHAN_MEDIA_RECONCILE_REPAIRS_PER_PASS", kind: InputKind::Number(0, 1_000_000), value: |c| c.media_reconcile_repairs_per_pass.to_string(), help: "Used when repair permission is enabled; caps repair work per pass." }, ]; /// Dedicated maintenance controls. static MAINTENANCE_SETTINGS: &[SettingDefinition] = &[ - SettingDefinition { key: "wal_checkpoint_interval_secs", label: "WAL checkpoint interval (seconds)", environment: "CHAN_WAL_CHECKPOINT_SECS", kind: InputKind::Number(0, 31_536_000), value: |c| c.wal_checkpoint_interval.to_string(), help: "0 disables the recurring checkpoint. Existing manual database actions remain available." }, - SettingDefinition { key: "auto_vacuum_interval_hours", label: "Automatic VACUUM interval (hours)", environment: "CHAN_AUTO_VACUUM_HOURS", kind: InputKind::Number(0, 8760), value: |c| c.auto_vacuum_interval_hours.to_string(), help: "0 disables recurring VACUUM. Manual VACUUM is available in database maintenance." }, - SettingDefinition { key: "poll_cleanup_interval_hours", label: "Expired poll-vote cleanup interval (hours)", environment: "CHAN_POLL_CLEANUP_HOURS", kind: InputKind::Number(0, 8760), value: |c| c.poll_cleanup_interval_hours.to_string(), help: "0 disables periodic cleanup. Default: 72 hours." }, - SettingDefinition { key: "db_warn_threshold_mb", label: "Database size warning threshold (MiB)", environment: "CHAN_DB_WARN_THRESHOLD_MB", kind: InputKind::Number(0, 1_048_576), value: |c| (c.db_warn_threshold_bytes / (1024 * 1024)).to_string(), help: "0 disables the admin warning. This warns about size; it does not cap or delete database contents." }, + SettingDefinition { application: crate::config::admin::ApplicationMode::Restart, key: "wal_checkpoint_interval_secs", label: "WAL checkpoint interval (seconds)", environment: "CHAN_WAL_CHECKPOINT_SECS", kind: InputKind::Number(0, 31_536_000), value: |c| c.wal_checkpoint_interval.to_string(), help: "0 disables the recurring checkpoint. Existing manual database actions remain available." }, + SettingDefinition { application: crate::config::admin::ApplicationMode::Restart, key: "auto_vacuum_interval_hours", label: "Automatic VACUUM interval (hours)", environment: "CHAN_AUTO_VACUUM_HOURS", kind: InputKind::Number(0, 8760), value: |c| c.auto_vacuum_interval_hours.to_string(), help: "0 disables recurring VACUUM. Manual VACUUM is available in database maintenance." }, + SettingDefinition { application: crate::config::admin::ApplicationMode::Restart, key: "poll_cleanup_interval_hours", label: "Expired poll-vote cleanup interval (hours)", environment: "CHAN_POLL_CLEANUP_HOURS", kind: InputKind::Number(0, 8760), value: |c| c.poll_cleanup_interval_hours.to_string(), help: "0 disables periodic cleanup. Default: 72 hours." }, + SettingDefinition { application: crate::config::admin::ApplicationMode::Restart, key: "db_warn_threshold_mb", label: "Database size warning threshold (MiB)", environment: "CHAN_DB_WARN_THRESHOLD_MB", kind: InputKind::Number(0, 1_048_576), value: |c| (c.db_warn_threshold_bytes / (1024 * 1024)).to_string(), help: "0 disables the admin warning. This warns about size; it does not cap or delete database contents." }, ]; /// Dedicated system controls. static SYSTEM_SETTINGS: &[SettingDefinition] = &[ - SettingDefinition { key: "blocking_threads", label: "Tokio blocking-pool size (threads)", environment: "CHAN_BLOCKING_THREADS", kind: InputKind::Number(0, 4096), value: |c| c.blocking_threads.to_string(), help: "0 selects logical CPUs × 4 (16 if CPU detection is unavailable). Active and saved state show the resolved count. Restart required." }, - SettingDefinition { key: "db_pool_size", label: "SQLite connection-pool size", environment: "CHAN_DB_POOL_SIZE", kind: InputKind::Number(1, 128), value: |c| c.db_pool_size.to_string(), help: "Default: 8 connections. Each connection has roughly 32 MiB of page cache; increasing this can substantially raise memory use." }, + SettingDefinition { application: crate::config::admin::ApplicationMode::Restart, key: "blocking_threads", label: "Tokio blocking-pool size (threads)", environment: "CHAN_BLOCKING_THREADS", kind: InputKind::Number(0, 4096), value: |c| c.blocking_threads.to_string(), help: "0 selects logical CPUs × 4 (16 if CPU detection is unavailable). Active and saved state show the resolved count. Restart required." }, + SettingDefinition { application: crate::config::admin::ApplicationMode::Restart, key: "db_pool_size", label: "SQLite connection-pool size", environment: "CHAN_DB_POOL_SIZE", kind: InputKind::Number(1, 128), value: |c| c.db_pool_size.to_string(), help: "Default: 8 connections. Each connection has roughly 32 MiB of page cache; increasing this can substantially raise memory use." }, ]; #[cfg(test)] @@ -313,8 +317,8 @@ mod tests { ); } ensure!( - keys.len() == 63, - "registry must cover 39 inventoried root controls, 12 TLS leaves and 12 new operator choices" + keys.len() == 62, + "registry must cover 38 inventoried root controls, 12 TLS leaves and 12 new operator choices" ); let config = resolve_file("", &Environment::Values(&BTreeMap::new()))?; ensure!( @@ -351,7 +355,7 @@ mod tests { RuntimeSection::Logging => ("log_filter", "warn"), RuntimeSection::Timeouts => ("read_timeout_secs", "45"), }; - form.insert(key.to_owned(), replacement.to_owned()); + let _previous_value = form.insert(key.to_owned(), replacement.to_owned()); let updates = parse_settings_form(section.definitions(), &form)?; save_root_at( &path, @@ -414,7 +418,7 @@ mod tests { (RuntimeSection::System, "db_pool_size", "0"), ] { let mut form = form_for(section)?; - form.insert(key.to_owned(), invalid.to_owned()); + let _previous_value = form.insert(key.to_owned(), invalid.to_owned()); let result = parse_settings_form(section.definitions(), &form).and_then(|updates| { save_root_at( &path, @@ -467,7 +471,6 @@ mod tests { ); } validate_tool_path("ffmpeg", "ffmpeg")?; - validate_tool_path("ffprobe", "ffprobe")?; Ok(()) } diff --git a/src/config/backup_storage.rs b/src/config/backup_storage.rs index f810c135..946a71ad 100644 --- a/src/config/backup_storage.rs +++ b/src/config/backup_storage.rs @@ -50,6 +50,11 @@ fn absolute_resolved_path(path: &Path) -> anyhow::Result { super::resolve_storage_dir(&absolute, "protected storage path") } +/// Check saved backup paths without creating directories or changing permissions. +pub(super) fn validate_backup_directory(path: &Path, config: &Config) -> anyhow::Result<()> { + validate_backup_children(&resolve_backup_directory(path, config)?) +} + /// Validate and prepare an absolute private backup root and legacy subdirectories. /// /// # Errors @@ -188,10 +193,10 @@ mod tests { let existing = temp.path().join("existing"); std::fs::create_dir(&existing)?; std::fs::write(existing.join("operator-file"), "keep")?; - let result = prepare_and_persist(&existing, &config, |_| { + let preserve_existing_result = prepare_and_persist(&existing, &config, |_| { anyhow::bail!("injected persistence failure") }); - ensure!(result.is_err()); + ensure!(preserve_existing_result.is_err()); ensure!(std::fs::read_to_string(existing.join("operator-file"))? == "keep"); ensure!( std::fs::read_dir(&existing)?.count() == 1, diff --git a/src/config/operator.rs b/src/config/operator.rs index 6631d4f6..d6db178e 100644 --- a/src/config/operator.rs +++ b/src/config/operator.rs @@ -2,7 +2,7 @@ use super::Environment; use crate::config::admin::{InputKind, SettingDefinition}; -use anyhow::ensure; +use anyhow::{ensure, Context as _}; /// Additional policies consumed by authentication, paging and request boundaries. #[derive(Debug, Clone)] @@ -157,7 +157,8 @@ impl OperatorSettings { if let Some(filter) = &self.log_filter { ensure!(filter.len() <= 1024, "log filter is too long"); tracing_subscriber::EnvFilter::try_new(filter) - .map_err(|_| anyhow::anyhow!("invalid tracing filter"))?; + .context("invalid tracing filter") + .map(|_validated_value| ())?; } Ok(()) } @@ -185,17 +186,18 @@ impl SavedOperatorSettings { /// Additional access settings. pub static ACCESS_SETTINGS: &[SettingDefinition] = &[ - SettingDefinition { key: "admin_login_fail_limit", label: "Administrator failed-login allowance", help: "Failed login attempts per visitor before lockout; account reauthentication uses the same protection.", environment: "CHAN_ADMIN_LOGIN_FAIL_LIMIT", kind: InputKind::Number(1, 100), value: |c| c.operator.admin_login_fail_limit.to_string() }, - SettingDefinition { key: "admin_login_fail_window_secs", label: "Administrator failed-login window (seconds)", help: "Separate from browsing limits. Counter resets after this interval.", environment: "CHAN_ADMIN_LOGIN_FAIL_WINDOW_SECS", kind: InputKind::Number(30, 86400), value: |c| c.operator.admin_login_fail_window_secs.to_string() }, - SettingDefinition { key: "board_password_fail_limit", label: "Board-password failed-attempt allowance", help: "Per visitor and board; leaves browsing and administrator protections separate.", environment: "CHAN_BOARD_PASSWORD_FAIL_LIMIT", kind: InputKind::Number(1, 100), value: |c| c.operator.board_password_fail_limit.to_string() }, - SettingDefinition { key: "board_password_fail_window_secs", label: "Board-password failed-attempt window (seconds)", help: "Board unlock counters expire after this interval.", environment: "CHAN_BOARD_PASSWORD_FAIL_WINDOW_SECS", kind: InputKind::Number(30, 86400), value: |c| c.operator.board_password_fail_window_secs.to_string() }, - SettingDefinition { key: "board_access_cookie_days", label: "Board-password access lifetime (days)", help: "Browser cookie lifetime for new password grants. Existing browser cookies retain their issued expiry; password or secret changes revoke grants.", environment: "CHAN_BOARD_ACCESS_COOKIE_DAYS", kind: InputKind::Number(1, 365), value: |c| c.operator.board_access_cookie_days.to_string() }, - SettingDefinition { key: "self_action_window_secs", label: "Self-edit and self-delete window (seconds)", help: "Signed ownership grants and server authorization use this window. Board permissions still apply; legacy edit_window_secs remains inactive.", environment: "CHAN_SELF_ACTION_WINDOW_SECS", kind: InputKind::Number(1, 3600), value: |c| c.operator.self_action_window_secs.to_string() }, + SettingDefinition { application: crate::config::admin::ApplicationMode::Restart, key: "admin_login_fail_limit", label: "Administrator failed-login allowance", help: "Failed login attempts per visitor before lockout; account reauthentication uses the same protection.", environment: "CHAN_ADMIN_LOGIN_FAIL_LIMIT", kind: InputKind::Number(1, 100), value: |c| c.operator.admin_login_fail_limit.to_string() }, + SettingDefinition { application: crate::config::admin::ApplicationMode::Restart, key: "admin_login_fail_window_secs", label: "Administrator failed-login window (seconds)", help: "Separate from browsing limits. Counter resets after this interval.", environment: "CHAN_ADMIN_LOGIN_FAIL_WINDOW_SECS", kind: InputKind::Number(30, 86400), value: |c| c.operator.admin_login_fail_window_secs.to_string() }, + SettingDefinition { application: crate::config::admin::ApplicationMode::Restart, key: "board_password_fail_limit", label: "Board-password failed-attempt allowance", help: "Per visitor and board; leaves browsing and administrator protections separate.", environment: "CHAN_BOARD_PASSWORD_FAIL_LIMIT", kind: InputKind::Number(1, 100), value: |c| c.operator.board_password_fail_limit.to_string() }, + SettingDefinition { application: crate::config::admin::ApplicationMode::Restart, key: "board_password_fail_window_secs", label: "Board-password failed-attempt window (seconds)", help: "Board unlock counters expire after this interval.", environment: "CHAN_BOARD_PASSWORD_FAIL_WINDOW_SECS", kind: InputKind::Number(30, 86400), value: |c| c.operator.board_password_fail_window_secs.to_string() }, + SettingDefinition { application: crate::config::admin::ApplicationMode::Restart, key: "board_access_cookie_days", label: "Board-password access lifetime (days)", help: "Browser cookie lifetime for new password grants. Existing browser cookies retain their issued expiry; password or secret changes revoke grants.", environment: "CHAN_BOARD_ACCESS_COOKIE_DAYS", kind: InputKind::Number(1, 365), value: |c| c.operator.board_access_cookie_days.to_string() }, + SettingDefinition { application: crate::config::admin::ApplicationMode::Restart, key: "self_action_window_secs", label: "Self-edit and self-delete window (seconds)", help: "Signed ownership grants and server authorization use this window. Board permissions still apply; legacy edit_window_secs remains inactive.", environment: "CHAN_SELF_ACTION_WINDOW_SECS", kind: InputKind::Number(1, 3600), value: |c| c.operator.self_action_window_secs.to_string() }, ]; /// Public index display preferences. pub static DISPLAY_SETTINGS: &[SettingDefinition] = &[ SettingDefinition { + application: crate::config::admin::ApplicationMode::Restart, key: "index_threads_per_page", label: "Board index threads per page", help: "Index pagination only. Catalog behavior is unchanged.", @@ -204,6 +206,7 @@ pub static DISPLAY_SETTINGS: &[SettingDefinition] = &[ value: |c| c.operator.index_threads_per_page.to_string(), }, SettingDefinition { + application: crate::config::admin::ApplicationMode::Restart, key: "index_reply_previews", label: "Board index reply previews", help: "Latest reply previews per thread; 0 hides previews.", @@ -216,6 +219,7 @@ pub static DISPLAY_SETTINGS: &[SettingDefinition] = &[ /// Normal request deadline preferences. pub static TIMEOUT_SETTINGS: &[SettingDefinition] = &[ SettingDefinition { + application: crate::config::admin::ApplicationMode::Restart, key: "read_timeout_secs", label: "Normal GET/HEAD timeout (seconds)", help: "Uploads, backup, restore and maintenance retain their existing timeout exemptions.", @@ -224,6 +228,7 @@ pub static TIMEOUT_SETTINGS: &[SettingDefinition] = &[ value: |c| c.operator.read_timeout_secs.to_string(), }, SettingDefinition { + application: crate::config::admin::ApplicationMode::Restart, key: "write_timeout_secs", label: "Normal write timeout (seconds)", help: "Other methods use this timeout; existing operation exemptions remain available.", @@ -235,5 +240,5 @@ pub static TIMEOUT_SETTINGS: &[SettingDefinition] = &[ /// Startup tracing filter. pub static LOG_SETTINGS: &[SettingDefinition] = &[ - SettingDefinition { key: "log_filter", label: "Log filter / verbosity", help: "Tracing directives such as info,rustchan=debug. Blank keeps the built-in INFO application/dependency filter. RUST_LOG takes precedence. Restart required; excessive detail can increase log volume.", environment: "RUST_LOG", kind: InputKind::OptionalText, value: |c| c.operator.log_filter.clone().unwrap_or_else(|| "Built-in INFO filter".to_owned()) }, + SettingDefinition { application: crate::config::admin::ApplicationMode::Restart, key: "log_filter", label: "Log filter / verbosity", help: "Tracing directives such as info,rustchan=debug. Blank keeps the built-in INFO application/dependency filter. RUST_LOG takes precedence. Restart required; excessive detail can increase log volume.", environment: "RUST_LOG", kind: InputKind::OptionalText, value: |c| c.operator.log_filter.clone().unwrap_or_else(|| "Built-in INFO filter".to_owned()) }, ]; diff --git a/src/config/template.rs b/src/config/template.rs index 452403ba..4bc7a92a 100644 --- a/src/config/template.rs +++ b/src/config/template.rs @@ -142,8 +142,6 @@ ffmpeg_timeout_secs = 600 # Optional explicit ffmpeg binary path. Leave unset to use PATH lookup. # ffmpeg_path = "/usr/local/bin/ffmpeg" -# Optional explicit ffprobe binary path. Leave unset to use PATH lookup. -# ffprobe_path = "/usr/local/bin/ffprobe" # Automatically prune oldest full-size post media when active stored post media # exceeds the configured byte cap. Thumbnails are kept when possible so old diff --git a/src/db/accounts.rs b/src/db/accounts.rs index 22acb15d..5d12892f 100644 --- a/src/db/accounts.rs +++ b/src/db/accounts.rs @@ -58,7 +58,7 @@ pub fn manage_account( super::get_admin_by_username(&tx, username)?.is_none(), "username already exists" ); - super::create_admin(&tx, username, &hash)?; + super::create_admin(&tx, username, &hash).map(|_created_id| ())?; } AccountAction::Password => { let target = @@ -67,7 +67,8 @@ pub fn manage_account( tx.execute( "DELETE FROM admin_sessions WHERE admin_id = ?1", [target.id], - )?; + ) + .map(|_affected_rows| ())?; } } tx.commit()?; @@ -123,9 +124,10 @@ mod tests { super::super::get_session(&conn, "actor-session")?.is_some(), "other sessions must survive" ); - let target = super::super::get_admin_by_username(&conn, "target")?.context("target")?; + let updated_target = + super::super::get_admin_by_username(&conn, "target")?.context("target")?; ensure!( - crate::utils::crypto::verify_password("Replacement123", &target.password_hash)?, + crate::utils::crypto::verify_password("Replacement123", &updated_target.password_hash)?, "stored password must match" ); ensure!( diff --git a/src/db/admin.rs b/src/db/admin.rs index 64873cee..603b3085 100644 --- a/src/db/admin.rs +++ b/src/db/admin.rs @@ -191,7 +191,8 @@ pub fn create_session( "INSERT INTO admin_sessions (id, admin_id, expires_at) VALUES (?1, ?2, ?3)", params![session_id, admin_id, expires_at], ) - .context("Failed to create admin session")?; + .context("Failed to create admin session") + .map(|_affected_rows| ())?; Ok(()) } @@ -221,7 +222,8 @@ pub fn delete_session(conn: &rusqlite::Connection, session_id: &str) -> Result<( conn.execute( "DELETE FROM admin_sessions WHERE id = ?1", params![session_id], - )?; + ) + .map(|_affected_rows| ())?; Ok(()) } @@ -361,23 +363,22 @@ pub fn add_word_filter( /// # Errors /// Returns an error if the database operation fails. pub fn remove_word_filter(conn: &rusqlite::Connection, id: i64) -> Result<()> { - conn.execute("DELETE FROM word_filters WHERE id = ?1", params![id])?; + conn.execute("DELETE FROM word_filters WHERE id = ?1", params![id]) + .map(|_affected_rows| ())?; Ok(()) } // Reports /// Return whether an `SQLite` error represents the open-report uniqueness guard. fn is_open_report_unique_violation(error: &rusqlite::Error) -> bool { - match error { - rusqlite::Error::SqliteFailure(inner, message) => { - inner.code == rusqlite::ErrorCode::ConstraintViolation - && message.as_deref().is_some_and(|text| { - text.contains("idx_reports_open_unique") - || (text.contains("reports.post_id") - && text.contains("reports.reporter_hash")) - }) - } - _ => false, + if let rusqlite::Error::SqliteFailure(inner, message) = error { + inner.code == rusqlite::ErrorCode::ConstraintViolation + && message.as_deref().is_some_and(|text| { + text.contains("idx_reports_open_unique") + || (text.contains("reports.post_id") && text.contains("reports.reporter_hash")) + }) + } else { + false } } @@ -505,7 +506,8 @@ pub fn log_mod_action( board_short, detail ], - )?; + ) + .map(|_affected_rows| ())?; Ok(()) } @@ -662,7 +664,8 @@ pub fn accept_ban_appeal(conn: &rusqlite::Connection, appeal_id: i64) -> Result< .context("Failed to accept ban appeal")? .with_context(|| format!("Ban appeal id {appeal_id} not found or already handled"))?; conn.execute("DELETE FROM bans WHERE ip_hash=?1", params![ip_hash]) - .context("Failed to lift ban during appeal acceptance")?; + .context("Failed to lift ban during appeal acceptance") + .map(|_affected_rows| ())?; Ok(ip_hash) })(); @@ -755,7 +758,7 @@ pub fn get_posts_by_ip_hash( /// The raw PRAGMA `wal_checkpoint` pragma returns three columns in this order: /// col 0 — busy: 1 if a checkpoint could not complete due to an active reader/writer /// col 1 — log: total pages in the WAL file -/// col 2 — checkpointed: pages actually written back to the database +/// col 2 — checkpointed: pages actually written back to the database. /// /// This function returns `(log_pages, checkpointed_pages, busy)` — intentionally /// reordered so the two informational values come first and the error flag last. @@ -1102,7 +1105,7 @@ mod tests { fn ban_appeal_submission_is_deduplicated_within_window() -> Result<()> { let pool = crate::db::init_test_pool()?; let conn = pool.get()?; - crate::db::add_ban(&conn, "hash1", "reason", None)?; + crate::db::add_ban(&conn, "hash1", "reason", None).map(|_created_id| ())?; let first = file_ban_appeal(&conn, "hash1", "please unban")?; let second = file_ban_appeal(&conn, "hash1", "second try")?; @@ -1146,8 +1149,8 @@ mod tests { fn appeal_resolution_uses_the_stored_address_and_is_single_transition() -> Result<()> { let pool = crate::db::init_test_pool()?; let conn = pool.get()?; - crate::db::add_ban(&conn, "appealed-hash", "appealed", None)?; - crate::db::add_ban(&conn, "other-hash", "other", None)?; + crate::db::add_ban(&conn, "appealed-hash", "appealed", None).map(|_created_id| ())?; + crate::db::add_ban(&conn, "other-hash", "other", None).map(|_created_id| ())?; assert_eq!( file_ban_appeal(&conn, "appealed-hash", "please unban")?, BanAppealSubmission::Filed @@ -1392,7 +1395,7 @@ mod tests { let conn = pool.get()?; let ip_hash = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; - create_board(&conn, "test", "Test", "", false)?; + create_board(&conn, "test", "Test", "", false).map(|_created_id| ())?; let board = get_board_by_short(&conn, "test")?.context("test board should exist")?; let post = NewPost { thread_id: 0, @@ -1427,19 +1430,22 @@ mod tests { )?; let posts = get_posts_by_ip_hash(&conn, ip_hash, 25, 0)?; - let (post, board_short) = posts + let (history_post, board_short) = posts .first() .context("IP history should contain the post")?; assert_eq!(posts.len(), 1, "exactly one post should match the IP hash"); - assert_eq!(post.id, post_id, "the created post should be returned"); assert_eq!( - post.media_processing_state.as_deref(), + history_post.id, post_id, + "the created post should be returned" + ); + assert_eq!( + history_post.media_processing_state.as_deref(), Some("pending"), "media processing state should be decoded" ); assert_eq!( - post.media_processing_error.as_deref(), + history_post.media_processing_error.as_deref(), Some("transcoding"), "media processing error should be decoded" ); diff --git a/src/db/banners.rs b/src/db/banners.rs index 9acd3ade..1304e68a 100644 --- a/src/db/banners.rs +++ b/src/db/banners.rs @@ -111,7 +111,7 @@ pub fn next_banner_sort_order( params![board_id], |row| row.get(0), )?, - _ => conn.query_row( + BannerScope::Global | BannerScope::Home => conn.query_row( "SELECT COALESCE(MAX(sort_order) + 1, 1) FROM banner_assets WHERE scope_type = ?1", @@ -219,7 +219,8 @@ pub fn delete_banner_asset(conn: &rusqlite::Connection, banner_id: i64) -> Resul conn.execute( "DELETE FROM banner_assets WHERE id = ?1", params![banner_id], - )?; + ) + .map(|_affected_rows| ())?; Ok(asset) } @@ -235,7 +236,8 @@ pub fn delete_board_banner_assets( conn.execute( "DELETE FROM banner_assets WHERE scope_type = 'board' AND board_id = ?1", params![board_id], - )?; + ) + .map(|_affected_rows| ())?; Ok(assets) } @@ -275,10 +277,10 @@ pub fn move_banner_asset( .ok_or_else(|| anyhow::anyhow!("Banner id {banner_id} not found"))?; let swap_with = if move_up { index.checked_sub(1) - } else if index + 1 < ordered_ids.len() { - Some(index + 1) } else { - None + index + .checked_add(1) + .filter(|next| *next < ordered_ids.len()) }; let Some(target_index) = swap_with else { tx.commit()?; @@ -289,9 +291,13 @@ pub fn move_banner_asset( let mut update = tx.prepare_cached("UPDATE banner_assets SET sort_order = ?1 WHERE id = ?2")?; for (position, id) in ordered_ids.iter().enumerate() { - let sort_order = - i64::try_from(position).context("banner sort_order must fit in i64")? + 1; - update.execute(params![sort_order, id])?; + let sort_order = i64::try_from(position) + .ok() + .and_then(|value| value.checked_add(1)) + .context("banner sort_order must fit in i64")?; + update + .execute(params![sort_order, id]) + .map(|_affected_rows| ())?; } } tx.commit()?; diff --git a/src/db/boards.rs b/src/db/boards.rs index 1427ebfc..25fbc4fc 100644 --- a/src/db/boards.rs +++ b/src/db/boards.rs @@ -142,9 +142,13 @@ fn normalize_board_group_order( let mut update = conn.prepare_cached("UPDATE boards SET display_order = ?1 WHERE id = ?2")?; for (position, board_id) in ordered_ids.iter().enumerate() { - let display_order = - i64::try_from(position).context("board display_order index must fit in i64")? + 1; - update.execute(params![display_order, board_id])?; + let display_order = i64::try_from(position) + .ok() + .and_then(|value| value.checked_add(1)) + .context("board display_order index must fit in i64")?; + update + .execute(params![display_order, board_id]) + .map(|_affected_rows| ())?; } Ok(()) } @@ -174,7 +178,8 @@ pub fn set_site_setting(conn: &rusqlite::Connection, key: &str, value: &str) -> ON CONFLICT(key) DO UPDATE SET value = excluded.value", params![key, value], ) - .context("Failed to upsert site setting")?; + .context("Failed to upsert site setting") + .map(|_affected_rows| ())?; Ok(()) } @@ -410,13 +415,11 @@ pub fn count_new_threads_for_boards( return Ok(HashMap::new()); } - let values_sql = markers - .iter() - .enumerate() - .map(|(index, _)| { - let base = index * 3; - format!("(?{}, ?{}, ?{})", base + 1, base + 2, base + 3) - }) + let parameter_count = markers + .len() + .checked_mul(3) + .context("board marker parameter count overflow")?; + let values_sql = std::iter::repeat_n("(?, ?, ?)", markers.len()) .collect::>() .join(", "); let sql = format!( @@ -437,7 +440,7 @@ pub fn count_new_threads_for_boards( GROUP BY t.board_id" ); - let mut params = Vec::with_capacity(markers.len() * 3); + let mut params = Vec::with_capacity(parameter_count); for marker in markers { params.push(rusqlite::types::Value::Integer(marker.board_id)); params.push(rusqlite::types::Value::Integer( @@ -456,7 +459,7 @@ pub fn count_new_threads_for_boards( let mut counts = HashMap::new(); for row in rows { let (board_id, count) = row?; - counts.insert(board_id, count); + let _previous_value = counts.insert(board_id, count); } Ok(counts) } @@ -486,13 +489,11 @@ pub fn count_new_replies_for_boards( return Ok(HashMap::new()); } - let values_sql = markers - .iter() - .enumerate() - .map(|(index, _)| { - let base = index * 2; - format!("(?{}, ?{})", base + 1, base + 2) - }) + let parameter_count = markers + .len() + .checked_mul(2) + .context("reply marker parameter count overflow")?; + let values_sql = std::iter::repeat_n("(?, ?)", markers.len()) .collect::>() .join(", "); let sql = format!( @@ -506,7 +507,7 @@ pub fn count_new_replies_for_boards( GROUP BY t.board_id" ); - let mut params = Vec::with_capacity(markers.len() * 2); + let mut params = Vec::with_capacity(parameter_count); for marker in markers { params.push(rusqlite::types::Value::Integer(marker.thread_id)); params.push(rusqlite::types::Value::Integer( @@ -523,7 +524,7 @@ pub fn count_new_replies_for_boards( for row in rows { let (board_id, count) = row?; if count > 0 { - counts.insert(board_id, count); + let _previous_value = counts.insert(board_id, count); } } Ok(counts) @@ -657,7 +658,7 @@ pub fn move_board(conn: &mut rusqlite::Connection, id: i64, move_up: bool) -> Re let board_nsfw: bool = tx.query_row( "SELECT nsfw FROM boards WHERE id = ?1", params![id], - |row| row.get::<_, i32>(0).map(|value| value != 0), + |row| row.get::<_, i32>(0).map(|value| value != 0_i32), )?; let mut stmt = tx.prepare_cached(&format!( "SELECT id FROM boards WHERE nsfw = ?1 ORDER BY {BOARD_GROUP_ORDER_SQL}" @@ -674,10 +675,10 @@ pub fn move_board(conn: &mut rusqlite::Connection, id: i64, move_up: bool) -> Re let swap_with = if move_up { index.checked_sub(1) - } else if index + 1 < ordered_ids.len() { - Some(index + 1) } else { - None + index + .checked_add(1) + .filter(|next| *next < ordered_ids.len()) }; let Some(target_index) = swap_with else { @@ -690,9 +691,13 @@ pub fn move_board(conn: &mut rusqlite::Connection, id: i64, move_up: bool) -> Re { let mut update = tx.prepare_cached("UPDATE boards SET display_order = ?1 WHERE id = ?2")?; for (position, board_id) in ordered_ids.iter().enumerate() { - let display_order = - i64::try_from(position).context("board display_order index must fit in i64")? + 1; - update.execute(params![display_order, board_id])?; + let display_order = i64::try_from(position) + .ok() + .and_then(|value| value.checked_add(1)) + .context("board display_order index must fit in i64")?; + update + .execute(params![display_order, board_id]) + .map(|_affected_rows| ())?; } } tx.commit()?; @@ -752,7 +757,7 @@ pub fn update_board_settings( let current_nsfw: bool = tx.query_row( "SELECT nsfw FROM boards WHERE id = ?1", params![id], - |row| row.get::<_, i32>(0).map(|value| value != 0), + |row| row.get::<_, i32>(0).map(|value| value != 0_i32), )?; let affected = if current_nsfw == nsfw { @@ -1169,14 +1174,16 @@ mod tests { "INSERT INTO boards (id, short_name, name, created_at) VALUES (1, 'test', 'Test', 1_700_000_000)", [], - )?; + ) + .map(|_affected_rows| ())?; conn.execute( "INSERT INTO threads (id, board_id, subject, archived) VALUES (1, 1, 'visible one', 0), (2, 1, 'visible two', 0), (3, 1, 'archived', 1)", [], - )?; + ) + .map(|_affected_rows| ())?; let stats = get_all_boards_with_stats(&conn)?; let board_stats = stats @@ -1230,11 +1237,13 @@ mod tests { conn.execute( "INSERT INTO boards (id, short_name, name) VALUES (1, 'test', 'Test')", [], - )?; + ) + .map(|_affected_rows| ())?; conn.execute( "INSERT INTO threads (id, board_id, subject) VALUES (1, 1, 'test thread')", [], - )?; + ) + .map(|_affected_rows| ())?; conn.execute( "INSERT INTO posts ( @@ -1250,7 +1259,8 @@ mod tests { (4, 1, 1, 'text post', '

text

', 'tok4', 0, NULL, NULL, NULL, NULL, NULL)", [], - )?; + ) + .map(|_affected_rows| ())?; conn.execute( "UPDATE posts SET audio_file_path = 'test/track.flac', @@ -1259,7 +1269,8 @@ mod tests { audio_mime_type = 'audio/flac' WHERE id = 2", [], - )?; + ) + .map(|_affected_rows| ())?; let stats = get_site_stats(&conn)?; assert_eq!(stats.total_posts, 4, "all posts should be counted"); @@ -1288,13 +1299,15 @@ mod tests { conn.execute( "INSERT INTO boards (id, short_name, name) VALUES (1, 'test', 'Test')", [], - )?; + ) + .map(|_affected_rows| ())?; conn.execute( "INSERT INTO threads (id, board_id, subject, archived) VALUES (1, 1, 'live thread', 0), (2, 1, 'archived thread', 1)", [], - )?; + ) + .map(|_affected_rows| ())?; conn.execute( "INSERT INTO posts ( id, thread_id, board_id, body, body_html, deletion_token, is_op, @@ -1303,7 +1316,7 @@ mod tests { (1, 1, 1, 'live', '

live

', 'tok1', 0, 'live.webp', 'live.webp', 100), (2, 2, 1, 'archived', '

archived

', 'tok2', 0, 'archived.webp', 'archived.webp', 900)", [], - )?; + ).map(|_affected_rows| ())?; let stats = get_site_stats(&conn)?; assert_eq!( @@ -1371,7 +1384,8 @@ mod tests { true, true, true, - )?; + ) + .map(|_created_id| ())?; let board = get_board_by_short(&conn, "audio")?.context("audio board should exist")?; assert!(board.allow_images, "image uploads should be enabled"); @@ -1411,7 +1425,7 @@ mod tests { let pool = crate::db::init_test_pool()?; let conn = pool.get()?; - create_board(&conn, "fresh", "Fresh", "", false)?; + create_board(&conn, "fresh", "Fresh", "", false).map(|_created_id| ())?; let board = get_board_by_short(&conn, "fresh")?.context("fresh board should exist")?; assert_eq!( @@ -1485,7 +1499,8 @@ mod tests { conn.execute( "INSERT INTO threads (id, board_id, subject) VALUES (1, ?1, 'delete me')", rusqlite::params![board_id], - )?; + ) + .map(|_affected_rows| ())?; conn.execute( "INSERT INTO posts ( id, thread_id, board_id, body, body_html, deletion_token, is_op, @@ -1494,7 +1509,8 @@ mod tests { (1, 1, ?1, 'body', '

body

', 'tok', 1, 'gone/file.webp', 'file.webp', 4, 'gone/thumbs/file.webp')", rusqlite::params![board_id], - )?; + ) + .map(|_affected_rows| ())?; let deleted = delete_board(&conn, board_id)?; assert_eq!( diff --git a/src/db/diagnostics.rs b/src/db/diagnostics.rs new file mode 100644 index 00000000..d87f2452 --- /dev/null +++ b/src/db/diagnostics.rs @@ -0,0 +1,146 @@ +//! Timing-only diagnostics; never record SQL parameters or private content. + +use anyhow::{Context as _, Result}; +use std::{ + sync::atomic::{AtomicU64, Ordering}, + time::{Duration, Instant}, +}; + +/// Slow operation threshold; normal checkouts and short writes stay quiet. +const SLOW: Duration = Duration::from_millis(250); +/// Minimum interval between warnings in each pool-event category. +const WARNING_INTERVAL_MS: u64 = 30_000; + +/// Pool-wide events avoid changing the existing pool type or checkout API. +#[derive(Debug)] +pub(super) struct PoolEvents { + /// Monotonic reference time for rate limiting. + started: Instant, + /// Last slow checkout warning timestamp. + checkout_warning: AtomicU64, + /// Last long connection lifetime warning timestamp. + hold_warning: AtomicU64, + /// Last pool exhaustion warning timestamp. + timeout_warning: AtomicU64, +} + +impl PoolEvents { + /// Initialize independent per-pool warning budgets. + pub(super) fn new() -> Self { + Self { + started: Instant::now(), + checkout_warning: AtomicU64::new(0), + hold_warning: AtomicU64::new(0), + timeout_warning: AtomicU64::new(0), + } + } + + /// Atomically grant at most one warning per interval and category. + fn should_warn(&self, last: &AtomicU64) -> bool { + let now = u64::try_from(self.started.elapsed().as_millis()) + .unwrap_or(u64::MAX) + .saturating_add(1); + let previous = last.load(Ordering::Relaxed); + (previous == 0 || now.saturating_sub(previous) >= WARNING_INTERVAL_MS) + && last + .compare_exchange(previous, now, Ordering::Relaxed, Ordering::Relaxed) + .is_ok() + } +} + +impl r2d2::HandleEvent for PoolEvents { + fn handle_checkout(&self, event: r2d2::event::CheckoutEvent) { + if event.duration() >= SLOW && self.should_warn(&self.checkout_warning) { + tracing::warn!(target: "db", event = "slow_checkout", + connection_id = event.connection_id(), duration_ms = %event.duration().as_millis(), + "database pool checkout waited longer than 250 ms"); + } + } + + fn handle_checkin(&self, event: r2d2::event::CheckinEvent) { + if event.duration() >= SLOW && self.should_warn(&self.hold_warning) { + tracing::warn!(target: "db", event = "long_connection_hold", + connection_id = event.connection_id(), duration_ms = %event.duration().as_millis(), + "database connection held longer than 250 ms"); + } + } + + fn handle_timeout(&self, event: r2d2::event::TimeoutEvent) { + if self.should_warn(&self.timeout_warning) { + tracing::warn!(target: "db", event = "pool_timeout", timeout_ms = %event.timeout().as_millis(), + "database pool checkout timed out"); + } + } +} + +/// Measure lock acquisition separately from the posting write-lock window. +#[derive(Debug)] +pub(super) struct WriteTiming { + /// Fixed operation name, never request data. + operation: &'static str, + /// Time the write lock was acquired. + acquired: Instant, + /// Time spent waiting to begin. + wait: Duration, +} + +impl WriteTiming { + /// Acquire the immediate lock and start transaction timing. + pub(super) fn begin(conn: &rusqlite::Connection, operation: &'static str) -> Result { + let started = Instant::now(); + let begin = conn.execute_batch("BEGIN IMMEDIATE"); + let wait = started.elapsed(); + if begin.is_err() || wait >= SLOW { + tracing::debug!(target: "db", event = "write_lock_wait", operation, + wait_ms = %wait.as_millis(), failed = begin.is_err(), + "SQLite immediate write-lock acquisition"); + } + begin.context(operation)?; + Ok(Self { + operation, + acquired: Instant::now(), + wait, + }) + } +} + +impl Drop for WriteTiming { + fn drop(&mut self) { + let held = self.acquired.elapsed(); + if held >= SLOW || self.wait >= SLOW { + // Explicit db=debug enables detailed write diagnostics; overload + // does not produce an unbounded stream at normal info/warn levels. + tracing::debug!(target: "db", event = "posting_transaction", operation = self.operation, + wait_ms = %self.wait.as_millis(), transaction_ms = %held.as_millis(), + "SQLite posting write-lock duration"); + } + } +} + +/// Fixed-label SQL timing for the measured read paths; parameters stay private. +pub(super) struct QueryTiming { + /// Static operation label. + operation: &'static str, + /// Monotonic query start. + started: Instant, +} + +impl QueryTiming { + /// Begin a read measurement without additional queries or allocations. + pub(super) fn start(operation: &'static str) -> Self { + Self { + operation, + started: Instant::now(), + } + } +} + +impl Drop for QueryTiming { + fn drop(&mut self) { + let elapsed = self.started.elapsed(); + if elapsed >= SLOW { + tracing::debug!(target: "db", event = "slow_query", operation = self.operation, + duration_ms = %elapsed.as_millis(), "SQLite read query exceeded 250 ms"); + } + } +} diff --git a/src/db/fs_ops.rs b/src/db/fs_ops.rs index db6798fa..8f7954b1 100644 --- a/src/db/fs_ops.rs +++ b/src/db/fs_ops.rs @@ -47,7 +47,8 @@ pub fn insert_pending_fs_op( "INSERT INTO pending_fs_ops (id, kind, payload_json) VALUES (?1, ?2, ?3)", rusqlite::params![op.id, op.kind, op.payload_json], ) - .context("Insert pending_fs_op failed")?; + .context("Insert pending_fs_op failed") + .map(|_affected_rows| ())?; Ok(()) } @@ -60,7 +61,8 @@ pub fn delete_pending_fs_op(conn: &rusqlite::Connection, id: &str) -> Result<()> "DELETE FROM pending_fs_ops WHERE id = ?1", rusqlite::params![id], ) - .with_context(|| format!("Delete pending_fs_op {id} failed"))?; + .with_context(|| format!("Delete pending_fs_op {id} failed")) + .map(|_affected_rows| ())?; Ok(()) } diff --git a/src/db/migrations.rs b/src/db/migrations.rs index aca6a4bc..3e6871f0 100644 --- a/src/db/migrations.rs +++ b/src/db/migrations.rs @@ -57,7 +57,8 @@ pub(super) fn stamp_schema_version_in_transaction(conn: &rusqlite::Connection) - "INSERT INTO schema_version (version) VALUES (?1)", rusqlite::params![BASELINE_SCHEMA_VERSION], ) - .with_context(|| format!("Failed to set schema_version to {BASELINE_SCHEMA_VERSION}"))?; + .with_context(|| format!("Failed to set schema_version to {BASELINE_SCHEMA_VERSION}")) + .map(|_affected_rows| ())?; Ok(()) } diff --git a/src/db/mod.rs b/src/db/mod.rs index 4e1616db..c3d16b84 100644 --- a/src/db/mod.rs +++ b/src/db/mod.rs @@ -13,6 +13,8 @@ pub mod admin; pub mod banners; /// Board configuration, statistics, and deletion operations. pub mod boards; +/// Thresholded timing for pooled connections and atomic posting writes. +mod diagnostics; /// Durable filesystem-operation records. mod fs_ops; /// Database schema-version bookkeeping. @@ -33,6 +35,9 @@ pub mod threads; mod types; /// Anonymous per-thread display preferences. mod user_thread_prefs; +#[cfg(test)] +/// Explicitly invoked populated SQLite workload and query-plan evidence. +mod workload; pub use pool::{first_run_check, has_no_admin, init_pool}; pub use types::{CachedFile, DbPool, NewPost}; @@ -219,7 +224,8 @@ pub fn paths_safe_to_delete( "DELETE FROM file_hashes WHERE file_path = ?1", params![file_path], ) - .context("Delete stale file_hashes row failed")?; + .context("Delete stale file_hashes row failed") + .map(|_affected_rows| ())?; } } } diff --git a/src/db/pool.rs b/src/db/pool.rs index b1583054..0cbde1f0 100644 --- a/src/db/pool.rs +++ b/src/db/pool.rs @@ -8,9 +8,9 @@ use super::schema::install_or_migrate_schema; use super::types::DbPool; /// Pragmas applied to every pooled `SQLite` connection. -const CONNECTION_PRAGMAS: &str = " +pub(super) const CONNECTION_PRAGMAS: &str = " PRAGMA journal_mode = WAL; - PRAGMA synchronous = NORMAL; + PRAGMA synchronous = FULL; PRAGMA foreign_keys = ON; PRAGMA cache_size = -32000; PRAGMA temp_store = MEMORY; @@ -21,6 +21,21 @@ const CONNECTION_PRAGMAS: &str = " /// Maximum time callers wait for a pooled connection. const POOL_CONNECTION_TIMEOUT: Duration = Duration::from_secs(1); +/// Require one initialized connection before startup; fill spare capacity asynchronously. +fn build_pool(manager: SqliteConnectionManager, pool_size: u32) -> Result { + // r2d2's build() waits for max_size connections by default. That couples + // first-run startup to every spare initializer's scheduling even when a + // usable connection is ready. Keep eager refill and the one-second request + // deadline, but validate readiness through a real checkout instead. + let pool = Pool::builder() + .max_size(pool_size) + .event_handler(Box::new(super::diagnostics::PoolEvents::new())) + .connection_timeout(POOL_CONNECTION_TIMEOUT) + .build_unchecked(manager); + drop(pool.get().context("Failed to initialize database pool")?); + Ok(pool) +} + /// Initialise the `SQLite` connection pool and ensure the schema exists. /// /// # Errors @@ -37,17 +52,15 @@ pub fn init_pool() -> Result { .with_init(|conn| conn.execute_batch(CONNECTION_PRAGMAS)); let pool_size = CONFIG.db_pool_size; - let pool = Pool::builder() - .max_size(pool_size) - .connection_timeout(POOL_CONNECTION_TIMEOUT) - .build(manager) - .context("Failed to build database pool")?; + let pool = build_pool(manager, pool_size)?; let conn = pool.get().context("Failed to get DB connection")?; install_or_migrate_schema(&conn)?; super::upsert_builtin_themes(&conn)?; - tracing::info!(target: "db", path = db_path, "Database initialised"); + tracing::info!(target: "db", path = db_path, pool_size, + checkout_timeout_ms = %POOL_CONNECTION_TIMEOUT.as_millis(), + "Database initialised"); Ok(pool) } @@ -91,7 +104,7 @@ pub fn first_run_check(pool: &DbPool) -> Result<()> { if board_count == 0 { tracing::info!( target: "startup", - boards = 0, + boards = 0_i32, admins = admin_count, "No boards found — create boards via admin panel or: rustchan-cli admin create-board" ); @@ -117,6 +130,86 @@ pub fn has_no_admin(pool: &DbPool) -> bool { mod tests { use anyhow::Result; + /// Holds spare connection initializers until the test releases them. + #[derive(Debug, Default)] + struct InitializationGate { + /// Whether spare initializers can finish. + released: parking_lot::Mutex, + /// Wakes blocked initializers when the test has observed startup. + wake: parking_lot::Condvar, + } + + /// Releases blocked initializers even when the startup regression fails. + #[derive(Debug)] + struct ReleaseInitializers(std::sync::Arc); + + impl Drop for ReleaseInitializers { + fn drop(&mut self) { + *self.0.released.lock() = true; + let _waiting_threads = self.0.wake.notify_all(); + } + } + + #[test] + fn startup_uses_a_ready_connection_while_spare_initializers_are_blocked() -> Result<()> { + let directory = tempfile::tempdir()?; + let gate = std::sync::Arc::new(InitializationGate::default()); + let release = ReleaseInitializers(std::sync::Arc::clone(&gate)); + let openings = std::sync::atomic::AtomicUsize::new(0); + let manager = + r2d2_sqlite::SqliteConnectionManager::file(directory.path().join("startup.db")) + .with_init(move |conn| { + if openings.fetch_add(1, std::sync::atomic::Ordering::Relaxed) != 0 { + let mut released = gate.released.lock(); + while !*released { + gate.wake.wait(&mut released); + } + } + conn.execute_batch(super::CONNECTION_PRAGMAS) + }); + let result = super::build_pool(manager, 4); + drop(release); + let pool = result?; + anyhow::ensure!(pool.max_size() == 4, "startup changed pool capacity"); + anyhow::ensure!( + pool.connection_timeout() == super::POOL_CONNECTION_TIMEOUT, + "startup changed request checkout timeout" + ); + let conn = pool.get()?; + super::install_or_migrate_schema(&conn)?; + crate::db::verify_database_schema(&conn)?; + Ok(()) + } + + #[test] + fn startup_rejects_a_pool_with_no_usable_connection() -> Result<()> { + let directory = tempfile::tempdir()?; + let manager = r2d2_sqlite::SqliteConnectionManager::file(directory.path()); + anyhow::ensure!( + super::build_pool(manager, 4).is_err(), + "startup accepted an unusable database" + ); + Ok(()) + } + + #[test] + fn every_pool_connection_enforces_wal_full_and_foreign_keys() -> Result<()> { + let pool = super::init_test_pool()?; + let connections = (0_i32..4_i32) + .map(|_| pool.get()) + .collect::, _>>()?; + for conn in connections { + let mode: String = conn.query_row("PRAGMA journal_mode", [], |row| row.get(0))?; + let synchronous: i64 = conn.query_row("PRAGMA synchronous", [], |row| row.get(0))?; + let foreign_keys: bool = conn.query_row("PRAGMA foreign_keys", [], |row| row.get(0))?; + anyhow::ensure!( + mode == "wal" && synchronous == 2 && foreign_keys, + "pooled connection weakened durability or relationship enforcement" + ); + } + Ok(()) + } + #[test] #[expect( clippy::panic_in_result_fn, @@ -138,4 +231,121 @@ mod tests { ); Ok(()) } + + #[test] + fn exhausted_pool_returns_a_bounded_retryable_error() -> Result<()> { + let pool = super::init_test_pool()?; + let held = (0_i32..4_i32) + .map(|_| pool.get()) + .collect::, _>>()?; + let started = std::time::Instant::now(); + let error = pool + .get() + .err() + .ok_or_else(|| anyhow::anyhow!("exhausted pool supplied a connection"))?; + anyhow::ensure!( + started.elapsed() < std::time::Duration::from_secs(3), + "pool timeout exceeded its bound" + ); + anyhow::ensure!( + matches!( + crate::error::AppError::from(error), + crate::error::AppError::DbBusy + ), + "pool exhaustion must be retryable" + ); + drop(held); + anyhow::ensure!( + pool.get()?.is_autocommit(), + "returned connection inherited a transaction" + ); + Ok(()) + } + + #[test] + fn wal_readers_remain_usable_and_writer_contention_is_bounded() -> Result<()> { + let pool = super::init_test_pool()?; + let writer = pool.get()?; + writer.execute_batch("BEGIN IMMEDIATE")?; + writer + .execute( + "INSERT INTO site_settings(key,value) VALUES('uncommitted','value')", + [], + ) + .map(|_affected_rows| ())?; + let reader = pool.get()?; + let visible: i64 = reader.query_row( + "SELECT COUNT(*) FROM site_settings WHERE key='uncommitted'", + [], + |row| row.get(0), + )?; + anyhow::ensure!(visible == 0, "reader observed an uncommitted write"); + let started = std::time::Instant::now(); + let error = reader + .execute( + "INSERT INTO site_settings(key,value) VALUES('contended','value')", + [], + ) + .err() + .ok_or_else(|| anyhow::anyhow!("competing write unexpectedly succeeded"))?; + anyhow::ensure!( + started.elapsed() < std::time::Duration::from_secs(3), + "busy wait exceeded its bound" + ); + anyhow::ensure!( + matches!( + crate::error::AppError::from(error), + crate::error::AppError::DbBusy + ), + "busy writer must be retryable" + ); + writer.execute_batch("ROLLBACK")?; + anyhow::ensure!( + reader.is_autocommit(), + "busy failure left a transaction active" + ); + reader + .execute( + "INSERT INTO site_settings(key,value) VALUES('recovered','value')", + [], + ) + .map(|_affected_rows| ())?; + Ok(()) + } + + #[test] + fn active_read_snapshot_bounds_checkpoint_and_wal_backup_stays_complete() -> Result<()> { + let pool = super::init_test_pool()?; + let writer = pool.get()?; + crate::db::create_board(&writer, "wal", "WAL", "", false).map(|_created_id| ())?; + let reader = pool.get()?; + reader.execute_batch("BEGIN; SELECT COUNT(*) FROM boards;")?; + crate::db::create_board(&writer, "next", "Next", "", false).map(|_created_id| ())?; + let (log, checkpointed, busy) = crate::db::run_wal_checkpoint(&writer)?; + anyhow::ensure!( + busy == 1 && checkpointed < log, + "active reader must prevent a complete truncate checkpoint" + ); + let directory = tempfile::tempdir()?; + let path = directory.path().join("backup.sqlite3"); + writer + .execute( + "VACUUM INTO ?1", + [path + .to_str() + .ok_or_else(|| anyhow::anyhow!("backup path"))?], + ) + .map(|_affected_rows| ())?; + let snapshot = rusqlite::Connection::open(&path)?; + let boards: i64 = snapshot.query_row("SELECT COUNT(*) FROM boards", [], |r| r.get(0))?; + anyhow::ensure!(boards == 2, "WAL snapshot missed a committed board"); + crate::db::verify_database_schema(&snapshot)?; + reader.execute_batch("ROLLBACK")?; + let (_, _, recovered_busy) = crate::db::run_wal_checkpoint(&writer)?; + anyhow::ensure!( + recovered_busy == 0, + "checkpoint did not recover after reader release" + ); + Ok(()) + } } diff --git a/src/db/posts.rs b/src/db/posts.rs index 88f7d572..7cd2dcdf 100644 --- a/src/db/posts.rs +++ b/src/db/posts.rs @@ -121,7 +121,7 @@ pub struct RecentBackgroundJob { /// 5 subject 13 `mime_type` 21 `audio_mime_type` /// 6 body 14 `created_at` 22 `edited_at` /// 7 `body_html` 15 `deletion_token` 23 `media_processing_state` -/// 24 `media_processing_error` +/// 24 `media_processing_error`. /// /// # Errors /// Returns an error if the database operation fails. @@ -225,10 +225,7 @@ pub fn get_posts_by_ids_in_thread( return Ok(Vec::new()); } - let placeholders = post_ids - .iter() - .enumerate() - .map(|(index, _)| format!("?{}", index + 3)) + let placeholders = std::iter::repeat_n("?", post_ids.len()) .collect::>() .join(", "); let sql = format!( @@ -265,13 +262,9 @@ pub fn get_preview_posts_for_threads( return Ok(HashMap::new()); } - let placeholders = thread_ids - .iter() - .enumerate() - .map(|(index, _)| format!("?{}", index + 1)) + let placeholders = std::iter::repeat_n("?", thread_ids.len()) .collect::>() .join(", "); - let limit_param = thread_ids.len() + 1; let sql = format!( "SELECT {POST_SELECT_COLUMNS} FROM ( @@ -283,7 +276,7 @@ pub fn get_preview_posts_for_threads( FROM posts WHERE is_op = 0 AND thread_id IN ({placeholders}) ) - WHERE preview_rank <= ?{limit_param} + WHERE preview_rank <= ? ORDER BY thread_id ASC, created_at ASC, id ASC" ); let mut stmt = conn.prepare_cached(&sql)?; @@ -382,7 +375,7 @@ pub fn get_post_submission( Ok(PostSubmissionRecord { thread_id: row.get(0)?, post_id: row.get(1)?, - is_thread: row.get::<_, i32>(2)? != 0, + is_thread: row.get::<_, i32>(2)? != 0_i32, }) }, ) @@ -419,13 +412,15 @@ pub fn record_post_submission( i32::from(is_thread) ], ) - .context("Failed to record unique post submission token")?; + .context("Failed to record unique post submission token") + .map(|_affected_rows| ())?; conn.execute( "DELETE FROM post_submissions WHERE created_at < unixepoch() - 604800", [], ) - .context("Failed to prune expired post submission tokens")?; + .context("Failed to prune expired post submission tokens") + .map(|_affected_rows| ())?; Ok(()) } @@ -459,7 +454,8 @@ pub(super) fn create_poll_inner( text, i64::try_from(i).context("poll option index overflow")? ]) - .context("Failed to insert poll option")?; + .context("Failed to insert poll option") + .map(|_affected_rows| ())?; } Ok(poll_id) @@ -540,7 +536,7 @@ fn delete_post_reply_in_tx( .query_row(params![post_id], |r| { Ok(( r.get::<_, i64>(0)?, - r.get::<_, i32>(1)? != 0, + r.get::<_, i32>(1)? != 0_i32, r.get::<_, Option>(2)?, r.get::<_, Option>(3)?, r.get::<_, Option>(4)?, @@ -646,7 +642,12 @@ pub fn post_thread_allows_self_actions( JOIN threads t ON t.id = p.thread_id WHERE p.id = ?1", params![post_id], - |row| Ok((row.get::<_, i32>(0)? != 0, row.get::<_, i32>(1)? != 0)), + |row| { + Ok(( + row.get::<_, i32>(0)? != 0_i32, + row.get::<_, i32>(1)? != 0_i32, + )) + }, ) .optional()?; Ok(flags.is_some_and(|(locked, archived)| !locked && !archived)) @@ -681,11 +682,11 @@ pub fn self_delete_post( |r| { Ok(( r.get::<_, i64>(0)?, - r.get::<_, i32>(1)? != 0, + r.get::<_, i32>(1)? != 0_i32, r.get::<_, String>(2)?, r.get::<_, i64>(3)?, - r.get::<_, i32>(4)? != 0, - r.get::<_, i32>(5)? != 0, + r.get::<_, i32>(4)? != 0_i32, + r.get::<_, i32>(5)? != 0_i32, )) }, ) @@ -784,8 +785,8 @@ pub fn edit_post( Ok(( r.get(0)?, r.get(1)?, - r.get::<_, i32>(2)? != 0, - r.get::<_, i32>(3)? != 0, + r.get::<_, i32>(2)? != 0_i32, + r.get::<_, i32>(3)? != 0_i32, )) }, ) @@ -811,7 +812,8 @@ pub fn edit_post( conn.execute( "UPDATE posts SET body = ?1, body_html = ?2, edited_at = ?3 WHERE id = ?4", params![new_body, new_body_html, now, post_id], - )?; + ) + .map(|_affected_rows| ())?; // Belt-and-suspenders: confirm the row was actually written. Ok(conn.changes() > 0) @@ -903,23 +905,24 @@ pub fn search_posts( limit: i64, offset: i64, ) -> Result> { + let _timing = super::diagnostics::QueryTiming::start("search_posts"); let Some(fts_query) = to_fts_query(query) else { return Ok(Vec::new()); }; - let mut stmt = conn.prepare_cached( - "SELECT posts.id, posts.thread_id, posts.board_id, posts.name, posts.tripcode, + // Prefix MATCH queries drive from FTS so each matching row is looked up + // once, instead of running an FTS prefix probe for every board post. + let sql = "SELECT posts.id, posts.thread_id, posts.board_id, posts.name, posts.tripcode, posts.subject, posts.body, posts.body_html, posts.ip_hash, posts.file_path, posts.file_name, posts.file_size, posts.thumb_path, posts.mime_type, posts.created_at, posts.deletion_token, posts.is_op, posts.media_type, posts.audio_file_path, posts.audio_file_name, posts.audio_file_size, posts.audio_mime_type, posts.edited_at, posts.media_processing_state, posts.media_processing_error - FROM posts - JOIN posts_fts ON posts_fts.rowid = posts.id + FROM posts_fts CROSS JOIN posts ON posts.id = posts_fts.rowid WHERE posts.board_id = ?1 AND posts_fts MATCH ?2 ORDER BY posts.created_at DESC, posts.id DESC - LIMIT ?3 OFFSET ?4", - )?; + LIMIT ?3 OFFSET ?4"; + let mut stmt = conn.prepare_cached(sql)?; let posts = stmt .query_map(params![board_id, fts_query, limit, offset], map_post)? .collect::>>()?; @@ -933,17 +936,17 @@ pub fn count_search_results( board_id: i64, query: &str, ) -> Result { + let _timing = super::diagnostics::QueryTiming::start("count_search_results"); let Some(fts_query) = to_fts_query(query) else { return Ok(0); }; - Ok(conn.query_row( - "SELECT COUNT(*) - FROM posts - JOIN posts_fts ON posts_fts.rowid = posts.id - WHERE posts.board_id = ?1 AND posts_fts MATCH ?2", - params![board_id, fts_query], - |r| r.get(0), - )?) + Ok(conn + .prepare_cached( + "SELECT COUNT(*) + FROM posts_fts CROSS JOIN posts ON posts.id = posts_fts.rowid + WHERE posts.board_id = ?1 AND posts_fts MATCH ?2", + )? + .query_row(params![board_id, fts_query], |r| r.get(0))?) } // File deduplication @@ -1012,7 +1015,8 @@ pub fn record_file_hash( "INSERT OR REPLACE INTO file_hashes (sha256, file_path, thumb_path, mime_type) VALUES (?1, ?2, ?3, ?4)", params![sha256, file_path, thumb_path, mime_type], - )?; + ) + .map(|_affected_rows| ())?; Ok(()) } @@ -1327,7 +1331,8 @@ pub fn persist_media_job( updated_at = unixepoch() WHERE id = ?1 AND status IN ('pending', 'running')", params![id], - )?; + ) + .map(|_affected_rows| ())?; if status == "pending" { pending_jobs_resolved = pending_jobs_resolved.saturating_add(1); } @@ -1352,7 +1357,8 @@ pub fn persist_media_job( expected_source, MEDIA_PROCESSING_FAILED ], - )?; + ) + .map(|_affected_rows| ())?; } return Ok(MediaJobSchedule { job_id, @@ -1444,7 +1450,8 @@ pub fn reject_media_job_at_capacity( expected_source, MEDIA_PROCESSING_FAILED ], - )?; + ) + .map(|_affected_rows| ())?; Ok(None) })(); match result { @@ -1522,7 +1529,8 @@ pub(crate) fn persist_thread_prune_intent_in_tx( WHERE id = ?1 AND status = 'pending'", params![job_id, payload], ) - .context("Refresh coalesced board prune intent failed")?; + .context("Refresh coalesced board prune intent failed") + .map(|_affected_rows| ())?; for duplicate_id in duplicate_ids { conn.execute( @@ -1533,7 +1541,8 @@ pub(crate) fn persist_thread_prune_intent_in_tx( WHERE id = ?1 AND status = 'pending'", params![duplicate_id], ) - .context("Resolve duplicate legacy board prune intent failed")?; + .context("Resolve duplicate legacy board prune intent failed") + .map(|_affected_rows| ())?; } return Ok(ThreadPruneSchedule { @@ -1865,7 +1874,8 @@ pub fn recover_interrupted_background_jobs( WHERE id = ?1 AND media_processing_state IN (?2, 'running')", params![post_id, MEDIA_PROCESSING_PENDING], - )?; + ) + .map(|_affected_rows| ())?; } } } @@ -1971,16 +1981,24 @@ fn interrupted_media_job_disposition( media_post_id: None, }); }; - let post_id = match &target { - InterruptedMediaTarget::Video { post_id, .. } - | InterruptedMediaTarget::Audio { post_id, .. } => *post_id, + let target_post_id = match &target { + InterruptedMediaTarget::Video { + post_id, + source_path: _, + expected_output_path: _, + } + | InterruptedMediaTarget::Audio { + post_id, + source_path: _, + expected_thumb_path: _, + } => *post_id, }; let post = conn .query_row( "SELECT file_path, thumb_path, media_processing_state FROM posts WHERE id = ?1", - rusqlite::params![post_id], + rusqlite::params![target_post_id], |row| { Ok(( row.get::<_, Option>(0)?, @@ -2160,7 +2178,8 @@ pub fn set_post_media_processing_state( media_processing_error = ?2 WHERE id = ?3", params![normalized_state, normalized_error, post_id], - )?; + ) + .map(|_affected_rows| ())?; Ok(()) } @@ -2287,7 +2306,8 @@ pub fn complete_media_job_without_output_in_tx( WHERE id = ?1 AND file_path = ?2 AND media_processing_state IN (?3, 'running')", params![post_id, expected_source, MEDIA_PROCESSING_PENDING], - )?; + ) + .map(|_affected_rows| ())?; complete_job(conn, job_id) } @@ -2344,7 +2364,8 @@ pub fn fail_media_job_in_tx( WHERE id = ?2 AND file_path = ?3 AND media_processing_state IN (?1, 'running')", params![MEDIA_PROCESSING_PENDING, post_id, expected_source], - )?; + ) + .map(|_affected_rows| ())?; } JobFailureState::PermanentlyFailed => { let error: String = error.trim().chars().take(512).collect(); @@ -2360,7 +2381,8 @@ pub fn fail_media_job_in_tx( expected_source, MEDIA_PROCESSING_PENDING ], - )?; + ) + .map(|_affected_rows| ())?; } } Ok(Some(failure_state)) @@ -2509,7 +2531,8 @@ pub fn finish_waveform_media_job_in_tx( "UPDATE file_hashes SET thumb_path = ?1 WHERE sha256 = ?2 AND file_path = ?3", params![output_path, source_sha256, expected_source], - )?; + ) + .map(|_affected_rows| ())?; let cleanup = if previous_thumb .as_deref() .is_some_and(|path| path != output_path) @@ -2590,7 +2613,8 @@ pub fn replace_transcoded_media( conn.execute( "DELETE FROM file_hashes WHERE file_path = ?1", params![old_path], - )?; + ) + .map(|_affected_rows| ())?; record_file_hash(conn, new_sha256, new_path, &thumb_path, new_mime)?; Ok(()) })(); @@ -2645,7 +2669,8 @@ pub fn delete_file_hash_by_path(conn: &rusqlite::Connection, file_path: &str) -> conn.execute( "DELETE FROM file_hashes WHERE file_path = ?1", params![file_path], - )?; + ) + .map(|_affected_rows| ())?; Ok(()) } @@ -2666,7 +2691,7 @@ mod tests { }; use crate::error::AppError; use anyhow::{Context as _, Result}; - use rusqlite::Connection; + use rusqlite::{params, Connection}; fn test_conn() -> Result { let conn = Connection::open_in_memory()?; @@ -2675,7 +2700,7 @@ mod tests { } fn seed_search_post(conn: &Connection, board_short: &str, body: &str) -> Result { - create_board(conn, board_short, board_short, "", false)?; + create_board(conn, board_short, board_short, "", false).map(|_created_id| ())?; let board = get_board_by_short(conn, board_short)?.context("seeded search board should exist")?; let post = NewPost { @@ -2707,7 +2732,7 @@ mod tests { } fn seed_media_post(conn: &Connection, board_short: &str, file_path: &str) -> Result { - create_board(conn, board_short, board_short, "", false)?; + create_board(conn, board_short, board_short, "", false).map(|_created_id| ())?; let board = get_board_by_short(conn, board_short)?.context("seeded media board should exist")?; let post = NewPost { @@ -2775,7 +2800,8 @@ mod tests { insert_background_job(&conn, "spam_check", payload, "failed", 3, Some("older"))?; let newer = insert_background_job(&conn, "thread_prune", payload, "failed", 2, Some("newer"))?; - insert_background_job(&conn, "spam_check", payload, "pending", 0, None)?; + insert_background_job(&conn, "spam_check", payload, "pending", 0, None) + .map(|_created_id| ())?; let jobs = recent_background_jobs(&conn, "failed", 1)?; @@ -2809,7 +2835,8 @@ mod tests { fn failed_background_job_acknowledgement_preserves_history() -> Result<()> { let conn = test_conn()?; let payload = r#"{"t":"SpamCheck","d":{"post_id":1,"ip_hash":"hash","body_len":5}}"#; - insert_background_job(&conn, "spam_check", payload, "failed", 3, Some("older"))?; + insert_background_job(&conn, "spam_check", payload, "failed", 3, Some("older")) + .map(|_created_id| ())?; let acknowledged = insert_background_job(&conn, "thread_prune", payload, "failed", 3, Some("newer"))?; @@ -2841,7 +2868,8 @@ mod tests { "failed", 3, Some("new failure"), - )?; + ) + .map(|_created_id| ())?; assert_eq!( background_job_summary(&conn)?.failed, 1, @@ -2910,7 +2938,7 @@ mod tests { )] fn search_posts_reads_joined_fts_rows_without_ambiguous_columns() -> Result<()> { let conn = test_conn()?; - seed_search_post(&conn, "tech", "rust search body")?; + seed_search_post(&conn, "tech", "rust search body").map(|_created_id| ())?; let board = get_board_by_short(&conn, "tech")?.context("tech board should exist")?; let posts = search_posts(&conn, board.id, "rust", 20, 0)?; @@ -2931,8 +2959,8 @@ mod tests { )] fn search_posts_stays_scoped_to_board() -> Result<()> { let conn = test_conn()?; - seed_search_post(&conn, "tech", "shared rust term")?; - seed_search_post(&conn, "meta", "shared rust term")?; + seed_search_post(&conn, "tech", "shared rust term").map(|_created_id| ())?; + seed_search_post(&conn, "meta", "shared rust term").map(|_created_id| ())?; let tech = get_board_by_short(&conn, "tech")?.context("tech board should exist")?; let posts = search_posts(&conn, tech.id, "rust", 20, 0)?; @@ -2959,7 +2987,7 @@ mod tests { )] fn search_posts_matches_case_insensitively() -> Result<()> { let conn = test_conn()?; - seed_search_post(&conn, "tech", "AI will find this")?; + seed_search_post(&conn, "tech", "AI will find this").map(|_created_id| ())?; let board = get_board_by_short(&conn, "tech")?.context("tech board should exist")?; let posts = search_posts(&conn, board.id, "ai", 20, 0)?; @@ -2997,6 +3025,59 @@ mod tests { Ok(()) } + #[test] + fn search_prefixes_preserve_board_order_offset_and_counts() -> Result<()> { + let conn = test_conn()?; + let op = seed_search_post(&conn, "search", "common needle")?; + let board = get_board_by_short(&conn, "search")?.context("search board")?; + let thread = get_post(&conn, op)?.context("search OP")?.thread_id; + conn.execute( + "WITH RECURSIVE n(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM n WHERE x<1100) + INSERT INTO posts(thread_id,board_id,body,body_html,deletion_token,created_at) + SELECT ?1,?2,CASE WHEN x%101=0 THEN 'common needle' ELSE 'common' END, + 'body','delete',1700000000+x%7 FROM n", + params![thread, board.id], + ) + .map(|_affected_rows| ())?; + for term in ["common", "needle", "absent"] { + let total: i64 = conn.query_row( + "SELECT COUNT(*) FROM posts WHERE board_id=?1 AND body LIKE ?2", + params![board.id, format!("%{term}%")], + |row| row.get(0), + )?; + anyhow::ensure!( + count_search_results(&conn, board.id, term)? == total, + "search count changed" + ); + for offset in [0, 7, 1095] { + let mut statement = conn.prepare( + "SELECT id FROM posts WHERE board_id=?1 AND body LIKE ?2 + ORDER BY created_at DESC,id DESC LIMIT 20 OFFSET ?3", + )?; + let expected = statement + .query_map(params![board.id, format!("%{term}%"), offset], |row| { + row.get::<_, i64>(0) + })? + .collect::>>()?; + let posts = search_posts(&conn, board.id, term, 20, offset)?; + anyhow::ensure!( + expected == posts.iter().map(|post| post.id).collect::>(), + "search pagination changed" + ); + anyhow::ensure!( + search_posts(&conn, board.id + 1, term, 20, offset)?.is_empty(), + "search escaped board" + ); + } + } + conn.execute( + "INSERT INTO posts_fts(posts_fts,rank) VALUES('integrity-check',1)", + [], + ) + .map(|_affected_rows| ())?; + Ok(()) + } + #[test] #[expect( clippy::panic_in_result_fn, @@ -3297,7 +3378,8 @@ mod tests { "running", 1, Some("old error"), - )?; + ) + .map(|_created_id| ())?; let recovery = recover_interrupted_background_jobs(&conn)?; @@ -3335,7 +3417,8 @@ mod tests { conn.execute( "UPDATE posts SET file_path = 'applied/video.webm' WHERE id = ?1", rusqlite::params![post_id], - )?; + ) + .map(|_affected_rows| ())?; let payload = format!( r#"{{"t":"VideoTranscode","d":{{"post_id":{post_id},"file_path":"applied/video.mp4","board_short":"applied"}}}}"# ); @@ -3398,7 +3481,8 @@ mod tests { media_processing_state = ?2 WHERE id = ?1", rusqlite::params![post_id, MEDIA_PROCESSING_PENDING], - )?; + ) + .map(|_affected_rows| ())?; let payload = format!( r#"{{"t":"VideoTranscode","d":{{"post_id":{post_id},"file_path":"newer/old.mp4","board_short":"newer"}}}}"# ); @@ -3448,7 +3532,8 @@ mod tests { media_processing_state = 'running' WHERE id = ?1", rusqlite::params![post_id], - )?; + ) + .map(|_affected_rows| ())?; let payload = format!( r#"{{"t":"AudioWaveform","d":{{"post_id":{post_id},"file_path":"audio/track.mp3","board_short":"audio"}}}}"# ); @@ -3503,7 +3588,7 @@ mod tests { let payload = r#"{"t":"SpamCheck","d":{"post_id":42,"ip_hash":"hash","body_len":5}}"#; let job_id = insert_background_job(&conn, "spam_check", payload, "running", 1, None)?; - recover_interrupted_background_jobs(&conn)?; + recover_interrupted_background_jobs(&conn).map(|_operation_summary| ())?; let claimed = claim_next_job(&conn)?.context("recovered job should be claimable")?; assert_eq!( @@ -3773,7 +3858,7 @@ mod tests { "reply creation should increment the thread count" ); - super::delete_post(&conn, reply_id)?; + super::delete_post(&conn, reply_id).map(|_operation_summary| ())?; let after_count: i64 = conn.query_row( "SELECT reply_count FROM threads WHERE id = ?1", @@ -3944,11 +4029,12 @@ mod tests { deletion_token: "reply-token".to_owned(), is_op: false, }; - create_reply_with_thread_update(&conn, &reply, "", false, None)?; + create_reply_with_thread_update(&conn, &reply, "", false, None).map(|_created_id| ())?; conn.execute( "UPDATE threads SET reply_count = 0 WHERE id = ?1", [thread_id], - )?; + ) + .map(|_affected_rows| ())?; let (outcome, deleted) = self_delete_post(&conn, op_id, "op-token", 60)?; diff --git a/src/db/schema.rs b/src/db/schema.rs index e27b0f13..f5ce1a2f 100644 --- a/src/db/schema.rs +++ b/src/db/schema.rs @@ -278,6 +278,10 @@ const INDEX_SCHEMA_SQL: &str = " ON threads(board_id, sticky DESC, bumped_at DESC); CREATE INDEX IF NOT EXISTS idx_posts_thread ON posts(thread_id, created_at ASC); + CREATE INDEX IF NOT EXISTS idx_posts_thread_live + ON posts(thread_id, id); + CREATE INDEX IF NOT EXISTS idx_posts_board_ip_created + ON posts(board_id, ip_hash, created_at DESC); CREATE INDEX IF NOT EXISTS idx_posts_board ON posts(board_id, created_at DESC); CREATE INDEX IF NOT EXISTS idx_bans_ip @@ -336,7 +340,9 @@ const INDEX_SCHEMA_SQL: &str = " /// Obsolete theme index accepted only during the known legacy repair path. const LEGACY_THEME_SORT_INDEX: &str = "idx_themes_enabled_sort"; /// Additive indexes introduced after the first package-version baseline. -const ADDITIVE_BASELINE_INDEXES: [&str; 9] = [ +const ADDITIVE_BASELINE_INDEXES: [&str; 11] = [ + "idx_posts_thread_live", + "idx_posts_board_ip_created", "idx_posts_file_path", "idx_posts_thumb_path", "idx_posts_audio_file_path", @@ -1020,7 +1026,7 @@ fn themes_table_is_legacy_repairable(expected: &TableShape, actual: &TableShape) } let mut actual_indexes = actual.indexes.clone(); - actual_indexes.remove(LEGACY_THEME_SORT_INDEX); + let _previous_value = actual_indexes.remove(LEGACY_THEME_SORT_INDEX); actual_indexes == expected.indexes } @@ -1333,12 +1339,14 @@ fn restore_autoincrement_sequence( .with_context(|| format!("Read rebuilt {table} maximum id failed"))?; let sequence = prior_sequence.max(current_max); conn.execute("DELETE FROM sqlite_sequence WHERE name = ?1", [table]) - .with_context(|| format!("Clear rebuilt {table} AUTOINCREMENT sequence failed"))?; + .with_context(|| format!("Clear rebuilt {table} AUTOINCREMENT sequence failed")) + .map(|_affected_rows| ())?; conn.execute( "INSERT INTO sqlite_sequence (name, seq) VALUES (?1, ?2)", rusqlite::params![table, sequence], ) - .with_context(|| format!("Restore rebuilt {table} AUTOINCREMENT sequence failed"))?; + .with_context(|| format!("Restore rebuilt {table} AUTOINCREMENT sequence failed")) + .map(|_affected_rows| ())?; Ok(()) } @@ -1636,7 +1644,8 @@ fn schema_version_metadata_issues(conn: &rusqlite::Connection) -> Result Result<()> { conn.query_row("SELECT 1", [], |row| row.get::<_, i64>(0)) - .context("Database did not answer a readiness query")?; + .context("Database did not answer a readiness query") + .map(|_readiness_value| ())?; let version: String = conn .query_row( "SELECT CAST(version AS TEXT) FROM schema_version", @@ -1678,7 +1687,7 @@ fn schema_shape(conn: &rusqlite::Connection) -> Result { let mut tables = BTreeMap::new(); for (name, object) in &objects { if object.kind == "table" { - tables.insert(name.clone(), table_shape(conn, name)?); + let _previous_value = tables.insert(name.clone(), table_shape(conn, name)?); } } Ok(SchemaShape { objects, tables }) @@ -1714,7 +1723,7 @@ fn schema_objects(conn: &rusqlite::Connection) -> Result Result Vec { let mut current = String::new(); let mut chars = sql.chars().peekable(); - let flush_current = |current: &mut String, tokens: &mut Vec| { - if !current.is_empty() { - tokens.push(std::mem::take(current)); + let flush_current = |fragment: &mut String, output: &mut Vec| { + if !fragment.is_empty() { + output.push(std::mem::take(fragment)); } }; @@ -2154,28 +2164,36 @@ fn check_constraints_from_tokens(tokens: &[String]) -> BTreeSet { while let Some(token) = tokens.get(index) { let next_index = index.saturating_add(1); if token != "check" || tokens.get(next_index).map(String::as_str) != Some("(") { - index += 1; + index = next_index; continue; } let start = next_index; - let mut depth = 0_i64; + let mut depth = 0_usize; let mut end = start; while let Some(expression_token) = tokens.get(end) { match expression_token.as_str() { - "(" => depth += 1, + "(" => { + let Some(next_depth) = depth.checked_add(1) else { + break; + }; + depth = next_depth; + } ")" => { - depth -= 1; + let Some(next_depth) = depth.checked_sub(1) else { + break; + }; + depth = next_depth; if depth == 0 { if let Some(expression) = tokens.get(start..=end) { - constraints.insert(expression.join(" ")); + let _inserted = constraints.insert(expression.join(" ")); } break; } } _ => {} } - end += 1; + end = end.saturating_add(1); } index = end.saturating_add(1); } @@ -2245,7 +2263,8 @@ mod tests { conn.execute( "INSERT INTO schema_version (version) VALUES (?1)", [version], - )?; + ) + .map(|_affected_rows| ())?; Ok(()) } @@ -2413,22 +2432,25 @@ mod tests { let results = std::thread::scope(|scope| { let mut handles = Vec::new(); - for _ in 0..2 { + for _ in 0_i32..2_i32 { let path = database_path.clone(); let barrier = std::sync::Arc::clone(&barrier); handles.push(scope.spawn(move || -> Result<()> { let conn = rusqlite::Connection::open(path)?; conn.busy_timeout(std::time::Duration::from_secs(5))?; - barrier.wait(); + let _barrier_state = barrier.wait(); install_or_migrate_schema(&conn) })); } handles .into_iter() .map(|handle| { - handle - .join() - .map_err(|_| anyhow::anyhow!("schema installer thread panicked"))? + handle.join().map_err(|panic_payload| { + anyhow::anyhow!( + "schema installer thread panicked: {}", + crate::media::process::panic_message(panic_payload.as_ref()) + ) + })? }) .collect::>>() })?; @@ -2458,11 +2480,13 @@ mod tests { conn.execute( "INSERT INTO schema_version (version) VALUES (?1)", [baseline_schema_version()], - )?; + ) + .map(|_affected_rows| ())?; conn.execute( "INSERT INTO boards (id, short_name, name) VALUES (1, 'b', 'Random')", [], - )?; + ) + .map(|_affected_rows| ())?; normalize_database_schema_version(&conn)?; @@ -2623,7 +2647,8 @@ mod tests { ) VALUES (1, 'b', 'Random', 0, 1, 0, 1, 'override', 'public')", [], - )?; + ) + .map(|_affected_rows| ())?; rebuild_boards_with_historical_v41_shape(&conn)?; conn.execute_batch( "CREATE INDEX idx_themes_enabled_sort @@ -2746,7 +2771,8 @@ mod tests { fn unknown_schema_object_fails_closed() -> Result<()> { let conn = rusqlite::Connection::open_in_memory()?; install_or_migrate_schema(&conn)?; - conn.execute("CREATE TABLE operator_notes (body TEXT)", [])?; + conn.execute("CREATE TABLE operator_notes (body TEXT)", []) + .map(|_affected_rows| ())?; let error = normalize_database_schema_version(&conn) .err() @@ -2772,7 +2798,8 @@ mod tests { conn.execute( "INSERT INTO boards (short_name, name) VALUES ('fresh', 'Fresh Board')", [], - )?; + ) + .map(|_affected_rows| ())?; let flags: (i64, i64, i64, i64, i64) = conn .query_row( @@ -2844,17 +2871,20 @@ mod tests { conn.execute( "INSERT INTO boards (id, short_name, name) VALUES (1, 'b', 'Random')", [], - )?; + ) + .map(|_affected_rows| ())?; conn.execute( "INSERT INTO threads (id, board_id, subject) VALUES (10, 1, 'thread')", [], - )?; + ) + .map(|_affected_rows| ())?; conn.execute( "INSERT INTO posts (id, thread_id, board_id, body, body_html, deletion_token, is_op) VALUES (100, 10, 1, 'body', 'body', 'token', 1)", [], - )?; + ) + .map(|_affected_rows| ())?; for sql in [ "UPDATE boards SET nsfw = 2 WHERE id = 1", @@ -2905,7 +2935,8 @@ mod tests { "INSERT INTO background_jobs (id, job_type, payload) VALUES (1, 'spam_check', '{}')", [], - )?; + ) + .map(|_affected_rows| ())?; assert_sql_rejected( &conn, "UPDATE background_jobs SET status = 'stuck' WHERE id = 1", @@ -3080,7 +3111,8 @@ mod tests { "INSERT INTO boards (id, short_name, name, max_threads) VALUES (1, 'b', 'Random', 0)", [], - )?; + ) + .map(|_affected_rows| ())?; let error = normalize_database_schema_version(&conn) .err() @@ -3116,7 +3148,8 @@ mod tests { conn.execute( "INSERT INTO boards (id, short_name, name) VALUES (1, 'b', 'Random')", [], - )?; + ) + .map(|_affected_rows| ())?; let access_trigger: String = conn.query_row( "SELECT sql FROM sqlite_schema WHERE type = 'trigger' AND name = 'boards_access_mode_update'", @@ -3127,7 +3160,8 @@ mod tests { conn.execute( "UPDATE boards SET access_mode = 'unknown_mode' WHERE id = 1", [], - )?; + ) + .map(|_affected_rows| ())?; conn.execute_batch(&access_trigger)?; conn.execute_batch( "CREATE TABLE schema_version (version TEXT NOT NULL PRIMARY KEY); @@ -3191,7 +3225,8 @@ mod tests { next_id == 501, "a rebuild must not reuse an AUTOINCREMENT id issued before deletion" ); - conn.execute("DELETE FROM boards WHERE id = 1", [])?; + conn.execute("DELETE FROM boards WHERE id = 1", []) + .map(|_affected_rows| ())?; let child_count = conn.query_row("SELECT COUNT(*) FROM threads WHERE id = 10", [], |row| { row.get::<_, i64>(0) diff --git a/src/db/setup.rs b/src/db/setup.rs index d64e902d..bc7e54fc 100644 --- a/src/db/setup.rs +++ b/src/db/setup.rs @@ -128,7 +128,8 @@ pub fn close_reopened_setup(conn: &rusqlite::Connection) -> Result<()> { "DELETE FROM site_settings WHERE key IN (?1, ?2)", params![SETUP_REOPENED_AT_KEY, SETUP_REOPENED_BY_KEY], ) - .context("clear setup reopen marker")?; + .context("clear setup reopen marker") + .map(|_affected_rows| ())?; Ok(()) } @@ -144,7 +145,8 @@ pub fn mark_setup_complete(conn: &rusqlite::Connection) -> Result<()> { ON CONFLICT(key) DO UPDATE SET value = excluded.value", params![SETUP_COMPLETED_AT_KEY, now], ) - .context("write setup completion marker")?; + .context("write setup completion marker") + .map(|_affected_rows| ())?; close_reopened_setup(conn) })(); finish_savepoint( @@ -237,7 +239,7 @@ mod tests { fn admin_without_marker_blocks_setup_as_initialized() -> Result<()> { let pool = crate::db::init_test_pool()?; let conn = pool.get()?; - crate::db::create_admin(&conn, "admin", "hash")?; + crate::db::create_admin(&conn, "admin", "hash").map(|_created_id| ())?; let state = setup_state(&conn)?; @@ -265,7 +267,7 @@ mod tests { fn admin_reopen_makes_setup_available_but_admin_authenticated() -> Result<()> { let pool = crate::db::init_test_pool()?; let conn = pool.get()?; - crate::db::create_admin(&conn, "admin", "hash")?; + crate::db::create_admin(&conn, "admin", "hash").map(|_created_id| ())?; reopen_setup(&conn, 1)?; let state = setup_state(&conn)?; diff --git a/src/db/themes.rs b/src/db/themes.rs index 4108ad17..3d594215 100644 --- a/src/db/themes.rs +++ b/src/db/themes.rs @@ -119,7 +119,7 @@ pub fn upsert_builtin_themes(conn: &rusqlite::Connection) -> Result<()> { theme.sort_order, ], ) - .context("Failed to upsert built-in theme")?; + .context("Failed to upsert built-in theme").map(|_affected_rows| ())?; } Ok(()) } @@ -155,7 +155,7 @@ pub fn create_custom_theme( custom_css, ], ) - .context("Failed to create custom theme")?; + .context("Failed to create custom theme").map(|_affected_rows| ())?; Ok(()) } @@ -223,13 +223,15 @@ pub fn update_theme( "UPDATE boards SET default_theme = ?1 WHERE lower(default_theme) = lower(?2)", params![new_slug, existing_slug], ) - .context("Failed to update board theme references")?; + .context("Failed to update board theme references") + .map(|_affected_rows| ())?; tx.execute( "UPDATE site_settings SET value = ?1 WHERE key = 'default_theme' AND lower(value) = lower(?2)", params![new_slug, existing_slug], ) - .context("Failed to update site default theme reference")?; + .context("Failed to update site default theme reference") + .map(|_affected_rows| ())?; } tx.commit()?; Ok(()) @@ -266,12 +268,14 @@ pub fn delete_custom_theme(conn: &mut rusqlite::Connection, slug: &str) -> Resul tx.execute( "UPDATE boards SET default_theme = '' WHERE lower(default_theme) = lower(?1)", params![slug], - )?; + ) + .map(|_affected_rows| ())?; tx.execute( "UPDATE site_settings SET value = ?2 WHERE key = 'default_theme' AND lower(value) = lower(?1)", params![slug, crate::theme::HARD_DEFAULT_THEME], - )?; + ) + .map(|_affected_rows| ())?; tx.commit()?; Ok(()) } @@ -547,7 +551,8 @@ mod tests { let pool = crate::db::init_test_pool()?; let mut conn = pool.get()?; let board_short = "thmup"; - crate::db::boards::create_board(&conn, board_short, "Theme Update", "", false)?; + crate::db::boards::create_board(&conn, board_short, "Theme Update", "", false) + .map(|_created_id| ())?; super::create_custom_theme( &conn, "guided-forest", @@ -560,7 +565,8 @@ mod tests { conn.execute( "UPDATE boards SET default_theme = 'guided-forest' WHERE short_name = ?1", params![board_short], - )?; + ) + .map(|_affected_rows| ())?; crate::db::set_site_setting(&conn, "default_theme", "guided-forest")?; super::update_theme( @@ -600,7 +606,8 @@ mod tests { let pool = crate::db::init_test_pool()?; let mut conn = pool.get()?; let board_short = "thmdel"; - crate::db::boards::create_board(&conn, board_short, "Theme Delete", "", false)?; + crate::db::boards::create_board(&conn, board_short, "Theme Delete", "", false) + .map(|_created_id| ())?; super::create_custom_theme( &conn, "guided-forest", @@ -613,7 +620,8 @@ mod tests { conn.execute( "UPDATE boards SET default_theme = 'guided-forest' WHERE short_name = ?1", params![board_short], - )?; + ) + .map(|_affected_rows| ())?; crate::db::set_site_setting(&conn, "default_theme", "guided-forest")?; super::delete_custom_theme(&mut conn, "guided-forest")?; diff --git a/src/db/threads.rs b/src/db/threads.rs index 29915f21..5dd525e0 100644 --- a/src/db/threads.rs +++ b/src/db/threads.rs @@ -49,7 +49,7 @@ pub struct PollInsert<'a> { /// 1 `t.board_id` 5 t.locked 9 `op.file_path` 13 op.id (`op_id`) /// 2 t.subject 6 t.sticky 10 `op.thumb_path` 14 t.archived /// 3 `t.created_at` 7 `t.reply_count` 11 op.name 15 `image_count` -/// 16 `last_post_at` +/// 16 `last_post_at`. fn map_thread(row: &rusqlite::Row<'_>) -> rusqlite::Result { Ok(Thread { id: row.get(0)?, @@ -127,7 +127,7 @@ fn collect_thread_file_paths( /// /// A left-join aggregation computes image counts in one pass. Callers must /// group by thread to preserve one output row per thread. -const THREAD_SELECT: &str = " +pub(super) const THREAD_SELECT: &str = " SELECT t.id, t.board_id, t.subject, t.created_at, t.bumped_at, t.locked, t.sticky, t.reply_count, op.body, op.file_path, op.thumb_path, op.name, op.tripcode, op.id, @@ -149,16 +149,52 @@ pub fn get_threads_for_board( limit: i64, offset: i64, ) -> Result> { - let sql = format!( - "{THREAD_SELECT} - WHERE t.board_id = ?1 AND t.archived = 0 - GROUP BY t.id, op.id - ORDER BY t.sticky DESC, t.bumped_at DESC - LIMIT ?2 OFFSET ?3" - ); + load_thread_page(conn, board_id, limit, offset, false) +} + +/// Paginate threads before computing reply aggregates; exclude missing OPs first. +pub(super) fn thread_page_sql(archived: bool) -> String { + // Preserve the legacy aggregate's tie order for each listing, including + // deterministic boundaries when many replies bump in the same second. + let order = if archived { + "t.bumped_at DESC, t.id ASC" + } else { + "t.sticky DESC, t.bumped_at DESC, t.id DESC" + }; + format!( + "WITH page AS MATERIALIZED ( + SELECT t.* FROM threads t + WHERE t.board_id = ?1 AND t.archived = ?4 + AND EXISTS (SELECT 1 FROM posts op WHERE op.thread_id=t.id AND op.is_op=1) + ORDER BY {order} LIMIT ?2 OFFSET ?3 + ) + SELECT t.id, t.board_id, t.subject, t.created_at, t.bumped_at, + t.locked, t.sticky, t.reply_count, + op.body, op.file_path, op.thumb_path, op.name, op.tripcode, op.id, + t.archived, + (SELECT COUNT(*) FROM posts fp WHERE fp.thread_id=t.id AND fp.file_path IS NOT NULL), + COALESCE((SELECT MAX(fp.created_at) FROM posts fp WHERE fp.thread_id=t.id), t.created_at) + FROM page t JOIN posts op ON op.thread_id=t.id AND op.is_op=1 + ORDER BY {order}" + ) +} + +/// Execute an indexed page and its bounded per-thread aggregate probes. +fn load_thread_page( + conn: &rusqlite::Connection, + board_id: i64, + limit: i64, + offset: i64, + archived: bool, +) -> Result> { + let _timing = super::diagnostics::QueryTiming::start("thread_page"); + let sql = thread_page_sql(archived); let mut stmt = conn.prepare_cached(&sql)?; let threads = stmt - .query_map(params![board_id, limit, offset], map_thread)? + .query_map( + params![board_id, limit, offset, i64::from(archived)], + map_thread, + )? .collect::>>()?; Ok(threads) } @@ -218,13 +254,15 @@ pub(crate) enum PostCreationOutcome { #[derive(Debug, Clone, Copy)] /// Filesystem lifecycle context validated in a public post transaction. -pub(crate) struct PostFilesystemCommit<'a> { +pub(crate) struct PostFilesystemCommit<'a, F = fn(&rusqlite::Connection) -> Result<()>> { /// Durable publication operation for newly staged files. pending_fs_op: Option<&'a crate::pending_fs::PendingFsOpInsert>, /// Prior dedup cache hits that must still be valid under the write lock. deduplicated_paths: &'a [&'a str], /// Whether a new thread must atomically persist board-prune work. schedule_thread_prune: bool, + /// Posting policy checked after token replay and while holding the write lock. + validate: F, } impl<'a> PostFilesystemCommit<'a> { @@ -234,11 +272,29 @@ impl<'a> PostFilesystemCommit<'a> { pending_fs_op: Option<&'a crate::pending_fs::PendingFsOpInsert>, deduplicated_paths: &'a [&'a str], schedule_thread_prune: bool, + ) -> Self { + Self::new_with_validation( + pending_fs_op, + deduplicated_paths, + schedule_thread_prune, + |_| Ok(()), + ) + } +} + +impl<'a, F> PostFilesystemCommit<'a, F> { + /// Build creation context with request validation under the write lock. + pub(crate) const fn new_with_validation( + pending_fs_op: Option<&'a crate::pending_fs::PendingFsOpInsert>, + deduplicated_paths: &'a [&'a str], + schedule_thread_prune: bool, + validate: F, ) -> Self { Self { pending_fs_op, deduplicated_paths, schedule_thread_prune, + validate, } } } @@ -286,14 +342,13 @@ pub(crate) fn create_thread_submission( post: &super::NewPost, submission_token: &str, poll: Option<&PollInsert<'_>>, - filesystem: PostFilesystemCommit<'_>, + filesystem: PostFilesystemCommit<'_, impl FnOnce(&rusqlite::Connection) -> Result<()>>, ) -> Result)>> { // BEGIN IMMEDIATE acquires the write lock upfront to avoid SQLITE_BUSY // during the lock-upgrade step that DEFERRED transactions perform on first // write. With &Connection (not &mut Connection) we cannot use rusqlite's // typed Transaction::new(Immediate), so we issue the pragma directly. - conn.execute_batch("BEGIN IMMEDIATE") - .context("Failed to begin IMMEDIATE transaction for create_thread_with_optional_poll")?; + let _timing = super::diagnostics::WriteTiming::begin(conn, "create_thread")?; let result: Result)>> = (|| { if let Some(ip_hash) = post.ip_hash.as_deref() { @@ -304,6 +359,7 @@ pub(crate) fn create_thread_submission( } } + (filesystem.validate)(conn)?; validate_deduplicated_paths(conn, filesystem.deduplicated_paths)?; let thread_id: i64 = conn.query_row( @@ -346,7 +402,8 @@ pub(crate) fn create_thread_submission( } if filesystem.schedule_thread_prune { super::posts::persist_thread_prune_intent_in_tx(conn, board_id) - .context("Persist required board prune intent failed")?; + .context("Persist required board prune intent failed") + .map(|_operation_summary| ())?; } Ok(PostCreationOutcome::Created((thread_id, post_id, poll_id))) @@ -401,10 +458,10 @@ pub(crate) fn create_reply_submission( post: &super::NewPost, submission_token: &str, should_bump: bool, - filesystem: PostFilesystemCommit<'_>, + filesystem: PostFilesystemCommit<'_, impl FnOnce(&rusqlite::Connection) -> Result<()>>, ) -> Result> { - conn.execute_batch("BEGIN IMMEDIATE") - .context("Failed to begin create_reply_with_thread_update transaction")?; + // Serialize token replay, policy/thread validation, count and bump decisions. + let _timing = super::diagnostics::WriteTiming::begin(conn, "create_reply")?; let result: Result> = (|| { if let Some(ip_hash) = post.ip_hash.as_deref() { @@ -415,6 +472,7 @@ pub(crate) fn create_reply_submission( } } + (filesystem.validate)(conn)?; validate_deduplicated_paths(conn, filesystem.deduplicated_paths)?; let flags = conn @@ -423,7 +481,12 @@ pub(crate) fn create_reply_submission( FROM threads WHERE id = ?1 AND board_id = ?2", params![post.thread_id, post.board_id], - |row| Ok((row.get::<_, i32>(0)? != 0, row.get::<_, i32>(1)? != 0)), + |row| { + Ok(( + row.get::<_, i32>(0)? != 0_i32, + row.get::<_, i32>(1)? != 0_i32, + )) + }, ) .optional()?; let Some((locked, archived)) = flags else { @@ -443,7 +506,9 @@ pub(crate) fn create_reply_submission( let updated = if should_bump { conn.execute( "UPDATE threads - SET bumped_at = unixepoch(), + SET bumped_at = CASE + WHEN reply_count < (SELECT bump_limit FROM boards WHERE id = ?2) + THEN unixepoch() ELSE bumped_at END, reply_count = reply_count + 1 WHERE id = ?1 AND board_id = ?2 AND locked = 0 AND archived = 0", params![post.thread_id, post.board_id], @@ -697,7 +762,8 @@ pub fn archive_old_threads(conn: &rusqlite::Connection, board_id: i64, max: i64) let sql = format!("UPDATE threads SET archived = 1, locked = 1 WHERE id IN ({placeholders})"); conn.execute(&sql, rusqlite::params_from_iter(&ids)) - .context("Failed to bulk archive threads")?; + .context("Failed to bulk archive threads") + .map(|_affected_rows| ())?; Ok(count) })(); @@ -776,7 +842,8 @@ pub fn prune_old_threads( .join(", "); let sql = format!("DELETE FROM threads WHERE id IN ({placeholders})"); conn.execute(&sql, rusqlite::params_from_iter(&ids)) - .context("Failed to bulk delete pruned threads")?; + .context("Failed to bulk delete pruned threads") + .map(|_affected_rows| ())?; // Determine safe paths INSIDE the transaction so the check sees the // post-delete state before any concurrent writer can insert new references. @@ -853,7 +920,8 @@ pub fn prune_old_archived_threads( .join(", "); let sql = format!("DELETE FROM threads WHERE id IN ({placeholders})"); conn.execute(&sql, rusqlite::params_from_iter(&ids)) - .context("Failed to bulk delete archived threads")?; + .context("Failed to bulk delete archived threads") + .map(|_affected_rows| ())?; let safe = super::paths_safe_to_delete(conn, candidates)?; let pending_fs_op = super::build_delete_files_pending_op(&safe)?; @@ -892,18 +960,7 @@ pub fn get_archived_threads_for_board( limit: i64, offset: i64, ) -> Result> { - let sql = format!( - "{THREAD_SELECT} - WHERE t.board_id = ?1 AND t.archived = 1 - GROUP BY t.id, op.id - ORDER BY t.bumped_at DESC - LIMIT ?2 OFFSET ?3" - ); - let mut stmt = conn.prepare_cached(&sql)?; - let threads = stmt - .query_map(params![board_id, limit, offset], map_thread)? - .collect::>>()?; - Ok(threads) + load_thread_page(conn, board_id, limit, offset, true) } /// Count archived threads for a board (used for archive pagination). @@ -1053,11 +1110,13 @@ mod tests { conn.execute( "UPDATE threads SET created_at = 100, bumped_at = 100 WHERE id = ?1", params![thread_id], - )?; + ) + .map(|_affected_rows| ())?; conn.execute( "UPDATE posts SET created_at = 100 WHERE thread_id = ?1", params![thread_id], - )?; + ) + .map(|_affected_rows| ())?; let thread = super::get_thread(&conn, thread_id)?.context("opening thread should exist")?; anyhow::ensure!( thread.last_post_at == 100 && thread.image_count == 0, @@ -1077,16 +1136,17 @@ mod tests { false, None, )?; - for (post_id, timestamp) in [(image_id, 200), (audio_id, 300), (text_id, 400)] { + for (post_id, timestamp) in [(image_id, 200_i32), (audio_id, 300_i32), (text_id, 400_i32)] { conn.execute( "UPDATE posts SET created_at = ?1 WHERE id = ?2", params![timestamp, post_id], - )?; + ) + .map(|_affected_rows| ())?; } - let thread = + let replied_thread = super::get_thread(&conn, thread_id)?.context("thread should exist after replies")?; anyhow::ensure!( - thread.last_post_at == 400 && thread.bumped_at == 100 && thread.image_count == 1, + replied_thread.last_post_at == 400 && replied_thread.bumped_at == 100 && replied_thread.image_count == 1, "sage text replies update last-post time while bump time and image counts remain correct" ); let active = super::get_threads_for_board(&conn, board_id, 20, 0)?; @@ -1099,7 +1159,8 @@ mod tests { conn.execute( "UPDATE threads SET archived = 1 WHERE id = ?1", params![thread_id], - )?; + ) + .map(|_affected_rows| ())?; let archived = super::get_archived_threads_for_board(&conn, board_id, 20, 0)?; anyhow::ensure!( archived.len() == 1, @@ -1171,7 +1232,8 @@ mod tests { SET file_path = 'b/file.webp', media_processing_state = ?1 WHERE id = ?2", params![crate::db::MEDIA_ORIGINAL_PRUNE_PENDING, post_id], - )?; + ) + .map(|_affected_rows| ())?; assert!(validate_deduplicated_paths(&conn, &["b/file.webp"]).is_err()); assert!(crate::db::find_file_by_hash(&conn, "hash")?.is_none()); @@ -1186,8 +1248,8 @@ mod tests { fn prune_old_threads_commits_even_when_no_files_are_safe() -> Result<()> { let conn = test_conn()?; let board_id = create_board(&conn, "prune", "Prune", "", false)?; - create_plain_thread(&conn, board_id, "old thread")?; - create_plain_thread(&conn, board_id, "new thread")?; + create_plain_thread(&conn, board_id, "old thread").map(|_created_id| ())?; + create_plain_thread(&conn, board_id, "new thread").map(|_created_id| ())?; let board = get_board_by_short(&conn, "prune")?.context("prune board should exist")?; assert_eq!( count_threads_for_board(&conn, board.id)?, @@ -1225,7 +1287,8 @@ mod tests { conn.execute( "UPDATE threads SET archived = 1 WHERE id IN (?1, ?2)", params![first, second], - )?; + ) + .map(|_affected_rows| ())?; let deleted = prune_old_archived_threads(&conn, board_id, 1)?; assert!( @@ -1287,7 +1350,7 @@ mod tests { deletion_token: "token".to_owned(), is_op: false, }; - create_reply_with_thread_update(&conn, &reply, "", false, None)?; + create_reply_with_thread_update(&conn, &reply, "", false, None).map(|_created_id| ())?; let deleted = delete_thread(&conn, thread_id)?; assert!( @@ -1343,7 +1406,8 @@ mod tests { conn.execute( "UPDATE threads SET locked = 1 WHERE id = ?1", params![thread_id], - )?; + ) + .map(|_affected_rows| ())?; let reply = plain_reply(board_id, thread_id); let pending_op = pending_upload_op("locked-reply-upload"); @@ -1390,7 +1454,8 @@ mod tests { conn.execute( "UPDATE threads SET archived = 1 WHERE id = ?1", params![thread_id], - )?; + ) + .map(|_affected_rows| ())?; let reply = plain_reply(board_id, thread_id); let pending_op = pending_upload_op("archived-reply-upload"); @@ -1479,7 +1544,7 @@ mod tests { deletion_token: "token".to_owned(), is_op: false, }; - create_reply_with_thread_update(&conn, &reply, "", false, None)?; + create_reply_with_thread_update(&conn, &reply, "", false, None).map(|_created_id| ())?; let deleted = delete_thread(&conn, thread_id)?; assert!( @@ -1502,4 +1567,143 @@ mod tests { ); Ok(()) } + + #[test] + fn paged_thread_aggregates_match_the_unbounded_baseline() -> Result<()> { + let conn = test_conn()?; + let board = create_board(&conn, "page", "Page", "", false)?; + for number in 0_i32..35_i32 { + let thread = create_plain_thread(&conn, board, &format!("thread {number}"))?; + conn.execute( + "UPDATE threads SET bumped_at=?1, sticky=?2, archived=?3 WHERE id=?4", + params![ + 1_700_000_000_i32 + number / 7_i32, + i64::from(number % 11_i32 == 0_i32), + i64::from(number % 5_i32 == 0_i32), + thread + ], + ) + .map(|_affected_rows| ())?; + for reply in 0_i32..number % 7_i32 { + conn.execute("INSERT INTO posts(thread_id,board_id,body,body_html,deletion_token,file_path,created_at) + VALUES(?1,?2,'reply','reply','delete',?3,?4)", + params![thread,board,(reply%2_i32==0_i32).then(|| format!("page/file-{number}-{reply}")),1_700_000_100_i32+reply]).map(|_affected_rows| ())?; + } + } + // A missing OP must be excluded before pagination, matching the old JOIN. + conn.execute( + "INSERT INTO threads(board_id,bumped_at,sticky) VALUES(?1,1900000000,1)", + [board], + ) + .map(|_affected_rows| ())?; + for archived in [false, true] { + let order = if archived { + "t.bumped_at DESC" + } else { + "t.sticky DESC,t.bumped_at DESC" + }; + let old = format!( + "{} WHERE t.board_id=?1 AND t.archived=?4 GROUP BY t.id,op.id ORDER BY {order} LIMIT ?2 OFFSET ?3", + super::THREAD_SELECT + ); + for (limit, offset) in [ + (1, 0), + (10, 0), + (10, 10), + (100, 0), + (5, 99), + (0, 0), + (-1, 2), + ] { + let baseline = conn + .prepare(&old)? + .query_map( + params![board, limit, offset, i64::from(archived)], + super::map_thread, + )? + .collect::>>()?; + let paged = super::load_thread_page(&conn, board, limit, offset, archived)?; + let baseline_ids: Vec<_> = baseline.iter().map(|thread| thread.id).collect(); + let paged_ids: Vec<_> = paged.iter().map(|thread| thread.id).collect(); + anyhow::ensure!( + serde_json::to_value(baseline)? == serde_json::to_value(paged)?, + "page aggregates changed: archived={archived}, limit={limit}, offset={offset}, baseline={baseline_ids:?}, paged={paged_ids:?}" + ); + } + } + Ok(()) + } + + #[test] + fn concurrent_bump_requests_use_the_current_count_and_limit() -> Result<()> { + let pool = crate::db::init_test_pool()?; + let conn = pool.get()?; + let board = create_board(&conn, "bump", "Bump", "", false)?; + conn.execute("UPDATE boards SET bump_limit=1 WHERE id=?1", [board]) + .map(|_affected_rows| ())?; + let thread = create_plain_thread(&conn, board, "target")?; + let op = crate::db::get_posts_for_thread(&conn, thread)? + .into_iter() + .next() + .context("OP")?; + let mut post = NewPost { + thread_id: thread, + board_id: board, + name: "anon".into(), + tripcode: None, + subject: None, + body: "reply".into(), + body_html: "reply".into(), + ip_hash: None, + file_path: None, + file_name: None, + file_size: None, + thumb_path: None, + mime_type: None, + media_type: None, + audio_file_path: None, + audio_file_name: None, + audio_file_size: None, + audio_mime_type: None, + deletion_token: op.deletion_token, + is_op: false, + }; + post.ip_hash = Some("bump-actor".into()); + drop(conn); + let barrier = std::sync::Barrier::new(2); + std::thread::scope(|scope| -> Result<()> { + let handles:Vec<_>=(0_i32..2_i32).map(|number| { + let pool=&pool; let post=&post; let barrier=&barrier; + scope.spawn(move || -> Result<()> { + let _barrier_state = barrier.wait(); + let worker_conn=pool.get()?; + super::create_reply_submission(&worker_conn,post,&format!("bump-{number}"),true, + PostFilesystemCommit::new_with_validation(None,&[],false,|validation_conn:&Connection| { + // Give the second serialized reply a distinguishable previous + // timestamp; no sleep or wall-clock assumption is required. + validation_conn.execute("UPDATE threads SET bumped_at=42 WHERE id=?1 AND reply_count=1",[thread]).map(|_affected_rows| ())?; + Ok(()) + })).map(|_operation_summary| ())?; + Ok(()) + }) + }).collect(); + for handle in handles { + handle.join().map_err(|panic_payload| { + anyhow::anyhow!( + "reply worker panicked: {}", + crate::media::process::panic_message(panic_payload.as_ref()) + ) + })??; + } + Ok(()) + })?; + let recovered_conn = pool.get()?; + let state = crate::db::get_thread(&recovered_conn, thread)?.context("thread")?; + anyhow::ensure!( + state.reply_count == 2 && state.bumped_at == 42, + "concurrent replies exceeded the bump limit" + ); + crate::db::verify_database_schema(&recovered_conn)?; + Ok(()) + } } diff --git a/src/db/user_thread_prefs.rs b/src/db/user_thread_prefs.rs index c093002c..534eaedc 100644 --- a/src/db/user_thread_prefs.rs +++ b/src/db/user_thread_prefs.rs @@ -17,7 +17,8 @@ fn cleanup_if_default(conn: &rusqlite::Connection, user_hash: &str, thread_id: i "DELETE FROM user_thread_preferences WHERE user_hash = ?1 AND thread_id = ?2 AND pinned = 0 AND hidden = 0", params![user_hash, thread_id], - )?; + ) + .map(|_affected_rows| ())?; Ok(()) } @@ -38,7 +39,8 @@ pub fn set_thread_hidden( hidden = excluded.hidden, updated_at = unixepoch()", params![user_hash, thread_id, i32::from(hidden)], - )?; + ) + .map(|_affected_rows| ())?; cleanup_if_default(conn, user_hash, thread_id)?; Ok(()) } @@ -60,7 +62,8 @@ pub fn set_thread_pinned( pinned = excluded.pinned, updated_at = unixepoch()", params![user_hash, thread_id, i32::from(pinned)], - )?; + ) + .map(|_affected_rows| ())?; cleanup_if_default(conn, user_hash, thread_id)?; Ok(()) } @@ -81,8 +84,8 @@ pub fn get_thread_preference( params![user_hash, thread_id], |row| { Ok(UserThreadPreference { - pinned: row.get::<_, i32>(0)? != 0, - hidden: row.get::<_, i32>(1)? != 0, + pinned: row.get::<_, i32>(0)? != 0_i32, + hidden: row.get::<_, i32>(1)? != 0_i32, }) }, ) @@ -112,8 +115,8 @@ pub fn get_preferences_for_board( Ok(( row.get::<_, i64>(0)?, UserThreadPreference { - pinned: row.get::<_, i32>(1)? != 0, - hidden: row.get::<_, i32>(2)? != 0, + pinned: row.get::<_, i32>(1)? != 0_i32, + hidden: row.get::<_, i32>(2)? != 0_i32, }, )) })?; @@ -121,7 +124,7 @@ pub fn get_preferences_for_board( let mut prefs = HashMap::new(); for row in rows { let (thread_id, pref) = row?; - prefs.insert(thread_id, pref); + let _previous_value = prefs.insert(thread_id, pref); } Ok(prefs) } @@ -181,7 +184,7 @@ mod tests { let pinned_barrier = Arc::clone(&barrier); let pinned = scope.spawn(move || -> Result<()> { let conn = pinned_pool.get()?; - pinned_barrier.wait(); + let _barrier_state = pinned_barrier.wait(); set_thread_pinned(&conn, "viewer", thread_id, true) }); @@ -189,16 +192,22 @@ mod tests { let hidden_barrier = Arc::clone(&barrier); let hidden = scope.spawn(move || -> Result<()> { let conn = hidden_pool.get()?; - hidden_barrier.wait(); + let _barrier_state = hidden_barrier.wait(); set_thread_hidden(&conn, "viewer", thread_id, true) }); - pinned - .join() - .map_err(|_| anyhow::anyhow!("pinned preference thread panicked"))??; - hidden - .join() - .map_err(|_| anyhow::anyhow!("hidden preference thread panicked"))??; + pinned.join().map_err(|panic_payload| { + anyhow::anyhow!( + "pinned preference thread panicked: {}", + crate::media::process::panic_message(panic_payload.as_ref()) + ) + })??; + hidden.join().map_err(|panic_payload| { + anyhow::anyhow!( + "hidden preference thread panicked: {}", + crate::media::process::panic_message(panic_payload.as_ref()) + ) + })??; Result::<()>::Ok(()) })?; diff --git a/src/db/workload.rs b/src/db/workload.rs new file mode 100644 index 00000000..a6522c0f --- /dev/null +++ b/src/db/workload.rs @@ -0,0 +1,921 @@ +//! Offline, opt-in measurements against the bundled SQLite and real DB helpers. + +use anyhow::{ensure, Context as _, Result}; +use rusqlite::{params, Connection}; +use serde::Serialize; +use std::{path::Path, sync::Arc, time::Instant}; + +/// Number of concurrent callers used for each measured case. +const WORKERS: u32 = 8; +/// Number of measured operations per caller. +const OPERATIONS: u32 = 400; + +/// One operation's elapsed times in microseconds. +#[derive(Debug, Serialize)] +struct Sample { + /// Full checkout plus operation latency. + elapsed_us: u128, + /// Time waiting for a pooled connection. + checkout_us: u128, + /// Time acquiring an immediate SQLite write lock (zero for reads). + write_wait_us: u128, + /// Time holding a write transaction after lock acquisition. + transaction_us: u128, + /// Whether SQLite or the pool rejected the operation. + failed: bool, + /// Whether the rejection was SQLite busy/locked. + busy: bool, + /// Whether the pool checkout timed out. + pool_timeout: bool, +} + +/// Nearest-rank latency summary, with integer units avoiding precision loss. +#[derive(Debug, Serialize)] +struct Latencies { + /// Median microseconds. + #[serde(rename = "p50_us")] + p50: u128, + /// 95th percentile microseconds. + #[serde(rename = "p95_us")] + p95: u128, + /// 99th percentile microseconds. + #[serde(rename = "p99_us")] + p99: u128, + /// Maximum microseconds. + #[serde(rename = "max_us")] + max: u128, +} + +/// Result for one pool/durability/workload combination. +#[derive(Debug, Serialize)] +struct Measurement { + /// Configured maximum connections. + pool_size: u32, + /// Explicit SQLite synchronous mode. + synchronous: String, + /// Workload traffic mix. + workload: String, + /// Number of attempted operations. + operations: u32, + /// Successful operations per wall-clock second. + operations_per_second: f64, + /// Total request latency. + latency: Latencies, + /// Read-only operation latency including checkout. + read_latency: Latencies, + /// Write operation latency including checkout and lock acquisition. + write_latency: Latencies, + /// Pool checkout latency. + checkout: Latencies, + /// Immediate lock acquisition latency for writes only. + write_wait: Latencies, + /// Write lock duration after acquisition, including commit. + transaction: Latencies, + /// Requests whose checkout exceeded 100 microseconds. + checkout_over_100us: usize, + /// Failed operations of all kinds. + failures: u32, + /// SQLite busy or locked errors. + busy_errors: usize, + /// Pool timeouts. + pool_timeouts: usize, + /// Database bytes, including pages resident in WAL. + database_bytes: i64, + /// WAL file bytes before passive checkpoint. + wal_bytes: u64, + /// WAL pages and progress reported by a passive checkpoint. + checkpoint: (i64, i64, i64), + /// Passive checkpoint elapsed microseconds after workers release snapshots. + checkpoint_us: u128, + /// Resident memory in KiB, if the host provides ps. + rss_kib: Option, + /// Process CPU seconds during this case, if the host provides ps. + cpu_seconds: Option, +} + +/// Explain output for a named production query shape. +#[derive(Debug, Serialize)] +struct Plan { + /// Stable description of the query. + name: String, + /// Planner details from the populated database. + details: Vec, +} + +/// Generate deterministic representative content, including FTS and media rows. +fn seed(conn: &Connection) -> Result<()> { + super::schema::install_or_migrate_schema(conn)?; + conn.execute_batch( + "BEGIN IMMEDIATE; + WITH RECURSIVE n(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM n WHERE x<8) + INSERT INTO boards(id,short_name,name) SELECT x,'b'||x,'Board '||x FROM n; + WITH RECURSIVE n(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM n WHERE x<800) + INSERT INTO threads(id,board_id,subject,archived,reply_count,bumped_at) + SELECT x,1+(x-1)%8,'Thread '||x,x%5=0,49,1700000000+x FROM n; + WITH RECURSIVE n(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM n WHERE x<40000) + INSERT INTO posts(id,thread_id,board_id,body,body_html,ip_hash,deletion_token, + is_op,created_at,file_path,thumb_path,mime_type,media_type,file_size) + SELECT x,1+(x-1)/50,1+((x-1)/50)%8, + CASE WHEN x%101=0 THEN 'rare benchmark needle' ELSE 'representative post text' END, + 'representative post text','actor-'||(x%1000),'fixture-token',(x-1)%50=0, + 1700000000+x,CASE WHEN x%4=0 THEN 'b'||(1+((x-1)/50)%8)||'/src/'||x||'.png' END, + CASE WHEN x%4=0 THEN 'b'||(1+((x-1)/50)%8)||'/thumb/'||x||'.png' END, + CASE WHEN x%4=0 THEN 'image/png' END,CASE WHEN x%4=0 THEN 'image' END, + CASE WHEN x%4=0 THEN 1024 END FROM n; + INSERT INTO file_hashes(sha256,file_path,thumb_path,mime_type) + SELECT printf('%064x',id),file_path,thumb_path,mime_type FROM posts WHERE file_path IS NOT NULL; + INSERT INTO bans(ip_hash,reason) SELECT DISTINCT ip_hash,'fixture' FROM posts WHERE id%101=0; + INSERT INTO admin_users(id,username,password_hash) VALUES(1,'fixture-admin','fixture-hash'); + INSERT INTO reports(post_id,thread_id,board_id,reason,reporter_hash) + SELECT id,thread_id,board_id,'fixture','reporter-'||id FROM posts WHERE id%83=0; + INSERT INTO mod_log(admin_id,admin_name,action,target_type,target_id) + SELECT 1,'fixture-admin','fixture','post',id FROM posts WHERE id%37=0; + INSERT INTO polls(thread_id,question,expires_at) + SELECT id,'fixture poll',unixepoch()+86400 FROM threads WHERE id%7=0; + INSERT INTO poll_options(poll_id,text,position) SELECT id,'yes',0 FROM polls; + INSERT INTO poll_options(poll_id,text,position) SELECT id,'no',1 FROM polls; + INSERT INTO poll_votes(poll_id,option_id,ip_hash) + SELECT poll_id,id,'fixture-voter' FROM poll_options WHERE position=0; + INSERT INTO background_jobs(job_type,payload,status) + SELECT 'spam_check','{}',CASE WHEN id%2=0 THEN 'done' ELSE 'pending' END FROM posts WHERE id%31=0; + COMMIT;", + )?; + Ok(()) +} + +/// Create warmed file-backed production-configured connections for a case. +fn pool(path: &Path, size: u32, synchronous: &'static str) -> Result { + let manager = r2d2_sqlite::SqliteConnectionManager::file(path).with_init(move |conn| { + conn.execute_batch(super::pool::CONNECTION_PRAGMAS)?; + conn.pragma_update(None, "synchronous", synchronous) + }); + Ok(r2d2::Pool::builder() + .max_size(size) + .min_idle(Some(size)) + .connection_timeout(std::time::Duration::from_secs(1)) + .build(manager)?) +} + +/// Run production read helpers, covering board/catalog/thread/polling/FTS/admin. +fn read(conn: &Connection, operation: u32) -> Result<()> { + let thread = i64::from( + (operation % 800) + .checked_add(1) + .context("workload thread index overflow")?, + ); + let board = thread + .checked_sub(1) + .context("invalid workload thread index")? + .rem_euclid(8) + .checked_add(1) + .context("workload board index overflow")?; + match operation % 7 { + 0 => drop(super::get_threads_for_board(conn, board, 10, 0)?), + 1 => drop(super::get_threads_for_board(conn, board, 100, 0)?), + 2 => { + drop(super::get_thread(conn, thread)?); + drop(super::get_posts_for_thread(conn, thread)?); + } + 3 => drop(super::get_new_posts_since( + conn, + board, + thread, + thread + .checked_mul(50) + .and_then(|value| value.checked_sub(5)) + .context("workload post ID overflow")?, + 100, + )?), + 4 => drop(super::search_posts(conn, board, "needle", 20, 0)?), + 5 => drop(super::get_open_reports(conn)?), + _ => { + drop(super::get_poll_for_thread(conn, thread, "fixture-voter")?); + drop(super::find_file_by_hash( + conn, + &format!( + "{:064x}", + operation + .checked_mul(4) + .and_then(|value| value.checked_add(4)) + .context("workload hash index overflow")? + ), + )?); + } + } + Ok(()) +} + +/// Run a short post transaction with FTS, counters and submission-token work. +fn write(conn: &Connection, worker: u32, operation: u32) -> Result<()> { + let index = operation + .checked_mul(WORKERS) + .and_then(|value| value.checked_add(worker)) + .context("workload operation index overflow")?; + let thread = i64::from( + (index % 799) + .checked_add(1) + .context("workload thread index overflow")?, + ); + let board = thread + .checked_sub(1) + .context("invalid workload thread index")? + .rem_euclid(8) + .checked_add(1) + .context("workload board index overflow")?; + conn.execute( + "INSERT INTO posts(thread_id,board_id,body,body_html,ip_hash,deletion_token) + VALUES(?1,?2,'workload reply','workload reply',?3,'workload-delete')", + params![thread, board, format!("workload-{worker}")], + ) + .map(|_affected_rows| ())?; + let post_id = conn.last_insert_rowid(); + conn.execute( + "UPDATE threads SET reply_count=reply_count+1,bumped_at=unixepoch() WHERE id=?1", + [thread], + ) + .map(|_affected_rows| ())?; + super::record_post_submission( + conn, + &format!("workload-{worker}-{operation}"), + &format!("workload-{worker}"), + board, + thread, + post_id, + false, + )?; + if operation.is_multiple_of(17) { + super::enqueue_job(conn, "spam_check", "{}").map(|_created_id| ())?; + if let Some((job, _)) = super::claim_next_job(conn)? { + super::complete_job(conn, job)?; + } + } + if operation.is_multiple_of(23) { + conn.execute( + "INSERT OR IGNORE INTO poll_votes(poll_id,option_id,ip_hash) + SELECT poll_id,id,?1 FROM poll_options WHERE position=0 LIMIT 1", + [format!("workload-{worker}-{operation}")], + ) + .map(|_affected_rows| ())?; + } + if operation.is_multiple_of(41) { + conn.execute( + "DELETE FROM background_jobs WHERE id IN + (SELECT id FROM background_jobs WHERE status='done' LIMIT 4)", + [], + ) + .map(|_affected_rows| ())?; + } + Ok(()) +} + +/// Record a single operation without hiding failures behind retries. +fn sample(pool: &super::DbPool, worker: u32, operation: u32, write_percent: u32) -> Sample { + let started = Instant::now(); + let acquired = pool.get(); + let checkout_us = started.elapsed().as_micros(); + let pool_timeout = acquired.is_err(); + let mut write_wait_us = 0; + let mut transaction_us = 0; + let result = acquired.map_err(anyhow::Error::from).and_then(|conn| { + let mix_index = operation + .checked_mul(37) + .and_then(|value| { + worker + .checked_mul(13) + .and_then(|worker_offset| value.checked_add(worker_offset)) + }) + .context("workload mix index overflow")?; + if mix_index % 100 >= write_percent { + return read(&conn, operation); + } + let wait = Instant::now(); + let begin = conn.execute_batch("BEGIN IMMEDIATE"); + write_wait_us = wait.elapsed().as_micros().max(1); + begin?; + let locked = Instant::now(); + let result = write(&conn, worker, operation) + .and_then(|()| super::commit_transaction(&conn, "commit workload reply")); + if result.is_err() { + drop(conn.execute_batch("ROLLBACK")); + } + transaction_us = locked.elapsed().as_micros().max(1); + result + }); + let busy = result.as_ref().err().is_some_and(|error| { + error.downcast_ref::().is_some_and(|error| { + matches!(error, rusqlite::Error::SqliteFailure(code, _) + if matches!(code.code, rusqlite::ErrorCode::DatabaseBusy | rusqlite::ErrorCode::DatabaseLocked)) + }) + }); + Sample { + elapsed_us: started.elapsed().as_micros(), + checkout_us, + write_wait_us, + transaction_us, + failed: result.is_err(), + busy, + pool_timeout, + } +} + +/// Compute nearest-rank percentiles over one elapsed-time series. +fn latencies(values: impl Iterator) -> Latencies { + let mut values: Vec<_> = values.collect(); + values.sort_unstable(); + let percentile = |percent: usize| { + u128::try_from(values.len()) + .ok() + .and_then(|length| { + u128::try_from(percent) + .ok() + .and_then(|percent| length.checked_mul(percent)) + }) + .and_then(|rank| usize::try_from(rank.div_ceil(100).saturating_sub(1)).ok()) + .and_then(|index| values.get(index)) + .copied() + .unwrap_or(0) + }; + Latencies { + p50: percentile(50), + p95: percentile(95), + p99: percentile(99), + max: values.last().copied().unwrap_or(0), + } +} + +/// Read this process's resident memory using an optional host utility. +fn rss_kib() -> Option { + let output = std::process::Command::new("ps") + .args(["-o", "rss=", "-p", &std::process::id().to_string()]) + .output() + .ok()?; + String::from_utf8(output.stdout).ok()?.trim().parse().ok() +} + +/// Read accumulated process CPU time without adding a platform dependency. +fn cpu_seconds() -> Option { + let output = std::process::Command::new("ps") + .args(["-o", "time=", "-p", &std::process::id().to_string()]) + .output() + .ok()?; + let time = String::from_utf8(output.stdout).ok()?; + time.trim().split(':').try_fold(0.0_f64, |total, part| { + part.parse::() + .ok() + .map(|part| total.mul_add(60.0, part)) + }) +} + +/// Verify persisted relationships, FTS and counters after concurrent operations. +fn verify_stress_state(conn: &Connection) -> Result<()> { + let integrity: String = conn.query_row("PRAGMA integrity_check", [], |r| r.get(0))?; + ensure!(integrity == "ok", "stress integrity check: {integrity}"); + ensure!( + !conn.prepare("PRAGMA foreign_key_check")?.exists([])?, + "stress foreign-key violation" + ); + conn.execute( + "INSERT INTO posts_fts(posts_fts,rank) VALUES('integrity-check',1)", + [], + ) + .map(|_affected_rows| ())?; + super::verify_database_schema(conn)?; + let incorrect_counts: bool = conn.query_row( + "SELECT EXISTS(SELECT 1 FROM threads t WHERE t.reply_count != + (SELECT COUNT(*) FROM posts p WHERE p.thread_id=t.id AND p.is_op=0))", + [], + |row| row.get(0), + )?; + ensure!( + !incorrect_counts, + "stress transaction left an incorrect reply count" + ); + Ok(()) +} + +/// Stress one mix; verify database and FTS before returning the measurement. +fn measure(path: &Path, size: u32, sync: &'static str, mix: (&str, u32)) -> Result { + let pool = pool(path, size, sync)?; + let barrier = Arc::new(std::sync::Barrier::new(usize::try_from(WORKERS)?)); + let cpu_started = cpu_seconds(); + let started = Instant::now(); + let samples = std::thread::scope(|scope| -> Result> { + let handles: Vec<_> = (0..WORKERS) + .map(|worker| { + let pool = &pool; + let barrier = Arc::clone(&barrier); + scope.spawn(move || { + let _barrier_state = barrier.wait(); + (0..OPERATIONS) + .map(|operation| sample(pool, worker, operation, mix.1)) + .collect::>() + }) + }) + .collect(); + let mut samples = Vec::new(); + for handle in handles { + samples.extend(handle.join().map_err(|panic_payload| { + anyhow::anyhow!( + "workload worker panicked: {}", + crate::media::process::panic_message(panic_payload.as_ref()) + ) + })?); + } + Ok(samples) + })?; + let elapsed = started.elapsed(); + let cpu_elapsed = cpu_started + .zip(cpu_seconds()) + .map(|(before, after)| (after - before).max(0.0)); + let conn = pool.get()?; + let wal_bytes = std::fs::metadata(path.with_extension("sqlite3-wal")).map_or(0, |m| m.len()); + let checkpoint_started = Instant::now(); + let checkpoint = conn.query_row("PRAGMA wal_checkpoint(PASSIVE)", [], |r| { + Ok((r.get(0)?, r.get(1)?, r.get(2)?)) + })?; + let checkpoint_us = checkpoint_started.elapsed().as_micros(); + verify_stress_state(&conn)?; + let failures = u32::try_from(samples.iter().filter(|s| s.failed).count())?; + Ok(Measurement { + pool_size: size, + synchronous: sync.to_owned(), + workload: mix.0.to_owned(), + operations: WORKERS * OPERATIONS, + operations_per_second: f64::from( + WORKERS + .checked_mul(OPERATIONS) + .and_then(|count| count.checked_sub(failures)) + .context("workload successful operation count overflow")?, + ) / elapsed.as_secs_f64(), + latency: latencies(samples.iter().map(|s| s.elapsed_us)), + read_latency: latencies( + samples + .iter() + .filter(|s| !s.pool_timeout && s.write_wait_us == 0) + .map(|s| s.elapsed_us), + ), + write_latency: latencies( + samples + .iter() + .filter(|s| s.write_wait_us > 0) + .map(|s| s.elapsed_us), + ), + checkout: latencies(samples.iter().map(|s| s.checkout_us)), + write_wait: latencies( + samples + .iter() + .filter(|s| s.write_wait_us > 0) + .map(|s| s.write_wait_us), + ), + transaction: latencies( + samples + .iter() + .filter(|s| s.transaction_us > 0) + .map(|s| s.transaction_us), + ), + checkout_over_100us: samples.iter().filter(|s| s.checkout_us > 100).count(), + failures, + busy_errors: samples.iter().filter(|s| s.busy).count(), + pool_timeouts: samples.iter().filter(|s| s.pool_timeout).count(), + database_bytes: super::get_db_size_bytes(&conn)?, + wal_bytes, + checkpoint, + checkpoint_us, + rss_kib: rss_kib(), + cpu_seconds: cpu_elapsed, + }) +} + +/// Profile production query shapes on the exact generated schema and data. +fn plans(conn: &Connection) -> Result> { + let board = super::threads::thread_page_sql(false) + .replace("?1", "1") + .replace("?2", "10") + .replace("?3", "0") + .replace("?4", "0"); + let catalog = super::threads::thread_page_sql(false) + .replace("?1", "1") + .replace("?2", "100") + .replace("?3", "0") + .replace("?4", "0"); + let queries = [ + ("board_index", board.as_str()), + ("catalog", catalog.as_str()), + ("thread_loading", "SELECT * FROM posts WHERE thread_id=1 ORDER BY created_at ASC,id ASC"), + ("latest_post", "SELECT created_at,id FROM posts WHERE board_id=1 ORDER BY created_at DESC,id DESC LIMIT 1"), + ("thread_bump", "UPDATE threads SET reply_count=reply_count+1,bumped_at=CASE WHEN reply_count<(SELECT bump_limit FROM boards WHERE id=1) THEN unixepoch() ELSE bumped_at END WHERE id=1 AND board_id=1 AND locked=0 AND archived=0"), + ("admin_overview", "SELECT (SELECT COUNT(*) FROM posts),(SELECT COUNT(*) FROM threads),(SELECT COUNT(*) FROM reports WHERE status='open')"), + ("report_lookup", "SELECT id FROM reports WHERE post_id=83 AND reporter_hash='reporter-83' AND status='open'"), + ("backup_board_inventory", "SELECT id,file_path,thumb_path,audio_file_path FROM posts WHERE board_id=1 ORDER BY id"), + ( + "live_poll", + "SELECT * FROM posts WHERE board_id=1 AND thread_id=1 AND id>45 ORDER BY id LIMIT 100", + ), + ( + "cooldown", + "SELECT MAX(created_at) FROM posts WHERE board_id=1 AND ip_hash='actor-1'", + ), + ( + "latest_thread", + "SELECT created_at,id FROM threads WHERE board_id=1 AND archived=0 ORDER BY created_at DESC,id DESC LIMIT 1", + ), + ( + "prune", + "SELECT id FROM threads WHERE board_id=1 AND archived=0 AND sticky=0 ORDER BY bumped_at DESC LIMIT -1 OFFSET 100", + ), + ( + "search_selective", + "SELECT p.id FROM posts_fts CROSS JOIN posts p ON p.id=posts_fts.rowid WHERE p.board_id=1 AND posts_fts MATCH '\"needle\"*' ORDER BY p.created_at DESC,p.id DESC LIMIT 20", + ), + ( + "reports", + "SELECT * FROM reports WHERE status='open' ORDER BY created_at DESC LIMIT 20", + ), + ( + "bans", + "SELECT reason FROM bans WHERE ip_hash='actor-1' AND (expires_at IS NULL OR expires_at>unixepoch())", + ), + ( + "job_claim", + "SELECT id FROM background_jobs WHERE status='pending' AND attempts<3 ORDER BY priority DESC,created_at LIMIT 1", + ), + ( + "file_refs", + "SELECT 1 FROM posts WHERE file_path='b1/src/4.png' OR thumb_path='b1/src/4.png' OR audio_file_path='b1/src/4.png' LIMIT 1", + ), + ( + "dedup", + "SELECT * FROM file_hashes WHERE sha256=printf('%064x',4)", + ), + ( + "mod_log", + "SELECT * FROM mod_log ORDER BY created_at DESC LIMIT 50", + ), + ]; + queries + .into_iter() + .map(|(name, sql)| { + let details = conn + .prepare(&format!("EXPLAIN QUERY PLAN {sql}"))? + .query_map([], |r| r.get::<_, String>(3))? + .collect::>()?; + Ok(Plan { + name: name.to_owned(), + details, + }) + }) + .collect() +} + +/// Explicitly invoked benchmark; temporary databases never enter the repository. +#[test] +#[ignore = "populated offline performance matrix; run explicitly with RUSTCHAN_DB_EVIDENCE set"] +fn benchmark() -> Result<()> { + let output = std::env::var("RUSTCHAN_DB_EVIDENCE") + .context("set RUSTCHAN_DB_EVIDENCE to a JSON result path")?; + let directory = tempfile::tempdir()?; + let seed_path = directory.path().join("seed.sqlite3"); + let seed_conn = Connection::open(&seed_path)?; + seed_conn.execute_batch(super::pool::CONNECTION_PRAGMAS)?; + seed(&seed_conn)?; + let query_plans = plans(&seed_conn)?; + seed_conn.execute_batch("PRAGMA wal_checkpoint(TRUNCATE)")?; + drop(seed_conn); + let mut measurements = Vec::new(); + for sync in ["NORMAL", "FULL"] { + for size in [2, 4, 8, 12, 16] { + for mix in [("read_heavy", 5), ("write_burst", 70), ("mixed", 25)] { + let path = directory + .path() + .join(format!("{sync}-{size}-{}.sqlite3", mix.0)); + std::fs::copy(&seed_path, &path).map(|_bytes_copied| ())?; + measurements.push(measure(&path, size, sync, mix)?); + std::fs::write( + &output, + serde_json::to_vec_pretty(&serde_json::json!({ + "sqlite_version":rusqlite::version(),"workers":WORKERS, + "fixture":{"boards":8_i32,"threads":800_i32,"posts":40_000_i32}, + "query_plans":query_plans,"measurements":measurements, + "limitations":"SQL/database workload; media CPU and HTTP excluded; RSS sampled at case end; ps optional; process-kill is not power loss" + }))?, + )?; + } + } + } + std::fs::write( + output, + serde_json::to_vec_pretty(&serde_json::json!({ + "sqlite_version": rusqlite::version(), "workers":WORKERS, + "fixture":{"boards":8_i32,"threads":800_i32,"posts":40_000_i32}, + "query_plans":query_plans, "measurements":measurements, + "limitations":"SQL/database workload; media CPU and HTTP excluded; RSS sampled at case end; ps optional; process-kill is not power loss" + }))?, + )?; + Ok(()) +} + +/// Child process guard prevents a failing parent test from leaving a writer alive. +#[derive(Debug)] +struct CrashChild(std::process::Child); + +impl Drop for CrashChild { + fn drop(&mut self) { + drop(self.0.kill()); + drop(self.0.wait()); + } +} + +/// Child-only workload; parent kills it with an open WAL write transaction. +#[test] +#[ignore = "invoked only by the crash/restart parent regression"] +fn crash_writer_child() -> Result<()> { + let path = std::path::PathBuf::from(std::env::var("RUSTCHAN_DB_CRASH_PATH")?); + ensure!(!path.exists(), "crash fixture must be a fresh database"); + let conn = Connection::open(&path)?; + conn.execute_batch(super::pool::CONNECTION_PRAGMAS)?; + conn.pragma_update( + None, + "synchronous", + std::env::var("RUSTCHAN_DB_CRASH_SYNC")?, + )?; + super::schema::install_or_migrate_schema(&conn)?; + conn.execute_batch( + r#"BEGIN IMMEDIATE; + INSERT INTO boards(id,short_name,name) VALUES(1,'crash','Crash'); + INSERT INTO threads(id,board_id) VALUES(1,1); + INSERT INTO posts(thread_id,board_id,body,body_html,deletion_token,is_op) + VALUES(1,1,'op','op','delete',1); + INSERT INTO background_jobs(job_type,payload,status,attempts) + VALUES('spam_check','{}','running',1); + INSERT INTO pending_fs_ops(id,kind,payload_json) + VALUES('crash-delete','delete_files','{"paths":["crash/src/remove.txt"],"dirs":[]}'); + COMMIT;"#, + )?; + let root = path + .parent() + .context("crash fixture parent")? + .join("uploads"); + std::fs::create_dir_all(root.join("crash/src"))?; + std::fs::write(root.join("crash/src/remove.txt"), b"recoverable cleanup")?; + // The readiness marker is created only inside a later write transaction, + // after committed data, FTS content, a job and a filesystem intent exist. + for number in 0_u64.. { + conn.execute_batch("BEGIN IMMEDIATE")?; + conn.execute( + "INSERT INTO posts(thread_id,board_id,body,body_html,deletion_token) + VALUES(1,1,'crash reply','crash reply','delete')", + [], + ) + .map(|_affected_rows| ())?; + conn.execute( + "UPDATE threads SET reply_count=reply_count+1 WHERE id=1", + [], + ) + .map(|_affected_rows| ())?; + if number == 100 { + std::fs::write(path.with_extension("ready"), b"open transaction")?; + // Give the parent a deterministic open transaction to interrupt. + std::thread::sleep(std::time::Duration::from_secs(30)); + anyhow::bail!("crash parent failed to terminate the child within its deadline"); + } + super::commit_transaction(&conn, "commit crash workload")?; + } + Ok(()) +} + +/// Verify process-crash recovery for NORMAL and FULL without simulating power loss. +#[test] +fn abrupt_writer_termination_recovers_wal_fts_jobs_and_fs_intents() -> Result<()> { + for synchronous in ["NORMAL", "FULL"] { + let directory = tempfile::tempdir()?; + let path = directory.path().join("crash.sqlite3"); + let mut child = CrashChild( + std::process::Command::new(std::env::current_exe()?) + .args(["--ignored", "--exact", "db::workload::crash_writer_child"]) + .env("RUSTCHAN_DB_CRASH_PATH", &path) + .env("RUSTCHAN_DB_CRASH_SYNC", synchronous) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .spawn()?, + ); + let deadline = Instant::now() + std::time::Duration::from_secs(20); + while !path.with_extension("ready").exists() { + ensure!( + child.0.try_wait()?.is_none(), + "crash writer exited before readiness" + ); + ensure!( + Instant::now() < deadline, + "crash writer readiness timed out" + ); + std::thread::sleep(std::time::Duration::from_millis(10)); + } + child.0.kill()?; + child.0.wait().map(|_exit_status| ())?; + let pool = pool(&path, 2, synchronous)?; + let conn = pool.get()?; + super::schema::install_or_migrate_schema(&conn)?; + super::verify_database_schema(&conn)?; + conn.execute( + "INSERT INTO posts_fts(posts_fts,rank) VALUES('integrity-check',1)", + [], + ) + .map(|_affected_rows| ())?; + let (replies, count): (i64, i64) = conn.query_row( + "SELECT (SELECT COUNT(*) FROM posts WHERE is_op=0),reply_count FROM threads WHERE id=1", + [], + |r| Ok((r.get(0)?, r.get(1)?)), + )?; + ensure!( + replies == 100 && count == replies, + "crash transaction was partially committed" + ); + ensure!( + conn.is_autocommit(), + "restart inherited an active transaction" + ); + super::recover_interrupted_background_jobs(&conn).map(|_operation_summary| ())?; + let status: String = + conn.query_row("SELECT status FROM background_jobs WHERE id=1", [], |r| { + r.get(0) + })?; + ensure!(status == "pending", "interrupted job was not recovered"); + drop(conn); + let uploads = directory.path().join("uploads"); + crate::pending_fs::reconcile_pending_fs_ops( + &pool, + uploads.to_str().context("uploads path")?, + )?; + ensure!( + !uploads.join("crash/src/remove.txt").exists(), + "durable filesystem intent did not replay" + ); + let recovered_conn = pool.get()?; + ensure!( + super::list_pending_fs_ops(&recovered_conn)?.is_empty(), + "filesystem intent was not completed" + ); + recovered_conn.execute_batch("BEGIN IMMEDIATE; ROLLBACK")?; + super::verify_database_schema(&recovered_conn)?; + } + Ok(()) +} + +/// Time one cached query and retain IDs to prove result equivalence. +fn timed_query(conn: &Connection, sql: &str) -> Result<(Latencies, Vec)> { + let mut timings = Vec::new(); + let mut ids = Vec::new(); + let mut statement = conn.prepare_cached(sql)?; + for _ in 0_i32..25_i32 { + let started = Instant::now(); + ids = statement + .query_map([], |r| r.get::<_, i64>(0))? + .collect::>()?; + timings.push(started.elapsed().as_micros()); + } + Ok((latencies(timings.into_iter()), ids)) +} + +/// Collect the actual access plan for a parameter-free profiling query. +fn explain(conn: &Connection, sql: &str) -> Result> { + let mut statement = conn.prepare(&format!("EXPLAIN QUERY PLAN {sql}"))?; + let details = statement + .query_map([], |r| r.get(3))? + .collect::>()?; + Ok(details) +} + +/// Compare measured candidate queries/indexes against their baseline counterparts. +#[test] +#[ignore = "explicit populated query and index evidence; set RUSTCHAN_DB_QUERY_EVIDENCE"] +fn query_profile() -> Result<()> { + let output = std::env::var("RUSTCHAN_DB_QUERY_EVIDENCE")?; + let directory = tempfile::tempdir()?; + let conn = Connection::open(directory.path().join("profile.sqlite3"))?; + conn.execute_batch(super::pool::CONNECTION_PRAGMAS)?; + seed(&conn)?; + let mut evidence = Vec::new(); + for term in ["needle", "representative", "missingword"] { + for selection in ["p.id", "COUNT(*)"] { + let tail = if selection == "p.id" { + "ORDER BY p.created_at DESC,p.id DESC LIMIT 20" + } else { + "" + }; + let before = format!( + "SELECT {selection} FROM posts p JOIN posts_fts f ON f.rowid=p.id + WHERE p.board_id=1 AND posts_fts MATCH '\"{term}\"*' {tail}" + ); + let after = format!( + "SELECT {selection} FROM posts_fts CROSS JOIN posts p ON p.id=posts_fts.rowid + WHERE p.board_id=1 AND posts_fts MATCH '\"{term}\"*' {tail}" + ); + let (before_time, before_ids) = timed_query(&conn, &before)?; + let (after_time, after_ids) = timed_query(&conn, &after)?; + ensure!(before_ids == after_ids, "FTS query changed results"); + evidence.push(serde_json::json!({"query":format!("fts_{selection}_{term}"), + "before":before_time,"after":after_time,"before_plan":explain(&conn,&before)?,"after_plan":explain(&conn,&after)?})); + } + } + evidence.extend(profile_search(&conn)?); + let before_board = format!( + "{} WHERE t.board_id=1 AND t.archived=0 GROUP BY t.id,op.id + ORDER BY t.sticky DESC,t.bumped_at DESC LIMIT 10", + super::threads::THREAD_SELECT + ); + let after_board = super::threads::thread_page_sql(false) + .replace("?1", "1") + .replace("?2", "10") + .replace("?3", "0") + .replace("?4", "0"); + let (before_time, before_ids) = timed_query(&conn, &before_board)?; + let (after_time, after_ids) = timed_query(&conn, &after_board)?; + ensure!(before_ids == after_ids, "board page changed results"); + evidence.push( + serde_json::json!({"query":"board_page","before":before_time,"after":after_time, + "before_plan":explain(&conn,&before_board)?,"after_plan":explain(&conn,&after_board)?}), + ); + // Reconstruct the pre-index fixture, including reclaiming dropped pages, + // before measuring index storage and write cost. + conn.execute_batch( + "DROP INDEX IF EXISTS idx_posts_thread_live; + DROP INDEX IF EXISTS idx_posts_board_ip_created; + VACUUM;", + )?; + let queries = [ + ( + "live_poll", + "SELECT id FROM posts WHERE board_id=1 AND thread_id=1 AND id>45 ORDER BY id LIMIT 100", + ), + ( + "cooldown", + "SELECT MAX(created_at) FROM posts WHERE board_id=1 AND ip_hash='actor-1'", + ), + ]; + let before = queries + .iter() + .map(|(_, sql)| Ok((timed_query(&conn, sql)?, explain(&conn, sql)?))) + .collect::>>()?; + let write_before = profile_writes(&conn, 0)?; + let size_before = super::get_db_size_bytes(&conn)?; + conn.execute_batch("CREATE INDEX idx_posts_thread_live ON posts(thread_id,id); + CREATE INDEX idx_posts_board_ip_created ON posts(board_id,ip_hash,created_at DESC);")?; + let size_after = super::get_db_size_bytes(&conn)?; + for ((name, sql), ((baseline_time, _), before_plan)) in queries.into_iter().zip(before) { + let (indexed_time, _) = timed_query(&conn, sql)?; + evidence.push( + serde_json::json!({"query":name,"before":baseline_time,"after":indexed_time, + "before_plan":before_plan,"after_plan":explain(&conn,sql)?}), + ); + } + let write_after = profile_writes(&conn, 100)?; + std::fs::write( + output, + serde_json::to_vec_pretty(&serde_json::json!({"sqlite":rusqlite::version(), + "queries":evidence,"write_before":write_before,"write_after":write_after, + "database_bytes_before":size_before,"database_bytes_after":size_after}))?, + )?; + Ok(()) +} + +/// Compare full post projections using the actual sanitized prefix syntax. +fn profile_search(conn: &Connection) -> Result> { + let mut evidence = Vec::new(); + for term in ["needle", "representative", "missingword"] { + let before = format!("SELECT p.* FROM posts p JOIN posts_fts f ON f.rowid=p.id + WHERE p.board_id=1 AND posts_fts MATCH '\"{term}\"*' ORDER BY p.created_at DESC,p.id DESC LIMIT 20"); + let (before_time, ids) = timed_query(conn, &before)?; + let mut times = Vec::new(); + for _ in 0_i32..25_i32 { + let started = Instant::now(); + let posts = super::search_posts(conn, 1, term, 20, 0)?; + times.push(started.elapsed().as_micros()); + ensure!( + ids == posts.iter().map(|post| post.id).collect::>(), + "production FTS results changed" + ); + } + evidence.push( + serde_json::json!({"query":format!("production_fts_{term}"),"before":before_time, + "after":latencies(times.into_iter())}), + ); + } + Ok(evidence) +} + +/// Measure representative write cost before and after candidate indexes. +fn profile_writes(conn: &Connection, first: u32) -> Result { + let mut timings = Vec::new(); + for operation in first + ..first + .checked_add(100) + .context("workload profile range overflow")? + { + let started = Instant::now(); + conn.execute_batch("BEGIN IMMEDIATE")?; + write(conn, 0, operation)?; + super::commit_transaction(conn, "profile writes")?; + timings.push(started.elapsed().as_micros()); + } + Ok(latencies(timings.into_iter())) +} diff --git a/src/detect.rs b/src/detect.rs index f6f7ff05..65ca25b1 100644 --- a/src/detect.rs +++ b/src/detect.rs @@ -22,13 +22,15 @@ pub struct WebmEncoderStatus { pub vp9: bool, /// Whether the `libopus` audio encoder is available. pub opus: bool, + /// Whether the `WebM` output muxer is available. + pub muxer: bool, } impl WebmEncoderStatus { - /// Returns whether both required `WebM` encoders are available. + /// Returns whether both selected encoders and the `WebM` muxer are available. #[must_use] pub const fn is_available(self) -> bool { - self.vp9 && self.opus + self.vp9 && self.opus && self.muxer } } @@ -67,127 +69,10 @@ pub fn detect_ffmpeg(require_ffmpeg: bool) -> ToolStatus { } } -/// Probe the configured `ffprobe` path at startup so bogus explicit paths are -/// detected immediately instead of only failing later on the first `WebM` probe. -pub fn detect_ffprobe() -> bool { - let ok = probe_tool(&crate::config::CONFIG.ffprobe_path); - - if ok { - tracing::info!( - target: "rustchan::detect", - available = true, - "ffprobe detected — WebM codec inspection enabled" - ); - } else { - tracing::warn!( - target: "rustchan::detect", - available = false, - ffprobe_path = %crate::config::CONFIG.ffprobe_path, - "ffprobe not detected — WebM codec inspection will fail for uploads that need it" - ); - } - - ok -} - -/// Probe whether the detected ffmpeg has `libwebp` compiled in. -pub fn detect_webp_encoder(ffmpeg_ok: bool) -> bool { - if !ffmpeg_ok { - return false; - } - - let has_webp = crate::media::ffmpeg::check_webp_encoder(); - - if has_webp { - tracing::info!( - target: "rustchan::detect", - webp = true, - "ffmpeg libwebp encoder available — image to WebP conversion enabled" - ); - } else { - tracing::warn!( - target: "rustchan::detect", - webp = false, - "ffmpeg libwebp encoder missing — JPEG/PNG/BMP/TIFF stored in original format" - ); - if crate::logging::is_tty() { - crate::logging::console_print_raw(&webp_install_hint()); - } - } - - has_webp -} - -/// Detects the available external PDF thumbnail renderers in priority order. +/// Reports the built-in pure-Rust PDF preview renderer. +#[must_use] pub fn detect_pdf_thumbnail_renderers() -> Vec { - let renderers = crate::media::thumbnail::detect_pdf_renderers(); - - if renderers.is_empty() { - tracing::warn!( - target: "rustchan::detect", - available = false, - "no PDF thumbnail renderer detected — PDF uploads still work and will use the built-in generic PDF thumbnail. Install Poppler pdftoppm, MuPDF mutool, or use macOS qlmanage to enable real first-page thumbnails" - ); - if crate::logging::is_tty() { - crate::logging::console_print_raw( - " PDF uploads still work with a built-in generic thumbnail.\n Install Poppler `pdftoppm`, MuPDF `mutool`, or use macOS `qlmanage` for real first-page thumbnails.\n\n", - ); - } - } else { - let detected = renderers - .iter() - .map(|renderer| renderer.binary_name()) - .collect::>() - .join(", "); - let selected = renderers - .first() - .map_or("unknown", |renderer| renderer.binary_name()); - tracing::info!( - target: "rustchan::detect", - available = true, - renderers = %detected, - selected = selected, - "PDF thumbnail renderer detected" - ); - } - - renderers -} - -/// Builds the platform-specific `WebP` encoder installation hint. -fn webp_install_hint() -> String { - let mut s = String::new(); - - #[cfg(target_os = "macos")] - { - s.push_str( - " ── macOS: reinstall ffmpeg with libwebp ─────────────────────────────\n\ - \n\ - \x1b[2m brew uninstall ffmpeg\n\ - \x1b[2m brew tap homebrew-ffmpeg/ffmpeg\n\ - \x1b[2m brew install homebrew-ffmpeg/ffmpeg/ffmpeg --with-webp\x1b[0m\n\n", - ); - } - #[cfg(target_os = "linux")] - { - s.push_str( - " ── Linux: install ffmpeg with libwebp ───────────────────────────────\n\ - \n\ - \x1b[2m sudo apt update && sudo apt install ffmpeg libwebp-dev\x1b[0m\n\n", - ); - } - #[cfg(not(any(target_os = "macos", target_os = "linux")))] - { - s.push_str( - " Reinstall ffmpeg with libwebp support. See: https://ffmpeg.org/download.html\n\n", - ); - } - - if !crate::logging::is_tty() { - // Strip any ANSI codes we added for TTY mode - s.retain(|c| c != '\x1b'); - } - s + crate::media::thumbnail::detect_pdf_renderers() } /// Probes an external tool using its conventional version argument. @@ -198,7 +83,7 @@ fn probe_tool(program: &str) -> bool { /// Probes an external tool with explicit arguments and a bounded timeout. fn probe_tool_with_args(program: &str, args: &[&str]) -> bool { let mut command = Command::new(program); - command + let _configured = command .args(args) .stdout(Stdio::null()) .stderr(Stdio::null()); @@ -228,14 +113,23 @@ pub fn detect_webm_encoder(ffmpeg_ok: bool) -> WebmEncoderStatus { return WebmEncoderStatus { vp9: false, opus: false, + muxer: false, }; } - let has_vp9 = crate::media::ffmpeg::check_vp9_encoder(); - let has_opus = crate::media::ffmpeg::check_opus_encoder(); + let caps = crate::media::ffmpeg::video_capabilities(); + tracing::info!( + target: "rustchan::detect", + av1_decoders = ?caps.av1.decoders, + av1_encoders = ?caps.av1.encoders, + "ffmpeg AV1 build capabilities; RustChan output policy remains VP9 + Opus" + ); + let has_vp9 = caps.vp9; + let has_opus = caps.opus; let status = WebmEncoderStatus { vp9: has_vp9, opus: has_opus, + muxer: caps.webm_muxer, }; if status.is_available() { @@ -256,10 +150,15 @@ pub fn detect_webm_encoder(ffmpeg_ok: bool) -> WebmEncoderStatus { target: "rustchan::detect", vp9 = has_vp9, opus = has_opus, - "ffmpeg VP9/Opus encoders missing — MP4 uploads stored as MP4" + webm_muxer = caps.webm_muxer, + "ffmpeg VP9/Opus encoders or WebM muxer missing — videos stored as uploaded" ); if crate::logging::is_tty() { - crate::logging::console_print_raw(&webm_install_hint(has_vp9, has_opus)); + crate::logging::console_print_raw(&webm_install_hint( + has_vp9, + has_opus, + caps.webm_muxer, + )); } } @@ -267,7 +166,7 @@ pub fn detect_webm_encoder(ffmpeg_ok: bool) -> WebmEncoderStatus { } /// Builds a platform-specific installation hint for missing `WebM` encoders. -fn webm_install_hint(has_vp9: bool, has_opus: bool) -> String { +fn webm_install_hint(has_vp9: bool, has_opus: bool, has_muxer: bool) -> String { let mut s = String::new(); if !has_vp9 { s.push_str(" Missing: libvpx-vp9 (VP9 video encoder)\n"); @@ -275,6 +174,9 @@ fn webm_install_hint(has_vp9: bool, has_opus: bool) -> String { if !has_opus { s.push_str(" Missing: libopus (Opus audio encoder)\n"); } + if !has_muxer { + s.push_str(" Missing: webm (WebM output muxer)\n"); + } s.push('\n'); #[cfg(target_os = "macos")] @@ -361,7 +263,7 @@ pub fn tor_stream_token_identity( struct TokenGuard(SocketAddr); impl Drop for TokenGuard { fn drop(&mut self) { - TOR_STREAM_TOKENS.remove(&self.0); + let _previous_value = TOR_STREAM_TOKENS.remove(&self.0); } } @@ -398,7 +300,7 @@ pub fn detect_tor( loop { tracing::info!( target: "rustchan::detect", - attempt = attempt + 1, + attempt = attempt.saturating_add(1), "Starting Tor" ); let run_start = Instant::now(); @@ -473,12 +375,12 @@ async fn run_arti( let cache_dir = crate::config::runtime_tor_cache_dir(); let state_dir = crate::config::runtime_tor_state_dir(); - let key_dir = crate::config::runtime_tor_hidden_service_keys_dir(); + let service_key_dir = crate::config::runtime_tor_hidden_service_keys_dir(); tracing::info!( target: "rustchan::detect", cache_dir = %cache_dir.display(), state_dir = %state_dir.display(), - key_dir = %key_dir.display(), + key_dir = %service_key_dir.display(), "Tor: bootstrapping — first run downloads ~2 MB of directory data" ); @@ -501,11 +403,11 @@ async fn run_arti( tracing::info!(target: "rustchan::detect", "Tor: connected to the Tor network"); // Configurable nicknames distinguish instances that share Arti state. - let key_dir = crate::config::runtime_tor_hidden_service_keys_dir(); + let onion_key_dir = crate::config::runtime_tor_hidden_service_keys_dir(); tracing::info!( target: "rustchan::detect", nickname = %crate::config::CONFIG.tor_service_nickname, - key_dir = %key_dir.display(), + key_dir = %onion_key_dir.display(), "Tor: launching onion service" ); let svc_config = OnionServiceConfigBuilder::default() @@ -627,7 +529,7 @@ fn spawn_tor_stream_proxy( permit: tokio::sync::OwnedSemaphorePermit, local_addr: Arc, ) { - tokio::spawn(async move { + drop(tokio::spawn(async move { let result = proxy_tor_stream(stream_req, &local_addr).await; drop(permit); if let Err(e) = result { @@ -648,7 +550,7 @@ fn spawn_tor_stream_proxy( ); } } - }); + })); } // Connection proxy @@ -678,10 +580,12 @@ async fn proxy_tor_stream( Arc::from(format!("tor:{}", hex::encode(bytes)).as_str()) }; // _guard removes the map entry when this task ends (connection closed or error). - TOR_STREAM_TOKENS.insert(local_peer, Arc::clone(&token)); + let _previous_value = TOR_STREAM_TOKENS.insert(local_peer, Arc::clone(&token)); let _guard = TokenGuard(local_peer); - tokio::io::copy_bidirectional(&mut tor_stream, &mut local).await?; + tokio::io::copy_bidirectional(&mut tor_stream, &mut local) + .await + .map(|_completed_value| ())?; Ok(()) } @@ -691,7 +595,7 @@ async fn proxy_tor_stream( /// [`HsId`] does not implement `std::fmt::Display` in arti-client. /// Encoded manually using `HsId: AsRef<[u8; 32]>`. /// -/// Format: `base32( pubkey || sha3_256(".onion checksum" || pubkey || version)[..2] || version )` +/// Format: `base32( pubkey || sha3_256(".onion checksum" || pubkey || version)[..2] || version )`. fn hsid_to_onion_address(hsid: HsId) -> String { use sha3::{Digest as _, Sha3_256}; @@ -814,7 +718,7 @@ mod tests { use std::os::unix::fs::symlink; let tempdir = tempfile::tempdir().context("create temporary directory")?; - let script = tempdir.path().join("ffprobe"); + let script = tempdir.path().join("fixture-probe"); symlink("/usr/bin/true", &script).context("symlink true as the probe executable")?; assert!( @@ -826,7 +730,7 @@ mod tests { "the explicit executable path must be probed" ); assert!( - !probe_tool("/definitely/not/a/real/ffprobe"), + !probe_tool("/definitely/not/a/real/fixture-probe"), "a nonexistent executable path must fail the probe" ); Ok(()) diff --git a/src/error.rs b/src/error.rs index 34f4c99c..e4eb0156 100644 --- a/src/error.rs +++ b/src/error.rs @@ -8,11 +8,11 @@ use tracing::error; #[derive(Debug, Error)] /// Error returned by application operations and HTTP handlers. pub enum AppError { - /// 404 — board or thread not found + /// 404 — board or thread not found. #[error("Not found: {0}")] NotFound(String), - /// 400 — bad input from user + /// 400 — bad input from user. #[error("Bad request: {0}")] BadRequest(String), @@ -30,19 +30,19 @@ pub enum AppError { csrf_token: String, }, - /// 413 — upload body too large + /// 413 — upload body too large. #[error("Upload too large: {0}")] UploadTooLarge(String), - /// 415 — MIME type not accepted + /// 415 — MIME type not accepted. #[error("Invalid media type: {0}")] InvalidMediaType(String), - /// 409 — resource already exists or snapshot already imported + /// 409 — resource already exists or snapshot already imported. #[error("Conflict: {0}")] Conflict(String), - /// 503 — database write contention; client should retry + /// 503 — database write contention; client should retry. #[error("Database busy — please retry")] DbBusy, @@ -202,7 +202,7 @@ impl IntoResponse for AppError { ) .into_response(); if retry_after { - response + let _previous_value = response .headers_mut() .insert(header::RETRY_AFTER, HeaderValue::from_static("1")); } diff --git a/src/handlers/admin/accounts.rs b/src/handlers/admin/accounts.rs index 98868fd0..568c6cb5 100644 --- a/src/handlers/admin/accounts.rs +++ b/src/handlers/admin/accounts.rs @@ -43,13 +43,13 @@ pub(in crate::server) async fn update_account( } tokio::task::spawn_blocking(move || -> Result { let mut conn = state.db.get()?; - require_admin_session_sid(&conn, session.as_deref())?; + require_admin_session_sid(&conn, session.as_deref()).map(|_completed_value| ())?; if form.new_password != form.confirm_password { return Ok(super::admin_panel_error_redirect_anchor("Passwords did not match. No account changes saved.", "accounts").into_response()); } let result = db::manage_account(&mut conn, session.as_deref().unwrap_or_default(), action, form.username.trim(), &form.current_password, &form.new_password); match result { Ok(true) => Ok(super::admin_panel_redirect_anchor("Account saved. Password changes revoke all sessions for that account; sign in again if you changed your own password.", "accounts").into_response()), Ok(false) => { - super::auth::record_login_fail(&key); + let _failure_count = super::auth::record_login_fail(&key); Ok(super::admin_panel_error_redirect_anchor("Current password did not verify. No account changes saved.", "accounts").into_response()) } Err(error) => Ok(super::admin_panel_error_redirect_anchor(&format!("No account changes saved: {error}"), "accounts").into_response()), @@ -87,7 +87,7 @@ pub(in crate::server) async fn rotate_secret( let actor = db::get_admin_name_by_id(&conn, actor)?.ok_or_else(|| AppError::Forbidden("Administrator missing".into()))?; let user = db::get_admin_by_username(&conn, &actor)?.ok_or_else(|| AppError::Forbidden("Administrator missing".into()))?; if password.len() > 1024 || !crate::utils::crypto::verify_password(password, &user.password_hash)? { - super::auth::record_login_fail(&key); + let _failure_count = super::auth::record_login_fail(&key); return Ok(super::admin_panel_error_redirect_anchor("Current password did not verify. No secret changes saved.", "storage").into_response()); } let rotation = crate::config::admin::management::rotate_secret(); diff --git a/src/handlers/admin/auth.rs b/src/handlers/admin/auth.rs index 3d87ee85..a02c9a34 100644 --- a/src/handlers/admin/auth.rs +++ b/src/handlers/admin/auth.rs @@ -51,7 +51,7 @@ fn admin_login_fail_window_secs() -> u64 { /// Admin login CSRF scope used by this handler. const ADMIN_LOGIN_CSRF_SCOPE: &str = "admin-login"; -/// `ip_hash` → (`fail_count`, `window_start_secs`) +/// `ip_hash` → (`fail_count`, `window_start_secs`). static ADMIN_LOGIN_FAILS: LazyLock> = LazyLock::new(DashMap::new); static LOGIN_CLEANUP_SECS: AtomicU64 = AtomicU64::new(0); @@ -123,7 +123,7 @@ pub(super) fn record_login_fail(ip_key: &str) -> u32 { } fn clear_login_fails(ip_key: &str) { - ADMIN_LOGIN_FAILS.remove(ip_key); + drop(ADMIN_LOGIN_FAILS.remove(ip_key)); } /// Remove login-fail entries whose window has expired. @@ -184,7 +184,7 @@ async fn render_admin_login_response( } }) .await - .map_err(|error| AppError::Internal(anyhow::anyhow!(error)))??; + .map_err(|join_error| AppError::Internal(anyhow::anyhow!(join_error)))??; Ok(( jar, Html(templates::admin_login_page( @@ -346,12 +346,17 @@ pub(in crate::server) async fn admin_login( clear_login_fails(&ip_key); let session_id = new_session_id(); let bootstrap_session_id = session_id.clone(); - let expires_at = Utc::now().timestamp() + CONFIG.session_duration; + let expires_at = Utc::now() + .timestamp() + .checked_add(CONFIG.session_duration) + .ok_or_else(|| { + AppError::Internal(anyhow::anyhow!("administrator session expiry overflows")) + })?; let sid_clone = session_id.clone(); tokio::task::spawn_blocking({ - let pool = state.db.clone(); + let session_pool = state.db.clone(); move || -> Result<()> { - let conn = pool.get()?; + let conn = session_pool.get()?; db::create_session(&conn, &sid_clone, admin_id, expires_at)?; Ok(()) } @@ -433,7 +438,6 @@ mod tests { routing::post, Router, }; - use axum_extra::extract::cookie::{Cookie, CookieJar}; use tower::ServiceExt as _; const TEST_CSRF_COOKIE: &str = "csrf123"; @@ -481,7 +485,9 @@ mod tests { .db .get() .context("get database connection for test board")?; - db::create_board(&conn, "test", "Test", "", false).context("create test board")?; + db::create_board(&conn, "test", "Test", "", false) + .context("create test board") + .map(|_completed_value| ())?; Ok(()) } @@ -492,8 +498,12 @@ mod tests { .context("get database connection for test administrator")?; let password_hash = crate::utils::crypto::hash_password("hunter2") .context("hash test administrator password")?; - db::create_admin(&conn, "admin", &password_hash).context("create test administrator")?; - db::create_board(&conn, "test", "Test", "", false).context("create test board")?; + db::create_admin(&conn, "admin", &password_hash) + .context("create test administrator") + .map(|_completed_value| ())?; + db::create_board(&conn, "test", "Test", "", false) + .context("create test board") + .map(|_completed_value| ())?; Ok(()) } @@ -542,39 +552,42 @@ mod tests { fn locked_after_exceeding_fail_limit() { let key = login_ip_key("test-lock-unique-99887766"); // Clean up any residue from a previous run - ADMIN_LOGIN_FAILS.remove(&key); + drop(ADMIN_LOGIN_FAILS.remove(&key)); let now = login_now_secs(); - ADMIN_LOGIN_FAILS.insert(key.clone(), (admin_login_fail_limit(), now)); + let _previous_failure_window = + ADMIN_LOGIN_FAILS.insert(key.clone(), (admin_login_fail_limit(), now)); assert!(is_login_locked(&key)); // Cleanup - ADMIN_LOGIN_FAILS.remove(&key); + drop(ADMIN_LOGIN_FAILS.remove(&key)); } #[test] fn not_locked_below_fail_limit() { let key = login_ip_key("test-below-limit-11223344"); - ADMIN_LOGIN_FAILS.remove(&key); + drop(ADMIN_LOGIN_FAILS.remove(&key)); let now = login_now_secs(); - ADMIN_LOGIN_FAILS.insert(key.clone(), (admin_login_fail_limit() - 1, now)); + let _previous_failure_window = + ADMIN_LOGIN_FAILS.insert(key.clone(), (admin_login_fail_limit() - 1, now)); assert!(!is_login_locked(&key)); - ADMIN_LOGIN_FAILS.remove(&key); + drop(ADMIN_LOGIN_FAILS.remove(&key)); } #[test] fn expired_window_is_not_locked() { let key = login_ip_key("test-expired-window-55667788"); - ADMIN_LOGIN_FAILS.remove(&key); + drop(ADMIN_LOGIN_FAILS.remove(&key)); // window_start far in the past, beyond admin_login_fail_window_secs() let old_ts = login_now_secs().saturating_sub(admin_login_fail_window_secs() + 60); - ADMIN_LOGIN_FAILS.insert(key.clone(), (admin_login_fail_limit() + 10, old_ts)); + let _previous_failure_window = + ADMIN_LOGIN_FAILS.insert(key.clone(), (admin_login_fail_limit() + 10, old_ts)); assert!(!is_login_locked(&key)); - ADMIN_LOGIN_FAILS.remove(&key); + drop(ADMIN_LOGIN_FAILS.remove(&key)); } #[tokio::test] @@ -583,8 +596,9 @@ mod tests { create_test_board(&state)?; let ip_key = login_ip_key("192.0.2.44"); - ADMIN_LOGIN_FAILS.remove(&ip_key); - ADMIN_LOGIN_FAILS.insert(ip_key.clone(), (admin_login_fail_limit(), login_now_secs())); + drop(ADMIN_LOGIN_FAILS.remove(&ip_key)); + let _previous_failure_window = + ADMIN_LOGIN_FAILS.insert(ip_key.clone(), (admin_login_fail_limit(), login_now_secs())); let router = Router::new() .route("/admin/login", post(admin_login)) @@ -594,18 +608,15 @@ mod tests { "username=admin&password=wrong&_csrf={}", signed_admin_csrf() ))?; - request - .extensions_mut() - .insert(axum::extract::ConnectInfo(std::net::SocketAddr::from(( - [192, 0, 2, 44], - 41000, - )))); + let _previous_peer = request.extensions_mut().insert(axum::extract::ConnectInfo( + std::net::SocketAddr::from(([192, 0, 2, 44], 41000)), + )); let response = router .oneshot(request) .await .context("send locked-out admin login request")?; - ADMIN_LOGIN_FAILS.remove(&ip_key); + drop(ADMIN_LOGIN_FAILS.remove(&ip_key)); anyhow::ensure!( response.status() == StatusCode::OK, @@ -636,7 +647,7 @@ mod tests { "username=admin&password=hunter2&_csrf={}", signed_admin_csrf() ))?; - request + let _previous_transport = request .extensions_mut() .insert(crate::middleware::RequestTransport { direct_https: true }); let response = router diff --git a/src/handlers/admin/backup.rs b/src/handlers/admin/backup.rs index b353d752..5589db21 100644 --- a/src/handlers/admin/backup.rs +++ b/src/handlers/admin/backup.rs @@ -10,6 +10,7 @@ use crate::{ config::CONFIG, db, error::{AppError, Result}, + handlers::is_xml_http_request, middleware::AppState, models::{BackupInfo, BoardAccessMode}, utils::crypto::{new_session_id, verify_password}, @@ -105,10 +106,10 @@ use downloads::{prune_stale_temp_board_downloads, write_temp_board_download_toke #[cfg(test)] use http::admin_xhr_error_response; use http::{ - is_xml_http_request, log_restore_upload_started, redirect_page_response, - restore_auth_preflight, restore_error_redirect_target, restore_failure_response, - restore_start_response, restore_success_redirect_target, restore_upload_parse_response, - sanitize_board_short_value, stream_restore_upload_to_tempfile, validate_backup_zip_filename, + log_restore_upload_started, redirect_page_response, restore_auth_preflight, + restore_error_redirect_target, restore_failure_response, restore_start_response, + restore_success_redirect_target, restore_upload_parse_response, sanitize_board_short_value, + stream_restore_upload_to_tempfile, validate_backup_zip_filename, validate_saved_backup_reference, validate_streamed_restore_upload, RestoreKind, }; pub(in crate::server) use listing::latest_verified_full_backup_modified_time; @@ -142,7 +143,7 @@ pub(in crate::server) async fn admin_backup( let pool = state.db.clone(); move || -> Result<(PathBuf, String, u64)> { let conn = pool.get()?; - require_admin_session_sid(&conn, session_id.as_deref())?; + require_admin_session_sid(&conn, session_id.as_deref()).map(|_completed_value| ())?; let uploads_base = Path::new(&upload_dir); progress.reset(crate::middleware::backup_phase::SNAPSHOT_DB); @@ -229,8 +230,9 @@ pub(in crate::server) async fn admin_backup( .map_err(|e| AppError::Internal(anyhow::anyhow!("Stream DB to zip: {e}")))?; drop(db_src); drop(std::fs::remove_file(&temp_db)); - progress.files_done.fetch_add(1, Ordering::Relaxed); - progress.bytes_done.fetch_add(copied, Ordering::Relaxed); + let _files_before_snapshot = progress.files_done.fetch_add(1, Ordering::Relaxed); + let _bytes_before_snapshot = + progress.bytes_done.fetch_add(copied, Ordering::Relaxed); log_backup_progress(&progress); // Upload files (streamed file-by-file via io::copy) @@ -315,10 +317,10 @@ pub(in crate::server) async fn admin_backup( // Schedule temp-file cleanup after a generous window so even slow clients finish. let cleanup_path = tmp_path; - tokio::spawn(async move { + drop(tokio::spawn(async move { tokio::time::sleep(Duration::from_mins(10)).await; drop(tokio::fs::remove_file(cleanup_path).await); - }); + })); let disposition = format!("attachment; filename=\"{filename}\""); Ok(( @@ -349,11 +351,11 @@ fn count_files_in_dir(dir: &Path) -> u64 { if metadata.file_type().is_symlink() { acc } else if metadata.file_type().is_dir() { - acc + count_files_in_dir(&p) + acc.saturating_add(count_files_in_dir(&p)) } else if metadata.file_type().is_file() && crate::utils::fs_security::assert_regular_file_no_symlink(&p).is_ok() { - acc + 1 + acc.saturating_add(1) } else { acc } @@ -364,7 +366,7 @@ fn count_files_in_dir(dir: &Path) -> u64 { /// /// MEM-FIX: Uses `std::io::copy` with the zip writer directly, streaming each /// file through a kernel buffer (~8 KiB) instead of reading the whole file -/// into a Vec first. Peak RAM per file = `io::copy`'s 8 KiB stack buffer. +/// into a `Vec` first. Peak RAM per file = `io::copy`'s 8 KiB stack buffer. /// /// Progress tracking: increments `progress.files_done` and `progress.bytes_done` /// after each file is written to the zip. @@ -424,8 +426,9 @@ pub(super) fn add_dir_to_zip_with_prefix( let copied = std::io::copy(&mut src, zip).map_err(|e| { AppError::Internal(anyhow::anyhow!("copy {} to zip: {}", path.display(), e)) })?; - progress.files_done.fetch_add(1, Ordering::Relaxed); - progress.bytes_done.fetch_add(copied, Ordering::Relaxed); + let _files_before_archive_entry = progress.files_done.fetch_add(1, Ordering::Relaxed); + let _bytes_before_archive_entry = + progress.bytes_done.fetch_add(copied, Ordering::Relaxed); log_backup_progress(progress); } } @@ -518,7 +521,7 @@ fn unique_backup_filename(dir: &Path, base_name: &str) -> String { } } -/// rustchan-data/runtime/tmp/board-downloads/ +/// rustchan-data/runtime/tmp/board-downloads/. fn temp_board_download_dir() -> PathBuf { crate::config::runtime_temp_board_downloads_dir() } @@ -557,7 +560,7 @@ pub(in crate::server) async fn board_backup( let safe_board = safe_board.clone(); move || -> Result { let conn = pool.get()?; - require_admin_session_sid(&conn, session_id.as_deref())?; + require_admin_session_sid(&conn, session_id.as_deref()).map(|_completed_value| ())?; conn.query_row( "SELECT 1 FROM boards WHERE short_name = ?1", params![safe_board], @@ -625,6 +628,7 @@ mod tests { }; use crate::error::AppError; use crate::models::BackupBoardSummary; + use crate::test_support::admin_signed_csrf; use anyhow::{bail, ensure, Context as _, Result as TestResult}; use axum::{ body::{to_bytes, Body}, @@ -666,13 +670,17 @@ mod tests { ); install_admin_session(&state)?; for path in ["", "relative", "%00"] { - let body = format!("_csrf={}&backup_directory={path}", admin_signed_csrf()); - let response = app + let invalid_update_body = + format!("_csrf={}&backup_directory={path}", admin_signed_csrf()); + let invalid_update_response = app .clone() - .oneshot(admin_form_post("/admin/backup/settings", body)?) + .oneshot(admin_form_post( + "/admin/backup/settings", + invalid_update_body, + )?) .await?; - ensure!(response.status() == StatusCode::BAD_REQUEST); - ensure!(response_body_string(response) + ensure!(invalid_update_response.status() == StatusCode::BAD_REQUEST); + ensure!(response_body_string(invalid_update_response) .await? .contains("backup_directory")); } @@ -701,14 +709,14 @@ mod tests { let temp = tempfile::tempdir()?; let data = temp.path().join("data"); let mut command = std::process::Command::new(std::env::current_exe()?); - command + let _base_environment = command .args(["--exact", name, "--nocapture"]) .env("RUSTCHAN_TEST_BACKUP_DATA", &data) .env("CHAN_TOR_SUPPORT", "0") .env_remove("CHAN_BACKUP_DIRECTORY"); let custom = temp.path().join("disk/custom-backups"); if source == "environment" { - command.env("CHAN_BACKUP_DIRECTORY", &custom); + let _custom_backup_environment = command.env("CHAN_BACKUP_DIRECTORY", &custom); } else if source == "settings" { std::fs::create_dir(&data)?; std::fs::write( @@ -774,9 +782,11 @@ mod tests { )?; let saved = root.join(&filename); ensure!(saved.is_dir()); - super::storage::verify_saved_backup(&saved, &[super::storage::BackupScope::FullSite])?; - let listed = super::list_backup_files(&full_backup_dir(), BackupListKind::Full); - ensure!(listed + super::storage::verify_saved_backup(&saved, &[super::storage::BackupScope::FullSite]) + .map(|_completed_value| ())?; + let updated_listing = + super::list_backup_files(&full_backup_dir(), BackupListKind::Full); + ensure!(updated_listing .iter() .any(|backup| backup.backup_ref == filename && backup.verified)); ensure!(latest_verified_full_backup_modified_time().is_some()); @@ -795,7 +805,8 @@ mod tests { "Storage test", "Storage test", "Storage test", - )?; + ) + .map(|_completed_value| ())?; } let removed = super::enforce_full_backup_retention(1)?; ensure!(!removed.is_empty()); @@ -841,7 +852,10 @@ mod tests { .write_all(body) .with_context(|| format!("write ZIP entry {name}"))?; } - writer.finish().context("finish ZIP archive")?; + writer + .finish() + .context("finish ZIP archive") + .map(|_completed_value| ())?; } cursor.set_position(0); zip::ZipArchive::new(cursor).context("parse ZIP archive") @@ -942,20 +956,12 @@ mod tests { &conn, "session123", admin_id, - chrono::Utc::now().timestamp() + 3600, + chrono::Utc::now().timestamp().saturating_add(3600), ) .context("create test admin session")?; Ok(()) } - fn admin_signed_csrf() -> String { - crate::utils::crypto::make_scoped_csrf_form_token( - "csrf123", - &crate::config::CONFIG.cookie_secret, - "session123", - ) - } - fn admin_cookie_jar() -> CookieJar { CookieJar::new() .add(Cookie::new("csrf_token", "csrf123")) @@ -999,7 +1005,8 @@ mod tests { { let conn = state.db.get().context("get database connection")?; crate::db::create_board(&conn, &board_short, "Board", "", false) - .context("create board")?; + .context("create board") + .map(|_completed_value| ())?; } let app = Router::new() .route("/admin/board/backup/{board}", get(super::board_backup)) @@ -1170,7 +1177,8 @@ mod tests { { let conn = state.db.get().context("get database connection")?; crate::db::create_board(&conn, &marker_board, "Maintenance Marker", "", false) - .context("create marker board")?; + .context("create marker board") + .map(|_completed_value| ())?; } std::fs::create_dir_all(full_backup_dir()).context("create full backup directory")?; @@ -1347,8 +1355,8 @@ mod tests { std::fs::write(&backup_path, b"not-a-zip").context("write invalid ZIP")?; let mut headers = HeaderMap::new(); - headers.insert(header::HOST, HeaderValue::from_static("localhost")); - headers.insert(header::ORIGIN, HeaderValue::from_static("http://localhost")); + drop(headers.insert(header::HOST, HeaderValue::from_static("localhost"))); + drop(headers.insert(header::ORIGIN, HeaderValue::from_static("http://localhost"))); let response = super::restore_saved_full_backup( axum::extract::State(state), @@ -1570,8 +1578,8 @@ mod tests { admin_cookie_jar(), { let mut headers = HeaderMap::new(); - headers.insert(header::HOST, HeaderValue::from_static("localhost")); - headers.insert(header::ORIGIN, HeaderValue::from_static("http://localhost")); + drop(headers.insert(header::HOST, HeaderValue::from_static("localhost"))); + drop(headers.insert(header::ORIGIN, HeaderValue::from_static("http://localhost"))); headers }, crate::test_support::connect_info(), @@ -1817,7 +1825,8 @@ mod tests { .get() .context("get source database connection")?; crate::db::create_board(&source_conn, "tech", "Technology", "", false) - .context("create source board")?; + .context("create source board") + .map(|_completed_value| ())?; let mut manifest = build_board_backup_manifest(&source_conn, "tech")?; manifest.board.access_mode = "definitely_not_valid".to_owned(); @@ -1855,7 +1864,8 @@ mod tests { .get() .context("get source database connection")?; crate::db::create_board(&source_conn, "tech", "Technology", "", false) - .context("create source board")?; + .context("create source board") + .map(|_completed_value| ())?; let mut manifest = build_board_backup_manifest(&source_conn, "tech")?; manifest.board.access_mode = "view_password".to_owned(); manifest.board.access_password_hash.clear(); @@ -1900,7 +1910,8 @@ mod tests { "UPDATE boards SET allow_pdf = 1 WHERE id = ?1", params![board_id], ) - .context("enable PDF uploads")?; + .context("enable PDF uploads") + .map(|_completed_value| ())?; let manifest = build_board_backup_manifest(&source_conn, "tech")?; @@ -1921,7 +1932,8 @@ mod tests { "UPDATE boards SET allow_pdf = 1 WHERE id = ?1", params![board_id], ) - .context("enable PDF uploads")?; + .context("enable PDF uploads") + .map(|_completed_value| ())?; let manifest = build_board_backup_manifest(&source_conn, "tech")?; let target_pool = crate::db::init_test_pool().context("create target database pool")?; @@ -1941,7 +1953,8 @@ mod tests { "Test PDF setting restore", "Test PDF setting restore completed", ) - .context("restore board")?; + .context("restore board") + .map(|_completed_value| ())?; let restored = crate::db::get_board_by_short(&target_conn, "tech") .context("load restored board")? @@ -1975,7 +1988,8 @@ mod tests { true, None, ) - .context("create source reply")?; + .context("create source reply") + .map(|_completed_value| ())?; let mut manifest = build_board_backup_manifest(&source_conn, "tech")?; manifest .threads @@ -2000,7 +2014,8 @@ mod tests { "Test reply-count restore", "Test reply-count restore completed", ) - .context("restore board")?; + .context("restore board") + .map(|_completed_value| ())?; let restored_count = target_conn .query_row( @@ -2019,22 +2034,22 @@ mod tests { #[test] fn older_board_restore_manifests_default_pdf_uploads_off() -> TestResult<()> { let json = serde_json::json!({ - "version": 1, + "version": 1_i32, "board": { - "id": 1, + "id": 1_i32, "short_name": "tech", "name": "Technology", "description": "", "nsfw": false, - "max_threads": 100, - "max_archived_threads": 150, - "bump_limit": 300, + "max_threads": 100_i32, + "max_archived_threads": 150_i32, + "bump_limit": 300_i32, "allow_images": true, "allow_video": true, "allow_audio": true, "allow_any_files": false, "allow_tripcodes": true, - "edit_window_secs": 300, + "edit_window_secs": 300_i32, "allow_editing": true, "allow_self_delete": true, "allow_archive": true, @@ -2042,11 +2057,11 @@ mod tests { "allow_captcha": false, "show_poster_ids": false, "collapse_greentext": false, - "post_cooldown_secs": 0, + "post_cooldown_secs": 0_i32, "banner_mode": "inherit", "access_mode": "public", "access_password_hash": "", - "created_at": 1_700_000_000 + "created_at": 1_700_000_000_i32 }, "threads": [], "posts": [], @@ -2166,7 +2181,8 @@ mod tests { { let conn = pool.get().context("get database connection")?; crate::db::create_board(&conn, board_short, "Technology", "", false) - .context("create fixture board")?; + .context("create fixture board") + .map(|_completed_value| ())?; let board = crate::db::get_board_by_short(&conn, board_short) .context("load fixture board")? .context("fixture board not found")?; @@ -2201,7 +2217,8 @@ mod tests { None, None, ) - .context("create fixture thread")?; + .context("create fixture thread") + .map(|_completed_value| ())?; let db_path_str = db_path .to_str() @@ -2250,7 +2267,7 @@ mod tests { zip.start_file(format!("uploads/{board_short}/hello.txt"), options) .context("start upload ZIP entry")?; zip.write_all(b"hello").context("write upload ZIP entry")?; - zip.finish().context("finish backup ZIP")?; + drop(zip.finish().context("finish backup ZIP")?); } Ok(()) } @@ -2342,7 +2359,8 @@ mod tests { crate::handlers::admin::backup::storage::board_file_fixtures(), Some(b"sqlite".to_vec()), 4_102_444_800, - )?; + ) + .map(|_completed_value| ())?; std::thread::sleep(std::time::Duration::from_millis(20)); crate::handlers::admin::backup::storage::write_saved_backup_fixture( &newer_dir_mtime_dir, @@ -2350,7 +2368,8 @@ mod tests { crate::handlers::admin::backup::storage::board_file_fixtures(), Some(b"sqlite".to_vec()), 4_102_444_700, - )?; + ) + .map(|_completed_value| ())?; invalidate_backup_list_cache(&full_backup_dir(), BackupListKind::Full); let modified = latest_verified_full_backup_modified_time() @@ -2401,7 +2420,9 @@ mod tests { let upload_dir = tempfile::tempdir().context("create upload directory")?; let mut conn = pool.get().context("get database connection")?; - crate::db::create_board(&conn, "tech", "Technology", "", false).context("create board")?; + crate::db::create_board(&conn, "tech", "Technology", "", false) + .context("create board") + .map(|_completed_value| ())?; let tech_board = crate::db::get_board_by_short(&conn, "tech") .context("load board")? .context("tech board not found")?; @@ -2427,9 +2448,13 @@ mod tests { .context("create reply")?; let manifest = build_board_backup_manifest(&conn, "tech")?; - crate::db::delete_board(&conn, tech_board.id).context("delete board")?; + crate::db::delete_board(&conn, tech_board.id) + .context("delete board") + .map(|_completed_value| ())?; - crate::db::create_board(&conn, "b", "Random", "", false).context("create other board")?; + crate::db::create_board(&conn, "b", "Random", "", false) + .context("create other board") + .map(|_completed_value| ())?; let other_board = crate::db::get_board_by_short(&conn, "b") .context("load other board")? .context("other board not found")?; @@ -2458,7 +2483,8 @@ mod tests { "Test board restore", "Test board restore completed", ) - .context("restore board")?; + .context("restore board") + .map(|_completed_value| ())?; let restored_op = crate::db::get_post_on_board(&conn, "tech", op_post_id) .context("load restored OP")? @@ -2490,11 +2516,12 @@ mod tests { .get() .context("get source database connection")?; crate::db::create_board(&source_conn, "pad", "Padding", "", false) - .context("create padding board")?; + .context("create padding board") + .map(|_completed_value| ())?; let pad_board = crate::db::get_board_by_short(&source_conn, "pad") .context("load padding board")? .context("padding board not found")?; - for idx in 0..5 { + for idx in 0_i32..5_i32 { crate::db::create_thread_with_optional_poll( &source_conn, pad_board.id, @@ -2504,10 +2531,12 @@ mod tests { None, None, ) - .context("create padding thread")?; + .context("create padding thread") + .map(|_completed_value| ())?; } crate::db::create_board(&source_conn, "tech", "Technology", "", false) - .context("create source tech board")?; + .context("create source tech board") + .map(|_completed_value| ())?; let source_tech_board = crate::db::get_board_by_short(&source_conn, "tech") .context("load source tech board")? .context("source tech board not found")?; @@ -2543,7 +2572,8 @@ mod tests { .get() .context("get target database connection")?; crate::db::create_board(&target_conn, "b", "Random", "", false) - .context("create target board")?; + .context("create target board") + .map(|_completed_value| ())?; let target_b = crate::db::get_board_by_short(&target_conn, "b") .context("load target board")? .context("target board not found")?; @@ -2556,7 +2586,8 @@ mod tests { None, None, ) - .context("create target thread")?; + .context("create target thread") + .map(|_completed_value| ())?; let upload_dir = tempfile::tempdir().context("create upload directory")?; let upload_dir_str = upload_dir @@ -2571,7 +2602,8 @@ mod tests { "Test board restore high ids", "Test board restore high ids completed", ) - .context("restore board with high IDs")?; + .context("restore board with high IDs") + .map(|_completed_value| ())?; let restored_op = crate::db::get_post_on_board(&target_conn, "tech", source_op_id) .context("load restored OP")? @@ -2613,7 +2645,8 @@ mod tests { .get() .context("get source database connection")?; crate::db::create_board(&source_conn, "tech", "Technology", "", false) - .context("create source tech board")?; + .context("create source tech board") + .map(|_completed_value| ())?; let source_board = crate::db::get_board_by_short(&source_conn, "tech") .context("load source tech board")? .context("source tech board not found")?; @@ -2650,7 +2683,8 @@ mod tests { .get() .context("get target database connection")?; crate::db::create_board(&target_conn, "b", "Random", "", false) - .context("create target board")?; + .context("create target board") + .map(|_completed_value| ())?; let target_board = crate::db::get_board_by_short(&target_conn, "b") .context("load target board")? .context("target board not found")?; @@ -2687,7 +2721,8 @@ mod tests { "Test board restore remap", "Test board restore remap completed", ) - .context("restore board with ID collisions")?; + .context("restore board with ID collisions") + .map(|_completed_value| ())?; let restored_board_id: i64 = target_conn .query_row( diff --git a/src/handlers/admin/backup/archive.rs b/src/handlers/admin/backup/archive.rs index 95846cbf..e41a96bd 100644 --- a/src/handlers/admin/backup/archive.rs +++ b/src/handlers/admin/backup/archive.rs @@ -86,7 +86,8 @@ pub(super) fn extract_sqlite_db_from_full_backup_archive( .map_err(|error| AppError::Internal(anyhow::anyhow!("Copy {zip_path}: {error}"))) } +/// Finalize ZIP metadata and surface buffered-output flush failures to the caller. +fn finalize_archive(zip: zip::ZipWriter, destination: &Path) -> Result<()> { + let mut writer = zip.finish().map_err(|error| { + AppError::Internal(anyhow::anyhow!( + "Finalize {}: {error}", + destination.display() + )) + })?; + writer.flush().map_err(|error| { + AppError::Internal(anyhow::anyhow!("Flush {}: {error}", destination.display())) + }) +} + fn temporary_archive_path(prefix: &str) -> PathBuf { std::env::temp_dir().join(format!("{prefix}_{}.zip", uuid::Uuid::new_v4().simple())) } @@ -256,7 +271,17 @@ pub(super) fn convert_transfer_zip_to_full_restore_archive { upload_file_count = upload_file_count.saturating_add(1); } - _ => {} + storage::BackupFileKind::Db + | storage::BackupFileKind::Settings + | storage::BackupFileKind::BoardJson + | storage::BackupFileKind::ThreadExport + | storage::BackupFileKind::PostExport + | storage::BackupFileKind::FileInventoryExport + | storage::BackupFileKind::Audio + | storage::BackupFileKind::TorKey + | storage::BackupFileKind::Maintenance + | storage::BackupFileKind::PendingFsOps + | storage::BackupFileKind::Log => {} } } } else if let Some(rel) = name.strip_prefix("site-assets/favicon/") { @@ -296,7 +321,7 @@ pub(super) fn convert_transfer_zip_to_full_restore_archive Result Result {} + storage::BackupFileKind::Db + | storage::BackupFileKind::Settings + | storage::BackupFileKind::BoardJson + | storage::BackupFileKind::ThreadExport + | storage::BackupFileKind::PostExport + | storage::BackupFileKind::FileInventoryExport + | storage::BackupFileKind::Audio + | storage::BackupFileKind::TorKey + | storage::BackupFileKind::Maintenance + | storage::BackupFileKind::PendingFsOps + | storage::BackupFileKind::Log => {} } } } @@ -522,9 +557,7 @@ fn create_full_restore_archive(verified: &storage::VerifiedBackup) -> Result>); + + impl Write for FlushFailure { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + self.0.write(bytes) + } + + fn flush(&mut self) -> std::io::Result<()> { + Err(std::io::Error::other("injected final flush failure")) + } + } + + impl Seek for FlushFailure { + fn seek(&mut self, position: std::io::SeekFrom) -> std::io::Result { + self.0.seek(position) + } + } + + #[test] + fn archive_publication_reports_final_flush_failures() -> TestResult<()> { + let output = FlushFailure(std::io::Cursor::new(Vec::new())); + let zip = zip::ZipWriter::new(output); + let error = finalize_archive(zip, Path::new("test.zip")) + .err() + .context("final flush failure was hidden")?; + ensure!( + error.to_string().contains("injected final flush failure"), + "underlying flush error was lost" + ); + Ok(()) + } + use super::*; use anyhow::{ensure, Context as _, Result as TestResult}; @@ -692,7 +755,8 @@ mod tests { storage::board_file_fixtures(), Some(storage::database_snapshot_fixture()?), 1_715_010_000_i64, - )?; + ) + .map(|_completed_value| ())?; Ok((dir, root)) } @@ -705,7 +769,8 @@ mod tests { storage::board_file_fixtures(), None, 1_715_020_000_i64, - )?; + ) + .map(|_completed_value| ())?; Ok((dir, root)) } diff --git a/src/handlers/admin/backup/create.rs b/src/handlers/admin/backup/create.rs index 96a06a0e..109c39c2 100644 --- a/src/handlers/admin/backup/create.rs +++ b/src/handlers/admin/backup/create.rs @@ -30,11 +30,12 @@ pub(in crate::server) fn create_full_backup_to_server( storage_mode: BackupStorageMode, split_zip_part_size: u64, ) -> Result { - let conn = pool.get()?; + let auth_conn = pool.get()?; let automated = session_id.is_none(); if let Some(session_id) = session_id { - require_admin_session_sid(&conn, Some(session_id))?; + require_admin_session_sid(&auth_conn, Some(session_id)).map(|_completed_value| ())?; } + drop(auth_conn); let uploads_base = Path::new(&CONFIG.upload_dir); let global_favicon_dir = crate::favicon::global_backup_source_dir(); let mut tor_hidden_service_keys_dir = if include_tor_hidden_service_keys { @@ -88,7 +89,7 @@ pub(in crate::server) fn create_full_backup_to_server( } else { 0 }; - let include_tor_hidden_service_keys = tor_hidden_service_keys_dir.is_some(); + let tor_hidden_service_keys_included = tor_hidden_service_keys_dir.is_some(); let file_count = count_files_in_dir(uploads_base) .saturating_add(favicon_file_count) .saturating_add(banner_file_count) @@ -107,9 +108,21 @@ pub(in crate::server) fn create_full_backup_to_server( .to_str() .ok_or_else(|| AppError::Internal(anyhow::anyhow!("Backup DB path non-UTF-8")))? .replace('\'', "''"); - conn.execute_batch(&format!("VACUUM INTO '{db_snapshot_str}'")) + let snapshot_conn = pool.get()?; + if let Some(session_id) = session_id { + require_admin_session_sid(&snapshot_conn, Some(session_id)).map(|_completed_value| ())?; + } + snapshot_conn + .execute_batch(&format!("VACUUM INTO '{db_snapshot_str}'")) .map_err(|error| AppError::Internal(anyhow::anyhow!("VACUUM INTO: {error}")))?; + drop(snapshot_conn); restrict_backup_file(&db_snapshot_path)?; + // Export metadata from the same committed snapshot shipped in the backup; + // file copying and hashing no longer occupy a live pooled connection. + let export_conn = rusqlite::Connection::open_with_flags( + &db_snapshot_path, + rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY, + )?; let db_snapshot_size = std::fs::metadata(&db_snapshot_path) .map(|metadata| metadata.len()) .map_err(|error| AppError::Internal(anyhow::anyhow!("Stat DB snapshot: {error}")))?; @@ -126,7 +139,7 @@ pub(in crate::server) fn create_full_backup_to_server( db_snapshot_sha.clone(), ); - let boards = collect_backup_board_summaries(&conn)?; + let boards = collect_backup_board_summaries(&export_conn)?; let settings_path = crate::config::data_dir().join("settings.toml"); if settings_path.is_file() { let destination = config_dir.join("settings.toml"); @@ -150,7 +163,7 @@ pub(in crate::server) fn create_full_backup_to_server( for board in &boards { validate_board_short_name(&board.short_name)?; - let board_manifest = build_board_backup_manifest(&conn, &board.short_name)?; + let board_manifest = build_board_backup_manifest(&export_conn, &board.short_name)?; write_board_exports(&root_dir, &board_manifest, &mut files)?; } @@ -240,7 +253,7 @@ pub(in crate::server) fn create_full_backup_to_server( settings: settings_path.is_file(), uploads: true, thumbnails: true, - tor_keys: include_tor_hidden_service_keys, + tor_keys: tor_hidden_service_keys_included, board_exports: true, file_inventory: true, }, @@ -248,8 +261,8 @@ pub(in crate::server) fn create_full_backup_to_server( path: "db/rustchan.sqlite3".to_owned(), size: db_snapshot_size, sha256: db_snapshot_sha, - integrity_check: Some(snapshot_db_health_output(&conn, "integrity_check")), - foreign_key_check: Some(snapshot_db_health_output(&conn, "foreign_key_check")), + integrity_check: Some(snapshot_db_health_output(&export_conn, "integrity_check")), + foreign_key_check: Some(snapshot_db_health_output(&export_conn, "foreign_key_check")), }), files, parts: Vec::new(), @@ -258,7 +271,7 @@ pub(in crate::server) fn create_full_backup_to_server( if storage_mode == BackupStorageMode::SplitZip { materialize_split_zip_parts(&root_dir, &mut manifest, split_zip_part_size)?; } - drop(conn); + drop(export_conn); let backup_ref = finalize_saved_backup(&root_dir, manifest)?; invalidate_backup_list_cache(&full_backup_dir(), BackupListKind::Full); @@ -290,7 +303,7 @@ pub(in crate::server) fn create_full_backup_to_server( path = %root_dir.display(), bytes = size, automated = session_id.is_none(), - includes_tor_hidden_service_keys = include_tor_hidden_service_keys, + includes_tor_hidden_service_keys = tor_hidden_service_keys_included, "Backup v4 full backup created" ); progress @@ -311,7 +324,6 @@ pub(in crate::server::handlers::admin) fn create_pre_maintenance_backup_to_serve job_id: u64, reason: &str, ) -> Result { - let conn = pool.get()?; let backup_id = storage::build_backup_id(storage::BackupScope::PreMaintenance, "pre-repair-db"); let root_dir = storage::create_backup_root(&backup_id)?; let db_dir = root_dir.join("db"); @@ -329,9 +341,16 @@ pub(in crate::server::handlers::admin) fn create_pre_maintenance_backup_to_serve .to_str() .ok_or_else(|| AppError::Internal(anyhow::anyhow!("Backup DB path non-UTF-8")))? .replace('\'', "''"); - conn.execute_batch(&format!("VACUUM INTO '{db_snapshot_str}'")) + let snapshot_conn = pool.get()?; + snapshot_conn + .execute_batch(&format!("VACUUM INTO '{db_snapshot_str}'")) .map_err(|error| AppError::Internal(anyhow::anyhow!("VACUUM INTO: {error}")))?; + drop(snapshot_conn); restrict_backup_file(&db_snapshot_path)?; + let export_conn = rusqlite::Connection::open_with_flags( + &db_snapshot_path, + rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY, + )?; let db_snapshot_size = std::fs::metadata(&db_snapshot_path) .map(|metadata| metadata.len()) .map_err(|error| AppError::Internal(anyhow::anyhow!("Stat DB snapshot: {error}")))?; @@ -363,8 +382,8 @@ pub(in crate::server::handlers::admin) fn create_pre_maintenance_backup_to_serve ); } - let pre_integrity = snapshot_db_health_output(&conn, "integrity_check"); - let pre_foreign_key = snapshot_db_health_output(&conn, "foreign_key_check"); + let pre_integrity = snapshot_db_health_output(&export_conn, "integrity_check"); + let pre_foreign_key = snapshot_db_health_output(&export_conn, "foreign_key_check"); let repair_request_path = maintenance_dir.join("repair-request.json"); let repair_request = serde_json::json!({ @@ -411,7 +430,7 @@ pub(in crate::server::handlers::admin) fn create_pre_maintenance_backup_to_serve ); let schema_dump = { - let mut statement = conn + let mut statement = export_conn .prepare( "SELECT sql FROM sqlite_schema WHERE sql IS NOT NULL @@ -443,7 +462,7 @@ pub(in crate::server::handlers::admin) fn create_pre_maintenance_backup_to_serve storage::sha256_hex_for_bytes(schema_dump.as_bytes()), ); - let pending_fs_ops = db::list_pending_fs_ops(&conn) + let pending_fs_ops = db::list_pending_fs_ops(&export_conn) .map_err(|error| AppError::Internal(anyhow::anyhow!("List pending_fs_ops: {error}")))?; if !pending_fs_ops.is_empty() { let pending_fs_path = maintenance_dir.join("pending-fs-ops.json"); @@ -756,7 +775,8 @@ pub(in crate::server) async fn create_full_backup( include_tor_hidden_service_keys, storage_mode, split_zip_part_size, - )?; + ) + .map(|_completed_value| ())?; Ok(()) } }) @@ -815,7 +835,7 @@ pub(in crate::server) async fn create_board_backup( let pool = state.db.clone(); move || -> Result { let conn = pool.get()?; - require_admin_session_sid(&conn, session_id.as_deref())?; + require_admin_session_sid(&conn, session_id.as_deref()).map(|_completed_value| ())?; progress.reset(crate::middleware::backup_phase::SNAPSHOT_DB); log_backup_phase(crate::middleware::backup_phase::SNAPSHOT_DB); let manifest = build_board_backup_manifest(&conn, &board_short)?; @@ -1331,8 +1351,8 @@ where zip.write_all(manifest_json) .map_err(|error| AppError::Internal(anyhow::anyhow!("Write manifest: {error}")))?; if let Some(progress) = progress { - progress.files_done.fetch_add(1, Ordering::Relaxed); - progress.bytes_done.fetch_add( + let _files_before_manifest = progress.files_done.fetch_add(1, Ordering::Relaxed); + let _bytes_before_manifest = progress.bytes_done.fetch_add( u64::try_from(manifest_json.len()).unwrap_or(u64::MAX), Ordering::Relaxed, ); @@ -1492,11 +1512,16 @@ fn materialize_split_zip_parts( let parts_dir = root_dir.join(storage::PARTS_DIR_NAME); ensure_backup_dir(&parts_dir)?; let planned_parts = plan_split_zip_parts(&manifest.files, target_part_size); - let total_parts = u32::try_from(planned_parts.len()).unwrap_or(u32::MAX); + let total_parts = u32::try_from(planned_parts.len()).map_err(|error| { + AppError::Internal(anyhow::anyhow!("split part count exceeds u32: {error}")) + })?; let mut part_infos = Vec::with_capacity(planned_parts.len()); for (part_offset, planned) in planned_parts.iter().enumerate() { - let part_index = u32::try_from(part_offset + 1).unwrap_or(u32::MAX); + let part_index = u32::try_from(part_offset) + .ok() + .and_then(|part| part.checked_add(1)) + .ok_or_else(|| AppError::Internal(anyhow::anyhow!("split part index overflows")))?; let part_filename = format!("parts/part-{part_index:04}.zip"); let part_path = root_dir.join(&part_filename); let tmp_path = root_dir.join(format!("{part_filename}.tmp")); @@ -1528,12 +1553,14 @@ fn materialize_split_zip_parts( let mut source = std::fs::File::open(&source_path).map_err(|error| { AppError::Internal(anyhow::anyhow!("Open {}: {error}", source_path.display())) })?; - std::io::copy(&mut source, &mut zip).map_err(|error| { - AppError::Internal(anyhow::anyhow!( - "Copy {} into split ZIP: {error}", - source_path.display() - )) - })?; + std::io::copy(&mut source, &mut zip) + .map_err(|error| { + AppError::Internal(anyhow::anyhow!( + "Copy {} into split ZIP: {error}", + source_path.display() + )) + }) + .map(|_completed_value| ())?; } let writer = zip.finish().map_err(|error| { AppError::Internal(anyhow::anyhow!("Finalize {}: {error}", tmp_path.display())) @@ -1692,8 +1719,8 @@ where sha256, ); if let Some(progress) = progress { - progress.files_done.fetch_add(1, Ordering::Relaxed); - progress.bytes_done.fetch_add(size, Ordering::Relaxed); + let _files_before_upload = progress.files_done.fetch_add(1, Ordering::Relaxed); + let _bytes_before_upload = progress.bytes_done.fetch_add(size, Ordering::Relaxed); log_backup_progress(progress); } } @@ -1913,7 +1940,7 @@ mod tests { std::fs::write(&source, "cookie_secret = \"secret\"") .context("write source settings file")?; - copy_regular_file_to_backup(&source, &destination)?; + copy_regular_file_to_backup(&source, &destination).map(|_completed_value| ())?; let destination_mode = std::fs::metadata(&destination) .context("read destination metadata")? @@ -1952,7 +1979,8 @@ mod tests { let pool = crate::db::init_test_pool().context("create test pool")?; let conn = pool.get().context("get test database connection")?; crate::db::create_board(&conn, "tech", "Technology", "", false) - .context("create test board")?; + .context("create test board") + .map(|_completed_value| ())?; let db_path = temp_dir.path().join("snapshot.db"); let db_path_str = db_path @@ -2012,7 +2040,7 @@ mod tests { ) .context("add Tor keys to backup ZIP")?; } - zip.finish().context("finish backup ZIP")?; + drop(zip.finish().context("finish backup ZIP")?); Ok(()) } @@ -2021,7 +2049,8 @@ mod tests { let pool = crate::db::init_test_pool().context("create test pool")?; let conn = pool.get().context("get test database connection")?; crate::db::create_board(&conn, "tech", "Technology", "", false) - .context("create test board")?; + .context("create test board") + .map(|_completed_value| ())?; let manifest = build_full_backup_manifest(&conn, 1024, 5, 1, 2, false, 0)?; @@ -2138,7 +2167,8 @@ mod tests { let pool = crate::db::init_test_pool().context("create test pool")?; let conn = pool.get().context("get test database connection")?; crate::db::create_board(&conn, "tech", "Technology", "", false) - .context("create test board")?; + .context("create test board") + .map(|_completed_value| ())?; let invariant_trigger: String = conn .query_row( "SELECT sql FROM sqlite_schema @@ -2153,7 +2183,8 @@ mod tests { "UPDATE boards SET short_name = '../escape' WHERE short_name = 'tech'", [], ) - .context("corrupt stored board short name")?; + .context("corrupt stored board short name") + .map(|_completed_value| ())?; conn.execute_batch(&invariant_trigger) .context("restore board domain trigger")?; @@ -2233,7 +2264,18 @@ mod tests { crate::error::AppError::BadRequest(message) => { ensure!(message.contains("Tor hidden service keys were requested")); } - other => bail!("expected BadRequest, got {other:?}"), + other @ (crate::error::AppError::NotFound(_) + | crate::error::AppError::Forbidden(_) + | crate::error::AppError::BannedUser { + reason: _, + csrf_token: _, + } + | crate::error::AppError::UploadTooLarge(_) + | crate::error::AppError::InvalidMediaType(_) + | crate::error::AppError::Conflict(_) + | crate::error::AppError::DbBusy + | crate::error::AppError::Internal(_) + | crate::error::AppError::Tls(_)) => bail!("expected BadRequest, got {other:?}"), } Ok(()) } @@ -2265,7 +2307,18 @@ mod tests { crate::error::AppError::BadRequest(message) => { ensure!(message.contains("Tor hidden service key backups are not available")); } - other => bail!("expected BadRequest, got {other:?}"), + other @ (crate::error::AppError::NotFound(_) + | crate::error::AppError::Forbidden(_) + | crate::error::AppError::BannedUser { + reason: _, + csrf_token: _, + } + | crate::error::AppError::UploadTooLarge(_) + | crate::error::AppError::InvalidMediaType(_) + | crate::error::AppError::Conflict(_) + | crate::error::AppError::DbBusy + | crate::error::AppError::Internal(_) + | crate::error::AppError::Tls(_)) => bail!("expected BadRequest, got {other:?}"), } Ok(()) } @@ -2287,7 +2340,18 @@ mod tests { ensure!(message.contains("could not be read")); ensure!(message.contains(&missing.display().to_string())); } - other => bail!("expected BadRequest, got {other:?}"), + other @ (crate::error::AppError::NotFound(_) + | crate::error::AppError::Forbidden(_) + | crate::error::AppError::BannedUser { + reason: _, + csrf_token: _, + } + | crate::error::AppError::UploadTooLarge(_) + | crate::error::AppError::InvalidMediaType(_) + | crate::error::AppError::Conflict(_) + | crate::error::AppError::DbBusy + | crate::error::AppError::Internal(_) + | crate::error::AppError::Tls(_)) => bail!("expected BadRequest, got {other:?}"), } Ok(()) } diff --git a/src/handlers/admin/backup/downloads.rs b/src/handlers/admin/backup/downloads.rs index 7808350e..0fb5fdf7 100644 --- a/src/handlers/admin/backup/downloads.rs +++ b/src/handlers/admin/backup/downloads.rs @@ -171,7 +171,7 @@ pub(in crate::server) async fn download_backup( let pool = state.db.clone(); move || -> Result<()> { let conn = pool.get()?; - require_admin_session_sid(&conn, session_id.as_deref())?; + require_admin_session_sid(&conn, session_id.as_deref()).map(|_completed_value| ())?; Ok(()) } }) @@ -302,7 +302,7 @@ pub(in crate::server) async fn backup_progress_json( let pool = state.db.clone(); move || -> Result<()> { let conn = pool.get()?; - require_admin_session_sid(&conn, session_id.as_deref())?; + require_admin_session_sid(&conn, session_id.as_deref()).map(|_completed_value| ())?; Ok(()) } }) @@ -349,7 +349,7 @@ pub(in crate::server) async fn delete_backup( let pool = state.db.clone(); move || -> Result<()> { let conn = pool.get()?; - require_admin_session_sid(&conn, session_id.as_deref())?; + require_admin_session_sid(&conn, session_id.as_deref()).map(|_completed_value| ())?; let saved_root = crate::config::backups_dir().join(&safe_filename); let legacy_path = backup_dir.join(&safe_filename); @@ -360,8 +360,8 @@ pub(in crate::server) async fn delete_backup( invalidate_backup_list_cache(&backup_dir, backup_kind); tracing::info!(target: "admin", backup_ref = %safe_filename, "Backup directory deleted"); } else if legacy_path.exists() { - let legacy_path = safe_backup_file_path(&backup_dir, &safe_filename)?; - std::fs::remove_file(&legacy_path) + let validated_legacy_path = safe_backup_file_path(&backup_dir, &safe_filename)?; + std::fs::remove_file(&validated_legacy_path) .map_err(|e| AppError::Internal(anyhow::anyhow!("Delete backup: {e}")))?; invalidate_backup_list_cache(&backup_dir, backup_kind); tracing::info!(target: "admin", filename = %safe_filename, "Backup file deleted"); diff --git a/src/handlers/admin/backup/http.rs b/src/handlers/admin/backup/http.rs index a746ff56..c9726ea3 100644 --- a/src/handlers/admin/backup/http.rs +++ b/src/handlers/admin/backup/http.rs @@ -41,19 +41,15 @@ pub(in crate::server) async fn backup_request_logging_middleware( response } -pub(super) fn is_xml_http_request(headers: &HeaderMap) -> bool { - headers - .get("x-requested-with") - .and_then(|value| value.to_str().ok()) - .is_some_and(|value| value.eq_ignore_ascii_case("XMLHttpRequest")) -} - pub(super) fn admin_xhr_error_response(error: &AppError) -> Response { let handled = match error { AppError::NotFound(message) => Some((StatusCode::NOT_FOUND, message.clone())), AppError::BadRequest(message) => Some((StatusCode::BAD_REQUEST, message.clone())), AppError::Forbidden(message) => Some((StatusCode::FORBIDDEN, message.clone())), - AppError::BannedUser { reason, .. } => Some(( + AppError::BannedUser { + reason, + csrf_token: _, + } => Some(( StatusCode::FORBIDDEN, format!("You are banned. Reason: {reason}"), )), @@ -84,7 +80,17 @@ pub(super) fn admin_xhr_error_response(error: &AppError) -> Response { let (status, message) = match error { AppError::Internal(error) => (StatusCode::INTERNAL_SERVER_ERROR, error.to_string()), AppError::Tls(message) => (StatusCode::INTERNAL_SERVER_ERROR, message.clone()), - _ => ( + AppError::NotFound(_) + | AppError::BadRequest(_) + | AppError::Forbidden(_) + | AppError::BannedUser { + reason: _, + csrf_token: _, + } + | AppError::UploadTooLarge(_) + | AppError::InvalidMediaType(_) + | AppError::Conflict(_) + | AppError::DbBusy => ( StatusCode::INTERNAL_SERVER_ERROR, "Unexpected admin restore error.".to_owned(), ), @@ -118,16 +124,18 @@ pub(super) fn redirect_page_response(target: &str, message: &str) -> Response { ); let mut resp = Response::new(axum::body::Body::from(body)); - resp.extensions_mut().insert(content); + drop(resp.extensions_mut().insert(content)); *resp.status_mut() = StatusCode::OK; - resp.headers_mut().insert( + drop(resp.headers_mut().insert( header::CONTENT_TYPE, HeaderValue::from_static("text/html; charset=utf-8"), - ); - resp.headers_mut().insert( - header::HeaderName::from_static("refresh"), - HeaderValue::from_str(&format!("0; url={target}")) - .unwrap_or_else(|_| HeaderValue::from_static("0; url=/admin/panel")), + )); + drop( + resp.headers_mut().insert( + header::HeaderName::from_static("refresh"), + HeaderValue::from_str(&format!("0; url={target}")) + .unwrap_or_else(|_| HeaderValue::from_static("0; url=/admin/panel")), + ), ); resp } @@ -352,7 +360,8 @@ pub(super) async fn restore_auth_preflight( let session_id_for_task = session_id.clone(); tokio::task::spawn_blocking(move || -> Result<()> { let conn = pool.get()?; - require_admin_session_sid(&conn, session_id_for_task.as_deref())?; + require_admin_session_sid(&conn, session_id_for_task.as_deref()) + .map(|_completed_value| ())?; Ok(()) }) .await @@ -646,7 +655,9 @@ mod tests { async fn parse_full_restore_upload( mut multipart: Multipart, ) -> crate::error::Result<&'static str> { - stream_restore_upload_to_tempfile(RestoreKind::Full, &mut multipart).await?; + stream_restore_upload_to_tempfile(RestoreKind::Full, &mut multipart) + .await + .map(|_completed_value| ())?; Ok("ok") } diff --git a/src/handlers/admin/backup/listing.rs b/src/handlers/admin/backup/listing.rs index a56ba8d9..ec4236c4 100644 --- a/src/handlers/admin/backup/listing.rs +++ b/src/handlers/admin/backup/listing.rs @@ -57,7 +57,7 @@ fn current_source_modified(dir: &Path) -> Option { } pub(super) fn invalidate_backup_list_cache(dir: &Path, kind: BackupListKind) { - BACKUP_LIST_CACHE + let _previous_value = BACKUP_LIST_CACHE .lock() .remove(&backup_list_cache_key(dir, kind)); } @@ -457,7 +457,7 @@ pub(in crate::server) fn list_backup_files(dir: &Path, kind: BackupListKind) -> .then_with(|| right.backup_ref.cmp(&left.backup_ref)) }); - BACKUP_LIST_CACHE.lock().insert( + let _previous_value = BACKUP_LIST_CACHE.lock().insert( cache_key, BackupListCacheEntry { generated_at: Instant::now(), diff --git a/src/handlers/admin/backup/restore_board.rs b/src/handlers/admin/backup/restore_board.rs index f918ac71..b18d255d 100644 --- a/src/handlers/admin/backup/restore_board.rs +++ b/src/handlers/admin/backup/restore_board.rs @@ -216,7 +216,8 @@ fn sync_autoincrement_sequence( AppError::Internal(anyhow::anyhow!( "Advance sqlite_sequence for {table} during restore: {error}" )) - })?; + }) + .map(|_completed_value| ())?; Ok(()) } None => { @@ -228,7 +229,8 @@ fn sync_autoincrement_sequence( AppError::Internal(anyhow::anyhow!( "Insert sqlite_sequence for {table} during restore: {error}" )) - })?; + }) + .map(|_completed_value| ())?; Ok(()) } } @@ -389,17 +391,24 @@ where conn.execute_batch(&format!("VACUUM INTO '{db_snapshot_str}'")) .map_err(|error| AppError::Internal(anyhow::anyhow!("Snapshot board DB: {error}")))?; conn.execute("BEGIN IMMEDIATE", []) - .map_err(|error| AppError::Internal(anyhow::anyhow!("Begin tx: {error}")))?; + .map_err(|error| AppError::Internal(anyhow::anyhow!("Begin tx: {error}"))) + .map(|_completed_value| ())?; let restore_result = (|| -> Result<()> { let live_board_id: i64 = if let Some(existing_id) = existing_id { - conn.execute( - "DELETE FROM threads WHERE board_id = ?1", - params![existing_id], - ) - .map_err(|error| { - map_board_restore_sqlite_error(restore_label, &board_short, "Clear threads", &error) - })?; + let _removed_threads = conn + .execute( + "DELETE FROM threads WHERE board_id = ?1", + params![existing_id], + ) + .map_err(|error| { + map_board_restore_sqlite_error( + restore_label, + &board_short, + "Clear threads", + &error, + ) + })?; conn.execute( "UPDATE boards SET name=?1, description=?2, nsfw=?3, max_threads=?4, max_archived_threads=?5, bump_limit=?6, @@ -437,12 +446,13 @@ where existing_id, ], ) - .map_err(|error| AppError::Internal(anyhow::anyhow!("Update board: {error}")))?; + .map_err(|error| AppError::Internal(anyhow::anyhow!("Update board: {error}"))).map(|_completed_value| ())?; conn.execute( "DELETE FROM banner_assets WHERE scope_type = 'board' AND board_id = ?1", params![existing_id], ) - .map_err(|error| AppError::Internal(anyhow::anyhow!("Clear board banners: {error}")))?; + .map_err(|error| AppError::Internal(anyhow::anyhow!("Clear board banners: {error}"))) + .map(|_completed_value| ())?; existing_id } else { insert_returning_id( @@ -511,7 +521,8 @@ where banner.created_at, ], ) - .map_err(|error| AppError::Internal(anyhow::anyhow!("Insert board banner: {error}")))?; + .map_err(|error| AppError::Internal(anyhow::anyhow!("Insert board banner: {error}"))) + .map(|_completed_value| ())?; } let preserve_thread_ids = can_reuse_row_ids( @@ -572,7 +583,7 @@ where .map_err(|error| { AppError::Internal(anyhow::anyhow!("Insert thread {}: {error}", thread.id)) })?; - thread_id_map.insert(thread.id, new_thread_id); + let _previous_value = thread_id_map.insert(thread.id, new_thread_id); } if preserve_thread_ids { sync_autoincrement_sequence( @@ -658,7 +669,7 @@ where .map_err(|error| { AppError::Internal(anyhow::anyhow!("Insert post {}: {error}", post.id)) })?; - post_id_map.insert(post.id, new_post_id); + let _previous_value = post_id_map.insert(post.id, new_post_id); } if preserve_post_ids { sync_autoincrement_sequence( @@ -693,7 +704,8 @@ where AppError::Internal(anyhow::anyhow!( "Fixup quotelinks for post {new_post_id}: {error}" )) - })?; + }) + .map(|_completed_value| ())?; } } } @@ -738,7 +750,7 @@ where .map_err(|error| { AppError::Internal(anyhow::anyhow!("Insert poll {}: {error}", poll.id)) })?; - poll_id_map.insert(poll.id, new_poll_id); + let _previous_value = poll_id_map.insert(poll.id, new_poll_id); } if preserve_poll_ids { sync_autoincrement_sequence( @@ -777,7 +789,7 @@ where .map_err(|error| { AppError::Internal(anyhow::anyhow!("Insert option {}: {error}", option.id)) })?; - option_id_map.insert(option.id, new_option_id); + let _previous_value = option_id_map.insert(option.id, new_option_id); } if preserve_option_ids { sync_autoincrement_sequence( @@ -809,7 +821,8 @@ where ) .map_err(|error| { AppError::Internal(anyhow::anyhow!("Insert vote {}: {error}", vote.id)) - })?; + }) + .map(|_completed_value| ())?; } for file_hash in &manifest.file_hashes { @@ -833,7 +846,8 @@ where AppError::Internal(anyhow::anyhow!( "Recompute restored thread reply counts: {error}" )) - })?; + }) + .map(|_completed_value| ())?; db::insert_pending_fs_op(conn, &workspace.pending_restore_op)?; Ok(()) @@ -842,7 +856,8 @@ where match restore_result { Ok(()) => { conn.execute("COMMIT", []) - .map_err(|error| AppError::Internal(anyhow::anyhow!("Commit tx: {error}")))?; + .map_err(|error| AppError::Internal(anyhow::anyhow!("Commit tx: {error}"))) + .map(|_completed_value| ())?; if let Err(error) = crate::pending_fs::finalize_board_restore_payload( &workspace.pending_restore_payload, Path::new(upload_dir), @@ -938,7 +953,7 @@ pub(in crate::server) async fn extract_board_from_full_backup( let pool = state.db.clone(); move || -> Result { let mut conn = pool.get()?; - require_admin_session_sid(&conn, session_id.as_deref())?; + require_admin_session_sid(&conn, session_id.as_deref()).map(|_completed_value| ())?; let full_backup_dir_path = crate::config::backups_dir().join(&safe_filename); let full_backup_path = full_backup_dir().join(&safe_filename); @@ -1073,7 +1088,7 @@ pub(in crate::server) async fn restore_saved_board_backup( let pool = state.db.clone(); move || -> Result { let mut conn = pool.get()?; - require_admin_session_sid(&conn, session_id.as_deref())?; + require_admin_session_sid(&conn, session_id.as_deref()).map(|_completed_value| ())?; let root_dir = crate::config::backups_dir().join(&safe_filename); let legacy_zip_path = board_backup_dir().join(&safe_filename); let restore_archive_path = if root_dir.is_dir() { @@ -1183,7 +1198,8 @@ pub(in crate::server) async fn board_restore( use std::io::Read as _; let mut conn = pool.get()?; - require_admin_session_sid(&conn, session_id.as_deref())?; + require_admin_session_sid(&conn, session_id.as_deref()) + .map(|_completed_value| ())?; let mut magic = [0u8; 4]; let mut probe = zip_tmp diff --git a/src/handlers/admin/backup/restore_full.rs b/src/handlers/admin/backup/restore_full.rs index 8abe3a91..a0b2a06e 100644 --- a/src/handlers/admin/backup/restore_full.rs +++ b/src/handlers/admin/backup/restore_full.rs @@ -78,8 +78,8 @@ fn validate_full_restore_db_trust_boundary(conn: &rusqlite::Connection) -> Resul )) })?; validate_board_short_name(&short_name)?; - valid_board_shorts.insert(short_name.clone()); - board_ids_to_shorts.insert(board_id, short_name); + let _completed_value = valid_board_shorts.insert(short_name.clone()); + let _previous_value = board_ids_to_shorts.insert(board_id, short_name); } let mut post_stmt = conn @@ -223,14 +223,16 @@ fn recompute_restored_post_body_html(conn: &rusqlite::Connection) -> Result<()> AppError::Internal(anyhow::anyhow!( "Update restored body_html for post {post_id}: {error}" )) - })?; + }) + .map(|_completed_value| ())?; } Ok(()) } fn scrub_full_restore_runtime_state(conn: &rusqlite::Connection) -> Result<()> { conn.execute("DELETE FROM admin_sessions", []) - .map_err(|error| AppError::Internal(anyhow::anyhow!("Clear restored sessions: {error}")))?; + .map_err(|error| AppError::Internal(anyhow::anyhow!("Clear restored sessions: {error}"))) + .map(|_completed_value| ())?; Ok(()) } @@ -390,6 +392,22 @@ fn create_live_restore_snapshot( Ok(()) } +/// Validate expiry across Chrono's entire timestamp range before restore mutation. +/// This permits calculating the full lifetime at issuance even if the clock moves. +fn validate_restored_session_duration(duration: i64) -> Result<()> { + let _earliest_expiry = + restored_session_expiry(chrono::DateTime::::MIN_UTC.timestamp(), duration)?; + let _latest_expiry = + restored_session_expiry(chrono::DateTime::::MAX_UTC.timestamp(), duration)?; + Ok(()) +} + +/// Compute a checked replacement-session expiry at the actual issuance time. +fn restored_session_expiry(now: i64, duration: i64) -> Result { + now.checked_add(duration) + .ok_or_else(|| AppError::Internal(anyhow::anyhow!("restored session expiry overflows"))) +} + #[expect( clippy::cognitive_complexity, reason = "full restore validation, replacement, and rollback remain one fail-closed operation" @@ -413,6 +431,7 @@ pub(super) fn execute_full_restore( ) -> Result { validate_full_restore_archive_layout(archive)?; let manifest = verify_full_backup_archive(archive)?; + validate_restored_session_duration(CONFIG.session_duration)?; if restore_tor_hidden_service_keys && !manifest.tor_hidden_service_keys_included { return Err(AppError::BadRequest( "This backup does not include Tor hidden service keys.".into(), @@ -523,7 +542,8 @@ pub(super) fn execute_full_restore( RESTORE_TOTAL_EXTRACTED_MAX_BYTES, "Full restore archive", ) - .map_err(|error| AppError::Internal(anyhow::anyhow!("Write temp DB: {error}")))?; + .map_err(|error| AppError::Internal(anyhow::anyhow!("Write temp DB: {error}"))) + .map(|_completed_value| ())?; restrict_restore_temp_file(&temp_db)?; let mut header = [0u8; 16]; @@ -571,7 +591,8 @@ pub(super) fn execute_full_restore( ) .map_err(|error| { AppError::Internal(anyhow::anyhow!("Write {}: {error}", target.display())) - })?; + }) + .map(|_completed_value| ())?; } } else if let Some(rel_path) = validate_entry_path_under_prefix(&name, "favicon/")? { favicon_extracted = true; @@ -599,7 +620,8 @@ pub(super) fn execute_full_restore( ) .map_err(|error| { AppError::Internal(anyhow::anyhow!("Write {}: {error}", target.display())) - })?; + }) + .map(|_completed_value| ())?; } } else if let Some(rel) = name.strip_prefix("banner/") { if rel.is_empty() { @@ -636,7 +658,8 @@ pub(super) fn execute_full_restore( ) .map_err(|error| { AppError::Internal(anyhow::anyhow!("Write {}: {error}", target.display())) - })?; + }) + .map(|_completed_value| ())?; } else if let (Some(rel_path), Some(staged_tor_hidden_service_keys_dir)) = ( validate_entry_path_under_prefix(&name, safety::FULL_BACKUP_TOR_KEYS_ENTRY_PREFIX)?, staged_tor_hidden_service_keys_dir.as_ref(), @@ -665,7 +688,8 @@ pub(super) fn execute_full_restore( ) .map_err(|error| { AppError::Internal(anyhow::anyhow!("Write {}: {error}", target.display())) - })?; + }) + .map(|_completed_value| ())?; tor_hidden_service_key_files_extracted = tor_hidden_service_key_files_extracted.saturating_add(1); } @@ -829,7 +853,7 @@ pub(super) fn execute_full_restore( drop(std::fs::remove_file(&db_snapshot)); let fresh_sid = new_session_id(); - let expires_at = Utc::now().timestamp() + CONFIG.session_duration; + let expires_at = restored_session_expiry(Utc::now().timestamp(), CONFIG.session_duration)?; let session_result = match db::create_session(live_conn, &fresh_sid, admin_id, expires_at) { Ok(()) => { tracing::info!(target: "admin", admin_id = admin_id, "{completion_log}"); @@ -952,7 +976,8 @@ pub(in crate::server) async fn admin_restore( let session_id = restore_auth_preflight(&state, &headers, &jar, secure_context.peer).await?; let upload = stream_restore_upload_to_tempfile(RestoreKind::Full, &mut multipart).await?; - validate_streamed_restore_upload(RestoreKind::Full, &jar, &upload)?; + validate_streamed_restore_upload(RestoreKind::Full, &jar, &upload) + .map(|_completed_value| ())?; let zip_tmp = upload.temp_file; let restore_tor_hidden_service_keys = upload.restore_tor_hidden_service_keys; let uploaded_filename = upload.uploaded_filename; @@ -1144,6 +1169,66 @@ pub(in crate::server) async fn restore_saved_full_backup( #[cfg(test)] mod tests { + #[test] + fn restore_preflight_bounds_session_duration_for_any_clock_value() -> anyhow::Result<()> { + use anyhow::Context as _; + let minimum = chrono::DateTime::::MIN_UTC.timestamp(); + let maximum = chrono::DateTime::::MAX_UTC.timestamp(); + super::validate_restored_session_duration(3600)?; + let longest = i64::MAX + .checked_sub(maximum) + .context("maximum lifetime boundary overflow")?; + super::validate_restored_session_duration(longest)?; + anyhow::ensure!( + super::restored_session_expiry(maximum, longest)? == i64::MAX, + "maximum boundary changed" + ); + anyhow::ensure!( + super::validate_restored_session_duration( + longest + .checked_add(1) + .context("lifetime increment overflow")? + ) + .is_err(), + "overflowing future expiry was accepted" + ); + let shortest = i64::MIN + .checked_sub(minimum) + .context("minimum lifetime boundary overflow")?; + super::validate_restored_session_duration(shortest)?; + anyhow::ensure!( + super::restored_session_expiry(minimum, shortest)? == i64::MIN, + "minimum boundary changed" + ); + anyhow::ensure!( + super::validate_restored_session_duration( + shortest + .checked_sub(1) + .context("lifetime decrement overflow")? + ) + .is_err(), + "underflowing past expiry was accepted" + ); + Ok(()) + } + + #[test] + fn restored_session_expiry_rejects_overflow() -> anyhow::Result<()> { + anyhow::ensure!( + super::restored_session_expiry(100, 3600)? == 3700, + "ordinary session duration changed" + ); + anyhow::ensure!( + super::restored_session_expiry(i64::MAX, 1).is_err(), + "overflowing session expiry was accepted" + ); + anyhow::ensure!( + super::restored_session_expiry(i64::MIN, -1).is_err(), + "underflowing session expiry was accepted" + ); + Ok(()) + } + use super::{ execute_full_restore, full_restore_success_response, validate_full_restore_db_trust_boundary, @@ -1178,7 +1263,7 @@ mod tests { impl Drop for FullRestoreRuntimeTmpRootOverride { fn drop(&mut self) { - super::FULL_RESTORE_RUNTIME_TMP_ROOT_FOR_TEST + let _previous_value = super::FULL_RESTORE_RUNTIME_TMP_ROOT_FOR_TEST .with(|root| root.replace(self.previous.take())); } } @@ -1189,7 +1274,8 @@ mod tests { let pool = crate::db::init_test_pool().context("create test database pool")?; let conn = pool.get().context("get test database connection")?; crate::db::create_board(&conn, "tech", "Technology", "", false) - .context("create fixture board")?; + .context("create fixture board") + .map(|_completed_value| ())?; let db_path_str = db_path .to_str() .context("snapshot fixture path is not UTF-8")? @@ -1344,7 +1430,7 @@ mod tests { }); let manifest_json = if legacy_manifest { serde_json::json!({ - "version": 2, + "version": 2_i32, "generated_at": 1_700_000_000_i64, "rustchan_version": "1.1.3", "db_bytes": db_bytes.len(), @@ -1355,7 +1441,7 @@ mod tests { }) } else { serde_json::json!({ - "version": 3, + "version": 3_i32, "generated_at": 1_700_000_000_i64, "rustchan_version": "1.1.3", "db_bytes": db_bytes.len(), @@ -1395,7 +1481,7 @@ mod tests { .context("write Tor key entry")?; } } - zip.finish().context("finish backup ZIP")?; + drop(zip.finish().context("finish backup ZIP")?); Ok(()) } @@ -1460,7 +1546,8 @@ mod tests { "Test restore completed", "Test restore", "Test restore", - )?; + ) + .map(|_completed_value| ())?; ensure_full_restore_temp_dir_empty(&runtime_tmp_root) } @@ -1485,7 +1572,7 @@ mod tests { .replace('\\', "/"); let contents = std::fs::read_to_string(&path) .with_context(|| format!("read tree file {}", path.display()))?; - out.insert(rel, contents); + let _previous_value = out.insert(rel, contents); } } Ok(()) @@ -1502,7 +1589,7 @@ mod tests { fn saved_full_restore_success_response_sets_session_cookie_and_reopens_section( ) -> TestResult<()> { let mut headers = HeaderMap::new(); - headers.insert(header::HOST, HeaderValue::from_static("localhost")); + let _previous_value = headers.insert(header::HOST, HeaderValue::from_static("localhost")); let response = full_restore_success_response( CookieJar::new(), @@ -1574,7 +1661,8 @@ mod tests { "Test restore completed", "Test restore", "Test restore", - )?; + ) + .map(|_completed_value| ())?; let live_tree = read_tree(&tor_keys_dir)?; ensure!( @@ -1620,7 +1708,8 @@ mod tests { "Test restore completed", "Test restore", "Test restore", - )?; + ) + .map(|_completed_value| ())?; Ok(()) } @@ -1635,13 +1724,15 @@ mod tests { VALUES (1, 'restored-admin', 'restored-hash')", [], ) - .context("seed restored administrator")?; + .context("seed restored administrator") + .map(|_completed_value| ())?; conn.execute( "INSERT INTO admin_sessions (id, admin_id, expires_at) VALUES ('stale-session-from-backup', 1, unixepoch() + 86400)", [], ) - .context("seed stale session")?; + .context("seed stale session") + .map(|_completed_value| ())?; } let zip_path = temp_dir.path().join("backup.zip"); write_full_backup_zip_from_db(&zip_path, &db_path, None, false)?; @@ -1720,7 +1811,8 @@ mod tests { "Test restore completed", "Test restore", "Test restore", - )?; + ) + .map(|_completed_value| ())?; let body_html: String = live_conn .query_row( @@ -1751,7 +1843,8 @@ mod tests { conn.execute_batch("DROP TRIGGER boards_domain_update") .context("temporarily remove board domain trigger")?; conn.execute("UPDATE boards SET short_name = '../admin'", []) - .context("seed invalid board short name")?; + .context("seed invalid board short name") + .map(|_completed_value| ())?; conn.execute_batch(&invariant_trigger) .context("restore board domain trigger")?; } @@ -1785,7 +1878,18 @@ mod tests { crate::error::AppError::BadRequest(message) => { ensure!(message.contains("boards has 1 row(s)")); } - other => bail!("expected BadRequest, got {other:?}"), + other @ (crate::error::AppError::NotFound(_) + | crate::error::AppError::Forbidden(_) + | crate::error::AppError::BannedUser { + reason: _, + csrf_token: _, + } + | crate::error::AppError::UploadTooLarge(_) + | crate::error::AppError::InvalidMediaType(_) + | crate::error::AppError::Conflict(_) + | crate::error::AppError::DbBusy + | crate::error::AppError::Internal(_) + | crate::error::AppError::Tls(_)) => bail!("expected BadRequest, got {other:?}"), } Ok(()) } @@ -1808,7 +1912,7 @@ mod tests { VALUES (1, ?1, 'ghost', unixepoch(), unixepoch(), 0, 0, 0, 0)", [board_id], ) - .context("seed thread")?; + .context("seed thread").map(|_completed_value| ())?; conn.execute( "INSERT INTO posts (id, thread_id, board_id, name, body, body_html, file_path, file_name, file_size, @@ -1818,13 +1922,14 @@ mod tests { 'ghost/thumbs/doc.svg', 'application/pdf', 'pdf', unixepoch(), 'token', 1)", [board_id], ) - .context("seed invalid restored media path")?; + .context("seed invalid restored media path") + .map(|_completed_value| ())?; conn.execute( "INSERT INTO file_hashes (sha256, file_path, thumb_path, mime_type, created_at) VALUES ('ghost-hash', 'ghost/doc.pdf', 'ghost/thumbs/doc.svg', 'application/pdf', unixepoch())", [], ) - .context("seed invalid restored file-hash path")?; + .context("seed invalid restored file-hash path").map(|_completed_value| ())?; } let zip_path = temp_dir.path().join("backup.zip"); write_full_backup_zip_from_db(&zip_path, &db_path, None, false)?; @@ -1856,7 +1961,18 @@ mod tests { crate::error::AppError::BadRequest(message) => { ensure!(message.contains("points to unknown board /ghost/")); } - other => bail!("expected BadRequest, got {other:?}"), + other @ (crate::error::AppError::NotFound(_) + | crate::error::AppError::Forbidden(_) + | crate::error::AppError::BannedUser { + reason: _, + csrf_token: _, + } + | crate::error::AppError::UploadTooLarge(_) + | crate::error::AppError::InvalidMediaType(_) + | crate::error::AppError::Conflict(_) + | crate::error::AppError::DbBusy + | crate::error::AppError::Internal(_) + | crate::error::AppError::Tls(_)) => bail!("expected BadRequest, got {other:?}"), } Ok(()) } @@ -1868,7 +1984,8 @@ mod tests { { let conn = rusqlite::Connection::open(&db_path).context("open database snapshot")?; crate::db::create_board(&conn, "b", "Random", "", false) - .context("create second board")?; + .context("create second board") + .map(|_completed_value| ())?; let tech_board_id: i64 = conn .query_row( "SELECT id FROM boards WHERE short_name = 'tech'", @@ -1881,7 +1998,7 @@ mod tests { VALUES (1, ?1, 'doc', unixepoch(), unixepoch(), 0, 0, 0, 0)", [tech_board_id], ) - .context("seed thread")?; + .context("seed thread").map(|_completed_value| ())?; conn.execute( "INSERT INTO posts (id, thread_id, board_id, name, body, body_html, file_path, file_name, file_size, @@ -1891,13 +2008,14 @@ mod tests { 'b/thumbs/doc.svg', 'application/pdf', 'pdf', unixepoch(), 'token', 1)", [tech_board_id], ) - .context("seed cross-board thumbnail path")?; + .context("seed cross-board thumbnail path") + .map(|_completed_value| ())?; conn.execute( "INSERT INTO file_hashes (sha256, file_path, thumb_path, mime_type, created_at) VALUES ('cross-board-hash', 'tech/doc.pdf', 'b/thumbs/doc.svg', 'application/pdf', unixepoch())", [], ) - .context("seed cross-board file hash")?; + .context("seed cross-board file hash").map(|_completed_value| ())?; } let zip_path = temp_dir.path().join("backup.zip"); write_full_backup_zip_from_db(&zip_path, &db_path, None, false)?; @@ -1929,7 +2047,18 @@ mod tests { crate::error::AppError::BadRequest(message) => { ensure!(message.contains("escapes its board /tech/")); } - other => bail!("expected BadRequest, got {other:?}"), + other @ (crate::error::AppError::NotFound(_) + | crate::error::AppError::Forbidden(_) + | crate::error::AppError::BannedUser { + reason: _, + csrf_token: _, + } + | crate::error::AppError::UploadTooLarge(_) + | crate::error::AppError::InvalidMediaType(_) + | crate::error::AppError::Conflict(_) + | crate::error::AppError::DbBusy + | crate::error::AppError::Internal(_) + | crate::error::AppError::Tls(_)) => bail!("expected BadRequest, got {other:?}"), } Ok(()) } @@ -1943,7 +2072,8 @@ mod tests { VALUES ('generic-hash', 'tech/file.bin', '', 'application/octet-stream', unixepoch())", [], ) - .context("seed generic file hash")?; + .context("seed generic file hash") + .map(|_completed_value| ())?; validate_full_restore_db_trust_boundary(&conn)?; Ok(()) @@ -1953,13 +2083,15 @@ mod tests { fn full_restore_trust_boundary_rejects_cross_board_file_hash_pairing() -> TestResult<()> { let db_path = create_snapshot_db()?; let conn = rusqlite::Connection::open(&db_path).context("open database snapshot")?; - crate::db::create_board(&conn, "b", "Random", "", false).context("create second board")?; + crate::db::create_board(&conn, "b", "Random", "", false) + .context("create second board") + .map(|_completed_value| ())?; conn.execute( "INSERT INTO file_hashes (sha256, file_path, thumb_path, mime_type, created_at) VALUES ('cross-board-hash', 'tech/doc.pdf', 'b/thumbs/doc.svg', 'application/pdf', unixepoch())", [], ) - .context("seed cross-board file hash")?; + .context("seed cross-board file hash").map(|_completed_value| ())?; let error = validate_full_restore_db_trust_boundary(&conn) .err() @@ -1969,7 +2101,18 @@ mod tests { crate::error::AppError::BadRequest(message) => { ensure!(message.contains("mixes boards between file_path and thumb_path")); } - other => bail!("expected BadRequest, got {other:?}"), + other @ (crate::error::AppError::NotFound(_) + | crate::error::AppError::Forbidden(_) + | crate::error::AppError::BannedUser { + reason: _, + csrf_token: _, + } + | crate::error::AppError::UploadTooLarge(_) + | crate::error::AppError::InvalidMediaType(_) + | crate::error::AppError::Conflict(_) + | crate::error::AppError::DbBusy + | crate::error::AppError::Internal(_) + | crate::error::AppError::Tls(_)) => bail!("expected BadRequest, got {other:?}"), } Ok(()) } @@ -2014,7 +2157,18 @@ mod tests { crate::error::AppError::BadRequest(message) => { ensure!(message.contains("Tor hidden service key restore is not available")); } - other => bail!("expected BadRequest, got {other:?}"), + other @ (crate::error::AppError::NotFound(_) + | crate::error::AppError::Forbidden(_) + | crate::error::AppError::BannedUser { + reason: _, + csrf_token: _, + } + | crate::error::AppError::UploadTooLarge(_) + | crate::error::AppError::InvalidMediaType(_) + | crate::error::AppError::Conflict(_) + | crate::error::AppError::DbBusy + | crate::error::AppError::Internal(_) + | crate::error::AppError::Tls(_)) => bail!("expected BadRequest, got {other:?}"), } Ok(()) } @@ -2055,7 +2209,8 @@ mod tests { "Test restore completed", "Test restore", "Test restore", - )?; + ) + .map(|_completed_value| ())?; ensure!( read_tree(&tor_keys_dir)? @@ -2117,7 +2272,8 @@ mod tests { "Test restore completed", "Test restore", "Test restore", - )?; + ) + .map(|_completed_value| ())?; let live_tree = read_tree(&tor_keys_dir)?; ensure!( @@ -2264,7 +2420,18 @@ mod tests { crate::error::AppError::BadRequest(message) => { ensure!(message.contains("does not include Tor hidden service keys")); } - other => bail!("expected BadRequest, got {other:?}"), + other @ (crate::error::AppError::NotFound(_) + | crate::error::AppError::Forbidden(_) + | crate::error::AppError::BannedUser { + reason: _, + csrf_token: _, + } + | crate::error::AppError::UploadTooLarge(_) + | crate::error::AppError::InvalidMediaType(_) + | crate::error::AppError::Conflict(_) + | crate::error::AppError::DbBusy + | crate::error::AppError::Internal(_) + | crate::error::AppError::Tls(_)) => bail!("expected BadRequest, got {other:?}"), } Ok(()) } @@ -2298,7 +2465,8 @@ mod tests { "Test restore completed", "Test restore", "Test restore", - )?; + ) + .map(|_completed_value| ())?; ensure!( read_tree(&tor_keys_dir)? diff --git a/src/handlers/admin/backup/safety.rs b/src/handlers/admin/backup/safety.rs index ef41e02d..51835ce3 100644 --- a/src/handlers/admin/backup/safety.rs +++ b/src/handlers/admin/backup/safety.rs @@ -51,12 +51,14 @@ pub(super) fn resolve_tor_hidden_service_keys_availability( return Err(AppError::BadRequest(unavailable_message.to_owned())); }; - std::fs::read_dir(&dir).map_err(|error| { - AppError::BadRequest(format!( + std::fs::read_dir(&dir) + .map_err(|error| { + AppError::BadRequest(format!( "{unavailable_message} The configured identity directory {} could not be read: {error}", dir.display() )) - })?; + }) + .map(|_completed_value| ())?; Ok(TorHiddenServiceKeysAvailability::Available(dir)) } @@ -130,9 +132,13 @@ pub(super) fn log_backup_progress(progress: &BackupProgress) { return; } - let percent = done.saturating_mul(100) / total.max(1); + let Some(percent) = done.saturating_mul(100).checked_div(total) else { + return; + }; let prev_done = done.saturating_sub(1); - let prev_percent = prev_done.saturating_mul(100) / total.max(1); + let Some(prev_percent) = prev_done.saturating_mul(100).checked_div(total) else { + return; + }; let should_log = total <= 50 || done == 1 || done == total @@ -215,8 +221,8 @@ fn remap_numeric_references(body: &str, prefix: &str, pairs: &[(String, String)] break; } Some(rel) => { - let abs = pos + rel; - let after = abs + needle.len(); + let abs = pos.saturating_add(rel); + let after = abs.saturating_add(needle.len()); let next_is_digit = bytes.get(after).is_some_and(u8::is_ascii_digit); let Some(before_match) = remaining.get(..rel) else { break; @@ -345,9 +351,9 @@ pub(super) fn read_limited_bytes( label: &str, ) -> Result> { let mut bytes = Vec::new(); - copy_limited(reader, &mut bytes, max_bytes).map_err(|error| { - AppError::BadRequest(format!("{label} exceeds safe size limit: {error}")) - })?; + copy_limited(reader, &mut bytes, max_bytes) + .map_err(|error| AppError::BadRequest(format!("{label} exceeds safe size limit: {error}"))) + .map(|_completed_value| ())?; Ok(bytes) } @@ -401,7 +407,8 @@ pub(super) fn extract_uploads_to_dir( RESTORE_TOTAL_EXTRACTED_MAX_BYTES, "Restored uploads", ) - .map_err(|e| AppError::Internal(anyhow::anyhow!("Write {}: {e}", target.display())))?; + .map_err(|e| AppError::Internal(anyhow::anyhow!("Write {}: {e}", target.display()))) + .map(|_completed_value| ())?; } Ok(()) } @@ -427,7 +434,10 @@ pub(super) fn validate_restore_safe_entry_name(name: &str) -> Result<()> { for component in Path::new(name).components() { match component { std::path::Component::Normal(_) => {} - _ => { + std::path::Component::Prefix(_) + | std::path::Component::RootDir + | std::path::Component::CurDir + | std::path::Component::ParentDir => { return Err(AppError::BadRequest(format!( "Backup contains suspicious path '{name}'" ))); @@ -655,7 +665,7 @@ mod tests { zip.write_all(bytes) .with_context(|| format!("write ZIP entry {name}"))?; } - zip.finish().context("finish ZIP archive")?; + drop(zip.finish().context("finish ZIP archive")?); Ok(()) } @@ -664,7 +674,8 @@ mod tests { let db_path = temp_dir.path().join("partial.sqlite3"); let conn = rusqlite::Connection::open(&db_path).context("open SQLite database")?; conn.execute("CREATE TABLE boards (id INTEGER PRIMARY KEY)", []) - .context("create partial boards table")?; + .context("create partial boards table") + .map(|_completed_value| ())?; drop(conn); std::fs::read(db_path).context("read partial SQLite database") } @@ -717,7 +728,7 @@ mod tests { zip.start_file("uploads/../../escape.txt", options) .context("start suspicious ZIP entry")?; std::io::Write::write_all(&mut zip, b"bad").context("write suspicious ZIP entry")?; - zip.finish().context("finish ZIP archive")?; + drop(zip.finish().context("finish ZIP archive")?); } let file = std::fs::File::open(&zip_path).context("open ZIP file")?; @@ -767,7 +778,7 @@ mod tests { zip.start_file("uploads/test/ok.txt", options) .context("start valid ZIP entry")?; std::io::Write::write_all(&mut zip, b"ok").context("write valid ZIP entry")?; - zip.finish().context("finish ZIP archive")?; + drop(zip.finish().context("finish ZIP archive")?); } let file = std::fs::File::open(&zip_path).context("open ZIP file")?; @@ -922,7 +933,7 @@ mod tests { let zip_path = temp_dir.path().join("legacy-full.zip"); let db_bytes = super::super::storage::database_snapshot_fixture()?; let manifest = json!({ - "version": 2, + "version": 2_i32, "generated_at": 1_700_000_000_i64, "rustchan_version": "1.1.3", "db_bytes": u64::try_from(db_bytes.len()).context("convert database size")?, @@ -985,30 +996,30 @@ mod tests { let temp_dir = tempfile::tempdir().context("create temporary directory")?; let zip_path = temp_dir.path().join("board.zip"); let manifest = json!({ - "version": 1, + "version": 1_i32, "board": { - "id": 1, + "id": 1_i32, "short_name": "b", "name": "Random", "description": "", "nsfw": false, - "max_threads": 100, - "max_archived_threads": 150, - "bump_limit": 300, + "max_threads": 100_i32, + "max_archived_threads": 150_i32, + "bump_limit": 300_i32, "allow_images": true, "allow_video": true, "allow_audio": true, "allow_any_files": false, "allow_tripcodes": true, - "edit_window_secs": 300, + "edit_window_secs": 300_i32, "allow_editing": true, "allow_archive": true, "allow_video_embeds": true, "allow_captcha": false, "show_poster_ids": false, "collapse_greentext": true, - "post_cooldown_secs": 0, - "created_at": 1_700_000_000 + "post_cooldown_secs": 0_i32, + "created_at": 1_700_000_000_i32 }, "threads": [], "posts": [], diff --git a/src/handlers/admin/backup/storage.rs b/src/handlers/admin/backup/storage.rs index 98dc3286..0f23271c 100644 --- a/src/handlers/admin/backup/storage.rs +++ b/src/handlers/admin/backup/storage.rs @@ -4,7 +4,6 @@ use crate::error::{AppError, Result}; use serde::{Deserialize, Serialize}; use sha2::{Digest as _, Sha256}; use std::collections::{HashMap, HashSet}; -use std::fmt::Write as _; use std::io::{Read, Write}; use std::path::{Path, PathBuf}; @@ -291,7 +290,8 @@ pub(super) fn build_backup_id(_scope: BackupScope, scope_label: &str) -> String pub(super) fn create_backup_root(backup_id: &str) -> Result { if let Some(path) = &crate::config::CONFIG.backup_directory { crate::config::prepare_backup_directory(path, &crate::config::CONFIG) - .map_err(|error| AppError::BadRequest(format!("{error:#}")))?; + .map_err(|error| AppError::BadRequest(format!("{error:#}"))) + .map(|_completed_value| ())?; } let root = backups_root_dir().join(backup_id); crate::config::ensure_private_dir(&root).map_err(|error| { @@ -510,35 +510,50 @@ pub(super) fn build_readme( has_tor_keys: bool, ) -> String { let mut out = String::new(); - let _ = writeln!(out, "RustChan Backup v4"); - let _ = writeln!(out); - let _ = writeln!(out, "Backup ID: {}", manifest.backup_id); - let _ = writeln!(out, "Scope: {}", manifest.scope.slug()); - let _ = writeln!(out, "Mode: {}", metadata.storage_mode.display_name()); - let _ = writeln!(out, "Archive container: {}", manifest.archive_container); - let _ = writeln!( - out, - "Tor keys included: {}", - if has_tor_keys { "yes" } else { "no" } + crate::templates::append_html(&mut out, format_args!("RustChan Backup v4\n")); + crate::templates::append_html(&mut out, format_args!("\n")); + crate::templates::append_html( + &mut out, + format_args!("Backup ID: {}\n", manifest.backup_id), ); - let _ = writeln!(out); - let _ = writeln!(out, "Important files:"); - let _ = writeln!(out, "- {BACKUP_METADATA_FILE_NAME}"); - let _ = writeln!(out, "- {MANIFEST_FILE_NAME}"); - let _ = writeln!(out, "- {CHECKSUMS_FILE_NAME}"); + crate::templates::append_html(&mut out, format_args!("Scope: {}\n", manifest.scope.slug())); + crate::templates::append_html( + &mut out, + format_args!("Mode: {}\n", metadata.storage_mode.display_name()), + ); + crate::templates::append_html( + &mut out, + format_args!("Archive container: {}\n", manifest.archive_container), + ); + crate::templates::append_html( + &mut out, + format_args!( + "Tor keys included: {}\n", + if has_tor_keys { "yes" } else { "no" } + ), + ); + crate::templates::append_html(&mut out, format_args!("\n")); + crate::templates::append_html(&mut out, format_args!("Important files:\n")); + crate::templates::append_html(&mut out, format_args!("- {BACKUP_METADATA_FILE_NAME}\n")); + crate::templates::append_html(&mut out, format_args!("- {MANIFEST_FILE_NAME}\n")); + crate::templates::append_html(&mut out, format_args!("- {CHECKSUMS_FILE_NAME}\n")); if metadata.storage_mode == BackupStorageMode::SplitZip { - let _ = writeln!(out, "- {PARTS_DIR_NAME}/"); + crate::templates::append_html(&mut out, format_args!("- {PARTS_DIR_NAME}/\n")); } if has_tor_keys { - let _ = writeln!(out, "- tor-keys/"); - } - let _ = writeln!(out); - let _ = writeln!(out, "Keep all files in this backup folder together."); - let _ = writeln!(out, "Do not mix ZIP parts from different backups."); - let _ = writeln!( - out, - "Use RustChan restore for safe programmatic restore. Standard ZIP tools are useful only for manual inspection and emergency extraction." + crate::templates::append_html(&mut out, format_args!("- tor-keys/\n")); + } + crate::templates::append_html(&mut out, format_args!("\n")); + crate::templates::append_html( + &mut out, + format_args!("Keep all files in this backup folder together.\n"), + ); + crate::templates::append_html( + &mut out, + format_args!("Do not mix ZIP parts from different backups.\n"), ); + crate::templates::append_html(&mut out, format_args!("Use RustChan restore for safe programmatic restore. Standard ZIP tools are useful only for manual inspection and emergency extraction.\n" + )); out } @@ -555,7 +570,7 @@ pub(super) fn write_root_checksums(root_dir: &Path, extra_paths: &[&Path]) -> Re continue; } let sha256 = sha256_hex_for_file(&path)?; - let _ = writeln!(lines, "{sha256} {name}"); + crate::templates::append_html(&mut lines, format_args!("{sha256} {name}\n")); } for path in extra_paths { if !path.is_file() { @@ -563,7 +578,10 @@ pub(super) fn write_root_checksums(root_dir: &Path, extra_paths: &[&Path]) -> Re } let sha256 = sha256_hex_for_file(path)?; let rel = path.strip_prefix(root_dir).unwrap_or(path); - let _ = writeln!(lines, "{} {}", sha256, rel.to_string_lossy()); + crate::templates::append_html( + &mut lines, + format_args!("{} {}\n", sha256, rel.to_string_lossy()), + ); } write_text(&root_dir.join(CHECKSUMS_FILE_NAME), &lines) } @@ -967,9 +985,9 @@ pub(super) fn copy_verified_file_to_writer( let mut source = std::fs::File::open(path).map_err(|error| { AppError::Internal(anyhow::anyhow!("Open {}: {error}", path.display())) })?; - std::io::copy(&mut source, writer).map_err(|error| { - AppError::Internal(anyhow::anyhow!("Copy verified file: {error}")) - })?; + std::io::copy(&mut source, writer) + .map_err(|error| AppError::Internal(anyhow::anyhow!("Copy verified file: {error}"))) + .map(|_completed_value| ())?; } VerifiedFileSource::ZipEntry { part_path, @@ -987,11 +1005,13 @@ pub(super) fn copy_verified_file_to_writer( "Verified Backup v4 ZIP entry '{entry_path}' is no longer readable: {error}" )) })?; - std::io::copy(&mut entry, writer).map_err(|error| { - AppError::Internal(anyhow::anyhow!( - "Copy verified Backup v4 ZIP entry '{entry_path}': {error}" - )) - })?; + std::io::copy(&mut entry, writer) + .map_err(|error| { + AppError::Internal(anyhow::anyhow!( + "Copy verified Backup v4 ZIP entry '{entry_path}': {error}" + )) + }) + .map(|_completed_value| ())?; } } Ok(()) @@ -1116,7 +1136,7 @@ fn verify_split_part_path(root_dir: &Path, filename: &str) -> Result { "Backup v4 split part path '{filename}' must be an immediate file under parts/." ))); } - parse_split_part_index(filename)?; + parse_split_part_index(filename).map(|_completed_value| ())?; resolve_declared_file(root_dir, filename, "Backup v4 split ZIP part") } @@ -1245,8 +1265,8 @@ fn verify_split_zip_files( part.filename ))); } - allowed_files.insert(part.filename.clone()); - part_paths.insert(part.filename.clone(), part_path); + let _completed_value = allowed_files.insert(part.filename.clone()); + let _previous_value = part_paths.insert(part.filename.clone(), part_path); } for expected_index in 1..=expected_total { if !part_indexes.contains(&expected_index) { @@ -1303,7 +1323,7 @@ fn verify_split_zip_files( entry.size, &entry.sha256, )?; - verified_files.insert(entry.logical_path.clone(), verified); + let _previous_value = verified_files.insert(entry.logical_path.clone(), verified); } let mut sorted_part_paths: Vec<_> = part_paths.iter().collect(); @@ -1362,7 +1382,8 @@ fn verify_split_zip_files( entry_path: entry_name, }, )?; - verified_files.insert(manifest_entry.logical_path.clone(), verified); + let _previous_value = + verified_files.insert(manifest_entry.logical_path.clone(), verified); } let mut expected_entry_names: Vec<_> = expected_entries.keys().collect(); expected_entry_names.sort(); @@ -1540,7 +1561,7 @@ pub(super) fn verify_saved_backup( entry.size, &entry.sha256, )?; - verified_files.insert(entry.logical_path.clone(), verified); + let _previous_value = verified_files.insert(entry.logical_path.clone(), verified); } (verified_files, HashSet::new()) }; @@ -1626,7 +1647,7 @@ pub(super) fn verify_saved_backup( allowed_files.extend(split_part_allowed_files); if let Some(snapshot) = &db_snapshot { if verified_file_is_root_stored(&snapshot.file) { - allowed_files.insert(snapshot.file.logical_path.clone()); + let _completed_value = allowed_files.insert(snapshot.file.logical_path.clone()); } } @@ -1648,7 +1669,7 @@ pub(super) fn verify_saved_backup( sorted_verified_files.sort_by(|left, right| left.logical_path.cmp(&right.logical_path)); for verified in sorted_verified_files { if verified_file_is_root_stored(verified) { - allowed_files.insert(verified.logical_path.clone()); + let _completed_value = allowed_files.insert(verified.logical_path.clone()); } if let Some(board_short) = verified.board.as_deref() { if !included_boards.contains_key(board_short) { @@ -1783,7 +1804,7 @@ pub(super) fn verify_saved_backup( )) })?; validate_board_json_identity(&board_json_file, board_short, &board_summary.name)?; - boards.insert( + let _previous_value = boards.insert( board_short.clone(), VerifiedBoard { board_json: board_json_file, @@ -1918,7 +1939,9 @@ pub(super) fn write_saved_backup_fixture( .and_then(|name| name.to_str()) .context("saved-backup fixture root has no UTF-8 backup ID")? .to_owned(); - let created_at = completed_at - 60; + let created_at = completed_at + .checked_sub(60) + .context("backup fixture creation timestamp underflows")?; let mut manifest_files = Vec::new(); let includes_database = db_snapshot.is_some(); let db_snapshot_info = db_snapshot.as_deref().map(|bytes| DatabaseSnapshotInfo { @@ -2142,7 +2165,12 @@ mod tests { let total_parts = u32::try_from(groups.len()).context("split part count exceeds u32")?; for (offset, (file_indexes, oversized)) in groups.iter().enumerate() { - let part_index = u32::try_from(offset + 1).context("split part index exceeds u32")?; + let part_index = u32::try_from( + offset + .checked_add(1) + .context("split part index overflows")?, + ) + .context("split part index exceeds u32")?; let part_filename = format!("parts/part-{part_index:04}.zip"); let part_path = root.join(&part_filename); let file = std::fs::File::create(&part_path) @@ -2161,9 +2189,11 @@ mod tests { .context("start split-part ZIP entry")?; let mut source = std::fs::File::open(&source_path) .with_context(|| format!("open fixture file {}", source_path.display()))?; - std::io::copy(&mut source, &mut zip).context("copy split-part ZIP entry")?; + std::io::copy(&mut source, &mut zip) + .context("copy split-part ZIP entry") + .map(|_completed_value| ())?; } - zip.finish().context("finish split-part ZIP")?; + drop(zip.finish().context("finish split-part ZIP")?); for file_index in file_indexes { let entry = manifest .files @@ -2233,7 +2263,8 @@ mod tests { let mut bytes = Vec::new(); zip_entry .read_to_end(&mut bytes) - .context("read split-part ZIP entry contents")?; + .context("read split-part ZIP entry contents") + .map(|_completed_value| ())?; if name == entry_path { bytes = replacement.to_vec(); } @@ -2249,7 +2280,7 @@ mod tests { zip.write_all(&bytes) .context("write rewritten split-part entry")?; } - zip.finish().context("finish rewritten split part")?; + drop(zip.finish().context("finish rewritten split part")?); let part = manifest .parts @@ -2273,7 +2304,8 @@ mod tests { board_file_fixtures(), Some(database_snapshot_fixture()?), 1_715_000_000_i64, - )?; + ) + .map(|_completed_value| ())?; let verified = verify_saved_backup(&root, &[BackupScope::FullSite])?; ensure!(verified.completed_at == 1_715_000_000_i64); @@ -2397,7 +2429,8 @@ mod tests { board_file_fixtures(), Some(database_snapshot_fixture()?), 1_715_000_100_i64, - )?; + ) + .map(|_completed_value| ())?; std::fs::write(root.join("boards/tech/media/src/example.txt"), b"other") .context("tamper fixture file")?; @@ -2419,7 +2452,8 @@ mod tests { board_file_fixtures(), Some(database_snapshot_fixture()?), 1_715_000_200_i64, - )?; + ) + .map(|_completed_value| ())?; std::fs::remove_file(root.join("boards/tech/media/src/example.txt")) .context("remove declared fixture file")?; @@ -2440,8 +2474,9 @@ mod tests { board_file_fixtures(), Some(database_snapshot_fixture()?), 1_715_000_250_i64, - )?; - convert_fixture_to_split(&root, 16)?; + ) + .map(|_completed_value| ())?; + convert_fixture_to_split(&root, 16).map(|_completed_value| ())?; let verified = verify_saved_backup(&root, &[BackupScope::FullSite])?; ensure!(verified.metadata.storage_mode == BackupStorageMode::SplitZip); @@ -2513,7 +2548,7 @@ mod tests { #[test] fn verify_saved_backup_rejects_zip_backed_file_duplicate_at_root() -> TestResult<()> { let (_dir, root, _manifest) = saved_full_fixture("2026-05-06_full-site_duplicate-root")?; - convert_fixture_to_split(&root, 16)?; + convert_fixture_to_split(&root, 16).map(|_completed_value| ())?; let duplicate = root.join("boards/tech/media/src/example.txt"); let duplicate_parent = duplicate .parent() @@ -2643,7 +2678,7 @@ mod tests { fn verify_saved_backup_rejects_non_contiguous_split_part_index() -> TestResult<()> { let (_dir, root, _manifest) = saved_full_fixture("2026-05-06_full-site_part-gap")?; let mut manifest = convert_fixture_to_split(&root, 16)?; - manifest.parts.remove(0); + let _completed_value = manifest.parts.remove(0); let new_total = u32::try_from(manifest.parts.len()).context("part count exceeds u32")?; for part in &mut manifest.parts { part.total_parts = new_total; @@ -2676,7 +2711,9 @@ mod tests { .first_mut() .context("fixture manifest has no mutable split part")? .filename = replacement_filename.clone(); - std::fs::copy(original, root.join(replacement_filename)).context("copy split part")?; + std::fs::copy(original, root.join(replacement_filename)) + .context("copy split part") + .map(|_completed_value| ())?; rewrite_manifest(&root, &manifest)?; let error = verify_saved_backup(&root, &[BackupScope::FullSite]) @@ -2721,7 +2758,7 @@ mod tests { .context("start undeclared ZIP entry")?; zip.write_all(b"extra") .context("write undeclared ZIP entry")?; - zip.finish().context("finish split part")?; + drop(zip.finish().context("finish split part")?); let first_part = manifest .parts .first_mut() @@ -2756,7 +2793,7 @@ mod tests { zip.start_file("../escape.txt", zip::write::SimpleFileOptions::default()) .context("start unsafe ZIP entry")?; zip.write_all(b"escape").context("write unsafe ZIP entry")?; - zip.finish().context("finish split part")?; + drop(zip.finish().context("finish split part")?); let first_part = manifest .parts .first_mut() diff --git a/src/handlers/admin/content.rs b/src/handlers/admin/content.rs index 1133d415..a3a2e1fa 100644 --- a/src/handlers/admin/content.rs +++ b/src/handlers/admin/content.rs @@ -144,7 +144,8 @@ pub(in crate::server) async fn create_board( let pool = state.db.clone(); move || -> Result<()> { let conn = pool.get()?; - super::require_admin_session_sid(&conn, session_id.as_deref())?; + super::require_admin_session_sid(&conn, session_id.as_deref()) + .map(|_completed_value| ())?; if db::get_board_by_short(&conn, &short)?.is_some() { return Err(AppError::Conflict(format!( "Board /{short}/ already exists." @@ -159,7 +160,8 @@ pub(in crate::server) async fn create_board( true, true, allow_audio, - )?; + ) + .map(|_completed_value| ())?; tracing::info!(target: "admin", board = %short, "Created board"); // Refresh live board list so the top bar on any subsequent error // page includes the newly created board. @@ -210,7 +212,7 @@ pub(in crate::server) async fn delete_board( let pool = state.db.clone(); move || -> Result<()> { let conn = pool.get()?; - super::require_admin_session_sid(&conn, session_id.as_deref())?; + super::require_admin_session_sid(&conn, session_id.as_deref()).map(|_completed_value| ())?; // Fetch the board's short_name before deletion so we can remove // its upload directory entirely after cleaning tracked files. @@ -310,7 +312,8 @@ pub(in crate::server) async fn reorder_board( let pool = state.db.clone(); move || -> Result<()> { let mut conn = pool.get()?; - super::require_admin_session_sid(&conn, session_id.as_deref())?; + super::require_admin_session_sid(&conn, session_id.as_deref()) + .map(|_completed_value| ())?; db::move_board(&mut conn, board_id, move_up)?; crate::templates::set_live_boards(db::get_all_boards(&conn)?); Ok(()) @@ -624,7 +627,7 @@ mod tests { use super::*; use anyhow::{bail, ensure, Context as _}; use axum::http::{header, StatusCode}; - use axum_extra::extract::cookie::{Cookie, CookieJar}; + use axum_extra::extract::cookie::Cookie; fn admin_signed_csrf() -> String { crate::utils::crypto::make_scoped_csrf_form_token( @@ -642,41 +645,41 @@ mod tests { fn admin_headers() -> axum::http::HeaderMap { let mut headers = axum::http::HeaderMap::new(); - headers.insert( + drop(headers.insert( header::HOST, axum::http::HeaderValue::from_static("localhost"), - ); - headers.insert( + )); + drop(headers.insert( header::ORIGIN, axum::http::HeaderValue::from_static("http://localhost"), - ); + )); headers } fn tunneled_admin_headers() -> axum::http::HeaderMap { let mut headers = axum::http::HeaderMap::new(); - headers.insert( + drop(headers.insert( header::HOST, axum::http::HeaderValue::from_static("demo.serveo.net"), - ); - headers.insert(header::ORIGIN, axum::http::HeaderValue::from_static("null")); - headers.insert( + )); + drop(headers.insert(header::ORIGIN, axum::http::HeaderValue::from_static("null"))); + drop(headers.insert( header::REFERER, axum::http::HeaderValue::from_static("https://demo.serveo.net/test/thread/1"), - ); + )); headers } fn cross_origin_headers() -> axum::http::HeaderMap { let mut headers = axum::http::HeaderMap::new(); - headers.insert( + drop(headers.insert( header::HOST, axum::http::HeaderValue::from_static("demo.serveo.net"), - ); - headers.insert( + )); + drop(headers.insert( header::ORIGIN, axum::http::HeaderValue::from_static("https://attacker.test"), - ); + )); headers } @@ -690,10 +693,12 @@ mod tests { &conn, "session123", admin_id, - chrono::Utc::now().timestamp() + 3600, + chrono::Utc::now().timestamp().saturating_add(3600), ) .context("create admin session")?; - db::create_board(&conn, "test", "Test", "", false).context("create board")?; + db::create_board(&conn, "test", "Test", "", false) + .context("create board") + .map(|_completed_value| ())?; let board = db::get_board_by_short(&conn, "test") .context("load board")? .context("test board does not exist")?; diff --git a/src/handlers/admin/mod.rs b/src/handlers/admin/mod.rs index 7b5acee8..acae7a7f 100644 --- a/src/handlers/admin/mod.rs +++ b/src/handlers/admin/mod.rs @@ -15,6 +15,8 @@ mod accounts; pub(in crate::server) mod auth; +/// Explicit administrator settings restarts. +pub(in crate::server) mod restart; /// Administrator-only software update handlers. pub(in crate::server) mod updates; pub(in crate::server) use accounts::*; @@ -281,9 +283,9 @@ pub(super) fn ensure_admin_csrf(jar: CookieJar, secure: bool) -> Result<(CookieJ .map(str::to_owned); let mut jar = jar; let raw = raw.unwrap_or_else(|| { - let raw = new_csrf_token(); - jar = std::mem::take(&mut jar).add(admin_csrf_cookie(raw.clone(), secure)); - raw + let generated_token = new_csrf_token(); + jar = std::mem::take(&mut jar).add(admin_csrf_cookie(generated_token.clone(), secure)); + generated_token }); let session_id = jar .get(SESSION_COOKIE) @@ -614,8 +616,7 @@ struct AdminPanelSnapshot { media_auto_prune_enabled: bool, media_max_active_content_size_bytes: u64, ffmpeg_available: bool, - ffprobe_available: bool, - ffmpeg_webp_available: bool, + ffmpeg_av1: crate::media::ffmpeg::Av1Capabilities, ffmpeg_vp9_available: bool, ffmpeg_vp9_encoder_available: bool, ffmpeg_opus_available: bool, @@ -814,8 +815,7 @@ struct MaintenanceDomainData { media_auto_prune_enabled: bool, media_max_active_content_size_bytes: u64, ffmpeg_available: bool, - ffprobe_available: bool, - ffmpeg_webp_available: bool, + ffmpeg_av1: crate::media::ffmpeg::Av1Capabilities, ffmpeg_vp9_available: bool, ffmpeg_vp9_encoder_available: bool, ffmpeg_opus_available: bool, @@ -1134,15 +1134,23 @@ fn fmt_epoch(timestamp: i64) -> String { fn db_integrity_status(status: &crate::middleware::DbMaintenanceJobStatus) -> String { match status { - crate::middleware::DbMaintenanceJobStatus::Finished { report, .. } => { + crate::middleware::DbMaintenanceJobStatus::Finished { report, job_id: _ } => { if report.after.as_ref().unwrap_or(&report.before).ok() { "passed at last check".to_owned() } else { "failed at last check".to_owned() } } - crate::middleware::DbMaintenanceJobStatus::Running { .. } => "check running".to_owned(), - crate::middleware::DbMaintenanceJobStatus::Failed { .. } => "last check failed".to_owned(), + crate::middleware::DbMaintenanceJobStatus::Running { + job_id: _, + started_at: _, + phase: _, + } => "check running".to_owned(), + crate::middleware::DbMaintenanceJobStatus::Failed { + job_id: _, + finished_at: _, + message: _, + } => "last check failed".to_owned(), crate::middleware::DbMaintenanceJobStatus::Idle => "not checked".to_owned(), } } @@ -1197,16 +1205,16 @@ fn safe_dir_size(root: &Path) -> Option { }; for entry in entries.flatten() { let entry_path = entry.path(); - let Ok(metadata) = std::fs::symlink_metadata(&entry_path) else { + let Ok(entry_metadata) = std::fs::symlink_metadata(&entry_path) else { continue; }; - if metadata.file_type().is_symlink() { + if entry_metadata.file_type().is_symlink() { continue; } - if metadata.is_dir() { + if entry_metadata.is_dir() { pending.push_back(entry_path); - } else if metadata.is_file() { - total = total.saturating_add(metadata.len()); + } else if entry_metadata.is_file() { + total = total.saturating_add(entry_metadata.len()); } } } @@ -1306,8 +1314,7 @@ fn load_maintenance_domain_data( media_auto_prune_enabled: db::get_media_auto_prune_enabled(conn), media_max_active_content_size_bytes: db::get_media_max_active_content_size_bytes(conn), ffmpeg_available: state.ffmpeg_available, - ffprobe_available: state.ffprobe_available, - ffmpeg_webp_available: state.ffmpeg_webp_available, + ffmpeg_av1: state.ffmpeg_av1.clone(), ffmpeg_vp9_available: state.ffmpeg_vp9_available, ffmpeg_vp9_encoder_available: state.ffmpeg_vp9_encoder_available, ffmpeg_opus_available: state.ffmpeg_opus_available, @@ -1386,8 +1393,7 @@ fn load_admin_panel_snapshot( media_max_active_content_size_bytes: maintenance_domain .media_max_active_content_size_bytes, ffmpeg_available: maintenance_domain.ffmpeg_available, - ffprobe_available: maintenance_domain.ffprobe_available, - ffmpeg_webp_available: maintenance_domain.ffmpeg_webp_available, + ffmpeg_av1: maintenance_domain.ffmpeg_av1, ffmpeg_vp9_available: maintenance_domain.ffmpeg_vp9_available, ffmpeg_vp9_encoder_available: maintenance_domain.ffmpeg_vp9_encoder_available, ffmpeg_opus_available: maintenance_domain.ffmpeg_opus_available, @@ -1664,17 +1670,16 @@ fn dashboard_dependency_status( ffmpeg_required: bool, ) -> (String, String, crate::templates::AdminDashboardState) { let ffmpeg = detection_word(maintenance.ffmpeg_available); - let ffprobe = detection_word(maintenance.ffprobe_available); - let state = if maintenance.ffmpeg_available && maintenance.ffprobe_available { + let state = if maintenance.ffmpeg_vp9_available { crate::templates::AdminDashboardState::Ok - } else if ffmpeg_required { + } else if ffmpeg_required && !maintenance.ffmpeg_available { crate::templates::AdminDashboardState::ActionNeeded } else { crate::templates::AdminDashboardState::Informational }; - let status = if maintenance.ffmpeg_available && maintenance.ffprobe_available { + let status = if maintenance.ffmpeg_vp9_available { "ready" - } else if ffmpeg_required { + } else if ffmpeg_required && !maintenance.ffmpeg_available { "required tool missing" } else { "limited" @@ -1682,10 +1687,11 @@ fn dashboard_dependency_status( ( status.to_owned(), format!( - "ffmpeg {ffmpeg}; ffprobe {ffprobe}; WebP {}; VP9 {}; Opus {}.", - detection_word(maintenance.ffmpeg_webp_available), + "WebP/images built in (Rust); ffmpeg video {ffmpeg}; VP9 {}; Opus {}; AV1 decoding {}; AV1 encoding {} (output policy: VP9 + Opus).", detection_word(maintenance.ffmpeg_vp9_encoder_available), - detection_word(maintenance.ffmpeg_opus_available) + detection_word(maintenance.ffmpeg_opus_available), + detection_word(!maintenance.ffmpeg_av1.decoders.is_empty()), + detection_word(!maintenance.ffmpeg_av1.encoders.is_empty()) ), state, ) @@ -1827,7 +1833,7 @@ fn render_runtime_workspaces(snapshot: &AdminPanelSnapshot, csrf_token: &str) -> .iter() .map(|section| { let fields = crate::config::admin::runtime::section_snapshot(*section) - .map_err(|_| "unavailable".to_owned()); + .map_err(|snapshot_error| { tracing::warn!(error = %snapshot_error, "operator settings snapshot unavailable"); "unavailable".to_owned() }); crate::templates::admin::render_runtime_settings(*section, &fields, csrf_token) }) .collect(); @@ -1836,7 +1842,10 @@ fn render_runtime_workspaces(snapshot: &AdminPanelSnapshot, csrf_token: &str) -> "{runtime_html}{}{}{application_html}", crate::templates::admin::render_accounts(&snapshot.accounts, csrf_token), crate::templates::admin::render_management( - &crate::config::admin::management::snapshot().map_err(|_| "unavailable".to_owned()), + &crate::config::admin::management::snapshot().map_err(|snapshot_error| { + tracing::warn!(error = %snapshot_error, "operator settings snapshot unavailable"); + "unavailable".to_owned() + }), csrf_token ) ) @@ -1856,8 +1865,10 @@ fn render_admin_panel_from_snapshot( is_error: *is_error, message, }); - let network_fields = - crate::config::admin::network_snapshot().map_err(|_| "unavailable".to_owned()); + let network_fields = crate::config::admin::network_snapshot().map_err(|snapshot_error| { + tracing::warn!(error = %snapshot_error, "operator settings snapshot unavailable"); + "unavailable".to_owned() + }); let view = crate::templates::AdminPanelViewModel { network_html: crate::templates::admin::render_network_settings(&network_fields, csrf_token), runtime_html: render_runtime_workspaces(snapshot, csrf_token), @@ -1916,16 +1927,8 @@ fn render_admin_panel_from_snapshot( } else { crate::templates::AdminDetectionStatus::Missing }, - ffprobe: if snapshot.ffprobe_available { - crate::templates::AdminDetectionStatus::Detected - } else { - crate::templates::AdminDetectionStatus::Missing - }, - webp_encoder: if snapshot.ffmpeg_webp_available { - crate::templates::AdminDetectionStatus::Detected - } else { - crate::templates::AdminDetectionStatus::Missing - }, + av1_decoder: detection_status(!snapshot.ffmpeg_av1.decoders.is_empty()), + av1_encoder: detection_status(!snapshot.ffmpeg_av1.encoders.is_empty()), vp9_pipeline: if snapshot.ffmpeg_vp9_available { crate::templates::AdminDetectionStatus::Detected } else { @@ -2003,10 +2006,10 @@ fn build_site_health_view<'a>( tor_detail: &snapshot.dashboard.tor_detail, dependency_summary: crate::templates::AdminSiteHealthDependencySummary { ffmpeg: detection_status(snapshot.ffmpeg_available), - ffprobe: detection_status(snapshot.ffprobe_available), - webp: detection_status(snapshot.ffmpeg_webp_available), vp9: detection_status(snapshot.ffmpeg_vp9_encoder_available), opus: detection_status(snapshot.ffmpeg_opus_available), + av1_decoder: detection_status(!snapshot.ffmpeg_av1.decoders.is_empty()), + av1_encoder: detection_status(!snapshot.ffmpeg_av1.encoders.is_empty()), }, running_jobs: snapshot.site_health.running_jobs, queued_jobs: snapshot.site_health.queued_jobs, @@ -2048,8 +2051,10 @@ fn build_diagnostics_text(snapshot: &AdminPanelSnapshot, tor_address: Option<&st Database schema: {schema}\n\ OS: {os}-{arch}\n\ SQLite: {sqlite}\n\ - ffmpeg: {ffmpeg}\n\ - ffprobe: {ffprobe}\n\ + WebP/images: built in (Rust), including animation\n\ + ffmpeg video: {ffmpeg}\n\ + WebM conversion (VP9 + Opus): {webm}\n\ + AV1 decoding: {av1_decoding}; AV1 encoding: {av1_encoding}\n\ Tor enabled: {tor_enabled} ({tor_detail})\n\ TLS enabled: {tls_enabled}\n\ Reverse proxy: {reverse_proxy}\n\ @@ -2063,7 +2068,9 @@ fn build_diagnostics_text(snapshot: &AdminPanelSnapshot, tor_address: Option<&st arch = std::env::consts::ARCH, sqlite = rusqlite::version(), ffmpeg = detection_word(snapshot.ffmpeg_available), - ffprobe = detection_word(snapshot.ffprobe_available), + webm = detection_word(snapshot.ffmpeg_vp9_available), + av1_decoding = detection_word(!snapshot.ffmpeg_av1.decoders.is_empty()), + av1_encoding = detection_word(!snapshot.ffmpeg_av1.encoders.is_empty()), warnings = indent_diagnostics_block(&snapshot.site_health.recent_warnings), ) } @@ -2132,8 +2139,11 @@ pub(in crate::server) async fn admin_panel( } else { None }; - updates::authorize(&state, &jar).await?; + updates::authorize(&state, &jar) + .await + .map(|_completed_value| ())?; let update_status = updates::snapshot().await; + let restart_view = restart::snapshot(Some(&update_status)).await; let auto_full_backup_settings = state.auto_full_backup_settings.snapshot(); let html = tokio::task::spawn_blocking({ let pool = state.db.clone(); @@ -2144,7 +2154,8 @@ pub(in crate::server) async fn admin_panel( // Auth check inside blocking task let sid = session_id.ok_or_else(|| AppError::Forbidden("Not logged in.".into()))?; db::get_session(&conn, &sid)? - .ok_or_else(|| AppError::Forbidden("Session expired or invalid.".into()))?; + .ok_or_else(|| AppError::Forbidden("Session expired or invalid.".into())) + .map(|_completed_value| ())?; let (snapshot, tor_address) = load_admin_panel_snapshot( &conn, @@ -2166,7 +2177,8 @@ pub(in crate::server) async fn admin_panel( .await .map_err(|e| AppError::Internal(anyhow::anyhow!(e)))??; - let html = html.replace("", &crate::templates::admin::render_software_updates(&update_status, &csrf, crate::updates::managed(), crate::updates::container_managed())) + let html = html.replace("", &crate::templates::admin::render_restart(&restart_view, &csrf)) + .replace("", &crate::templates::admin::render_software_updates(&update_status, &csrf, crate::updates::managed(), crate::updates::container_managed())) .replace("", &crate::templates::admin::render_update_backups(&update_status)) .replace("", if update_status.available_release().is_some() { r#" Update available"# } else { "" }); Ok((jar, Html(html))) @@ -2182,7 +2194,7 @@ pub(in crate::server) async fn admin_site_health_jobs( let state = state.clone(); move || -> Result { let conn = state.db.get()?; - require_admin_session_sid(&conn, session_id.as_deref())?; + require_admin_session_sid(&conn, session_id.as_deref()).map(|_completed_value| ())?; Ok(load_site_health_jobs_snapshot(&conn, &state)) } }) @@ -2271,7 +2283,7 @@ pub(in crate::server) async fn admin_live_log( let pool = state.db.clone(); move || -> Result { let conn = pool.get()?; - require_admin_session_sid(&conn, session_id.as_deref())?; + require_admin_session_sid(&conn, session_id.as_deref()).map(|_completed_value| ())?; let logs_dir = crate::config::logs_dir(); @@ -2364,7 +2376,8 @@ fn admin_bootstrap_now_secs() -> u64 { pub(super) fn create_admin_session_bootstrap(session_id: &str) -> String { let token = crate::utils::crypto::new_session_id(); let expires_at = admin_bootstrap_now_secs().saturating_add(ADMIN_BOOTSTRAP_TTL_SECS); - ADMIN_SESSION_BOOTSTRAPS.insert(token.clone(), (session_id.to_owned(), expires_at)); + let _previous_value = + ADMIN_SESSION_BOOTSTRAPS.insert(token.clone(), (session_id.to_owned(), expires_at)); token } @@ -2453,7 +2466,10 @@ fn build_application_state(snapshot: &AdminPanelSnapshot) -> String { &live, snapshot.saved_ffmpeg_timeout.as_deref(), ) - .map_err(|_| "unavailable".to_owned()), + .map_err(|snapshot_error| { + tracing::warn!(error = %snapshot_error, "operator settings snapshot unavailable"); + "unavailable".to_owned() + }), ) } @@ -2473,6 +2489,7 @@ mod tests { }; use crate::error::AppError; use crate::middleware::SecureCookieContext; + use crate::test_support::admin_signed_csrf; use anyhow::{bail, ensure, Context as _}; use axum::{ body::to_bytes, @@ -2498,7 +2515,7 @@ mod tests { cookie_header.push_str(raw); } let mut headers = HeaderMap::new(); - headers.insert(header::COOKIE, HeaderValue::from_str(&cookie_header)?); + drop(headers.insert(header::COOKIE, HeaderValue::from_str(&cookie_header)?)); let (jar, token) = ensure_admin_csrf(CookieJar::from_headers(&headers), secure)?; let raw = jar.get("csrf_token").context("CSRF cookie is missing")?; ensure!(!raw.value().is_empty(), "CSRF cookie must not be empty"); @@ -2576,7 +2593,8 @@ mod tests { VALUES (?1, 'active', ?2, ?2, 0)", rusqlite::params![board_id, now], ) - .context("insert active thread")?; + .context("insert active thread") + .map(|_completed_value| ())?; let active_thread_id = conn.last_insert_rowid(); conn.execute( "INSERT INTO posts @@ -2589,7 +2607,8 @@ mod tests { 7, 'audio/ogg', ?3, 'delete-active', 1)", rusqlite::params![active_thread_id, board_id, now], ) - .context("insert active post")?; + .context("insert active post") + .map(|_completed_value| ())?; let active_post_id = conn.last_insert_rowid(); conn.execute( @@ -2597,7 +2616,8 @@ mod tests { VALUES (?1, 'archived', ?2, ?2, 1)", rusqlite::params![board_id, now - (8 * 24 * 60 * 60)], ) - .context("insert archived thread")?; + .context("insert archived thread") + .map(|_completed_value| ())?; let archived_thread_id = conn.last_insert_rowid(); conn.execute( "INSERT INTO posts @@ -2608,14 +2628,16 @@ mod tests { 1000, 'dash/video-thumb.webp', 'video/webm', 'video', ?3, 'delete-archived', 1)", rusqlite::params![archived_thread_id, board_id, now - (8 * 24 * 60 * 60)], ) - .context("insert archived post")?; + .context("insert archived post") + .map(|_completed_value| ())?; conn.execute( "INSERT INTO reports (post_id, thread_id, board_id, reason, reporter_hash, created_at) VALUES (?1, ?2, ?3, 'needs review', 'reporter', ?4)", rusqlite::params![active_post_id, active_thread_id, board_id, now], ) - .context("insert report")?; + .context("insert report") + .map(|_completed_value| ())?; let activity = load_dashboard_activity_snapshot(&conn, 1); @@ -2755,7 +2777,7 @@ mod tests { assert_eq!(state, crate::templates::AdminDashboardState::Failure); } - fn maintenance_with_media_tools(ffmpeg: bool, ffprobe: bool) -> MaintenanceDomainData { + fn maintenance_with_media_tools(ffmpeg: bool) -> MaintenanceDomainData { MaintenanceDomainData { db_size_bytes: 0, db_size_warning: false, @@ -2763,8 +2785,7 @@ mod tests { media_auto_prune_enabled: false, media_max_active_content_size_bytes: 0, ffmpeg_available: ffmpeg, - ffprobe_available: ffprobe, - ffmpeg_webp_available: false, + ffmpeg_av1: crate::media::ffmpeg::Av1Capabilities::default(), ffmpeg_vp9_available: false, ffmpeg_vp9_encoder_available: false, ffmpeg_opus_available: false, @@ -2774,7 +2795,10 @@ mod tests { #[test] fn dashboard_optional_media_tools_are_informational_not_warning() { - let maintenance = maintenance_with_media_tools(false, false); + let maintenance = maintenance_with_media_tools(false); + let detail = dashboard_dependency_status(&maintenance, false).1; + assert!(detail.contains("WebP/images built in (Rust)")); + assert!(detail.contains("AV1 decoding missing; AV1 encoding missing")); assert_eq!( dashboard_dependency_status(&maintenance, false).2, @@ -2786,6 +2810,15 @@ mod tests { ); } + #[test] + fn admin_av1_status_distinguishes_decoding_from_encoding() { + let mut maintenance = maintenance_with_media_tools(true); + maintenance.ffmpeg_av1.decoders.push("libdav1d".to_owned()); + let detail = dashboard_dependency_status(&maintenance, false).1; + assert!(detail.contains("AV1 decoding found; AV1 encoding missing")); + assert!(detail.contains("output policy: VP9 + Opus")); + } + #[test] fn dashboard_tor_bootstrap_is_pending_and_disabled_is_neutral() { assert_eq!( @@ -2813,7 +2846,7 @@ mod tests { fn same_origin_headers(host: &str) -> HeaderMap { let mut headers = HeaderMap::new(); if let Ok(host) = HeaderValue::from_str(host) { - headers.insert(header::HOST, host); + drop(headers.insert(header::HOST, host)); } headers } @@ -2845,44 +2878,44 @@ mod tests { #[test] fn same_origin_request_accepts_loopback_aliases_with_matching_port() { let mut headers = same_origin_headers("127.0.0.1:8080"); - headers.insert( + drop(headers.insert( header::ORIGIN, HeaderValue::from_static("http://localhost:8080"), - ); + )); assert!(require_same_origin_request(&headers, None).is_ok()); - let mut headers = same_origin_headers("[::1]:8080"); - headers.insert( + let mut ipv6_headers = same_origin_headers("[::1]:8080"); + drop(ipv6_headers.insert( header::ORIGIN, HeaderValue::from_static("http://127.0.0.1:8080"), - ); - assert!(require_same_origin_request(&headers, None).is_ok()); + )); + assert!(require_same_origin_request(&ipv6_headers, None).is_ok()); } #[test] fn same_origin_request_accepts_ipv6_loopback_bracket_format() { let mut headers = same_origin_headers("[::1]:8080"); - headers.insert( + drop(headers.insert( header::ORIGIN, HeaderValue::from_static("http://[::1]:8080"), - ); + )); assert!(require_same_origin_request(&headers, None).is_ok()); } #[test] fn same_origin_request_accepts_referer_when_origin_is_missing() { let mut headers = same_origin_headers("localhost:8080"); - headers.insert( + drop(headers.insert( header::REFERER, HeaderValue::from_static("http://127.0.0.1:8080/admin"), - ); + )); assert!(require_same_origin_request(&headers, None).is_ok()); } #[test] fn same_origin_request_accepts_valid_onion_http_origin() { let mut headers = same_origin_headers(TEST_ONION_HOST); - headers.insert(header::ORIGIN, HeaderValue::from_static(TEST_ONION_ORIGIN)); + drop(headers.insert(header::ORIGIN, HeaderValue::from_static(TEST_ONION_ORIGIN))); assert!(require_same_origin_request(&headers, None).is_ok()); } @@ -2890,13 +2923,13 @@ mod tests { #[test] fn same_origin_request_accepts_valid_onion_http_referer() { let mut headers = same_origin_headers(TEST_ONION_HOST); - headers.insert( + drop(headers.insert( header::REFERER, HeaderValue::from_static(concat!( "http://aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaam2dqd.onion", "/admin/panel" )), - ); + )); assert!(require_same_origin_request(&headers, None).is_ok()); } @@ -2904,12 +2937,12 @@ mod tests { #[test] fn same_origin_request_rejects_spoofed_onion_origin() { let mut headers = same_origin_headers(TEST_ONION_HOST); - headers.insert( + drop(headers.insert( header::ORIGIN, HeaderValue::from_static( "http://bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbm2dqd.onion", ), - ); + )); assert!(require_same_origin_request(&headers, None).is_err()); } @@ -2917,7 +2950,7 @@ mod tests { #[test] fn same_origin_request_keeps_onion_http_when_tls_uses_default_https_port() { let mut headers = same_origin_headers(TEST_ONION_HOST); - headers.insert(header::ORIGIN, HeaderValue::from_static(TEST_ONION_ORIGIN)); + drop(headers.insert(header::ORIGIN, HeaderValue::from_static(TEST_ONION_ORIGIN))); assert_eq!( request_scheme_for_same_origin_with_config( @@ -2953,14 +2986,14 @@ mod tests { #[test] fn same_origin_request_accepts_missing_origin_and_referer_with_same_origin_fetch_metadata() { let mut headers = same_origin_headers("demo.serveo.net"); - headers.insert("sec-fetch-site", HeaderValue::from_static("same-origin")); + drop(headers.insert("sec-fetch-site", HeaderValue::from_static("same-origin"))); assert!(require_same_origin_request(&headers, None).is_ok()); } #[test] fn same_origin_request_rejects_missing_origin_and_referer_with_cross_site_fetch_metadata() { let mut headers = same_origin_headers("demo.serveo.net"); - headers.insert("sec-fetch-site", HeaderValue::from_static("cross-site")); + drop(headers.insert("sec-fetch-site", HeaderValue::from_static("cross-site"))); assert!(require_same_origin_request(&headers, None).is_err()); } @@ -2979,41 +3012,41 @@ mod tests { #[test] fn same_origin_or_valid_csrf_rejects_cross_origin_post_with_valid_csrf() { let mut headers = same_origin_headers("demo.serveo.net"); - headers.insert( + drop(headers.insert( header::ORIGIN, HeaderValue::from_static("https://evil.test"), - ); + )); assert!(require_same_origin_or_valid_csrf(&headers, None, true).is_err()); } #[test] fn same_origin_request_accepts_null_origin_with_same_origin_referer_on_https_tunnel() { let mut headers = same_origin_headers("demo.serveo.net"); - headers.insert(header::ORIGIN, HeaderValue::from_static("null")); - headers.insert( + drop(headers.insert(header::ORIGIN, HeaderValue::from_static("null"))); + drop(headers.insert( header::REFERER, HeaderValue::from_static("https://demo.serveo.net/admin"), - ); + )); assert!(require_same_origin_request(&headers, None).is_ok()); } #[test] fn same_origin_request_accepts_same_host_https_origin_on_https_tunnel() { let mut headers = same_origin_headers("rustchan.serveousercontent.com"); - headers.insert( + drop(headers.insert( header::ORIGIN, HeaderValue::from_static("https://rustchan.serveousercontent.com"), - ); + )); assert!(require_same_origin_request(&headers, None).is_ok()); } #[test] fn admin_post_csrf_accepts_scoped_token_on_https_tunnel_host() { let mut headers = same_origin_headers("rustchan.serveousercontent.com"); - headers.insert( + drop(headers.insert( header::ORIGIN, HeaderValue::from_static("https://rustchan.serveousercontent.com"), - ); + )); let token = crate::utils::crypto::make_scoped_csrf_form_token( "csrf123", &crate::config::CONFIG.cookie_secret, @@ -3035,31 +3068,31 @@ mod tests { #[test] fn same_origin_request_rejects_null_origin_for_non_loopback_targets() { let mut headers = same_origin_headers("192.168.1.20:8080"); - headers.insert(header::ORIGIN, HeaderValue::from_static("null")); + drop(headers.insert(header::ORIGIN, HeaderValue::from_static("null"))); assert!(require_same_origin_request(&headers, None).is_err()); - let mut headers = same_origin_headers("board-admin-exampleonion123.onion"); - headers.insert(header::ORIGIN, HeaderValue::from_static("null")); - assert!(require_same_origin_request(&headers, None).is_err()); + let mut onion_headers = same_origin_headers("board-admin-exampleonion123.onion"); + drop(onion_headers.insert(header::ORIGIN, HeaderValue::from_static("null"))); + assert!(require_same_origin_request(&onion_headers, None).is_err()); } #[test] fn same_origin_request_rejects_default_https_origin_with_explicit_http_port() { let mut headers = same_origin_headers("example.test:8080"); - headers.insert( + drop(headers.insert( header::ORIGIN, HeaderValue::from_static("https://example.test"), - ); + )); assert!(require_same_origin_request(&headers, None).is_err()); } #[test] fn same_origin_request_rejects_port_mismatch_even_for_loopback_aliases() { let mut headers = same_origin_headers("localhost:8080"); - headers.insert( + drop(headers.insert( header::ORIGIN, HeaderValue::from_static("http://127.0.0.1:3000"), - ); + )); assert!(require_same_origin_request(&headers, None).is_err()); } @@ -3104,43 +3137,43 @@ mod tests { #[test] fn same_origin_request_rejects_userinfo_bypass_shapes() { let mut headers = same_origin_headers("127.0.0.1:8080"); - headers.insert( + drop(headers.insert( header::ORIGIN, HeaderValue::from_static("http://127.0.0.1@evil.com:8080"), - ); + )); assert!(require_same_origin_request(&headers, None).is_err()); - let mut headers = same_origin_headers("127.0.0.1:8080"); - headers.insert( + let mut userinfo_headers = same_origin_headers("127.0.0.1:8080"); + drop(userinfo_headers.insert( header::ORIGIN, HeaderValue::from_static("http://evil.com@127.0.0.1:8080"), - ); - assert!(require_same_origin_request(&headers, None).is_err()); + )); + assert!(require_same_origin_request(&userinfo_headers, None).is_err()); } #[test] fn same_origin_request_rejects_non_loopback_null_origin_lookalikes() { let mut headers = same_origin_headers("localhost.evil.com:8080"); - headers.insert(header::ORIGIN, HeaderValue::from_static("null")); + drop(headers.insert(header::ORIGIN, HeaderValue::from_static("null"))); assert!(require_same_origin_request(&headers, None).is_err()); - let mut headers = same_origin_headers("192.168.1.20:8080"); - headers.insert(header::ORIGIN, HeaderValue::from_static("null")); - assert!(require_same_origin_request(&headers, None).is_err()); + let mut lan_headers = same_origin_headers("192.168.1.20:8080"); + drop(lan_headers.insert(header::ORIGIN, HeaderValue::from_static("null"))); + assert!(require_same_origin_request(&lan_headers, None).is_err()); - let mut headers = same_origin_headers("examplehiddenservice.onion"); - headers.insert(header::ORIGIN, HeaderValue::from_static("null")); - assert!(require_same_origin_request(&headers, None).is_err()); + let mut onion_headers = same_origin_headers("examplehiddenservice.onion"); + drop(onion_headers.insert(header::ORIGIN, HeaderValue::from_static("null"))); + assert!(require_same_origin_request(&onion_headers, None).is_err()); } #[test] fn https_host_port_marks_request_secure() -> anyhow::Result<()> { let mut headers = HeaderMap::new(); let host = format!("example.test:{}", crate::config::CONFIG.tls.port); - headers.insert( + drop(headers.insert( header::HOST, HeaderValue::from_str(&host).context("build HTTPS host header")?, - ); + )); ensure!(host_header_uses_https_port_with_config( &headers, crate::config::CONFIG.tls.port @@ -3151,7 +3184,7 @@ mod tests { #[test] fn http_host_port_does_not_mark_request_secure() { let mut headers = HeaderMap::new(); - headers.insert(header::HOST, HeaderValue::from_static("example.test:8080")); + drop(headers.insert(header::HOST, HeaderValue::from_static("example.test:8080"))); assert!(!host_header_uses_https_port_with_config( &headers, crate::config::CONFIG.tls.port @@ -3161,33 +3194,33 @@ mod tests { #[test] fn https_origin_marks_tunneled_request_secure() { let mut headers = HeaderMap::new(); - headers.insert(header::HOST, HeaderValue::from_static("demo.serveo.net")); - headers.insert( + drop(headers.insert(header::HOST, HeaderValue::from_static("demo.serveo.net"))); + drop(headers.insert( header::REFERER, HeaderValue::from_static("https://demo.serveo.net/admin"), - ); + )); assert!(request_origin_uses_https(&headers)); } #[test] fn mismatched_https_origin_does_not_mark_request_secure() { let mut headers = HeaderMap::new(); - headers.insert(header::HOST, HeaderValue::from_static("demo.serveo.net")); - headers.insert( + drop(headers.insert(header::HOST, HeaderValue::from_static("demo.serveo.net"))); + drop(headers.insert( header::ORIGIN, HeaderValue::from_static("https://evil.example"), - ); + )); assert!(!request_origin_uses_https(&headers)); } #[test] fn secure_cookie_decision_ignores_spoofed_https_origin_on_plain_http() { let mut headers = HeaderMap::new(); - headers.insert(header::HOST, HeaderValue::from_static("localhost:8080")); - headers.insert( + drop(headers.insert(header::HOST, HeaderValue::from_static("localhost:8080"))); + drop(headers.insert( header::ORIGIN, HeaderValue::from_static("https://localhost:8080"), - ); + )); let context = SecureCookieContext::new( Some(std::net::SocketAddr::from(([127, 0, 0, 1], 41_000))), false, @@ -3203,10 +3236,10 @@ mod tests { { let mut headers = HeaderMap::new(); let host = format!("example.test:{}", crate::config::CONFIG.tls.port); - headers.insert( + drop(headers.insert( header::HOST, HeaderValue::from_str(&host).context("build HTTPS-port host header")?, - ); + )); let context = SecureCookieContext::new( Some(std::net::SocketAddr::from(([127, 0, 0, 1], 41_000))), false, @@ -3221,7 +3254,7 @@ mod tests { #[test] fn secure_cookie_decision_keeps_onion_plain_http_cookie_insecure() { let mut headers = same_origin_headers(TEST_ONION_HOST); - headers.insert(header::ORIGIN, HeaderValue::from_static(TEST_ONION_ORIGIN)); + drop(headers.insert(header::ORIGIN, HeaderValue::from_static(TEST_ONION_ORIGIN))); let context = SecureCookieContext::new( Some(std::net::SocketAddr::from(([127, 0, 0, 1], 41_000))), false, @@ -3235,10 +3268,10 @@ mod tests { #[test] fn secure_cookie_decision_marks_direct_onion_https_cookie_secure() { let mut headers = same_origin_headers(TEST_ONION_HOST); - headers.insert( + drop(headers.insert( header::ORIGIN, HeaderValue::from_static(TEST_ONION_HTTPS_ORIGIN), - ); + )); let context = SecureCookieContext::new( Some(std::net::SocketAddr::from(([127, 0, 0, 1], 41_000))), true, @@ -3252,8 +3285,8 @@ mod tests { #[test] fn secure_cookie_decision_requires_trusted_proxy_for_forwarded_proto() { let mut headers = HeaderMap::new(); - headers.insert(header::HOST, HeaderValue::from_static("localhost:8080")); - headers.insert("x-forwarded-proto", HeaderValue::from_static("https")); + drop(headers.insert(header::HOST, HeaderValue::from_static("localhost:8080"))); + drop(headers.insert("x-forwarded-proto", HeaderValue::from_static("https"))); let trusted = SecureCookieContext::new( Some(std::net::SocketAddr::from(([127, 0, 0, 1], 41_000))), false, @@ -3374,20 +3407,12 @@ mod tests { &conn, "session123", admin_id, - chrono::Utc::now().timestamp() + 3600, + chrono::Utc::now().timestamp().saturating_add(3600), ) .context("create admin session")?; Ok(()) } - fn admin_signed_csrf() -> String { - crate::utils::crypto::make_scoped_csrf_form_token( - "csrf123", - &crate::config::CONFIG.cookie_secret, - "session123", - ) - } - #[tokio::test] async fn live_log_requires_admin_auth() -> anyhow::Result<()> { let state = crate::test_support::app_state(); @@ -3430,7 +3455,8 @@ mod tests { "INSERT INTO threads (board_id, subject) VALUES (?1, 'job thread')", rusqlite::params![board_id], ) - .context("insert job thread")?; + .context("insert job thread") + .map(|_completed_value| ())?; let thread_id = conn.last_insert_rowid(); conn.execute( "INSERT INTO posts @@ -3438,7 +3464,8 @@ mod tests { VALUES (?1, ?2, 'job body', 'job body', 'delete-token', 1)", rusqlite::params![thread_id, board_id], ) - .context("insert job post")?; + .context("insert job post") + .map(|_completed_value| ())?; let post_id = conn.last_insert_rowid(); expected_post_id = post_id; expected_post_url = format!("/test/thread/{thread_id}#p{post_id}"); @@ -3447,7 +3474,7 @@ mod tests { "d": { "post_id": post_id, "ip_hash": "hash", - "body_len": 8 + "body_len": 8_i32 } }) .to_string(); @@ -3462,7 +3489,8 @@ mod tests { "failed reading /Users/example/private.txt with token=abc123 ".repeat(8) ], ) - .context("insert background jobs")?; + .context("insert background jobs") + .map(|_completed_value| ())?; } let response = admin_site_health_jobs( State(state), @@ -3507,7 +3535,7 @@ mod tests { .and_then(|jobs| jobs.first()) .context("site-health payload omitted failed job detail")?; ensure!(failed_job["name"] == "Spam check"); - ensure!(failed_job["attempts"] == 3); + ensure!(failed_job["attempts"] == 3_i32); ensure!(failed_job["post_id"] == expected_post_id); ensure!(failed_job["post_url"] == expected_post_url); let error = failed_job["error"] @@ -3534,7 +3562,8 @@ mod tests { ('video_transcode', '{}', 'failed', 3, 'ffmpeg failed', unixepoch())", [], ) - .context("insert failed job")?; + .context("insert failed job") + .map(|_completed_value| ())?; ensure!( crate::db::background_job_summary(&conn) .context("load summary before dismiss")? @@ -3543,7 +3572,7 @@ mod tests { ); } let mut headers = same_origin_headers("localhost"); - headers.insert(header::ORIGIN, HeaderValue::from_static("http://localhost")); + drop(headers.insert(header::ORIGIN, HeaderValue::from_static("http://localhost"))); let response = dismiss_failed_site_health_jobs( State(state.clone()), CookieJar::new() diff --git a/src/handlers/admin/moderation.rs b/src/handlers/admin/moderation.rs index 9dba6542..e61f7533 100644 --- a/src/handlers/admin/moderation.rs +++ b/src/handlers/admin/moderation.rs @@ -40,7 +40,11 @@ pub(in crate::server) async fn add_ban( .filter(|&h| h > 0) // Cap at 87_600 hours (10 years) to prevent overflow in h * 3600. // Permanent bans are represented by None (duration_hours absent or zero). - .map(|h| Utc::now().timestamp() + h.min(87_600).saturating_mul(3600)); + .map(|h| { + Utc::now() + .timestamp() + .saturating_add(h.min(87_600).saturating_mul(3600)) + }); let ip_hash_log = form.ip_hash.chars().take(8).collect::(); @@ -50,7 +54,8 @@ pub(in crate::server) async fn add_ban( let conn = pool.get()?; let (admin_id, admin_name) = super::require_admin_session_with_name(&conn, session_id.as_deref())?; - db::add_ban(&conn, &form.ip_hash, &form.reason, expires_at)?; + db::add_ban(&conn, &form.ip_hash, &form.reason, expires_at) + .map(|_completed_value| ())?; if let Err(error) = db::log_mod_action( &conn, admin_id, @@ -101,7 +106,8 @@ pub(in crate::server) async fn remove_ban( let pool = state.db.clone(); move || -> Result<()> { let conn = pool.get()?; - super::require_admin_session_sid(&conn, session_id.as_deref())?; + super::require_admin_session_sid(&conn, session_id.as_deref()) + .map(|_completed_value| ())?; db::remove_ban(&conn, form.ban_id)?; Ok(()) } @@ -155,10 +161,11 @@ pub(in crate::server) async fn admin_ban_and_delete( .filter(|r| !r.is_empty()) .unwrap_or_else(|| "Rule violation".to_owned()); - let expires_at = form - .duration_hours - .filter(|&h| h > 0) - .map(|h| Utc::now().timestamp() + h.min(87_600).saturating_mul(3600)); + let expires_at = form.duration_hours.filter(|&h| h > 0).map(|h| { + Utc::now() + .timestamp() + .saturating_add(h.min(87_600).saturating_mul(3600)) + }); let ip_hash_log = form.ip_hash.chars().take(8).collect::(); let post_id = form.post_id; @@ -187,7 +194,7 @@ pub(in crate::server) async fn admin_ban_and_delete( } // Ban first so the IP cannot re-post before the delete lands - db::add_ban(&conn, &form.ip_hash, &reason, expires_at)?; + db::add_ban(&conn, &form.ip_hash, &reason, expires_at).map(|_completed_value| ())?; if let Err(error) = db::log_mod_action( &conn, admin_id, @@ -326,7 +333,8 @@ pub(in crate::server) async fn dismiss_appeal( let pool = state.db.clone(); move || -> Result<()> { let conn = pool.get()?; - super::require_admin_session_sid(&conn, session_id.as_deref())?; + super::require_admin_session_sid(&conn, session_id.as_deref()) + .map(|_completed_value| ())?; db::dismiss_ban_appeal(&conn, form.appeal_id)?; Ok(()) } @@ -429,8 +437,9 @@ pub(in crate::server) async fn add_filter( let pool = state.db.clone(); move || -> Result<()> { let conn = pool.get()?; - super::require_admin_session_sid(&conn, session_id.as_deref())?; - db::add_word_filter(&conn, &pattern, &replacement)?; + super::require_admin_session_sid(&conn, session_id.as_deref()) + .map(|_completed_value| ())?; + db::add_word_filter(&conn, &pattern, &replacement).map(|_completed_value| ())?; Ok(()) } }) @@ -462,7 +471,8 @@ pub(in crate::server) async fn remove_filter( let pool = state.db.clone(); move || -> Result<()> { let conn = pool.get()?; - super::require_admin_session_sid(&conn, session_id.as_deref())?; + super::require_admin_session_sid(&conn, session_id.as_deref()) + .map(|_completed_value| ())?; db::remove_word_filter(&conn, form.filter_id)?; Ok(()) } @@ -520,7 +530,8 @@ pub(in crate::server) async fn admin_ip_history( move || -> Result { const PER_PAGE: i64 = 25; let conn = pool.get()?; - super::require_admin_session_sid(&conn, session_id.as_deref())?; + super::require_admin_session_sid(&conn, session_id.as_deref()) + .map(|_completed_value| ())?; let total = db::count_posts_by_ip_hash(&conn, &ip_hash)?; let pagination = crate::models::Pagination::new(page, PER_PAGE, total); @@ -724,17 +735,9 @@ pub(in crate::server) async fn admin_ip_report( mod tests { use super::super::{admin_panel_redirect_anchor_open, SESSION_COOKIE}; use super::*; + use crate::test_support::admin_signed_csrf; use anyhow::{ensure, Context as _}; - use axum::extract::State; - use axum_extra::extract::cookie::{Cookie, CookieJar}; - - fn admin_signed_csrf() -> String { - crate::utils::crypto::make_scoped_csrf_form_token( - "csrf123", - &crate::config::CONFIG.cookie_secret, - "session123", - ) - } + use axum_extra::extract::cookie::Cookie; fn build_admin_jar() -> CookieJar { CookieJar::new() @@ -744,14 +747,14 @@ mod tests { fn admin_headers() -> axum::http::HeaderMap { let mut headers = axum::http::HeaderMap::new(); - headers.insert( + drop(headers.insert( axum::http::header::HOST, axum::http::HeaderValue::from_static("localhost"), - ); - headers.insert( + )); + drop(headers.insert( axum::http::header::ORIGIN, axum::http::HeaderValue::from_static("http://localhost"), - ); + )); headers } @@ -830,7 +833,9 @@ mod tests { db::create_admin(&conn, "admin", &password_hash).context("create test admin")?; db::create_session(&conn, "session123", admin_id, Utc::now().timestamp() + 3600) .context("create test admin session")?; - db::create_board(&conn, "test", "Test", "", false).context("create test board")?; + db::create_board(&conn, "test", "Test", "", false) + .context("create test board") + .map(|_completed_value| ())?; let board = db::get_board_by_short(&conn, "test") .context("load test board")? .context("test board was not persisted")?; @@ -875,8 +880,8 @@ mod tests { "administrator report redirect did not target the reports anchor" ); - let conn = state.db.get().context("get verification connection")?; - let (reporter_hash, reason): (String, String) = conn + let verification_conn = state.db.get().context("get verification connection")?; + let (reporter_hash, reason): (String, String) = verification_conn .query_row( "SELECT reporter_hash, reason FROM reports WHERE post_id = ?1", rusqlite::params![reply_id], @@ -936,7 +941,8 @@ pub(in crate::server) async fn mod_log_page( move || -> Result { const PER_PAGE: i64 = 50; let conn = pool.get()?; - super::require_admin_session_sid(&conn, session_id.as_deref())?; + super::require_admin_session_sid(&conn, session_id.as_deref()) + .map(|_completed_value| ())?; let total = db::count_mod_log(&conn)?; let pagination = crate::models::Pagination::new(page, PER_PAGE, total); diff --git a/src/handlers/admin/restart.rs b/src/handlers/admin/restart.rs new file mode 100644 index 00000000..0643f4fc --- /dev/null +++ b/src/handlers/admin/restart.rs @@ -0,0 +1,309 @@ +//! Administrator-only POST restarts with closed forms and existing CSRF/origin protections. + +use super::{require_admin_post_origin_and_csrf, updates}; +use crate::{ + error::{AppError, Result}, + middleware::AppState, + restart::View, +}; +use axum::{ + extract::{ConnectInfo, Form, State}, + http::HeaderMap, + response::{IntoResponse as _, Response}, + Json, +}; +use axum_extra::extract::cookie::CookieJar; + +/// CSRF is the only accepted form parameter; service/command/path parameters are rejected. +#[derive(Debug, serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub(in crate::server) struct RestartForm { + /// Session-scoped signed CSRF token. + #[serde(rename = "_csrf")] + csrf: Option, +} + +/// Derive pending state from actual saved/active settings, after authorization. +pub(super) async fn snapshot(native_status: Option<&crate::updates::Status>) -> View { + let managed = crate::updates::managed(); + let local = if crate::restart::container_restart_enabled() { + tokio::task::spawn_blocking(crate::restart::container_status) + .await + .ok() + .and_then(std::result::Result::ok) + } else { + None + }; + let status = if managed { + native_status + } else { + local.as_ref() + }; + let pending = tokio::task::spawn_blocking(crate::config::admin::restart_pending) + .await + .ok() + .and_then(std::result::Result::ok); + let in_progress = status.is_some_and(|s| s.phase.active()); + let supported = (managed || crate::restart::container_restart_enabled()) && pending.is_some(); + let message = if pending.is_none() { + "Cannot safely read configuration state. Restart is disabled." + } else if let Some(status) = status + .filter(|s| s.operation == crate::updates::Operation::SettingsRestart || s.phase.active()) + { + &status.message + } else if pending == Some(true) { + "Settings saved. Restart RustChan to apply these changes." + } else { + "Saved startup settings are active. No restart is required." + }; + View { + pending: pending.unwrap_or(true) || in_progress, + supported, + in_progress, + message: if !supported && pending == Some(true) { + format!("{message} This standalone deployment has no configured restart supervisor. Use the supplied managed systemd service or container restart policy.") + } else { + message.to_owned() + }, + instance: *crate::restart::INSTANCE, + } +} + +/// Administrator-only bounded progress endpoint for reconnect polling. +pub(in crate::server) async fn status( + State(state): State, + jar: CookieJar, +) -> Result { + updates::authorize(&state, &jar) + .await + .map(|_completed_value| ())?; + let status = updates::snapshot().await; + Ok(Json(snapshot(Some(&status)).await).into_response()) +} + +/// Request the single predefined restart operation; merely saving never reaches this route. +pub(in crate::server) async fn restart( + State(state): State, + jar: CookieJar, + headers: HeaderMap, + ConnectInfo(peer): ConnectInfo, + Form(form): Form, +) -> Result { + require_admin_post_origin_and_csrf(&jar, &headers, Some(peer), form.csrf.as_deref())?; + let administrator = updates::authorize(&state, &jar).await?; + let guard = state.maintenance_gate.try_begin("Settings restart")?; + let pending = tokio::task::spawn_blocking(crate::config::admin::restart_pending) + .await + .map_err(|error| AppError::Internal(error.into()))??; + if !pending { + return Err(AppError::BadRequest( + "No saved settings require a restart.".into(), + )); + } + if crate::updates::managed() { + let result = crate::updates::request(&crate::updates::Request::Restart { + instance: *crate::restart::INSTANCE, + administrator, + }) + .await; + let acknowledged_rejection = result.as_ref().is_ok_and(|reply| reply.error.is_some()); + if !acknowledged_rejection { + let cancel = state.job_queue.cancel.clone(); + drop(tokio::spawn(async move { + let _guard = guard; + loop { + if cancel.is_cancelled() { + break; + } + if crate::updates::request(&crate::updates::Request::Status) + .await + .is_ok_and(|reply| { + reply.error.is_none() && reply.ready && !reply.status.phase.active() + }) + { + break; + } + tokio::time::sleep(std::time::Duration::from_secs(2)).await; + } + })); + } + match result { + Ok(reply) if reply.error.is_none() => {}, + Ok(_) => return Ok(outcome(&headers, Some(false), "Restart rejected. Another operation may be running, or configuration/service preflight failed. Review the saved state and retry.")), + Err(_) => return Ok(outcome(&headers, None, "Restart acknowledgment timed out. Check restart status before retrying; the supervisor may have accepted it.")), + } + } else if crate::restart::container_restart_enabled() { + tokio::task::spawn_blocking(move || crate::restart::request_container(administrator)) + .await + .map_err(|error| AppError::Internal(error.into()))??; + let cancel = state.job_queue.cancel.clone(); + drop(tokio::spawn(async move { + let _guard = guard; + // Permit the 303 response to reach the browser before closing listeners. + tokio::time::sleep(std::time::Duration::from_secs(1)).await; + cancel.cancel(); + cancel.cancelled().await; + tokio::time::sleep(crate::restart::SHUTDOWN_TIMEOUT).await; + })); + } else { + return Err(AppError::BadRequest( + "This deployment has no configured restart supervisor.".into(), + )); + } + Ok(outcome(&headers, Some(true), "Restart accepted. RustChan is reconnecting; this page will show the verified result when it returns.")) +} + +/// AJAX receives the acknowledgment before shutdown; ordinary forms retain POST/redirect/GET. +fn outcome(headers: &HeaderMap, accepted: Option, message: &str) -> Response { + if headers + .get(axum::http::header::ACCEPT) + .and_then(|value| value.to_str().ok()) + .is_some_and(|value| { + value + .split(',') + .any(|part| part.trim() == "application/json") + }) + { + return ( + if accepted == Some(false) { + axum::http::StatusCode::CONFLICT + } else { + axum::http::StatusCode::ACCEPTED + }, + Json(serde_json::json!({ "accepted": accepted, "message": message })), + ) + .into_response(); + } + if accepted == Some(true) { + super::admin_panel_redirect_anchor(message, "settings-restart").into_response() + } else { + super::admin_panel_error_redirect_anchor(message, "settings-restart").into_response() + } +} + +#[cfg(test)] +/// Real route extraction/authentication/CSRF tests with no active process supervisor. +mod tests { + use super::*; + use axum::{ + body::Body, + http::{Request, StatusCode}, + routing::{get, post}, + Router, + }; + use tower::ServiceExt as _; + + /// Enhanced requests receive a bounded acknowledgment without following a redirect into downtime. + #[tokio::test] + async fn enhanced_acknowledgment_preserves_acceptance_and_rejection() -> anyhow::Result<()> { + let mut headers = HeaderMap::new(); + drop(headers.insert( + "accept", + axum::http::HeaderValue::from_static("application/json"), + )); + for (accepted, expected) in [ + (Some(true), StatusCode::ACCEPTED), + (None, StatusCode::ACCEPTED), + (Some(false), StatusCode::CONFLICT), + ] { + let response = outcome(&headers, accepted, "Restart state"); + anyhow::ensure!( + response.status() == expected && !response.headers().contains_key("location"), + "JSON acknowledgment must not redirect" + ); + let bytes = axum::body::to_bytes(response.into_body(), 4096).await?; + let body: serde_json::Value = serde_json::from_slice(&bytes)?; + anyhow::ensure!( + body.get("accepted") == Some(&serde_json::json!(accepted)), + "uncertain acceptance must remain explicit" + ); + } + let native = outcome(&HeaderMap::new(), Some(true), "Restart accepted"); + anyhow::ensure!( + native.status() == StatusCode::SEE_OTHER && native.headers().contains_key("location"), + "ordinary forms retain POST/redirect/GET" + ); + Ok(()) + } + + /// Exercise the production route handlers with isolated application/database fixtures. + #[tokio::test] + async fn restart_routes_reject_anonymous_users_methods_csrf_and_arbitrary_fields( + ) -> anyhow::Result<()> { + let state = crate::test_support::app_state(); + let app = Router::new() + .route("/admin/restart", post(restart)) + .route("/admin/restart/status", get(status)) + .with_state(state); + for (method, route, body, expected) in [ + ("GET", "/admin/restart", "", StatusCode::METHOD_NOT_ALLOWED), + ("GET", "/admin/restart/status", "", StatusCode::FORBIDDEN), + ( + "POST", + "/admin/restart", + "_csrf=invalid", + StatusCode::FORBIDDEN, + ), + ( + "POST", + "/admin/restart", + "command=systemctl&service=other", + StatusCode::UNPROCESSABLE_ENTITY, + ), + ] { + let request = Request::builder() + .method(method) + .uri(route) + .header("host", "localhost") + .header("origin", "http://localhost") + .header("content-type", "application/x-www-form-urlencoded") + .extension(crate::test_support::connect_info()) + .body(Body::from(body))?; + let response = app.clone().oneshot(request).await?; + anyhow::ensure!( + response.status() == expected, + "{method} {route} must reject with {expected}" + ); + } + Ok(()) + } + /// A valid signature is insufficient without a full administrator session. + #[tokio::test] + async fn signed_csrf_does_not_authorize_a_non_admin_restart() -> anyhow::Result<()> { + let state = crate::test_support::app_state(); + let session = "not-an-admin-session"; + let token = crate::utils::crypto::make_scoped_csrf_form_token( + "raw", + &crate::config::CONFIG.cookie_secret, + session, + ); + let jar = CookieJar::new() + .add(axum_extra::extract::cookie::Cookie::new( + "csrf_token", + "raw", + )) + .add(axum_extra::extract::cookie::Cookie::new( + super::super::SESSION_COOKIE, + session, + )); + let mut headers = HeaderMap::new(); + drop(headers.insert("host", axum::http::HeaderValue::from_static("localhost"))); + drop(headers.insert( + "origin", + axum::http::HeaderValue::from_static("http://localhost"), + )); + let result = restart( + State(state), + jar, + headers, + crate::test_support::connect_info(), + Form(RestartForm { csrf: Some(token) }), + ) + .await; + anyhow::ensure!( + matches!(result, Err(AppError::Forbidden(_))), + "valid CSRF cannot substitute for admin authorization" + ); + Ok(()) + } +} diff --git a/src/handlers/admin/settings.rs b/src/handlers/admin/settings.rs index 0772a10e..f5c5b931 100644 --- a/src/handlers/admin/settings.rs +++ b/src/handlers/admin/settings.rs @@ -134,7 +134,7 @@ pub(in crate::server) async fn update_visitor_defaults( let session = jar.get(SESSION_COOKIE).map(|c| c.value().to_owned()); tokio::task::spawn_blocking(move || -> Result<()> { let conn = state.db.get()?; - require_admin_session_sid(&conn, session.as_deref())?; + require_admin_session_sid(&conn, session.as_deref()).map(|_completed_value| ())?; db::set_site_setting( &conn, "default_hide_nsfw_boards", diff --git a/src/handlers/admin/settings/appearance.rs b/src/handlers/admin/settings/appearance.rs index f68597b0..6506ac35 100644 --- a/src/handlers/admin/settings/appearance.rs +++ b/src/handlers/admin/settings/appearance.rs @@ -30,7 +30,7 @@ pub(in crate::server) async fn clear_board_favicon_override( let pool = state.db.clone(); move || -> Result { let conn = pool.get()?; - require_admin_session_sid(&conn, session_id.as_deref())?; + require_admin_session_sid(&conn, session_id.as_deref()).map(|_completed_value| ())?; let board_short: String = conn.query_row( "SELECT short_name FROM boards WHERE id = ?1", rusqlite::params![form.board_id], @@ -92,7 +92,7 @@ pub(in crate::server) async fn update_site_favicon( let pool = state.db.clone(); move || -> Result<()> { let conn = pool.get()?; - require_admin_session_sid(&conn, session_id.as_deref())?; + require_admin_session_sid(&conn, session_id.as_deref()).map(|_completed_value| ())?; crate::favicon::write_favicon_set( crate::favicon::FaviconScope::Global, &favicon_bytes, @@ -162,7 +162,7 @@ pub(in crate::server) async fn update_board_favicon( let pool = state.db.clone(); move || -> Result { let conn = pool.get()?; - require_admin_session_sid(&conn, session_id.as_deref())?; + require_admin_session_sid(&conn, session_id.as_deref()).map(|_completed_value| ())?; let board_short: String = conn.query_row( "SELECT short_name FROM boards WHERE id = ?1", rusqlite::params![board_id], diff --git a/src/handlers/admin/settings/backup_settings.rs b/src/handlers/admin/settings/backup_settings.rs index c044c9eb..e0d03030 100644 --- a/src/handlers/admin/settings/backup_settings.rs +++ b/src/handlers/admin/settings/backup_settings.rs @@ -52,18 +52,19 @@ fn parse_full_backup_settings_form( let storage_mode_value = match storage_mode { crate::handlers::admin::BackupStorageMode::Directory => "directory", crate::handlers::admin::BackupStorageMode::SplitZip => "split_zip", - _ => { + crate::handlers::admin::BackupStorageMode::SingleZip + | crate::handlers::admin::BackupStorageMode::LegacyZip => { return Err(AppError::BadRequest( "Unsupported automatic backup storage mode.".into(), )); } }; - let split_zip_part_size_gib = form + let requested_part_size_gib = form .auto_full_backup_split_zip_part_size_gib .as_deref() .and_then(|value| value.parse::().ok()); let split_zip_part_size = - crate::handlers::admin::backup::parse_split_zip_part_size_gib(split_zip_part_size_gib)?; + crate::handlers::admin::backup::parse_split_zip_part_size_gib(requested_part_size_gib)?; let split_zip_part_size_gib = crate::handlers::admin::backup::split_zip_part_size_gib(split_zip_part_size); @@ -90,7 +91,7 @@ pub(in crate::server) async fn update_full_backup_settings( if let Some(directory) = form.backup_directory { tokio::task::spawn_blocking(move || -> Result<()> { let conn = state.db.get()?; - require_admin_session_sid(&conn, session_id.as_deref())?; + require_admin_session_sid(&conn, session_id.as_deref()).map(|_completed_value| ())?; crate::config::update_settings_file_backup_directory(std::path::Path::new(&directory)) .map_err(|error| AppError::BadRequest(format!("{error:#}")))?; Ok(()) @@ -98,7 +99,7 @@ pub(in crate::server) async fn update_full_backup_settings( .await .map_err(|error| AppError::Internal(anyhow::anyhow!(error)))??; return Ok(admin_panel_redirect_anchor( - "Backup directory saved. Restart RustChan to apply it. Existing backups have not been moved. CHAN_BACKUP_DIRECTORY, if set, takes precedence.", + if crate::config::admin::restart_pending().unwrap_or(true) { "Backup directory saved. Restart RustChan to apply it. Existing backups have not been moved; environment overrides take precedence." } else { "Backup directory saved. The effective configuration does not require a restart." }, "full-backup-restore", ).into_response()); } @@ -116,7 +117,7 @@ pub(in crate::server) async fn update_full_backup_settings( let auto_backup_settings = state.auto_full_backup_settings.clone(); move || -> Result<()> { let conn = pool.get()?; - require_admin_session_sid(&conn, session_id.as_deref())?; + require_admin_session_sid(&conn, session_id.as_deref()).map(|_completed_value| ())?; crate::config::update_settings_file_auto_full_backup( interval_hours, copies_to_keep, diff --git a/src/handlers/admin/settings/banners.rs b/src/handlers/admin/settings/banners.rs index b3e9e754..86ccc6ec 100644 --- a/src/handlers/admin/settings/banners.rs +++ b/src/handlers/admin/settings/banners.rs @@ -174,7 +174,8 @@ fn restore_board_banner_inheritance_if_empty( conn.execute( "UPDATE boards SET banner_mode = 'inherit' WHERE id = ?1 AND banner_mode = 'override'", rusqlite::params![board_id], - )?; + ) + .map(|_completed_value| ())?; } Ok(()) } @@ -211,7 +212,7 @@ fn delete_banner_asset_safely( let tx = conn.unchecked_transaction()?; let asset = db::get_banner_asset(&tx, banner_id)? .ok_or_else(|| AppError::BadRequest("Banner not found.".into()))?; - db::delete_banner_asset(&tx, banner_id)?; + db::delete_banner_asset(&tx, banner_id).map(|_completed_value| ())?; if asset.scope == BannerScope::Board { restore_board_banner_inheritance_if_empty(&tx, asset.board_id)?; } @@ -240,11 +241,12 @@ fn clear_board_banner_assets_safely( let tx = conn.unchecked_transaction()?; let assets = db::list_banner_assets_for_board(&tx, board_id)?; let pending_op = banner_cleanup_payload(&assets)?; - db::delete_board_banner_assets(&tx, board_id)?; + db::delete_board_banner_assets(&tx, board_id).map(|_completed_value| ())?; tx.execute( "UPDATE boards SET banner_mode = 'inherit' WHERE id = ?1 AND banner_mode = 'override'", rusqlite::params![board_id], - )?; + ) + .map(|_completed_value| ())?; if let Some(op) = pending_op.as_ref() { db::insert_pending_fs_op(&tx, op)?; } @@ -270,7 +272,7 @@ async fn upload_banner_for_scope( ) -> Result { tokio::task::spawn_blocking(move || -> Result { let mut conn = state.db.get()?; - require_admin_session_sid(&conn, session_id.as_deref())?; + require_admin_session_sid(&conn, session_id.as_deref()).map(|_completed_value| ())?; let (target_type, target_value) = resolve_banner_target_selection( &parsed.target_type, parsed.target_value.as_deref(), @@ -508,7 +510,7 @@ pub(in crate::server) async fn update_banner_meta( let pool = state.db.clone(); move || -> Result { let conn = pool.get()?; - require_admin_session_sid(&conn, session_id.as_deref())?; + require_admin_session_sid(&conn, session_id.as_deref()).map(|_completed_value| ())?; let asset = db::get_banner_asset(&conn, form.banner_id)? .ok_or_else(|| AppError::BadRequest("Banner not found.".into()))?; let (target_type, target_value) = resolve_banner_target_selection( @@ -576,7 +578,7 @@ pub(in crate::server) async fn delete_banner( let pool = state.db.clone(); move || -> Result { let conn = pool.get()?; - require_admin_session_sid(&conn, session_id.as_deref())?; + require_admin_session_sid(&conn, session_id.as_deref()).map(|_completed_value| ())?; let asset = delete_banner_asset_safely(&conn, form.banner_id)?; Ok(banner::banner_admin_anchor( asset.scope, @@ -618,7 +620,7 @@ pub(in crate::server) async fn move_banner( let pool = state.db.clone(); move || -> Result { let mut conn = pool.get()?; - require_admin_session_sid(&conn, session_id.as_deref())?; + require_admin_session_sid(&conn, session_id.as_deref()).map(|_completed_value| ())?; let asset = db::get_banner_asset(&conn, form.banner_id)? .ok_or_else(|| AppError::BadRequest("Banner not found.".into()))?; db::move_banner_asset(&mut conn, form.banner_id, move_up)?; @@ -653,7 +655,7 @@ pub(in crate::server) async fn clear_board_banner_override( let pool = state.db.clone(); move || -> Result { let conn = pool.get()?; - require_admin_session_sid(&conn, session_id.as_deref())?; + require_admin_session_sid(&conn, session_id.as_deref()).map(|_completed_value| ())?; let (board_short, _assets) = clear_board_banner_assets_safely(&conn, form.board_id)?; Ok(board_short) } @@ -695,7 +697,8 @@ mod tests { "UPDATE boards SET banner_mode = 'override' WHERE id = ?1", rusqlite::params![board_id], ) - .context("set override mode")?; + .context("set override mode") + .map(|_completed_value| ())?; restore_board_banner_inheritance_if_empty(&conn, Some(board_id))?; @@ -713,7 +716,8 @@ mod tests { "UPDATE boards SET banner_mode = 'override' WHERE id = ?1", rusqlite::params![board_id], ) - .context("set override mode")?; + .context("set override mode") + .map(|_completed_value| ())?; crate::db::insert_banner_asset( &conn, crate::models::BannerScope::Board, @@ -729,7 +733,8 @@ mod tests { true, true, ) - .context("insert board banner")?; + .context("insert board banner") + .map(|_completed_value| ())?; restore_board_banner_inheritance_if_empty(&conn, Some(board_id))?; diff --git a/src/handlers/admin/settings/board.rs b/src/handlers/admin/settings/board.rs index 5db57a9c..520d2684 100644 --- a/src/handlers/admin/settings/board.rs +++ b/src/handlers/admin/settings/board.rs @@ -164,7 +164,7 @@ pub(in crate::server) async fn update_board_settings( let pool = state.db.clone(); move || -> Result { let mut conn = pool.get()?; - require_admin_session_sid(&conn, session_id.as_deref())?; + require_admin_session_sid(&conn, session_id.as_deref()).map(|_completed_value| ())?; let board_short: String = conn.query_row( "SELECT short_name FROM boards WHERE id = ?1", rusqlite::params![board_id], diff --git a/src/handlers/admin/settings/maintenance.rs b/src/handlers/admin/settings/maintenance.rs index ddb7e511..80970568 100644 --- a/src/handlers/admin/settings/maintenance.rs +++ b/src/handlers/admin/settings/maintenance.rs @@ -62,6 +62,69 @@ fn create_pre_repair_backup( ) } +/// Run repair under the caller's maintenance guard, retaining stale-job diagnostics. +fn run_db_repair_job( + pool: &db::DbPool, + progress: &std::sync::Arc, + jobs: &crate::middleware::DbMaintenanceJobs, + job_id: u64, +) { + if !jobs.mark_phase(job_id, crate::middleware::DbMaintenanceJobPhase::Backup) { + tracing::debug!(target: "admin", job_id, "Ignored backup phase update for stale database repair job"); + } + let backup_result = create_pre_repair_backup(pool, progress, job_id); + + let conn = match pool.get() { + Ok(conn) => conn, + Err(error) => { + if !jobs.mark_failed( + job_id, + format!("Could not open database connection after pre-repair backup: {error}"), + ) { + tracing::debug!(target: "admin", job_id, %error, "Ignored failure update for stale database repair job"); + } + return; + } + }; + + if !jobs.mark_phase(job_id, crate::middleware::DbMaintenanceJobPhase::Repair) { + tracing::debug!(target: "admin", job_id, "Ignored repair phase update for stale database repair job"); + } + let report = match backup_result { + Ok(backup_id) => db::attempt_db_repair( + &conn, + Some(db::DbRepairBackup { + backup_id: backup_id.clone(), + backup_type: "DB + config".to_owned(), + backup_path: crate::config::backups_dir() + .join(&backup_id) + .display() + .to_string(), + verified: true, + }), + ), + Err(error) => { + let backup_error = error.to_string(); + db::db_repair_aborted_for_backup_failure(&conn, &backup_error) + } + }; + tracing::info!( + target: "admin", + before_ok = report.before.ok(), + after_ok = report.after.as_ref().map(db::DbHealthSnapshot::ok), + steps = report.repair_steps.len(), + backup = report + .repair_backup + .as_ref() + .map(|backup| backup.backup_id.as_str()), + backup_error = report.repair_backup_error.as_deref(), + "Admin ran database repair attempt" + ); + if !jobs.mark_finished(job_id, report) { + tracing::debug!(target: "admin", job_id, "Ignored completion update for stale database repair job"); + } +} + fn parse_ffmpeg_timeout_secs_input(input: Option<&str>) -> Result { let raw = input .map(str::trim) @@ -155,7 +218,7 @@ pub(in crate::server) async fn update_media_settings( let pool = state.db.clone(); move || -> Result<()> { let mut conn = pool.get()?; - require_admin_session_sid(&conn, session_id.as_deref())?; + require_admin_session_sid(&conn, session_id.as_deref()).map(|_completed_value| ())?; let tx = conn.transaction().map_err(anyhow::Error::from)?; db::set_media_prune_settings(&tx, prune_enabled, prune_max_bytes)?; db::set_site_setting(&tx, "ffmpeg_timeout_secs", &timeout_secs.to_string())?; @@ -207,7 +270,7 @@ pub(in crate::server) async fn admin_vacuum( let csrf_clone = csrf.clone(); move || -> Result { let conn = pool.get()?; - require_admin_session_sid(&conn, session_id.as_deref())?; + require_admin_session_sid(&conn, session_id.as_deref()).map(|_completed_value| ())?; let size_before = db::get_db_size_bytes(&conn).unwrap_or(0); @@ -259,7 +322,7 @@ pub(in crate::server) async fn admin_db_check( let csrf_clone = csrf.clone(); move || -> Result { let conn = pool.get()?; - require_admin_session_sid(&conn, session_id.as_deref())?; + require_admin_session_sid(&conn, session_id.as_deref()).map(|_completed_value| ())?; let report = db::check_db_health(&conn); tracing::info!( @@ -303,7 +366,7 @@ pub(in crate::server) async fn admin_db_repair( let pool = state.db.clone(); move || -> Result<()> { let conn = pool.get()?; - require_admin_session_sid(&conn, session_id.as_deref())?; + require_admin_session_sid(&conn, session_id.as_deref()).map(|_completed_value| ())?; Ok(()) } }) @@ -318,67 +381,22 @@ pub(in crate::server) async fn admin_db_repair( let pool = state.db.clone(); let db_maintenance_jobs = state.db_maintenance_jobs.clone(); - tokio::spawn(async move { + drop(tokio::spawn(async move { let job_status = db_maintenance_jobs.clone(); let join_result = tokio::task::spawn_blocking(move || { let _maintenance_guard = maintenance_guard; - let _ = db_maintenance_jobs - .mark_phase(job_id, crate::middleware::DbMaintenanceJobPhase::Backup); - let backup_result = create_pre_repair_backup(&pool, &progress, job_id); - - let conn = match pool.get() { - Ok(conn) => conn, - Err(error) => { - let _ = db_maintenance_jobs.mark_failed( - job_id, - format!( - "Could not open database connection after pre-repair backup: {error}" - ), - ); - return; - } - }; - - let _ = db_maintenance_jobs - .mark_phase(job_id, crate::middleware::DbMaintenanceJobPhase::Repair); - let report = match backup_result { - Ok(backup_id) => db::attempt_db_repair( - &conn, - Some(db::DbRepairBackup { - backup_id: backup_id.clone(), - backup_type: "DB + config".to_owned(), - backup_path: crate::config::backups_dir() - .join(&backup_id) - .display() - .to_string(), - verified: true, - }), - ), - Err(error) => { - let backup_error = error.to_string(); - db::db_repair_aborted_for_backup_failure(&conn, &backup_error) - } - }; - tracing::info!( - target: "admin", - before_ok = report.before.ok(), - after_ok = report.after.as_ref().map(db::DbHealthSnapshot::ok), - steps = report.repair_steps.len(), - backup = report - .repair_backup - .as_ref() - .map(|backup| backup.backup_id.as_str()), - backup_error = report.repair_backup_error.as_deref(), - "Admin ran database repair attempt" - ); - let _ = db_maintenance_jobs.mark_finished(job_id, report); + run_db_repair_job(&pool, &progress, &db_maintenance_jobs, job_id); }) .await; if let Err(error) = join_result { - let _ = job_status.mark_failed(job_id, format!("Database repair task failed: {error}")); + let failure_recorded = + job_status.mark_failed(job_id, format!("Database repair task failed: {error}")); + if !failure_recorded { + tracing::debug!(target: "admin", job_id, %error, "Ignored task failure update for stale database repair job"); + } } - }); + })); Ok(render_db_repair_entry_response( &jar, @@ -398,7 +416,7 @@ pub(in crate::server) async fn admin_db_repair_progress_json( let pool = state.db.clone(); move || -> Result<()> { let conn = pool.get()?; - require_admin_session_sid(&conn, session_id.as_deref())?; + require_admin_session_sid(&conn, session_id.as_deref()).map(|_completed_value| ())?; Ok(()) } }) @@ -454,7 +472,7 @@ fn db_repair_progress_payload( crate::middleware::DbMaintenanceJobStatus::Running { job_id, phase: crate::middleware::DbMaintenanceJobPhase::Starting, - .. + started_at: _, } => serde_json::json!({ "state": "running", "job_id": job_id, @@ -465,7 +483,7 @@ fn db_repair_progress_payload( crate::middleware::DbMaintenanceJobStatus::Running { job_id, phase: crate::middleware::DbMaintenanceJobPhase::Backup, - .. + started_at: _, } => { let backup_percent = backup_percent(backup_phase, backup_files_done, backup_files_total); @@ -481,7 +499,7 @@ fn db_repair_progress_payload( crate::middleware::DbMaintenanceJobStatus::Running { job_id, phase: crate::middleware::DbMaintenanceJobPhase::Repair, - .. + started_at: _, } => serde_json::json!({ "state": "running", "job_id": job_id, @@ -489,16 +507,20 @@ fn db_repair_progress_payload( "percent": 82, "done": false, }), - crate::middleware::DbMaintenanceJobStatus::Finished { job_id, .. } => serde_json::json!({ - "state": "finished", - "job_id": job_id, - "label": "Maintenance rebuild complete. Opening report...", - "percent": 100, - "done": true, - "redirect_url": db_repair_status_url(Some(job_id)), - }), + crate::middleware::DbMaintenanceJobStatus::Finished { job_id, report: _ } => { + serde_json::json!({ + "state": "finished", + "job_id": job_id, + "label": "Maintenance rebuild complete. Opening report...", + "percent": 100, + "done": true, + "redirect_url": db_repair_status_url(Some(job_id)), + }) + } crate::middleware::DbMaintenanceJobStatus::Failed { - job_id, message, .. + job_id, + message, + finished_at: _, } => serde_json::json!({ "state": "failed", "job_id": job_id, @@ -517,7 +539,7 @@ fn backup_percent(phase: u64, files_done: u64, files_total: u64) -> u64 { crate::middleware::backup_phase::COMPRESS => files_done .saturating_mul(30) .checked_div(files_total) - .map_or(55, |percent| 55 + percent), + .map_or(55, |percent| 55_u64.saturating_add(percent)), crate::middleware::backup_phase::DONE => 78, _ => 10, } @@ -564,7 +586,7 @@ pub(in crate::server) async fn admin_db_repair_status( let pool = state.db.clone(); move || -> Result<()> { let conn = pool.get()?; - require_admin_session_sid(&conn, session_id.as_deref())?; + require_admin_session_sid(&conn, session_id.as_deref()).map(|_completed_value| ())?; Ok(()) } }) @@ -605,7 +627,7 @@ fn render_db_repair_running_response( ) .into_response(); if let Ok(refresh) = HeaderValue::from_str(&refresh) { - response.headers_mut().insert(header::REFRESH, refresh); + let _previous_value = response.headers_mut().insert(header::REFRESH, refresh); } response } @@ -618,11 +640,20 @@ fn render_db_repair_entry_response( let current_theme = crate::handlers::board::current_theme_from_jar(jar); match status { crate::middleware::DbMaintenanceJobStatus::Running { - job_id, started_at, .. + job_id, + started_at, + phase: _, } => render_db_repair_running_response(jar, csrf, *job_id, *started_at), crate::middleware::DbMaintenanceJobStatus::Idle - | crate::middleware::DbMaintenanceJobStatus::Finished { .. } - | crate::middleware::DbMaintenanceJobStatus::Failed { .. } => ( + | crate::middleware::DbMaintenanceJobStatus::Finished { + job_id: _, + report: _, + } + | crate::middleware::DbMaintenanceJobStatus::Failed { + job_id: _, + finished_at: _, + message: _, + } => ( jar.clone(), Html(crate::templates::admin_db_repair_idle_page( csrf, @@ -665,7 +696,9 @@ fn render_db_repair_status_response( ) .into_response(), crate::middleware::DbMaintenanceJobStatus::Running { - job_id, started_at, .. + job_id, + started_at, + phase: _, } => render_db_repair_running_response(jar, csrf, *job_id, *started_at), crate::middleware::DbMaintenanceJobStatus::Finished { job_id, report } => ( jar.clone(), @@ -702,6 +735,7 @@ mod tests { admin_db_repair, admin_db_repair_status, admin_vacuum, parse_ffmpeg_timeout_secs_input, parse_media_prune_size_input, update_media_settings, PRE_REPAIR_BACKUP_FAILURE, }; + use crate::test_support::admin_signed_csrf; use anyhow::{bail, ensure, Context as _}; use axum::{ body::{to_bytes, Body}, @@ -725,20 +759,12 @@ mod tests { &conn, "session123", admin_id, - chrono::Utc::now().timestamp() + 3600, + chrono::Utc::now().timestamp().saturating_add(3600), ) .context("create admin session")?; Ok(()) } - fn admin_signed_csrf() -> String { - crate::utils::crypto::make_scoped_csrf_form_token( - "csrf123", - &crate::config::CONFIG.cookie_secret, - "session123", - ) - } - fn posts_ai_trigger_sql(state: &crate::middleware::AppState) -> anyhow::Result { let conn = state.db.get().context("get database connection")?; conn.query_row( @@ -789,7 +815,8 @@ mod tests { None, None, ) - .context("create repair-test thread")?; + .context("create repair-test thread") + .map(|_completed_value| ())?; conn.execute_batch(&format!( "PRAGMA foreign_keys=OFF; BEGIN; DELETE FROM boards WHERE short_name='{board_short}'; COMMIT; PRAGMA foreign_keys=ON;" @@ -856,7 +883,7 @@ mod tests { } async fn wait_for_repair_result(router: &Router) -> anyhow::Result { - for _ in 0..100 { + for _ in 0_i32..100_i32 { let body = repair_status_body(router).await?; if !body.contains("maintenance rebuild running") { return Ok(body); @@ -1002,9 +1029,9 @@ media_max_active_content_size_bytes = 0 } { - let mut failure = PRE_REPAIR_BACKUP_FAILURE - .lock() - .map_err(|_| anyhow::anyhow!("backup failure mutex was poisoned"))?; + let mut failure = PRE_REPAIR_BACKUP_FAILURE.lock().map_err(|poison_error| { + anyhow::anyhow!("backup failure mutex was poisoned: {poison_error}") + })?; *failure = Some("simulated pre-repair backup failure".to_owned()); } @@ -1051,9 +1078,9 @@ media_max_active_content_size_bytes = 0 let body = wait_for_repair_result(&router).await?; { - let mut failure = PRE_REPAIR_BACKUP_FAILURE - .lock() - .map_err(|_| anyhow::anyhow!("backup failure mutex was poisoned"))?; + let mut failure = PRE_REPAIR_BACKUP_FAILURE.lock().map_err(|poison_error| { + anyhow::anyhow!("backup failure mutex was poisoned: {poison_error}") + })?; *failure = None; } diff --git a/src/handlers/admin/settings/network.rs b/src/handlers/admin/settings/network.rs index ad0453a9..b452011a 100644 --- a/src/handlers/admin/settings/network.rs +++ b/src/handlers/admin/settings/network.rs @@ -27,12 +27,12 @@ pub(in crate::server) async fn update_network_settings( .map(|cookie| cookie.value().to_owned()); tokio::task::spawn_blocking(move || -> Result { let conn = state.db.get()?; - require_admin_session_sid(&conn, session_id.as_deref())?; + require_admin_session_sid(&conn, session_id.as_deref()).map(|_completed_value| ())?; let save_result = crate::config::admin::save_network(&form); match save_result { - Ok(()) => Ok(admin_panel_redirect_anchor("Network settings saved. Restart required; environment and launcher overrides still take precedence.", "network-security").into_response()), + Ok(()) => Ok(admin_panel_redirect_anchor(if crate::config::admin::restart_pending().unwrap_or(true) { "Settings saved. Restart required: use Restart RustChan to apply these changes." } else { "Settings saved. No restart is required for the effective configuration." }, "network-security").into_response()), Err(error) => { - let mut fields = crate::config::admin::network_snapshot().map_err(|_| "unavailable".to_owned()); + let mut fields = crate::config::admin::network_snapshot().map_err(|snapshot_error| { tracing::warn!(error = %snapshot_error, "operator settings snapshot unavailable"); "unavailable".to_owned() }); if let Ok(fields) = &mut fields { for field in fields { if let Some(value) = form.get(field.definition.key) { field.input.clone_from(value); } diff --git a/src/handlers/admin/settings/runtime.rs b/src/handlers/admin/settings/runtime.rs index 95ca2e99..fe0b958b 100644 --- a/src/handlers/admin/settings/runtime.rs +++ b/src/handlers/admin/settings/runtime.rs @@ -29,12 +29,12 @@ pub(in crate::server) async fn update_runtime_settings( .map(|cookie| cookie.value().to_owned()); tokio::task::spawn_blocking(move || -> Result { let conn = state.db.get()?; - require_admin_session_sid(&conn, session_id.as_deref())?; + require_admin_session_sid(&conn, session_id.as_deref()).map(|_completed_value| ())?; let save_result = crate::config::admin::runtime::save_section(section, &form); match save_result { - Ok(()) => Ok(admin_panel_redirect_anchor("Configuration saved. Restart required; environment and launcher overrides still take precedence.", section.key()).into_response()), + Ok(()) => Ok(admin_panel_redirect_anchor(if crate::config::admin::restart_pending().unwrap_or(true) { "Settings saved. Restart required: use Restart RustChan to apply these changes." } else { "Settings saved. No restart is required for the effective configuration." }, section.key()).into_response()), Err(error) => { - let mut fields = crate::config::admin::runtime::section_snapshot(section).map_err(|_| "unavailable".to_owned()); + let mut fields = crate::config::admin::runtime::section_snapshot(section).map_err(|snapshot_error| { tracing::warn!(error = %snapshot_error, "operator settings snapshot unavailable"); "unavailable".to_owned() }); if let Ok(fields) = &mut fields { for field in fields { if let Some(value) = form.get(field.definition.key) { field.input.clone_from(value); } diff --git a/src/handlers/admin/settings/site.rs b/src/handlers/admin/settings/site.rs index 9c058e3c..b04c8684 100644 --- a/src/handlers/admin/settings/site.rs +++ b/src/handlers/admin/settings/site.rs @@ -10,7 +10,7 @@ use serde::Deserialize; pub(in crate::server) struct SiteSettingsForm { #[serde(rename = "_csrf")] pub csrf: Option, - /// Custom site name (replaces [ `RustChan` ] on home page and footer). + /// Custom site name (replaces `RustChan` on home page and footer). pub site_name: Option, /// Custom home page subtitle line below the site name. pub site_subtitle: Option, @@ -81,10 +81,12 @@ pub(in crate::server) async fn update_site_settings( tokio::task::spawn_blocking({ let pool = state.db.clone(); move || -> Result<()> { - let mut conn = pool.get()?; - let tx = conn.transaction().map_err(anyhow::Error::from)?; + let mut pooled_connection = pool.get()?; + let tx = pooled_connection + .transaction() + .map_err(anyhow::Error::from)?; let conn = &tx; - require_admin_session_sid(conn, session_id.as_deref())?; + require_admin_session_sid(conn, session_id.as_deref()).map(|_completed_value| ())?; let homepage_new_thread_badges_enabled = resolved_checkbox_setting( form.homepage_new_thread_badges_enabled.as_deref(), db::get_homepage_new_thread_badges_enabled(conn), diff --git a/src/handlers/admin/settings/themes.rs b/src/handlers/admin/settings/themes.rs index 588c184d..c71c7aec 100644 --- a/src/handlers/admin/settings/themes.rs +++ b/src/handlers/admin/settings/themes.rs @@ -397,7 +397,7 @@ pub(in crate::server) async fn create_theme( let pool = state.db.clone(); move || -> Result<()> { let conn = pool.get()?; - require_admin_session_sid(&conn, session_id.as_deref())?; + require_admin_session_sid(&conn, session_id.as_deref()).map(|_completed_value| ())?; let slug = db::sanitize_theme_slug(&form.slug); if slug.is_empty() { return Err(AppError::BadRequest("Theme slug is required.".into())); @@ -434,7 +434,18 @@ pub(in crate::server) async fn create_theme( .into_response(), ) } - other => Err(other), + other @ (AppError::NotFound(_) + | AppError::Forbidden(_) + | AppError::BannedUser { + reason: _, + csrf_token: _, + } + | AppError::UploadTooLarge(_) + | AppError::InvalidMediaType(_) + | AppError::Conflict(_) + | AppError::DbBusy + | AppError::Internal(_) + | AppError::Tls(_)) => Err(other), }) } @@ -451,7 +462,7 @@ pub(in crate::server) async fn update_theme( let pool = state.db.clone(); move || -> Result<()> { let mut conn = pool.get()?; - require_admin_session_sid(&conn, session_id.as_deref())?; + require_admin_session_sid(&conn, session_id.as_deref()).map(|_completed_value| ())?; let existing_slug = db::sanitize_theme_slug(&form.existing_slug); let theme = db::get_theme(&conn, &existing_slug)? .ok_or_else(|| AppError::BadRequest("Theme not found.".into()))?; @@ -507,7 +518,18 @@ pub(in crate::server) async fn update_theme( .into_response(), ) } - other => Err(other), + other @ (AppError::NotFound(_) + | AppError::Forbidden(_) + | AppError::BannedUser { + reason: _, + csrf_token: _, + } + | AppError::UploadTooLarge(_) + | AppError::InvalidMediaType(_) + | AppError::Conflict(_) + | AppError::DbBusy + | AppError::Internal(_) + | AppError::Tls(_)) => Err(other), }) } @@ -531,7 +553,7 @@ pub(in crate::server) async fn delete_theme( let pool = state.db.clone(); move || -> Result<()> { let mut conn = pool.get()?; - require_admin_session_sid(&conn, session_id.as_deref())?; + require_admin_session_sid(&conn, session_id.as_deref()).map(|_completed_value| ())?; let slug = db::sanitize_theme_slug(&form.slug); db::delete_custom_theme(&mut conn, &slug)?; db::sync_live_theme_state(&conn)?; diff --git a/src/handlers/admin/updates.rs b/src/handlers/admin/updates.rs index 790922e6..b9ac78f9 100644 --- a/src/handlers/admin/updates.rs +++ b/src/handlers/admin/updates.rs @@ -69,7 +69,10 @@ pub(super) async fn snapshot() -> Status { .ok() .and_then(|file| { let mut bytes = Vec::new(); - file.take(512 * 1024 + 1).read_to_end(&mut bytes).ok()?; + file.take(512 * 1024 + 1) + .read_to_end(&mut bytes) + .ok() + .map(|_completed_value| ())?; (bytes.len() <= 512 * 1024).then_some(bytes) }) .and_then(|bytes| serde_json::from_slice::(&bytes).ok()); @@ -87,7 +90,7 @@ pub(in crate::server) async fn status( State(state): State, jar: CookieJar, ) -> Result { - authorize(&state, &jar).await?; + authorize(&state, &jar).await.map(|_completed_value| ())?; Ok(Json(snapshot().await).into_response()) } @@ -100,7 +103,7 @@ pub(in crate::server) async fn check( Form(form): Form, ) -> Result { require_admin_post_origin_and_csrf(&jar, &headers, Some(peer), form.csrf.as_deref())?; - authorize(&state, &jar).await?; + authorize(&state, &jar).await.map(|_completed_value| ())?; let _guard = state.maintenance_gate.try_begin("Software update check")?; let outcome = if updates::managed() { updates::request(&Request::Check).await.and_then(|reply| { @@ -170,7 +173,7 @@ pub(in crate::server) async fn install( if password.len() > 1024 || !crate::utils::crypto::verify_password(&password, &user.password_hash)? { - super::auth::record_login_fail(&key); + let _failure_count = super::auth::record_login_fail(&key); return Err(AppError::Forbidden( "Current password did not verify. No update started.".to_owned(), )); @@ -191,7 +194,7 @@ pub(in crate::server) async fn install( }) .await; let acknowledged = outcome.is_ok(); - tokio::spawn(async move { + drop(tokio::spawn(async move { let _guard = guard; loop { if updates::request(&Request::Status) @@ -202,7 +205,7 @@ pub(in crate::server) async fn install( } tokio::time::sleep(std::time::Duration::from_secs(2)).await; } - }); + })); let reply = outcome.map_err(|_error| { AppError::BadRequest( "Updater unavailable; reload Software Updates to check the persisted result." diff --git a/src/handlers/banner.rs b/src/handlers/banner.rs index d8f55e4a..aa734fc1 100644 --- a/src/handlers/banner.rs +++ b/src/handlers/banner.rs @@ -115,7 +115,8 @@ pub(in crate::server) async fn serve_banner_asset( |_| StatusCode::INTERNAL_SERVER_ERROR.into_response(), |resp| { let mut resp = resp.map(axum::body::Body::new); - resp.headers_mut() + let _previous_value = resp + .headers_mut() .insert(header::CONTENT_TYPE, HeaderValue::from_static(content_type)); crate::cache::set_cache_control( resp.headers_mut(), @@ -267,7 +268,8 @@ mod tests { "UPDATE boards SET access_mode = ?1, access_password_hash = ?2 WHERE id = ?3", rusqlite::params!["view_password", password_hash, board_id], ) - .context("protect board")?; + .context("protect board") + .map(|_completed_value| ())?; let banner_id = crate::db::insert_banner_asset( &conn, crate::models::BannerScope::Board, @@ -354,7 +356,8 @@ mod tests { "UPDATE boards SET access_mode = ?1, access_password_hash = ?2 WHERE id = ?3", rusqlite::params!["view_password", password_hash, board_id], ) - .context("protect board")?; + .context("protect board") + .map(|_completed_value| ())?; let admin_hash = crate::utils::crypto::hash_password("hunter2").context("hash admin password")?; let admin_id = diff --git a/src/handlers/board.rs b/src/handlers/board.rs index 17fd02ec..f04badb4 100644 --- a/src/handlers/board.rs +++ b/src/handlers/board.rs @@ -13,7 +13,7 @@ use crate::{ config::CONFIG, db::{self}, error::{AppError, Result}, - handlers::{parse_post_multipart, posting, render, PostFormData}, + handlers::{is_xml_http_request, parse_post_multipart, posting, render, PostFormData}, middleware::{validate_csrf, validate_signed_csrf, AppState, SecureCookieContext}, models::{Board, Pagination, SearchQuery, SEARCH_QUERY_MAX_CHARS}, templates, @@ -153,19 +153,15 @@ fn thread_unread_counts( .iter() .filter_map(|thread| { let marker = markers.get(&thread.id)?; - let unread = (thread.reply_count - marker.seen_reply_count).max(0); + let unread = thread + .reply_count + .saturating_sub(marker.seen_reply_count) + .max(0); (unread > 0).then_some((thread.id, unread)) }) .collect() } -fn is_xml_http_request(headers: &HeaderMap) -> bool { - headers - .get("x-requested-with") - .and_then(|value| value.to_str().ok()) - .is_some_and(|value| value.eq_ignore_ascii_case("XMLHttpRequest")) -} - #[derive(Serialize)] struct XhrErrorPayload<'a> { error: &'a str, @@ -176,10 +172,10 @@ fn json_response(status: StatusCode, payload: &T) -> Result Result { let mut response = json_response(response_status, &XhrErrorPayload { error: message })?; - response.headers_mut().insert( - "x-rustchan-error-status", - HeaderValue::from_str(error_status.as_str()) - .map_err(|error| AppError::Internal(anyhow::anyhow!(error)))?, + drop( + response.headers_mut().insert( + "x-rustchan-error-status", + HeaderValue::from_str(error_status.as_str()) + .map_err(|error| AppError::Internal(anyhow::anyhow!(error)))?, + ), ); if error_status == StatusCode::SERVICE_UNAVAILABLE { - response - .headers_mut() - .insert(header::RETRY_AFTER, HeaderValue::from_static("1")); + drop( + response + .headers_mut() + .insert(header::RETRY_AFTER, HeaderValue::from_static("1")), + ); } Ok(response) } @@ -217,10 +217,12 @@ pub(super) fn xhr_handled_error_response(status: StatusCode, message: &str) -> R pub(super) fn xhr_redirect_response(target: &str) -> Result { let mut response = Response::new(axum::body::Body::empty()); *response.status_mut() = StatusCode::NO_CONTENT; - response.headers_mut().insert( - X_RUSTCHAN_REDIRECT_HEADER, - HeaderValue::from_str(target) - .map_err(|error| AppError::Internal(anyhow::anyhow!(error)))?, + drop( + response.headers_mut().insert( + X_RUSTCHAN_REDIRECT_HEADER, + HeaderValue::from_str(target) + .map_err(|error| AppError::Internal(anyhow::anyhow!(error)))?, + ), ); Ok(response) } @@ -240,9 +242,10 @@ pub(super) fn xhr_post_error_response(error: AppError) -> Result { xhr_handled_error_response(StatusCode::UNPROCESSABLE_ENTITY, &message) } AppError::Forbidden(message) => xhr_handled_error_response(StatusCode::FORBIDDEN, &message), - AppError::BannedUser { reason, .. } => { - xhr_redirect_response(&banned_page_redirect_url(&reason)) - } + AppError::BannedUser { + reason, + csrf_token: _, + } => xhr_redirect_response(&banned_page_redirect_url(&reason)), AppError::UploadTooLarge(message) => { xhr_handled_error_response(StatusCode::PAYLOAD_TOO_LARGE, &message) } @@ -278,7 +281,16 @@ pub(super) fn handled_post_error_status( AppError::BadRequest(message) => Ok((StatusCode::UNPROCESSABLE_ENTITY, message)), AppError::UploadTooLarge(message) => Ok((StatusCode::PAYLOAD_TOO_LARGE, message)), AppError::InvalidMediaType(message) => Ok((StatusCode::UNSUPPORTED_MEDIA_TYPE, message)), - other => Err(other), + other @ (AppError::NotFound(_) + | AppError::Forbidden(_) + | AppError::BannedUser { + reason: _, + csrf_token: _, + } + | AppError::Conflict(_) + | AppError::DbBusy + | AppError::Internal(_) + | AppError::Tls(_)) => Err(other), } } @@ -611,7 +623,7 @@ fn board_unlock_retry_after_secs(attempt_key: &str) -> Option { let (count, window_start) = *BOARD_UNLOCK_FAILS.get(attempt_key)?; let elapsed = now_secs.saturating_sub(window_start); if elapsed > board_password_fail_window_secs() { - BOARD_UNLOCK_FAILS.remove(attempt_key); + let _previous_value = BOARD_UNLOCK_FAILS.remove(attempt_key); return None; } if count < board_password_fail_limit() { @@ -637,7 +649,7 @@ fn record_board_unlock_failure(attempt_key: &str) { } fn clear_board_unlock_failures(attempt_key: &str) { - BOARD_UNLOCK_FAILS.remove(attempt_key); + let _previous_value = BOARD_UNLOCK_FAILS.remove(attempt_key); } fn board_unlock_rate_limit_message(retry_after_secs: u64) -> String { @@ -678,13 +690,13 @@ fn board_access_page_response( ) -> Response { let mut resp = Html(html).into_response(); *resp.status_mut() = status; - resp.headers_mut().insert( + drop(resp.headers_mut().insert( header::CACHE_CONTROL, HeaderValue::from_static(crate::cache::CACHE_CONTROL_PRIVATE_NO_STORE), - ); + )); if let Some(retry_after_secs) = retry_after_secs { if let Ok(retry_after) = HeaderValue::from_str(&retry_after_secs.to_string()) { - resp.headers_mut().insert(header::RETRY_AFTER, retry_after); + drop(resp.headers_mut().insert(header::RETRY_AFTER, retry_after)); } } (jar, resp).into_response() @@ -750,7 +762,7 @@ fn split_catalog_threads( for thread in threads { if let Some(pref) = prefs.get(&thread.id) { if pref.pinned { - pinned_ids.insert(thread.id); + let _completed_value = pinned_ids.insert(thread.id); } if pref.hidden { hidden.push(thread); diff --git a/src/handlers/board/access_preferences.rs b/src/handlers/board/access_preferences.rs index ad8c5582..53f20a05 100644 --- a/src/handlers/board/access_preferences.rs +++ b/src/handlers/board/access_preferences.rs @@ -29,6 +29,13 @@ const OWNED_POSTS_COOKIE_MAX_LEN: usize = 3_800; pub(in crate::server::handlers) fn self_action_window_secs() -> i64 { CONFIG.operator.self_action_window_secs } + +/// Bound self-action grants without extending a malformed restored timestamp on overflow. +pub(in crate::server::handlers) fn self_action_expiry(created_at: i64) -> Result { + created_at + .checked_add(self_action_window_secs()) + .ok_or_else(|| AppError::Internal(anyhow::anyhow!("self-action expiry timestamp overflow"))) +} /// Board activity cookie used by this handler. const BOARD_ACTIVITY_COOKIE: &str = "rustchan_board_activity"; /// Thread activity cookie used by this handler. @@ -142,7 +149,7 @@ fn prune_owned_post_grants_for_cookie(mut grants: Vec) -> Vec HashMap let mut markers = HashMap::new(); for entry in parse_activity_cookie_entries(value) { if let Some(marker) = parse_board_activity_marker(entry, now) { - markers.insert(marker.board_id, marker); + let _previous_value = markers.insert(marker.board_id, marker); } } markers @@ -307,7 +314,7 @@ fn parse_thread_activity_cookie(value: &str) -> HashMap anyhow::Result<()> { + anyhow::ensure!( + super::self_action_expiry(i64::MAX).is_err(), + "overflowed expiry must fail closed" + ); + let created_at = 1_700_000_000_i64; + let expected = created_at + .checked_add(self_action_window_secs()) + .ok_or_else(|| anyhow::anyhow!("test expiry overflow"))?; + anyhow::ensure!( + super::self_action_expiry(created_at)? == expected, + "valid expiry must preserve the configured window" + ); + Ok(()) + } use super::{ board_activity_markers_from_jar, ensure_csrf_with_secure, owned_post_grants_from_jar, owned_posts_cookie, owned_posts_cookie_value, prune_board_activity_markers, @@ -1207,7 +1231,7 @@ mod tests { async fn user_preferences_remain_settable_on_plain_http_onion_transport() -> AnyResult<()> { let jar = CookieJar::new().add(TestCookie::new("csrf_token", "csrf123")); let mut headers = HeaderMap::new(); - headers.insert( + let _previous_value = headers.insert( header::HOST, HeaderValue::from_static( "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaam2dqd.onion", @@ -1324,7 +1348,7 @@ mod tests { chrono::Utc::now().timestamp() - 1, ); - assert!(owned_post_grants_from_jar(&jar).is_empty()); + assert_eq!(owned_post_grants_from_jar(&jar).len(), 0); } #[test] @@ -1387,7 +1411,7 @@ mod tests { value, )); - assert!(owned_post_grants_from_jar(&jar).is_empty()); + assert_eq!(owned_post_grants_from_jar(&jar).len(), 0); } } diff --git a/src/handlers/board/catalog.rs b/src/handlers/board/catalog.rs index 5a04feb4..5e96ea7b 100644 --- a/src/handlers/board/catalog.rs +++ b/src/handlers/board/catalog.rs @@ -217,14 +217,14 @@ pub(in crate::server) async fn catalog( .status(StatusCode::NOT_MODIFIED) .body(axum::body::Body::empty()) .map_err(|e| AppError::Internal(anyhow::anyhow!(e)))?; - resp.headers_mut().insert( + drop(resp.headers_mut().insert( "etag", HeaderValue::from_str(&etag).unwrap_or_else(|_| HeaderValue::from_static("\"0\"")), - ); - resp.headers_mut().insert( + )); + drop(resp.headers_mut().insert( header::CACHE_CONTROL, HeaderValue::from_static(activity_html_cache_control(activity_markers_enabled)), - ); + )); crate::cache::insert_vary_cookie(resp.headers_mut()); return Ok((jar, resp).into_response()); } @@ -255,12 +255,12 @@ pub(in crate::server) async fn catalog( ); let mut resp = Html(html).into_response(); if let Ok(v) = HeaderValue::from_str(&etag) { - resp.headers_mut().insert("etag", v); + drop(resp.headers_mut().insert("etag", v)); } - resp.headers_mut().insert( + drop(resp.headers_mut().insert( header::CACHE_CONTROL, HeaderValue::from_static(activity_html_cache_control(activity_markers_enabled)), - ); + )); crate::cache::insert_vary_cookie(resp.headers_mut()); Ok((jar, resp).into_response()) } diff --git a/src/handlers/board/create_thread.rs b/src/handlers/board/create_thread.rs index 878fe4a4..8adb8817 100644 --- a/src/handlers/board/create_thread.rs +++ b/src/handlers/board/create_thread.rs @@ -2,12 +2,11 @@ use super::{ board_access_cookie_from_jar, board_access_preflight, current_theme_from_jar, db, handled_post_error_status, identity_key, index_reply_previews, index_threads_per_page, is_xml_http_request, make_scoped_csrf_form_token, parse_post_multipart, posting, - remember_owned_post_until_with_secure, render, self_action_window_secs, - should_set_public_secure_cookie, unlock_redirect_url, user_preferences_from_jar, - xhr_error_response, xhr_post_error_response, xhr_redirect_response, AppError, AppState, - BoardAccessDecision, BoardAccessRequirement, CookieJar, HashMap, HeaderMap, Html, Multipart, - Path, PostFormData, Redirect, Response, Result, SecureCookieContext, State, - ADMIN_SESSION_COOKIE, CONFIG, + remember_owned_post_until_with_secure, render, should_set_public_secure_cookie, + unlock_redirect_url, user_preferences_from_jar, xhr_error_response, xhr_post_error_response, + xhr_redirect_response, AppError, AppState, BoardAccessDecision, BoardAccessRequirement, + CookieJar, HashMap, HeaderMap, Html, Multipart, Path, PostFormData, Redirect, Response, Result, + SecureCookieContext, State, ADMIN_SESSION_COOKIE, CONFIG, }; use axum::response::IntoResponse as _; @@ -40,7 +39,7 @@ pub(in crate::server) async fn create_thread( &state, &board_short, admin_session_id.clone(), - access_cookie, + access_cookie.clone(), BoardAccessRequirement::Post, format!("/{board_short}"), ) @@ -130,16 +129,13 @@ pub(in crate::server) async fn create_thread( let pool = state.db.clone(); let job_queue = std::sync::Arc::clone(&state.job_queue); let ffmpeg_available = state.ffmpeg_available; - let ffprobe_available = state.ffprobe_available; - let ffmpeg_webp_available = state.ffmpeg_webp_available; move || -> Result { // `spawn_blocking` work is not cancelled when its join handle is // dropped. Keep the permit inside this closure so a disconnected // request cannot release the media gate while parsing continues. let _media_upload_guard = media_upload_guard; - let conn = pool.get()?; posting::submit_post( - &conn, + &pool, &job_queue, posting::SubmitPostCommand { mode: posting::SubmitPostMode::NewThread { @@ -152,6 +148,7 @@ pub(in crate::server) async fn create_thread( identity_key, cookie_secret: CONFIG.cookie_secret.clone(), admin_session_id, + access_cookie, ban_csrf_token, submission_token, name, @@ -165,8 +162,6 @@ pub(in crate::server) async fn create_thread( upload_dir: CONFIG.upload_dir.clone(), thumb_size: CONFIG.thumb_size, ffmpeg_available, - ffprobe_available, - ffmpeg_webp_available, }, ) } @@ -265,7 +260,7 @@ pub(in crate::server) async fn create_thread( submit_result.thread_id, submit_result.post_id, &submit_result.deletion_token, - submit_result.created_at + self_action_window_secs(), + super::self_action_expiry(submit_result.created_at)?, should_set_public_secure_cookie(&req_headers, secure_context), ); diff --git a/src/handlers/board/media.rs b/src/handlers/board/media.rs index d1f24bfd..6f0a499f 100644 --- a/src/handlers/board/media.rs +++ b/src/handlers/board/media.rs @@ -68,17 +68,48 @@ fn stale_webm_redirect_path(base: &std::path::Path, media_path: &str) -> Option< let path = std::path::Path::new(media_path); if !path .extension() - .is_some_and(|ext| ext.eq_ignore_ascii_case("mp4")) + .is_some_and(|ext| ext.eq_ignore_ascii_case("mp4") || ext.eq_ignore_ascii_case("mkv")) { return None; } let stem = media_path.get(..media_path.len().saturating_sub(4))?; let webm_path = format!("{stem}.webm"); - safe_board_media_file(base, &webm_path).ok()?; - Some(format!("/boards/{webm_path}")) + safe_board_media_file(base, &webm_path) + .ok() + .map(|_completed_value| ())?; + let location = format!("/boards/{webm_path}"); + HeaderValue::from_str(&location) + .ok() + .map(|_completed_value| ())?; + axum::http::Uri::try_from(location.as_str()) + .ok() + .map(|_completed_value| ())?; + Some(location) } -// Legacy `.mp4` links redirect permanently to transcoded `.webm` files; all +/// A page can still request its SVG just after the worker replaces that file. +/// Resolve only generated thumbnail paths and validate the PNG under the root. +fn stale_waveform_redirect_path(base: &std::path::Path, media_path: &str) -> Option { + if !is_generated_svg_placeholder_thumb(media_path) { + return None; + } + let png_path = std::path::Path::new(media_path).with_extension("png"); + let relative = png_path.to_str()?; + safe_board_media_file(base, relative) + .ok() + .map(|_completed_value| ())?; + let location = format!("/boards/{relative}"); + // Validate untrusted stored paths before constructing an HTTP redirect. + HeaderValue::from_str(&location) + .ok() + .map(|_completed_value| ())?; + axum::http::Uri::try_from(location.as_str()) + .ok() + .map(|_completed_value| ())?; + Some(location) +} + +// Legacy `.mp4` and `.mkv` links redirect to transcoded `.webm` files; all // other validated paths are served directly. #[expect( @@ -91,7 +122,6 @@ pub(in crate::server) async fn serve_board_media( jar: CookieJar, req: axum::extract::Request, ) -> Response { - use axum::http::StatusCode; use std::path::PathBuf; use tower::ServiceExt as _; use tower_http::services::ServeFile; @@ -148,74 +178,108 @@ pub(in crate::server) async fn serve_board_media( .and_then(|ext| ext.to_str()) .is_some_and(|ext| ext.eq_ignore_ascii_case("pdf")); - let target = match safe_board_media_file(&base, &media_path) { + let updated_target = match safe_board_media_file(&base, &media_path) { Ok(path) => Some(path), Err(error) if is_not_found_error(&error) - && std::path::Path::new(&media_path) + && (std::path::Path::new(&media_path) .extension() - .is_some_and(|ext| ext.eq_ignore_ascii_case("mp4")) => + .is_some_and(|ext| { + ext.eq_ignore_ascii_case("mp4") || ext.eq_ignore_ascii_case("mkv") + }) + || is_generated_svg_placeholder_thumb(&media_path)) => { None } Err(_) => return StatusCode::NOT_FOUND.into_response(), }; - if let Some(target) = target { + if let Some(updated_target) = updated_target { // File present — forward the real request (with Range, ETag, etc.) to // ServeFile so it can respond with 206 Partial Content when needed. // iOS Safari requires Range request support to play video — dropping // the request headers caused it to receive 200 instead of 206 and // refuse playback on videos it tried to stream in chunks. let req = req.map(|_| axum::body::Body::empty()); - ServeFile::new(&target).oneshot(req).await.map_or_else( - |_| StatusCode::INTERNAL_SERVER_ERROR.into_response(), - |resp| { - let mut resp = resp.map(axum::body::Body::new); - crate::cache::set_cache_control( - resp.headers_mut(), - board_media_cache_control( - access_context.board.access_mode.requires_view_password(), - is_board_favicon, - has_version, - ), - ); - if is_generated_svg_placeholder_thumb(&media_path) { - resp.headers_mut() - .insert(CONTENT_TYPE, HeaderValue::from_static("image/svg+xml")); - resp.headers_mut() - .insert(X_CONTENT_TYPE_OPTIONS, HeaderValue::from_static("nosniff")); - resp.headers_mut().insert( - CONTENT_SECURITY_POLICY, - HeaderValue::from_static("default-src 'none'; script-src 'none'"), - ); - } else if let Some(ct) = media_content_type(&target) { - resp.headers_mut() - .insert(CONTENT_TYPE, HeaderValue::from_static(ct)); - } else { - resp.headers_mut().insert( - CONTENT_TYPE, - HeaderValue::from_static("application/octet-stream"), - ); - resp.headers_mut() - .insert(X_CONTENT_TYPE_OPTIONS, HeaderValue::from_static("nosniff")); - let filename = target - .file_name() - .and_then(|name| name.to_str()) - .unwrap_or("download.bin") - .replace(['\\', '"'], "_"); - if let Ok(value) = - HeaderValue::from_str(&format!("attachment; filename=\"{filename}\"")) + ServeFile::new(&updated_target) + .oneshot(req) + .await + .map_or_else( + |_| StatusCode::INTERNAL_SERVER_ERROR.into_response(), + |resp| { + // The waveform worker can remove the SVG between our path check + // and ServeFile opening it. Resolve the completed PNG again. + if let Some(path) = (resp.status() == StatusCode::NOT_FOUND) + .then(|| stale_waveform_redirect_path(&base, &media_path)) + .flatten() { - resp.headers_mut().insert(CONTENT_DISPOSITION, value); + return Redirect::temporary(&path).into_response(); } - } - if is_pdf { - apply_pdf_embed_headers(resp.headers_mut()); - } - resp.into_response() - }, - ) + if let Some(path) = (resp.status() == StatusCode::NOT_FOUND) + .then(|| stale_webm_redirect_path(&base, &media_path)) + .flatten() + { + return Redirect::permanent(&path).into_response(); + } + let mut resp = resp.map(axum::body::Body::new); + crate::cache::set_cache_control( + resp.headers_mut(), + board_media_cache_control( + access_context.board.access_mode.requires_view_password(), + is_board_favicon, + has_version, + ), + ); + if is_generated_svg_placeholder_thumb(&media_path) { + drop( + resp.headers_mut() + .insert(CONTENT_TYPE, HeaderValue::from_static("image/svg+xml")), + ); + drop( + resp.headers_mut().insert( + X_CONTENT_TYPE_OPTIONS, + HeaderValue::from_static("nosniff"), + ), + ); + drop(resp.headers_mut().insert( + CONTENT_SECURITY_POLICY, + HeaderValue::from_static("default-src 'none'; script-src 'none'"), + )); + } else if let Some(ct) = media_content_type(&updated_target) { + drop( + resp.headers_mut() + .insert(CONTENT_TYPE, HeaderValue::from_static(ct)), + ); + } else { + drop(resp.headers_mut().insert( + CONTENT_TYPE, + HeaderValue::from_static("application/octet-stream"), + )); + drop( + resp.headers_mut().insert( + X_CONTENT_TYPE_OPTIONS, + HeaderValue::from_static("nosniff"), + ), + ); + let filename = updated_target + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or("download.bin") + .replace(['\\', '"'], "_"); + let disposition = format!("attachment; filename=\"{filename}\""); + let parsed_disposition = HeaderValue::from_str(&disposition); + if let Ok(value) = parsed_disposition { + drop(resp.headers_mut().insert(CONTENT_DISPOSITION, value)); + } + } + if is_pdf { + apply_pdf_embed_headers(resp.headers_mut()); + } + resp.into_response() + }, + ) + } else if let Some(redirect_path) = stale_waveform_redirect_path(&base, &media_path) { + Redirect::temporary(&redirect_path).into_response() } else if let Some(redirect_path) = stale_webm_redirect_path(&base, &media_path) { Redirect::permanent(&redirect_path).into_response() } else { @@ -239,13 +303,13 @@ const fn board_media_cache_control( } fn apply_pdf_embed_headers(headers: &mut HeaderMap) { - headers.insert(X_FRAME_OPTIONS, HeaderValue::from_static("SAMEORIGIN")); - headers.insert( + drop(headers.insert(X_FRAME_OPTIONS, HeaderValue::from_static("SAMEORIGIN"))); + drop(headers.insert( CONTENT_SECURITY_POLICY, HeaderValue::from_static( "default-src 'none'; frame-ancestors 'self'; sandbox allow-same-origin allow-scripts", ), - ); + )); } // GET /api/post/{board}/{post_id} @@ -355,8 +419,6 @@ pub(in crate::server) async fn redirect_to_post( Path((board_short, post_id)): Path<(String, i64)>, jar: CookieJar, ) -> impl axum::response::IntoResponse { - use axum::response::Redirect; - let board_short_for_url = board_short.clone(); let admin_session_id = jar .get(ADMIN_SESSION_COOKIE) @@ -408,6 +470,25 @@ mod tests { use super::stale_webm_redirect_path; use anyhow::{ensure, Context as _, Result as AnyResult}; + #[test] + fn stale_waveform_svg_redirects_only_to_an_existing_generated_png() -> AnyResult<()> { + let dir = tempfile::tempdir()?; + let thumbs = dir.path().join("test/thumbs"); + std::fs::create_dir_all(&thumbs)?; + let path = "test/thumbs/clip.svg"; + ensure!(super::stale_waveform_redirect_path(dir.path(), path).is_none()); + std::fs::write(thumbs.join("clip.png"), b"fixture")?; + ensure!( + super::stale_waveform_redirect_path(dir.path(), path).as_deref() + == Some("/boards/test/thumbs/clip.png") + ); + ensure!(super::stale_waveform_redirect_path(dir.path(), "test/clip.svg").is_none()); + ensure!( + super::stale_waveform_redirect_path(dir.path(), "../test/thumbs/clip.svg").is_none() + ); + Ok(()) + } + #[test] fn stale_mp4_redirect_path_accepts_valid_webm_sibling() -> AnyResult<()> { let tempdir = tempfile::tempdir().context("create temporary upload root")?; @@ -416,10 +497,13 @@ mod tests { std::fs::create_dir_all(&board_dir).context("create board directory")?; std::fs::write(board_dir.join("clip.webm"), b"webm").context("write WebM fixture")?; - ensure!( - stale_webm_redirect_path(&upload_root, "test/clip.mp4").as_deref() - == Some("/boards/test/clip.webm") - ); + for path in ["test/clip.mp4", "test/clip.mkv", "test/clip.MKV"] { + ensure!( + stale_webm_redirect_path(&upload_root, path).as_deref() + == Some("/boards/test/clip.webm") + ); + } + ensure!(stale_webm_redirect_path(&upload_root, "test/clip.wav").is_none()); Ok(()) } @@ -438,6 +522,7 @@ mod tests { unix_fs::symlink(&outside, board_dir.join("link")).context("create escaping symlink")?; ensure!(stale_webm_redirect_path(&upload_root, "test/link/clip.mp4").is_none()); + ensure!(stale_webm_redirect_path(&upload_root, "test/link/clip.mkv").is_none()); Ok(()) } diff --git a/src/handlers/board/pages.rs b/src/handlers/board/pages.rs index e88814b2..a1dc6c02 100644 --- a/src/handlers/board/pages.rs +++ b/src/handlers/board/pages.rs @@ -210,10 +210,10 @@ pub(in crate::server) async fn index( )) .into_response(); let activity_markers_enabled = homepage_thread_badges_enabled || homepage_reply_badges_enabled; - response.headers_mut().insert( + drop(response.headers_mut().insert( header::CACHE_CONTROL, HeaderValue::from_static(activity_html_cache_control(activity_markers_enabled)), - ); + )); crate::cache::insert_vary_cookie(response.headers_mut()); Ok((jar, response).into_response()) } @@ -334,7 +334,11 @@ pub(in crate::server) async fn board_index( .iter() .filter_map(|summary| { let marker = thread_activity_markers.get(&summary.thread.id)?; - let unread = (summary.thread.reply_count - marker.seen_reply_count).max(0); + let unread = summary + .thread + .reply_count + .saturating_sub(marker.seen_reply_count) + .max(0); (unread > 0).then_some((summary.thread.id, unread)) }) .collect::>() @@ -412,14 +416,14 @@ pub(in crate::server) async fn board_index( .status(StatusCode::NOT_MODIFIED) .body(axum::body::Body::empty()) .map_err(|e| AppError::Internal(anyhow::anyhow!(e)))?; - resp.headers_mut().insert( + drop(resp.headers_mut().insert( "etag", HeaderValue::from_str(&etag).unwrap_or_else(|_| HeaderValue::from_static("\"0\"")), - ); - resp.headers_mut().insert( + )); + drop(resp.headers_mut().insert( header::CACHE_CONTROL, HeaderValue::from_static(activity_html_cache_control(activity_markers_enabled)), - ); + )); crate::cache::insert_vary_cookie(resp.headers_mut()); return Ok((jar, resp).into_response()); } @@ -449,12 +453,12 @@ pub(in crate::server) async fn board_index( .map_err(|e| AppError::Internal(anyhow::anyhow!(e)))?; let mut resp = Html(html).into_response(); if let Ok(v) = HeaderValue::from_str(&etag) { - resp.headers_mut().insert("etag", v); + drop(resp.headers_mut().insert("etag", v)); } - resp.headers_mut().insert( + drop(resp.headers_mut().insert( header::CACHE_CONTROL, HeaderValue::from_static(activity_html_cache_control(activity_markers_enabled)), - ); + )); crate::cache::insert_vary_cookie(resp.headers_mut()); Ok((jar, resp).into_response()) } diff --git a/src/handlers/board/reports.rs b/src/handlers/board/reports.rs index 27f38843..74098c4c 100644 --- a/src/handlers/board/reports.rs +++ b/src/handlers/board/reports.rs @@ -84,7 +84,8 @@ pub(in crate::server) async fn file_report( } // Use the DB's thread_id for the redirect — not the user-submitted value. let authoritative_thread_id = post.thread_id; - let _ = db::file_report(&conn, post_id, &reason, &ip_hash)?; + // The report ID does not affect the authoritative thread redirect. + db::file_report(&conn, post_id, &reason, &ip_hash).map(|_report_id| ())?; Ok(authoritative_thread_id) } }) diff --git a/src/handlers/board/tests.rs b/src/handlers/board/tests.rs index d8856b67..f4186967 100644 --- a/src/handlers/board/tests.rs +++ b/src/handlers/board/tests.rs @@ -54,7 +54,7 @@ fn seed_post_password_board( "UPDATE boards SET access_mode = ?1, access_password_hash = ?2, allow_editing = 1, allow_self_delete = 1 WHERE id = ?3", rusqlite::params!["post_password", password_hash, board_id], ) - .context("update board access")?; + .context("update board access").map(|_completed_value| ())?; let post = crate::db::NewPost { thread_id: 0, board_id, @@ -80,7 +80,7 @@ fn seed_post_password_board( let poll = crate::db::threads::PollInsert { question: "pick one", options: &["yes".to_owned(), "no".to_owned()], - expires_at: chrono::Utc::now().timestamp() + 3600, + expires_at: chrono::Utc::now().timestamp().saturating_add(3600), }; let (thread_id, post_id, poll_id) = crate::db::create_thread_with_optional_poll( &conn, @@ -257,7 +257,8 @@ fn create_reply_on_thread( is_op: false, }; crate::db::create_reply_with_thread_update(&conn, &reply, "", true, None) - .context("create reply")?; + .context("create reply") + .map(|_completed_value| ())?; Ok(()) } @@ -290,9 +291,9 @@ fn update_cookie_store(store: &mut HashMap, headers: &HeaderMap) continue; }; if cookie_value.is_empty() { - store.remove(name); + let _previous_value = store.remove(name); } else { - store.insert(name.to_owned(), cookie_value.to_owned()); + let _previous_value = store.insert(name.to_owned(), cookie_value.to_owned()); } } } @@ -322,7 +323,7 @@ fn activity_restore_js_uses_explicit_page_markers() -> anyhow::Result<()> { #[test] fn board_activity_cookie_removal_keeps_root_path_attributes() -> anyhow::Result<()> { let mut headers = HeaderMap::new(); - headers.insert( + let _previous_value = headers.insert( header::COOKIE, "rustchan_board_activity=v1|1.100.1.200" .parse() @@ -348,7 +349,7 @@ fn board_activity_cookie_removal_keeps_root_path_attributes() -> anyhow::Result< #[test] fn thread_activity_cookie_removal_keeps_root_path_attributes() -> anyhow::Result<()> { let mut headers = HeaderMap::new(); - headers.insert( + let _previous_value = headers.insert( header::COOKIE, "rustchan_thread_activity=v1|1.2.200" .parse() @@ -490,7 +491,7 @@ async fn post_password_board_write_actions_require_unlock() -> anyhow::Result<() Some("/secret/unlock?return_to=%2Fsecret") ); - let (boundary, body) = + let (response_boundary, resolved_body) = crate::test_support::multipart_body(&[("_csrf", "csrf123"), ("body", "reply")], None); let reply_response = router .clone() @@ -500,11 +501,11 @@ async fn post_password_board_write_actions_require_unlock() -> anyhow::Result<() .uri(format!("/secret/thread/{thread_id}")) .header( header::CONTENT_TYPE, - format!("multipart/form-data; boundary={boundary}"), + format!("multipart/form-data; boundary={response_boundary}"), ) .header(header::COOKIE, "csrf_token=csrf123") .extension(crate::test_support::connect_info()) - .body(Body::from(body)) + .body(Body::from(resolved_body)) .context("request")?, ) .await @@ -609,7 +610,8 @@ async fn self_delete_requires_owned_post_cookie() -> anyhow::Result<()> { "UPDATE boards SET allow_self_delete = 1 WHERE id = ?1", rusqlite::params![board_id], ) - .context("enable self delete")?; + .context("enable self delete") + .map(|_completed_value| ())?; let op = crate::db::NewPost { thread_id: 0, board_id, @@ -753,7 +755,8 @@ async fn search_returns_results_without_500() -> anyhow::Result<()> { is_op: true, }; crate::db::create_thread_with_optional_poll(&conn, board_id, None, &post, "", None, None) - .context("create thread")?; + .context("create thread") + .map(|_completed_value| ())?; } let router = Router::new() @@ -788,7 +791,9 @@ async fn search_without_q_param_returns_empty_results_page() -> anyhow::Result<( let state = crate::test_support::app_state(); { let conn = state.db.get().context("db connection")?; - crate::db::create_board(&conn, "test", "Test", "", false).context("create board")?; + crate::db::create_board(&conn, "test", "Test", "", false) + .context("create board") + .map(|_completed_value| ())?; } let router = Router::new() @@ -823,14 +828,16 @@ async fn locked_board_search_returns_forbidden_unlock_page() -> anyhow::Result<( let state = crate::test_support::app_state(); { let conn = state.db.get().context("db connection")?; - crate::db::create_board(&conn, "slock", "Secret", "", false).context("create board")?; + crate::db::create_board(&conn, "slock", "Secret", "", false) + .context("create board") + .map(|_completed_value| ())?; let password_hash = crate::utils::crypto::hash_password("swordfish").context("hash password")?; conn.execute( "UPDATE boards SET access_mode = ?1, access_password_hash = ?2 WHERE short_name = 'slock'", rusqlite::params!["view_password", password_hash], ) - .context("update board access")?; + .context("update board access").map(|_completed_value| ())?; } let router = Router::new() @@ -872,7 +879,9 @@ async fn create_thread_accepts_valid_multipart_submission() -> anyhow::Result<() let state = crate::test_support::app_state(); { let conn = state.db.get().context("db connection")?; - crate::db::create_board(&conn, "test", "Test", "", false).context("create board")?; + crate::db::create_board(&conn, "test", "Test", "", false) + .context("create board") + .map(|_completed_value| ())?; } let router = Router::new() @@ -913,7 +922,9 @@ async fn create_thread_xhr_returns_explicit_redirect_header() -> anyhow::Result< let state = crate::test_support::app_state(); { let conn = state.db.get().context("db connection")?; - crate::db::create_board(&conn, "test", "Test", "", false).context("create board")?; + crate::db::create_board(&conn, "test", "Test", "", false) + .context("create board") + .map(|_completed_value| ())?; } let router = Router::new() @@ -968,7 +979,9 @@ async fn create_thread_xhr_validation_failure_returns_json_error() -> anyhow::Re let state = crate::test_support::app_state(); { let conn = state.db.get().context("db connection")?; - crate::db::create_board(&conn, "test", "Test", "", false).context("create board")?; + crate::db::create_board(&conn, "test", "Test", "", false) + .context("create board") + .map(|_completed_value| ())?; } let router = Router::new() @@ -1011,14 +1024,14 @@ async fn create_thread_xhr_validation_failure_returns_json_error() -> anyhow::Re Some(StatusCode::UNPROCESSABLE_ENTITY.as_str()) ); - let body = String::from_utf8( + let resolved_body = String::from_utf8( to_bytes(response.into_body(), usize::MAX) .await .context("response body")? .to_vec(), ) .context("utf8 body")?; - anyhow::ensure!(body.contains("\"error\"")); + anyhow::ensure!(resolved_body.contains("\"error\"")); Ok(()) } @@ -1123,7 +1136,9 @@ async fn create_thread_rejects_mime_mismatch_with_415_inline_error() -> anyhow:: let state = crate::test_support::app_state(); { let conn = state.db.get().context("db connection")?; - crate::db::create_board(&conn, "test", "Test", "", false).context("create board")?; + crate::db::create_board(&conn, "test", "Test", "", false) + .context("create board") + .map(|_completed_value| ())?; } let router = Router::new() @@ -1152,15 +1167,15 @@ async fn create_thread_rejects_mime_mismatch_with_415_inline_error() -> anyhow:: .context("response")?; ensure_eq!(response.status(), StatusCode::UNSUPPORTED_MEDIA_TYPE); - let body = String::from_utf8( + let resolved_body = String::from_utf8( to_bytes(response.into_body(), usize::MAX) .await .context("response body")? .to_vec(), ) .context("utf8 body")?; - anyhow::ensure!(body.contains("post-error-banner")); - anyhow::ensure!(body.contains("File type not allowed")); + anyhow::ensure!(resolved_body.contains("post-error-banner")); + anyhow::ensure!(resolved_body.contains("File type not allowed")); Ok(()) } @@ -1169,7 +1184,9 @@ async fn create_thread_rejects_truncated_png_with_422_inline_error() -> anyhow:: let state = crate::test_support::app_state(); { let conn = state.db.get().context("db connection")?; - crate::db::create_board(&conn, "test", "Test", "", false).context("create board")?; + crate::db::create_board(&conn, "test", "Test", "", false) + .context("create board") + .map(|_completed_value| ())?; } let router = Router::new() @@ -1199,15 +1216,15 @@ async fn create_thread_rejects_truncated_png_with_422_inline_error() -> anyhow:: .context("response")?; ensure_eq!(response.status(), StatusCode::UNPROCESSABLE_ENTITY); - let body = String::from_utf8( + let resolved_body = String::from_utf8( to_bytes(response.into_body(), usize::MAX) .await .context("response body")? .to_vec(), ) .context("utf8 body")?; - anyhow::ensure!(body.contains("post-error-banner")); - anyhow::ensure!(body.contains("image header is malformed")); + anyhow::ensure!(resolved_body.contains("post-error-banner")); + anyhow::ensure!(resolved_body.contains("image header is malformed")); Ok(()) } @@ -1277,7 +1294,8 @@ async fn absent_homepage_reply_badge_setting_defaults_to_enabled() -> anyhow::Re "DELETE FROM site_settings WHERE key = 'homepage_new_reply_badges_enabled'", [], ) - .context("delete setting")?; + .context("delete setting") + .map(|_completed_value| ())?; anyhow::ensure!(crate::db::get_homepage_new_reply_badges_enabled(&conn)); Ok(()) @@ -1304,7 +1322,7 @@ async fn homepage_reply_toggle_off_suppresses_only_homepage_reply_badges() -> an .await .context("response")?; update_cookie_store(&mut cookies, baseline.headers()); - create_thread_on_board(&state, board_id, "new thread")?; + create_thread_on_board(&state, board_id, "new thread").map(|_completed_value| ())?; create_reply_on_thread(&state, board_id, thread_id, "reply")?; let home_response = router @@ -1433,7 +1451,7 @@ async fn homepage_thread_toggle_off_does_not_suppress_homepage_reply_badges() -> .context("response")?; update_cookie_store(&mut cookies, baseline.headers()); - create_thread_on_board(&state, board_id, "new thread")?; + create_thread_on_board(&state, board_id, "new thread").map(|_completed_value| ())?; create_reply_on_thread(&state, board_id, thread_id, "reply")?; let response = router @@ -1574,7 +1592,7 @@ async fn new_thread_after_board_baseline_shows_homepage_badge() -> anyhow::Resul .context("response")?; update_cookie_store(&mut cookies, baseline.headers()); - create_thread_on_board(&state, board_id, "new thread")?; + create_thread_on_board(&state, board_id, "new thread").map(|_completed_value| ())?; let response = router .oneshot( @@ -1663,7 +1681,7 @@ async fn homepage_thread_and_reply_badges_can_render_together() -> anyhow::Resul .context("response")?; update_cookie_store(&mut cookies, baseline.headers()); - create_thread_on_board(&state, board_id, "new thread")?; + create_thread_on_board(&state, board_id, "new thread").map(|_completed_value| ())?; create_reply_on_thread(&state, board_id, thread_id, "reply")?; let response = router @@ -1707,7 +1725,7 @@ async fn board_index_visit_clears_homepage_new_thread_badge() -> anyhow::Result< .await .context("response")?; update_cookie_store(&mut cookies, baseline.headers()); - create_thread_on_board(&state, board_id, "new thread")?; + create_thread_on_board(&state, board_id, "new thread").map(|_completed_value| ())?; let clear_response = router .clone() @@ -1761,7 +1779,7 @@ async fn board_catalog_visit_clears_homepage_new_thread_badge() -> anyhow::Resul .await .context("response")?; update_cookie_store(&mut cookies, baseline.headers()); - create_thread_on_board(&state, board_id, "new thread")?; + create_thread_on_board(&state, board_id, "new thread").map(|_completed_value| ())?; let clear_response = router .clone() @@ -1815,7 +1833,7 @@ async fn thread_visit_clears_homepage_new_thread_badge() -> anyhow::Result<()> { .await .context("response")?; update_cookie_store(&mut cookies, baseline.headers()); - create_thread_on_board(&state, board_id, "new thread")?; + create_thread_on_board(&state, board_id, "new thread").map(|_completed_value| ())?; let clear_response = router .clone() @@ -1877,7 +1895,7 @@ async fn conditional_thread_visit_that_marks_activity_read_returns_full_response .context("thread etag")?; update_cookie_store(&mut cookies, baseline.headers()); - create_thread_on_board(&state, board_id, "new thread")?; + create_thread_on_board(&state, board_id, "new thread").map(|_completed_value| ())?; let clear_response = router .clone() @@ -1917,7 +1935,7 @@ async fn conditional_board_activity_pages_return_full_response_when_tracking_ena ) -> anyhow::Result<()> { let state = crate::test_support::app_state(); set_new_activity_settings(&state, true, true, true)?; - seed_board_with_thread(&state, "tech", "op")?; + seed_board_with_thread(&state, "tech", "op").map(|_completed_value| ())?; let router = activity_router(state); for uri in ["/tech", "/tech/catalog"] { @@ -2478,7 +2496,7 @@ async fn thread_updates_clear_homepage_new_thread_badge_cookie() -> anyhow::Resu .context("response")?; update_cookie_store(&mut cookies, baseline.headers()); - create_thread_on_board(&state, board_id, "new thread")?; + create_thread_on_board(&state, board_id, "new thread").map(|_completed_value| ())?; let badge_home = router .clone() @@ -2540,7 +2558,8 @@ async fn password_protected_board_does_not_leak_homepage_new_activity_badge() -> "UPDATE boards SET access_mode = ?1, access_password_hash = ?2 WHERE id = ?3", rusqlite::params!["view_password", password_hash, board_id], ) - .context("update board access")?; + .context("update board access") + .map(|_completed_value| ())?; } let router = activity_router(state); let cookie = format!( @@ -2578,7 +2597,8 @@ async fn thread_updates_rejects_thread_id_from_other_board() -> anyhow::Result<( "UPDATE boards SET access_mode = ?1, access_password_hash = ?2 WHERE id = ?3", rusqlite::params!["view_password", password_hash, secret_board_id], ) - .context("protect secret board")?; + .context("protect secret board") + .map(|_completed_value| ())?; } create_reply_on_thread( &state, @@ -2733,7 +2753,7 @@ async fn catalog_baseline_tracks_only_highest_priority_threads_within_cookie_lim set_new_activity_settings(&state, true, true, true)?; let (board_id, first_thread_id) = seed_board_with_thread(&state, "tech", "op")?; let mut created_thread_ids = vec![first_thread_id]; - for index in 0..120 { + for index in 0_i32..120_i32 { created_thread_ids.push(create_thread_on_board( &state, board_id, @@ -2768,7 +2788,7 @@ async fn catalog_baseline_tracks_only_highest_priority_threads_within_cookie_lim }) .context("thread activity cookie")?; let mut cookie_headers = HeaderMap::new(); - cookie_headers.insert( + let _previous_value = cookie_headers.insert( header::COOKIE, format!("rustchan_thread_activity={cookie_value}") .parse() @@ -2803,14 +2823,17 @@ async fn create_thread_xhr_banned_user_redirects_to_banned_page() -> anyhow::Res let state = crate::test_support::app_state(); { let conn = state.db.get().context("db connection")?; - crate::db::create_board(&conn, "test", "Test", "", false).context("create board")?; + crate::db::create_board(&conn, "test", "Test", "", false) + .context("create board") + .map(|_completed_value| ())?; crate::db::add_ban( &conn, &crate::utils::crypto::hash_ip("127.0.0.1", &crate::config::CONFIG.cookie_secret), "testing ban", None, ) - .context("add ban")?; + .context("add ban") + .map(|_completed_value| ())?; } let router = Router::new() @@ -2855,12 +2878,15 @@ async fn create_thread_xhr_captcha_failure_returns_inline_json_error() -> anyhow let state = crate::test_support::app_state(); { let conn = state.db.get().context("db connection")?; - crate::db::create_board(&conn, "test", "Test", "", false).context("create board")?; + crate::db::create_board(&conn, "test", "Test", "", false) + .context("create board") + .map(|_completed_value| ())?; conn.execute( "UPDATE boards SET allow_captcha = 1 WHERE short_name = 'test'", [], ) - .context("enable captcha")?; + .context("enable captcha") + .map(|_completed_value| ())?; } let router = Router::new() @@ -2905,14 +2931,14 @@ async fn create_thread_xhr_captcha_failure_returns_inline_json_error() -> anyhow Some(StatusCode::UNPROCESSABLE_ENTITY.as_str()) ); - let body = String::from_utf8( + let resolved_body = String::from_utf8( to_bytes(response.into_body(), usize::MAX) .await .context("response body")? .to_vec(), ) .context("utf8 body")?; - anyhow::ensure!(body.contains("CAPTCHA verification failed")); + anyhow::ensure!(resolved_body.contains("CAPTCHA verification failed")); Ok(()) } @@ -2956,7 +2982,7 @@ async fn duplicate_report_redirects_back_without_500() -> anyhow::Result<()> { .route("/report", post(super::file_report)) .with_state(state.clone()); - for _ in 0..2 { + for _ in 0_i32..2_i32 { let response = router .clone() .oneshot( @@ -3053,7 +3079,9 @@ async fn banned_actor_cannot_file_reports_and_unban_restores_reporting() -> anyh let ip_hash = loopback_ip_hash(); { let conn = state.db.get().context("db connection")?; - crate::db::add_ban(&conn, &ip_hash, "report ban reason", None).context("add ban")?; + crate::db::add_ban(&conn, &ip_hash, "report ban reason", None) + .context("add ban") + .map(|_completed_value| ())?; } let router = Router::new() @@ -3105,7 +3133,8 @@ async fn banned_actor_cannot_file_reports_and_unban_restores_reporting() -> anyh "DELETE FROM bans WHERE ip_hash = ?1", rusqlite::params![ip_hash], ) - .context("remove ban")?; + .context("remove ban") + .map(|_completed_value| ())?; } let after_unban = router @@ -3153,7 +3182,7 @@ fn seed_votable_poll(state: &crate::middleware::AppState) -> anyhow::Result let poll = crate::db::threads::PollInsert { question: "pick one", options: &["yes".to_owned(), "no".to_owned()], - expires_at: chrono::Utc::now().timestamp() + 3600, + expires_at: chrono::Utc::now().timestamp().saturating_add(3600), }; let (_, _, poll_id) = crate::db::create_thread_with_optional_poll( &conn, @@ -3180,7 +3209,9 @@ async fn banned_actor_cannot_vote_and_unban_restores_voting() -> anyhow::Result< let ip_hash = loopback_ip_hash(); { let conn = state.db.get().context("db connection")?; - crate::db::add_ban(&conn, &ip_hash, "vote ban reason", None).context("add ban")?; + crate::db::add_ban(&conn, &ip_hash, "vote ban reason", None) + .context("add ban") + .map(|_completed_value| ())?; } let router = Router::new() @@ -3230,7 +3261,8 @@ async fn banned_actor_cannot_vote_and_unban_restores_voting() -> anyhow::Result< "DELETE FROM bans WHERE ip_hash = ?1", rusqlite::params![ip_hash], ) - .context("remove ban")?; + .context("remove ban") + .map(|_completed_value| ())?; } let after_unban = router @@ -3251,7 +3283,9 @@ async fn create_thread_rejects_uploads_on_upload_disabled_board() -> anyhow::Res let state = crate::test_support::app_state(); { let mut conn = state.db.get().context("db connection")?; - crate::db::create_board(&conn, "test", "Test", "", false).context("create board")?; + crate::db::create_board(&conn, "test", "Test", "", false) + .context("create board") + .map(|_completed_value| ())?; crate::db::update_board_settings( &mut conn, 1, @@ -3325,14 +3359,16 @@ async fn view_locked_catalog_renders_unlock_page() -> anyhow::Result<()> { let state = crate::test_support::app_state(); { let conn = state.db.get().context("db connection")?; - crate::db::create_board(&conn, "secret", "Secret", "", false).context("create board")?; + crate::db::create_board(&conn, "secret", "Secret", "", false) + .context("create board") + .map(|_completed_value| ())?; let password_hash = crate::utils::crypto::hash_password("swordfish").context("hash password")?; conn.execute( "UPDATE boards SET access_mode = ?1, access_password_hash = ?2 WHERE short_name = 'secret'", rusqlite::params!["view_password", password_hash], ) - .context("update board access")?; + .context("update board access").map(|_completed_value| ())?; } let router = Router::new() @@ -3369,14 +3405,16 @@ async fn unlock_board_access_sets_cookie_and_redirects() -> anyhow::Result<()> { let state = crate::test_support::app_state(); { let conn = state.db.get().context("db connection")?; - crate::db::create_board(&conn, "secret", "Secret", "", false).context("create board")?; + crate::db::create_board(&conn, "secret", "Secret", "", false) + .context("create board") + .map(|_completed_value| ())?; let password_hash = crate::utils::crypto::hash_password("swordfish").context("hash password")?; conn.execute( "UPDATE boards SET access_mode = ?1, access_password_hash = ?2 WHERE short_name = 'secret'", rusqlite::params!["view_password", password_hash], ) - .context("update board access")?; + .context("update board access").map(|_completed_value| ())?; } let router = Router::new() @@ -3424,14 +3462,16 @@ async fn unlock_board_access_rejects_malformed_return_to_and_uses_board_default( let state = crate::test_support::app_state(); { let conn = state.db.get().context("db connection")?; - crate::db::create_board(&conn, "secret", "Secret", "", false).context("create board")?; + crate::db::create_board(&conn, "secret", "Secret", "", false) + .context("create board") + .map(|_completed_value| ())?; let password_hash = crate::utils::crypto::hash_password("swordfish").context("hash password")?; conn.execute( "UPDATE boards SET access_mode = ?1, access_password_hash = ?2 WHERE short_name = 'secret'", rusqlite::params!["view_password", password_hash], ) - .context("update board access")?; + .context("update board access").map(|_completed_value| ())?; } let router = Router::new() @@ -3470,14 +3510,16 @@ async fn changing_board_password_invalidates_existing_unlock_cookie() -> anyhow: let state = crate::test_support::app_state(); { let conn = state.db.get().context("db connection")?; - crate::db::create_board(&conn, "secret", "Secret", "", false).context("create board")?; + crate::db::create_board(&conn, "secret", "Secret", "", false) + .context("create board") + .map(|_completed_value| ())?; let password_hash = crate::utils::crypto::hash_password("swordfish").context("hash password")?; conn.execute( "UPDATE boards SET access_mode = ?1, access_password_hash = ?2 WHERE short_name = 'secret'", rusqlite::params!["view_password", password_hash], ) - .context("update board access")?; + .context("update board access").map(|_completed_value| ())?; } let router = Router::new() @@ -3519,10 +3561,11 @@ async fn changing_board_password_invalidates_existing_unlock_cookie() -> anyhow: "UPDATE boards SET access_password_hash = ?1 WHERE short_name = 'secret'", rusqlite::params![password_hash], ) - .context("change board password")?; + .context("change board password") + .map(|_completed_value| ())?; } - let response = router + let resolved_response = router .oneshot( Request::builder() .method("GET") @@ -3535,7 +3578,7 @@ async fn changing_board_password_invalidates_existing_unlock_cookie() -> anyhow: .await .context("catalog response")?; - ensure_eq!(response.status(), StatusCode::FORBIDDEN); + ensure_eq!(resolved_response.status(), StatusCode::FORBIDDEN); Ok(()) } @@ -3618,7 +3661,7 @@ async fn theme_redirect_persists_no_js_theme_without_csrf() -> anyhow::Result<() ensure_eq!(rejected.status(), StatusCode::FORBIDDEN); anyhow::ensure!(rejected.headers().get(header::SET_COOKIE).is_none()); - let accepted = router + let resolved_accepted = router .oneshot( Request::builder() .method("GET") @@ -3630,15 +3673,15 @@ async fn theme_redirect_persists_no_js_theme_without_csrf() -> anyhow::Result<() .await .context("accepted response")?; - ensure_eq!(accepted.status(), StatusCode::SEE_OTHER); + ensure_eq!(resolved_accepted.status(), StatusCode::SEE_OTHER); ensure_eq!( - accepted + resolved_accepted .headers() .get(header::LOCATION) .and_then(|value| value.to_str().ok()), Some("/secret/catalog") ); - anyhow::ensure!(accepted + anyhow::ensure!(resolved_accepted .headers() .get_all(header::SET_COOKIE) .iter() @@ -3650,7 +3693,7 @@ async fn theme_redirect_persists_no_js_theme_without_csrf() -> anyhow::Result<() #[test] fn user_preferences_from_jar_defaults_and_ignores_invalid_values() -> anyhow::Result<()> { let mut headers = HeaderMap::new(); - headers.insert( + let _previous_value = headers.insert( header::COOKIE, "rustchan_hide_nsfw=maybe; rustchan_video_audio=loud; rustchan_preferred_view=grid; rustchan_activity_badges=maybe" .parse() @@ -3834,7 +3877,7 @@ async fn set_user_preferences_accepts_admin_scoped_csrf_from_admin_panel() -> an async fn preferences_theme_cookie_drives_rendered_theme_after_reload() -> anyhow::Result<()> { let state = crate::test_support::app_state(); install_preference_test_themes(); - seed_board_with_thread(&state, "tech", "op")?; + seed_board_with_thread(&state, "tech", "op").map(|_completed_value| ())?; let router = Router::new() .route("/preferences", post(super::set_user_preferences)) .route("/{board}", get(super::board_index)) @@ -3977,7 +4020,7 @@ async fn partial_preference_updates_preserve_unrelated_cookies() -> anyhow::Resu async fn user_theme_overrides_configured_default_and_changes_etag() -> anyhow::Result<()> { let state = crate::test_support::app_state(); install_preference_test_themes(); - seed_board_with_thread(&state, "tech", "op")?; + seed_board_with_thread(&state, "tech", "op").map(|_completed_value| ())?; crate::templates::set_live_default_theme("forest"); let router = activity_router(state); @@ -4119,14 +4162,20 @@ async fn preference_specific_html_responses_vary_on_cookie() -> anyhow::Result<( async fn thread_updates_nav_uses_cookie_preferences() -> anyhow::Result<()> { let state = crate::test_support::app_state(); let conn = state.db.get().context("db connection")?; - crate::db::create_board(&conn, "tech", "Tech", "", false).context("create sfw board")?; - crate::db::create_board(&conn, "x", "Adult", "", true).context("create nsfw board")?; + crate::db::create_board(&conn, "tech", "Tech", "", false) + .context("create sfw board") + .map(|_completed_value| ())?; + crate::db::create_board(&conn, "x", "Adult", "", true) + .context("create nsfw board") + .map(|_completed_value| ())?; drop(conn); let (board_id, thread_id) = seed_board_with_thread(&state, "chat", "op")?; create_reply_on_thread(&state, board_id, thread_id, "reply")?; { - let conn = state.db.get().context("db connection")?; - crate::templates::set_live_boards(crate::db::get_all_boards(&conn).context("load boards")?); + let recovered_conn = state.db.get().context("db connection")?; + crate::templates::set_live_boards( + crate::db::get_all_boards(&recovered_conn).context("load boards")?, + ); } let router = activity_router(state); @@ -4164,12 +4213,14 @@ async fn malformed_board_password_hash_renders_misconfiguration_message() -> any let state = crate::test_support::app_state(); { let conn = state.db.get().context("db connection")?; - crate::db::create_board(&conn, "broken", "Broken", "", false).context("create board")?; + crate::db::create_board(&conn, "broken", "Broken", "", false) + .context("create board") + .map(|_completed_value| ())?; conn.execute( "UPDATE boards SET access_mode = ?1, access_password_hash = ?2 WHERE short_name = 'broken'", rusqlite::params!["view_password", "not-a-phc-string"], ) - .context("update board access")?; + .context("update board access").map(|_completed_value| ())?; } let router = Router::new() @@ -4211,14 +4262,16 @@ async fn unlock_board_access_rate_limits_repeated_failures() -> anyhow::Result<( let state = crate::test_support::app_state(); { let conn = state.db.get().context("db connection")?; - crate::db::create_board(&conn, "srate", "Secret", "", false).context("create board")?; + crate::db::create_board(&conn, "srate", "Secret", "", false) + .context("create board") + .map(|_completed_value| ())?; let password_hash = crate::utils::crypto::hash_password("swordfish").context("hash password")?; conn.execute( "UPDATE boards SET access_mode = ?1, access_password_hash = ?2 WHERE short_name = 'srate'", rusqlite::params!["view_password", password_hash], ) - .context("update board access")?; + .context("update board access").map(|_completed_value| ())?; } let router = Router::new() @@ -4282,14 +4335,16 @@ async fn locked_board_media_requires_unlock() -> anyhow::Result<()> { let state = crate::test_support::app_state(); { let conn = state.db.get().context("db connection")?; - crate::db::create_board(&conn, "secret", "Secret", "", false).context("create board")?; + crate::db::create_board(&conn, "secret", "Secret", "", false) + .context("create board") + .map(|_completed_value| ())?; let password_hash = crate::utils::crypto::hash_password("swordfish").context("hash password")?; conn.execute( "UPDATE boards SET access_mode = ?1, access_password_hash = ?2 WHERE short_name = 'secret'", rusqlite::params!["view_password", password_hash], ) - .context("update board access")?; + .context("update board access").map(|_completed_value| ())?; } let router = Router::new() @@ -4322,7 +4377,8 @@ async fn submit_appeal_is_rate_limited_to_one_open_window() -> anyhow::Result<() "test ban", None, ) - .context("add ban")?; + .context("add ban") + .map(|_completed_value| ())?; } let router = Router::new() diff --git a/src/handlers/captcha.rs b/src/handlers/captcha.rs index 10667626..2cbb923c 100644 --- a/src/handlers/captcha.rs +++ b/src/handlers/captcha.rs @@ -28,13 +28,14 @@ pub(in crate::server) async fn serve_captcha_image( )?; let mut headers = HeaderMap::new(); - headers.insert(header::CONTENT_TYPE, HeaderValue::from_static("image/png")); - headers.insert( + let _previous_value = + headers.insert(header::CONTENT_TYPE, HeaderValue::from_static("image/png")); + drop(headers.insert( header::CACHE_CONTROL, HeaderValue::from_static("private, no-store, no-cache, max-age=0"), - ); - headers.insert(header::PRAGMA, HeaderValue::from_static("no-cache")); - headers.insert(header::EXPIRES, HeaderValue::from_static("0")); + )); + drop(headers.insert(header::PRAGMA, HeaderValue::from_static("no-cache"))); + drop(headers.insert(header::EXPIRES, HeaderValue::from_static("0"))); Ok((StatusCode::OK, headers, png).into_response()) } diff --git a/src/handlers/favicon.rs b/src/handlers/favicon.rs index 39ebba32..1f84cf8a 100644 --- a/src/handlers/favicon.rs +++ b/src/handlers/favicon.rs @@ -37,7 +37,7 @@ async fn serve_named_global_favicon( |_| StatusCode::INTERNAL_SERVER_ERROR.into_response(), |resp| { let mut resp = resp.map(axum::body::Body::new); - resp.headers_mut().insert( + let _previous_value = resp.headers_mut().insert( header::CONTENT_TYPE, HeaderValue::from_static(favicon_content_type(file_name)), ); diff --git a/src/handlers/mod.rs b/src/handlers/mod.rs index 2f8ad974..76ae5ac8 100644 --- a/src/handlers/mod.rs +++ b/src/handlers/mod.rs @@ -31,6 +31,15 @@ use std::collections::HashSet; use std::time::Duration; use tokio::io::AsyncWriteExt as _; +/// Classify enhanced requests; authentication and CSRF checks remain independent. +#[must_use] +pub(crate) fn is_xml_http_request(headers: &axum::http::HeaderMap) -> bool { + headers + .get("x-requested-with") + .and_then(|value| value.to_str().ok()) + .is_some_and(|value| value.eq_ignore_ascii_case("XMLHttpRequest")) +} + /// MIME sniff bytes used by this handler. const MIME_SNIFF_BYTES: usize = 512; /// Text multipart field max bytes used by this handler. @@ -606,7 +615,10 @@ async fn discard_rejected_upload_field( budget: &mut PublicMultipartBudget, initial_excess_bytes: usize, ) -> bool { - let deadline = tokio::time::Instant::now() + REJECTED_UPLOAD_DISCARD_TIMEOUT; + let Some(deadline) = tokio::time::Instant::now().checked_add(REJECTED_UPLOAD_DISCARD_TIMEOUT) + else { + return false; + }; budget.rejected_upload = Some(RejectedUploadRecovery { bytes_seen: initial_excess_bytes, deadline, @@ -740,7 +752,7 @@ async fn read_upload_field( let submitted_filename = field.file_name().map(str::to_owned); let fname = submitted_filename .as_deref() - .filter(|name| !name.is_empty()) + .filter(|filename| !filename.is_empty()) .unwrap_or(default_name) .to_owned(); let upload = stream_field_to_temp_file( @@ -755,7 +767,7 @@ async fn read_upload_field( if upload.size_bytes == 0 { if submitted_filename .as_deref() - .is_some_and(|name| !name.is_empty()) + .is_some_and(|filename| !filename.is_empty()) { return Err(AppError::BadRequest("Uploaded file is empty.".into())); } @@ -789,7 +801,7 @@ pub(crate) struct PostFormData { // Poll fields are used only when creating a new thread. pub poll_question: String, pub poll_options: Vec, - /// Duration in seconds (parsed from value + unit) + /// Duration in seconds (parsed from value + unit). pub poll_duration_secs: Option, /// Sage — when true the reply must not bump the thread. pub sage: bool, @@ -823,7 +835,7 @@ pub(crate) fn native_post_error_response( false, "/", ); - response + let _previous_value = response .extensions_mut() .insert(crate::error::ErrorPage::Content { title: "Posting error".into(), @@ -928,7 +940,9 @@ async fn recover_rejected_post_tail( capture_post_draft(&name, &value, draft)?; } "file" | "audio_file" | "image_file" => { - draft.had_attachments |= field.file_name().is_some_and(|name| !name.is_empty()); + draft.had_attachments |= field + .file_name() + .is_some_and(|filename| !filename.is_empty()); loop { let next_chunk = field.chunk().await.map_err(|error| { multipart_read_error("rejected upload tail", &error) @@ -1100,7 +1114,9 @@ pub(crate) async fn parse_post_multipart( "Duplicate upload field 'file'.".into(), )); } - draft.had_attachments |= field.file_name().is_some_and(|name| !name.is_empty()); + draft.had_attachments |= field + .file_name() + .is_some_and(|filename| !filename.is_empty()); let upload_result = read_upload_field( field, max_image_size @@ -1125,7 +1141,9 @@ pub(crate) async fn parse_post_multipart( "Duplicate upload field 'audio_file'.".into(), )); } - draft.had_attachments |= field.file_name().is_some_and(|name| !name.is_empty()); + draft.had_attachments |= field + .file_name() + .is_some_and(|filename| !filename.is_empty()); let upload_result = read_upload_field( field, max_audio_size, @@ -1147,7 +1165,9 @@ pub(crate) async fn parse_post_multipart( "Duplicate upload field 'image_file'.".into(), )); } - draft.had_attachments |= field.file_name().is_some_and(|name| !name.is_empty()); + draft.had_attachments |= field + .file_name() + .is_some_and(|filename| !filename.is_empty()); let upload_result = read_upload_field( field, max_image_size, @@ -1221,7 +1241,7 @@ pub(crate) async fn parse_post_multipart( /// • "Insufficient disk space" → 413 `UploadTooLarge` /// • "File type not allowed" → 415 `InvalidMediaType` /// • "Not an audio file" → 415 `InvalidMediaType` -/// • anything else → 400 `BadRequest` +/// • anything else → 400 `BadRequest`. pub(crate) fn classify_upload_error(e: &anyhow::Error) -> AppError { let msg = e.to_string(); // Compare lower-cased so minor wording changes in save_upload don't silently @@ -1265,7 +1285,7 @@ use crate::models::Board; pub(crate) fn process_primary_upload( file_data: Option<(TempUpload, String)>, board: &Board, - conn: &rusqlite::Connection, + mut lookup: impl FnMut(&str) -> Result>, upload_dir: &str, save_root: &str, thumb_size: u32, @@ -1274,8 +1294,6 @@ pub(crate) fn process_primary_upload( max_audio_size: usize, max_pdf_size: usize, ffmpeg_available: bool, - ffprobe_available: bool, - ffmpeg_webp_available: bool, ) -> Result<(Option, Option)> { let Some((upload, fname)) = file_data else { return Ok((None, None)); @@ -1285,7 +1303,6 @@ pub(crate) fn process_primary_upload( let detected_mime = crate::utils::files::classify_upload_mime( upload.temp_file.path(), &upload.sniff_bytes, - ffprobe_available, allow_any_files, ) .map_err(|error| classify_upload_error(&error))?; @@ -1343,8 +1360,6 @@ pub(crate) fn process_primary_upload( max_audio_size, max_pdf_size, ffmpeg_available, - ffprobe_available, - ffmpeg_webp_available, allow_any_files, }; let validated = crate::utils::files::storage::validate_upload_for_storage( @@ -1368,7 +1383,7 @@ pub(crate) fn process_primary_upload( // record_file_hash uses INSERT OR REPLACE, so the cache entry is // automatically refreshed to point at the newly saved files. let hash = sha256_file_hex(upload.temp_file.path())?; - if let Some(cached) = crate::db::find_file_by_hash(conn, &hash)? { + if let Some(cached) = lookup(&hash)? { let same_board_cache = cached_paths_belong_to_board(&cached, &board.short_name); let file_ok = std::path::Path::new(upload_dir) .join(&cached.file_path) @@ -1426,15 +1441,10 @@ fn upload_path_belongs_to_board(path: &str, board_short: &str) -> bool { path.split('/').next() == Some(board_short) } -fn temp_upload_mime( - upload: &TempUpload, - ffprobe_available: bool, - allow_any_files: bool, -) -> Result { +fn temp_upload_mime(upload: &TempUpload, allow_any_files: bool) -> Result { crate::utils::files::classify_upload_mime( upload.temp_file.path(), &upload.sniff_bytes, - ffprobe_available, allow_any_files, ) .map_err(|error| classify_upload_error(&error)) @@ -1451,7 +1461,6 @@ pub(crate) fn process_audio_combo( board: &Board, upload_dir: &str, max_audio_size: usize, - ffprobe_available: bool, ) -> Result> { let Some((audio_upload, aud_fname)) = audio_file_data else { return Ok(None); @@ -1480,7 +1489,6 @@ pub(crate) fn process_audio_combo( upload_dir, &board.short_name, max_audio_size, - ffprobe_available, ) .map_err(|e| classify_upload_error(&e))?; @@ -1500,7 +1508,7 @@ pub(crate) fn process_audio_first_uploads( image_file_data: Option<(TempUpload, String)>, fallback_file_data: Option<(TempUpload, String)>, board: &Board, - conn: &rusqlite::Connection, + mut lookup: impl FnMut(&str) -> Result>, upload_dir: &str, save_root_str: &str, thumb_size: u32, @@ -1509,8 +1517,6 @@ pub(crate) fn process_audio_first_uploads( max_audio_size: usize, max_pdf_size: usize, ffmpeg_available: bool, - ffprobe_available: bool, - ffmpeg_webp_available: bool, ) -> Result<( Option, Option, @@ -1519,11 +1525,11 @@ pub(crate) fn process_audio_first_uploads( let allow_any_files = crate::config::CONFIG.enable_any_file_uploads_feature && board.allow_any_files; let has_audio_or_image_upload = audio_file_data.is_some() || image_file_data.is_some(); - let save_primary = |file_data| { + let mut save_primary = |file_data| { process_primary_upload( file_data, board, - conn, + &mut lookup, upload_dir, save_root_str, thumb_size, @@ -1532,8 +1538,6 @@ pub(crate) fn process_audio_first_uploads( max_audio_size, max_pdf_size, ffmpeg_available, - ffprobe_available, - ffmpeg_webp_available, ) }; @@ -1553,14 +1557,13 @@ pub(crate) fn process_audio_first_uploads( board, save_root_str, max_audio_size, - ffprobe_available, )?; return Ok((primary, audio, primary_hash)); } if let Some((audio_upload, audio_name)) = audio_file_data { - let audio_mime = temp_upload_mime(&audio_upload, ffprobe_available, allow_any_files)?; + let audio_mime = temp_upload_mime(&audio_upload, allow_any_files)?; if crate::models::MediaType::from_mime(&audio_mime) != crate::models::MediaType::Audio { return Err(AppError::BadRequest( "The audio slot only accepts audio files.".into(), @@ -1600,7 +1603,7 @@ fn sha256_file_hex(path: &std::path::Path) -> Result { /// post. Shared by `create_thread` and `post_reply`. pub(crate) fn enqueue_post_jobs( job_queue: &JobQueue, - conn: &rusqlite::Connection, + pool: &crate::db::DbPool, post_id: i64, ip_hash: &str, body_len: usize, @@ -1626,7 +1629,9 @@ pub(crate) fn enqueue_post_jobs( | crate::models::MediaType::Other => None, }; if let Some(j) = job { - match job_queue.enqueue_media(conn, &j) { + let conn = pool.get()?; + let enqueued = job_queue.enqueue_media(&conn, &j); + match enqueued { Ok(crate::workers::EnqueueOutcome::Enqueued(job_id)) => tracing::debug!( target: "workers", post_id, @@ -1683,6 +1688,7 @@ mod tests { parse_post_multipart, process_audio_first_uploads, MultipartEnvelopeScanner, TempUpload, PUBLIC_MULTIPART_ENVELOPE_LIMIT_MARKER, PUBLIC_MULTIPART_ENVELOPE_MAX_BYTES, }; + use crate::test_support::valid_pdf; use anyhow::{bail, ensure, Context as _}; use axum::{ body::Body, @@ -1719,19 +1725,6 @@ mod tests { )) } - fn valid_pdf() -> &'static [u8] { - b"%PDF-1.4 -1 0 obj << /Type /Catalog /Pages 2 0 R >> endobj -2 0 obj << /Type /Pages /Kids [3 0 R] /Count 1 >> endobj -3 0 obj << /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] /Resources << >> /Contents 4 0 R >> endobj -4 0 obj << /Length 0 >> stream - -endstream endobj -trailer << /Root 1 0 R >> -%%EOF -" - } - async fn multipart_from_bytes( boundary: &str, body: Vec, @@ -1788,19 +1781,23 @@ trailer << /Root 1 0 R >> ); ensure!(budget.bytes_seen == 2048); - let (boundary, body) = + let (response_boundary, resolved_body) = multipart_body_with_files(&[], &[("file", "oversized.png", &[1; 2048], "image/png")]); - let mut multipart = multipart_from_bytes(&boundary, body).await?; - let mut field = multipart + let mut resolved_multipart = + multipart_from_bytes(&response_boundary, resolved_body).await?; + let mut resolved_field = resolved_multipart .next_field() .await? .context("missing upload field")?; - let mut budget = super::PublicMultipartBudget { + let mut resolved_budget = super::PublicMultipartBudget { fields_seen: 0, bytes_seen: super::PUBLIC_MULTIPART_AGGREGATE_MAX_BYTES, ..super::PublicMultipartBudget::default() }; - ensure!(!super::discard_rejected_upload_field(&mut field, &mut budget, 0).await); + ensure!( + !super::discard_rejected_upload_field(&mut resolved_field, &mut resolved_budget, 0) + .await + ); Ok(()) } @@ -2067,7 +2064,8 @@ trailer << /Root 1 0 R >> )", [], ) - .context("create file_hashes table")?; + .context("create file_hashes table") + .map(|_completed_value| ())?; Ok(()) } @@ -2117,7 +2115,8 @@ trailer << /Root 1 0 R >> draft: &mut crate::templates::forms::PostFormState::default(), }, ) - .await?; + .await + .map(|_completed_value| ())?; Ok("ok") } @@ -2209,7 +2208,8 @@ trailer << /Root 1 0 R >> draft: &mut crate::templates::forms::PostFormState::default(), }, ) - .await?; + .await + .map(|_completed_value| ())?; Ok("ok") } @@ -2426,26 +2426,26 @@ trailer << /Root 1 0 R >> ensure!(response.status() == StatusCode::OK); let over_pdf = vec![b'p'; 2_049]; - let (boundary, body) = multipart_body_with_files( + let (response_boundary, resolved_body) = multipart_body_with_files( &[("_csrf", "csrf123"), ("body", "pdf")], &[("file", "over.pdf", &over_pdf, "application/pdf")], ); - let response = router + let resolved_response = router .oneshot( Request::builder() .method("POST") .uri("/parse") .header( header::CONTENT_TYPE, - format!("multipart/form-data; boundary={boundary}"), + format!("multipart/form-data; boundary={response_boundary}"), ) - .body(Body::from(body)) + .body(Body::from(resolved_body)) .context("build over-limit PDF multipart request")?, ) .await .context("receive over-limit PDF multipart response")?; - ensure!(response.status() == StatusCode::PAYLOAD_TOO_LARGE); + ensure!(resolved_response.status() == StatusCode::PAYLOAD_TOO_LARGE); Ok(()) } @@ -2484,9 +2484,9 @@ trailer << /Root 1 0 R >> } ensure!(budget.note_field().is_err()); - let mut budget = super::PublicMultipartBudget::default(); - budget.note_chunk(super::PUBLIC_MULTIPART_AGGREGATE_MAX_BYTES)?; - ensure!(budget.note_chunk(1).is_err()); + let mut resolved_budget = super::PublicMultipartBudget::default(); + resolved_budget.note_chunk(super::PUBLIC_MULTIPART_AGGREGATE_MAX_BYTES)?; + ensure!(resolved_budget.note_chunk(1).is_err()); Ok(()) } @@ -2505,7 +2505,7 @@ trailer << /Root 1 0 R >> None, Some(other), &board, - &conn, + |candidate_hash| Ok(crate::db::find_file_by_hash(&conn, candidate_hash)?), boards_dir .path() .to_str() @@ -2520,8 +2520,6 @@ trailer << /Root 1 0 R >> 1024 * 1024, 1024 * 1024, false, - false, - false, ); let Err(error) = result else { @@ -2569,7 +2567,7 @@ trailer << /Root 1 0 R >> let result = super::process_primary_upload( Some(upload), &board, - &conn, + |candidate_hash| Ok(crate::db::find_file_by_hash(&conn, candidate_hash)?), uploads_dir .path() .to_str() @@ -2584,8 +2582,6 @@ trailer << /Root 1 0 R >> 1024 * 1024, 1024 * 1024, false, - false, - false, ); let Err(error) = result else { @@ -2636,7 +2632,7 @@ trailer << /Root 1 0 R >> let (uploaded, primary_hash) = super::process_primary_upload( Some(temp_upload("doc.pdf", pdf)?), &board, - &conn, + |candidate_hash| Ok(crate::db::find_file_by_hash(&conn, candidate_hash)?), uploads_dir .path() .to_str() @@ -2651,8 +2647,6 @@ trailer << /Root 1 0 R >> 1024 * 1024, 1024 * 1024, false, - false, - false, ) .context("accept PDF upload")?; let uploaded = uploaded.context("PDF upload result was empty")?; @@ -2678,7 +2672,7 @@ trailer << /Root 1 0 R >> let result = super::process_primary_upload( Some(temp_upload("doc.pdf", valid_pdf())?), &board, - &conn, + |candidate_hash| Ok(crate::db::find_file_by_hash(&conn, candidate_hash)?), uploads_dir .path() .to_str() @@ -2693,8 +2687,6 @@ trailer << /Root 1 0 R >> 1024 * 1024, 1024 * 1024, false, - false, - false, ); let Err(error) = result else { @@ -2722,7 +2714,7 @@ trailer << /Root 1 0 R >> let result = super::process_primary_upload( Some(temp_upload("doc.pdf", valid_pdf())?), &board, - &conn, + |candidate_hash| Ok(crate::db::find_file_by_hash(&conn, candidate_hash)?), uploads_dir .path() .to_str() @@ -2737,8 +2729,6 @@ trailer << /Root 1 0 R >> 1024 * 1024, board.max_pdf_size_bytes(), false, - false, - false, ); let Err(error) = result else { @@ -2762,7 +2752,7 @@ trailer << /Root 1 0 R >> let result = super::process_primary_upload( Some(temp_upload("not-really.pdf", b"plain text")?), &board, - &conn, + |candidate_hash| Ok(crate::db::find_file_by_hash(&conn, candidate_hash)?), uploads_dir .path() .to_str() @@ -2777,8 +2767,6 @@ trailer << /Root 1 0 R >> 1024 * 1024, 1024 * 1024, false, - false, - false, ); let Err(error) = result else { @@ -2806,7 +2794,7 @@ trailer << /Root 1 0 R >> let (uploaded, _) = super::process_primary_upload( Some(temp_upload("doc.pdf", valid_pdf())?), &board, - &conn, + |candidate_hash| Ok(crate::db::find_file_by_hash(&conn, candidate_hash)?), uploads_dir .path() .to_str() @@ -2821,8 +2809,6 @@ trailer << /Root 1 0 R >> 1024 * 1024, 1024 * 1024, false, - false, - false, ) .context("accept PDF upload")?; let uploaded = uploaded.context("PDF upload result was empty")?; diff --git a/src/handlers/posting.rs b/src/handlers/posting.rs index aae654cc..5b8de2d0 100644 --- a/src/handlers/posting.rs +++ b/src/handlers/posting.rs @@ -36,6 +36,7 @@ pub(super) struct SubmitPostCommand { pub identity_key: String, pub cookie_secret: String, pub admin_session_id: Option, + pub access_cookie: Option, pub ban_csrf_token: String, pub submission_token: String, pub name: String, @@ -49,8 +50,6 @@ pub(super) struct SubmitPostCommand { pub upload_dir: String, pub thumb_size: u32, pub ffmpeg_available: bool, - pub ffprobe_available: bool, - pub ffmpeg_webp_available: bool, } pub(super) struct SubmitPostResult { @@ -94,8 +93,6 @@ struct UploadConfig<'a> { pub max_audio_size: usize, pub max_pdf_size: usize, pub ffmpeg_available: bool, - pub ffprobe_available: bool, - pub ffmpeg_webp_available: bool, } #[derive(Clone)] @@ -111,11 +108,7 @@ struct ProcessedUploads { } impl ProcessedUploads { - pub(crate) fn rollback_new_files( - &self, - conn: &rusqlite::Connection, - upload_dir: &str, - ) -> Result<()> { + pub(crate) fn rollback_new_files(&self, pool: &db::DbPool, upload_dir: &str) -> Result<()> { if let Some(pending) = self.pending_finalize.as_ref() { let stage_dir = std::path::Path::new(&pending.payload.stage_dir); if stage_dir.exists() { @@ -141,7 +134,9 @@ impl ProcessedUploads { } match crate::utils::files::delete_file_checked(upload_dir, &primary.file_path) { Ok(()) => { - if let Err(error) = db::delete_file_hash_by_path(conn, &primary.file_path) { + if let Err(error) = + db::delete_file_hash_by_path(&*pool.get()?, &primary.file_path) + { return Err(AppError::Internal(error)); } } @@ -210,7 +205,7 @@ fn build_upload_finalize_payload( relative_paths.dedup(); for relative_path in &relative_paths { - artifact_sha256.insert( + let _previous_value = artifact_sha256.insert( relative_path.clone(), crate::pending_fs::staged_upload_artifact_sha256(stage_dir, relative_path, "required") .map_err(AppError::Internal)?, @@ -223,7 +218,7 @@ fn build_upload_finalize_payload( match std::fs::symlink_metadata(&staged_path) { Ok(metadata) if metadata.file_type().is_file() => { optional_paths.push(file.thumb_path.clone()); - artifact_sha256.insert( + let _previous_value = artifact_sha256.insert( file.thumb_path.clone(), crate::pending_fs::staged_upload_artifact_sha256( stage_dir, @@ -314,17 +309,13 @@ fn build_pending_upload_op( })) } -fn finalize_pending_uploads( - conn: &rusqlite::Connection, - upload_dir: &str, - uploads: &ProcessedUploads, -) { +fn finalize_pending_uploads(pool: &db::DbPool, upload_dir: &str, uploads: &ProcessedUploads) { let Some(pending) = uploads.pending_finalize.as_ref() else { return; }; - match crate::pending_fs::finalize_upload_payload_for_op( - conn, + match crate::pending_fs::finalize_upload_payload_with_pool( + pool, upload_dir, Some(&pending.op_id), &pending.payload, @@ -340,6 +331,97 @@ fn finalize_pending_uploads( } } +/// Clean uncommitted staged files if the final pool checkout fails. +fn checkout_post_connection( + pool: &db::DbPool, + uploads: &ProcessedUploads, + upload_dir: &str, +) -> Result> { + match pool.get() { + Ok(conn) => Ok(conn), + Err(error) => { + uploads.rollback_new_files(pool, upload_dir)?; + Err(error.into()) + } + } +} + +/// Preserve a transaction's typed public rejection. +fn post_creation_error(error: anyhow::Error) -> AppError { + match error.downcast::() { + Ok(error) => error, + Err(error) => error.into(), + } +} + +/// Check mutable request policy while the post transaction owns the write lock. +fn revalidate_posting_policy( + conn: &rusqlite::Connection, + prepared: &Board, + prepared_filters: &[(String, String)], + admin_session_id: Option<&str>, + access_cookie: Option<&str>, + ip_hash: &str, + ban_csrf_token: &str, +) -> Result<()> { + let context = crate::handlers::board::load_board_access_context( + conn, + &prepared.short_name, + admin_session_id, + access_cookie, + )?; + let current = &context.board; + if current.id != prepared.id || !context.can_post { + return Err(AppError::Forbidden( + "Board posting access changed. Reload the board.".into(), + )); + } + crate::handlers::board::ensure_actor_not_banned(conn, ip_hash, ban_csrf_token.to_owned())?; + // These settings determined media acceptance and body preparation. Refuse + // stale preparation rather than publishing under a different policy. + let media_policy = |board: &Board| { + ( + ( + board.allow_images, + board.allow_video, + board.allow_audio, + board.allow_pdf, + board.allow_any_files, + ), + ( + board.max_image_size, + board.max_video_size, + board.max_audio_size, + board.max_pdf_size, + ), + ( + board.allow_tripcodes, + board.allow_captcha, + board.collapse_greentext, + ), + ) + }; + if media_policy(current) != media_policy(prepared) + || load_word_filters(conn)? != prepared_filters + { + return Err(AppError::Conflict( + "Board posting settings changed. Reload the board and try again.".into(), + )); + } + if !context.is_admin && current.post_cooldown_secs > 0 { + if let Some(seconds) = db::get_seconds_since_last_post(conn, current.id, ip_hash)? { + let remaining = current.post_cooldown_secs.saturating_sub(seconds); + if remaining > 0 { + return Err(AppError::BadRequest(format!( + "Please wait {remaining} more second{} before posting again.", + if remaining == 1 { "" } else { "s" } + ))); + } + } + } + Ok(()) +} + pub(super) fn is_admin_session( conn: &rusqlite::Connection, admin_session_id: Option<&str>, @@ -393,7 +475,7 @@ fn process_uploads( file_data: Option<(crate::handlers::TempUpload, String)>, audio_file_data: Option<(crate::handlers::TempUpload, String)>, board: &Board, - conn: &rusqlite::Connection, + pool: &db::DbPool, config: &UploadConfig<'_>, ) -> Result { let stage_root = @@ -413,7 +495,10 @@ fn process_uploads( image_file_data, file_data, board, - conn, + |hash| { + let conn = pool.get()?; + Ok(db::find_file_by_hash(&conn, hash)?) + }, config.upload_dir, save_root_str, config.thumb_size, @@ -422,8 +507,6 @@ fn process_uploads( config.max_audio_size, config.max_pdf_size, config.ffmpeg_available, - config.ffprobe_available, - config.ffmpeg_webp_available, ); let (mut primary, mut audio, primary_hash) = match processed { @@ -434,17 +517,24 @@ fn process_uploads( } }; - let pending_finalize = stage_root + let pending_payload = stage_root .as_ref() .map(|stage_dir| { build_upload_finalize_payload(stage_dir, primary.as_mut(), audio.as_mut(), primary_hash) }) - .transpose()? - .flatten() - .map(|payload| PendingUploadFinalize { - op_id: uuid::Uuid::new_v4().to_string(), - payload, - }); + .transpose(); + let pending_finalize = match pending_payload { + Ok(payload) => payload, + Err(error) => { + cleanup_unused_upload_stage(stage_root.as_deref()); + return Err(error); + } + } + .flatten() + .map(|payload| PendingUploadFinalize { + op_id: uuid::Uuid::new_v4().to_string(), + payload, + }); if pending_finalize.is_none() { cleanup_unused_upload_stage(stage_root.as_deref()); @@ -501,6 +591,15 @@ fn build_new_post( } } +/// Submit with independently scoped preparation, persistence and promotion. +pub(super) fn submit_post( + pool: &db::DbPool, + job_queue: &crate::workers::JobQueue, + command: SubmitPostCommand, +) -> Result { + submit_post_with_preparation(pool, job_queue, command, || Ok(())) +} + #[expect( clippy::cognitive_complexity, reason = "post validation and transactional creation share one consistency boundary" @@ -509,10 +608,12 @@ fn build_new_post( clippy::too_many_lines, reason = "validation, transactional insertion, attachment updates, and job enqueueing share one boundary" )] -pub(super) fn submit_post( - conn: &rusqlite::Connection, +/// The preparation callback permits deterministic interleaving in regression tests. +fn submit_post_with_preparation( + pool: &db::DbPool, job_queue: &crate::workers::JobQueue, command: SubmitPostCommand, + prepare: impl FnOnce() -> Result<()>, ) -> Result { let SubmitPostCommand { mode, @@ -520,6 +621,7 @@ pub(super) fn submit_post( identity_key, cookie_secret, admin_session_id, + access_cookie, ban_csrf_token, submission_token, name, @@ -533,11 +635,10 @@ pub(super) fn submit_post( upload_dir, thumb_size, ffmpeg_available, - ffprobe_available, - ffmpeg_webp_available, } = command; - let board = db::get_board_by_short(conn, &board_short)? + let conn = pool.get()?; + let board = db::get_board_by_short(&conn, &board_short)? .ok_or_else(|| AppError::NotFound(format!("Board /{board_short}/ not found")))?; let effective_max_image_size = board.max_image_size_bytes(); let effective_max_video_size = board.max_video_size_bytes(); @@ -546,7 +647,7 @@ pub(super) fn submit_post( let reply_context = match &mode { SubmitPostMode::Reply { thread_id, sage } => { - let thread = db::get_thread(conn, *thread_id)? + let thread = db::get_thread(&conn, *thread_id)? .ok_or_else(|| AppError::NotFound("Thread not found.".into()))?; if thread.board_id != board.id { @@ -559,20 +660,25 @@ pub(super) fn submit_post( return Err(AppError::Forbidden("This thread is archived.".into())); } - Some((*thread_id, *sage, thread.reply_count)) + Some((*thread_id, *sage)) } - SubmitPostMode::NewThread { .. } => None, + SubmitPostMode::NewThread { + subject: _, + poll_question: _, + poll_options: _, + poll_duration_secs: _, + } => None, }; let ip_hash = hash_ip(&identity_key, &cookie_secret); - crate::handlers::board::ensure_actor_not_banned(conn, &ip_hash, ban_csrf_token)?; - if let Some(existing) = db::get_post_submission(conn, &submission_token, &ip_hash, board.id)? { - return existing_submission_result(conn, board.short_name, existing); + crate::handlers::board::ensure_actor_not_banned(&conn, &ip_hash, ban_csrf_token.clone())?; + if let Some(existing) = db::get_post_submission(&conn, &submission_token, &ip_hash, board.id)? { + return existing_submission_result(&conn, board.short_name, existing); } - let is_admin = is_admin_session(conn, admin_session_id.as_deref()); + let is_admin = is_admin_session(&conn, admin_session_id.as_deref()); if board.post_cooldown_secs > 0 && !is_admin { - let elapsed = db::get_seconds_since_last_post(conn, board.id, &ip_hash)?; + let elapsed = db::get_seconds_since_last_post(&conn, board.id, &ip_hash)?; if let Some(secs) = elapsed { let remaining = board.post_cooldown_secs.saturating_sub(secs); if remaining > 0 { @@ -589,7 +695,9 @@ pub(super) fn submit_post( .map_err(|error| AppError::BadRequest(error.user_message().to_owned()))?; } - let filters = load_word_filters(conn)?; + let filters = load_word_filters(&conn)?; + drop(conn); + prepare()?; let (name, tripcode) = resolve_post_identity(&name, board.allow_tripcodes); let board_allows_media = board.allow_images || board.allow_video @@ -610,7 +718,7 @@ pub(super) fn submit_post( file_data, audio_file_data, &board, - conn, + pool, &UploadConfig { upload_dir: &upload_dir, thumb_size, @@ -619,12 +727,16 @@ pub(super) fn submit_post( max_audio_size: effective_max_audio_size, max_pdf_size: effective_max_pdf_size, ffmpeg_available, - ffprobe_available, - ffmpeg_webp_available, }, )?; let deletion_token = resolve_deletion_token(&deletion_token); - let pending_upload_op = build_pending_upload_op(&uploads)?; + let pending_upload_op = match build_pending_upload_op(&uploads) { + Ok(op) => op, + Err(error) => { + uploads.rollback_new_files(pool, &upload_dir)?; + return Err(error); + } + }; let deduplicated_paths: Vec<&str> = uploads .primary .iter() @@ -632,6 +744,18 @@ pub(super) fn submit_post( .map(|upload| upload.file_path.as_str()) .collect(); + let validate = |validation_conn: &rusqlite::Connection| { + revalidate_posting_policy( + validation_conn, + &board, + &filters, + admin_session_id.as_deref(), + access_cookie.as_deref(), + &ip_hash, + &ban_csrf_token, + ) + .map_err(anyhow::Error::new) + }; let (post_id, thread_id, redirect_url, prune_board_id) = match mode { SubmitPostMode::NewThread { subject, @@ -663,19 +787,19 @@ pub(super) fn submit_post( None } else { if q.is_empty() { - uploads.rollback_new_files(conn, &upload_dir)?; + uploads.rollback_new_files(pool, &upload_dir)?; return Err(AppError::BadRequest( "Polls need a question and at least two options.".into(), )); } if valid_opts.len() < 2 { - uploads.rollback_new_files(conn, &upload_dir)?; + uploads.rollback_new_files(pool, &upload_dir)?; return Err(AppError::BadRequest( "Polls need a question and at least two options.".into(), )); } let Some(secs) = poll_duration_secs else { - uploads.rollback_new_files(conn, &upload_dir)?; + uploads.rollback_new_files(pool, &upload_dir)?; return Err(AppError::BadRequest( "A duration is required when creating a poll.".into(), )); @@ -688,28 +812,37 @@ pub(super) fn submit_post( expires_at, }) }; + let recovered_conn = checkout_post_connection(pool, &uploads, &upload_dir)?; let create_result = db::threads::create_thread_submission( - conn, + &recovered_conn, board.id, subject.as_deref(), &new_post, &submission_token, poll_insert.as_ref(), - db::threads::PostFilesystemCommit::new( + db::threads::PostFilesystemCommit::new_with_validation( pending_upload_op.as_ref(), &deduplicated_paths, true, + validate, ), ); let (thread_id, post_id, _) = match create_result { - Ok(db::threads::PostCreationOutcome::Created(ids)) => ids, + Ok(db::threads::PostCreationOutcome::Created(ids)) => { + drop(recovered_conn); + ids + } Ok(db::threads::PostCreationOutcome::Replayed(existing)) => { - uploads.rollback_new_files(conn, &upload_dir)?; - return existing_submission_result(conn, board.short_name, existing); + let result = + existing_submission_result(&recovered_conn, board.short_name, existing); + drop(recovered_conn); + uploads.rollback_new_files(pool, &upload_dir)?; + return result; } Err(error) => { - uploads.rollback_new_files(conn, &upload_dir)?; - return Err(error.into()); + drop(recovered_conn); + uploads.rollback_new_files(pool, &upload_dir)?; + return Err(post_creation_error(error)); } }; ( @@ -719,10 +852,13 @@ pub(super) fn submit_post( Some(board.id), ) } - SubmitPostMode::Reply { .. } => { - let (thread_id, sage, reply_count) = reply_context + SubmitPostMode::Reply { + thread_id: _, + sage: _, + } => { + let (thread_id, sage) = reply_context .ok_or_else(|| AppError::Internal(anyhow::anyhow!("Missing reply context")))?; - let should_bump = !sage && reply_count < board.bump_limit; + let should_bump = !sage; let new_post = build_new_post( thread_id, board.id, @@ -736,28 +872,37 @@ pub(super) fn submit_post( deletion_token.clone(), false, ); + let recovered_conn = checkout_post_connection(pool, &uploads, &upload_dir)?; let post_id = match db::threads::create_reply_submission( - conn, + &recovered_conn, &new_post, &submission_token, should_bump, - db::threads::PostFilesystemCommit::new( + db::threads::PostFilesystemCommit::new_with_validation( pending_upload_op.as_ref(), &deduplicated_paths, false, + validate, ), ) { - Ok(db::threads::PostCreationOutcome::Created(post_id)) => post_id, + Ok(db::threads::PostCreationOutcome::Created(post_id)) => { + drop(recovered_conn); + post_id + } Ok(db::threads::PostCreationOutcome::Replayed(existing)) => { - uploads.rollback_new_files(conn, &upload_dir)?; - return existing_submission_result(conn, board.short_name, existing); + let result = + existing_submission_result(&recovered_conn, board.short_name, existing); + drop(recovered_conn); + uploads.rollback_new_files(pool, &upload_dir)?; + return result; } Err(error) => { - uploads.rollback_new_files(conn, &upload_dir)?; + drop(recovered_conn); + uploads.rollback_new_files(pool, &upload_dir)?; if let Some(closed) = error.downcast_ref::() { return Err(AppError::Forbidden(closed.message().to_owned())); } - return Err(error.into()); + return Err(post_creation_error(error)); } }; ( @@ -769,10 +914,10 @@ pub(super) fn submit_post( } }; - finalize_pending_uploads(conn, &upload_dir, &uploads); + finalize_pending_uploads(pool, &upload_dir, &uploads); crate::handlers::enqueue_post_jobs( job_queue, - conn, + pool, post_id, &ip_hash, body_text.len(), @@ -780,7 +925,8 @@ pub(super) fn submit_post( &board.short_name, )?; if let Some(prune_board_id) = prune_board_id { - if let Err(error) = job_queue.notify_persisted_thread_prune(conn) { + let recovered_conn = pool.get()?; + if let Err(error) = job_queue.notify_persisted_thread_prune(&recovered_conn) { tracing::warn!( target: "workers", board = %board.short_name, @@ -798,7 +944,7 @@ pub(super) fn submit_post( } else { tracing::info!(target: "board", post_id = post_id, thread_id = thread_id, board = %board.short_name, "Reply posted"); } - if let Err(error) = crate::media::prune::run_configured_prune(conn, &upload_dir) { + if let Err(error) = crate::media::prune::run_configured_prune_with_pool(pool, &upload_dir) { tracing::warn!( target: "media_prune", post_id, @@ -807,7 +953,8 @@ pub(super) fn submit_post( ); } - let stored_post = db::get_post(conn, post_id)? + let recovered_conn = pool.get()?; + let stored_post = db::get_post(&recovered_conn, post_id)? .ok_or_else(|| AppError::NotFound("Posted row not found.".into()))?; Ok(SubmitPostResult { @@ -857,7 +1004,7 @@ mod tests { }}; } - const TEST_BOARD: &str = "test"; + pub(super) const TEST_BOARD: &str = "test"; const TEST_COOKIE_SECRET: &str = "cookie-secret"; const TEST_IDENTITY_KEY: &str = "127.0.0.1"; @@ -892,7 +1039,7 @@ mod tests { } } - fn thread_command( + pub(super) fn thread_command( board_short: &str, submission_token: &str, body: &str, @@ -909,6 +1056,7 @@ mod tests { identity_key: TEST_IDENTITY_KEY.to_owned(), cookie_secret: TEST_COOKIE_SECRET.to_owned(), admin_session_id: None, + access_cookie: None, ban_csrf_token: "ban-csrf".to_owned(), submission_token: submission_token.to_owned(), name: "anon".to_owned(), @@ -922,8 +1070,6 @@ mod tests { upload_dir: upload_dir.to_owned(), thumb_size: 250, ffmpeg_available: false, - ffprobe_available: false, - ffmpeg_webp_available: false, } } @@ -947,6 +1093,7 @@ mod tests { identity_key: TEST_IDENTITY_KEY.to_owned(), cookie_secret: TEST_COOKIE_SECRET.to_owned(), admin_session_id: None, + access_cookie: None, ban_csrf_token: "ban-csrf".to_owned(), submission_token: submission_token.to_owned(), name: "anon".to_owned(), @@ -960,8 +1107,6 @@ mod tests { upload_dir: upload_dir.to_owned(), thumb_size: 250, ffmpeg_available: false, - ffprobe_available: false, - ffmpeg_webp_available: false, } } @@ -981,6 +1126,7 @@ mod tests { identity_key: TEST_IDENTITY_KEY.to_owned(), cookie_secret: TEST_COOKIE_SECRET.to_owned(), admin_session_id: None, + access_cookie: None, ban_csrf_token: "ban-csrf".to_owned(), submission_token: submission_token.to_owned(), name: "anon".to_owned(), @@ -994,12 +1140,10 @@ mod tests { upload_dir: upload_dir.to_owned(), thumb_size: 250, ffmpeg_available: false, - ffprobe_available: false, - ffmpeg_webp_available: false, } } - fn temp_upload(name: &str, bytes: &[u8]) -> Result<(TempUpload, String)> { + pub(super) fn temp_upload(name: &str, bytes: &[u8]) -> Result<(TempUpload, String)> { let temp_file = tempfile::Builder::new() .prefix("rustchan-posting-test-upload-") .tempfile() @@ -1027,7 +1171,7 @@ mod tests { } fn flac_header_bytes() -> Vec { - b"fLaC\x00\x00\x00\x22tiny test flac bytes".to_vec() + include_bytes!("../../tests/fixtures/media/tone.flac").to_vec() } fn malformed_aac_bytes() -> Result> { @@ -1043,7 +1187,7 @@ mod tests { } fn mp4_header_bytes() -> Vec { - b"\x00\x00\x00\x18ftypisom\x00\x00\x02\x00isomiso2mp41".to_vec() + include_bytes!("../../tests/fixtures/media/video.mp4").to_vec() } fn pending_upload_stage_count(upload_dir: &std::path::Path) -> Result { @@ -1083,17 +1227,19 @@ mod tests { let pool = state.db.clone(); let job_queue = std::sync::Arc::clone(&job_queue); handles.push(scope.spawn(move || { - barrier.wait(); - let conn = pool.get()?; - submit_post(&conn, job_queue.as_ref(), command) + let _completed_value = barrier.wait(); + submit_post(&pool, job_queue.as_ref(), command) })); } let mut results = Vec::with_capacity(handles.len()); for handle in handles { - let result = handle - .join() - .map_err(|_| anyhow::anyhow!("concurrent submission worker panicked"))?; + let result = handle.join().map_err(|panic_payload| { + anyhow::anyhow!( + "concurrent submission worker panicked: {}", + crate::media::process::panic_message(panic_payload.as_ref()) + ) + })?; results.push(result.context("concurrent submission failed")?); } Ok(results) @@ -1168,7 +1314,7 @@ mod tests { if path.is_dir() { visit(&path, files)?; } else if path.is_file() { - files.insert(path); + let _completed_value = files.insert(path); } } Ok(()) @@ -1188,14 +1334,16 @@ mod tests { .get() .context("failed to get database connection")?; crate::db::create_board(&conn, TEST_BOARD, "Test", "", false) - .context("failed to create board")?; + .context("failed to create board") + .map(|_completed_value| ())?; let ip_hash = crate::utils::crypto::hash_ip(TEST_IDENTITY_KEY, TEST_COOKIE_SECRET); crate::db::add_ban(&conn, &ip_hash, "posting blocked", None) - .context("failed to add ban")?; + .context("failed to add ban") + .map(|_completed_value| ())?; let error = match submit_post( - &conn, + &state.db, state.job_queue.as_ref(), thread_command( TEST_BOARD, @@ -1219,7 +1367,15 @@ mod tests { assert_eq!(reason, "posting blocked"); assert_eq!(csrf_token, "ban-csrf"); } - other => bail!("expected BannedUser, got {other:?}"), + other @ (AppError::NotFound(_) + | AppError::BadRequest(_) + | AppError::Forbidden(_) + | AppError::UploadTooLarge(_) + | AppError::InvalidMediaType(_) + | AppError::Conflict(_) + | AppError::DbBusy + | AppError::Internal(_) + | AppError::Tls(_)) => bail!("expected BannedUser, got {other:?}"), } let thread_count: i64 = conn @@ -1243,7 +1399,8 @@ mod tests { "UPDATE boards SET post_cooldown_secs = 60 WHERE short_name = ?1", rusqlite::params![TEST_BOARD], ) - .context("failed to enable cooldown")?; + .context("failed to enable cooldown") + .map(|_completed_value| ())?; let ip_hash = crate::utils::crypto::hash_ip(TEST_IDENTITY_KEY, TEST_COOKIE_SECRET); let (thread_id, _, _) = crate::db::create_thread_with_optional_poll( @@ -1258,7 +1415,7 @@ mod tests { .context("failed to create thread")?; let error = match submit_post( - &conn, + &state.db, state.job_queue.as_ref(), reply_command( TEST_BOARD, @@ -1280,7 +1437,18 @@ mod tests { assert!(message.contains("Please wait")); assert!(message.contains("before posting again.")); } - other => bail!("expected BadRequest, got {other:?}"), + other @ (AppError::NotFound(_) + | AppError::Forbidden(_) + | AppError::BannedUser { + reason: _, + csrf_token: _, + } + | AppError::UploadTooLarge(_) + | AppError::InvalidMediaType(_) + | AppError::Conflict(_) + | AppError::DbBusy + | AppError::Internal(_) + | AppError::Tls(_)) => bail!("expected BadRequest, got {other:?}"), } Ok(()) } @@ -1294,15 +1462,17 @@ mod tests { .get() .context("failed to get database connection")?; crate::db::create_board(&conn, TEST_BOARD, "Test", "", false) - .context("failed to create board")?; + .context("failed to create board") + .map(|_completed_value| ())?; conn.execute( "UPDATE boards SET allow_captcha = 1 WHERE short_name = ?1", rusqlite::params![TEST_BOARD], ) - .context("failed to enable CAPTCHA")?; + .context("failed to enable CAPTCHA") + .map(|_completed_value| ())?; let error = match submit_post( - &conn, + &state.db, state.job_queue.as_ref(), thread_command( TEST_BOARD, @@ -1325,7 +1495,18 @@ mod tests { "CAPTCHA verification failed. Enter the text from the image and try again." ); } - other => bail!("expected BadRequest, got {other:?}"), + other @ (AppError::NotFound(_) + | AppError::Forbidden(_) + | AppError::BannedUser { + reason: _, + csrf_token: _, + } + | AppError::UploadTooLarge(_) + | AppError::InvalidMediaType(_) + | AppError::Conflict(_) + | AppError::DbBusy + | AppError::Internal(_) + | AppError::Tls(_)) => bail!("expected BadRequest, got {other:?}"), } Ok(()) } @@ -1339,12 +1520,14 @@ mod tests { .get() .context("failed to get database connection")?; crate::db::create_board(&conn, TEST_BOARD, "Test", "", false) - .context("failed to create board")?; + .context("failed to create board") + .map(|_completed_value| ())?; conn.execute( "UPDATE boards SET allow_captcha = 1 WHERE short_name = ?1", rusqlite::params![TEST_BOARD], ) - .context("failed to enable CAPTCHA")?; + .context("failed to enable CAPTCHA") + .map(|_completed_value| ())?; let captcha_id = "00000000000000000000000000000007"; crate::captcha::testing::insert_challenge_for_test( TEST_BOARD, @@ -1365,7 +1548,7 @@ mod tests { command.captcha_id = captcha_id.to_owned(); command.captcha_answer = "abc23".to_owned(); - let result = submit_post(&conn, state.job_queue.as_ref(), command) + let result = submit_post(&state.db, state.job_queue.as_ref(), command) .context("failed to submit post with valid CAPTCHA")?; assert_eq!(result.board_short, TEST_BOARD); @@ -1380,10 +1563,10 @@ mod tests { ); replay.captcha_id = captcha_id.to_owned(); replay.captcha_answer = "ABC23".to_owned(); - let error = match submit_post(&conn, state.job_queue.as_ref(), replay) { - Ok(result) => bail!( + let error = match submit_post(&state.db, state.job_queue.as_ref(), replay) { + Ok(replay_result) => bail!( "expected captcha replay rejection, got {}", - result.redirect_url + replay_result.redirect_url ), Err(error) => error, }; @@ -1400,12 +1583,14 @@ mod tests { .get() .context("failed to get database connection")?; crate::db::create_board(&conn, TEST_BOARD, "Test", "", false) - .context("failed to create board")?; + .context("failed to create board") + .map(|_completed_value| ())?; conn.execute( "UPDATE boards SET allow_captcha = 1 WHERE short_name = ?1", rusqlite::params![TEST_BOARD], ) - .context("failed to enable CAPTCHA")?; + .context("failed to enable CAPTCHA") + .map(|_completed_value| ())?; let captcha_id = "00000000000000000000000000000008"; crate::captcha::testing::insert_challenge_for_test( TEST_BOARD, @@ -1425,7 +1610,7 @@ mod tests { command.captcha_id = captcha_id.to_owned(); command.captcha_answer = "ABC23".to_owned(); - let error = match submit_post(&conn, state.job_queue.as_ref(), command) { + let error = match submit_post(&state.db, state.job_queue.as_ref(), command) { Ok(result) => bail!( "expected expired captcha rejection, got {}", result.redirect_url @@ -1445,10 +1630,11 @@ mod tests { .get() .context("failed to get database connection")?; crate::db::create_board(&conn, TEST_BOARD, "Test", "", false) - .context("failed to create board")?; + .context("failed to create board") + .map(|_completed_value| ())?; let error = match submit_post( - &conn, + &state.db, state.job_queue.as_ref(), thread_command_with_poll( TEST_BOARD, @@ -1474,7 +1660,18 @@ mod tests { AppError::BadRequest(message) => { assert_eq!(message, "Polls need a question and at least two options."); } - other => bail!("expected BadRequest, got {other:?}"), + other @ (AppError::NotFound(_) + | AppError::Forbidden(_) + | AppError::BannedUser { + reason: _, + csrf_token: _, + } + | AppError::UploadTooLarge(_) + | AppError::InvalidMediaType(_) + | AppError::Conflict(_) + | AppError::DbBusy + | AppError::Internal(_) + | AppError::Tls(_)) => bail!("expected BadRequest, got {other:?}"), } Ok(()) } @@ -1487,7 +1684,8 @@ mod tests { .get() .context("failed to get database connection")?; crate::db::create_board(&conn, TEST_BOARD, "Test", "", false) - .context("failed to create board")?; + .context("failed to create board") + .map(|_completed_value| ())?; let board = crate::db::get_board_by_short(&conn, TEST_BOARD) .context("failed to load board")? .context("test board did not exist")?; @@ -1500,7 +1698,7 @@ mod tests { None, Some(bad_audio), &board, - &conn, + &state.db, &UploadConfig { upload_dir: upload_dir .path() @@ -1512,8 +1710,6 @@ mod tests { max_audio_size: 1024 * 1024, max_pdf_size: 1024 * 1024, ffmpeg_available: false, - ffprobe_available: false, - ffmpeg_webp_available: false, }, ); @@ -1531,12 +1727,14 @@ mod tests { .get() .context("failed to get database connection")?; crate::db::create_board(&conn, TEST_BOARD, "Test", "", false) - .context("failed to create board")?; + .context("failed to create board") + .map(|_completed_value| ())?; conn.execute( "UPDATE boards SET max_image_size = ?1 WHERE short_name = ?2", rusqlite::params![64_i64, TEST_BOARD], ) - .context("failed to shrink image limit")?; + .context("failed to shrink image limit") + .map(|_completed_value| ())?; let mut command = thread_command( TEST_BOARD, "board-image-cap", @@ -1548,7 +1746,7 @@ mod tests { ); command.file_data = Some(temp_upload("cover.png", &one_pixel_png()?)?); - let error = match submit_post(&conn, state.job_queue.as_ref(), command) { + let error = match submit_post(&state.db, state.job_queue.as_ref(), command) { Ok(result) => bail!( "board-specific image cap should reject upload, got {}", result.redirect_url @@ -1560,7 +1758,18 @@ mod tests { AppError::UploadTooLarge(message) => { assert!(message.contains("Maximum image upload size is 64 B.")); } - other => bail!("expected UploadTooLarge, got {other:?}"), + other @ (AppError::NotFound(_) + | AppError::BadRequest(_) + | AppError::Forbidden(_) + | AppError::BannedUser { + reason: _, + csrf_token: _, + } + | AppError::InvalidMediaType(_) + | AppError::Conflict(_) + | AppError::DbBusy + | AppError::Internal(_) + | AppError::Tls(_)) => bail!("expected UploadTooLarge, got {other:?}"), } Ok(()) } @@ -1574,7 +1783,8 @@ mod tests { .get() .context("failed to get database connection")?; crate::db::create_board(&conn, TEST_BOARD, "Test", "", false) - .context("failed to create board")?; + .context("failed to create board") + .map(|_completed_value| ())?; let mut command = thread_command( TEST_BOARD, "audio-disabled", @@ -1586,7 +1796,7 @@ mod tests { ); command.file_data = Some(temp_upload("tiny.flac", &flac_header_bytes())?); - let error = match submit_post(&conn, state.job_queue.as_ref(), command) { + let error = match submit_post(&state.db, state.job_queue.as_ref(), command) { Ok(result) => bail!( "audio-disabled board should reject, got {}", result.redirect_url @@ -1598,7 +1808,18 @@ mod tests { AppError::BadRequest(message) => { assert!(message.contains("Audio uploads are disabled")); } - other => bail!("expected BadRequest, got {other:?}"), + other @ (AppError::NotFound(_) + | AppError::Forbidden(_) + | AppError::BannedUser { + reason: _, + csrf_token: _, + } + | AppError::UploadTooLarge(_) + | AppError::InvalidMediaType(_) + | AppError::Conflict(_) + | AppError::DbBusy + | AppError::Internal(_) + | AppError::Tls(_)) => bail!("expected BadRequest, got {other:?}"), } assert_eq!(pending_upload_stage_count(upload_dir.path())?, 0); assert!(!upload_dir.path().join(TEST_BOARD).exists()); @@ -1615,12 +1836,14 @@ mod tests { .get() .context("failed to get database connection")?; crate::db::create_board(&conn, TEST_BOARD, "Test", "", false) - .context("failed to create board")?; + .context("failed to create board") + .map(|_completed_value| ())?; conn.execute( "UPDATE boards SET allow_audio = 1 WHERE short_name = ?1", rusqlite::params![TEST_BOARD], ) - .context("failed to enable audio")?; + .context("failed to enable audio") + .map(|_completed_value| ())?; let mut command = thread_command( TEST_BOARD, "malformed-aac", @@ -1633,16 +1856,30 @@ mod tests { command.file_data = Some(temp_upload("broken.aac", &malformed_aac_bytes()?)?); command.ffmpeg_available = true; - let error = match submit_post(&conn, state.job_queue.as_ref(), command) { + let error = match submit_post(&state.db, state.job_queue.as_ref(), command) { Ok(result) => bail!("malformed AAC should reject, got {}", result.redirect_url), Err(error) => error, }; match error { AppError::BadRequest(message) => { - assert!(message.contains("ADTS stream is malformed")); + assert!( + message.contains("ADTS stream is malformed") + || message.contains("could not validate") + ); } - other => bail!("expected BadRequest, got {other:?}"), + other @ (AppError::NotFound(_) + | AppError::Forbidden(_) + | AppError::BannedUser { + reason: _, + csrf_token: _, + } + | AppError::UploadTooLarge(_) + | AppError::InvalidMediaType(_) + | AppError::Conflict(_) + | AppError::DbBusy + | AppError::Internal(_) + | AppError::Tls(_)) => bail!("expected BadRequest, got {other:?}"), } assert_eq!(pending_upload_stage_count(upload_dir.path())?, 0); assert!(!upload_dir.path().join(TEST_BOARD).exists()); @@ -1663,12 +1900,14 @@ mod tests { .get() .context("failed to get database connection")?; crate::db::create_board(&conn, TEST_BOARD, "Test", "", false) - .context("failed to create board")?; + .context("failed to create board") + .map(|_completed_value| ())?; conn.execute( "UPDATE boards SET allow_video = 0 WHERE short_name = ?1", rusqlite::params![TEST_BOARD], ) - .context("failed to disable video")?; + .context("failed to disable video") + .map(|_completed_value| ())?; let mut command = thread_command( TEST_BOARD, "video-disabled", @@ -1680,7 +1919,7 @@ mod tests { ); command.file_data = Some(temp_upload("tiny.mp4", &mp4_header_bytes())?); - let error = match submit_post(&conn, state.job_queue.as_ref(), command) { + let error = match submit_post(&state.db, state.job_queue.as_ref(), command) { Ok(result) => bail!( "video-disabled board should reject, got {}", result.redirect_url @@ -1692,7 +1931,18 @@ mod tests { AppError::BadRequest(message) => { assert!(message.contains("Video uploads are disabled")); } - other => bail!("expected BadRequest, got {other:?}"), + other @ (AppError::NotFound(_) + | AppError::Forbidden(_) + | AppError::BannedUser { + reason: _, + csrf_token: _, + } + | AppError::UploadTooLarge(_) + | AppError::InvalidMediaType(_) + | AppError::Conflict(_) + | AppError::DbBusy + | AppError::Internal(_) + | AppError::Tls(_)) => bail!("expected BadRequest, got {other:?}"), } assert_eq!(pending_upload_stage_count(upload_dir.path())?, 0); assert!(!upload_dir.path().join(TEST_BOARD).exists()); @@ -1709,12 +1959,14 @@ mod tests { .get() .context("failed to get database connection")?; crate::db::create_board(&conn, TEST_BOARD, "Test", "", false) - .context("failed to create board")?; + .context("failed to create board") + .map(|_completed_value| ())?; conn.execute( "UPDATE boards SET allow_audio = 1, max_audio_size = 8 WHERE short_name = ?1", rusqlite::params![TEST_BOARD], ) - .context("failed to shrink audio limit")?; + .context("failed to shrink audio limit") + .map(|_completed_value| ())?; let mut command = thread_command( TEST_BOARD, "audio-overlimit", @@ -1726,7 +1978,7 @@ mod tests { ); command.file_data = Some(temp_upload("tiny.flac", &flac_header_bytes())?); - let error = match submit_post(&conn, state.job_queue.as_ref(), command) { + let error = match submit_post(&state.db, state.job_queue.as_ref(), command) { Ok(result) => bail!( "over-limit audio should reject, got {}", result.redirect_url @@ -1738,7 +1990,18 @@ mod tests { AppError::UploadTooLarge(message) => { assert!(message.contains("Maximum audio upload size is 8 B.")); } - other => bail!("expected UploadTooLarge, got {other:?}"), + other @ (AppError::NotFound(_) + | AppError::BadRequest(_) + | AppError::Forbidden(_) + | AppError::BannedUser { + reason: _, + csrf_token: _, + } + | AppError::InvalidMediaType(_) + | AppError::Conflict(_) + | AppError::DbBusy + | AppError::Internal(_) + | AppError::Tls(_)) => bail!("expected UploadTooLarge, got {other:?}"), } assert_eq!(pending_upload_stage_count(upload_dir.path())?, 0); assert!(!upload_dir.path().join(TEST_BOARD).exists()); @@ -1755,12 +2018,14 @@ mod tests { .get() .context("failed to get database connection")?; crate::db::create_board(&conn, TEST_BOARD, "Test", "", false) - .context("failed to create board")?; + .context("failed to create board") + .map(|_completed_value| ())?; conn.execute( "UPDATE boards SET max_video_size = 8 WHERE short_name = ?1", rusqlite::params![TEST_BOARD], ) - .context("failed to shrink video limit")?; + .context("failed to shrink video limit") + .map(|_completed_value| ())?; let mut command = thread_command( TEST_BOARD, "video-overlimit", @@ -1772,7 +2037,7 @@ mod tests { ); command.file_data = Some(temp_upload("tiny.mp4", &mp4_header_bytes())?); - let error = match submit_post(&conn, state.job_queue.as_ref(), command) { + let error = match submit_post(&state.db, state.job_queue.as_ref(), command) { Ok(result) => bail!( "over-limit video should reject, got {}", result.redirect_url @@ -1784,7 +2049,18 @@ mod tests { AppError::UploadTooLarge(message) => { assert!(message.contains("Maximum video upload size is 8 B.")); } - other => bail!("expected UploadTooLarge, got {other:?}"), + other @ (AppError::NotFound(_) + | AppError::BadRequest(_) + | AppError::Forbidden(_) + | AppError::BannedUser { + reason: _, + csrf_token: _, + } + | AppError::InvalidMediaType(_) + | AppError::Conflict(_) + | AppError::DbBusy + | AppError::Internal(_) + | AppError::Tls(_)) => bail!("expected UploadTooLarge, got {other:?}"), } assert_eq!(pending_upload_stage_count(upload_dir.path())?, 0); assert!(!upload_dir.path().join(TEST_BOARD).exists()); @@ -1801,7 +2077,8 @@ mod tests { .get() .context("failed to get database connection")?; crate::db::create_board(&conn, TEST_BOARD, "Test", "", false) - .context("failed to create board")?; + .context("failed to create board") + .map(|_completed_value| ())?; let mut command = thread_command_with_poll( TEST_BOARD, @@ -1817,7 +2094,7 @@ mod tests { ); command.file_data = Some(temp_upload("cover.png", &one_pixel_png()?)?); - let error = match submit_post(&conn, state.job_queue.as_ref(), command) { + let error = match submit_post(&state.db, state.job_queue.as_ref(), command) { Ok(result) => bail!( "poll validation should reject submission, got {}", result.redirect_url @@ -1829,7 +2106,18 @@ mod tests { AppError::BadRequest(message) => { assert_eq!(message, "Polls need a question and at least two options."); } - other => bail!("expected BadRequest, got {other:?}"), + other @ (AppError::NotFound(_) + | AppError::Forbidden(_) + | AppError::BannedUser { + reason: _, + csrf_token: _, + } + | AppError::UploadTooLarge(_) + | AppError::InvalidMediaType(_) + | AppError::Conflict(_) + | AppError::DbBusy + | AppError::Internal(_) + | AppError::Tls(_)) => bail!("expected BadRequest, got {other:?}"), } assert_eq!(pending_upload_stage_count(upload_dir.path())?, 0); @@ -1850,12 +2138,14 @@ mod tests { .get() .context("failed to get database connection")?; crate::db::create_board(&conn, TEST_BOARD, "Test", "", false) - .context("failed to create board")?; + .context("failed to create board") + .map(|_completed_value| ())?; conn.execute( "UPDATE boards SET max_image_size = ?1 WHERE short_name = ?2", rusqlite::params![1024 * 1024_i64, TEST_BOARD], ) - .context("failed to raise image limit")?; + .context("failed to raise image limit") + .map(|_completed_value| ())?; let mut command = thread_command( TEST_BOARD, "board-image-raised-cap", @@ -1867,7 +2157,7 @@ mod tests { ); command.file_data = Some(temp_upload("cover.png", &one_pixel_png()?)?); - let result = submit_post(&conn, state.job_queue.as_ref(), command) + let result = submit_post(&state.db, state.job_queue.as_ref(), command) .context("board-specific raised image cap should allow upload")?; assert_eq!(result.board_short, TEST_BOARD); @@ -1882,7 +2172,8 @@ mod tests { .get() .context("failed to get database connection")?; crate::db::create_board(&conn, TEST_BOARD, "Test", "", false) - .context("failed to create board")?; + .context("failed to create board") + .map(|_completed_value| ())?; let board = crate::db::get_board_by_short(&conn, TEST_BOARD) .context("failed to load board")? .context("test board did not exist")?; @@ -1911,7 +2202,7 @@ mod tests { Some(temp_upload("same-but-renamed.jpg", &bytes)?), None, &board, - &conn, + &state.db, &UploadConfig { upload_dir: upload_dir .path() @@ -1923,8 +2214,6 @@ mod tests { max_audio_size: 1024 * 1024, max_pdf_size: 1024 * 1024, ffmpeg_available: false, - ffprobe_available: false, - ffmpeg_webp_available: false, }, ) .context("failed to process deduplicated upload")?; @@ -1980,7 +2269,7 @@ mod tests { fn thumbnail_failures_preserve_original_and_omit_derived_intent_metadata() -> Result<()> { let state = crate::test_support::app_state(); let conn = state.db.get().context("get test database connection")?; - crate::db::create_board(&conn, TEST_BOARD, "Test", "", false)?; + crate::db::create_board(&conn, TEST_BOARD, "Test", "", false).map(|_completed_value| ())?; let board = crate::db::get_board_by_short(&conn, TEST_BOARD)? .context("test board did not exist")?; let upload_dir = tempfile::tempdir().context("create upload directory")?; @@ -2001,7 +2290,7 @@ mod tests { Some(temp_upload(&format!("failure-{index}.png"), &png)?), None, &board, - &conn, + &state.db, &UploadConfig { upload_dir: upload_dir.path().to_str().context("UTF-8 upload root")?, thumb_size: 64, @@ -2010,8 +2299,6 @@ mod tests { max_audio_size: 1024 * 1024, max_pdf_size: 1024 * 1024, ffmpeg_available: false, - ffprobe_available: false, - ffmpeg_webp_available: false, }, )?; @@ -2055,10 +2342,11 @@ mod tests { .get() .context("failed to get database connection")?; crate::db::create_board(&conn, TEST_BOARD, "Test", "", false) - .context("failed to create board")?; + .context("failed to create board") + .map(|_completed_value| ())?; submit_post( - &conn, + &state.db, state.job_queue.as_ref(), thread_command( TEST_BOARD, @@ -2070,7 +2358,8 @@ mod tests { .context("upload directory was not valid UTF-8")?, ), ) - .context("failed to make first submission")?; + .context("failed to make first submission") + .map(|_completed_value| ())?; let original_created_at = chrono::Utc::now().timestamp() - crate::handlers::board::self_action_window_secs() - 1; @@ -2078,10 +2367,11 @@ mod tests { "UPDATE posts SET created_at = ?1", rusqlite::params![original_created_at], ) - .context("failed to age original post")?; + .context("failed to age original post") + .map(|_completed_value| ())?; let duplicate = submit_post( - &conn, + &state.db, state.job_queue.as_ref(), thread_command( TEST_BOARD, @@ -2149,7 +2439,7 @@ mod tests { .context("failed to create thread")?; submit_post( - &conn, + &state.db, state.job_queue.as_ref(), reply_command( TEST_BOARD, @@ -2162,10 +2452,11 @@ mod tests { .context("upload directory was not valid UTF-8")?, ), ) - .context("failed to create first reply")?; + .context("failed to create first reply") + .map(|_completed_value| ())?; let duplicate = submit_post( - &conn, + &state.db, state.job_queue.as_ref(), reply_command( TEST_BOARD, @@ -2208,7 +2499,7 @@ mod tests { #[test] fn concurrent_thread_submissions_share_one_atomic_result_on_fresh_databases() -> Result<()> { - for repetition in 0..3 { + for repetition in 0_i32..3_i32 { let state = crate::test_support::app_state(); let upload_dir = tempfile::tempdir().context("failed to create upload directory")?; let conn = state @@ -2216,7 +2507,8 @@ mod tests { .get() .context("failed to get setup database connection")?; crate::db::create_board(&conn, TEST_BOARD, "Test", "", false) - .context("failed to create board")?; + .context("failed to create board") + .map(|_completed_value| ())?; drop(conn); let token = format!("thread-race-{repetition}"); @@ -2225,7 +2517,7 @@ mod tests { .path() .to_str() .context("upload directory was not valid UTF-8")?; - let commands = (0..8) + let commands = (0_i32..8_i32) .map(|_| thread_command(TEST_BOARD, &token, &body, upload_path)) .collect(); let results = submit_concurrently(&state, commands)?; @@ -2241,23 +2533,23 @@ mod tests { assert_eq!(canonical_locations.len(), 1); assert_eq!(canonical_posts.len(), 1); - let conn = state + let recovered_conn = state .db .get() .context("failed to get verification database connection")?; - let post_count: i64 = conn.query_row( + let post_count: i64 = recovered_conn.query_row( "SELECT COUNT(*) FROM posts WHERE body = ?1", rusqlite::params![body], |row| row.get(0), )?; - let thread_count: i64 = conn.query_row( + let thread_count: i64 = recovered_conn.query_row( "SELECT COUNT(*) FROM threads WHERE board_id = ( SELECT id FROM boards WHERE short_name = ?1 )", rusqlite::params![TEST_BOARD], |row| row.get(0), )?; - let token_count: i64 = conn.query_row( + let token_count: i64 = recovered_conn.query_row( "SELECT COUNT(*) FROM post_submissions WHERE submission_token = ?1", rusqlite::params![token], |row| row.get(0), @@ -2265,14 +2557,14 @@ mod tests { assert_eq!(post_count, 1); assert_eq!(thread_count, 1); assert_eq!(token_count, 1); - assert_database_integrity(&conn, upload_dir.path())?; + assert_database_integrity(&recovered_conn, upload_dir.path())?; } Ok(()) } #[test] fn concurrent_reply_submissions_share_one_atomic_result_on_fresh_databases() -> Result<()> { - for repetition in 0..3 { + for repetition in 0_i32..3_i32 { let state = crate::test_support::app_state(); let upload_dir = tempfile::tempdir().context("failed to create upload directory")?; let conn = state @@ -2298,7 +2590,7 @@ mod tests { .path() .to_str() .context("upload directory was not valid UTF-8")?; - let commands = (0..8) + let commands = (0_i32..8_i32) .map(|_| reply_command(TEST_BOARD, thread_id, &token, &body, upload_path)) .collect(); let results = submit_concurrently(&state, commands)?; @@ -2314,21 +2606,21 @@ mod tests { assert_eq!(canonical_locations.len(), 1); assert_eq!(canonical_posts.len(), 1); - let conn = state + let recovered_conn = state .db .get() .context("failed to get verification database connection")?; - let reply_count: i64 = conn.query_row( + let reply_count: i64 = recovered_conn.query_row( "SELECT COUNT(*) FROM posts WHERE thread_id = ?1 AND is_op = 0", rusqlite::params![thread_id], |row| row.get(0), )?; - let stored_reply_count: i64 = conn.query_row( + let stored_reply_count: i64 = recovered_conn.query_row( "SELECT reply_count FROM threads WHERE id = ?1", rusqlite::params![thread_id], |row| row.get(0), )?; - let token_count: i64 = conn.query_row( + let token_count: i64 = recovered_conn.query_row( "SELECT COUNT(*) FROM post_submissions WHERE submission_token = ?1", rusqlite::params![token], |row| row.get(0), @@ -2336,7 +2628,7 @@ mod tests { assert_eq!(reply_count, 1); assert_eq!(stored_reply_count, 1); assert_eq!(token_count, 1); - assert_database_integrity(&conn, upload_dir.path())?; + assert_database_integrity(&recovered_conn, upload_dir.path())?; } Ok(()) } @@ -2350,7 +2642,8 @@ mod tests { .get() .context("failed to get setup database connection")?; crate::db::create_board(&conn, TEST_BOARD, "Test", "", false) - .context("failed to create board")?; + .context("failed to create board") + .map(|_completed_value| ())?; drop(conn); let png = one_pixel_png()?; @@ -2358,7 +2651,7 @@ mod tests { .path() .to_str() .context("upload directory was not valid UTF-8")?; - let commands = (0..8) + let commands = (0_i32..8_i32) .map(|index| { let mut command = thread_command( TEST_BOARD, @@ -2380,28 +2673,28 @@ mod tests { 1 ); - let conn = state + let recovered_conn = state .db .get() .context("failed to get verification database connection")?; - let (file_path, thumb_path): (String, String) = conn.query_row( + let (file_path, thumb_path): (String, String) = recovered_conn.query_row( "SELECT file_path, thumb_path FROM posts WHERE body = 'thread media race body'", [], |row| Ok((row.get(0)?, row.get(1)?)), )?; - let post_count: i64 = conn.query_row( + let post_count: i64 = recovered_conn.query_row( "SELECT COUNT(*) FROM posts WHERE body = 'thread media race body'", [], |row| row.get(0), )?; - let token_count: i64 = conn.query_row( + let token_count: i64 = recovered_conn.query_row( "SELECT COUNT(*) FROM post_submissions WHERE submission_token = 'thread-media-race'", [], |row| row.get(0), )?; let file_hash_count: i64 = - conn.query_row("SELECT COUNT(*) FROM file_hashes", [], |row| row.get(0))?; + recovered_conn.query_row("SELECT COUNT(*) FROM file_hashes", [], |row| row.get(0))?; assert_eq!(post_count, 1); assert_eq!(token_count, 1); assert_eq!(file_hash_count, 1); @@ -2412,7 +2705,7 @@ mod tests { .map(|path| upload_dir.path().join(path)) .collect::>(); assert_eq!(regular_files(upload_dir.path())?, expected_files); - assert_database_integrity(&conn, upload_dir.path())?; + assert_database_integrity(&recovered_conn, upload_dir.path())?; Ok(()) } @@ -2436,7 +2729,7 @@ mod tests { .context("failed to create board")?; let first_thread = submit_post( - &conn, + &state.db, state.job_queue.as_ref(), thread_command( TEST_BOARD, @@ -2446,7 +2739,7 @@ mod tests { ), )?; let replayed_thread = submit_post( - &conn, + &state.db, state.job_queue.as_ref(), thread_command( TEST_BOARD, @@ -2461,7 +2754,7 @@ mod tests { let distinct_threads = submit_concurrently( &state, - (0..8) + (0_i32..8_i32) .map(|index| { thread_command( TEST_BOARD, @@ -2481,12 +2774,12 @@ mod tests { 8 ); - let conn = state + let retry_conn = state .db .get() .context("failed to get reply setup database connection")?; let (reply_thread_id, _, _) = crate::db::create_thread_with_optional_poll( - &conn, + &retry_conn, board_id, Some("reply target"), &sample_post( @@ -2501,7 +2794,7 @@ mod tests { None, )?; let first_reply = submit_post( - &conn, + &state.db, state.job_queue.as_ref(), reply_command( TEST_BOARD, @@ -2512,7 +2805,7 @@ mod tests { ), )?; let replayed_reply = submit_post( - &conn, + &state.db, state.job_queue.as_ref(), reply_command( TEST_BOARD, @@ -2524,11 +2817,11 @@ mod tests { )?; assert_eq!(first_reply.redirect_url, replayed_reply.redirect_url); assert_eq!(first_reply.post_id, replayed_reply.post_id); - drop(conn); + drop(retry_conn); let distinct_replies = submit_concurrently( &state, - (0..8) + (0_i32..8_i32) .map(|index| { reply_command( TEST_BOARD, @@ -2549,12 +2842,12 @@ mod tests { 8 ); - let conn = state + let recovered_conn = state .db .get() .context("failed to get rollback database connection")?; - conn.execute_batch( - "CREATE TEMP TRIGGER fail_submission_post + recovered_conn.execute_batch( + "CREATE TRIGGER fail_submission_post BEFORE INSERT ON posts WHEN NEW.body IN ('force thread rollback', 'force reply rollback') BEGIN @@ -2562,7 +2855,7 @@ mod tests { END;", )?; let failed_thread = submit_post( - &conn, + &state.db, state.job_queue.as_ref(), thread_command( TEST_BOARD, @@ -2573,7 +2866,7 @@ mod tests { ); assert!(failed_thread.is_err()); let failed_reply = submit_post( - &conn, + &state.db, state.job_queue.as_ref(), reply_command( TEST_BOARD, @@ -2584,17 +2877,17 @@ mod tests { ), ); assert!(failed_reply.is_err()); - let failed_token_count: i64 = conn.query_row( + let failed_token_count: i64 = recovered_conn.query_row( "SELECT COUNT(*) FROM post_submissions WHERE submission_token IN ('rollback-thread', 'rollback-reply')", [], |row| row.get(0), )?; assert_eq!(failed_token_count, 0); - conn.execute_batch("DROP TRIGGER fail_submission_post")?; + recovered_conn.execute_batch("DROP TRIGGER fail_submission_post")?; let corrected_thread = submit_post( - &conn, + &state.db, state.job_queue.as_ref(), thread_command( TEST_BOARD, @@ -2604,7 +2897,7 @@ mod tests { ), )?; let corrected_reply = submit_post( - &conn, + &state.db, state.job_queue.as_ref(), reply_command( TEST_BOARD, @@ -2617,17 +2910,17 @@ mod tests { assert!(corrected_thread.post_id > 0); assert!(corrected_reply.post_id > 0); - let distinct_thread_count: i64 = conn.query_row( + let distinct_thread_count: i64 = recovered_conn.query_row( "SELECT COUNT(*) FROM posts WHERE body LIKE 'distinct thread body %'", [], |row| row.get(0), )?; - let distinct_reply_count: i64 = conn.query_row( + let distinct_reply_count: i64 = recovered_conn.query_row( "SELECT COUNT(*) FROM posts WHERE body LIKE 'distinct reply body %'", [], |row| row.get(0), )?; - let corrected_token_count: i64 = conn.query_row( + let corrected_token_count: i64 = recovered_conn.query_row( "SELECT COUNT(*) FROM post_submissions WHERE submission_token IN ('rollback-thread', 'rollback-reply')", [], @@ -2636,7 +2929,262 @@ mod tests { assert_eq!(distinct_thread_count, 8); assert_eq!(distinct_reply_count, 8); assert_eq!(corrected_token_count, 2); - assert_database_integrity(&conn, upload_dir.path())?; + assert_database_integrity(&recovered_conn, upload_dir.path())?; + Ok(()) + } + + /// Replace the test state's pool with one connection to expose nested borrowing. + fn single_connection_state() -> Result { + let mut state = crate::test_support::app_state(); + let path = { + let conn = state.db.get()?; + conn.path().context("test database has no path")?.to_owned() + }; + let manager = r2d2_sqlite::SqliteConnectionManager::file(path).with_init(|conn| { + conn.execute_batch("PRAGMA foreign_keys=ON; PRAGMA busy_timeout=1000;") + }); + state.db = r2d2::Pool::builder() + .max_size(1) + .connection_timeout(std::time::Duration::from_secs(1)) + .build(manager)?; + state.job_queue = std::sync::Arc::new(crate::workers::JobQueue::new(state.db.clone())); + Ok(state) + } + + #[test] + fn independent_preparation_releases_the_only_pool_connection() -> Result<()> { + let state = single_connection_state()?; + let uploads = tempfile::tempdir()?; + crate::db::create_board(&*state.db.get()?, TEST_BOARD, "Test", "", false) + .map(|_completed_value| ())?; + let command = thread_command( + TEST_BOARD, + "single-pool-post", + "body", + uploads.path().to_str().context("upload path")?, + ); + let result = + super::submit_post_with_preparation(&state.db, &state.job_queue, command, || { + let conn = state + .db + .try_get() + .context("preparation must release the sole connection")?; + crate::db::database_ready_probe(&conn)?; + Ok(()) + })?; + let conn = state + .db + .try_get() + .context("submission must return its connection")?; + assert!(crate::db::get_post(&conn, result.post_id)?.is_some()); + assert!( + crate::db::pending_job_count(&conn)? > 0, + "spam scheduling must not exhaust a one-connection pool" + ); + Ok(()) + } + + #[test] + fn reply_rechecks_thread_state_after_independent_preparation() -> Result<()> { + for archived in [false, true] { + let state = single_connection_state()?; + let uploads = tempfile::tempdir()?; + let conn = state.db.get()?; + let board = crate::db::create_board(&conn, TEST_BOARD, "Test", "", false)?; + let (thread, _, _) = crate::db::create_thread_with_optional_poll( + &conn, + board, + None, + &sample_post(board, 0, "op", true, None), + "", + None, + None, + )?; + drop(conn); + let command = reply_command( + TEST_BOARD, + thread, + "state-race", + "reply", + uploads.path().to_str().context("upload path")?, + ); + let result = + super::submit_post_with_preparation(&state.db, &state.job_queue, command, || { + let recovered_conn = state.db.try_get().context("preparation held the pool")?; + if archived { + crate::db::set_thread_archived(&recovered_conn, thread, true)?; + } else { + crate::db::set_thread_locked(&recovered_conn, thread, true)?; + } + Ok(()) + }); + assert!(matches!(result, Err(AppError::Forbidden(_)))); + let recovered_conn = state.db.get()?; + assert_eq!( + crate::db::get_posts_for_thread(&recovered_conn, thread)?.len(), + 1 + ); + assert_eq!( + crate::db::get_thread(&recovered_conn, thread)? + .context("thread")? + .reply_count, + 0 + ); + assert!(recovered_conn.is_autocommit()); + } + Ok(()) + } + + #[test] + fn posting_rechecks_access_media_and_bans_at_commit() -> Result<()> { + for mutation in ["access", "media", "ban", "captcha", "cooldown", "filters"] { + let state = single_connection_state()?; + let uploads = tempfile::tempdir()?; + crate::db::create_board(&*state.db.get()?, TEST_BOARD, "Test", "", false) + .map(|_completed_value| ())?; + let command = thread_command( + TEST_BOARD, + "policy-race", + "body", + uploads.path().to_str().context("upload path")?, + ); + let result = super::submit_post_with_preparation( + &state.db, + &state.job_queue, + command, + || { + let conn = state.db.try_get().context("preparation held the pool")?; + match mutation { + "access" => { + conn.execute("UPDATE boards SET access_mode='post_password',access_password_hash='changed'", []).map(|_completed_value| ())?; + } + "media" => { + conn.execute("UPDATE boards SET allow_images=0", []) + .map(|_completed_value| ())?; + } + "captcha" => { + conn.execute("UPDATE boards SET allow_captcha=1", []) + .map(|_completed_value| ())?; + } + "filters" => { + conn.execute("INSERT INTO word_filters(pattern,replacement) VALUES('body','filtered')", []).map(|_completed_value| ())?; + } + "cooldown" => { + let board = crate::db::get_board_by_short(&conn, TEST_BOARD)? + .context("board")?; + conn.execute("UPDATE boards SET post_cooldown_secs=60", []) + .map(|_completed_value| ())?; + crate::db::create_thread_with_optional_poll( + &conn, + board.id, + None, + &sample_post( + board.id, + 0, + "other post", + true, + Some(crate::utils::crypto::hash_ip( + TEST_IDENTITY_KEY, + TEST_COOKIE_SECRET, + )), + ), + "", + None, + None, + ) + .map(|_completed_value| ())?; + } + _ => { + crate::db::add_ban( + &conn, + &crate::utils::crypto::hash_ip( + TEST_IDENTITY_KEY, + TEST_COOKIE_SECRET, + ), + "new ban", + None, + ) + .map(|_completed_value| ())?; + } + } + Ok(()) + }, + ); + assert!( + result.is_err(), + "concurrent policy change {mutation} must reject the write" + ); + let conn = state.db.get()?; + assert!(conn.is_autocommit()); + let count: i64 = conn.query_row( + "SELECT COUNT(*) FROM post_submissions WHERE submission_token='policy-race'", + [], + |r| r.get(0), + )?; + assert_eq!(count, 0); + } + Ok(()) + } + + #[test] + fn concurrent_cooldown_submissions_cannot_both_commit() -> Result<()> { + let state = crate::test_support::app_state(); + let uploads = tempfile::tempdir()?; + let conn = state.db.get()?; + crate::db::create_board(&conn, TEST_BOARD, "Test", "", false).map(|_completed_value| ())?; + conn.execute("UPDATE boards SET post_cooldown_secs=60", []) + .map(|_completed_value| ())?; + drop(conn); + let barrier = std::sync::Barrier::new(2); + let results = std::thread::scope(|scope| -> Result> { + let mut handles = Vec::new(); + for number in 0_i32..2_i32 { + let command = thread_command( + TEST_BOARD, + &format!("cooldown-{number}"), + "body", + uploads.path().to_str().context("upload path")?, + ); + let state = &state; + let barrier = &barrier; + handles.push(scope.spawn(move || { + super::submit_post_with_preparation( + &state.db, + &state.job_queue, + command, + || { + let _completed_value = barrier.wait(); + Ok(()) + }, + ) + })); + } + handles + .into_iter() + .map(|handle| { + handle.join().map_err(|panic_payload| { + anyhow::anyhow!( + "posting worker panicked: {}", + crate::media::process::panic_message(panic_payload.as_ref()) + ) + }) + }) + .collect() + })?; + assert_eq!(results.iter().filter(|result| result.is_ok()).count(), 1); + assert_eq!( + results + .iter() + .filter(|result| matches!(result, Err(AppError::BadRequest(_)))) + .count(), + 1 + ); + let recovered_conn = state.db.get()?; + crate::db::verify_database_schema(&recovered_conn)?; Ok(()) } } + +#[cfg(test)] +/// Opt-in posting/media connection-lifetime measurements. +mod performance; diff --git a/src/handlers/posting/performance.rs b/src/handlers/posting/performance.rs new file mode 100644 index 00000000..757b6a1d --- /dev/null +++ b/src/handlers/posting/performance.rs @@ -0,0 +1,214 @@ +//! Explicit actual-media benchmark for pooled connection ownership. + +use super::tests::{temp_upload, thread_command, TEST_BOARD}; +use super::{submit_post, SubmitPostCommand, SubmitPostResult}; +use anyhow::{Context as _, Result}; + +/// Event subscriber used only by the explicit posting benchmark. +#[derive(Debug, Clone, Default)] +struct PostingPoolProbe { + /// Connection hold durations on producer threads. + holds: std::sync::Arc>>, + /// Checkout durations on reader threads. + waits: std::sync::Arc>>, + /// Checkout timeouts across all benchmark threads. + timeouts: std::sync::Arc, +} + +impl r2d2::HandleEvent for PostingPoolProbe { + fn handle_checkin(&self, event: r2d2::event::CheckinEvent) { + if std::thread::current().name() == Some("db-post-benchmark") { + self.holds.lock().push(event.duration().as_micros()); + } + } + fn handle_checkout(&self, event: r2d2::event::CheckoutEvent) { + if std::thread::current().name() == Some("db-read-benchmark") { + self.waits.lock().push(event.duration().as_micros()); + } + } + fn handle_timeout(&self, _event: r2d2::event::TimeoutEvent) { + let _previous_count = self + .timeouts + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + } +} + +/// Summarize an elapsed-time series in microseconds. +fn posting_latencies(mut values: Vec) -> serde_json::Value { + values.sort_unstable(); + let percentile = |percent: usize| { + values + .get( + values + .len() + .saturating_mul(percent) + .div_ceil(100) + .saturating_sub(1), + ) + .copied() + .unwrap_or(0) + }; + serde_json::json!({"p50_us":percentile(50),"p95_us":percentile(95),"p99_us":percentile(99), + "max_us":values.last().copied().unwrap_or(0),"samples":values.len()}) +} + +/// Production entry point shared by the current and archived-baseline harness. +fn benchmark_submit( + pool: &crate::db::DbPool, + queue: &crate::workers::JobQueue, + command: SubmitPostCommand, +) -> crate::error::Result { + submit_post(pool, queue, command) +} + +/// Run actual PNG submissions alongside pooled reads; preparation is excluded from timing. +fn media_posting_case(size: u32, synchronous: &'static str) -> Result { + let mut state = crate::test_support::app_state(); + let upload_dir = tempfile::tempdir()?; + let path = { + let conn = state.db.get()?; + conn.path().context("DB path")?.to_owned() + }; + let probe = PostingPoolProbe::default(); + let manager = r2d2_sqlite::SqliteConnectionManager::file(path).with_init(move |conn| { + conn.execute_batch( + "PRAGMA journal_mode=WAL; PRAGMA foreign_keys=ON; PRAGMA busy_timeout=1000;", + )?; + conn.pragma_update(None, "synchronous", synchronous) + }); + state.db = r2d2::Pool::builder() + .max_size(size) + .min_idle(Some(size)) + .connection_timeout(std::time::Duration::from_secs(1)) + .event_handler(Box::new(probe.clone())) + .build(manager)?; + state.job_queue = std::sync::Arc::new(crate::workers::JobQueue::new(state.db.clone())); + crate::db::create_board(&*state.db.get()?, TEST_BOARD, "Test", "", false) + .map(|_completed_value| ())?; + let mut commands = Vec::new(); + for number in 0_u8..8 { + let pixels = image::RgbImage::from_fn(768, 768, |x, y| { + image::Rgb([ + u8::try_from(x % 256).unwrap_or(0), + u8::try_from(y % 256).unwrap_or(0), + number, + ]) + }); + let mut bytes = std::io::Cursor::new(Vec::new()); + image::DynamicImage::ImageRgb8(pixels).write_to(&mut bytes, image::ImageFormat::Png)?; + let (upload, name) = temp_upload(&format!("bench-{number}.png"), bytes.get_ref())?; + let mut command = thread_command( + TEST_BOARD, + &format!("media-bench-{number}"), + "media body", + upload_dir.path().to_str().context("uploads path")?, + ); + command.file_data = Some((upload, name)); + commands.push(command); + } + let mut result = run_media_posting_case(&state, commands, &probe, size)?; + let _previous_value = result + .as_object_mut() + .context("media benchmark object")? + .insert("synchronous".into(), serde_json::json!(synchronous)); + Ok(result) +} + +/// Coordinate measured readers/producers without background network activity. +fn run_media_posting_case( + state: &crate::middleware::AppState, + commands: Vec, + probe: &PostingPoolProbe, + size: u32, +) -> Result { + let barrier = std::sync::Barrier::new(6); + let remaining = std::sync::atomic::AtomicU64::new(2); + let post_times = parking_lot::Mutex::new(Vec::new()); + let reads = std::sync::atomic::AtomicU64::new(0); + let started = std::time::Instant::now(); + std::thread::scope(|scope| -> Result<()> { + let mut handles = Vec::new(); + let mut commands = commands.into_iter(); + for _ in 0_i32..2_i32 { + let batch: Vec<_> = commands.by_ref().take(4).collect(); + let barrier = &barrier; + let remaining = &remaining; + let post_times = &post_times; + handles.push( + std::thread::Builder::new() + .name("db-post-benchmark".into()) + .spawn_scoped(scope, move || -> Result<()> { + let _completed_value = barrier.wait(); + let result = (|| -> Result<()> { + for command in batch { + let post_started = std::time::Instant::now(); + benchmark_submit(&state.db, &state.job_queue, command) + .map(|_posted_identity| ())?; + post_times.lock().push(post_started.elapsed().as_micros()); + } + Ok(()) + })(); + let _previous_count = + remaining.fetch_sub(1, std::sync::atomic::Ordering::Relaxed); + result + })?, + ); + } + for _ in 0_i32..4_i32 { + let barrier = &barrier; + let remaining = &remaining; + let reads = &reads; + handles.push( + std::thread::Builder::new() + .name("db-read-benchmark".into()) + .spawn_scoped(scope, move || -> Result<()> { + let _completed_value = barrier.wait(); + while remaining.load(std::sync::atomic::Ordering::Relaxed) > 0 { + if let Ok(conn) = state.db.get() { + crate::db::database_ready_probe(&conn)?; + let _previous_count = + reads.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + } + std::thread::sleep(std::time::Duration::from_millis(2)); + } + Ok(()) + })?, + ); + } + for handle in handles { + handle.join().map_err(|panic_payload| { + anyhow::anyhow!( + "media benchmark worker panicked: {}", + crate::media::process::panic_message(panic_payload.as_ref()) + ) + })??; + } + Ok(()) + })?; + let elapsed = started.elapsed(); + let conn = state.db.get()?; + crate::db::verify_database_schema(&conn)?; + let pending = crate::db::list_pending_fs_ops(&conn)?.len(); + let posts: i64 = conn.query_row("SELECT COUNT(*) FROM posts", [], |r| r.get(0))?; + Ok( + serde_json::json!({"pool_size":size,"posts":posts,"reads":reads.load(std::sync::atomic::Ordering::Relaxed), + "seconds":elapsed.as_secs_f64(),"posting":posting_latencies(post_times.into_inner()), + "posting_connection_holds":posting_latencies(probe.holds.lock().clone()), + "reader_checkout":posting_latencies(probe.waits.lock().clone()), + "pool_timeouts":probe.timeouts.load(std::sync::atomic::Ordering::Relaxed),"pending_fs_ops":pending}), + ) +} + +#[test] +#[ignore = "explicit actual-media/connection-lifetime benchmark; set RUSTCHAN_DB_POST_EVIDENCE"] +fn posting_connection_workload() -> Result<()> { + let output = std::env::var("RUSTCHAN_DB_POST_EVIDENCE")?; + let mut measurements = Vec::new(); + for synchronous in ["NORMAL", "FULL"] { + for size in [2, 4, 8] { + measurements.push(media_posting_case(size, synchronous)?); + } + } + std::fs::write(output, serde_json::to_vec_pretty(&measurements)?)?; + Ok(()) +} diff --git a/src/handlers/render.rs b/src/handlers/render.rs index d5e8eab5..448f361d 100644 --- a/src/handlers/render.rs +++ b/src/handlers/render.rs @@ -121,8 +121,10 @@ pub(super) fn load_board_page_data( // The preview map is consumed here, so move each list out instead // of cloning every preview post. let preview_posts = previews.remove(&thread.id).unwrap_or_default(); - let omitted = - (thread.reply_count - i64::try_from(preview_posts.len()).unwrap_or(0)).max(0); + let omitted = thread + .reply_count + .saturating_sub(i64::try_from(preview_posts.len()).unwrap_or(0)) + .max(0); ThreadSummary { thread, preview_posts, @@ -309,7 +311,7 @@ mod tests { audio_file_name: None, audio_file_size: None, audio_mime_type: None, - created_at: 100 + id, + created_at: 100_i64.saturating_add(id), deletion_token: "token".into(), is_op: id == 1, edited_at: None, @@ -343,7 +345,7 @@ mod tests { vote_count: vote_counts[1], }, ], - total_votes: vote_counts[0] + vote_counts[1], + total_votes: vote_counts[0].saturating_add(vote_counts[1]), user_voted_option, is_expired: false, } diff --git a/src/handlers/setup.rs b/src/handlers/setup.rs index 22648852..83a9cf5e 100644 --- a/src/handlers/setup.rs +++ b/src/handlers/setup.rs @@ -13,7 +13,6 @@ use axum::{ }; use axum_extra::extract::cookie::{Cookie, CookieJar, SameSite}; use serde::{Deserialize, Serialize}; -use std::fmt::Write as _; /// Setup CSRF scope used by this handler. const SETUP_CSRF_SCOPE: &str = "first-run-setup"; @@ -696,9 +695,11 @@ pub(in crate::server) async fn setup_finish( move || -> Result<()> { let mut conn = pool.get()?; let tx = conn.transaction()?; - db::ensure_setup_available(&tx).map_err(|_setup_state_error| { - AppError::NotFound("Setup wizard is not available.".into()) - })?; + db::ensure_setup_available(&tx) + .map_err(|_setup_state_error| { + AppError::NotFound("Setup wizard is not available.".into()) + }) + .map(|_completed_value| ())?; if db::admin_count(&tx)? == 0 { let username = parsed.admin_username.as_deref().ok_or_else(|| { AppError::BadRequest("Initial admin username is required.".into()) @@ -711,7 +712,7 @@ pub(in crate::server) async fn setup_finish( })?; crypto::hash_password(password)? }; - db::create_admin(&tx, username, &password_hash)?; + db::create_admin(&tx, username, &password_hash).map(|_completed_value| ())?; } if db::board_slug_exists(&tx, &parsed.board_slug)? { return Err(AppError::Conflict(format!( @@ -741,9 +742,9 @@ pub(in crate::server) async fn setup_finish( access_password_hash = '' WHERE id = ?17", rusqlite::params![ - if parsed.allow_posting { 150 } else { 0 }, - 150, - 500, + if parsed.allow_posting { 150_i32 } else { 0_i32 }, + 150_i32, + 500_i32, parsed.image_limit_bytes, parsed.video_limit_bytes, parsed.audio_limit_bytes, @@ -759,7 +760,8 @@ pub(in crate::server) async fn setup_finish( parsed.board_visibility, board_id, ], - )?; + ) + .map(|_completed_value| ())?; db::set_site_setting(&tx, "site_name", &parsed.site_name)?; db::set_site_setting(&tx, "site_subtitle", &parsed.site_subtitle)?; db::set_site_setting(&tx, "default_theme", &parsed.default_theme)?; @@ -918,7 +920,8 @@ pub(in crate::server) async fn admin_close_setup( let pool = state.db.clone(); move || -> Result<()> { let conn = pool.get()?; - crate::handlers::admin::require_admin_session_sid(&conn, session_id.as_deref())?; + crate::handlers::admin::require_admin_session_sid(&conn, session_id.as_deref()) + .map(|_completed_value| ())?; db::close_reopened_setup(&conn)?; Ok(()) } @@ -1020,10 +1023,12 @@ fn setup_form_page( alerts.push_str(r#"
Admin setup session. Setup was reopened by an administrator. Existing credentials will not be replaced.
"#); } if let Some(warning) = transport_warning { - let _ = write!( - alerts, - r#"
{}
"#, - escape_html(warning) + templates::append_html( + &mut alerts, + format_args!( + r#"
{}
"#, + escape_html(warning) + ), ); } if !errors.is_empty() { @@ -1031,7 +1036,7 @@ fn setup_form_page( r#""); } @@ -1054,11 +1059,6 @@ fn setup_form_page( } else { "missing" }; - let ffprobe = if app_state.ffprobe_available { - "detected" - } else { - "missing" - }; let body = format!( r#"
@@ -1105,7 +1105,7 @@ fn setup_form_page(

The slug becomes the board URL and must contain 1–8 lowercase letters or digits.

Step 6

Uploads and media

Enable only the formats this instance should accept and set whole-MiB limits.

-

Runtime detection: ffmpeg {ffmpeg}; ffprobe {ffprobe}.

+

Runtime detection: ffmpeg {ffmpeg}; image/audio inspection built in.

@@ -1169,7 +1169,6 @@ fn setup_form_page( allow_self_delete = checked(form.allow_self_delete.as_deref()), allow_archive = checked(form.allow_archive.as_deref()), ffmpeg = ffmpeg, - ffprobe = ffprobe, allow_uploads = checked(form.allow_uploads.as_deref()), allow_video = checked(form.allow_video.as_deref()), allow_audio = checked(form.allow_audio.as_deref()), @@ -1248,10 +1247,10 @@ fn setup_review_page( admin = admin_line, slug = escape_html(&parsed.board_slug), board = escape_html(&parsed.board_name), - image = parsed.image_limit_bytes / i64::try_from(MIB).unwrap_or(1), - video = parsed.video_limit_bytes / i64::try_from(MIB).unwrap_or(1), - audio = parsed.audio_limit_bytes / i64::try_from(MIB).unwrap_or(1), - pdf = parsed.pdf_limit_bytes / i64::try_from(MIB).unwrap_or(1), + image = parsed.image_limit_bytes / 1_048_576_i64, + video = parsed.video_limit_bytes / 1_048_576_i64, + audio = parsed.audio_limit_bytes / 1_048_576_i64, + pdf = parsed.pdf_limit_bytes / 1_048_576_i64, tor = if parsed.enable_tor { "enabled" } else { @@ -1303,17 +1302,19 @@ fn preset_options(selected: &str) -> String { SetupPreset::Private => "Restricted board visibility and conservative access.", SetupPreset::Local => "Loopback-focused defaults for local evaluation.", }; - let _ = write!( - out, - r#"

{description}

load defaults
"#, - value = preset.as_str(), - checked = if selected == preset.as_str() { - " checked" - } else { - "" - }, - label = preset.label(), - description = description, + templates::append_html( + &mut out, + format_args!( + r#"

{description}

load defaults
"#, + value = preset.as_str(), + checked = if selected == preset.as_str() { + " checked" + } else { + "" + }, + label = preset.label(), + description = description, + ), ); } out @@ -1326,10 +1327,12 @@ fn visibility_options(selected: &str) -> String { ("view_password", "Require password to view"), ("post_password", "Require password to post"), ] { - let _ = write!( - out, - r#""#, - selected_attr = if selected == value { " selected" } else { "" }, + templates::append_html( + &mut out, + format_args!( + r#""#, + selected_attr = if selected == value { " selected" } else { "" }, + ), ); } out @@ -1449,7 +1452,7 @@ mod tests { use anyhow::{bail, ensure, Context as _, Result as AnyResult}; use axum::{ body::{to_bytes, Body}, - http::{header, Request, StatusCode}, + http::Request, routing::{get, post}, Router, }; @@ -1598,7 +1601,9 @@ mod tests { let state = crate::test_support::app_state(); { let conn = state.db.get().context("get database connection")?; - db::create_admin(&conn, "admin", "hash").context("create admin fixture")?; + db::create_admin(&conn, "admin", "hash") + .context("create admin fixture") + .map(|_completed_value| ())?; } let app = Router::new() .route("/setup", get(setup_get)) @@ -1617,11 +1622,12 @@ mod tests { .context("serve setup request")?; ensure!(response.status() == StatusCode::NOT_FOUND); - let body = to_bytes(response.into_body(), usize::MAX) + let response_body = to_bytes(response.into_body(), usize::MAX) .await .context("read setup response body")?; - let body = String::from_utf8(body.to_vec()).context("decode setup response body")?; - ensure!(body.contains("Setup wizard is not available")); + let response_body = + String::from_utf8(response_body.to_vec()).context("decode setup response body")?; + ensure!(response_body.contains("Setup wizard is not available")); Ok(()) } @@ -1719,13 +1725,14 @@ mod tests { .context("serve setup review request")?; ensure!(response.status() == StatusCode::OK); - let body = to_bytes(response.into_body(), usize::MAX) + let review_body = to_bytes(response.into_body(), usize::MAX) .await .context("read setup review response")?; - let body = String::from_utf8(body.to_vec()).context("decode setup review response")?; - ensure!(body.contains(r#"data-active-theme="blue-sky""#)); - ensure!(body.contains(r#"data-theme="blue-sky""#)); - ensure!(body.contains(r#"name="default_theme" value="blue-sky""#)); + let review_body = + String::from_utf8(review_body.to_vec()).context("decode setup review response")?; + ensure!(review_body.contains(r#"data-active-theme="blue-sky""#)); + ensure!(review_body.contains(r#"data-theme="blue-sky""#)); + ensure!(review_body.contains(r#"name="default_theme" value="blue-sky""#)); Ok(()) } @@ -1774,7 +1781,8 @@ mod tests { { let conn = state.db.get().context("get database connection")?; db::create_board(&conn, "b", "Existing", "", false) - .context("create conflicting board fixture")?; + .context("create conflicting board fixture") + .map(|_completed_value| ())?; } let app = Router::new() .route("/setup/finish", post(setup_finish)) diff --git a/src/handlers/thread.rs b/src/handlers/thread.rs index 1c693b0e..95d63071 100644 --- a/src/handlers/thread.rs +++ b/src/handlers/thread.rs @@ -15,7 +15,7 @@ use crate::{ admin_scoped_csrf_token, check_csrf_jar, ensure_csrf_for_request, ensure_csrf_with_secure, BoardAccessContext, }, - parse_post_multipart, posting, render, PostFormData, + is_xml_http_request, parse_post_multipart, posting, render, PostFormData, }, middleware::AppState, utils::crypto::hash_ip, @@ -38,13 +38,6 @@ type ThreadViewLoadResult = ( Option<(i64, i64)>, ); -fn is_xml_http_request(headers: &HeaderMap) -> bool { - headers - .get("x-requested-with") - .and_then(|value| value.to_str().ok()) - .is_some_and(|value| value.eq_ignore_ascii_case("XMLHttpRequest")) -} - // GET /:board/thread/:id #[expect( clippy::too_many_lines, @@ -108,7 +101,7 @@ pub(in crate::server) async fn view_thread( &CONFIG.cookie_secret, is_admin, )?; - let is_admin = page_data.is_admin; + let effective_is_admin = page_data.is_admin; let thread_badges_enabled = db::get_thread_new_reply_badges_enabled(&conn); let homepage_thread_badges_enabled = db::get_homepage_new_thread_badges_enabled(&conn); let homepage_reply_badges_enabled = db::get_homepage_new_reply_badges_enabled(&conn); @@ -116,7 +109,7 @@ pub(in crate::server) async fn view_thread( Ok(( render::thread_page_etag_signature(&page_data), page_data, - is_admin, + effective_is_admin, thread_badges_enabled, homepage_thread_badges_enabled, homepage_reply_badges_enabled, @@ -222,16 +215,16 @@ pub(in crate::server) async fn view_thread( .status(StatusCode::NOT_MODIFIED) .body(axum::body::Body::empty()) .map_err(|e| AppError::Internal(anyhow::anyhow!(e)))?; - resp.headers_mut().insert( + drop(resp.headers_mut().insert( "etag", HeaderValue::from_str(&etag).unwrap_or_else(|_| HeaderValue::from_static("\"0\"")), - ); - resp.headers_mut().insert( + )); + drop(resp.headers_mut().insert( header::CACHE_CONTROL, HeaderValue::from_static(crate::handlers::board::activity_html_cache_control( activity_markers_enabled, )), - ); + )); crate::cache::insert_vary_cookie(resp.headers_mut()); return Ok((jar, resp).into_response()); } @@ -260,14 +253,14 @@ pub(in crate::server) async fn view_thread( .map_err(|e| AppError::Internal(anyhow::anyhow!(e)))?; let mut resp = Html(html).into_response(); if let Ok(v) = HeaderValue::from_str(&etag) { - resp.headers_mut().insert("etag", v); + drop(resp.headers_mut().insert("etag", v)); } - resp.headers_mut().insert( + drop(resp.headers_mut().insert( header::CACHE_CONTROL, HeaderValue::from_static(crate::handlers::board::activity_html_cache_control( activity_markers_enabled, )), - ); + )); crate::cache::insert_vary_cookie(resp.headers_mut()); Ok((jar, resp).into_response()) } @@ -300,7 +293,7 @@ pub(in crate::server) async fn post_reply( &state, &board_short, admin_session_id.clone(), - access_cookie, + access_cookie.clone(), crate::handlers::board::BoardAccessRequirement::Post, format!("/{board_short}/thread/{thread_id}"), ) @@ -396,16 +389,13 @@ pub(in crate::server) async fn post_reply( let pool = state.db.clone(); let job_queue = std::sync::Arc::clone(&state.job_queue); let ffmpeg_available = state.ffmpeg_available; - let ffprobe_available = state.ffprobe_available; - let ffmpeg_webp_available = state.ffmpeg_webp_available; move || -> Result { // `spawn_blocking` work is not cancelled when its join handle is // dropped. Keep the permit inside this closure so a disconnected // request cannot release the media gate while parsing continues. let _media_upload_guard = media_upload_guard; - let conn = pool.get()?; posting::submit_post( - &conn, + &pool, &job_queue, posting::SubmitPostCommand { mode: posting::SubmitPostMode::Reply { thread_id, sage }, @@ -413,6 +403,7 @@ pub(in crate::server) async fn post_reply( identity_key, cookie_secret: CONFIG.cookie_secret.clone(), admin_session_id, + access_cookie, ban_csrf_token, submission_token, name, @@ -426,8 +417,6 @@ pub(in crate::server) async fn post_reply( upload_dir: CONFIG.upload_dir.clone(), thumb_size: CONFIG.thumb_size, ffmpeg_available, - ffprobe_available, - ffmpeg_webp_available, }, ) } @@ -512,7 +501,7 @@ pub(in crate::server) async fn post_reply( submit_result.thread_id, submit_result.post_id, &submit_result.deletion_token, - submit_result.created_at + crate::handlers::board::self_action_window_secs(), + crate::handlers::board::self_action_expiry(submit_result.created_at)?, crate::handlers::board::should_set_public_secure_cookie(&req_headers, secure_context), ); @@ -627,10 +616,10 @@ async fn render_edit_post_error_page( let mut response = Html(html).into_response(); *response.status_mut() = StatusCode::UNPROCESSABLE_ENTITY; - response.headers_mut().insert( + drop(response.headers_mut().insert( header::CACHE_CONTROL, HeaderValue::from_static(crate::cache::CACHE_CONTROL_PRIVATE_NO_STORE), - ); + )); Ok((jar, response).into_response()) } @@ -681,12 +670,7 @@ pub(in crate::server) async fn edit_post_get( } let now = chrono::Utc::now().timestamp(); - if now - > context - .post - .created_at - .saturating_add(crate::handlers::board::self_action_window_secs()) - { + if now > crate::handlers::board::self_action_expiry(context.post.created_at)? { return Err(AppError::Forbidden( "The 60-second edit window for this post has closed.".into(), )); @@ -717,10 +701,10 @@ pub(in crate::server) async fn edit_post_get( None, ); let mut response = Html(html).into_response(); - response.headers_mut().insert( + drop(response.headers_mut().insert( header::CACHE_CONTROL, HeaderValue::from_static(crate::cache::CACHE_CONTROL_PRIVATE_NO_STORE), - ); + )); Ok((jar, response).into_response()) } @@ -961,12 +945,7 @@ pub(in crate::server) async fn delete_post_get( } let now = chrono::Utc::now().timestamp(); - if now - > context - .post - .created_at - .saturating_add(crate::handlers::board::self_action_window_secs()) - { + if now > crate::handlers::board::self_action_expiry(context.post.created_at)? { return Err(AppError::Forbidden( "The 60-second self-delete window for this post has closed.".into(), )); @@ -998,10 +977,10 @@ pub(in crate::server) async fn delete_post_get( None, ); let mut response = Html(html).into_response(); - response.headers_mut().insert( + drop(response.headers_mut().insert( header::CACHE_CONTROL, HeaderValue::from_static(crate::cache::CACHE_CONTROL_PRIVATE_NO_STORE), - ); + )); Ok((jar, response).into_response()) } @@ -1227,8 +1206,9 @@ pub(in crate::server) async fn vote_handler( // Voting is persisted participation in the thread, so the same ban // gate as posting applies; CSRF was validated above. crate::handlers::board::ensure_actor_not_banned(&conn, &ip_hash, ban_csrf_token)?; - let (poll_id, thread_id, board_short) = db::get_poll_context(&conn, option_id)? - .ok_or_else(|| AppError::NotFound("Poll option not found.".into()))?; + let (poll_id, resolved_thread_id, resolved_board_short) = + db::get_poll_context(&conn, option_id)? + .ok_or_else(|| AppError::NotFound("Poll option not found.".into()))?; let recorded = db::cast_vote(&conn, poll_id, option_id, &ip_hash)?; if !recorded { @@ -1242,7 +1222,9 @@ pub(in crate::server) async fn vote_handler( option_id = option_id, "Vote cast" ); - Ok(format!("/{board_short}/thread/{thread_id}#poll")) + Ok(format!( + "/{resolved_board_short}/thread/{resolved_thread_id}#poll" + )) } }) .await @@ -1577,10 +1559,8 @@ mod tests { } fn flac_fixture(size: usize) -> Vec { - let mut bytes = vec![0_u8; size.max(4)]; - if let Some(prefix) = bytes.get_mut(..4) { - prefix.copy_from_slice(b"fLaC"); - } + let mut bytes = include_bytes!("../../tests/fixtures/media/tone.flac").to_vec(); + bytes.resize(size.max(bytes.len()), 0); bytes } @@ -1595,14 +1575,15 @@ mod tests { "UPDATE boards SET allow_audio = 1, max_audio_size = ?1 WHERE id = ?2", rusqlite::params![max_audio_size, board_id], ) - .context("failed to enable audio board")?; + .context("failed to enable audio board") + .map(|_completed_value| ())?; Ok(()) } #[tokio::test] async fn create_thread_and_reply_accept_audio_within_board_limit() -> Result<()> { let state = crate::test_support::app_state(); - seed_audio_board(&state, 5_000)?; + seed_audio_board(&state, 20_000)?; let router = Router::new() .route("/{board}", post(crate::handlers::board::create_thread)) @@ -1673,13 +1654,13 @@ mod tests { #[tokio::test] async fn create_thread_rejects_audio_over_board_limit_with_413() -> Result<()> { let state = crate::test_support::app_state(); - seed_audio_board(&state, 5_000)?; + seed_audio_board(&state, 20_000)?; let router = Router::new() .route("/{board}", post(crate::handlers::board::create_thread)) .with_state(state); - let audio = flac_fixture(5_001); + let audio = flac_fixture(20_001); let (boundary, body) = crate::test_support::multipart_body( &[("_csrf", "csrf123"), ("body", "")], Some(("audio_file", "too-large.flac", &audio, "audio/flac")), @@ -1776,15 +1757,15 @@ mod tests { .context("failed to receive response")?; assert_eq!(response.status(), StatusCode::UNSUPPORTED_MEDIA_TYPE); - let body = String::from_utf8( + let resolved_body = String::from_utf8( to_bytes(response.into_body(), usize::MAX) .await .context("failed to read response body")? .to_vec(), ) .context("response body was not valid UTF-8")?; - assert!(body.contains("post-error-banner")); - assert!(body.contains("File type not allowed")); + assert!(resolved_body.contains("post-error-banner")); + assert!(resolved_body.contains("File type not allowed")); Ok(()) } @@ -1808,7 +1789,8 @@ mod tests { board_id ], ) - .context("failed to update board toggles")?; + .context("failed to update board toggles") + .map(|_completed_value| ())?; let post = crate::db::NewPost { thread_id: 0, board_id, @@ -1838,9 +1820,13 @@ mod tests { if age_secs > 0 { conn.execute( "UPDATE posts SET created_at = ?1 WHERE id = ?2", - rusqlite::params![chrono::Utc::now().timestamp() - age_secs, post_id], + rusqlite::params![ + chrono::Utc::now().timestamp().saturating_sub(age_secs), + post_id + ], ) - .context("failed to age post")?; + .context("failed to age post") + .map(|_completed_value| ())?; } drop(conn); @@ -1850,7 +1836,9 @@ mod tests { thread_id, post_id, "edit-token", - chrono::Utc::now().timestamp() + crate::handlers::board::self_action_window_secs(), + chrono::Utc::now() + .timestamp() + .saturating_add(crate::handlers::board::self_action_window_secs()), ); let cookie = jar .get("rustchan_owned_posts") @@ -1874,7 +1862,8 @@ mod tests { "UPDATE threads SET locked = ?1, archived = ?2 WHERE id = ?3", rusqlite::params![i64::from(locked), i64::from(archived), thread_id], ) - .context("failed to update thread state")?; + .context("failed to update thread state") + .map(|_completed_value| ())?; Ok(()) } @@ -1887,7 +1876,8 @@ mod tests { .get() .context("failed to get database connection")?; crate::db::create_board(&conn, "test", "Test", "", false) - .context("failed to create board")?; + .context("failed to create board") + .map(|_completed_value| ())?; } let router = Router::new() @@ -1985,7 +1975,8 @@ mod tests { .get() .context("failed to get database connection")?; crate::db::create_board(&conn, "test", "Test", "", false) - .context("failed to create board")?; + .context("failed to create board") + .map(|_completed_value| ())?; } let router = Router::new() @@ -2084,7 +2075,8 @@ mod tests { .get() .context("failed to get database connection")?; crate::db::create_board(&conn, "test", "Test", "", false) - .context("failed to create board")?; + .context("failed to create board") + .map(|_completed_value| ())?; } let router = Router::new() @@ -2171,58 +2163,62 @@ mod tests { Ok(()) } + /// Seed the same-client thread whose cooldown should reject a subsequent reply. + fn seed_cooldown_thread(state: &crate::middleware::AppState) -> Result { + let conn = state + .db + .get() + .context("failed to get database connection")?; + let board_id = crate::db::create_board(&conn, "test", "Test", "", false) + .context("failed to create board")?; + conn.execute( + "UPDATE boards SET post_cooldown_secs = 60 WHERE short_name = 'test'", + [], + ) + .context("failed to enable cooldown") + .map(|_completed_value| ())?; + + let ip_hash = + crate::utils::crypto::hash_ip("127.0.0.1", &crate::config::CONFIG.cookie_secret); + let post = crate::db::NewPost { + thread_id: 0, + board_id, + name: "anon".to_owned(), + tripcode: None, + subject: Some("subject".to_owned()), + body: "op body".to_owned(), + body_html: "op body".to_owned(), + ip_hash: Some(ip_hash), + file_path: None, + file_name: None, + file_size: None, + thumb_path: None, + mime_type: None, + media_type: None, + audio_file_path: None, + audio_file_name: None, + audio_file_size: None, + audio_mime_type: None, + deletion_token: "token".to_owned(), + is_op: true, + }; + crate::db::create_thread_with_optional_poll( + &conn, + board_id, + Some("subject"), + &post, + "", + None, + None, + ) + .map(|(created_thread_id, _, _)| created_thread_id) + .context("failed to create thread") + } + #[tokio::test] async fn reply_cooldown_failure_rerenders_thread_inline() -> Result<()> { let state = crate::test_support::app_state(); - let thread_id = { - let conn = state - .db - .get() - .context("failed to get database connection")?; - let board_id = crate::db::create_board(&conn, "test", "Test", "", false) - .context("failed to create board")?; - conn.execute( - "UPDATE boards SET post_cooldown_secs = 60 WHERE short_name = 'test'", - [], - ) - .context("failed to enable cooldown")?; - - let ip_hash = - crate::utils::crypto::hash_ip("127.0.0.1", &crate::config::CONFIG.cookie_secret); - let post = crate::db::NewPost { - thread_id: 0, - board_id, - name: "anon".to_owned(), - tripcode: None, - subject: Some("subject".to_owned()), - body: "op body".to_owned(), - body_html: "op body".to_owned(), - ip_hash: Some(ip_hash), - file_path: None, - file_name: None, - file_size: None, - thumb_path: None, - mime_type: None, - media_type: None, - audio_file_path: None, - audio_file_name: None, - audio_file_size: None, - audio_mime_type: None, - deletion_token: "token".to_owned(), - is_op: true, - }; - let (thread_id, _, _) = crate::db::create_thread_with_optional_poll( - &conn, - board_id, - Some("subject"), - &post, - "", - None, - None, - ) - .context("failed to create thread")?; - thread_id - }; + let thread_id = seed_cooldown_thread(&state)?; let router = Router::new() .route("/{board}/thread/{id}", post(super::post_reply)) diff --git a/src/lib.rs b/src/lib.rs index 3d995b9e..7e250149 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -50,6 +50,8 @@ pub mod test_fixtures; #[cfg(test)] /// Shared state and request builders for crate-local tests. pub(crate) use test_fixtures as test_support; +/// Settings restart coordination and supervisor recovery. +pub mod restart; /// Built-in theme metadata. pub mod theme; /// Custom theme configuration and CSS generation. diff --git a/src/logging.rs b/src/logging.rs index fd1864e2..32e6825b 100644 --- a/src/logging.rs +++ b/src/logging.rs @@ -13,7 +13,7 @@ use std::fmt; use std::io::{self, Read as _, Seek as _, SeekFrom, Write as _}; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::{LazyLock, OnceLock}; +use std::sync::LazyLock; use std::time::{Duration, Instant}; use tracing::field::{Field, Visit}; @@ -48,10 +48,12 @@ static CONSOLE_MUTEX: LazyLock> = // requiring callers to thread it through their own state, and without changing // the public `init_logging(&Path)` signature. /// Keeps the main log writer's background worker alive for the process lifetime. -static MAIN_FILE_GUARD: OnceLock = OnceLock::new(); +static MAIN_FILE_GUARD: parking_lot::Mutex> = + parking_lot::Mutex::new(None); /// Keeps the dependency log writer's background worker alive for the process lifetime. -static DEPENDENCY_FILE_GUARD: OnceLock = - OnceLock::new(); +static DEPENDENCY_FILE_GUARD: parking_lot::Mutex< + Option, +> = parking_lot::Mutex::new(None); /// Fallback main-log filename used when daily rotation cannot be initialized. pub const MAIN_LOG_FALLBACK_FILE_NAME: &str = "rustchan.log"; @@ -74,8 +76,9 @@ pub fn is_tty() -> bool { } /// Returns `true` when stdout is an interactive terminal that can consume ANSI +/// escape sequences. /// -/// escape sequences. On Windows this also probes/enables virtual terminal +/// On Windows this also probes/enables virtual terminal /// processing so raw colour sequences are not printed literally. pub fn ansi_enabled() -> bool { ANSI_ENABLED.load(Ordering::Relaxed) @@ -185,20 +188,22 @@ pub fn read_log_tail(path: &Path, max_bytes: usize) -> Result<(String, bool), St // complete line instead of dropping it as an assumed partial prefix. let read_start = start.saturating_sub(1); file.seek(SeekFrom::Start(read_start)) - .map_err(|error| format!("Seek log: {error}"))?; + .map_err(|error| format!("Seek log: {error}")) + .map(|_completed_value| ())?; let mut bytes = Vec::with_capacity(usize::try_from(length.saturating_sub(read_start)).unwrap_or(max_bytes)); io::copy( &mut file.take(length.saturating_sub(read_start)), &mut bytes, ) - .map_err(|error| format!("Read log: {error}"))?; + .map_err(|error| format!("Read log: {error}")) + .map(|_completed_value| ())?; let truncated = start > 0; let content = if truncated { let prefix = bytes .iter() .position(|byte| *byte == b'\n') - .map_or(1, |index| index + 1); + .map_or(1, |index| index.saturating_add(1)); bytes.get(prefix..).unwrap_or_default() } else { bytes.as_slice() @@ -242,7 +247,7 @@ fn is_dependency_log_target(target: &str) -> bool { /// `rustchan::server::server` → `server` /// `rustchan::db::mod` → `db` /// `rustchan::workers` → `workers` -/// `tower_http::trace` → `trace` +/// `tower_http::trace` → `trace`. fn extract_component(target: &str) -> &str { let mut parts = target.rsplit("::"); let last = parts.next().unwrap_or(target); @@ -475,14 +480,14 @@ fn parse_compound_duration(value: &str) -> Option { .find(|ch: char| !ch.is_ascii_digit()) .filter(|&index| index > 0)?; let (number, unit) = part.split_at(split); - let value = number.parse::().ok()?; + let duration_value = number.parse::().ok()?; let nanos = match unit { - "h" => value.saturating_mul(3_600_000_000_000), - "m" => value.saturating_mul(60_000_000_000), - "s" => value.saturating_mul(1_000_000_000), - "ms" => value.saturating_mul(1_000_000), - "us" | "µs" => value.saturating_mul(1_000), - "ns" => value, + "h" => duration_value.saturating_mul(3_600_000_000_000), + "m" => duration_value.saturating_mul(60_000_000_000), + "s" => duration_value.saturating_mul(1_000_000_000), + "ms" => duration_value.saturating_mul(1_000_000), + "us" | "µs" => duration_value.saturating_mul(1_000), + "ns" => duration_value, _ => return None, }; total_nanos = total_nanos.saturating_add(nanos); @@ -857,7 +862,7 @@ fn rewrite_message(target: &str, file: Option<&str>, fields: &mut LogEventFields "reactor" => { if message.eq_ignore_ascii_case("removing circuit leg") { fields.message = Some("Tor circuit closed".to_owned()); - remove_field(&mut fields.fields, "tunnel_id"); + let _previous_value = remove_field(&mut fields.fields, "tunnel_id"); } else if message.contains("descriptor upload") || message.contains("Unable to upload") || message.contains("publish") @@ -1024,10 +1029,10 @@ fn env_filter() -> EnvFilter { /// Writes one compact line per log event to the terminal. /// /// TTY mode (local dev): -/// `14:22:01.123 [INFO ] [server ] HTTP server listening addr=0.0.0.0:8080` +/// `14:22:01.123 [INFO ] [server ] HTTP server listening addr=0.0.0.0:8080`. /// /// Non-TTY mode (piped / systemd / Docker): -/// `2026-03-18 14:22:01.123 [INFO ] [server ] HTTP server listening addr=0.0.0.0:8080` +/// `2026-03-18 14:22:01.123 [INFO ] [server ] HTTP server listening addr=0.0.0.0:8080`. /// /// Columns are fixed-width so the message text always starts at the same /// horizontal position, making it easy to scan down a busy log stream. @@ -1076,7 +1081,7 @@ where /// /// Format: /// `2026-03-18 14:22:01.123 [INFO ] [server ] HTTP server listening addr=0.0.0.0:8080` -/// `2026-03-18 14:22:01.456 [ERROR] [error ] DB query failed err=no such table (src/db/posts.rs:79)` +/// `2026-03-18 14:22:01.456 [ERROR] [error ] DB query failed err=no such table (src/db/posts.rs:79)`. /// /// Differences from the terminal format: /// • Always UTC with the full date — the file is an archive, not a live view. @@ -1244,7 +1249,7 @@ pub fn init_logging(log_dir: &Path) { // The WorkerGuard is stored in MAIN_FILE_GUARD so it lives for the entire // process — see the comment on that static for why this matters. let (main_file_writer, main_guard) = tracing_appender::non_blocking(rolling); - drop(MAIN_FILE_GUARD.set(main_guard)); + *MAIN_FILE_GUARD.lock() = Some(main_guard); let file_layer = tracing_subscriber::fmt::layer() .event_format(FileFormatter) @@ -1254,7 +1259,7 @@ pub fn init_logging(log_dir: &Path) { let dependency_log = tracing_appender::rolling::never(log_dir, DEPENDENCY_LOG_FILE_NAME); let (dependency_file_writer, dependency_guard) = tracing_appender::non_blocking(dependency_log); - drop(DEPENDENCY_FILE_GUARD.set(dependency_guard)); + *DEPENDENCY_FILE_GUARD.lock() = Some(dependency_guard); let dependency_file_layer = tracing_subscriber::fmt::layer() .event_format(FileFormatter) @@ -1274,12 +1279,6 @@ pub fn init_logging(log_dir: &Path) { // correctly with the tracing terminal layer. Use these instead of // `println!`/`print!` in `console.rs` and `detect.rs`. -/// Print `msg` followed by a newline to stdout, under the console lock. -pub fn console_println(msg: &str) { - let _guard = CONSOLE_MUTEX.lock(); - drop(writeln!(io::stdout(), "{msg}")); -} - /// Write a raw pre-formatted block exactly as provided (no trailing newline added). /// /// Used for banners, install-hint blocks, and stats sections that are already @@ -1301,6 +1300,12 @@ pub fn console_prompt(msg: &str) { // _guard dropped here — stdin read happens outside the lock } +/// Drain queued file logging at the end of the executable lifecycle. +pub fn shutdown() { + drop(MAIN_FILE_GUARD.lock().take()); + drop(DEPENDENCY_FILE_GUARD.lock().take()); +} + #[cfg(test)] mod tests { use super::{ @@ -1368,7 +1373,7 @@ mod tests { .iter() .filter_map(|(name, value)| { let clean = normalize_field_value(name, value); - (!super::should_hide_field(name, &clean)).then(|| ((*name).to_string(), clean)) + (!super::should_hide_field(name, &clean)).then(|| ((*name).to_owned(), clean)) }) .collect(), }; @@ -1580,17 +1585,17 @@ mod tests { vec![("retry_in".to_owned(), "30s".to_owned())] ); - let fields = rewrite_for_test( + let disabled_guard_fields = rewrite_for_test( "tor_guardmgr::guard", "Disabling guard: 72.5% of circuits died under mysterious circumstances, exceeding threshold of 70.0%", &[("guard", "GuardId(abc)")], ); assert_eq!( - fields.message.as_deref(), + disabled_guard_fields.message.as_deref(), Some("Tor disabled an unstable guard") ); assert_eq!( - fields.fields, + disabled_guard_fields.fields, vec![("failure_rate".to_owned(), "72.5%".to_owned())] ); } @@ -1617,16 +1622,19 @@ mod tests { )] ); - let fields = rewrite_for_test( + let bootstrap_failure_fields = rewrite_for_test( "tor_dirmgr::bootstrap", "We failed 3 times to bootstrap a directory. We're going to give up.", &[], ); assert_eq!( - fields.message.as_deref(), + bootstrap_failure_fields.message.as_deref(), Some("Tor directory bootstrap failed") ); - assert_eq!(fields.fields, vec![("attempts".to_owned(), "3".to_owned())]); + assert_eq!( + bootstrap_failure_fields.fields, + vec![("attempts".to_owned(), "3".to_owned())] + ); } #[test] @@ -1641,14 +1649,17 @@ mod tests { Some("Tor onion-service circuits are failing; waiting before retrying") ); - let fields = rewrite_for_test( + let circuit_fields = rewrite_for_test( "tor_proto::client::reactor", "removing circuit leg", &[("tunnel_id", "Circ 8.98"), ("reason", "closed")], ); - assert_eq!(fields.message.as_deref(), Some("Tor circuit closed")); assert_eq!( - fields.fields, + circuit_fields.message.as_deref(), + Some("Tor circuit closed") + ); + assert_eq!( + circuit_fields.fields, vec![("reason".to_owned(), "closed".to_owned())] ); } diff --git a/src/main.rs b/src/main.rs index 4a67f6bb..d3b901d0 100644 --- a/src/main.rs +++ b/src/main.rs @@ -53,6 +53,8 @@ pub mod test_fixtures; #[cfg(test)] /// Shared state and request builders for crate-local tests. pub(crate) use test_fixtures as test_support; +/// Settings restart coordination and supervisor recovery. +pub mod restart; /// Built-in theme metadata. pub mod theme; /// Custom theme configuration and CSS generation. @@ -153,7 +155,7 @@ fn main() -> anyhow::Result<()> { "{}", serde_json::json!({ "version": env!("CARGO_PKG_VERSION"), "target": updates::platform_target(), - "schema": db::baseline_schema_version(), "minimum_schema": "1.5.0", "updater_protocol": 1 + "schema": db::baseline_schema_version(), "minimum_schema": "1.5.0", "updater_protocol": 1_u32 }) )?; return Ok(()); @@ -204,6 +206,9 @@ fn main() -> anyhow::Result<()> { } generate_settings_file_if_missing(); + if matches!(&cli.command, None | Some(server::cli::Command::Serve)) { + restart::prepare_startup()?; + } logging::init_logging(&log_dir); // Install a panic hook that restores the terminal before printing the @@ -238,7 +243,7 @@ fn main() -> anyhow::Result<()> { .build() .map_err(|e| anyhow::anyhow!("Failed to build Tokio runtime: {e}"))?; - rt.block_on(async move { + let result = rt.block_on(async move { // Install the ring crypto provider once before anything else accesses // rustls. ok() = harmless if already installed (tests, re-runs). drop(rustls::crypto::ring::default_provider().install_default()); @@ -258,5 +263,8 @@ fn main() -> anyhow::Result<()> { Ok(()) } } - }) + }); + rt.shutdown_timeout(std::time::Duration::from_secs(10)); + logging::shutdown(); + result } diff --git a/src/media/audio.rs b/src/media/audio.rs new file mode 100644 index 00000000..c8933c5d --- /dev/null +++ b/src/media/audio.rs @@ -0,0 +1,1004 @@ +//! Incremental PCM decoding and deterministic, bounded-memory waveform images. + +use anyhow::{ensure, Context as _, Result}; +use image::{Rgba, RgbaImage}; +#[path = "audio_mp4.rs"] +pub(super) mod mp4; +#[path = "audio_opus.rs"] +mod opus; +use std::path::Path; +use std::time::{Duration, Instant}; +use symphonia::core::codecs::audio::well_known::CODEC_ID_OPUS; +use symphonia::core::codecs::audio::{AudioCodecParameters, AudioDecoder, AudioDecoderOptions}; +use symphonia::core::formats::TrackType; +use symphonia::core::io::{BitReaderLtr, FiniteBitStream as _, ReadBitsLtr as _}; +use symphonia::core::packet::Packet; +use tokio_util::sync::CancellationToken; + +/// Distinguish unavailable codecs from malformed input and resource failures. +pub(crate) fn is_unsupported(error: &anyhow::Error) -> bool { + // Symphonia also calls missing MP4 atoms and excessive MKV geometry + // Unsupported. Only audited codec limitations may select compatibility. + // Opus reconstruction and packet errors fail closed. + error.chain().any(|cause| { + matches!( + cause.downcast_ref::(), + Some(symphonia::core::errors::Error::Unsupported( + "Opus mapping family requires the compatibility decoder" + | "AAC profile needs the compatibility decoder" + | "audio codec needs the compatibility decoder" + | "Speex requires the compatibility decoder" + | "aac: aac too complex" + | "aac: spectral band replication is unsupported" + | "audio edit list requires the compatibility decoder" + )) + ) + }) +} + +/// Maximum interleaved samples retained for a single decoded audio packet. +const MAX_PACKET_SAMPLES: usize = 4 * 1024 * 1024; + +/// Audio codecs share one packet-to-interleaved-PCM boundary. +enum PacketDecoder { + /// Mature general-purpose pure-Rust audio decoders. + Symphonia(Box), + /// Bounded mono/stereo/multistream Opus with explicit channel mappings. + Opus(Box), +} + +impl PacketDecoder { + /// Select a decoder from validated container metadata. + fn new(params: &AudioCodecParameters, channels: usize) -> Result { + if params.codec != CODEC_ID_OPUS { + if let Some(frames) = params.max_frames_per_packet { + ensure!( + u128::from(frames) + .checked_mul(u128::try_from(channels)?) + .context("audio sample budget overflow")? + <= u128::try_from(MAX_PACKET_SAMPLES)?, + "audio packet exceeds sample budget" + ); + } + return Ok(Self::Symphonia( + symphonia::default::get_codecs() + .make_audio_decoder(params, &AudioDecoderOptions::default())?, + )); + } + let header = params + .extra_data + .as_deref() + .context("Opus header missing")?; + Ok(Self::Opus(Box::new(opus::Decoder::new(header, channels)?))) + } + + /// Decode one packet into a reused bounded float buffer. + fn decode( + &mut self, + packet: &Packet, + channels: usize, + samples: &mut Vec, + budget: &AudioBudget<'_>, + ) -> Result<()> { + match self { + Self::Symphonia(decoder) => { + let pcm = decoder.decode(packet).context("decode audio packet")?; + let count = pcm.samples_interleaved(); + ensure!( + count <= MAX_PACKET_SAMPLES, + "decoded audio packet exceeds sample budget" + ); + ensure!( + pcm.spec().channels().count() == channels, + "audio channel count changed" + ); + samples.resize(count, 0.0); + pcm.copy_to_slice_interleaved(samples.as_mut_slice()); + } + Self::Opus(decoder) => { + let _decoded_frames = decoder.decode(&packet.data, samples, || budget.check())?; + } + } + ensure!( + samples.iter().all(|sample| sample.is_finite()), + "non-finite PCM samples" + ); + Ok(()) + } +} + +/// Validate a discrete Opus identification header without decoding PCM. +pub(crate) fn validate_discrete_opus_header( + header: &[u8], + head: &opus_pure::OpusHead, +) -> Result<()> { + let channels = usize::from(head.channel_count); + opus::Decoder::validate(header, channels)?; + ensure!( + header.get(18) == Some(&head.mapping_family) + && header.get(19) == Some(&head.stream_count) + && header.get(20) == Some(&head.coupled_count) + && header.get(21..21 + channels) == Some(head.channel_mapping.as_slice()) + && header.get(10..12) == Some(head.pre_skip.to_le_bytes().as_slice()) + && header.get(16..18) == Some(head.output_gain_q8.to_le_bytes().as_slice()), + "Opus headers changed during validation" + ); + Ok(()) +} + +/// Stream discrete Ogg layouts through the same bounded, explicitly mapped decoder. +fn visit_discrete_opus( + input: &Path, + header: &[u8], + budget: &AudioBudget<'_>, + mut visit: impl FnMut(f32) -> Result<()>, +) -> Result { + use opus_pure::{OggOpusReader, OggPacket, Trim}; + let (source, reads) = super::probe::open_discrete_opus_source( + input, + budget.started, + budget.timeout, + budget.cancel, + )?; + let mut reader = OggOpusReader::new(source)?; + validate_discrete_opus_header(header, reader.head())?; + let channels = usize::from(reader.head().channel_count); + let mut decoder = opus::Decoder::new(header, channels)?; + let mut trim = Trim::new(reader.head(), 48_000, channels)?; + let gain = 10.0_f32.powf(f32::from(reader.head().output_gain_q8) / (256.0 * 20.0)); + let mut packet = OggPacket::default(); + let mut samples = Vec::new(); + let mut frames = 0_u64; + loop { + budget.check()?; + reads.start_packet(); + if !reader.read_packet_into(&mut packet)? { + break; + } + let _decoded_frames = decoder.decode(&packet.data, &mut samples, || budget.check())?; + ensure!( + samples.iter().all(|sample| sample.is_finite()), + "non-finite Opus PCM" + ); + for frame in trim.keep(&packet, &samples).chunks_exact(channels) { + let amplitude = frame_amplitude(frame, gain)?; + visit(amplitude)?; + frames = frames + .checked_add(1) + .context("Opus sample count overflow")?; + } + } + budget.check()?; + ensure!(frames > 0, "Opus contains no decodable samples"); + Ok(frames) +} + +/// Opus presentation metadata is independent of the PCM decoder state. +struct OpusPresentation { + /// Exact remaining delay from the identification header, in 48 kHz samples. + delay: usize, + /// Linear output gain from the signed Q8 decibel header field. + gain: f32, + /// Container timestamp units for packet trimming. + time_base: symphonia::core::units::TimeBase, +} + +impl OpusPresentation { + /// Read metadata from the already validated stream parameters. + fn new( + params: &AudioCodecParameters, + time_base: Option, + ) -> Result { + let header = params + .extra_data + .as_deref() + .context("Opus header missing")?; + let delay: [u8; 2] = header + .get(10..12) + .context("Opus delay missing")? + .try_into()?; + let gain: [u8; 2] = header + .get(16..18) + .context("Opus gain missing")? + .try_into()?; + Ok(Self { + delay: usize::from(u16::from_le_bytes(delay)), + // OpusHead gain is Q8 dB (RFC 7845 section 5.1). + gain: 10.0_f32.powf(f32::from(i16::from_le_bytes(gain)) / (256.0 * 20.0)), + time_base: time_base.context("Opus time base missing")?, + }) + } + + /// Return playable frames, removing exact header delay and container padding. + fn samples<'a>( + &mut self, + packet: &Packet, + samples: &'a [f32], + channels: usize, + ) -> Result<&'a [f32]> { + let to_frames = |duration: symphonia::core::units::Duration| -> Result { + let numerator = u128::from(duration.get()) + .checked_mul(u128::from(self.time_base.numer.get())) + .and_then(|numerator| numerator.checked_mul(48_000)) + .context("Opus timestamp conversion overflow")?; + let denominator = u128::from(self.time_base.denom.get()); + Ok(usize::try_from( + numerator + .checked_add(denominator / 2) + .and_then(|rounded| rounded.checked_div(denominator)) + .context("invalid Opus timestamp denominator or overflow")?, + )?) + }; + // Ogg supplies exact trimming; Matroska can round delay to milliseconds. + let start = to_frames(packet.trim_start)?.max(self.delay).min( + samples + .len() + .checked_div(channels) + .context("Opus has no channels")?, + ); + self.delay = self.delay.saturating_sub(start); + let end = to_frames(packet.trim_end)?; + let first = start.checked_mul(channels).context("Opus trim overflow")?; + let last = samples + .len() + .checked_sub(end.checked_mul(channels).context("Opus trim overflow")?) + .context("Opus padding exceeds packet")?; + samples.get(first..last).context("Opus trim exceeds packet") + } +} + +/// Deadline/cancellation policy shared by both streaming waveform passes. +struct AudioBudget<'a> { + /// Time at the start of the job. + started: Instant, + /// Existing operator media-processing timeout. + timeout: Duration, + /// Worker shutdown cancellation token. + cancel: &'a CancellationToken, +} + +/// Detect backward-compatible HE-AAC signaling after an AAC-LC configuration. +/// +/// Symphonia 0.6.1 skips this sync extension when no explicit SBR prefix was +/// present, decoding only the low-rate core while reporting AAC-LC. Inspect the +/// bounded configuration, including extension frequency, before choosing it. +fn aac_needs_compatibility(extra: &[u8]) -> Result { + ensure!(extra.len() <= 64, "AAC configuration exceeds safety budget"); + let mut bits = BitReaderLtr::new(extra); + let profile = bits.read_bits_leq32(5)?; + ensure!(matches!(profile, 1..=5 | 29), "unvalidated AAC profile"); + read_aac_frequency(&mut bits)?; + let channels = bits.read_bits_leq32(4)?; + ensure!(channels <= 7, "invalid AAC channel configuration"); + let explicit_sbr = matches!(profile, 5 | 29); + if explicit_sbr { + read_aac_frequency(&mut bits)?; + ensure!(bits.read_bits_leq32(5)? == 2, "invalid HE-AAC core profile"); + } + let _short_frame = bits.read_bool()?; + if bits.read_bool()? { + bits.ignore_bits(14)?; + } + if bits.read_bool()? { + let _extension_flag3 = bits.read_bool()?; + } + if explicit_sbr || profile != 2 { + return Ok(true); + } + // PCE channel layouts are handled by the decoder's explicit limitation. + if channels == 0 || bits.bits_left() < 16 { + return Ok(false); + } + if bits.read_bits_leq32(11)? != 0x2b7 { + return Ok(false); + } + ensure!( + bits.read_bits_leq32(5)? == 5, + "invalid AAC SBR extension type" + ); + let sbr = bits.read_bool()?; + if sbr { + read_aac_frequency(&mut bits)?; + if bits.bits_left() >= 12 && bits.read_bits_leq32(11)? == 0x548 { + let _parametric_stereo = bits.read_bool()?; + } + } + Ok(sbr) +} + +/// Validate a sampling-frequency index or a nonzero explicit 24-bit frequency. +fn read_aac_frequency(bits: &mut BitReaderLtr<'_>) -> Result<()> { + let index = bits.read_bits_leq32(4)?; + if index == 15 { + ensure!( + bits.read_bits_leq32(24)? > 0, + "invalid AAC sampling frequency" + ); + } else { + ensure!(index <= 12, "invalid AAC sampling frequency index"); + } + Ok(()) +} + +impl AudioBudget<'_> { + /// Check the deadline before decoding another packet or publishing pixels. + fn check(&self) -> Result<()> { + ensure!( + !self.cancel.is_cancelled(), + "audio processing cancelled during shutdown" + ); + ensure!( + self.started.elapsed() < self.timeout, + "audio processing timed out" + ); + Ok(()) + } +} + +/// Reject audited unavailable codecs while distinguishing malformed configurations. +fn validate_audio_codec(params: &AudioCodecParameters) -> Result { + if params.codec == symphonia::core::codecs::audio::well_known::CODEC_ID_AAC + && params.extra_data.as_deref().map_or_else( + || Ok(params.profile.is_some_and(|profile| { + profile != symphonia::core::codecs::audio::well_known::profiles::CODEC_PROFILE_AAC_LC + })), + aac_needs_compatibility, + )? + { + return Err(symphonia::core::errors::Error::Unsupported( + "AAC profile needs the compatibility decoder", + ) + .into()); + } + let is_opus = params.codec == CODEC_ID_OPUS; + if !is_opus + && symphonia::default::get_codecs() + .get_audio_decoder(params.codec) + .is_none() + { + return Err(symphonia::core::errors::Error::Unsupported( + "audio codec needs the compatibility decoder", + ) + .into()); + } + Ok(is_opus) +} + +/// Mean absolute channel amplitude avoids cancellation of out-of-phase stereo. +fn frame_amplitude(frame: &[f32], gain: f32) -> Result { + let channels = u16::try_from(frame.len())?; + ensure!(channels > 0, "empty audio channel frame"); + Ok(frame + .iter() + .map(|sample| (sample * gain).abs().min(1.0)) + .sum::() + / f32::from(channels)) +} + +/// Walk decoded channel frames without retaining the clip in memory. +fn visit_samples( + input: &Path, + budget: &AudioBudget<'_>, + mut visit: impl FnMut(f32) -> Result<()>, +) -> Result { + budget.check()?; + if super::probe::is_speex(input, budget.started, budget.timeout, Some(budget.cancel))? { + return Err(symphonia::core::errors::Error::Unsupported( + "Speex requires the compatibility decoder", + ) + .into()); + } + if let Some(header) = super::probe::discrete_opus_header( + input, + budget.started, + budget.timeout, + Some(budget.cancel), + )? { + return visit_discrete_opus(input, &header, budget, visit); + } + let (mut format, read_budget) = + super::probe::open_audio_format(input, budget.started, budget.timeout, budget.cancel)?; + let track = format + .first_track(TrackType::Audio) + .context("media has no audio track")?; + let track_id = track.id; + let params = track + .codec_params + .as_ref() + .and_then(symphonia::core::codecs::CodecParameters::audio) + .context("audio codec parameters missing")?; + let mut movie_presentation = if format.format_info().short_name == "isomp4" { + mp4::read( + input, + track_id, + params + .sample_rate + .context("MP4 audio sample rate missing")?, + || budget.check(), + )? + } else { + None + }; + let is_opus = validate_audio_codec(params)?; + let mut opus = if is_opus { + Some(OpusPresentation::new(params, track.time_base)?) + } else { + None + }; + let gain = opus.as_ref().map_or(1.0, |presentation| presentation.gain); + let channels = params + .channels + .as_ref() + .context("audio channels missing")? + .count(); + ensure!( + (1..=255).contains(&channels), + "audio channels exceed safety limit" + ); + let mut decoder = PacketDecoder::new(params, channels)?; + if is_opus && movie_presentation.is_some() { + return Err(symphonia::core::errors::Error::Unsupported( + "audio edit list requires the compatibility decoder", + ) + .into()); + } + let mut samples = Vec::new(); + let mut frames = 0_u64; + loop { + budget.check()?; + read_budget.start_packet(); + let Some(packet) = format.next_packet().context("read audio packet")? else { + break; + }; + if packet.track_id != track_id { + continue; + } + decoder.decode(&packet, channels, &mut samples, budget)?; + let presentation = if let Some(opus) = &mut opus { + opus.samples(&packet, &samples, channels)? + } else { + samples.as_slice() + }; + // Mean absolute channel amplitude avoids cancelling out-of-phase stereo. + for frame in presentation.chunks_exact(channels) { + let amplitude = frame_amplitude(frame, gain)?; + let emitted = if let Some(movie_timeline) = &mut movie_presentation { + movie_timeline.frame(amplitude, &mut visit, || budget.check())? + } else { + visit(amplitude)?; + 1 + }; + frames = frames + .checked_add(emitted) + .context("audio sample count overflow")?; + } + } + if let Some(presentation) = &mut movie_presentation { + frames = frames + .checked_add(presentation.finish(&mut visit, || budget.check())?) + .context("audio sample count overflow")?; + } + ensure!(frames > 0, "audio contains no decodable samples"); + budget.check()?; + Ok(frames) +} + +/// Render an audio waveform with exact bucket boundaries and bounded memory. +/// +/// A first streaming pass measures decoded duration; a second fills width-sized +/// peak buckets. This also handles containers without reliable duration metadata. +/// +/// # Errors +/// Returns an error for malformed/unsupported audio, resource limits, cancellation, +/// or output I/O failures. Call on a blocking worker, not an async executor. +pub fn render_waveform( + input: &Path, + output: &Path, + width: u32, + height: u32, + cancel: &CancellationToken, +) -> Result<()> { + super::images::validate_dimensions(width, height)?; + let budget = AudioBudget { + started: Instant::now(), + timeout: Duration::from_secs(crate::config::ffmpeg_timeout_secs()), + cancel, + }; + let total = visit_samples(input, &budget, |_| Ok(()))?; + let mut peaks = vec![0.0_f32; usize::try_from(width)?]; + let mut position = 0_u64; + let actual = visit_samples(input, &budget, |amplitude| { + let bucket = u128::from(position) + .checked_mul(u128::from(width)) + .and_then(|scaled| scaled.checked_div(u128::from(total))) + .context("invalid waveform sample range")?; + let index = usize::try_from(bucket.min(u128::from( + width.checked_sub(1).context("waveform width is empty")?, + )))?; + let peak = peaks + .get_mut(index) + .context("waveform bucket out of range")?; + *peak = peak.max(amplitude); + position = position + .checked_add(1) + .context("waveform position overflow")?; + Ok(()) + })?; + ensure!(actual == total, "audio changed between waveform passes"); + let pixels = render_peaks(&peaks, width, height)?; + budget.check()?; + pixels + .save_with_format(output, image::ImageFormat::Png) + .context("write waveform PNG") +} + +/// Draw a symmetric gray amplitude envelope on a transparent canvas. +#[expect( + clippy::cast_possible_truncation, + clippy::cast_sign_loss, + clippy::as_conversions, + reason = "clamped normalized amplitudes map to a bounded pixel radius" +)] +fn render_peaks(peaks: &[f32], width: u32, height: u32) -> Result { + let mut image = RgbaImage::new(width, height); + let middle = height / 2; + let max_radius = middle.saturating_sub(1); + let radius_scale = f32::from(u16::try_from(max_radius).context("waveform is too tall")?); + for (column, peak) in peaks.iter().enumerate() { + let radius = (peak.clamp(0.0, 1.0) * radius_scale).round() as u32; + let x = u32::try_from(column)?; + for y in middle.saturating_sub(radius) + ..=middle + .saturating_add(radius) + .min(height.checked_sub(1).context("waveform height is empty")?) + { + image.put_pixel(x, y, Rgba([0x88, 0x88, 0x88, 255])); + } + } + Ok(image) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn compatibility_errors_exclude_malformed_packets_io_and_resource_failures() { + assert!(is_unsupported( + &symphonia::core::errors::Error::Unsupported( + "audio codec needs the compatibility decoder" + ) + .into() + )); + for error in [ + anyhow::Error::from(opus_pure::Error::Internal("transform failure")), + anyhow::Error::from(opus_pure::Error::InvalidPacket("invalid framing")), + anyhow::Error::from(opus_pure::Error::InvalidArgument("output buffer budget")), + anyhow::Error::from(symphonia::core::errors::Error::DecodeError( + "invalid header", + )), + anyhow::Error::from(std::io::Error::new( + std::io::ErrorKind::TimedOut, + "deadline", + )), + anyhow::anyhow!("audio packet exceeds sample budget"), + ] { + assert!(!is_unsupported(&error)); + } + for message in [ + "core (probe): no suitable format reader found", + "isomp4: missing ftyp atom", + "isomp4: missing moov atom", + "mkv: missing segment element", + "mkv: video width too large", + "mkv: timestamp scale too large (report this)", + "ogg: page is not marked as first", + "aac: sample rate is required", + "aac: coupling channel element", + "aac: program config", + "aac: predictor data", + "aac: gain control data", + "adts: only 1 aac frame per adts packet is supported", + ] { + assert!( + !is_unsupported(&symphonia::core::errors::Error::Unsupported(message).into()), + "invalid/resource error selected fallback: {message}" + ); + } + } + + #[test] + fn all_supported_audio_fixtures_produce_deterministic_visible_waveforms() -> Result<()> { + let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/media"); + let dir = tempfile::tempdir()?; + let first = dir.path().join("first.png"); + let second = dir.path().join("second.png"); + for file in [ + "tone.wav", + "tone.mp3", + "tone.flac", + "tone.ogg", + "tone.opus", + "speech-mode.opus", + "surround.opus", + "tone.aac", + "tone.m4a", + "tone-alac.m4a", + "audio.webm", + "audio.mkv", + ] { + render_waveform( + &root.join(file), + &first, + 250, + 125, + &CancellationToken::new(), + ) + .with_context(|| format!("waveform for {file}"))?; + render_waveform( + &root.join(file), + &second, + 250, + 125, + &CancellationToken::new(), + )?; + ensure!( + std::fs::read(&first)? == std::fs::read(&second)?, + "{file} waveform is not deterministic" + ); + let pixels = image::open(&first)?.into_rgba8(); + ensure!( + pixels.dimensions() == (250, 125), + "waveform dimensions changed" + ); + ensure!( + pixels + .pixels() + .filter(|pixel| pixel.0.get(3) == Some(&255)) + .count() + > 250, + "{file} waveform looks silent" + ); + } + Ok(()) + } + + /// Build valid PCM without any executable or codec fixture generator. + fn write_pcm(path: &Path, channels: u16, frames: u32, silence: bool) -> Result<()> { + use std::io::Write as _; + let data_len = frames + .checked_mul(u32::from(channels) * 2) + .context("PCM length")?; + let mut file = std::fs::File::create(path)?; + file.write_all(b"RIFF")?; + file.write_all( + &data_len + .checked_add(36) + .context("PCM RIFF length overflow")? + .to_le_bytes(), + )?; + file.write_all(b"WAVEfmt ")?; + file.write_all(&16_u32.to_le_bytes())?; + file.write_all(&1_u16.to_le_bytes())?; + file.write_all(&channels.to_le_bytes())?; + file.write_all(&8000_u32.to_le_bytes())?; + file.write_all( + &u32::from(channels) + .checked_mul(16_000) + .context("PCM byte rate overflow")? + .to_le_bytes(), + )?; + file.write_all( + &channels + .checked_mul(2) + .context("PCM block size overflow")? + .to_le_bytes(), + )?; + file.write_all(&16_u16.to_le_bytes())?; + file.write_all(b"data")?; + file.write_all(&data_len.to_le_bytes())?; + let mut samples = Vec::with_capacity(usize::try_from(data_len)?); + for frame in 0..frames { + for channel in 0..channels { + let sample = if silence { + 0_i16 + } else if frame + .checked_add(u32::from(channel)) + .context("PCM frame position overflow")? + % 2 + == 0 + { + 16_000_i16 + } else { + -16_000_i16 + }; + samples.extend_from_slice(&sample.to_le_bytes()); + } + } + file.write_all(&samples)?; + Ok(()) + } + + #[test] + fn actual_silence_single_frame_surround_and_long_pcm_are_streamed() -> Result<()> { + let dir = tempfile::tempdir()?; + let input = dir.path().join("sound.wav"); + let output = dir.path().join("wave.png"); + for (channels, frames, silence) in [ + (1, 1, false), + (6, 800, false), + (2, 480, true), + (1, 960_000, true), + ] { + write_pcm(&input, channels, frames, silence)?; + render_waveform(&input, &output, 250, 125, &CancellationToken::new())?; + let pixels = image::open(&output)?.into_rgba8(); + let ink = pixels + .pixels() + .filter(|pixel| pixel.0.get(3) == Some(&255)) + .count(); + ensure!( + pixels.dimensions() == (250, 125), + "PCM waveform dimensions changed" + ); + if silence { + ensure!(ink == 250, "silence should be a flat line"); + } else { + ensure!(ink > 250, "opposite-phase/single-frame audio disappeared"); + } + } + Ok(()) + } + + #[test] + fn opus_encoder_delay_and_end_padding_do_not_extend_the_waveform() -> Result<()> { + let input = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/media/tone.opus"); + let cancel = CancellationToken::new(); + let budget = AudioBudget { + started: Instant::now(), + timeout: Duration::from_secs(30), + cancel: &cancel, + }; + ensure!( + visit_samples(&input, &budget, |_| Ok(()))? == 5760, + "120ms Opus clip includes encoder delay/padding" + ); + Ok(()) + } + + #[test] + fn silence_malformed_and_cancellation_are_safe() -> Result<()> { + let image = render_peaks(&[0.0, 0.0, 0.0], 3, 2)?; + ensure!( + image + .pixels() + .filter(|pixel| pixel.0.get(3) == Some(&255)) + .count() + == 3, + "silence should be a flat center line" + ); + let dir = tempfile::tempdir()?; + let source = dir.path().join("bad.wav"); + let output = dir.path().join("wave.png"); + std::fs::write(&source, b"RIFF\xff\xff\xff\xffWAVE")?; + ensure!( + render_waveform(&source, &output, 250, 125, &CancellationToken::new()).is_err(), + "malformed audio decoded" + ); + ensure!(!output.exists(), "malformed audio published output"); + let cancel = CancellationToken::new(); + cancel.cancel(); + ensure!( + render_waveform(&source, &output, 250, 125, &cancel).is_err(), + "cancelled decoding continued" + ); + Ok(()) + } + + #[test] + fn malformed_container_errors_cannot_select_compatibility() -> Result<()> { + let dir = tempfile::tempdir()?; + let source = dir.path().join("bad.m4a"); + let output = dir.path().join("wave.png"); + for bytes in [ + b"not an audio container".as_slice(), + b"\0\0\0\x18ftypisom\0\0\x02\0isomiso2", + b"OggS\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", + b"\x1a\x45\xdf\xa3\x01", + ] { + std::fs::write(&source, bytes)?; + let error = render_waveform(&source, &output, 250, 125, &CancellationToken::new()) + .err() + .context("malformed audio was decoded")?; + ensure!( + !is_unsupported(&error), + "malformed input selected fallback: {error:#}" + ); + ensure!(!output.exists(), "malformed input published a waveform"); + } + Ok(()) + } + + /// libopus reference samples verify exact duration, mapping, padding and gain. + #[test] + fn mapped_opus_matches_independent_pcm_without_external_tools() -> Result<()> { + let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/media"); + let cancel = CancellationToken::new(); + for name in ["opus-surround-distinct", "opus-surround-padded-gain"] { + let expected = std::fs::read(root.join(format!("{name}-native.f32")))?; + ensure!( + expected.len().is_multiple_of(24), + "invalid six-channel reference" + ); + let expected = expected + .as_chunks::<24>() + .0 + .iter() + .map(|frame| { + frame + .as_chunks::<4>() + .0 + .iter() + .map(|sample| f32::from_le_bytes(*sample).abs().min(1.0)) + .sum::() + / 6.0 + }) + .collect::>(); + let mut actual = Vec::new(); + let budget = AudioBudget { + started: Instant::now(), + timeout: Duration::from_secs(10), + cancel: &cancel, + }; + let frames = visit_samples(&root.join(format!("{name}.opus")), &budget, |amplitude| { + actual.push(amplitude); + Ok(()) + }) + .with_context(|| format!("decode {name}"))?; + ensure!( + frames == 11_520 && actual.len() == expected.len(), + "Opus presentation duration changed for {name}" + ); + ensure!( + actual + .iter() + .zip(&expected) + .all(|(a, b)| (a - b).abs() <= 0.000_04), + "Opus amplitude differs from libopus for {name}" + ); + } + Ok(()) + } + + /// The native reference applies the movie edit list, including encoder delay. + #[test] + fn aac_lc_movie_edits_preserve_presentation_duration_and_waveform() -> Result<()> { + let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/media"); + let bytes = std::fs::read(root.join("aac-lc-native.f32"))?; + let expected = bytes + .as_chunks::<8>() + .0 + .iter() + .map(|frame| { + frame + .as_chunks::<4>() + .0 + .iter() + .map(|sample| f32::from_le_bytes(*sample).abs()) + .sum::() + / 2.0 + }) + .collect::>(); + let cancel = CancellationToken::new(); + let budget = AudioBudget { + started: Instant::now(), + timeout: Duration::from_secs(10), + cancel: &cancel, + }; + let mut actual = Vec::new(); + let frames = visit_samples(&root.join("tone.m4a"), &budget, |amplitude| { + actual.push(amplitude); + Ok(()) + })?; + ensure!( + frames == 5_760 && actual.len() == expected.len(), + "AAC encoder padding changed movie presentation" + ); + ensure!( + actual + .iter() + .zip(&expected) + .all(|(a, b)| (a - b).abs() <= 0.002), + "AAC movie waveform differs from independent decoder" + ); + Ok(()) + } + + #[test] + fn implicit_he_aac_is_unsupported_and_truncated_extensions_are_invalid() -> Result<()> { + ensure!( + aac_needs_compatibility(&[0x13, 0x10, 0x56, 0xe5, 0x98])?, + "implicit SBR was ignored" + ); + ensure!( + !aac_needs_compatibility(&[0x11, 0x90])?, + "AAC-LC selected compatibility" + ); + let error = aac_needs_compatibility(&[0x13, 0x10, 0x56, 0xe5]) + .err() + .context("truncated SBR configuration was accepted")?; + ensure!(!is_unsupported(&error), "truncated SBR selected fallback"); + for bytes in [&[0x28][..], &[0x00, 0x90], &[0x17, 0x10]] { + let invalid_config_error = aac_needs_compatibility(bytes) + .err() + .context("invalid AAC config accepted")?; + ensure!( + !is_unsupported(&invalid_config_error), + "invalid AAC config selected fallback" + ); + } + Ok(()) + } + #[test] + fn in_band_sbr_never_publishes_core_only_waveforms() -> Result<()> { + let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/media"); + let input = root.join("he-aac-inband.aac"); + let dir = tempfile::tempdir()?; + let output = dir.path().join("wave.png"); + let cancel = CancellationToken::new(); + let error = render_waveform(&input, &output, 100, 50, &cancel) + .err() + .context("in-band HE-AAC was decoded as core-only AAC-LC")?; + ensure!( + is_unsupported(&error), + "SBR did not select explicit compatibility: {error:#}" + ); + ensure!(!output.exists(), "incomplete SBR waveform was published"); + let (mut format, _) = super::super::probe::open_audio_format( + &input, + Instant::now(), + Duration::from_secs(5), + &cancel, + )?; + let track = format + .first_track(TrackType::Audio) + .context("no AAC track")?; + let params = track + .codec_params + .as_ref() + .and_then(symphonia::core::codecs::CodecParameters::audio) + .context("no AAC parameters")? + .clone(); + let channels = params.channels.as_ref().context("no AAC channels")?.count(); + let packet = format.next_packet()?.context("no AAC packet")?; + let budget = AudioBudget { + started: Instant::now(), + timeout: Duration::from_secs(5), + cancel: &cancel, + }; + for removed in 1..=3 { + let mut truncated = packet.clone(); + truncated.data = packet + .data + .get( + ..packet + .data + .len() + .checked_sub(removed) + .context("AAC packet too short")?, + ) + .context("AAC fixture truncation outside packet")? + .to_vec() + .into_boxed_slice(); + let truncated_packet_error = PacketDecoder::new(¶ms, channels)? + .decode(&truncated, channels, &mut Vec::new(), &budget) + .err() + .context("truncated AAC raw-data block was accepted")?; + ensure!( + !is_unsupported(&truncated_packet_error), + "truncated SBR payload selected compatibility: {truncated_packet_error:#}" + ); + } + Ok(()) + } +} diff --git a/src/media/audio_mp4.rs b/src/media/audio_mp4.rs new file mode 100644 index 00000000..4f231114 --- /dev/null +++ b/src/media/audio_mp4.rs @@ -0,0 +1,684 @@ +//! Bounded MP4 metadata supplements: edit timing and video codec declarations. + +use anyhow::{ensure, Context as _, Result}; +use std::fs::File; +use std::io::{Read as _, Seek as _, SeekFrom}; +use std::path::Path; + +/// Maximum framing boxes visited, regardless of their declared payload size. +const MAX_BOXES: usize = 4096; +/// Maximum retained presentation segments. +const MAX_EDITS: usize = 64; + +/// One explicitly declared segment of the movie's audio presentation. +enum Edit { + /// Empty edits introduce real presentation silence. + Silence(u64), + /// Unit-rate, forward-moving range of decoded media frames. + Audio { + /// First raw decoded frame included in this edit. + start: u64, + /// Exclusive end of this edit's raw decoded range. + end: u64, + }, +} + +/// Streaming selection without retaining PCM or allocating silence buffers. +pub(super) struct Presentation { + /// Bounded edit plan, expressed in decoded sample frames. + edits: Vec, + /// Next edit to emit. + next: usize, + /// Current raw decoded frame position. + position: u64, +} + +impl Presentation { + /// Emit explicit empty edits with deadline checks throughout long silence. + fn silence( + &mut self, + visit: &mut impl FnMut(f32) -> Result<()>, + check: &mut impl FnMut() -> Result<()>, + ) -> Result { + let mut emitted = 0_u64; + while let Some(Edit::Silence(count)) = self.edits.get(self.next) { + for frame in 0..*count { + if frame % 1024 == 0 { + check()?; + } + visit(0.0)?; + } + emitted = emitted + .checked_add(*count) + .context("MP4 presentation length overflow")?; + self.next = self + .next + .checked_add(1) + .context("MP4 edit cursor overflow")?; + } + Ok(emitted) + } + + /// Select one raw frame and advance every completed edit exactly once. + pub(super) fn frame( + &mut self, + amplitude: f32, + visit: &mut impl FnMut(f32) -> Result<()>, + mut check: impl FnMut() -> Result<()>, + ) -> Result { + let mut emitted = self.silence(visit, &mut check)?; + if let Some(Edit::Audio { start, end }) = self.edits.get(self.next) { + if self.position >= *start && self.position < *end { + visit(amplitude)?; + emitted = emitted + .checked_add(1) + .context("MP4 presentation length overflow")?; + } + if self + .position + .checked_add(1) + .context("MP4 frame position overflow")? + >= *end + { + self.next = self + .next + .checked_add(1) + .context("MP4 edit cursor overflow")?; + } + } + self.position = self + .position + .checked_add(1) + .context("MP4 frame position overflow")?; + Ok(emitted) + } + + /// Reject a range past decoded media rather than inventing missing samples. + pub(super) fn finish( + &mut self, + visit: &mut impl FnMut(f32) -> Result<()>, + mut check: impl FnMut() -> Result<()>, + ) -> Result { + let emitted = self.silence(visit, &mut check)?; + ensure!( + self.next == self.edits.len(), + "MP4 edit extends beyond decoded audio" + ); + Ok(emitted) + } +} + +/// Checked framing locations; encoded sample payloads are never retained. +#[derive(Clone, Copy)] +struct Atom { + /// Four-byte container box type. + kind: [u8; 4], + /// First byte after the normal or extended framing header. + body: u64, + /// Exclusive boundary, checked against its enclosing box. + end: u64, +} + +/// Deadline-aware metadata walker with a fixed allocation ceiling. +struct Reader { + /// Validated regular upload file. + file: File, + /// Total framing boxes inspected across every nesting level. + boxes: usize, + /// Job deadline and cancellation check. + check: F, +} + +impl Result<()>> Reader { + /// Read one fixed-size field after checking its container boundary. + fn field(&mut self, atom: Atom, offset: u64) -> Result<[u8; N]> { + (self.check)()?; + let start = atom + .body + .checked_add(offset) + .context("MP4 field offset overflow")?; + ensure!( + start + .checked_add(u64::try_from(N)?) + .is_some_and(|end| end <= atom.end), + "truncated MP4 timing field" + ); + let _position = self.file.seek(SeekFrom::Start(start))?; + let mut bytes = [0; N]; + self.file.read_exact(&mut bytes)?; + Ok(bytes) + } + + /// Walk only direct child headers, skipping bounded payload extents. + fn children(&mut self, start: u64, end: u64) -> Result> { + let mut cursor = start; + let mut atoms = Vec::new(); + while cursor < end { + (self.check)()?; + ensure!( + self.boxes < MAX_BOXES + && end + .checked_sub(cursor) + .is_some_and(|remaining| remaining >= 8), + "MP4 timing box budget or extent invalid" + ); + self.boxes = self + .boxes + .checked_add(1) + .context("MP4 box count overflow")?; + let outer = Atom { + kind: [0; 4], + body: cursor, + end, + }; + let size = u32::from_be_bytes(self.field::<4>(outer, 0)?); + let kind = self.field::<4>(outer, 4)?; + let (size, header) = match size { + 0 => ( + end.checked_sub(cursor) + .context("invalid MP4 box boundary")?, + 8, + ), + 1 => (u64::from_be_bytes(self.field::<8>(outer, 8)?), 16), + size => (u64::from(size), 8), + }; + let next = cursor + .checked_add(size) + .context("MP4 box extent overflow")?; + ensure!( + size >= header && next <= end, + "invalid MP4 timing box extent" + ); + atoms.push(Atom { + kind, + body: cursor + .checked_add(header) + .context("MP4 box header overflow")?, + end: next, + }); + cursor = next; + } + Ok(atoms) + } + + /// Decode one bounded edit, validating media time before rate compatibility. + fn edit_entry(&mut self, elst: Atom, offset: u64, version: u8) -> Result<(u64, i64, bool)> { + let (duration, start, rate_offset) = if version == 0 { + ( + u64::from(u32::from_be_bytes(self.field::<4>(elst, offset)?)), + i64::from(i32::from_be_bytes( + self.field::<4>( + elst, + offset + .checked_add(4) + .context("MP4 edit field offset overflow")?, + )?, + )), + offset + .checked_add(8) + .context("MP4 edit field offset overflow")?, + ) + } else { + ( + u64::from_be_bytes(self.field::<8>(elst, offset)?), + i64::from_be_bytes( + self.field::<8>( + elst, + offset + .checked_add(8) + .context("MP4 edit field offset overflow")?, + )?, + ), + offset + .checked_add(16) + .context("MP4 edit field offset overflow")?, + ) + }; + ensure!(start >= -1, "invalid MP4 edit media time"); + let non_unit_rate = self.field::<4>(elst, rate_offset)? != [0, 1, 0, 0]; + Ok((duration, start, non_unit_rate)) + } + + /// Validate full-box version and select its 32/64-bit timestamp fields. + fn timestamp_offset(&mut self, atom: Atom) -> Result { + match self.field::<1>(atom, 0)? { + [0] => Ok(12), + [1] => Ok(20), + _ => anyhow::bail!("invalid MP4 timing version"), + } + } + + /// Decode a nonzero movie or media timescale. + fn timescale(&mut self, atom: Atom) -> Result { + let offset = self.timestamp_offset(atom)?; + let scale = u32::from_be_bytes(self.field::<4>(atom, offset)?); + ensure!(scale > 0, "zero MP4 timescale"); + Ok(scale) + } +} + +/// Require exactly one structural timing box rather than choosing duplicates. +fn one(atoms: &[Atom], kind: [u8; 4]) -> Result { + let mut found = atoms.iter().filter(|atom| atom.kind == kind); + let atom = *found.next().context("missing MP4 timing box")?; + ensure!(found.next().is_none(), "duplicate MP4 timing box"); + Ok(atom) +} + +/// Recover a declared AV1/VP8/VP9 sample-entry codec for one validated MP4 track. +/// Symphonia 0.6 recognizes these entries but leaves their codec ID unset. +/// Walk structural boxes by track ID; never scan media payloads for codec tags. +pub(in crate::media) fn declared_video_codec( + path: &Path, + track_id: u32, + check: impl FnMut() -> Result<()>, +) -> Result> { + let file = File::open(path)?; + let metadata = file.metadata()?; + ensure!(metadata.is_file(), "MP4 codec source is not a regular file"); + let mut reader = Reader { + file, + boxes: 0, + check, + }; + let root = reader.children(0, metadata.len())?; + let moov = one(&root, *b"moov")?; + let movie = reader.children(moov.body, moov.end)?; + let mut chosen = None; + for track in movie.iter().filter(|atom| atom.kind == *b"trak") { + let children = reader.children(track.body, track.end)?; + let tkhd = one(&children, *b"tkhd")?; + let offset = reader.timestamp_offset(tkhd)?; + if u32::from_be_bytes(reader.field::<4>(tkhd, offset)?) == track_id { + ensure!(chosen.is_none(), "duplicate MP4 video track identifier"); + chosen = Some(children); + } + } + let mut children = chosen.context("MP4 video codec track missing")?; + for kind in [*b"mdia", *b"minf", *b"stbl"] { + let parent = one(&children, kind)?; + children = reader.children(parent.body, parent.end)?; + } + let stsd = one(&children, *b"stsd")?; + ensure!( + reader.field::<1>(stsd, 0)? == [0], + "invalid MP4 sample description version" + ); + ensure!( + u32::from_be_bytes(reader.field::<4>(stsd, 4)?) == 1, + "ambiguous MP4 video sample descriptions" + ); + let entries = reader.children( + stsd.body + .checked_add(8) + .context("MP4 stsd offset overflow")?, + stsd.end, + )?; + ensure!( + entries.len() == 1, + "invalid MP4 video sample description extent" + ); + let entry = entries.first().context("MP4 video sample entry missing")?; + let codec = match &entry.kind { + b"av01" => "av1", + b"vp08" => "vp8", + b"vp09" => "vp9", + _ => return Ok(None), + }; + ensure!( + entry + .end + .checked_sub(entry.body) + .is_some_and(|size| size >= 78), + "truncated MP4 visual sample entry" + ); + Ok(Some(codec)) +} + +/// Convert rational timestamps to the nearest whole decoded sample frame. +fn frames(units: u64, scale: u32, rate: u32) -> Result { + ensure!(scale > 0 && rate > 0, "invalid MP4 timing scale"); + Ok(u64::try_from( + u128::from(units) + .checked_mul(u128::from(rate)) + .and_then(|numerator| numerator.checked_add(u128::from(scale) / 2)) + .and_then(|rounded| rounded.checked_div(u128::from(scale))) + .context("MP4 sample frame conversion overflow")?, + )?) +} + +/// Compute the checked byte offset of an edit-table entry after its header. +fn edit_offset(index: u64, size: u64) -> Result { + index + .checked_mul(size) + .and_then(|bytes| bytes.checked_add(8)) + .context("MP4 edit table offset overflow") +} + +/// Valid but unimplemented temporal operations retain an explicit compatibility path. +fn unsupported() -> anyhow::Error { + symphonia::core::errors::Error::Unsupported( + "audio edit list requires the compatibility decoder", + ) + .into() +} + +/// Read unit-rate ordered edits for the track selected by the content-based demuxer. +pub(super) fn read( + path: &Path, + track_id: u32, + rate: u32, + check: impl FnMut() -> Result<()>, +) -> Result> { + let file = File::open(path)?; + let metadata = file.metadata()?; + ensure!( + metadata.is_file(), + "MP4 timing source is not a regular file" + ); + let mut reader = Reader { + file, + boxes: 0, + check, + }; + let root = reader.children(0, metadata.len())?; + let moov = one(&root, *b"moov")?; + let movie = reader.children(moov.body, moov.end)?; + let movie_scale = reader.timescale(one(&movie, *b"mvhd")?)?; + let mut chosen = None; + for track in movie.iter().filter(|atom| atom.kind == *b"trak") { + let children = reader.children(track.body, track.end)?; + let header = one(&children, *b"tkhd")?; + let offset = reader.timestamp_offset(header)?; + if u32::from_be_bytes(reader.field::<4>(header, offset)?) == track_id { + ensure!(chosen.is_none(), "duplicate MP4 track identifier"); + chosen = Some(children); + } + } + let track = chosen.context("MP4 audio timing track missing")?; + let Some(edit_container) = track.iter().find(|atom| atom.kind == *b"edts") else { + return Ok(None); + }; + ensure!( + track.iter().filter(|atom| atom.kind == *b"edts").count() == 1, + "duplicate MP4 edit container" + ); + let edit_atoms = reader.children(edit_container.body, edit_container.end)?; + let elst = one(&edit_atoms, *b"elst")?; + let media_container = one(&track, *b"mdia")?; + let media = reader.children(media_container.body, media_container.end)?; + let media_scale = reader.timescale(one(&media, *b"mdhd")?)?; + let [version] = reader.field::<1>(elst, 0)?; + ensure!(version <= 1, "invalid MP4 edit version"); + let count = usize::try_from(u32::from_be_bytes(reader.field::<4>(elst, 4)?))?; + ensure!(count <= MAX_EDITS, "MP4 edit budget exceeded"); + let size = if version == 0 { 12 } else { 20 }; + ensure!( + elst.end.checked_sub(elst.body) == Some(edit_offset(u64::try_from(count)?, size)?), + "invalid MP4 edit table size" + ); + let mut edits = Vec::with_capacity(count); + let mut movie_end = 0_u64; + let mut previous_end = 0_u64; + let mut compatibility = false; + for index in 0..count { + let offset = edit_offset(u64::try_from(index)?, size)?; + let (duration, start, non_unit_rate) = reader.edit_entry(elst, offset, version)?; + compatibility |= non_unit_rate; + let old_end = frames(movie_end, movie_scale, rate)?; + movie_end = movie_end + .checked_add(duration) + .context("MP4 movie duration overflow")?; + let duration_frames = frames(movie_end, movie_scale, rate)? + .checked_sub(old_end) + .context("MP4 edit frame range is reversed")?; + if duration_frames == 0 { + continue; + } + if start == -1 { + edits.push(Edit::Silence(duration_frames)); + } else { + let start = frames(u64::try_from(start)?, media_scale, rate)?; + let end = start + .checked_add(duration_frames) + .context("MP4 media duration overflow")?; + compatibility |= start < previous_end; + previous_end = end; + edits.push(Edit::Audio { start, end }); + } + } + if compatibility { + return Err(unsupported()); + } + Ok((!edits.is_empty()).then_some(Presentation { + edits, + next: 0, + position: 0, + })) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn video_codec_declarations_obey_track_identity_and_box_boundaries() -> Result<()> { + let original = include_bytes!("../../tests/fixtures/media/av1.mp4"); + let directory = tempfile::tempdir()?; + let path = directory.path().join("misleading.webm"); + std::fs::write(&path, original)?; + ensure!( + declared_video_codec(&path, 1, || Ok(()))? == Some("av1"), + "AV1 declaration lost" + ); + ensure!( + declared_video_codec(&path, 999, || Ok(())).is_err(), + "wrong track matched" + ); + ensure!( + declared_video_codec(&path, 1, || anyhow::bail!("cancelled")).is_err(), + "cancelled inspection continued" + ); + + let stsd = original + .windows(4) + .position(|bytes| bytes == b"stsd") + .context("missing sample descriptions")?; + let entry = stsd + .checked_add(16) + .context("sample entry offset overflow")?; + ensure!( + original.get(entry..entry + 4) == Some(b"av01"), + "fixture layout changed" + ); + let mut unrelated_tag = original.to_vec(); + unrelated_tag + .get_mut(entry..entry + 4) + .context("sample entry bounds")? + .copy_from_slice(b"zzzz"); + // An AV1-looking tag inside an unrelated free-box payload is not metadata. + unrelated_tag.extend_from_slice(b"\0\0\0\x0cfreeav01"); + std::fs::write(&path, unrelated_tag)?; + ensure!( + declared_video_codec(&path, 1, || Ok(()))?.is_none(), + "payload tag became a codec" + ); + + let mut excessive_entries = original.to_vec(); + excessive_entries + .get_mut(stsd + 8..stsd + 12) + .context("stsd count bounds")? + .copy_from_slice(&u32::MAX.to_be_bytes()); + std::fs::write(&path, excessive_entries)?; + ensure!( + declared_video_codec(&path, 1, || Ok(())).is_err(), + "unbounded sample descriptions accepted" + ); + std::fs::write( + &path, + original + .get(..entry + 3) + .context("truncated fixture bounds")?, + )?; + ensure!( + declared_video_codec(&path, 1, || Ok(())).is_err(), + "truncated movie accepted" + ); + Ok(()) + } + + /// Empty edits, gaps, and multiple forward ranges preserve presentation order. + #[test] + fn silence_and_forward_ranges_are_streamed_without_padding() -> Result<()> { + let mut plan = Presentation { + edits: vec![ + Edit::Silence(2), + Edit::Audio { start: 1, end: 3 }, + Edit::Audio { start: 5, end: 6 }, + Edit::Silence(1), + ], + next: 0, + position: 0, + }; + let mut samples = Vec::new(); + let mut visit = |sample| { + samples.push(sample); + Ok(()) + }; + let mut total = 0; + for value in 0_u16..7 { + total += plan.frame(f32::from(value), &mut visit, || Ok(()))?; + } + total += plan.finish(&mut visit, || Ok(()))?; + ensure!( + total == 6 && samples == [0.0, 0.0, 1.0, 2.0, 5.0, 0.0], + "MP4 timeline selection changed" + ); + let mut missing = Presentation { + edits: vec![Edit::Audio { start: 0, end: 2 }], + next: 0, + position: 0, + }; + let _decoded_frames = missing.frame(0.5, &mut |_| Ok(()), || Ok(()))?; + ensure!( + missing.finish(&mut |_| Ok(()), || Ok(())).is_err(), + "missing media was invented" + ); + let mut long = Presentation { + edits: vec![Edit::Silence(u64::MAX)], + next: 0, + position: 0, + }; + ensure!( + long.frame(0.0, &mut |_| Ok(()), || anyhow::bail!("cancelled")) + .is_err(), + "unbounded silence ignored cancellation" + ); + Ok(()) + } + + /// A genuine AAC movie protects framing, versions and bounded edit allocations. + #[test] + fn malformed_movie_timing_is_rejected_before_allocation() -> Result<()> { + let source = include_bytes!("../../tests/fixtures/media/tone.m4a"); + let elst = source + .windows(4) + .position(|bytes| bytes == b"elst") + .context("fixture edit list missing")?; + let directory = tempfile::tempdir()?; + let path = directory.path().join("misnamed.bin"); + for (offset, bytes) in [ + (elst + 8, u32::MAX.to_be_bytes().to_vec()), + (elst + 4, vec![2]), + (elst - 4, u32::MAX.to_be_bytes().to_vec()), + (elst + 16, (-2_i32).to_be_bytes().to_vec()), + ] { + let mut invalid = source.to_vec(); + invalid + .get_mut(offset..offset + bytes.len()) + .context("fixture offset missing")? + .copy_from_slice(&bytes); + std::fs::write(&path, invalid)?; + let error = read(&path, 1, 48_000, || Ok(())) + .err() + .context("invalid edit metadata accepted")?; + ensure!( + !super::super::is_unsupported(&error), + "malformed edits selected compatibility" + ); + } + let mut dwell = source.to_vec(); + dwell + .get_mut(elst + 20..elst + 24) + .context("fixture rate missing")? + .fill(0); + std::fs::write(&path, dwell)?; + let error = read(&path, 1, 48_000, || Ok(())) + .err() + .context("unimplemented dwell was ignored")?; + ensure!( + super::super::is_unsupported(&error), + "valid dwell lost its explicit compatibility path" + ); + ensure!( + read(&path, 1, 48_000, || anyhow::bail!("deadline")).is_err(), + "timing parser ignored deadline" + ); + Ok(()) + } + /// An unsupported first edit must not hide malformed metadata in later entries. + #[test] + fn malformed_edit_after_valid_dwell_cannot_select_compatibility() -> Result<()> { + let source = include_bytes!("../../tests/fixtures/media/tone.m4a"); + let elst = source + .windows(4) + .position(|bytes| bytes == b"elst") + .context("fixture edit list missing")?; + ensure!( + source.get(elst + 8..elst + 12) == Some(&1_u32.to_be_bytes()), + "fixture must have one version-zero edit" + ); + let mut second = source + .get(elst + 12..elst + 24) + .context("fixture entry missing")? + .to_vec(); + second + .get_mut(4..8) + .context("media time missing")? + .copy_from_slice(&(-2_i32).to_be_bytes()); + let mut invalid = source.to_vec(); + invalid + .get_mut(elst + 8..elst + 12) + .context("count missing")? + .copy_from_slice(&2_u32.to_be_bytes()); + invalid + .get_mut(elst + 20..elst + 24) + .context("rate missing")? + .fill(0); + for kind in [b"moov", b"trak", b"edts", b"elst"] { + let kind_offset = source + .windows(4) + .position(|bytes| bytes == kind) + .context("fixture ancestor missing")?; + let field = invalid + .get_mut(kind_offset - 4..kind_offset) + .context("fixture size missing")?; + let size = u32::from_be_bytes(field.try_into()?) + .checked_add(12) + .context("fixture size overflow")?; + field.copy_from_slice(&size.to_be_bytes()); + } + drop(invalid.splice(elst + 24..elst + 24, second)); + let directory = tempfile::tempdir()?; + let path = directory.path().join("dwell-then-invalid.m4a"); + std::fs::write(&path, invalid)?; + let error = read(&path, 1, 48_000, || Ok(())) + .err() + .context("malformed later edit accepted")?; + ensure!( + !super::super::is_unsupported(&error), + "unsupported first edit hid malformed later media time" + ); + Ok(()) + } +} diff --git a/src/media/audio_opus.rs b/src/media/audio_opus.rs new file mode 100644 index 00000000..7579f5bb --- /dev/null +++ b/src/media/audio_opus.rs @@ -0,0 +1,421 @@ +//! Bounded Opus multistream decoding with RFC 6716 Appendix B framing. +//! +//! Codec reconstruction comes from `opus-pure`. This adapter preserves the +//! container's explicit mapping and padding, including discrete-channel layouts. + +use anyhow::{ensure, Context as _, Result}; +use opus_pure::{OpusDecoder, MAX_PACKET_SAMPLES}; + +/// One decoder per coded mono/stereo stream, with explicit output-channel mapping. +#[derive(Debug)] +pub(super) struct Decoder { + /// Independent state for each coded mono or coupled stereo stream. + streams: Vec, + /// Number of initial streams that each reconstruct two channels. + coupled: usize, + /// Output-channel indices into coded PCM; 255 represents silence. + mapping: Vec, + /// Reused single-stream interleaved reconstruction buffer. + pcm: Vec, + /// Reused ordinary packet after removing self-delimiting framing. + packet: Vec, +} + +impl Decoder { + /// Validate the identification header before allocating decoder state. + pub(super) fn new(header: &[u8], channels: usize) -> Result { + let (streams, coupled, mapping) = Self::layout(header, channels)?; + let streams = (0..streams) + .map(|index| OpusDecoder::new(48_000, if index < coupled { 2 } else { 1 })) + .collect::, _>>()?; + Ok(Self { + streams, + coupled, + mapping, + pcm: Vec::new(), + packet: Vec::new(), + }) + } + + /// Validate metadata without allocating codec reconstruction state during probing. + pub(super) fn validate(header: &[u8], channels: usize) -> Result<()> { + let _validated_layout = Self::layout(header, channels)?; + Ok(()) + } + + /// Decode the bounded header layout, including arbitrary and silent channel mappings. + fn layout(header: &[u8], channels: usize) -> Result<(usize, usize, Vec)> { + ensure!((1..=255).contains(&channels), "invalid Opus channel count"); + ensure!(header.get(..8) == Some(b"OpusHead"), "invalid Opus header"); + ensure!( + header.get(8).is_some_and(|version| version >> 4_i32 == 0), + "unsupported Opus header version" + ); + ensure!( + header.get(9).copied().map(usize::from) == Some(channels), + "Opus channels disagree with container" + ); + let family = *header.get(18).context("truncated Opus mapping")?; + let (streams, coupled, mapping) = if family == 0 { + ensure!( + (1..=2).contains(&channels), + "invalid single-stream Opus channels" + ); + ( + 1, + channels + .checked_sub(1) + .context("invalid Opus single-stream channels")?, + if channels == 1 { vec![0] } else { vec![0, 1] }, + ) + } else { + let streams = usize::from(*header.get(19).context("truncated Opus stream count")?); + let coupled = usize::from(*header.get(20).context("truncated Opus coupled count")?); + let mapping = header + .get( + 21..21_usize + .checked_add(channels) + .context("Opus channel map size overflow")?, + ) + .context("truncated Opus channel map")?; + let coded_channels = streams + .checked_add(coupled) + .context("Opus coded channel count overflow")?; + ensure!( + streams > 0 && coupled <= streams && coded_channels <= 255, + "invalid Opus stream mapping" + ); + ensure!( + mapping + .iter() + .all(|slot| *slot == 255 || usize::from(*slot) < coded_channels), + "invalid Opus channel mapping" + ); + if !matches!(family, 1 | 2 | 255) { + return Err(symphonia::core::errors::Error::Unsupported( + "Opus mapping family requires the compatibility decoder", + ) + .into()); + } + ensure!( + family != 1 || channels <= 8, + "invalid surround Opus channel count" + ); + (streams, coupled, mapping.to_vec()) + }; + Ok((streams, coupled, mapping)) + } + + /// Decode every stream and reject differing packet durations before publication. + pub(super) fn decode( + &mut self, + data: &[u8], + output: &mut Vec, + mut check: impl FnMut() -> Result<()>, + ) -> Result { + ensure!(!data.is_empty(), "empty Opus packet"); + let channels = self.mapping.len(); + output.clear(); + output.resize( + MAX_PACKET_SAMPLES + .checked_mul(channels) + .context("Opus buffer overflow")?, + 0.0, + ); + let mut remaining = data; + let mut frames = None; + let count = self.streams.len(); + for (index, decoder) in self.streams.iter_mut().enumerate() { + check()?; + let stream_channels = if index < self.coupled { 2 } else { 1 }; + self.pcm.resize( + MAX_PACKET_SAMPLES + .checked_mul(stream_channels) + .context("Opus PCM buffer overflow")?, + 0.0, + ); + let encoded = if index.checked_add(1) == Some(count) { + remaining + } else { + let consumed = normalize_self_delimited(remaining, &mut self.packet)?; + remaining = remaining + .get(consumed..) + .context("invalid Opus packet extent")?; + &self.packet + }; + let decoded = decoder.decode(encoded, MAX_PACKET_SAMPLES, &mut self.pcm)?; + ensure!( + decoded > 0 && decoded <= MAX_PACKET_SAMPLES, + "invalid decoded Opus duration" + ); + if let Some(expected) = frames { + ensure!(decoded == expected, "Opus streams have different durations"); + } + frames = Some(decoded); + let base = if index < self.coupled { + index + .checked_mul(2) + .context("Opus coupled channel offset overflow")? + } else { + index + .checked_add(self.coupled) + .context("Opus channel offset overflow")? + }; + let end = base + .checked_add(stream_channels) + .context("Opus stream channel range overflow")?; + for (channel, mapped) in self.mapping.iter().copied().enumerate() { + let mapped = usize::from(mapped); + if mapped >= base && mapped < end { + let stream_channel = mapped + .checked_sub(base) + .context("invalid Opus channel offset")?; + for (target, source) in output + .chunks_exact_mut(channels) + .take(decoded) + .zip(self.pcm.chunks_exact(stream_channels)) + { + *target + .get_mut(channel) + .context("Opus output channel is missing")? = *source + .get(stream_channel) + .context("Opus coded channel is missing")?; + } + } + } + } + let frames = frames.context("Opus packet has no streams")?; + output.truncate( + frames + .checked_mul(channels) + .context("Opus output buffer overflow")?, + ); + Ok(frames) + } +} + +/// Remove Appendix B's extra frame-length field, retaining ordinary framing/padding. +fn normalize_self_delimited(data: &[u8], output: &mut Vec) -> Result { + let toc = *data.first().context("empty self-delimited Opus packet")?; + let mut cursor = 1; + let mut padding = 0_usize; + let mut payload = 0_usize; + let (count, cbr) = match toc & 3 { + 0 => (1, false), + 1 => (2, true), + 2 => { + payload = read_size(data, &mut cursor)?; + (2, false) + } + _ => { + let flags = read_byte(data, &mut cursor)?; + let count = usize::from(flags & 63); + ensure!((1..=48).contains(&count), "invalid Opus frame count"); + if flags & 64 != 0 { + loop { + let byte = read_byte(data, &mut cursor)?; + padding = padding + .checked_add(if byte == 255 { 254 } else { usize::from(byte) }) + .context("Opus padding overflow")?; + ensure!(padding <= data.len(), "truncated Opus padding"); + if byte != 255 { + break; + } + } + } + let cbr = flags & 128 == 0; + if !cbr { + for _ in 1..count { + payload = payload + .checked_add(read_size(data, &mut cursor)?) + .context("Opus payload overflow")?; + } + } + (count, cbr) + } + }; + let samples_per_frame = if toc & 128 != 0 { + 120 << ((toc >> 3_i32) & 3) + } else if toc & 0x60 == 0x60 { + if toc & 8 != 0 { + 960 + } else { + 480 + } + } else { + *[480, 960, 1920, 2880] + .get(usize::from((toc >> 3_i32) & 3)) + .context("invalid Opus TOC")? + }; + ensure!( + count + .checked_mul(samples_per_frame) + .is_some_and(|samples| samples <= MAX_PACKET_SAMPLES), + "Opus packet exceeds duration limit" + ); + let extra_start = cursor; + let last = read_size(data, &mut cursor)?; + payload = if cbr { + last.checked_mul(count) + } else { + payload.checked_add(last) + } + .context("Opus payload overflow")?; + let end = cursor + .checked_add(payload) + .and_then(|size| size.checked_add(padding)) + .context("Opus packet size overflow")?; + ensure!(end <= data.len(), "truncated self-delimited Opus packet"); + output.clear(); + output.extend_from_slice( + data.get(..extra_start) + .context("invalid Opus header extent")?, + ); + output.extend_from_slice( + data.get(cursor..end) + .context("invalid Opus payload extent")?, + ); + Ok(end) +} + +/// Read one bounded length byte. +fn read_byte(data: &[u8], cursor: &mut usize) -> Result { + let value = *data.get(*cursor).context("truncated Opus packet header")?; + *cursor = cursor + .checked_add(1) + .context("Opus packet cursor overflow")?; + Ok(value) +} + +/// RFC 6716 Section 3.2.1 one- or two-byte frame size, always at most 1275. +fn read_size(data: &[u8], cursor: &mut usize) -> Result { + let first = read_byte(data, cursor)?; + Ok(if first < 252 { + usize::from(first) + } else { + usize::from(read_byte(data, cursor)?) + .checked_mul(4) + .and_then(|high| high.checked_add(usize::from(first))) + .context("Opus frame size overflow")? + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Retain code 0/1/2/3 framing, VBR/CBR, multi-byte lengths and trailing padding. + #[test] + fn self_delimited_packet_lengths_and_padding_are_exact() -> Result<()> { + for (source, expected, consumed) in [ + (vec![0, 3, 1, 2, 3, 99], vec![0, 1, 2, 3], 5), + (vec![1, 2, 1, 2, 3, 4, 99], vec![1, 1, 2, 3, 4], 6), + (vec![2, 1, 2, 1, 2, 3, 99], vec![2, 1, 1, 2, 3], 6), + (vec![3, 65, 2, 1, 7, 0, 0, 99], vec![3, 65, 2, 7, 0, 0], 7), + (vec![3, 130, 1, 2, 7, 8, 9, 99], vec![3, 130, 1, 7, 8, 9], 7), + ] { + let mut actual = Vec::new(); + ensure!( + normalize_self_delimited(&source, &mut actual)? == consumed, + "packet boundary changed" + ); + ensure!(actual == expected, "ordinary framing changed"); + } + let mut source = vec![0, 252, 0]; + source.extend_from_slice(&[7; 252]); + source.push(99); + let mut output = Vec::new(); + ensure!( + normalize_self_delimited(&source, &mut output)? == 255, + "two-byte frame size changed" + ); + ensure!(output.len() == 253, "length field was retained"); + Ok(()) + } + + /// Compare each mapped channel against a separate libopus reference decoder. + #[test] + fn channel_mapping_padding_and_gain_match_independent_pcm() -> Result<()> { + use opus_pure::{OggOpusReader, Trim}; + use std::fs::File; + use std::path::Path; + + let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/media"); + for name in [ + "opus-surround-distinct", + "opus-surround-padded-gain", + "opus-surround-discrete", + ] { + let mut reader = OggOpusReader::new(File::open(root.join(format!("{name}.opus")))?)?; + let head = reader.head().clone(); + let encoded = std::fs::read(root.join(format!("{name}.opus")))?; + let offset = 27 + usize::from(*encoded.get(26).context("missing fixture lacing")?); + let length = usize::from(*encoded.get(27).context("missing fixture header size")?); + let header = encoded + .get(offset..offset + length) + .context("truncated fixture header")?; + let mut decoder = Decoder::new(header, 6)?; + let mut trim = Trim::new(&head, 48_000, 6)?; + let gain = 10.0_f32.powf(f32::from(head.output_gain_q8) / (256.0 * 20.0)); + let reference = std::fs::read(root.join(format!("{name}-native.f32")))?; + let reference = reference + .as_chunks::<4>() + .0 + .iter() + .map(|bytes| f32::from_le_bytes(*bytes)) + .collect::>(); + let mut output = Vec::new(); + let mut actual = Vec::new(); + for packet in reader.packets() { + let packet = packet?; + let _decoded_frames = decoder.decode(&packet.data, &mut output, || Ok(()))?; + actual.extend( + trim.keep(&packet, &output) + .iter() + .map(|sample| sample * gain), + ); + } + ensure!( + actual.len() == 69_120 && reference.len() == actual.len(), + "duration changed for {name}" + ); + // FFmpeg's libopus wrapper reorders all six-channel output to SMPTE, + // including family 255. The adapter preserves the explicit wire map. + let order = [0, 2, 1, 5, 3, 4]; + for (a, b) in actual + .as_chunks::<6>() + .0 + .iter() + .zip(reference.as_chunks::<6>().0.iter()) + { + for (channel, index) in order.iter().copied().enumerate() { + ensure!( + (a.get(index).context("missing mapped channel")? + - b.get(channel).context("missing reference channel")?) + .abs() + <= 0.000_04, + "channel {channel} changed for {name}" + ); + } + } + } + Ok(()) + } + + /// Missing lengths, impossible padding, frame counts and durations fail closed. + #[test] + fn malformed_multistream_framing_is_rejected() { + for source in [ + vec![], + vec![0], + vec![0, 252], + vec![0, 4, 1], + vec![3, 0], + vec![3, 65, 255], + vec![3, 63], + vec![27, 3, 0], + ] { + assert!(normalize_self_delimited(&source, &mut Vec::new()).is_err()); + } + } +} diff --git a/src/media/convert.rs b/src/media/convert.rs index 040c5eb2..eba01f18 100644 --- a/src/media/convert.rs +++ b/src/media/convert.rs @@ -2,8 +2,8 @@ // Per-format conversion logic. // // Conversion rules (from project spec): -// jpg / jpeg → WebP (quality 85, metadata stripped) -// gif → WebP (quality 85, -loop 0 preserves animation if libwebp supports it) +// jpg / jpeg → WebP (lossless, metadata stripped) +// gif → animated WebP (composited frames, timing and repeats preserved) // heic/heif → WebP // bmp → WebP // tiff → WebP @@ -14,8 +14,7 @@ // all audio → keep as-is // mp4 → keep as-is (background worker handles MP4→WebM separately) // -// All conversion functions require ffmpeg. Callers must check -// `MediaProcessor::ffmpeg_available` before calling into this module. +// Image decoding and WebP encoding run internally in Rust. // On failure, all functions log a warning and the caller falls back to // storing the original bytes. @@ -23,7 +22,7 @@ use anyhow::{Context as _, Result}; use std::path::{Path, PathBuf}; use uuid::Uuid; -use super::ffmpeg; +use super::images; /// Describes what action the conversion pipeline should take for a given /// source MIME type. @@ -44,10 +43,6 @@ pub enum ConversionAction { #[must_use] pub fn conversion_action(mime: &str) -> ConversionAction { match mime { - // GIF → animated WebP: keeps the media type as Image so it renders in - // an tag rather than a
"#, - backup_id = escape_html(&bf.backup_id), - backup_ref = escape_html(&bf.backup_ref), - scope = escape_html(&bf.scope), - mode = escape_html(&bf.mode), - size = size_fmt, - modified = escape_html(&bf.modified), - status = status_html, - csrf = escape_html(view.csrf_token), - download_link = download_link, - manifest_path = escape_html(&bf.manifest_path), - server_path = escape_html(&bf.server_path), + backup_id = escape_html(&bf.backup_id), + backup_ref = escape_html(&bf.backup_ref), + scope = escape_html(&bf.scope), + mode = escape_html(&bf.mode), + size = size_fmt, + modified = escape_html(&bf.modified), + status = status_html, + csrf = escape_html(view.csrf_token), + download_link = download_link, + manifest_path = escape_html(&bf.manifest_path), + server_path = escape_html(&bf.server_path), + ), ); } board_backup_rows @@ -441,9 +448,9 @@ fn split_zip_part_size_options(selected_gib: u64) -> String { } else { "" }; - let _ = write!( - options, - r#""# + crate::templates::append_html( + &mut options, + format_args!(r#""#), ); } options diff --git a/src/templates/admin/control_center.rs b/src/templates/admin/control_center.rs index 06cec5b8..bd42e4ae 100644 --- a/src/templates/admin/control_center.rs +++ b/src/templates/admin/control_center.rs @@ -2,7 +2,6 @@ use super::{escape_html, AdminDashboardState, AdminPanelDashboardView, AdminPanelViewModel}; use std::cmp::Reverse; -use std::fmt::Write as _; /// One operational signal displayed by the Control Center. #[derive(Clone, Copy)] @@ -225,9 +224,10 @@ fn render_attention(signals: &DashboardSignals<'_>) -> String { let mut rows = String::new(); for signal in &alerts { - let _ = write!( - rows, - r#"
  • + crate::templates::append_html( + &mut rows, + format_args!( + r#"
  • {label} {value} @@ -236,13 +236,14 @@ fn render_attention(signals: &DashboardSignals<'_>) -> String { {pill} {action}
    • "#, - state = state_class(signal.state), - key = escape_html(signal.key), - label = escape_html(signal.label), - value = escape_html(signal.value), - detail = escape_html(signal.detail), - pill = render_state_pill(signal.state, state_label(signal.state)), - action = section_action_link(signal.open_section, signal.anchor, signal.action), + state = state_class(signal.state), + key = escape_html(signal.key), + label = escape_html(signal.label), + value = escape_html(signal.value), + detail = escape_html(signal.detail), + pill = render_state_pill(signal.state, state_label(signal.state)), + action = section_action_link(signal.open_section, signal.anchor, signal.action), + ), ); } format!( @@ -482,17 +483,19 @@ fn render_task_group( fn render_status_rows(signals: &[DashboardSignal<'_>]) -> String { let mut rows = String::new(); for signal in signals { - let _ = write!( - rows, - r#"
  • + crate::templates::append_html( + &mut rows, + format_args!( + r#"
  • {label}{value} {pill}
    • "#, - state = state_class(signal.state), - key = escape_html(signal.key), - label = escape_html(signal.label), - value = escape_html(signal.value), - pill = render_state_pill(signal.state, state_label(signal.state)), + state = state_class(signal.state), + key = escape_html(signal.key), + label = escape_html(signal.label), + value = escape_html(signal.value), + pill = render_state_pill(signal.state, state_label(signal.state)), + ), ); } format!(r#"
      {rows}
    "#) @@ -505,11 +508,13 @@ fn render_fact_rows(facts: &[(&str, &str)]) -> String { } let mut rows = String::new(); for (label, value) in facts { - let _ = write!( - rows, - r"
    {label}
    {value}
    ", - label = escape_html(label), - value = escape_html(value), + crate::templates::append_html( + &mut rows, + format_args!( + r"
    {label}
    {value}
    ", + label = escape_html(label), + value = escape_html(value), + ), ); } format!(r#"
    {rows}
    "#) @@ -530,11 +535,13 @@ fn render_system_details(view: &AdminPanelViewModel<'_>, signals: &DashboardSign ]; let mut detail_rows = String::new(); for signal in details { - let _ = write!( - detail_rows, - r"
    {label}
    {detail}
    ", - label = escape_html(signal.label), - detail = escape_html(signal.detail), + crate::templates::append_html( + &mut detail_rows, + format_args!( + r"
    {label}
    {detail}
    ", + label = escape_html(signal.label), + detail = escape_html(signal.detail), + ), ); } let technical_facts = render_fact_rows(&[ diff --git a/src/templates/admin/layout.rs b/src/templates/admin/layout.rs index de9f278d..f4ec6811 100644 --- a/src/templates/admin/layout.rs +++ b/src/templates/admin/layout.rs @@ -58,6 +58,7 @@ pub(super) fn render(view: &AdminPanelViewModel<'_>) -> String { + {settings_search}
    {section_index} diff --git a/src/templates/admin/maintenance.rs b/src/templates/admin/maintenance.rs index e20203e0..92050e7c 100644 --- a/src/templates/admin/maintenance.rs +++ b/src/templates/admin/maintenance.rs @@ -1,7 +1,6 @@ //! Media-pipeline and database-maintenance sections of the admin panel. use super::{escape_html, format_file_size, AdminPanelViewModel}; -use std::fmt::Write as _; /// Pre-rendered values interpolated into the maintenance section. struct MaintenanceSectionView<'a> { @@ -69,10 +68,12 @@ old boards to prevent query performance degradation. } else { let mut addresses = String::new(); if let Some(addr) = view.tor_address { - let _ = write!( - addresses, - r#"

    Onion address: {addr}

    "#, - addr = escape_html(addr) + crate::templates::append_html( + &mut addresses, + format_args!( + r#"

    Onion address: {addr}

    "#, + addr = escape_html(addr) + ), ); } addresses @@ -170,22 +171,27 @@ fn render_media_detection_cards(view: &AdminPanelViewModel<'_>) -> String { ( "ffmpeg", view.maintenance.media_detection.ffmpeg.is_detected(), - "video thumbnails, waveform jobs, and transcoding entrypoint", + "video thumbnails and transcoding entrypoint", ), ( - "ffprobe", - view.maintenance.media_detection.ffprobe.is_detected(), - "WebM codec inspection for uploads that need it", - ), - ( - "WebP encoder", - view.maintenance.media_detection.webp_encoder.is_detected(), - "image to WebP conversion", + "WebP/images", + true, + "built in (Rust), including animated WebP", ), ( "VP9/WebM pipeline", view.maintenance.media_detection.vp9_pipeline.is_detected(), - "MP4 to WebM transcoding with VP9 + Opus", + "video conversion to WebM with VP9 + Opus", + ), + ( + "AV1 decoding", + view.maintenance.media_detection.av1_decoder.is_detected(), + "AV1 video inputs; independent of AV1 encoding", + ), + ( + "AV1 encoding", + view.maintenance.media_detection.av1_encoder.is_detected(), + "installed build capability; RustChan output uses VP9 + Opus", ), ( "PDF thumbnails", @@ -196,23 +202,25 @@ fn render_media_detection_cards(view: &AdminPanelViewModel<'_>) -> String { &pdf_detail, ), ] { - let _ = write!( - cards, - r#"
    + crate::templates::append_html( + &mut cards, + format_args!( + r#"

    {label}

    {status}

    {detail}

    "#, - label = escape_html(label), - pill_class = if ok { - "admin-detection-pill-ok" - } else { - "admin-detection-pill-missing" - }, - status = if ok { "detected" } else { "missing" }, - detail = escape_html(detail), + label = escape_html(label), + pill_class = if ok { + "admin-detection-pill-ok" + } else { + "admin-detection-pill-missing" + }, + status = if ok { "detected" } else { "missing" }, + detail = escape_html(detail), + ), ); } cards diff --git a/src/templates/admin/management.rs b/src/templates/admin/management.rs index f94aa25c..b69077cb 100644 --- a/src/templates/admin/management.rs +++ b/src/templates/admin/management.rs @@ -1,7 +1,5 @@ //! Actionable offline relocation and secret rotation without exposing material. -use std::fmt::Write as _; - use crate::config::admin::management::ManagementState; use crate::utils::sanitize::escape_html; @@ -34,7 +32,7 @@ pub(super) fn render(state: &Result, csrf: &str) -> Str s.uploads_source.as_str(), ), ] { - let _ = write!(paths, "{label}{}{}{}{source}
    Restart required for changes; paths alone do not migrate data.", escape_html(active), escape_html(saved), escape_html(next), source = escape_html(source)); + crate::templates::append_html(&mut paths, format_args!( "{label}{}{}{}{source}
    Restart required for changes; paths alone do not migrate data.", escape_html(active), escape_html(saved), escape_html(next), source = escape_html(source))); } let saved_secret = if s.secret_saved { "configured material withheld" diff --git a/src/templates/admin/moderation.rs b/src/templates/admin/moderation.rs index cb6ea9d0..d658eb59 100644 --- a/src/templates/admin/moderation.rs +++ b/src/templates/admin/moderation.rs @@ -1,7 +1,6 @@ //! Report, appeal, ban, and word-filter sections of the admin panel. use super::{escape_html, fmt_ts, AdminPanelViewModel}; -use std::fmt::Write as _; /// Renders the complete moderation tab of the admin panel. pub(super) fn render(view: &AdminPanelViewModel<'_>) -> String { @@ -45,9 +44,10 @@ fn render_ban_rows(view: &AdminPanelViewModel<'_>) -> String { let expires = ban .expires_at .map_or_else(|| "permanent".to_owned(), fmt_ts); - let _ = write!( - ban_rows, - r#" + crate::templates::append_html( + &mut ban_rows, + format_args!( + r#" {}{}{}
    @@ -57,11 +57,12 @@ fn render_ban_rows(view: &AdminPanelViewModel<'_>) -> String {
    "#, - escape_html(ban.ip_hash.get(..16).unwrap_or(&ban.ip_hash)), - escape_html(ban.reason.as_deref().unwrap_or("")), - escape_html(&expires), - csrf = escape_html(view.csrf_token), - id = ban.id + escape_html(ban.ip_hash.get(..16).unwrap_or(&ban.ip_hash)), + escape_html(ban.reason.as_deref().unwrap_or("")), + escape_html(&expires), + csrf = escape_html(view.csrf_token), + id = ban.id + ), ); } ban_rows @@ -71,9 +72,10 @@ fn render_ban_rows(view: &AdminPanelViewModel<'_>) -> String { fn render_filter_rows(view: &AdminPanelViewModel<'_>) -> String { let mut filter_rows = String::new(); for f in view.moderation.filters { - let _ = write!( - filter_rows, - r#" + crate::templates::append_html( + &mut filter_rows, + format_args!( + r#" {}{}
    @@ -83,10 +85,11 @@ fn render_filter_rows(view: &AdminPanelViewModel<'_>) -> String {
    "#, - escape_html(&f.pattern), - escape_html(&f.replacement), - csrf = escape_html(view.csrf_token), - id = f.id + escape_html(&f.pattern), + escape_html(&f.replacement), + csrf = escape_html(view.csrf_token), + id = f.id + ), ); } filter_rows @@ -124,9 +127,10 @@ fn render_report_rows(view: &AdminPanelViewModel<'_>) -> String { ) }, ); - let _ = write!( - report_rows, - r#" + crate::templates::append_html( + &mut report_rows, + format_args!( + r#" /{board}/ No.{pid} {user_info} {preview} @@ -140,15 +144,16 @@ fn render_report_rows(view: &AdminPanelViewModel<'_>) -> String { "#, - board = escape_html(&rc.board_short), - tid = rc.report.thread_id, - pid = rc.report.post_id, - user_info = user_info, - preview = preview, - reason = reason, - age = escape_html(&age), - csrf = escape_html(view.csrf_token), - rid = rc.report.id + board = escape_html(&rc.board_short), + tid = rc.report.thread_id, + pid = rc.report.post_id, + user_info = user_info, + preview = preview, + reason = reason, + age = escape_html(&age), + csrf = escape_html(view.csrf_token), + rid = rc.report.id + ), ); } report_rows @@ -167,9 +172,10 @@ fn render_appeal_rows(csrf_token: &str, appeals: &[crate::models::BanAppeal]) -> let age = fmt_ts(a.created_at); let ip_short = a.ip_hash.get(..16).unwrap_or(&a.ip_hash); let ip_short = escape_html(ip_short); - let _ = write!( - appeal_rows, - r#" + crate::templates::append_html( + &mut appeal_rows, + format_args!( + r#" {ip_short}… {reason} {age} @@ -188,12 +194,13 @@ fn render_appeal_rows(csrf_token: &str, appeals: &[crate::models::BanAppeal]) -> "#, - ip_short = ip_short, - reason = reason, - age = escape_html(&age), - csrf = escape_html(csrf_token), - aid = a.id, - ip_hash = escape_html(&a.ip_hash) + ip_short = ip_short, + reason = reason, + age = escape_html(&age), + csrf = escape_html(csrf_token), + aid = a.id, + ip_hash = escape_html(&a.ip_hash) + ), ); } appeal_rows diff --git a/src/templates/admin/network.rs b/src/templates/admin/network.rs index 594ac25d..7fe214d7 100644 --- a/src/templates/admin/network.rs +++ b/src/templates/admin/network.rs @@ -1,7 +1,5 @@ //! Dedicated Network & Security forms with honest startup configuration state. -use std::fmt::Write as _; - use crate::config::admin::{InputKind, SettingField}; use crate::utils::sanitize::escape_html; @@ -42,11 +40,11 @@ pub(super) fn render_section( r#"

    {label}

    {description}

    {capabilities} -

    Save writes settings.toml atomically. These settings require a restart; running listeners and workers keep their active values. Environment and launcher overrides take precedence on restart.

    +

    Save writes settings.toml atomically. Controls marked restart required use process initialization; running listeners and workers keep their active values. Environment and launcher overrides take precedence on restart.

    {controls} -
    Review saved values and overrides before restarting your service.
    +
    Review saved values, then use Restart RustChan in Configuration restart.
    "#, label = escape_html(label), description = escape_html(description), @@ -103,7 +101,6 @@ fn render_control_groups(fields: &[SettingField], section: &str) -> String { &[ "require_ffmpeg", "ffmpeg_path", - "ffprobe_path", "thumb_size", "job_queue_capacity", "waveform_cache_max_mb", @@ -136,7 +133,7 @@ fn render_control_groups(fields: &[SettingField], section: &str) -> String { .filter(|f| keys.contains(&f.definition.key)) .map(render_field) .collect::(); - let _ = write!(output, "
    {label}
    {controls}
    "); + crate::templates::append_html(&mut output, format_args!( "
    {label}
    {controls}
    ")); } output } @@ -176,7 +173,9 @@ fn render_field(field: &SettingField) -> String { option("reads", "Reads: GET and HEAD", &field.input) ), }; - let state = if field.overridden { + let state = if definition.application == crate::config::admin::ApplicationMode::Live { + "Applies live through its existing save control" + } else if field.overridden { "Override prevents the saved value taking effect; manage the environment or launcher" } else if field.pending { "Saved configuration differs — restart required; check overrides" @@ -185,7 +184,7 @@ fn render_field(field: &SettingField) -> String { }; format!( r#"
    -{control} +{application}{control}

    {}

    Active
    {}
    Saved
    {}
    Next restart
    {}
    Source
    {}
    Application
    {state}
    "#, @@ -197,7 +196,12 @@ fn render_field(field: &SettingField) -> String { escape_html(&field.active), escape_html(&field.saved), escape_html(&field.next_start), - escape_html(&field.source) + escape_html(&field.source), + application = if definition.application == crate::config::admin::ApplicationMode::Restart { + "Restart required" + } else { + "Applies live" + } ) } diff --git a/src/templates/admin/restart.rs b/src/templates/admin/restart.rs new file mode 100644 index 00000000..b9d9c01a --- /dev/null +++ b/src/templates/admin/restart.rs @@ -0,0 +1,22 @@ +//! Compact configuration restart status within the existing administrator panel. + +use crate::{restart::View, utils::sanitize::escape_html}; + +/// Preserve a fully usable POST form and refresh link with JavaScript disabled. +pub(super) fn render(view: &View, csrf: &str) -> String { + let action = if view.pending && view.supported && !view.in_progress { + format!( + r#"
    Stops gracefully and verifies startup. Unsaved form edits are not included.
    "#, + escape_html(csrf) + ) + } else { + String::new() + }; + format!( + r#"

    Configuration restart

    {}

    {action}

    Refresh restart status

    "#, + view.instance, + view.pending, + view.in_progress, + escape_html(&view.message) + ) +} diff --git a/src/templates/admin/search.rs b/src/templates/admin/search.rs index 2b4e0a9c..71bf4dda 100644 --- a/src/templates/admin/search.rs +++ b/src/templates/admin/search.rs @@ -1,7 +1,5 @@ //! Server-rendered settings search, including no-JavaScript operation. -use std::fmt::Write as _; - use crate::config::admin::{application, runtime, NETWORK_SETTINGS}; use crate::utils::sanitize::escape_html; @@ -31,7 +29,7 @@ pub(super) fn render(query: Option<&str>) -> String { "Application state", application::SETTINGS, ))); - let mut count = 0; + let mut count = 0_usize; for (section, label, definitions) in sections { for definition in definitions { let haystack = format!( @@ -40,17 +38,19 @@ pub(super) fn render(query: Option<&str>) -> String { ) .to_lowercase(); if terms.iter().all(|term| haystack.contains(term)) { - count += 1; - let _ = write!( - results, - "
  • {} — {}
    • ", - if section == "configuration-state" { - format!("application-{}", definition.key) - } else { - format!("setting-{}", definition.key) - }, - escape_html(label), - escape_html(definition.label) + count = count.saturating_add(1); + crate::templates::append_html( + &mut results, + format_args!( + "
  • {} — {}
    • ", + if section == "configuration-state" { + format!("application-{}", definition.key) + } else { + format!("setting-{}", definition.key) + }, + escape_html(label), + escape_html(definition.label) + ), ); } } @@ -67,7 +67,7 @@ pub(super) fn render(query: Option<&str>) -> String { ("moderation", "Moderation", "moderation reports appeals bans filters words IP"), ] { let haystack = synonyms.to_lowercase(); - if terms.iter().all(|term| haystack.contains(term)) { count += 1; let _ = write!(results, "
  • {label}
    • "); } + if terms.iter().all(|term| haystack.contains(term)) { count = count.saturating_add(1); crate::templates::append_html(&mut results, format_args!( "
  • {label}
    • ")); } } if count == 0 { results.push_str("
  • No matching runtime settings. Try proxy, Serveo, slow down or upload limit.
    • "); diff --git a/src/templates/admin/site_health.rs b/src/templates/admin/site_health.rs index ea840162..4acb6a0c 100644 --- a/src/templates/admin/site_health.rs +++ b/src/templates/admin/site_health.rs @@ -1,7 +1,6 @@ //! Runtime health and dependency diagnostics for the admin panel. use super::{escape_html, AdminDetectionStatus, AdminPanelViewModel}; -use std::fmt::Write as _; /// Renders the complete site-health section. pub(super) fn render(view: &AdminPanelViewModel<'_>) -> String { @@ -138,11 +137,13 @@ fn append_job_rows(rows: &mut String, view: &AdminPanelViewModel<'_>) { /// Appends one escaped label-value health row. fn append_health_row(out: &mut String, label: &str, value: &str) { - let _ = write!( + crate::templates::append_html( out, - r#"
    {label}{value}
    "#, - label = escape_html(label), - value = escape_html(value), + format_args!( + r#"
    {label}{value}
    "#, + label = escape_html(label), + value = escape_html(value), + ), ); } @@ -154,33 +155,39 @@ fn append_health_job_row(out: &mut String, label: &str, value: &str, key: &str, } else { "" }; - let _ = write!( + crate::templates::append_html( out, - r#"
    "#, - label = escape_html(label), - key = escape_html(key), - value = escape_html(value), - csrf = escape_html(csrf_token), - disabled_attr = disabled_attr, + format_args!( + r#"
    "#, + label = escape_html(label), + key = escape_html(key), + value = escape_html(value), + csrf = escape_html(csrf_token), + disabled_attr = disabled_attr, + ), ); return; } if key == "recent_completed_jobs" { - let _ = write!( + crate::templates::append_html( out, - r#"
    "#, - label = escape_html(label), - key = escape_html(key), - value = escape_html(value), + format_args!( + r#"
    "#, + label = escape_html(label), + key = escape_html(key), + value = escape_html(value), + ), ); return; } - let _ = write!( + crate::templates::append_html( out, - r#"
    {label}{value}
    "#, - label = escape_html(label), - key = escape_html(key), - value = escape_html(value), + format_args!( + r#"
    {label}{value}
    "#, + label = escape_html(label), + key = escape_html(key), + value = escape_html(value), + ), ); } @@ -211,10 +218,17 @@ fn render_dependency_summary(view: &AdminPanelViewModel<'_>) -> String { let mut rows = String::new(); for (label, status) in [ ("ffmpeg", dependencies.ffmpeg), - ("ffprobe", dependencies.ffprobe), - ("WebP support", dependencies.webp), + ( + "WebP/images (built in, Rust)", + AdminDetectionStatus::Detected, + ), ("VP9 support", dependencies.vp9), ("Opus support", dependencies.opus), + ("AV1 decoding", dependencies.av1_decoder), + ( + "AV1 encoding (output policy: VP9)", + dependencies.av1_encoder, + ), ] { append_health_row(&mut rows, label, detection_label(status)); } diff --git a/src/templates/admin/updates.rs b/src/templates/admin/updates.rs index 7e5da0a2..d5d4af4e 100644 --- a/src/templates/admin/updates.rs +++ b/src/templates/admin/updates.rs @@ -2,7 +2,6 @@ use crate::updates::{Discovery, Status}; use crate::utils::sanitize::escape_html; -use std::fmt::Write as _; /// Render native update status or a clear deployment-managed check-only workflow. pub(super) fn render(status: &Status, csrf: &str, managed: bool, container: bool) -> String { @@ -35,11 +34,13 @@ pub(super) fn render(status: &Status, csrf: &str, managed: bool, container: bool .available_release() .is_some_and(|release| release.compatible) }) { - let _ = write!( - controls, - r#"

    Install verified update

    A verified backup is mandatory. RustChan will briefly restart. Failed upgrades restore the previous software, database, configuration and persistent files.

    Approval expires after 10 minutes and can be used once.

    "#, - escape_html(csrf), - escape_html(approval) + crate::templates::append_html( + &mut controls, + format_args!( + r#"

    Install verified update

    A verified backup is mandatory. RustChan will briefly restart. Failed upgrades restore the previous software, database, configuration and persistent files.

    Approval expires after 10 minutes and can be used once.

    "#, + escape_html(csrf), + escape_html(approval) + ), ); } } else { @@ -48,7 +49,10 @@ pub(super) fn render(status: &Status, csrf: &str, managed: bool, container: bool } else { "Installation is deployment-managed on this system. Install the official release using your deployment tools. Native self-updates require the supported Linux updater service." }; - let _ = write!(controls, "

    {note}

    "); + crate::templates::append_html( + &mut controls, + format_args!("

    {note}

    "), + ); } format!( r#"

    Software Updates

    Stable releases from the official RustChan repository. Update checks and installation details are visible only to administrators.

    Running version
    {}
    Latest stable version
    {latest}
    Last check
    {}
    Verification
    {verification}
    Previous result / current progress
    {}
    {controls}

    If the server restarts, reconnect to Software Updates to see the persisted result. JavaScript reconnects automatically while an installation is active.

    {}
    "#, @@ -78,18 +82,20 @@ fn backup_summary(status: &Status) -> String { } let mut rows = String::new(); for backup in status.backups.iter().take(20) { - let _ = write!( - rows, - "{}{} → {}{}{}", - escape_html(&backup.created_at), - escape_html(&backup.previous_version), - escape_html(&backup.target_version), - backup.size, - if backup.verified { - "Verified pre-upgrade backup" - } else { - "Verification failed" - } + crate::templates::append_html( + &mut rows, + format_args!( + "{}{} → {}{}{}", + escape_html(&backup.created_at), + escape_html(&backup.previous_version), + escape_html(&backup.target_version), + backup.size, + if backup.verified { + "Verified pre-upgrade backup" + } else { + "Verification failed" + } + ), ); } format!("
    {rows}
    CreatedUpgradeBytesVerification
    ") diff --git a/src/templates/board.rs b/src/templates/board.rs index 80569ee7..daf94810 100644 --- a/src/templates/board.rs +++ b/src/templates/board.rs @@ -3,7 +3,6 @@ use crate::models::{Board, Pagination, Post, Thread, ThreadSummary, SEARCH_QUERY_MAX_CHARS}; use crate::utils::sanitize::escape_html; use std::collections::{HashMap, HashSet}; -use std::fmt::Write as _; use super::{ base_layout, base_layout_with_preferences, compress_modal_script, embed_thumb_from_body, @@ -642,7 +641,7 @@ fn board_cards( admin_csrf_token, show_reorder_controls, index == 0, - index + 1 == list.len(), + index.checked_add(1) == Some(list.len()), user_preferences, )); } @@ -747,9 +746,11 @@ pub fn index_page( let mut access_links = String::new(); if let Some(addr) = onion_address { let escaped_addr = escape_html(addr); - let _ = write!( - access_links, - r#"

    {escaped_addr}

    "# + crate::templates::append_html( + &mut access_links, + format_args!( + r#"

    {escaped_addr}

    "# + ), ); } let onion_html = if access_links.is_empty() { @@ -884,17 +885,20 @@ pub fn board_page( let admin_form_csrf = admin_csrf_token.unwrap_or(csrf_token); if let Some(msg) = error { - let _ = write!( - body, - r#"
    ⚠ {}
    "#, - escape_html(msg) + crate::templates::append_html( + &mut body, + format_args!( + r#"
    ⚠ {}
    "#, + escape_html(msg) + ), ); } if is_admin { - let _ = write!( - body, - r#"
    + crate::templates::append_html( + &mut body, + format_args!( + r#"
    ■ ADMIN
    @@ -902,8 +906,9 @@ pub fn board_page(
    "#, - csrf = escape_html(admin_form_csrf), - board = escape_html(&board.short_name) + csrf = escape_html(admin_form_csrf), + board = escape_html(&board.short_name) + ), ); } @@ -917,41 +922,45 @@ pub fn board_page( } else { String::new() }; - let _ = write!( - body, - r#"

    /{short}/ — {name}{access_badge}

    {desc}

    + crate::templates::append_html( + &mut body, + format_args!( + r#"

    /{short}/ — {name}{access_badge}

    {desc}

    {board_banner_html}
    [Index][Catalog]{nav_archive}
    "# + ), ); } if can_post { let show_post_form = error.is_some() || new_thread_prefill.is_some(); - let _ = write!( - body, - r##"
    + crate::templates::append_html( + &mut body, + format_args!( + r##"
    [ Post a New Thread ]
    {}
    "##, - super::forms::new_thread_form( - &board.short_name, - csrf_token, - board, - new_thread_prefill, - &format!("/{}", board.short_name), + super::forms::new_thread_form( + &board.short_name, + csrf_token, + board, + new_thread_prefill, + &format!("/{}", board.short_name), + ), + post_form_class = if show_post_form { + "post-form-wrap is-open" + } else { + "post-form-wrap is-collapsed" + }, + post_form_style = if show_post_form { + "display:block" + } else { + "display:none" + }, ), - post_form_class = if show_post_form { - "post-form-wrap is-open" - } else { - "post-form-wrap is-collapsed" - }, - post_form_style = if show_post_form { - "display:block" - } else { - "display:none" - }, ); } else if board.access_mode.requires_unlock_for_posting() { body.push_str(&render_post_access_gate( @@ -1038,62 +1047,72 @@ fn render_thread_summary( "" }; - let _ = write!( - html, - r#"
    + crate::templates::append_html( + &mut html, + format_args!( + r#"
    "#, - tid = t.id, - op_id = t.op_id.unwrap_or(0) + tid = t.id, + op_id = t.op_id.unwrap_or(0) + ), ); let thread_state_badges = super::thread::render_thread_state_badges(t.sticky, t.locked); if let (Some(_file), Some(thumb)) = (&t.op_file, &t.op_thumb) { - let _ = write!( - html, - r#"
    image{badges}
    "#, - board = escape_html(board_short), - tid = t.id, - th = escape_html(thumb), - badges = thread_state_badges + crate::templates::append_html( + &mut html, + format_args!( + r#"
    image{badges}
    "#, + board = escape_html(board_short), + tid = t.id, + th = escape_html(thumb), + badges = thread_state_badges + ), ); } else if let Some(embed_thumb) = t.op_body.as_deref().and_then(embed_thumb_from_body) { - let _ = write!( - html, - r#"
    video thumbnail{badges}
    "#, - board = escape_html(board_short), - tid = t.id, - src = escape_html(&embed_thumb), - badges = thread_state_badges + crate::templates::append_html( + &mut html, + format_args!( + r#"
    video thumbnail{badges}
    "#, + board = escape_html(board_short), + tid = t.id, + src = escape_html(&embed_thumb), + badges = thread_state_badges + ), ); } - let _ = write!( - html, - r#"
    + crate::templates::append_html( + &mut html, + format_args!( + r#"
    {sticky}{locked} {name} {time} No.{op_id} [ #{tid} ]
    "#, - sticky = sticky_label, - locked = locked_label, - name = escape_html(t.op_name.as_deref().unwrap_or("Anonymous")), - ts = t.created_at, - time = fmt_ts_short(t.created_at), - board = escape_html(board_short), - tid = t.id, - op_id = t.op_id.unwrap_or(0) + sticky = sticky_label, + locked = locked_label, + name = escape_html(t.op_name.as_deref().unwrap_or("Anonymous")), + ts = t.created_at, + time = fmt_ts_short(t.created_at), + board = escape_html(board_short), + tid = t.id, + op_id = t.op_id.unwrap_or(0) + ), ); if let Some(subject) = &t.subject { - let _ = write!( - html, - r#"
    {s}
    "#, - b = escape_html(board_short), - tid = t.id, - s = escape_html(subject) + crate::templates::append_html( + &mut html, + format_args!( + r#"
    {s}
    "#, + b = escape_html(board_short), + tid = t.id, + s = escape_html(subject) + ), ); } @@ -1113,7 +1132,10 @@ fn render_thread_summary( } else { escape_html(body) }; - let _ = write!(html, r#"
    {truncated}
    "#); + crate::templates::append_html( + &mut html, + format_args!(r#"
    {truncated}
    "#), + ); } let activity_badge = unread_reply_count @@ -1126,24 +1148,26 @@ fn render_thread_summary( }) .unwrap_or_default(); if !activity_badge.is_empty() { - let _ = write!( - html, - r#"
    {activity_badge}
    "# + crate::templates::append_html( + &mut html, + format_args!(r#"
    {activity_badge}
    "#), ); } - let _ = write!( - html, - r#"
    + crate::templates::append_html( + &mut html, + format_args!( + r#"
    [reply] ({n} {word})"#, - board = escape_html(board_short), - tid = t.id, - n = t.reply_count, - word = if t.reply_count == 1 { - "reply" - } else { - "replies" - }, + board = escape_html(board_short), + tid = t.id, + n = t.reply_count, + word = if t.reply_count == 1 { + "reply" + } else { + "replies" + }, + ), ); if is_admin { @@ -1159,9 +1183,10 @@ fn render_thread_summary( } else { "🔒 lock" }; - let _ = write!( - html, - r#"
    + crate::templates::append_html( + &mut html, + format_args!( + r#" @@ -1182,25 +1207,28 @@ fn render_thread_summary(
    "#, - csrf = escape_html(admin_form_csrf), - tid = t.id, - board = escape_html(board_short), - sticky_act = sticky_act, - sticky_lbl = sticky_lbl, - lock_act = lock_act, - lock_lbl = lock_lbl + csrf = escape_html(admin_form_csrf), + tid = t.id, + board = escape_html(board_short), + sticky_act = sticky_act, + sticky_lbl = sticky_lbl, + lock_act = lock_act, + lock_lbl = lock_lbl + ), ); } html.push_str("
    \n
    "); if summary.omitted > 0 { - let _ = write!( - html, - r#"
    {} posts omitted. view thread
    "#, - summary.omitted, - b = escape_html(board_short), - tid = t.id + crate::templates::append_html( + &mut html, + format_args!( + r#"
    {} posts omitted. view thread
    "#, + summary.omitted, + b = escape_html(board_short), + tid = t.id + ), ); } @@ -1269,9 +1297,10 @@ pub fn catalog_page( let admin_form_csrf = admin_csrf_token.unwrap_or(csrf_token); if is_admin { - let _ = write!( - body, - r#"
    + crate::templates::append_html( + &mut body, + format_args!( + r#"
    ■ ADMIN
    @@ -1279,8 +1308,9 @@ pub fn catalog_page(
    "#, - csrf = escape_html(admin_form_csrf), - board = escape_html(&board.short_name) + csrf = escape_html(admin_form_csrf), + board = escape_html(&board.short_name) + ), ); } @@ -1305,9 +1335,10 @@ pub fn catalog_page( }; let access_badge = board_access_badge(board); - let _ = write!( - body, - r#"
    + crate::templates::append_html( + &mut body, + format_args!( + r#"

    /{bs}/ — {bn}{access_badge}{title_suffix}

    {desc}

    @@ -1334,36 +1365,39 @@ pub fn catalog_page(
    [Index][Catalog]{nav_archive}{hidden_nav}
    "#, - bs = bs, - bn = bn, - access_badge = access_badge, - title_suffix = title_suffix, - desc = escape_html(&board.description), - board_banner_html = board_banner_html, - catalog_active = if hidden_view { "" } else { " active" }, - nav_archive = nav_archive, - hidden_nav = hidden_nav, + bs = bs, + bn = bn, + access_badge = access_badge, + title_suffix = title_suffix, + desc = escape_html(&board.description), + board_banner_html = board_banner_html, + catalog_active = if hidden_view { "" } else { " active" }, + nav_archive = nav_archive, + hidden_nav = hidden_nav, + ), ); if can_post { - let _ = write!( - body, - r##"
    + crate::templates::append_html( + &mut body, + format_args!( + r##"
    [ Start a New Thread ]
    "##, - form = super::forms::new_thread_form( - &board.short_name, - csrf_token, - board, - None, - &if hidden_view { - format!("/{}/hidden", board.short_name) - } else { - format!("/{}/catalog", board.short_name) - }, - ) + form = super::forms::new_thread_form( + &board.short_name, + csrf_token, + board, + None, + &if hidden_view { + format!("/{}/hidden", board.short_name) + } else { + format!("/{}/catalog", board.short_name) + }, + ) + ), ); } else if board.access_mode.requires_unlock_for_posting() { body.push_str(&render_post_access_gate( @@ -1419,10 +1453,12 @@ pub fn catalog_page( } if threads.is_empty() { - let _ = write!( - body, - r#"

    {}

    "#, - escape_html(empty_message) + crate::templates::append_html( + &mut body, + format_args!( + r#"

    {}

    "#, + escape_html(empty_message) + ), ); } @@ -1505,10 +1541,12 @@ pub fn search_page( r#"

    no results found. try a different query.

    "#, ); } else { - let _ = write!( - body, - r#"

    {}

    "#, - escape_html(&result_label) + crate::templates::append_html( + &mut body, + format_args!( + r#"

    {}

    "#, + escape_html(&result_label) + ), ); for post in posts { body.push_str(&super::thread::render_search_post( @@ -1746,7 +1784,11 @@ mod tests { thread_count: 4, }; let mut badges = HashMap::new(); - badges.insert(stats.board.id, 2); + assert_eq!( + badges.insert(stats.board.id, 2), + None, + "the board badge fixture must be unique" + ); let html = board_cards( &[&stats], @@ -1918,7 +1960,10 @@ mod tests { let board = sample_board(); let thread = sample_thread(); let mut pinned_ids = HashSet::new(); - pinned_ids.insert(thread.id); + assert!( + pinned_ids.insert(thread.id), + "the pinned thread fixture must be unique" + ); let html = catalog_page( &board, diff --git a/src/templates/forms.rs b/src/templates/forms.rs index 939601b2..b8c1f549 100644 --- a/src/templates/forms.rs +++ b/src/templates/forms.rs @@ -6,7 +6,6 @@ use crate::config::CONFIG; use crate::models::Board; use crate::utils::sanitize::escape_html; -use std::fmt::Write as _; /// User-entered fields preserved when a post form must be rendered again. #[derive(Clone, Debug, Default, PartialEq, Eq)] @@ -48,7 +47,7 @@ pub fn post_error_recovery_body(message: &str, prefill: Option<&PostFormState>) let mut options = String::new(); for (index, option) in poll.options.iter().enumerate() { // Writing formatted text to a String cannot fail. - let _ = write!(options, "

    Option {}: {}

    ", index + 1, escape_html(option)); + crate::templates::append_html(&mut options, format_args!( "

    Option {}: {}

    ", index.saturating_add(1), escape_html(option))); } format!("

    Poll: {}

    {options}

    Duration: {} {}

    ", escape_html(&poll.question), escape_html(&poll.duration_value), escape_html(&poll.duration_unit)) @@ -141,7 +140,7 @@ fn render_captcha_row(board_short: &str, reply_suffix: &str, refresh_href: &str) (path, Some(fragment)) }); let query_separator = if refresh_path.contains('?') { '&' } else { '?' }; - let refresh_href = format!( + let refreshed_href = format!( "{refresh_path}{query_separator}captcha_refresh={captcha_id}{}", fragment.map_or_else(String::new, |value| format!("#{value}")) ); @@ -151,7 +150,7 @@ fn render_captcha_row(board_short: &str, reply_suffix: &str, refresh_href: &str)
    CAPTCHA challenge image - new challenge + new challenge
    @@ -163,7 +162,7 @@ fn render_captcha_row(board_short: &str, reply_suffix: &str, refresh_href: &str) answer_id = escape_html(&answer_id), image_src = escape_html(&image_src), captcha_id = escape_html(&captcha_id), - refresh_href = escape_html(&refresh_href), + refreshed_href = escape_html(&refreshed_href), ) } @@ -321,12 +320,13 @@ fn render_poll_creator(poll: Option<&PollFormState>) -> String { .map(|number| { render_poll_option_row( number, - poll.and_then(|state| state.options.get(number - 1)) + poll.and_then(|state| state.options.get(number.saturating_sub(1))) .map_or("", String::as_str), ) }) .collect(); - let extra_poll_option_rows: String = (visible_option_count + 1..=POLL_OPTION_MAX_COUNT) + let extra_poll_option_rows: String = (visible_option_count.saturating_add(1) + ..=POLL_OPTION_MAX_COUNT) .map(|number| render_poll_option_row(number, "")) .collect(); let poll_question = poll.map_or("", |state| state.question.as_str()); diff --git a/src/templates/mod.rs b/src/templates/mod.rs index 1fe62e82..6981ab1c 100644 --- a/src/templates/mod.rs +++ b/src/templates/mod.rs @@ -24,6 +24,17 @@ pub use admin::*; pub use board::*; pub use thread::*; +/// Appends formatted HTML without allocating an intermediate string. +/// +/// The `String` sink is infallible; an error indicates that a supplied `Display` +/// implementation rejected formatting. Report that failure instead of silently +/// discarding the result, while preserving the already rendered page fragment. +pub(crate) fn append_html(html: &mut String, arguments: std::fmt::Arguments<'_>) { + if let Err(error) = html.write_fmt(arguments) { + tracing::error!(%error, "HTML formatting failed"); + } +} + /// Selects the default page used by links to a board. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum PreferredBoardView { @@ -138,7 +149,9 @@ static STATIC_ASSET_VERSION: LazyLock = LazyLock::new(compute_static_ass pub fn set_live_boards(boards: Vec) { *LIVE_BOARDS.write() = Arc::new(boards); // Thread-page ETags include this version so navigation changes invalidate them. - LIVE_BOARDS_VERSION.fetch_add(1, Ordering::Relaxed); + // The previous version is irrelevant; fetch_add intentionally wraps this + // cache invalidation counter after every representable version is used. + let _previous_version = LIVE_BOARDS_VERSION.fetch_add(1, Ordering::Relaxed); rebuild_live_board_nav(); } @@ -211,7 +224,7 @@ pub fn set_live_site_subtitle(subtitle: &str) { /// Pass an empty string to clear the admin override and fall back to the hard default. pub fn set_live_default_theme(theme: &str) { *LIVE_DEFAULT_THEME.write() = Arc::from(theme); - LIVE_THEME_VERSION.fetch_add(1, Ordering::Relaxed); + let _previous_version = LIVE_THEME_VERSION.fetch_add(1, Ordering::Relaxed); } /// Read the current live default theme slug. @@ -222,7 +235,7 @@ pub fn live_default_theme() -> Arc { /// Replaces the in-memory theme snapshot. pub fn set_live_themes(themes: Vec) { *LIVE_THEMES.write() = Arc::new(themes); - LIVE_THEME_VERSION.fetch_add(1, Ordering::Relaxed); + let _previous_version = LIVE_THEME_VERSION.fetch_add(1, Ordering::Relaxed); } /// Returns a shared snapshot of the live themes. @@ -373,9 +386,9 @@ fn mobile_board_group_html( for board in boards { let short = escape_html(&board.short_name); let href = board_href(&board.short_name, preferences); - let _ = write!( - items, - r#"/{short}/"#, + append_html( + &mut items, + format_args!(r#"/{short}/"#), ); } format!( @@ -588,7 +601,7 @@ pub const fn thread_autoupdate_script() -> &'static str { pub fn fmt_ts(ts: i64) -> String { match Local.timestamp_opt(ts, 0) { chrono::LocalResult::Single(dt) => dt.format("%Y-%m-%d %H:%M:%S").to_string(), - _ => "unknown".to_owned(), + chrono::LocalResult::None | chrono::LocalResult::Ambiguous(_, _) => "unknown".to_owned(), } } @@ -597,7 +610,7 @@ pub fn fmt_ts(ts: i64) -> String { pub fn fmt_ts_short(ts: i64) -> String { match Local.timestamp_opt(ts, 0) { chrono::LocalResult::Single(dt) => dt.format("%m/%d/%y(%a)%H:%M:%S").to_string(), - _ => "?".to_owned(), + chrono::LocalResult::None | chrono::LocalResult::Ambiguous(_, _) => "?".to_owned(), } } @@ -631,22 +644,29 @@ pub fn render_pagination(p: &Pagination, base_url: &str) -> String { let mut html = String::from(r#"